From 9928bdc89674fca644c56b5e676db0db87c66107 Mon Sep 17 00:00:00 2001
From: Manuel Guenther <manuel.guenther@idiap.ch>
Date: Mon, 22 Jun 2015 11:37:53 +0200
Subject: [PATCH] Re-implemented the hook for database directories
---
bob/bio/base/database/DatabaseBob.py | 35 ++++++++++++++++++++++++++++
bob/bio/base/script/resources.py | 23 +++++++++++++++++-
bob/bio/base/tools/command_line.py | 33 +++-----------------------
bob/bio/base/utils/resources.py | 19 +++++++++++++++
setup.py | 3 ++-
5 files changed, 81 insertions(+), 32 deletions(-)
diff --git a/bob/bio/base/database/DatabaseBob.py b/bob/bio/base/database/DatabaseBob.py
index 5fce8342..5eed659f 100644
--- a/bob/bio/base/database/DatabaseBob.py
+++ b/bob/bio/base/database/DatabaseBob.py
@@ -1,4 +1,5 @@
from .Database import Database, DatabaseZT
+import os
class DatabaseBob (Database):
"""This class can be used whenever you have a database that follows the default Bob database interface."""
@@ -51,6 +52,10 @@ class DatabaseBob (Database):
self.database = database
self.original_directory = database.original_directory
+ try:
+ self.annotation_directory = database.annotation_directory
+ except AttributeError:
+ pass
self.all_files_options = all_files_options
self.extractor_training_options = extractor_training_options
@@ -73,6 +78,36 @@ class DatabaseBob (Database):
return "%s(%s)" % (str(self.__class__), params)
+ def replace_directories(self, replacements = None):
+ """This function replaces the original_directory and the annotation_directory with the directories read from the given replacement file."""
+ if replacements is None:
+ return
+ if isinstance(replacements, str):
+ if not os.path.exists(replacements):
+ return
+ # Open the database replacement file and reads its content
+ with open(replacements) as f:
+ replacements = {}
+ for line in f:
+ if line.strip() and not line.startswith("#"):
+ splits = line.split("=")
+ assert len(splits) == 2
+ replacements[splits[0].strip()] = splits[1].strip()
+
+ assert isinstance(replacements, dict)
+
+ if self.original_directory in replacements:
+ self.original_directory = replacements[self.original_directory]
+ self.database.original_directory = replacements[self.database.original_directory]
+
+ try:
+ if self.annotation_directory in replacements:
+ self.annotation_directory = replacements[self.annotation_directory]
+ self.database.annotation_directory = replacements[self.database.annotation_directory]
+ except AttributeError:
+ pass
+
+
def uses_probe_file_sets(self):
"""Defines if, for the current protocol, the database uses several probe files to generate a score."""
return self.protocol != 'None' and self.database.provides_file_set_for_protocol(self.protocol)
diff --git a/bob/bio/base/script/resources.py b/bob/bio/base/script/resources.py
index f06f230d..4ba3887c 100644
--- a/bob/bio/base/script/resources.py
+++ b/bob/bio/base/script/resources.py
@@ -2,8 +2,9 @@
from __future__ import print_function
import bob.bio.base
+import os
-def main():
+def resources():
import argparse
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter)
@@ -35,3 +36,23 @@ def main():
print (bob.bio.base.list_resources('grid'))
print()
+
+def databases():
+ import argparse
+ database_replacement = "/idiap/home/%s/.bob_bio_databases.txt" % os.environ["USER"] if os.path.isdir("/idiap") else "/home/%s/.bob_bio_databases.txt" % os.environ["USER"]
+
+ parser = argparse.ArgumentParser(description="Prints a list of directories for registered databases", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+ parser.add_argument('-D', '--database-directories-file', metavar = 'FILE', default = database_replacement, help = 'The file, where database directories are stored (to avoid changing the database configurations)')
+
+ args = parser.parse_args()
+
+ # get registered databases
+ databases = bob.bio.base.utils.resources.database_directories(replacements=args.database_directories_file)
+
+ # print directories for all databases
+ for d in sorted(databases):
+ print ("\n%s:" % d)
+
+ print ("Original data: %s" % databases[d][0])
+ if len(databases[d]) > 1:
+ print ("Annotations: %s" % databases[d][1])
diff --git a/bob/bio/base/tools/command_line.py b/bob/bio/base/tools/command_line.py
index 68c83797..ebb6e9f7 100644
--- a/bob/bio/base/tools/command_line.py
+++ b/bob/bio/base/tools/command_line.py
@@ -67,7 +67,7 @@ def command_line_parser(description=__doc__, exclude_resources_from=[]):
help = 'The database file in which the submitted jobs will be written; relative to the current directory (only valid with the --grid option).')
file_group.add_argument('--experiment-info-file', metavar = 'FILE', default = 'Experiment.info',
help = 'The file where the configuration of all parts of the experiments are written; relative to te --result-directory.')
- file_group.add_argument('--database-directories-file', metavar = 'FILE', default = database_replacement,
+ file_group.add_argument('-D', '--database-directories-file', metavar = 'FILE', default = database_replacement,
help = 'An optional file, where database directories are stored (to avoid changing the database configurations)')
sub_dir_group = parser.add_argument_group('\nSubdirectories of certain parts of the tool chain. You can specify directories in case you want to reuse parts of the experiments (e.g. extracted features) in other experiments. Please note that these directories are relative to the --temp-directory, but you can also specify absolute paths')
@@ -98,7 +98,7 @@ def command_line_parser(description=__doc__, exclude_resources_from=[]):
help = 'Try to recursively stop the dependent jobs from the SGE grid queue, when a job failed')
flag_group.add_argument('-X', '--external-dependencies', type=int, default = [], nargs='+',
help = 'The jobs submitted to the grid have dependencies on the given job ids.')
- flag_group.add_argument('-D', '--timer', choices=('real', 'system', 'user'), nargs = '*',
+ flag_group.add_argument('-B', '--timer', choices=('real', 'system', 'user'), nargs = '*',
help = 'Measure and report the time required by the execution of the tool chain (only on local machine)')
flag_group.add_argument('-L', '--run-local-scheduler', action='store_true',
help = 'Starts the local scheduler after submitting the jobs to the local queue (by default, local jobs must be started by hand, e.g., using ./bin/jman --local -vv run-scheduler -x)')
@@ -169,7 +169,6 @@ def initialize(parsers, command_line_parameters = None, skips = []):
args.grid_log_directory = os.path.join(args.temp_directory, args.grid_log_directory)
-
# protocol command line override
if args.protocol is not None:
args.database.protocol = args.protocol
@@ -185,34 +184,8 @@ def initialize(parsers, command_line_parameters = None, skips = []):
enroller_sub_dir = protocol if args.database.training_depends_on_protocol and args.algorithm.requires_enroller_training else projector_sub_dir
model_sub_dir = protocol if args.database.models_depend_on_protocol else enroller_sub_dir
-
# Database directories, which should be automatically replaced
- if os.path.exists(args.database_directories_file):
- #
- replacements = {}
- with open(args.database_directories_file) as f:
- for line in f:
- if line.strip() and not line.startswith("#"):
- splits = line.split("=")
- assert len(splits) == 2
- replacements[splits[0].strip()] = splits[1].strip()
-
- try:
- for d in replacements:
- if args.database.original_directory == d:
- args.database.original_directory = replacements[d]
- args.database.database.original_directory = replacements[d]
- except AttributeError:
- pass
-
- try:
- for d in replacements:
- if args.database.annotation_directory == d:
- args.database.annotation_directory = replacements[d]
- args.database.database.annotation_directory = replacements[d]
- except AttributeError:
- pass
-
+ args.database.replace_directories(args.database_directories_file)
# initialize the file selector
FileSelector.create(
diff --git a/bob/bio/base/utils/resources.py b/bob/bio/base/utils/resources.py
index deecd562..57af5f21 100644
--- a/bob/bio/base/utils/resources.py
+++ b/bob/bio/base/utils/resources.py
@@ -186,3 +186,22 @@ def list_resources(keyword, strip=['dummy']):
else:
retval += " + %s --> %s\n" % (entry_point.name + " "*(length - len(entry_point.name)), entry_point.module_name)
return retval
+
+
+def database_directories(strip=['dummy'], replacements = None):
+ """Returns a dictionary of original directories for all registered databases."""
+ entry_points = _get_entry_points('database', strip)
+
+ dirs = {}
+ for entry_point in sorted(entry_points):
+ try:
+ db = load_resource(entry_point.name, 'database')
+ db.replace_directories(replacements)
+ dirs[entry_point.name] = [db.original_directory]
+# import ipdb; ipdb.set_trace()
+ if db.annotation_directory is not None:
+ dirs[entry_point.name].append(db.annotation_directory)
+ except (AttributeError, ValueError):
+ pass
+
+ return dirs
diff --git a/setup.py b/setup.py
index 6811620e..406e6ef5 100644
--- a/setup.py
+++ b/setup.py
@@ -104,7 +104,8 @@ setup(
# scripts should be declared using this entry:
'console_scripts' : [
'verify.py = bob.bio.base.script.verify:main',
- 'resources.py = bob.bio.base.script.resources:main',
+ 'resources.py = bob.bio.base.script.resources:resources',
+ 'databases.py = bob.bio.base.script.resources:databases',
'evaluate.py = bob.bio.base.script.evaluate:main',
'collect_results.py = bob.bio.base.script.collect_results:main',
'grid_search.py = bob.bio.base.script.grid_search:main',
--
GitLab