diff --git a/bob/bio/base/script/grid_search.py b/bob/bio/base/script/grid_search.py
index ea2dc5e8594291b8008a9bc06322b78ff4b84a75..d04b2b7038e88a387f8e05c67b2278ec61550c5b 100755
--- a/bob/bio/base/script/grid_search.py
+++ b/bob/bio/base/script/grid_search.py
@@ -406,7 +406,7 @@ def main(command_line_parameters = None):
command_line_options(command_line_parameters)
global configuration, place_holder_key
- configuration = utils.read_config_file(args.configuration_file)
+ configuration = utils.read_config_file([args.configuration_file])
place_holder_key = args.place_holder_key
if args.preprocessor:
diff --git a/bob/bio/base/script/resources.py b/bob/bio/base/script/resources.py
index ffeaedac6a8d6d6e9a813a5fdd4c08fc3c66c782..15fc021158bd2c40e72f807574e742f31d0a1ad3 100644
--- a/bob/bio/base/script/resources.py
+++ b/bob/bio/base/script/resources.py
@@ -9,8 +9,8 @@ def resources(command_line_parameters = None):
import argparse
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("--types", '-t', nargs = '+',
- choices = ('d', 'database', 'p', 'preprocessor', 'e', 'extractor', 'a', 'algorithm', 'g', 'grid'),
- default = ('d', 'p', 'e', 'a', 'g'),
+ choices = ('d', 'database', 'p', 'preprocessor', 'e', 'extractor', 'a', 'algorithm', 'g', 'grid', 'c', 'config'),
+ default = ('d', 'p', 'e', 'a', 'g', 'c'),
help = "Select the resource types that should be listed.")
parser.add_argument("--details", '-d', action='store_true', help = "Prints the complete configuration for all resources")
@@ -47,6 +47,10 @@ def resources(command_line_parameters = None):
print ("\nList of registered grid configurations:")
print (bob.bio.base.list_resources('grid', **kwargs))
+ if 'c' in args.types or 'config' in args.types:
+ print ("\nList of registered configurations:")
+ print (bob.bio.base.list_resources('config', **kwargs))
+
print()
def databases(command_line_parameters = None):
diff --git a/bob/bio/base/test/dummy/config.py b/bob/bio/base/test/dummy/config.py
new file mode 100644
index 0000000000000000000000000000000000000000..c2e3b48aacd5970e664d700afb64eb5c7dcad509
--- /dev/null
+++ b/bob/bio/base/test/dummy/config.py
@@ -0,0 +1,7 @@
+from .database import database
+from .preprocessor import preprocessor
+from .extractor import extractor
+from .algorithm import algorithm
+zt_norm = True
+verbose = 1
+sub_directory = "test_dummy"
diff --git a/bob/bio/base/test/dummy/config2.py b/bob/bio/base/test/dummy/config2.py
new file mode 100644
index 0000000000000000000000000000000000000000..ff740ebc8bde53fe72633d989a1e7e81253e729a
--- /dev/null
+++ b/bob/bio/base/test/dummy/config2.py
@@ -0,0 +1,2 @@
+verbose = 2
+sub_directory = "test_dummy2"
diff --git a/bob/bio/base/test/test_config_file.py b/bob/bio/base/test/test_config_file.py
index 39e6c347ccd2d97edf48e6bcf57a771f1ee25d40..d00e1c33c4504da03fe9d2212536a9be30e962e5 100644
--- a/bob/bio/base/test/test_config_file.py
+++ b/bob/bio/base/test/test_config_file.py
@@ -76,7 +76,7 @@ def test_basic():
'result_directory = "%s"' % test_dir,
])
- args = parse_arguments(['-c', test_config_file.name])
+ args = parse_arguments([test_config_file.name])
assert args.zt_norm is True
assert args.verbose == 1
@@ -117,7 +117,7 @@ def test_compare_to_cmdline_basic():
'result_directory = "%s"' % test_dir,
])
- args_file = parse_arguments(['-c', test_config_file.name])
+ args_file = parse_arguments([test_config_file.name])
# now do the same with command-line arguments, ensure result is equal
args_cmdline = parse_arguments([
@@ -159,7 +159,7 @@ def test_compare_to_cmdline_resources():
'preferred_package = "bob.bio.base"',
])
- args_file = parse_arguments(['-c', test_config_file.name])
+ args_file = parse_arguments([test_config_file.name])
# now do the same with command-line arguments, ensure result is equal
args_cmdline = parse_arguments([
@@ -204,7 +204,7 @@ def test_compare_to_cmdline_skip():
'preferred_package = "bob.bio.base"',
])
- args_file = parse_arguments(['-c', test_config_file.name])
+ args_file = parse_arguments([test_config_file.name])
# now do the same with command-line arguments, ensure result is equal
args_cmdline = parse_arguments([
@@ -227,3 +227,107 @@ def test_compare_to_cmdline_skip():
finally:
if test_dir: shutil.rmtree(test_dir)
if test_config_file: del test_config_file
+
+
+def test_from_resource():
+
+ test_dir = None
+
+ try:
+ test_dir = tempfile.mkdtemp(prefix='bobtest_')
+ args = parse_arguments(['dummy'])
+
+ assert args.sub_directory.endswith('test_dummy')
+ assert args.allow_missing_files is False
+ assert args.zt_norm is True
+ assert args.verbose == 1
+
+ from bob.bio.base.test.dummy.database import DummyDatabase
+ assert isinstance(args.database, DummyDatabase)
+ from bob.bio.base.test.dummy.preprocessor import DummyPreprocessor
+ assert isinstance(args.preprocessor, DummyPreprocessor)
+ from bob.bio.base.test.dummy.extractor import DummyExtractor
+ assert isinstance(args.extractor, DummyExtractor)
+ from bob.bio.base.test.dummy.algorithm import DummyAlgorithm
+ assert isinstance(args.algorithm, DummyAlgorithm)
+
+ finally:
+ if test_dir: shutil.rmtree(test_dir)
+
+
+def test_from_module():
+
+ test_dir = None
+
+ try:
+ test_dir = tempfile.mkdtemp(prefix='bobtest_')
+ args = parse_arguments(['bob.bio.base.test.dummy.config'])
+
+ assert args.sub_directory.endswith('test_dummy')
+ assert args.allow_missing_files is False
+ assert args.zt_norm is True
+ assert args.verbose == 1
+
+ from bob.bio.base.test.dummy.database import DummyDatabase
+ assert isinstance(args.database, DummyDatabase)
+ from bob.bio.base.test.dummy.preprocessor import DummyPreprocessor
+ assert isinstance(args.preprocessor, DummyPreprocessor)
+ from bob.bio.base.test.dummy.extractor import DummyExtractor
+ assert isinstance(args.extractor, DummyExtractor)
+ from bob.bio.base.test.dummy.algorithm import DummyAlgorithm
+ assert isinstance(args.algorithm, DummyAlgorithm)
+
+ finally:
+ if test_dir: shutil.rmtree(test_dir)
+
+
+def test_order():
+
+ test_dir = None
+
+ try:
+ test_dir = tempfile.mkdtemp(prefix='bobtest_')
+ args = parse_arguments(['dummy', 'dummy2'])
+
+ assert args.sub_directory.endswith('test_dummy2')
+ assert args.allow_missing_files is False
+ assert args.zt_norm is True
+ assert args.verbose == 2
+
+ from bob.bio.base.test.dummy.database import DummyDatabase
+ assert isinstance(args.database, DummyDatabase)
+ from bob.bio.base.test.dummy.preprocessor import DummyPreprocessor
+ assert isinstance(args.preprocessor, DummyPreprocessor)
+ from bob.bio.base.test.dummy.extractor import DummyExtractor
+ assert isinstance(args.extractor, DummyExtractor)
+ from bob.bio.base.test.dummy.algorithm import DummyAlgorithm
+ assert isinstance(args.algorithm, DummyAlgorithm)
+
+ finally:
+ if test_dir: shutil.rmtree(test_dir)
+
+
+def test_order_inverse():
+
+ test_dir = None
+
+ try:
+ test_dir = tempfile.mkdtemp(prefix='bobtest_')
+ args = parse_arguments(['dummy2', 'dummy'])
+
+ assert args.sub_directory.endswith('test_dummy')
+ assert args.allow_missing_files is False
+ assert args.zt_norm is True
+ assert args.verbose == 1
+
+ from bob.bio.base.test.dummy.database import DummyDatabase
+ assert isinstance(args.database, DummyDatabase)
+ from bob.bio.base.test.dummy.preprocessor import DummyPreprocessor
+ assert isinstance(args.preprocessor, DummyPreprocessor)
+ from bob.bio.base.test.dummy.extractor import DummyExtractor
+ assert isinstance(args.extractor, DummyExtractor)
+ from bob.bio.base.test.dummy.algorithm import DummyAlgorithm
+ assert isinstance(args.algorithm, DummyAlgorithm)
+
+ finally:
+ if test_dir: shutil.rmtree(test_dir)
diff --git a/bob/bio/base/tools/command_line.py b/bob/bio/base/tools/command_line.py
index ba4379960571bdf35e23f117b5c34c3e0e94a56d..33f0ed662fe7cb153cb48f339afa58889f2a5953 100644
--- a/bob/bio/base/tools/command_line.py
+++ b/bob/bio/base/tools/command_line.py
@@ -47,8 +47,7 @@ def command_line_parser(description=__doc__, exclude_resources_from=[]):
#######################################################################################
############## options that are required to be specified #######################
config_group = parser.add_argument_group('\nParameters defining the experiment. Most of these parameters can be a registered resource, a configuration file, or even a string that defines a newly created object')
- config_group.add_argument('-c', '--configuration-file',
- help = 'A configuration file containing one or more of "database", "preprocessor", "extractor", "algorithm" and/or "grid"')
+ config_group.add_argument('configuration_file', metavar='PATH', nargs='*', help = 'A configuration file containing one or more of "database", "preprocessor", "extractor", "algorithm" and/or "grid"')
config_group.add_argument('-d', '--database', metavar = 'x', nargs = '+',
help = 'Database and the protocol; registered databases are: %s' % utils.resource_keys('database', exclude_resources_from))
config_group.add_argument('-p', '--preprocessor', metavar = 'x', nargs = '+',
@@ -169,7 +168,7 @@ def _take_from_config_or_command_line(args, config, keyword, default, required=T
setattr(args, keyword, val)
elif required:
- raise ValueError("Please specify a %s either on command line (via --%s) or in the configuration file (via --configuration-file)" %(keyword, keyword))
+ raise ValueError("Please specify a %s either on command line (via --%s) or in a configuration file" %(keyword, keyword))
def initialize(parsers, command_line_parameters = None, skips = []):
@@ -217,7 +216,7 @@ def initialize(parsers, command_line_parameters = None, skips = []):
args = parser.parse_args(command_line_parameters)
# first, read the configuration file and set everything from the config file to the args -- as long as not overwritten on command line
- config = utils.read_config_file(args.configuration_file) if args.configuration_file is not None else None
+ config = utils.read_config_file(args.configuration_file) if args.configuration_file else None
for keyword in ("database", "preprocessor", "extractor", "algorithm"):
_take_from_config_or_command_line(args, config, keyword,
parser.get_default(keyword))
diff --git a/bob/bio/base/utils/resources.py b/bob/bio/base/utils/resources.py
index 8be8e0304d5dc243ef17b8919dc73557c19a3a4b..f49652ff833863aa5bf125a4e0fd9c9b0b3735a9 100644
--- a/bob/bio/base/utils/resources.py
+++ b/bob/bio/base/utils/resources.py
@@ -20,10 +20,74 @@ logger = logging.getLogger("bob.bio.base")
#: Keywords for which resources are defined.
-valid_keywords = ('database', 'preprocessor', 'extractor', 'algorithm', 'grid')
+valid_keywords = ('database', 'preprocessor', 'extractor', 'algorithm', 'grid', 'config')
-def read_config_file(filename, keyword = None):
- """read_config_file(filename, keyword = None) -> config
+
+def _collect_config(paths):
+ '''Collect all python file resources into a module
+
+ This function recursively loads python modules (in a Python 3-compatible way)
+ so the last loaded module corresponds to the final state of the loading. In
+ this way, we load the first file, resolve its symbols, overwrite with the
+ second file and so on. We return a temporarily created module containing all
+ resolved variables, respecting the input order.
+
+
+ Parameters:
+
+ paths : [str]
+ A list of resources, modules or files (in order) to collect resources from
+
+
+ Returns: module
+
+ A valid Python module you can use to configure your tool
+
+ '''
+
+ def _attach_resources(src, dst):
+ for k in dir(src):
+ setattr(dst, k, getattr(src, k))
+
+ import random
+
+ name = "".join(random.sample(ascii_letters, 10))
+ retval = imp.new_module(name)
+
+ for path in paths:
+ # execute the module code on the context of previously import modules
+ for ep in pkg_resources.iter_entry_points('bob.bio.config'):
+ if ep.name == path:
+ tmp = ep.load() # loads the pointed module
+ _attach_resources(tmp, retval)
+ break
+ else:
+
+ # if you get to this point, then it is not a resource, maybe it is a module?
+ try:
+ tmp = __import__(path, retval.__dict__, retval.__dict__, ['*'])
+ _attach_resources(tmp, retval)
+ continue
+ except ImportError:
+ # module does not exist, ignore it
+ pass
+ except Exception as e:
+ raise IOError("The configuration module '%s' could not be loaded: %s" % (path, e))
+
+ # if you get to this point, then its not a resource nor a loadable module, is
+ # it on the file system?
+ if not os.path.exists(path):
+ raise IOError("The configuration file, resource or module '%s' could not be found, loaded or imported" % path)
+
+ name = "".join(random.sample(ascii_letters, 10))
+ tmp = imp.load_source(name, path)
+ _attach_resources(tmp, retval)
+
+ return retval
+
+
+def read_config_file(filenames, keyword = None):
+ """read_config_file(filenames, keyword = None) -> config
Use this function to read the given configuration file.
If a keyword is specified, only the configuration according to this keyword is returned.
@@ -31,8 +95,9 @@ def read_config_file(filename, keyword = None):
**Parameters:**
- filename : str
- The name of the configuration file to read.
+ filenames : [str]
+ A list (pontentially empty) of configuration files or resources to read
+ running options from
keyword : str or ``None``
If specified, only the contents of the variable with the given name is returned.
@@ -45,19 +110,18 @@ def read_config_file(filename, keyword = None):
Otherwise, the whole configuration is returned (as a local namespace).
"""
- if not os.path.exists(filename):
- raise IOError("The given configuration file '%s' could not be found" % filename)
+ if not filenames:
+ raise RuntimeError("At least one configuration file, resource or " \
+ "module name must be passed")
- import string
- import random
- tmp_config = "".join(random.sample(ascii_letters, 10))
- config = imp.load_source(tmp_config, filename)
+ config = _collect_config(filenames)
if not keyword:
return config
if not hasattr(config, keyword):
- raise ImportError("The desired keyword '%s' does not exist in your configuration file '%s'." %(keyword, filename))
+ raise ImportError("The desired keyword '%s' does not exist in any of " \
+ "your configuration files: %s" %(keyword, ', '.join(filenames)))
return getattr(config, keyword)
@@ -103,7 +167,7 @@ def load_resource(resource, keyword, imports = ['bob.bio.base'], package_prefix=
# first, look if the resource is a file name
if os.path.isfile(resource):
- return read_config_file(resource, keyword)
+ return read_config_file([resource], keyword)
if keyword not in valid_keywords:
raise ValueError("The given keyword '%s' is not valid. Please use one of %s!" % (str(keyword), str(valid_keywords)))
diff --git a/develop.cfg b/develop.cfg
index b9e28af7b73d926e6b0b824f405ae171b1c62965..ce6a1b19c5e6608bc8c09ff59722bd5c73007465 100644
--- a/develop.cfg
+++ b/develop.cfg
@@ -5,6 +5,8 @@
[buildout]
parts = scripts
eggs = bob.bio.base
+ bob.db.atnt
+ bob.io.image
gridtk
extensions = bob.buildout
diff --git a/doc/implementation.rst b/doc/implementation.rst
index 984742d6b8be59c719b7bd9d98929b211f8d2a6f..b7bdcf7d27b300e493fbc62c53b374205c4ebf45 100644
--- a/doc/implementation.rst
+++ b/doc/implementation.rst
@@ -307,7 +307,7 @@ Resources
---------
Finally, some of the configuration files, which sit in the ``bob/bio/*/config`` directories, are registered as *resources*.
-This means that a resource is nothing else than a short name for a registered instance of one of the tools (database, preprocessor, extractor, algorithm or grid configuration) of ``bob.bio``, which has a pre-defined set of parameters.
+A resource is nothing else than a short name for a registered instance of one of the tools (database, preprocessor, extractor, algorithm or grid configuration) of ``bob.bio`` or a python module which has a pre-defined set of parameters.
The process of registering a resource is relatively easy.
We use the SetupTools_ mechanism of registering so-called entry points in the ``setup.py`` file of the according ``bob.bio`` package.
@@ -318,6 +318,8 @@ Particularly, we use a specific list of entry points, which are:
* ``bob.bio.extractor`` to register an instance of a (derivation of a) :py:class:`bob.bio.base.extractor.Extractor`
* ``bob.bio.algorithm`` to register an instance of a (derivation of a) :py:class:`bob.bio.base.algorithm.Algorithm`
* ``bob.bio.grid`` to register an instance of the :py:class:`bob.bio.base.grid.Grid`
+* ``bob.bio.config`` to register a Python module that contains the values of
+ resources and parameters to use for an experiment
For each of the tools, several resources are defined, which you can list with the ``./bin/resources.py`` command line.
diff --git a/setup.py b/setup.py
index f10b8155a0fcb8c951a4fcd5cc398304436d5249..9a893c860ac49880f1114466419ed1d1a7259248 100644
--- a/setup.py
+++ b/setup.py
@@ -83,6 +83,11 @@ setup(
'fuse_scores.py = bob.bio.base.script.fuse_scores:main',
],
+ 'bob.bio.config': [
+ 'dummy = bob.bio.base.test.dummy.config', # for test purposes only
+ 'dummy2 = bob.bio.base.test.dummy.config2', # for test purposes only
+ ],
+
'bob.bio.database': [
'dummy = bob.bio.base.test.dummy.database:database', # for test purposes only
],