Fixes and tests for the config-file feature

bob/bio/base/test/test_config_file.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+'''Tests for the configuration-file command line options'''
+
+import os
+import shutil
+import tempfile
+
+from ..script.verify import parse_arguments
+
+
+def tmp_file(contents):
+  '''Generates a temporary configuration file with the contents on the input'''
+
+  retval = tempfile.NamedTemporaryFile()
+  retval.write('\n'.join(contents) + '\n')
+  retval.flush()
+  return retval
+
+
+def check_parameters(args_file, args_cmdline):
+  '''Checks parameters generated from a configuration file or command-line
+  are as similar they can be'''
+
+  from bob.bio.base.test.dummy.database import DummyDatabase
+  assert isinstance(args_file.database, DummyDatabase)
+  assert isinstance(args_cmdline.database, DummyDatabase)
+  from bob.bio.base.test.dummy.preprocessor import DummyPreprocessor
+  assert isinstance(args_file.preprocessor, DummyPreprocessor)
+  assert isinstance(args_cmdline.preprocessor, DummyPreprocessor)
+  from bob.bio.base.test.dummy.extractor import DummyExtractor
+  assert isinstance(args_file.extractor, DummyExtractor)
+  assert isinstance(args_cmdline.extractor, DummyExtractor)
+  from bob.bio.base.test.dummy.algorithm import DummyAlgorithm
+  assert isinstance(args_file.algorithm, DummyAlgorithm)
+  assert isinstance(args_cmdline.algorithm, DummyAlgorithm)
+
+  # elements checked otherwise or not comparable between the two settings
+  skip_check = (
+      'configuration_file',
+      'imports',
+      'database',
+      'preprocessor',
+      'extractor',
+      'algorithm',
+      )
+
+  for attr in [k for k in dir(args_file) if not k.startswith('_')]:
+    if attr in skip_check: continue
+    assert hasattr(args_cmdline, attr)
+    attr_cmdline = getattr(args_cmdline, attr)
+    attr_file = getattr(args_file, attr)
+    if (isinstance(attr_file, (bool, str, int, list))) or (attr_file is None):
+      assert attr_cmdline == attr_file, '(%s) %r != %r' % \
+          (attr, attr_cmdline, attr_file)
+    else:
+      print '(%s) %r == %r?' % (attr, attr_cmdline, attr_file),
+      print attr_cmdline == attr_file, type(attr_cmdline)
+
+
+def test_basic():
+
+  test_dir = None
+  test_config_file = None
+  try:
+    test_dir = tempfile.mkdtemp(prefix='bobtest_')
+    test_config_file = tmp_file([
+      'from bob.bio.base.test.dummy.database import database',
+      'from bob.bio.base.test.dummy.preprocessor import preprocessor',
+      'from bob.bio.base.test.dummy.extractor import extractor',
+      'from bob.bio.base.test.dummy.algorithm import algorithm',
+      'zt_norm = True',
+      'verbose = 1',
+      'sub_directory = "test_config"',
+      'temp_directory = "%s"' % test_dir,
+      'result_directory = "%s"' % test_dir,
+      ])
+
+    args = parse_arguments(['-c', test_config_file.name])
+
+    assert args.zt_norm is True
+    assert args.verbose == 1
+    assert args.sub_directory.endswith('test_config')
+    assert args.temp_directory.startswith(test_dir)
+    assert args.result_directory.startswith(test_dir)
+    assert args.allow_missing_files is False
+
+    from bob.bio.base.test.dummy.database import DummyDatabase
+    assert isinstance(args.database, DummyDatabase)
+    from bob.bio.base.test.dummy.preprocessor import DummyPreprocessor
+    assert isinstance(args.preprocessor, DummyPreprocessor)
+    from bob.bio.base.test.dummy.extractor import DummyExtractor
+    assert isinstance(args.extractor, DummyExtractor)
+    from bob.bio.base.test.dummy.algorithm import DummyAlgorithm
+    assert isinstance(args.algorithm, DummyAlgorithm)
+
+  finally:
+    if test_dir: shutil.rmtree(test_dir)
+    if test_config_file: del test_config_file
+
+
+def test_compare_to_cmdline_basic():
+
+  test_dir = None
+  test_config_file = None
+  try:
+    test_dir = tempfile.mkdtemp(prefix='bobtest_')
+    test_config_file = tmp_file([
+      'from bob.bio.base.test.dummy.database import database',
+      'from bob.bio.base.test.dummy.preprocessor import preprocessor',
+      'from bob.bio.base.test.dummy.extractor import extractor',
+      'from bob.bio.base.test.dummy.algorithm import algorithm',
+      'zt_norm = True',
+      'verbose = 1',
+      'sub_directory = "test_config"',
+      'temp_directory = "%s"' % test_dir,
+      'result_directory = "%s"' % test_dir,
+      ])
+
+    args_file = parse_arguments(['-c', test_config_file.name])
+
+    # now do the same with command-line arguments, ensure result is equal
+    args_cmdline = parse_arguments([
+      '-d', 'bob.bio.base.test.dummy.database.DummyDatabase()',
+      '-p', 'bob.bio.base.test.dummy.preprocessor.DummyPreprocessor()',
+      '-e', 'bob.bio.base.test.dummy.extractor.DummyExtractor()',
+      '-a', 'bob.bio.base.test.dummy.algorithm.DummyAlgorithm()',
+      '--zt-norm',
+      '-vs', 'test_config',
+      '--temp-directory', test_dir,
+      '--result-directory', test_dir,
+      '--imports', 'bob.bio.base.test.dummy',
+      ])
+
+    check_parameters(args_file, args_cmdline)
+
+  finally:
+    if test_dir: shutil.rmtree(test_dir)
+    if test_config_file: del test_config_file
+
+
+def test_compare_to_cmdline_resources():
+
+  test_dir = None
+  test_config_file = None
+  try:
+    test_dir = tempfile.mkdtemp(prefix='bobtest_')
+    test_config_file = tmp_file([
+      'database = "dummy"',
+      'preprocessor = "dummy"',
+      'extractor = "dummy"',
+      'algorithm = "dummy"',
+      'zt_norm = True',
+      'allow_missing_files = True',
+      'verbose = 1',
+      'sub_directory = "test_config"',
+      'temp_directory = "%s"' % test_dir,
+      'result_directory = "%s"' % test_dir,
+      'preferred_package = "bob.bio.base"',
+      ])
+
+    args_file = parse_arguments(['-c', test_config_file.name])
+
+    # now do the same with command-line arguments, ensure result is equal
+    args_cmdline = parse_arguments([
+      '-d', 'dummy',
+      '-p', 'dummy',
+      '-e', 'dummy',
+      '-a', 'dummy',
+      '--zt-norm',
+      '--allow-missing-files',
+      '-vs', 'test_config',
+      '--temp-directory', test_dir,
+      '--result-directory', test_dir,
+      '--preferred-package', 'bob.bio.base',
+      ])
+
+    check_parameters(args_file, args_cmdline)
+
+  finally:
+    if test_dir: shutil.rmtree(test_dir)
+    if test_config_file: del test_config_file
+
+
+def test_compare_to_cmdline_skip():
+
+  test_dir = None
+  test_config_file = None
+  try:
+    test_dir = tempfile.mkdtemp(prefix='bobtest_')
+    test_config_file = tmp_file([
+      'database = "dummy"',
+      'preprocessor = "dummy"',
+      'extractor = "dummy"',
+      'skip_preprocessing = True',
+      'skip_extraction = True',
+      'algorithm = "dummy"',
+      'zt_norm = True',
+      'allow_missing_files = True',
+      'verbose = 1',
+      'sub_directory = "test_config"',
+      'temp_directory = "%s"' % test_dir,
+      'result_directory = "%s"' % test_dir,
+      'preferred_package = "bob.bio.base"',
+      ])
+
+    args_file = parse_arguments(['-c', test_config_file.name])
+
+    # now do the same with command-line arguments, ensure result is equal
+    args_cmdline = parse_arguments([
+      '-d', 'dummy',
+      '-p', 'dummy',
+      '-e', 'dummy',
+      '-a', 'dummy',
+      '--zt-norm',
+      '--allow-missing-files',
+      '--skip-preprocessing',
+      '--skip-extraction',
+      '-vs', 'test_config',
+      '--temp-directory', test_dir,
+      '--result-directory', test_dir,
+      '--preferred-package', 'bob.bio.base',
+      ])
+
+    check_parameters(args_file, args_cmdline)
+
+  finally:
+    if test_dir: shutil.rmtree(test_dir)
+    if test_config_file: del test_config_file

bob/bio/base/tools/command_line.py

@@ -63,7 +63,7 @@ def command_line_parser(description=__doc__, exclude_resources_from=[]):
       help = 'If one of your configuration files is an actual command, please specify the lists of required libraries (imports) to execute this command')
   config_group.add_argument('-W', '--preferred-package', metavar = 'LIB',
       help = 'If resources with identical names are defined in several packages, prefer the one from the given package')
-  config_group.add_argument('-s', '--sub-directory', metavar = 'DIR', required = True,
+  config_group.add_argument('-s', '--sub-directory', metavar = 'DIR',
       help = 'The sub-directory where the files of the current experiment should be stored. Please specify a directory name with a name describing your experiment')
   config_group.add_argument('--groups', metavar = 'GROUP', nargs = '+', default = ['dev'],
       help = "The groups (i.e., 'dev', 'eval') for which the models and scores should be generated; by default, only the 'dev' group is evaluated")
@@ -155,21 +155,23 @@ def command_line_parser(description=__doc__, exclude_resources_from=[]):
   }
 
 
-def _take_from_config_or_command_line(args, config, keyword, required=True, is_resource=True):
-  if getattr(args, keyword) is not None:
+def _take_from_config_or_command_line(args, config, keyword, default, required=True, is_resource=True):
+
+  if getattr(args, keyword) != default:
     if is_resource:
       setattr(args, keyword, utils.load_resource(' '.join(getattr(args, keyword)), keyword, imports = args.imports, preferred_package = args.preferred_package))
+
   elif config is not None and hasattr(config, keyword):
+
     val = getattr(config, keyword)
     if isinstance(val, str) and is_resource:
       val = utils.load_resource(val, keyword, imports = args.imports, preferred_package = args.preferred_package)
     setattr(args, keyword, val)
+
   elif required:
     raise ValueError("Please specify a %s either on command line (via --%s) or in the configuration file (via --configuration-file)" %(keyword, keyword))
 
 
-
-
 def initialize(parsers, command_line_parameters = None, skips = []):
   """initialize(parsers, command_line_parameters = None, skips = []) -> args
 
@@ -211,21 +213,28 @@ def initialize(parsers, command_line_parameters = None, skips = []):
     skip_group.add_argument('-o', '--execute-only', nargs = '+', choices = skips, help = 'If specified, executes only the given parts of the tool chain.')
 
   # parse the arguments
-  args = parsers['main'].parse_args(command_line_parameters)
+  parser = parsers['main']
+  args = parser.parse_args(command_line_parameters)
 
   # first, read the configuration file and set everything from the config file to the args -- as long as not overwritten on command line
   config = utils.read_config_file(args.configuration_file) if args.configuration_file is not None else None
   for keyword in ("database", "preprocessor", "extractor", "algorithm"):
-    _take_from_config_or_command_line(args, config, keyword)
+    _take_from_config_or_command_line(args, config, keyword,
+        parser.get_default(keyword))
+
+  _take_from_config_or_command_line(args, config, "grid",
+      parser.get_default(keyword), required=False)
+
+  _take_from_config_or_command_line(args, config, "sub_directory",
+      parser.get_default(keyword), is_resource=False)
 
-  _take_from_config_or_command_line(args, config, "grid", required=False)
+  skip_keywords = tuple(['skip_' + k.replace('-', '_') for k in skips])
 
   for keyword in (
       "protocol",
       "groups",
       "parallel",
       "preferred_package",
-      "sub_directory",
       "temp_directory",
       "result_directory",
       "extractor_file",
@@ -255,18 +264,9 @@ def initialize(parsers, command_line_parameters = None, skips = []):
       "zt_norm",
       "allow_missing_files",
       "env",
-      "skip_preprocessing",
-      "skip_extractor_training",
-      "skip_extraction",
-      "skip_projector_training",
-      "skip_projection",
-      "skip_enroller_training",
-      "skip_enrollment",
-      "skip_score_computation",
-      "skip_concatenation",
-      "skip_calibration",
-      ):
-    _take_from_config_or_command_line(args, config, keyword, required=False, is_resource=False)
+      ) + skip_keywords:
+    _take_from_config_or_command_line(args, config, keyword,
+        parser.get_default(keyword), required=False, is_resource=False)
 
   # evaluate skips
   if skips is not None and args.execute_only is not None: