From 0414a71b5d5dd0734f8c2d01687281ca3be484be Mon Sep 17 00:00:00 2001 From: Amir MOHAMMADI Date: Sun, 11 Mar 2018 11:43:53 +0100 Subject: [PATCH 1/5] Add support for listing annotator configs --- bob/bio/base/script/resources.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/bob/bio/base/script/resources.py b/bob/bio/base/script/resources.py index 15fc021..c68d6f2 100644 --- a/bob/bio/base/script/resources.py +++ b/bob/bio/base/script/resources.py @@ -9,8 +9,8 @@ def resources(command_line_parameters = None): import argparse parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("--types", '-t', nargs = '+', - choices = ('d', 'database', 'p', 'preprocessor', 'e', 'extractor', 'a', 'algorithm', 'g', 'grid', 'c', 'config'), - default = ('d', 'p', 'e', 'a', 'g', 'c'), + choices = ('d', 'database', 'p', 'preprocessor', 'e', 'extractor', 'a', 'algorithm', 'g', 'grid', 'c', 'config', 'an', 'annotator'), + default = ('d', 'p', 'e', 'a', 'g', 'c', 'an'), help = "Select the resource types that should be listed.") parser.add_argument("--details", '-d', action='store_true', help = "Prints the complete configuration for all resources") @@ -51,6 +51,10 @@ def resources(command_line_parameters = None): print ("\nList of registered configurations:") print (bob.bio.base.list_resources('config', **kwargs)) + if 'an' in args.types or 'annotator' in args.types: + print ("\nList of registered annotators:") + print (bob.bio.base.list_resources('annotator', **kwargs)) + print() def databases(command_line_parameters = None): -- GitLab From 54a71013612586b653accc56e4b617bf4c921567 Mon Sep 17 00:00:00 2001 From: Amir MOHAMMADI Date: Sun, 11 Mar 2018 12:18:28 +0100 Subject: [PATCH 2/5] Annotations could be empty --- bob/bio/base/annotator/FailSafe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bob/bio/base/annotator/FailSafe.py b/bob/bio/base/annotator/FailSafe.py index 4ba32ad..88af4dd 100644 --- a/bob/bio/base/annotator/FailSafe.py +++ b/bob/bio/base/annotator/FailSafe.py @@ -38,7 +38,7 @@ class FailSafe(Annotator): if not annotations: logger.debug( "Annotator `%s' returned empty annotations.", annotator) - kwargs['annotations'].update(annotations) + kwargs['annotations'].update(annotations or {}) # check if we have all the required annotations if all(key in kwargs['annotations'] for key in self.required_keys): break -- GitLab From 23d69e5157ec72c56e5c7497ffaf8fc52e44de3a Mon Sep 17 00:00:00 2001 From: Amir MOHAMMADI Date: Sun, 11 Mar 2018 12:20:09 +0100 Subject: [PATCH 3/5] Replace directories --- bob/bio/base/script/annotate.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/bob/bio/base/script/annotate.py b/bob/bio/base/script/annotate.py index e9dc135..2918590 100644 --- a/bob/bio/base/script/annotate.py +++ b/bob/bio/base/script/annotate.py @@ -3,7 +3,7 @@ import logging import json import click -from os.path import dirname, isfile +from os.path import dirname, isfile, expanduser from bob.extension.scripts.click_helper import ( verbosity_option, ConfigCommand, ResourceOption) from bob.io.base import create_directories_safe @@ -19,9 +19,12 @@ logger = logging.getLogger(__name__) entry_point_group='bob.bio.annotator') @click.option('--output-dir', '-o', required=True, cls=ResourceOption) @click.option('--force', '-f', is_flag=True, cls=ResourceOption) -@click.option('--array', type=click.INT, default=1,) +@click.option('--array', type=click.INT, default=1, cls=ResourceOption) +@click.option('--database-directories-file', cls=ResourceOption, + default=expanduser('~/.bob_bio_databases.txt')) @verbosity_option(cls=ResourceOption) -def annotate(database, annotator, output_dir, force, array, **kwargs): +def annotate(database, annotator, output_dir, force, array, + database_directories_file, **kwargs): """Annotates a database. The annotations are written in text file (json) format which can be read back using :any:`bob.db.base.read_annotation_file` (annotation_type='json') @@ -60,8 +63,12 @@ def annotate(database, annotator, output_dir, force, array, **kwargs): logger.debug('force: %s', force) logger.debug('output_dir: %s', output_dir) logger.debug('array: %s', array) + logger.debug('database_directories_file: %s', database_directories_file) logger.debug('kwargs: %s', kwargs) + # Some databases need their original_directory to be replaced + database.replace_directories(database_directories_file) + biofiles = database.objects(groups=None, protocol=database.protocol) biofiles = sorted(biofiles) -- GitLab From 53962a8beb946b97b0348711e671c9364a2e3c7a Mon Sep 17 00:00:00 2001 From: Amir MOHAMMADI Date: Sun, 11 Mar 2018 12:20:32 +0100 Subject: [PATCH 4/5] Account for unicode in Python 2 --- bob/bio/base/database/database.py | 4 ++-- bob/bio/base/database/filelist/query.py | 2 +- bob/bio/base/grid.py | 4 ++-- bob/bio/base/tools/command_line.py | 7 ++++--- bob/bio/base/utils/__init__.py | 2 +- bob/bio/base/utils/resources.py | 3 ++- conda/meta.yaml | 2 ++ requirements.txt | 1 + 8 files changed, 15 insertions(+), 10 deletions(-) diff --git a/bob/bio/base/database/database.py b/bob/bio/base/database/database.py index d20f76f..97d73e4 100644 --- a/bob/bio/base/database/database.py +++ b/bob/bio/base/database/database.py @@ -86,7 +86,7 @@ class BioDatabase(six.with_metaclass(abc.ABCMeta, bob.db.base.FileDatabase)): **kwargs ): - assert isinstance(name, str) + assert isinstance(name, six.string_types) super(BioDatabase, self).__init__( original_directory=original_directory, @@ -180,7 +180,7 @@ class BioDatabase(six.with_metaclass(abc.ABCMeta, bob.db.base.FileDatabase)): """ if replacements is None: return - if isinstance(replacements, str): + if isinstance(replacements, six.string_types): if not os.path.exists(replacements): return # Open the database replacement file and reads its content diff --git a/bob/bio/base/database/filelist/query.py b/bob/bio/base/database/filelist/query.py index 4bb6487..1f55662 100644 --- a/bob/bio/base/database/filelist/query.py +++ b/bob/bio/base/database/filelist/query.py @@ -823,7 +823,7 @@ class FileListBioDatabase(ZTBioDatabase): The full path of the original data file. """ - if isinstance(self.original_extension, str): + if isinstance(self.original_extension, six.string_types): # extract file name file_name = file.make_path(self.original_directory, self.original_extension) if check_existence and os.path.exists(file_name): diff --git a/bob/bio/base/grid.py b/bob/bio/base/grid.py index 6390661..60aa012 100644 --- a/bob/bio/base/grid.py +++ b/bob/bio/base/grid.py @@ -2,7 +2,7 @@ # vim: set fileencoding=utf-8 : # @author: Manuel Guenther # @date: Tue Oct 2 12:12:39 CEST 2012 - +import six PREDEFINED_QUEUES = { 'default' : {}, @@ -130,7 +130,7 @@ class Grid (object): """ if self.is_local(): return {} - if isinstance(params, str) and params in PREDEFINED_QUEUES: + if isinstance(params, six.string_types) and params in PREDEFINED_QUEUES: return PREDEFINED_QUEUES[params] elif isinstance(params, dict): return params diff --git a/bob/bio/base/tools/command_line.py b/bob/bio/base/tools/command_line.py index e766070..d8bb712 100644 --- a/bob/bio/base/tools/command_line.py +++ b/bob/bio/base/tools/command_line.py @@ -5,6 +5,7 @@ import argparse import os import socket import sys +import six import bob.core import bob.extension @@ -242,7 +243,7 @@ def take_from_config_or_command_line(args, config, keyword, default, required=Tr elif config is not None and hasattr(config, keyword): val = getattr(config, keyword) - if isinstance(val, str) and is_resource: + if isinstance(val, six.string_types) and is_resource: val = utils.load_resource(val, keyword, imports=args.imports, package_prefix=args.package_prefix, preferred_package=args.preferred_package) setattr(args, keyword, val) @@ -294,10 +295,10 @@ def parse_config_file(parsers, args, args_dictionary, keywords, skips): take_from_config_or_command_line(args, config, "sub_directory", parser.get_default("sub_directory"), is_resource=False) - + take_from_config_or_command_line(args, config, "env", parser.get_default("env"), is_resource=False) - + skip_keywords = tuple(['skip_' + k.replace('-', '_') for k in skips]) for keyword in keywords + skip_keywords + ('execute_only',): diff --git a/bob/bio/base/utils/__init__.py b/bob/bio/base/utils/__init__.py index 75737bd..6738ad5 100644 --- a/bob/bio/base/utils/__init__.py +++ b/bob/bio/base/utils/__init__.py @@ -56,7 +56,7 @@ def selected_elements(list_of_elements, desired_number_of_elements = None): def pretty_print(obj, kwargs): """Returns a pretty-print of the parameters to the constructor of a class, which should be able to copy-paste on the command line to create the object (with few exceptions).""" - return "%s(%s)" % (str(obj.__class__), ", ".join(["%s='%s'" % (key,value) if isinstance(value, str) else "%s=%s" % (key, value) for key,value in kwargs.items() if value is not None])) + return "%s(%s)" % (str(obj.__class__), ", ".join(["%s='%s'" % (key,value) if isinstance(value, six.string_types) else "%s=%s" % (key, value) for key,value in kwargs.items() if value is not None])) def is_argument_available(argument, method): diff --git a/bob/bio/base/utils/resources.py b/bob/bio/base/utils/resources.py index 5f6f4c1..fa5e3c4 100644 --- a/bob/bio/base/utils/resources.py +++ b/bob/bio/base/utils/resources.py @@ -14,6 +14,7 @@ if sys.version_info[0] == 2: from string import letters as ascii_letters else: from string import ascii_letters +import six import logging logger = logging.getLogger("bob.bio.base") @@ -173,7 +174,7 @@ def load_resource(resource, keyword, imports = ['bob.bio.base'], package_prefix= for i in imports: exec ("import %s"%i) # now, evaluate the resource (re-evaluate if the resource is still a string) - while isinstance(resource, str): + while isinstance(resource, six.string_types): resource = eval(resource) return resource diff --git a/conda/meta.yaml b/conda/meta.yaml index 126977b..fa9ebae 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -44,10 +44,12 @@ requirements: - bob.measure - bob.sp - scipy {{ scipy }} + - six {{ six }} run: - python - setuptools - scipy + - six test: imports: diff --git a/requirements.txt b/requirements.txt index bc8552c..774aac2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,3 +14,4 @@ click-plugins numpy scipy setuptools +six -- GitLab From 874aa4d13e0a3932d5aafe9bafe8f5d5174678b4 Mon Sep 17 00:00:00 2001 From: Amir MOHAMMADI Date: Sun, 11 Mar 2018 12:24:21 +0100 Subject: [PATCH 5/5] Mention how you can find annotators --- doc/annotations.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/annotations.rst b/doc/annotations.rst index 851a2f2..7dfe74e 100644 --- a/doc/annotations.rst +++ b/doc/annotations.rst @@ -27,3 +27,10 @@ The script can also be run in parallel using :ref:`gridtk`: $ jman submit --array 64 -- bob bio annotate /path/to/config.py --array 64 The number that is given to the ``--array`` options should match. + +You can find the list of readily available annotator configurations using the +``resources.py`` command: + +.. code-block:: sh + + $ resources.py --types annotator -- GitLab