Commit 1f560d43 authored by Amir MOHAMMADI's avatar Amir MOHAMMADI

Add a common_keyword to config.load

Fixes #64
parent 769a2a94
Pipeline #21790 failed with stage
in 17 minutes and 31 seconds
......@@ -68,7 +68,15 @@ def _get_module_filename(module_name):
return loader.filename
def _resolve_entry_point_or_modules(paths, entry_point_group):
def _object_name(path, common_name):
path = path.rsplit(':', 1)
name = path[1] if len(path) > 1 else common_name
path = path[0]
return path, name
def _resolve_entry_point_or_modules(paths, entry_point_group,
"""Resolves a mixture of paths, entry point names, and module names to just
paths. For example paths can be:
``paths = ['/tmp/', 'config1', 'bob.extension.config2']``.
......@@ -80,6 +88,9 @@ def _resolve_entry_point_or_modules(paths, entry_point_group):
names, or are module names.
entry_point_group : str
The entry point group name to search in entry points.
common_name : None or str
It will be used as a default name for object names. See the
common_keyword parameter from :any:`load`.
......@@ -90,20 +101,24 @@ def _resolve_entry_point_or_modules(paths, entry_point_group):
paths : [str]
The resolved paths pointing to existing files.
names : [str]
module_names : [str]
The valid python module names to bind each of the files to
object_names : [str]
The name of objects that are supposed to be picked from paths.
entries = { e for e in
files = []
names = []
module_names = []
object_names = []
for i, path in enumerate(paths):
old_path = path
module_name = 'user_config' # fixed module name for files with full paths
path, object_name = _object_name(path, common_name)
# if it already points to a file
if isfile(path):
......@@ -111,7 +126,9 @@ def _resolve_entry_point_or_modules(paths, entry_point_group):
# If it is an entry point name, collect path and module name
elif path in entries:
module_name = entries[path].module_name
entry = entries[path]
module_name = entry.module_name
object_name = entry.attrs[0] if entry.attrs else common_name
path = _get_module_filename(module_name)
if not isfile(path):
raise ValueError(
......@@ -131,12 +148,13 @@ def _resolve_entry_point_or_modules(paths, entry_point_group):
old_path, path, entry_point_group or ''))
return files, names
return files, module_names, object_names
def load(paths, context=None, entry_point_group=None):
def load(paths, context=None, entry_point_group=None, common_keyword=None):
'''Loads a set of configuration files, in sequence
This method will load one or more configuration files. Every time a
......@@ -157,17 +175,37 @@ def load(paths, context=None, entry_point_group=None):
entry_point_group : :py:class:`str`, optional
If provided, it will treat non-existing file paths as entry point names
under the ``entry_point_group`` name.
common_keyword : None or str
If provided, will look for the common_keyword variable inside the loaded
files. Paths ending with `some_path:variable_name` can override the
common_keyword. The entry_point_group must provided as well
common_keyword is not None.
mod : :any:`module`
mod : :any:`module` or object
A module representing the resolved context, after loading the provided
modules and resolving all variables.
modules and resolving all variables. If common_keyword is given, the
object with the common_keyword name (or the name provided by user) is
returned instead of the module.
If common_keyword is given but the object does not exist in the paths.
If common_keyword is given but entry_point_group is not given.
if common_keyword and not entry_point_group:
raise ValueError(
"entry_point_group must be provided when using the "
"common_keyword parameter.")
# resolve entry points to paths
if entry_point_group is not None:
paths, names = _resolve_entry_point_or_modules(paths, entry_point_group)
paths, names, object_names = _resolve_entry_point_or_modules(
paths, entry_point_group, common_keyword)
names = len(paths) * ['user_config']
......@@ -192,7 +230,18 @@ def load(paths, context=None, entry_point_group=None):
ctxt = _load_context(k, mod)
return mod
if not common_keyword:
return mod
# We pick the last object_name here. Normally users should provide just one
# path when enabling the common_keyword parameter.
common_keyword = object_names[-1]
if not hasattr(mod, common_keyword):
raise ImportError(
"The desired variable '%s' does not exist in any of "
"your configuration files: %s" % (common_keyword, ', '.join(paths)))
return getattr(mod, common_keyword)
def mod_to_context(mod):
......@@ -14,12 +14,12 @@ yyy : callable
# database = None
'''Required parameter: database (--database, -d)
bla bla bla Can be a ``bob.extension.test_config_load`` entry point, a module name, or a path to a Python file which contains a variable named `database`.
bla bla bla Can be a ``bob.extension.test_dump_config`` entry point, a module name, or a path to a Python file which contains a variable named `database`.
Registered entries are: ['basic_config', 'resource_config', 'subpackage_config']'''
# annotator = None
'''Required parameter: annotator (--annotator, -a)
bli bli bli Can be a ``bob.extension.test_config_load`` entry point, a module name, or a path to a Python file which contains a variable named `annotator`.
bli bli bli Can be a ``bob.extension.test_dump_config`` entry point, a module name, or a path to a Python file which contains a variable named `annotator`.
Registered entries are: ['basic_config', 'resource_config', 'subpackage_config']'''
# output_dir = None
......@@ -339,10 +339,10 @@ class ResourceOption(click.Option):
self).full_process_value(ctx, value)
if self.entry_point_group is not None:
keyword = self.entry_point_group.split('.')[-1]
common_keyword = self.entry_point_group.split('.')[-1]
while isinstance(value, basestring):
value = load([value], entry_point_group=self.entry_point_group)
value = getattr(value, keyword)
value = load([value], entry_point_group=self.entry_point_group,
return value
......@@ -216,11 +216,11 @@ def test_config_dump2():
def cli():
@cli.command(cls=ConfigCommand, entry_point_group='bob.extension.test_config_load')
@cli.command(cls=ConfigCommand, entry_point_group='bob.extension.test_dump_config')
@click.option('--database', '-d', required=True, cls=ResourceOption,
entry_point_group='bob.extension.test_config_load', help="bla bla bla")
entry_point_group='bob.extension.test_dump_config', help="bla bla bla")
@click.option('--annotator', '-a', required=True, cls=ResourceOption,
entry_point_group='bob.extension.test_config_load', help="bli bli bli")
entry_point_group='bob.extension.test_dump_config', help="bli bli bli")
@click.option('--output-dir', '-o', required=True, cls=ResourceOption,
help="blo blo blo")
@click.option('--force', '-f', is_flag=True, cls=ResourceOption,
......@@ -58,3 +58,27 @@ def test_entry_point_configs():
assert hasattr(c, "a") and c.a == 1
assert hasattr(c, "b") and c.b == 3
assert hasattr(c, "rc")
def test_load_resource():
for p, ref in [
(os.path.join(path, ''), 1),
(os.path.join(path, ''), 1),
(os.path.join(path, ''), 2),
('resource1', 1),
('resource2', 2),
('', 1),
('', 1),
('', 2),
c = load([p], entry_point_group='bob.extension.test_config_load',
assert c == ref, c
assert False, 'The code above should have raised an ImportError'
except ImportError:
......@@ -123,6 +123,32 @@ to provide the group name of the entry points:
b = 6
Resource Loading
The function :py:func:`bob.extension.config.load` can also only return
variables from paths. To do this, you need provide a common_keyword. For
example, given the following config file:
.. literalinclude:: ../bob/extension/data/
:caption: "" with two variables inside
:language: python
The loaded value can be either 1 or 2:
.. doctest:: load_resource
>>> group = 'bob.extension.test_config_load' # the group name of entry points
>>> common_keyword = 'test_config_load' # the common variable name
>>> value = load([''], entry_point_group=group, common_keyword=common_keyword)
>>> value == 1
>>> value = load([''], entry_point_group=group, common_keyword=common_keyword)
>>> value == 2
.. _bob.extension.processors:
Stacked Processing
......@@ -37,6 +37,13 @@ setup(
'basic_config =',
'resource_config =',
'subpackage_config =',
'resource1 =',
'resource2 =',
'bob.extension.test_dump_config': [
'basic_config =',
'resource_config =',
'subpackage_config =',
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment