Commit aa8832c1 authored by Amir MOHAMMADI's avatar Amir MOHAMMADI

[click_helper][ResourceOption] Improvements

- Use self.name as the attribute_name in loading of config files
- Improve docs
parent e7292440
Pipeline #41005 passed with stage
in 6 minutes and 32 seconds
...@@ -6,10 +6,6 @@ import logging ...@@ -6,10 +6,6 @@ import logging
import traceback import traceback
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
try:
basestring
except NameError:
basestring = str
def bool_option(name, short_name, desc, dflt=False, **kwargs): def bool_option(name, short_name, desc, dflt=False, **kwargs):
...@@ -179,7 +175,7 @@ def verbosity_option(**kwargs): ...@@ -179,7 +175,7 @@ def verbosity_option(**kwargs):
class ConfigCommand(click.Command): class ConfigCommand(click.Command):
"""A click.Command that can take options both form command line options and """A click.Command that can take options both form command line options and
configuration files. In order to use this class, you have to use the configuration files. In order to use this class, you **have to** use the
:any:`ResourceOption` class also. :any:`ResourceOption` class also.
Attributes Attributes
...@@ -299,18 +295,30 @@ file.""".format( ...@@ -299,18 +295,30 @@ file.""".format(
class ResourceOption(click.Option): class ResourceOption(click.Option):
"""A click.Option that automatically loads resources. """An extended click.Option that automatically loads resources from config
It uses :any:`load` to convert provided strings in the command line into Python files.
objects.
It also integrates with the ConfigCommand class. This class comes with two different functionalities that are independent and
could be combined:
1. If used in commands that are inherited from :any:`ConfigCommand`, it will
lookup inside the config files (that are provided as argument to the
command) to resolve its value. Values given explicitly in the command
line take precedence.
2. If `entry_point_group` is provided, it will treat values given to it (by
any means) as resources to be loaded. Loading is done using :any:`load`.
See :ref:`bob.extension.config.resource` for more information. The final
value cannot be a string.
You may use this class in three ways: You may use this class in three ways:
1. Using this class without using :any:`ConfigCommand` AND providing 1. Using this class (without using :any:`ConfigCommand`) AND (providing
`entry_point_group`. `entry_point_group`).
2. Using this class with :any:`ConfigCommand` AND providing `entry_point_group`. 2. Using this class (with :any:`ConfigCommand`) AND (providing
3. Using this class with :any:`ConfigCommand` AND without providing `entry_point_group`).
`entry_point_group`. 3. Using this class (with :any:`ConfigCommand`) AND (without providing
`entry_point_group`).
Using this class without :any:`ConfigCommand` and without providing Using this class without :any:`ConfigCommand` and without providing
`entry_point_group` does nothing and is not allowed. `entry_point_group` does nothing and is not allowed.
...@@ -381,11 +389,7 @@ class ResourceOption(click.Option): ...@@ -381,11 +389,7 @@ class ResourceOption(click.Option):
# true. # true.
if hasattr(ctx, "config_context"): if hasattr(ctx, "config_context"):
value = ctx.config_context.get(self.name) value = ctx.config_context.get(self.name)
else:
logger.debug(
"config_context attribute not found in context. Did you mean to "
"use the ConfigCommand class with this ResourceOption class?"
)
# if not from config files, lookup the environment variables # if not from config files, lookup the environment variables
if value is None: if value is None:
value = self.value_from_envvar(ctx) value = self.value_from_envvar(ctx)
...@@ -399,12 +403,11 @@ class ResourceOption(click.Option): ...@@ -399,12 +403,11 @@ class ResourceOption(click.Option):
# if the value is a string and an entry_point_group is provided, load it # if the value is a string and an entry_point_group is provided, load it
if self.entry_point_group is not None: if self.entry_point_group is not None:
attribute_name = self.entry_point_group.split(".")[-1] while isinstance(value, str):
while isinstance(value, basestring):
value = load( value = load(
[value], [value],
entry_point_group=self.entry_point_group, entry_point_group=self.entry_point_group,
attribute_name=attribute_name, attribute_name=self.name,
) )
return value return value
......
This diff is collapsed.
...@@ -63,22 +63,22 @@ def test_entry_point_configs(): ...@@ -63,22 +63,22 @@ def test_entry_point_configs():
def test_load_resource(): def test_load_resource():
for p, ref in [ for p, ref in [
(os.path.join(path, 'resource_config2.py'), 1), (os.path.join(path, 'resource_config2.py'), 1),
(os.path.join(path, 'resource_config2.py:test_config_load'), 1), (os.path.join(path, 'resource_config2.py:a'), 1),
(os.path.join(path, 'resource_config2.py:b'), 2), (os.path.join(path, 'resource_config2.py:b'), 2),
('resource1', 1), ('resource1', 1),
('resource2', 2), ('resource2', 2),
('bob.extension.data.resource_config2', 1), ('bob.extension.data.resource_config2', 1),
('bob.extension.data.resource_config2:test_config_load', 1), ('bob.extension.data.resource_config2:a', 1),
('bob.extension.data.resource_config2:b', 2), ('bob.extension.data.resource_config2:b', 2),
]: ]:
c = load([p], entry_point_group='bob.extension.test_config_load', c = load([p], entry_point_group='bob.extension.test_config_load',
attribute_name='test_config_load') attribute_name='a')
assert c == ref, c assert c == ref, c
try: try:
load(['bob.extension.data.resource_config2:c'], load(['bob.extension.data.resource_config2:c'],
entry_point_group='bob.extension.test_config_load', entry_point_group='bob.extension.test_config_load',
attribute_name='test_config_load') attribute_name='a')
assert False, 'The code above should have raised an ImportError' assert False, 'The code above should have raised an ImportError'
except ImportError: except ImportError:
pass pass
"""A script to help annotate databases. """A script to help annotate databases.
""" """
# Avoid importing packages here! Importing packages here will slowdown your command
# line's --help option and its auto-complete feature in terminal (if enabled). Instead,
# put your imports inside the function.
import logging import logging
import click import click
from bob.extension.scripts.click_helper import ( from bob.extension.scripts.click_helper import (
...@@ -8,16 +11,13 @@ from bob.extension.scripts.click_helper import ( ...@@ -8,16 +11,13 @@ from bob.extension.scripts.click_helper import (
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
ANNOTATE_EPILOG = '''\b @click.command(entry_point_group='bob.bio.config', cls=ConfigCommand,
epilog='''\b
Examples: Examples:
$ bob bio annotate -vvv -d <database> -a <annotator> -o /tmp/annotations $ bob bio annotate -vvv -d <database> -a <annotator> -o /tmp/annotations
$ jman submit --array 64 -- bob bio annotate ... --array 64 $ jman submit --array 64 -- bob bio annotate ... --array 64
''' ''')
@click.command(entry_point_group='bob.bio.config', cls=ConfigCommand,
epilog=ANNOTATE_EPILOG)
@click.option('--database', '-d', required=True, cls=ResourceOption, @click.option('--database', '-d', required=True, cls=ResourceOption,
entry_point_group='bob.bio.database', entry_point_group='bob.bio.database',
help='''The database that you want to annotate.''') help='''The database that you want to annotate.''')
...@@ -40,3 +40,8 @@ def annotate(database, annotator, output_dir, force, array, **kwargs): ...@@ -40,3 +40,8 @@ def annotate(database, annotator, output_dir, force, array, **kwargs):
back using :any:`bob.db.base.read_annotation_file` (annotation_type='json') back using :any:`bob.db.base.read_annotation_file` (annotation_type='json')
""" """
log_parameters(logger) log_parameters(logger)
# Add imports needed for your code here:
import numpy as np
np.zeros(10)
...@@ -17,7 +17,7 @@ Python-based Configuration System ...@@ -17,7 +17,7 @@ Python-based Configuration System
--------------------------------- ---------------------------------
This package also provides a configuration system that can be used by packages This package also provides a configuration system that can be used by packages
in the |project|-echosystem to load *run-time* configuration for applications in the |project|-ecosystem to load *run-time* configuration for applications
(for package-level static variable configuration use :ref:`bob.extension.rc`). (for package-level static variable configuration use :ref:`bob.extension.rc`).
It can be used to accept complex configurations from users through It can be used to accept complex configurations from users through
command-line. command-line.
...@@ -59,7 +59,7 @@ Then, the object ``configuration`` would look like this: ...@@ -59,7 +59,7 @@ Then, the object ``configuration`` would look like this:
.. doctest:: .. doctest::
>>> print("a = %d\nb = %d"%(configuration.a, configuration.b)) >>> print(f"a = {configuration.a}\nb = {configuration.b}")
a = 1 a = 1
b = 3 b = 3
...@@ -96,9 +96,10 @@ Then, one can chain-load them like this: ...@@ -96,9 +96,10 @@ Then, one can chain-load them like this:
>>> file1 = os.path.join(path, 'basic_config.py') >>> file1 = os.path.join(path, 'basic_config.py')
>>> file2 = os.path.join(path, 'load_config.py') >>> file2 = os.path.join(path, 'load_config.py')
>>> configuration = load([file1, file2]) >>> configuration = load([file1, file2])
>>> print("a = %d \nb = %d"%(configuration.a, configuration.b)) # doctest: +NORMALIZE_WHITESPACE >>> print(f"a = {configuration.a} \nb = {configuration.b} \nc = {configuration.c}") # doctest: +NORMALIZE_WHITESPACE
a = 1 a = 1
b = 6 b = 6
c = 4
The user wanting to override the values needs to manage the overriding and the The user wanting to override the values needs to manage the overriding and the
...@@ -123,6 +124,8 @@ to provide the group name of the entry points: ...@@ -123,6 +124,8 @@ to provide the group name of the entry points:
b = 6 b = 6
.. _bob.extension.config.resource:
Resource Loading Resource Loading
================ ================
...@@ -140,73 +143,16 @@ The loaded value can be either 1 or 2: ...@@ -140,73 +143,16 @@ The loaded value can be either 1 or 2:
.. doctest:: load_resource .. doctest:: load_resource
>>> group = 'bob.extension.test_config_load' # the group name of entry points >>> group = 'bob.extension.test_config_load' # the group name of entry points
>>> attribute_name = 'test_config_load' # the common variable name >>> attribute_name = 'a' # the common variable name
>>> value = load(['bob.extension.data.resource_config2'], entry_point_group=group, attribute_name=attribute_name) >>> value = load(['bob.extension.data.resource_config2'], entry_point_group=group, attribute_name=attribute_name)
>>> value == 1 >>> value == 1
True True
>>> # attribute_name can be ovverriden using the `path:attribute_name` syntax
>>> value = load(['bob.extension.data.resource_config2:b'], entry_point_group=group, attribute_name=attribute_name) >>> value = load(['bob.extension.data.resource_config2:b'], entry_point_group=group, attribute_name=attribute_name)
>>> value == 2 >>> value == 2
True True
.. _bob.extension.processors:
Stacked Processing
------------------
:any:`bob.extension.processors.SequentialProcessor` and
:any:`bob.extension.processors.ParallelProcessor` are provided to help you
build complex processing mechanisms. You can use these processors to apply a
chain of processes on your data. For example,
:any:`bob.extension.processors.SequentialProcessor` accepts a list of callables
and applies them on the data one by one sequentially. :
.. doctest::
>>> import numpy as np; from numpy import array
>>> from functools import partial
>>> from bob.extension.processors import SequentialProcessor
>>> raw_data = np.array([[1, 2, 3], [1, 2, 3]])
>>> seq_processor = SequentialProcessor(
... [np.cast['float64'], lambda x: x / 2, partial(np.mean, axis=1)])
>>> np.allclose(seq_processor(raw_data),
... array([ 1., 1.]))
True
>>> np.all(seq_processor(raw_data) ==
... np.mean(np.cast['float64'](raw_data) / 2, axis=1))
True
:any:`bob.extension.processors.ParallelProcessor` accepts a list of callables
and applies each them on the data independently and returns all the results.
For example:
.. doctest::
>>> from bob.extension.processors import ParallelProcessor
>>> raw_data = np.array([[1, 2, 3], [1, 2, 3]])
>>> parallel_processor = ParallelProcessor(
... [np.cast['float64'], lambda x: x / 2.0])
>>> np.allclose(list(parallel_processor(raw_data)),
... [array([[ 1., 2., 3.],
... [ 1., 2., 3.]]),
... array([[ 0.5, 1. , 1.5],
... [ 0.5, 1. , 1.5]])])
True
The data may be further processed using a
:any:`bob.extension.processors.SequentialProcessor`:
.. doctest::
>>> total_processor = SequentialProcessor(
... [parallel_processor, list, partial(np.concatenate, axis=1)])
>>> np.allclose(total_processor(raw_data),
... array([[ 1. , 2. , 3. , 0.5, 1. , 1.5],
... [ 1. , 2. , 3. , 0.5, 1. , 1.5]]))
True
.. _bob.extension.cli: .. _bob.extension.cli:
Unified Command Line Mechanism Unified Command Line Mechanism
...@@ -228,11 +174,6 @@ commands by default:: ...@@ -228,11 +174,6 @@ commands by default::
config The manager for bob's global configuration. config The manager for bob's global configuration.
... ...
.. warning::
This feature is experimental and most probably will break compatibility.
If you are not willing to fix your code after changes are made here,
please do not use this feature.
This command line is implemented using click_. You can extend the commands of This command line is implemented using click_. You can extend the commands of
this script through setuptools entry points (this is implemented using this script through setuptools entry points (this is implemented using
...@@ -243,15 +184,13 @@ independently; then, advertise it as a command under bob script using the ...@@ -243,15 +184,13 @@ independently; then, advertise it as a command under bob script using the
.. note:: .. note::
If you are still not sure how this must be done, maybe you don't know how If you are still not sure how this must be done, maybe you don't know how
to use click_ yet. to use click_ and `click-plugins`_ yet.
This feature is experimental and may change and break compatibility in future.
For a best practice example, please look at how the ``bob config`` command is For a best practice example, please look at how the ``bob config`` command is
implemented: implemented:
.. literalinclude:: ../bob/extension/scripts/config.py .. literalinclude:: ../bob/extension/scripts/config.py
:caption: "bob/extension/scripts/config.py" implementation of the ``bob :caption: "bob/extension/scripts/config.py" implementation of the ``bob config`` command.
config`` command.
:language: python :language: python
...@@ -366,5 +305,63 @@ module names and maybe have them provide simple options like ``--verbose`` or ...@@ -366,5 +305,63 @@ module names and maybe have them provide simple options like ``--verbose`` or
``--force`` through the command line options. ``--force`` through the command line options.
.. _bob.extension.processors:
Stacked Processing
------------------
:any:`bob.extension.processors.SequentialProcessor` and
:any:`bob.extension.processors.ParallelProcessor` are provided to help you
build complex processing mechanisms. You can use these processors to apply a
chain of processes on your data. For example,
:any:`bob.extension.processors.SequentialProcessor` accepts a list of callables
and applies them on the data one by one sequentially. :
.. doctest::
>>> import numpy as np; from numpy import array
>>> from functools import partial
>>> from bob.extension.processors import SequentialProcessor
>>> raw_data = np.array([[1, 2, 3], [1, 2, 3]])
>>> seq_processor = SequentialProcessor(
... [np.cast['float64'], lambda x: x / 2, partial(np.mean, axis=1)])
>>> np.allclose(seq_processor(raw_data),
... array([ 1., 1.]))
True
>>> np.all(seq_processor(raw_data) ==
... np.mean(np.cast['float64'](raw_data) / 2, axis=1))
True
:any:`bob.extension.processors.ParallelProcessor` accepts a list of callables
and applies each them on the data independently and returns all the results.
For example:
.. doctest::
>>> from bob.extension.processors import ParallelProcessor
>>> raw_data = np.array([[1, 2, 3], [1, 2, 3]])
>>> parallel_processor = ParallelProcessor(
... [np.cast['float64'], lambda x: x / 2.0])
>>> np.allclose(list(parallel_processor(raw_data)),
... [array([[ 1., 2., 3.],
... [ 1., 2., 3.]]),
... array([[ 0.5, 1. , 1.5],
... [ 0.5, 1. , 1.5]])])
True
The data may be further processed using a
:any:`bob.extension.processors.SequentialProcessor`:
.. doctest::
>>> total_processor = SequentialProcessor(
... [parallel_processor, list, partial(np.concatenate, axis=1)])
>>> np.allclose(total_processor(raw_data),
... array([[ 1. , 2. , 3. , 0.5, 1. , 1.5],
... [ 1. , 2. , 3. , 0.5, 1. , 1.5]]))
True
.. include:: links.rst .. include:: links.rst
4.0.1b0 5.0.0b0
\ No newline at end of file
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment