Skip to content
Snippets Groups Projects
Commit 799c31e7 authored by André Anjos's avatar André Anjos :speech_balloon:
Browse files

Merge branch 'click-helper' into 'master' 

[click_helper][ResourceOption] Improvements

See merge request !115
parents e7292440 aa8832c1
No related branches found
No related tags found
1 merge request!115[click_helper][ResourceOption] Improvements
Pipeline #41006 passed
Stage: build
Stage: deploy
Show whitespace changes
Inline Side-by-side
bob/extension/data/resource_config2.py
+ 1
− 1
View file @ 799c31e7
test_config_load = 1 a = 1
b = 2 b = 2
bob/extension/scripts/click_helper.py
+ 25
− 22
View file @ 799c31e7
...@@ -6,10 +6,6 @@ import logging ...@@ -6,10 +6,6 @@ import logging
import traceback import traceback
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
try:
basestring
except NameError:
basestring = str
def bool_option(name, short_name, desc, dflt=False, **kwargs): def bool_option(name, short_name, desc, dflt=False, **kwargs):
...@@ -179,7 +175,7 @@ def verbosity_option(**kwargs): ...@@ -179,7 +175,7 @@ def verbosity_option(**kwargs):
class ConfigCommand(click.Command): class ConfigCommand(click.Command):
"""A click.Command that can take options both form command line options and """A click.Command that can take options both form command line options and
configuration files. In order to use this class, you have to use the configuration files. In order to use this class, you **have to** use the
:any:`ResourceOption` class also. :any:`ResourceOption` class also.
Attributes Attributes
...@@ -299,18 +295,30 @@ file.""".format( ...@@ -299,18 +295,30 @@ file.""".format(
class ResourceOption(click.Option): class ResourceOption(click.Option):
"""A click.Option that automatically loads resources. """An extended click.Option that automatically loads resources from config
It uses :any:`load` to convert provided strings in the command line into Python files.
objects.
It also integrates with the ConfigCommand class. This class comes with two different functionalities that are independent and
could be combined:
1. If used in commands that are inherited from :any:`ConfigCommand`, it will
lookup inside the config files (that are provided as argument to the
command) to resolve its value. Values given explicitly in the command
line take precedence.
2. If `entry_point_group` is provided, it will treat values given to it (by
any means) as resources to be loaded. Loading is done using :any:`load`.
See :ref:`bob.extension.config.resource` for more information. The final
value cannot be a string.
You may use this class in three ways: You may use this class in three ways:
1. Using this class without using :any:`ConfigCommand` AND providing 1. Using this class (without using :any:`ConfigCommand`) AND (providing
`entry_point_group`. `entry_point_group`).
2. Using this class with :any:`ConfigCommand` AND providing `entry_point_group`. 2. Using this class (with :any:`ConfigCommand`) AND (providing
3. Using this class with :any:`ConfigCommand` AND without providing `entry_point_group`).
`entry_point_group`. 3. Using this class (with :any:`ConfigCommand`) AND (without providing
`entry_point_group`).
Using this class without :any:`ConfigCommand` and without providing Using this class without :any:`ConfigCommand` and without providing
`entry_point_group` does nothing and is not allowed. `entry_point_group` does nothing and is not allowed.
...@@ -381,11 +389,7 @@ class ResourceOption(click.Option): ...@@ -381,11 +389,7 @@ class ResourceOption(click.Option):
# true. # true.
if hasattr(ctx, "config_context"): if hasattr(ctx, "config_context"):
value = ctx.config_context.get(self.name) value = ctx.config_context.get(self.name)
else:
logger.debug(
"config_context attribute not found in context. Did you mean to "
"use the ConfigCommand class with this ResourceOption class?"
)
# if not from config files, lookup the environment variables # if not from config files, lookup the environment variables
if value is None: if value is None:
value = self.value_from_envvar(ctx) value = self.value_from_envvar(ctx)
...@@ -399,12 +403,11 @@ class ResourceOption(click.Option): ...@@ -399,12 +403,11 @@ class ResourceOption(click.Option):
# if the value is a string and an entry_point_group is provided, load it # if the value is a string and an entry_point_group is provided, load it
if self.entry_point_group is not None: if self.entry_point_group is not None:
attribute_name = self.entry_point_group.split(".")[-1] while isinstance(value, str):
while isinstance(value, basestring):
value = load( value = load(
[value], [value],
entry_point_group=self.entry_point_group, entry_point_group=self.entry_point_group,
attribute_name=attribute_name, attribute_name=self.name,
) )
return value return value
......
bob/extension/test_click_helper.py
+ 134
− 91
View file @ 799c31e7
...@@ -3,19 +3,25 @@ import time ...@@ -3,19 +3,25 @@ import time
import pkg_resources import pkg_resources
from click.testing import CliRunner from click.testing import CliRunner
from bob.extension.scripts.click_helper import ( from bob.extension.scripts.click_helper import (
verbosity_option, bool_option, list_float_option, verbosity_option,
ConfigCommand, ResourceOption, AliasedGroup, assert_click_runner_result) bool_option,
list_float_option,
ConfigCommand,
ResourceOption,
AliasedGroup,
assert_click_runner_result,
)
def test_verbosity_option(): def test_verbosity_option():
for VERBOSITY, OPTIONS in zip([0, 1, 2, 3], for VERBOSITY, OPTIONS in zip([0, 1, 2, 3], [[], ["-v"], ["-vv"], ["-vvv"]]):
[[], ['-v'], ['-vv'], ['-vvv']]):
@click.command() @click.command()
@verbosity_option() @verbosity_option()
def cli(verbose): def cli(verbose):
ctx = click.get_current_context() ctx = click.get_current_context()
verbose = ctx.meta['verbosity'] verbose = ctx.meta["verbosity"]
assert verbose == VERBOSITY, verbose assert verbose == VERBOSITY, verbose
runner = CliRunner() runner = CliRunner()
...@@ -24,20 +30,19 @@ def test_verbosity_option(): ...@@ -24,20 +30,19 @@ def test_verbosity_option():
def test_bool_option(): def test_bool_option():
@click.command() @click.command()
@bool_option('i-am-test', 'T', 'test test test', True) @bool_option("i-am-test", "T", "test test test", True)
def cli(i_am_test): def cli(i_am_test):
ctx = click.get_current_context() ctx = click.get_current_context()
is_test = ctx.meta['i_am_test'] is_test = ctx.meta["i_am_test"]
assert i_am_test == is_test assert i_am_test == is_test
assert is_test assert is_test
@click.command() @click.command()
@bool_option('i-am-test', 'T', 'test test test', False) @bool_option("i-am-test", "T", "test test test", False)
def cli2(i_am_test): def cli2(i_am_test):
ctx = click.get_current_context() ctx = click.get_current_context()
is_test = ctx.meta['i_am_test'] is_test = ctx.meta["i_am_test"]
assert i_am_test == is_test assert i_am_test == is_test
assert not is_test assert not is_test
...@@ -50,107 +55,106 @@ def test_bool_option(): ...@@ -50,107 +55,106 @@ def test_bool_option():
def test_list_float_option(): def test_list_float_option():
@click.command() @click.command()
@list_float_option('test-list', 'T', 'Test list') @list_float_option("test-list", "T", "Test list")
def cli(test_list): def cli(test_list):
ctx = click.get_current_context() ctx = click.get_current_context()
test = ctx.meta['test_list'] test = ctx.meta["test_list"]
assert test == test_list assert test == test_list
assert test == [1, 2, 3] assert test == [1, 2, 3]
runner = CliRunner() runner = CliRunner()
result = runner.invoke(cli, ['-T', '1,2,3']) result = runner.invoke(cli, ["-T", "1,2,3"])
assert_click_runner_result(result) assert_click_runner_result(result)
def test_list_float_option_empty(): def test_list_float_option_empty():
@click.command() @click.command()
@list_float_option('test-list', 'T', 'Test list') @list_float_option("test-list", "T", "Test list")
def cli(test_list): def cli(test_list):
ctx = click.get_current_context() ctx = click.get_current_context()
test = ctx.meta['test_list'] test = ctx.meta["test_list"]
assert test is None assert test is None
runner = CliRunner() runner = CliRunner()
result = runner.invoke(cli, ['-T', ' ']) result = runner.invoke(cli, ["-T", " "])
assert_click_runner_result(result) assert_click_runner_result(result)
def test_commands_with_config_1(): def test_commands_with_config_1():
# random test # random test
@click.command( @click.command(
cls=ConfigCommand, entry_point_group='bob.extension.test_config_load') cls=ConfigCommand, entry_point_group="bob.extension.test_config_load"
)
def cli(**kwargs): def cli(**kwargs):
pass pass
runner = CliRunner() runner = CliRunner()
result = runner.invoke(cli, ['basic_config']) result = runner.invoke(cli, ["basic_config"])
assert_click_runner_result(result) assert_click_runner_result(result)
def test_commands_with_config_2(): def test_commands_with_config_2():
# test option with valid default value # test option with valid default value
@click.command( @click.command(
cls=ConfigCommand, entry_point_group='bob.extension.test_config_load') cls=ConfigCommand, entry_point_group="bob.extension.test_config_load"
@click.option( )
'-a', cls=ResourceOption, default=3) @click.option("-a", cls=ResourceOption, default=3)
def cli(a, **kwargs): def cli(a, **kwargs):
click.echo('{}'.format(a)) click.echo("{}".format(a))
runner = CliRunner() runner = CliRunner()
result = runner.invoke(cli, []) result = runner.invoke(cli, [])
assert_click_runner_result(result) assert_click_runner_result(result)
assert result.output.strip() == '3', result.output assert result.output.strip() == "3", result.output
result = runner.invoke(cli, ['basic_config']) result = runner.invoke(cli, ["basic_config"])
assert_click_runner_result(result) assert_click_runner_result(result)
assert result.output.strip() == '1', result.output assert result.output.strip() == "1", result.output
result = runner.invoke(cli, ['-a', 2]) result = runner.invoke(cli, ["-a", 2])
assert_click_runner_result(result) assert_click_runner_result(result)
assert result.output.strip() == '2', result.output assert result.output.strip() == "2", result.output
result = runner.invoke(cli, ['-a', 3, 'basic_config']) result = runner.invoke(cli, ["-a", 3, "basic_config"])
assert_click_runner_result(result) assert_click_runner_result(result)
assert result.output.strip() == '3', result.output assert result.output.strip() == "3", result.output
result = runner.invoke(cli, ['basic_config', '-a', 3]) result = runner.invoke(cli, ["basic_config", "-a", 3])
assert_click_runner_result(result) assert_click_runner_result(result)
assert result.output.strip() == '3', result.output assert result.output.strip() == "3", result.output
def test_commands_with_config_3(): def test_commands_with_config_3():
# test required options # test required options
@click.command( @click.command(
cls=ConfigCommand, entry_point_group='bob.extension.test_config_load') cls=ConfigCommand, entry_point_group="bob.extension.test_config_load"
@click.option( )
'-a', cls=ResourceOption, required=True) @click.option("-a", cls=ResourceOption, required=True)
def cli(a, **kwargs): def cli(a, **kwargs):
click.echo('{}'.format(a)) click.echo("{}".format(a))
runner = CliRunner() runner = CliRunner()
result = runner.invoke(cli, []) result = runner.invoke(cli, [])
assert_click_runner_result(result, exit_code=2) assert_click_runner_result(result, exit_code=2)
result = runner.invoke(cli, ['basic_config']) result = runner.invoke(cli, ["basic_config"])
assert_click_runner_result(result) assert_click_runner_result(result)
assert result.output.strip() == '1', result.output assert result.output.strip() == "1", result.output
result = runner.invoke(cli, ['-a', 2]) result = runner.invoke(cli, ["-a", 2])
assert_click_runner_result(result) assert_click_runner_result(result)
assert result.output.strip() == '2', result.output assert result.output.strip() == "2", result.output
result = runner.invoke(cli, ['-a', 3, 'basic_config']) result = runner.invoke(cli, ["-a", 3, "basic_config"])
assert_click_runner_result(result) assert_click_runner_result(result)
assert result.output.strip() == '3', result.output assert result.output.strip() == "3", result.output
result = runner.invoke(cli, ['basic_config', '-a', 3]) result = runner.invoke(cli, ["basic_config", "-a", 3])
assert_click_runner_result(result) assert_click_runner_result(result)
assert result.output.strip() == '3', result.output assert result.output.strip() == "3", result.output
def test_prefix_aliasing(): def test_prefix_aliasing():
...@@ -162,31 +166,33 @@ def test_prefix_aliasing(): ...@@ -162,31 +166,33 @@ def test_prefix_aliasing():
def test(): def test():
click.echo("OK") click.echo("OK")
@cli.command(name='test-aaa') @cli.command(name="test-aaa")
def test_aaa(): def test_aaa():
click.echo("AAA") click.echo("AAA")
runner = CliRunner() runner = CliRunner()
result = runner.invoke(cli, ['te'], catch_exceptions=False) result = runner.invoke(cli, ["te"], catch_exceptions=False)
assert result.exit_code != 0, (result.exit_code, result.output) assert result.exit_code != 0, (result.exit_code, result.output)
result = runner.invoke(cli, ['test'], catch_exceptions=False) result = runner.invoke(cli, ["test"], catch_exceptions=False)
assert_click_runner_result(result) assert_click_runner_result(result)
assert 'OK' in result.output, (result.exit_code, result.output) assert "OK" in result.output, (result.exit_code, result.output)
result = runner.invoke(cli, ['test-a'], catch_exceptions=False) result = runner.invoke(cli, ["test-a"], catch_exceptions=False)
assert_click_runner_result(result) assert_click_runner_result(result)
assert 'AAA' in result.output, (result.exit_code, result.output) assert "AAA" in result.output, (result.exit_code, result.output)
def _assert_config_dump(ref, ref_date): def _assert_config_dump(ref, ref_date):
today = time.strftime("%d/%m/%Y") today = time.strftime("%d/%m/%Y")
# uncomment below to re-write tests # uncomment below to re-write tests
# open(ref, 'wt').write(open('TEST_CONF').read()) # open(ref, 'wt').write(open('TEST_CONF').read())
with open('TEST_CONF', 'r') as f, open(ref, 'r') as f2: with open("TEST_CONF", "r") as f, open(ref, "r") as f2:
text = f.read() text = f.read()
ref_text = f2.read().replace(ref_date, today) ref_text = f2.read().replace(ref_date, today)
assert text == ref_text, '\n'.join([text, "########################\n"*2, ref_text]) assert text == ref_text, "\n".join(
[text, "########################\n" * 2, ref_text]
)
def test_config_dump(): def test_config_dump():
...@@ -194,21 +200,28 @@ def test_config_dump(): ...@@ -194,21 +200,28 @@ def test_config_dump():
def cli(): def cli():
pass pass
@cli.command(cls=ConfigCommand, epilog='Examples!') @cli.command(cls=ConfigCommand, epilog="Examples!")
@click.option('-t', '--test', required=True, default="/my/path/test.txt", @click.option(
help="Path leading to test blablabla", cls=ResourceOption) "-t",
"--test",
required=True,
default="/my/path/test.txt",
help="Path leading to test blablabla",
cls=ResourceOption,
)
@verbosity_option(cls=ResourceOption) @verbosity_option(cls=ResourceOption)
def test(config, test, **kwargs): def test(config, test, **kwargs):
"""Test command""" """Test command"""
pass pass
runner = CliRunner() runner = CliRunner()
with runner.isolated_filesystem(): with runner.isolated_filesystem():
result = runner.invoke( result = runner.invoke(cli, ["test", "-H", "TEST_CONF"], catch_exceptions=False)
cli, ['test', '-H', 'TEST_CONF'], catch_exceptions=False) ref = pkg_resources.resource_filename(
ref = pkg_resources.resource_filename('bob.extension', "bob.extension", "data/test_dump_config.py"
'data/test_dump_config.py') )
assert_click_runner_result(result) assert_click_runner_result(result)
_assert_config_dump(ref, '08/07/2018') _assert_config_dump(ref, "08/07/2018")
def test_config_dump2(): def test_config_dump2():
...@@ -216,19 +229,38 @@ def test_config_dump2(): ...@@ -216,19 +229,38 @@ def test_config_dump2():
def cli(): def cli():
pass pass
@cli.command(cls=ConfigCommand, entry_point_group='bob.extension.test_dump_config') @cli.command(cls=ConfigCommand, entry_point_group="bob.extension.test_dump_config")
@click.option('--database', '-d', required=True, cls=ResourceOption, @click.option(
entry_point_group='bob.extension.test_dump_config', help="bla bla bla") "--database",
@click.option('--annotator', '-a', required=True, cls=ResourceOption, "-d",
entry_point_group='bob.extension.test_dump_config', help="bli bli bli") required=True,
@click.option('--output-dir', '-o', required=True, cls=ResourceOption, cls=ResourceOption,
help="blo blo blo") entry_point_group="bob.extension.test_dump_config",
@click.option('--force', '-f', is_flag=True, cls=ResourceOption, help="bla bla bla",
help="lalalalalala") )
@click.option('--array', type=click.INT, default=1, cls=ResourceOption, @click.option(
help="lililili") "--annotator",
@click.option('--database-directories-file', cls=ResourceOption, "-a",
default='~/databases.txt', help="lklklklk") required=True,
cls=ResourceOption,
entry_point_group="bob.extension.test_dump_config",
help="bli bli bli",
)
@click.option(
"--output-dir", "-o", required=True, cls=ResourceOption, help="blo blo blo"
)
@click.option(
"--force", "-f", is_flag=True, cls=ResourceOption, help="lalalalalala"
)
@click.option(
"--array", type=click.INT, default=1, cls=ResourceOption, help="lililili"
)
@click.option(
"--database-directories-file",
cls=ResourceOption,
default="~/databases.txt",
help="lklklklk",
)
@verbosity_option(cls=ResourceOption) @verbosity_option(cls=ResourceOption)
def test(**kwargs): def test(**kwargs):
"""Blablabla bli blo """Blablabla bli blo
...@@ -246,55 +278,66 @@ def test_config_dump2(): ...@@ -246,55 +278,66 @@ def test_config_dump2():
runner = CliRunner() runner = CliRunner()
with runner.isolated_filesystem(): with runner.isolated_filesystem():
result = runner.invoke( result = runner.invoke(cli, ["test", "-H", "TEST_CONF"], catch_exceptions=False)
cli, ['test', '-H', 'TEST_CONF'], catch_exceptions=False) ref = pkg_resources.resource_filename(
ref = pkg_resources.resource_filename('bob.extension', "bob.extension", "data/test_dump_config2.py"
'data/test_dump_config2.py') )
assert_click_runner_result(result) assert_click_runner_result(result)
_assert_config_dump(ref, '08/07/2018') _assert_config_dump(ref, "08/07/2018")
def test_config_command_with_callback_options(): def test_config_command_with_callback_options():
@click.command(
@click.command(cls=ConfigCommand, entry_point_group='bob.extension.test_config_load') cls=ConfigCommand, entry_point_group="bob.extension.test_config_load"
@verbosity_option(cls=ResourceOption, envvar='VERBOSE') )
@verbosity_option(cls=ResourceOption, envvar="VERBOSE")
@click.pass_context @click.pass_context
def cli(ctx, **kwargs): def cli(ctx, **kwargs):
verbose = ctx.meta['verbosity'] verbose = ctx.meta["verbosity"]
assert verbose == 2, verbose assert verbose == 2, verbose
runner = CliRunner() runner = CliRunner()
result = runner.invoke(cli, ['verbose_config'], catch_exceptions=False) result = runner.invoke(cli, ["verbose_config"], catch_exceptions=False)
assert_click_runner_result(result) assert_click_runner_result(result)
runner = CliRunner(env=dict(VERBOSE='1')) runner = CliRunner(env=dict(VERBOSE="1"))
result = runner.invoke(cli, ['verbose_config'], catch_exceptions=False) result = runner.invoke(cli, ["verbose_config"], catch_exceptions=False)
assert_click_runner_result(result) assert_click_runner_result(result)
runner = CliRunner(env=dict(VERBOSE='2')) runner = CliRunner(env=dict(VERBOSE="2"))
result = runner.invoke(cli, catch_exceptions=False) result = runner.invoke(cli, catch_exceptions=False)
assert_click_runner_result(result) assert_click_runner_result(result)
def test_resource_option(): def test_resource_option():
# tests of ResourceOption used with ConfigCommand are done in other tests. # tests of ResourceOption used with ConfigCommand are done in other tests.
# test usage without ConfigCommand and with entry_point_group # test usage without ConfigCommand and with entry_point_group
@click.command() @click.command()
@click.option('-a', '--a', cls=ResourceOption, entry_point_group='bob.extension.test_config_load') @click.option(
"-a",
"--a",
cls=ResourceOption,
entry_point_group="bob.extension.test_config_load",
)
def cli(a): def cli(a):
assert a == 1, a assert a == 1, a
runner = CliRunner() runner = CliRunner()
result = runner.invoke(cli, ['-a', 'bob.extension.data.resource_config2'], catch_exceptions=False) result = runner.invoke(
cli,
["-a", "bob.extension.data.resource_config2"],
catch_exceptions=False,
)
assert_click_runner_result(result) assert_click_runner_result(result)
# test usage without ConfigCommand and without entry_point_group # test usage without ConfigCommand and without entry_point_group
# should raise a TypeError # should raise a TypeError
@click.command() @click.command()
@click.option('-a', '--a', cls=ResourceOption) @click.option("-a", "--a", cls=ResourceOption)
def cli(a): def cli(a):
raise ValueError("Should not have reached here!") raise ValueError("Should not have reached here!")
runner = CliRunner() runner = CliRunner()
result = runner.invoke(cli, ['-a', '1'], catch_exceptions=True) result = runner.invoke(cli, ["-a", "1"], catch_exceptions=True)
assert_click_runner_result(result, exit_code=1, exception_type=TypeError) assert_click_runner_result(result, exit_code=1, exception_type=TypeError)
bob/extension/test_config.py
+ 4
− 4
View file @ 799c31e7
...@@ -63,22 +63,22 @@ def test_entry_point_configs(): ...@@ -63,22 +63,22 @@ def test_entry_point_configs():
def test_load_resource(): def test_load_resource():
for p, ref in [ for p, ref in [
(os.path.join(path, 'resource_config2.py'), 1), (os.path.join(path, 'resource_config2.py'), 1),
(os.path.join(path, 'resource_config2.py:test_config_load'), 1), (os.path.join(path, 'resource_config2.py:a'), 1),
(os.path.join(path, 'resource_config2.py:b'), 2), (os.path.join(path, 'resource_config2.py:b'), 2),
('resource1', 1), ('resource1', 1),
('resource2', 2), ('resource2', 2),
('bob.extension.data.resource_config2', 1), ('bob.extension.data.resource_config2', 1),
('bob.extension.data.resource_config2:test_config_load', 1), ('bob.extension.data.resource_config2:a', 1),
('bob.extension.data.resource_config2:b', 2), ('bob.extension.data.resource_config2:b', 2),
]: ]:
c = load([p], entry_point_group='bob.extension.test_config_load', c = load([p], entry_point_group='bob.extension.test_config_load',
attribute_name='test_config_load') attribute_name='a')
assert c == ref, c assert c == ref, c
try: try:
load(['bob.extension.data.resource_config2:c'], load(['bob.extension.data.resource_config2:c'],
entry_point_group='bob.extension.test_config_load', entry_point_group='bob.extension.test_config_load',
attribute_name='test_config_load') attribute_name='a')
assert False, 'The code above should have raised an ImportError' assert False, 'The code above should have raised an ImportError'
except ImportError: except ImportError:
pass pass
doc/annotate.py
+ 11
− 6
View file @ 799c31e7
"""A script to help annotate databases. """A script to help annotate databases.
""" """
# Avoid importing packages here! Importing packages here will slowdown your command
# line's --help option and its auto-complete feature in terminal (if enabled). Instead,
# put your imports inside the function.
import logging import logging
import click import click
from bob.extension.scripts.click_helper import ( from bob.extension.scripts.click_helper import (
...@@ -8,16 +11,13 @@ from bob.extension.scripts.click_helper import ( ...@@ -8,16 +11,13 @@ from bob.extension.scripts.click_helper import (
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
ANNOTATE_EPILOG = '''\b @click.command(entry_point_group='bob.bio.config', cls=ConfigCommand,
epilog='''\b
Examples: Examples:
$ bob bio annotate -vvv -d <database> -a <annotator> -o /tmp/annotations $ bob bio annotate -vvv -d <database> -a <annotator> -o /tmp/annotations
$ jman submit --array 64 -- bob bio annotate ... --array 64 $ jman submit --array 64 -- bob bio annotate ... --array 64
''' ''')
@click.command(entry_point_group='bob.bio.config', cls=ConfigCommand,
epilog=ANNOTATE_EPILOG)
@click.option('--database', '-d', required=True, cls=ResourceOption, @click.option('--database', '-d', required=True, cls=ResourceOption,
entry_point_group='bob.bio.database', entry_point_group='bob.bio.database',
help='''The database that you want to annotate.''') help='''The database that you want to annotate.''')
...@@ -40,3 +40,8 @@ def annotate(database, annotator, output_dir, force, array, **kwargs): ...@@ -40,3 +40,8 @@ def annotate(database, annotator, output_dir, force, array, **kwargs):
back using :any:`bob.db.base.read_annotation_file` (annotation_type='json') back using :any:`bob.db.base.read_annotation_file` (annotation_type='json')
""" """
log_parameters(logger) log_parameters(logger)
# Add imports needed for your code here:
import numpy as np
np.zeros(10)
doc/framework.rst
+ 68
− 71
View file @ 799c31e7
...@@ -17,7 +17,7 @@ Python-based Configuration System ...@@ -17,7 +17,7 @@ Python-based Configuration System
--------------------------------- ---------------------------------
This package also provides a configuration system that can be used by packages This package also provides a configuration system that can be used by packages
in the |project|-echosystem to load *run-time* configuration for applications in the |project|-ecosystem to load *run-time* configuration for applications
(for package-level static variable configuration use :ref:`bob.extension.rc`). (for package-level static variable configuration use :ref:`bob.extension.rc`).
It can be used to accept complex configurations from users through It can be used to accept complex configurations from users through
command-line. command-line.
...@@ -59,7 +59,7 @@ Then, the object ``configuration`` would look like this: ...@@ -59,7 +59,7 @@ Then, the object ``configuration`` would look like this:
.. doctest:: .. doctest::
>>> print("a = %d\nb = %d"%(configuration.a, configuration.b)) >>> print(f"a = {configuration.a}\nb = {configuration.b}")
a = 1 a = 1
b = 3 b = 3
...@@ -96,9 +96,10 @@ Then, one can chain-load them like this: ...@@ -96,9 +96,10 @@ Then, one can chain-load them like this:
>>> file1 = os.path.join(path, 'basic_config.py') >>> file1 = os.path.join(path, 'basic_config.py')
>>> file2 = os.path.join(path, 'load_config.py') >>> file2 = os.path.join(path, 'load_config.py')
>>> configuration = load([file1, file2]) >>> configuration = load([file1, file2])
>>> print("a = %d \nb = %d"%(configuration.a, configuration.b)) # doctest: +NORMALIZE_WHITESPACE >>> print(f"a = {configuration.a} \nb = {configuration.b} \nc = {configuration.c}") # doctest: +NORMALIZE_WHITESPACE
a = 1 a = 1
b = 6 b = 6
c = 4
The user wanting to override the values needs to manage the overriding and the The user wanting to override the values needs to manage the overriding and the
...@@ -123,6 +124,8 @@ to provide the group name of the entry points: ...@@ -123,6 +124,8 @@ to provide the group name of the entry points:
b = 6 b = 6
.. _bob.extension.config.resource:
Resource Loading Resource Loading
================ ================
...@@ -140,73 +143,16 @@ The loaded value can be either 1 or 2: ...@@ -140,73 +143,16 @@ The loaded value can be either 1 or 2:
.. doctest:: load_resource .. doctest:: load_resource
>>> group = 'bob.extension.test_config_load' # the group name of entry points >>> group = 'bob.extension.test_config_load' # the group name of entry points
>>> attribute_name = 'test_config_load' # the common variable name >>> attribute_name = 'a' # the common variable name
>>> value = load(['bob.extension.data.resource_config2'], entry_point_group=group, attribute_name=attribute_name) >>> value = load(['bob.extension.data.resource_config2'], entry_point_group=group, attribute_name=attribute_name)
>>> value == 1 >>> value == 1
True True
>>> # attribute_name can be ovverriden using the `path:attribute_name` syntax
>>> value = load(['bob.extension.data.resource_config2:b'], entry_point_group=group, attribute_name=attribute_name) >>> value = load(['bob.extension.data.resource_config2:b'], entry_point_group=group, attribute_name=attribute_name)
>>> value == 2 >>> value == 2
True True
.. _bob.extension.processors:
Stacked Processing
------------------
:any:`bob.extension.processors.SequentialProcessor` and
:any:`bob.extension.processors.ParallelProcessor` are provided to help you
build complex processing mechanisms. You can use these processors to apply a
chain of processes on your data. For example,
:any:`bob.extension.processors.SequentialProcessor` accepts a list of callables
and applies them on the data one by one sequentially. :
.. doctest::
>>> import numpy as np; from numpy import array
>>> from functools import partial
>>> from bob.extension.processors import SequentialProcessor
>>> raw_data = np.array([[1, 2, 3], [1, 2, 3]])
>>> seq_processor = SequentialProcessor(
... [np.cast['float64'], lambda x: x / 2, partial(np.mean, axis=1)])
>>> np.allclose(seq_processor(raw_data),
... array([ 1., 1.]))
True
>>> np.all(seq_processor(raw_data) ==
... np.mean(np.cast['float64'](raw_data) / 2, axis=1))
True
:any:`bob.extension.processors.ParallelProcessor` accepts a list of callables
and applies each them on the data independently and returns all the results.
For example:
.. doctest::
>>> from bob.extension.processors import ParallelProcessor
>>> raw_data = np.array([[1, 2, 3], [1, 2, 3]])
>>> parallel_processor = ParallelProcessor(
... [np.cast['float64'], lambda x: x / 2.0])
>>> np.allclose(list(parallel_processor(raw_data)),
... [array([[ 1., 2., 3.],
... [ 1., 2., 3.]]),
... array([[ 0.5, 1. , 1.5],
... [ 0.5, 1. , 1.5]])])
True
The data may be further processed using a
:any:`bob.extension.processors.SequentialProcessor`:
.. doctest::
>>> total_processor = SequentialProcessor(
... [parallel_processor, list, partial(np.concatenate, axis=1)])
>>> np.allclose(total_processor(raw_data),
... array([[ 1. , 2. , 3. , 0.5, 1. , 1.5],
... [ 1. , 2. , 3. , 0.5, 1. , 1.5]]))
True
.. _bob.extension.cli: .. _bob.extension.cli:
Unified Command Line Mechanism Unified Command Line Mechanism
...@@ -228,11 +174,6 @@ commands by default:: ...@@ -228,11 +174,6 @@ commands by default::
config The manager for bob's global configuration. config The manager for bob's global configuration.
... ...
.. warning::
This feature is experimental and most probably will break compatibility.
If you are not willing to fix your code after changes are made here,
please do not use this feature.
This command line is implemented using click_. You can extend the commands of This command line is implemented using click_. You can extend the commands of
this script through setuptools entry points (this is implemented using this script through setuptools entry points (this is implemented using
...@@ -243,15 +184,13 @@ independently; then, advertise it as a command under bob script using the ...@@ -243,15 +184,13 @@ independently; then, advertise it as a command under bob script using the
.. note:: .. note::
If you are still not sure how this must be done, maybe you don't know how If you are still not sure how this must be done, maybe you don't know how
to use click_ yet. to use click_ and `click-plugins`_ yet.
This feature is experimental and may change and break compatibility in future.
For a best practice example, please look at how the ``bob config`` command is For a best practice example, please look at how the ``bob config`` command is
implemented: implemented:
.. literalinclude:: ../bob/extension/scripts/config.py .. literalinclude:: ../bob/extension/scripts/config.py
:caption: "bob/extension/scripts/config.py" implementation of the ``bob :caption: "bob/extension/scripts/config.py" implementation of the ``bob config`` command.
config`` command.
:language: python :language: python
...@@ -366,5 +305,63 @@ module names and maybe have them provide simple options like ``--verbose`` or ...@@ -366,5 +305,63 @@ module names and maybe have them provide simple options like ``--verbose`` or
``--force`` through the command line options. ``--force`` through the command line options.
.. _bob.extension.processors:
Stacked Processing
------------------
:any:`bob.extension.processors.SequentialProcessor` and
:any:`bob.extension.processors.ParallelProcessor` are provided to help you
build complex processing mechanisms. You can use these processors to apply a
chain of processes on your data. For example,
:any:`bob.extension.processors.SequentialProcessor` accepts a list of callables
and applies them on the data one by one sequentially. :
.. doctest::
>>> import numpy as np; from numpy import array
>>> from functools import partial
>>> from bob.extension.processors import SequentialProcessor
>>> raw_data = np.array([[1, 2, 3], [1, 2, 3]])
>>> seq_processor = SequentialProcessor(
... [np.cast['float64'], lambda x: x / 2, partial(np.mean, axis=1)])
>>> np.allclose(seq_processor(raw_data),
... array([ 1., 1.]))
True
>>> np.all(seq_processor(raw_data) ==
... np.mean(np.cast['float64'](raw_data) / 2, axis=1))
True
:any:`bob.extension.processors.ParallelProcessor` accepts a list of callables
and applies each them on the data independently and returns all the results.
For example:
.. doctest::
>>> from bob.extension.processors import ParallelProcessor
>>> raw_data = np.array([[1, 2, 3], [1, 2, 3]])
>>> parallel_processor = ParallelProcessor(
... [np.cast['float64'], lambda x: x / 2.0])
>>> np.allclose(list(parallel_processor(raw_data)),
... [array([[ 1., 2., 3.],
... [ 1., 2., 3.]]),
... array([[ 0.5, 1. , 1.5],
... [ 0.5, 1. , 1.5]])])
True
The data may be further processed using a
:any:`bob.extension.processors.SequentialProcessor`:
.. doctest::
>>> total_processor = SequentialProcessor(
... [parallel_processor, list, partial(np.concatenate, axis=1)])
>>> np.allclose(total_processor(raw_data),
... array([[ 1. , 2. , 3. , 0.5, 1. , 1.5],
... [ 1. , 2. , 3. , 0.5, 1. , 1.5]]))
True
.. include:: links.rst .. include:: links.rst
version.txt
+ 1
− 1
View file @ 799c31e7
4.0.1b0 5.0.0b0
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment