Commit 769a2a94 authored by André Anjos's avatar André Anjos

Merge branch 'dumpconfig' into 'master'

Dump config file

Closes #61

See merge request !86
parents 13a70cfc ae895002
Pipeline #21736 passed with stages
in 26 minutes and 53 seconds
......@@ -5,10 +5,10 @@
'''
import imp
import pkg_resources
import pkgutil
from os.path import isfile
import logging
import pkg_resources
logger = logging.getLogger(__name__)
......@@ -211,3 +211,29 @@ def mod_to_context(mod):
"""
return {k: v for k, v in mod.__dict__.items()
if not (k.startswith('__') and k.endswith('__'))}
def resource_keys(entry_point_group, exclude_packages=[], strip=['dummy']):
"""Reads and returns all resources that are registered with the given
entry_point_group. Entry points from the given ``exclude_packages`` are
ignored.
Parameters
----------
entry_point_group : str
The entry point group name.
exclude_packages : :any:`list`, optional
List of packages to exclude when finding resources.
strip : :any:`list`, optional
Entrypoint names that start with any value in ``strip`` will be ignored.
Returns
-------
:any:`list`
List of found resources.
"""
ret_list = [entry_point.name for entry_point in
pkg_resources.iter_entry_points(entry_point_group)
if (entry_point.dist.project_name not in exclude_packages and
not entry_point.name.startswith(tuple(strip)))]
return sorted(ret_list)
'''Configuration file automatically generated at 08/07/2018
cli test
Test command
Examples!'''
# test = /my/path/test.txt
'''Required parameter: test (-t, --test)
Path leading to test blablabla'''
# verbose = 0
'''Optional parameter: verbose (-v, --verbose) [default: 0]
Increase the verbosity level from 0 (only error messages) to 1 (warnings), 2 (log messages), 3 (debug information) by adding the --verbose option as often as desired (e.g. '-vvv' for debug).'''
'''Configuration file automatically generated at 08/07/2018
cli test
Blablabla bli blo
Parameters
----------
xxx : :any:`list`
blabla blablo
yyy : callable
bli bla blo bla bla bla
[CONFIG]... BLA BLA BLA BLA'''
# database = None
'''Required parameter: database (--database, -d)
bla bla bla Can be a ``bob.extension.test_config_load`` entry point, a module name, or a path to a Python file which contains a variable named `database`.
Registered entries are: ['basic_config', 'resource_config', 'subpackage_config']'''
# annotator = None
'''Required parameter: annotator (--annotator, -a)
bli bli bli Can be a ``bob.extension.test_config_load`` entry point, a module name, or a path to a Python file which contains a variable named `annotator`.
Registered entries are: ['basic_config', 'resource_config', 'subpackage_config']'''
# output_dir = None
'''Required parameter: output_dir (--output-dir, -o)
blo blo blo'''
# force = False
'''Optional parameter: force (--force, -f) [default: False]
lalalalalala'''
# array = 1
'''Optional parameter: array (--array) [default: 1]
lililili'''
# database_directories_file = ~/databases.txt
'''Optional parameter: database_directories_file (--database-directories-file) [default: ~/databases.txt]
lklklklk'''
# verbose = 0
'''Optional parameter: verbose (-v, --verbose) [default: 0]
Increase the verbosity level from 0 (only error messages) to 1 (warnings), 2 (log messages), 3 (debug information) by adding the --verbose option as often as desired (e.g. '-vvv' for debug).'''
from ..log import set_verbosity_level
from ..config import load, mod_to_context
from ..config import load, mod_to_context, resource_keys
import time
import click
import logging
......@@ -137,7 +138,6 @@ def verbosity_option(**kwargs):
return value
return click.option(
'-v', '--verbose', count=True,
expose_value=False,
help="Increase the verbosity level from 0 (only error messages) to 1 "
"(warnings), 2 (log messages), 3 (debug information) by adding the "
"--verbose option as often as desired (e.g. '-vvv' for debug).",
......@@ -165,6 +165,17 @@ class ConfigCommand(click.Command):
config_argument_name='CONFIG', **kwargs):
self.config_argument_name = config_argument_name
self.entry_point_group = entry_point_group
# Augment help for the config file argument
self.extra_help = '''\n\nIt is possible to pass one or several Python files
(or names of ``{entry_point_group}`` entry points or module names) as {CONFIG}
arguments to the command line which contain the parameters listed below as
Python variables. The options through the command-line (see below) will
override the values of configuration files. You can run this command with
``<COMMAND> -H example_config.py`` to create a template config
file.'''.format(CONFIG=config_argument_name,
entry_point_group=entry_point_group)
help = (help or '').rstrip() + self.extra_help
# kwargs['help'] = help
click.Command.__init__(
self, name, context_settings=context_settings, callback=callback,
params=params, help=help, epilog=epilog, short_help=short_help,
......@@ -172,15 +183,29 @@ class ConfigCommand(click.Command):
**kwargs)
# Add the config argument to the command
click.argument(config_argument_name, nargs=-1)(self)
# Option for config file generation
click.option('-H', '--dump-config', type=click.File(mode='wt'),
help="Name of the config file to be generated")(self)
def is_resource(self, param, ctx):
"""Checks if the param is an option and is also in the current context."""
return (param.name in ctx.params and
param.name != 'dump_config' and
isinstance(param, click.Option))
def invoke(self, ctx):
dump_file = ctx.params.get('dump_config')
if dump_file is not None:
click.echo("Configuration file '{}' was written; exiting".format(
dump_file.name))
return self.dump_config(ctx)
config_files = ctx.params[self.config_argument_name.lower()]
# load and normalize context from config files
config_context = load(
config_files, entry_point_group=self.entry_point_group)
config_context = mod_to_context(config_context)
for param in self.params:
if param.name not in ctx.params:
if not self.is_resource(param, ctx):
continue
value = ctx.params[param.name]
if not hasattr(param, 'user_provided'):
......@@ -203,6 +228,58 @@ class ConfigCommand(click.Command):
return super(ConfigCommand, self).invoke(ctx)
def dump_config(self, ctx):
"""Generate configuration file from parameters and context
Parameters
----------
ctx : object
Click context
"""
config_file = ctx.params['dump_config']
logger.debug("Generating configuration file `%s'...", config_file)
config_file.write("'''")
config_file.write('Configuration file automatically generated at '
'%s\n%s\n' % (time.strftime("%d/%m/%Y"),
ctx.command_path))
if self.help:
h = self.help.replace(self.extra_help, '').replace('\b\n', '')
config_file.write('\n{}'.format(h.rstrip()))
if self.epilog:
config_file.write('\n\n{}'.format(self.epilog.replace('\b\n', '')))
config_file.write("'''\n")
for param in self.params:
if not self.is_resource(param, ctx):
continue
config_file.write('\n# %s = %s\n' % (param.name,
str(ctx.params[param.name])))
config_file.write("'''")
if param.required or (isinstance(param, ResourceOption) and
param.real_required):
begin, dflt = 'Required parameter', ''
else:
begin, dflt = 'Optional parameter', ' [default: {}]'.format(
param.default)
config_file.write(
"%s: %s (%s)%s" % (
begin, param.name, ', '.join(param.opts), dflt))
if param.help is not None:
config_file.write("\n%s" % param.help)
if isinstance(param, ResourceOption) and \
param.entry_point_group is not None:
config_file.write("\nRegistered entries are: {}".format(
resource_keys(param.entry_point_group)))
config_file.write("'''\n")
class ResourceOption(click.Option):
"""A click.Option that is aware if the user actually provided this option
......@@ -231,6 +308,13 @@ class ResourceOption(click.Option):
self.entry_point_group = entry_point_group
self.real_required = required
kwargs['required'] = False
if entry_point_group is not None:
name, _, _ = self._parse_decls(param_decls, kwargs.get('expose_value'))
help = help or ''
help += (
' Can be a ``{entry_point_group}`` entry point, a module name, or '
'a path to a Python file which contains a variable named `{name}`.')
help = help.format(entry_point_group=entry_point_group, name=name)
click.Option.__init__(
self, param_decls=param_decls, show_default=show_default,
prompt=prompt, confirmation_prompt=confirmation_prompt,
......@@ -259,7 +343,6 @@ class ResourceOption(click.Option):
while isinstance(value, basestring):
value = load([value], entry_point_group=self.entry_point_group)
value = getattr(value, keyword)
return value
......@@ -286,3 +369,26 @@ class AliasedGroup(click.Group):
elif len(matches) == 1:
return click.Group.get_command(self, ctx, matches[0])
ctx.fail('Too many matches: %s' % ', '.join(sorted(matches)))
def log_parameters(logger_handle):
"""Logs the click parameters with the logging module.
Parameters
----------
logger_handle : object
The logger handle to write debug information into.
"""
ctx = click.get_current_context()
# do not sort the ctx.params dict. The insertion order is kept in Python 3
# and is useful (but not necessary so works on Python 2 too).
for k, v in ctx.params.items():
logger_handle.debug('%s: %s', k, v)
def assert_click_runner_result(result, exit_code=0):
"""Helper for asserting click runner results"""
m = ("Click command exited with code `{}' and exception:\n{}"
"\nThe output was:\n{}")
m = m.format(result.exit_code, result.exception, result.output)
assert result.exit_code == exit_code, m
......@@ -13,7 +13,7 @@ logger = logging.getLogger(__name__)
@click.group(cls=AliasedGroup)
@verbosity_option()
def config():
def config(**kwargs):
"""The manager for bob's global configuration."""
# Load the config file again. This may be needed since the environment
# variable might change the config path during the tests. Otherwise, this
......
import click
import time
import pkg_resources
from click.testing import CliRunner
from bob.extension.scripts.click_helper import (
verbosity_option, bool_option, list_float_option,
open_file_mode_option, ConfigCommand, ResourceOption, AliasedGroup)
ConfigCommand, ResourceOption, AliasedGroup)
def test_verbosity_option():
......@@ -11,7 +13,7 @@ def test_verbosity_option():
[[], ['-v'], ['-vv'], ['-vvv']]):
@click.command()
@verbosity_option()
def cli():
def cli(verbose):
ctx = click.get_current_context()
verbose = ctx.meta['verbosity']
assert verbose == VERBOSITY, verbose
......@@ -20,6 +22,7 @@ def test_verbosity_option():
result = runner.invoke(cli, OPTIONS, catch_exceptions=False)
assert result.exit_code == 0, (result.exit_code, result.output)
def test_bool_option():
@click.command()
......@@ -45,6 +48,7 @@ def test_bool_option():
result = runner.invoke(cli2)
assert result.exit_code == 0, (result.exit_code, result.output)
def test_list_float_option():
@click.command()
......@@ -59,6 +63,7 @@ def test_list_float_option():
result = runner.invoke(cli, ['-T', '1,2,3'])
assert result.exit_code == 0, (result.exit_code, result.output)
def test_list_float_option_empty():
@click.command()
......@@ -72,6 +77,7 @@ def test_list_float_option_empty():
result = runner.invoke(cli, ['-T', ' '])
assert result.exit_code == 0, (result.exit_code, result.output)
def test_commands_with_config_1():
# random test
@click.command(
......@@ -146,6 +152,7 @@ def test_commands_with_config_3():
assert result.exit_code == 0, (result.exit_code, result.output)
assert result.output.strip() == '3', result.output
def test_prefix_aliasing():
@click.group(cls=AliasedGroup)
def cli():
......@@ -159,7 +166,6 @@ def test_prefix_aliasing():
def test_aaa():
click.echo("AAA")
runner = CliRunner()
result = runner.invoke(cli, ['te'], catch_exceptions=False)
assert result.exit_code != 0, (result.exit_code, result.output)
......@@ -171,3 +177,78 @@ def test_prefix_aliasing():
result = runner.invoke(cli, ['test_a'], catch_exceptions=False)
assert result.exit_code == 0, (result.exit_code, result.output)
assert 'AAA' in result.output, (result.exit_code, result.output)
def _assert_config_dump(ref, ref_date):
today = time.strftime("%d/%m/%Y")
# uncomment below to re-write tests
# open(ref, 'wt').write(open('TEST_CONF').read())
with open('TEST_CONF', 'r') as f, open(ref, 'r') as f2:
text = f.read()
ref_text = f2.read().replace(ref_date, today)
assert text == ref_text, '\n'.join([text, ref_text])
def test_config_dump():
@click.group(cls=AliasedGroup)
def cli():
pass
@cli.command(cls=ConfigCommand, epilog='Examples!')
@click.option('-t', '--test', required=True, default="/my/path/test.txt",
help="Path leading to test blablabla", cls=ResourceOption)
@verbosity_option()
def test(config, test, **kwargs):
"""Test command"""
pass
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(
cli, ['test', '-H', 'TEST_CONF'], catch_exceptions=False)
ref = pkg_resources.resource_filename('bob.extension',
'data/test_dump_config.py')
assert result.exit_code == 0, (result.exit_code, result.output)
_assert_config_dump(ref, '08/07/2018')
def test_config_dump2():
@click.group(cls=AliasedGroup)
def cli():
pass
@cli.command(cls=ConfigCommand, entry_point_group='bob.extension.test_config_load')
@click.option('--database', '-d', required=True, cls=ResourceOption,
entry_point_group='bob.extension.test_config_load', help="bla bla bla")
@click.option('--annotator', '-a', required=True, cls=ResourceOption,
entry_point_group='bob.extension.test_config_load', help="bli bli bli")
@click.option('--output-dir', '-o', required=True, cls=ResourceOption,
help="blo blo blo")
@click.option('--force', '-f', is_flag=True, cls=ResourceOption,
help="lalalalalala")
@click.option('--array', type=click.INT, default=1, cls=ResourceOption,
help="lililili")
@click.option('--database-directories-file', cls=ResourceOption,
default='~/databases.txt', help="lklklklk")
@verbosity_option(cls=ResourceOption)
def test(**kwargs):
"""Blablabla bli blo
Parameters
----------
xxx : :any:`list`
blabla blablo
yyy : callable
bli bla blo bla bla bla
[CONFIG]... BLA BLA BLA BLA
"""
pass
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(
cli, ['test', '-H', 'TEST_CONF'], catch_exceptions=False)
ref = pkg_resources.resource_filename('bob.extension',
'data/test_dump_config2.py')
assert result.exit_code == 0, (result.exit_code, result.output)
_assert_config_dump(ref, '08/07/2018')
......@@ -35,7 +35,7 @@ def _run(package, run_call):
assert os.path.exists(_bin('python'))
# nosetests
subprocess.call(['python', _bin('nosetests'), '-sv'])
subprocess.call(['python', _bin('nosetests'), '-sv', 'bob.example.{0}'.format(package)])
# check that the call is working
subprocess.call(['python', _bin(run_call[0])] + run_call[1:])
......
......@@ -2,6 +2,7 @@
from .rc_config import _loadrc, ENVNAME
from .scripts import main_cli
from .scripts.click_helper import assert_click_runner_result
from click.testing import CliRunner
import os
import pkg_resources
......@@ -27,19 +28,19 @@ def test_bob_config():
# test config show
result = runner.invoke(main_cli, ['config', 'show'])
assert result.exit_code == 0, result.exit_code
assert_click_runner_result(result, 0)
assert 'defaults-config' in result.output, result.output
assert open(defaults_config).read() in result.output, result.output
# test config get (existing key)
result = runner.invoke(main_cli,
['config', 'get', 'bob.db.atnt.directory'])
assert result.exit_code == 0, result.exit_code
assert_click_runner_result(result, 0)
assert result.output == '/home/bob/databases/atnt\n', result.output
# test config get (non-existing key)
result = runner.invoke(main_cli, ['config', 'get', 'bob.db.atnt'])
assert result.exit_code == 1, result.exit_code
assert_click_runner_result(result, 1)
# test config set
runner = CliRunner()
......@@ -53,14 +54,14 @@ def test_bob_config():
env={
ENVNAME: bobrcfile
})
assert result.exit_code == 0, result.exit_code
assert_click_runner_result(result, 0)
# read the config back to make sure it is ok.
result = runner.invoke(
main_cli, ['config', 'show'], env={
ENVNAME: bobrcfile
})
assert result.exit_code == 0, result.exit_code
assert_click_runner_result(result, 0)
expected_output = '''Displaying `bobrc':
{
"bob.db.atnt.directory": "/home/bob/databases/orl_faces"
......
......@@ -33,6 +33,10 @@ test:
- {{ name }}
commands:
- bob_dependecy_graph.py --help
- bob -h
- bob --help
- bob config -h
- bob config --help
- nosetests --with-coverage --cover-package={{ name }} -sv {{ name }} --exclude=test_extensions
- sphinx-build -aEW {{ project_dir }}/doc {{ project_dir }}/sphinx
- sphinx-build -aEb doctest {{ project_dir }}/doc sphinx
......
......@@ -3,53 +3,40 @@
import logging
import click
from bob.extension.scripts.click_helper import (
verbosity_option, ConfigCommand, ResourceOption)
verbosity_option, ConfigCommand, ResourceOption, log_parameters)
logger = logging.getLogger(__name__)
@click.command(entry_point_group='bob.bio.config', cls=ConfigCommand)
ANNOTATE_EPILOG = '''\b
Examples:
$ bob bio annotate -vvv -d <database> -a <annotator> -o /tmp/annotations
$ jman submit --array 64 -- bob bio annotate ... --array 64
'''
@click.command(entry_point_group='bob.bio.config', cls=ConfigCommand,
epilog=ANNOTATE_EPILOG)
@click.option('--database', '-d', required=True, cls=ResourceOption,
entry_point_group='bob.bio.database')
entry_point_group='bob.bio.database',
help='''The database that you want to annotate.''')
@click.option('--annotator', '-a', required=True, cls=ResourceOption,
entry_point_group='bob.bio.annotator')
@click.option('--output-dir', '-o', required=True, cls=ResourceOption)
@click.option('--force', '-f', is_flag=True, cls=ResourceOption)
entry_point_group='bob.bio.annotator',
help='A callable that takes the database and a sample (biofile) '
'of the database and returns the annotations in a dictionary.')
@click.option('--output-dir', '-o', required=True, cls=ResourceOption,
help='The directory to save the annotations.')
@click.option('--force', '-f', is_flag=True, cls=ResourceOption,
help='Whether to overwrite existing annotations.')
@click.option('--array', type=click.INT, default=1, cls=ResourceOption,
help='Use this option alongside gridtk to submit this script as '
'an array job.')
@verbosity_option(cls=ResourceOption)
def annotate(database, annotator, output_dir, force, **kwargs):
def annotate(database, annotator, output_dir, force, array, **kwargs):
"""Annotates a database.
The annotations are written in text file (json) format which can be read
back using :any:`bob.db.base.read_annotation_file` (annotation_type='json')
\b
Parameters
----------
database : :any:`bob.bio.database`
The database that you want to annotate. Can be a ``bob.bio.database``
entry point or a path to a Python file which contains a variable
named `database`.
annotator : callable
A function that takes the database and a sample (biofile) of the
database and returns the annotations in a dictionary. Can be a
``bob.bio.annotator`` entry point or a path to a Python file which
contains a variable named `annotator`.
output_dir : str
The directory to save the annotations.
force : bool, optional
Wether to overwrite existing annotations.
verbose : int, optional
Increases verbosity (see help for --verbose).
\b
[CONFIG]... Configuration files. It is possible to pass one or
several Python files (or names of ``bob.bio.config``
entry points) which contain the parameters listed
above as Python variables. The options through the
command-line (see below) will override the values of
configuration files.
"""
print('database', database)
print('annotator', annotator)
print('force', force)
print('output_dir', output_dir)
print('kwargs', kwargs)
log_parameters(logger)
......@@ -245,47 +245,49 @@ example:
This will produce the following help message to the users::
Usage: bob annotate [OPTIONS] [CONFIG]...
Annotates a database. The annotations are written in text file (json)
format which can be read back using
:any:`bob.db.base.read_annotation_file` (annotation_type='json')
Parameters
----------
database : :any:`bob.bio.database`
The database that you want to annotate. Can be a ``bob.bio.database``
entry point or a path to a Python file which contains a variable
named `database`.
annotator : callable
A function that takes the database and a sample (biofile) of the
database and returns the annotations in a dictionary. Can be a
``bob.bio.annotator`` entry point or a path to a Python file which
contains a variable named `annotator`.
output_dir : str
The directory to save the annotations.
force : bool, optional
Wether to overwrite existing annotations.
verbose : int, optional
Increases verbosity (see help for --verbose).
[CONFIG]... Configuration files. It is possible to pass one or
several Python files (or names of ``bob.bio.config``
entry points) which contain the parameters listed
above as Python variables. The options through the
command-line (see below) will override the values of
configuration files.
Options:
-d, --database TEXT
-a, --annotator TEXT
-o, --output-dir TEXT
-f, --force
-v, --verbose Increase the verbosity level from 0 (only error
messages) to 1 (warnings), 2 (log messages), 3 (debug
information) by adding the --verbose option as often
as desired (e.g. '-vvv' for debug).
--help Show this message and exit.
Usage: bob bio annotate [OPTIONS] [CONFIG]...
Annotates a database.
The annotations are written in text file (json) format which can be read
back using :any:`bob.db.base.read_annotation_file`
(annotation_type='json')
It is possible to pass one or several Python files (or names of
``bob.bio.config`` entry points or module names) as CONFIG arguments to
the command line which contain the parameters listed below as Python
variables. The options through the command-line (see below) will override
the values of configuration files. You can run this command with
``<COMMAND> -H example_config.py`` to create a template config file.
Options:
-d, --database TEXT The database that you want to annotate. Can
be a ``bob.bio.database`` entry point, a
module name, or a path to a Python file
which contains a variable named `database`.
-a, --annotator TEXT A callable that takes the database and a
sample (biofile) of the database and returns
the annotations in a dictionary. Can be a
``bob.bio.annotator`` entry point, a module
name, or a path to a Python file which
contains a variable named `annotator`.
-o, --output-dir TEXT The directory to save the annotations.
-f, --force Whether to overwrite existing annotations.
--array INTEGER Use this option alongside gridtk to submit
this script as an array job.
databases.
-v, --verbose Increase the verbosity level from 0 (only
error messages) to 1 (warnings), 2 (log
messages), 3 (debug information) by adding
the --verbose option as often as desired
(e.g. '-vvv' for debug).
-H, --dump-config FILENAME Name of the config file to be generated
-?, -h, --help Show this message and exit.
Examples:
$ bob bio annotate -vvv -d <database> -a <annotator> -o /tmp/annotations
$ jman submit --array 64 -- bob bio annotate ... --array 64
This script takes configuration files (``CONFIG``) and command line options
......@@ -328,6 +330,8 @@ Below you can see several ways that this script can be invoked:
$ bob annotate bob.package.config_with_all_parameters -o /tmp
# below, each resource option can be loaded through config loading mechanism too.
$ bob annotate -d /path/to/config/database.py -a bob.package.annotate.config --output /tmp
# Using the command below users can generate a template config file
$ bob annotate -H example_config.py
As you can see the command line interface can accept its inputs through several
different mechanism. Normally to keep things simple, you would encourage users
......
......@@ -77,6 +77,8 @@ Scripts
bob.extension.scripts.click_helper.list_float_option
bob.extension.scripts.click_helper.open_file_mode_option
bob.extension.scripts.click_helper.AliasedGroup
bob.extension.scripts.click_helper.log_parameters
bob.extension.scripts.click_helper.assert_click_runner_result
Core Functionality
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment