diff --git a/bob/extension/__init__.py b/bob/extension/__init__.py index 036945ea9d3b8c30dcb42b1d4804ab2802ea825f..3a3fc7dea2f3ba7b2722a3eaf455e3154c7fd622 100644 --- a/bob/extension/__init__.py +++ b/bob/extension/__init__.py @@ -20,7 +20,7 @@ from .pkgconfig import pkgconfig from .boost import boost from .utils import uniq, uniq_paths, find_executable, find_library from .cmake import CMakeListsGenerator -from .config import _loadrc +from .rc_config import _loadrc __version__ = pkg_resources.require(__name__)[0].version @@ -762,6 +762,8 @@ def get_config(package=__name__, externals=None, api_version=None): # Loads the rc user preferences rc = _loadrc() +"""The content of the global configuration file loaded as a dictionary. +The value for any non-existing key is ``None``.""" # gets sphinx autodoc done right - don't remove it __all__ = [_ for _ in dir() if not _.startswith('_')] diff --git a/bob/extension/config.py b/bob/extension/config.py index 70995873908d3e715d2ce83514fb06777e8eab3c..ee8cf80ee2c933e5e96e5b6d538de6bc4ee8f7f5 100644 --- a/bob/extension/config.py +++ b/bob/extension/config.py @@ -1,21 +1,14 @@ #!/usr/bin/env python # vim: set fileencoding=utf-8 : -'''Functionality to implement config file parsing and loading''' +'''Functionality to implement python-based config file parsing and loading. +''' -import os import imp -import copy import logging logger = logging.getLogger(__name__) -ENVNAME = 'BOBRC' -"""Name of environment variable to look for an alternative for the RC file""" - -RCFILENAME = '.bobrc.py' -"""Default name to be used for the RC file to load""" - def _load_context(path, mod): '''Loads the Python file as module, returns a resolved context @@ -77,7 +70,8 @@ def load(paths, context=None): ''' mod = imp.new_module('config') - if context is not None: mod.__dict__.update(context) + if context is not None: + mod.__dict__.update(context) for k in paths: logger.debug("Loading configuration file `%s'...", k) @@ -85,43 +79,4 @@ def load(paths, context=None): # notice context.__dict__ will be gone as soon as the module is deleted # we need to shallow-copy it to prevent this - return dict((k,v) for k,v in mod.__dict__.items() if not k.startswith('_')) - - -def _loadrc(context=None): - '''Loads the default configuration file, or an override if provided - - This method will load **exactly** one (global) resource configuration file in - this fixed order of preference: - - 1. A path pointed by the environment variable BOBRC - 2. A file named :py:attr:`RCFILENAME` on your HOME directory - - - Parameters: - - context (:py:class:`dict`, Optional): A dictionary that establishes the - context (existing variables) in which the RC file will be loaded. By - default, this value is set to ``None`` which indicates no previous - context. - - - Returns: - - dict: A dictionary of key-values representing the resolved context, after - loading the provided modules and resolving all variables. - - ''' - - if 'BOBRC' in os.environ: - path = os.environ['BOBRC'] - elif os.path.exists(os.path.expanduser('~' + os.sep + RCFILENAME)): - path = os.path.expanduser('~' + os.sep + RCFILENAME) - else: - logger.debug("No RC file found") - return {} - - logger.debug("Loading RC file `%s'...", path) - mod = imp.new_module('rc') - if context is not None: mod.__dict__.update(context) - return _load_context(path, mod) + return dict((k, v) for k, v in mod.__dict__.items() if not k.startswith('_')) diff --git a/bob/extension/data/defaults-config b/bob/extension/data/defaults-config new file mode 100644 index 0000000000000000000000000000000000000000..f332e2b115a3091f8162ae94d5404bbd03ad614e --- /dev/null +++ b/bob/extension/data/defaults-config @@ -0,0 +1,4 @@ +{ + "bob.db.atnt.directory": "/home/bob/databases/atnt", + "bob.db.mobio.directory": "/home/bob/databases/mobio" +} diff --git a/bob/extension/data/defaults-config.py b/bob/extension/data/defaults-config.py deleted file mode 100644 index 24edac37e61e2e72ba432592642431b1e0f2e842..0000000000000000000000000000000000000000 --- a/bob/extension/data/defaults-config.py +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env python -# vim: set fileencoding=utf-8 : - -'''Advanced configuration for my jet pack''' - -import os as _os -# Objects whose name start with an underscore are not returned by ``load()`` -_model = _os.path.expanduser('~/.jet-pack-model.hdf5') - -bob_db_atnt = { - 'directory': '/directory/to/root/of/atnt-database', - 'extension': '.ppm', -} diff --git a/bob/extension/data/load-config.py b/bob/extension/data/load-config.py index 5bf961b1a6acde5c43a4e3f58bce84a0d0cde139..063cdb7fd067e022d0d0dd7e14776fe377d6953b 100644 --- a/bob/extension/data/load-config.py +++ b/bob/extension/data/load-config.py @@ -1 +1 @@ -bob_db_atnt['extension'] = '.hdf5' +b = b + 3 diff --git a/bob/extension/rc_config.py b/bob/extension/rc_config.py new file mode 100644 index 0000000000000000000000000000000000000000..3b5eb97020b51fe527b68ab421773bd398e47239 --- /dev/null +++ b/bob/extension/rc_config.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python +# vim: set fileencoding=utf-8 : + +'''Implements a global configuration system for bob using json.''' + +from collections import defaultdict +import json +import logging +import os + +logger = logging.getLogger(__name__) + +ENVNAME = 'BOBRC' +"""Name of environment variable to look for an alternative for the RC file""" + +RCFILENAME = '.bobrc' +"""Default name to be used for the RC file to load""" + + +def _get_rc_path(): + """Returns the path to the bob rc file. + This method will return the path to **exactly** one (global) resource + configuration file in this fixed order of preference: + + 1. A path pointed by the environment variable BOBRC + 2. A file named :py:attr:`RCFILENAME` on your HOME directory + + Returns + ------- + str + The path to the rc file. + """ + if 'BOBRC' in os.environ: + path = os.environ['BOBRC'] + else: + path = os.path.expanduser('~' + os.sep + RCFILENAME) + + return path + + +def _loadrc(): + '''Loads the default configuration file, or an override if provided + + This method will load **exactly** one (global) resource configuration file as + returned by :py:func:`_get_rc_path`. + + Returns: + + dict: A dictionary of key-values representing the resolved context, after + loading the provided modules and resolving all variables. + + ''' + + def _default_none_dict(dct): + dct2 = defaultdict(lambda: None) + dct2.update(dct) + return dct2 + + path = _get_rc_path() + if not os.path.exists(path): + logger.debug("No RC file found") + return _default_none_dict({}) + + logger.debug("Loading RC file `%s'...", path) + + with open(path, 'rt') as f: + context = json.load(f, object_hook=_default_none_dict) + return context + + +def _dumprc(context, f): + """Saves the context into the global rc file. + + Parameters + ---------- + context : dict + All the configurations to save into the rc file. + f : obj + An object that provides a ``f.write()`` function. + """ + json.dump(context, f, sort_keys=True, indent=4, separators=(',', ': ')) + + +def _saverc(context): + """Saves the context into the global rc file. + + Parameters + ---------- + context : dict + All the configurations to save into the rc file. + """ + path = _get_rc_path() + with open(path, 'wt') as f: + _dumprc(context, f) diff --git a/bob/extension/scripts/__init__.py b/bob/extension/scripts/__init__.py index 4b9f68dec6a521b9d90c5e3f5b9f38a7e8561580..15233a7c5772918e1c44783748a0f426e9625359 100644 --- a/bob/extension/scripts/__init__.py +++ b/bob/extension/scripts/__init__.py @@ -1,5 +1,6 @@ from .new_version import main as new_version from .dependency_graph import main as dependency_graph +from .main_cli import main as main_cli # gets sphinx autodoc done right - don't remove it __all__ = [_ for _ in dir() if not _.startswith('_')] diff --git a/bob/extension/scripts/config.py b/bob/extension/scripts/config.py new file mode 100644 index 0000000000000000000000000000000000000000..004448e43627cd218eb6b77ec42bf40f9b19683c --- /dev/null +++ b/bob/extension/scripts/config.py @@ -0,0 +1,102 @@ +"""The manager for bob's main configuration. +""" +from __future__ import (absolute_import, division, + print_function, unicode_literals) +from .. import rc +from ..rc_config import _saverc, _dumprc, _get_rc_path +from argparse import RawDescriptionHelpFormatter +import logging +import sys + + +def setup_parser(parser): + from . import __doc__ as docs + # creates a top-level parser for this database + top_level = parser.add_parser('config', + formatter_class=RawDescriptionHelpFormatter, + help=docs) + + subparsers = top_level.add_subparsers(title="subcommands") + + # add commands + show_command(subparsers) + get_command(subparsers) + set_command(subparsers) + + return subparsers + + +def show(arguments=None): + """Shows the content of bob's global configuration file. + """ + print("The configuration is located at {}".format(_get_rc_path())) + print("It's content are:") + _dumprc(rc, sys.stdout) + + +def show_command(subparsers): + parser = subparsers.add_parser('show', help=show.__doc__) + parser.set_defaults(func=show) + return parser + + +def get(arguments): + """Gets the specified configuration from bob's global configuration file. + + Parameters + ---------- + arguments : argparse.Namespace + A set of arguments passed by the command-line parser + + + Returns + ------- + int + A POSIX compliant return value of ``0`` if the key exists, or ``1`` + otherwise. + """ + value = rc[arguments.key] + if value is None: + return 1 + print(value) + return 0 + + +def get_command(subparsers): + parser = subparsers.add_parser('get', help=get.__doc__) + parser.add_argument("key", help="The requested key.") + parser.set_defaults(func=get) + return parser + + +def set(arguments): + """Sets the specified configuration to the provided value in bob's global + configuration file. + + Parameters + ---------- + arguments : argparse.Namespace + A set of arguments passed by the command-line parser + + + Returns + ------- + int + A POSIX compliant return value of ``0`` if the operation is successful, + or ``1`` otherwise. + """ + try: + rc[arguments.key] = arguments.value + _saverc(rc) + except Exception: + logging.warn("Could not configure the rc file", exc_info=True) + return 1 + return 0 + + +def set_command(subparsers): + parser = subparsers.add_parser('set', help=set.__doc__) + parser.add_argument("key", help="The key.") + parser.add_argument("value", help="The value.") + parser.set_defaults(func=set) + return parser diff --git a/bob/extension/scripts/main_cli.py b/bob/extension/scripts/main_cli.py new file mode 100644 index 0000000000000000000000000000000000000000..0f120cf7aa214740687ee9d6f17881473b4f1b5b --- /dev/null +++ b/bob/extension/scripts/main_cli.py @@ -0,0 +1,47 @@ +"""This is the main entry to bob's scripts. +As of now it just supports `bob config`. +""" + +from __future__ import (absolute_import, division, + print_function, unicode_literals) +import argparse + +epilog = """ For a list of available commands: + >>> %(prog)s --help + + For a list of actions on each command: + >>> %(prog)s <command> --help +""" + + +def create_parser(**kwargs): + """Creates a parser for the central manager taking into consideration the + options for every module that can provide those.""" + + parser = argparse.ArgumentParser(**kwargs) + subparsers = parser.add_subparsers(title='commands') + + return parser, subparsers + + +def main(argv=None): + from argparse import RawDescriptionHelpFormatter + parser, subparsers = create_parser( + description=__doc__, epilog=epilog, + formatter_class=RawDescriptionHelpFormatter) + + # for now there is only the config command so we'll just add it here. + # Normally, this would be added in a better way in future. Maybe something + # similar to bob_dbmanage.py + from .config import setup_parser + setup_parser(subparsers) + + args = parser.parse_args(args=argv) + if hasattr(args, 'func'): + return args.func(args) + else: + return parser.parse_args(args=['--help']) + + +if __name__ == '__main__': + main() diff --git a/bob/extension/test_config.py b/bob/extension/test_config.py index 66c7ed52903702a0168cd8cde858ce7aa584ff90..c503e25952b5f82fb9abfe8ba6d2423a460beb6b 100644 --- a/bob/extension/test_config.py +++ b/bob/extension/test_config.py @@ -1,15 +1,14 @@ #!/usr/bin/env python # vim: set fileencoding=utf-8 : -'''Tests for the config module and functionality''' +'''Tests for the python-based config functionality''' +from .config import load import os import pkg_resources path = pkg_resources.resource_filename('bob.extension', 'data') -from .config import load, _loadrc, ENVNAME - def test_basic(): @@ -23,23 +22,9 @@ def test_basic_with_context(): assert c == {'a': 1, 'b': 3, 'd': 35} -def test_defaults(): - - c = load([os.path.join(path, 'defaults-config.py')]) - assert c == {'bob_db_atnt': {'directory': '/directory/to/root/of/atnt-database', 'extension': '.ppm'} } - - def test_chain_loading(): - file1 = os.path.join(path, 'defaults-config.py') + file1 = os.path.join(path, 'basic-config.py') file2 = os.path.join(path, 'load-config.py') c = load([file1, file2]) - assert c == {'bob_db_atnt': {'directory': '/directory/to/root/of/atnt-database', 'extension': '.hdf5'} } - - -def test_rc_env(): - - os.environ[ENVNAME] = os.path.join(path, 'basic-config.py') - c = _loadrc() #should load from environment variable - assert c.a == 1 - assert c.b == 3 + assert c == {'a': 1, 'b': 6} diff --git a/bob/extension/test_rc.py b/bob/extension/test_rc.py new file mode 100644 index 0000000000000000000000000000000000000000..27f6e360624c91c2ff091b6f4265220f0c395cf4 --- /dev/null +++ b/bob/extension/test_rc.py @@ -0,0 +1,31 @@ +'''Tests for the global bob's configuration functionality''' + +from .rc_config import _loadrc, ENVNAME +from .scripts import main_cli +import os +import pkg_resources +import tempfile +path = pkg_resources.resource_filename('bob.extension', 'data') + + +def test_rc_env(): + + os.environ[ENVNAME] = os.path.join(path, 'defaults-config') + c = _loadrc() # should load from environment variable + REFERENCE = { + "bob.db.atnt.directory": "/home/bob/databases/atnt", + "bob.db.mobio.directory": "/home/bob/databases/mobio" + } + + assert c == REFERENCE + assert c['random'] is None + + +def test_bob_config(): + os.environ[ENVNAME] = os.path.join(path, 'defaults-config') + main_cli(['config', 'get', 'bob.db.atnt.directory']) + with tempfile.NamedTemporaryFile('wt') as f: + os.environ[ENVNAME] = f.name + main_cli(['config', 'set', 'bob.db.atnt.directory', + '/home/bob/databases/atnt']) + main_cli(['config', 'show']) diff --git a/doc/config.rst b/doc/config.rst index 0bf0bcabfdf394a48318379147af463367bcaa63..a484d589e15021db7aecafdf875a8f94e8b09842 100644 --- a/doc/config.rst +++ b/doc/config.rst @@ -1,8 +1,8 @@ .. _bob.extension.config: -====================== - Configuration System -====================== +=================================== + Python-based Configuration System +=================================== This package also provides a configuration system that can be used by packages in the |project|-echosystem. The configuration system is pretty simple and uses @@ -50,7 +50,7 @@ Then, the object ``configuration`` would look like this: The configuration file does not have to limit itself to simple Pythonic -operations, you can import modules and more. +operations, you can import modules, define functions and more. .. note:: @@ -63,16 +63,6 @@ operations, you can import modules and more. configuration file. -.. note:: - - Since configuration files are written in Python, you can execute full python - programs while configuring, import modules, create classes and more. - **However**, it is recommended that you keep the configuration files simple. - Avoid defining classes in the configuration files. A recommended - configuration file normally would only contain ``dict, list, tuple, str, - int, float, True, False, None`` Python objects. - - Chain Loading ------------- @@ -81,20 +71,20 @@ passing iterables with more than one filename to :py:func:`bob.extension.config.load`. Suppose we have two configuration files which must be loaded in sequence: -.. literalinclude:: ../bob/extension/data/defaults-config.py - :caption: "defaults-config.py" (first to be loaded) +.. literalinclude:: ../bob/extension/data/basic-config.py + :caption: "basic-config.py" (first to be loaded) :language: python :linenos: .. literalinclude:: ../bob/extension/data/load-config.py - :caption: "load-config.py" (loaded after defaults-config.py) + :caption: "load-config.py" (loaded after basic-config.py) :language: python :linenos: Then, one can chain-load them like this: -.. testsetup:: defaults-config +.. testsetup:: basic-config import os import pkg_resources @@ -104,71 +94,17 @@ Then, one can chain-load them like this: from bob.extension.config import load -.. doctest:: defaults-config +.. doctest:: basic-config >>> #the variable `path` points to <path-to-bob.extension's root>/data - >>> file1 = os.path.join(path, 'defaults-config.py') + >>> file1 = os.path.join(path, 'basic-config.py') >>> file2 = os.path.join(path, 'load-config.py') >>> configuration = load([file1, file2]) >>> print(json.dumps(configuration, indent=2, sort_keys=True)) # doctest: +NORMALIZE_WHITESPACE { - "bob_db_atnt": { - "directory": "/directory/to/root/of/atnt-database", - "extension": ".hdf5" - } + "a": 1, + "b": 6 } -The user wanting to override defaults needs to manage the overriding and the +The user wanting to override the values needs to manage the overriding and the order in which the override happens. - - -Defaults and RC Parameters --------------------------- - -When this package loads, it will automatically search for a file named -``${HOME}/.bobrc.py``. If it finds it, it will load this python module and make -its contents available inside the built-in module ``bob.extension.rc``. The -path of the file that will be loaded can be overridden by an environment -variable named ``${BOBRC}``. - -While this configuration system by itself does not make assumptions about your -rc strategy, we recommend to organize values in a sensible manner which relates -to the package name. Package-based defaults may be, for example, the directory -where raw data files for a particular ``bob.db`` are installed or the -verbosity-level logging messages should have. - -Here is a possible example for the package ``bob.db.atnt``: - -.. literalinclude:: ../bob/extension/data/defaults-config.py - :caption: "defaults-config.py" - :language: python - :linenos: - - -.. testsetup:: rc-config - - import os - import pkg_resources - path = pkg_resources.resource_filename('bob.extension', 'data') - import json - - from bob.extension.config import _loadrc - - -When loaded, this configuration file produces the following result: - -.. doctest:: rc-config - - >>> #the variable `path` points to <path-to-bob.extension's root>/data - >>> os.environ['BOBRC'] = os.path.join(path, 'defaults-config.py') - >>> mod = _loadrc() - >>> #notice `mod` is a normal python module - >>> print(mod) - <module 'rc'...> - >>> print(json.dumps(dict((k,v) for k,v in mod.__dict__.items() if not k.startswith('_')), indent=2, sort_keys=True)) # doctest: +NORMALIZE_WHITESPACE - { - "bob_db_atnt": { - "directory": "/directory/to/root/of/atnt-database", - "extension": ".ppm" - } - } diff --git a/doc/index.rst b/doc/index.rst index c9ec962f695fca0fc1b0630c9081b0b1226c2067..f6104328e119f8a2073bcb19e886d744473ee8c1 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -39,6 +39,7 @@ Documentation cplusplus_library documenting additional + rc config py_api cpp_api diff --git a/doc/py_api.rst b/doc/py_api.rst index 7ae3103a4919cc98fd0974f12c7e627698dd76d1..eb75b8989d18e3f6a3edfdeb2074dd7242a88727 100644 --- a/doc/py_api.rst +++ b/doc/py_api.rst @@ -18,6 +18,7 @@ Configuration ------------- .. automodule:: bob.extension.config +.. automodule:: bob.extension.rc_config Scripts diff --git a/doc/rc.rst b/doc/rc.rst new file mode 100644 index 0000000000000000000000000000000000000000..7271f3637de680133cf17525cde413cb0347100c --- /dev/null +++ b/doc/rc.rst @@ -0,0 +1,75 @@ +.. _bob.extension.rc: + +========================================= + |project|'s Global Configuration System +========================================= + +|project| provides a global configuration system for users. +The configuration file is located in ``${HOME}/.bobrc``. +The path of the file can be overridden by an environment variable named +``${BOBRC}``. +This configuration can be used to customize the behavior of |project| libraries +and also to specify the location of extra data on user's computers. +For example, the extra data can be the location of the raw data of a database +that you may want to access in |project|. + +The configuration file is accessible through the ``bob config`` command line:: + + $ bob config --help + +You can view the content of the configuration file:: + + $ bob config show + +You can view the content of a specific variable in the configuration file:: + + $ bob config get bob.db.atnt.directory + /home/bobuser/databases/atnt + +You can change the value of a specific variable in the configuration file:: + + $ bob config set bob.db.atnt.directory /home/bobuser/databases/orl_faces + + +The rest of this guide explains how developers of |project| packages should +take advantage of the configuration system. + + +For Developers +-------------- + +The configuration file is automatically loaded and is available as +:py:attr:`bob.extension.rc`. +It's main usage (for now) is to automatically load find where the databases +are located. +Here is an example on how the configuration system can be potentially used: + +.. doctest:: rc-config + + >>> from bob.extension import rc + >>> class AtntDatabase: + ... def __init__(self, original_directory=None): + ... if original_directory is None: + ... original_directory = rc['bob.db.atnt.directory'] + ... self.original_directory = original_directory + +:py:attr:`bob.extension.rc` is a dictionary which returns ``None`` for +non-existing keys so you don't have to worry about exception handling for non- +existing keys. + +.. warning:: + + The variables of each package **must** start with the name of package. For + example, if the variable is used in ``bob.db.atnt``, its name should be + ``bob.db.atnt.<name>``. This is required to avoid variable name clashes + between hundreds of |project| package. Remember that your package is **not** + special and **should** follow this rule. + +In the documentation of your package do not explain how the configuration +system works. Just provide an example command on how the variable should be +configured:: + + $ bob config set bob.db.mydatabase.directory /path/to/mydatabase + +And point to this page for more information. You can point to this page using +the ``ref`` command: ``:ref:`bob.extension.rc``` diff --git a/setup.py b/setup.py index 4f96474a18e9ecb07c0f40ae1b9a2a21ff860855..745adf7705743bb0cf34ca240df2bee6b57b0d12 100644 --- a/setup.py +++ b/setup.py @@ -37,6 +37,7 @@ setup( entry_points = { 'console_scripts': [ + 'bob = bob.extension.scripts:main_cli', 'bob_new_version.py = bob.extension.scripts:new_version', 'bob_dependecy_graph.py = bob.extension.scripts:dependency_graph', ],