Implementation of the bob script using click

bob/extension/log.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+# Andre Anjos <andre.anjos@idiap.ch>
+# Sat 19 Oct 12:51:01 2013
+"""Sets-up logging, centrally for Bob.
+"""
+
+import sys
+import logging
+
+# get the default root logger of Bob
+_logger = logging.getLogger('bob')
+
+# by default, warning and error messages should be written to sys.stderr
+_warn_err = logging.StreamHandler(sys.stderr)
+_warn_err.setLevel(logging.WARNING)
+_logger.addHandler(_warn_err)
+
+# debug and info messages are written to sys.stdout
+
+
+class _InfoFilter:
+  def filter(self, record):
+    return record.levelno <= logging.INFO
+
+
+_debug_info = logging.StreamHandler(sys.stdout)
+_debug_info.setLevel(logging.DEBUG)
+_debug_info.addFilter(_InfoFilter())
+_logger.addHandler(_debug_info)
+
+
+# helper functions to instantiate and set-up logging
+def setup(logger_name,
+          format="%(name)s@%(asctime)s -- %(levelname)s: %(message)s"):
+  """This function returns a logger object that is set up to perform logging
+  using Bob loggers.
+
+  Parameters
+  ----------
+  logger_name : str
+      The name of the module to generate logs for
+  format : :obj:`str`, optional
+      The format of the logs, see :py:class:`logging.LogRecord` for more
+      details. By default, the log contains the logger name, the log time, the
+      log level and the massage.
+
+  Returns
+  -------
+  logger : :py:class:`logging.Logger`
+      The logger configured for logging. The same logger can be retrieved using
+      the :py:func:`logging.getLogger` function.
+  """
+  # generate new logger object
+  logger = logging.getLogger(logger_name)
+
+  # add log the handlers if not yet done
+  if not logger_name.startswith("bob") and not logger.handlers:
+    logger.addHandler(_warn_err)
+    logger.addHandler(_debug_info)
+
+  # this formats the logger to print the desired information
+  formatter = logging.Formatter(format)
+  # we have to set the formatter to all handlers registered in the current
+  # logger
+  for handler in logger.handlers:
+    handler.setFormatter(formatter)
+
+  # set the same formatter for bob loggers
+  for handler in _logger.handlers:
+    handler.setFormatter(formatter)
+
+  return logger
+
+
+def set_verbosity_level(logger, level):
+  """Sets the log level for the given logger.
+
+  Parameters
+  ----------
+  logger : :py:class:`logging.Logger` or str
+      The logger to generate logs for, or the name  of the module to generate
+      logs for.
+  level : int
+      Possible log levels are: 0: Error; 1: Warning; 2: Info; 3: Debug.
+
+  Raises
+  ------
+  ValueError
+      If the level is not in range(0, 4).
+  """
+  if level not in range(0, 4):
+    raise ValueError(
+        "The verbosity level %d does not exist. Please reduce the number of "
+        "'--verbose' parameters in your command line" % level)
+  # set up the verbosity level of the logging system
+  log_level = {
+      0: logging.ERROR,
+      1: logging.WARNING,
+      2: logging.INFO,
+      3: logging.DEBUG
+  }[level]
+
+  # set this log level to the logger with the specified name
+  if isinstance(logger, str):
+    logger = logging.getLogger(logger)
+  logger.setLevel(log_level)
+  # set the same log level for the bob logger
+  _logger.setLevel(log_level)
+
+
+__all__ = [_ for _ in dir() if not _.startswith('_')]

bob/extension/rc_config.py

@@ -67,18 +67,21 @@ def _loadrc():
     return json.load(f, object_hook=_default_none_dict)
 
 
-def _dumprc(context, f):
-  """Saves the context into the global rc file.
+def _rc_to_str(context):
+  """Converts the configurations into a pretty JSON formatted string.
 
   Parameters
   ----------
   context : dict
       All the configurations to save into the rc file.
-  f : obj
-      An object that provides a ``f.write()`` function.
+
+  Returns
+  -------
+  str
+      The configurations in a JSON formatted string.
   """
 
-  json.dump(context, f, sort_keys=True, indent=4, separators=(',', ': '))
+  return json.dumps(context, sort_keys=True, indent=4, separators=(',', ': '))
 
 
 def _saverc(context):
@@ -92,4 +95,4 @@ def _saverc(context):
 
   path = _get_rc_path()
   with open(path, 'wt') as f:
-    _dumprc(context, f)
+    f.write(_rc_to_str(context))

bob/extension/scripts/config.py

 """The manager for bob's main configuration.
 """
-from __future__ import (absolute_import, division,
-                        print_function, unicode_literals)
 from .. import rc
-from ..rc_config import _saverc, _dumprc, _get_rc_path
-from argparse import RawDescriptionHelpFormatter
+from ..rc_config import _saverc, _rc_to_str, _get_rc_path
 import logging
-import sys
+import click
 
+logger = logging.getLogger(__name__)
 
-def setup_parser(parser):
-    from . import __doc__ as docs
-    # creates a top-level parser for this database
-    top_level = parser.add_parser('config',
-                                  formatter_class=RawDescriptionHelpFormatter,
-                                  help=docs)
 
-    subparsers = top_level.add_subparsers(title="subcommands")
+@click.group()
+def config():
+    """The manager for bob's global configuration."""
+    # Load the config file again. This may be needed since the environment
+    # variable might change the config path during the tests. Otherwise, this
+    # should not be important.
+    logger.debug('Reloading the global configuration file.')
+    from ..rc_config import _loadrc
+    rc.clear()
+    rc.update(_loadrc())
 
-    # add commands
-    show_command(subparsers)
-    get_command(subparsers)
-    set_command(subparsers)
 
-    return subparsers
+@config.command()
+def show():
+    """Shows the configuration.
 
-
-def show(arguments=None):
-    """Shows the content of bob's global configuration file.
+    Displays the content of bob's global configuration file.
     """
-    print("Displaying `{}':".format(_get_rc_path()))
-    _dumprc(rc, sys.stdout)
-    print()
-
+    log_file = click.get_current_context().meta['log_file']
+    click.echo("Displaying `{}':".format(_get_rc_path()), log_file)
+    click.echo(_rc_to_str(rc), log_file)
 
-def show_command(subparsers):
-    parser = subparsers.add_parser('show', help=show.__doc__)
-    parser.set_defaults(func=show)
-    return parser
 
+@config.command()
+@click.argument('key')
+def get(key):
+    """Prints a key.
 
-def get(arguments):
-    """Gets the specified configuration from bob's global configuration file.
+    Retrieves the value of the requested key and displays it.
 
-    Parameters
-    ----------
-    arguments : argparse.Namespace
-        A set of arguments passed by the command-line parser
+    \b
+    Arguments
+    ---------
+    key : str
+        The key to return its value from the configuration.
 
-
-    Returns
-    -------
-    int
-        A POSIX compliant return value of ``0`` if the key exists, or ``1``
-        otherwise.
+    \b
+    Fails
+    -----
+    * If the key is not found.
     """
-    value = rc[arguments.key]
+    log_file = click.get_current_context().meta['log_file']
+    value = rc[key]
     if value is None:
-        return 1
-    print(value)
-    return 0
-
+        raise click.ClickException(
+            "The requested key `{}' does not exist".format(key))
+    click.echo(value, log_file)
 
-def get_command(subparsers):
-    parser = subparsers.add_parser('get', help=get.__doc__)
-    parser.add_argument("key", help="The requested key.")
-    parser.set_defaults(func=get)
-    return parser
 
+@config.command()
+@click.argument('key')
+@click.argument('value')
+def set(key, value):
+    """Sets the value for a key.
 
-def set(arguments):
-    """Sets the specified configuration to the provided value in bob's global
+    Sets the value of the specified configuration key in bob's global
     configuration file.
 
-    Parameters
-    ----------
-    arguments : argparse.Namespace
-        A set of arguments passed by the command-line parser
-
-
-    Returns
-    -------
-    int
-        A POSIX compliant return value of ``0`` if the operation is successful,
-        or ``1`` otherwise.
+    \b
+    Arguments
+    ---------
+    key : str
+        The key to set the value for.
+    value : str
+        The value of the key.
+
+    \b
+    Fails
+    -----
+    * If something goes wrong.
     """
     try:
-        rc[arguments.key] = arguments.value
+        rc[key] = value
         _saverc(rc)
     except Exception:
-        logging.warn("Could not configure the rc file", exc_info=True)
-        return 1
-    return 0
-
-
-def set_command(subparsers):
-    parser = subparsers.add_parser('set', help=set.__doc__)
-    parser.add_argument("key", help="The key.")
-    parser.add_argument("value", help="The value.")
-    parser.set_defaults(func=set)
-    return parser
+        logger.error("Could not configure the rc file", exc_info=True)
+        raise click.ClickException("Failed to change the configuration.")

bob/extension/scripts/main_cli.py

 """This is the main entry to bob's scripts.
-As of now it just supports `bob config`.
 """
-
-from __future__ import (absolute_import, division,
-                        print_function, unicode_literals)
-import argparse
-
-epilog = """  For a list of available commands:
-  >>> %(prog)s --help
-
-  For a list of actions on each command:
-  >>> %(prog)s <command> --help
-"""
-
-
-def create_parser(**kwargs):
-    """Creates a parser for the central manager taking into consideration the
-    options for every module that can provide those."""
-
-    parser = argparse.ArgumentParser(**kwargs)
-    subparsers = parser.add_subparsers(title='commands')
-
-    return parser, subparsers
-
-
-def main(argv=None):
-    from argparse import RawDescriptionHelpFormatter
-    parser, subparsers = create_parser(
-        description=__doc__, epilog=epilog,
-        formatter_class=RawDescriptionHelpFormatter)
-
-    # for now there is only the config command so we'll just add it here.
-    # Normally, this would be added in a better way in future. Maybe something
-    # similar to bob_dbmanage.py
-    from .config import setup_parser
-    setup_parser(subparsers)
-
-    args = parser.parse_args(args=argv)
-    if hasattr(args, 'func'):
-        return args.func(args)
-    else:
-        return parser.parse_args(args=['--help'])
-
-
-if __name__ == '__main__':
-    main()
+import pkg_resources
+import click
+from click_plugins import with_plugins
+from ..log import setup, set_verbosity_level
+logger = setup('bob')
+
+
+@with_plugins(pkg_resources.iter_entry_points('bob.cli'))
+@click.group()
+@click.option(
+    '-v',
+    '--verbose',
+    count=True,
+    help="Increase the verbosity level from 0 (only error messages) to 1 "
+    "(warnings), 2 (log messages), 3 (debug information) by adding the "
+    "--verbose option as often as desired (e.g. '-vvv' for debug).")
+@click.option(
+    '--log',
+    type=click.File('wb'),
+    help='Redirects the prints of the scripts to FILENAME.')
+def main(verbose, log):
+  """The main command line interface for bob.
+  Look below for available commands."""
+  set_verbosity_level(logger, verbose)
+  logger.debug("Logging of the `bob' logger was set to %d", verbose)
+  ctx = click.get_current_context()
+  ctx.meta['verbosity'] = verbose
+  ctx.meta['log_file'] = log

bob/extension/test_rc.py

@@ -2,30 +2,68 @@
 
 from .rc_config import _loadrc, ENVNAME
 from .scripts import main_cli
+from click.testing import CliRunner
 import os
 import pkg_resources
-import tempfile
 path = pkg_resources.resource_filename('bob.extension', 'data')
 
 
 def test_rc_env():
 
-    os.environ[ENVNAME] = os.path.join(path, 'defaults-config')
-    c = _loadrc()  # should load from environment variable
-    REFERENCE = {
-        "bob.db.atnt.directory": "/home/bob/databases/atnt",
-        "bob.db.mobio.directory": "/home/bob/databases/mobio"
-    }
+  os.environ[ENVNAME] = os.path.join(path, 'defaults-config')
+  c = _loadrc()  # should load from environment variable
+  REFERENCE = {
+      "bob.db.atnt.directory": "/home/bob/databases/atnt",
+      "bob.db.mobio.directory": "/home/bob/databases/mobio"
+  }
 
-    assert c == REFERENCE
-    assert c['random'] is None
+  assert c == REFERENCE
+  assert c['random'] is None
 
 
 def test_bob_config():
-    os.environ[ENVNAME] = os.path.join(path, 'defaults-config')
-    main_cli(['config', 'get', 'bob.db.atnt.directory'])
-    with tempfile.NamedTemporaryFile('wt') as f:
-        os.environ[ENVNAME] = f.name
-        main_cli(['config', 'set', 'bob.db.atnt.directory',
-                  '/home/bob/databases/atnt'])
-        main_cli(['config', 'show'])
+  defaults_config = os.path.join(path, 'defaults-config')
+  runner = CliRunner(env={ENVNAME: defaults_config})
+
+  # test config show
+  result = runner.invoke(main_cli, ['config', 'show'])
+  assert result.exit_code == 0, result.exit_code
+  assert 'defaults-config' in result.output, result.output
+  assert open(defaults_config).read() in result.output, result.output
+
+  # test config get (existing key)
+  result = runner.invoke(main_cli,
+                         ['config', 'get', 'bob.db.atnt.directory'])
+  assert result.exit_code == 0, result.exit_code
+  assert result.output == '/home/bob/databases/atnt\n', result.output
+
+  # test config get (non-existing key)
+  result = runner.invoke(main_cli, ['config', 'get', 'bob.db.atnt'])
+  assert result.exit_code == 1, result.exit_code
+
+  # test config set
+  runner = CliRunner()
+  with runner.isolated_filesystem():
+    bobrcfile = 'bobrc'
+    result = runner.invoke(
+        main_cli, [
+            'config', 'set', 'bob.db.atnt.directory',
+            '/home/bob/databases/orl_faces'
+        ],
+        env={
+            ENVNAME: bobrcfile
+        })
+    assert result.exit_code == 0, result.exit_code
+
+    # read the config back to make sure it is ok.
+    result = runner.invoke(
+        main_cli, ['config', 'show'], env={
+            ENVNAME: bobrcfile
+        })
+    assert result.exit_code == 0, result.exit_code
+    expected_output = '''Displaying `bobrc':
+{
+    "bob.db.atnt.directory": "/home/bob/databases/orl_faces"
+}
+'''
+    assert expected_output == result.output, result.output

bob/extension/utils.py

@@ -516,6 +516,7 @@ def link_documentation(additional_packages = ['python', 'numpy'], requirements_f
   _add_index('docopt', 'http://docopt.readthedocs.io/en/latest/')
   _add_index('scikit-image', 'http://scikit-image.org/docs/dev/')
   _add_index('pillow', 'http://pillow.readthedocs.io/en/latest/')
+  _add_index('click', 'http://click.pocoo.org/')
 
 
   # get the server for the other packages

doc/extra-intersphinx.txt

 python
+click
 bob.blitz
 bob.db.base
 bob.io.image

doc/framework.rst

@@ -180,5 +180,51 @@ The data may be further processed using a
           [ 1. ,  2. ,  3. ,  0.5,  1. ,  1.5]])
 
 
+.. _bob.extension.cli:
+
+Unified Command Line Mechanism
+------------------------------
+
+|project| comes with a command line called ``bob`` which provides a set of
+commands by default::
+
+   $ bob --help
+   Usage: bob [OPTIONS] COMMAND [ARGS]...
+
+     The main command line interface for bob. Look below for available
+     commands.
+
+   Options:
+     -v, --verbose   Increase the verbosity level from 0 (only error messages) to
+                     1 (warnings), 2 (log messages), 3 (debug information) by
+                     adding the --verbose option as often as desired (e.g. '-vvv'
+                     for debug).
+     --log FILENAME  Redirects the prints of the scripts to FILENAME.
+     --help          Show this message and exit.
+
+   Commands:
+     config  The manager for bob's global configuration.
+     ...
+
+This command line is implemented using click_. You can extend the commands of
+this script through setuptools entry points (this is implemented using
+`click-plugins`_). To do so you implement your command-line using click_
+independently; then, advertise it as a command under bob script using the
+``bob.cli`` entry point.
+
+.. note::
+
+   If you are still not sure how this must be done, maybe you don't know how
+   to use click_ yet.
+
+This feature is experimental and may change and break compatibility in future.
+For a best practice example, please look at how the ``bob config`` command is
+implemented:
+
+.. literalinclude:: ../bob/extension/scripts/config.py
+   :caption: "bob/extension/scripts/config.py" implementation of the ``bob config`` command.
+   :language: python
+
+
 .. include:: links.rst
 

doc/links.rst

@@ -8,6 +8,8 @@
 .. _bob: https://www.idiap.ch/software/bob
 .. _bob's installation: https://www.idiap.ch/software/bob/install
 .. _c++: http://www2.research.att.com/~bs/C++.html
+.. _click: http://click.pocoo.org/
+.. _click-plugins: https://github.com/click-contrib/click-plugins
 .. _distutils: https://docs.python.org/distutils/
 .. _git: http://git-scm.com/
 .. _gitlab: https://gitlab.idiap.ch/bob/

doc/nitpick-exceptions.txt

+# Not available in Python 2.7, but ok in Python 3.x
+py:exc ValueError
+
 # we don't link against setuptools manual
 py:class setuptools.extension.Extension
 py:class distutils.extension.Extension

doc/py_api.rst

@@ -96,6 +96,12 @@ Stacked Processors
     :special-members: __init__, __call__
 
 
+Logging
+-------
+
+.. automodule:: bob.extension.log
+
+
 Scripts
 -------
 

setup.py

@@ -2,7 +2,6 @@
 # vim: set fileencoding=utf-8 :
 # Andre Anjos <andre.anjos@idiap.ch>
 # Thu 20 Sep 2012 14:43:19 CEST
-
 """A package that contains a helper for Bob/Python C++ extension development
 """
 
@@ -11,13 +10,7 @@ from setuptools import setup, find_packages
 # Define package version
 version = open("version.txt").read().rstrip()
 
-requires = ['setuptools']
-import sys
-if sys.version_info[0] == 2 and sys.version_info[1] <= 6:
-  requires.append('importlib')
-
 setup(
-
     name="bob.extension",
     version=version,
     description="Building of Python/C++ extensions for Bob",
@@ -31,33 +24,32 @@ setup(
     include_package_data=True,
     zip_safe=False,
 
-
-
-    install_requires=requires,
-
-    entry_points = {
-      'console_scripts': [
-        'bob = bob.extension.scripts:main_cli',
-        'bob_new_version.py = bob.extension.scripts:new_version',
-        'bob_dependecy_graph.py = bob.extension.scripts:dependency_graph',
-      ],
-      # some test entry_points
-      'bob.extension.test_config_load': [
-        'basic_config = bob.extension.data.basic_config',
-        'resource_config = bob.extension.data.resource_config',
-        'subpackage_config = bob.extension.data.subpackage.config',
-      ],
+    install_requires=['setuptools', 'click'],
+
+    entry_points={
+        'console_scripts': [
+            'bob = bob.extension.scripts:main_cli',
+            'bob_new_version.py = bob.extension.scripts:new_version',
+            'bob_dependecy_graph.py = bob.extension.scripts:dependency_graph',
+        ],
+        'bob.cli': [
+            'config = bob.extension.scripts.config:config',
+        ],
+        # some test entry_points
+        'bob.extension.test_config_load': [
+            'basic_config = bob.extension.data.basic_config',
+            'resource_config = bob.extension.data.resource_config',
+            'subpackage_config = bob.extension.data.subpackage.config',
+        ],
     },
-
-    classifiers = [
-      'Framework :: Bob',
-      'Development Status :: 4 - Beta',
-      'Intended Audience :: Developers',
-      'License :: OSI Approved :: BSD License',
-      'Natural Language :: English',
-      'Programming Language :: Python',
-      'Programming Language :: Python :: 3',
-      'Topic :: Software Development :: Libraries :: Python Modules',
+    classifiers=[
+        'Framework :: Bob',
+        'Development Status :: 4 - Beta',
+        'Intended Audience :: Developers',
+        'License :: OSI Approved :: BSD License',
+        'Natural Language :: English',
+        'Programming Language :: Python',
+        'Programming Language :: Python :: 3',
+        'Topic :: Software Development :: Libraries :: Python Modules',
     ],
-
 )