Simplifies configuration system

bob/extension/config.py

@@ -5,6 +5,7 @@
 
 import os
 import imp
+import six
 import collections
 
 
@@ -12,8 +13,8 @@ RCFILENAME = '.bobrc.py'
 """Default name to be used for the RC file to load"""
 
 
-def _load_module(path, variables):
-  '''Loads the Python file as module, returns a proper Python module
+def _load_context(path, context):
+  '''Loads the Python file as module, returns a resolved context
 
   This function is implemented in a way that is both Python 2 and Python 3
   compatible. It does not directly load the python file, but reads its contents
@@ -25,121 +26,110 @@ def _load_module(path, variables):
     path (str): The full path of the Python file to load the module contents
       from
 
-    variables (dict): A mapping which indicates name -> object relationship to
-      be established within the file before loading it
+    context (dict): A mapping which indicates name -> object relationship to
+      be established within the file before loading it. This dictionary
+      establishes the context in which the module loading is executed, i.e.,
+      previously existing variables when the readout of the new module starts.
 
 
   Returns:
 
-    module: A valid Python module you can use in an Algorithm or Library.
+    dict: A python dictionary with the new, fully resolved context.
 
   '''
 
   retval = imp.new_module('config')
 
   # defines symbols
-  for k, v in variables.items(): retval.__dict__[k] = v
+  for k, v in context.items(): retval.__dict__[k] = v
 
   # executes the module code on the context of previously import modules
   exec(compile(open(path, "rb").read(), path, 'exec'), retval.__dict__)
 
-  return retval
+  # notice retval.__dict__ is deleted when we return
+  return dict([(k,v) for k,v in retval.__dict__.items() if not k.startswith('_')])
 
 
-def update(d, u):
-  '''Updates dictionary ``d`` with sparse values from ``u``
-
-  This function updates the base dictionary ``d`` with values from the
-  dictionary ``u``, with possible dictionary nesting. Matching keys that
-  existing in ``d`` and ``u`` will be updated. Others will be added to ``d``.
-
-  If the type of value in ``u`` is not the same as in ``d``, ``d``'s value is
-  *overriden* with the new value from ``u``.
-
-  This procedure does **not** delete any existing keys in ``d``
+def load(path, context=None):
+  '''Loads a set of configuration files, in sequence
 
+  This method will load one or more configuration files. Everytime a
+  configuration file is loaded, the context (variables) loaded from the
+  previous file is made available, so the new configuration file can override
+  or modify this context.
 
   Parameters:
 
-    d (dict): Dictionary that will be updated
+    path (:py:class:`str`, :py:class:`list`): The full path of the Python file
+      to load the module contents from. If an iterable is passed, then it is
+      iterated and each configuration file is loaded by creating/modifying the
+      context generated after each file readout.
 
-    u (dict): Dictionary with the updates.
+    context (:py:class:`dict`, Optional): If passed, start the readout of the
+      first configuration file with the given context. Otherwise, create a new
+      internal context.
 
 
   Returns:
 
-    dict: The input dictionary ``d``, updated
+    dict: A dictionary of key-values representing the resolved context, after
+    loading the provided modules and resolving all variables.
 
   '''
 
-  for k, v in u.items():
-    if isinstance(v, collections.Mapping):
-      d[k] = update(d.get(k, {}), v)
+  if isinstance(path, six.string_types):
+
+    if context is None:
+      context = dict(defaults={})
     else:
-      d[k] = v
+      if 'defaults' not in context:
+        context['defaults'] = {}
+
+    retval = _load_context(os.path.realpath(os.path.expanduser(path)), context)
+
+    return retval
 
-  return d
+  elif isinstance(path, collections.Iterable):
 
+    retval = None
+    for k in path: retval = load(k, retval)
+    return retval
 
-def load(path=None):
+  else:
+
+    raise TypeError('path must be either a string or iterable over strings')
+
+
+def loadrc(context=None):
   '''Loads the default configuration file, or an override if provided
 
-  This method will load **exactly** one configuration file in this order or
-  preference:
+  This method will load **exactly** one (global) resource configuration file in
+  this fixed order of preference:
 
-  1. The value passed in ``path``, if it exists
-  2. A file named :py:attr:`RCFILENAME` on the current directory
-  3. A file named :py:attr:`RCFILENAME` on your HOME directory
+  1. A file named :py:attr:`RCFILENAME` on the current directory
+  2. A file named :py:attr:`RCFILENAME` on your HOME directory
 
-  This function will be available in the global context of the loaded
-  configuration file. You can use it by calling ``load(path)`` to load objects
-  from another configuration file.
 
   Parameters:
 
-    path (:py:class:`str`, Optional): The full path of the Python file to load
-      the module contents from.
+    context (:py:class:`dict`, Optional): A dictionary that establishes the
+      context (existing variables) in which the RC file will be loaded. By
+      default, this value is set to ``None`` which indicates no previous
+      context.
 
 
   Returns:
 
-    dict: A dictionary of key-values after loading the provided module and
-    resolving all variables.
+    dict: A dictionary of key-values representing the resolved context, after
+    loading the provided modules and resolving all variables.
 
   '''
 
-  if path is None:
-    if os.path.exists(RCFILENAME):
-      path = os.path.realpath(RCFILENAME)
-    elif os.path.exists(os.path.expanduser('~' + os.sep + RCFILENAME)):
-      path = os.path.expanduser('~' + os.sep + RCFILENAME)
+  if os.path.exists(RCFILENAME):
+    path = os.path.realpath(RCFILENAME)
+  elif os.path.exists(os.path.expanduser('~' + os.sep + RCFILENAME)):
+    path = os.path.expanduser('~' + os.sep + RCFILENAME)
   else:
-    # if path is relative, make it relative to the current module
-    if not os.path.isabs(path):
-      import inspect
-      f = inspect.currentframe().f_back
-      if f.f_back is not None:
-        # this is a call from another module, use that as base for relpath
-        basedir = os.path.dirname(f.f_code.co_filename)
-        path = os.path.join(basedir, path)
-
-  if path is None: return {}
-
-  # symbols that will exist (even if not imported) in every config file
-  symbols = {
-      'load': load,
-      'update': update,
-      'defaults': {},
-      }
-
-  mod = _load_module(os.path.realpath(os.path.expanduser(path)), symbols)
-  retval = mod.__dict__
-
-  # cleans-up
-  for key in symbols:
-    if key == 'defaults': continue
-    if key in retval: del retval[key]
-  for key in list(retval.keys()):
-    if key.startswith('_'): del retval[key]
-
-  return retval
+    return {}
+
+  return load(path, context)

bob/extension/data/config1.py

-var1 = 'hello'
-var2 = 'world'
-
-import logging as _L
-defaults['bob.core'] = {'verbosity': _L.WARNING}
-
-var3 = {'crazy': 'dictionary', 'to be': 'replaced'}

bob/extension/data/config2.py

-var1 = 'howdy'
-var3 = 'foo'
-
-defaults['bob.db.atnt'] = {'extension': '.jpg'}

bob/extension/data/defaults-config.py

@@ -13,4 +13,3 @@ defaults['bob.db.atnt'] = {
     'directory': '/directory/to/root/of/atnt-database',
     'extension': '.ppm',
     }
-

bob/extension/data/load-config.py

-# relative paths are considered w.r.t. location of the caller
-# the following will load the file ``advanced-config.py`` which
-# is located alongside this file
-defaults = load('defaults-config.py')['defaults']
-
-# overrides a particular default or sets it for the first time
-update(defaults, {'bob.db.atnt': {'extension': '.hdf5'}})
+defaults['bob.db.atnt']['extension'] = '.hdf5'

doc/config.rst

@@ -32,7 +32,7 @@ objects from a given configuration file, like this:
 If the function :py:func:`bob.extension.config.load` succeeds, it returns a
 python dictionary containing strings as keys and objects (of any kind) which
 represent the configuration resource. For example, if the file
-``example-config.py`` contained:
+``basic-config.py`` contained:
 
 .. literalinclude:: ../bob/extension/data/basic-config.py
    :language: python
@@ -44,7 +44,7 @@ Then, the object ``configuration`` would look like this:
 
 .. doctest:: basic-config
 
-   >>> print(json.dumps(configuration, indent=2, sort_keys=True))
+   >>> print(json.dumps(configuration, indent=2, sort_keys=True)) # doctest: +NORMALIZE_WHITESPACE
    {
      "a": 1,
      "b": 3,
@@ -55,6 +55,19 @@ Then, the object ``configuration`` would look like this:
 The configuration file does not have to limit itself to simple Pythonic
 operations, you can import modules and more.
 
+There is a special function to load global configuration resources, typically
+called *run commands* (or "rc" for short files). The function is called
+:py:func:`bob.extension.config.loadrc` file and automatically searches for an
+RC file named :py:func:`bob.extension.config.RCFILENAME` on the current
+directory and, if that does not exist, reads the file with the same name
+located on the root of your home directory (or whatever ``${HOME}/.bobrc.py``
+points to).
+
+Configurable resources in each |project| package should be clearly named so you
+can correctly configure them. The next section hints on how to organize such
+global resources so they are configured homogeneously across packages in the
+|project| echo-system.
+
 
 Package Defaults
 ----------------
@@ -78,7 +91,7 @@ elements at the start of the configuration file loading. Here is an example:
 
 .. testsetup:: defaults-config
 
-   from bob.extension.config import load, update
+   from bob.extension.config import load
 
 
 When loaded, this configuration file produces the result:
@@ -108,31 +121,33 @@ When loaded, this configuration file produces the result:
    configuration file.
 
 
-Value Overrides
----------------
+Chain Loading
+-------------
 
 It is possible to implement chain configuration loading and overriding by
-either calling :py:func:`bob.extension.config.load` many times or by nesting
-calls to ``load()`` within the same configuration file. Here is an example of
-the latter:
+either calling :py:func:`bob.extension.config.load` many times or by passing
+iterables with filenames to that function. Suppose we have two configuration
+files which must be loaded in sequence:
 
-.. literalinclude:: ../bob/extension/data/load-config.py
-   :caption: "load-config.py"
+.. literalinclude:: ../bob/extension/data/defaults-config.py
+   :caption: "defaults-config.py" (first to be loaded)
    :language: python
    :linenos:
 
+.. literalinclude:: ../bob/extension/data/load-config.py
+   :caption: "load-config.py" (loaded after defaults-config.py)
+   :language: python
+   :linenos:
 
-The function :py:func:`bob.extension.config.update` is also bound to the
-configuration readout and appears as an object called ``update`` within the
-configuration file. It provides an easier handle to update the ``defaults``
-dictionary.
 
-This would produce the following result:
+Then, one can chain-load them like this:
 
 .. doctest:: defaults-config
 
    >>> #the variable `path` points to <path-to-bob.extension's root>/data
-   >>> configuration = load(os.path.join(path, 'load-config.py'))
+   >>> file1 = os.path.join(path, 'defaults-config.py')
+   >>> file2 = os.path.join(path, 'load-config.py')
+   >>> configuration = load([file1, file2])
    >>> print(json.dumps(configuration, indent=2, sort_keys=True)) # doctest: +NORMALIZE_WHITESPACE
    {
      "defaults": {
@@ -145,50 +160,3 @@ This would produce the following result:
 
 The user wanting to override defaults needs to manage the overriding and the
 order in which the override happens.
-
-It is possible to implement the same override technique programmatically. For
-example, suppose a program that receives various configuration files to read as
-input and must override values set, one after the other:
-
-.. code-block:: sh
-
-   # example application call
-   $ ./my-application.py config1.py config2.py
-
-
-The configuration files contain settings like these:
-
-.. literalinclude:: ../bob/extension/data/config1.py
-   :caption: "config1.py"
-   :language: python
-   :linenos:
-
-
-.. literalinclude:: ../bob/extension/data/config2.py
-   :caption: "config2.py"
-   :language: python
-   :linenos:
-
-
-Programmatically, the application and implement the update of the configuration
-using :py:func:`bob.extension.config.update`:
-
-.. doctest:: defaults-config
-
-   >>> #the variable `path` points to <path-to-bob.extension's root>/data
-   >>> configuration = load(os.path.join(path, 'config1.py'))
-   >>> _ = update(configuration, load(os.path.join(path, 'config2.py')))
-   >>> print(json.dumps(configuration, indent=2, sort_keys=True)) # doctest: +NORMALIZE_WHITESPACE
-   {
-     "defaults": {
-       "bob.core": {
-         "verbosity": 30
-       },
-       "bob.db.atnt": {
-         "extension": ".jpg"
-       }
-     },
-     "var1": "howdy",
-     "var2": "world",
-     "var3": "foo"
-   }

setup.py

@@ -11,7 +11,7 @@ from setuptools import setup, find_packages
 # Define package version
 version = open("version.txt").read().rstrip()
 
-requires = ['setuptools']
+requires = ['setuptools', 'six']
 import sys
 if sys.version_info[0] == 2 and sys.version_info[1] <= 6:
   requires.append('importlib')