Implement automatic rc module loading; Multiple simplifications

bob/extension/__init__.py

@@ -20,6 +20,7 @@ from .pkgconfig import pkgconfig
 from .boost import boost
 from .utils import uniq, uniq_paths, find_executable, find_library
 from .cmake import CMakeListsGenerator
+from .config import _loadrc
 
 __version__ = pkg_resources.require(__name__)[0].version
 
@@ -759,5 +760,8 @@ def get_config(package=__name__, externals=None, api_version=None):
 
   return retval.strip()
 
+# Loads the rc user preferences
+rc = _loadrc()
+
 # gets sphinx autodoc done right - don't remove it
 __all__ = [_ for _ in dir() if not _.startswith('_')]

bob/extension/config.py

@@ -5,7 +5,7 @@
 
 import os
 import imp
-import collections
+import copy
 import logging
 
 logger = logging.getLogger(__name__)
@@ -17,7 +17,7 @@ RCFILENAME = '.bobrc.py'
 """Default name to be used for the RC file to load"""
 
 
-def _load_context(path, context):
+def _load_context(path, mod):
   '''Loads the Python file as module, returns a resolved context
 
   This function is implemented in a way that is both Python 2 and Python 3
@@ -30,26 +30,23 @@ def _load_context(path, context):
     path (str): The full path of the Python file to load the module contents
       from
 
-    context (dict): A mapping which indicates name -> object relationship to
-      be established within the file before loading it. This dictionary
-      establishes the context in which the module loading is executed, i.e.,
-      previously existing variables when the readout of the new module starts.
+    mod (module): A preloaded module to use as context for the next module
+      loading. You can create a new module using :py:mod:`imp` as in ``m =
+      imp.new_module('name'); m.__dict__.update(ctxt)`` where ``ctxt`` is a
+      python dictionary with string -> object values representing the contents
+      of the module to be created.
 
 
   Returns:
 
-    dict: A python dictionary with the new, fully resolved context.
+    module: A python module with the fully resolved context
 
   '''
 
-  retval = imp.new_module('config')
-  retval.__dict__.update(context)
+  # executes the module code on the context of previously imported modules
+  exec(compile(open(path, "rb").read(), path, 'exec'), mod.__dict__)
 
-  # executes the module code on the context of previously import modules
-  exec(compile(open(path, "rb").read(), path, 'exec'), retval.__dict__)
-
-  # notice retval.__dict__ is deleted when we return
-  return dict((k,v) for k,v in retval.__dict__.items() if not k.startswith('_'))
+  return mod
 
 
 def load(paths, context=None):
@@ -79,23 +76,26 @@ def load(paths, context=None):
 
   '''
 
-  if context is None: context = dict()
+  mod = imp.new_module('config')
+  if context is not None: mod.__dict__.update(context)
 
   for k in paths:
-    context = _load_context(os.path.realpath(os.path.expanduser(k)), context)
+    logger.debug("Loading configuration file `%s'...", k)
+    mod = _load_context(k, mod)
 
-  return context
+  # notice context.__dict__ will be gone as soon as the module is deleted
+  # we need to shallow-copy it to prevent this
+  return dict((k,v) for k,v in mod.__dict__.items() if not k.startswith('_'))
 
 
-def loadrc(context=None):
+def _loadrc(context=None):
   '''Loads the default configuration file, or an override if provided
 
   This method will load **exactly** one (global) resource configuration file in
   this fixed order of preference:
 
   1. A path pointed by the environment variable BOBRC
-  2. A file named :py:attr:`RCFILENAME` on the current directory
-  3. A file named :py:attr:`RCFILENAME` on your HOME directory
+  2. A file named :py:attr:`RCFILENAME` on your HOME directory
 
 
   Parameters:
@@ -115,13 +115,13 @@ def loadrc(context=None):
 
   if 'BOBRC' in os.environ:
     path = os.environ['BOBRC']
-  elif os.path.exists(RCFILENAME):
-    path = os.path.realpath(RCFILENAME)
   elif os.path.exists(os.path.expanduser('~' + os.sep + RCFILENAME)):
     path = os.path.expanduser('~' + os.sep + RCFILENAME)
   else:
-    logger.debug("No RC file found", path)
+    logger.debug("No RC file found")
     return {}
 
   logger.debug("Loading RC file `%s'...", path)
-  return load([path], context)
+  mod = imp.new_module('rc')
+  if context is not None: mod.__dict__.update(context)
+  return _load_context(path, mod)

bob/extension/test_config.py

@@ -8,7 +8,7 @@ import os
 import pkg_resources
 path = pkg_resources.resource_filename('bob.extension', 'data')
 
-from .config import load, loadrc, ENVNAME
+from .config import load, _loadrc, ENVNAME
 
 
 def test_basic():
@@ -17,6 +17,12 @@ def test_basic():
   assert c == {'a': 1, 'b': 3}
 
 
+def test_basic_with_context():
+
+  c = load([os.path.join(path, 'basic-config.py')], {'d': 35, 'a': 0})
+  assert c == {'a': 1, 'b': 3, 'd': 35}
+
+
 def test_defaults():
 
   c = load([os.path.join(path, 'defaults-config.py')])
@@ -34,5 +40,6 @@ def test_chain_loading():
 def test_rc_env():
 
   os.environ[ENVNAME] = os.path.join(path, 'basic-config.py')
-  c = loadrc() #should load from environment variable
-  assert c == {'a': 1, 'b': 3}
+  c = _loadrc() #should load from environment variable
+  assert c.a == 1
+  assert c.b == 3

doc/config.rst

@@ -52,112 +52,123 @@ Then, the object ``configuration`` would look like this:
 The configuration file does not have to limit itself to simple Pythonic
 operations, you can import modules and more.
 
+
 .. note::
 
-   Since configuration files are written in Python, you can execute full python
-   programs while configuring, import modules, create classes and more.
-   **However**, it is recommended that you keep the configuration files simple.
-   Avoid defining classes in the configuration files. A recommended
-   configuration file normally would only contain
-   ``dict, list, tuple, str, int, float, True, False, None`` Python objects.
+   Variables starting with an underscore (``_``) are automatically removed from
+   the list of returned values by :py:func:`bob.extension.config.load`.
 
+   If you want to use temporary values on your configuration file either name
+   them starting with an underscore or delete the object before the end of the
+   configuration file.
 
-There is a special function to load global configuration resources, typically
-called *run commands* (or "rc" for short files). The function is called
-:py:func:`bob.extension.config.loadrc` file and automatically searches for an
-RC file named :py:func:`bob.extension.config.RCFILENAME` on the current
-directory and, if that does not exist, reads the file with the same name
-located on the root of your home directory (or whatever ``${HOME}/.bobrc.py``
-points to). The path of the file that will be loaded can be overridden by an
-environment variable named :py:attr:`bob.extension.config.ENVNAME`.
 
-Configurable resources in each |project| package should be clearly named so you
-can correctly configure them. The next section hints on how to organize such
-global resources so they are configured homogeneously across packages in the
-|project| echo-system.
+.. note::
 
+   Since configuration files are written in Python, you can execute full python
+   programs while configuring, import modules, create classes and more.
+   **However**, it is recommended that you keep the configuration files simple.
+   Avoid defining classes in the configuration files. A recommended
+   configuration file normally would only contain ``dict, list, tuple, str,
+   int, float, True, False, None`` Python objects.
 
-Package Defaults
-----------------
 
-While the configuration system by itself does not make assumptions about your
-configuration strategy, we recommend to organize values in a sensible manner
-which relates to the package name. Package-based defaults may be, for example,
-the directory where raw data files for a particular ``bob.db`` are installed or
-the verbosity-level logging messages should have.
+Chain Loading
+-------------
 
-Here is an example for the package ``bob.db.atnt``:
+It is possible to implement chain configuration loading and overriding by
+passing iterables with more than one filename to
+:py:func:`bob.extension.config.load`. Suppose we have two configuration files
+which must be loaded in sequence:
 
 .. literalinclude:: ../bob/extension/data/defaults-config.py
-   :caption: "defaults-config.py"
+   :caption: "defaults-config.py" (first to be loaded)
+   :language: python
+   :linenos:
+
+.. literalinclude:: ../bob/extension/data/load-config.py
+   :caption: "load-config.py" (loaded after defaults-config.py)
    :language: python
    :linenos:
 
 
+Then, one can chain-load them like this:
+
 .. testsetup:: defaults-config
 
-   from bob.extension.config import load
+   import os
+   import pkg_resources
+   path = pkg_resources.resource_filename('bob.extension', 'data')
+   import json
 
+   from bob.extension.config import load
 
-When loaded, this configuration file produces the result:
 
 .. doctest:: defaults-config
 
    >>> #the variable `path` points to <path-to-bob.extension's root>/data
-   >>> configuration = load([os.path.join(path, 'defaults-config.py')])
+   >>> file1 = os.path.join(path, 'defaults-config.py')
+   >>> file2 = os.path.join(path, 'load-config.py')
+   >>> configuration = load([file1, file2])
    >>> print(json.dumps(configuration, indent=2, sort_keys=True)) # doctest: +NORMALIZE_WHITESPACE
    {
      "bob_db_atnt": {
        "directory": "/directory/to/root/of/atnt-database",
-       "extension": ".ppm"
+       "extension": ".hdf5"
      }
    }
 
+The user wanting to override defaults needs to manage the overriding and the
+order in which the override happens.
 
-.. note::
-
-   Variables starting with an underscore (``_``) are automatically removed from
-   the list of returned values by :py:func:`bob.extension.config.load`.
 
-   If you want to use temporary values on your configuration file either name
-   them starting with an underscore or delete the object before the end of the
-   configuration file.
+Defaults and RC Parameters
+--------------------------
 
+When this package loads, it will automatically search for a file named
+``${HOME}/.bobrc.py``. If it finds it, it will load this python module and make
+its contents available inside the built-in module ``bob.extension.rc``. The
+path of the file that will be loaded can be overridden by an environment
+variable named ``${BOBRC}``.
 
-Chain Loading
--------------
+While this configuration system by itself does not make assumptions about your
+rc strategy, we recommend to organize values in a sensible manner which relates
+to the package name. Package-based defaults may be, for example, the directory
+where raw data files for a particular ``bob.db`` are installed or the
+verbosity-level logging messages should have.
 
-It is possible to implement chain configuration loading and overriding by
-passing iterables with more than one filename to
-:py:func:`bob.extension.config.load`. Suppose we have two configuration files
-which must be loaded in sequence:
+Here is a possible example for the package ``bob.db.atnt``:
 
 .. literalinclude:: ../bob/extension/data/defaults-config.py
-   :caption: "defaults-config.py" (first to be loaded)
+   :caption: "defaults-config.py"
    :language: python
    :linenos:
 
-.. literalinclude:: ../bob/extension/data/load-config.py
-   :caption: "load-config.py" (loaded after defaults-config.py)
-   :language: python
-   :linenos:
 
+.. testsetup:: rc-config
 
-Then, one can chain-load them like this:
+   import os
+   import pkg_resources
+   path = pkg_resources.resource_filename('bob.extension', 'data')
+   import json
 
-.. doctest:: defaults-config
+   from bob.extension.config import _loadrc
+
+
+When loaded, this configuration file produces the following result:
+
+.. doctest:: rc-config
 
    >>> #the variable `path` points to <path-to-bob.extension's root>/data
-   >>> file1 = os.path.join(path, 'defaults-config.py')
-   >>> file2 = os.path.join(path, 'load-config.py')
-   >>> configuration = load([file1, file2])
-   >>> print(json.dumps(configuration, indent=2, sort_keys=True)) # doctest: +NORMALIZE_WHITESPACE
+   >>> os.environ['BOBRC'] = os.path.join(path, 'defaults-config.py')
+   >>> mod = _loadrc()
+   >>> #notice `mod` is a normal python module
+   >>> print(mod)
+   <module 'rc' (built-in)>
+   >>> print(json.dumps(dict((k,v) for k,v in mod.__dict__.items() if not k.startswith('_')), indent=2, sort_keys=True)) # doctest: +NORMALIZE_WHITESPACE
    {
      "bob_db_atnt": {
        "directory": "/directory/to/root/of/atnt-database",
-       "extension": ".hdf5"
+       "extension": ".ppm"
      }
    }
-
-The user wanting to override defaults needs to manage the overriding and the
-order in which the override happens.