Add support for loading configs through entrypoint names

bob/extension/config.py

@@ -5,11 +5,15 @@
 '''
 
 import imp
+import pkg_resources
+import pkgutil
+from os.path import isfile
 import logging
 
 logger = logging.getLogger(__name__)
 
-loaded_configs = []
+LOADED_CONFIGS = []
+
 
 def _load_context(path, mod):
   '''Loads the Python file as module, returns a resolved context
@@ -18,23 +22,22 @@ def _load_context(path, mod):
   compatible. It does not directly load the python file, but reads its contents
   in memory before Python-compiling it. It leaves no traces on the file system.
 
-
-  Parameters:
-
-    path (str): The full path of the Python file to load the module contents
+  Parameters
+  ----------
+  path : str
+      The full path of the Python file to load the module contents
       from
-
-    mod (module): A preloaded module to use as context for the next module
+  mod : module
+      A preloaded module to use as context for the next module
       loading. You can create a new module using :py:mod:`imp` as in ``m =
       imp.new_module('name'); m.__dict__.update(ctxt)`` where ``ctxt`` is a
       python dictionary with string -> object values representing the contents
       of the module to be created.
 
-
-  Returns:
-
-    module: A python module with the fully resolved context
-
+  Returns
+  -------
+  mod : :any:`module`
+      A python module with the fully resolved context
   '''
 
   # executes the module code on the context of previously imported modules
@@ -43,43 +46,120 @@ def _load_context(path, mod):
   return mod
 
 
-def load(paths, context=None):
+def _get_module_filename(module_name):
+  """Resolves a module name to an actual Python file.
+
+  Parameters
+  ----------
+  module_name : str
+      The name of the module
+
+  Returns
+  -------
+  str
+      The Python files that corresponds to the module name.
+  """
+  loader = pkgutil.get_loader(module_name)
+  if loader is None:
+    return ''
+  return loader.filename
+
+
+def _resolve_entry_point_or_modules(paths, entry_point_group):
+  """Resolves a mixture of paths, entry point names, and module names to just
+  paths. For example paths can be:
+  ``paths = ['/tmp/config.py', 'config1', 'bob.extension.config2']``.
+
+  Parameters
+  ----------
+  paths : [str]
+      An iterable strings that either point to actual files, are entry point
+      names, or are module names.
+  entry_point_group : str
+      The entry point group name to search in entry points.
+
+  Raises
+  ------
+  ValueError
+      If one of the paths cannot be resolved to an actual path to a file.
+
+  Returns
+  -------
+  paths : [str]
+      The resolved paths pointing to existing files.
+  """
+  entries = {e.name: e for e in
+             pkg_resources.iter_entry_points(entry_point_group)}
+  files = []
+
+  for i, path in enumerate(paths):
+    old_path = path
+    # if it is already a file
+    if isfile(path):
+      pass
+    # If it is an entry point name
+    elif path in entries:
+      module_name = entries[path].module_name
+      path = _get_module_filename(module_name)
+      if not isfile(path):
+        raise ValueError(
+            "The specified entry point: `{}' pointing to module: `{}' and "
+            "resolved to: `{}' does not point to an existing "
+            "file.".format(old_path, module_name, path))
+    # If it is not a path nor an entry point name, it is a module name then?
+    else:
+      path = _get_module_filename(path)
+      if not isfile(path):
+        raise ValueError(
+            "The specified path: `{}' resolved to: `{}' is not a file, entry "
+            "point name, or a module name".format(old_path, path))
+    files.append(path)
+  return files
+
+
+def load(paths, context=None, entry_point_group=None):
   '''Loads a set of configuration files, in sequence
 
-  This method will load one or more configuration files. Everytime a
+  This method will load one or more configuration files. Every time a
   configuration file is loaded, the context (variables) loaded from the
   previous file is made available, so the new configuration file can override
   or modify this context.
 
-  Parameters:
-
-    paths (:py:class:`list`): A list or iterable containing paths (relative or
-      absolute) of configuration files that need to be loaded in sequence.
-      Each configuration file is loaded by creating/modifying the context
-      generated after each file readout.
-
-    context (:py:class:`dict`, Optional): If passed, start the readout of the
-      first configuration file with the given context. Otherwise, create a new
-      internal context.
-
-
-  Returns:
-
-    dict: A dictionary of key-values representing the resolved context, after
-    loading the provided modules and resolving all variables.
-
+  Parameters
+  ----------
+  paths : [str]
+      A list or iterable containing paths (relative or absolute) of
+      configuration files that need to be loaded in sequence. Each
+      configuration file is loaded by creating/modifying the context generated
+      after each file readout.
+  context : :py:class:`dict`, optional
+      If provided, start the readout of the first configuration file with the
+      given context. Otherwise, create a new internal context.
+  entry_point_group : :py:class:`str`, optional
+      If provided, it will treat non-existing file paths as entry point names
+      under the ``entry_point_group`` name.
+
+  Returns
+  -------
+  mod : :any:`module`
+      A module representing the resolved context, after loading the provided
+      modules and resolving all variables.
   '''
 
   mod = imp.new_module('config')
   if context is not None:
     mod.__dict__.update(context)
 
+  # resolve entry points to paths
+  if entry_point_group is not None:
+    paths = _resolve_entry_point_or_modules(paths, entry_point_group)
+
   for k in paths:
     logger.debug("Loading configuration file `%s'...", k)
     mod = _load_context(k, mod)
 
   # Small gambiarra (https://www.urbandictionary.com/define.php?term=Gambiarra)
-  # to avoid the gc to collect some already imported modules
-  loaded_configs.append(mod)
-  
+  # to avoid the garbage collector to collect some already imported modules.
+  LOADED_CONFIGS.append(mod)
+
   return mod

bob/extension/data/load-config.py

-b = b + 3

bob/extension/data/load_config.py

+# the b variable from the last config file is available here
+c = b + 1
+b = b + 3

bob/extension/test_config.py

@@ -13,14 +13,14 @@ path = pkg_resources.resource_filename('bob.extension', 'data')
 
 def test_basic():
 
-  c = load([os.path.join(path, 'basic-config.py')])
+  c = load([os.path.join(path, 'basic_config.py')])
   assert hasattr(c, "a") and c.a == 1
   assert hasattr(c, "b") and c.b == 3
-  
+
 
 def test_basic_with_context():
 
-  c = load([os.path.join(path, 'basic-config.py')], {'d': 35, 'a': 0})
+  c = load([os.path.join(path, 'basic_config.py')], {'d': 35, 'a': 0})
   assert hasattr(c, "a") and c.a == 1
   assert hasattr(c, "b") and c.b == 3
   assert hasattr(c, "d") and c.d == 35
@@ -28,15 +28,27 @@ def test_basic_with_context():
 
 def test_chain_loading():
 
-  file1 = os.path.join(path, 'basic-config.py')
-  file2 = os.path.join(path, 'load-config.py')
+  file1 = os.path.join(path, 'basic_config.py')
+  file2 = os.path.join(path, 'load_config.py')
   c = load([file1, file2])
   assert hasattr(c, "a") and c.a == 1
   assert hasattr(c, "b") and c.b == 6
 
-  
+
 def test_config_with_module():
 
-  c = load([os.path.join(path, 'config-with-module.py')])
-  assert hasattr(c, "return_zeros") and numpy.allclose(c.return_zeros(), numpy.zeros(shape=(2,)))
+  c = load([os.path.join(path, 'config_with_module.py')])
+  assert hasattr(c, "return_zeros") and numpy.allclose(
+      c.return_zeros(), numpy.zeros(shape=(2,)))
+
 
+def test_entry_point_configs():
+
+  # test when all kinds of paths
+  c = load([
+      os.path.join(path, 'basic_config.py'),
+      'basic_config',
+      'bob.extension.data.basic_config',
+  ], entry_point_group='bob.extension.test_config_load')
+  assert hasattr(c, "a") and c.a == 1
+  assert hasattr(c, "b") and c.b == 3

doc/config.rst

@@ -21,30 +21,31 @@ objects from one or more configuration files, like this:
    import pkg_resources
    path = pkg_resources.resource_filename('bob.extension', 'data')
    import json
+   from bob.extension.config import load
 
-.. doctest:: basic-config
+.. doctest::
 
    >>> from bob.extension.config import load
    >>> #the variable `path` points to <path-to-bob.extension's root>/data
-   >>> configuration = load([os.path.join(path, 'basic-config.py')])
+   >>> configuration = load([os.path.join(path, 'basic_config.py')])
 
 
 If the function :py:func:`bob.extension.config.load` succeeds, it returns a
 python dictionary containing strings as keys and objects (of any kind) which
 represent the configuration resource. For example, if the file
-``basic-config.py`` contained:
+``basic_config.py`` contained:
 
-.. literalinclude:: ../bob/extension/data/basic-config.py
+.. literalinclude:: ../bob/extension/data/basic_config.py
    :language: python
    :linenos:
-   :caption: "basic-config.py"
+   :caption: "basic_config.py"
 
 
 Then, the object ``configuration`` would look like this:
 
-.. doctest:: basic-config
+.. doctest::
 
-   >>> print("a = %d\nb = %d"%(configuration.a, configuration.b)) # doctest: +NORMALIZE_WHITESPACE
+   >>> print("a = %d\nb = %d"%(configuration.a, configuration.b))
    a = 1
    b = 3
 
@@ -62,34 +63,24 @@ passing iterables with more than one filename to
 :py:func:`bob.extension.config.load`. Suppose we have two configuration files
 which must be loaded in sequence:
 
-.. literalinclude:: ../bob/extension/data/basic-config.py
-   :caption: "basic-config.py" (first to be loaded)
+.. literalinclude:: ../bob/extension/data/basic_config.py
+   :caption: "basic_config.py" (first to be loaded)
    :language: python
    :linenos:
 
-.. literalinclude:: ../bob/extension/data/load-config.py
-   :caption: "load-config.py" (loaded after basic-config.py)
+.. literalinclude:: ../bob/extension/data/load_config.py
+   :caption: "load_config.py" (loaded after basic_config.py)
    :language: python
    :linenos:
 
 
 Then, one can chain-load them like this:
 
-.. testsetup:: basic-config
-
-   import os
-   import pkg_resources
-   path = pkg_resources.resource_filename('bob.extension', 'data')
-   import json
-
-   from bob.extension.config import load
-
-
-.. doctest:: basic-config
+.. doctest::
 
    >>> #the variable `path` points to <path-to-bob.extension's root>/data
-   >>> file1 = os.path.join(path, 'basic-config.py')
-   >>> file2 = os.path.join(path, 'load-config.py')
+   >>> file1 = os.path.join(path, 'basic_config.py')
+   >>> file2 = os.path.join(path, 'load_config.py')
    >>> configuration = load([file1, file2])
    >>> print("a = %d \nb = %d"%(configuration.a, configuration.b)) # doctest: +NORMALIZE_WHITESPACE
    a = 1
@@ -98,3 +89,23 @@ Then, one can chain-load them like this:
 
 The user wanting to override the values needs to manage the overriding and the
 order in which the override happens.
+
+
+Entry Points
+------------
+
+The function :py:func:`bob.extension.config.load` can also load config files
+through `Setuptools`_ entry points and module names. It is only needed
+to provide the group name of the entry points:
+
+.. doctest:: entry_point
+
+   >>> group = 'bob.extension.test_config_load'  # the group name of entry points
+   >>> file1 = 'basic_config'  # an entry point name
+   >>> file2 = 'bob.extension.data.load_config' # module name
+   >>> configuration = load([file1, file2], entry_point_group=group)
+   >>> print("a = %d \nb = %d"%(configuration.a, configuration.b)) # doctest: +NORMALIZE_WHITESPACE
+   a = 1
+   b = 6
+
+.. include:: links.rst

doc/links.rst

@@ -18,6 +18,7 @@
 .. _python: http://www.python.org
 .. _pypi: http://pypi.python.org
 .. _bob packages: https://www.idiap.ch/software/bob/packages
+.. _setuptools documentation:
 .. _setuptools: https://setuptools.readthedocs.io
 .. _sphinx: http://sphinx.pocoo.org
 .. _zc.buildout: http://pypi.python.org/pypi/zc.buildout/

doc/pure_python.rst

@@ -114,7 +114,7 @@ In detail, it defines the name and the version of this package, which files
 belong to the package (those files are automatically collected by the
 ``find_packages`` function), other packages that we depend on, namespaces and
 console scripts. The full set of options can be inspected in the
-`Setuptools documentation <https://setuptools.readthedocs.io>`_.
+`Setuptools documentation`_.
 
 .. warning::
 

setup.py

@@ -41,6 +41,10 @@ setup(
         'bob_new_version.py = bob.extension.scripts:new_version',
         'bob_dependecy_graph.py = bob.extension.scripts:dependency_graph',
       ],
+      # some test entry_points
+      'bob.extension.test_config_load': [
+        'basic_config = bob.extension.data.basic_config',
+      ],
     },
 
     classifiers = [