Merge branch 'click-config' into 'master'

List available entry points in config-based click commands

See merge request !144

bob/extension/config.py

@@ -8,6 +8,7 @@ import logging
 import pkgutil
 import types
 
+from collections import defaultdict
 from os.path import isfile
 
 import pkg_resources
@@ -280,7 +281,12 @@ def mod_to_context(mod):
     }
 
 
-def resource_keys(entry_point_group, exclude_packages=[], strip=["dummy"]):
+def resource_keys(
+    entry_point_group,
+    exclude_packages=[],
+    strip=["dummy"],
+    with_project_names=False,
+):
     """Reads and returns all resources that are registered with the given
     entry_point_group. Entry points from the given ``exclude_packages`` are
     ignored.
@@ -293,18 +299,37 @@ def resource_keys(entry_point_group, exclude_packages=[], strip=["dummy"]):
         List of packages to exclude when finding resources.
     strip : :any:`list`, optional
         Entrypoint names that start with any value in ``strip`` will be ignored.
+    with_project_names : :any:`bool`, optional
+        If True, will return a list of tuples with the project name and the entry point name.
 
     Returns
     -------
     :any:`list`
-        List of found resources.
+        List of found entry point names. If ``with_project_names`` is True, will return
+        a list of tuples with the project name and the entry point name.
     """
-    ret_list = [
-        entry_point.name
-        for entry_point in pkg_resources.iter_entry_points(entry_point_group)
-        if (
+    if with_project_names:
+        ret_list = defaultdict(list)
+    else:
+        ret_list = []
+
+    for entry_point in pkg_resources.iter_entry_points(entry_point_group):
+        if not (
             entry_point.dist.project_name not in exclude_packages
             and not entry_point.name.startswith(tuple(strip))
-        )
-    ]
-    return sorted(ret_list)
+        ):
+            continue
+        if with_project_names:
+            ret_list[str(entry_point.dist.project_name)].append(
+                entry_point.name
+            )
+        else:
+            ret_list.append(entry_point.name)
+
+    if with_project_names:
+        # sort each list inside the dict
+        ret_list = {k: sorted(v) for k, v in ret_list.items()}
+    else:
+        ret_list = sorted(ret_list)
+
+    return ret_list

bob/extension/data/test_dump_config.py

-"""Configuration file automatically generated at 08/07/2018
+"""Configuration file automatically generated at 19/05/2022
 cli test
 
 Test command
 
+It is possible to pass one or several Python files
+(or names of ``None`` entry points or module names i.e. import
+paths) as CONFIG arguments to this command line which contain the parameters
+listed below as Python variables. Available entry points are:
+
+The options through the command-line (see below) will
+override the values of argument provided configuration files. You can run this
+command with ``<COMMAND> -H example_config.py`` to create a template config
+file.
+
 Examples!"""
 
 # test = /my/path/test.txt
@@ -11,4 +21,6 @@ Path leading to test blablabla"""
 
 # verbose = 0
 """Optional parameter: verbose (-v, --verbose) [default: 0]
-Increase the verbosity level from 0 (only error messages) to 1 (warnings), 2 (log messages), 3 (debug information) by adding the --verbose option as often as desired (e.g. '-vvv' for debug)."""
+Increase the verbosity level from 0 (only error messages) to 1 (warnings), 2
+(log messages), 3 (debug information) by adding the --verbose option as often as
+desired (e.g. '-vvv' for debug)."""

bob/extension/data/test_dump_config2.py

-"""Configuration file automatically generated at 08/07/2018
+"""Configuration file automatically generated at 19/05/2022
 cli test
 
 Blablabla bli blo
 
-        Parameters
-        ----------
-        xxx : :any:`list`
-            blabla blablo
-        yyy : callable
-            bli bla blo bla bla bla
+Parameters
+----------
+xxx : :any:`list`
+    blabla blablo
+yyy : callable
+    bli bla blo bla bla bla
 
-        [CONFIG]...           BLA BLA BLA BLA"""
+[CONFIG]...           BLA BLA BLA BLA
+
+It is possible to pass one or several Python files
+(or names of ``bob.extension.test_dump_config`` entry points or module names i.e. import
+paths) as CONFIG arguments to this command line which contain the parameters
+listed below as Python variables. Available entry points are:
+
+**bob.extension** entry points are: basic_config, resource_config,
+subpackage_config
+
+The options through the command-line (see below) will
+override the values of argument provided configuration files. You can run this
+command with ``<COMMAND> -H example_config.py`` to create a template config
+file."""
 
 # database = None
 """Required parameter: database (--database, -d)
-bla bla bla Can be a ``bob.extension.test_dump_config`` entry point, a module name, or a path to a Python file which contains a variable named `database`.
-Registered entries are: ['basic_config', 'resource_config', 'subpackage_config']"""
+bla bla bla Can be a ``bob.extension.test_dump_config`` entry point, a module
+name, or a path to a Python file which contains a variable named `database`.
+Available entry points are:
+
+**bob.extension** entry points are: basic_config,
+resource_config,
+subpackage_config"""
 
 # annotator = None
 """Required parameter: annotator (--annotator, -a)
-bli bli bli Can be a ``bob.extension.test_dump_config`` entry point, a module name, or a path to a Python file which contains a variable named `annotator`.
-Registered entries are: ['basic_config', 'resource_config', 'subpackage_config']"""
+bli bli bli Can be a ``bob.extension.test_dump_config`` entry point, a module
+name, or a path to a Python file which contains a variable named `annotator`.
+Available entry points are:
+
+**bob.extension** entry points are: basic_config,
+resource_config,
+subpackage_config"""
 
 # output_dir = None
 """Required parameter: output_dir (--output-dir, -o)
@@ -40,4 +63,6 @@ lklklklk"""
 
 # verbose = 0
 """Optional parameter: verbose (-v, --verbose) [default: 0]
-Increase the verbosity level from 0 (only error messages) to 1 (warnings), 2 (log messages), 3 (debug information) by adding the --verbose option as often as desired (e.g. '-vvv' for debug)."""
+Increase the verbosity level from 0 (only error messages) to 1 (warnings), 2
+(log messages), 3 (debug information) by adding the --verbose option as often as
+desired (e.g. '-vvv' for debug)."""

bob/extension/scripts/click_helper.py

 import logging
+import textwrap
 import time
 import traceback
 
@@ -177,6 +178,23 @@ def verbosity_option(**kwargs):
     return custom_verbosity_option
 
 
+def _prepare_entry_points(entry_point_group):
+    if not entry_point_group:
+        return ""
+    ret = ""
+    for prj_name, prj_entry_points in resource_keys(
+        entry_point_group, with_project_names=True
+    ).items():
+        ret += f"\n\n**{prj_name}** entry points are: "
+        ret += ", ".join(prj_entry_points)
+
+    # wrap ret to 80 chars
+    ret = "\n".join(
+        textwrap.wrap(ret, 80, break_on_hyphens=False, replace_whitespace=False)
+    )
+    return ret
+
+
 class ConfigCommand(click.Command):
     """A click.Command that can take options both form command line options and
     configuration files. In order to use this class, you **have to** use the
@@ -197,13 +215,16 @@ class ConfigCommand(click.Command):
         configs_argument_name = "CONFIG"
         # Augment help for the config file argument
         self.extra_help = """\n\nIt is possible to pass one or several Python files
-(or names of ``{entry_point_group}`` entry points or module names) as {CONFIG}
-arguments to the command line which contain the parameters listed below as
-Python variables. The options through the command-line (see below) will
-override the values of configuration files. You can run this command with
-``<COMMAND> -H example_config.py`` to create a template config
+(or names of ``{entry_point_group}`` entry points or module names i.e. import
+paths) as {CONFIG} arguments to this command line which contain the parameters
+listed below as Python variables. Available entry points are: {entry_points}
+\nThe options through the command-line (see below) will
+override the values of argument provided configuration files. You can run this
+command with ``<COMMAND> -H example_config.py`` to create a template config
 file.""".format(
-            CONFIG=configs_argument_name, entry_point_group=entry_point_group
+            CONFIG=configs_argument_name,
+            entry_point_group=entry_point_group,
+            entry_points=_prepare_entry_points(entry_point_group),
         )
         help = (help or "").rstrip() + self.extra_help
         super().__init__(name, *args, help=help, **kwargs)
@@ -254,7 +275,7 @@ file.""".format(
         )
 
         if self.help:
-            h = self.help.replace(self.extra_help, "").replace("\b\n", "")
+            h = self.help.replace("\b\n", "")
             config_file.write("\n{}".format(h.rstrip()))
 
         if self.epilog:
@@ -284,24 +305,24 @@ file.""".format(
             )
 
             if param.help is not None:
-                config_file.write("\n%s" % param.help)
-
-            if (
-                isinstance(param, ResourceOption)
-                and param.entry_point_group is not None
-            ):
                 config_file.write(
-                    "\nRegistered entries are: {}".format(
-                        resource_keys(param.entry_point_group)
+                    "\n%s"
+                    % "\n".join(
+                        textwrap.wrap(
+                            param.help,
+                            80,
+                            break_on_hyphens=False,
+                            replace_whitespace=False,
+                        )
                     )
                 )
 
             config_file.write("'''\n")
-            click.echo(
-                "Configuration file '{}' was written; exiting".format(
-                    config_file.name
-                )
+        click.echo(
+            "Configuration file '{}' was written; exiting".format(
+                config_file.name
             )
+        )
 
         config_file.close()
         ctx.exit()
@@ -387,9 +408,14 @@ class ResourceOption(click.Option):
             help = help or ""
             help += (
                 " Can be a ``{entry_point_group}`` entry point, a module name, or "
-                "a path to a Python file which contains a variable named `{name}`."
+                "a path to a Python file which contains a variable named `{name}`. "
+                "Available entry points are: {entry_points}"
+            )
+            help = help.format(
+                entry_point_group=entry_point_group,
+                entry_points=_prepare_entry_points(entry_point_group),
+                name=name,
             )
-            help = help.format(entry_point_group=entry_point_group, name=name)
         super().__init__(
             param_decls=param_decls,
             show_default=show_default,

bob/extension/test_click_helper.py

@@ -190,14 +190,16 @@ def _assert_config_dump(ref, ref_date):
     # uncomment below to re-write tests
     # open(ref, 'wt').write(open('TEST_CONF').read())
     with open("TEST_CONF", "r") as f, open(ref, "r") as f2:
-        text = f.read().replace("'''", '"""')
-        ref_text = f2.read().replace(ref_date, today)
-        # remove the starting whitespace of each line so the tests are more relaxed
-        text = "\n".join(line.lstrip() for line in text.splitlines())
-        ref_text = "\n".join(line.lstrip() for line in ref_text.splitlines())
-        assert text == ref_text, "\n".join(
-            [text, "########################\n" * 2, ref_text]
-        )
+        text = f.read()
+        ref_text = f2.read()
+    ref_text = ref_text.replace(ref_date, today)
+    # remove the starting and final whitespace of each line so the tests are more relaxed
+    text = "\n".join(line.strip() for line in text.splitlines())
+    ref_text = "\n".join(line.strip() for line in ref_text.splitlines())
+    # replace ''' with """ so tests are more relaxed
+    text = text.replace("'''", '"""')
+    ref_text = ref_text.replace("'''", '"""')
+    assert text == ref_text
 
 
 def test_config_dump():
@@ -228,7 +230,7 @@ def test_config_dump():
             "bob.extension", "data/test_dump_config.py"
         )
         assert_click_runner_result(result)
-        _assert_config_dump(ref, "08/07/2018")
+        _assert_config_dump(ref, "19/05/2022")
 
 
 def test_config_dump2():
@@ -302,7 +304,7 @@ def test_config_dump2():
             "bob.extension", "data/test_dump_config2.py"
         )
         assert_click_runner_result(result)
-        _assert_config_dump(ref, "08/07/2018")
+        _assert_config_dump(ref, "19/05/2022")
 
 
 def test_config_command_with_callback_options():