[graph] Re-implement dependence graph generation as per my instructions to @tiago.pereira

bob/devtools/scripts/build.py

@@ -20,7 +20,10 @@ from ..build import (
     get_docserver_setup,
     get_env_directory,
     get_output_path,
+    remove_conda_loggers,
 )
+remove_conda_loggers()
+
 from ..constants import (
     CONDA_BUILD_CONFIG,
     CONDA_RECIPE_APPEND,
@@ -182,7 +185,7 @@ def build(
     from bob.devtools.bootstrap import do_hack
     project_dir = os.path.dirname(recipe_dir[0])
     do_hack(project_dir)
-    
+
 
     # get potential channel upload and other auxiliary channels
     channels = get_channels(
@@ -235,7 +238,7 @@ def build(
     for d in recipe_dir:
 
         if not os.path.exists(d):
-            raise RuntimeError("The directory %s does not exist" % recipe_dir)
+            raise RuntimeError("The directory %s does not exist" % d)
 
         version_candidate = os.path.join(d, "..", "version.txt")
         if os.path.exists(version_candidate):

bob/devtools/scripts/graph.py

 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
+import sys
+
+import yaml
 import click
 from click_plugins import with_plugins
 
-from ..graph import compute_dependency_graph, get_graphviz_dependency_graph
+from . import bdt
+from ..constants import (
+    CONDA_BUILD_CONFIG,
+    CONDA_RECIPE_APPEND,
+    SERVER,
+    MATPLOTLIB_RCDIR,
+    BASE_CONDARC,
+)
+from ..build import make_conda_config
+from ..bootstrap import set_environment, get_channels
+from ..release import get_gitlab_instance
+from ..graph import compute_adjencence_matrix, generate_graph
 
 from ..log import verbosity_option, get_logger, echo_info
-
 logger = get_logger(__name__)
 
 
 @click.command(
     epilog="""
-Example:
+Examples:
 
-   bdt graph bob.bio.face graph
+  1. Draws the graph of a package
+
+     $ bdt gitlab graph bob/bob.bio.face
 
 
 """
 )
-@click.argument("package_name", required=True)
-@click.argument("output_file", required=True)
+@click.argument("package", required=True)
 @click.option(
-    "-c",
-    "--channel",
-    default=None,
-    help="Define a target channel for conda serch. If not set, will use what is set in .condarc",
+    "-p",
+    "--python",
+    default=("%d.%d" % sys.version_info[:2]),
+    show_default=True,
+    help="Version of python to build the environment for",
 )
 @click.option(
-    "-p",
-    "--prefix",
-    default="bob.",
-    help="It will recursivelly look into dependencies whose package name matches the prefix. Default 'bob.'",
+    "-r",
+    "--condarc",
+    help="Use custom conda configuration file instead of our own",
+)
+@click.option(
+    "-m",
+    "--config",
+    "--variant-config-files",
+    show_default=True,
+    default=CONDA_BUILD_CONFIG,
+    help="overwrites the path leading to " "variant configuration file to use",
+)
+@click.option(
+    "-a",
+    "--append-file",
+    show_default=True,
+    default=CONDA_RECIPE_APPEND,
+    help="overwrites the path leading to " "appended configuration file to use",
+)
+@click.option(
+    "-S",
+    "--server",
+    show_default=True,
+    default=SERVER,
+    help="Server used for downloading conda packages and documentation "
+    "indexes of required packages",
+)
+@click.option(
+    "-P",
+    "--private/--no-private",
+    default=False,
+    help="Set this to **include** private channels on your search - "
+    "you **must** be at Idiap to execute this build in this case - "
+    "you **must** also use the correct server name through --server - "
+    "notice this option has no effect to conda if you also pass --condarc",
+)
+@click.option(
+    "-X",
+    "--stable/--no-stable",
+    default=False,
+    help="Set this to **exclude** beta channels from your build - "
+    "notice this option has no effect if you also pass --condarc",
+)
+@click.option(
+    "-C",
+    "--ci/--no-ci",
+    default=False,
+    hidden=True,
+    help="Use this flag to indicate the graph will be running on the CI",
+)
+@click.option(
+    "-n",
+    "--name",
+    show_default=True,
+    default="graph",
+    help="set the graph name",
+)
+@click.option(
+    "-f",
+    "--format",
+    show_default=True,
+    default="svg",
+    help="determines the type of output to expect",
 )
+@click.option(
+    "-w",
+    "--whitelist",
+    show_default=True,
+    default="^(bob|beat|batl|gridtk)(\.)?(?!-).*$",
+    help="package regular expression to preserve in the graph, "
+    "use .* for keeping all packages, including non-maintained ones.  The "
+    "current expression accepts most of our packages, excluding bob/beat-devel")
 @verbosity_option()
-def graph(package_name, output_file, channel, prefix):
+@bdt.raise_on_error
+def graph(package, python, condarc, config, append_file, server, private,
+        stable, ci, name, format, whitelist):
     """
-    Compute the dependency graph of a conda package and save it in an SVG file using graphviz.
+    Computes the dependency graph of a gitlab package (via its conda recipe)
+    and outputs an dot file that can be used by graphviz to draw a direct
+    acyclic graph (DAG) of package dependencies.
+
+    This command uses the conda-build API to resolve the package dependencies.
     """
-    logger.info(f"Computing dependency graph")
-    graph_dict = compute_dependency_graph(package_name, channel=channel, prefix=prefix)
-    logger.info("Generating SVG")
-    get_graphviz_dependency_graph(graph_dict, output_file, prefix=prefix)
+
+    if "/" not in package:
+        raise RuntimeError('PACKAGE should be specified as "group/name"')
+
+    package_group, package_name = package.split('/', 1)
+
+    gl = get_gitlab_instance()
+
+    # get potential channel upload and other auxiliary channels
+    channels = get_channels(
+        public=(not private),
+        stable=stable,
+        server=server,
+        intranet=ci,
+        group=package_group,
+    )
+
+    if condarc is not None:
+        logger.info("Loading CONDARC file from %s...", condarc)
+        with open(condarc, "rb") as f:
+            condarc_options = yaml.load(f, Loader=yaml.FullLoader)
+    else:
+        # use default and add channels
+        all_channels = []
+        all_channels += channels + ["defaults"]
+        condarc_options = yaml.load(BASE_CONDARC, Loader=yaml.FullLoader)
+        logger.info(
+            "Using the following channels during build:\n  - %s",
+            "\n  - ".join(all_channels),
+        )
+        condarc_options["channels"] = all_channels
+
+    conda_config = make_conda_config(
+        config, python, append_file, condarc_options
+    )
+
+    set_environment("MATPLOTLIBRC", MATPLOTLIB_RCDIR)
+
+    # setup BOB_DOCUMENTATION_SERVER environment variable (used for bob.extension
+    # and derived documentation building via Sphinx)
+    set_environment("DOCSERVER", server)
+    set_environment("BOB_DOCUMENTATION_SERVER", "/not/set")
+
+    adj_matrix = compute_adjencence_matrix(gl, package, conda_config,
+            channels[0])
+
+    graph = generate_graph(adj_matrix, deptypes=[], whitelist=whitelist)
+    graph.render(name, format=format, cleanup=True)

bob/devtools/scripts/test.py

@@ -15,7 +15,10 @@ from ..build import (
     make_conda_config,
     get_docserver_setup,
     get_env_directory,
+    remove_conda_loggers,
 )
+remove_conda_loggers()
+
 from ..constants import (
     CONDA_BUILD_CONFIG,
     CONDA_RECIPE_APPEND,