From a700c950e25e2737f13f71d048c8a7cdc76d30ca Mon Sep 17 00:00:00 2001
From: Tiago Freitas Pereira <tiagofrepereira@gmail.com>
Date: Fri, 1 Nov 2019 07:55:07 +0100
Subject: [PATCH] Implemented a mechanism to run a dependency graph
---
bob/devtools/graph.py | 120 ++++++++++++++++++++++++++++++++++
bob/devtools/scripts/graph.py | 45 +++++++++++++
conda/meta.yaml | 2 +
setup.py | 1 +
4 files changed, 168 insertions(+)
create mode 100644 bob/devtools/graph.py
create mode 100644 bob/devtools/scripts/graph.py
diff --git a/bob/devtools/graph.py b/bob/devtools/graph.py
new file mode 100644
index 00000000..70077ef2
--- /dev/null
+++ b/bob/devtools/graph.py
@@ -0,0 +1,120 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import conda.cli.python_api
+import json
+
+from .log import verbosity_option, get_logger, echo_info
+
+logger = get_logger(__name__)
+
+
+from graphviz import Digraph
+
+
+def get_graphviz_dependency_graph(
+ graph_dict,
+ file_name,
+ prefix="bob.",
+ black_list=["python", "setuptools", "libcxx", "numpy", "libblitz", "boost"],
+):
+ """
+ Given a dictionary with the dependency graph, compute the graphviz DAG and save it
+ in SVG
+ """
+
+ d = Digraph(format="svg", engine="dot")
+
+ for i in graph_dict:
+ for j in graph_dict[i]:
+ # Conections to python, setuptools....gets very messy
+ if j in black_list:
+ continue
+
+ if prefix in j:
+ d.attr("node", shape="box")
+ else:
+ d.attr("node", shape="ellipse")
+ d.edge(i, j)
+ d.render(file_name)
+
+
+def compute_dependency_graph(
+ package_name, channel=None, selected_packages=[], prefix="bob.", dependencies=dict()
+):
+ """
+ Given a target package, returns an adjacency matrix with its dependencies returned via the command `conda search xxxx --info`
+
+ **Parameters**
+
+ package_name:
+ Name of the package
+
+ channel:
+ Name of the channel to be sent via `-c` option. If None `conda search` will use what is in .condarc
+
+ selected_packages:
+ List of target packages. If set, the returned adjacency matrix will be in terms of this list.
+
+ prefix:
+ Only seach for deep dependencies under the prefix. This would avoid to go deeper in
+ dependencies not maintained by us, such as, numpy, matplotlib, etc..
+
+ dependencies:
+ Dictionary controlling the state of each search
+
+ """
+
+ if package_name in dependencies:
+ return dependencies
+
+ dependencies[package_name] = fetch_dependencies(
+ package_name, channel, selected_packages
+ )
+ logger.info(f" >> Searching dependencies of {package_name}")
+ for d in dependencies[package_name]:
+ if prefix in d:
+ compute_dependency_graph(
+ d, channel, selected_packages, prefix, dependencies
+ )
+ return dependencies
+
+
+def fetch_dependencies(package_name, channel=None, selected_packages=[]):
+ """
+ conda search the dependencies of a package
+
+ **Parameters**
+ packge_name:
+ channel:
+ selected_packages:
+ """
+
+ # Running conda search and returns to a json file
+ if channel is None:
+ package_description = conda.cli.python_api.run_command(
+ conda.cli.python_api.Commands.SEARCH, package_name, "--info", "--json"
+ )
+ else:
+ package_description = conda.cli.python_api.run_command(
+ conda.cli.python_api.Commands.SEARCH,
+ package_name,
+ "--info",
+ "-c",
+ channel,
+ "--json",
+ )
+
+ # TODO: Fix that
+ package_description = json.loads(package_description[0])
+
+ # Fetching the dependencies of the most updated package
+ all_dependencies = [
+ p.split(" ")[0] for p in package_description[package_name][-1]["depends"]
+ ]
+
+ if len(selected_packages) > 0:
+ # Filtering the dependencies
+ return [d for d in selected_packages if d in all_dependencies]
+
+ return all_dependencies
diff --git a/bob/devtools/scripts/graph.py b/bob/devtools/scripts/graph.py
new file mode 100644
index 00000000..3415eacb
--- /dev/null
+++ b/bob/devtools/scripts/graph.py
@@ -0,0 +1,45 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import click
+from click_plugins import with_plugins
+
+from ..graph import compute_dependency_graph, get_graphviz_dependency_graph
+
+from ..log import verbosity_option, get_logger, echo_info
+
+logger = get_logger(__name__)
+
+
+@click.command(
+ epilog="""
+Example:
+
+ bdt graph bob.bio.face graph
+
+
+"""
+)
+@click.argument("package_name", required=True)
+@click.argument("output_file", required=True)
+@click.option(
+ "-c",
+ "--channel",
+ default=None,
+ help="Define a target channel for conda serch. If not set, will use what is set in .condarc",
+)
+@click.option(
+ "-p",
+ "--prefix",
+ default="bob.",
+ help="It will recursivelly look into dependencies whose package name matches the prefix. Default 'bob.'",
+)
+@verbosity_option()
+def graph(package_name, output_file, channel, prefix):
+ """
+ Compute the dependency graph of a conda package and save it in an SVG file using graphviz.
+ """
+ logger.info(f"Computing dependency graph")
+ graph_dict = compute_dependency_graph(package_name, channel=channel, prefix=prefix)
+ logger.info("Generating SVG")
+ get_graphviz_dependency_graph(graph_dict, output_file, prefix=prefix)
diff --git a/conda/meta.yaml b/conda/meta.yaml
index 43c37974..40ea6b0a 100644
--- a/conda/meta.yaml
+++ b/conda/meta.yaml
@@ -49,6 +49,7 @@ requirements:
- termcolor
- psutil
- tabulate
+ - python-graphviz
test:
requires:
@@ -104,6 +105,7 @@ test:
- bdt dav upload --help
- bdt gitlab process-pipelines --help
- bdt gitlab get-pipelines --help
+ - bdt graph --help
- sphinx-build -aEW ${PREFIX}/share/doc/{{ name }}/doc sphinx
- if [ -n "${CI_PROJECT_DIR}" ]; then mv sphinx "${CI_PROJECT_DIR}/"; fi
diff --git a/setup.py b/setup.py
index 1e729138..87e08fd0 100644
--- a/setup.py
+++ b/setup.py
@@ -56,6 +56,7 @@ setup(
'dav = bob.devtools.scripts.dav:dav',
'local = bob.devtools.scripts.local:local',
'gitlab = bob.devtools.scripts.gitlab:gitlab',
+ 'graph = bob.devtools.scripts.graph:graph'
],
'bdt.gitlab.cli': [
--
GitLab