From 29b6740fe3e0552e5ed809635096534adeac9d44 Mon Sep 17 00:00:00 2001
From: Amir MOHAMMADI <amir.mohammadi@idiap.ch>
Date: Fri, 14 Feb 2020 15:28:53 +0100
Subject: [PATCH] Add tools to maintain bob/bob
---
bob/devtools/scripts/common_options.py | 18 ++++++
bob/devtools/scripts/getpath.py | 10 +--
bob/devtools/scripts/gitlab.py | 2 -
bob/devtools/scripts/update_bob.py | 88 ++++++++++++++++++++++++++
setup.py | 3 +-
5 files changed, 110 insertions(+), 11 deletions(-)
create mode 100644 bob/devtools/scripts/common_options.py
create mode 100644 bob/devtools/scripts/update_bob.py
diff --git a/bob/devtools/scripts/common_options.py b/bob/devtools/scripts/common_options.py
new file mode 100644
index 00000000..df211a09
--- /dev/null
+++ b/bob/devtools/scripts/common_options.py
@@ -0,0 +1,18 @@
+import click
+
+
+def ref_option(**kwargs):
+ """An option for getting branch name."""
+
+ def custom_ref_option(func):
+
+ return click.option(
+ "-r",
+ "--ref",
+ default="master",
+ show_default=True,
+ help="Download path from the provided git reference (may be a branch, tag or commit hash)",
+ **kwargs
+ )(func)
+
+ return custom_ref_option
diff --git a/bob/devtools/scripts/getpath.py b/bob/devtools/scripts/getpath.py
index 04cba771..a7aede79 100644
--- a/bob/devtools/scripts/getpath.py
+++ b/bob/devtools/scripts/getpath.py
@@ -6,7 +6,7 @@ import click
from . import bdt
from ..release import get_gitlab_instance, download_path
-
+from .common_options import ref_option
from ..log import verbosity_option, get_logger
logger = get_logger(__name__)
@@ -34,13 +34,7 @@ Examples:
@click.argument("package")
@click.argument("path")
@click.argument("output", type=click.Path(exists=False), required=False)
-@click.option(
- "-r",
- "--ref",
- default="master",
- show_default=True,
- help="Download path from the provided git reference (may be a branch, tag or commit hash)",
-)
+@ref_option()
@verbosity_option()
@bdt.raise_on_error
def getpath(package, path, output, ref):
diff --git a/bob/devtools/scripts/gitlab.py b/bob/devtools/scripts/gitlab.py
index 90afda51..7e66d36e 100644
--- a/bob/devtools/scripts/gitlab.py
+++ b/bob/devtools/scripts/gitlab.py
@@ -1,7 +1,5 @@
#!/usr/bin/env python
-import os
-
import pkg_resources
import click
diff --git a/bob/devtools/scripts/update_bob.py b/bob/devtools/scripts/update_bob.py
new file mode 100644
index 00000000..f068088b
--- /dev/null
+++ b/bob/devtools/scripts/update_bob.py
@@ -0,0 +1,88 @@
+#!/usr/bin/env python
+
+
+import click
+
+from . import bdt
+from .common_options import ref_option
+
+from ..log import verbosity_option, get_logger
+
+logger = get_logger(__name__)
+
+
+@click.command(
+ epilog="""
+Examples:
+
+bdt gitlab update-bob -vv
+"""
+)
+@ref_option()
+@verbosity_option()
+@bdt.raise_on_error
+def update_bob(ref):
+ """Updates the Bob meta package with new packages.
+ """
+ import tempfile
+ from ..ci import read_packages
+ from ..release import get_gitlab_instance, download_path
+
+ gl = get_gitlab_instance()
+
+ # download order.txt form bob.nightlies and get the list of packages
+ nightlies = gl.projects.get("bob/bob.nightlies")
+
+ with tempfile.NamedTemporaryFile() as f:
+ download_path(nightlies, "order.txt", f.name, ref=ref)
+ packages = read_packages(f.name)
+
+ # find the list of public packages
+ public_packages, private_packages = [], []
+ for n, (package, branch) in enumerate(packages):
+
+ if package == "bob/bob":
+ continue
+
+ # determine package visibility
+ use_package = gl.projects.get(package)
+ is_public = use_package.attributes["visibility"] == "public"
+
+ if is_public:
+ public_packages.append(package.replace("bob/", ""))
+ else:
+ private_packages.append(package.replace("bob/", ""))
+
+ logger.debug("%s is %s", package, "public" if is_public else "not public")
+
+ logger.info("Found %d public packages", len(public_packages))
+ logger.info(
+ "The following packages were not public:\n%s", "\n".join(private_packages)
+ )
+
+ # modify conda/meta.yaml and requirements.txt in bob/bob
+
+ logger.info("Updating conda/meta.yaml")
+ start_tag = "# LIST OF BOB PACKAGES - START"
+ end_tag = "# LIST OF BOB PACKAGES - END"
+
+ with open("conda/meta.yaml", "r+") as f:
+ lines = f.read()
+ i1 = lines.find(start_tag) + len(start_tag)
+ i2 = lines.find(end_tag)
+
+ lines = (
+ lines[:i1] + "\n - ".join([""] + public_packages) + "\n " + lines[i2:]
+ )
+
+ f.seek(0)
+ f.write(lines)
+
+ logger.info("Updating requirements.txt")
+ with open("requirements.txt", "w") as f:
+ f.write("\n".join(public_packages) + "\n")
+
+ click.echo(
+ "You may need to add the ` # [linux]` tag in front of linux only "
+ "packages in conda/meta.yaml"
+ )
diff --git a/setup.py b/setup.py
index f9ac0ca9..449ba6d3 100644
--- a/setup.py
+++ b/setup.py
@@ -69,7 +69,8 @@ setup(
'getpath = bob.devtools.scripts.getpath:getpath',
'process-pipelines = bob.devtools.scripts.pipelines:process_pipelines',
'get-pipelines- = bob.devtools.scripts.pipelines:get_pipelines',
- 'graph = bob.devtools.scripts.graph:graph'
+ 'graph = bob.devtools.scripts.graph:graph',
+ 'update-bob = bob.devtools.scripts.update_bob:update_bob',
],
'bdt.ci.cli': [
--
GitLab