From e3e7ac3d4e3887507bc2f489dd2e255de4df9251 Mon Sep 17 00:00:00 2001
From: Andre Anjos <andre.dos.anjos@gmail.com>
Date: Wed, 9 Jan 2019 10:26:34 +0100
Subject: [PATCH] [scripts] Add getpath utility
---
bob/devtools/release.py | 41 +++++++++++++++++++++++
bob/devtools/scripts/build.py | 2 +-
bob/devtools/scripts/changelog.py | 2 +-
bob/devtools/scripts/getpath.py | 54 ++++++++++++++++++++++++++++++
bob/devtools/scripts/lasttag.py | 2 +-
bob/devtools/scripts/release.py | 2 +-
bob/devtools/scripts/visibility.py | 6 ++--
conda/meta.yaml | 1 +
doc/release.rst | 4 +--
setup.py | 1 +
10 files changed, 106 insertions(+), 9 deletions(-)
create mode 100644 bob/devtools/scripts/getpath.py
diff --git a/bob/devtools/release.py b/bob/devtools/release.py
index be87b1b5..2dad02a8 100644
--- a/bob/devtools/release.py
+++ b/bob/devtools/release.py
@@ -4,6 +4,7 @@
import os
import re
import time
+import shutil
import gitlab
import logging
@@ -12,6 +13,46 @@ logger = logging.getLogger(__name__)
from distutils.version import StrictVersion
+def download_path(package, path, output=None, ref='master'):
+ '''Downloads paths from gitlab, with an optional recurse
+
+ This method will download an archive of the repository from chosen
+ reference, and then it will search insize the zip blob for the path to be
+ copied into output. It uses :py:class:`zipfile.ZipFile` to do this search.
+ This method will not be very efficient for larger repository references,
+ but works recursively by default.
+
+ Args:
+
+ package: the gitlab package object to use (should be pre-fetched)
+ path: the path on the project to download
+ output: where to place the path to be downloaded - if not provided, use
+ the basename of ``path`` as storage point with respect to the current
+ directory
+ ref: the name of the git reference (branch, tag or commit hash) to use
+
+ '''
+ from io import BytesIO
+ import tarfile
+ import tempfile
+
+ output = output or os.path.realpath(os.curdir)
+
+ logger.debug('Downloading archive of "%s" from "%s"...', ref,
+ package.attributes['path_with_namespace'])
+ archive = package.repository_archive(ref=ref)
+ logger.debug('Archive has %d bytes', len(archive))
+ logger.debug('Searching for "%s" within archive...', path)
+
+ with tempfile.TemporaryDirectory() as d:
+ with tarfile.open(fileobj=BytesIO(archive), mode='r:gz') as f:
+ f.extractall(path=d)
+
+ # move stuff to "output"
+ basedir = os.listdir(d)[0]
+ shutil.move(os.path.join(d, basedir, path), output)
+
+
def get_gitlab_instance():
'''Returns an instance of the gitlab object for remote operations'''
diff --git a/bob/devtools/scripts/build.py b/bob/devtools/scripts/build.py
index e17335d9..b2ad1ca3 100644
--- a/bob/devtools/scripts/build.py
+++ b/bob/devtools/scripts/build.py
@@ -68,7 +68,7 @@ Examples:
@bdt.raise_on_error
def build(recipe_dir, python, condarc, config, channel, no_test, append_file,
docserver, dry_run):
- """Runs conda-build with a standard configuration and environment
+ """Builds package through conda-build with stock configuration
This command wraps the execution of conda-build so that you use the same
``condarc`` and ``conda_build_config.yaml`` file we use for our CI. It
diff --git a/bob/devtools/scripts/changelog.py b/bob/devtools/scripts/changelog.py
index baac1ff0..1ad42af8 100644
--- a/bob/devtools/scripts/changelog.py
+++ b/bob/devtools/scripts/changelog.py
@@ -44,7 +44,7 @@ Examples:
5. Generates a complete list of changelogs for a list of packages (one per line:
\b
- $ curl -o order.txt https://gitlab.idiap.ch/bob/bob.nightlies/raw/master/order.txt
+ $ bdt getpath bob/bob.nightlies order.txt
$ bdt lasttag bob/bob
# copy and paste date to next command
$ bdt changelog --since="2018-07-17 10:23:40" order.txt changelog.md
diff --git a/bob/devtools/scripts/getpath.py b/bob/devtools/scripts/getpath.py
new file mode 100644
index 00000000..d70be09a
--- /dev/null
+++ b/bob/devtools/scripts/getpath.py
@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+
+import os
+import logging
+logger = logging.getLogger(__name__)
+
+import click
+
+from . import bdt
+from ..log import verbosity_option
+from ..release import get_gitlab_instance, download_path
+
+
+@click.command(epilog='''
+Examples:
+
+ 1. Get the file ``order.txt`` from bob.nightlies master branch:
+
+ $ bdt getpath bob/bob.nightlies order.txt
+
+
+ 2. Get the file ``order.txt`` from a different branch ``2.x``:
+
+ $ bdt getpath --ref=2.x bob/bob.nightlies order.txt
+
+
+ 3. Get the directory ``gitlab`` (and eventual sub-directories) from bob.admin, save outputs in directory ``_ci``:
+
+ $ bdt getpath bob/bob.admin master gitlab _ci
+''')
+@click.argument('package')
+@click.argument('path')
+@click.argument('output', type=click.Path(exists=False), required=False)
+@click.option('-r', '--ref', default='master', show_default=True,
+ help='Download path from the provided git reference (may be a branch, tag or commit hash)')
+@verbosity_option()
+@bdt.raise_on_error
+def getpath(package, path, output, ref):
+ """Downloads files and directories from gitlab
+
+ Files are downloaded and stored. Directories are recursed and fully
+ downloaded to the client.
+ """
+
+ if '/' not in package:
+ raise RuntimeError('PACKAGE should be specified as "group/name"')
+
+ gl = get_gitlab_instance()
+
+ # we lookup the gitlab package once
+ use_package = gl.projects.get(package)
+ logger.info('Found gitlab project %s (id=%d)',
+ use_package.attributes['path_with_namespace'], use_package.id)
+ download_path(use_package, path, output, ref=ref)
diff --git a/bob/devtools/scripts/lasttag.py b/bob/devtools/scripts/lasttag.py
index d8129bd1..63a4d224 100644
--- a/bob/devtools/scripts/lasttag.py
+++ b/bob/devtools/scripts/lasttag.py
@@ -37,7 +37,7 @@ def lasttag(package):
gl = get_gitlab_instance()
- # we lookup the gitlab group once
+ # we lookup the gitlab package once
use_package = gl.projects.get(package)
logger.info('Found gitlab project %s (id=%d)',
use_package.attributes['path_with_namespace'], use_package.id)
diff --git a/bob/devtools/scripts/release.py b/bob/devtools/scripts/release.py
index 057be1b1..2573a79a 100644
--- a/bob/devtools/scripts/release.py
+++ b/bob/devtools/scripts/release.py
@@ -46,7 +46,7 @@ Examples:
)
@click.argument('changelog', type=click.File('rt', lazy=False))
@click.option('-g', '--group', default='bob', show_default=True,
- help='Group name where all packages are located')
+ help='Group name where all packages are located (if not provided with the package)')
@click.option('-p', '--package',
help='If the name of a package is provided, then this package will be ' \
'found in the changelog file and the release will resume from it ' \
diff --git a/bob/devtools/scripts/visibility.py b/bob/devtools/scripts/visibility.py
index 290f50d8..ed682f01 100644
--- a/bob/devtools/scripts/visibility.py
+++ b/bob/devtools/scripts/visibility.py
@@ -25,7 +25,7 @@ Examples:
2. Checks the visibility of all packages in a file list:
\b
- $ curl -o order.txt https://gitlab.idiap.ch/bob/bob.nightlies/raw/master/order.txt
+ $ bdt getpath bob/bob.nightlies order.txt
$ bdt visibility order.txt
''')
@click.argument('target')
@@ -36,13 +36,13 @@ Examples:
@verbosity_option()
@bdt.raise_on_error
def visibility(target, group):
- """Checks if the gitlab repository is visible to the current user
+ '''Reports visibility of gitlab repository
This command checks if the named package is visible to the currently logged
in user, and reports its visibility level ('public', 'internal',
'private'). If the package does not exist or it is private to the current
user, it says 'unknown' instead.
- """
+ '''
gl = get_gitlab_instance()
diff --git a/conda/meta.yaml b/conda/meta.yaml
index 8a47f1ff..25f80dbd 100644
--- a/conda/meta.yaml
+++ b/conda/meta.yaml
@@ -59,6 +59,7 @@ test:
- bdt dumpsphinx --help
- bdt bootstrap --help
- bdt build --help
+ - bdt getpath --help
- sphinx-build -aEW ${PREFIX}/share/doc/{{ name }}/doc {{ project_dir }}/sphinx
about:
diff --git a/doc/release.rst b/doc/release.rst
index 595aa4ff..edeabada 100644
--- a/doc/release.rst
+++ b/doc/release.rst
@@ -87,8 +87,8 @@ Here are the instructions to release Bob meta package:
.. code-block:: sh
- $ curl -o order.txt https://gitlab.idiap.ch/bob/bob.nightlies/raw/master/order.txt
- $ bdt -vvv visibility order.txt
+ $ bdt getpath bob/bob.nightlies order.txt
+ $ bdt visibility order.txt
* Put the list of public packages in ../../bob/requirements.txt
* Run ``bdt changelog`` first:
diff --git a/setup.py b/setup.py
index fdc190d4..f2b9f8ab 100644
--- a/setup.py
+++ b/setup.py
@@ -47,6 +47,7 @@ setup(
'dumpsphinx = bob.devtools.scripts.dumpsphinx:dumpsphinx',
'bootstrap = bob.devtools.scripts.bootstrap:bootstrap',
'build = bob.devtools.scripts.build:build',
+ 'getpath = bob.devtools.scripts.getpath:getpath',
],
},
classifiers=[
--
GitLab