[scripts] Add getpath utility

bob/devtools/release.py

@@ -4,6 +4,7 @@
 import os
 import re
 import time
+import shutil
 import gitlab
 
 import logging
@@ -12,6 +13,46 @@ logger = logging.getLogger(__name__)
 from distutils.version import StrictVersion
 
 
+def download_path(package, path, output=None, ref='master'):
+    '''Downloads paths from gitlab, with an optional recurse
+
+    This method will download an archive of the repository from chosen
+    reference, and then it will search insize the zip blob for the path to be
+    copied into output.  It uses :py:class:`zipfile.ZipFile` to do this search.
+    This method will not be very efficient for larger repository references,
+    but works recursively by default.
+
+    Args:
+
+      package: the gitlab package object to use (should be pre-fetched)
+      path: the path on the project to download
+      output: where to place the path to be downloaded - if not provided, use
+        the basename of ``path`` as storage point with respect to the current
+        directory
+      ref: the name of the git reference (branch, tag or commit hash) to use
+
+    '''
+    from io import BytesIO
+    import tarfile
+    import tempfile
+
+    output = output or os.path.realpath(os.curdir)
+
+    logger.debug('Downloading archive of "%s" from "%s"...', ref,
+        package.attributes['path_with_namespace'])
+    archive = package.repository_archive(ref=ref)
+    logger.debug('Archive has %d bytes', len(archive))
+    logger.debug('Searching for "%s" within archive...', path)
+
+    with tempfile.TemporaryDirectory() as d:
+      with tarfile.open(fileobj=BytesIO(archive), mode='r:gz') as f:
+        f.extractall(path=d)
+
+      # move stuff to "output"
+      basedir = os.listdir(d)[0]
+      shutil.move(os.path.join(d, basedir, path), output)
+
+
 def get_gitlab_instance():
     '''Returns an instance of the gitlab object for remote operations'''
 

bob/devtools/scripts/build.py

@@ -68,7 +68,7 @@ Examples:
 @bdt.raise_on_error
 def build(recipe_dir, python, condarc, config, channel, no_test, append_file,
     docserver, dry_run):
-  """Runs conda-build with a standard configuration and environment
+  """Builds package through conda-build with stock configuration
 
   This command wraps the execution of conda-build so that you use the same
   ``condarc`` and ``conda_build_config.yaml`` file we use for our CI.  It

bob/devtools/scripts/changelog.py

@@ -44,7 +44,7 @@ Examples:
   5. Generates a complete list of changelogs for a list of packages (one per line:
 
 \b
-     $ curl -o order.txt https://gitlab.idiap.ch/bob/bob.nightlies/raw/master/order.txt
+     $ bdt getpath bob/bob.nightlies order.txt
      $ bdt lasttag bob/bob
      # copy and paste date to next command
      $ bdt changelog --since="2018-07-17 10:23:40" order.txt changelog.md

bob/devtools/scripts/getpath.py

+#!/usr/bin/env python
+
+import os
+import logging
+logger = logging.getLogger(__name__)
+
+import click
+
+from . import bdt
+from ..log import verbosity_option
+from ..release import get_gitlab_instance, download_path
+
+
+@click.command(epilog='''
+Examples:
+
+  1. Get the file ``order.txt`` from bob.nightlies master branch:
+
+     $ bdt getpath bob/bob.nightlies order.txt
+
+
+  2. Get the file ``order.txt`` from a different branch ``2.x``:
+
+     $ bdt getpath --ref=2.x bob/bob.nightlies order.txt
+
+
+  3. Get the directory ``gitlab`` (and eventual sub-directories) from bob.admin, save outputs in directory ``_ci``:
+
+     $ bdt getpath bob/bob.admin master gitlab _ci
+''')
+@click.argument('package')
+@click.argument('path')
+@click.argument('output', type=click.Path(exists=False), required=False)
+@click.option('-r', '--ref', default='master', show_default=True,
+    help='Download path from the provided git reference (may be a branch, tag or commit hash)')
+@verbosity_option()
+@bdt.raise_on_error
+def getpath(package, path, output, ref):
+    """Downloads files and directories from gitlab
+
+    Files are downloaded and stored.  Directories are recursed and fully
+    downloaded to the client.
+    """
+
+    if '/' not in package:
+        raise RuntimeError('PACKAGE should be specified as "group/name"')
+
+    gl = get_gitlab_instance()
+
+    # we lookup the gitlab package once
+    use_package = gl.projects.get(package)
+    logger.info('Found gitlab project %s (id=%d)',
+        use_package.attributes['path_with_namespace'], use_package.id)
+    download_path(use_package, path, output, ref=ref)

bob/devtools/scripts/lasttag.py

@@ -37,7 +37,7 @@ def lasttag(package):
 
     gl = get_gitlab_instance()
 
-    # we lookup the gitlab group once
+    # we lookup the gitlab package once
     use_package = gl.projects.get(package)
     logger.info('Found gitlab project %s (id=%d)',
         use_package.attributes['path_with_namespace'], use_package.id)

bob/devtools/scripts/release.py

@@ -46,7 +46,7 @@ Examples:
 )
 @click.argument('changelog', type=click.File('rt', lazy=False))
 @click.option('-g', '--group', default='bob', show_default=True,
-    help='Group name where all packages are located')
+    help='Group name where all packages are located (if not provided with the package)')
 @click.option('-p', '--package',
     help='If the name of a package is provided, then this package will be ' \
         'found in the changelog file and the release will resume from it ' \

bob/devtools/scripts/visibility.py

@@ -25,7 +25,7 @@ Examples:
   2. Checks the visibility of all packages in a file list:
 
 \b
-     $ curl -o order.txt https://gitlab.idiap.ch/bob/bob.nightlies/raw/master/order.txt
+     $ bdt getpath bob/bob.nightlies order.txt
      $ bdt visibility order.txt
 ''')
 @click.argument('target')
@@ -36,13 +36,13 @@ Examples:
 @verbosity_option()
 @bdt.raise_on_error
 def visibility(target, group):
-    """Checks if the gitlab repository is visible to the current user
+    '''Reports visibility of gitlab repository
 
     This command checks if the named package is visible to the currently logged
     in user, and reports its visibility level ('public', 'internal',
     'private').  If the package does not exist or it is private to the current
     user, it says 'unknown' instead.
-    """
+    '''
 
     gl = get_gitlab_instance()
 

conda/meta.yaml

@@ -59,6 +59,7 @@ test:
     - bdt dumpsphinx --help
     - bdt bootstrap --help
     - bdt build --help
+    - bdt getpath --help
     - sphinx-build -aEW ${PREFIX}/share/doc/{{ name }}/doc {{ project_dir }}/sphinx
 
 about:

doc/release.rst

@@ -87,8 +87,8 @@ Here are the instructions to release Bob meta package:
 
   .. code-block:: sh
 
-     $ curl -o order.txt https://gitlab.idiap.ch/bob/bob.nightlies/raw/master/order.txt
-     $ bdt -vvv visibility order.txt
+     $ bdt getpath bob/bob.nightlies order.txt
+     $ bdt visibility order.txt
 
 * Put the list of public packages in ../../bob/requirements.txt
 * Run ``bdt changelog`` first:

setup.py

@@ -47,6 +47,7 @@ setup(
             'dumpsphinx = bob.devtools.scripts.dumpsphinx:dumpsphinx',
             'bootstrap = bob.devtools.scripts.bootstrap:bootstrap',
             'build = bob.devtools.scripts.build:build',
+            'getpath = bob.devtools.scripts.getpath:getpath',
         ],
     },
     classifiers=[