Commit e3e7ac3d authored by André Anjos's avatar André Anjos 💬

[scripts] Add getpath utility

parent 4a390926
Pipeline #25754 passed with stages
in 4 minutes and 31 seconds
......@@ -4,6 +4,7 @@
import os
import re
import time
import shutil
import gitlab
import logging
......@@ -12,6 +13,46 @@ logger = logging.getLogger(__name__)
from distutils.version import StrictVersion
def download_path(package, path, output=None, ref='master'):
'''Downloads paths from gitlab, with an optional recurse
This method will download an archive of the repository from chosen
reference, and then it will search insize the zip blob for the path to be
copied into output. It uses :py:class:`zipfile.ZipFile` to do this search.
This method will not be very efficient for larger repository references,
but works recursively by default.
Args:
package: the gitlab package object to use (should be pre-fetched)
path: the path on the project to download
output: where to place the path to be downloaded - if not provided, use
the basename of ``path`` as storage point with respect to the current
directory
ref: the name of the git reference (branch, tag or commit hash) to use
'''
from io import BytesIO
import tarfile
import tempfile
output = output or os.path.realpath(os.curdir)
logger.debug('Downloading archive of "%s" from "%s"...', ref,
package.attributes['path_with_namespace'])
archive = package.repository_archive(ref=ref)
logger.debug('Archive has %d bytes', len(archive))
logger.debug('Searching for "%s" within archive...', path)
with tempfile.TemporaryDirectory() as d:
with tarfile.open(fileobj=BytesIO(archive), mode='r:gz') as f:
f.extractall(path=d)
# move stuff to "output"
basedir = os.listdir(d)[0]
shutil.move(os.path.join(d, basedir, path), output)
def get_gitlab_instance():
'''Returns an instance of the gitlab object for remote operations'''
......
......@@ -68,7 +68,7 @@ Examples:
@bdt.raise_on_error
def build(recipe_dir, python, condarc, config, channel, no_test, append_file,
docserver, dry_run):
"""Runs conda-build with a standard configuration and environment
"""Builds package through conda-build with stock configuration
This command wraps the execution of conda-build so that you use the same
``condarc`` and ``conda_build_config.yaml`` file we use for our CI. It
......
......@@ -44,7 +44,7 @@ Examples:
5. Generates a complete list of changelogs for a list of packages (one per line:
\b
$ curl -o order.txt https://gitlab.idiap.ch/bob/bob.nightlies/raw/master/order.txt
$ bdt getpath bob/bob.nightlies order.txt
$ bdt lasttag bob/bob
# copy and paste date to next command
$ bdt changelog --since="2018-07-17 10:23:40" order.txt changelog.md
......
#!/usr/bin/env python
import os
import logging
logger = logging.getLogger(__name__)
import click
from . import bdt
from ..log import verbosity_option
from ..release import get_gitlab_instance, download_path
@click.command(epilog='''
Examples:
1. Get the file ``order.txt`` from bob.nightlies master branch:
$ bdt getpath bob/bob.nightlies order.txt
2. Get the file ``order.txt`` from a different branch ``2.x``:
$ bdt getpath --ref=2.x bob/bob.nightlies order.txt
3. Get the directory ``gitlab`` (and eventual sub-directories) from bob.admin, save outputs in directory ``_ci``:
$ bdt getpath bob/bob.admin master gitlab _ci
''')
@click.argument('package')
@click.argument('path')
@click.argument('output', type=click.Path(exists=False), required=False)
@click.option('-r', '--ref', default='master', show_default=True,
help='Download path from the provided git reference (may be a branch, tag or commit hash)')
@verbosity_option()
@bdt.raise_on_error
def getpath(package, path, output, ref):
"""Downloads files and directories from gitlab
Files are downloaded and stored. Directories are recursed and fully
downloaded to the client.
"""
if '/' not in package:
raise RuntimeError('PACKAGE should be specified as "group/name"')
gl = get_gitlab_instance()
# we lookup the gitlab package once
use_package = gl.projects.get(package)
logger.info('Found gitlab project %s (id=%d)',
use_package.attributes['path_with_namespace'], use_package.id)
download_path(use_package, path, output, ref=ref)
......@@ -37,7 +37,7 @@ def lasttag(package):
gl = get_gitlab_instance()
# we lookup the gitlab group once
# we lookup the gitlab package once
use_package = gl.projects.get(package)
logger.info('Found gitlab project %s (id=%d)',
use_package.attributes['path_with_namespace'], use_package.id)
......
......@@ -46,7 +46,7 @@ Examples:
)
@click.argument('changelog', type=click.File('rt', lazy=False))
@click.option('-g', '--group', default='bob', show_default=True,
help='Group name where all packages are located')
help='Group name where all packages are located (if not provided with the package)')
@click.option('-p', '--package',
help='If the name of a package is provided, then this package will be ' \
'found in the changelog file and the release will resume from it ' \
......
......@@ -25,7 +25,7 @@ Examples:
2. Checks the visibility of all packages in a file list:
\b
$ curl -o order.txt https://gitlab.idiap.ch/bob/bob.nightlies/raw/master/order.txt
$ bdt getpath bob/bob.nightlies order.txt
$ bdt visibility order.txt
''')
@click.argument('target')
......@@ -36,13 +36,13 @@ Examples:
@verbosity_option()
@bdt.raise_on_error
def visibility(target, group):
"""Checks if the gitlab repository is visible to the current user
'''Reports visibility of gitlab repository
This command checks if the named package is visible to the currently logged
in user, and reports its visibility level ('public', 'internal',
'private'). If the package does not exist or it is private to the current
user, it says 'unknown' instead.
"""
'''
gl = get_gitlab_instance()
......
......@@ -59,6 +59,7 @@ test:
- bdt dumpsphinx --help
- bdt bootstrap --help
- bdt build --help
- bdt getpath --help
- sphinx-build -aEW ${PREFIX}/share/doc/{{ name }}/doc {{ project_dir }}/sphinx
about:
......
......@@ -87,8 +87,8 @@ Here are the instructions to release Bob meta package:
.. code-block:: sh
$ curl -o order.txt https://gitlab.idiap.ch/bob/bob.nightlies/raw/master/order.txt
$ bdt -vvv visibility order.txt
$ bdt getpath bob/bob.nightlies order.txt
$ bdt visibility order.txt
* Put the list of public packages in ../../bob/requirements.txt
* Run ``bdt changelog`` first:
......
......@@ -47,6 +47,7 @@ setup(
'dumpsphinx = bob.devtools.scripts.dumpsphinx:dumpsphinx',
'bootstrap = bob.devtools.scripts.bootstrap:bootstrap',
'build = bob.devtools.scripts.build:build',
'getpath = bob.devtools.scripts.getpath:getpath',
],
},
classifiers=[
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment