From e92948658866333e405b5bf60485c377d679a89d Mon Sep 17 00:00:00 2001
From: Andre Anjos <andre.dos.anjos@gmail.com>
Date: Mon, 4 Mar 2019 08:42:30 +0100
Subject: [PATCH] [deploy] Add new deployment module and applies DRY to
scripts.ci (closes #18)
---
bob/devtools/deploy.py | 134 +++++++++++++++++++++++++++++
bob/devtools/scripts/ci.py | 171 ++++++++++---------------------------
doc/api.rst | 2 +
3 files changed, 179 insertions(+), 128 deletions(-)
create mode 100644 bob/devtools/deploy.py
diff --git a/bob/devtools/deploy.py b/bob/devtools/deploy.py
new file mode 100644
index 00000000..e67e57ab
--- /dev/null
+++ b/bob/devtools/deploy.py
@@ -0,0 +1,134 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+'''Deployment utilities for conda packages and documentation via webDAV'''
+
+
+
+from .constants import WEBDAV_PATHS, SERVER
+from .log import get_logger
+logger = get_logger(__name__)
+
+
+def _setup_webdav_client(server, root, username, password):
+ '''Configures and checks the webdav client'''
+
+ # setup webdav connection
+ webdav_options = dict(webdav_hostname=server, webdav_root=root,
+ webdav_login=username, webdav_password=password)
+
+ from .webdav3 import client as webdav
+
+ retval = webdav.Client(webdav_options)
+ assert retval.valid()
+
+ return retval
+
+
+def deploy_conda_package(package, stable, public, username, password,
+ overwrite, dry_run):
+ '''Deploys a single conda package on the appropriate path
+
+ Args:
+
+ package (str): Path leading to the conda package to be deployed
+ stable (bool): Indicates if the package should be deployed on a stable
+ (``True``) or beta (``False``) channel
+ public (bool): Indicates if the package is supposed to be distributed
+ publicly or privatly (within Idiap network)
+ username (str): The name of the user on the webDAV server to use for
+ uploading the package
+ password (str): The password of the user on the webDAV server to use for
+ uploading the package
+ overwrite (bool): If we should overwrite a package with equal name existing
+ on the destination directory. Otherwise, an exception is raised.
+ dry_run (bool): If we're supposed to really do the actions, or just log
+ messages.
+
+ '''
+
+ server_info = WEBDAV_PATHS[stable][public]
+ davclient = _setup_webdav_client(SERVER, server_info['root'], username,
+ password)
+
+ logger.info('Deploying %s to %s%s%s...', package, SERVER,
+ server_info['root'], server_info['conda'])
+
+ basename = os.path.basename(package)
+ remote_path = '%s/%s/%s' % (server_info['conda'], arch, basename)
+
+ if davclient.check(remote_path):
+ if not overwrite:
+ raise RuntimeError('The file %s/%s already exists on the server ' \
+ '- this can be due to more than one build with deployment ' \
+ 'running at the same time. Re-running the broken builds ' \
+ 'normally fixes it' % (SERVER, remote_path))
+
+ else:
+ logger.info('[dav] rm -f %s%s%s', SERVER, server_info['root'],
+ remote_path)
+ if not dry_run:
+ davclient.clean(remote_path)
+
+ logger.info('[dav] %s -> %s%s%s', package, SERVER, server_info['root'],
+ remote_path)
+ if not dry_run:
+ davclient.upload(local_path=package, remote_path=remote_path)
+
+ logger.debug('Removing local copy (%s) to avoid re-uploads', k)
+ os.unlink(package)
+
+
+def deploy_documentation(path, package, stable, public, branch, tag, username,
+ password, dry_run):
+ '''Deploys sphinx documentation to the appropriate webdav locations
+
+ Args:
+
+ path (str): Path leading to the root of the documentation to be deployed
+ package (str): Full name (with namespace) of the package being treated
+ stable (bool): Indicates if the documentation corresponds to the latest
+ stable build
+ public (bool): Indicates if the documentation is supposed to be distributed
+ publicly or privatly (within Idiap network)
+ branch (str): The name of the branch for the current build
+ tag (str): The name of the tag currently built (may be ``None``)
+ username (str): The name of the user on the webDAV server to use for
+ uploading the package
+ password (str): The password of the user on the webDAV server to use for
+ uploading the package
+ dry_run (bool): If we're supposed to really do the actions, or just log
+ messages.
+
+ '''
+
+ # uploads documentation artifacts
+ if not os.path.exists(path):
+ raise RuntimeError('Documentation is not available at %s - ' \
+ 'ensure documentation is being produced for your project!' % path)
+
+ server_info = WEBDAV_PATHS[stable][public]
+ davclient = _setup_webdav_client(SERVER, server_info['root'], username,
+ password)
+
+ remote_path_prefix = '%s/%s' % (server_info['docs'], package)
+
+ # finds out the correct mixture of sub-directories we should deploy to.
+ # 1. if ref-name is a tag, don't forget to publish to 'master' as well -
+ # all tags are checked to come from that branch
+ # 2. if ref-name is a branch name, deploy to it
+ # 3. in case a tag is being published, make sure to deploy to the special
+ # "stable" subdir as well
+ deploy_docs_to = set([branch])
+ if stable:
+ deploy_docs_to.add('master')
+ if tag is not None:
+ deploy_docs_to.add(tag)
+ deploy_docs_to.add('stable')
+
+ for k in deploy_docs_to:
+ remote_path = '%s/%s' % (remote_path_prefix, k)
+ logger.info('[dav] %s -> %s%s%s', path, SERVER, server_info['root'],
+ remote_path)
+ if not dry_run:
+ davclient.upload_directory(local_path=path, remote_path=remote_path)
diff --git a/bob/devtools/scripts/ci.py b/bob/devtools/scripts/ci.py
index 9f753fa5..f6cc0b33 100644
--- a/bob/devtools/scripts/ci.py
+++ b/bob/devtools/scripts/ci.py
@@ -14,6 +14,7 @@ from click_plugins import with_plugins
from . import bdt
from ..constants import SERVER, CONDA_BUILD_CONFIG, CONDA_RECIPE_APPEND, \
WEBDAV_PATHS
+from ..deploy import deploy_conda_package, deploy_documentation
from ..log import verbosity_option, get_logger, echo_normal
logger = get_logger(__name__)
@@ -58,47 +59,25 @@ def base_deploy(dry_run):
logger.warn('Nothing is being deployed to server')
package = os.environ['CI_PROJECT_PATH']
-
- server_info = WEBDAV_PATHS[True][True] #stable=True, visible=True
-
- logger.info('Deploying dependence packages to %s%s%s...', SERVER,
- server_info['root'], server_info['conda'])
-
- # setup webdav connection
- webdav_options = {
- 'webdav_hostname': SERVER,
- 'webdav_root': server_info['root'],
- 'webdav_login': os.environ['DOCUSER'],
- 'webdav_password': os.environ['DOCPASS'],
- }
- from ..webdav3 import client as webdav
- davclient = webdav.Client(webdav_options)
- assert davclient.valid()
-
group, name = package.split('/')
- # uploads conda package artificats
+ # deploys all conda package artefacts currently available (erases them
+ # afterwards)
for arch in ('linux-64', 'osx-64', 'noarch'):
# finds conda dependencies and uploads what we can find
package_path = os.path.join(os.environ['CONDA_ROOT'], 'conda-bld', arch,
'*.tar.bz2')
deploy_packages = glob.glob(package_path)
+
for k in deploy_packages:
- basename = os.path.basename(k)
- if basename.startswith(name):
+
+ if os.path.basename(k).startswith(name):
logger.debug('Skipping deploying of %s - not a base package', k)
continue
- remote_path = '%s/%s/%s' % (server_info['conda'], arch, basename)
- if davclient.check(remote_path):
- raise RuntimeError('The file %s/%s already exists on the server ' \
- '- this can be due to more than one build with deployment ' \
- 'running at the same time. Re-running the broken builds ' \
- 'normally fixes it' % (SERVER, remote_path))
- logger.info('[dav] %s -> %s%s%s', k, SERVER, server_info['root'],
- remote_path)
- if not dry_run:
- davclient.upload(local_path=k, remote_path=remote_path)
+ deploy_conda_package(k, stable=True, public=True,
+ username=os.environ['DOCUSER'], password=os.environ['DOCPASS'],
+ overwrite=False, dry_run=dry_run)
@ci.command(epilog='''
@@ -132,79 +111,29 @@ def deploy(dry_run):
logger.warn('Nothing is being deployed to server')
package = os.environ['CI_PROJECT_PATH']
+ group, name = package.split('/')
- # determine project visibility
- visible = (os.environ['CI_PROJECT_VISIBILITY'] == 'public')
-
- # determine if building branch or tag
+ # determine if building branch or tag, and project visibility
stable = ('CI_COMMIT_TAG' in os.environ)
+ public = (os.environ['CI_PROJECT_VISIBILITY'] == 'public')
- server_info = WEBDAV_PATHS[stable][visible]
-
- logger.info('Deploying conda packages to %s%s%s...', SERVER,
- server_info['root'], server_info['conda'])
-
- # setup webdav connection
- webdav_options = {
- 'webdav_hostname': SERVER,
- 'webdav_root': server_info['root'],
- 'webdav_login': os.environ['DOCUSER'],
- 'webdav_password': os.environ['DOCPASS'],
- }
- from ..webdav3 import client as webdav
- davclient = webdav.Client(webdav_options)
- assert davclient.valid()
-
- group, name = package.split('/')
-
- # uploads conda package artificats
+ # deploys all conda package artefacts currently available (erases them
+ # afterwards)
for arch in ('linux-64', 'osx-64', 'noarch'):
# finds conda packages and uploads what we can find
package_path = os.path.join(os.environ['CONDA_ROOT'], 'conda-bld', arch,
name + '*.tar.bz2')
deploy_packages = glob.glob(package_path)
for k in deploy_packages:
- remote_path = '%s/%s/%s' % (server_info['conda'], arch,
- os.path.basename(k))
- if davclient.check(remote_path):
- raise RuntimeError('The file %s/%s already exists on the server ' \
- '- this can be due to more than one build with deployment ' \
- 'running at the same time. Re-running the broken builds ' \
- 'normally fixes it' % (SERVER, remote_path))
- logger.info('[dav] %s -> %s%s%s', k, SERVER, server_info['root'],
- remote_path)
- if not dry_run:
- davclient.upload(local_path=k, remote_path=remote_path)
-
- # uploads documentation artifacts
+ deploy_conda_package(k, stable=stable, public=visible,
+ username=os.environ['DOCUSER'], password=os.environ['DOCPASS'],
+ overwrite=False, dry_run=dry_run)
+
local_docs = os.path.join(os.environ['CI_PROJECT_DIR'], 'sphinx')
- if not os.path.exists(local_docs):
- raise RuntimeError('Documentation is not available at %s - ' \
- 'ensure documentation is being produced for your project!' % \
- local_docs)
-
- remote_path_prefix = '%s/%s' % (server_info['docs'], package)
-
- # finds out the correct mixture of sub-directories we should deploy to.
- # 1. if ref-name is a tag, don't forget to publish to 'master' as well -
- # all tags are checked to come from that branch
- # 2. if ref-name is a branch name, deploy to it
- # 3. in case a tag is being published, make sure to deploy to the special
- # "stable" subdir as well
- deploy_docs_to = set([os.environ['CI_COMMIT_REF_NAME']])
- if stable:
- deploy_docs_to.add('master')
- if os.environ.get('CI_COMMIT_TAG') is not None:
- deploy_docs_to.add(os.environ['CI_COMMIT_TAG'])
- deploy_docs_to.add('stable')
-
- for k in deploy_docs_to:
- remote_path = '%s/%s' % (remote_path_prefix, k)
- logger.info('[dav] %s -> %s%s%s', local_docs, SERVER,
- server_info['root'], remote_path)
- if not dry_run:
- davclient.upload_directory(local_path=local_docs,
- remote_path=remote_path)
+ deploy_documentation(local_docs, package, stable=stable, public=public,
+ branch=os.environ['CI_COMMIT_REF_NAME'],
+ tag=os.environ.get('CI_COMMIT_TAG'), username=os.environ['DOCUSER'],
+ password=os.environ['DOCPASS'], dry_run=dry_run)
@ci.command(epilog='''
@@ -580,43 +509,29 @@ def nightlies(ctx, order, dry_run):
ci=True,
)
- sphinx_output = os.path.join(os.environ['CI_PROJECT_DIR'], 'sphinx')
- if os.path.exists(sphinx_output):
- logger.debug('Sphinx output was generated during test/rebuild of %s - ' \
- 'Erasing...', package)
- shutil.rmtree(sphinx_output)
-
- # re-deploys a new conda package if it was rebuilt
+ local_docs = os.path.join(os.environ['CI_PROJECT_DIR'], 'sphinx')
+ is_master = os.environ['CI_COMMIT_REF_NAME'] == 'master'
+
+ # re-deploys freshly built documentation if we're on the master branch
+ # otherwise, removes the documentation
+ if is_master:
+ deploy_documentation(local_docs, package, stable=stable,
+ public=(not private), branch='master', tag=None,
+ username=os.environ['DOCUSER'], password=os.environ['DOCPASS'],
+ dry_run=dry_run)
+ elif os.path.exists(local_docs):
+ logger.debug('Sphinx output was generated during test/rebuild ' \
+ 'of %s - Erasing...', package)
+ shutil.rmtree(local_docs)
+
+ # re-deploys a new conda package if it was rebuilt and it is the master
+ # branch
# n.b.: can only arrive here if dry_run was ``False`` (no need to check
# again)
- if 'BDT_REBUILD' in os.environ:
-
+ if 'BDT_REBUILD' in os.environ and is_master:
tarball = os.environ['BDT_REBUILD']
del os.environ['BDT_REBUILD']
- server_info = WEBDAV_PATHS[stable][not private]
-
- logger.info('Deploying conda package to %s%s%s...', SERVER,
- server_info['root'], server_info['conda'])
-
- # setup webdav connection
- webdav_options = {
- 'webdav_hostname': SERVER,
- 'webdav_root': server_info['root'],
- 'webdav_login': os.environ['DOCUSER'],
- 'webdav_password': os.environ['DOCPASS'],
- }
- from ..webdav3 import client as webdav
- davclient = webdav.Client(webdav_options)
- assert davclient.valid()
-
- remote_path = '%s/%s/%s' % (server_info['conda'], arch,
- os.path.basename(tarball))
- if davclient.check(remote_path):
- raise RuntimeError('The file %s/%s already exists on the server ' \
- '- this can be due to more than one rebuild with deployment ' \
- 'running at the same time. Re-running the broken builds ' \
- 'normally fixes it' % (SERVER, remote_path))
- logger.info('[dav] %s -> %s%s%s', k, SERVER, server_info['root'],
- remote_path)
- davclient.upload(local_path=tarball, remote_path=remote_path)
+ deploy_conda_package(tarball, stable=stable, public=(not private),
+ username=os.environ['DOCUSER'], password=os.environ['DOCPASS'],
+ overwrite=False, dry_run=dry_run)
diff --git a/doc/api.rst b/doc/api.rst
index 3ed4bd75..0d20dd0e 100644
--- a/doc/api.rst
+++ b/doc/api.rst
@@ -33,6 +33,8 @@ Detailed Information
.. automodule:: bob.devtools.build
+.. automodule:: bob.devtools.deploy
+
WebDAV Python Client
--------------------
--
GitLab