[ci] New deployment subcommand

.gitlab-ci.yml

@@ -77,7 +77,7 @@ build_macosx_36:
   script:
     - source ${CONDA_ROOT}/etc/profile.d/conda.sh
     - conda activate myenv
-    - bdt --help
+    - bdt ci deploy -vv --dry-run
   dependencies:
     - build_linux_36
     - build_macosx_36

LICENSE

-Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/
+Copyright (c) 2018 Idiap Research Institute, http://www.idiap.ch/
 Written by Andre Anjos <andre.anjos@idiap.ch>
 
 Redistribution and use in source and binary forms, with or without
@@ -25,3 +25,36 @@ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+------------------------------------------------------------------------------
+
+Code from the "webdav3" directory was copied from the Github repository
+https://github.com/ezhov-evgeny/webdav-client-python-3, but later modified and
+repackaged as part of this package.
+
+The authors asked to reproduce the following license text.
+
+
+COPYRIGHT AND PERMISSION NOTICE
+-------------------------------
+
+Copyright (c) 2016, The WDC Project, and many contributors, see the THANKS
+file.
+
+All rights reserved.
+
+Permission to use, copy, modify, and distribute this software for any purpose
+with or without fee is hereby granted, provided that the above copyright notice
+and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. IN
+NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
+OR OTHER DEALINGS IN THE SOFTWARE.
+
+Except as contained in this notice, the name of a copyright holder shall not be
+used in advertising or otherwise to promote the sale, use or other dealings in
+this Software without prior written authorization of the copyright holder.

bob/devtools/ci.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+'''Tools to help CI-based builds and artifact deployment'''
+
+
+import os
+
+import logging
+logger = logging.getLogger(__name__)
+
+import git
+import packaging.version
+
+
+def is_master(refname, tag):
+  '''Tells if we're on the master branch via ref_name or tag
+
+  This function checks if the name of the branch being built is "master".  If a
+  tag is set, then it checks if the tag is on the master branch.  If so, then
+  also returns ``True``, otherwise, ``False``.
+
+  Args:
+
+    refname: The value of the environment variable ``CI_COMMIT_REF_NAME``
+    tag: The value of the environment variable ``CI_COMMIT_TAG`` - (may be
+      ``None``)
+
+  Returns: a boolean, indicating we're building the master branch **or** that
+  the tag being built was issued on the master branch.
+  '''
+
+  if tag is not None:
+    repo = git.Repo(os.environ['CI_PROJECT_DIR'])
+    _tag = repo.tag('refs/tags/%s' % tag)
+    return _tag.commit in repo.iter_commits(rev='master')
+
+  return refname == 'master'
+
+
+def is_visible_outside(package, visibility):
+  '''Determines if the project is visible outside Idiap'''
+
+  logger.info('Project %s visibility is "%s"', package, visibility)
+
+  if visibility == 'internal':
+    visibility = 'private' #the same thing for this command
+    logger.warn('Project %s visibility switched to "%s".  ' \
+        'For this command, it all boils down to the same...', package,
+        visibility)
+
+  return visibility == 'public'
+
+
+def is_stable(package, refname, tag):
+  '''Determines if the package being published is stable
+
+  This is done by checking if a tag was set for the package.  If that is the
+  case, we still cross-check the tag is on the "master" branch.  If everything
+  checks out, we return ``True``.  Else, ``False``.
+
+  Args:
+
+    package: Package name in the format "group/name"
+    refname: The current value of the environment ``CI_COMMIT_REF_NAME``
+    tag: The current value of the enviroment ``CI_COMMIT_TAG`` (may be
+      ``None``)
+
+  Returns: a boolean, indicating if the current build is for a stable release
+  '''
+
+  if tag is not None:
+    logger.info('Project %s tag is "%s"', package, tag)
+    parsed_tag = packaging.version.Version(tag)
+
+    if parsed_tag.is_prerelease:
+      logger.warn('Pre-release detected - not publishing to stable channels')
+      return False
+
+    if is_master(os.environ['CI_COMMIT_REF_NAME'], tag):
+      return True
+    else:
+      logger.warn('Tag %s in non-master branch will be ignored', tag)
+      return False
+
+  logger.info('No tag information available at build')
+  logger.info('Considering this to be a pre-release build')
+  return False

bob/devtools/constants.py

@@ -29,6 +29,48 @@ SERVER = 'http://www.idiap.ch'
 '''This is the default server use use to store data and build artifacts'''
 
 
+CONDA_CHANNELS = {
+    True: {  #stable?
+      False: '/private/conda',  #visible outside?
+      True: '/public/conda',
+      },
+    False: {
+      False: '/private/conda/label/beta',  #visible outside?
+      True: '/public/conda/label/beta',
+      },
+    }
+'''Default locations of our stable, beta, public and private conda channels'''
+
+
+WEBDAV_PATHS = {
+    True: {  #stable?
+      False: {  #visible?
+        'root': '/private-upload',
+        'conda': '/conda',
+        'docs': '/docs',
+        },
+      True: {  #visible?
+        'root': '/public-upload',
+        'conda': '/conda',
+        'docs': '/docs',
+        },
+      },
+    False: {  #stable?
+      False: {  #visible?
+        'root': '/private-upload',
+        'conda': '/conda/label/beta',
+        'docs': '/docs',
+        },
+      True: {  #visible?
+        'root': '/public-upload',
+        'conda': '/conda/label/beta',
+        'docs': '/docs',
+        },
+      },
+    }
+'''Default locations of our webdav upload paths'''
+
+
 IDIAP_ROOT_CA = b'''
 Idiap Root CA 2016 - for internal use
 =====================================

bob/devtools/release.py

@@ -61,9 +61,13 @@ def get_gitlab_instance():
     cfgs = [os.path.expanduser(k) for k in cfgs]
     if any([os.path.exists(k) for k in cfgs]):
         gl = gitlab.Gitlab.from_config('idiap', cfgs)
-    else: #ask the user for a token
+    else: #ask the user for a token or use one from the current runner
         server = "https://gitlab.idiap.ch"
-        token = input("%s token: " % server)
+        token = os.environ.get('CI_JOB_TOKEN')
+        if token is None:
+          logger.debug('Did not find any of %s nor CI_JOB_TOKEN is defined. ' \
+              'Asking for user token on the command line...', '|'.join(cfgs))
+          token = input("Your %s (private) token: " % server)
         gl = gitlab.Gitlab(server, private_token=token, api_version=4)
 
     return gl

bob/devtools/scripts/ci.py

+#!/usr/bin/env python
+
+import os
+import re
+import glob
+import logging
+logger = logging.getLogger(__name__)
+
+import click
+import pkg_resources
+from click_plugins import with_plugins
+
+from . import bdt
+from ..log import verbosity_option
+from ..ci import is_stable, is_visible_outside
+from ..constants import SERVER, WEBDAV_PATHS
+from .. import webdav3
+
+
+@with_plugins(pkg_resources.iter_entry_points('bdt.ci.cli'))
+@click.group(cls=bdt.AliasedGroup)
+def ci():
+  """Commands for building packages and handling CI activities
+
+  Commands defined here are supposed to run on our CI, where a number of
+  variables that define their behavior is correctly defined.  Do **NOT**
+  attempt to run these commands in your own installation.  Unexpected errors
+  may occur.
+  """
+  pass
+
+
+@ci.command(epilog='''
+Examples:
+
+  1. Deploys current build artifacts to the appropriate channels:
+
+     $ bdt ci deploy -vv
+
+''')
+@click.option('-d', '--dry-run/--no-dry-run', default=False,
+    help='Only goes through the actions, but does not execute them ' \
+        '(combine with the verbosity flags - e.g. ``-vvv``) to enable ' \
+        'printing to help you understand what will be done')
+@verbosity_option()
+@bdt.raise_on_error
+def deploy(dry_run):
+    """Deploys build artifacts (conda packages and sphinx documentation)
+
+    Deployment happens at the "right" locations - conda packages which do not
+    represent stable releases are deployed to our conda "beta" channel, while
+    stable packages to our root channel.  Sphinx documentation from unstable
+    builds (typically the master branch) is deployed to the documentation
+    server in a subdirectory named after the current branch name, while stable
+    documentation is deployed to a special subdirectory named "stable" and to
+    the respective tag name.
+    """
+
+    if dry_run:
+        logger.warn('!!!! DRY RUN MODE !!!!')
+        logger.warn('Nothing is being deployed to server')
+
+    package = os.environ['CI_PROJECT_PATH']
+
+    # determine project visibility
+    visible = is_visible_outside(package, os.environ['CI_PROJECT_VISIBILITY'])
+
+    # determine if building master branch or tag - and if tag is on master
+    tag = os.environ.get('CI_COMMIT_TAG')
+    stable = is_stable(package, os.environ['CI_COMMIT_REF_NAME'], tag)
+
+    server_info = WEBDAV_PATHS[stable][visible]
+
+    logger.info('Deploying conda packages to %s/%s%s...', SERVER,
+        server_info['root'], server_info['conda'])
+
+    # setup webdav connection
+    webdav_options = {
+        'webdav_hostname': SERVER,
+        'webdav_root': server_info['root'],
+        'webdav_login': os.environ['DOCUSER'],
+        'webdav_password': os.environ['DOCPASS'],
+        }
+    davclient = Client(options)
+    assert davclient.valid()
+
+    group, name = package.split('/')
+
+    # uploads conda package artificats
+    for arch in ('linux-64', 'osx-64', 'noarch'):
+      # finds conda packages and uploads what we can find
+      package_path = os.path.join(os.environ['CONDA_ROOT'], 'conda-bld', arch,
+          name + '*.tar.bz2')
+      deploy_packages = glob.glob(package_path)
+      if len(deploy_packages):
+        logger.info('Deploying %d conda package(s) for %s',
+            len(deploy_packages), arch)
+      for k in deploy_packages:
+        remote_path = '%s%s/%s' % (server_info['root'], server_info['conda'],
+            os.path.basename(k))
+        if davclient.check(remote_path):
+          raise RuntimeError('The file %s/%s already exists on the server ' \
+              '- this can be due to more than one build with deployment ' \
+              'running at the same time.  Re-running the broken builds ' \
+              'normally fixes it' % (SERVER, remote_path))
+        if not dry_run:
+          davclient.upload(local_path=k, remote_path=remote_path)
+
+    # uploads documentation artifacts
+    local_docs = os.path.join(os.environ['CI_PROJECT_DIR'], 'sphinx')
+    if not os.path.exists(local_docs):
+      raise RuntimeError('Documentation is not available at %s - ' \
+          'ensure documentation is being produced for your project!' % \
+          local_docs)
+
+    remote_path_prefix = '%s%s/%s' % (server_info['root'], server_info['docs'],
+        package)
+
+    # finds out the correct mixture of sub-directories we should deploy to.
+    # 1. if ref-name is a tag, don't forget to publish to 'master' as well -
+    # all tags are checked to come from that branch
+    # 2. if ref-name is a branch name, deploy to it
+    # 3. in case a tag is being published, make sure to deploy to the special
+    # "stable" subdir as well
+    deploy_docs_to = set([os.environ['CI_COMMIT_REF_NAME']])
+    if stable:
+      deploy_docs_to.add('master')
+      if os.environ.get('CI_COMMIT_TAG') is not None:
+        deploy_docs_to.add(os.environ['CI_COMMIT_TAG'])
+      deploy_docs_to.add('stable')
+
+    for k in deploy_docs_to:
+      remote_path = '%s/%s' % (remote_path_prefix, k)
+      logger.info('Deploying package documentation to %s/%s...', SERVER,
+          remote_path)
+      if not dry_run:
+        client.upload_directory(local_path=local_docs, remote_path=remote_path)

bob/devtools/webdav3/connection.py

+from os.path import exists
+
+from .exceptions import *
+from .urn import Urn
+
+
+class ConnectionSettings:
+    def is_valid(self):
+        pass
+
+    def valid(self):
+
+        try:
+            self.is_valid()
+        except OptionNotValid:
+            return False
+        else:
+            return True
+
+
+class WebDAVSettings(ConnectionSettings):
+    ns = "webdav:"
+    prefix = "webdav_"
+    keys = {'hostname', 'login', 'password', 'token', 'root', 'cert_path', 'key_path', 'recv_speed', 'send_speed',
+            'verbose'}
+
+    hostname = None
+    login = None
+    password = None
+    token = None
+    root = None
+    cert_path = None
+    key_path = None
+    recv_speed = None
+    send_speed = None
+    verbose = None
+
+    def __init__(self, options):
+
+        self.options = dict()
+
+        for key in self.keys:
+            value = options.get(key, '')
+            self.options[key] = value
+            self.__dict__[key] = value
+
+        self.root = Urn(self.root).quote() if self.root else ''
+        self.root = self.root.rstrip(Urn.separate)
+
+    def is_valid(self):
+
+        if not self.hostname:
+            raise OptionNotValid(name="hostname", value=self.hostname, ns=self.ns)
+
+        if self.cert_path and not exists(self.cert_path):
+            raise OptionNotValid(name="cert_path", value=self.cert_path, ns=self.ns)
+
+        if self.key_path and not exists(self.key_path):
+            raise OptionNotValid(name="key_path", value=self.key_path, ns=self.ns)
+
+        if self.key_path and not self.cert_path:
+            raise OptionNotValid(name="cert_path", value=self.cert_path, ns=self.ns)
+
+        if self.password and not self.login:
+            raise OptionNotValid(name="login", value=self.login, ns=self.ns)
+
+        if not self.token and not self.login:
+            raise OptionNotValid(name="login", value=self.login, ns=self.ns)
+
+
+class ProxySettings(ConnectionSettings):
+    ns = "proxy:"
+    prefix = "proxy_"
+    keys = {'hostname', 'login', 'password'}
+
+    hostname = None
+    login = None
+    password = None
+
+    def __init__(self, options):
+
+        self.options = dict()
+
+        for key in self.keys:
+            value = options.get(key, '')
+            self.options[key] = value
+            self.__dict__[key] = value
+
+    def is_valid(self):
+
+        if self.password and not self.login:
+            raise OptionNotValid(name="login", value=self.login, ns=self.ns)
+
+        if self.login or self.password:
+            if not self.hostname:
+                raise OptionNotValid(name="hostname", value=self.hostname, ns=self.ns)

bob/devtools/webdav3/exceptions.py

+class WebDavException(Exception):
+    pass
+
+
+class NotValid(WebDavException):
+    pass
+
+
+class OptionNotValid(NotValid):
+    def __init__(self, name, value, ns=""):
+        self.name = name
+        self.value = value
+        self.ns = ns
+
+    def __str__(self):
+        return "Option ({ns}{name}={value}) have invalid name or value".format(ns=self.ns, name=self.name,
+                                                                               value=self.value)
+
+
+class CertificateNotValid(NotValid):
+    pass
+
+
+class NotFound(WebDavException):
+    pass
+
+
+class LocalResourceNotFound(NotFound):
+    def __init__(self, path):
+        self.path = path
+
+    def __str__(self):
+        return "Local file: {path} not found".format(path=self.path)
+
+
+class RemoteResourceNotFound(NotFound):
+    def __init__(self, path):
+        self.path = path
+
+    def __str__(self):
+        return "Remote resource: {path} not found".format(path=self.path)
+
+
+class RemoteParentNotFound(NotFound):
+    def __init__(self, path):
+        self.path = path
+
+    def __str__(self):
+        return "Remote parent for: {path} not found".format(path=self.path)
+
+
+class ResourceTooBig(WebDavException):
+    def __init__(self, path, size, max_size):
+        self.path = path
+        self.size = size
+        self.max_size = max_size
+
+    def __str__(self):
+        return "Resource {path} is too big, it should be less then {max_size} but actually: {size}".format(
+            path=self.path,
+            max_size=self.max_size,
+            size=self.size)
+
+
+class MethodNotSupported(WebDavException):
+    def __init__(self, name, server):
+        self.name = name
+        self.server = server
+
+    def __str__(self):
+        return "Method {name} not supported for {server}".format(name=self.name, server=self.server)
+
+
+class ConnectionException(WebDavException):
+    def __init__(self, exception):
+        self.exception = exception
+
+    def __str__(self):
+        return self.exception.__str__()
+
+
+class NoConnection(WebDavException):
+    def __init__(self, hostname):
+        self.hostname = hostname
+
+    def __str__(self):
+        return "Not connection with {hostname}".format(hostname=self.hostname)
+
+
+# This exception left only for supporting original library interface.
+class NotConnection(WebDavException):
+    def __init__(self, hostname):
+        self.hostname = hostname
+
+    def __str__(self):
+        return "No connection with {hostname}".format(hostname=self.hostname)
+
+
+class ResponseErrorCode(WebDavException):
+    def __init__(self, url, code, message):
+        self.url = url
+        self.code = code
+        self.message = message
+
+    def __str__(self):
+        return "Request to {url} failed with code {code} and message: {message}".format(url=self.url, code=self.code,
+                                                                                        message=self.message)
+
+
+class NotEnoughSpace(WebDavException):
+    def __init__(self):
+        pass
+
+    def __str__(self):
+        return "Not enough space on the server"

bob/devtools/webdav3/urn.py

+try:
+    from urllib.parse import unquote, quote, urlsplit
+except ImportError:
+    from urllib import unquote, quote
+    from urlparse import urlsplit
+
+from re import sub
+
+
+class Urn(object):
+
+    separate = "/"
+
+    def __init__(self, path, directory=False):
+
+        self._path = quote(path)
+        expressions = "/\.+/", "/+"
+        for expression in expressions:
+            self._path = sub(expression, Urn.separate, self._path)
+
+        if not self._path.startswith(Urn.separate):
+            self._path = "{begin}{end}".format(begin=Urn.separate, end=self._path)
+
+        if directory and not self._path.endswith(Urn.separate):
+            self._path = "{begin}{end}".format(begin=self._path, end=Urn.separate)
+
+    def __str__(self):
+        return self.path()
+
+    def path(self):
+        return unquote(self._path)
+
+    def quote(self):
+        return self._path
+
+    def filename(self):
+
+        path_split = self._path.split(Urn.separate)
+        name = path_split[-2] + Urn.separate if path_split[-1] == '' else path_split[-1]
+        return unquote(name)
+
+    def parent(self):
+
+        path_split = self._path.split(Urn.separate)
+        nesting_level = self.nesting_level()
+        parent_path_split = path_split[:nesting_level]
+        parent = self.separate.join(parent_path_split) if nesting_level != 1 else Urn.separate
+        if not parent.endswith(Urn.separate):
+            return unquote(parent + Urn.separate)
+        else:
+            return unquote(parent)
+
+    def nesting_level(self):
+        return self._path.count(Urn.separate, 0, -1)
+
+    def is_dir(self):
+        return self._path[-1] == Urn.separate
+
+    @staticmethod
+    def normalize_path(path):
+        result = sub('/{2,}', '/', path)
+        return result if len(result) < 1 or result[-1] != Urn.separate else result[:-1]
+
+    @staticmethod
+    def compare_path(path_a, href):
+        unqouted_path = Urn.separate + unquote(urlsplit(href).path)
+        return Urn.normalize_path(path_a) == Urn.normalize_path(unqouted_path)

conda/meta.yaml

@@ -44,6 +44,8 @@ requirements:
     - sphinx
     - pyyaml
     - twine
+    - packaging
+    - lxml
 
 test:
   requires:

setup.py

@@ -17,6 +17,8 @@ requires = [
     'sphinx',
     'pyyaml',
     'twine',
+    'packaging',
+    'lxml',
     ]
 
 setup(
@@ -41,16 +43,21 @@ setup(
             'bdt = bob.devtools.scripts.bdt:main',
         ],
         'bdt.cli': [
-            'release = bob.devtools.scripts.release:release',
-            'changelog = bob.devtools.scripts.changelog:changelog',
-            'lasttag = bob.devtools.scripts.lasttag:lasttag',
-            'visibility = bob.devtools.scripts.visibility:visibility',
-            'dumpsphinx = bob.devtools.scripts.dumpsphinx:dumpsphinx',
-            'bootstrap = bob.devtools.scripts.bootstrap:bootstrap',
-            'build = bob.devtools.scripts.build:build',
-            'getpath = bob.devtools.scripts.getpath:getpath',
-            'caupdate = bob.devtools.scripts.caupdate:caupdate',
-        ],
+          'release = bob.devtools.scripts.release:release',
+          'changelog = bob.devtools.scripts.changelog:changelog',
+          'lasttag = bob.devtools.scripts.lasttag:lasttag',
+          'visibility = bob.devtools.scripts.visibility:visibility',
+          'dumpsphinx = bob.devtools.scripts.dumpsphinx:dumpsphinx',
+          'bootstrap = bob.devtools.scripts.bootstrap:bootstrap',
+          'build = bob.devtools.scripts.build:build',
+          'getpath = bob.devtools.scripts.getpath:getpath',
+          'caupdate = bob.devtools.scripts.caupdate:caupdate',
+          'ci = bob.devtools.scripts.ci:ci',
+          ],
+
+        'bdt.ci.cli': [
+          'deploy = bob.devtools.scripts.ci:deploy',
+          ],
     },
     classifiers=[
         'Framework :: Bob',