Commit 9536df17 authored by André Anjos's avatar André Anjos 💬

[ci] New deployment subcommand

parent ea5f5829
Pipeline #25949 failed with stages
in 5 minutes and 47 seconds
......@@ -77,7 +77,7 @@ build_macosx_36:
script:
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- conda activate myenv
- bdt --help
- bdt ci deploy -vv --dry-run
dependencies:
- build_linux_36
- build_macosx_36
......
Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/
Copyright (c) 2018 Idiap Research Institute, http://www.idiap.ch/
Written by Andre Anjos <andre.anjos@idiap.ch>
Redistribution and use in source and binary forms, with or without
......@@ -25,3 +25,36 @@ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
------------------------------------------------------------------------------
Code from the "webdav3" directory was copied from the Github repository
https://github.com/ezhov-evgeny/webdav-client-python-3, but later modified and
repackaged as part of this package.
The authors asked to reproduce the following license text.
COPYRIGHT AND PERMISSION NOTICE
-------------------------------
Copyright (c) 2016, The WDC Project, and many contributors, see the THANKS
file.
All rights reserved.
Permission to use, copy, modify, and distribute this software for any purpose
with or without fee is hereby granted, provided that the above copyright notice
and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. IN
NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
OR OTHER DEALINGS IN THE SOFTWARE.
Except as contained in this notice, the name of a copyright holder shall not be
used in advertising or otherwise to promote the sale, use or other dealings in
this Software without prior written authorization of the copyright holder.
#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''Tools to help CI-based builds and artifact deployment'''
import os
import logging
logger = logging.getLogger(__name__)
import git
import packaging.version
def is_master(refname, tag):
'''Tells if we're on the master branch via ref_name or tag
This function checks if the name of the branch being built is "master". If a
tag is set, then it checks if the tag is on the master branch. If so, then
also returns ``True``, otherwise, ``False``.
Args:
refname: The value of the environment variable ``CI_COMMIT_REF_NAME``
tag: The value of the environment variable ``CI_COMMIT_TAG`` - (may be
``None``)
Returns: a boolean, indicating we're building the master branch **or** that
the tag being built was issued on the master branch.
'''
if tag is not None:
repo = git.Repo(os.environ['CI_PROJECT_DIR'])
_tag = repo.tag('refs/tags/%s' % tag)
return _tag.commit in repo.iter_commits(rev='master')
return refname == 'master'
def is_visible_outside(package, visibility):
'''Determines if the project is visible outside Idiap'''
logger.info('Project %s visibility is "%s"', package, visibility)
if visibility == 'internal':
visibility = 'private' #the same thing for this command
logger.warn('Project %s visibility switched to "%s". ' \
'For this command, it all boils down to the same...', package,
visibility)
return visibility == 'public'
def is_stable(package, refname, tag):
'''Determines if the package being published is stable
This is done by checking if a tag was set for the package. If that is the
case, we still cross-check the tag is on the "master" branch. If everything
checks out, we return ``True``. Else, ``False``.
Args:
package: Package name in the format "group/name"
refname: The current value of the environment ``CI_COMMIT_REF_NAME``
tag: The current value of the enviroment ``CI_COMMIT_TAG`` (may be
``None``)
Returns: a boolean, indicating if the current build is for a stable release
'''
if tag is not None:
logger.info('Project %s tag is "%s"', package, tag)
parsed_tag = packaging.version.Version(tag)
if parsed_tag.is_prerelease:
logger.warn('Pre-release detected - not publishing to stable channels')
return False
if is_master(os.environ['CI_COMMIT_REF_NAME'], tag):
return True
else:
logger.warn('Tag %s in non-master branch will be ignored', tag)
return False
logger.info('No tag information available at build')
logger.info('Considering this to be a pre-release build')
return False
......@@ -29,6 +29,48 @@ SERVER = 'http://www.idiap.ch'
'''This is the default server use use to store data and build artifacts'''
CONDA_CHANNELS = {
True: { #stable?
False: '/private/conda', #visible outside?
True: '/public/conda',
},
False: {
False: '/private/conda/label/beta', #visible outside?
True: '/public/conda/label/beta',
},
}
'''Default locations of our stable, beta, public and private conda channels'''
WEBDAV_PATHS = {
True: { #stable?
False: { #visible?
'root': '/private-upload',
'conda': '/conda',
'docs': '/docs',
},
True: { #visible?
'root': '/public-upload',
'conda': '/conda',
'docs': '/docs',
},
},
False: { #stable?
False: { #visible?
'root': '/private-upload',
'conda': '/conda/label/beta',
'docs': '/docs',
},
True: { #visible?
'root': '/public-upload',
'conda': '/conda/label/beta',
'docs': '/docs',
},
},
}
'''Default locations of our webdav upload paths'''
IDIAP_ROOT_CA = b'''
Idiap Root CA 2016 - for internal use
=====================================
......
......@@ -61,9 +61,13 @@ def get_gitlab_instance():
cfgs = [os.path.expanduser(k) for k in cfgs]
if any([os.path.exists(k) for k in cfgs]):
gl = gitlab.Gitlab.from_config('idiap', cfgs)
else: #ask the user for a token
else: #ask the user for a token or use one from the current runner
server = "https://gitlab.idiap.ch"
token = input("%s token: " % server)
token = os.environ.get('CI_JOB_TOKEN')
if token is None:
logger.debug('Did not find any of %s nor CI_JOB_TOKEN is defined. ' \
'Asking for user token on the command line...', '|'.join(cfgs))
token = input("Your %s (private) token: " % server)
gl = gitlab.Gitlab(server, private_token=token, api_version=4)
return gl
......
#!/usr/bin/env python
import os
import re
import glob
import logging
logger = logging.getLogger(__name__)
import click
import pkg_resources
from click_plugins import with_plugins
from . import bdt
from ..log import verbosity_option
from ..ci import is_stable, is_visible_outside
from ..constants import SERVER, WEBDAV_PATHS
from .. import webdav3
@with_plugins(pkg_resources.iter_entry_points('bdt.ci.cli'))
@click.group(cls=bdt.AliasedGroup)
def ci():
"""Commands for building packages and handling CI activities
Commands defined here are supposed to run on our CI, where a number of
variables that define their behavior is correctly defined. Do **NOT**
attempt to run these commands in your own installation. Unexpected errors
may occur.
"""
pass
@ci.command(epilog='''
Examples:
1. Deploys current build artifacts to the appropriate channels:
$ bdt ci deploy -vv
''')
@click.option('-d', '--dry-run/--no-dry-run', default=False,
help='Only goes through the actions, but does not execute them ' \
'(combine with the verbosity flags - e.g. ``-vvv``) to enable ' \
'printing to help you understand what will be done')
@verbosity_option()
@bdt.raise_on_error
def deploy(dry_run):
"""Deploys build artifacts (conda packages and sphinx documentation)
Deployment happens at the "right" locations - conda packages which do not
represent stable releases are deployed to our conda "beta" channel, while
stable packages to our root channel. Sphinx documentation from unstable
builds (typically the master branch) is deployed to the documentation
server in a subdirectory named after the current branch name, while stable
documentation is deployed to a special subdirectory named "stable" and to
the respective tag name.
"""
if dry_run:
logger.warn('!!!! DRY RUN MODE !!!!')
logger.warn('Nothing is being deployed to server')
package = os.environ['CI_PROJECT_PATH']
# determine project visibility
visible = is_visible_outside(package, os.environ['CI_PROJECT_VISIBILITY'])
# determine if building master branch or tag - and if tag is on master
tag = os.environ.get('CI_COMMIT_TAG')
stable = is_stable(package, os.environ['CI_COMMIT_REF_NAME'], tag)
server_info = WEBDAV_PATHS[stable][visible]
logger.info('Deploying conda packages to %s/%s%s...', SERVER,
server_info['root'], server_info['conda'])
# setup webdav connection
webdav_options = {
'webdav_hostname': SERVER,
'webdav_root': server_info['root'],
'webdav_login': os.environ['DOCUSER'],
'webdav_password': os.environ['DOCPASS'],
}
davclient = Client(options)
assert davclient.valid()
group, name = package.split('/')
# uploads conda package artificats
for arch in ('linux-64', 'osx-64', 'noarch'):
# finds conda packages and uploads what we can find
package_path = os.path.join(os.environ['CONDA_ROOT'], 'conda-bld', arch,
name + '*.tar.bz2')
deploy_packages = glob.glob(package_path)
if len(deploy_packages):
logger.info('Deploying %d conda package(s) for %s',
len(deploy_packages), arch)
for k in deploy_packages:
remote_path = '%s%s/%s' % (server_info['root'], server_info['conda'],
os.path.basename(k))
if davclient.check(remote_path):
raise RuntimeError('The file %s/%s already exists on the server ' \
'- this can be due to more than one build with deployment ' \
'running at the same time. Re-running the broken builds ' \
'normally fixes it' % (SERVER, remote_path))
if not dry_run:
davclient.upload(local_path=k, remote_path=remote_path)
# uploads documentation artifacts
local_docs = os.path.join(os.environ['CI_PROJECT_DIR'], 'sphinx')
if not os.path.exists(local_docs):
raise RuntimeError('Documentation is not available at %s - ' \
'ensure documentation is being produced for your project!' % \
local_docs)
remote_path_prefix = '%s%s/%s' % (server_info['root'], server_info['docs'],
package)
# finds out the correct mixture of sub-directories we should deploy to.
# 1. if ref-name is a tag, don't forget to publish to 'master' as well -
# all tags are checked to come from that branch
# 2. if ref-name is a branch name, deploy to it
# 3. in case a tag is being published, make sure to deploy to the special
# "stable" subdir as well
deploy_docs_to = set([os.environ['CI_COMMIT_REF_NAME']])
if stable:
deploy_docs_to.add('master')
if os.environ.get('CI_COMMIT_TAG') is not None:
deploy_docs_to.add(os.environ['CI_COMMIT_TAG'])
deploy_docs_to.add('stable')
for k in deploy_docs_to:
remote_path = '%s/%s' % (remote_path_prefix, k)
logger.info('Deploying package documentation to %s/%s...', SERVER,
remote_path)
if not dry_run:
client.upload_directory(local_path=local_docs, remote_path=remote_path)
This diff is collapsed.
from os.path import exists
from .exceptions import *
from .urn import Urn
class ConnectionSettings:
def is_valid(self):
pass
def valid(self):
try:
self.is_valid()
except OptionNotValid:
return False
else:
return True
class WebDAVSettings(ConnectionSettings):
ns = "webdav:"
prefix = "webdav_"
keys = {'hostname', 'login', 'password', 'token', 'root', 'cert_path', 'key_path', 'recv_speed', 'send_speed',
'verbose'}
hostname = None
login = None
password = None
token = None
root = None
cert_path = None
key_path = None
recv_speed = None
send_speed = None
verbose = None
def __init__(self, options):
self.options = dict()
for key in self.keys:
value = options.get(key, '')
self.options[key] = value
self.__dict__[key] = value
self.root = Urn(self.root).quote() if self.root else ''
self.root = self.root.rstrip(Urn.separate)
def is_valid(self):
if not self.hostname:
raise OptionNotValid(name="hostname", value=self.hostname, ns=self.ns)
if self.cert_path and not exists(self.cert_path):
raise OptionNotValid(name="cert_path", value=self.cert_path, ns=self.ns)
if self.key_path and not exists(self.key_path):
raise OptionNotValid(name="key_path", value=self.key_path, ns=self.ns)
if self.key_path and not self.cert_path:
raise OptionNotValid(name="cert_path", value=self.cert_path, ns=self.ns)
if self.password and not self.login:
raise OptionNotValid(name="login", value=self.login, ns=self.ns)
if not self.token and not self.login:
raise OptionNotValid(name="login", value=self.login, ns=self.ns)
class ProxySettings(ConnectionSettings):
ns = "proxy:"
prefix = "proxy_"
keys = {'hostname', 'login', 'password'}
hostname = None
login = None
password = None
def __init__(self, options):
self.options = dict()
for key in self.keys:
value = options.get(key, '')
self.options[key] = value
self.__dict__[key] = value
def is_valid(self):
if self.password and not self.login:
raise OptionNotValid(name="login", value=self.login, ns=self.ns)
if self.login or self.password:
if not self.hostname:
raise OptionNotValid(name="hostname", value=self.hostname, ns=self.ns)
class WebDavException(Exception):
pass
class NotValid(WebDavException):
pass
class OptionNotValid(NotValid):
def __init__(self, name, value, ns=""):
self.name = name
self.value = value
self.ns = ns
def __str__(self):
return "Option ({ns}{name}={value}) have invalid name or value".format(ns=self.ns, name=self.name,
value=self.value)
class CertificateNotValid(NotValid):
pass
class NotFound(WebDavException):
pass
class LocalResourceNotFound(NotFound):
def __init__(self, path):
self.path = path
def __str__(self):
return "Local file: {path} not found".format(path=self.path)
class RemoteResourceNotFound(NotFound):
def __init__(self, path):
self.path = path
def __str__(self):
return "Remote resource: {path} not found".format(path=self.path)
class RemoteParentNotFound(NotFound):
def __init__(self, path):
self.path = path
def __str__(self):
return "Remote parent for: {path} not found".format(path=self.path)
class ResourceTooBig(WebDavException):
def __init__(self, path, size, max_size):
self.path = path
self.size = size
self.max_size = max_size
def __str__(self):
return "Resource {path} is too big, it should be less then {max_size} but actually: {size}".format(
path=self.path,
max_size=self.max_size,
size=self.size)
class MethodNotSupported(WebDavException):
def __init__(self, name, server):
self.name = name
self.server = server
def __str__(self):
return "Method {name} not supported for {server}".format(name=self.name, server=self.server)
class ConnectionException(WebDavException):
def __init__(self, exception):
self.exception = exception
def __str__(self):
return self.exception.__str__()
class NoConnection(WebDavException):
def __init__(self, hostname):
self.hostname = hostname
def __str__(self):
return "Not connection with {hostname}".format(hostname=self.hostname)
# This exception left only for supporting original library interface.
class NotConnection(WebDavException):
def __init__(self, hostname):
self.hostname = hostname
def __str__(self):
return "No connection with {hostname}".format(hostname=self.hostname)
class ResponseErrorCode(WebDavException):
def __init__(self, url, code, message):
self.url = url
self.code = code
self.message = message
def __str__(self):
return "Request to {url} failed with code {code} and message: {message}".format(url=self.url, code=self.code,
message=self.message)
class NotEnoughSpace(WebDavException):
def __init__(self):
pass
def __str__(self):
return "Not enough space on the server"
try:
from urllib.parse import unquote, quote, urlsplit
except ImportError:
from urllib import unquote, quote
from urlparse import urlsplit
from re import sub
class Urn(object):
separate = "/"
def __init__(self, path, directory=False):
self._path = quote(path)
expressions = "/\.+/", "/+"
for expression in expressions:
self._path = sub(expression, Urn.separate, self._path)
if not self._path.startswith(Urn.separate):
self._path = "{begin}{end}".format(begin=Urn.separate, end=self._path)
if directory and not self._path.endswith(Urn.separate):
self._path = "{begin}{end}".format(begin=self._path, end=Urn.separate)
def __str__(self):
return self.path()
def path(self):
return unquote(self._path)
def quote(self):
return self._path
def filename(self):
path_split = self._path.split(Urn.separate)
name = path_split[-2] + Urn.separate if path_split[-1] == '' else path_split[-1]
return unquote(name)
def parent(self):
path_split = self._path.split(Urn.separate)
nesting_level = self.nesting_level()
parent_path_split = path_split[:nesting_level]
parent = self.separate.join(parent_path_split) if nesting_level != 1 else Urn.separate
if not parent.endswith(Urn.separate):
return unquote(parent + Urn.separate)
else:
return unquote(parent)
def nesting_level(self):
return self._path.count(Urn.separate, 0, -1)
def is_dir(self):
return self._path[-1] == Urn.separate
@staticmethod
def normalize_path(path):
result = sub('/{2,}', '/', path)
return result if len(result) < 1 or result[-1] != Urn.separate else result[:-1]
@staticmethod
def compare_path(path_a, href):
unqouted_path = Urn.separate + unquote(urlsplit(href).path)
return Urn.normalize_path(path_a) == Urn.normalize_path(unqouted_path)
......@@ -44,6 +44,8 @@ requirements:
- sphinx
- pyyaml
- twine
- packaging
- lxml
test:
requires:
......
......@@ -17,6 +17,8 @@ requires = [
'sphinx',
'pyyaml',
'twine',
'packaging',
'lxml',
]
setup(
......@@ -41,16 +43,21 @@ setup(
'bdt = bob.devtools.scripts.bdt:main',
],
'bdt.cli': [
'release = bob.devtools.scripts.release:release',
'changelog = bob.devtools.scripts.changelog:changelog',
'lasttag = bob.devtools.scripts.lasttag:lasttag',
'visibility = bob.devtools.scripts.visibility:visibility',
'dumpsphinx = bob.devtools.scripts.dumpsphinx:dumpsphinx',
'bootstrap = bob.devtools.scripts.bootstrap:bootstrap',
'build = bob.devtools.scripts.build:build',
'getpath = bob.devtools.scripts.getpath:getpath',
'caupdate = bob.devtools.scripts.caupdate:caupdate',
],
'release = bob.devtools.scripts.release:release',
'changelog = bob.devtools.scripts.changelog:changelog',
'lasttag = bob.devtools.scripts.lasttag:lasttag',
'visibility = bob.devtools.scripts.visibility:visibility',
'dumpsphinx = bob.devtools.scripts.dumpsphinx:dumpsphinx',
'bootstrap = bob.devtools.scripts.bootstrap:bootstrap',
'build = bob.devtools.scripts.build:build',
'getpath = bob.devtools.scripts.getpath:getpath',
'caupdate = bob.devtools.scripts.caupdate:caupdate',
'ci = bob.devtools.scripts.ci:ci',
],
'bdt.ci.cli': [
'deploy = bob.devtools.scripts.ci:deploy',
],
},
classifiers=[
'Framework :: Bob',
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment