Commit 9536df17 authored by André Anjos's avatar André Anjos 💬

[ci] New deployment subcommand

parent ea5f5829
Pipeline #25949 failed with stages
in 5 minutes and 47 seconds
...@@ -77,7 +77,7 @@ build_macosx_36: ...@@ -77,7 +77,7 @@ build_macosx_36:
script: script:
- source ${CONDA_ROOT}/etc/profile.d/conda.sh - source ${CONDA_ROOT}/etc/profile.d/conda.sh
- conda activate myenv - conda activate myenv
- bdt --help - bdt ci deploy -vv --dry-run
dependencies: dependencies:
- build_linux_36 - build_linux_36
- build_macosx_36 - build_macosx_36
......
Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/ Copyright (c) 2018 Idiap Research Institute, http://www.idiap.ch/
Written by Andre Anjos <andre.anjos@idiap.ch> Written by Andre Anjos <andre.anjos@idiap.ch>
Redistribution and use in source and binary forms, with or without Redistribution and use in source and binary forms, with or without
...@@ -25,3 +25,36 @@ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER ...@@ -25,3 +25,36 @@ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
------------------------------------------------------------------------------
Code from the "webdav3" directory was copied from the Github repository
https://github.com/ezhov-evgeny/webdav-client-python-3, but later modified and
repackaged as part of this package.
The authors asked to reproduce the following license text.
COPYRIGHT AND PERMISSION NOTICE
-------------------------------
Copyright (c) 2016, The WDC Project, and many contributors, see the THANKS
file.
All rights reserved.
Permission to use, copy, modify, and distribute this software for any purpose
with or without fee is hereby granted, provided that the above copyright notice
and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. IN
NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
OR OTHER DEALINGS IN THE SOFTWARE.
Except as contained in this notice, the name of a copyright holder shall not be
used in advertising or otherwise to promote the sale, use or other dealings in
this Software without prior written authorization of the copyright holder.
#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''Tools to help CI-based builds and artifact deployment'''
import os
import logging
logger = logging.getLogger(__name__)
import git
import packaging.version
def is_master(refname, tag):
'''Tells if we're on the master branch via ref_name or tag
This function checks if the name of the branch being built is "master". If a
tag is set, then it checks if the tag is on the master branch. If so, then
also returns ``True``, otherwise, ``False``.
Args:
refname: The value of the environment variable ``CI_COMMIT_REF_NAME``
tag: The value of the environment variable ``CI_COMMIT_TAG`` - (may be
``None``)
Returns: a boolean, indicating we're building the master branch **or** that
the tag being built was issued on the master branch.
'''
if tag is not None:
repo = git.Repo(os.environ['CI_PROJECT_DIR'])
_tag = repo.tag('refs/tags/%s' % tag)
return _tag.commit in repo.iter_commits(rev='master')
return refname == 'master'
def is_visible_outside(package, visibility):
'''Determines if the project is visible outside Idiap'''
logger.info('Project %s visibility is "%s"', package, visibility)
if visibility == 'internal':
visibility = 'private' #the same thing for this command
logger.warn('Project %s visibility switched to "%s". ' \
'For this command, it all boils down to the same...', package,
visibility)
return visibility == 'public'
def is_stable(package, refname, tag):
'''Determines if the package being published is stable
This is done by checking if a tag was set for the package. If that is the
case, we still cross-check the tag is on the "master" branch. If everything
checks out, we return ``True``. Else, ``False``.
Args:
package: Package name in the format "group/name"
refname: The current value of the environment ``CI_COMMIT_REF_NAME``
tag: The current value of the enviroment ``CI_COMMIT_TAG`` (may be
``None``)
Returns: a boolean, indicating if the current build is for a stable release
'''
if tag is not None:
logger.info('Project %s tag is "%s"', package, tag)
parsed_tag = packaging.version.Version(tag)
if parsed_tag.is_prerelease:
logger.warn('Pre-release detected - not publishing to stable channels')
return False
if is_master(os.environ['CI_COMMIT_REF_NAME'], tag):
return True
else:
logger.warn('Tag %s in non-master branch will be ignored', tag)
return False
logger.info('No tag information available at build')
logger.info('Considering this to be a pre-release build')
return False
...@@ -29,6 +29,48 @@ SERVER = 'http://www.idiap.ch' ...@@ -29,6 +29,48 @@ SERVER = 'http://www.idiap.ch'
'''This is the default server use use to store data and build artifacts''' '''This is the default server use use to store data and build artifacts'''
CONDA_CHANNELS = {
True: { #stable?
False: '/private/conda', #visible outside?
True: '/public/conda',
},
False: {
False: '/private/conda/label/beta', #visible outside?
True: '/public/conda/label/beta',
},
}
'''Default locations of our stable, beta, public and private conda channels'''
WEBDAV_PATHS = {
True: { #stable?
False: { #visible?
'root': '/private-upload',
'conda': '/conda',
'docs': '/docs',
},
True: { #visible?
'root': '/public-upload',
'conda': '/conda',
'docs': '/docs',
},
},
False: { #stable?
False: { #visible?
'root': '/private-upload',
'conda': '/conda/label/beta',
'docs': '/docs',
},
True: { #visible?
'root': '/public-upload',
'conda': '/conda/label/beta',
'docs': '/docs',
},
},
}
'''Default locations of our webdav upload paths'''
IDIAP_ROOT_CA = b''' IDIAP_ROOT_CA = b'''
Idiap Root CA 2016 - for internal use Idiap Root CA 2016 - for internal use
===================================== =====================================
......
...@@ -61,9 +61,13 @@ def get_gitlab_instance(): ...@@ -61,9 +61,13 @@ def get_gitlab_instance():
cfgs = [os.path.expanduser(k) for k in cfgs] cfgs = [os.path.expanduser(k) for k in cfgs]
if any([os.path.exists(k) for k in cfgs]): if any([os.path.exists(k) for k in cfgs]):
gl = gitlab.Gitlab.from_config('idiap', cfgs) gl = gitlab.Gitlab.from_config('idiap', cfgs)
else: #ask the user for a token else: #ask the user for a token or use one from the current runner
server = "https://gitlab.idiap.ch" server = "https://gitlab.idiap.ch"
token = input("%s token: " % server) token = os.environ.get('CI_JOB_TOKEN')
if token is None:
logger.debug('Did not find any of %s nor CI_JOB_TOKEN is defined. ' \
'Asking for user token on the command line...', '|'.join(cfgs))
token = input("Your %s (private) token: " % server)
gl = gitlab.Gitlab(server, private_token=token, api_version=4) gl = gitlab.Gitlab(server, private_token=token, api_version=4)
return gl return gl
......
#!/usr/bin/env python
import os
import re
import glob
import logging
logger = logging.getLogger(__name__)
import click
import pkg_resources
from click_plugins import with_plugins
from . import bdt
from ..log import verbosity_option
from ..ci import is_stable, is_visible_outside
from ..constants import SERVER, WEBDAV_PATHS
from .. import webdav3
@with_plugins(pkg_resources.iter_entry_points('bdt.ci.cli'))
@click.group(cls=bdt.AliasedGroup)
def ci():
"""Commands for building packages and handling CI activities
Commands defined here are supposed to run on our CI, where a number of
variables that define their behavior is correctly defined. Do **NOT**
attempt to run these commands in your own installation. Unexpected errors
may occur.
"""
pass
@ci.command(epilog='''
Examples:
1. Deploys current build artifacts to the appropriate channels:
$ bdt ci deploy -vv
''')
@click.option('-d', '--dry-run/--no-dry-run', default=False,
help='Only goes through the actions, but does not execute them ' \
'(combine with the verbosity flags - e.g. ``-vvv``) to enable ' \
'printing to help you understand what will be done')
@verbosity_option()
@bdt.raise_on_error
def deploy(dry_run):
"""Deploys build artifacts (conda packages and sphinx documentation)
Deployment happens at the "right" locations - conda packages which do not
represent stable releases are deployed to our conda "beta" channel, while
stable packages to our root channel. Sphinx documentation from unstable
builds (typically the master branch) is deployed to the documentation
server in a subdirectory named after the current branch name, while stable
documentation is deployed to a special subdirectory named "stable" and to
the respective tag name.
"""
if dry_run:
logger.warn('!!!! DRY RUN MODE !!!!')
logger.warn('Nothing is being deployed to server')
package = os.environ['CI_PROJECT_PATH']
# determine project visibility
visible = is_visible_outside(package, os.environ['CI_PROJECT_VISIBILITY'])
# determine if building master branch or tag - and if tag is on master
tag = os.environ.get('CI_COMMIT_TAG')
stable = is_stable(package, os.environ['CI_COMMIT_REF_NAME'], tag)
server_info = WEBDAV_PATHS[stable][visible]
logger.info('Deploying conda packages to %s/%s%s...', SERVER,
server_info['root'], server_info['conda'])
# setup webdav connection
webdav_options = {
'webdav_hostname': SERVER,
'webdav_root': server_info['root'],
'webdav_login': os.environ['DOCUSER'],
'webdav_password': os.environ['DOCPASS'],
}
davclient = Client(options)
assert davclient.valid()
group, name = package.split('/')
# uploads conda package artificats
for arch in ('linux-64', 'osx-64', 'noarch'):
# finds conda packages and uploads what we can find
package_path = os.path.join(os.environ['CONDA_ROOT'], 'conda-bld', arch,
name + '*.tar.bz2')
deploy_packages = glob.glob(package_path)
if len(deploy_packages):
logger.info('Deploying %d conda package(s) for %s',
len(deploy_packages), arch)
for k in deploy_packages:
remote_path = '%s%s/%s' % (server_info['root'], server_info['conda'],
os.path.basename(k))
if davclient.check(remote_path):
raise RuntimeError('The file %s/%s already exists on the server ' \
'- this can be due to more than one build with deployment ' \
'running at the same time. Re-running the broken builds ' \
'normally fixes it' % (SERVER, remote_path))
if not dry_run:
davclient.upload(local_path=k, remote_path=remote_path)
# uploads documentation artifacts
local_docs = os.path.join(os.environ['CI_PROJECT_DIR'], 'sphinx')
if not os.path.exists(local_docs):
raise RuntimeError('Documentation is not available at %s - ' \
'ensure documentation is being produced for your project!' % \
local_docs)
remote_path_prefix = '%s%s/%s' % (server_info['root'], server_info['docs'],
package)
# finds out the correct mixture of sub-directories we should deploy to.
# 1. if ref-name is a tag, don't forget to publish to 'master' as well -
# all tags are checked to come from that branch
# 2. if ref-name is a branch name, deploy to it
# 3. in case a tag is being published, make sure to deploy to the special
# "stable" subdir as well
deploy_docs_to = set([os.environ['CI_COMMIT_REF_NAME']])
if stable:
deploy_docs_to.add('master')
if os.environ.get('CI_COMMIT_TAG') is not None:
deploy_docs_to.add(os.environ['CI_COMMIT_TAG'])
deploy_docs_to.add('stable')
for k in deploy_docs_to:
remote_path = '%s/%s' % (remote_path_prefix, k)
logger.info('Deploying package documentation to %s/%s...', SERVER,
remote_path)
if not dry_run:
client.upload_directory(local_path=local_docs, remote_path=remote_path)
# -*- coding: utf-8
import functools
import logging
import os
import shutil
import threading
from io import BytesIO
from re import sub
import lxml.etree as etree
import requests
from .connection import *
from .exceptions import *
from .urn import Urn
try:
from urllib.parse import unquote, urlsplit
except ImportError:
from urllib import unquote
from urlparse import urlsplit
__version__ = "0.2"
log = logging.getLogger(__name__)
def listdir(directory):
"""Returns list of nested files and directories for local directory by path
:param directory: absolute or relative path to local directory
:return: list nested of file or directory names
"""
file_names = list()
for filename in os.listdir(directory):
file_path = os.path.join(directory, filename)
if os.path.isdir(file_path):
filename = "{filename}{separate}".format(filename=filename, separate=os.path.sep)
file_names.append(filename)
return file_names
def get_options(option_type, from_options):
"""Extract options for specified option type from all options
:param option_type: the object of specified type of options
:param from_options: all options dictionary
:return: the dictionary of options for specified type, each option can be filled by value from all options
dictionary or blank in case the option for specified type is not exist in all options dictionary
"""
_options = dict()
for key in option_type.keys:
key_with_prefix = "{prefix}{key}".format(prefix=option_type.prefix, key=key)
if key not in from_options and key_with_prefix not in from_options:
_options[key] = ""
elif key in from_options:
_options[key] = from_options.get(key)
else:
_options[key] = from_options.get(key_with_prefix)
return _options
def wrap_connection_error(fn):
@functools.wraps(fn)
def _wrapper(self, *args, **kw):
log.debug("Requesting %s(%s, %s)", fn, args, kw)
try:
res = fn(self, *args, **kw)
except requests.ConnectionError:
raise NoConnection(self.webdav.hostname)
except requests.RequestException as re:
raise ConnectionException(re)
else:
return res
return _wrapper
class Client(object):
"""The client for WebDAV servers provides an ability to control files on remote WebDAV server.
"""
# path to root directory of WebDAV
root = '/'
# Max size of file for uploading
large_size = 2 * 1024 * 1024 * 1024
# request timeout in seconds
timeout = 30
# HTTP headers for different actions
http_header = {
'list': ["Accept: */*", "Depth: 1"],
'free': ["Accept: */*", "Depth: 0", "Content-Type: text/xml"],
'copy': ["Accept: */*"],
'move': ["Accept: */*"],
'mkdir': ["Accept: */*", "Connection: Keep-Alive"],
'clean': ["Accept: */*", "Connection: Keep-Alive"],
'check': ["Accept: */*"],
'info': ["Accept: */*", "Depth: 1"],
'get_property': ["Accept: */*", "Depth: 1", "Content-Type: application/x-www-form-urlencoded"],
'set_property': ["Accept: */*", "Depth: 1", "Content-Type: application/x-www-form-urlencoded"]
}
def get_headers(self, action, headers_ext=None):
"""Returns HTTP headers of specified WebDAV actions.
:param action: the identifier of action.
:param headers_ext: (optional) the addition headers list witch sgould be added to basic HTTP headers for
the specified action.
:return: the dictionary of headers for specified action.
"""
if action in Client.http_header:
try:
headers = Client.http_header[action].copy()
except AttributeError:
headers = Client.http_header[action][:]
else:
headers = list()
if headers_ext:
headers.extend(headers_ext)
if self.webdav.token:
webdav_token = "Authorization: OAuth {token}".format(token=self.webdav.token)
headers.append(webdav_token)
return dict([map(lambda s: s.strip(), i.split(':')) for i in headers])
def get_url(self, path):
"""Generates url by uri path.
:param path: uri path.
:return: the url string.
"""
url = {'hostname': self.webdav.hostname, 'root': self.webdav.root, 'path': path}
return "{hostname}{root}{path}".format(**url)
def get_full_path(self, urn):
"""Generates full path to remote resource exclude hostname.
:param urn: the URN to resource.
:return: full path to resource with root path.
"""
return "{root}{path}".format(root=self.webdav.root, path=urn.path())
def execute_request(self, action, path, data=None, headers_ext=None):
"""Generate request to WebDAV server for specified action and path and execute it.
:param action: the action for WebDAV server which should be executed.
:param path: the path to resource for action
:param data: (optional) Dictionary or list of tuples ``[(key, value)]`` (will be form-encoded), bytes,
or file-like object to send in the body of the :class:`Request`.
:param headers_ext: (optional) the addition headers list witch should be added to basic HTTP headers for
the specified action.
:return: HTTP response of request.
"""
response = requests.request(
method=Client.requests[action],
url=self.get_url(path),
auth=(self.webdav.login, self.webdav.password),
headers=self.get_headers(action, headers_ext),
timeout=self.timeout,
data=data
)
if response.status_code == 507:
raise NotEnoughSpace()
if response.status_code >= 400:
raise ResponseErrorCode(url=self.get_url(path), code=response.status_code, message=response.content)
return response
# mapping of actions to WebDAV methods
requests = {
'download': "GET",
'upload': "PUT",
'copy': "COPY",
'move': "MOVE",
'mkdir': "MKCOL",
'clean': "DELETE",
'check': "HEAD",
'list': "PROPFIND",
'free': "PROPFIND",
'info': "PROPFIND",
'publish': "PROPPATCH",
'unpublish': "PROPPATCH",
'published': "PROPPATCH",
'get_property': "PROPFIND",
'set_property': "PROPPATCH"
}
meta_xmlns = {
'https://webdav.yandex.ru': "urn:yandex:disk:meta",
}
def __init__(self, options):
"""Constructor of WebDAV client
:param options: the dictionary of connection options to WebDAV can include proxy server options.
WebDev settings:
`webdav_hostname`: url for WebDAV server should contain protocol and ip address or domain name.
Example: `https://webdav.server.com`.
`webdav_login`: (optional) login name for WebDAV server can be empty in case using of token auth.
`webdav_password`: (optional) password for WebDAV server can be empty in case using of token auth.
`webdav_token': (optional) token for WebDAV server can be empty in case using of login/password auth.
`webdav_root`: (optional) root directory of WebDAV server. Defaults is `/`.
`webdav_cert_path`: (optional) path to certificate.
`webdav_key_path`: (optional) path to private key.
`webdav_recv_speed`: (optional) rate limit data download speed in Bytes per second.
Defaults to unlimited speed.
`webdav_send_speed`: (optional) rate limit data upload speed in Bytes per second.
Defaults to unlimited speed.
`webdav_verbose`: (optional) set verbose mode on.off. By default verbose mode is off.
Proxy settings (optional):
`proxy_hostname`: url to proxy server should contain protocol and ip address or domain name and if needed
port. Example: `https://proxy.server.com:8383`.
`proxy_login`: login name for proxy server.
`proxy_password`: password for proxy server.
"""
webdav_options = get_options(option_type=WebDAVSettings, from_options=options)
proxy_options = get_options(option_type=ProxySettings, from_options=options)
self.webdav = WebDAVSettings(webdav_options)
self.proxy = ProxySettings(proxy_options)
self.default_options = {}
def valid(self):
"""Validates of WebDAV and proxy settings.
:return: True in case settings are valid and False otherwise.
"""
return True if self.webdav.valid() and self.proxy.valid() else False
@wrap_connection_error
def list(self, remote_path=root):
"""Returns list of nested files and directories for remote WebDAV directory by path.
More information you can find by link http://webdav.org/specs/rfc4918.html#METHOD_PROPFIND
:param remote_path: path to remote directory.
:return: list of nested file or directory names.
"""
directory_urn = Urn(remote_path, directory=True)
if directory_urn.path() != Client.root:
if not self.check(directory_urn.path()):
raise RemoteResourceNotFound(directory_urn.path())
response = self.execute_request(action='list', path=directory_urn.quote())
urns = WebDavXmlUtils.parse_get_list_response(response.content)
path = Urn.normalize_path(self.get_full_path(directory_urn))
return [urn.filename() for urn in urns if Urn.compare_path(path, urn.path()) is False]
@wrap_connection_error
def free(self):
"""Returns an amount of free space on remote WebDAV server.
More information you can find by link http://webdav.org/specs/rfc4918.html#METHOD_PROPFIND
:return: an amount of free space in bytes.
"""
data = WebDavXmlUtils.create_free_space_request_content()
response = self.execute_request(action='free', path='', data=data)
return WebDavXmlUtils.parse_free_space_response(response.content, self.webdav.hostname)
@wrap_connection_error
def check(self, remote_path=root):
"""Checks an existence of remote resource on WebDAV server by remote path.
More information you can find by link http://webdav.org/specs/rfc4918.html#rfc.section.9.4
:param remote_path: (optional) path to resource on WebDAV server. Defaults is root directory of WebDAV.
:return: True if resource is exist or False otherwise
"""
urn = Urn(remote_path)
try:
response = self.execute_request(action='check', path=urn.quote())
except ResponseErrorCode:
return False
if int(response.status_code) == 200:
return True
return False
@wrap_connection_error
def mkdir(self, remote_path):
"""Makes new directory on WebDAV server.
More information you can find by link http://webdav.org/specs/rfc4918.html#METHOD_MKCOL
:param remote_path: path to directory
:return: True if request executed with code 200 or 201 and False otherwise.
"""
directory_urn = Urn(remote_path, directory=True)
if not self.check(directory_urn.parent()):
raise RemoteParentNotFound(directory_urn.path())
response = self.execute_request(action='mkdir', path=directory_urn.quote())
return response.status_code in (200, 201)
@wrap_connection_error
def download_from(self, buff, remote_path):
"""Downloads file from WebDAV and writes it in buffer.
:param buff: buffer object for writing of downloaded file content.
:param remote_path: path to file on WebDAV server.
"""
urn = Urn(remote_path)
if self.is_dir(urn.path()):
raise OptionNotValid(name="remote_path", value=remote_path)
if not self.check(urn.path()):
raise RemoteResourceNotFound(urn.path())
response = self.execute_request(action='download', path=urn.quote())
buff.write(response.content)
def download(self, remote_path, local_path, progress=None):
"""Downloads remote resource from WebDAV and save it in local path.
More information you can find by link http://webdav.org/specs/rfc4918.html#rfc.section.9.4
:param remote_path: the path to remote resource for downloading can be file and directory.
:param local_path: the path to save resource locally.
:param progress: progress function. Not supported now.
"""
urn = Urn(remote_path)
if self.is_dir(urn.path()):
self.download_directory(local_path=local_path, remote_path=remote_path, progress=progress)
else:
self.download_file(local_path=local_path, remote_path=remote_path, progress=progress)
def download_directory(self, remote_path, local_path, progress=None):
"""Downloads directory and downloads all nested files and directories from remote WebDAV to local.
If there is something on local path it deletes directories and files then creates new.
:param remote_path: the path to directory for downloading form WebDAV server.
:param local_path: the path to local directory for saving downloaded files and directories.
:param progress: Progress function. Not supported now.
"""
urn = Urn(remote_path, directory=True)
if not self.is_dir(urn.path()):
raise OptionNotValid(name="remote_path", value=remote_path)
if os.path.exists(local_path):
shutil.rmtree(local_path)
os.makedirs(local_path)
for resource_name in self.list(urn.path()):
_remote_path = "{parent}{name}".format(parent=urn.path(), name=resource_name)
_local_path = os.path.join(local_path, resource_name)
self.download(local_path=_local_path, remote_path=_remote_path, progress=progress)
@wrap_connection_error
def download_file(self, remote_path, local_path, progress=None):
"""Downloads file from WebDAV server and save it locally.
More information you can find by link http://webdav.org/specs/rfc4918.html#rfc.section.9.4
:param remote_path: the path to remote file for downloading.
:param local_path: the path to save file locally.
:param progress: progress function. Not supported now.
"""
urn = Urn(remote_path)