From e7ac23c94326c4641152484304b07d4336cece17 Mon Sep 17 00:00:00 2001
From: Andre Anjos <andre.dos.anjos@gmail.com>
Date: Tue, 12 Feb 2019 09:21:49 +0100
Subject: [PATCH] Support for python 3.7 and installation on the "base" conda
environment
---
.gitlab-ci.yml | 25 ++++++++--
bob/devtools/build.py | 97 +++++++++++++++++++++++++++++++++++-
bob/devtools/scripts/ci.py | 72 +++++++++++++++++++++++++-
conda/meta.yaml | 1 +
deps/python-gitlab/meta.yaml | 46 +++++++++++++++++
setup.py | 1 +
6 files changed, 235 insertions(+), 7 deletions(-)
create mode 100644 deps/python-gitlab/meta.yaml
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 6ec7e0f7..89ed3071 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -56,6 +56,12 @@ build_linux_36:
<<: *linux_build_job
variables:
PYTHON_VERSION: "3.6"
+
+
+build_linux_37:
+ <<: *linux_build_job
+ variables:
+ PYTHON_VERSION: "3.7"
BUILD_EGG: "true"
script:
- python3 ./bob/devtools/bootstrap.py -vv build
@@ -76,18 +82,27 @@ build_macosx_36:
PYTHON_VERSION: "3.6"
+build_macosx_37:
+ <<: *macosx_build_job
+ variables:
+ PYTHON_VERSION: "3.7"
+
+
# Deploy targets
.deploy_template: &deploy_job
stage: deploy
script:
- - python3 ./bob/devtools/bootstrap.py -vv local bdt
+ - python3 ./bob/devtools/bootstrap.py -vv local base
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- - conda activate bdt
+ - conda activate base
+ - bdt ci base-deploy -vv
- bdt ci deploy -vv
- bdt ci clean -vv
dependencies:
- build_linux_36
+ - build_linux_37
- build_macosx_36
+ - build_macosx_37
tags:
- docker
cache: &build_caches
@@ -121,14 +136,16 @@ pypi:
except:
- branches
script:
- - python3 ./bob/devtools/bootstrap.py -vv local bdt
+ - python3 ./bob/devtools/bootstrap.py -vv local base
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- - conda activate bdt
+ - conda activate base
- bdt ci pypi -vv dist/*.zip
- bdt ci clean -vv
dependencies:
- build_linux_36
+ - build_linux_37
- build_macosx_36
+ - build_macosx_37
tags:
- docker
cache: &build_caches
diff --git a/bob/devtools/build.py b/bob/devtools/build.py
index ab73d6f8..a5b32a9a 100644
--- a/bob/devtools/build.py
+++ b/bob/devtools/build.py
@@ -154,6 +154,43 @@ def get_parsed_recipe(metadata):
return yaml.load(output)
+def exists_on_channel(channel_url, name, version, build_number):
+ """Checks on the given channel if a package with the specs exist
+
+ Args:
+
+ channel_url: The URL where to look for packages clashes (normally a beta
+ channel)
+ name: The name of the package
+ version: The version of the package
+ build_number: The build number of the package
+
+ Returns: ``True``, if the package already exists in the channel or ``False``
+ otherwise
+
+ """
+
+ from conda.exports import get_index
+
+ # get the channel index
+ logger.debug('Downloading channel index from %s', channel_url)
+ index = get_index(channel_urls=[channel_url], prepend=False)
+
+ logger.info('Checking for %s-%s_*_%s...', name, version, build_number)
+ for dist in index:
+
+ if dist.name == name and dist.version == version and \
+ dist.build_string.endswith('_%s' % build_number):
+ match = re.match('py[2-9][0-9]+', dist.build_string)
+ logger.info('Found matching package (%s-%s_%s)', dist.name, dist.version,
+ dist.build_string)
+ return True
+
+ logger.info('No matches for %s-%s_%s found among %d packages',
+ name, version, build_number, len(index))
+ return False
+
+
def remove_pins(deps):
return [l.split()[0] for l in deps]
@@ -407,6 +444,52 @@ def git_clean_build(runner, verbose):
['--exclude=%s' % k for k in exclude_from_cleanup])
+def base_build(server, intranet, recipe_dir, config):
+ '''Builds a non-beat/bob software dependence that does not exist on defaults
+
+ This function will build a software dependence that is required for our
+ software stack, but does not (yet) exist on the defaults channels. It first
+ check if the build should run for the current architecture, checks if the
+ package is not already built on our public channel and, if that is true, then
+ proceeds with the build of the dependence.
+
+
+ Args:
+
+ server: The base address of the server containing our conda channels
+ intranet: Boolean indicating if we should add "private"/"public" prefixes
+ on the returned paths
+ recipe_dir: The directory containing the recipe's ``meta.yaml`` file
+ config: A dictionary containing the merged configuration, as produced by
+ conda-build API's ``get_or_merge_config()`` function
+
+ '''
+
+ # if you get to this point, tries to build the package
+ public_channel = bootstrap.get_channels(public=True, stable=True,
+ server=server, intranet=intranet)[0]
+ metadata = get_rendered_metadata(recipe_dir, config)
+ recipe = get_parsed_recipe(metadata)
+
+ if recipe is None:
+ logger.warn('Skipping build for %s - rendering returned None', recipe_dir)
+ continue
+
+ if exists_on_channel(public_channel, recipe['package']['name'],
+ recipe['package']['version'], recipe['build']['number']):
+ logger.warn('Skipping build for %s-%s_%s - exists on channel already',
+ recipe['package']['name'], recipe['package']['version'],
+ recipe['build']['number'])
+ continue
+
+ # if you get to this point, just builds the package
+ arch = conda_arch()
+ logger.info('Building %s-%s (build: %d) for %s',
+ recipe['package']['name'], recipe['package']['version'],
+ recipe['build']['number'], arch)
+ conda_build.api.build(recipe_dir, config=conda_config)
+
+
if __name__ == '__main__':
import argparse
@@ -477,7 +560,8 @@ if __name__ == '__main__':
condarc_options = yaml.load(f)
# notice this condarc typically will only contain the defaults channel - we
- # need to boost this up with more channels to get it right.
+ # need to boost this up with more channels to get it right for this package's
+ # build
public = ( args.visibility == 'public' )
channels = bootstrap.get_channels(public=public, stable=(not is_prerelease),
server=server, intranet=(not args.internet))
@@ -492,6 +576,15 @@ if __name__ == '__main__':
conda_config = make_conda_config(conda_build_config, args.python_version,
recipe_append, condarc_options)
+ # builds all dependencies in the 'deps' subdirectory - or at least checks
+ # these dependencies are already available; these dependencies go directly to
+ # the public channel once built
+ for recipe in glob.glob(os.path.join('deps', '*')):
+ if not os.path.exists(os.path.join(recipe, 'meta.yaml')):
+ # ignore - not a conda package
+ continue
+ base_build(server, not args.internet, recipe, conda_config)
+
# retrieve the current build number for this build
build_number, _ = next_build_number(channels[0], args.name, version,
args.python_version)
@@ -517,4 +610,4 @@ if __name__ == '__main__':
else:
logger.info('twine check (a.k.a. readme check) %s: OK', package[0])
- git_clean_build(bootstrap.run_cmdline, verbose=(args.verbose >= 2))
+ git_clean_build(bootstrap.run_cmdline, verbose=(args.verbose >= 3))
diff --git a/bob/devtools/scripts/ci.py b/bob/devtools/scripts/ci.py
index f05c4a42..ced32008 100644
--- a/bob/devtools/scripts/ci.py
+++ b/bob/devtools/scripts/ci.py
@@ -30,6 +30,76 @@ def ci():
pass
+@ci.command(epilog='''
+Examples:
+
+ 1. Deploys base build artifacts (dependencies) to the appropriate channels:
+
+ $ bdt ci base-deploy -vv
+
+''')
+@click.option('-d', '--dry-run/--no-dry-run', default=False,
+ help='Only goes through the actions, but does not execute them ' \
+ '(combine with the verbosity flags - e.g. ``-vvv``) to enable ' \
+ 'printing to help you understand what will be done')
+@verbosity_option()
+@bdt.raise_on_error
+def base_deploy(dry_run):
+ """Deploys dependencies not available at the defaults channel
+
+ Deployment happens to our public channel directly, as these are
+ dependencies are required for proper bob/beat package runtime environments.
+ """
+
+ if dry_run:
+ logger.warn('!!!! DRY RUN MODE !!!!')
+ logger.warn('Nothing is being deployed to server')
+
+ package = os.environ['CI_PROJECT_PATH']
+
+ from ..constants import WEBDAV_PATHS
+ server_info = WEBDAV_PATHS[True][True] #stable=True, visible=True
+
+ logger.info('Deploying dependence packages to %s%s%s...', SERVER,
+ server_info['root'], server_info['conda'])
+
+ # setup webdav connection
+ webdav_options = {
+ 'webdav_hostname': SERVER,
+ 'webdav_root': server_info['root'],
+ 'webdav_login': os.environ['DOCUSER'],
+ 'webdav_password': os.environ['DOCPASS'],
+ }
+ from ..webdav3 import client as webdav
+ davclient = webdav.Client(webdav_options)
+ assert davclient.valid()
+
+ group, name = package.split('/')
+
+ # uploads conda package artificats
+ for arch in ('linux-64', 'osx-64', 'noarch'):
+ # finds conda dependencies and uploads what we can find
+ package_path = os.path.join(os.environ['CONDA_ROOT'], 'conda-bld', arch,
+ '*.tar.bz2')
+ deploy_packages = glob.glob(package_path)
+ for k in deploy_packages:
+ basename = os.path.basename(k)
+ if basename.startswith(name):
+ logger.debug('Skipping deploying of %s - not a base package', k)
+ continue
+
+ remote_path = '%s/%s/%s' % (server_info['conda'], arch, basename)
+ if davclient.check(remote_path):
+ raise RuntimeError('The file %s/%s already exists on the server ' \
+ '- this can be due to more than one build with deployment ' \
+ 'running at the same time. Re-running the broken builds ' \
+ 'normally fixes it' % (SERVER, remote_path))
+ logger.info('[dav] %s -> %s%s%s', k, SERVER, server_info['root'],
+ remote_path)
+ if not dry_run:
+ davclient.upload(local_path=k, remote_path=remote_path)
+
+
@ci.command(epilog='''
Examples:
@@ -291,4 +361,4 @@ def clean(ctx):
from ..build import git_clean_build
from ..bootstrap import run_cmdline
- git_clean_build(run_cmdline, verbose=(ctx.meta['verbosity']>=2))
+ git_clean_build(run_cmdline, verbose=(ctx.meta['verbosity']>=3))
diff --git a/conda/meta.yaml b/conda/meta.yaml
index deb63262..0cc376da 100644
--- a/conda/meta.yaml
+++ b/conda/meta.yaml
@@ -68,6 +68,7 @@ test:
- bdt new --help
- bdt ci --help
- bdt ci build --help
+ - bdt ci base-deploy --help
- bdt ci deploy --help
- bdt ci pypi --help
- bdt ci readme --help
diff --git a/deps/python-gitlab/meta.yaml b/deps/python-gitlab/meta.yaml
new file mode 100644
index 00000000..8cce6e82
--- /dev/null
+++ b/deps/python-gitlab/meta.yaml
@@ -0,0 +1,46 @@
+{% set name = "python-gitlab" %}
+{% set version = "1.7.0" %}
+
+package:
+ name: {{ name|lower }}
+ version: {{ version }}
+
+source:
+ url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name }}-{{ version }}.tar.gz
+ sha256: 401ef8929db4dcb5b08e0a2263a0a70599fc7e5b27615f956ac26d245802d09e
+
+build:
+ number: 0
+ script: "{{ PYTHON }} -m pip install . -vv"
+ entry_points:
+ - gitlab = gitlab.cli:main
+
+requirements:
+ host:
+ - python
+ - pip
+ run:
+ - python
+ - requests
+ - six
+
+test:
+ imports:
+ - gitlab
+
+about:
+ home: https://github.com/python-gitlab/python-gitlab
+ license: LGPL-3.0
+ license_family: LGPL
+ license_file: COPYING
+ summary: 'Python wrapper for the GitLab API'
+ description: |
+ python-gitlab is a Python package providing access to the GitLab
+ server API. It supports the v4 API of GitLab, and provides a CLI
+ tool (gitlab).
+ doc_url: https://python-gitlab.readthedocs.io/
+ dev_url: https://github.com/python-gitlab/python-gitlab
+
+extra:
+ recipe-maintainers:
+ - anjos
diff --git a/setup.py b/setup.py
index 178fd465..01918fb2 100644
--- a/setup.py
+++ b/setup.py
@@ -59,6 +59,7 @@ setup(
'bdt.ci.cli': [
'build = bob.devtools.scripts.ci:build',
'clean = bob.devtools.scripts.ci:clean',
+ 'base-deploy = bob.devtools.scripts.ci:base_deploy',
'deploy = bob.devtools.scripts.ci:deploy',
'readme = bob.devtools.scripts.ci:readme',
'pypi = bob.devtools.scripts.ci:pypi',
--
GitLab