diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 6ec7e0f754be049fcaffbd52e713f51a4ad4b1d0..89ed30711b73ff8ce160031f6adb7880110b66a8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -56,6 +56,12 @@ build_linux_36:
<<: *linux_build_job
variables:
PYTHON_VERSION: "3.6"
+
+
+build_linux_37:
+ <<: *linux_build_job
+ variables:
+ PYTHON_VERSION: "3.7"
BUILD_EGG: "true"
script:
- python3 ./bob/devtools/bootstrap.py -vv build
@@ -76,18 +82,27 @@ build_macosx_36:
PYTHON_VERSION: "3.6"
+build_macosx_37:
+ <<: *macosx_build_job
+ variables:
+ PYTHON_VERSION: "3.7"
+
+
# Deploy targets
.deploy_template: &deploy_job
stage: deploy
script:
- - python3 ./bob/devtools/bootstrap.py -vv local bdt
+ - python3 ./bob/devtools/bootstrap.py -vv local base
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- - conda activate bdt
+ - conda activate base
+ - bdt ci base-deploy -vv
- bdt ci deploy -vv
- bdt ci clean -vv
dependencies:
- build_linux_36
+ - build_linux_37
- build_macosx_36
+ - build_macosx_37
tags:
- docker
cache: &build_caches
@@ -121,14 +136,16 @@ pypi:
except:
- branches
script:
- - python3 ./bob/devtools/bootstrap.py -vv local bdt
+ - python3 ./bob/devtools/bootstrap.py -vv local base
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- - conda activate bdt
+ - conda activate base
- bdt ci pypi -vv dist/*.zip
- bdt ci clean -vv
dependencies:
- build_linux_36
+ - build_linux_37
- build_macosx_36
+ - build_macosx_37
tags:
- docker
cache: &build_caches
diff --git a/MANIFEST.in b/MANIFEST.in
index 64a24146322e9c0e5a7bc873132dc1eb99bbc583..fd350028a9bb270daf9c4fc8f001544a83998fa6 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,4 +1,4 @@
include LICENSE README.rst buildout.cfg version.txt
recursive-include doc conf.py *.rst *.sh *.png *.ico
recursive-include bob/devtools/data *.md *.yaml *.pem matplotlibrc
-recursive-include bob/devtools/templates conf.py *.rst *.png *.ico LICENSE COPYING .gitlab-ci.yml .gitignore *.cfg *.txt *.py
+recursive-include bob/devtools/templates conf.py *.rst *.png *.ico LICENSE COPYING MANIFEST.in .gitlab-ci.yml .gitignore *.cfg *.txt *.py
diff --git a/bob/devtools/bootstrap.py b/bob/devtools/bootstrap.py
index bb338ce0074f97d5913fd4e40bfc90656fb7db80..bcb086b8d29bf453addb81df9dc253ff82c1c31c 100644
--- a/bob/devtools/bootstrap.py
+++ b/bob/devtools/bootstrap.py
@@ -383,7 +383,7 @@ if __name__ == '__main__':
f.write(_BASE_CONDARC)
conda_version = '4'
- conda_build_version = '3'
+ conda_build_version = '3.16'
conda_verbosity = []
#if args.verbose >= 2:
diff --git a/bob/devtools/build.py b/bob/devtools/build.py
index ab73d6f87d5ca605cdb00b311724644089b05dae..c33c0d55bc0d0cd2a16ed69581da5cca25ffaaf3 100644
--- a/bob/devtools/build.py
+++ b/bob/devtools/build.py
@@ -154,6 +154,61 @@ def get_parsed_recipe(metadata):
return yaml.load(output)
+def exists_on_channel(channel_url, name, version, build_number,
+ python_version):
+ """Checks on the given channel if a package with the specs exist
+
+ Args:
+
+ channel_url: The URL where to look for packages clashes (normally a beta
+ channel)
+ name: The name of the package
+ version: The version of the package
+ build_number: The build number of the package
+ python_version: The current version of python we're building for
+
+ Returns: A complete package name, version and build string, if the package
+ already exists in the channel or ``None`` otherwise.
+
+ """
+
+ from conda.exports import get_index
+
+ # no dot in py_ver
+ py_ver = python_version.replace('.', '')
+
+ # get the channel index
+ logger.debug('Downloading channel index from %s', channel_url)
+ index = get_index(channel_urls=[channel_url], prepend=False)
+
+ logger.info('Checking for %s-%s-%s...', name, version, build_number)
+
+ for dist in index:
+
+ if dist.name == name and dist.version == version and \
+ dist.build_string.endswith('_%s' % build_number):
+
+ # two possible options must be checked - (i) the package build_string
+ # starts with ``py``, which means it is a python specific package so we
+ # must also check for the matching python version. (ii) the package is
+ # not a python-specific package and a simple match will do
+ if dist.build_string.startswith('py'):
+ match = re.match('py[2-9][0-9]+', dist.build_string)
+ if match and match.group() == 'py{}'.format(py_ver):
+ logger.debug('Found matching package (%s-%s-%s)', dist.name,
+ dist.version, dist.build_string)
+ return (dist.name, dist.version, dist.build_string)
+
+ else:
+ logger.debug('Found matching package (%s-%s-%s)', dist.name,
+ dist.version, dist.build_string)
+ return (dist.name, dist.version, dist.build_string)
+
+ logger.info('No matches for %s-%s-(py%s_?)%s found among %d packages',
+ name, version, py_ver, build_number, len(index))
+ return
+
+
def remove_pins(deps):
return [l.split()[0] for l in deps]
@@ -407,6 +462,69 @@ def git_clean_build(runner, verbose):
['--exclude=%s' % k for k in exclude_from_cleanup])
+def base_build(server, intranet, recipe_dir, conda_build_config,
+ python_version, condarc_options):
+ '''Builds a non-beat/bob software dependence that does not exist on defaults
+
+ This function will build a software dependence that is required for our
+ software stack, but does not (yet) exist on the defaults channels. It first
+ check if the build should run for the current architecture, checks if the
+ package is not already built on our public channel and, if that is true, then
+ proceeds with the build of the dependence.
+
+
+ Args:
+
+ server: The base address of the server containing our conda channels
+ intranet: Boolean indicating if we should add "private"/"public" prefixes
+ on the returned paths
+ recipe_dir: The directory containing the recipe's ``meta.yaml`` file
+ conda_build_config: Path to the ``conda_build_config.yaml`` file to use
+ python_version: String with the python version to build for, in the format
+ ``x.y`` (should be passed even if not building a python package)
+ condarc_options: Pre-parsed condarc options loaded from the respective YAML
+ file
+
+ '''
+
+ # if you get to this point, tries to build the package
+ public_channels = bootstrap.get_channels(public=True, stable=True,
+ server=server, intranet=intranet)
+
+ logger.info('Using the following channels during (potential) build:\n - %s',
+ '\n - '.join(public_channels + ['defaults']))
+ condarc_options['channels'] = public_channels + ['defaults']
+
+ logger.info('Merging conda configuration files...')
+ conda_config = make_conda_config(conda_build_config, python_version,
+ None, condarc_options)
+
+ metadata = get_rendered_metadata(recipe_dir, conda_config)
+ recipe = get_parsed_recipe(metadata)
+
+ if recipe is None:
+ logger.info('Skipping build for %s - rendering returned None', recipe_dir)
+ return
+
+ # no dot in py_ver
+ py_ver = python_version.replace('.', '')
+ arch = conda_arch()
+
+ candidate = exists_on_channel(public_channels[0], recipe['package']['name'],
+ recipe['package']['version'], recipe['build']['number'],
+ python_version)
+ if candidate is not None:
+ logger.info('Skipping build for %s-%s-(py%s_?)%s for %s - exists ' \
+ 'on channel', candidate[0], candidate[1], candidate[2], py_ver, arch)
+ return
+
+ # if you get to this point, just builds the package
+ logger.info('Building %s-%s-(py%s_?)%s for %s',
+ recipe['package']['name'], recipe['package']['version'],
+ recipe['build']['number'], py_ver, arch)
+ conda_build.api.build(recipe_dir, config=conda_config)
+
+
if __name__ == '__main__':
import argparse
@@ -476,8 +594,22 @@ if __name__ == '__main__':
with open(condarc, 'rb') as f:
condarc_options = yaml.load(f)
+ # dump packages at conda_root
+ condarc_options['croot'] = os.path.join(args.conda_root, 'conda-bld')
+
+ # builds all dependencies in the 'deps' subdirectory - or at least checks
+ # these dependencies are already available; these dependencies go directly to
+ # the public channel once built
+ for recipe in glob.glob(os.path.join('deps', '*')):
+ if not os.path.exists(os.path.join(recipe, 'meta.yaml')):
+ # ignore - not a conda package
+ continue
+ base_build(server, not args.internet, recipe, conda_build_config,
+ args.python_version, condarc_options)
+
# notice this condarc typically will only contain the defaults channel - we
- # need to boost this up with more channels to get it right.
+ # need to boost this up with more channels to get it right for this package's
+ # build
public = ( args.visibility == 'public' )
channels = bootstrap.get_channels(public=public, stable=(not is_prerelease),
server=server, intranet=(not args.internet))
@@ -485,18 +617,15 @@ if __name__ == '__main__':
'\n - '.join(channels + ['defaults']))
condarc_options['channels'] = channels + ['defaults']
- # dump packages at conda_root
- condarc_options['croot'] = os.path.join(args.conda_root, 'conda-bld')
-
- logger.info('Merging conda configuration files...')
- conda_config = make_conda_config(conda_build_config, args.python_version,
- recipe_append, condarc_options)
-
# retrieve the current build number for this build
build_number, _ = next_build_number(channels[0], args.name, version,
args.python_version)
bootstrap.set_environment('BOB_BUILD_NUMBER', str(build_number))
+ logger.info('Merging conda configuration files...')
+ conda_config = make_conda_config(conda_build_config, args.python_version,
+ recipe_append, condarc_options)
+
# runs the build using the conda-build API
arch = conda_arch()
logger.info('Building %s-%s-py%s (build: %d) for %s',
@@ -517,4 +646,4 @@ if __name__ == '__main__':
else:
logger.info('twine check (a.k.a. readme check) %s: OK', package[0])
- git_clean_build(bootstrap.run_cmdline, verbose=(args.verbose >= 2))
+ git_clean_build(bootstrap.run_cmdline, verbose=(args.verbose >= 3))
diff --git a/bob/devtools/data/conda_build_config.yaml b/bob/devtools/data/conda_build_config.yaml
index 46fa5c044415da9dda30394f4e0a6cba6865cb69..0302a7dee0226a1dd2a8892df3a2638ccaa60c9f 100644
--- a/bob/devtools/data/conda_build_config.yaml
+++ b/bob/devtools/data/conda_build_config.yaml
@@ -32,6 +32,7 @@ pin_run_as_build:
## the dependencies that we build against multiple versions
python:
- 3.6
+ - 3.7
zip_keys:
- # [win]
diff --git a/bob/devtools/data/gitlab-ci/single-package.yaml b/bob/devtools/data/gitlab-ci/single-package.yaml
index 5b045a479856f0567d312e8decd6c2cc4d3e5b32..f1cb7a09a2fce60587f134ceb6123468049f8be0 100644
--- a/bob/devtools/data/gitlab-ci/single-package.yaml
+++ b/bob/devtools/data/gitlab-ci/single-package.yaml
@@ -20,9 +20,9 @@ stages:
stage: build
script:
- curl --silent "${BOOTSTRAP}" --output "bootstrap.py"
- - python3 bootstrap.py -vv channel bdt
+ - python3 bootstrap.py -vv channel base
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- - conda activate bdt
+ - conda activate base
- bdt ci build -vv
- bdt ci clean -vv
cache: &build_caches
@@ -66,9 +66,9 @@ build_linux_36:
BUILD_EGG: "true"
script:
- curl --silent "${BOOTSTRAP}" --output "bootstrap.py"
- - python3 bootstrap.py -vv channel bdt
+ - python3 bootstrap.py -vv channel base
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- - conda activate bdt
+ - conda activate base
- bdt ci build -vv
- bdt ci readme -vv dist/*.zip
- bdt ci clean -vv
@@ -91,9 +91,9 @@ build_macosx_36:
stage: deploy
script:
- curl --silent "${BOOTSTRAP}" --output "bootstrap.py"
- - python3 bootstrap.py channel bdt
+ - python3 bootstrap.py channel base
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- - conda activate bdt
+ - conda activate base
- bdt ci deploy -vv
- bdt ci clean -vv
dependencies:
@@ -133,9 +133,9 @@ pypi:
- branches
script:
- curl --silent "${BOOTSTRAP}" --output "bootstrap.py"
- - python3 bootstrap.py -vv channel bdt
+ - python3 bootstrap.py -vv channel base
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- - conda activate bdt
+ - conda activate base
- bdt ci pypi -vv dist/*.zip
- bdt ci clean -vv
dependencies:
diff --git a/bob/devtools/scripts/ci.py b/bob/devtools/scripts/ci.py
index f05c4a425591f90621dbee407d7e77f2e6e0efef..ced320084cae86ffdb8fb24996747f842e509707 100644
--- a/bob/devtools/scripts/ci.py
+++ b/bob/devtools/scripts/ci.py
@@ -30,6 +30,76 @@ def ci():
pass
+@ci.command(epilog='''
+Examples:
+
+ 1. Deploys base build artifacts (dependencies) to the appropriate channels:
+
+ $ bdt ci base-deploy -vv
+
+''')
+@click.option('-d', '--dry-run/--no-dry-run', default=False,
+ help='Only goes through the actions, but does not execute them ' \
+ '(combine with the verbosity flags - e.g. ``-vvv``) to enable ' \
+ 'printing to help you understand what will be done')
+@verbosity_option()
+@bdt.raise_on_error
+def base_deploy(dry_run):
+ """Deploys dependencies not available at the defaults channel
+
+ Deployment happens to our public channel directly, as these are
+ dependencies are required for proper bob/beat package runtime environments.
+ """
+
+ if dry_run:
+ logger.warn('!!!! DRY RUN MODE !!!!')
+ logger.warn('Nothing is being deployed to server')
+
+ package = os.environ['CI_PROJECT_PATH']
+
+ from ..constants import WEBDAV_PATHS
+ server_info = WEBDAV_PATHS[True][True] #stable=True, visible=True
+
+ logger.info('Deploying dependence packages to %s%s%s...', SERVER,
+ server_info['root'], server_info['conda'])
+
+ # setup webdav connection
+ webdav_options = {
+ 'webdav_hostname': SERVER,
+ 'webdav_root': server_info['root'],
+ 'webdav_login': os.environ['DOCUSER'],
+ 'webdav_password': os.environ['DOCPASS'],
+ }
+ from ..webdav3 import client as webdav
+ davclient = webdav.Client(webdav_options)
+ assert davclient.valid()
+
+ group, name = package.split('/')
+
+ # uploads conda package artificats
+ for arch in ('linux-64', 'osx-64', 'noarch'):
+ # finds conda dependencies and uploads what we can find
+ package_path = os.path.join(os.environ['CONDA_ROOT'], 'conda-bld', arch,
+ '*.tar.bz2')
+ deploy_packages = glob.glob(package_path)
+ for k in deploy_packages:
+ basename = os.path.basename(k)
+ if basename.startswith(name):
+ logger.debug('Skipping deploying of %s - not a base package', k)
+ continue
+
+ remote_path = '%s/%s/%s' % (server_info['conda'], arch, basename)
+ if davclient.check(remote_path):
+ raise RuntimeError('The file %s/%s already exists on the server ' \
+ '- this can be due to more than one build with deployment ' \
+ 'running at the same time. Re-running the broken builds ' \
+ 'normally fixes it' % (SERVER, remote_path))
+ logger.info('[dav] %s -> %s%s%s', k, SERVER, server_info['root'],
+ remote_path)
+ if not dry_run:
+ davclient.upload(local_path=k, remote_path=remote_path)
+
+
@ci.command(epilog='''
Examples:
@@ -291,4 +361,4 @@ def clean(ctx):
from ..build import git_clean_build
from ..bootstrap import run_cmdline
- git_clean_build(run_cmdline, verbose=(ctx.meta['verbosity']>=2))
+ git_clean_build(run_cmdline, verbose=(ctx.meta['verbosity']>=3))
diff --git a/conda/meta.yaml b/conda/meta.yaml
index deb63262b5fc1e768835a2139cd3a9bf9dec0f88..0cc376da3ab5a2eedcc71f43480f49f8bad98279 100644
--- a/conda/meta.yaml
+++ b/conda/meta.yaml
@@ -68,6 +68,7 @@ test:
- bdt new --help
- bdt ci --help
- bdt ci build --help
+ - bdt ci base-deploy --help
- bdt ci deploy --help
- bdt ci pypi --help
- bdt ci readme --help
diff --git a/deps/python-gitlab/meta.yaml b/deps/python-gitlab/meta.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..8cce6e8296470bc5ea3d891f0ef6436c48ded162
--- /dev/null
+++ b/deps/python-gitlab/meta.yaml
@@ -0,0 +1,46 @@
+{% set name = "python-gitlab" %}
+{% set version = "1.7.0" %}
+
+package:
+ name: {{ name|lower }}
+ version: {{ version }}
+
+source:
+ url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name }}-{{ version }}.tar.gz
+ sha256: 401ef8929db4dcb5b08e0a2263a0a70599fc7e5b27615f956ac26d245802d09e
+
+build:
+ number: 0
+ script: "{{ PYTHON }} -m pip install . -vv"
+ entry_points:
+ - gitlab = gitlab.cli:main
+
+requirements:
+ host:
+ - python
+ - pip
+ run:
+ - python
+ - requests
+ - six
+
+test:
+ imports:
+ - gitlab
+
+about:
+ home: https://github.com/python-gitlab/python-gitlab
+ license: LGPL-3.0
+ license_family: LGPL
+ license_file: COPYING
+ summary: 'Python wrapper for the GitLab API'
+ description: |
+ python-gitlab is a Python package providing access to the GitLab
+ server API. It supports the v4 API of GitLab, and provides a CLI
+ tool (gitlab).
+ doc_url: https://python-gitlab.readthedocs.io/
+ dev_url: https://github.com/python-gitlab/python-gitlab
+
+extra:
+ recipe-maintainers:
+ - anjos
diff --git a/setup.py b/setup.py
index 178fd46511dc7f630499511bcee83ddd0b802bd6..01918fb2ffcbeb365f69dbfa00430be17ff57aa5 100644
--- a/setup.py
+++ b/setup.py
@@ -59,6 +59,7 @@ setup(
'bdt.ci.cli': [
'build = bob.devtools.scripts.ci:build',
'clean = bob.devtools.scripts.ci:clean',
+ 'base-deploy = bob.devtools.scripts.ci:base_deploy',
'deploy = bob.devtools.scripts.ci:deploy',
'readme = bob.devtools.scripts.ci:readme',
'pypi = bob.devtools.scripts.ci:pypi',