From 6e6b148e2ea3c8a0b287c3588c2d78c4a5810954 Mon Sep 17 00:00:00 2001
From: Andre Anjos <andre.dos.anjos@gmail.com>
Date: Thu, 14 Feb 2019 10:39:22 +0100
Subject: [PATCH] [build] Support multi-python base builds
---
bob/devtools/build.py | 51 +++++++++++++++------
bob/devtools/data/gitlab-ci/base-build.yaml | 12 ++---
bob/devtools/scripts/ci.py | 26 ++++++++---
3 files changed, 62 insertions(+), 27 deletions(-)
diff --git a/bob/devtools/build.py b/bob/devtools/build.py
index 709529a4..2045b8a1 100644
--- a/bob/devtools/build.py
+++ b/bob/devtools/build.py
@@ -86,7 +86,10 @@ def next_build_number(channel_url, name, version, python):
for dist in index:
if dist.name == name and dist.version == version:
- match = re.match('py[2-9][0-9]+', dist.build_string)
+ if py_ver:
+ match = re.match('py[2-9][0-9]+', dist.build_string)
+ else:
+ match = re.match('py', dist.build_string)
if match and match.group() == 'py{}'.format(py_ver):
logger.debug("Found match at %s for %s-%s-py%s", index[dist].url,
@@ -165,7 +168,9 @@ def exists_on_channel(channel_url, name, version, build_number,
name: The name of the package
version: The version of the package
build_number: The build number of the package
- python_version: The current version of python we're building for
+ python_version: The current version of python we're building for. May be
+ ``noarch``, to check for "noarch" packages or ``None``, in which case we
+ don't check for the python version
Returns: A complete package name, version and build string, if the package
already exists in the channel or ``None`` otherwise.
@@ -174,8 +179,10 @@ def exists_on_channel(channel_url, name, version, build_number,
from conda.exports import get_index
- # no dot in py_ver
- py_ver = python_version.replace('.', '')
+ # handles different cases as explained on the description of
+ # ``python_version``
+ py_ver = python_version.replace('.', '') if python_version else None
+ if py_ver == 'noarch': py_ver = ''
# get the channel index
logger.debug('Downloading channel index from %s', channel_url)
@@ -204,8 +211,12 @@ def exists_on_channel(channel_url, name, version, build_number,
dist.version, dist.build_string)
return (dist.name, dist.version, dist.build_string)
- logger.info('No matches for %s-%s-(py%s_?)%s found among %d packages',
- name, version, py_ver, build_number, len(index))
+ if py_ver is None:
+ logger.info('No matches for %s-%s-%s found among %d packages',
+ name, version, build_number, len(index))
+ else:
+ logger.info('No matches for %s-%s-py%s_%s found among %d packages',
+ name, version, py_ver, build_number, len(index))
return
@@ -483,7 +494,10 @@ def base_build(bootstrap, server, intranet, recipe_dir, conda_build_config,
recipe_dir: The directory containing the recipe's ``meta.yaml`` file
conda_build_config: Path to the ``conda_build_config.yaml`` file to use
python_version: String with the python version to build for, in the format
- ``x.y`` (should be passed even if not building a python package)
+ ``x.y`` (should be passed even if not building a python package). It
+ can also be set to ``noarch``, or ``None``. If set to ``None``, then we
+ don't assume there is a python-specific version being built. If set to
+ ``noarch``, then it is a python package without a specific build.
condarc_options: Pre-parsed condarc options loaded from the respective YAML
file
@@ -498,8 +512,12 @@ def base_build(bootstrap, server, intranet, recipe_dir, conda_build_config,
condarc_options['channels'] = public_channels + ['defaults']
logger.info('Merging conda configuration files...')
- conda_config = make_conda_config(conda_build_config, python_version,
- None, condarc_options)
+ if python_version not in ('noarch', None):
+ conda_config = make_conda_config(conda_build_config, python_version,
+ None, condarc_options)
+ else:
+ conda_config = make_conda_config(conda_build_config, None, None,
+ condarc_options)
metadata = get_rendered_metadata(recipe_dir, conda_config)
recipe = get_parsed_recipe(metadata)
@@ -508,8 +526,10 @@ def base_build(bootstrap, server, intranet, recipe_dir, conda_build_config,
logger.info('Skipping build for %s - rendering returned None', recipe_dir)
return
- # no dot in py_ver
- py_ver = python_version.replace('.', '')
+ # handles different cases as explained on the description of
+ # ``python_version``
+ py_ver = python_version.replace('.', '') if python_version else None
+ if py_ver == 'noarch': py_ver = ''
arch = conda_arch()
candidate = exists_on_channel(public_channels[0], recipe['package']['name'],
@@ -521,9 +541,14 @@ def base_build(bootstrap, server, intranet, recipe_dir, conda_build_config,
return
# if you get to this point, just builds the package
- logger.info('Building %s-%s-(py%s_?)%s for %s',
+ if py_ver is None:
+ logger.info('Building %s-%s-%s for %s',
recipe['package']['name'], recipe['package']['version'],
- recipe['build']['number'], py_ver, arch)
+ recipe['build']['number'], arch)
+ else:
+ logger.info('Building %s-%s-py%s_%s for %s',
+ recipe['package']['name'], recipe['package']['version'], py_ver,
+ recipe['build']['number'], arch)
conda_build.api.build(recipe_dir, config=conda_config)
diff --git a/bob/devtools/data/gitlab-ci/base-build.yaml b/bob/devtools/data/gitlab-ci/base-build.yaml
index 8fdc9374..a9f616d9 100644
--- a/bob/devtools/data/gitlab-ci/base-build.yaml
+++ b/bob/devtools/data/gitlab-ci/base-build.yaml
@@ -21,7 +21,8 @@ stages:
- python3 bootstrap.py -vv channel base
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- conda activate base
- - bdt ci base-build -vv order.txt
+ - bdt ci base-build -vv nopython.txt
+ - bdt ci base-build -vv --python=3.6 python.txt
- '[ "${CI_COMMIT_REF_NAME}" = "master" ] && bdt ci base-deploy -vv'
- bdt ci clean -vv
cache: &build_caches
@@ -31,10 +32,8 @@ stages:
- ${CONDA_ROOT}/pkgs/urls.txt
-build_linux_36:
+build_linux:
<<: *build_job
- variables:
- PYTHON_VERSION: "3.6"
tags:
- docker
image: continuumio/conda-concourse-ci
@@ -42,10 +41,9 @@ build_linux_36:
<<: *build_caches
key: "linux-cache"
-build_macosx_36:
+
+build_macosx:
<<: *build_job
- variables:
- PYTHON_VERSION: "3.6"
tags:
- macosx
cache:
diff --git a/bob/devtools/scripts/ci.py b/bob/devtools/scripts/ci.py
index de5aea19..af9aff04 100644
--- a/bob/devtools/scripts/ci.py
+++ b/bob/devtools/scripts/ci.py
@@ -310,13 +310,16 @@ Examples:
''')
@click.argument('order', required=True, type=click.Path(file_okay=True,
dir_okay=False, exists=True), nargs=1)
+@click.option('-p', '--python', multiple=True,
+ help='Versions of python in the format "x.y" we should build for. Pass ' \
+ 'various times this option to build for multiple python versions')
@click.option('-d', '--dry-run/--no-dry-run', default=False,
help='Only goes through the actions, but does not execute them ' \
'(combine with the verbosity flags - e.g. ``-vvv``) to enable ' \
'printing to help you understand what will be done')
@verbosity_option()
@bdt.raise_on_error
-def base_build(order, dry_run):
+def base_build(order, python, dry_run):
"""Builds base (dependence) packages
This command builds dependence packages (packages that are not Bob/BEAT
@@ -343,17 +346,26 @@ def base_build(order, dry_run):
line = line.partition('#')[0].strip()
if line: recipes.append(line)
+ import itertools
from .. import bootstrap
- for order, recipe in enumerate(recipes):
- click.echo('\n' + (60*'='))
- click.echo('Building "%s" (%d/%d)' % (recipe, order+1, len(recipes)))
- click.echo((60*'=') + '\n')
+ # combine all versions of python with recipes
+ if python:
+ recipes = list(itertools.product(python, recipes))
+ else:
+ recipes = list(itertools.product([None], recipes))
+
+ for order, (pyver, recipe) in enumerate(recipes):
+ click.echo('\n' + (80*'='))
+ pytext = 'for python-%s' % pyver if pyver is not None else ''
+ click.echo('Building "%s" %s(%d/%d)' % \
+ (recipe, pytext, order+1, total_recipes))
+ click.echo((80*'=') + '\n')
if not os.path.exists(os.path.join(recipe, 'meta.yaml')):
logger.info('Ignoring directory "%s" - no meta.yaml found' % recipe)
continue
- _build(bootstrap, SERVER, True, recipe, CONDA_BUILD_CONFIG,
- os.environ['PYTHON_VERSION'], condarc_options)
+ _build(bootstrap, SERVER, True, recipe, CONDA_BUILD_CONFIG, pyver,
+ condarc_options)
@ci.command(epilog='''
--
GitLab