Commit 6e6b148e authored by André Anjos's avatar André Anjos 💬

[build] Support multi-python base builds

parent e98012f0
......@@ -86,7 +86,10 @@ def next_build_number(channel_url, name, version, python):
for dist in index:
if dist.name == name and dist.version == version:
match = re.match('py[2-9][0-9]+', dist.build_string)
if py_ver:
match = re.match('py[2-9][0-9]+', dist.build_string)
else:
match = re.match('py', dist.build_string)
if match and match.group() == 'py{}'.format(py_ver):
logger.debug("Found match at %s for %s-%s-py%s", index[dist].url,
......@@ -165,7 +168,9 @@ def exists_on_channel(channel_url, name, version, build_number,
name: The name of the package
version: The version of the package
build_number: The build number of the package
python_version: The current version of python we're building for
python_version: The current version of python we're building for. May be
``noarch``, to check for "noarch" packages or ``None``, in which case we
don't check for the python version
Returns: A complete package name, version and build string, if the package
already exists in the channel or ``None`` otherwise.
......@@ -174,8 +179,10 @@ def exists_on_channel(channel_url, name, version, build_number,
from conda.exports import get_index
# no dot in py_ver
py_ver = python_version.replace('.', '')
# handles different cases as explained on the description of
# ``python_version``
py_ver = python_version.replace('.', '') if python_version else None
if py_ver == 'noarch': py_ver = ''
# get the channel index
logger.debug('Downloading channel index from %s', channel_url)
......@@ -204,8 +211,12 @@ def exists_on_channel(channel_url, name, version, build_number,
dist.version, dist.build_string)
return (dist.name, dist.version, dist.build_string)
logger.info('No matches for %s-%s-(py%s_?)%s found among %d packages',
name, version, py_ver, build_number, len(index))
if py_ver is None:
logger.info('No matches for %s-%s-%s found among %d packages',
name, version, build_number, len(index))
else:
logger.info('No matches for %s-%s-py%s_%s found among %d packages',
name, version, py_ver, build_number, len(index))
return
......@@ -483,7 +494,10 @@ def base_build(bootstrap, server, intranet, recipe_dir, conda_build_config,
recipe_dir: The directory containing the recipe's ``meta.yaml`` file
conda_build_config: Path to the ``conda_build_config.yaml`` file to use
python_version: String with the python version to build for, in the format
``x.y`` (should be passed even if not building a python package)
``x.y`` (should be passed even if not building a python package). It
can also be set to ``noarch``, or ``None``. If set to ``None``, then we
don't assume there is a python-specific version being built. If set to
``noarch``, then it is a python package without a specific build.
condarc_options: Pre-parsed condarc options loaded from the respective YAML
file
......@@ -498,8 +512,12 @@ def base_build(bootstrap, server, intranet, recipe_dir, conda_build_config,
condarc_options['channels'] = public_channels + ['defaults']
logger.info('Merging conda configuration files...')
conda_config = make_conda_config(conda_build_config, python_version,
None, condarc_options)
if python_version not in ('noarch', None):
conda_config = make_conda_config(conda_build_config, python_version,
None, condarc_options)
else:
conda_config = make_conda_config(conda_build_config, None, None,
condarc_options)
metadata = get_rendered_metadata(recipe_dir, conda_config)
recipe = get_parsed_recipe(metadata)
......@@ -508,8 +526,10 @@ def base_build(bootstrap, server, intranet, recipe_dir, conda_build_config,
logger.info('Skipping build for %s - rendering returned None', recipe_dir)
return
# no dot in py_ver
py_ver = python_version.replace('.', '')
# handles different cases as explained on the description of
# ``python_version``
py_ver = python_version.replace('.', '') if python_version else None
if py_ver == 'noarch': py_ver = ''
arch = conda_arch()
candidate = exists_on_channel(public_channels[0], recipe['package']['name'],
......@@ -521,9 +541,14 @@ def base_build(bootstrap, server, intranet, recipe_dir, conda_build_config,
return
# if you get to this point, just builds the package
logger.info('Building %s-%s-(py%s_?)%s for %s',
if py_ver is None:
logger.info('Building %s-%s-%s for %s',
recipe['package']['name'], recipe['package']['version'],
recipe['build']['number'], py_ver, arch)
recipe['build']['number'], arch)
else:
logger.info('Building %s-%s-py%s_%s for %s',
recipe['package']['name'], recipe['package']['version'], py_ver,
recipe['build']['number'], arch)
conda_build.api.build(recipe_dir, config=conda_config)
......
......@@ -21,7 +21,8 @@ stages:
- python3 bootstrap.py -vv channel base
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- conda activate base
- bdt ci base-build -vv order.txt
- bdt ci base-build -vv nopython.txt
- bdt ci base-build -vv --python=3.6 python.txt
- '[ "${CI_COMMIT_REF_NAME}" = "master" ] && bdt ci base-deploy -vv'
- bdt ci clean -vv
cache: &build_caches
......@@ -31,10 +32,8 @@ stages:
- ${CONDA_ROOT}/pkgs/urls.txt
build_linux_36:
build_linux:
<<: *build_job
variables:
PYTHON_VERSION: "3.6"
tags:
- docker
image: continuumio/conda-concourse-ci
......@@ -42,10 +41,9 @@ build_linux_36:
<<: *build_caches
key: "linux-cache"
build_macosx_36:
build_macosx:
<<: *build_job
variables:
PYTHON_VERSION: "3.6"
tags:
- macosx
cache:
......
......@@ -310,13 +310,16 @@ Examples:
''')
@click.argument('order', required=True, type=click.Path(file_okay=True,
dir_okay=False, exists=True), nargs=1)
@click.option('-p', '--python', multiple=True,
help='Versions of python in the format "x.y" we should build for. Pass ' \
'various times this option to build for multiple python versions')
@click.option('-d', '--dry-run/--no-dry-run', default=False,
help='Only goes through the actions, but does not execute them ' \
'(combine with the verbosity flags - e.g. ``-vvv``) to enable ' \
'printing to help you understand what will be done')
@verbosity_option()
@bdt.raise_on_error
def base_build(order, dry_run):
def base_build(order, python, dry_run):
"""Builds base (dependence) packages
This command builds dependence packages (packages that are not Bob/BEAT
......@@ -343,17 +346,26 @@ def base_build(order, dry_run):
line = line.partition('#')[0].strip()
if line: recipes.append(line)
import itertools
from .. import bootstrap
for order, recipe in enumerate(recipes):
click.echo('\n' + (60*'='))
click.echo('Building "%s" (%d/%d)' % (recipe, order+1, len(recipes)))
click.echo((60*'=') + '\n')
# combine all versions of python with recipes
if python:
recipes = list(itertools.product(python, recipes))
else:
recipes = list(itertools.product([None], recipes))
for order, (pyver, recipe) in enumerate(recipes):
click.echo('\n' + (80*'='))
pytext = 'for python-%s' % pyver if pyver is not None else ''
click.echo('Building "%s" %s(%d/%d)' % \
(recipe, pytext, order+1, total_recipes))
click.echo((80*'=') + '\n')
if not os.path.exists(os.path.join(recipe, 'meta.yaml')):
logger.info('Ignoring directory "%s" - no meta.yaml found' % recipe)
continue
_build(bootstrap, SERVER, True, recipe, CONDA_BUILD_CONFIG,
os.environ['PYTHON_VERSION'], condarc_options)
_build(bootstrap, SERVER, True, recipe, CONDA_BUILD_CONFIG, pyver,
condarc_options)
@ci.command(epilog='''
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment