Commit 6e6b148e authored by André Anjos's avatar André Anjos 💬

[build] Support multi-python base builds

parent e98012f0
...@@ -86,7 +86,10 @@ def next_build_number(channel_url, name, version, python): ...@@ -86,7 +86,10 @@ def next_build_number(channel_url, name, version, python):
for dist in index: for dist in index:
if dist.name == name and dist.version == version: if dist.name == name and dist.version == version:
match = re.match('py[2-9][0-9]+', dist.build_string) if py_ver:
match = re.match('py[2-9][0-9]+', dist.build_string)
else:
match = re.match('py', dist.build_string)
if match and match.group() == 'py{}'.format(py_ver): if match and match.group() == 'py{}'.format(py_ver):
logger.debug("Found match at %s for %s-%s-py%s", index[dist].url, logger.debug("Found match at %s for %s-%s-py%s", index[dist].url,
...@@ -165,7 +168,9 @@ def exists_on_channel(channel_url, name, version, build_number, ...@@ -165,7 +168,9 @@ def exists_on_channel(channel_url, name, version, build_number,
name: The name of the package name: The name of the package
version: The version of the package version: The version of the package
build_number: The build number of the package build_number: The build number of the package
python_version: The current version of python we're building for python_version: The current version of python we're building for. May be
``noarch``, to check for "noarch" packages or ``None``, in which case we
don't check for the python version
Returns: A complete package name, version and build string, if the package Returns: A complete package name, version and build string, if the package
already exists in the channel or ``None`` otherwise. already exists in the channel or ``None`` otherwise.
...@@ -174,8 +179,10 @@ def exists_on_channel(channel_url, name, version, build_number, ...@@ -174,8 +179,10 @@ def exists_on_channel(channel_url, name, version, build_number,
from conda.exports import get_index from conda.exports import get_index
# no dot in py_ver # handles different cases as explained on the description of
py_ver = python_version.replace('.', '') # ``python_version``
py_ver = python_version.replace('.', '') if python_version else None
if py_ver == 'noarch': py_ver = ''
# get the channel index # get the channel index
logger.debug('Downloading channel index from %s', channel_url) logger.debug('Downloading channel index from %s', channel_url)
...@@ -204,8 +211,12 @@ def exists_on_channel(channel_url, name, version, build_number, ...@@ -204,8 +211,12 @@ def exists_on_channel(channel_url, name, version, build_number,
dist.version, dist.build_string) dist.version, dist.build_string)
return (dist.name, dist.version, dist.build_string) return (dist.name, dist.version, dist.build_string)
logger.info('No matches for %s-%s-(py%s_?)%s found among %d packages', if py_ver is None:
name, version, py_ver, build_number, len(index)) logger.info('No matches for %s-%s-%s found among %d packages',
name, version, build_number, len(index))
else:
logger.info('No matches for %s-%s-py%s_%s found among %d packages',
name, version, py_ver, build_number, len(index))
return return
...@@ -483,7 +494,10 @@ def base_build(bootstrap, server, intranet, recipe_dir, conda_build_config, ...@@ -483,7 +494,10 @@ def base_build(bootstrap, server, intranet, recipe_dir, conda_build_config,
recipe_dir: The directory containing the recipe's ``meta.yaml`` file recipe_dir: The directory containing the recipe's ``meta.yaml`` file
conda_build_config: Path to the ``conda_build_config.yaml`` file to use conda_build_config: Path to the ``conda_build_config.yaml`` file to use
python_version: String with the python version to build for, in the format python_version: String with the python version to build for, in the format
``x.y`` (should be passed even if not building a python package) ``x.y`` (should be passed even if not building a python package). It
can also be set to ``noarch``, or ``None``. If set to ``None``, then we
don't assume there is a python-specific version being built. If set to
``noarch``, then it is a python package without a specific build.
condarc_options: Pre-parsed condarc options loaded from the respective YAML condarc_options: Pre-parsed condarc options loaded from the respective YAML
file file
...@@ -498,8 +512,12 @@ def base_build(bootstrap, server, intranet, recipe_dir, conda_build_config, ...@@ -498,8 +512,12 @@ def base_build(bootstrap, server, intranet, recipe_dir, conda_build_config,
condarc_options['channels'] = public_channels + ['defaults'] condarc_options['channels'] = public_channels + ['defaults']
logger.info('Merging conda configuration files...') logger.info('Merging conda configuration files...')
conda_config = make_conda_config(conda_build_config, python_version, if python_version not in ('noarch', None):
None, condarc_options) conda_config = make_conda_config(conda_build_config, python_version,
None, condarc_options)
else:
conda_config = make_conda_config(conda_build_config, None, None,
condarc_options)
metadata = get_rendered_metadata(recipe_dir, conda_config) metadata = get_rendered_metadata(recipe_dir, conda_config)
recipe = get_parsed_recipe(metadata) recipe = get_parsed_recipe(metadata)
...@@ -508,8 +526,10 @@ def base_build(bootstrap, server, intranet, recipe_dir, conda_build_config, ...@@ -508,8 +526,10 @@ def base_build(bootstrap, server, intranet, recipe_dir, conda_build_config,
logger.info('Skipping build for %s - rendering returned None', recipe_dir) logger.info('Skipping build for %s - rendering returned None', recipe_dir)
return return
# no dot in py_ver # handles different cases as explained on the description of
py_ver = python_version.replace('.', '') # ``python_version``
py_ver = python_version.replace('.', '') if python_version else None
if py_ver == 'noarch': py_ver = ''
arch = conda_arch() arch = conda_arch()
candidate = exists_on_channel(public_channels[0], recipe['package']['name'], candidate = exists_on_channel(public_channels[0], recipe['package']['name'],
...@@ -521,9 +541,14 @@ def base_build(bootstrap, server, intranet, recipe_dir, conda_build_config, ...@@ -521,9 +541,14 @@ def base_build(bootstrap, server, intranet, recipe_dir, conda_build_config,
return return
# if you get to this point, just builds the package # if you get to this point, just builds the package
logger.info('Building %s-%s-(py%s_?)%s for %s', if py_ver is None:
logger.info('Building %s-%s-%s for %s',
recipe['package']['name'], recipe['package']['version'], recipe['package']['name'], recipe['package']['version'],
recipe['build']['number'], py_ver, arch) recipe['build']['number'], arch)
else:
logger.info('Building %s-%s-py%s_%s for %s',
recipe['package']['name'], recipe['package']['version'], py_ver,
recipe['build']['number'], arch)
conda_build.api.build(recipe_dir, config=conda_config) conda_build.api.build(recipe_dir, config=conda_config)
......
...@@ -21,7 +21,8 @@ stages: ...@@ -21,7 +21,8 @@ stages:
- python3 bootstrap.py -vv channel base - python3 bootstrap.py -vv channel base
- source ${CONDA_ROOT}/etc/profile.d/conda.sh - source ${CONDA_ROOT}/etc/profile.d/conda.sh
- conda activate base - conda activate base
- bdt ci base-build -vv order.txt - bdt ci base-build -vv nopython.txt
- bdt ci base-build -vv --python=3.6 python.txt
- '[ "${CI_COMMIT_REF_NAME}" = "master" ] && bdt ci base-deploy -vv' - '[ "${CI_COMMIT_REF_NAME}" = "master" ] && bdt ci base-deploy -vv'
- bdt ci clean -vv - bdt ci clean -vv
cache: &build_caches cache: &build_caches
...@@ -31,10 +32,8 @@ stages: ...@@ -31,10 +32,8 @@ stages:
- ${CONDA_ROOT}/pkgs/urls.txt - ${CONDA_ROOT}/pkgs/urls.txt
build_linux_36: build_linux:
<<: *build_job <<: *build_job
variables:
PYTHON_VERSION: "3.6"
tags: tags:
- docker - docker
image: continuumio/conda-concourse-ci image: continuumio/conda-concourse-ci
...@@ -42,10 +41,9 @@ build_linux_36: ...@@ -42,10 +41,9 @@ build_linux_36:
<<: *build_caches <<: *build_caches
key: "linux-cache" key: "linux-cache"
build_macosx_36:
build_macosx:
<<: *build_job <<: *build_job
variables:
PYTHON_VERSION: "3.6"
tags: tags:
- macosx - macosx
cache: cache:
......
...@@ -310,13 +310,16 @@ Examples: ...@@ -310,13 +310,16 @@ Examples:
''') ''')
@click.argument('order', required=True, type=click.Path(file_okay=True, @click.argument('order', required=True, type=click.Path(file_okay=True,
dir_okay=False, exists=True), nargs=1) dir_okay=False, exists=True), nargs=1)
@click.option('-p', '--python', multiple=True,
help='Versions of python in the format "x.y" we should build for. Pass ' \
'various times this option to build for multiple python versions')
@click.option('-d', '--dry-run/--no-dry-run', default=False, @click.option('-d', '--dry-run/--no-dry-run', default=False,
help='Only goes through the actions, but does not execute them ' \ help='Only goes through the actions, but does not execute them ' \
'(combine with the verbosity flags - e.g. ``-vvv``) to enable ' \ '(combine with the verbosity flags - e.g. ``-vvv``) to enable ' \
'printing to help you understand what will be done') 'printing to help you understand what will be done')
@verbosity_option() @verbosity_option()
@bdt.raise_on_error @bdt.raise_on_error
def base_build(order, dry_run): def base_build(order, python, dry_run):
"""Builds base (dependence) packages """Builds base (dependence) packages
This command builds dependence packages (packages that are not Bob/BEAT This command builds dependence packages (packages that are not Bob/BEAT
...@@ -343,17 +346,26 @@ def base_build(order, dry_run): ...@@ -343,17 +346,26 @@ def base_build(order, dry_run):
line = line.partition('#')[0].strip() line = line.partition('#')[0].strip()
if line: recipes.append(line) if line: recipes.append(line)
import itertools
from .. import bootstrap from .. import bootstrap
for order, recipe in enumerate(recipes): # combine all versions of python with recipes
click.echo('\n' + (60*'=')) if python:
click.echo('Building "%s" (%d/%d)' % (recipe, order+1, len(recipes))) recipes = list(itertools.product(python, recipes))
click.echo((60*'=') + '\n') else:
recipes = list(itertools.product([None], recipes))
for order, (pyver, recipe) in enumerate(recipes):
click.echo('\n' + (80*'='))
pytext = 'for python-%s' % pyver if pyver is not None else ''
click.echo('Building "%s" %s(%d/%d)' % \
(recipe, pytext, order+1, total_recipes))
click.echo((80*'=') + '\n')
if not os.path.exists(os.path.join(recipe, 'meta.yaml')): if not os.path.exists(os.path.join(recipe, 'meta.yaml')):
logger.info('Ignoring directory "%s" - no meta.yaml found' % recipe) logger.info('Ignoring directory "%s" - no meta.yaml found' % recipe)
continue continue
_build(bootstrap, SERVER, True, recipe, CONDA_BUILD_CONFIG, _build(bootstrap, SERVER, True, recipe, CONDA_BUILD_CONFIG, pyver,
os.environ['PYTHON_VERSION'], condarc_options) condarc_options)
@ci.command(epilog=''' @ci.command(epilog='''
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment