From 5fb2b39b5c397748a5b0fff5d02e868eabe0b884 Mon Sep 17 00:00:00 2001
From: Andre Anjos <andre.dos.anjos@gmail.com>
Date: Mon, 25 Feb 2019 07:49:44 +0100
Subject: [PATCH] [scripts][build] Use get_output_path() for more accurately
searching for the next build number
---
bob/devtools/build.py | 156 +++++++++++++++-------------------
bob/devtools/scripts/build.py | 33 ++++---
2 files changed, 86 insertions(+), 103 deletions(-)
diff --git a/bob/devtools/build.py b/bob/devtools/build.py
index 7ad4acb6..52a5ac57 100644
--- a/bob/devtools/build.py
+++ b/bob/devtools/build.py
@@ -48,58 +48,54 @@ def should_skip_build(metadata_tuples):
return all(m[0].skip() for m in metadata_tuples)
-def next_build_number(channel_url, name, version, python):
+def next_build_number(channel_url, basename):
"""Calculates the next build number of a package given the channel
This function returns the next build number (integer) for a package given its
- recipe, dependencies, name, version and python version. It looks on the
- channel URL provided and figures out if any clash would happen and what would
- be the highest build number available for that configuration.
+ resulting tarball base filename (can be obtained with
+ :py:func:`get_output_path`).
Args:
channel_url: The URL where to look for packages clashes (normally a beta
channel)
- name: The name of the package
- version: The version of the package
- python: The version of python as 2 digits (e.g.: "2.7" or "3.6")
+ basename: The tarball basename to check on the channel
Returns: The next build number with the current configuration. Zero (0) is
returned if no match is found. Also returns the URLs of the packages it
- finds with matches on the name, version and python-version.
+ finds with matches on the name, version and python-version, ordered by
+ (reversed) build-number.
"""
from conda.exports import get_index
- # no dot in py_ver
- py_ver = python.replace('.', '')
-
# get the channel index
logger.debug('Downloading channel index from %s', channel_url)
index = get_index(channel_urls=[channel_url], prepend=False)
- # search if package with the same version exists
- build_number = 0
- urls = []
- for dist in index:
+ # remove .tar.bz2 from name, then split from the end twice, on '-'
+ name, version, build = basename[:-8].rsplit('-', 2)
- if dist.name == name and dist.version == version:
- if py_ver:
- match = re.match('py[2-9][0-9]+', dist.build_string)
- else:
- match = re.match('py', dist.build_string)
+ # remove the build number as we're looking for the next value
+ build_variant = build.rsplit('_', 1)[0]
- if match and match.group() == 'py{}'.format(py_ver):
- logger.debug("Found match at %s for %s-%s-py%s", index[dist].url,
- name, version, py_ver)
- build_number = max(build_number, dist.build_number + 1)
- urls.append(index[dist].url)
+ # search if package with the same characteristics
+ urls = {}
+ build_number = 0
+ for dist in index:
+ if dist.name == name and dist.version == version and \
+ dist.build_string.startswith(build_variant): #match!
+ url = index[dist].url
+ logger.debug("Found match at %s for %s-%s-%s", url,
+ name, version, build_variant)
+ build_number = max(build_number, dist.build_number + 1)
+ urls[dist.build_number] = url.replace(channel_url, '')
- urls = [url.replace(channel_url, '') for url in urls]
+ sorted_urls = [urls[k] for k in reversed(list(urls.keys()))]
- return build_number, urls
+ return build_number, sorted_urls
def make_conda_config(config, python, append_file, condarc_options):
@@ -142,6 +138,13 @@ def make_conda_config(config, python, append_file, condarc_options):
return retval
+def get_output_path(metadata, config):
+ '''Renders the recipe and returns the interpreted YAML file'''
+
+ from conda_build.api import get_output_file_path
+ return get_output_file_paths(metadata, config=config)[0]
+
+
def get_rendered_metadata(recipe_dir, config):
'''Renders the recipe and returns the interpreted YAML file'''
@@ -157,66 +160,35 @@ def get_parsed_recipe(metadata):
return yaml.load(output)
-def exists_on_channel(channel_url, name, version, build_number,
- python_version):
+def exists_on_channel(channel_url, basename):
"""Checks on the given channel if a package with the specs exist
Args:
channel_url: The URL where to look for packages clashes (normally a beta
channel)
- name: The name of the package
- version: The version of the package
- build_number: The build number of the package
- python_version: The current version of python we're building for. May be
- ``noarch``, to check for "noarch" packages or ``None``, in which case we
- don't check for the python version
+ basename: The basename of the tarball to search for
- Returns: A complete package name, version and build string, if the package
- already exists in the channel or ``None`` otherwise.
+ Returns: A complete package url, if the package already exists in the channel
+ or ``None`` otherwise.
"""
from conda.exports import get_index
- # handles different cases as explained on the description of
- # ``python_version``
- py_ver = python_version.replace('.', '') if python_version else None
- if py_ver == 'noarch': py_ver = ''
-
# get the channel index
logger.debug('Downloading channel index from %s', channel_url)
index = get_index(channel_urls=[channel_url], prepend=False)
- logger.info('Checking for %s-%s-%s...', name, version, build_number)
+ logger.info('Checking for %s...', basename)
for dist in index:
+ url = index[dist].url
+ if url.endswith(basename):
+ logger.debug('Found matching package (%s) at %s', basename, url)
+ return url
- if dist.name == name and dist.version == version and \
- dist.build_string.endswith('_%s' % build_number):
-
- # two possible options must be checked - (i) the package build_string
- # starts with ``py``, which means it is a python specific package so we
- # must also check for the matching python version. (ii) the package is
- # not a python-specific package and a simple match will do
- if dist.build_string.startswith('py'):
- match = re.match('py[2-9][0-9]+', dist.build_string)
- if match and match.group() == 'py{}'.format(py_ver):
- logger.debug('Found matching package (%s-%s-%s)', dist.name,
- dist.version, dist.build_string)
- return (dist.name, dist.version, dist.build_string)
-
- else:
- logger.debug('Found matching package (%s-%s-%s)', dist.name,
- dist.version, dist.build_string)
- return (dist.name, dist.version, dist.build_string)
-
- if py_ver is None:
- logger.info('No matches for %s-%s-%s found among %d packages',
- name, version, build_number, len(index))
- else:
- logger.info('No matches for %s-%s-py%s_%s found among %d packages',
- name, version, py_ver, build_number, len(index))
+ logger.debug('No matches for %s', path)
return
@@ -543,26 +515,16 @@ def base_build(bootstrap, server, intranet, group, recipe_dir,
'on %s', recipe_dir, python_version, arch)
return
- recipe = get_parsed_recipe(metadata)
+ path = get_output_path(metadata, conda_config)
- candidate = exists_on_channel(public_channels[0], recipe['package']['name'],
- recipe['package']['version'], recipe['build']['number'],
- python_version)
- if candidate is not None:
- logger.info('Skipping build for %s-%s-%s for %s - exists ' \
- 'on channel', candidate[0], candidate[1], candidate[2], arch)
+ url = exists_on_channel(public_channels[0], os.path.basename(path))
+ if url is not None:
+ logger.info('Skipping build for %s as it exists (at %s)', path, url)
return
# if you get to this point, just builds the package
- if py_ver is None:
- logger.info('Building %s-%s-%s for %s',
- recipe['package']['name'], recipe['package']['version'],
- recipe['build']['number'], arch)
- else:
- logger.info('Building %s-%s-py%s_%s for %s',
- recipe['package']['name'], recipe['package']['version'], py_ver,
- recipe['build']['number'], arch)
- conda_build.api.build(recipe_dir, config=conda_config)
+ logger.info('Building %s', path)
+ conda_build.api.build(metadata, config=conda_config)
if __name__ == '__main__':
@@ -659,20 +621,36 @@ if __name__ == '__main__':
'\n - '.join(channels + ['defaults']))
condarc_options['channels'] = channels + ['defaults']
- # retrieve the current build number for this build
- build_number, _ = next_build_number(channels[0], args.name, version,
- args.python_version)
- bootstrap.set_environment('BOB_BUILD_NUMBER', str(build_number))
-
logger.info('Merging conda configuration files...')
conda_config = make_conda_config(conda_build_config, args.python_version,
recipe_append, condarc_options)
+ metadata = get_rendered_metadata(os.path.join(args.work_dir, 'conda'),
+ conda_config)
+ path = get_output_path(metadata, conda_config)
+
+ # asserts we're building at the right location
+ assert path.startswith(os.path.join(args.conda_root, 'conda-bld')), \
+ 'Output path for build (%s) does not start with "%s" - this ' \
+ 'typically means this build is running on a shared builder and ' \
+ 'the file ~/.conda/environments.txt is polluted with other ' \
+ 'environment paths. To fix, empty that file and set its mode ' \
+ 'to read-only for all.' % (path, os.path.join(args.conda_root,
+ 'conda-bld'))
+
+ # retrieve the current build number for this build
+ build_number, _ = next_build_number(channels[0], os.path.basename(path))
+
# runs the build using the conda-build API
arch = conda_arch()
logger.info('Building %s-%s-py%s (build: %d) for %s',
args.name, version, args.python_version.replace('.',''), build_number,
arch)
+
+ # notice we cannot build from the pre-parsed metadata because it has already
+ # resolved the "wrong" build number. We'll have to reparse after setting the
+ # environment variable BOB_BUILD_NUMBER.
+ bootstrap.set_environment('BOB_BUILD_NUMBER', str(build_number))
conda_build.api.build(os.path.join(args.work_dir, 'conda'),
config=conda_config)
diff --git a/bob/devtools/scripts/build.py b/bob/devtools/scripts/build.py
index 29aef646..059aae25 100644
--- a/bob/devtools/scripts/build.py
+++ b/bob/devtools/scripts/build.py
@@ -12,7 +12,7 @@ import conda_build.api
from . import bdt
from ..build import next_build_number, conda_arch, should_skip_build, \
get_rendered_metadata, get_parsed_recipe, make_conda_config, \
- get_docserver_setup, get_env_directory
+ get_docserver_setup, get_env_directory, get_output_path
from ..constants import CONDA_BUILD_CONFIG, CONDA_RECIPE_APPEND, \
SERVER, MATPLOTLIB_RCDIR, BASE_CONDARC
from ..bootstrap import set_environment, get_channels
@@ -125,6 +125,8 @@ def build(recipe_dir, python, condarc, config, no_test, append_file,
server=server, intranet=ci, group=group)
set_environment('BOB_DOCUMENTATION_SERVER', doc_urls)
+ arch = conda_arch()
+
for d in recipe_dir:
if not os.path.exists(d):
@@ -135,24 +137,27 @@ def build(recipe_dir, python, condarc, config, no_test, append_file,
version = open(version_candidate).read().rstrip()
set_environment('BOB_PACKAGE_VERSION', version)
- # pre-renders the recipe - figures out package name and version
+ # pre-renders the recipe - figures out the destination
metadata = get_rendered_metadata(d, conda_config)
- # checks we should actually build this recipe
- arch = conda_arch()
- if should_skip_build(metadata):
- logger.warn('Skipping UNSUPPORTED build of "%s" for py%s on %s',
- d, python.replace('.',''), arch)
- return 0
-
- # converts the metadata output into parsed yaml and continues the process
rendered_recipe = get_parsed_recipe(metadata)
- # if a channel URL was passed, set the build number
- build_number, _ = next_build_number(channels[0],
- rendered_recipe['package']['name'],
- rendered_recipe['package']['version'], python)
+ path = get_output_path(metadata, conda_config)
+ # checks if we should actually build this recipe
+ if should_skip_build(metadata):
+ logger.info('Skipping UNSUPPORTED build of %s-%s-py%s for %s',
+ rendered_recipe['package']['name'],
+ rendered_recipe['package']['version'], python.replace('.',''),
+ arch)
+ continue
+
+ # gets the next build number
+ build_number, _ = next_build_number(channels[0], os.path.basename(path))
+
+ # notice we cannot build from the pre-parsed metadata because it has
+ # already resolved the "wrong" build number. We'll have to reparse after
+ # setting the environment variable BOB_BUILD_NUMBER.
set_environment('BOB_BUILD_NUMBER', str(build_number))
logger.info('Building %s-%s-py%s (build: %d) for %s',
--
GitLab