diff --git a/bob/devtools/build.py b/bob/devtools/build.py
index 7ad4acb69ca00c9878dc68b725f651aa03b1e02d..d71e6bfb694e197a7e88ab518c07f58ad3a58b56 100644
--- a/bob/devtools/build.py
+++ b/bob/devtools/build.py
@@ -48,58 +48,54 @@ def should_skip_build(metadata_tuples):
return all(m[0].skip() for m in metadata_tuples)
-def next_build_number(channel_url, name, version, python):
+def next_build_number(channel_url, basename):
"""Calculates the next build number of a package given the channel
This function returns the next build number (integer) for a package given its
- recipe, dependencies, name, version and python version. It looks on the
- channel URL provided and figures out if any clash would happen and what would
- be the highest build number available for that configuration.
+ resulting tarball base filename (can be obtained with
+ :py:func:`get_output_path`).
Args:
channel_url: The URL where to look for packages clashes (normally a beta
channel)
- name: The name of the package
- version: The version of the package
- python: The version of python as 2 digits (e.g.: "2.7" or "3.6")
+ basename: The tarball basename to check on the channel
Returns: The next build number with the current configuration. Zero (0) is
returned if no match is found. Also returns the URLs of the packages it
- finds with matches on the name, version and python-version.
+ finds with matches on the name, version and python-version, ordered by
+ (reversed) build-number.
"""
from conda.exports import get_index
- # no dot in py_ver
- py_ver = python.replace('.', '')
-
# get the channel index
logger.debug('Downloading channel index from %s', channel_url)
index = get_index(channel_urls=[channel_url], prepend=False)
- # search if package with the same version exists
- build_number = 0
- urls = []
- for dist in index:
+ # remove .tar.bz2 from name, then split from the end twice, on '-'
+ name, version, build = basename[:-8].rsplit('-', 2)
- if dist.name == name and dist.version == version:
- if py_ver:
- match = re.match('py[2-9][0-9]+', dist.build_string)
- else:
- match = re.match('py', dist.build_string)
+ # remove the build number as we're looking for the next value
+ build_variant = build.rsplit('_', 1)[0]
- if match and match.group() == 'py{}'.format(py_ver):
- logger.debug("Found match at %s for %s-%s-py%s", index[dist].url,
- name, version, py_ver)
- build_number = max(build_number, dist.build_number + 1)
- urls.append(index[dist].url)
+ # search if package with the same characteristics
+ urls = {}
+ build_number = 0
+ for dist in index:
+ if dist.name == name and dist.version == version and \
+ dist.build_string.startswith(build_variant): #match!
+ url = index[dist].url
+ logger.debug("Found match at %s for %s-%s-%s", url,
+ name, version, build_variant)
+ build_number = max(build_number, dist.build_number + 1)
+ urls[dist.build_number] = url.replace(channel_url, '')
- urls = [url.replace(channel_url, '') for url in urls]
+ sorted_urls = [urls[k] for k in reversed(list(urls.keys()))]
- return build_number, urls
+ return build_number, sorted_urls
def make_conda_config(config, python, append_file, condarc_options):
@@ -142,6 +138,13 @@ def make_conda_config(config, python, append_file, condarc_options):
return retval
+def get_output_path(metadata, config):
+ '''Renders the recipe and returns the interpreted YAML file'''
+
+ from conda_build.api import get_output_file_paths
+ return get_output_file_paths(metadata, config=config)[0]
+
+
def get_rendered_metadata(recipe_dir, config):
'''Renders the recipe and returns the interpreted YAML file'''
@@ -157,66 +160,35 @@ def get_parsed_recipe(metadata):
return yaml.load(output)
-def exists_on_channel(channel_url, name, version, build_number,
- python_version):
+def exists_on_channel(channel_url, basename):
"""Checks on the given channel if a package with the specs exist
Args:
channel_url: The URL where to look for packages clashes (normally a beta
channel)
- name: The name of the package
- version: The version of the package
- build_number: The build number of the package
- python_version: The current version of python we're building for. May be
- ``noarch``, to check for "noarch" packages or ``None``, in which case we
- don't check for the python version
+ basename: The basename of the tarball to search for
- Returns: A complete package name, version and build string, if the package
- already exists in the channel or ``None`` otherwise.
+ Returns: A complete package url, if the package already exists in the channel
+ or ``None`` otherwise.
"""
from conda.exports import get_index
- # handles different cases as explained on the description of
- # ``python_version``
- py_ver = python_version.replace('.', '') if python_version else None
- if py_ver == 'noarch': py_ver = ''
-
# get the channel index
logger.debug('Downloading channel index from %s', channel_url)
index = get_index(channel_urls=[channel_url], prepend=False)
- logger.info('Checking for %s-%s-%s...', name, version, build_number)
+ logger.info('Checking for %s...', basename)
for dist in index:
+ url = index[dist].url
+ if url.endswith(basename):
+ logger.debug('Found matching package (%s) at %s', basename, url)
+ return url
- if dist.name == name and dist.version == version and \
- dist.build_string.endswith('_%s' % build_number):
-
- # two possible options must be checked - (i) the package build_string
- # starts with ``py``, which means it is a python specific package so we
- # must also check for the matching python version. (ii) the package is
- # not a python-specific package and a simple match will do
- if dist.build_string.startswith('py'):
- match = re.match('py[2-9][0-9]+', dist.build_string)
- if match and match.group() == 'py{}'.format(py_ver):
- logger.debug('Found matching package (%s-%s-%s)', dist.name,
- dist.version, dist.build_string)
- return (dist.name, dist.version, dist.build_string)
-
- else:
- logger.debug('Found matching package (%s-%s-%s)', dist.name,
- dist.version, dist.build_string)
- return (dist.name, dist.version, dist.build_string)
-
- if py_ver is None:
- logger.info('No matches for %s-%s-%s found among %d packages',
- name, version, build_number, len(index))
- else:
- logger.info('No matches for %s-%s-py%s_%s found among %d packages',
- name, version, py_ver, build_number, len(index))
+ logger.debug('No matches for %s', path)
return
@@ -543,26 +515,16 @@ def base_build(bootstrap, server, intranet, group, recipe_dir,
'on %s', recipe_dir, python_version, arch)
return
- recipe = get_parsed_recipe(metadata)
+ path = get_output_path(metadata, conda_config)
- candidate = exists_on_channel(public_channels[0], recipe['package']['name'],
- recipe['package']['version'], recipe['build']['number'],
- python_version)
- if candidate is not None:
- logger.info('Skipping build for %s-%s-%s for %s - exists ' \
- 'on channel', candidate[0], candidate[1], candidate[2], arch)
+ url = exists_on_channel(public_channels[0], os.path.basename(path))
+ if url is not None:
+ logger.info('Skipping build for %s as it exists (at %s)', path, url)
return
# if you get to this point, just builds the package
- if py_ver is None:
- logger.info('Building %s-%s-%s for %s',
- recipe['package']['name'], recipe['package']['version'],
- recipe['build']['number'], arch)
- else:
- logger.info('Building %s-%s-py%s_%s for %s',
- recipe['package']['name'], recipe['package']['version'], py_ver,
- recipe['build']['number'], arch)
- conda_build.api.build(recipe_dir, config=conda_config)
+ logger.info('Building %s', path)
+ conda_build.api.build(metadata, config=conda_config)
if __name__ == '__main__':
@@ -659,20 +621,36 @@ if __name__ == '__main__':
'\n - '.join(channels + ['defaults']))
condarc_options['channels'] = channels + ['defaults']
- # retrieve the current build number for this build
- build_number, _ = next_build_number(channels[0], args.name, version,
- args.python_version)
- bootstrap.set_environment('BOB_BUILD_NUMBER', str(build_number))
-
logger.info('Merging conda configuration files...')
conda_config = make_conda_config(conda_build_config, args.python_version,
recipe_append, condarc_options)
+ metadata = get_rendered_metadata(os.path.join(args.work_dir, 'conda'),
+ conda_config)
+ path = get_output_path(metadata, conda_config)
+
+ # asserts we're building at the right location
+ assert path.startswith(os.path.join(args.conda_root, 'conda-bld')), \
+ 'Output path for build (%s) does not start with "%s" - this ' \
+ 'typically means this build is running on a shared builder and ' \
+ 'the file ~/.conda/environments.txt is polluted with other ' \
+ 'environment paths. To fix, empty that file and set its mode ' \
+ 'to read-only for all.' % (path, os.path.join(args.conda_root,
+ 'conda-bld'))
+
+ # retrieve the current build number for this build
+ build_number, _ = next_build_number(channels[0], os.path.basename(path))
+
# runs the build using the conda-build API
arch = conda_arch()
logger.info('Building %s-%s-py%s (build: %d) for %s',
args.name, version, args.python_version.replace('.',''), build_number,
arch)
+
+ # notice we cannot build from the pre-parsed metadata because it has already
+ # resolved the "wrong" build number. We'll have to reparse after setting the
+ # environment variable BOB_BUILD_NUMBER.
+ bootstrap.set_environment('BOB_BUILD_NUMBER', str(build_number))
conda_build.api.build(os.path.join(args.work_dir, 'conda'),
config=conda_config)
diff --git a/bob/devtools/data/gitlab-ci/nightlies.yaml b/bob/devtools/data/gitlab-ci/nightlies.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..1309dfdbc0c3a3fbd045838c6eae8b25e7e705a2
--- /dev/null
+++ b/bob/devtools/data/gitlab-ci/nightlies.yaml
@@ -0,0 +1,35 @@
+# This YAML file contains descriptions for the CI of nightly builds of Bob and
+# BEAT.
+
+stages:
+ - build
+
+.build_template:
+ variables:
+ CONDA_ROOT: "${CI_PROJECT_DIR}/miniconda"
+ PYTHON_VERSION: "3.6"
+ PYTHONUNBUFFERED: 1
+ stage: build
+ script:
+ - curl --silent "${BOOTSTRAP}" --output "bootstrap.py"
+ - python3 bootstrap.py -vv channel base
+ - source ${CONDA_ROOT}/etc/profile.d/conda.sh
+ - conda activate base
+ - bdt ci nightlies -vv order.txt
+ - bdt ci clean -vv
+ cache:
+ key: "$CI_JOB_NAME"
+ paths:
+ - miniconda.sh
+ - ${CONDA_ROOT}/pkgs/*.tar.bz2
+ - ${CONDA_ROOT}/pkgs/urls.txt
+
+linux:
+ extends: .build_template
+ tags:
+ - docker
+
+macosx:
+ extends: .build_template
+ tags:
+ - macosx
diff --git a/bob/devtools/scripts/build.py b/bob/devtools/scripts/build.py
index 29aef6465c1866f1dd888cf0ac6249dd1e33cb5c..059aae25a14e08b3c648a51f0962690060dcad10 100644
--- a/bob/devtools/scripts/build.py
+++ b/bob/devtools/scripts/build.py
@@ -12,7 +12,7 @@ import conda_build.api
from . import bdt
from ..build import next_build_number, conda_arch, should_skip_build, \
get_rendered_metadata, get_parsed_recipe, make_conda_config, \
- get_docserver_setup, get_env_directory
+ get_docserver_setup, get_env_directory, get_output_path
from ..constants import CONDA_BUILD_CONFIG, CONDA_RECIPE_APPEND, \
SERVER, MATPLOTLIB_RCDIR, BASE_CONDARC
from ..bootstrap import set_environment, get_channels
@@ -125,6 +125,8 @@ def build(recipe_dir, python, condarc, config, no_test, append_file,
server=server, intranet=ci, group=group)
set_environment('BOB_DOCUMENTATION_SERVER', doc_urls)
+ arch = conda_arch()
+
for d in recipe_dir:
if not os.path.exists(d):
@@ -135,24 +137,27 @@ def build(recipe_dir, python, condarc, config, no_test, append_file,
version = open(version_candidate).read().rstrip()
set_environment('BOB_PACKAGE_VERSION', version)
- # pre-renders the recipe - figures out package name and version
+ # pre-renders the recipe - figures out the destination
metadata = get_rendered_metadata(d, conda_config)
- # checks we should actually build this recipe
- arch = conda_arch()
- if should_skip_build(metadata):
- logger.warn('Skipping UNSUPPORTED build of "%s" for py%s on %s',
- d, python.replace('.',''), arch)
- return 0
-
- # converts the metadata output into parsed yaml and continues the process
rendered_recipe = get_parsed_recipe(metadata)
- # if a channel URL was passed, set the build number
- build_number, _ = next_build_number(channels[0],
- rendered_recipe['package']['name'],
- rendered_recipe['package']['version'], python)
+ path = get_output_path(metadata, conda_config)
+ # checks if we should actually build this recipe
+ if should_skip_build(metadata):
+ logger.info('Skipping UNSUPPORTED build of %s-%s-py%s for %s',
+ rendered_recipe['package']['name'],
+ rendered_recipe['package']['version'], python.replace('.',''),
+ arch)
+ continue
+
+ # gets the next build number
+ build_number, _ = next_build_number(channels[0], os.path.basename(path))
+
+ # notice we cannot build from the pre-parsed metadata because it has
+ # already resolved the "wrong" build number. We'll have to reparse after
+ # setting the environment variable BOB_BUILD_NUMBER.
set_environment('BOB_BUILD_NUMBER', str(build_number))
logger.info('Building %s-%s-py%s (build: %d) for %s',
diff --git a/bob/devtools/scripts/ci.py b/bob/devtools/scripts/ci.py
index 0a21c74106fb9022ed5e1e3d47b2732d07a3e5ab..6139da6a72625cc0105195b502801530cf0334f6 100644
--- a/bob/devtools/scripts/ci.py
+++ b/bob/devtools/scripts/ci.py
@@ -11,7 +11,7 @@ import conda_build.api
from click_plugins import with_plugins
from . import bdt
-from ..constants import SERVER
+from ..constants import SERVER, CONDA_BUILD_CONFIG, CONDA_RECIPE_APPEND
from ..log import verbosity_option, get_logger, echo_normal
logger = get_logger(__name__)
@@ -317,8 +317,7 @@ Examples:
''')
@click.argument('order', required=True, type=click.Path(file_okay=True,
dir_okay=False, exists=True), nargs=1)
-@click.option('-g', '--group', show_default=True,
- default=os.environ['CI_PROJECT_NAMESPACE'],
+@click.option('-g', '--group', show_default=True, default='bob',
help='Group of packages (gitlab namespace) this package belongs to')
@click.option('-p', '--python', multiple=True,
help='Versions of python in the format "x.y" we should build for. Pass ' \
@@ -337,8 +336,6 @@ def base_build(order, group, python, dry_run):
this context.
"""
- from ..constants import CONDA_BUILD_CONFIG
-
condarc = os.path.join(os.environ['CONDA_ROOT'], 'condarc')
logger.info('Loading (this build\'s) CONDARC file from %s...', condarc)
with open(condarc, 'rb') as f:
@@ -400,8 +397,6 @@ def test(ctx, dry_run):
to be used outside this context.
"""
- from ..constants import CONDA_BUILD_CONFIG, CONDA_RECIPE_APPEND
-
group = os.environ['CI_PROJECT_NAMESPACE']
if group not in ('bob', 'beat'):
# defaults back to bob - no other server setups are available as of now
@@ -445,8 +440,6 @@ def build(ctx, dry_run):
to be used outside this context.
"""
- from ..constants import CONDA_BUILD_CONFIG, CONDA_RECIPE_APPEND
-
group = os.environ['CI_PROJECT_NAMESPACE']
if group not in ('bob', 'beat'):
# defaults back to bob - no other server setups are available as of now
@@ -491,3 +484,95 @@ def clean(ctx):
from ..bootstrap import run_cmdline
git_clean_build(run_cmdline, verbose=(ctx.meta['verbosity']>=3))
+
+
+@ci.command(epilog='''
+Examples:
+
+ 1. Runs the nightly builds following a list of packages in a file:
+
+ $ bdt ci nightlies -vv order.txt
+
+''')
+@click.argument('order', required=True, type=click.Path(file_okay=True,
+ dir_okay=False, exists=True), nargs=1)
+@click.option('-d', '--dry-run/--no-dry-run', default=False,
+ help='Only goes through the actions, but does not execute them ' \
+ '(combine with the verbosity flags - e.g. ``-vvv``) to enable ' \
+ 'printing to help you understand what will be done')
+@verbosity_option()
+@bdt.raise_on_error
+@click.pass_context
+def nightlies(ctx, order, dry_run):
+ """Runs nightly builds
+
+ This command can run nightly builds for packages listed on a file.
+
+ The build or each package happens in a few phases:
+
+ 1. Package is checked out and switched to the requested branch (master if not
+ set otherwise)
+ 2. A build string is calculated from current dependencies. If the package
+ has already been compiled, it is downloaded from the respective conda
+ channel and tested. If the test does not pass, the package is completely
+ rebuilt
+ 3. If the rebuild is successful, the new package is uploaded to the
+ respective conda channel, and the program continues with the next package
+
+ Dependencies are searched with priority to locally built packages. For this
+ reason, the input file **must** be provided in the right dependence order.
+ """
+
+ # loads dirnames from order file (accepts # comments and empty lines)
+ packages = []
+ with open(order, 'rt') as f:
+ for line in f:
+ line = line.partition('#')[0].strip()
+ if line:
+ if ',' in line: #user specified a branch
+ path, branch = [k.strip() for k in line.split(',', 1)]
+ packages.append((path, branch))
+ else:
+ packages.apend((line, 'master'))
+
+ import git
+ from .rebuild import rebuild
+ from urllib.request import urlopen
+
+ # loaded all recipes, now cycle through them implementing what is described
+ # in the documentation of this function
+ for n, (package, branch) in enumerate(packages):
+
+ echo_normal('\n' + (80*'='))
+ echo_normal('Testing/Re-building %s@%s (%d/%d)' % (package, branch, n+1,
+ len(packages))
+ echo_normal((80*'=') + '\n')
+
+ group, name = package.split('/', 1)
+
+ clone_to = os.path.join(os.environ['CI_PROJECT_DIR'], 'src', group, name)
+ dirname = os.path.dirname(clone_to)
+ if not os.path.exists(dirname):
+ os.makedirs(dirname)
+
+ # clone the repo, shallow version, on the specified branch
+ logger.info('Cloning "%s", branch "%s" (depth=1)...', package, branch)
+ git.Repo.clone_from('https://gitlab-ci-token:%s@gitlab.idiap.ch/%s' % \
+ (token, package), clone_to, branch=branch, depth=1)
+
+ # determine package visibility
+ private = urlopen('https://gitlab.idiap.ch/%s' % package).getcode() != 200
+
+ ctx.invoke(rebuild,
+ recipe_dir=[os.path.join(clone_to, 'conda')],
+ python=os.environ['PYTHON_VERSION'], #python version
+ condarc=None, #custom build configuration
+ config=CONDA_BUILD_CONFIG,
+ append_file=CONDA_RECIPE_APPEND,
+ server=SERVER,
+ group=group,
+ private=private,
+ stable='STABLE' in os.environ,
+ dry_run=dry_run,
+ ci=True,
+ )
diff --git a/bob/devtools/scripts/rebuild.py b/bob/devtools/scripts/rebuild.py
new file mode 100644
index 0000000000000000000000000000000000000000..4187b1950f94e0f4f81501f695d0f10de0a536c5
--- /dev/null
+++ b/bob/devtools/scripts/rebuild.py
@@ -0,0 +1,192 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+import sys
+import urllib.request
+
+import yaml
+import click
+import pkg_resources
+import conda_build.api
+
+from . import bdt
+from ..build import next_build_number, conda_arch, should_skip_build, \
+ get_rendered_metadata, get_parsed_recipe, make_conda_config, \
+ get_docserver_setup, get_env_directory, get_output_path
+from ..constants import CONDA_BUILD_CONFIG, CONDA_RECIPE_APPEND, \
+ SERVER, MATPLOTLIB_RCDIR, BASE_CONDARC
+from ..bootstrap import set_environment, get_channels
+
+from ..log import verbosity_option, get_logger, echo_normal
+logger = get_logger(__name__)
+
+
+@click.command(epilog='''
+Examples:
+ 1. Rebuilds a recipe from one of our packages, checked-out at "bob/bob.extension", for python 3.6:
+
+\b
+ $ bdt rebuild -vv --python=3.6 bob/bob.extension/conda
+
+
+ 2. To rebuild multiple recipes, just pass the paths to them:
+
+\b
+ $ bdt rebuild -vv --python=3.6 path/to/recipe-dir1 path/to/recipe-dir2
+''')
+@click.argument('recipe-dir', required=False, type=click.Path(file_okay=False,
+ dir_okay=True, exists=True), nargs=-1)
+@click.option('-p', '--python', default=('%d.%d' % sys.version_info[:2]),
+ show_default=True, help='Version of python to build the environment for')
+@click.option('-r', '--condarc',
+ help='Use custom conda configuration file instead of our own',)
+@click.option('-m', '--config', '--variant-config-files', show_default=True,
+ default=CONDA_BUILD_CONFIG, help='overwrites the path leading to ' \
+ 'variant configuration file to use')
+@click.option('-a', '--append-file', show_default=True,
+ default=CONDA_RECIPE_APPEND, help='overwrites the path leading to ' \
+ 'appended configuration file to use')
+@click.option('-S', '--server', show_default=True, default=SERVER,
+ help='Server used for downloading conda packages and documentation ' \
+ 'indexes of required packages')
+@click.option('-g', '--group', show_default=True, default='bob',
+ help='Group of packages (gitlab namespace) this package belongs to')
+@click.option('-P', '--private/--no-private', default=False,
+ help='Set this to **include** private channels on your build - ' \
+ 'you **must** be at Idiap to execute this build in this case - ' \
+ 'you **must** also use the correct server name through --server - ' \
+ 'notice this option has no effect to conda if you also pass --condarc')
+@click.option('-X', '--stable/--no-stable', default=False,
+ help='Set this to **exclude** beta channels from your build - ' \
+ 'notice this option has no effect if you also pass --condarc')
+@click.option('-d', '--dry-run/--no-dry-run', default=False,
+ help='Only goes through the actions, but does not execute them ' \
+ '(combine with the verbosity flags - e.g. ``-vvv``) to enable ' \
+ 'printing to help you understand what will be done')
+@click.option('-C', '--ci/--no-ci', default=False, hidden=True,
+ help='Use this flag to indicate the build will be running on the CI')
+@verbosity_option()
+@bdt.raise_on_error
+def rebuild(recipe_dir, python, condarc, config, append_file,
+ server, group, private, stable, dry_run, ci):
+ """Tests and rebuilds packages through conda-build with stock configuration
+
+ This command wraps the execution of conda-build in two stages: first, from
+ the original package recipe and some channel look-ups, it figures out what is
+ the lastest version of the package available. It downloads such file and
+ runs a test. If the test suceeds, then it proceeds to the next recipe.
+ Otherwise, it rebuilds the package and uploads a new version to the channel.
+ """
+
+ # if we are in a dry-run mode, let's let it be known
+ if dry_run:
+ logger.warn('!!!! DRY RUN MODE !!!!')
+ logger.warn('Nothing will be really built')
+
+ recipe_dir = recipe_dir or [os.path.join(os.path.realpath('.'), 'conda')]
+
+ logger.debug('This package is considered part of group "%s" - tunning ' \
+ 'conda package and documentation URLs for this...', group)
+
+ # get potential channel upload and other auxiliary channels
+ channels = get_channels(public=(not private), stable=stable, server=server,
+ intranet=ci, group=group)
+
+ if condarc is not None:
+ logger.info('Loading CONDARC file from %s...', condarc)
+ with open(condarc, 'rb') as f:
+ condarc_options = yaml.load(f)
+ else:
+ # use default and add channels
+ condarc_options = yaml.load(BASE_CONDARC) #n.b.: no channels
+ logger.info('Using the following channels during build:\n - %s',
+ '\n - '.join(channels + ['defaults']))
+ condarc_options['channels'] = channels + ['defaults']
+
+ # dump packages at base environment
+ prefix = get_env_directory(os.environ['CONDA_EXE'], 'base')
+ condarc_options['croot'] = os.path.join(prefix, 'conda-bld')
+
+ conda_config = make_conda_config(config, python, append_file,
+ condarc_options)
+
+ set_environment('MATPLOTLIBRC', MATPLOTLIB_RCDIR)
+
+ # setup BOB_DOCUMENTATION_SERVER environment variable (used for bob.extension
+ # and derived documentation building via Sphinx)
+ set_environment('DOCSERVER', server)
+ doc_urls = get_docserver_setup(public=(not private), stable=stable,
+ server=server, intranet=ci, group=group)
+ set_environment('BOB_DOCUMENTATION_SERVER', doc_urls)
+
+ arch = conda_arch()
+
+ for d in recipe_dir:
+
+ if not os.path.exists(d):
+ raise RuntimeError("The directory %s does not exist" % recipe_dir)
+
+ version_candidate = os.path.join(d, '..', 'version.txt')
+ if os.path.exists(version_candidate):
+ version = open(version_candidate).read().rstrip()
+ set_environment('BOB_PACKAGE_VERSION', version)
+
+ # pre-renders the recipe - figures out the destination
+ metadata = get_rendered_metadata(d, conda_config)
+
+ rendered_recipe = get_parsed_recipe(metadata)
+
+ path = get_output_path(metadata, conda_config)
+
+ # checks if we should actually build this recipe
+ if should_skip_build(metadata):
+ logger.info('Skipping UNSUPPORTED build of %s-%s-py%s for %s',
+ rendered_recipe['package']['name'],
+ rendered_recipe['package']['version'], python.replace('.',''),
+ arch)
+ continue
+
+ # Get the latest build number
+ build_number, existing = next_build_number(channels[0],
+ os.path.basename(path))
+
+ should_build = True
+
+ if existing: #other builds exist, get the latest and see if it still works
+
+ destpath = os.path.join(condarc_options['croot'], arch,
+ os.path.basename(existing[0]))
+ logger.info('Downloading %s -> %s', existing[0], destpath)
+ urllib.request.urlretrieve(existing[0], destpath)
+
+ try:
+ logger.info('Testing %s', existing[0])
+ conda_build.api.test(destpath, config=conda_config)
+ should_build = False
+ logger.info('Test for %s: SUCCESS', existing[0])
+ except Exception as error:
+ logger.exception(error)
+ logger.warn('Test for %s: FAILED. Building...', existing[0])
+
+
+ if should_build: #something wrong happened, run a full build
+
+ logger.info('Building %s-%s-py%s (build: %d) for %s',
+ rendered_recipe['package']['name'],
+ rendered_recipe['package']['version'], python.replace('.',''),
+ build_number, arch)
+
+ # notice we cannot build from the pre-parsed metadata because it has
+ # already resolved the "wrong" build number. We'll have to reparse after
+ # setting the environment variable BOB_BUILD_NUMBER.
+ set_environment('BOB_BUILD_NUMBER', str(build_number))
+
+ if not dry_run:
+ conda_build.api.build(d, config=conda_config, notest=no_test)
+
+ else: #skip build, test worked
+ logger.info('Skipping build of %s-%s-py%s (build: %d) for %s',
+ rendered_recipe['package']['name'],
+ rendered_recipe['package']['version'], python.replace('.',''),
+ build_number, arch)
diff --git a/conda/meta.yaml b/conda/meta.yaml
index f5fdc7b3731619c9a812b8246d213fedfaa62f76..6424fb310e0c0e5c16240c3a894264d8b454cce5 100644
--- a/conda/meta.yaml
+++ b/conda/meta.yaml
@@ -59,6 +59,7 @@ test:
- bdt dumpsphinx https://docs.python.org/3/objects.inv > /dev/null
- bdt create --help
- bdt build --help
+ - bdt rebuild --help
- bdt test --help
- bdt caupdate --help
- bdt new --help
@@ -87,6 +88,7 @@ test:
- bdt ci pypi --help
- bdt ci readme --help
- bdt ci clean --help
+ - bdt ci nightlies --help
- sphinx-build -aEW ${PREFIX}/share/doc/{{ name }}/doc sphinx
- if [ -n "${CI_PROJECT_DIR}" ]; then mv sphinx "${CI_PROJECT_DIR}/"; fi
diff --git a/setup.py b/setup.py
index 2e852fa1440d33e3c4c3c188c450cb6c018bdb05..6866872593d60f48862aabe7fc63edc8926075b3 100644
--- a/setup.py
+++ b/setup.py
@@ -50,6 +50,7 @@ setup(
'dumpsphinx = bob.devtools.scripts.dumpsphinx:dumpsphinx',
'create = bob.devtools.scripts.create:create',
'build = bob.devtools.scripts.build:build',
+ 'rebuild = bob.devtools.scripts.rebuild:rebuild',
'test = bob.devtools.scripts.test:test',
'caupdate = bob.devtools.scripts.caupdate:caupdate',
'ci = bob.devtools.scripts.ci:ci',
@@ -76,6 +77,7 @@ setup(
'deploy = bob.devtools.scripts.ci:deploy',
'readme = bob.devtools.scripts.ci:readme',
'pypi = bob.devtools.scripts.ci:pypi',
+ 'nightlies = bob.devtools.scripts.ci:nightlies',
],
},