Merge branch 'nightlies' into 'master'

Nightlies

See merge request !28

bob/devtools/build.py

@@ -48,58 +48,54 @@ def should_skip_build(metadata_tuples):
   return all(m[0].skip() for m in metadata_tuples)
 
 
-def next_build_number(channel_url, name, version, python):
+def next_build_number(channel_url, basename):
   """Calculates the next build number of a package given the channel
 
   This function returns the next build number (integer) for a package given its
-  recipe, dependencies, name, version and python version.  It looks on the
-  channel URL provided and figures out if any clash would happen and what would
-  be the highest build number available for that configuration.
+  resulting tarball base filename (can be obtained with
+  :py:func:`get_output_path`).
 
 
   Args:
 
     channel_url: The URL where to look for packages clashes (normally a beta
       channel)
-    name: The name of the package
-    version: The version of the package
-    python: The version of python as 2 digits (e.g.: "2.7" or "3.6")
+    basename: The tarball basename to check on the channel
 
   Returns: The next build number with the current configuration.  Zero (0) is
   returned if no match is found.  Also returns the URLs of the packages it
-  finds with matches on the name, version and python-version.
+  finds with matches on the name, version and python-version, ordered by
+  (reversed) build-number.
 
   """
 
   from conda.exports import get_index
 
-  # no dot in py_ver
-  py_ver = python.replace('.', '')
-
   # get the channel index
   logger.debug('Downloading channel index from %s', channel_url)
   index = get_index(channel_urls=[channel_url], prepend=False)
 
-  # search if package with the same version exists
-  build_number = 0
-  urls = []
-  for dist in index:
+  # remove .tar.bz2 from name, then split from the end twice, on '-'
+  name, version, build = basename[:-8].rsplit('-', 2)
 
-    if dist.name == name and dist.version == version:
-      if py_ver:
-        match = re.match('py[2-9][0-9]+', dist.build_string)
-      else:
-        match = re.match('py', dist.build_string)
+  # remove the build number as we're looking for the next value
+  build_variant = build.rsplit('_', 1)[0]
 
-      if match and match.group() == 'py{}'.format(py_ver):
-        logger.debug("Found match at %s for %s-%s-py%s", index[dist].url,
-            name, version, py_ver)
-        build_number = max(build_number, dist.build_number + 1)
-        urls.append(index[dist].url)
+  # search if package with the same characteristics
+  urls = {}
+  build_number = 0
+  for dist in index:
+    if dist.name == name and dist.version == version and \
+        dist.build_string.startswith(build_variant):  #match!
+      url = index[dist].url
+      logger.debug("Found match at %s for %s-%s-%s", url,
+          name, version, build_variant)
+      build_number = max(build_number, dist.build_number + 1)
+      urls[dist.build_number] = url.replace(channel_url, '')
 
-  urls = [url.replace(channel_url, '') for url in urls]
+  sorted_urls = [urls[k] for k in reversed(list(urls.keys()))]
 
-  return build_number, urls
+  return build_number, sorted_urls
 
 
 def make_conda_config(config, python, append_file, condarc_options):
@@ -142,6 +138,13 @@ def make_conda_config(config, python, append_file, condarc_options):
   return retval
 
 
+def get_output_path(metadata, config):
+  '''Renders the recipe and returns the interpreted YAML file'''
+
+  from conda_build.api import get_output_file_paths
+  return get_output_file_paths(metadata, config=config)[0]
+
+
 def get_rendered_metadata(recipe_dir, config):
   '''Renders the recipe and returns the interpreted YAML file'''
 
@@ -157,66 +160,35 @@ def get_parsed_recipe(metadata):
   return yaml.load(output)
 
 
-def exists_on_channel(channel_url, name, version, build_number,
-    python_version):
+def exists_on_channel(channel_url, basename):
   """Checks on the given channel if a package with the specs exist
 
   Args:
 
     channel_url: The URL where to look for packages clashes (normally a beta
       channel)
-    name: The name of the package
-    version: The version of the package
-    build_number: The build number of the package
-    python_version: The current version of python we're building for.  May be
-      ``noarch``, to check for "noarch" packages or ``None``, in which case we
-      don't check for the python version
+    basename: The basename of the tarball to search for
 
-  Returns: A complete package name, version and build string, if the package
-  already exists in the channel or ``None`` otherwise.
+  Returns: A complete package url, if the package already exists in the channel
+  or ``None`` otherwise.
 
   """
 
   from conda.exports import get_index
 
-  # handles different cases as explained on the description of
-  # ``python_version``
-  py_ver = python_version.replace('.', '') if python_version else None
-  if py_ver == 'noarch': py_ver = ''
-
   # get the channel index
   logger.debug('Downloading channel index from %s', channel_url)
   index = get_index(channel_urls=[channel_url], prepend=False)
 
-  logger.info('Checking for %s-%s-%s...', name, version, build_number)
+  logger.info('Checking for %s...', basename)
 
   for dist in index:
+    url = index[dist].url
+    if url.endswith(basename):
+      logger.debug('Found matching package (%s) at %s', basename, url)
+      return url
 
-    if dist.name == name and dist.version == version and \
-        dist.build_string.endswith('_%s' % build_number):
-
-      # two possible options must be checked - (i) the package build_string
-      # starts with ``py``, which means it is a python specific package so we
-      # must also check for the matching python version.  (ii) the package is
-      # not a python-specific package and a simple match will do
-      if dist.build_string.startswith('py'):
-        match = re.match('py[2-9][0-9]+', dist.build_string)
-        if match and match.group() == 'py{}'.format(py_ver):
-          logger.debug('Found matching package (%s-%s-%s)', dist.name,
-              dist.version, dist.build_string)
-          return (dist.name, dist.version, dist.build_string)
-
-      else:
-        logger.debug('Found matching package (%s-%s-%s)', dist.name,
-            dist.version, dist.build_string)
-        return (dist.name, dist.version, dist.build_string)
-
-  if py_ver is None:
-    logger.info('No matches for %s-%s-%s found among %d packages',
-        name, version, build_number, len(index))
-  else:
-    logger.info('No matches for %s-%s-py%s_%s found among %d packages',
-        name, version, py_ver, build_number, len(index))
+  logger.debug('No matches for %s', path)
   return
 
 
@@ -543,26 +515,16 @@ def base_build(bootstrap, server, intranet, group, recipe_dir,
           'on %s', recipe_dir, python_version, arch)
     return
 
-  recipe = get_parsed_recipe(metadata)
+  path = get_output_path(metadata, conda_config)
 
-  candidate = exists_on_channel(public_channels[0], recipe['package']['name'],
-      recipe['package']['version'], recipe['build']['number'],
-      python_version)
-  if candidate is not None:
-    logger.info('Skipping build for %s-%s-%s for %s - exists ' \
-        'on channel', candidate[0], candidate[1], candidate[2], arch)
+  url = exists_on_channel(public_channels[0], os.path.basename(path))
+  if url is not None:
+    logger.info('Skipping build for %s as it exists (at %s)', path, url)
     return
 
   # if you get to this point, just builds the package
-  if py_ver is None:
-    logger.info('Building %s-%s-%s for %s',
-      recipe['package']['name'], recipe['package']['version'],
-      recipe['build']['number'], arch)
-  else:
-    logger.info('Building %s-%s-py%s_%s for %s',
-      recipe['package']['name'], recipe['package']['version'], py_ver,
-      recipe['build']['number'], arch)
-  conda_build.api.build(recipe_dir, config=conda_config)
+  logger.info('Building %s', path)
+  conda_build.api.build(metadata, config=conda_config)
 
 
 if __name__ == '__main__':
@@ -659,20 +621,36 @@ if __name__ == '__main__':
       '\n  - '.join(channels + ['defaults']))
   condarc_options['channels'] = channels + ['defaults']
 
-  # retrieve the current build number for this build
-  build_number, _ = next_build_number(channels[0], args.name, version,
-      args.python_version)
-  bootstrap.set_environment('BOB_BUILD_NUMBER', str(build_number))
-
   logger.info('Merging conda configuration files...')
   conda_config = make_conda_config(conda_build_config, args.python_version,
       recipe_append, condarc_options)
 
+  metadata = get_rendered_metadata(os.path.join(args.work_dir, 'conda'),
+      conda_config)
+  path = get_output_path(metadata, conda_config)
+
+  # asserts we're building at the right location
+  assert path.startswith(os.path.join(args.conda_root, 'conda-bld')), \
+      'Output path for build (%s) does not start with "%s" - this ' \
+      'typically means this build is running on a shared builder and ' \
+      'the file ~/.conda/environments.txt is polluted with other ' \
+      'environment paths.  To fix, empty that file and set its mode ' \
+      'to read-only for all.' % (path, os.path.join(args.conda_root,
+        'conda-bld'))
+
+  # retrieve the current build number for this build
+  build_number, _ = next_build_number(channels[0], os.path.basename(path))
+
   # runs the build using the conda-build API
   arch = conda_arch()
   logger.info('Building %s-%s-py%s (build: %d) for %s',
       args.name, version, args.python_version.replace('.',''), build_number,
       arch)
+
+  # notice we cannot build from the pre-parsed metadata because it has already
+  # resolved the "wrong" build number.  We'll have to reparse after setting the
+  # environment variable BOB_BUILD_NUMBER.
+  bootstrap.set_environment('BOB_BUILD_NUMBER', str(build_number))
   conda_build.api.build(os.path.join(args.work_dir, 'conda'),
       config=conda_config)
 

bob/devtools/data/gitlab-ci/nightlies.yaml

+# This YAML file contains descriptions for the CI of nightly builds of Bob and
+# BEAT.
+
+stages:
+  - build
+
+.build_template:
+  variables:
+    CONDA_ROOT: "${CI_PROJECT_DIR}/miniconda"
+    PYTHON_VERSION: "3.6"
+    PYTHONUNBUFFERED: 1
+  stage: build
+    script:
+    - curl --silent "${BOOTSTRAP}" --output "bootstrap.py"
+    - python3 bootstrap.py -vv channel base
+    - source ${CONDA_ROOT}/etc/profile.d/conda.sh
+    - conda activate base
+    - bdt ci nightlies -vv order.txt
+    - bdt ci clean -vv
+  cache:
+    key: "$CI_JOB_NAME"
+    paths:
+      - miniconda.sh
+      - ${CONDA_ROOT}/pkgs/*.tar.bz2
+      - ${CONDA_ROOT}/pkgs/urls.txt
+
+linux:
+  extends: .build_template
+  tags:
+    - docker
+
+macosx:
+  extends: .build_template
+  tags:
+    - macosx

bob/devtools/scripts/build.py

@@ -12,7 +12,7 @@ import conda_build.api
 from . import bdt
 from ..build import next_build_number, conda_arch, should_skip_build, \
     get_rendered_metadata, get_parsed_recipe, make_conda_config, \
-    get_docserver_setup, get_env_directory
+    get_docserver_setup, get_env_directory, get_output_path
 from ..constants import CONDA_BUILD_CONFIG, CONDA_RECIPE_APPEND, \
     SERVER, MATPLOTLIB_RCDIR, BASE_CONDARC
 from ..bootstrap import set_environment, get_channels
@@ -125,6 +125,8 @@ def build(recipe_dir, python, condarc, config, no_test, append_file,
       server=server, intranet=ci, group=group)
   set_environment('BOB_DOCUMENTATION_SERVER', doc_urls)
 
+  arch = conda_arch()
+
   for d in recipe_dir:
 
     if not os.path.exists(d):
@@ -135,24 +137,27 @@ def build(recipe_dir, python, condarc, config, no_test, append_file,
       version = open(version_candidate).read().rstrip()
       set_environment('BOB_PACKAGE_VERSION', version)
 
-    # pre-renders the recipe - figures out package name and version
+    # pre-renders the recipe - figures out the destination
     metadata = get_rendered_metadata(d, conda_config)
 
-    # checks we should actually build this recipe
-    arch = conda_arch()
-    if should_skip_build(metadata):
-      logger.warn('Skipping UNSUPPORTED build of "%s" for py%s on %s',
-          d, python.replace('.',''), arch)
-      return 0
-
-    # converts the metadata output into parsed yaml and continues the process
     rendered_recipe = get_parsed_recipe(metadata)
 
-    # if a channel URL was passed, set the build number
-    build_number, _ = next_build_number(channels[0],
-        rendered_recipe['package']['name'],
-        rendered_recipe['package']['version'], python)
+    path = get_output_path(metadata, conda_config)
 
+    # checks if we should actually build this recipe
+    if should_skip_build(metadata):
+      logger.info('Skipping UNSUPPORTED build of %s-%s-py%s for %s',
+          rendered_recipe['package']['name'],
+          rendered_recipe['package']['version'], python.replace('.',''),
+          arch)
+      continue
+
+    # gets the next build number
+    build_number, _ = next_build_number(channels[0], os.path.basename(path))
+
+    # notice we cannot build from the pre-parsed metadata because it has
+    # already resolved the "wrong" build number.  We'll have to reparse after
+    # setting the environment variable BOB_BUILD_NUMBER.
     set_environment('BOB_BUILD_NUMBER', str(build_number))
 
     logger.info('Building %s-%s-py%s (build: %d) for %s',

bob/devtools/scripts/ci.py

@@ -11,7 +11,7 @@ import conda_build.api
 from click_plugins import with_plugins
 
 from . import bdt
-from ..constants import SERVER
+from ..constants import SERVER, CONDA_BUILD_CONFIG, CONDA_RECIPE_APPEND
 
 from ..log import verbosity_option, get_logger, echo_normal
 logger = get_logger(__name__)
@@ -317,8 +317,7 @@ Examples:
 ''')
 @click.argument('order', required=True, type=click.Path(file_okay=True,
   dir_okay=False, exists=True), nargs=1)
-@click.option('-g', '--group', show_default=True,
-    default=os.environ['CI_PROJECT_NAMESPACE'],
+@click.option('-g', '--group', show_default=True, default='bob',
     help='Group of packages (gitlab namespace) this package belongs to')
 @click.option('-p', '--python', multiple=True,
     help='Versions of python in the format "x.y" we should build for.  Pass ' \
@@ -337,8 +336,6 @@ def base_build(order, group, python, dry_run):
   this context.
   """
 
-  from ..constants import CONDA_BUILD_CONFIG
-
   condarc = os.path.join(os.environ['CONDA_ROOT'], 'condarc')
   logger.info('Loading (this build\'s) CONDARC file from %s...', condarc)
   with open(condarc, 'rb') as f:
@@ -400,8 +397,6 @@ def test(ctx, dry_run):
   to be used outside this context.
   """
 
-  from ..constants import CONDA_BUILD_CONFIG, CONDA_RECIPE_APPEND
-
   group = os.environ['CI_PROJECT_NAMESPACE']
   if group not in ('bob', 'beat'):
     # defaults back to bob - no other server setups are available as of now
@@ -445,8 +440,6 @@ def build(ctx, dry_run):
   to be used outside this context.
   """
 
-  from ..constants import CONDA_BUILD_CONFIG, CONDA_RECIPE_APPEND
-
   group = os.environ['CI_PROJECT_NAMESPACE']
   if group not in ('bob', 'beat'):
     # defaults back to bob - no other server setups are available as of now
@@ -491,3 +484,95 @@ def clean(ctx):
   from ..bootstrap import run_cmdline
 
   git_clean_build(run_cmdline, verbose=(ctx.meta['verbosity']>=3))
+
+
+@ci.command(epilog='''
+Examples:
+
+  1. Runs the nightly builds following a list of packages in a file:
+
+     $ bdt ci nightlies -vv order.txt
+
+''')
+@click.argument('order', required=True, type=click.Path(file_okay=True,
+  dir_okay=False, exists=True), nargs=1)
+@click.option('-d', '--dry-run/--no-dry-run', default=False,
+    help='Only goes through the actions, but does not execute them ' \
+        '(combine with the verbosity flags - e.g. ``-vvv``) to enable ' \
+        'printing to help you understand what will be done')
+@verbosity_option()
+@bdt.raise_on_error
+@click.pass_context
+def nightlies(ctx, order, dry_run):
+  """Runs nightly builds
+
+  This command can run nightly builds for packages listed on a file.
+
+  The build or each package happens in a few phases:
+
+  1. Package is checked out and switched to the requested branch (master if not
+     set otherwise)
+  2. A build string is calculated from current dependencies.  If the package
+     has already been compiled, it is downloaded from the respective conda
+     channel and tested.  If the test does not pass, the package is completely
+     rebuilt
+  3. If the rebuild is successful, the new package is uploaded to the
+     respective conda channel, and the program continues with the next package
+
+  Dependencies are searched with priority to locally built packages.  For this
+  reason, the input file **must** be provided in the right dependence order.
+  """
+
+  # loads dirnames from order file (accepts # comments and empty lines)
+  packages = []
+  with open(order, 'rt') as f:
+    for line in f:
+      line = line.partition('#')[0].strip()
+      if line:
+        if ',' in line:  #user specified a branch
+          path, branch = [k.strip() for k in line.split(',', 1)]
+          packages.append((path, branch))
+        else:
+          packages.apend((line, 'master'))
+
+  import git
+  from .rebuild import rebuild
+  from urllib.request import urlopen
+
+  # loaded all recipes, now cycle through them implementing what is described
+  # in the documentation of this function
+  for n, (package, branch) in enumerate(packages):
+
+    echo_normal('\n' + (80*'='))
+    echo_normal('Testing/Re-building %s@%s (%d/%d)' % (package, branch, n+1,
+      len(packages))
+    echo_normal((80*'=') + '\n')
+
+    group, name = package.split('/', 1)
+
+    clone_to = os.path.join(os.environ['CI_PROJECT_DIR'], 'src', group, name)
+    dirname = os.path.dirname(clone_to)
+    if not os.path.exists(dirname):
+      os.makedirs(dirname)
+
+    # clone the repo, shallow version, on the specified branch
+    logger.info('Cloning "%s", branch "%s" (depth=1)...', package, branch)
+    git.Repo.clone_from('https://gitlab-ci-token:%s@gitlab.idiap.ch/%s' % \
+        (token, package), clone_to, branch=branch, depth=1)
+
+    # determine package visibility
+    private = urlopen('https://gitlab.idiap.ch/%s' % package).getcode() != 200
+
+    ctx.invoke(rebuild,
+        recipe_dir=[os.path.join(clone_to, 'conda')],
+        python=os.environ['PYTHON_VERSION'],  #python version
+        condarc=None,  #custom build configuration
+        config=CONDA_BUILD_CONFIG,
+        append_file=CONDA_RECIPE_APPEND,
+        server=SERVER,
+        group=group,
+        private=private,
+        stable='STABLE' in os.environ,
+        dry_run=dry_run,
+        ci=True,
+        )

bob/devtools/scripts/rebuild.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+import sys
+import urllib.request
+
+import yaml
+import click
+import pkg_resources
+import conda_build.api
+
+from . import bdt
+from ..build import next_build_number, conda_arch, should_skip_build, \
+    get_rendered_metadata, get_parsed_recipe, make_conda_config, \
+    get_docserver_setup, get_env_directory, get_output_path
+from ..constants import CONDA_BUILD_CONFIG, CONDA_RECIPE_APPEND, \
+    SERVER, MATPLOTLIB_RCDIR, BASE_CONDARC
+from ..bootstrap import set_environment, get_channels
+
+from ..log import verbosity_option, get_logger, echo_normal
+logger = get_logger(__name__)
+
+
+@click.command(epilog='''
+Examples:
+  1. Rebuilds a recipe from one of our packages, checked-out at "bob/bob.extension", for python 3.6:
+
+\b
+     $ bdt rebuild -vv --python=3.6 bob/bob.extension/conda
+
+
+  2. To rebuild multiple recipes, just pass the paths to them:
+
+\b
+     $ bdt rebuild -vv --python=3.6 path/to/recipe-dir1 path/to/recipe-dir2
+''')
+@click.argument('recipe-dir', required=False, type=click.Path(file_okay=False,
+  dir_okay=True, exists=True), nargs=-1)
+@click.option('-p', '--python', default=('%d.%d' % sys.version_info[:2]),
+    show_default=True, help='Version of python to build the environment for')
+@click.option('-r', '--condarc',
+    help='Use custom conda configuration file instead of our own',)
+@click.option('-m', '--config', '--variant-config-files', show_default=True,
+    default=CONDA_BUILD_CONFIG, help='overwrites the path leading to ' \
+        'variant configuration file to use')
+@click.option('-a', '--append-file', show_default=True,
+    default=CONDA_RECIPE_APPEND, help='overwrites the path leading to ' \
+        'appended configuration file to use')
+@click.option('-S', '--server', show_default=True, default=SERVER,
+    help='Server used for downloading conda packages and documentation ' \
+        'indexes of required packages')
+@click.option('-g', '--group', show_default=True, default='bob',
+    help='Group of packages (gitlab namespace) this package belongs to')
+@click.option('-P', '--private/--no-private', default=False,
+    help='Set this to **include** private channels on your build - ' \
+        'you **must** be at Idiap to execute this build in this case - ' \
+        'you **must** also use the correct server name through --server - ' \
+        'notice this option has no effect to conda if you also pass --condarc')
+@click.option('-X', '--stable/--no-stable', default=False,
+    help='Set this to **exclude** beta channels from your build - ' \
+        'notice this option has no effect if you also pass --condarc')
+@click.option('-d', '--dry-run/--no-dry-run', default=False,
+    help='Only goes through the actions, but does not execute them ' \
+        '(combine with the verbosity flags - e.g. ``-vvv``) to enable ' \
+        'printing to help you understand what will be done')
+@click.option('-C', '--ci/--no-ci', default=False, hidden=True,
+    help='Use this flag to indicate the build will be running on the CI')
+@verbosity_option()
+@bdt.raise_on_error
+def rebuild(recipe_dir, python, condarc, config, append_file,
+    server, group, private, stable, dry_run, ci):
+  """Tests and rebuilds packages through conda-build with stock configuration
+
+  This command wraps the execution of conda-build in two stages: first, from
+  the original package recipe and some channel look-ups, it figures out what is
+  the lastest version of the package available.  It downloads such file and
+  runs a test.  If the test suceeds, then it proceeds to the next recipe.
+  Otherwise, it rebuilds the package and uploads a new version to the channel.
+  """
+
+  # if we are in a dry-run mode, let's let it be known
+  if dry_run:
+      logger.warn('!!!! DRY RUN MODE !!!!')
+      logger.warn('Nothing will be really built')
+
+  recipe_dir = recipe_dir or [os.path.join(os.path.realpath('.'), 'conda')]
+
+  logger.debug('This package is considered part of group "%s" - tunning ' \
+      'conda package and documentation URLs for this...', group)
+
+  # get potential channel upload and other auxiliary channels
+  channels = get_channels(public=(not private), stable=stable, server=server,
+      intranet=ci, group=group)
+
+  if condarc is not None:
+    logger.info('Loading CONDARC file from %s...', condarc)
+    with open(condarc, 'rb') as f:
+      condarc_options = yaml.load(f)
+  else:
+    # use default and add channels
+    condarc_options = yaml.load(BASE_CONDARC)  #n.b.: no channels
+    logger.info('Using the following channels during build:\n  - %s',
+        '\n  - '.join(channels + ['defaults']))
+    condarc_options['channels'] = channels + ['defaults']
+
+  # dump packages at base environment
+  prefix = get_env_directory(os.environ['CONDA_EXE'], 'base')
+  condarc_options['croot'] = os.path.join(prefix, 'conda-bld')
+
+  conda_config = make_conda_config(config, python, append_file,
+      condarc_options)
+
+  set_environment('MATPLOTLIBRC', MATPLOTLIB_RCDIR)
+
+  # setup BOB_DOCUMENTATION_SERVER environment variable (used for bob.extension
+  # and derived documentation building via Sphinx)
+  set_environment('DOCSERVER', server)
+  doc_urls = get_docserver_setup(public=(not private), stable=stable,
+      server=server, intranet=ci, group=group)
+  set_environment('BOB_DOCUMENTATION_SERVER', doc_urls)
+
+  arch = conda_arch()
+
+  for d in recipe_dir:
+
+    if not os.path.exists(d):
+      raise RuntimeError("The directory %s does not exist" % recipe_dir)
+
+    version_candidate = os.path.join(d, '..', 'version.txt')
+    if os.path.exists(version_candidate):
+      version = open(version_candidate).read().rstrip()
+      set_environment('BOB_PACKAGE_VERSION', version)
+
+    # pre-renders the recipe - figures out the destination
+    metadata = get_rendered_metadata(d, conda_config)
+
+    rendered_recipe = get_parsed_recipe(metadata)
+
+    path = get_output_path(metadata, conda_config)
+
+    # checks if we should actually build this recipe
+    if should_skip_build(metadata):
+      logger.info('Skipping UNSUPPORTED build of %s-%s-py%s for %s',
+          rendered_recipe['package']['name'],
+          rendered_recipe['package']['version'], python.replace('.',''),
+          arch)
+      continue
+
+    # Get the latest build number
+    build_number, existing = next_build_number(channels[0],
+        os.path.basename(path))
+
+    should_build = True
+
+    if existing:  #other builds exist, get the latest and see if it still works
+
+      destpath = os.path.join(condarc_options['croot'], arch,
+          os.path.basename(existing[0]))
+      logger.info('Downloading %s -> %s', existing[0], destpath)
+      urllib.request.urlretrieve(existing[0], destpath)
+
+      try:
+        logger.info('Testing %s', existing[0])
+        conda_build.api.test(destpath, config=conda_config)
+        should_build = False
+        logger.info('Test for %s: SUCCESS', existing[0])
+      except Exception as error:
+        logger.exception(error)
+        logger.warn('Test for %s: FAILED. Building...', existing[0])
+
+
+    if should_build:  #something wrong happened, run a full build
+
+      logger.info('Building %s-%s-py%s (build: %d) for %s',
+          rendered_recipe['package']['name'],
+          rendered_recipe['package']['version'], python.replace('.',''),
+          build_number, arch)
+
+      # notice we cannot build from the pre-parsed metadata because it has
+      # already resolved the "wrong" build number.  We'll have to reparse after
+      # setting the environment variable BOB_BUILD_NUMBER.
+      set_environment('BOB_BUILD_NUMBER', str(build_number))
+
+      if not dry_run:
+        conda_build.api.build(d, config=conda_config, notest=no_test)
+
+    else:  #skip build, test worked
+      logger.info('Skipping build of %s-%s-py%s (build: %d) for %s',
+          rendered_recipe['package']['name'],
+          rendered_recipe['package']['version'], python.replace('.',''),
+          build_number, arch)

conda/meta.yaml

@@ -59,6 +59,7 @@ test:
     - bdt dumpsphinx https://docs.python.org/3/objects.inv > /dev/null
     - bdt create --help
     - bdt build --help
+    - bdt rebuild --help
     - bdt test --help
     - bdt caupdate --help
     - bdt new --help
@@ -87,6 +88,7 @@ test:
     - bdt ci pypi --help
     - bdt ci readme --help
     - bdt ci clean --help
+    - bdt ci nightlies --help
     - sphinx-build -aEW ${PREFIX}/share/doc/{{ name }}/doc sphinx
     - if [ -n "${CI_PROJECT_DIR}" ]; then mv sphinx "${CI_PROJECT_DIR}/"; fi
 

setup.py

@@ -50,6 +50,7 @@ setup(
           'dumpsphinx = bob.devtools.scripts.dumpsphinx:dumpsphinx',
           'create = bob.devtools.scripts.create:create',
           'build = bob.devtools.scripts.build:build',
+          'rebuild = bob.devtools.scripts.rebuild:rebuild',
           'test = bob.devtools.scripts.test:test',
           'caupdate = bob.devtools.scripts.caupdate:caupdate',
           'ci = bob.devtools.scripts.ci:ci',
@@ -76,6 +77,7 @@ setup(
           'deploy = bob.devtools.scripts.ci:deploy',
           'readme = bob.devtools.scripts.ci:readme',
           'pypi = bob.devtools.scripts.ci:pypi',
+          'nightlies = bob.devtools.scripts.ci:nightlies',
           ],
 
     },