diff --git a/bob/devtools/build.py b/bob/devtools/build.py
index b04b7becf6cdc20b999a6fffdbdbec23be1cd086..9881ca58a0668518ced1ff557a4e39a5bc1a2745 100644
--- a/bob/devtools/build.py
+++ b/bob/devtools/build.py
@@ -314,6 +314,77 @@ def get_docserver_setup(public, stable, server, intranet):
return '|'.join(entries)
+def check_version(workdir, envtag):
+ '''Checks if the version being built and the value reported match
+
+ This method will read the contents of the file ``version.txt`` and compare it
+ to the potentially set ``envtag`` (may be ``None``). If the value of
+ ``envtag`` is different than ``None``, ensure it matches the value in
+ ``version.txt`` or raises an exception.
+
+
+ Args:
+
+ workdir: The work directory where the repo of the package being built was
+ checked-out
+ envtag: The output of os.environ.get('CI_COMMIT_TAG') (may be ``None``)
+
+
+ Returns: A tuple with the version of the package that we're currently
+ building and a boolean flag indicating if the version number represents a
+ pre-release or a stable release.
+ '''
+
+ version = open(os.path.join(workdir, "version.txt"), 'rt').read().rstrip()
+
+ # if we're building a stable release, ensure a tag is set
+ parsed_version = distutils.version.LooseVersion(version).version
+ is_prerelease = any([isinstance(k, str) for k in parsed_version])
+ if is_prerelease:
+ if envtag is not None:
+ raise EnvironmentError('"version.txt" indicates version is a ' \
+ 'pre-release (v%s) - but os.environ["CI_COMMIT_TAG"]="%s", ' \
+ 'which indicates this is a **stable** build. ' \
+ 'Have you created the tag using ``bdt release``?' % (version,
+ envtag))
+ else: #it is a stable build
+ if envtag is None:
+ raise EnvironmentError('"version.txt" indicates version is a ' \
+ 'stable build (v%s) - but there is no os.environ["CI_COMMIT_TAG"] ' \
+ 'variable defined, which indicates this is **not** ' \
+ 'a tagged build. Use ``bdt release`` to create stable releases' % \
+ (version,))
+ if envtag[1:] != version:
+ raise EnvironmentError('"version.txt" and the value of ' \
+ 'os.environ["CI_COMMIT_TAG"] do **NOT** agree - the former ' \
+ 'reports version %s, the latter, %s' % (version, envtag[1:]))
+
+ return version, is_prerelease
+
+
+def git_clean_build(runner, arch):
+ '''Runs git-clean to clean-up build products
+
+ Args:
+
+ runner: A pointer to the ``run_cmdline()`` function
+
+ '''
+
+ # runs git clean to clean everything that is not needed. This helps to keep
+ # the disk usage on CI machines to a minimum.
+ exclude_from_cleanup = [
+ "miniconda.sh", #the installer, cached
+ "miniconda/pkgs/*.tar.bz2", #downloaded packages, cached
+ "miniconda/pkgs/urls.txt", #download index, cached
+ "miniconda/conda-bld/%s/*.tar.bz2" % (arch,), #build artifact -- conda
+ "dist/*.zip", #build artifact -- pypi package
+ "sphinx", #build artifact -- documentation
+ ]
+ runner(['git', 'clean', '-qffdx'] + \
+ ['--exclude=%s' % k for k in exclude_from_cleanup])
+
+
if __name__ == '__main__':
# loads the "adjacent" bootstrap module
@@ -338,40 +409,18 @@ if __name__ == '__main__':
pyver = os.environ['PYTHON_VERSION']
logger.info('os.environ["%s"] = %s', 'PYTHON_VERSION', pyver)
- bootstrap.set_environment('DOCSERVER', bootstrap._SERVER, os.environ,
- verbose=True)
- bootstrap.set_environment('LANG', 'en_US.UTF-8', os.environ,
- verbose=True)
- bootstrap.set_environment('LC_ALL', os.environ['LANG'], os.environ,
- verbose=True)
+ bootstrap.set_environment('DOCSERVER', bootstrap._SERVER, verbose=True)
+ bootstrap.set_environment('LANG', 'en_US.UTF-8', verbose=True)
+ bootstrap.set_environment('LC_ALL', os.environ['LANG'], verbose=True)
# get information about the version of the package being built
- version = open("version.txt").read().rstrip()
- os.environ['BOB_PACKAGE_VERSION'] = version
- logger.info('os.environ["%s"] = %s', 'BOB_PACKAGE_VERSION', version)
-
- # if we're building a stable release, ensure a tag is set
- parsed_version = distutils.version.LooseVersion(version).version
- is_prerelease = any([isinstance(k, str) for k in parsed_version])
- if is_prerelease:
- if os.environ.get('CI_COMMIT_TAG') is not None:
- raise EnvironmentError('"version.txt" indicates version is a ' \
- 'pre-release (v%s) - but os.environ["CI_COMMIT_TAG"]="%s", ' \
- 'which indicates this is a **stable** build. ' \
- 'Have you created the tag using ``bdt release``?', version,
- os.environ['CI_COMMIT_TAG'])
- else: #it is a stable build
- if os.environ.get('CI_COMMIT_TAG') is None:
- raise EnvironmentError('"version.txt" indicates version is a ' \
- 'stable build (v%s) - but there is no os.environ["CI_COMMIT_TAG"] ' \
- 'variable defined, which indicates this is **not** ' \
- 'a tagged build. Use ``bdt release`` to create stable releases',
- version)
+ version, is_prerelease = check_version(workdir,
+ os.environ.get('CI_COMMIT_TAG'))
+ bootstrap.set_environment('BOB_PACKAGE_VERSION', version, verbose=True)
# create the build configuration
conda_build_config = os.path.join(mydir, 'data', 'conda_build_config.yaml')
recipe_append = os.path.join(mydir, 'data', 'recipe_append.yaml')
- logger.info('Merging conda configuration files...')
condarc = os.path.join(prefix, 'condarc')
logger.info('Loading (this build\'s) CONDARC file from %s...', condarc)
@@ -380,13 +429,14 @@ if __name__ == '__main__':
# notice this condarc typically will only contain the defaults channel - we
# need to boost this up with more channels to get it right.
- channels = bootstrap.get_channels(
- public=(os.environ['CI_PROJECT_VISIBILITY']=='public'),
- stable=(not is_prerelease), server=bootstrap._SERVER, intranet=True)
+ public = ( os.environ['CI_PROJECT_VISIBILITY']=='public' )
+ channels = bootstrap.get_channels(public=public, stable=(not is_prerelease),
+ server=bootstrap._SERVER, intranet=True)
logger.info('Using the following channels during build:\n - %s',
'\n - '.join(channels + ['defaults']))
condarc_options['channels'] = channels + ['defaults']
+ logger.info('Merging conda configuration files...')
conda_config = make_conda_config(conda_build_config, pyver, recipe_append,
condarc_options)
@@ -401,15 +451,4 @@ if __name__ == '__main__':
name, version, pyver.replace('.',''), build_number, arch)
conda_build.api.build(os.path.join(workdir, 'conda'), config=conda_config)
- # runs git clean to clean everything that is not needed. This helps to keep
- # the disk usage on CI machines to a minimum.
- exclude_from_cleanup = [
- "miniconda.sh", #the installer, cached
- "miniconda/pkgs/*.tar.bz2", #downloaded packages, cached
- "miniconda/pkgs/urls.txt", #download index, cached
- "miniconda/conda-bld/%s/*.tar.bz2" % (arch,), #build artifact -- conda
- "dist/*.zip", #build artifact -- pypi package
- "sphinx", #build artifact -- documentation
- ]
- bootstrap.run_cmdline(['git', 'clean', '-qffdx'] + \
- ['--exclude=%s' % k for k in exclude_from_cleanup])
+ git_clean_build(bootstrap.run_cmdline, arch)
diff --git a/bob/devtools/scripts/build.py b/bob/devtools/scripts/build.py
index 2242ccee4e51fabdb3fa512e9a882653e39dff73..6e7f2038efcdc1f09e740304f9dd769bb0e9d5bc 100644
--- a/bob/devtools/scripts/build.py
+++ b/bob/devtools/scripts/build.py
@@ -6,9 +6,10 @@ import sys
import logging
logger = logging.getLogger(__name__)
-import pkg_resources
-import click
import yaml
+import click
+import pkg_resources
+import conda_build.api
from . import bdt
from ..log import verbosity_option
@@ -107,16 +108,16 @@ def build(recipe_dir, python, condarc, config, no_test, append_file,
conda_config = make_conda_config(config, python, append_file, condarc_options)
- set_environment('LANG', 'en_US.UTF-8', os.environ)
- set_environment('LC_ALL', os.environ['LANG'], os.environ)
- set_environment('MATPLOTLIBRC', MATPLOTLIB_RCDIR, os.environ)
+ set_environment('LANG', 'en_US.UTF-8', verbose=True)
+ set_environment('LC_ALL', os.environ['LANG'], verbose=True)
+ set_environment('MATPLOTLIBRC', MATPLOTLIB_RCDIR, verbose=True)
# setup BOB_DOCUMENTATION_SERVER environment variable (used for bob.extension
# and derived documentation building via Sphinx)
- set_environment('DOCSERVER', server, os.environ)
+ set_environment('DOCSERVER', server, verbose=True)
doc_urls = get_docserver_setup(public=(not private), stable=stable,
server=server, intranet=private)
- set_environment('BOB_DOCUMENTATION_SERVER', doc_urls, server=server)
+ set_environment('BOB_DOCUMENTATION_SERVER', doc_urls, verbose=True)
for d in recipe_dir:
@@ -126,7 +127,7 @@ def build(recipe_dir, python, condarc, config, no_test, append_file,
version_candidate = os.path.join(d, '..', 'version.txt')
if os.path.exists(version_candidate):
version = open(version_candidate).read().rstrip()
- set_environment('BOB_PACKAGE_VERSION', version, os.environ)
+ set_environment('BOB_PACKAGE_VERSION', version, verbose=True)
# pre-renders the recipe - figures out package name and version
metadata = get_rendered_metadata(d, conda_config)
@@ -146,12 +147,11 @@ def build(recipe_dir, python, condarc, config, no_test, append_file,
rendered_recipe['package']['name'],
rendered_recipe['package']['version'], python)
- set_environment('BOB_BUILD_NUMBER', str(build_number), os.environ)
+ set_environment('BOB_BUILD_NUMBER', str(build_number), verbose=True)
logger.info('Building %s-%s-py%s (build: %d) for %s',
rendered_recipe['package']['name'],
rendered_recipe['package']['version'], python.replace('.',''),
build_number, arch)
if not dry_run:
- from conda_build.api import build
- build(d, config=conda_config, notest=no_test)
+ conda_build.api.build(d, config=conda_config, notest=no_test)
diff --git a/bob/devtools/scripts/ci.py b/bob/devtools/scripts/ci.py
index fed7cf008aa279bd6ad66c0967c976997d115bfb..8e10099f2da46d27cc532da38cd4947c5fcd57db 100644
--- a/bob/devtools/scripts/ci.py
+++ b/bob/devtools/scripts/ci.py
@@ -13,9 +13,15 @@ from click_plugins import with_plugins
from . import bdt
from ..log import verbosity_option
from ..ci import is_stable, is_visible_outside
-from ..constants import SERVER, WEBDAV_PATHS, CACERT
from ..webdav3 import client as webdav
+from ..constants import SERVER, WEBDAV_PATHS, CACERT, CONDA_BUILD_CONFIG, \
+ CONDA_RECIPE_APPEND, MATPLOTLIB_RCDIR, BASE_CONDARC
+from ..build import next_build_number, conda_arch, should_skip_build, \
+ get_rendered_metadata, get_parsed_recipe, make_conda_config, \
+ get_docserver_setup, check_version, git_clean_build
+from ..bootstrap import set_environment, get_channels, run_cmdline
+
@with_plugins(pkg_resources.iter_entry_points('bdt.ci.cli'))
@click.group(cls=bdt.AliasedGroup)
@@ -201,3 +207,90 @@ def pypi(dry_run):
from twine.commands.upload import upload
upload(settings, zip_files)
logger.info('Deployment to PyPI successful')
+
+
+@ci.command(epilog='''
+Examples:
+
+ 1. Builds the current package
+
+ $ bdt ci build -vv
+
+''')
+@click.option('-d', '--dry-run/--no-dry-run', default=False,
+ help='Only goes through the actions, but does not execute them ' \
+ '(combine with the verbosity flags - e.g. ``-vvv``) to enable ' \
+ 'printing to help you understand what will be done')
+@verbosity_option()
+@bdt.raise_on_error
+def build(dry_run):
+ """Builds packages
+
+ This command builds packages in the CI infrastructure. It is **not** meant
+ to be used outside this context.
+ """
+
+ if dry_run:
+ logger.warn('!!!! DRY RUN MODE !!!!')
+ logger.warn('Nothing is being built')
+
+ prefix = os.environ['CONDA_ROOT']
+ logger.info('os.environ["%s"] = %s', 'CONDA_ROOT', prefix)
+
+ workdir = os.environ['CI_PROJECT_DIR']
+ logger.info('os.environ["%s"] = %s', 'CI_PROJECT_DIR', workdir)
+
+ name = os.environ['CI_PROJECT_NAME']
+ logger.info('os.environ["%s"] = %s', 'CI_PROJECT_NAME', name)
+
+ pyver = os.environ['PYTHON_VERSION']
+ logger.info('os.environ["%s"] = %s', 'PYTHON_VERSION', pyver)
+
+ set_environment('LANG', 'en_US.UTF-8', os.environ, verbose=True)
+ set_environment('LC_ALL', os.environ['LANG'], os.environ, verbose=True)
+ set_environment('MATPLOTLIBRC', MATPLOTLIB_RCDIR, verbose=True)
+
+ # setup BOB_DOCUMENTATION_SERVER environment variable (used for bob.extension
+ # and derived documentation building via Sphinx)
+ set_environment('DOCSERVER', SERVER, os.environ, verbose=True)
+ public = ( os.environ['CI_PROJECT_VISIBILITY']=='public' )
+ doc_urls = get_docserver_setup(public=public, stable=(not is_prerelease),
+ server=SERVER, intranet=True)
+ set_environment('BOB_DOCUMENTATION_SERVER', doc_urls, verbose=True)
+
+ # get information about the version of the package being built
+ version, is_prerelease = check_version(workdir,
+ os.environ.get('CI_COMMIT_TAG'))
+ set_environment('BOB_PACKAGE_VERSION', version, verbose=True)
+
+ condarc = os.path.join(prefix, 'condarc')
+ logger.info('Loading (this build\'s) CONDARC file from %s...', condarc)
+ with open(condarc, 'rb') as f:
+ condarc_options = yaml.load(f)
+
+ # notice this condarc typically will only contain the defaults channel - we
+ # need to boost this up with more channels to get it right.
+ channels = bootstrap.get_channels(public=public, stable=(not is_prerelease),
+ server=SERVER, intranet=True)
+ logger.info('Using the following channels during build:\n - %s',
+ '\n - '.join(channels + ['defaults']))
+ condarc_options['channels'] = channels + ['defaults']
+
+ # create the build configuration
+ logger.info('Merging conda configuration files...')
+ conda_config = make_conda_config(CONDA_BUILD_CONFIG, pyver,
+ CONDA_RECIPE_APPEND, condarc_options)
+
+ # retrieve the current build number for this build
+ build_number, _ = next_build_number(channels[0], name, version, pyver)
+ set_environment('BOB_BUILD_NUMBER', str(build_number), verbose=True)
+
+ # runs the build using the conda-build API
+ arch = conda_arch()
+ logger.info('Building %s-%s-py%s (build: %d) for %s',
+ name, version, pyver.replace('.',''), build_number, arch)
+
+ if not dry_run:
+ conda_build.api.build(os.path.join(workdir, 'conda'), config=conda_config)
+
+ git_clean_build(run_cmdline, arch)