Commit f1a72301 authored by André Anjos's avatar André Anjos 💬
Browse files

[ci] Add build instructions

parent 58476350
Pipeline #26051 passed with stages
in 7 minutes and 50 seconds
...@@ -314,6 +314,77 @@ def get_docserver_setup(public, stable, server, intranet): ...@@ -314,6 +314,77 @@ def get_docserver_setup(public, stable, server, intranet):
return '|'.join(entries) return '|'.join(entries)
def check_version(workdir, envtag):
'''Checks if the version being built and the value reported match
This method will read the contents of the file ``version.txt`` and compare it
to the potentially set ``envtag`` (may be ``None``). If the value of
``envtag`` is different than ``None``, ensure it matches the value in
``version.txt`` or raises an exception.
Args:
workdir: The work directory where the repo of the package being built was
checked-out
envtag: The output of os.environ.get('CI_COMMIT_TAG') (may be ``None``)
Returns: A tuple with the version of the package that we're currently
building and a boolean flag indicating if the version number represents a
pre-release or a stable release.
'''
version = open(os.path.join(workdir, "version.txt"), 'rt').read().rstrip()
# if we're building a stable release, ensure a tag is set
parsed_version = distutils.version.LooseVersion(version).version
is_prerelease = any([isinstance(k, str) for k in parsed_version])
if is_prerelease:
if envtag is not None:
raise EnvironmentError('"version.txt" indicates version is a ' \
'pre-release (v%s) - but os.environ["CI_COMMIT_TAG"]="%s", ' \
'which indicates this is a **stable** build. ' \
'Have you created the tag using ``bdt release``?' % (version,
envtag))
else: #it is a stable build
if envtag is None:
raise EnvironmentError('"version.txt" indicates version is a ' \
'stable build (v%s) - but there is no os.environ["CI_COMMIT_TAG"] ' \
'variable defined, which indicates this is **not** ' \
'a tagged build. Use ``bdt release`` to create stable releases' % \
(version,))
if envtag[1:] != version:
raise EnvironmentError('"version.txt" and the value of ' \
'os.environ["CI_COMMIT_TAG"] do **NOT** agree - the former ' \
'reports version %s, the latter, %s' % (version, envtag[1:]))
return version, is_prerelease
def git_clean_build(runner, arch):
'''Runs git-clean to clean-up build products
Args:
runner: A pointer to the ``run_cmdline()`` function
'''
# runs git clean to clean everything that is not needed. This helps to keep
# the disk usage on CI machines to a minimum.
exclude_from_cleanup = [
"miniconda.sh", #the installer, cached
"miniconda/pkgs/*.tar.bz2", #downloaded packages, cached
"miniconda/pkgs/urls.txt", #download index, cached
"miniconda/conda-bld/%s/*.tar.bz2" % (arch,), #build artifact -- conda
"dist/*.zip", #build artifact -- pypi package
"sphinx", #build artifact -- documentation
]
runner(['git', 'clean', '-qffdx'] + \
['--exclude=%s' % k for k in exclude_from_cleanup])
if __name__ == '__main__': if __name__ == '__main__':
# loads the "adjacent" bootstrap module # loads the "adjacent" bootstrap module
...@@ -338,40 +409,18 @@ if __name__ == '__main__': ...@@ -338,40 +409,18 @@ if __name__ == '__main__':
pyver = os.environ['PYTHON_VERSION'] pyver = os.environ['PYTHON_VERSION']
logger.info('os.environ["%s"] = %s', 'PYTHON_VERSION', pyver) logger.info('os.environ["%s"] = %s', 'PYTHON_VERSION', pyver)
bootstrap.set_environment('DOCSERVER', bootstrap._SERVER, os.environ, bootstrap.set_environment('DOCSERVER', bootstrap._SERVER, verbose=True)
verbose=True) bootstrap.set_environment('LANG', 'en_US.UTF-8', verbose=True)
bootstrap.set_environment('LANG', 'en_US.UTF-8', os.environ, bootstrap.set_environment('LC_ALL', os.environ['LANG'], verbose=True)
verbose=True)
bootstrap.set_environment('LC_ALL', os.environ['LANG'], os.environ,
verbose=True)
# get information about the version of the package being built # get information about the version of the package being built
version = open("version.txt").read().rstrip() version, is_prerelease = check_version(workdir,
os.environ['BOB_PACKAGE_VERSION'] = version os.environ.get('CI_COMMIT_TAG'))
logger.info('os.environ["%s"] = %s', 'BOB_PACKAGE_VERSION', version) bootstrap.set_environment('BOB_PACKAGE_VERSION', version, verbose=True)
# if we're building a stable release, ensure a tag is set
parsed_version = distutils.version.LooseVersion(version).version
is_prerelease = any([isinstance(k, str) for k in parsed_version])
if is_prerelease:
if os.environ.get('CI_COMMIT_TAG') is not None:
raise EnvironmentError('"version.txt" indicates version is a ' \
'pre-release (v%s) - but os.environ["CI_COMMIT_TAG"]="%s", ' \
'which indicates this is a **stable** build. ' \
'Have you created the tag using ``bdt release``?', version,
os.environ['CI_COMMIT_TAG'])
else: #it is a stable build
if os.environ.get('CI_COMMIT_TAG') is None:
raise EnvironmentError('"version.txt" indicates version is a ' \
'stable build (v%s) - but there is no os.environ["CI_COMMIT_TAG"] ' \
'variable defined, which indicates this is **not** ' \
'a tagged build. Use ``bdt release`` to create stable releases',
version)
# create the build configuration # create the build configuration
conda_build_config = os.path.join(mydir, 'data', 'conda_build_config.yaml') conda_build_config = os.path.join(mydir, 'data', 'conda_build_config.yaml')
recipe_append = os.path.join(mydir, 'data', 'recipe_append.yaml') recipe_append = os.path.join(mydir, 'data', 'recipe_append.yaml')
logger.info('Merging conda configuration files...')
condarc = os.path.join(prefix, 'condarc') condarc = os.path.join(prefix, 'condarc')
logger.info('Loading (this build\'s) CONDARC file from %s...', condarc) logger.info('Loading (this build\'s) CONDARC file from %s...', condarc)
...@@ -380,13 +429,14 @@ if __name__ == '__main__': ...@@ -380,13 +429,14 @@ if __name__ == '__main__':
# notice this condarc typically will only contain the defaults channel - we # notice this condarc typically will only contain the defaults channel - we
# need to boost this up with more channels to get it right. # need to boost this up with more channels to get it right.
channels = bootstrap.get_channels( public = ( os.environ['CI_PROJECT_VISIBILITY']=='public' )
public=(os.environ['CI_PROJECT_VISIBILITY']=='public'), channels = bootstrap.get_channels(public=public, stable=(not is_prerelease),
stable=(not is_prerelease), server=bootstrap._SERVER, intranet=True) server=bootstrap._SERVER, intranet=True)
logger.info('Using the following channels during build:\n - %s', logger.info('Using the following channels during build:\n - %s',
'\n - '.join(channels + ['defaults'])) '\n - '.join(channels + ['defaults']))
condarc_options['channels'] = channels + ['defaults'] condarc_options['channels'] = channels + ['defaults']
logger.info('Merging conda configuration files...')
conda_config = make_conda_config(conda_build_config, pyver, recipe_append, conda_config = make_conda_config(conda_build_config, pyver, recipe_append,
condarc_options) condarc_options)
...@@ -401,15 +451,4 @@ if __name__ == '__main__': ...@@ -401,15 +451,4 @@ if __name__ == '__main__':
name, version, pyver.replace('.',''), build_number, arch) name, version, pyver.replace('.',''), build_number, arch)
conda_build.api.build(os.path.join(workdir, 'conda'), config=conda_config) conda_build.api.build(os.path.join(workdir, 'conda'), config=conda_config)
# runs git clean to clean everything that is not needed. This helps to keep git_clean_build(bootstrap.run_cmdline, arch)
# the disk usage on CI machines to a minimum.
exclude_from_cleanup = [
"miniconda.sh", #the installer, cached
"miniconda/pkgs/*.tar.bz2", #downloaded packages, cached
"miniconda/pkgs/urls.txt", #download index, cached
"miniconda/conda-bld/%s/*.tar.bz2" % (arch,), #build artifact -- conda
"dist/*.zip", #build artifact -- pypi package
"sphinx", #build artifact -- documentation
]
bootstrap.run_cmdline(['git', 'clean', '-qffdx'] + \
['--exclude=%s' % k for k in exclude_from_cleanup])
...@@ -6,9 +6,10 @@ import sys ...@@ -6,9 +6,10 @@ import sys
import logging import logging
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
import pkg_resources
import click
import yaml import yaml
import click
import pkg_resources
import conda_build.api
from . import bdt from . import bdt
from ..log import verbosity_option from ..log import verbosity_option
...@@ -107,16 +108,16 @@ def build(recipe_dir, python, condarc, config, no_test, append_file, ...@@ -107,16 +108,16 @@ def build(recipe_dir, python, condarc, config, no_test, append_file,
conda_config = make_conda_config(config, python, append_file, condarc_options) conda_config = make_conda_config(config, python, append_file, condarc_options)
set_environment('LANG', 'en_US.UTF-8', os.environ) set_environment('LANG', 'en_US.UTF-8', verbose=True)
set_environment('LC_ALL', os.environ['LANG'], os.environ) set_environment('LC_ALL', os.environ['LANG'], verbose=True)
set_environment('MATPLOTLIBRC', MATPLOTLIB_RCDIR, os.environ) set_environment('MATPLOTLIBRC', MATPLOTLIB_RCDIR, verbose=True)
# setup BOB_DOCUMENTATION_SERVER environment variable (used for bob.extension # setup BOB_DOCUMENTATION_SERVER environment variable (used for bob.extension
# and derived documentation building via Sphinx) # and derived documentation building via Sphinx)
set_environment('DOCSERVER', server, os.environ) set_environment('DOCSERVER', server, verbose=True)
doc_urls = get_docserver_setup(public=(not private), stable=stable, doc_urls = get_docserver_setup(public=(not private), stable=stable,
server=server, intranet=private) server=server, intranet=private)
set_environment('BOB_DOCUMENTATION_SERVER', doc_urls, server=server) set_environment('BOB_DOCUMENTATION_SERVER', doc_urls, verbose=True)
for d in recipe_dir: for d in recipe_dir:
...@@ -126,7 +127,7 @@ def build(recipe_dir, python, condarc, config, no_test, append_file, ...@@ -126,7 +127,7 @@ def build(recipe_dir, python, condarc, config, no_test, append_file,
version_candidate = os.path.join(d, '..', 'version.txt') version_candidate = os.path.join(d, '..', 'version.txt')
if os.path.exists(version_candidate): if os.path.exists(version_candidate):
version = open(version_candidate).read().rstrip() version = open(version_candidate).read().rstrip()
set_environment('BOB_PACKAGE_VERSION', version, os.environ) set_environment('BOB_PACKAGE_VERSION', version, verbose=True)
# pre-renders the recipe - figures out package name and version # pre-renders the recipe - figures out package name and version
metadata = get_rendered_metadata(d, conda_config) metadata = get_rendered_metadata(d, conda_config)
...@@ -146,12 +147,11 @@ def build(recipe_dir, python, condarc, config, no_test, append_file, ...@@ -146,12 +147,11 @@ def build(recipe_dir, python, condarc, config, no_test, append_file,
rendered_recipe['package']['name'], rendered_recipe['package']['name'],
rendered_recipe['package']['version'], python) rendered_recipe['package']['version'], python)
set_environment('BOB_BUILD_NUMBER', str(build_number), os.environ) set_environment('BOB_BUILD_NUMBER', str(build_number), verbose=True)
logger.info('Building %s-%s-py%s (build: %d) for %s', logger.info('Building %s-%s-py%s (build: %d) for %s',
rendered_recipe['package']['name'], rendered_recipe['package']['name'],
rendered_recipe['package']['version'], python.replace('.',''), rendered_recipe['package']['version'], python.replace('.',''),
build_number, arch) build_number, arch)
if not dry_run: if not dry_run:
from conda_build.api import build conda_build.api.build(d, config=conda_config, notest=no_test)
build(d, config=conda_config, notest=no_test)
...@@ -13,9 +13,15 @@ from click_plugins import with_plugins ...@@ -13,9 +13,15 @@ from click_plugins import with_plugins
from . import bdt from . import bdt
from ..log import verbosity_option from ..log import verbosity_option
from ..ci import is_stable, is_visible_outside from ..ci import is_stable, is_visible_outside
from ..constants import SERVER, WEBDAV_PATHS, CACERT
from ..webdav3 import client as webdav from ..webdav3 import client as webdav
from ..constants import SERVER, WEBDAV_PATHS, CACERT, CONDA_BUILD_CONFIG, \
CONDA_RECIPE_APPEND, MATPLOTLIB_RCDIR, BASE_CONDARC
from ..build import next_build_number, conda_arch, should_skip_build, \
get_rendered_metadata, get_parsed_recipe, make_conda_config, \
get_docserver_setup, check_version, git_clean_build
from ..bootstrap import set_environment, get_channels, run_cmdline
@with_plugins(pkg_resources.iter_entry_points('bdt.ci.cli')) @with_plugins(pkg_resources.iter_entry_points('bdt.ci.cli'))
@click.group(cls=bdt.AliasedGroup) @click.group(cls=bdt.AliasedGroup)
...@@ -201,3 +207,90 @@ def pypi(dry_run): ...@@ -201,3 +207,90 @@ def pypi(dry_run):
from twine.commands.upload import upload from twine.commands.upload import upload
upload(settings, zip_files) upload(settings, zip_files)
logger.info('Deployment to PyPI successful') logger.info('Deployment to PyPI successful')
@ci.command(epilog='''
Examples:
1. Builds the current package
$ bdt ci build -vv
''')
@click.option('-d', '--dry-run/--no-dry-run', default=False,
help='Only goes through the actions, but does not execute them ' \
'(combine with the verbosity flags - e.g. ``-vvv``) to enable ' \
'printing to help you understand what will be done')
@verbosity_option()
@bdt.raise_on_error
def build(dry_run):
"""Builds packages
This command builds packages in the CI infrastructure. It is **not** meant
to be used outside this context.
"""
if dry_run:
logger.warn('!!!! DRY RUN MODE !!!!')
logger.warn('Nothing is being built')
prefix = os.environ['CONDA_ROOT']
logger.info('os.environ["%s"] = %s', 'CONDA_ROOT', prefix)
workdir = os.environ['CI_PROJECT_DIR']
logger.info('os.environ["%s"] = %s', 'CI_PROJECT_DIR', workdir)
name = os.environ['CI_PROJECT_NAME']
logger.info('os.environ["%s"] = %s', 'CI_PROJECT_NAME', name)
pyver = os.environ['PYTHON_VERSION']
logger.info('os.environ["%s"] = %s', 'PYTHON_VERSION', pyver)
set_environment('LANG', 'en_US.UTF-8', os.environ, verbose=True)
set_environment('LC_ALL', os.environ['LANG'], os.environ, verbose=True)
set_environment('MATPLOTLIBRC', MATPLOTLIB_RCDIR, verbose=True)
# setup BOB_DOCUMENTATION_SERVER environment variable (used for bob.extension
# and derived documentation building via Sphinx)
set_environment('DOCSERVER', SERVER, os.environ, verbose=True)
public = ( os.environ['CI_PROJECT_VISIBILITY']=='public' )
doc_urls = get_docserver_setup(public=public, stable=(not is_prerelease),
server=SERVER, intranet=True)
set_environment('BOB_DOCUMENTATION_SERVER', doc_urls, verbose=True)
# get information about the version of the package being built
version, is_prerelease = check_version(workdir,
os.environ.get('CI_COMMIT_TAG'))
set_environment('BOB_PACKAGE_VERSION', version, verbose=True)
condarc = os.path.join(prefix, 'condarc')
logger.info('Loading (this build\'s) CONDARC file from %s...', condarc)
with open(condarc, 'rb') as f:
condarc_options = yaml.load(f)
# notice this condarc typically will only contain the defaults channel - we
# need to boost this up with more channels to get it right.
channels = bootstrap.get_channels(public=public, stable=(not is_prerelease),
server=SERVER, intranet=True)
logger.info('Using the following channels during build:\n - %s',
'\n - '.join(channels + ['defaults']))
condarc_options['channels'] = channels + ['defaults']
# create the build configuration
logger.info('Merging conda configuration files...')
conda_config = make_conda_config(CONDA_BUILD_CONFIG, pyver,
CONDA_RECIPE_APPEND, condarc_options)
# retrieve the current build number for this build
build_number, _ = next_build_number(channels[0], name, version, pyver)
set_environment('BOB_BUILD_NUMBER', str(build_number), verbose=True)
# runs the build using the conda-build API
arch = conda_arch()
logger.info('Building %s-%s-py%s (build: %d) for %s',
name, version, pyver.replace('.',''), build_number, arch)
if not dry_run:
conda_build.api.build(os.path.join(workdir, 'conda'), config=conda_config)
git_clean_build(run_cmdline, arch)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment