Commit 7eadf62d authored by André Anjos's avatar André Anjos 💬
Browse files

A lot of DRY, but mostly for the ci biuld

parent cb44686e
Pipeline #26093 failed with stages
in 7 minutes and 44 seconds
......@@ -7,7 +7,7 @@ This command uses a bare-minimum python3 installation (with SSL support) to
bootstrap a new miniconda installation preset for the defined activity. It is
primarily intended for CI operation and prefixes build and deployment steps.
Usage: python3 %s <cmd> build|local|beta|stable [<name>]
Usage: python3 %s <cmd> build|local|channel [<name>]
......@@ -15,8 +15,7 @@ Arguments:
build to build bob.devtools
local to bootstrap deploy|pypi stages for bob.devtools builds
beta to bootstrap CI environment for beta builds
stable to bootstrap CI environment for stable builds
channel to bootstrap CI environment for beta/stable builds
test to locally test this bootstrap script
<name> (optional) if command is one of ``local|beta|stable`` provide the
......@@ -363,6 +362,8 @@ if __name__ == '__main__':'(create) %s', condarc)
with open(condarc, 'wt') as f:
# we just add the "defaults" channels to the stock condarc
add_channels_condarc(['defaults'], condarc)
conda_version = '4'
conda_build_version = '3'
......@@ -370,7 +371,6 @@ if __name__ == '__main__':
if sys.argv[1] in ('build', 'test'):
# simple - just use the defaults channels when self building
add_channels_condarc(['defaults'], condarc)
run_cmdline([conda_bin, 'install', '-n', 'base',
'conda=%s' % conda_version,
......@@ -389,23 +389,29 @@ if __name__ == '__main__':
run_cmdline([conda_bin, 'index', conda_bld_path])
# add the locally build directory before defaults, boot from there
channels = get_channels(public=True, stable=True, server=_SERVER,
add_channels_condarc(channels + [conda_bld_path, 'defaults'], condarc)
run_cmdline([conda_bin, 'create', '-n', sys.argv[2], 'bob.devtools'])
intranet=True) + ['defaults']
channels = ['--override-channels'] + \
['--channel=' + conda_bld_path] + \
['--channel=%s' % k for k in channels]
run_cmdline([conda_bin, 'create'] + channels + \
['-n', sys.argv[2], 'bob.devtools'])
elif sys.argv[1] in ('beta', 'stable'):
elif sys.argv[1] == 'channel':
# installs from channel
channels = get_channels(
public=os.environ['CI_PROJECT_VISIBILITY'] == 'public',
stable=os.environ.get('CI_COMMIT_TAG') is not None,
public=(os.environ['CI_PROJECT_VISIBILITY'] == 'public'),
stable=('CI_COMMIT_TAG' in os.environ),
server=_SERVER, intranet=True)
channels = ['--override-channels'] + \
['--channel=%s' % k for k in channels]
add_channels_condarc(channels + ['defaults'], condarc)
run_cmdline([conda_bin, 'create', '-n', sys.argv[2], 'bob.devtools'])
run_cmdline([conda_bin, 'create'] + channels + \
['-n', sys.argv[2], 'bob.devtools'])
logger.error("Bootstrap with 'build', or 'local|beta|stable <name>'")
logger.error("Bootstrap with 'build', or 'local|channel <name>'")
logger.error("The value '%s' is not currently supported", sys.argv[1])
......@@ -38,20 +38,6 @@ def is_master(refname, tag):
return refname == 'master'
def is_visible_outside(package, visibility):
'''Determines if the project is visible outside Idiap''''Project %s visibility is "%s"', package, visibility)
if visibility == 'internal':
visibility = 'private' #the same thing for this command
logger.warn('Project %s visibility switched to "%s". ' \
'For this command, it all boils down to the same...', package,
return visibility == 'public'
def is_stable(package, refname, tag):
'''Determines if the package being published is stable
......@@ -109,7 +109,8 @@ def build(recipe_dir, python, condarc, config, no_test, append_file,
prefix = get_env_directory(os.environ['CONDA_EXE'], 'base')
condarc_options['croot'] = os.path.join(prefix, 'conda-bld')
conda_config = make_conda_config(config, python, append_file, condarc_options)
conda_config = make_conda_config(config, python, append_file,
set_environment('MATPLOTLIBRC', MATPLOTLIB_RCDIR, verbose=True)
......@@ -14,15 +14,9 @@ from click_plugins import with_plugins
from . import bdt
from ..log import verbosity_option
from import is_stable, is_visible_outside
from ..webdav3 import client as webdav
from import next_build_number, conda_arch, should_skip_build, \
get_rendered_metadata, get_parsed_recipe, make_conda_config, \
get_docserver_setup, check_version, git_clean_build
from ..bootstrap import set_environment, get_channels, run_cmdline
from ..constants import SERVER, WEBDAV_PATHS, CACERT
from ..bootstrap import set_environment, run_cmdline
......@@ -72,11 +66,12 @@ def deploy(dry_run):
package = os.environ['CI_PROJECT_PATH']
# determine project visibility
visible = is_visible_outside(package, os.environ['CI_PROJECT_VISIBILITY'])
visible = (os.environ['CI_PROJECT_VISIBILITY'] == 'public')
# determine if building master branch or tag - and if tag is on master
tag = os.environ.get('CI_COMMIT_TAG')
stable = is_stable(package, os.environ['CI_COMMIT_REF_NAME'], tag)
stable = is_stable(package,
server_info = WEBDAV_PATHS[stable][visible]
......@@ -174,7 +169,7 @@ def pypi(dry_run):
package = os.environ['CI_PROJECT_PATH']
# determine project visibility
visible = is_visible_outside(package, os.environ['CI_PROJECT_VISIBILITY'])
visible = (os.environ['CI_PROJECT_VISIBILITY'] == 'public')
if not visible:
raise RuntimeError('The repository %s is not public - a package ' \
......@@ -233,84 +228,23 @@ def build(dry_run):
to be used outside this context.
if dry_run:
logger.warn('!!!! DRY RUN MODE !!!!')
logger.warn('Nothing is being built')
prefix = os.environ['CONDA_ROOT']'os.environ["%s"] = %s', 'CONDA_ROOT', prefix)
workdir = os.environ['CI_PROJECT_DIR']'os.environ["%s"] = %s', 'CI_PROJECT_DIR', workdir)
pyver = os.environ['PYTHON_VERSION']'os.environ["%s"] = %s', 'PYTHON_VERSION', pyver)
set_environment('MATPLOTLIBRC', MATPLOTLIB_RCDIR, verbose=True)
# get information about the version of the package being built
version, is_prerelease = check_version(workdir,
set_environment('BOB_PACKAGE_VERSION', version, verbose=True)
# setup BOB_DOCUMENTATION_SERVER environment variable (used for bob.extension
# and derived documentation building via Sphinx)
set_environment('DOCSERVER', SERVER, os.environ, verbose=True)
public = ( os.environ['CI_PROJECT_VISIBILITY']=='public' )
doc_urls = get_docserver_setup(public=public, stable=(not is_prerelease),
server=SERVER, intranet=True)
set_environment('BOB_DOCUMENTATION_SERVER', doc_urls, verbose=True)
condarc = os.path.join(prefix, 'condarc')'Loading (this build\'s) CONDARC file from %s...', condarc)
with open(condarc, 'rb') as f:
condarc_options = yaml.load(f)
# notice this condarc typically will only contain the defaults channel - we
# need to boost this up with more channels to get it right.
channels = get_channels(public=public, stable=(not is_prerelease),
server=SERVER, intranet=True)'Using the following channels during build:\n - %s',
'\n - '.join(channels + ['defaults']))
condarc_options['channels'] = channels + ['defaults']
# dump packages at conda_root
condarc_options['croot'] = os.path.join(prefix, 'conda-bld')
# create the build configuration'Merging conda configuration files...')
conda_config = make_conda_config(CONDA_BUILD_CONFIG, pyver,
CONDA_RECIPE_APPEND, condarc_options)
recipe_dir = os.path.join(workdir, 'conda')
if not os.path.exists(recipe_dir):
raise RuntimeError("The directory %s does not exist" % recipe_dir)
# pre-renders the recipe - figures out package name and version
metadata = get_rendered_metadata(recipe_dir, conda_config)
arch = conda_arch()
if should_skip_build(metadata):
logger.warn('Skipping UNSUPPORTED build of "%s" for py%s on %s',
d, python.replace('.',''), arch)
return 0
# converts the metadata output into parsed yaml and continues the process
rendered_recipe = get_parsed_recipe(metadata)
# retrieve the current build number for this build
build_number, _ = next_build_number(channels[0],
rendered_recipe['package']['version'], python)
set_environment('BOB_BUILD_NUMBER', str(build_number), verbose=True)
# runs the build using the conda-build API'Building %s-%s-py%s (build: %d) for %s',
rendered_recipe['package']['version'], python.replace('.',''),
build_number, arch)
if not dry_run:, config=conda_config)
from import git_clean_build
from ..bootstrap import run_cmdline
from .build import build
recipe_dir=[os.path.join(os.path.realpath(os.curdir), 'conda')],
python=os.environ['PYTHON_VERSION'], #python version
condarc=None, #custom build configuration
private=(os.environ['CI_PROJECT_VISIBILITY'] != 'public'),
stable='CI_COMMIT_TAG' in os.environ,
git_clean_build(run_cmdline, arch)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment