Commit 7eadf62d authored by André Anjos's avatar André Anjos 💬
Browse files

A lot of DRY, but mostly for the ci biuld

parent cb44686e
Pipeline #26093 failed with stages
in 7 minutes and 44 seconds
...@@ -7,7 +7,7 @@ This command uses a bare-minimum python3 installation (with SSL support) to ...@@ -7,7 +7,7 @@ This command uses a bare-minimum python3 installation (with SSL support) to
bootstrap a new miniconda installation preset for the defined activity. It is bootstrap a new miniconda installation preset for the defined activity. It is
primarily intended for CI operation and prefixes build and deployment steps. primarily intended for CI operation and prefixes build and deployment steps.
Usage: python3 %s <cmd> build|local|beta|stable [<name>] Usage: python3 %s <cmd> build|local|channel [<name>]
Arguments: Arguments:
...@@ -15,8 +15,7 @@ Arguments: ...@@ -15,8 +15,7 @@ Arguments:
build to build bob.devtools build to build bob.devtools
local to bootstrap deploy|pypi stages for bob.devtools builds local to bootstrap deploy|pypi stages for bob.devtools builds
beta to bootstrap CI environment for beta builds channel to bootstrap CI environment for beta/stable builds
stable to bootstrap CI environment for stable builds
test to locally test this bootstrap script test to locally test this bootstrap script
<name> (optional) if command is one of ``local|beta|stable`` provide the <name> (optional) if command is one of ``local|beta|stable`` provide the
...@@ -363,6 +362,8 @@ if __name__ == '__main__': ...@@ -363,6 +362,8 @@ if __name__ == '__main__':
logger.info('(create) %s', condarc) logger.info('(create) %s', condarc)
with open(condarc, 'wt') as f: with open(condarc, 'wt') as f:
f.write(_BASE_CONDARC) f.write(_BASE_CONDARC)
# we just add the "defaults" channels to the stock condarc
add_channels_condarc(['defaults'], condarc)
conda_version = '4' conda_version = '4'
conda_build_version = '3' conda_build_version = '3'
...@@ -370,7 +371,6 @@ if __name__ == '__main__': ...@@ -370,7 +371,6 @@ if __name__ == '__main__':
if sys.argv[1] in ('build', 'test'): if sys.argv[1] in ('build', 'test'):
# simple - just use the defaults channels when self building # simple - just use the defaults channels when self building
add_channels_condarc(['defaults'], condarc)
run_cmdline([conda_bin, 'install', '-n', 'base', run_cmdline([conda_bin, 'install', '-n', 'base',
'python', 'python',
'conda=%s' % conda_version, 'conda=%s' % conda_version,
...@@ -389,23 +389,29 @@ if __name__ == '__main__': ...@@ -389,23 +389,29 @@ if __name__ == '__main__':
run_cmdline([conda_bin, 'index', conda_bld_path]) run_cmdline([conda_bin, 'index', conda_bld_path])
# add the locally build directory before defaults, boot from there # add the locally build directory before defaults, boot from there
channels = get_channels(public=True, stable=True, server=_SERVER, channels = get_channels(public=True, stable=True, server=_SERVER,
intranet=True) intranet=True) + ['defaults']
add_channels_condarc(channels + [conda_bld_path, 'defaults'], condarc) channels = ['--override-channels'] + \
run_cmdline([conda_bin, 'create', '-n', sys.argv[2], 'bob.devtools']) ['--channel=' + conda_bld_path] + \
['--channel=%s' % k for k in channels]
run_cmdline([conda_bin, 'create'] + channels + \
['-n', sys.argv[2], 'bob.devtools'])
elif sys.argv[1] in ('beta', 'stable'): elif sys.argv[1] == 'channel':
# installs from channel # installs from channel
channels = get_channels( channels = get_channels(
public=os.environ['CI_PROJECT_VISIBILITY'] == 'public', public=(os.environ['CI_PROJECT_VISIBILITY'] == 'public'),
stable=os.environ.get('CI_COMMIT_TAG') is not None, stable=('CI_COMMIT_TAG' in os.environ),
server=_SERVER, intranet=True) server=_SERVER, intranet=True)
channels = ['--override-channels'] + \
['--channel=%s' % k for k in channels]
add_channels_condarc(channels + ['defaults'], condarc) add_channels_condarc(channels + ['defaults'], condarc)
run_cmdline([conda_bin, 'create', '-n', sys.argv[2], 'bob.devtools']) run_cmdline([conda_bin, 'create'] + channels + \
['-n', sys.argv[2], 'bob.devtools'])
else: else:
logger.error("Bootstrap with 'build', or 'local|beta|stable <name>'") logger.error("Bootstrap with 'build', or 'local|channel <name>'")
logger.error("The value '%s' is not currently supported", sys.argv[1]) logger.error("The value '%s' is not currently supported", sys.argv[1])
sys.exit(1) sys.exit(1)
......
...@@ -38,20 +38,6 @@ def is_master(refname, tag): ...@@ -38,20 +38,6 @@ def is_master(refname, tag):
return refname == 'master' return refname == 'master'
def is_visible_outside(package, visibility):
'''Determines if the project is visible outside Idiap'''
logger.info('Project %s visibility is "%s"', package, visibility)
if visibility == 'internal':
visibility = 'private' #the same thing for this command
logger.warn('Project %s visibility switched to "%s". ' \
'For this command, it all boils down to the same...', package,
visibility)
return visibility == 'public'
def is_stable(package, refname, tag): def is_stable(package, refname, tag):
'''Determines if the package being published is stable '''Determines if the package being published is stable
......
...@@ -109,7 +109,8 @@ def build(recipe_dir, python, condarc, config, no_test, append_file, ...@@ -109,7 +109,8 @@ def build(recipe_dir, python, condarc, config, no_test, append_file,
prefix = get_env_directory(os.environ['CONDA_EXE'], 'base') prefix = get_env_directory(os.environ['CONDA_EXE'], 'base')
condarc_options['croot'] = os.path.join(prefix, 'conda-bld') condarc_options['croot'] = os.path.join(prefix, 'conda-bld')
conda_config = make_conda_config(config, python, append_file, condarc_options) conda_config = make_conda_config(config, python, append_file,
condarc_options)
set_environment('MATPLOTLIBRC', MATPLOTLIB_RCDIR, verbose=True) set_environment('MATPLOTLIBRC', MATPLOTLIB_RCDIR, verbose=True)
......
...@@ -14,15 +14,9 @@ from click_plugins import with_plugins ...@@ -14,15 +14,9 @@ from click_plugins import with_plugins
from . import bdt from . import bdt
from ..log import verbosity_option from ..log import verbosity_option
from ..ci import is_stable, is_visible_outside
from ..webdav3 import client as webdav from ..webdav3 import client as webdav
from ..constants import SERVER, WEBDAV_PATHS, CACERT
from ..constants import SERVER, WEBDAV_PATHS, CACERT, CONDA_BUILD_CONFIG, \ from ..bootstrap import set_environment, run_cmdline
CONDA_RECIPE_APPEND, MATPLOTLIB_RCDIR, BASE_CONDARC
from ..build import next_build_number, conda_arch, should_skip_build, \
get_rendered_metadata, get_parsed_recipe, make_conda_config, \
get_docserver_setup, check_version, git_clean_build
from ..bootstrap import set_environment, get_channels, run_cmdline
@with_plugins(pkg_resources.iter_entry_points('bdt.ci.cli')) @with_plugins(pkg_resources.iter_entry_points('bdt.ci.cli'))
...@@ -72,11 +66,12 @@ def deploy(dry_run): ...@@ -72,11 +66,12 @@ def deploy(dry_run):
package = os.environ['CI_PROJECT_PATH'] package = os.environ['CI_PROJECT_PATH']
# determine project visibility # determine project visibility
visible = is_visible_outside(package, os.environ['CI_PROJECT_VISIBILITY']) visible = (os.environ['CI_PROJECT_VISIBILITY'] == 'public')
# determine if building master branch or tag - and if tag is on master # determine if building master branch or tag - and if tag is on master
tag = os.environ.get('CI_COMMIT_TAG') stable = is_stable(package,
stable = is_stable(package, os.environ['CI_COMMIT_REF_NAME'], tag) os.environ['CI_COMMIT_REF_NAME'],
os.environ['CI_COMMIT_TAG'])
server_info = WEBDAV_PATHS[stable][visible] server_info = WEBDAV_PATHS[stable][visible]
...@@ -174,7 +169,7 @@ def pypi(dry_run): ...@@ -174,7 +169,7 @@ def pypi(dry_run):
package = os.environ['CI_PROJECT_PATH'] package = os.environ['CI_PROJECT_PATH']
# determine project visibility # determine project visibility
visible = is_visible_outside(package, os.environ['CI_PROJECT_VISIBILITY']) visible = (os.environ['CI_PROJECT_VISIBILITY'] == 'public')
if not visible: if not visible:
raise RuntimeError('The repository %s is not public - a package ' \ raise RuntimeError('The repository %s is not public - a package ' \
...@@ -233,84 +228,23 @@ def build(dry_run): ...@@ -233,84 +228,23 @@ def build(dry_run):
to be used outside this context. to be used outside this context.
""" """
if dry_run: from ..constants import CONDA_BUILD_CONFIG, CONDA_RECIPE_APPEND
logger.warn('!!!! DRY RUN MODE !!!!') from ..build import git_clean_build
logger.warn('Nothing is being built') from ..bootstrap import run_cmdline
prefix = os.environ['CONDA_ROOT'] from .build import build
logger.info('os.environ["%s"] = %s', 'CONDA_ROOT', prefix) build(
recipe_dir=[os.path.join(os.path.realpath(os.curdir), 'conda')],
workdir = os.environ['CI_PROJECT_DIR'] python=os.environ['PYTHON_VERSION'], #python version
logger.info('os.environ["%s"] = %s', 'CI_PROJECT_DIR', workdir) condarc=None, #custom build configuration
config=CONDA_BUILD_CONFIG,
pyver = os.environ['PYTHON_VERSION'] no_test=False,
logger.info('os.environ["%s"] = %s', 'PYTHON_VERSION', pyver) append_file=CONDA_RECIPE_APPEND,
server=SERVER,
set_environment('MATPLOTLIBRC', MATPLOTLIB_RCDIR, verbose=True) private=(os.environ['CI_PROJECT_VISIBILITY'] != 'public'),
stable='CI_COMMIT_TAG' in os.environ,
# get information about the version of the package being built dry_run=dry_run,
version, is_prerelease = check_version(workdir, verbosity=verbosity,
os.environ.get('CI_COMMIT_TAG')) )
set_environment('BOB_PACKAGE_VERSION', version, verbose=True)
# setup BOB_DOCUMENTATION_SERVER environment variable (used for bob.extension
# and derived documentation building via Sphinx)
set_environment('DOCSERVER', SERVER, os.environ, verbose=True)
public = ( os.environ['CI_PROJECT_VISIBILITY']=='public' )
doc_urls = get_docserver_setup(public=public, stable=(not is_prerelease),
server=SERVER, intranet=True)
set_environment('BOB_DOCUMENTATION_SERVER', doc_urls, verbose=True)
condarc = os.path.join(prefix, 'condarc')
logger.info('Loading (this build\'s) CONDARC file from %s...', condarc)
with open(condarc, 'rb') as f:
condarc_options = yaml.load(f)
# notice this condarc typically will only contain the defaults channel - we
# need to boost this up with more channels to get it right.
channels = get_channels(public=public, stable=(not is_prerelease),
server=SERVER, intranet=True)
logger.info('Using the following channels during build:\n - %s',
'\n - '.join(channels + ['defaults']))
condarc_options['channels'] = channels + ['defaults']
# dump packages at conda_root
condarc_options['croot'] = os.path.join(prefix, 'conda-bld')
# create the build configuration
logger.info('Merging conda configuration files...')
conda_config = make_conda_config(CONDA_BUILD_CONFIG, pyver,
CONDA_RECIPE_APPEND, condarc_options)
recipe_dir = os.path.join(workdir, 'conda')
if not os.path.exists(recipe_dir):
raise RuntimeError("The directory %s does not exist" % recipe_dir)
# pre-renders the recipe - figures out package name and version
metadata = get_rendered_metadata(recipe_dir, conda_config)
arch = conda_arch()
if should_skip_build(metadata):
logger.warn('Skipping UNSUPPORTED build of "%s" for py%s on %s',
d, python.replace('.',''), arch)
return 0
# converts the metadata output into parsed yaml and continues the process
rendered_recipe = get_parsed_recipe(metadata)
# retrieve the current build number for this build
build_number, _ = next_build_number(channels[0],
rendered_recipe['package']['name'],
rendered_recipe['package']['version'], python)
set_environment('BOB_BUILD_NUMBER', str(build_number), verbose=True)
# runs the build using the conda-build API
logger.info('Building %s-%s-py%s (build: %d) for %s',
rendered_recipe['package']['name'],
rendered_recipe['package']['version'], python.replace('.',''),
build_number, arch)
if not dry_run:
conda_build.api.build(recipe_dir, config=conda_config)
git_clean_build(run_cmdline, arch) git_clean_build(run_cmdline, arch)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment