Commit 8b2a97da authored by André Anjos's avatar André Anjos 💬
Browse files

DRY; Move boot/build from ci dir into the package

parent 80b4a8d1
Pipeline #26013 failed with stages
in 3 minutes and 5 seconds
......@@ -16,7 +16,9 @@ stages:
before_script:
- python3 ./bob/devtools/bootstrap.py build
script:
- ./ci/build.sh
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- conda activate base
- python3 ./bob/devtools/build.py
cache: &build_caches
paths:
- miniconda.sh
......
include LICENSE README.rst buildout.cfg version.txt
recursive-include doc conf.py *.rst *.sh
recursive-include bob/devtools/data *.md *.yaml *condarc *.pem matplotlibrc
recursive-include bob/devtools/data *.md *.yaml *.pem matplotlibrc
......@@ -24,7 +24,7 @@ Arguments:
'''
BASE_CONDARC = '''\
_BASE_CONDARC = '''\
default_channels:
- https://repo.anaconda.com/pkgs/main
- https://repo.anaconda.com/pkgs/free
......@@ -39,6 +39,17 @@ anaconda_upload: false #!final
ssl_verify: false #!final
'''
_SERVER = 'http://www.idiap.ch'
_INTERVALS = (
('weeks', 604800), # 60 * 60 * 24 * 7
('days', 86400), # 60 * 60 * 24
('hours', 3600), # 60 * 60
('minutes', 60),
('seconds', 1),
)
'''Time intervals that make up human readable time slots'''
import os
import sys
......@@ -49,16 +60,25 @@ import platform
import subprocess
import logging
logger = logging.getLogger('bootstrap')
logger = logging.getLogger(__name__)
_INTERVALS = (
('weeks', 604800), # 60 * 60 * 24 * 7
('days', 86400), # 60 * 60 * 24
('hours', 3600), # 60 * 60
('minutes', 60),
('seconds', 1),
)
def set_environment(name, value, env=os.environ):
'''Function to setup the environment variable and print debug message
Args:
name: The name of the environment variable to set
value: The value to set the environment variable to
env: Optional environment (dictionary) where to set the variable at
'''
if name in env:
logger.warn('Overriding existing environment variable ${%s} (was: "%s")',
name, env[name])
env[name] = value
logger.debug('$ export %s="%s"', name, value)
def human_time(seconds, granularity=2):
'''Returns a human readable time string like "1 day, 2 hours"'''
......@@ -228,7 +248,7 @@ def install_miniconda(prefix):
shutil.rmtree(cached)
def get_channels(public, stable):
def get_channels(public, stable, server, intranet):
'''Returns the relevant conda channels to consider if building project
The subset of channels to be returned depends on the visibility and stability
......@@ -249,22 +269,31 @@ def get_channels(public, stable):
channels
stable: Boolean indicating if we're supposed to include only stable
channels
server: The base address of the server containing our conda channels
intranet: Boolean indicating if we should add "private"/"public" prefixes
on the conda paths
Returns: a list of channels that need to be considered.
'''
server = "http://www.idiap.ch"
channels = []
if (not public) and (not intranet):
raise RuntimeError('You cannot request for private channels and set' \
' intranet=False (server=%s) - these are conflicting options' % server)
if not public:
prefix = '/private' if intranet else ''
if not stable: #allowed private channels
channels += [server + '/private/conda/label/beta'] #allowed betas
channels += [server + '/private/conda']
channels += [server + prefix + '/conda/label/beta'] #allowed betas
channels += [server + prefix + '/conda']
prefix = '/public' if intranet else ''
if not stable:
channels += [server + '/public/conda/label/beta'] #allowed betas
channels += [server + '/public/conda']
channels += [server + prefix + '/conda/label/beta'] #allowed betas
channels += [server + prefix + '/conda']
return channels
......@@ -278,10 +307,10 @@ def add_channels_condarc(channels, condarc):
f.write(' - %s\n' % k)
with open(condarc, 'rt') as f:
logger.info('Contents of $CONDARC:\n%s', f.read())
logger.info('Contents of installed CONDARC:\n%s', f.read())
def setup_logger():
def setup_logger(logger):
'''Sets-up the logging for this command at level ``INFO``'''
warn_err = logging.StreamHandler(sys.stderr)
......@@ -313,14 +342,14 @@ if __name__ == '__main__':
print(__doc__ % sys.argv[0])
sys.exit(1)
setup_logger()
setup_logger(logger)
if sys.argv[1] == 'test':
# sets up local variables for testing
os.environ['CI_PROJECT_DIR'] = os.path.realpath(os.curdir)
os.environ['CI_PROJECT_NAME'] = 'bob.devtools'
os.environ['CONDA_ROOT'] = os.path.join(os.environ['CI_PROJECT_DIR'],
'miniconda')
set_environment('CI_PROJECT_DIR', os.path.realpath(os.curdir))
set_environment('CI_PROJECT_NAME', 'bob.devtools')
set_environment('CONDA_ROOT', os.path.join(os.environ['CI_PROJECT_DIR'],
'miniconda'))
prefix = os.environ['CONDA_ROOT']
logger.info('os.environ["%s"] = %s', 'CONDA_ROOT', prefix)
......@@ -335,9 +364,7 @@ if __name__ == '__main__':
condarc = os.path.join(prefix, 'condarc')
logger.info('(create) %s', condarc)
with open(condarc, 'wt') as f:
f.write(BASE_CONDARC)
os.environ['CONDARC'] = condarc
logger.info('os.environ["%s"] = %s', 'CONDARC', condarc)
f.write(_BASE_CONDARC)
conda_version = '4'
conda_build_version = '3'
......@@ -363,7 +390,8 @@ if __name__ == '__main__':
conda_bld_path = os.path.join(prefix, 'conda-bld')
run_cmdline([conda_bin, 'index', conda_bld_path])
# add the locally build directory before defaults, boot from there
channels = get_channels(public=True, stable=True)
channels = get_channels(public=True, stable=True, server=_SERVER,
intranet=True)
add_channels_condarc(channels + [conda_bld_path, 'defaults'], condarc)
run_cmdline([conda_bin, 'create', '-n', sys.argv[2], 'bob.devtools'])
......@@ -372,7 +400,8 @@ if __name__ == '__main__':
# installs from channel
channels = get_channels(
public=os.environ['CI_PROJECT_VISIBILITY'] == 'public',
stable=os.environ.get('CI_COMMIT_TAG') is not None)
stable=os.environ.get('CI_COMMIT_TAG') is not None,
server=_SERVER, intranet=True)
add_channels_condarc(channels + ['defaults'], condarc)
run_cmdline([conda_bin, 'create', '-n', sys.argv[2], 'bob.devtools'])
......
#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''This is a copy of bob/devtools/conda:next_build_number with a CLI'''
'''Tools for self-building and other utilities
This script, if called in standalone format, can be used to build the current
package. It contains various functions and utilities that can be used by
modules inside the package itself. It assumes a base installation for the
build is operational (i.e., the python package for ``conda-build`` is
installed).
'''
import os
import re
import sys
import json
import shutil
import platform
import subprocess
from conda.exports import get_index
import logging
logger = logging.getLogger(__name__)
import yaml
import packaging
import conda_build.api
if __name__ == '__main__':
channel_url = sys.argv[1]
name = sys.argv[2]
version = sys.argv[3]
python = sys.argv[4]
def osname():
"""Returns the current OS name as recognized by conda"""
r = 'unknown'
if platform.system().lower() == 'linux':
r = 'linux'
elif platform.system().lower() == 'darwin':
r = 'osx'
else:
raise RuntimeError('Unsupported system "%s"' % platform.system())
if platform.machine().lower() == 'x86_64':
r += '-64'
else:
raise RuntimeError('Unsupported machine type "%s"' % platform.machine())
return r
def should_skip_build(metadata_tuples):
"""Takes the output of render_recipe as input and evaluates if this
recipe's build should be skipped.
"""
return all(m[0].skip() for m in metadata_tuples)
def next_build_number(channel_url, name, version, python):
"""Calculates the next build number of a package given the channel
This function returns the next build number (integer) for a package given its
recipe, dependencies, name, version and python version. It looks on the
channel URL provided and figures out if any clash would happen and what would
be the highest build number available for that configuration.
Args:
channel_url: The URL where to look for packages clashes (normally a beta
channel)
name: The name of the package
version: The version of the package
python: The version of python as 2 digits (e.g.: "2.7" or "3.6")
Returns: The next build number with the current configuration. Zero (0) is
returned if no match is found. Also returns the URLs of the packages it
finds with matches on the name, version and python-version.
"""
from conda.exports import get_index
# no dot in py_ver
py_ver = python.replace('.', '')
# get the channel index
logger.debug('Downloading channel index from %s', channel_url)
index = get_index(channel_urls=[channel_url], prepend=False)
# search if package with the same version exists
......@@ -32,9 +94,225 @@ if __name__ == '__main__':
match = re.match('py[2-9][0-9]+', dist.build_string)
if match and match.group() == 'py{}'.format(py_ver):
logger.debug("Found match at %s for %s-%s-py%s", index[dist].url,
name, version, py_ver)
build_number = max(build_number, dist.build_number + 1)
urls.append(index[dist].url)
urls = [url.replace(channel_url, '') for url in urls]
print(build_number)
return build_number, urls
def make_conda_config(config, python, append_file, condarc_options):
from conda_build.api import get_or_merge_config
from conda_build.conda_interface import url_path
retval = get_or_merge_config(None, variant_config_files=config,
python=python, append_sections_file=append_file, **condarc_options)
retval.channel_urls = []
for url in condarc_options['channels']:
# allow people to specify relative or absolute paths to local channels
# These channels still must follow conda rules - they must have the
# appropriate platform-specific subdir (e.g. win-64)
if os.path.isdir(url):
if not os.path.isabs(url):
url = os.path.normpath(os.path.abspath(os.path.join(os.getcwd(), url)))
url = url_path(url)
retval.channel_urls.append(url)
return retval
def get_rendered_metadata(recipe_dir, config):
'''Renders the recipe and returns the interpreted YAML file'''
from conda_build.api import render
return render(recipe_dir, config=config)
def get_parsed_recipe(metadata):
'''Renders the recipe and returns the interpreted YAML file'''
from conda_build.api import output_yaml
output = output_yaml(metadata[0][0])
return yaml.load(output)
def remove_pins(deps):
return [l.split()[0] for l in deps]
def parse_dependencies(recipe_dir, config):
metadata = get_rendered_metadata(recipe_dir, config)
recipe = get_parsed_recipe(metadata)
return remove_pins(recipe['requirements'].get('build', [])) + \
remove_pins(recipe['requirements'].get('host', [])) + \
recipe['requirements'].get('run', []) + \
recipe.get('test', {}).get('requires', []) + \
['bob.buildout', 'mr.developer', 'ipdb']
# by last, packages required for local dev
def get_env_directory(conda, name):
cmd = [conda, 'env', 'list', '--json']
output = subprocess.check_output(cmd)
data = json.loads(output)
retval = [k for k in data.get('envs', []) if k.endswith(os.sep + name)]
if retval:
return retval[0]
return None
def conda_create(conda, name, overwrite, condarc, packages, dry_run, use_local):
'''Creates a new conda environment following package specifications
This command can create a new conda environment following the list of input
packages. It will overwrite an existing environment if indicated.
Args:
conda: path to the main conda executable of the installation
name: the name of the environment to create or overwrite
overwrite: if set to ```True``, overwrite potentially existing environments
with the same name
condarc: a dictionary of options for conda, including channel urls
packages: the package list specification
dry_run: if set, then don't execute anything, just print stuff
use_local: include the local conda-bld directory as a possible installation
channel (useful for testing multiple interdependent recipes that are
built locally)
'''
from .bootstrap import run_cmdline
specs = []
for k in packages:
k = ' '.join(k.split()[:2]) # remove eventual build string
if any(elem in k for elem in '><|'):
specs.append(k.replace(' ', ''))
else:
specs.append(k.replace(' ', '='))
# if the current environment exists, delete it first
envdir = get_env_directory(conda, name)
if envdir is not None:
if overwrite:
cmd = [conda, 'env', 'remove', '--yes', '--name', name]
logger.debug('$ ' + ' '.join(cmd))
if not dry_run:
run_cmdline(cmd)
else:
raise RuntimeError('environment `%s\' exists in `%s\' - use '
'--overwrite to overwrite' % (name, envdir))
cmdline_channels = ['--channel=%s' % k for k in condarc['channels']]
cmd = [conda, 'create', '--yes', '--name', name, '--override-channels'] + \
cmdline_channels
if dry_run:
cmd.append('--dry-run')
if use_local:
cmd.append('--use-local')
cmd.extend(sorted(specs))
run_cmdline(cmd)
# creates a .condarc file to sediment the just created environment
if not dry_run:
# get envdir again - it may just be created!
envdir = get_env_directory(conda, name)
destrc = os.path.join(envdir, '.condarc')
logger.info('Creating %s...', destrc)
with open(destrc, 'w') as f:
yaml.dump(condarc, f, indent=2)
if __name__ == '__main__':
# loads the "adjacent" bootstrap module
import importlib.util
mydir = os.path.dirname(os.path.realpath(sys.argv[0]))
bootstrap_file = os.path.join(mydir, 'bootstrap.py')
spec = importlib.util.spec_from_file_location("bootstrap", bootstrap_file)
bootstrap = importlib.util.module_from_spec(spec)
spec.loader.exec_module(bootstrap)
bootstrap.setup_logger(logger)
prefix = os.environ['CONDA_ROOT']
logger.info('os.environ["%s"] = %s', 'CONDA_ROOT', prefix)
workdir = os.environ['CI_PROJECT_DIR']
logger.info('os.environ["%s"] = %s', 'CI_PROJECT_DIR', workdir)
name = os.environ['CI_PROJECT_NAME']
logger.info('os.environ["%s"] = %s', 'CI_PROJECT_NAME', name)
pyver = os.environ['PYTHON_VERSION']
logger.info('os.environ["%s"] = %s', 'PYTHON_VERSION', pyver)
set_environment('DOCSERVER', bootstrap._SERVER, os.environ)
set_environment('LANG', 'en_US.UTF-8', os.environ)
set_environment('LC_ALL', os.environ['LANG'], os.environ)
# create the build configuration
conda_build_config = os.path.join(mydir, 'data', 'conda_build_config.yaml')
recipe_append = os.path.join(mydir, 'data', 'recipe_append.yaml')
logger.info('Merging conda configuration files...')
condarc = os.path.join(prefix, 'condarc')
logger.info('Loading (this build\'s) CONDARC file from %s...', condarc)
with open(condarc, 'rb') as f:
condarc_options = yaml.load(f)
conda_config = make_conda_config(conda_build_config, pyver, recipe_append,
condarc_options)
version = open("version.txt").read().rstrip()
os.environ['BOB_PACKAGE_VERSION'] = version
logger.info('os.environ["%s"] = %s', 'BOB_PACKAGE_VERSION', version)
# if we're build a stable release, ensure a tag is set
parsed_version = packaging.version.Version(version)
if parsed_version.is_prerelease:
if os.environ.get('CI_COMMIT_TAG') is not None:
raise EnvironmentError('"version.txt" indicates version is a ' \
'pre-release (v%s) - but os.environ["CI_COMMIT_TAG"]="%s", ' \
'which indicates this is a **stable** build. ' \
'Have you created the tag using ``bdt release``?', version,
os.environ['CI_COMMIT_TAG'])
else: #it is a stable build
if os.environ.get('CI_COMMIT_TAG') is None:
raise EnvironmentError('"version.txt" indicates version is a ' \
'stable build (v%s) - but there is no os.environ["CI_COMMIT_TAG"] ' \
'variable defined, which indicates this is **not** ' \
'a tagged build. Use ``bdt release`` to create stable releases',
version)
build_number = next_build_number(channels[0], name, version, python)
os.environ['BOB_BUILD_NUMBER'] = build_number
logger.info('os.environ["%s"] = %s', 'BOB_BUILD_NUMBER', build_number)
# runs the build using the conda-build API
arch = osname()
logger.info('Building %s-%s-py%s (build: %d) for %s',
rendered_recipe['package']['name'],
rendered_recipe['package']['version'], pyver.replace('.',''),
build_number, arch)
conda_build.api.build(d, config=conda_config)
# runs git clean to clean everything that is not needed. This helps to keep
# the disk usage on CI machines to a minimum.
exclude_from_cleanup = [
"miniconda.sh", #the installer, cached
"miniconda/pkgs/*.tar.bz2", #downloaded packages, cached
"miniconda/pkgs/urls.txt", #download index, cached
"miniconda/conda-bld/${_os}-64/*.tar.bz2", #build artifact -- conda
"dist/*.zip", #build artifact -- pypi package
"sphinx", #build artifact -- documentation
]
bootstrap.run_cmdline(['git', 'clean', '-ffdx'] + \
['--exclude=%s' % k for k in exclude_from_cleanup])
......@@ -9,79 +9,3 @@ import platform
logger = logging.getLogger(__name__)
def osname():
"""Returns the current OS name as recognized by conda"""
r = 'unknown'
if platform.system().lower() == 'linux':
r = 'linux'
elif platform.system().lower() == 'darwin':
r = 'osx'
else:
raise RuntimeError('Unsupported system "%s"' % platform.system())
if platform.machine().lower() == 'x86_64':
r += '-64'
else:
raise RuntimeError('Unsupported machine type "%s"' % platform.machine())
return r
def should_skip_build(metadata_tuples):
"""Takes the output of render_recipe as input and evaluates if this
recipe's build should be skipped.
"""
return all(m[0].skip() for m in metadata_tuples)
def next_build_number(channel_url, name, version, python):
"""Calculates the next build number of a package given the channel
This function returns the next build number (integer) for a package given its
recipe, dependencies, name, version and python version. It looks on the
channel URL provided and figures out if any clash would happen and what would
be the highest build number available for that configuration.
Args:
channel_url: The URL where to look for packages clashes (normally a beta
channel)
name: The name of the package
version: The version of the package
python: The version of python as 2 digits (e.g.: "2.7" or "3.6")
Returns: The next build number with the current configuration. Zero (0) is
returned if no match is found. Also returns the URLs of the packages it
finds with matches on the name, version and python-version.
"""
from conda.exports import get_index
# no dot in py_ver
py_ver = python.replace('.', '')
# get the channel index
logger.debug('Downloading channel index from %s', channel_url)
index = get_index(channel_urls=[channel_url], prepend=False)
# search if package with the same version exists
build_number = 0
urls = []
for dist in index:
if dist.name == name and dist.version == version:
match = re.match('py[2-9][0-9]+', dist.build_string)
if match and match.group() == 'py{}'.format(py_ver):
logger.debug("Found match at %s for %s-%s-py%s", index[dist].url,
name, version, py_ver)
build_number = max(build_number, dist.build_number + 1)
urls.append(index[dist].url)
urls = [url.replace(channel_url, '') for url in urls]
return build_number, urls
......@@ -9,10 +9,11 @@ import pkg_resources
import logging
logger = logging.getLogger(__name__)
from . import bootstrap
CONDARC = pkg_resources.resource_filename(__name__,
os.path.join('data', 'build-condarc'))
'''The .condarc to use for building and creating new environments'''
BASE_CONDARC = bootstrap._BASE_CONDARC
'''Default setup for conda builds'''
CONDA_BUILD_CONFIG = pkg_resources.resource_filename(__name__,
......@@ -25,23 +26,10 @@ CONDA_RECIPE_APPEND = pkg_resources.resource_filename(__name__,
'''Extra information to be appended to every recipe upon building'''
SERVER = 'http://www.idiap.ch'
SERVER = bootstrap._SERVER
'''This is the default server use use to store data and build artifacts'''
CONDA_CHANNELS = {
True: { #stable?
False: '/private/conda', #visible outside?
True: '/public/conda',
},