Commit 80b4a8d1 authored by André Anjos's avatar André Anjos 💬

DRY

parent 1808f37b
Pipeline #26000 failed with stages
in 2 minutes and 18 seconds
......@@ -14,7 +14,7 @@ stages:
.build_template: &build_job
stage: build
before_script:
- python3 ./ci/bootstrap.py build
- python3 ./bob/devtools/bootstrap.py build
script:
- ./ci/build.sh
cache: &build_caches
......@@ -74,7 +74,7 @@ build_macosx_36:
.deploy_template: &deploy_job
stage: deploy
before_script:
- python3 ./ci/bootstrap.py local myenv
- python3 ./bob/devtools/bootstrap.py local myenv
script:
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- conda activate myenv
......@@ -115,7 +115,7 @@ pypi:
except:
- branches
before_script:
- python3 ./ci/bootstrap.py local myenv
- python3 ./bob/devtools/bootstrap.py local myenv
script:
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- conda activate myenv
......
This diff is collapsed.
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
'''Methods to create working environments based on conda-packages'''
import os
import json
import shutil
import subprocess
import logging
logger = logging.getLogger(__name__)
import yaml
def make_conda_config(config, python, append_file, condarc):
from conda_build.api import get_or_merge_config
from conda_build.conda_interface import url_path
with open(condarc, 'rb') as f:
condarc_options = yaml.load(f)
retval = get_or_merge_config(None, variant_config_files=config,
python=python, append_sections_file=append_file, **condarc_options)
retval.channel_urls = []
for url in condarc_options['channels']:
# allow people to specify relative or absolute paths to local channels
# These channels still must follow conda rules - they must have the
# appropriate platform-specific subdir (e.g. win-64)
if os.path.isdir(url):
if not os.path.isabs(url):
url = os.path.normpath(os.path.abspath(os.path.join(os.getcwd(), url)))
url = url_path(url)
retval.channel_urls.append(url)
return retval
def get_rendered_metadata(recipe_dir, config):
'''Renders the recipe and returns the interpreted YAML file'''
from conda_build.api import render
return render(recipe_dir, config=config)
def get_parsed_recipe(metadata):
'''Renders the recipe and returns the interpreted YAML file'''
from conda_build.api import output_yaml
output = output_yaml(metadata[0][0])
return yaml.load(output)
def remove_pins(deps):
return [l.split()[0] for l in deps]
def parse_dependencies(recipe_dir, config):
metadata = get_rendered_metadata(recipe_dir, config)
recipe = get_parsed_recipe(metadata)
return remove_pins(recipe['requirements'].get('build', [])) + \
remove_pins(recipe['requirements'].get('host', [])) + \
recipe['requirements'].get('run', []) + \
recipe.get('test', {}).get('requires', []) + \
['bob.buildout', 'mr.developer', 'ipdb']
# by last, packages required for local dev
def get_env_directory(conda, name):
cmd = [conda, 'env', 'list', '--json']
output = subprocess.check_output(cmd)
data = json.loads(output)
retval = [k for k in data.get('envs', []) if k.endswith(os.sep + name)]
if retval:
return retval[0]
return None
def conda_create(conda, name, overwrite, condarc, packages, dry_run, use_local):
specs = []
for k in packages:
k = ' '.join(k.split()[:2]) # remove eventual build string
if any(elem in k for elem in '><|'):
specs.append(k.replace(' ', ''))
else:
specs.append(k.replace(' ', '='))
# if the current environment exists, delete it first
envdir = get_env_directory(conda, name)
if envdir is not None:
if overwrite:
cmd = [conda, 'env', 'remove', '--yes', '--name', name]
logger.debug('$ ' + ' '.join(cmd))
if not dry_run:
status = subprocess.call(cmd)
if status != 0:
return status
else:
raise RuntimeError('environment `%s\' exists in `%s\' - use '
'--overwrite to overwrite' % (name, envdir))
cmd = [conda, 'create', '--yes', '--name', name]
if dry_run:
cmd.append('--dry-run')
if use_local:
cmd.append('--use-local')
cmd.extend(sorted(specs))
logger.debug('$ ' + ' '.join(cmd))
status = subprocess.call(cmd)
if status != 0:
return status
# copy the used condarc file to the just created environment
if not dry_run:
# get envdir again - it may just be created!
envdir = get_env_directory(conda, name)
destrc = os.path.join(envdir, '.condarc')
logger.debug('$ cp %s -> %s' % (condarc, destrc))
shutil.copy2(condarc, destrc)
return status
......@@ -12,7 +12,8 @@ import click
from . import bdt
from ..log import verbosity_option
from ..conda import next_build_number, osname
from ..bootstrap import get_rendered_metadata, get_parsed_recipe
from ..create import get_rendered_metadata, get_parsed_recipe, \
make_conda_config
from ..constants import CONDARC, CONDA_BUILD_CONFIG, CONDA_RECIPE_APPEND, \
SERVER, MATPLOTLIB_RCDIR, set_environment
......@@ -85,7 +86,6 @@ def build(recipe_dir, python, condarc, config, channel, no_test, append_file,
logger.debug("CONDARC=%s", condarc)
from ..bootstrap import make_conda_config
conda_config = make_conda_config(config, python, append_file, condarc)
set_environment('LANG', 'en_US.UTF-8', os.environ)
......
......@@ -12,7 +12,7 @@ import yaml
from . import bdt
from ..log import verbosity_option
from ..bootstrap import parse_dependencies, conda_create, make_conda_config
from ..create import parse_dependencies, conda_create, make_conda_config
from ..constants import CONDARC, CONDA_BUILD_CONFIG, CONDA_RECIPE_APPEND, \
SERVER
......@@ -24,7 +24,7 @@ Examples:
\b
$ cd bob.package.foo
$ bdt bootstrap -vv myenv
$ bdt create -vv myenv
The above command assumes the directory `conda' exists on the current directory and that it contains a file called `meta.yaml' containing the recipe for the package you want to create a development environment for.
......@@ -33,12 +33,12 @@ Examples:
2. By default, we use the native python version of your conda installation as the default python version to use for the newly created environment. You may select a different one with `--python=X.Y':
$ bdt bootstrap -vv --python=3.6 myenv
$ bdt create -vv --python=3.6 myenv
3. By default, we use our own condarc and `conda_build_config.yaml` files that are used in creating packages for our CI/CD system. If you wish to use your own, specify them on the command line:
$ bdt bootstrap -vv --python=3.6 --config=config.yaml --condarc=~/.condarc myenv
$ bdt create -vv --python=3.6 --config=config.yaml --condarc=~/.condarc myenv
Notice the condarc file **must** end in `condarc', or conda will complain.
......@@ -49,7 +49,7 @@ Examples:
shell will be printed:
$ bdt bootstrap -vvv --dry-run myenv
$ bdt create -vvv --dry-run myenv
''')
@click.argument('name')
@click.argument('recipe-dir', required=False, type=click.Path(file_okay=False,
......@@ -80,7 +80,7 @@ Examples:
help='Allow the use of local channels for package retrieval')
@verbosity_option()
@bdt.raise_on_error
def bootstrap(name, recipe_dir, python, overwrite, condarc, config,
def create(name, recipe_dir, python, overwrite, condarc, config,
append_file, docserver, dry_run, use_local):
"""Creates a development environment for a recipe
......
This diff is collapsed.
#!/usr/bin/env bash
# Bootstraps a new conda installation and prepares base environment
# if "local" is passed as parameter, then self installs an already built
# version of bob.devtools available on your conda-bld directory. If you pass
# "beta", then it bootstraps from the package installed on our conda beta
# channel. If you pass "stable", then it bootstraps installing the package
# available on the stable channel.
#
# If bootstrapping anything else than "build", then provide a second argument
# with the name of the environment that one wants to create with an
# installation of bob.devtools.
# datetime prefix for logging
log_datetime() {
echo "($(date +%T.%3N))"
}
# Functions for coloring echo commands
log_info() {
echo -e "$(log_datetime) \033[1;34m${@}\033[0m"
}
log_error() {
echo -e "$(log_datetime) \033[1;31mError: ${@}\033[0m" >&2
}
# Function for running command and echoing results
run_cmd() {
log_info "$ ${@}"
${@}
local status=$?
if [ ${status} != 0 ]; then
log_error "Command Failed \"${@}\""
exit ${status}
fi
}
# Checks just if the variable is defined and has non-zero length
check_defined() {
if [ -z "${!1+abc}" ]; then
log_error "Variable ${1} is undefined - aborting...";
exit 1
elif [ -z "${!1}" ]; then
log_error "Variable ${1} is zero-length - aborting...";
exit 1
fi
log_info "${1}=${!1}"
}
# merges conda cache folders
# $1: Path to the main cache to keep. The directory must exist.
# $2: Path to the extra cache to be merged into main cache
merge_conda_cache() {
if [ -e ${1} ]; then
_cached_urlstxt="${2}/pkgs/urls.txt"
_urlstxt="${1}/pkgs/urls.txt"
if [ -e ${2}/pkgs ]; then
log_info "Merging urls.txt and packages with cached files..."
mv ${2}/pkgs/*.tar.bz2 ${1}/pkgs
run_cmd rm -f ${1}/pkgs/${CI_PROJECT_NAME}-*-*_*.tar.bz2
cat ${_urlstxt} ${_cached_urlstxt} | sort | uniq > ${_urlstxt}
else
run_cmd mkdir -p ${1}/pkgs
run_cmd touch ${1}/pkgs/urls.txt
fi
run_cmd touch ${1}/pkgs/urls
if [ -d ${2}/conda-bld ]; then
log_info "Moving conda-bld packages (artifacts)..."
run_cmd mv ${2}/conda-bld ${1}
fi
fi
}
# installs a miniconda installation.
# $1: Path to where to install miniconda.
install_miniconda() {
log_info "Installing miniconda in ${1} ..."
# checks if miniconda.sh exists
if [ ! -e miniconda.sh ]; then
log_info "Downloading latest miniconda3 installer..."
# downloads the latest conda installation script
if [ "$(uname -s)" == "Linux" ]; then
_os="Linux"
else
_os="MacOSX"
fi
obj=https://repo.continuum.io/miniconda/Miniconda3-latest-${_os}-x86_64.sh
run_cmd curl --silent --output miniconda.sh ${obj}
else
log_info "Re-using cached miniconda3 installer..."
ls -l miniconda.sh
fi
# move cache to a different folder if it exists
if [ -e ${1} ]; then
run_cmd mv ${1} ${1}.cached
fi
# install miniconda
run_cmd bash miniconda.sh -b -p ${1}
# Put back cache and merge urls.txt
merge_conda_cache ${1} ${1}.cached
# remove the backup cache folder
rm -rf ${1}.cached
# List currently available packages on cache
# run_cmd ls -l ${1}/pkgs/
# run_cmd cat ${1}/pkgs/urls.txt
hash -r
}
check_defined CONDA_ROOT
check_defined CI_PROJECT_DIR
export CONDARC="${CONDA_ROOT}/condarc"
check_defined CONDARC
# checks if a conda installation exists. Otherwise, installs one
if [ ! -e ${CONDA_ROOT}/bin/conda ]; then
install_miniconda ${CONDA_ROOT}
fi
run_cmd cp -fv ${CI_PROJECT_DIR}/bob/devtools/data/base-condarc ${CONDARC}
echo "Contents of \`${CONDARC}':"
cat ${CONDARC}
# setup conda-channels
CONDA_CHANNEL_ROOT="http://www.idiap.ch/public/conda"
check_defined CONDA_CHANNEL_ROOT
CONDA_CLI_CHANNELS="-c ${CONDA_CHANNEL_ROOT} -c defaults"
# creates a base installation depending on the purpose
if [ "${1}" == "build" ]; then
run_cmd ${CONDA_ROOT}/bin/conda install -n base python conda=4 conda-build=3
elif [ "${1}" == "local" ]; then
# updates the base installation, installs conda-build
run_cmd ls -l ${CONDA_ROOT}/conda-bld
run_cmd ${CONDA_ROOT}/bin/conda install -n base python conda=4 conda-build=3
CONDA_CLI_CHANNELS="-c ${CONDA_ROOT}/conda-bld ${CONDA_CLI_CHANNELS}"
run_cmd ${CONDA_ROOT}/bin/conda index ${CONDA_ROOT}/conda-bld
run_cmd ls -l ${CONDA_ROOT}/conda-bld
run_cmd ls -l ${CONDA_ROOT}/conda-bld/noarch/
run_cmd ${CONDA_ROOT}/bin/conda create -n "${2}" --override-channels ${CONDA_CLI_CHANNELS} bob.devtools
elif [ "${1}" == "beta" ] || [ "${1}" == "stable" ]; then
run_cmd ${CONDA_ROOT}/bin/conda create -n "${2}" --override-channels ${CONDA_CLI_CHANNELS} bob.devtools
else
log_error "Bootstrap with 'build', or 'local|beta|stable <name>'"
log_error "The value '${1}' is not currently supported"
exit 1
fi
# cleans up
run_cmd ${CONDA_ROOT}/bin/conda clean --lock
# print conda information for debugging purposes
run_cmd ${CONDA_ROOT}/bin/conda info
......@@ -61,7 +61,7 @@ test:
- bdt release --help
- bdt visibility --help
- bdt dumpsphinx --help
- bdt bootstrap --help
- bdt create --help
- bdt build --help
- bdt getpath --help
- bdt caupdate --help
......
......@@ -12,7 +12,9 @@
bob.devtools.constants
bob.devtools.release
bob.devtools.changelog
bob.devtools.create
bob.devtools.bootstrap
bob.devtools.build
bob.devtools.webdav3.client
......@@ -31,8 +33,12 @@ Detailed Information
.. automodule:: bob.devtools.changelog
.. automodule:: bob.devtools.create
.. automodule:: bob.devtools.bootstrap
.. automodule:: bob.devtools.build
WebDAV Python Client
--------------------
......
......@@ -48,7 +48,7 @@ setup(
'lasttag = bob.devtools.scripts.lasttag:lasttag',
'visibility = bob.devtools.scripts.visibility:visibility',
'dumpsphinx = bob.devtools.scripts.dumpsphinx:dumpsphinx',
'bootstrap = bob.devtools.scripts.bootstrap:bootstrap',
'create = bob.devtools.scripts.create:create',
'build = bob.devtools.scripts.build:build',
'getpath = bob.devtools.scripts.getpath:getpath',
'caupdate = bob.devtools.scripts.caupdate:caupdate',
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment