Skip to content
Snippets Groups Projects
Commit 76d35e95 authored by André Anjos's avatar André Anjos :speech_balloon:
Browse files

[scripts] Add bootstrap support

parent bd190477
No related branches found
No related tags found
No related merge requests found
Pipeline #25719 failed
Stage: build
Stage: deploy
Hide whitespace changes
Inline Side-by-side
Showing
with 459 additions and 447 deletions
MANIFEST.in
+ 1
− 1
View file @ 76d35e95
include LICENSE README.rst buildout.cfg version.txt include LICENSE README.rst buildout.cfg version.txt
recursive-include doc conf.py *.rst recursive-include doc conf.py *.rst
recursive-include bob/devtools/data *.md recursive-include bob/devtools/data *.md *.yaml buid-condarc
bob/devtools/bootstrap.py 0 → 100755
+ 102
− 0
View file @ 76d35e95
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
import os
import json
import shutil
import subprocess
import logging
logger = logging.getLogger(__name__)
import yaml
from conda_build.api import get_or_merge_config, render, output_yaml
def get_rendered_recipe(conda, recipe_dir, python, config):
'''Renders the recipe and returns the interpreted YAML file'''
# equivalent command execute - in here we use the conda API
cmd = [
conda, 'render',
'--variant-config-files', config,
'--python', python,
recipe_dir,
]
logger.debug('$ ' + ' '.join(cmd))
# do the real job
config = get_or_merge_config(None, variant_config_files=config,
python=python)
metadata = render(recipe_dir, config=config)
output = output_yaml(metadata[0][0])
return yaml.load(output)
def remove_pins(deps):
return [l.split()[0] for l in deps]
def parse_dependencies(conda, recipe_dir, python, config):
recipe = get_rendered_recipe(conda, recipe_dir, python, config)
return remove_pins(recipe['requirements'].get('build', [])) + \
remove_pins(recipe['requirements'].get('host', [])) + \
recipe['requirements'].get('run', []) + \
recipe.get('test', {}).get('requires', []) + \
['bob.buildout', 'mr.developer', 'ipdb']
# by last, packages required for local dev
def get_env_directory(conda, name):
cmd = [conda, 'env', 'list', '--json']
output = subprocess.check_output(cmd)
data = json.loads(output)
retval = [k for k in data.get('envs', []) if k.endswith(os.sep + name)]
if retval:
return retval[0]
return None
def conda_create(conda, name, overwrite, condarc, packages, dry_run):
specs = []
for k in packages:
k = ' '.join(k.split()[:2]) # remove eventual build string
if any(elem in k for elem in '><|'):
specs.append(k.replace(' ', ''))
else:
specs.append(k.replace(' ', '='))
# if the current environment exists, delete it first
envdir = get_env_directory(conda, name)
if envdir is not None:
if overwrite:
cmd = [conda, 'env', 'remove', '--yes', '--name', name]
logger.debug('$ ' + ' '.join(cmd))
if not dry_run:
status = subprocess.call(cmd)
if status != 0:
return status
else:
raise RuntimeError('environment `%s\' exists in `%s\' - use '
'--overwrite to overwrite' % (name, envdir))
cmd = [conda, 'create', '--yes', '--name', name] + sorted(specs)
if dry_run:
cmd.insert(2, '--dry-run')
logger.debug('$ ' + ' '.join(cmd))
status = subprocess.call(cmd)
if status != 0:
return status
# copy the used condarc file to the just created environment
if not dry_run:
# get envdir again - it may just be created!
envdir = get_env_directory(conda, name)
destrc = os.path.join(envdir, '.condarc')
logger.debug('$ cp %s -> %s' % (condarc, destrc))
shutil.copy2(condarc, destrc)
return status
bob/devtools/data/build-condarc 0 → 100644
+ 14
− 0
View file @ 76d35e95
default_channels:
- https://repo.anaconda.com/pkgs/main
- https://repo.anaconda.com/pkgs/free
- https://repo.anaconda.com/pkgs/r
- https://repo.anaconda.com/pkgs/pro
add_pip_as_python_dependency: false
show_channel_urls: true
anaconda_upload: false
ssl_verify: false
quiet: true
channels:
- https://www.idiap.ch/software/bob/conda/label/beta
- https://www.idiap.ch/software/bob/conda
- defaults
bob/devtools/data/conda_build_config.yaml 0 → 100644
+ 223
− 0
View file @ 76d35e95
macos_min_version:
- 10.9
macos_machine:
- x86_64-apple-darwin13.4.0
MACOSX_DEPLOYMENT_TARGET:
- 10.9
CONDA_BUILD_SYSROOT: # [osx]
- /opt/MacOSX10.9.sdk # [osx]
# This helps CMAKE find the sysroot. See
# https://cmake.org/cmake/help/v3.11/variable/CMAKE_OSX_SYSROOT.html
SDKROOT: # [osx]
- /opt/MacOSX10.9.sdk # [osx]
# makes autotools verbose
VERBOSE_AT:
- V=1
# makes cmake verbose
VERBOSE_CM:
- VERBOSE=1
# the blas implementations that we build against
blas_impl:
- mkl
pin_run_as_build:
libboost:
max_pin: x.x.x
py_boost:
max_pin: x.x.x
boost:
max_pin: x.x.x
## the dependencies that we build against multiple versions
python:
- 3.6
zip_keys:
- # [win]
- vc # [win]
- c_compiler # [win]
- cxx_compiler # [win]
- fortran_compiler_version # [win]
- python # [win]
# Here is the version of dependencies are used when building packages (build
# and host requirements). We keep a list of **all of them** here to make sure
# everything goes as expected in our conda build process. For the version of
# packages that are used for testing packages, see the recipe of bob-devel.
# The version here do not necessarily match the versions in bob-devel.
# This version of bob-devel will be used at test time of packages:
bob_devel:
- 2018.12.29
# This version of beat-devel will be used at test time of packages. Notice it
# uses bob-devel and should have a version that is greater or equal its value
beat_devel:
- 2018.12.29
# The build time only dependencies (build requirements).
# Updating these to the latest version all the time is OK and a good idea.
# These versions should match the versions inside bob-devel as well (if they
# overlap) so update them in both places.
cmake:
- 3.12.2
pkg_config:
- 0.29.2
cython:
- 0.28.1
# The dependencies that we link against (host requirements).
# Ideally we want to build against the oldest possible version of packages when
# we are linking against them. It is best to keep this in sync with:
# https://github.com/AnacondaRecipes/aggregate/blob/master/conda_build_config.yaml
numpy:
- 1.14.5
boost:
- 1.65.1
cyvlfeat:
- 0.4.6
ffmpeg:
- 3.4
freetype:
- 2.8
giflib:
- 5.1.4
hdf5:
- 1.10.1
jpeg:
- 9b
libblitz:
- 1.0.1
libmatio:
- 1.5.11
libogg:
- 1.3.2
libpng:
- 1.6.35
libsvm:
- 3.22
libtiff:
- 4.0.9
mkl:
- 2018.0.3
openfst:
- 1.6.1
sox:
- 14.4.2
speex:
- 1.2.0
speexdsp:
- 1.2rc3
sqlite:
- 3.20.1
vlfeat:
- 0.9.21
xz:
- 5.2.3
zlib:
- 1.2.11
# The dependencies that are needed for runtime only (run requirements).
# These versions **should** match the versions inside bob-devel recipe.
caffe:
- 1.0
click:
- 6.7
click_plugins:
- 1.0.3
coverage:
- 4.5.1
dlib:
- 19.7
docopt:
- 0.6.2
jinja2:
- 2.10
kaldi:
- 2017.03.13
matplotlib:
- 2.2.3
menpo:
- 0.8.1
menpofit:
- 0.5.0
mne:
- 0.15.2
mr_developer:
- 1.38
nose:
- 1.3.7
opencv:
- 3.3.1
pillow:
- 5.2.0
pyedflib:
- 0.1.11
pytorch:
- 0.4.1
pyyaml:
- 3.13
requests:
- 2.19.1
schema:
- 0.6.7
scikit_image:
- 0.14
scikit_learn:
- 0.19.2
scipy:
- 1.1.0
setuptools:
- 40.2.0
six:
- 1.11.0
sphinx:
- 1.8.1
sphinx_rtd_theme:
- 0.4.1
sqlalchemy:
- 1.2.11
tabulate:
- 0.8.2
tensorflow:
- 1.9.0
torchvision:
- 0.2.1
zc_buildout:
- 2.12.2
zc_recipe_egg:
- 2.0.7
# The dependencies that are needed for runtime only (run requirements) of BEAT
# packages. These versions **should** match the versions inside beat-devel
# recipe.
docker_py:
- 3.6.0
jsonschema:
- 2.6.0
oset:
- 0.1.3
python_graphviz:
- 0.8.4
pyzmq:
- 17.1.2
simplejson:
- 3.16.0
termcolor:
- 1.1.0
sphinxcontrib_programoutput:
- 0.11
sphinxcontrib_httpdomain:
- 1.7.0
nodejs:
- 8.9.3
flask:
- 1.0.2
flask_cors:
- 3.0.7
flask_restful:
- 0.3.6
psycopg2:
- 2.7.6.1
bob/devtools/scripts/bootstrap.py 0 → 100644
+ 113
− 0
View file @ 76d35e95
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import sys
import logging
logger = logging.getLogger(__name__)
import pkg_resources
import click
from . import bdt
from ..log import verbosity_option
from ..bootstrap import parse_dependencies, conda_create
DEFAULT_CONDARC = pkg_resources.resource_filename(__name__,
os.path.join('..', 'data', 'build-condarc'))
DEFAULT_VARIANT = pkg_resources.resource_filename(__name__,
os.path.join('..', 'data', 'conda_build_config.yaml'))
@click.command(epilog='''
Examples:
1. Creates an environment called `myenv' for developing the currently checked-out package (N.B.: first activate the base environment):
\b
$ cd bob.package.foo
$ bdt bootstrap -vv myenv
The above command assumes the directory `conda' exists on the current directory and that it contains a file called `meta.yaml' containing the recipe for the package you want to create a development environment for.
If you get things right by default, the above form is the typical usage scenario of this app. Read-on to tweak special flags and settings.
2. By default, we use the native python version of your conda installation as the default python version to use for the newly created environment. You may select a different one with `--python=X.Y':
$ bdt bootstrap -vv --python=3.6 myenv
3. By default, we use our own condarc and `conda_build_config.yaml` files that are used in creating packages for our CI/CD system. If you wish to use your own, specify them on the command line:
$ bdt bootstrap -vv --python=3.6 --config=config.yaml --condarc=~/.condarc myenv
Notice the condarc file **must** end in `condarc', or conda will complain.
4. You can use the option `--dry-run' to simulate what would be installed
instead of actually creating a new environment. Combine with `-vvv` to
enable debug printing. Equivalent conda commands you can execute on the
shell will be printed:
$ bdt bootstrap -vvv --dry-run myenv
''')
@click.argument('name')
@click.argument('recipe-dir', required=False, type=click.Path(file_okay=False,
dir_okay=True, exists=True))
@click.option('-p', '--python', default=('%d.%d' % sys.version_info[:2]),
show_default=True, help='Version of python to build the ' \
'environment for [default: %(default)s]')
@click.option('-o', '--overwrite/--no-overwrite', default=False,
help='If set and an environment with the same name exists, ' \
'deletes it first before creating the new environment',
show_default=True)
@click.option('-r', '--condarc', default=DEFAULT_CONDARC, show_default=True,
help='overwrites the path leading to the condarc file to use',)
@click.option('-m', '--config', '--variant-config-files', show_default=True,
default=DEFAULT_VARIANT, help='overwrites the path leading to ' \
'variant configuration file to use')
@click.option('-d', '--dry-run/--no-dry-run', default=False,
help='Only goes through the actions, but does not execute them ' \
'(combine with the verbosity flags - e.g. ``-vvv``) to enable ' \
'printing to help you understand what will be done')
@verbosity_option()
@bdt.raise_on_error
def bootstrap(name, recipe_dir, python, overwrite, condarc, config, dry_run):
"""This program uses conda to build a development environment for a recipe
It uses the conda render API to render a recipe and install an environment
containing all build/host, run and test dependencies of a package. It does
**not** build the package itself, just install dependencies so you can build
the package by hand, possibly using buildout or similar. If you'd like to
conda-build your package, just use `conda build` instead.
Once the environment is created, a copy of the used `condarc' file is placed
on the root of the environment. Installing or updating packages on the newly
created environment should be possible without further configuration. Notice
that beta packages quickly get outdated and upgrading may no longer be
possible for aging development environments. You're advised to always re-use
this app and use the flag `--overwrite` to re-create from scratch the
development environment.
"""
recipe_dir = recipe_dir or os.path.join(os.path.realpath('.'), 'conda')
if not os.path.exists(recipe_dir):
raise RuntimeError("The directory %s does not exist" % recipe_dir)
conda = os.environ.get('CONDA_EXE')
if conda is None:
raise RuntimeError("Cannot find `conda' executable (${CONDA_EXEC}) - " \
"have you activated the build environment containing bob.devtools " \
"properly?")
# set condarc before continuing
logger.debug('$ export CONDARC=%s', condarc)
os.environ['CONDARC'] = condarc
deps = parse_dependencies(conda, recipe_dir, python, config)
status = conda_create(conda, name, overwrite, condarc, deps, dry_run)
click.echo('Execute on your shell: "conda activate %s"' % name)
conda/conda_build_config.yaml deleted 100644 → 0
+ 1
− 223
View file @ bd190477
macos_min_version:
- 10.9
macos_machine:
- x86_64-apple-darwin13.4.0
MACOSX_DEPLOYMENT_TARGET:
- 10.9
CONDA_BUILD_SYSROOT: # [osx]
- /opt/MacOSX10.9.sdk # [osx]
# This helps CMAKE find the sysroot. See
# https://cmake.org/cmake/help/v3.11/variable/CMAKE_OSX_SYSROOT.html
SDKROOT: # [osx]
- /opt/MacOSX10.9.sdk # [osx]
# makes autotools verbose
VERBOSE_AT:
- V=1
# makes cmake verbose
VERBOSE_CM:
- VERBOSE=1
# the blas implementations that we build against
blas_impl:
- mkl
pin_run_as_build:
libboost:
max_pin: x.x.x
py_boost:
max_pin: x.x.x
boost:
max_pin: x.x.x
## the dependencies that we build against multiple versions
python:
- 3.6
zip_keys:
- # [win]
- vc # [win]
- c_compiler # [win]
- cxx_compiler # [win]
- fortran_compiler_version # [win]
- python # [win]
# Here is the version of dependencies are used when building packages (build
# and host requirements). We keep a list of **all of them** here to make sure
# everything goes as expected in our conda build process. For the version of
# packages that are used for testing packages, see the recipe of bob-devel.
# The version here do not necessarily match the versions in bob-devel.
# This version of bob-devel will be used at test time of packages:
bob_devel:
- 2018.12.29
# This version of beat-devel will be used at test time of packages. Notice it
# uses bob-devel and should have a version that is greater or equal its value
beat_devel:
- 2018.12.29
# The build time only dependencies (build requirements).
# Updating these to the latest version all the time is OK and a good idea.
# These versions should match the versions inside bob-devel as well (if they
# overlap) so update them in both places.
cmake:
- 3.12.2
pkg_config:
- 0.29.2
cython:
- 0.28.1
# The dependencies that we link against (host requirements).
# Ideally we want to build against the oldest possible version of packages when
# we are linking against them. It is best to keep this in sync with:
# https://github.com/AnacondaRecipes/aggregate/blob/master/conda_build_config.yaml
numpy:
- 1.14.5
boost:
- 1.65.1
cyvlfeat:
- 0.4.6
ffmpeg:
- 3.4
freetype:
- 2.8
giflib:
- 5.1.4
hdf5:
- 1.10.1
jpeg:
- 9b
libblitz:
- 1.0.1
libmatio:
- 1.5.11
libogg:
- 1.3.2
libpng:
- 1.6.35
libsvm:
- 3.22
libtiff:
- 4.0.9
mkl:
- 2018.0.3
openfst:
- 1.6.1
sox:
- 14.4.2
speex:
- 1.2.0
speexdsp:
- 1.2rc3
sqlite:
- 3.20.1
vlfeat:
- 0.9.21
xz:
- 5.2.3
zlib:
- 1.2.11
# The dependencies that are needed for runtime only (run requirements).
# These versions **should** match the versions inside bob-devel recipe.
caffe:
- 1.0
click:
- 6.7
click_plugins:
- 1.0.3
coverage:
- 4.5.1
dlib:
- 19.7
docopt:
- 0.6.2
jinja2:
- 2.10
kaldi:
- 2017.03.13
matplotlib:
- 2.2.3
menpo:
- 0.8.1
menpofit:
- 0.5.0
mne:
- 0.15.2
mr_developer:
- 1.38
nose:
- 1.3.7
opencv:
- 3.3.1
pillow:
- 5.2.0
pyedflib:
- 0.1.11
pytorch:
- 0.4.1
pyyaml:
- 3.13
requests:
- 2.19.1
schema:
- 0.6.7
scikit_image:
- 0.14
scikit_learn:
- 0.19.2
scipy:
- 1.1.0
setuptools:
- 40.2.0
six:
- 1.11.0
sphinx:
- 1.8.1
sphinx_rtd_theme:
- 0.4.1
sqlalchemy:
- 1.2.11
tabulate:
- 0.8.2
tensorflow:
- 1.9.0
torchvision:
- 0.2.1
zc_buildout:
- 2.12.2
zc_recipe_egg:
- 2.0.7
# The dependencies that are needed for runtime only (run requirements) of BEAT
# packages. These versions **should** match the versions inside beat-devel
# recipe.
docker_py:
- 3.6.0
jsonschema:
- 2.6.0
oset:
- 0.1.3
python_graphviz:
- 0.8.4
pyzmq:
- 17.1.2
simplejson:
- 3.16.0
termcolor:
- 1.1.0
sphinxcontrib_programoutput:
- 0.11
sphinxcontrib_httpdomain:
- 1.7.0
nodejs:
- 8.9.3
flask:
- 1.0.2
flask_cors:
- 3.0.7
flask_restful:
- 0.3.6
psycopg2:
- 2.7.6.1
conda/conda_build_config.yaml 0 → 120000
+ 1
− 223
View file @ 76d35e95
../bob/devtools/data/conda_build_config.yaml
\ No newline at end of file
conda/meta.yaml
+ 2
− 0
View file @ 76d35e95
...@@ -40,6 +40,7 @@ requirements: ...@@ -40,6 +40,7 @@ requirements:
- python-gitlab - python-gitlab
- requests - requests
- sphinx - sphinx
- pyyaml
test: test:
requires: requires:
...@@ -55,6 +56,7 @@ test: ...@@ -55,6 +56,7 @@ test:
- bdt release --help - bdt release --help
- bdt visibility --help - bdt visibility --help
- bdt dumpsphinx --help - bdt dumpsphinx --help
- bdt bootstrap --help
- sphinx-build -aEW ${PREFIX}/share/doc/{{ name }}/doc {{ project_dir }}/sphinx - sphinx-build -aEW ${PREFIX}/share/doc/{{ name }}/doc {{ project_dir }}/sphinx
about: about:
......
setup.py
+ 2
− 0
View file @ 76d35e95
...@@ -14,6 +14,7 @@ requires = [ ...@@ -14,6 +14,7 @@ requires = [
'gitpython', 'gitpython',
'python-gitlab', 'python-gitlab',
'sphinx', 'sphinx',
'pyyaml',
] ]
setup( setup(
...@@ -44,6 +45,7 @@ setup( ...@@ -44,6 +45,7 @@ setup(
'lasttag = bob.devtools.scripts.lasttag:lasttag', 'lasttag = bob.devtools.scripts.lasttag:lasttag',
'visibility = bob.devtools.scripts.visibility:visibility', 'visibility = bob.devtools.scripts.visibility:visibility',
'dumpsphinx = bob.devtools.scripts.dumpsphinx:dumpsphinx', 'dumpsphinx = bob.devtools.scripts.dumpsphinx:dumpsphinx',
'bootstrap = bob.devtools.scripts.bootstrap:bootstrap',
], ],
}, },
classifiers=[ classifiers=[
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment