diff --git a/MANIFEST.in b/MANIFEST.in
index 09fc130a0450115612e5063b6a063347d2b9092a..c99bfdf4c4f9d4a01d05a514702d8027d1adc417 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,3 +1,3 @@
include LICENSE README.rst buildout.cfg version.txt
recursive-include doc conf.py *.rst
-recursive-include bob/devtools/data *.md
+recursive-include bob/devtools/data *.md *.yaml buid-condarc
diff --git a/bob/devtools/bootstrap.py b/bob/devtools/bootstrap.py
new file mode 100755
index 0000000000000000000000000000000000000000..90700df89be871be60e9f4ac7630ebff305fa671
--- /dev/null
+++ b/bob/devtools/bootstrap.py
@@ -0,0 +1,102 @@
+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+import os
+import json
+import shutil
+import subprocess
+
+import logging
+logger = logging.getLogger(__name__)
+
+import yaml
+from conda_build.api import get_or_merge_config, render, output_yaml
+
+
+def get_rendered_recipe(conda, recipe_dir, python, config):
+ '''Renders the recipe and returns the interpreted YAML file'''
+
+ # equivalent command execute - in here we use the conda API
+ cmd = [
+ conda, 'render',
+ '--variant-config-files', config,
+ '--python', python,
+ recipe_dir,
+ ]
+ logger.debug('$ ' + ' '.join(cmd))
+
+ # do the real job
+ config = get_or_merge_config(None, variant_config_files=config,
+ python=python)
+ metadata = render(recipe_dir, config=config)
+ output = output_yaml(metadata[0][0])
+ return yaml.load(output)
+
+
+def remove_pins(deps):
+ return [l.split()[0] for l in deps]
+
+
+def parse_dependencies(conda, recipe_dir, python, config):
+
+ recipe = get_rendered_recipe(conda, recipe_dir, python, config)
+ return remove_pins(recipe['requirements'].get('build', [])) + \
+ remove_pins(recipe['requirements'].get('host', [])) + \
+ recipe['requirements'].get('run', []) + \
+ recipe.get('test', {}).get('requires', []) + \
+ ['bob.buildout', 'mr.developer', 'ipdb']
+ # by last, packages required for local dev
+
+
+def get_env_directory(conda, name):
+
+ cmd = [conda, 'env', 'list', '--json']
+ output = subprocess.check_output(cmd)
+ data = json.loads(output)
+ retval = [k for k in data.get('envs', []) if k.endswith(os.sep + name)]
+ if retval:
+ return retval[0]
+ return None
+
+
+def conda_create(conda, name, overwrite, condarc, packages, dry_run):
+
+ specs = []
+ for k in packages:
+ k = ' '.join(k.split()[:2]) # remove eventual build string
+ if any(elem in k for elem in '><|'):
+ specs.append(k.replace(' ', ''))
+ else:
+ specs.append(k.replace(' ', '='))
+
+ # if the current environment exists, delete it first
+ envdir = get_env_directory(conda, name)
+ if envdir is not None:
+ if overwrite:
+ cmd = [conda, 'env', 'remove', '--yes', '--name', name]
+ logger.debug('$ ' + ' '.join(cmd))
+ if not dry_run:
+ status = subprocess.call(cmd)
+ if status != 0:
+ return status
+ else:
+ raise RuntimeError('environment `%s\' exists in `%s\' - use '
+ '--overwrite to overwrite' % (name, envdir))
+
+ cmd = [conda, 'create', '--yes', '--name', name] + sorted(specs)
+ if dry_run:
+ cmd.insert(2, '--dry-run')
+ logger.debug('$ ' + ' '.join(cmd))
+ status = subprocess.call(cmd)
+ if status != 0:
+ return status
+
+ # copy the used condarc file to the just created environment
+ if not dry_run:
+ # get envdir again - it may just be created!
+ envdir = get_env_directory(conda, name)
+ destrc = os.path.join(envdir, '.condarc')
+ logger.debug('$ cp %s -> %s' % (condarc, destrc))
+ shutil.copy2(condarc, destrc)
+
+ return status
diff --git a/bob/devtools/data/build-condarc b/bob/devtools/data/build-condarc
new file mode 100644
index 0000000000000000000000000000000000000000..aacfae4cbafcce6f1e0de3a84d0e826cdfe7a1cb
--- /dev/null
+++ b/bob/devtools/data/build-condarc
@@ -0,0 +1,14 @@
+default_channels:
+ - https://repo.anaconda.com/pkgs/main
+ - https://repo.anaconda.com/pkgs/free
+ - https://repo.anaconda.com/pkgs/r
+ - https://repo.anaconda.com/pkgs/pro
+add_pip_as_python_dependency: false
+show_channel_urls: true
+anaconda_upload: false
+ssl_verify: false
+quiet: true
+channels:
+ - https://www.idiap.ch/software/bob/conda/label/beta
+ - https://www.idiap.ch/software/bob/conda
+ - defaults
diff --git a/bob/devtools/data/conda_build_config.yaml b/bob/devtools/data/conda_build_config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..7c9e4ffd87a73e6d49982fb5f7e14ebf47786de5
--- /dev/null
+++ b/bob/devtools/data/conda_build_config.yaml
@@ -0,0 +1,223 @@
+macos_min_version:
+ - 10.9
+macos_machine:
+ - x86_64-apple-darwin13.4.0
+MACOSX_DEPLOYMENT_TARGET:
+ - 10.9
+CONDA_BUILD_SYSROOT: # [osx]
+ - /opt/MacOSX10.9.sdk # [osx]
+# This helps CMAKE find the sysroot. See
+# https://cmake.org/cmake/help/v3.11/variable/CMAKE_OSX_SYSROOT.html
+SDKROOT: # [osx]
+ - /opt/MacOSX10.9.sdk # [osx]
+# makes autotools verbose
+VERBOSE_AT:
+ - V=1
+# makes cmake verbose
+VERBOSE_CM:
+ - VERBOSE=1
+
+# the blas implementations that we build against
+blas_impl:
+ - mkl
+
+pin_run_as_build:
+ libboost:
+ max_pin: x.x.x
+ py_boost:
+ max_pin: x.x.x
+ boost:
+ max_pin: x.x.x
+
+## the dependencies that we build against multiple versions
+python:
+ - 3.6
+
+zip_keys:
+ - # [win]
+ - vc # [win]
+ - c_compiler # [win]
+ - cxx_compiler # [win]
+ - fortran_compiler_version # [win]
+ - python # [win]
+
+
+# Here is the version of dependencies are used when building packages (build
+# and host requirements). We keep a list of **all of them** here to make sure
+# everything goes as expected in our conda build process. For the version of
+# packages that are used for testing packages, see the recipe of bob-devel.
+# The version here do not necessarily match the versions in bob-devel.
+
+# This version of bob-devel will be used at test time of packages:
+bob_devel:
+ - 2018.12.29
+
+# This version of beat-devel will be used at test time of packages. Notice it
+# uses bob-devel and should have a version that is greater or equal its value
+beat_devel:
+ - 2018.12.29
+
+# The build time only dependencies (build requirements).
+# Updating these to the latest version all the time is OK and a good idea.
+# These versions should match the versions inside bob-devel as well (if they
+# overlap) so update them in both places.
+cmake:
+ - 3.12.2
+pkg_config:
+ - 0.29.2
+cython:
+ - 0.28.1
+
+# The dependencies that we link against (host requirements).
+# Ideally we want to build against the oldest possible version of packages when
+# we are linking against them. It is best to keep this in sync with:
+# https://github.com/AnacondaRecipes/aggregate/blob/master/conda_build_config.yaml
+numpy:
+ - 1.14.5
+boost:
+ - 1.65.1
+cyvlfeat:
+ - 0.4.6
+ffmpeg:
+ - 3.4
+freetype:
+ - 2.8
+giflib:
+ - 5.1.4
+hdf5:
+ - 1.10.1
+jpeg:
+ - 9b
+libblitz:
+ - 1.0.1
+libmatio:
+ - 1.5.11
+libogg:
+ - 1.3.2
+libpng:
+ - 1.6.35
+libsvm:
+ - 3.22
+libtiff:
+ - 4.0.9
+mkl:
+ - 2018.0.3
+openfst:
+ - 1.6.1
+sox:
+ - 14.4.2
+speex:
+ - 1.2.0
+speexdsp:
+ - 1.2rc3
+sqlite:
+ - 3.20.1
+vlfeat:
+ - 0.9.21
+xz:
+ - 5.2.3
+zlib:
+ - 1.2.11
+
+# The dependencies that are needed for runtime only (run requirements).
+# These versions **should** match the versions inside bob-devel recipe.
+caffe:
+ - 1.0
+click:
+ - 6.7
+click_plugins:
+ - 1.0.3
+coverage:
+ - 4.5.1
+dlib:
+ - 19.7
+docopt:
+ - 0.6.2
+jinja2:
+ - 2.10
+kaldi:
+ - 2017.03.13
+matplotlib:
+ - 2.2.3
+menpo:
+ - 0.8.1
+menpofit:
+ - 0.5.0
+mne:
+ - 0.15.2
+mr_developer:
+ - 1.38
+nose:
+ - 1.3.7
+opencv:
+ - 3.3.1
+pillow:
+ - 5.2.0
+pyedflib:
+ - 0.1.11
+pytorch:
+ - 0.4.1
+pyyaml:
+ - 3.13
+requests:
+ - 2.19.1
+schema:
+ - 0.6.7
+scikit_image:
+ - 0.14
+scikit_learn:
+ - 0.19.2
+scipy:
+ - 1.1.0
+setuptools:
+ - 40.2.0
+six:
+ - 1.11.0
+sphinx:
+ - 1.8.1
+sphinx_rtd_theme:
+ - 0.4.1
+sqlalchemy:
+ - 1.2.11
+tabulate:
+ - 0.8.2
+tensorflow:
+ - 1.9.0
+torchvision:
+ - 0.2.1
+zc_buildout:
+ - 2.12.2
+zc_recipe_egg:
+ - 2.0.7
+
+# The dependencies that are needed for runtime only (run requirements) of BEAT
+# packages. These versions **should** match the versions inside beat-devel
+# recipe.
+docker_py:
+ - 3.6.0
+jsonschema:
+ - 2.6.0
+oset:
+ - 0.1.3
+python_graphviz:
+ - 0.8.4
+pyzmq:
+ - 17.1.2
+simplejson:
+ - 3.16.0
+termcolor:
+ - 1.1.0
+sphinxcontrib_programoutput:
+ - 0.11
+sphinxcontrib_httpdomain:
+ - 1.7.0
+nodejs:
+ - 8.9.3
+flask:
+ - 1.0.2
+flask_cors:
+ - 3.0.7
+flask_restful:
+ - 0.3.6
+psycopg2:
+ - 2.7.6.1
diff --git a/bob/devtools/scripts/bootstrap.py b/bob/devtools/scripts/bootstrap.py
new file mode 100644
index 0000000000000000000000000000000000000000..d01cb3f5b9f71b55d3ea792173df62ea0e9a115d
--- /dev/null
+++ b/bob/devtools/scripts/bootstrap.py
@@ -0,0 +1,113 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+import sys
+import logging
+logger = logging.getLogger(__name__)
+
+import pkg_resources
+import click
+
+from . import bdt
+from ..log import verbosity_option
+from ..bootstrap import parse_dependencies, conda_create
+
+
+DEFAULT_CONDARC = pkg_resources.resource_filename(__name__,
+ os.path.join('..', 'data', 'build-condarc'))
+DEFAULT_VARIANT = pkg_resources.resource_filename(__name__,
+ os.path.join('..', 'data', 'conda_build_config.yaml'))
+
+
+@click.command(epilog='''
+Examples:
+
+ 1. Creates an environment called `myenv' for developing the currently checked-out package (N.B.: first activate the base environment):
+
+\b
+ $ cd bob.package.foo
+ $ bdt bootstrap -vv myenv
+
+ The above command assumes the directory `conda' exists on the current directory and that it contains a file called `meta.yaml' containing the recipe for the package you want to create a development environment for.
+
+ If you get things right by default, the above form is the typical usage scenario of this app. Read-on to tweak special flags and settings.
+
+
+ 2. By default, we use the native python version of your conda installation as the default python version to use for the newly created environment. You may select a different one with `--python=X.Y':
+
+ $ bdt bootstrap -vv --python=3.6 myenv
+
+
+ 3. By default, we use our own condarc and `conda_build_config.yaml` files that are used in creating packages for our CI/CD system. If you wish to use your own, specify them on the command line:
+
+ $ bdt bootstrap -vv --python=3.6 --config=config.yaml --condarc=~/.condarc myenv
+
+ Notice the condarc file **must** end in `condarc', or conda will complain.
+
+
+ 4. You can use the option `--dry-run' to simulate what would be installed
+ instead of actually creating a new environment. Combine with `-vvv` to
+ enable debug printing. Equivalent conda commands you can execute on the
+ shell will be printed:
+
+
+ $ bdt bootstrap -vvv --dry-run myenv
+''')
+@click.argument('name')
+@click.argument('recipe-dir', required=False, type=click.Path(file_okay=False,
+ dir_okay=True, exists=True))
+@click.option('-p', '--python', default=('%d.%d' % sys.version_info[:2]),
+ show_default=True, help='Version of python to build the ' \
+ 'environment for [default: %(default)s]')
+@click.option('-o', '--overwrite/--no-overwrite', default=False,
+ help='If set and an environment with the same name exists, ' \
+ 'deletes it first before creating the new environment',
+ show_default=True)
+@click.option('-r', '--condarc', default=DEFAULT_CONDARC, show_default=True,
+ help='overwrites the path leading to the condarc file to use',)
+@click.option('-m', '--config', '--variant-config-files', show_default=True,
+ default=DEFAULT_VARIANT, help='overwrites the path leading to ' \
+ 'variant configuration file to use')
+@click.option('-d', '--dry-run/--no-dry-run', default=False,
+ help='Only goes through the actions, but does not execute them ' \
+ '(combine with the verbosity flags - e.g. ``-vvv``) to enable ' \
+ 'printing to help you understand what will be done')
+@verbosity_option()
+@bdt.raise_on_error
+def bootstrap(name, recipe_dir, python, overwrite, condarc, config, dry_run):
+ """This program uses conda to build a development environment for a recipe
+
+ It uses the conda render API to render a recipe and install an environment
+ containing all build/host, run and test dependencies of a package. It does
+ **not** build the package itself, just install dependencies so you can build
+ the package by hand, possibly using buildout or similar. If you'd like to
+ conda-build your package, just use `conda build` instead.
+
+ Once the environment is created, a copy of the used `condarc' file is placed
+ on the root of the environment. Installing or updating packages on the newly
+ created environment should be possible without further configuration. Notice
+ that beta packages quickly get outdated and upgrading may no longer be
+ possible for aging development environments. You're advised to always re-use
+ this app and use the flag `--overwrite` to re-create from scratch the
+ development environment.
+ """
+
+ recipe_dir = recipe_dir or os.path.join(os.path.realpath('.'), 'conda')
+
+ if not os.path.exists(recipe_dir):
+ raise RuntimeError("The directory %s does not exist" % recipe_dir)
+
+ conda = os.environ.get('CONDA_EXE')
+ if conda is None:
+ raise RuntimeError("Cannot find `conda' executable (${CONDA_EXEC}) - " \
+ "have you activated the build environment containing bob.devtools " \
+ "properly?")
+
+ # set condarc before continuing
+ logger.debug('$ export CONDARC=%s', condarc)
+ os.environ['CONDARC'] = condarc
+
+ deps = parse_dependencies(conda, recipe_dir, python, config)
+ status = conda_create(conda, name, overwrite, condarc, deps, dry_run)
+ click.echo('Execute on your shell: "conda activate %s"' % name)
diff --git a/conda/conda_build_config.yaml b/conda/conda_build_config.yaml
deleted file mode 100644
index 7c9e4ffd87a73e6d49982fb5f7e14ebf47786de5..0000000000000000000000000000000000000000
--- a/conda/conda_build_config.yaml
+++ /dev/null
@@ -1,223 +0,0 @@
-macos_min_version:
- - 10.9
-macos_machine:
- - x86_64-apple-darwin13.4.0
-MACOSX_DEPLOYMENT_TARGET:
- - 10.9
-CONDA_BUILD_SYSROOT: # [osx]
- - /opt/MacOSX10.9.sdk # [osx]
-# This helps CMAKE find the sysroot. See
-# https://cmake.org/cmake/help/v3.11/variable/CMAKE_OSX_SYSROOT.html
-SDKROOT: # [osx]
- - /opt/MacOSX10.9.sdk # [osx]
-# makes autotools verbose
-VERBOSE_AT:
- - V=1
-# makes cmake verbose
-VERBOSE_CM:
- - VERBOSE=1
-
-# the blas implementations that we build against
-blas_impl:
- - mkl
-
-pin_run_as_build:
- libboost:
- max_pin: x.x.x
- py_boost:
- max_pin: x.x.x
- boost:
- max_pin: x.x.x
-
-## the dependencies that we build against multiple versions
-python:
- - 3.6
-
-zip_keys:
- - # [win]
- - vc # [win]
- - c_compiler # [win]
- - cxx_compiler # [win]
- - fortran_compiler_version # [win]
- - python # [win]
-
-
-# Here is the version of dependencies are used when building packages (build
-# and host requirements). We keep a list of **all of them** here to make sure
-# everything goes as expected in our conda build process. For the version of
-# packages that are used for testing packages, see the recipe of bob-devel.
-# The version here do not necessarily match the versions in bob-devel.
-
-# This version of bob-devel will be used at test time of packages:
-bob_devel:
- - 2018.12.29
-
-# This version of beat-devel will be used at test time of packages. Notice it
-# uses bob-devel and should have a version that is greater or equal its value
-beat_devel:
- - 2018.12.29
-
-# The build time only dependencies (build requirements).
-# Updating these to the latest version all the time is OK and a good idea.
-# These versions should match the versions inside bob-devel as well (if they
-# overlap) so update them in both places.
-cmake:
- - 3.12.2
-pkg_config:
- - 0.29.2
-cython:
- - 0.28.1
-
-# The dependencies that we link against (host requirements).
-# Ideally we want to build against the oldest possible version of packages when
-# we are linking against them. It is best to keep this in sync with:
-# https://github.com/AnacondaRecipes/aggregate/blob/master/conda_build_config.yaml
-numpy:
- - 1.14.5
-boost:
- - 1.65.1
-cyvlfeat:
- - 0.4.6
-ffmpeg:
- - 3.4
-freetype:
- - 2.8
-giflib:
- - 5.1.4
-hdf5:
- - 1.10.1
-jpeg:
- - 9b
-libblitz:
- - 1.0.1
-libmatio:
- - 1.5.11
-libogg:
- - 1.3.2
-libpng:
- - 1.6.35
-libsvm:
- - 3.22
-libtiff:
- - 4.0.9
-mkl:
- - 2018.0.3
-openfst:
- - 1.6.1
-sox:
- - 14.4.2
-speex:
- - 1.2.0
-speexdsp:
- - 1.2rc3
-sqlite:
- - 3.20.1
-vlfeat:
- - 0.9.21
-xz:
- - 5.2.3
-zlib:
- - 1.2.11
-
-# The dependencies that are needed for runtime only (run requirements).
-# These versions **should** match the versions inside bob-devel recipe.
-caffe:
- - 1.0
-click:
- - 6.7
-click_plugins:
- - 1.0.3
-coverage:
- - 4.5.1
-dlib:
- - 19.7
-docopt:
- - 0.6.2
-jinja2:
- - 2.10
-kaldi:
- - 2017.03.13
-matplotlib:
- - 2.2.3
-menpo:
- - 0.8.1
-menpofit:
- - 0.5.0
-mne:
- - 0.15.2
-mr_developer:
- - 1.38
-nose:
- - 1.3.7
-opencv:
- - 3.3.1
-pillow:
- - 5.2.0
-pyedflib:
- - 0.1.11
-pytorch:
- - 0.4.1
-pyyaml:
- - 3.13
-requests:
- - 2.19.1
-schema:
- - 0.6.7
-scikit_image:
- - 0.14
-scikit_learn:
- - 0.19.2
-scipy:
- - 1.1.0
-setuptools:
- - 40.2.0
-six:
- - 1.11.0
-sphinx:
- - 1.8.1
-sphinx_rtd_theme:
- - 0.4.1
-sqlalchemy:
- - 1.2.11
-tabulate:
- - 0.8.2
-tensorflow:
- - 1.9.0
-torchvision:
- - 0.2.1
-zc_buildout:
- - 2.12.2
-zc_recipe_egg:
- - 2.0.7
-
-# The dependencies that are needed for runtime only (run requirements) of BEAT
-# packages. These versions **should** match the versions inside beat-devel
-# recipe.
-docker_py:
- - 3.6.0
-jsonschema:
- - 2.6.0
-oset:
- - 0.1.3
-python_graphviz:
- - 0.8.4
-pyzmq:
- - 17.1.2
-simplejson:
- - 3.16.0
-termcolor:
- - 1.1.0
-sphinxcontrib_programoutput:
- - 0.11
-sphinxcontrib_httpdomain:
- - 1.7.0
-nodejs:
- - 8.9.3
-flask:
- - 1.0.2
-flask_cors:
- - 3.0.7
-flask_restful:
- - 0.3.6
-psycopg2:
- - 2.7.6.1
diff --git a/conda/conda_build_config.yaml b/conda/conda_build_config.yaml
new file mode 120000
index 0000000000000000000000000000000000000000..dbdbc0f110d96e2e6a1cdf7c1c1ae3c2bc9b0445
--- /dev/null
+++ b/conda/conda_build_config.yaml
@@ -0,0 +1 @@
+../bob/devtools/data/conda_build_config.yaml
\ No newline at end of file
diff --git a/conda/meta.yaml b/conda/meta.yaml
index 21ddbc1150e197cf94089cd35f486253a8608ce3..b46bad8903b09fc87d32f66d4cf06fd3ad28f36a 100644
--- a/conda/meta.yaml
+++ b/conda/meta.yaml
@@ -40,6 +40,7 @@ requirements:
- python-gitlab
- requests
- sphinx
+ - pyyaml
test:
requires:
@@ -55,6 +56,7 @@ test:
- bdt release --help
- bdt visibility --help
- bdt dumpsphinx --help
+ - bdt bootstrap --help
- sphinx-build -aEW ${PREFIX}/share/doc/{{ name }}/doc {{ project_dir }}/sphinx
about:
diff --git a/setup.py b/setup.py
index 103b99b713b4fd9b28a3af938305c0db41b4db39..d5a8e803dc9f10e924f8f392a909a842a68c31ee 100644
--- a/setup.py
+++ b/setup.py
@@ -14,6 +14,7 @@ requires = [
'gitpython',
'python-gitlab',
'sphinx',
+ 'pyyaml',
]
setup(
@@ -44,6 +45,7 @@ setup(
'lasttag = bob.devtools.scripts.lasttag:lasttag',
'visibility = bob.devtools.scripts.visibility:visibility',
'dumpsphinx = bob.devtools.scripts.dumpsphinx:dumpsphinx',
+ 'bootstrap = bob.devtools.scripts.bootstrap:bootstrap',
],
},
classifiers=[