[scripts] Add bootstrap support

MANIFEST.in

 include LICENSE README.rst buildout.cfg version.txt
 recursive-include doc conf.py *.rst
-recursive-include bob/devtools/data *.md
+recursive-include bob/devtools/data *.md *.yaml buid-condarc

bob/devtools/bootstrap.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+import os
+import json
+import shutil
+import subprocess
+
+import logging
+logger = logging.getLogger(__name__)
+
+import yaml
+from conda_build.api import get_or_merge_config, render, output_yaml
+
+
+def get_rendered_recipe(conda, recipe_dir, python, config):
+  '''Renders the recipe and returns the interpreted YAML file'''
+
+  # equivalent command execute - in here we use the conda API
+  cmd = [
+      conda, 'render',
+      '--variant-config-files', config,
+      '--python', python,
+      recipe_dir,
+      ]
+  logger.debug('$ ' + ' '.join(cmd))
+
+  # do the real job
+  config = get_or_merge_config(None, variant_config_files=config,
+                               python=python)
+  metadata = render(recipe_dir, config=config)
+  output = output_yaml(metadata[0][0])
+  return yaml.load(output)
+
+
+def remove_pins(deps):
+  return [l.split()[0] for l in deps]
+
+
+def parse_dependencies(conda, recipe_dir, python, config):
+
+  recipe = get_rendered_recipe(conda, recipe_dir, python, config)
+  return remove_pins(recipe['requirements'].get('build', [])) + \
+      remove_pins(recipe['requirements'].get('host', [])) + \
+      recipe['requirements'].get('run', []) + \
+      recipe.get('test', {}).get('requires', []) + \
+      ['bob.buildout', 'mr.developer', 'ipdb']
+      # by last, packages required for local dev
+
+
+def get_env_directory(conda, name):
+
+  cmd = [conda, 'env', 'list', '--json']
+  output = subprocess.check_output(cmd)
+  data = json.loads(output)
+  retval = [k for k in data.get('envs', []) if k.endswith(os.sep + name)]
+  if retval:
+    return retval[0]
+  return None
+
+
+def conda_create(conda, name, overwrite, condarc, packages, dry_run):
+
+  specs = []
+  for k in packages:
+    k = ' '.join(k.split()[:2])  # remove eventual build string
+    if any(elem in k for elem in '><|'):
+      specs.append(k.replace(' ', ''))
+    else:
+      specs.append(k.replace(' ', '='))
+
+  # if the current environment exists, delete it first
+  envdir = get_env_directory(conda, name)
+  if envdir is not None:
+    if overwrite:
+      cmd = [conda, 'env', 'remove', '--yes', '--name', name]
+      logger.debug('$ ' + ' '.join(cmd))
+      if not dry_run:
+        status = subprocess.call(cmd)
+        if status != 0:
+          return status
+    else:
+      raise RuntimeError('environment `%s\' exists in `%s\' - use '
+                         '--overwrite to overwrite' % (name, envdir))
+
+  cmd = [conda, 'create', '--yes', '--name', name] + sorted(specs)
+  if dry_run:
+    cmd.insert(2, '--dry-run')
+  logger.debug('$ ' + ' '.join(cmd))
+  status = subprocess.call(cmd)
+  if status != 0:
+    return status
+
+  # copy the used condarc file to the just created environment
+  if not dry_run:
+    # get envdir again - it may just be created!
+    envdir = get_env_directory(conda, name)
+    destrc = os.path.join(envdir, '.condarc')
+    logger.debug('$ cp %s -> %s' % (condarc, destrc))
+    shutil.copy2(condarc, destrc)
+
+  return status

bob/devtools/data/build-condarc

+default_channels:
+  - https://repo.anaconda.com/pkgs/main
+  - https://repo.anaconda.com/pkgs/free
+  - https://repo.anaconda.com/pkgs/r
+  - https://repo.anaconda.com/pkgs/pro
+add_pip_as_python_dependency: false
+show_channel_urls: true
+anaconda_upload: false
+ssl_verify: false
+quiet: true
+channels:
+  - https://www.idiap.ch/software/bob/conda/label/beta
+  - https://www.idiap.ch/software/bob/conda
+  - defaults

bob/devtools/data/conda_build_config.yaml

+macos_min_version:
+  - 10.9
+macos_machine:
+  - x86_64-apple-darwin13.4.0
+MACOSX_DEPLOYMENT_TARGET:
+  - 10.9
+CONDA_BUILD_SYSROOT:            # [osx]
+  - /opt/MacOSX10.9.sdk         # [osx]
+# This helps CMAKE find the sysroot. See
+# https://cmake.org/cmake/help/v3.11/variable/CMAKE_OSX_SYSROOT.html
+SDKROOT:                        # [osx]
+  - /opt/MacOSX10.9.sdk         # [osx]
+# makes autotools verbose
+VERBOSE_AT:
+  - V=1
+# makes cmake verbose
+VERBOSE_CM:
+  - VERBOSE=1
+
+# the blas implementations that we build against
+blas_impl:
+  - mkl
+
+pin_run_as_build:
+  libboost:
+    max_pin: x.x.x
+  py_boost:
+    max_pin: x.x.x
+  boost:
+    max_pin: x.x.x
+
+## the dependencies that we build against multiple versions
+python:
+  - 3.6
+
+zip_keys:
+  -                             # [win]
+    - vc                        # [win]
+    - c_compiler                # [win]
+    - cxx_compiler              # [win]
+    - fortran_compiler_version  # [win]
+    - python                    # [win]
+
+
+# Here is the version of dependencies are used when building packages (build
+# and host requirements). We keep a list of **all of them** here to make sure
+# everything goes as expected in our conda build process. For the version of
+# packages that are used for testing packages, see the recipe of bob-devel.
+# The version here do not necessarily match the versions in bob-devel.
+
+# This version of bob-devel will be used at test time of packages:
+bob_devel:
+  - 2018.12.29
+
+# This version of beat-devel will be used at test time of packages. Notice it
+# uses bob-devel and should have a version that is greater or equal its value
+beat_devel:
+  - 2018.12.29
+
+# The build time only dependencies (build requirements).
+# Updating these to the latest version all the time is OK and a good idea.
+# These versions should match the versions inside bob-devel as well (if they
+# overlap) so update them in both places.
+cmake:
+  - 3.12.2
+pkg_config:
+  - 0.29.2
+cython:
+  - 0.28.1
+
+# The dependencies that we link against (host requirements).
+# Ideally we want to build against the oldest possible version of packages when
+# we are linking against them. It is best to keep this in sync with:
+# https://github.com/AnacondaRecipes/aggregate/blob/master/conda_build_config.yaml
+numpy:
+  - 1.14.5
+boost:
+  - 1.65.1
+cyvlfeat:
+  - 0.4.6
+ffmpeg:
+  - 3.4
+freetype:
+  - 2.8
+giflib:
+  - 5.1.4
+hdf5:
+  - 1.10.1
+jpeg:
+  - 9b
+libblitz:
+  - 1.0.1
+libmatio:
+  - 1.5.11
+libogg:
+  - 1.3.2
+libpng:
+  - 1.6.35
+libsvm:
+  - 3.22
+libtiff:
+  - 4.0.9
+mkl:
+  - 2018.0.3
+openfst:
+  - 1.6.1
+sox:
+  - 14.4.2
+speex:
+  - 1.2.0
+speexdsp:
+  - 1.2rc3
+sqlite:
+  - 3.20.1
+vlfeat:
+  - 0.9.21
+xz:
+  - 5.2.3
+zlib:
+  - 1.2.11
+
+# The dependencies that are needed for runtime only (run requirements).
+# These versions **should** match the versions inside bob-devel recipe.
+caffe:
+  - 1.0
+click:
+  - 6.7
+click_plugins:
+  - 1.0.3
+coverage:
+  - 4.5.1
+dlib:
+  - 19.7
+docopt:
+  - 0.6.2
+jinja2:
+  - 2.10
+kaldi:
+  - 2017.03.13
+matplotlib:
+  - 2.2.3
+menpo:
+  - 0.8.1
+menpofit:
+  - 0.5.0
+mne:
+  - 0.15.2
+mr_developer:
+  - 1.38
+nose:
+  - 1.3.7
+opencv:
+  - 3.3.1
+pillow:
+  - 5.2.0
+pyedflib:
+  - 0.1.11
+pytorch:
+  - 0.4.1
+pyyaml:
+  - 3.13
+requests:
+  - 2.19.1
+schema:
+  - 0.6.7
+scikit_image:
+  - 0.14
+scikit_learn:
+  - 0.19.2
+scipy:
+  - 1.1.0
+setuptools:
+  - 40.2.0
+six:
+  - 1.11.0
+sphinx:
+  - 1.8.1
+sphinx_rtd_theme:
+  - 0.4.1
+sqlalchemy:
+  - 1.2.11
+tabulate:
+  - 0.8.2
+tensorflow:
+  - 1.9.0
+torchvision:
+  - 0.2.1
+zc_buildout:
+  - 2.12.2
+zc_recipe_egg:
+  - 2.0.7
+
+# The dependencies that are needed for runtime only (run requirements) of BEAT
+# packages. These versions **should** match the versions inside beat-devel
+# recipe.
+docker_py:
+  - 3.6.0
+jsonschema:
+  - 2.6.0
+oset:
+  - 0.1.3
+python_graphviz:
+  - 0.8.4
+pyzmq:
+  - 17.1.2
+simplejson:
+  - 3.16.0
+termcolor:
+  - 1.1.0
+sphinxcontrib_programoutput:
+  - 0.11
+sphinxcontrib_httpdomain:
+  - 1.7.0
+nodejs:
+  - 8.9.3
+flask:
+  - 1.0.2
+flask_cors:
+  - 3.0.7
+flask_restful:
+  - 0.3.6
+psycopg2:
+  - 2.7.6.1

bob/devtools/scripts/bootstrap.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+import sys
+import logging
+logger = logging.getLogger(__name__)
+
+import pkg_resources
+import click
+
+from . import bdt
+from ..log import verbosity_option
+from ..bootstrap import parse_dependencies, conda_create
+
+
+DEFAULT_CONDARC = pkg_resources.resource_filename(__name__,
+    os.path.join('..', 'data', 'build-condarc'))
+DEFAULT_VARIANT = pkg_resources.resource_filename(__name__,
+    os.path.join('..', 'data', 'conda_build_config.yaml'))
+
+
+@click.command(epilog='''
+Examples:
+
+  1. Creates an environment called `myenv' for developing the currently checked-out package (N.B.: first activate the base environment):
+
+\b
+     $ cd bob.package.foo
+     $ bdt bootstrap -vv myenv
+
+     The above command assumes the directory `conda' exists on the current directory and that it contains a file called `meta.yaml' containing the recipe for the package you want to create a development environment for.
+
+     If you get things right by default, the above form is the typical usage scenario of this app. Read-on to tweak special flags and settings.
+
+
+  2. By default, we use the native python version of your conda installation as the default python version to use for the newly created environment. You may select a different one with `--python=X.Y':
+
+     $ bdt bootstrap -vv --python=3.6 myenv
+
+
+  3. By default, we use our own condarc and `conda_build_config.yaml` files that are used in creating packages for our CI/CD system. If you wish to use your own, specify them on the command line:
+
+     $ bdt bootstrap -vv --python=3.6 --config=config.yaml --condarc=~/.condarc myenv
+
+     Notice the condarc file **must** end in `condarc', or conda will complain.
+
+
+  4. You can use the option `--dry-run' to simulate what would be installed
+  instead of actually creating a new environment.  Combine with `-vvv` to
+  enable debug printing.  Equivalent conda commands you can execute on the
+  shell will be printed:
+
+
+     $ bdt bootstrap -vvv --dry-run myenv
+''')
+@click.argument('name')
+@click.argument('recipe-dir', required=False, type=click.Path(file_okay=False,
+  dir_okay=True, exists=True))
+@click.option('-p', '--python', default=('%d.%d' % sys.version_info[:2]),
+    show_default=True, help='Version of python to build the ' \
+        'environment for [default: %(default)s]')
+@click.option('-o', '--overwrite/--no-overwrite', default=False,
+      help='If set and an environment with the same name exists, ' \
+          'deletes it first before creating the new environment',
+          show_default=True)
+@click.option('-r', '--condarc', default=DEFAULT_CONDARC, show_default=True,
+    help='overwrites the path leading to the condarc file to use',)
+@click.option('-m', '--config', '--variant-config-files', show_default=True,
+      default=DEFAULT_VARIANT, help='overwrites the path leading to ' \
+          'variant configuration file to use')
+@click.option('-d', '--dry-run/--no-dry-run', default=False,
+    help='Only goes through the actions, but does not execute them ' \
+        '(combine with the verbosity flags - e.g. ``-vvv``) to enable ' \
+        'printing to help you understand what will be done')
+@verbosity_option()
+@bdt.raise_on_error
+def bootstrap(name, recipe_dir, python, overwrite, condarc, config, dry_run):
+  """This program uses conda to build a development environment for a recipe
+
+  It uses the conda render API to render a recipe and install an environment
+  containing all build/host, run and test dependencies of a package. It does
+  **not** build the package itself, just install dependencies so you can build
+  the package by hand, possibly using buildout or similar. If you'd like to
+  conda-build your package, just use `conda build` instead.
+
+  Once the environment is created, a copy of the used `condarc' file is placed
+  on the root of the environment. Installing or updating packages on the newly
+  created environment should be possible without further configuration. Notice
+  that beta packages quickly get outdated and upgrading may no longer be
+  possible for aging development environments. You're advised to always re-use
+  this app and use the flag `--overwrite` to re-create from scratch the
+  development environment.
+  """
+
+  recipe_dir = recipe_dir or os.path.join(os.path.realpath('.'), 'conda')
+
+  if not os.path.exists(recipe_dir):
+    raise RuntimeError("The directory %s does not exist" % recipe_dir)
+
+  conda = os.environ.get('CONDA_EXE')
+  if conda is None:
+    raise RuntimeError("Cannot find `conda' executable (${CONDA_EXEC}) - " \
+        "have you activated the build environment containing bob.devtools " \
+        "properly?")
+
+  # set condarc before continuing
+  logger.debug('$ export CONDARC=%s', condarc)
+  os.environ['CONDARC'] = condarc
+
+  deps = parse_dependencies(conda, recipe_dir, python, config)
+  status = conda_create(conda, name, overwrite, condarc, deps, dry_run)
+  click.echo('Execute on your shell: "conda activate %s"' % name)

conda/conda_build_config.yaml

-macos_min_version:
-  - 10.9
-macos_machine:
-  - x86_64-apple-darwin13.4.0
-MACOSX_DEPLOYMENT_TARGET:
-  - 10.9
-CONDA_BUILD_SYSROOT:            # [osx]
-  - /opt/MacOSX10.9.sdk         # [osx]
-# This helps CMAKE find the sysroot. See
-# https://cmake.org/cmake/help/v3.11/variable/CMAKE_OSX_SYSROOT.html
-SDKROOT:                        # [osx]
-  - /opt/MacOSX10.9.sdk         # [osx]
-# makes autotools verbose
-VERBOSE_AT:
-  - V=1
-# makes cmake verbose
-VERBOSE_CM:
-  - VERBOSE=1
-
-# the blas implementations that we build against
-blas_impl:
-  - mkl
-
-pin_run_as_build:
-  libboost:
-    max_pin: x.x.x
-  py_boost:
-    max_pin: x.x.x
-  boost:
-    max_pin: x.x.x
-
-## the dependencies that we build against multiple versions
-python:
-  - 3.6
-
-zip_keys:
-  -                             # [win]
-    - vc                        # [win]
-    - c_compiler                # [win]
-    - cxx_compiler              # [win]
-    - fortran_compiler_version  # [win]
-    - python                    # [win]
-
-
-# Here is the version of dependencies are used when building packages (build
-# and host requirements). We keep a list of **all of them** here to make sure
-# everything goes as expected in our conda build process. For the version of
-# packages that are used for testing packages, see the recipe of bob-devel.
-# The version here do not necessarily match the versions in bob-devel.
-
-# This version of bob-devel will be used at test time of packages:
-bob_devel:
-  - 2018.12.29
-
-# This version of beat-devel will be used at test time of packages. Notice it
-# uses bob-devel and should have a version that is greater or equal its value
-beat_devel:
-  - 2018.12.29
-
-# The build time only dependencies (build requirements).
-# Updating these to the latest version all the time is OK and a good idea.
-# These versions should match the versions inside bob-devel as well (if they
-# overlap) so update them in both places.
-cmake:
-  - 3.12.2
-pkg_config:
-  - 0.29.2
-cython:
-  - 0.28.1
-
-# The dependencies that we link against (host requirements).
-# Ideally we want to build against the oldest possible version of packages when
-# we are linking against them. It is best to keep this in sync with:
-# https://github.com/AnacondaRecipes/aggregate/blob/master/conda_build_config.yaml
-numpy:
-  - 1.14.5
-boost:
-  - 1.65.1
-cyvlfeat:
-  - 0.4.6
-ffmpeg:
-  - 3.4
-freetype:
-  - 2.8
-giflib:
-  - 5.1.4
-hdf5:
-  - 1.10.1
-jpeg:
-  - 9b
-libblitz:
-  - 1.0.1
-libmatio:
-  - 1.5.11
-libogg:
-  - 1.3.2
-libpng:
-  - 1.6.35
-libsvm:
-  - 3.22
-libtiff:
-  - 4.0.9
-mkl:
-  - 2018.0.3
-openfst:
-  - 1.6.1
-sox:
-  - 14.4.2
-speex:
-  - 1.2.0
-speexdsp:
-  - 1.2rc3
-sqlite:
-  - 3.20.1
-vlfeat:
-  - 0.9.21
-xz:
-  - 5.2.3
-zlib:
-  - 1.2.11
-
-# The dependencies that are needed for runtime only (run requirements).
-# These versions **should** match the versions inside bob-devel recipe.
-caffe:
-  - 1.0
-click:
-  - 6.7
-click_plugins:
-  - 1.0.3
-coverage:
-  - 4.5.1
-dlib:
-  - 19.7
-docopt:
-  - 0.6.2
-jinja2:
-  - 2.10
-kaldi:
-  - 2017.03.13
-matplotlib:
-  - 2.2.3
-menpo:
-  - 0.8.1
-menpofit:
-  - 0.5.0
-mne:
-  - 0.15.2
-mr_developer:
-  - 1.38
-nose:
-  - 1.3.7
-opencv:
-  - 3.3.1
-pillow:
-  - 5.2.0
-pyedflib:
-  - 0.1.11
-pytorch:
-  - 0.4.1
-pyyaml:
-  - 3.13
-requests:
-  - 2.19.1
-schema:
-  - 0.6.7
-scikit_image:
-  - 0.14
-scikit_learn:
-  - 0.19.2
-scipy:
-  - 1.1.0
-setuptools:
-  - 40.2.0
-six:
-  - 1.11.0
-sphinx:
-  - 1.8.1
-sphinx_rtd_theme:
-  - 0.4.1
-sqlalchemy:
-  - 1.2.11
-tabulate:
-  - 0.8.2
-tensorflow:
-  - 1.9.0
-torchvision:
-  - 0.2.1
-zc_buildout:
-  - 2.12.2
-zc_recipe_egg:
-  - 2.0.7
-
-# The dependencies that are needed for runtime only (run requirements) of BEAT
-# packages. These versions **should** match the versions inside beat-devel
-# recipe.
-docker_py:
-  - 3.6.0
-jsonschema:
-  - 2.6.0
-oset:
-  - 0.1.3
-python_graphviz:
-  - 0.8.4
-pyzmq:
-  - 17.1.2
-simplejson:
-  - 3.16.0
-termcolor:
-  - 1.1.0
-sphinxcontrib_programoutput:
-  - 0.11
-sphinxcontrib_httpdomain:
-  - 1.7.0
-nodejs:
-  - 8.9.3
-flask:
-  - 1.0.2
-flask_cors:
-  - 3.0.7
-flask_restful:
-  - 0.3.6
-psycopg2:
-  - 2.7.6.1

conda/conda_build_config.yaml

+../bob/devtools/data/conda_build_config.yaml
\ No newline at end of file

conda/meta.yaml

@@ -40,6 +40,7 @@ requirements:
     - python-gitlab
     - requests
     - sphinx
+    - pyyaml
 
 test:
   requires:
@@ -55,6 +56,7 @@ test:
     - bdt release --help
     - bdt visibility --help
     - bdt dumpsphinx --help
+    - bdt bootstrap --help
     - sphinx-build -aEW ${PREFIX}/share/doc/{{ name }}/doc {{ project_dir }}/sphinx
 
 about:

setup.py

@@ -14,6 +14,7 @@ requires = [
     'gitpython',
     'python-gitlab',
     'sphinx',
+    'pyyaml',
     ]
 
 setup(
@@ -44,6 +45,7 @@ setup(
             'lasttag = bob.devtools.scripts.lasttag:lasttag',
             'visibility = bob.devtools.scripts.visibility:visibility',
             'dumpsphinx = bob.devtools.scripts.dumpsphinx:dumpsphinx',
+            'bootstrap = bob.devtools.scripts.bootstrap:bootstrap',
         ],
     },
     classifiers=[