DRY; Move boot/build from ci dir into the package

.gitlab-ci.yml

@@ -16,7 +16,9 @@ stages:
   before_script:
     - python3 ./bob/devtools/bootstrap.py build
   script:
-    - ./ci/build.sh
+    - source ${CONDA_ROOT}/etc/profile.d/conda.sh
+    - conda activate base
+    - python3 ./bob/devtools/build.py
   cache: &build_caches
     paths:
       - miniconda.sh

MANIFEST.in

 include LICENSE README.rst buildout.cfg version.txt
 recursive-include doc conf.py *.rst *.sh
-recursive-include bob/devtools/data *.md *.yaml *condarc *.pem matplotlibrc
+recursive-include bob/devtools/data *.md *.yaml *.pem matplotlibrc

bob/devtools/bootstrap.py

@@ -24,7 +24,7 @@ Arguments:
 '''
 
 
-BASE_CONDARC = '''\
+_BASE_CONDARC = '''\
 default_channels:
   - https://repo.anaconda.com/pkgs/main
   - https://repo.anaconda.com/pkgs/free
@@ -39,6 +39,17 @@ anaconda_upload: false #!final
 ssl_verify: false #!final
 '''
 
+_SERVER = 'http://www.idiap.ch'
+
+_INTERVALS = (
+    ('weeks', 604800),  # 60 * 60 * 24 * 7
+    ('days', 86400),    # 60 * 60 * 24
+    ('hours', 3600),    # 60 * 60
+    ('minutes', 60),
+    ('seconds', 1),
+    )
+'''Time intervals that make up human readable time slots'''
+
 
 import os
 import sys
@@ -49,16 +60,25 @@ import platform
 import subprocess
 
 import logging
-logger = logging.getLogger('bootstrap')
+logger = logging.getLogger(__name__)
 
 
-_INTERVALS = (
-    ('weeks', 604800),  # 60 * 60 * 24 * 7
-    ('days', 86400),    # 60 * 60 * 24
-    ('hours', 3600),    # 60 * 60
-    ('minutes', 60),
-    ('seconds', 1),
-    )
+def set_environment(name, value, env=os.environ):
+    '''Function to setup the environment variable and print debug message
+
+    Args:
+
+      name: The name of the environment variable to set
+      value: The value to set the environment variable to
+      env: Optional environment (dictionary) where to set the variable at
+    '''
+
+    if name in env:
+      logger.warn('Overriding existing environment variable ${%s} (was: "%s")',
+          name, env[name])
+    env[name] = value
+    logger.debug('$ export %s="%s"', name, value)
+
 
 def human_time(seconds, granularity=2):
   '''Returns a human readable time string like "1 day, 2 hours"'''
@@ -228,7 +248,7 @@ def install_miniconda(prefix):
     shutil.rmtree(cached)
 
 
-def get_channels(public, stable):
+def get_channels(public, stable, server, intranet):
   '''Returns the relevant conda channels to consider if building project
 
   The subset of channels to be returned depends on the visibility and stability
@@ -249,22 +269,31 @@ def get_channels(public, stable):
       channels
     stable: Boolean indicating if we're supposed to include only stable
       channels
+    server: The base address of the server containing our conda channels
+    intranet: Boolean indicating if we should add "private"/"public" prefixes
+      on the conda paths
 
 
   Returns: a list of channels that need to be considered.
 
   '''
 
-  server = "http://www.idiap.ch"
   channels = []
 
+  if (not public) and (not intranet):
+    raise RuntimeError('You cannot request for private channels and set' \
+        ' intranet=False (server=%s) - these are conflicting options' % server)
+
   if not public:
+    prefix = '/private' if intranet else ''
     if not stable:  #allowed private channels
-      channels += [server + '/private/conda/label/beta']  #allowed betas
-    channels += [server + '/private/conda']
+      channels += [server + prefix + '/conda/label/beta']  #allowed betas
+    channels += [server + prefix + '/conda']
+
+  prefix = '/public' if intranet else ''
   if not stable:
-    channels += [server + '/public/conda/label/beta']  #allowed betas
-  channels += [server + '/public/conda']
+    channels += [server + prefix + '/conda/label/beta']  #allowed betas
+  channels += [server + prefix + '/conda']
 
   return channels
 
@@ -278,10 +307,10 @@ def add_channels_condarc(channels, condarc):
       f.write('  - %s\n' % k)
 
   with open(condarc, 'rt') as f:
-    logger.info('Contents of $CONDARC:\n%s', f.read())
+    logger.info('Contents of installed CONDARC:\n%s', f.read())
 
 
-def setup_logger():
+def setup_logger(logger):
   '''Sets-up the logging for this command at level ``INFO``'''
 
   warn_err = logging.StreamHandler(sys.stderr)
@@ -313,14 +342,14 @@ if __name__ == '__main__':
     print(__doc__ % sys.argv[0])
     sys.exit(1)
 
-  setup_logger()
+  setup_logger(logger)
 
   if sys.argv[1] == 'test':
     # sets up local variables for testing
-    os.environ['CI_PROJECT_DIR'] = os.path.realpath(os.curdir)
-    os.environ['CI_PROJECT_NAME'] = 'bob.devtools'
-    os.environ['CONDA_ROOT'] = os.path.join(os.environ['CI_PROJECT_DIR'],
-        'miniconda')
+    set_environment('CI_PROJECT_DIR', os.path.realpath(os.curdir))
+    set_environment('CI_PROJECT_NAME', 'bob.devtools')
+    set_environment('CONDA_ROOT', os.path.join(os.environ['CI_PROJECT_DIR'],
+        'miniconda'))
 
   prefix = os.environ['CONDA_ROOT']
   logger.info('os.environ["%s"] = %s', 'CONDA_ROOT', prefix)
@@ -335,9 +364,7 @@ if __name__ == '__main__':
   condarc = os.path.join(prefix, 'condarc')
   logger.info('(create) %s', condarc)
   with open(condarc, 'wt') as f:
-    f.write(BASE_CONDARC)
-  os.environ['CONDARC'] = condarc
-  logger.info('os.environ["%s"] = %s', 'CONDARC', condarc)
+    f.write(_BASE_CONDARC)
 
   conda_version = '4'
   conda_build_version = '3'
@@ -363,7 +390,8 @@ if __name__ == '__main__':
     conda_bld_path = os.path.join(prefix, 'conda-bld')
     run_cmdline([conda_bin, 'index', conda_bld_path])
     # add the locally build directory before defaults, boot from there
-    channels = get_channels(public=True, stable=True)
+    channels = get_channels(public=True, stable=True, server=_SERVER,
+        intranet=True)
     add_channels_condarc(channels + [conda_bld_path, 'defaults'], condarc)
     run_cmdline([conda_bin, 'create', '-n', sys.argv[2], 'bob.devtools'])
 
@@ -372,7 +400,8 @@ if __name__ == '__main__':
     # installs from channel
     channels = get_channels(
         public=os.environ['CI_PROJECT_VISIBILITY'] == 'public',
-        stable=os.environ.get('CI_COMMIT_TAG') is not None)
+        stable=os.environ.get('CI_COMMIT_TAG') is not None,
+        server=_SERVER, intranet=True)
     add_channels_condarc(channels + ['defaults'], condarc)
     run_cmdline([conda_bin, 'create', '-n', sys.argv[2], 'bob.devtools'])
 

bob/devtools/build.py

 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-'''This is a copy of bob/devtools/conda:next_build_number with a CLI'''
+'''Tools for self-building and other utilities
+
+This script, if called in standalone format, can be used to build the current
+package.  It contains various functions and utilities that can be used by
+modules inside the package itself.  It assumes a base installation for the
+build is operational (i.e., the python package for ``conda-build`` is
+installed).
+'''
 
 import os
 import re
 import sys
+import json
+import shutil
+import platform
+import subprocess
 
-from conda.exports import get_index
+import logging
+logger = logging.getLogger(__name__)
 
+import yaml
+import packaging
+import conda_build.api
 
-if __name__ == '__main__':
 
-  channel_url = sys.argv[1]
-  name = sys.argv[2]
-  version = sys.argv[3]
-  python = sys.argv[4]
+def osname():
+  """Returns the current OS name as recognized by conda"""
+
+  r = 'unknown'
+  if platform.system().lower() == 'linux':
+    r = 'linux'
+  elif platform.system().lower() == 'darwin':
+    r = 'osx'
+  else:
+    raise RuntimeError('Unsupported system "%s"' % platform.system())
+
+  if platform.machine().lower() == 'x86_64':
+    r += '-64'
+  else:
+    raise RuntimeError('Unsupported machine type "%s"' % platform.machine())
+
+  return r
+
+
+def should_skip_build(metadata_tuples):
+  """Takes the output of render_recipe as input and evaluates if this
+  recipe's build should be skipped.
+  """
+
+  return all(m[0].skip() for m in metadata_tuples)
+
+
+def next_build_number(channel_url, name, version, python):
+  """Calculates the next build number of a package given the channel
+
+  This function returns the next build number (integer) for a package given its
+  recipe, dependencies, name, version and python version.  It looks on the
+  channel URL provided and figures out if any clash would happen and what would
+  be the highest build number available for that configuration.
+
+
+  Args:
+
+    channel_url: The URL where to look for packages clashes (normally a beta
+      channel)
+    name: The name of the package
+    version: The version of the package
+    python: The version of python as 2 digits (e.g.: "2.7" or "3.6")
+
+  Returns: The next build number with the current configuration.  Zero (0) is
+  returned if no match is found.  Also returns the URLs of the packages it
+  finds with matches on the name, version and python-version.
+
+  """
+
+  from conda.exports import get_index
 
   # no dot in py_ver
   py_ver = python.replace('.', '')
 
   # get the channel index
+  logger.debug('Downloading channel index from %s', channel_url)
   index = get_index(channel_urls=[channel_url], prepend=False)
 
   # search if package with the same version exists
@@ -32,9 +94,225 @@ if __name__ == '__main__':
       match = re.match('py[2-9][0-9]+', dist.build_string)
 
       if match and match.group() == 'py{}'.format(py_ver):
+        logger.debug("Found match at %s for %s-%s-py%s", index[dist].url,
+            name, version, py_ver)
         build_number = max(build_number, dist.build_number + 1)
         urls.append(index[dist].url)
 
   urls = [url.replace(channel_url, '') for url in urls]
 
-  print(build_number)
+  return build_number, urls
+
+
+def make_conda_config(config, python, append_file, condarc_options):
+
+  from conda_build.api import get_or_merge_config
+  from conda_build.conda_interface import url_path
+
+  retval = get_or_merge_config(None, variant_config_files=config,
+      python=python, append_sections_file=append_file, **condarc_options)
+
+  retval.channel_urls = []
+
+  for url in condarc_options['channels']:
+    # allow people to specify relative or absolute paths to local channels
+    #    These channels still must follow conda rules - they must have the
+    #    appropriate platform-specific subdir (e.g. win-64)
+    if os.path.isdir(url):
+      if not os.path.isabs(url):
+        url = os.path.normpath(os.path.abspath(os.path.join(os.getcwd(), url)))
+      url = url_path(url)
+    retval.channel_urls.append(url)
+
+  return retval
+
+
+def get_rendered_metadata(recipe_dir, config):
+  '''Renders the recipe and returns the interpreted YAML file'''
+
+  from conda_build.api import render
+  return render(recipe_dir, config=config)
+
+
+def get_parsed_recipe(metadata):
+  '''Renders the recipe and returns the interpreted YAML file'''
+
+  from conda_build.api import output_yaml
+  output = output_yaml(metadata[0][0])
+  return yaml.load(output)
+
+
+def remove_pins(deps):
+  return [l.split()[0] for l in deps]
+
+
+def parse_dependencies(recipe_dir, config):
+
+  metadata = get_rendered_metadata(recipe_dir, config)
+  recipe = get_parsed_recipe(metadata)
+  return remove_pins(recipe['requirements'].get('build', [])) + \
+      remove_pins(recipe['requirements'].get('host', [])) + \
+      recipe['requirements'].get('run', []) + \
+      recipe.get('test', {}).get('requires', []) + \
+      ['bob.buildout', 'mr.developer', 'ipdb']
+      # by last, packages required for local dev
+
+
+def get_env_directory(conda, name):
+
+  cmd = [conda, 'env', 'list', '--json']
+  output = subprocess.check_output(cmd)
+  data = json.loads(output)
+  retval = [k for k in data.get('envs', []) if k.endswith(os.sep + name)]
+  if retval:
+    return retval[0]
+  return None
+
+
+def conda_create(conda, name, overwrite, condarc, packages, dry_run, use_local):
+  '''Creates a new conda environment following package specifications
+
+  This command can create a new conda environment following the list of input
+  packages.  It will overwrite an existing environment if indicated.
+
+  Args:
+    conda: path to the main conda executable of the installation
+    name: the name of the environment to create or overwrite
+    overwrite: if set to ```True``, overwrite potentially existing environments
+      with the same name
+    condarc: a dictionary of options for conda, including channel urls
+    packages: the package list specification
+    dry_run: if set, then don't execute anything, just print stuff
+    use_local: include the local conda-bld directory as a possible installation
+      channel (useful for testing multiple interdependent recipes that are
+      built locally)
+  '''
+
+  from .bootstrap import run_cmdline
+
+  specs = []
+  for k in packages:
+    k = ' '.join(k.split()[:2])  # remove eventual build string
+    if any(elem in k for elem in '><|'):
+      specs.append(k.replace(' ', ''))
+    else:
+      specs.append(k.replace(' ', '='))
+
+  # if the current environment exists, delete it first
+  envdir = get_env_directory(conda, name)
+  if envdir is not None:
+    if overwrite:
+      cmd = [conda, 'env', 'remove', '--yes', '--name', name]
+      logger.debug('$ ' + ' '.join(cmd))
+      if not dry_run:
+        run_cmdline(cmd)
+    else:
+      raise RuntimeError('environment `%s\' exists in `%s\' - use '
+                         '--overwrite to overwrite' % (name, envdir))
+
+  cmdline_channels = ['--channel=%s' % k for k in condarc['channels']]
+  cmd = [conda, 'create', '--yes', '--name', name, '--override-channels'] + \
+      cmdline_channels
+  if dry_run:
+    cmd.append('--dry-run')
+  if use_local:
+     cmd.append('--use-local')
+  cmd.extend(sorted(specs))
+  run_cmdline(cmd)
+
+  # creates a .condarc file to sediment the just created environment
+  if not dry_run:
+    # get envdir again - it may just be created!
+    envdir = get_env_directory(conda, name)
+    destrc = os.path.join(envdir, '.condarc')
+    logger.info('Creating %s...', destrc)
+    with open(destrc, 'w') as f:
+      yaml.dump(condarc, f, indent=2)
+
+
+if __name__ == '__main__':
+
+  # loads the "adjacent" bootstrap module
+  import importlib.util
+  mydir = os.path.dirname(os.path.realpath(sys.argv[0]))
+  bootstrap_file = os.path.join(mydir, 'bootstrap.py')
+  spec = importlib.util.spec_from_file_location("bootstrap", bootstrap_file)
+  bootstrap = importlib.util.module_from_spec(spec)
+  spec.loader.exec_module(bootstrap)
+
+  bootstrap.setup_logger(logger)
+
+  prefix = os.environ['CONDA_ROOT']
+  logger.info('os.environ["%s"] = %s', 'CONDA_ROOT', prefix)
+
+  workdir = os.environ['CI_PROJECT_DIR']
+  logger.info('os.environ["%s"] = %s', 'CI_PROJECT_DIR', workdir)
+
+  name = os.environ['CI_PROJECT_NAME']
+  logger.info('os.environ["%s"] = %s', 'CI_PROJECT_NAME', name)
+
+  pyver = os.environ['PYTHON_VERSION']
+  logger.info('os.environ["%s"] = %s', 'PYTHON_VERSION', pyver)
+
+  set_environment('DOCSERVER', bootstrap._SERVER, os.environ)
+  set_environment('LANG', 'en_US.UTF-8', os.environ)
+  set_environment('LC_ALL', os.environ['LANG'], os.environ)
+
+  # create the build configuration
+  conda_build_config = os.path.join(mydir, 'data', 'conda_build_config.yaml')
+  recipe_append = os.path.join(mydir, 'data', 'recipe_append.yaml')
+  logger.info('Merging conda configuration files...')
+
+  condarc = os.path.join(prefix, 'condarc')
+  logger.info('Loading (this build\'s) CONDARC file from %s...', condarc)
+  with open(condarc, 'rb') as f:
+    condarc_options = yaml.load(f)
+
+  conda_config = make_conda_config(conda_build_config, pyver, recipe_append,
+      condarc_options)
+
+  version = open("version.txt").read().rstrip()
+  os.environ['BOB_PACKAGE_VERSION'] = version
+  logger.info('os.environ["%s"] = %s', 'BOB_PACKAGE_VERSION', version)
+
+  # if we're build a stable release, ensure a tag is set
+  parsed_version = packaging.version.Version(version)
+  if parsed_version.is_prerelease:
+    if os.environ.get('CI_COMMIT_TAG') is not None:
+      raise EnvironmentError('"version.txt" indicates version is a ' \
+          'pre-release (v%s) - but os.environ["CI_COMMIT_TAG"]="%s", ' \
+          'which indicates this is a **stable** build. ' \
+          'Have you created the tag using ``bdt release``?', version,
+          os.environ['CI_COMMIT_TAG'])
+  else:  #it is a stable build
+    if os.environ.get('CI_COMMIT_TAG') is None:
+      raise EnvironmentError('"version.txt" indicates version is a ' \
+          'stable build (v%s) - but there is no os.environ["CI_COMMIT_TAG"] ' \
+          'variable defined, which indicates this is **not** ' \
+          'a tagged build. Use ``bdt release`` to create stable releases',
+          version)
+
+  build_number = next_build_number(channels[0], name, version, python)
+  os.environ['BOB_BUILD_NUMBER'] = build_number
+  logger.info('os.environ["%s"] = %s', 'BOB_BUILD_NUMBER', build_number)
+
+  # runs the build using the conda-build API
+  arch = osname()
+  logger.info('Building %s-%s-py%s (build: %d) for %s',
+      rendered_recipe['package']['name'],
+      rendered_recipe['package']['version'], pyver.replace('.',''),
+      build_number, arch)
+  conda_build.api.build(d, config=conda_config)
+
+  # runs git clean to clean everything that is not needed. This helps to keep
+  # the disk usage on CI machines to a minimum.
+  exclude_from_cleanup = [
+      "miniconda.sh",   #the installer, cached
+      "miniconda/pkgs/*.tar.bz2",  #downloaded packages, cached
+      "miniconda/pkgs/urls.txt",  #download index, cached
+      "miniconda/conda-bld/${_os}-64/*.tar.bz2",  #build artifact -- conda
+      "dist/*.zip",  #build artifact -- pypi package
+      "sphinx",  #build artifact -- documentation
+      ]
+  bootstrap.run_cmdline(['git', 'clean', '-ffdx'] + \
+      ['--exclude=%s' % k for k in exclude_from_cleanup])

bob/devtools/conda.py

@@ -9,79 +9,3 @@ import platform
 logger = logging.getLogger(__name__)
 
 
-def osname():
-  """Returns the current OS name as recognized by conda"""
-
-  r = 'unknown'
-  if platform.system().lower() == 'linux':
-    r = 'linux'
-  elif platform.system().lower() == 'darwin':
-    r = 'osx'
-  else:
-    raise RuntimeError('Unsupported system "%s"' % platform.system())
-
-  if platform.machine().lower() == 'x86_64':
-    r += '-64'
-  else:
-    raise RuntimeError('Unsupported machine type "%s"' % platform.machine())
-
-  return r
-
-
-def should_skip_build(metadata_tuples):
-  """Takes the output of render_recipe as input and evaluates if this
-  recipe's build should be skipped.
-  """
-
-  return all(m[0].skip() for m in metadata_tuples)
-
-
-def next_build_number(channel_url, name, version, python):
-  """Calculates the next build number of a package given the channel
-
-  This function returns the next build number (integer) for a package given its
-  recipe, dependencies, name, version and python version.  It looks on the
-  channel URL provided and figures out if any clash would happen and what would
-  be the highest build number available for that configuration.
-
-
-  Args:
-
-    channel_url: The URL where to look for packages clashes (normally a beta
-      channel)
-    name: The name of the package
-    version: The version of the package
-    python: The version of python as 2 digits (e.g.: "2.7" or "3.6")
-
-  Returns: The next build number with the current configuration.  Zero (0) is
-  returned if no match is found.  Also returns the URLs of the packages it
-  finds with matches on the name, version and python-version.
-
-  """
-
-  from conda.exports import get_index
-
-  # no dot in py_ver
-  py_ver = python.replace('.', '')
-
-  # get the channel index
-  logger.debug('Downloading channel index from %s', channel_url)
-  index = get_index(channel_urls=[channel_url], prepend=False)
-
-  # search if package with the same version exists
-  build_number = 0
-  urls = []
-  for dist in index:
-
-    if dist.name == name and dist.version == version:
-      match = re.match('py[2-9][0-9]+', dist.build_string)
-
-      if match and match.group() == 'py{}'.format(py_ver):
-        logger.debug("Found match at %s for %s-%s-py%s", index[dist].url,
-            name, version, py_ver)
-        build_number = max(build_number, dist.build_number + 1)
-        urls.append(index[dist].url)
-
-  urls = [url.replace(channel_url, '') for url in urls]
-
-  return build_number, urls

bob/devtools/constants.py

@@ -9,10 +9,11 @@ import pkg_resources
 import logging
 logger = logging.getLogger(__name__)
 
+from . import bootstrap
 
-CONDARC = pkg_resources.resource_filename(__name__,
-    os.path.join('data', 'build-condarc'))
-'''The .condarc to use for building and creating new environments'''
+
+BASE_CONDARC = bootstrap._BASE_CONDARC
+'''Default setup for conda builds'''
 
 
 CONDA_BUILD_CONFIG = pkg_resources.resource_filename(__name__,
@@ -25,23 +26,10 @@ CONDA_RECIPE_APPEND = pkg_resources.resource_filename(__name__,
 '''Extra information to be appended to every recipe upon building'''
 
 
-SERVER = 'http://www.idiap.ch'
+SERVER = bootstrap._SERVER
 '''This is the default server use use to store data and build artifacts'''
 
 
-CONDA_CHANNELS = {
-    True: {  #stable?
-      False: '/private/conda',  #visible outside?
-      True: '/public/conda',
-      },
-    False: {
-      False: '/private/conda/label/beta',  #visible outside?
-      True: '/public/conda/label/beta',
-      },
-    }
-'''Default locations of our stable, beta, public and private conda channels'''
-
-
 WEBDAV_PATHS = {
     True: {  #stable?
       False: {  #visible?
@@ -136,20 +124,3 @@ MATPLOTLIB_RCDIR = pkg_resources.resource_filename(__name__, 'data')
 
 It is required for certain builds that use matplotlib functionality.
 '''
-
-
-def set_environment(name, value, env=os.environ):
-    '''Function to setup the environment variable and print debug message
-
-    Args:
-
-      name: The name of the environment variable to set
-      value: The value to set the environment variable to
-      env: Optional environment (dictionary) where to set the variable at
-    '''
-
-    if name in env:
-      logger.warn('Overriding existing environment variable ${%s} (was: "%s")',
-          name, env[name])
-    env[name] = value
-    logger.debug('$ export %s="%s"', name, value)

bob/devtools/create.py

-#!/usr/bin/env python
-# vim: set fileencoding=utf-8 :
-
-'''Methods to create working environments based on conda-packages'''
-
-import os
-import json
-import shutil
-import subprocess
-
-import logging
-logger = logging.getLogger(__name__)
-
-import yaml
-
-
-def make_conda_config(config, python, append_file, condarc):
-
-  from conda_build.api import get_or_merge_config
-  from conda_build.conda_interface import url_path
-
-  with open(condarc, 'rb') as f:
-    condarc_options = yaml.load(f)
-
-  retval = get_or_merge_config(None, variant_config_files=config,
-      python=python, append_sections_file=append_file, **condarc_options)
-
-  retval.channel_urls = []
-
-  for url in condarc_options['channels']:
-    # allow people to specify relative or absolute paths to local channels
-    #    These channels still must follow conda rules - they must have the
-    #    appropriate platform-specific subdir (e.g. win-64)
-    if os.path.isdir(url):
-      if not os.path.isabs(url):
-        url = os.path.normpath(os.path.abspath(os.path.join(os.getcwd(), url)))
-      url = url_path(url)
-    retval.channel_urls.append(url)
-
-  return retval
-
-
-def get_rendered_metadata(recipe_dir, config):
-  '''Renders the recipe and returns the interpreted YAML file'''
-
-  from conda_build.api import render
-  return render(recipe_dir, config=config)
-
-
-def get_parsed_recipe(metadata):
-  '''Renders the recipe and returns the interpreted YAML file'''
-
-  from conda_build.api import output_yaml
-  output = output_yaml(metadata[0][0])
-  return yaml.load(output)
-
-
-def remove_pins(deps):
-  return [l.split()[0] for l in deps]
-
-
-def parse_dependencies(recipe_dir, config):
-
-  metadata = get_rendered_metadata(recipe_dir, config)
-  recipe = get_parsed_recipe(metadata)
-  return remove_pins(recipe['requirements'].get('build', [])) + \
-      remove_pins(recipe['requirements'].get('host', [])) + \
-      recipe['requirements'].get('run', []) + \
-      recipe.get('test', {}).get('requires', []) + \
-      ['bob.buildout', 'mr.developer', 'ipdb']
-      # by last, packages required for local dev
-
-
-def get_env_directory(conda, name):
-
-  cmd = [conda, 'env', 'list', '--json']
-  output = subprocess.check_output(cmd)
-  data = json.loads(output)
-  retval = [k for k in data.get('envs', []) if k.endswith(os.sep + name)]
-  if retval:
-    return retval[0]
-  return None
-
-
-def conda_create(conda, name, overwrite, condarc, packages, dry_run, use_local):
-
-  specs = []
-  for k in packages:
-    k = ' '.join(k.split()[:2])  # remove eventual build string
-    if any(elem in k for elem in '><|'):
-      specs.append(k.replace(' ', ''))
-    else:
-      specs.append(k.replace(' ', '='))
-
-  # if the current environment exists, delete it first
-  envdir = get_env_directory(conda, name)
-  if envdir is not None:
-    if overwrite:
-      cmd = [conda, 'env', 'remove', '--yes', '--name', name]
-      logger.debug('$ ' + ' '.join(cmd))
-      if not dry_run:
-        status = subprocess.call(cmd)
-        if status != 0:
-          return status
-    else:
-      raise RuntimeError('environment `%s\' exists in `%s\' - use '
-                         '--overwrite to overwrite' % (name, envdir))
-
-  cmd = [conda, 'create', '--yes', '--name', name]
-  if dry_run:
-    cmd.append('--dry-run')
-  if use_local:
-     cmd.append('--use-local')
-  cmd.extend(sorted(specs))
-  logger.debug('$ ' + ' '.join(cmd))
-  status = subprocess.call(cmd)
-  if status != 0:
-    return status
-
-  # copy the used condarc file to the just created environment
-  if not dry_run:
-    # get envdir again - it may just be created!
-    envdir = get_env_directory(conda, name)
-    destrc = os.path.join(envdir, '.condarc')
-    logger.debug('$ cp %s -> %s' % (condarc, destrc))
-    shutil.copy2(condarc, destrc)
-
-  return status

bob/devtools/data/build-condarc

-default_channels:
-  - https://repo.anaconda.com/pkgs/main
-  - https://repo.anaconda.com/pkgs/free
-  - https://repo.anaconda.com/pkgs/r
-  - https://repo.anaconda.com/pkgs/pro
-add_pip_as_python_dependency: false
-show_channel_urls: true
-anaconda_upload: false
-ssl_verify: false
-quiet: true
-channels:
-  - https://www.idiap.ch/software/bob/conda/label/beta
-  - https://www.idiap.ch/software/bob/conda
-  - defaults

bob/devtools/scripts/bdt.py

@@ -55,7 +55,8 @@ def main():
     """Bob Development Tools - see available commands below"""
 
     #sets up basic environment variables required everywhere
-    from ..constants import CACERT, set_environment
+    from ..constants import CACERT
+    from ..bootstrap import set_environment
 
     set_environment('SSL_CERT_FILE', CACERT, os.environ)
     set_environment('REQUESTS_CA_BUNDLE', CACERT, os.environ)

bob/devtools/scripts/build.py

@@ -8,14 +8,15 @@ logger = logging.getLogger(__name__)
 
 import pkg_resources
 import click
+import yaml
 
 from . import bdt
 from ..log import verbosity_option
-from ..conda import next_build_number, osname
-from ..create import get_rendered_metadata, get_parsed_recipe, \
-    make_conda_config
-from ..constants import CONDARC, CONDA_BUILD_CONFIG, CONDA_RECIPE_APPEND, \
-    SERVER, MATPLOTLIB_RCDIR, set_environment
+from ..build import next_build_number, osname, should_skip_build, \
+    get_rendered_metadata, get_parsed_recipe, make_conda_config
+from ..constants import CONDA_BUILD_CONFIG, CONDA_RECIPE_APPEND, \
+    SERVER, MATPLOTLIB_RCDIR, BASE_CONDARC
+from ..bootstrap import set_environment, get_channels
 
 
 @click.command(epilog='''
@@ -23,55 +24,59 @@ Examples:
 
   1. Builds recipe from one of our build dependencies (inside bob.conda):
 
+\b
      $ cd bob.conda
      $ bdt build -vv conda/libblitz
 
 
-  2. Builds recipe from one of our packages, for Python 3.6 (if that is not
-     already the default for you):
+  2. Builds recipe from one of our packages, for Python 3.6 (if that is not already the default for you):
 
      $ bdt build --python=3.6 -vv path/to/conda/dir
 
 
   3. To build multiple recipes, just pass the paths to them:
 
-     $ bdt build --python=3.6 -vv path/to/recipe-dir/1 path/to/recipe-dir/2
+     $ bdt build --python=3.6 -vv path/to/recipe-dir1 path/to/recipe-dir2
 ''')
 @click.argument('recipe-dir', required=False, type=click.Path(file_okay=False,
   dir_okay=True, exists=True), nargs=-1)
 @click.option('-p', '--python', default=('%d.%d' % sys.version_info[:2]),
     show_default=True, help='Version of python to build the ' \
         'environment for [default: %(default)s]')
-@click.option('-r', '--condarc', default=CONDARC, show_default=True,
-    help='overwrites the path leading to the condarc file to use',)
+@click.option('-r', '--condarc',
+    help='Use custom conda configuration file instead of our own',)
 @click.option('-m', '--config', '--variant-config-files', show_default=True,
-      default=CONDA_BUILD_CONFIG, help='overwrites the path leading to ' \
-          'variant configuration file to use')
-@click.option('-c', '--channel', show_default=True,
-    default='https://www.idiap.ch/software/bob/conda/label/beta',
-    help='Channel URL where this package is meant to be uploaded to, ' \
-        'after a successful build - typically, this is a beta channel')
+    default=CONDA_BUILD_CONFIG, help='overwrites the path leading to ' \
+        'variant configuration file to use')
 @click.option('-n', '--no-test', is_flag=True,
     help='Do not test the package, only builds it')
 @click.option('-a', '--append-file', show_default=True,
-      default=CONDA_RECIPE_APPEND, help='overwrites the path leading to ' \
-          'appended configuration file to use')
-@click.option('-D', '--docserver', show_default=True,
-      default=SERVER, help='Server used for uploading artifacts ' \
-          'and other goodies')
+    default=CONDA_RECIPE_APPEND, help='overwrites the path leading to ' \
+        'appended configuration file to use')
+@click.option('-S', '--server', show_default=True,
+    default='https://www.idiap.ch/software/bob', help='Server used for ' \
+    'downloading conda packages and documentation indexes of required packages')
+@click.option('-P', '--private/--no-private', default=False,
+    help='Set this to **include** private channels on your build - ' \
+        'you **must** be at Idiap to execute this build in this case - ' \
+        'you **must** also use the correct server name through --server - ' \
+        'notice this option has no effect if you also pass --condarc')
+@click.option('-X', '--stable/--no-stable', default=False,
+    help='Set this to **exclude** beta channels from your build - ' \
+        'notice this option has no effect if you also pass --condarc')
 @click.option('-d', '--dry-run/--no-dry-run', default=False,
     help='Only goes through the actions, but does not execute them ' \
         '(combine with the verbosity flags - e.g. ``-vvv``) to enable ' \
         'printing to help you understand what will be done')
 @verbosity_option()
 @bdt.raise_on_error
-def build(recipe_dir, python, condarc, config, channel, no_test, append_file,
-    docserver, dry_run):
+def build(recipe_dir, python, condarc, config, no_test, append_file,
+    server, private, stable, dry_run):
   """Builds package through conda-build with stock configuration
 
   This command wraps the execution of conda-build so that you use the same
-  ``condarc`` and ``conda_build_config.yaml`` file we use for our CI.  It
-  always set ``--no-anaconda-upload``.
+  conda configuration we use for our CI.  It always set
+  ``--no-anaconda-upload``.
 
   Note that both files are embedded within bob.devtools - you may need to
   update your environment before trying this.
@@ -84,13 +89,25 @@ def build(recipe_dir, python, condarc, config, channel, no_test, append_file,
 
   recipe_dir = recipe_dir or [os.path.join(os.path.realpath('.'), 'conda')]
 
-  logger.debug("CONDARC=%s", condarc)
+  # get potential channel upload and other auxiliary channels
+  channels = get_channels(public=(not private), stable=stable, server=server,
+      intranet=private)
+  channel = channels[0]  # where we would upload this package
 
-  conda_config = make_conda_config(config, python, append_file, condarc)
+  if condarc is not None:
+    logger.info('Loading CONDARC file from %s...', condarc)
+    with open(condarc, 'rb') as f:
+      condarc_options = yaml.load(f)
+  else:
+    # use default and add channels
+    condarc_options = yaml.load(BASE_CONDARC)  #n.b.: no channels
+    condarc_options['channels'] = channels + ['defaults']
+
+  conda_config = make_conda_config(config, python, append_file, condarc_options)
 
   set_environment('LANG', 'en_US.UTF-8', os.environ)
   set_environment('LC_ALL', os.environ['LANG'], os.environ)
-  set_environment('DOCSERVER', docserver, os.environ)
+  set_environment('DOCSERVER', server, os.environ)
   set_environment('MATPLOTLIBRC', MATPLOTLIB_RCDIR, os.environ)
 
   for d in recipe_dir:
@@ -107,7 +124,6 @@ def build(recipe_dir, python, condarc, config, channel, no_test, append_file,
     metadata = get_rendered_metadata(d, conda_config)
 
     # checks we should actually build this recipe
-    from ..conda import should_skip_build
     if should_skip_build(metadata):
       logger.warn('Skipping UNSUPPORTED build of "%s" for py%s on %s',
           d, python.replace('.',''), osname())
@@ -126,8 +142,6 @@ def build(recipe_dir, python, condarc, config, channel, no_test, append_file,
 
     set_environment('BOB_BUILD_NUMBER', str(build_number), os.environ)
 
-    # we don't execute the following command, it is just here for logging
-    # purposes. we directly use the conda_build API.
     logger.info('Building %s-%s-py%s (build: %d) for %s',
         rendered_recipe['package']['name'],
         rendered_recipe['package']['version'], python.replace('.',''),

bob/devtools/scripts/create.py

@@ -12,9 +12,10 @@ import yaml
 
 from . import bdt
 from ..log import verbosity_option
-from ..create import parse_dependencies, conda_create, make_conda_config
-from ..constants import CONDARC, CONDA_BUILD_CONFIG, CONDA_RECIPE_APPEND, \
-    SERVER
+from ..build import parse_dependencies, conda_create, make_conda_config
+from ..constants import BASE_CONDARC, CONDA_BUILD_CONFIG, \
+    CONDA_RECIPE_APPEND, SERVER
+from ..bootstrap import set_environment, get_channels
 
 
 @click.command(epilog='''
@@ -61,27 +62,35 @@ Examples:
       help='If set and an environment with the same name exists, ' \
           'deletes it first before creating the new environment',
           show_default=True)
-@click.option('-r', '--condarc', default=CONDARC, show_default=True,
-    help='overwrites the path leading to the condarc file to use',)
+@click.option('-r', '--condarc',
+    help='Use custom conda configuration file instead of our own',)
+@click.option('-l', '--use-local', default=False,
+    help='Allow the use of local channels for package retrieval')
 @click.option('-m', '--config', '--variant-config-files', show_default=True,
       default=CONDA_BUILD_CONFIG, help='overwrites the path leading to ' \
           'variant configuration file to use')
 @click.option('-a', '--append-file', show_default=True,
       default=CONDA_RECIPE_APPEND, help='overwrites the path leading to ' \
           'appended configuration file to use')
-@click.option('-D', '--docserver', show_default=True,
-      default=SERVER, help='Server used for uploading artifacts ' \
-          'and other goodies')
+@click.option('-S', '--server', show_default=True,
+    default='https://www.idiap.ch/software/bob', help='Server used for ' \
+    'downloading conda packages and documentation indexes of required packages')
+@click.option('-P', '--private/--no-private', default=False,
+    help='Set this to **include** private channels on your build - ' \
+        'you **must** be at Idiap to execute this build in this case - ' \
+        'you **must** also use the correct server name through --server - ' \
+        'notice this option has no effect if you also pass --condarc')
+@click.option('-X', '--stable/--no-stable', default=False,
+    help='Set this to **exclude** beta channels from your build - ' \
+        'notice this option has no effect if you also pass --condarc')
 @click.option('-d', '--dry-run/--no-dry-run', default=False,
     help='Only goes through the actions, but does not execute them ' \
         '(combine with the verbosity flags - e.g. ``-vvv``) to enable ' \
         'printing to help you understand what will be done')
-@click.option('--use-local', default=False,
-    help='Allow the use of local channels for package retrieval')
 @verbosity_option()
 @bdt.raise_on_error
-def create(name, recipe_dir, python, overwrite, condarc, config,
-    append_file, docserver, dry_run, use_local):
+def create(name, recipe_dir, python, overwrite, condarc, use_local, config,
+    append_file, server, private, stable, dry_run):
   """Creates a development environment for a recipe
 
   It uses the conda render API to render a recipe and install an environment
@@ -112,12 +121,23 @@ def create(name, recipe_dir, python, overwrite, condarc, config,
         "properly?")
 
   # set some environment variables before continuing
-  set_environment('CONDARC', condarc, os.environ)
-  set_environment('SERVER', docserver, os.environ)
+  set_environment('DOCSERVER', server, os.environ)
   set_environment('LANG', 'en_US.UTF-8', os.environ)
   set_environment('LC_ALL', os.environ['LANG'], os.environ)
 
-  conda_config = make_conda_config(config, python, append_file, condarc)
+  if condarc is not None:
+    logger.info('Loading CONDARC file from %s...', condarc)
+    with open(condarc, 'rb') as f:
+      condarc_options = yaml.load(f)
+  else:
+    # use default and add channels
+    condarc_options = yaml.load(BASE_CONDARC)  #n.b.: no channels
+    channels = get_channels(public=(not private), stable=stable, server=server,
+        intranet=private)
+    condarc_options['channels'] = channels + ['defaults']
+
+  conda_config = make_conda_config(config, python, append_file, condarc_options)
   deps = parse_dependencies(recipe_dir, conda_config)
-  status = conda_create(conda, name, overwrite, condarc, deps, dry_run, use_local)
+  status = conda_create(conda, name, overwrite, condarc_options, deps,
+      dry_run, use_local)
   click.echo('Execute on your shell: "conda activate %s"' % name)

ci/build.sh

-#!/usr/bin/env bash
-
-# datetime prefix for logging
-log_datetime() {
-	echo "($(date +%T.%3N))"
-}
-
-# Functions for coloring echo commands
-log_info() {
-  echo -e "$(log_datetime) \033[1;34m${@}\033[0m"
-}
-
-
-log_error() {
-  echo -e "$(log_datetime) \033[1;31mError: ${@}\033[0m" >&2
-}
-
-# Function for running command and echoing results
-run_cmd() {
-  log_info "$ ${@}"
-  ${@}
-  local status=$?
-  if [ ${status} != 0 ]; then
-    log_error "Command Failed \"${@}\""
-    exit ${status}
-  fi
-}
-
-# Checks just if the variable is defined and has non-zero length
-check_defined() {
-  if [ -z "${!1+abc}" ]; then
-    log_error "Variable ${1} is undefined - aborting...";
-    exit 1
-  elif [ -z "${!1}" ]; then
-    log_error "Variable ${1} is zero-length - aborting...";
-    exit 1
-  fi
-  log_info "${1}=${!1}"
-}
-
-
-check_defined CONDA_ROOT
-check_defined CI_PROJECT_DIR
-check_defined CI_PROJECT_NAME
-check_defined PYTHON_VERSION
-
-export DOCSERVER=http://www.idiap.ch
-check_defined DOCSERVER
-
-export CONDARC="${CONDA_ROOT}/condarc"
-check_defined CONDARC
-
-export BOB_PACKAGE_VERSION=`cat version.txt | tr -d '\n'`;
-check_defined BOB_PACKAGE_VERSION
-
-# Makes sure we activate the base environment if available
-run_cmd source ${CONDA_ROOT}/etc/profile.d/conda.sh
-run_cmd conda activate base
-export PATH
-check_defined PATH
-
-CONDA_CHANNEL_ROOT="${DOCSERVER}/public/conda"
-check_defined CONDA_CHANNEL_ROOT
-if [ -z "${CI_COMMIT_TAG}" ]; then #building beta
-  UPLOAD_CHANNEL="${CONDA_CHANNEL_ROOT}/label/beta"
-else
-  UPLOAD_CHANNEL="${CONDA_CHANNEL_ROOT}"
-fi
-check_defined UPLOAD_CHANNEL
-
-log_info "$ ${CONDA_ROOT}/bin/python ${CI_PROJECT_DIR}/ci/nextbuild.py ${UPLOAD_CHANNEL} ${CI_PROJECT_NAME} ${BOB_PACKAGE_VERSION} ${PYTHON_VERSION}"
-export BOB_BUILD_NUMBER=$(${CONDA_ROOT}/bin/python ${CI_PROJECT_DIR}/ci/nextbuild.py ${UPLOAD_CHANNEL} ${CI_PROJECT_NAME} ${BOB_PACKAGE_VERSION} ${PYTHON_VERSION})
-check_defined BOB_BUILD_NUMBER
-
-# copy the recipe_append.yaml over before build
-run_cmd cp ${CI_PROJECT_DIR}/bob/devtools/data/recipe_append.yaml conda/
-run_cmd cp ${CI_PROJECT_DIR}/bob/devtools/data/conda_build_config.yaml conda/
-
-# to build, we only rely on the stable channel and defaults
-run_cmd ${CONDA_ROOT}/bin/conda build --override-channels -c "${CONDA_CHANNEL_ROOT} -c defaults --python=${PYTHON_VERSION} --no-anaconda-upload" conda
-
-# run git clean to clean everything that is not needed. This helps to keep the
-# disk usage on CI machines to minimum.
-if [ "$(uname -s)" == "Linux" ]; then
-  _os="linux"
-else
-  _os="osx"
-fi
-run_cmd git clean -ffdx \
-    --exclude="miniconda.sh" \
-    --exclude="miniconda/pkgs/*.tar.bz2" \
-    --exclude="miniconda/pkgs/urls.txt" \
-    --exclude="miniconda/conda-bld/${_os}-64/*.tar.bz2" \
-    --exclude="dist/*.zip" \
-    --exclude="sphinx"

doc/api.rst

@@ -12,7 +12,6 @@
    bob.devtools.constants
    bob.devtools.release
    bob.devtools.changelog
-   bob.devtools.create
    bob.devtools.bootstrap
    bob.devtools.build
    bob.devtools.webdav3.client
@@ -33,8 +32,6 @@ Detailed Information
 
 .. automodule:: bob.devtools.changelog
 
-.. automodule:: bob.devtools.create
-
 .. automodule:: bob.devtools.bootstrap
 
 .. automodule:: bob.devtools.build