[scritpts] Implement build support; Remove old cb-output script (incorporate into build)

bob/devtools/bootstrap.py

@@ -10,25 +10,45 @@ import logging
 logger = logging.getLogger(__name__)
 
 import yaml
-from conda_build.api import get_or_merge_config, render, output_yaml
 
 
-def get_rendered_recipe(conda, recipe_dir, python, config):
+def make_conda_config(config, python, append_file, condarc):
+
+  from conda_build.api import get_or_merge_config
+  from conda_build.conda_interface import url_path
+
+  with open(condarc, 'rb') as f:
+    condarc_options = yaml.load(f)
+
+  retval = get_or_merge_config(None, variant_config_files=config,
+      python=python, append_sections_file=append_file, **condarc_options)
+
+  retval.channel_urls = []
+
+  for url in condarc_options['channels']:
+    # allow people to specify relative or absolute paths to local channels
+    #    These channels still must follow conda rules - they must have the
+    #    appropriate platform-specific subdir (e.g. win-64)
+    if os.path.isdir(url):
+      if not os.path.isabs(url):
+        url = os.path.normpath(os.path.abspath(os.path.join(os.getcwd(), url)))
+      url = url_path(url)
+    retval.channel_urls.append(url)
+
+  return retval
+
+
+def get_rendered_metadata(recipe_dir, config):
   '''Renders the recipe and returns the interpreted YAML file'''
 
-  # equivalent command execute - in here we use the conda API
-  cmd = [
-      conda, 'render',
-      '--variant-config-files', config,
-      '--python', python,
-      recipe_dir,
-      ]
-  logger.debug('$ ' + ' '.join(cmd))
+  from conda_build.api import render
+  return render(recipe_dir, config=config)
+
+
+def get_parsed_recipe(metadata):
+  '''Renders the recipe and returns the interpreted YAML file'''
 
-  # do the real job
-  config = get_or_merge_config(None, variant_config_files=config,
-                               python=python)
-  metadata = render(recipe_dir, config=config)
+  from conda_build.api import output_yaml
   output = output_yaml(metadata[0][0])
   return yaml.load(output)
 
@@ -37,9 +57,10 @@ def remove_pins(deps):
   return [l.split()[0] for l in deps]
 
 
-def parse_dependencies(conda, recipe_dir, python, config):
+def parse_dependencies(recipe_dir, config):
 
-  recipe = get_rendered_recipe(conda, recipe_dir, python, config)
+  metadata = get_rendered_metadata(recipe_dir, config)
+  recipe = get_parsed_recipe(metadata)
   return remove_pins(recipe['requirements'].get('build', [])) + \
       remove_pins(recipe['requirements'].get('host', [])) + \
       recipe['requirements'].get('run', []) + \

bob/devtools/conda.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+'''Utilities for deadling with conda packages'''
+
+import re
+import logging
+import platform
+logger = logging.getLogger(__name__)
+
+
+def osname():
+  """Returns the current OS name as recognized by conda"""
+
+  r = 'unknown'
+  if platform.system().lower() == 'linux':
+    r = 'linux'
+  elif platform.system().lower() == 'darwin':
+    r = 'osx'
+  else:
+    raise RuntimeError('Unsupported system "%s"' % platform.system())
+
+  if platform.machine().lower() == 'x86_64':
+    r += '-64'
+  else:
+    raise RuntimeError('Unsupported machine type "%s"' % platform.machine())
+
+  return r
+
 
 def should_skip_build(metadata_tuples):
-    """Takes the output of render_recipe as input and evaluates if this
-    recipe's build should be skipped.
-    """
-    return all(m[0].skip() for m in metadata_tuples)
+  """Takes the output of render_recipe as input and evaluates if this
+  recipe's build should be skipped.
+  """
+
+  return all(m[0].skip() for m in metadata_tuples)
+
+
+def next_build_number(channel_url, name, version, python):
+  """Calculates the next build number of a package given the channel
+
+  This function returns the next build number (integer) for a package given its
+  recipe, dependencies, name, version and python version.  It looks on the
+  channel URL provided and figures out if any clash would happen and what would
+  be the highest build number available for that configuration.
+
+
+  Args:
+
+    channel_url: The URL where to look for packages clashes (normally a beta
+      channel)
+    name: The name of the package
+    version: The version of the package
+    python: The version of python as 2 digits (e.g.: "2.7" or "3.6")
+
+  Returns: The next build number with the current configuration.  Zero (0) is
+  returned if no match is found.  Also returns the URLs of the packages it
+  finds with matches on the name, version and python-version.
+
+  """
+
+  from conda.exports import get_index
+
+  # no dot in py_ver
+  py_ver = python.replace('.', '')
+
+  # get the channel index
+  logger.debug('Downloading channel index from %s', channel_url)
+  index = get_index(channel_urls=[channel_url], prepend=False)
+
+  # search if package with the same version exists
+  build_number = 0
+  urls = []
+  for dist in index:
+
+    if dist.name == name and dist.version == version:
+      match = re.match('py[2-9][0-9]+', dist.build_string)
+
+      if match and match.group() == 'py{}'.format(py_ver):
+        logger.debug("Found match at %s for %s-%s-py%s", index[dist].url,
+            name, version, py_ver)
+        build_number = max(build_number, dist.build_number + 1)
+        urls.append(index[dist].url)
+
+  urls = [url.replace(channel_url, '') for url in urls]
+
+  return build_number, urls

bob/devtools/data/recipe_append.yaml

+build:
+  script_env:
+    - DOCSERVER

bob/devtools/scripts/bootstrap.py

@@ -8,16 +8,20 @@ logger = logging.getLogger(__name__)
 
 import pkg_resources
 import click
+import yaml
 
 from . import bdt
 from ..log import verbosity_option
-from ..bootstrap import parse_dependencies, conda_create
+from ..bootstrap import parse_dependencies, conda_create, make_conda_config
 
 
 DEFAULT_CONDARC = pkg_resources.resource_filename(__name__,
     os.path.join('..', 'data', 'build-condarc'))
 DEFAULT_VARIANT = pkg_resources.resource_filename(__name__,
     os.path.join('..', 'data', 'conda_build_config.yaml'))
+DEFAULT_APPEND = pkg_resources.resource_filename(__name__,
+    os.path.join('..', 'data', 'recipe_append.yaml'))
+DEFAULT_DOCSERVER = 'http://www.idiap.ch'
 
 
 @click.command(epilog='''
@@ -69,14 +73,21 @@ Examples:
 @click.option('-m', '--config', '--variant-config-files', show_default=True,
       default=DEFAULT_VARIANT, help='overwrites the path leading to ' \
           'variant configuration file to use')
+@click.option('-a', '--append-file', show_default=True,
+      default=DEFAULT_APPEND, help='overwrites the path leading to ' \
+          'appended configuration file to use')
+@click.option('-D', '--docserver', show_default=True,
+      default=DEFAULT_DOCSERVER, help='Server used for uploading artifacts ' \
+          'and other goodies')
 @click.option('-d', '--dry-run/--no-dry-run', default=False,
     help='Only goes through the actions, but does not execute them ' \
         '(combine with the verbosity flags - e.g. ``-vvv``) to enable ' \
         'printing to help you understand what will be done')
 @verbosity_option()
 @bdt.raise_on_error
-def bootstrap(name, recipe_dir, python, overwrite, condarc, config, dry_run):
-  """This program uses conda to build a development environment for a recipe
+def bootstrap(name, recipe_dir, python, overwrite, condarc, config,
+    append_file, docserver, dry_run):
+  """Creates a development environment for a recipe
 
   It uses the conda render API to render a recipe and install an environment
   containing all build/host, run and test dependencies of a package. It does
@@ -106,9 +117,13 @@ def bootstrap(name, recipe_dir, python, overwrite, condarc, config, dry_run):
         "properly?")
 
   # set condarc before continuing
-  logger.debug('$ export CONDARC=%s', condarc)
+  logger.debug("[var] CONDARC=%s", condarc)
   os.environ['CONDARC'] = condarc
 
-  deps = parse_dependencies(conda, recipe_dir, python, config)
+  logger.debug("[var] DOCSERVER=%s", docserver)
+  os.environ['DOCSERVER'] = docserver
+
+  conda_config = make_conda_config(config, python, append_file, condarc)
+  deps = parse_dependencies(recipe_dir, conda_config)
   status = conda_create(conda, name, overwrite, condarc, deps, dry_run)
   click.echo('Execute on your shell: "conda activate %s"' % name)

bob/devtools/scripts/build.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+import sys
+import logging
+logger = logging.getLogger(__name__)
+
+import pkg_resources
+import click
+
+from . import bdt
+from ..log import verbosity_option
+from ..conda import next_build_number, osname
+from ..bootstrap import get_rendered_metadata, get_parsed_recipe
+
+
+from .bootstrap import DEFAULT_CONDARC, DEFAULT_VARIANT, DEFAULT_APPEND, \
+    DEFAULT_DOCSERVER
+
+
+@click.command(epilog='''
+Examples:
+
+  1. Builds recipe from one of our build dependencies (inside bob.conda):
+
+     $ cd bob.conda
+     $ bdt build -vv conda/libblitz
+
+
+  2. Builds recipe from one of our packages, for Python 3.6 (if that is not
+     already the default for you):
+
+     $ bdt build --python=3.6 -vv path/to/conda/dir
+
+
+  3. To build multiple recipes, just pass the paths to them:
+
+     $ bdt build --python=3.6 -vv path/to/recipe-dir/1 path/to/recipe-dir/2
+''')
+@click.argument('recipe-dir', required=False, type=click.Path(file_okay=False,
+  dir_okay=True, exists=True), nargs=-1)
+@click.option('-p', '--python', default=('%d.%d' % sys.version_info[:2]),
+    show_default=True, help='Version of python to build the ' \
+        'environment for [default: %(default)s]')
+@click.option('-r', '--condarc', default=DEFAULT_CONDARC, show_default=True,
+    help='overwrites the path leading to the condarc file to use',)
+@click.option('-m', '--config', '--variant-config-files', show_default=True,
+      default=DEFAULT_VARIANT, help='overwrites the path leading to ' \
+          'variant configuration file to use')
+@click.option('-c', '--channel', show_default=True,
+    default='https://www.idiap.ch/software/bob/conda/label/beta',
+    help='Channel URL where this package is meant to be uploaded to, ' \
+        'after a successful build - typically, this is a beta channel')
+@click.option('-n', '--no-test', is_flag=True,
+    help='Do not test the package, only builds it')
+@click.option('-a', '--append-file', show_default=True,
+      default=DEFAULT_APPEND, help='overwrites the path leading to ' \
+          'appended configuration file to use')
+@click.option('-D', '--docserver', show_default=True,
+      default=DEFAULT_DOCSERVER, help='Server used for uploading artifacts ' \
+          'and other goodies')
+@click.option('-d', '--dry-run/--no-dry-run', default=False,
+    help='Only goes through the actions, but does not execute them ' \
+        '(combine with the verbosity flags - e.g. ``-vvv``) to enable ' \
+        'printing to help you understand what will be done')
+@verbosity_option()
+@bdt.raise_on_error
+def build(recipe_dir, python, condarc, config, channel, no_test, append_file,
+    docserver, dry_run):
+  """Runs conda-build with a standard configuration and environment
+
+  This command wraps the execution of conda-build so that you use the same
+  ``condarc`` and ``conda_build_config.yaml`` file we use for our CI.  It
+  always set ``--no-anaconda-upload``.
+
+  Note that both files are embedded within bob.devtools - you may need to
+  update your environment before trying this.
+  """
+
+  # if we are in a dry-run mode, let's let it be known
+  if dry_run:
+      logger.warn('!!!! DRY RUN MODE !!!!')
+      logger.warn('Nothing will be really built')
+
+  recipe_dir = recipe_dir or [os.path.join(os.path.realpath('.'), 'conda')]
+
+  logger.debug("[var] CONDARC=%s", condarc)
+
+  from ..bootstrap import make_conda_config
+  conda_config = make_conda_config(config, python, append_file, condarc)
+
+  logger.debug("[var] DOCSERVER=%s", docserver)
+  os.environ['DOCSERVER'] = docserver
+
+  for d in recipe_dir:
+
+    if not os.path.exists(d):
+      raise RuntimeError("The directory %s does not exist" % recipe_dir)
+
+    version_candidate = os.path.join(d, '..', 'version.txt')
+    if os.path.exists(version_candidate):
+      version = open(version_candidate).read().rstrip()
+      logger.debug("[var] BOB_PACKAGE_VERSION=%s", version)
+      os.environ['BOB_PACKAGE_VERSION'] = version
+
+    # pre-renders the recipe - figures out package name and version
+    metadata = get_rendered_metadata(d, conda_config)
+
+    # checks we should actually build this recipe
+    from ..conda import should_skip_build
+    if should_skip_build(metadata):
+      logger.warn('Skipping UNSUPPORTED build of "%s" for py%s on %s',
+          d, python.replace('.',''), osname())
+      return 0
+
+    # converts the metadata output into parsed yaml and continues the process
+    rendered_recipe = get_parsed_recipe(metadata)
+
+    # if a channel URL was passed, set the build number
+    if channel:
+      build_number, _ = next_build_number(channel,
+          rendered_recipe['package']['name'],
+          rendered_recipe['package']['version'], python)
+    else:
+      build_number = 0
+
+    logger.debug("[var] BOB_BUILD_NUMBER=%s", build_number)
+    os.environ['BOB_BUILD_NUMBER'] = str(build_number)
+
+    # we don't execute the following command, it is just here for logging
+    # purposes. we directly use the conda_build API.
+    logger.info('Building %s-%s-py%s (build: %d) for %s',
+        rendered_recipe['package']['name'],
+        rendered_recipe['package']['version'], python.replace('.',''),
+        build_number, osname())
+    if not dry_run:
+      from conda_build.api import build
+      build(d, config=conda_config, notest=no_test)

bob/devtools/scripts/cb_output.py

-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-import logging
-logger = logging.getLogger(__name__)
-
-import click
-from click.testing import CliRunner
-import conda_build.api as cb
-
-from . import bdt
-from ..log import verbosity_option
-from ..conda import should_skip_build
-
-
-@click.command(context_settings=dict(
-    ignore_unknown_options=True,
-    allow_extra_args=True,
-    ),
-    epilog='''\b
-Examples:
-$ bdt cb-output conda_recipe_dir
-$ bdt cb-output ../bob.conda/conda/kaldi -m ../bob.admin/gitlab/conda_build_config.yaml --python 3.6
-'''
-)
-@click.argument('recipe_path')
-@click.option('-m', '--variant-config-files', help='see conda build --help')
-@click.option('--python', help='see conda build --help')
-@verbosity_option()
-@bdt.raise_on_error
-def cb_output(recipe_path, variant_config_files, python):
-  """Outputs name(s) of package(s) that would be generated by conda build.
-
-  This command accepts extra unknown arguments so you can give it the same
-  arguments that you would give to conda build.
-
-  As of now, it only parses -m/--variant_config_files and --python and other
-  arguments are ignored.
-  """
-  clirunner = CliRunner()
-  with clirunner.isolation():
-    # render
-    config = cb.get_or_merge_config(
-        None, variant_config_files=variant_config_files, python=python)
-    metadata_tuples = cb.render(recipe_path, config=config)
-
-    # check if build(s) should be skipped
-    if should_skip_build(metadata_tuples):
-      return 0
-
-    paths = cb.get_output_file_paths(metadata_tuples, config=config)
-  click.echo('\n'.join(sorted(paths)))

conda/meta.yaml

@@ -57,6 +57,7 @@ test:
     - bdt visibility --help
     - bdt dumpsphinx --help
     - bdt bootstrap --help
+    - bdt build --help
     - sphinx-build -aEW ${PREFIX}/share/doc/{{ name }}/doc {{ project_dir }}/sphinx
 
 about:

setup.py

@@ -39,13 +39,13 @@ setup(
             'bdt = bob.devtools.scripts.bdt:main',
         ],
         'bdt.cli': [
-            'cb-output = bob.devtools.scripts.cb_output:cb_output',
             'release = bob.devtools.scripts.release:release',
             'changelog = bob.devtools.scripts.changelog:changelog',
             'lasttag = bob.devtools.scripts.lasttag:lasttag',
             'visibility = bob.devtools.scripts.visibility:visibility',
             'dumpsphinx = bob.devtools.scripts.dumpsphinx:dumpsphinx',
             'bootstrap = bob.devtools.scripts.bootstrap:bootstrap',
+            'build = bob.devtools.scripts.build:build',
         ],
     },
     classifiers=[