From da9d8a52e7b9a54a3a1af0dba00baffaad47e3f3 Mon Sep 17 00:00:00 2001
From: Andre Anjos <andre.dos.anjos@gmail.com>
Date: Tue, 8 Jan 2019 22:16:19 +0100
Subject: [PATCH] [scritpts] Implement build support; Remove old cb-output
script (incorporate into build)
---
bob/devtools/bootstrap.py | 53 +++++++---
bob/devtools/conda.py | 89 ++++++++++++++++-
bob/devtools/data/recipe_append.yaml | 3 +
bob/devtools/scripts/bootstrap.py | 25 ++++-
bob/devtools/scripts/build.py | 139 +++++++++++++++++++++++++++
bob/devtools/scripts/cb_output.py | 52 ----------
conda/meta.yaml | 1 +
setup.py | 2 +-
8 files changed, 286 insertions(+), 78 deletions(-)
create mode 100644 bob/devtools/data/recipe_append.yaml
create mode 100644 bob/devtools/scripts/build.py
delete mode 100644 bob/devtools/scripts/cb_output.py
diff --git a/bob/devtools/bootstrap.py b/bob/devtools/bootstrap.py
index 90700df8..21436df0 100755
--- a/bob/devtools/bootstrap.py
+++ b/bob/devtools/bootstrap.py
@@ -10,25 +10,45 @@ import logging
logger = logging.getLogger(__name__)
import yaml
-from conda_build.api import get_or_merge_config, render, output_yaml
-def get_rendered_recipe(conda, recipe_dir, python, config):
+def make_conda_config(config, python, append_file, condarc):
+
+ from conda_build.api import get_or_merge_config
+ from conda_build.conda_interface import url_path
+
+ with open(condarc, 'rb') as f:
+ condarc_options = yaml.load(f)
+
+ retval = get_or_merge_config(None, variant_config_files=config,
+ python=python, append_sections_file=append_file, **condarc_options)
+
+ retval.channel_urls = []
+
+ for url in condarc_options['channels']:
+ # allow people to specify relative or absolute paths to local channels
+ # These channels still must follow conda rules - they must have the
+ # appropriate platform-specific subdir (e.g. win-64)
+ if os.path.isdir(url):
+ if not os.path.isabs(url):
+ url = os.path.normpath(os.path.abspath(os.path.join(os.getcwd(), url)))
+ url = url_path(url)
+ retval.channel_urls.append(url)
+
+ return retval
+
+
+def get_rendered_metadata(recipe_dir, config):
'''Renders the recipe and returns the interpreted YAML file'''
- # equivalent command execute - in here we use the conda API
- cmd = [
- conda, 'render',
- '--variant-config-files', config,
- '--python', python,
- recipe_dir,
- ]
- logger.debug('$ ' + ' '.join(cmd))
+ from conda_build.api import render
+ return render(recipe_dir, config=config)
+
+
+def get_parsed_recipe(metadata):
+ '''Renders the recipe and returns the interpreted YAML file'''
- # do the real job
- config = get_or_merge_config(None, variant_config_files=config,
- python=python)
- metadata = render(recipe_dir, config=config)
+ from conda_build.api import output_yaml
output = output_yaml(metadata[0][0])
return yaml.load(output)
@@ -37,9 +57,10 @@ def remove_pins(deps):
return [l.split()[0] for l in deps]
-def parse_dependencies(conda, recipe_dir, python, config):
+def parse_dependencies(recipe_dir, config):
- recipe = get_rendered_recipe(conda, recipe_dir, python, config)
+ metadata = get_rendered_metadata(recipe_dir, config)
+ recipe = get_parsed_recipe(metadata)
return remove_pins(recipe['requirements'].get('build', [])) + \
remove_pins(recipe['requirements'].get('host', [])) + \
recipe['requirements'].get('run', []) + \
diff --git a/bob/devtools/conda.py b/bob/devtools/conda.py
index 558c094e..70c04d49 100644
--- a/bob/devtools/conda.py
+++ b/bob/devtools/conda.py
@@ -1,6 +1,87 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+'''Utilities for deadling with conda packages'''
+
+import re
+import logging
+import platform
+logger = logging.getLogger(__name__)
+
+
+def osname():
+ """Returns the current OS name as recognized by conda"""
+
+ r = 'unknown'
+ if platform.system().lower() == 'linux':
+ r = 'linux'
+ elif platform.system().lower() == 'darwin':
+ r = 'osx'
+ else:
+ raise RuntimeError('Unsupported system "%s"' % platform.system())
+
+ if platform.machine().lower() == 'x86_64':
+ r += '-64'
+ else:
+ raise RuntimeError('Unsupported machine type "%s"' % platform.machine())
+
+ return r
+
def should_skip_build(metadata_tuples):
- """Takes the output of render_recipe as input and evaluates if this
- recipe's build should be skipped.
- """
- return all(m[0].skip() for m in metadata_tuples)
+ """Takes the output of render_recipe as input and evaluates if this
+ recipe's build should be skipped.
+ """
+
+ return all(m[0].skip() for m in metadata_tuples)
+
+
+def next_build_number(channel_url, name, version, python):
+ """Calculates the next build number of a package given the channel
+
+ This function returns the next build number (integer) for a package given its
+ recipe, dependencies, name, version and python version. It looks on the
+ channel URL provided and figures out if any clash would happen and what would
+ be the highest build number available for that configuration.
+
+
+ Args:
+
+ channel_url: The URL where to look for packages clashes (normally a beta
+ channel)
+ name: The name of the package
+ version: The version of the package
+ python: The version of python as 2 digits (e.g.: "2.7" or "3.6")
+
+ Returns: The next build number with the current configuration. Zero (0) is
+ returned if no match is found. Also returns the URLs of the packages it
+ finds with matches on the name, version and python-version.
+
+ """
+
+ from conda.exports import get_index
+
+ # no dot in py_ver
+ py_ver = python.replace('.', '')
+
+ # get the channel index
+ logger.debug('Downloading channel index from %s', channel_url)
+ index = get_index(channel_urls=[channel_url], prepend=False)
+
+ # search if package with the same version exists
+ build_number = 0
+ urls = []
+ for dist in index:
+
+ if dist.name == name and dist.version == version:
+ match = re.match('py[2-9][0-9]+', dist.build_string)
+
+ if match and match.group() == 'py{}'.format(py_ver):
+ logger.debug("Found match at %s for %s-%s-py%s", index[dist].url,
+ name, version, py_ver)
+ build_number = max(build_number, dist.build_number + 1)
+ urls.append(index[dist].url)
+
+ urls = [url.replace(channel_url, '') for url in urls]
+
+ return build_number, urls
diff --git a/bob/devtools/data/recipe_append.yaml b/bob/devtools/data/recipe_append.yaml
new file mode 100644
index 00000000..dcfb3076
--- /dev/null
+++ b/bob/devtools/data/recipe_append.yaml
@@ -0,0 +1,3 @@
+build:
+ script_env:
+ - DOCSERVER
diff --git a/bob/devtools/scripts/bootstrap.py b/bob/devtools/scripts/bootstrap.py
index f3ba9d3a..d8d7b115 100644
--- a/bob/devtools/scripts/bootstrap.py
+++ b/bob/devtools/scripts/bootstrap.py
@@ -8,16 +8,20 @@ logger = logging.getLogger(__name__)
import pkg_resources
import click
+import yaml
from . import bdt
from ..log import verbosity_option
-from ..bootstrap import parse_dependencies, conda_create
+from ..bootstrap import parse_dependencies, conda_create, make_conda_config
DEFAULT_CONDARC = pkg_resources.resource_filename(__name__,
os.path.join('..', 'data', 'build-condarc'))
DEFAULT_VARIANT = pkg_resources.resource_filename(__name__,
os.path.join('..', 'data', 'conda_build_config.yaml'))
+DEFAULT_APPEND = pkg_resources.resource_filename(__name__,
+ os.path.join('..', 'data', 'recipe_append.yaml'))
+DEFAULT_DOCSERVER = 'http://www.idiap.ch'
@click.command(epilog='''
@@ -69,14 +73,21 @@ Examples:
@click.option('-m', '--config', '--variant-config-files', show_default=True,
default=DEFAULT_VARIANT, help='overwrites the path leading to ' \
'variant configuration file to use')
+@click.option('-a', '--append-file', show_default=True,
+ default=DEFAULT_APPEND, help='overwrites the path leading to ' \
+ 'appended configuration file to use')
+@click.option('-D', '--docserver', show_default=True,
+ default=DEFAULT_DOCSERVER, help='Server used for uploading artifacts ' \
+ 'and other goodies')
@click.option('-d', '--dry-run/--no-dry-run', default=False,
help='Only goes through the actions, but does not execute them ' \
'(combine with the verbosity flags - e.g. ``-vvv``) to enable ' \
'printing to help you understand what will be done')
@verbosity_option()
@bdt.raise_on_error
-def bootstrap(name, recipe_dir, python, overwrite, condarc, config, dry_run):
- """This program uses conda to build a development environment for a recipe
+def bootstrap(name, recipe_dir, python, overwrite, condarc, config,
+ append_file, docserver, dry_run):
+ """Creates a development environment for a recipe
It uses the conda render API to render a recipe and install an environment
containing all build/host, run and test dependencies of a package. It does
@@ -106,9 +117,13 @@ def bootstrap(name, recipe_dir, python, overwrite, condarc, config, dry_run):
"properly?")
# set condarc before continuing
- logger.debug('$ export CONDARC=%s', condarc)
+ logger.debug("[var] CONDARC=%s", condarc)
os.environ['CONDARC'] = condarc
- deps = parse_dependencies(conda, recipe_dir, python, config)
+ logger.debug("[var] DOCSERVER=%s", docserver)
+ os.environ['DOCSERVER'] = docserver
+
+ conda_config = make_conda_config(config, python, append_file, condarc)
+ deps = parse_dependencies(recipe_dir, conda_config)
status = conda_create(conda, name, overwrite, condarc, deps, dry_run)
click.echo('Execute on your shell: "conda activate %s"' % name)
diff --git a/bob/devtools/scripts/build.py b/bob/devtools/scripts/build.py
new file mode 100644
index 00000000..e17335d9
--- /dev/null
+++ b/bob/devtools/scripts/build.py
@@ -0,0 +1,139 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+import sys
+import logging
+logger = logging.getLogger(__name__)
+
+import pkg_resources
+import click
+
+from . import bdt
+from ..log import verbosity_option
+from ..conda import next_build_number, osname
+from ..bootstrap import get_rendered_metadata, get_parsed_recipe
+
+
+from .bootstrap import DEFAULT_CONDARC, DEFAULT_VARIANT, DEFAULT_APPEND, \
+ DEFAULT_DOCSERVER
+
+
+@click.command(epilog='''
+Examples:
+
+ 1. Builds recipe from one of our build dependencies (inside bob.conda):
+
+ $ cd bob.conda
+ $ bdt build -vv conda/libblitz
+
+
+ 2. Builds recipe from one of our packages, for Python 3.6 (if that is not
+ already the default for you):
+
+ $ bdt build --python=3.6 -vv path/to/conda/dir
+
+
+ 3. To build multiple recipes, just pass the paths to them:
+
+ $ bdt build --python=3.6 -vv path/to/recipe-dir/1 path/to/recipe-dir/2
+''')
+@click.argument('recipe-dir', required=False, type=click.Path(file_okay=False,
+ dir_okay=True, exists=True), nargs=-1)
+@click.option('-p', '--python', default=('%d.%d' % sys.version_info[:2]),
+ show_default=True, help='Version of python to build the ' \
+ 'environment for [default: %(default)s]')
+@click.option('-r', '--condarc', default=DEFAULT_CONDARC, show_default=True,
+ help='overwrites the path leading to the condarc file to use',)
+@click.option('-m', '--config', '--variant-config-files', show_default=True,
+ default=DEFAULT_VARIANT, help='overwrites the path leading to ' \
+ 'variant configuration file to use')
+@click.option('-c', '--channel', show_default=True,
+ default='https://www.idiap.ch/software/bob/conda/label/beta',
+ help='Channel URL where this package is meant to be uploaded to, ' \
+ 'after a successful build - typically, this is a beta channel')
+@click.option('-n', '--no-test', is_flag=True,
+ help='Do not test the package, only builds it')
+@click.option('-a', '--append-file', show_default=True,
+ default=DEFAULT_APPEND, help='overwrites the path leading to ' \
+ 'appended configuration file to use')
+@click.option('-D', '--docserver', show_default=True,
+ default=DEFAULT_DOCSERVER, help='Server used for uploading artifacts ' \
+ 'and other goodies')
+@click.option('-d', '--dry-run/--no-dry-run', default=False,
+ help='Only goes through the actions, but does not execute them ' \
+ '(combine with the verbosity flags - e.g. ``-vvv``) to enable ' \
+ 'printing to help you understand what will be done')
+@verbosity_option()
+@bdt.raise_on_error
+def build(recipe_dir, python, condarc, config, channel, no_test, append_file,
+ docserver, dry_run):
+ """Runs conda-build with a standard configuration and environment
+
+ This command wraps the execution of conda-build so that you use the same
+ ``condarc`` and ``conda_build_config.yaml`` file we use for our CI. It
+ always set ``--no-anaconda-upload``.
+
+ Note that both files are embedded within bob.devtools - you may need to
+ update your environment before trying this.
+ """
+
+ # if we are in a dry-run mode, let's let it be known
+ if dry_run:
+ logger.warn('!!!! DRY RUN MODE !!!!')
+ logger.warn('Nothing will be really built')
+
+ recipe_dir = recipe_dir or [os.path.join(os.path.realpath('.'), 'conda')]
+
+ logger.debug("[var] CONDARC=%s", condarc)
+
+ from ..bootstrap import make_conda_config
+ conda_config = make_conda_config(config, python, append_file, condarc)
+
+ logger.debug("[var] DOCSERVER=%s", docserver)
+ os.environ['DOCSERVER'] = docserver
+
+ for d in recipe_dir:
+
+ if not os.path.exists(d):
+ raise RuntimeError("The directory %s does not exist" % recipe_dir)
+
+ version_candidate = os.path.join(d, '..', 'version.txt')
+ if os.path.exists(version_candidate):
+ version = open(version_candidate).read().rstrip()
+ logger.debug("[var] BOB_PACKAGE_VERSION=%s", version)
+ os.environ['BOB_PACKAGE_VERSION'] = version
+
+ # pre-renders the recipe - figures out package name and version
+ metadata = get_rendered_metadata(d, conda_config)
+
+ # checks we should actually build this recipe
+ from ..conda import should_skip_build
+ if should_skip_build(metadata):
+ logger.warn('Skipping UNSUPPORTED build of "%s" for py%s on %s',
+ d, python.replace('.',''), osname())
+ return 0
+
+ # converts the metadata output into parsed yaml and continues the process
+ rendered_recipe = get_parsed_recipe(metadata)
+
+ # if a channel URL was passed, set the build number
+ if channel:
+ build_number, _ = next_build_number(channel,
+ rendered_recipe['package']['name'],
+ rendered_recipe['package']['version'], python)
+ else:
+ build_number = 0
+
+ logger.debug("[var] BOB_BUILD_NUMBER=%s", build_number)
+ os.environ['BOB_BUILD_NUMBER'] = str(build_number)
+
+ # we don't execute the following command, it is just here for logging
+ # purposes. we directly use the conda_build API.
+ logger.info('Building %s-%s-py%s (build: %d) for %s',
+ rendered_recipe['package']['name'],
+ rendered_recipe['package']['version'], python.replace('.',''),
+ build_number, osname())
+ if not dry_run:
+ from conda_build.api import build
+ build(d, config=conda_config, notest=no_test)
diff --git a/bob/devtools/scripts/cb_output.py b/bob/devtools/scripts/cb_output.py
deleted file mode 100644
index cfebdb64..00000000
--- a/bob/devtools/scripts/cb_output.py
+++ /dev/null
@@ -1,52 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-import logging
-logger = logging.getLogger(__name__)
-
-import click
-from click.testing import CliRunner
-import conda_build.api as cb
-
-from . import bdt
-from ..log import verbosity_option
-from ..conda import should_skip_build
-
-
-@click.command(context_settings=dict(
- ignore_unknown_options=True,
- allow_extra_args=True,
- ),
- epilog='''\b
-Examples:
-$ bdt cb-output conda_recipe_dir
-$ bdt cb-output ../bob.conda/conda/kaldi -m ../bob.admin/gitlab/conda_build_config.yaml --python 3.6
-'''
-)
-@click.argument('recipe_path')
-@click.option('-m', '--variant-config-files', help='see conda build --help')
-@click.option('--python', help='see conda build --help')
-@verbosity_option()
-@bdt.raise_on_error
-def cb_output(recipe_path, variant_config_files, python):
- """Outputs name(s) of package(s) that would be generated by conda build.
-
- This command accepts extra unknown arguments so you can give it the same
- arguments that you would give to conda build.
-
- As of now, it only parses -m/--variant_config_files and --python and other
- arguments are ignored.
- """
- clirunner = CliRunner()
- with clirunner.isolation():
- # render
- config = cb.get_or_merge_config(
- None, variant_config_files=variant_config_files, python=python)
- metadata_tuples = cb.render(recipe_path, config=config)
-
- # check if build(s) should be skipped
- if should_skip_build(metadata_tuples):
- return 0
-
- paths = cb.get_output_file_paths(metadata_tuples, config=config)
- click.echo('\n'.join(sorted(paths)))
diff --git a/conda/meta.yaml b/conda/meta.yaml
index 21ac1915..566b7319 100644
--- a/conda/meta.yaml
+++ b/conda/meta.yaml
@@ -57,6 +57,7 @@ test:
- bdt visibility --help
- bdt dumpsphinx --help
- bdt bootstrap --help
+ - bdt build --help
- sphinx-build -aEW ${PREFIX}/share/doc/{{ name }}/doc {{ project_dir }}/sphinx
about:
diff --git a/setup.py b/setup.py
index d5a8e803..08acb341 100644
--- a/setup.py
+++ b/setup.py
@@ -39,13 +39,13 @@ setup(
'bdt = bob.devtools.scripts.bdt:main',
],
'bdt.cli': [
- 'cb-output = bob.devtools.scripts.cb_output:cb_output',
'release = bob.devtools.scripts.release:release',
'changelog = bob.devtools.scripts.changelog:changelog',
'lasttag = bob.devtools.scripts.lasttag:lasttag',
'visibility = bob.devtools.scripts.visibility:visibility',
'dumpsphinx = bob.devtools.scripts.dumpsphinx:dumpsphinx',
'bootstrap = bob.devtools.scripts.bootstrap:bootstrap',
+ 'build = bob.devtools.scripts.build:build',
],
},
classifiers=[
--
GitLab