Commit da9d8a52 authored by André Anjos's avatar André Anjos 💬

[scritpts] Implement build support; Remove old cb-output script (incorporate into build)

parent 090ac45b
......@@ -10,25 +10,45 @@ import logging
logger = logging.getLogger(__name__)
import yaml
from conda_build.api import get_or_merge_config, render, output_yaml
def get_rendered_recipe(conda, recipe_dir, python, config):
def make_conda_config(config, python, append_file, condarc):
from conda_build.api import get_or_merge_config
from conda_build.conda_interface import url_path
with open(condarc, 'rb') as f:
condarc_options = yaml.load(f)
retval = get_or_merge_config(None, variant_config_files=config,
python=python, append_sections_file=append_file, **condarc_options)
retval.channel_urls = []
for url in condarc_options['channels']:
# allow people to specify relative or absolute paths to local channels
# These channels still must follow conda rules - they must have the
# appropriate platform-specific subdir (e.g. win-64)
if os.path.isdir(url):
if not os.path.isabs(url):
url = os.path.normpath(os.path.abspath(os.path.join(os.getcwd(), url)))
url = url_path(url)
retval.channel_urls.append(url)
return retval
def get_rendered_metadata(recipe_dir, config):
'''Renders the recipe and returns the interpreted YAML file'''
# equivalent command execute - in here we use the conda API
cmd = [
conda, 'render',
'--variant-config-files', config,
'--python', python,
recipe_dir,
]
logger.debug('$ ' + ' '.join(cmd))
from conda_build.api import render
return render(recipe_dir, config=config)
def get_parsed_recipe(metadata):
'''Renders the recipe and returns the interpreted YAML file'''
# do the real job
config = get_or_merge_config(None, variant_config_files=config,
python=python)
metadata = render(recipe_dir, config=config)
from conda_build.api import output_yaml
output = output_yaml(metadata[0][0])
return yaml.load(output)
......@@ -37,9 +57,10 @@ def remove_pins(deps):
return [l.split()[0] for l in deps]
def parse_dependencies(conda, recipe_dir, python, config):
def parse_dependencies(recipe_dir, config):
recipe = get_rendered_recipe(conda, recipe_dir, python, config)
metadata = get_rendered_metadata(recipe_dir, config)
recipe = get_parsed_recipe(metadata)
return remove_pins(recipe['requirements'].get('build', [])) + \
remove_pins(recipe['requirements'].get('host', [])) + \
recipe['requirements'].get('run', []) + \
......
#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''Utilities for deadling with conda packages'''
import re
import logging
import platform
logger = logging.getLogger(__name__)
def osname():
"""Returns the current OS name as recognized by conda"""
r = 'unknown'
if platform.system().lower() == 'linux':
r = 'linux'
elif platform.system().lower() == 'darwin':
r = 'osx'
else:
raise RuntimeError('Unsupported system "%s"' % platform.system())
if platform.machine().lower() == 'x86_64':
r += '-64'
else:
raise RuntimeError('Unsupported machine type "%s"' % platform.machine())
return r
def should_skip_build(metadata_tuples):
"""Takes the output of render_recipe as input and evaluates if this
recipe's build should be skipped.
"""
return all(m[0].skip() for m in metadata_tuples)
"""Takes the output of render_recipe as input and evaluates if this
recipe's build should be skipped.
"""
return all(m[0].skip() for m in metadata_tuples)
def next_build_number(channel_url, name, version, python):
"""Calculates the next build number of a package given the channel
This function returns the next build number (integer) for a package given its
recipe, dependencies, name, version and python version. It looks on the
channel URL provided and figures out if any clash would happen and what would
be the highest build number available for that configuration.
Args:
channel_url: The URL where to look for packages clashes (normally a beta
channel)
name: The name of the package
version: The version of the package
python: The version of python as 2 digits (e.g.: "2.7" or "3.6")
Returns: The next build number with the current configuration. Zero (0) is
returned if no match is found. Also returns the URLs of the packages it
finds with matches on the name, version and python-version.
"""
from conda.exports import get_index
# no dot in py_ver
py_ver = python.replace('.', '')
# get the channel index
logger.debug('Downloading channel index from %s', channel_url)
index = get_index(channel_urls=[channel_url], prepend=False)
# search if package with the same version exists
build_number = 0
urls = []
for dist in index:
if dist.name == name and dist.version == version:
match = re.match('py[2-9][0-9]+', dist.build_string)
if match and match.group() == 'py{}'.format(py_ver):
logger.debug("Found match at %s for %s-%s-py%s", index[dist].url,
name, version, py_ver)
build_number = max(build_number, dist.build_number + 1)
urls.append(index[dist].url)
urls = [url.replace(channel_url, '') for url in urls]
return build_number, urls
build:
script_env:
- DOCSERVER
......@@ -8,16 +8,20 @@ logger = logging.getLogger(__name__)
import pkg_resources
import click
import yaml
from . import bdt
from ..log import verbosity_option
from ..bootstrap import parse_dependencies, conda_create
from ..bootstrap import parse_dependencies, conda_create, make_conda_config
DEFAULT_CONDARC = pkg_resources.resource_filename(__name__,
os.path.join('..', 'data', 'build-condarc'))
DEFAULT_VARIANT = pkg_resources.resource_filename(__name__,
os.path.join('..', 'data', 'conda_build_config.yaml'))
DEFAULT_APPEND = pkg_resources.resource_filename(__name__,
os.path.join('..', 'data', 'recipe_append.yaml'))
DEFAULT_DOCSERVER = 'http://www.idiap.ch'
@click.command(epilog='''
......@@ -69,14 +73,21 @@ Examples:
@click.option('-m', '--config', '--variant-config-files', show_default=True,
default=DEFAULT_VARIANT, help='overwrites the path leading to ' \
'variant configuration file to use')
@click.option('-a', '--append-file', show_default=True,
default=DEFAULT_APPEND, help='overwrites the path leading to ' \
'appended configuration file to use')
@click.option('-D', '--docserver', show_default=True,
default=DEFAULT_DOCSERVER, help='Server used for uploading artifacts ' \
'and other goodies')
@click.option('-d', '--dry-run/--no-dry-run', default=False,
help='Only goes through the actions, but does not execute them ' \
'(combine with the verbosity flags - e.g. ``-vvv``) to enable ' \
'printing to help you understand what will be done')
@verbosity_option()
@bdt.raise_on_error
def bootstrap(name, recipe_dir, python, overwrite, condarc, config, dry_run):
"""This program uses conda to build a development environment for a recipe
def bootstrap(name, recipe_dir, python, overwrite, condarc, config,
append_file, docserver, dry_run):
"""Creates a development environment for a recipe
It uses the conda render API to render a recipe and install an environment
containing all build/host, run and test dependencies of a package. It does
......@@ -106,9 +117,13 @@ def bootstrap(name, recipe_dir, python, overwrite, condarc, config, dry_run):
"properly?")
# set condarc before continuing
logger.debug('$ export CONDARC=%s', condarc)
logger.debug("[var] CONDARC=%s", condarc)
os.environ['CONDARC'] = condarc
deps = parse_dependencies(conda, recipe_dir, python, config)
logger.debug("[var] DOCSERVER=%s", docserver)
os.environ['DOCSERVER'] = docserver
conda_config = make_conda_config(config, python, append_file, condarc)
deps = parse_dependencies(recipe_dir, conda_config)
status = conda_create(conda, name, overwrite, condarc, deps, dry_run)
click.echo('Execute on your shell: "conda activate %s"' % name)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import sys
import logging
logger = logging.getLogger(__name__)
import pkg_resources
import click
from . import bdt
from ..log import verbosity_option
from ..conda import next_build_number, osname
from ..bootstrap import get_rendered_metadata, get_parsed_recipe
from .bootstrap import DEFAULT_CONDARC, DEFAULT_VARIANT, DEFAULT_APPEND, \
DEFAULT_DOCSERVER
@click.command(epilog='''
Examples:
1. Builds recipe from one of our build dependencies (inside bob.conda):
$ cd bob.conda
$ bdt build -vv conda/libblitz
2. Builds recipe from one of our packages, for Python 3.6 (if that is not
already the default for you):
$ bdt build --python=3.6 -vv path/to/conda/dir
3. To build multiple recipes, just pass the paths to them:
$ bdt build --python=3.6 -vv path/to/recipe-dir/1 path/to/recipe-dir/2
''')
@click.argument('recipe-dir', required=False, type=click.Path(file_okay=False,
dir_okay=True, exists=True), nargs=-1)
@click.option('-p', '--python', default=('%d.%d' % sys.version_info[:2]),
show_default=True, help='Version of python to build the ' \
'environment for [default: %(default)s]')
@click.option('-r', '--condarc', default=DEFAULT_CONDARC, show_default=True,
help='overwrites the path leading to the condarc file to use',)
@click.option('-m', '--config', '--variant-config-files', show_default=True,
default=DEFAULT_VARIANT, help='overwrites the path leading to ' \
'variant configuration file to use')
@click.option('-c', '--channel', show_default=True,
default='https://www.idiap.ch/software/bob/conda/label/beta',
help='Channel URL where this package is meant to be uploaded to, ' \
'after a successful build - typically, this is a beta channel')
@click.option('-n', '--no-test', is_flag=True,
help='Do not test the package, only builds it')
@click.option('-a', '--append-file', show_default=True,
default=DEFAULT_APPEND, help='overwrites the path leading to ' \
'appended configuration file to use')
@click.option('-D', '--docserver', show_default=True,
default=DEFAULT_DOCSERVER, help='Server used for uploading artifacts ' \
'and other goodies')
@click.option('-d', '--dry-run/--no-dry-run', default=False,
help='Only goes through the actions, but does not execute them ' \
'(combine with the verbosity flags - e.g. ``-vvv``) to enable ' \
'printing to help you understand what will be done')
@verbosity_option()
@bdt.raise_on_error
def build(recipe_dir, python, condarc, config, channel, no_test, append_file,
docserver, dry_run):
"""Runs conda-build with a standard configuration and environment
This command wraps the execution of conda-build so that you use the same
``condarc`` and ``conda_build_config.yaml`` file we use for our CI. It
always set ``--no-anaconda-upload``.
Note that both files are embedded within bob.devtools - you may need to
update your environment before trying this.
"""
# if we are in a dry-run mode, let's let it be known
if dry_run:
logger.warn('!!!! DRY RUN MODE !!!!')
logger.warn('Nothing will be really built')
recipe_dir = recipe_dir or [os.path.join(os.path.realpath('.'), 'conda')]
logger.debug("[var] CONDARC=%s", condarc)
from ..bootstrap import make_conda_config
conda_config = make_conda_config(config, python, append_file, condarc)
logger.debug("[var] DOCSERVER=%s", docserver)
os.environ['DOCSERVER'] = docserver
for d in recipe_dir:
if not os.path.exists(d):
raise RuntimeError("The directory %s does not exist" % recipe_dir)
version_candidate = os.path.join(d, '..', 'version.txt')
if os.path.exists(version_candidate):
version = open(version_candidate).read().rstrip()
logger.debug("[var] BOB_PACKAGE_VERSION=%s", version)
os.environ['BOB_PACKAGE_VERSION'] = version
# pre-renders the recipe - figures out package name and version
metadata = get_rendered_metadata(d, conda_config)
# checks we should actually build this recipe
from ..conda import should_skip_build
if should_skip_build(metadata):
logger.warn('Skipping UNSUPPORTED build of "%s" for py%s on %s',
d, python.replace('.',''), osname())
return 0
# converts the metadata output into parsed yaml and continues the process
rendered_recipe = get_parsed_recipe(metadata)
# if a channel URL was passed, set the build number
if channel:
build_number, _ = next_build_number(channel,
rendered_recipe['package']['name'],
rendered_recipe['package']['version'], python)
else:
build_number = 0
logger.debug("[var] BOB_BUILD_NUMBER=%s", build_number)
os.environ['BOB_BUILD_NUMBER'] = str(build_number)
# we don't execute the following command, it is just here for logging
# purposes. we directly use the conda_build API.
logger.info('Building %s-%s-py%s (build: %d) for %s',
rendered_recipe['package']['name'],
rendered_recipe['package']['version'], python.replace('.',''),
build_number, osname())
if not dry_run:
from conda_build.api import build
build(d, config=conda_config, notest=no_test)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import logging
logger = logging.getLogger(__name__)
import click
from click.testing import CliRunner
import conda_build.api as cb
from . import bdt
from ..log import verbosity_option
from ..conda import should_skip_build
@click.command(context_settings=dict(
ignore_unknown_options=True,
allow_extra_args=True,
),
epilog='''\b
Examples:
$ bdt cb-output conda_recipe_dir
$ bdt cb-output ../bob.conda/conda/kaldi -m ../bob.admin/gitlab/conda_build_config.yaml --python 3.6
'''
)
@click.argument('recipe_path')
@click.option('-m', '--variant-config-files', help='see conda build --help')
@click.option('--python', help='see conda build --help')
@verbosity_option()
@bdt.raise_on_error
def cb_output(recipe_path, variant_config_files, python):
"""Outputs name(s) of package(s) that would be generated by conda build.
This command accepts extra unknown arguments so you can give it the same
arguments that you would give to conda build.
As of now, it only parses -m/--variant_config_files and --python and other
arguments are ignored.
"""
clirunner = CliRunner()
with clirunner.isolation():
# render
config = cb.get_or_merge_config(
None, variant_config_files=variant_config_files, python=python)
metadata_tuples = cb.render(recipe_path, config=config)
# check if build(s) should be skipped
if should_skip_build(metadata_tuples):
return 0
paths = cb.get_output_file_paths(metadata_tuples, config=config)
click.echo('\n'.join(sorted(paths)))
......@@ -57,6 +57,7 @@ test:
- bdt visibility --help
- bdt dumpsphinx --help
- bdt bootstrap --help
- bdt build --help
- sphinx-build -aEW ${PREFIX}/share/doc/{{ name }}/doc {{ project_dir }}/sphinx
about:
......
......@@ -39,13 +39,13 @@ setup(
'bdt = bob.devtools.scripts.bdt:main',
],
'bdt.cli': [
'cb-output = bob.devtools.scripts.cb_output:cb_output',
'release = bob.devtools.scripts.release:release',
'changelog = bob.devtools.scripts.changelog:changelog',
'lasttag = bob.devtools.scripts.lasttag:lasttag',
'visibility = bob.devtools.scripts.visibility:visibility',
'dumpsphinx = bob.devtools.scripts.dumpsphinx:dumpsphinx',
'bootstrap = bob.devtools.scripts.bootstrap:bootstrap',
'build = bob.devtools.scripts.build:build',
],
},
classifiers=[
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment