Commit c48694da authored by André Anjos's avatar André Anjos 💬

Merge branch 'ci-test-only' into 'master'

New functionality to run test-only builds using conda packages

See merge request !21
parents f4bc6f1d ddb10658
Pipeline #27195 passed with stages
in 9 minutes and 26 seconds
......@@ -86,6 +86,42 @@ build_macosx_36:
PYTHON_VERSION: "3.6"
# Test targets (not normally used)
.test_template: &test_job
stage: test
script:
- curl --silent "${BOOTSTRAP}" --output "bootstrap.py"
- python3 bootstrap.py -vv channel base
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- conda activate base
- bdt ci test -vv
- bdt ci clean -vv
cache: &test_caches
paths:
- miniconda.sh
- ${CONDA_ROOT}/pkgs/*.tar.bz2
- ${CONDA_ROOT}/pkgs/urls.txt
.test_linux_template: &linux_test_job
<<: *test_job
tags:
- docker
image: continuumio/conda-concourse-ci
cache:
<<: *test_caches
key: "linux-cache"
.test_macosx_template: &macosx_test_job
<<: *test_job
tags:
- macosx
cache:
<<: *test_caches
key: "macosx-cache"
# Deploy targets
.deploy_template: &deploy_job
stage: deploy
......
......@@ -80,9 +80,6 @@ def build(recipe_dir, python, condarc, config, no_test, append_file,
This command wraps the execution of conda-build so that you use the same
conda configuration we use for our CI. It always set
``--no-anaconda-upload``.
Note that both files are embedded within bob.devtools - you may need to
update your environment before trying this.
"""
# if we are in a dry-run mode, let's let it be known
......
......@@ -375,6 +375,45 @@ def base_build(order, python, dry_run):
condarc_options)
@ci.command(epilog='''
Examples:
1. Tests the current package
$ bdt ci test -vv
''')
@click.option('-d', '--dry-run/--no-dry-run', default=False,
help='Only goes through the actions, but does not execute them ' \
'(combine with the verbosity flags - e.g. ``-vvv``) to enable ' \
'printing to help you understand what will be done')
@verbosity_option()
@bdt.raise_on_error
@click.pass_context
def test(ctx, dry_run):
"""Tests packages
This command tests packages in the CI infrastructure. It is **not** meant
to be used outside this context.
"""
from ..constants import CONDA_BUILD_CONFIG, CONDA_RECIPE_APPEND
from .test import test
ctx.invoke(test,
package = glob.glob(os.path.join(os.environ['CONDA_ROOT'], 'conda-bld',
arch, name + '*.tar.bz2')),
condarc=None, #custom build configuration
config=CONDA_BUILD_CONFIG,
append_file=CONDA_RECIPE_APPEND,
server=SERVER,
private=(os.environ['CI_PROJECT_VISIBILITY'] != 'public'),
stable='CI_COMMIT_TAG' in os.environ,
dry_run=dry_run,
ci=True,
)
@ci.command(epilog='''
Examples:
......
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import sys
import yaml
import click
import pkg_resources
import conda_build.api
from . import bdt
from ..build import conda_arch, make_conda_config, get_docserver_setup, \
get_env_directory
from ..constants import CONDA_BUILD_CONFIG, CONDA_RECIPE_APPEND, \
SERVER, MATPLOTLIB_RCDIR, BASE_CONDARC
from ..bootstrap import set_environment, get_channels
from ..log import verbosity_option, get_logger
logger = get_logger(__name__)
@click.command(epilog='''
Examples:
1. Builds recipe from one of our build dependencies (inside bob.conda):
\b
$ cd bob.conda
$ bdt build -vv conda/libblitz
2. Builds recipe from one of our packages, for Python 3.6 (if that is not already the default for you):
$ bdt build --python=3.6 -vv path/to/conda/dir
3. To build multiple recipes, just pass the paths to them:
$ bdt build --python=3.6 -vv path/to/recipe-dir1 path/to/recipe-dir2
''')
@click.argument('package', required=True, type=click.Path(file_okay=True,
dir_okay=False, exists=True), nargs=-1)
@click.option('-r', '--condarc',
help='Use custom conda configuration file instead of our own',)
@click.option('-m', '--config', '--variant-config-files', show_default=True,
default=CONDA_BUILD_CONFIG, help='overwrites the path leading to ' \
'variant configuration file to use')
@click.option('-a', '--append-file', show_default=True,
default=CONDA_RECIPE_APPEND, help='overwrites the path leading to ' \
'appended configuration file to use')
@click.option('-S', '--server', show_default=True,
default='https://www.idiap.ch/software/bob', help='Server used for ' \
'downloading conda packages and documentation indexes of required packages')
@click.option('-P', '--private/--no-private', default=False,
help='Set this to **include** private channels on your build - ' \
'you **must** be at Idiap to execute this build in this case - ' \
'you **must** also use the correct server name through --server - ' \
'notice this option has no effect if you also pass --condarc')
@click.option('-X', '--stable/--no-stable', default=False,
help='Set this to **exclude** beta channels from your build - ' \
'notice this option has no effect if you also pass --condarc')
@click.option('-d', '--dry-run/--no-dry-run', default=False,
help='Only goes through the actions, but does not execute them ' \
'(combine with the verbosity flags - e.g. ``-vvv``) to enable ' \
'printing to help you understand what will be done')
@click.option('-C', '--ci/--no-ci', default=False, hidden=True,
help='Use this flag to indicate the build will be running on the CI')
@verbosity_option()
@bdt.raise_on_error
def test(package, condarc, config, append_file, server, private, stable,
dry_run, ci):
"""Tests (pre-built) package through conda-build with stock configuration
This command wraps the execution of conda-build so that you use the same
conda configuration we use for our CI. It always set
``--no-anaconda-upload``.
"""
# if we are in a dry-run mode, let's let it be known
if dry_run:
logger.warn('!!!! DRY RUN MODE !!!!')
logger.warn('Nothing will be really built')
# get potential channel upload and other auxiliary channels
channels = get_channels(public=(not private), stable=stable, server=server,
intranet=ci)
if condarc is not None:
logger.info('Loading CONDARC file from %s...', condarc)
with open(condarc, 'rb') as f:
condarc_options = yaml.load(f)
else:
# use default and add channels
condarc_options = yaml.load(BASE_CONDARC) #n.b.: no channels
logger.info('Using the following channels during build:\n - %s',
'\n - '.join(channels + ['defaults']))
condarc_options['channels'] = channels + ['defaults']
# dump packages at base environment
prefix = get_env_directory(os.environ['CONDA_EXE'], 'base')
condarc_options['croot'] = os.path.join(prefix, 'conda-bld')
conda_config = make_conda_config(config, None, append_file,
condarc_options)
set_environment('MATPLOTLIBRC', MATPLOTLIB_RCDIR)
# setup BOB_DOCUMENTATION_SERVER environment variable (used for bob.extension
# and derived documentation building via Sphinx)
set_environment('DOCSERVER', server)
doc_urls = get_docserver_setup(public=(not private), stable=stable,
server=server, intranet=ci)
set_environment('BOB_DOCUMENTATION_SERVER', doc_urls)
arch = conda_arch()
for p in package:
logger.info('Testing %s at %s', p, arch)
if not dry_run:
conda_build.api.test(p, config=conda_config)
......@@ -71,6 +71,7 @@ test:
- bdt dumpsphinx https://docs.python.org/3/objects.inv > /dev/null
- bdt create --help
- bdt build --help
- bdt test --help
- bdt getpath --help
#- bdt getpath -vv bob/bob.devtools .gitignore
- bdt caupdate --help
......
......@@ -55,6 +55,7 @@ setup(
'dumpsphinx = bob.devtools.scripts.dumpsphinx:dumpsphinx',
'create = bob.devtools.scripts.create:create',
'build = bob.devtools.scripts.build:build',
'test = bob.devtools.scripts.test:test',
'getpath = bob.devtools.scripts.getpath:getpath',
'caupdate = bob.devtools.scripts.caupdate:caupdate',
'ci = bob.devtools.scripts.ci:ci',
......@@ -63,6 +64,7 @@ setup(
'bdt.ci.cli': [
'base-build = bob.devtools.scripts.ci:base_build',
'build = bob.devtools.scripts.ci:build',
'test = bob.devtools.scripts.ci:test',
'clean = bob.devtools.scripts.ci:clean',
'base-deploy = bob.devtools.scripts.ci:base_deploy',
'deploy = bob.devtools.scripts.ci:deploy',
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment