From 4f438f9b969aea4061a656c8f65a572509d66005 Mon Sep 17 00:00:00 2001
From: Andre Anjos <andre.dos.anjos@gmail.com>
Date: Mon, 18 Feb 2019 09:21:26 +0100
Subject: [PATCH] [scripts][test] New functionality to run test-only builds
using conda packages
---
.../data/gitlab-ci/single-package.yaml | 36 ++++++
bob/devtools/scripts/build.py | 3 -
bob/devtools/scripts/ci.py | 39 ++++++
bob/devtools/scripts/test.py | 120 ++++++++++++++++++
conda/meta.yaml | 1 +
setup.py | 2 +
6 files changed, 198 insertions(+), 3 deletions(-)
create mode 100644 bob/devtools/scripts/test.py
diff --git a/bob/devtools/data/gitlab-ci/single-package.yaml b/bob/devtools/data/gitlab-ci/single-package.yaml
index f1cb7a09..d9e7a18d 100644
--- a/bob/devtools/data/gitlab-ci/single-package.yaml
+++ b/bob/devtools/data/gitlab-ci/single-package.yaml
@@ -86,6 +86,42 @@ build_macosx_36:
PYTHON_VERSION: "3.6"
+# Test targets (not normally used)
+.test_template: &test_job
+ stage: test
+ script:
+ - curl --silent "${BOOTSTRAP}" --output "bootstrap.py"
+ - python3 bootstrap.py -vv channel base
+ - source ${CONDA_ROOT}/etc/profile.d/conda.sh
+ - conda activate base
+ - bdt ci test -vv
+ - bdt ci clean -vv
+ cache: &test_caches
+ paths:
+ - miniconda.sh
+ - ${CONDA_ROOT}/pkgs/*.tar.bz2
+ - ${CONDA_ROOT}/pkgs/urls.txt
+
+
+.test_linux_template: &linux_test_job
+ <<: *test_job
+ tags:
+ - docker
+ image: continuumio/conda-concourse-ci
+ cache:
+ <<: *test_caches
+ key: "linux-cache"
+
+
+.test_macosx_template: &macosx_test_job
+ <<: *test_job
+ tags:
+ - macosx
+ cache:
+ <<: *test_caches
+ key: "macosx-cache"
+
+
# Deploy targets
.deploy_template: &deploy_job
stage: deploy
diff --git a/bob/devtools/scripts/build.py b/bob/devtools/scripts/build.py
index 52d48411..72924f76 100644
--- a/bob/devtools/scripts/build.py
+++ b/bob/devtools/scripts/build.py
@@ -80,9 +80,6 @@ def build(recipe_dir, python, condarc, config, no_test, append_file,
This command wraps the execution of conda-build so that you use the same
conda configuration we use for our CI. It always set
``--no-anaconda-upload``.
-
- Note that both files are embedded within bob.devtools - you may need to
- update your environment before trying this.
"""
# if we are in a dry-run mode, let's let it be known
diff --git a/bob/devtools/scripts/ci.py b/bob/devtools/scripts/ci.py
index 3ad13b14..7e08d42e 100644
--- a/bob/devtools/scripts/ci.py
+++ b/bob/devtools/scripts/ci.py
@@ -375,6 +375,45 @@ def base_build(order, python, dry_run):
condarc_options)
+@ci.command(epilog='''
+Examples:
+
+ 1. Tests the current package
+
+ $ bdt ci test -vv
+
+''')
+@click.option('-d', '--dry-run/--no-dry-run', default=False,
+ help='Only goes through the actions, but does not execute them ' \
+ '(combine with the verbosity flags - e.g. ``-vvv``) to enable ' \
+ 'printing to help you understand what will be done')
+@verbosity_option()
+@bdt.raise_on_error
+@click.pass_context
+def test(ctx, dry_run):
+ """Tests packages
+
+ This command tests packages in the CI infrastructure. It is **not** meant
+ to be used outside this context.
+ """
+
+ from ..constants import CONDA_BUILD_CONFIG, CONDA_RECIPE_APPEND
+
+ from .test import test
+ ctx.invoke(test,
+ package = glob.glob(os.path.join(os.environ['CONDA_ROOT'], 'conda-bld',
+ arch, name + '*.tar.bz2')),
+ condarc=None, #custom build configuration
+ config=CONDA_BUILD_CONFIG,
+ append_file=CONDA_RECIPE_APPEND,
+ server=SERVER,
+ private=(os.environ['CI_PROJECT_VISIBILITY'] != 'public'),
+ stable='CI_COMMIT_TAG' in os.environ,
+ dry_run=dry_run,
+ ci=True,
+ )
+
+
@ci.command(epilog='''
Examples:
diff --git a/bob/devtools/scripts/test.py b/bob/devtools/scripts/test.py
new file mode 100644
index 00000000..f1630e8e
--- /dev/null
+++ b/bob/devtools/scripts/test.py
@@ -0,0 +1,120 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+import sys
+
+import yaml
+import click
+import pkg_resources
+import conda_build.api
+
+from . import bdt
+from ..build import conda_arch, make_conda_config, get_docserver_setup, \
+ get_env_directory
+from ..constants import CONDA_BUILD_CONFIG, CONDA_RECIPE_APPEND, \
+ SERVER, MATPLOTLIB_RCDIR, BASE_CONDARC
+from ..bootstrap import set_environment, get_channels
+
+from ..log import verbosity_option, get_logger
+logger = get_logger(__name__)
+
+
+@click.command(epilog='''
+Examples:
+
+ 1. Builds recipe from one of our build dependencies (inside bob.conda):
+
+\b
+ $ cd bob.conda
+ $ bdt build -vv conda/libblitz
+
+
+ 2. Builds recipe from one of our packages, for Python 3.6 (if that is not already the default for you):
+
+ $ bdt build --python=3.6 -vv path/to/conda/dir
+
+
+ 3. To build multiple recipes, just pass the paths to them:
+
+ $ bdt build --python=3.6 -vv path/to/recipe-dir1 path/to/recipe-dir2
+''')
+@click.argument('package', required=True, type=click.Path(file_okay=True,
+ dir_okay=False, exists=True), nargs=-1)
+@click.option('-r', '--condarc',
+ help='Use custom conda configuration file instead of our own',)
+@click.option('-m', '--config', '--variant-config-files', show_default=True,
+ default=CONDA_BUILD_CONFIG, help='overwrites the path leading to ' \
+ 'variant configuration file to use')
+@click.option('-a', '--append-file', show_default=True,
+ default=CONDA_RECIPE_APPEND, help='overwrites the path leading to ' \
+ 'appended configuration file to use')
+@click.option('-S', '--server', show_default=True,
+ default='https://www.idiap.ch/software/bob', help='Server used for ' \
+ 'downloading conda packages and documentation indexes of required packages')
+@click.option('-P', '--private/--no-private', default=False,
+ help='Set this to **include** private channels on your build - ' \
+ 'you **must** be at Idiap to execute this build in this case - ' \
+ 'you **must** also use the correct server name through --server - ' \
+ 'notice this option has no effect if you also pass --condarc')
+@click.option('-X', '--stable/--no-stable', default=False,
+ help='Set this to **exclude** beta channels from your build - ' \
+ 'notice this option has no effect if you also pass --condarc')
+@click.option('-d', '--dry-run/--no-dry-run', default=False,
+ help='Only goes through the actions, but does not execute them ' \
+ '(combine with the verbosity flags - e.g. ``-vvv``) to enable ' \
+ 'printing to help you understand what will be done')
+@click.option('-C', '--ci/--no-ci', default=False, hidden=True,
+ help='Use this flag to indicate the build will be running on the CI')
+@verbosity_option()
+@bdt.raise_on_error
+def test(package, condarc, config, append_file, server, private, stable,
+ dry_run, ci):
+ """Tests (pre-built_ package through conda-build with stock configuration
+
+ This command wraps the execution of conda-build so that you use the same
+ conda configuration we use for our CI. It always set
+ ``--no-anaconda-upload``.
+ """
+
+ # if we are in a dry-run mode, let's let it be known
+ if dry_run:
+ logger.warn('!!!! DRY RUN MODE !!!!')
+ logger.warn('Nothing will be really built')
+
+ # get potential channel upload and other auxiliary channels
+ channels = get_channels(public=(not private), stable=stable, server=server,
+ intranet=ci)
+
+ if condarc is not None:
+ logger.info('Loading CONDARC file from %s...', condarc)
+ with open(condarc, 'rb') as f:
+ condarc_options = yaml.load(f)
+ else:
+ # use default and add channels
+ condarc_options = yaml.load(BASE_CONDARC) #n.b.: no channels
+ logger.info('Using the following channels during build:\n - %s',
+ '\n - '.join(channels + ['defaults']))
+ condarc_options['channels'] = channels + ['defaults']
+
+ # dump packages at base environment
+ prefix = get_env_directory(os.environ['CONDA_EXE'], 'base')
+ condarc_options['croot'] = os.path.join(prefix, 'conda-bld')
+
+ conda_config = make_conda_config(config, None, append_file,
+ condarc_options)
+
+ set_environment('MATPLOTLIBRC', MATPLOTLIB_RCDIR)
+
+ # setup BOB_DOCUMENTATION_SERVER environment variable (used for bob.extension
+ # and derived documentation building via Sphinx)
+ set_environment('DOCSERVER', server)
+ doc_urls = get_docserver_setup(public=(not private), stable=stable,
+ server=server, intranet=ci)
+ set_environment('BOB_DOCUMENTATION_SERVER', doc_urls)
+
+ arch = conda_arch()
+ for p in package:
+ logger.info('Testing %s at %s', p, arch)
+ if not dry_run:
+ conda_build.api.test(p, config=conda_config)
diff --git a/conda/meta.yaml b/conda/meta.yaml
index 79ea7417..ca2dec95 100644
--- a/conda/meta.yaml
+++ b/conda/meta.yaml
@@ -71,6 +71,7 @@ test:
- bdt dumpsphinx https://docs.python.org/3/objects.inv > /dev/null
- bdt create --help
- bdt build --help
+ - bdt test --help
- bdt getpath --help
#- bdt getpath -vv bob/bob.devtools .gitignore
- bdt caupdate --help
diff --git a/setup.py b/setup.py
index b37d0922..f9ab5ce5 100644
--- a/setup.py
+++ b/setup.py
@@ -55,6 +55,7 @@ setup(
'dumpsphinx = bob.devtools.scripts.dumpsphinx:dumpsphinx',
'create = bob.devtools.scripts.create:create',
'build = bob.devtools.scripts.build:build',
+ 'test = bob.devtools.scripts.test:test',
'getpath = bob.devtools.scripts.getpath:getpath',
'caupdate = bob.devtools.scripts.caupdate:caupdate',
'ci = bob.devtools.scripts.ci:ci',
@@ -63,6 +64,7 @@ setup(
'bdt.ci.cli': [
'base-build = bob.devtools.scripts.ci:base_build',
'build = bob.devtools.scripts.ci:build',
+ 'test = bob.devtools.scripts.ci:test',
'clean = bob.devtools.scripts.ci:clean',
'base-deploy = bob.devtools.scripts.ci:base_deploy',
'deploy = bob.devtools.scripts.ci:deploy',
--
GitLab