From c25481da112bd29ed68697542463e104a30bb829 Mon Sep 17 00:00:00 2001
From: Andre Anjos <andre.dos.anjos@gmail.com>
Date: Thu, 12 Dec 2019 14:45:23 +0100
Subject: [PATCH] [build,scripts,data] Allows the environment variable
`$NOSE_EVAL_ATTR` to indicate which tests to skip during conda build (c.f.
bob/bob.nightlies#55)
---
bob/devtools/build.py | 43 +++++++++++++++++++---------
bob/devtools/data/recipe_append.yaml | 2 +-
bob/devtools/scripts/build.py | 15 +++++++++-
bob/devtools/scripts/ci.py | 38 +++++++++++++++---------
bob/devtools/scripts/create.py | 1 +
bob/devtools/scripts/graph.py | 3 ++
bob/devtools/scripts/rebuild.py | 12 ++++++++
bob/devtools/scripts/test.py | 12 ++++++++
8 files changed, 97 insertions(+), 29 deletions(-)
diff --git a/bob/devtools/build.py b/bob/devtools/build.py
index 631da43e..7228ed89 100644
--- a/bob/devtools/build.py
+++ b/bob/devtools/build.py
@@ -14,6 +14,7 @@ import platform
import subprocess
import logging
+
logger = logging.getLogger(__name__)
import yaml
@@ -23,13 +24,15 @@ import distutils.version
def remove_conda_loggers():
"""Cleans-up conda API logger handlers to avoid logging repetition"""
- z = logging.getLogger() #conda places their handlers inside root
+ z = logging.getLogger() # conda places their handlers inside root
if z.handlers:
handler = z.handlers[0]
z.removeHandler(handler)
logger.debug("Removed conda logger handler at %s", handler)
+
import conda_build.api
+
remove_conda_loggers()
@@ -94,6 +97,7 @@ def next_build_number(channel_url, basename):
"""
from conda.exports import get_index
+
remove_conda_loggers()
# get the channel index
@@ -101,13 +105,15 @@ def next_build_number(channel_url, basename):
index = get_index(channel_urls=[channel_url], prepend=False)
# remove .tar.bz2/.conda from name, then split from the end twice, on '-'
- if basename.endswith('.tar.bz2'):
+ if basename.endswith(".tar.bz2"):
name, version, build = basename[:-8].rsplit("-", 2)
- elif basename.endswith('.conda'):
+ elif basename.endswith(".conda"):
name, version, build = basename[:-6].rsplit("-", 2)
else:
- raise RuntimeError("Package name %s does not end in either " \
- ".tar.bz2 or .conda" % (basename,))
+ raise RuntimeError(
+ "Package name %s does not end in either "
+ ".tar.bz2 or .conda" % (basename,)
+ )
# remove the build number as we're looking for the next value
# examples to be coped with:
@@ -176,6 +182,7 @@ def make_conda_config(config, python, append_file, condarc_options):
"""
from conda_build.conda_interface import url_path
+
remove_conda_loggers()
retval = conda_build.api.get_or_merge_config(
@@ -244,13 +251,15 @@ def exists_on_channel(channel_url, basename):
# remove .tar.bz2/.conda from name, then split from the end twice, on
# '-'
- if name.endswith('.conda'):
+ if name.endswith(".conda"):
name, version, build = name[:-6].rsplit("-", 2)
- elif name.endswith('.tar.bz2'):
+ elif name.endswith(".tar.bz2"):
name, version, build = name[:-8].rsplit("-", 2)
else:
- raise RuntimeError("Package name %s does not end in either " \
- ".tar.bz2 or .conda" % (name,))
+ raise RuntimeError(
+ "Package name %s does not end in either "
+ ".tar.bz2 or .conda" % (name,)
+ )
# remove the build number as we're looking for the next value
# examples to be coped with:
@@ -265,13 +274,13 @@ def exists_on_channel(channel_url, basename):
self_build_number = _get_build_number(basename)
other_build_numbers = dict(
- [(k, _get_build_number(os.path.basename(k))) for k in urls]
- )
+ [(k, _get_build_number(os.path.basename(k))) for k in urls]
+ )
if self_build_number in other_build_numbers.values():
- pkg_type = '.conda' if basename.endswith('.conda') else '.tar.bz2'
+ pkg_type = ".conda" if basename.endswith(".conda") else ".tar.bz2"
for k, v in other_build_numbers.items():
- if k.endswith(pkg_type): #match
+ if k.endswith(pkg_type): # match
return "".join((channel_url, k))
@@ -737,6 +746,13 @@ if __name__ == "__main__":
"two ``-vv`` to enable information messages and three ``-vvv`` "
"to enable debug messages [default: %(default)s]",
)
+ parser.add_argument(
+ "--nose-eval-attr",
+ "-A",
+ default="",
+ help="Use this flag to avoid running certain tests during the build. "
+ "It forwards all settings to ``nosetests --eval-attr=<settings>``",
+ )
args = parser.parse_args()
@@ -755,6 +771,7 @@ if __name__ == "__main__":
bootstrap.set_environment("DOCSERVER", server)
bootstrap.set_environment("LANG", "en_US.UTF-8")
bootstrap.set_environment("LC_ALL", os.environ["LANG"])
+ bootstrap.set_environment("NOSE_EVAL_ATTR", args.nose_eval_attr)
# get information about the version of the package being built
version, is_prerelease = check_version(args.work_dir, args.tag)
diff --git a/bob/devtools/data/recipe_append.yaml b/bob/devtools/data/recipe_append.yaml
index 91bb2450..c0481152 100644
--- a/bob/devtools/data/recipe_append.yaml
+++ b/bob/devtools/data/recipe_append.yaml
@@ -1,4 +1,4 @@
build:
script_env:
- DOCSERVER
- - BOB_NOSE_SLOW
+ - NOSE_EVAL_ATTR
diff --git a/bob/devtools/scripts/build.py b/bob/devtools/scripts/build.py
index feac1b0a..8ca767cf 100644
--- a/bob/devtools/scripts/build.py
+++ b/bob/devtools/scripts/build.py
@@ -22,6 +22,7 @@ from ..build import (
get_output_path,
remove_conda_loggers,
)
+
remove_conda_loggers()
from ..constants import (
@@ -144,6 +145,14 @@ Examples:
hidden=True,
help="Use this flag to indicate the build will be running on the CI",
)
+@click.option(
+ "-A",
+ "--nose-eval-attr",
+ envvar="NOSE_EVAL_ATTR",
+ default="",
+ help="Use this flag to avoid running certain tests during the build. "
+ "It forwards all settings to ``nosetests --eval-attr=<settings>``",
+)
@verbosity_option()
@bdt.raise_on_error
def build(
@@ -159,6 +168,7 @@ def build(
stable,
dry_run,
ci,
+ nose_eval_attr,
):
"""Builds package through conda-build with stock configuration.
@@ -183,10 +193,10 @@ def build(
#### HACK to avoid ripgrep ignoring bin/ directories in our checkouts
# TODO: Remove this hack as soon as possible
from bob.devtools.bootstrap import do_hack
+
project_dir = os.path.dirname(recipe_dir[0])
do_hack(project_dir)
-
# get potential channel upload and other auxiliary channels
channels = get_channels(
public=(not private),
@@ -233,6 +243,9 @@ def build(
)
set_environment("BOB_DOCUMENTATION_SERVER", doc_urls)
+ # this is for testing and may limit which tests run
+ set_environ("NOSE_EVAL_ATTR", nose_eval_attr)
+
arch = conda_arch()
for d in recipe_dir:
diff --git a/bob/devtools/scripts/ci.py b/bob/devtools/scripts/ci.py
index 7aec5654..9677342d 100644
--- a/bob/devtools/scripts/ci.py
+++ b/bob/devtools/scripts/ci.py
@@ -422,8 +422,7 @@ def base_build(order, group, python, dry_run):
echo_normal("\n" + (80 * "="))
pytext = "for python-%s " % pyver if pyver is not None else ""
echo_normal(
- 'Building "%s" %s(%d/%d)'
- % (recipe, pytext, k + 1, len(recipes))
+ 'Building "%s" %s(%d/%d)' % (recipe, pytext, k + 1, len(recipes))
)
echo_normal((80 * "=") + "\n")
if not os.path.exists(os.path.join(recipe, "meta.yaml")):
@@ -505,13 +504,17 @@ def test(ctx, dry_run):
from .test import test
- base_path = os.path.join(os.environ["CONDA_ROOT"], "conda-bld", "*",
- os.environ["CI_PROJECT_NAME"])
+ base_path = os.path.join(
+ os.environ["CONDA_ROOT"],
+ "conda-bld",
+ "*",
+ os.environ["CI_PROJECT_NAME"],
+ )
ctx.invoke(
test,
- package=glob.glob(base_path + "*.conda") + \
- glob.glob(base_path + "*.tar.bz2"),
+ package=glob.glob(base_path + "*.conda")
+ + glob.glob(base_path + "*.tar.bz2"),
condarc=condarc,
config=variants_file,
append_file=append_file,
@@ -521,6 +524,7 @@ def test(ctx, dry_run):
stable="CI_COMMIT_TAG" in os.environ,
dry_run=dry_run,
ci=True,
+ nose_eval_attr=os.environ.get("NOSE_EVAL_ATTR", ""),
)
@@ -599,6 +603,7 @@ def build(ctx, dry_run, recipe_dir):
stable="CI_COMMIT_TAG" in os.environ,
dry_run=dry_run,
ci=True,
+ nose_eval_attr=os.environ.get("NOSE_EVAL_ATTR", ""),
)
@@ -754,6 +759,7 @@ def nightlies(ctx, order, dry_run):
stable=stable,
dry_run=dry_run,
ci=True,
+ nose_eval_attr=os.environ.get("NOSE_EVAL_ATTR", ""),
)
is_master = os.environ["CI_COMMIT_REF_NAME"] == "master"
@@ -944,7 +950,10 @@ def docs(ctx, requirement, dry_run):
f.write(data)
logger.info("Building documentation...")
- ctx.invoke(build, dry_run=dry_run)
+ ctx.invoke(
+ build, dry_run=dry_run,
+ nose_eval_attr=os.environ.get("NOSE_EVAL_ATTR", ""),
+ )
@ci.command(
@@ -991,14 +1000,15 @@ def clean_betas(dry_run):
logger.warn("Nothing is being executed on server.")
import re
+
if os.environ["CI_PROJECT_NAMESPACE"] == "beat":
- includes = re.compile(r'^beat.*')
+ includes = re.compile(r"^beat.*")
else:
- includes = re.compile(r'^(bob|batl|gridtk).*')
+ includes = re.compile(r"^(bob|batl|gridtk).*")
cleanup(
- dry_run=dry_run,
- username=os.environ["DOCUSER"],
- password=os.environ["DOCPASS"],
- includes=includes,
- )
+ dry_run=dry_run,
+ username=os.environ["DOCUSER"],
+ password=os.environ["DOCPASS"],
+ includes=includes,
+ )
diff --git a/bob/devtools/scripts/create.py b/bob/devtools/scripts/create.py
index 835064d0..c397945c 100644
--- a/bob/devtools/scripts/create.py
+++ b/bob/devtools/scripts/create.py
@@ -196,6 +196,7 @@ def create(
# set some environment variables before continuing
set_environment("DOCSERVER", server, os.environ)
+ set_environment("NOSE_EVAL_ATTR", "", os.environ)
logger.debug(
'This package is considered part of group "%s" - tunning '
diff --git a/bob/devtools/scripts/graph.py b/bob/devtools/scripts/graph.py
index 12f654bf..b7b1ee36 100644
--- a/bob/devtools/scripts/graph.py
+++ b/bob/devtools/scripts/graph.py
@@ -191,6 +191,9 @@ def graph(package, python, condarc, config, append_file, server, private,
set_environment("DOCSERVER", server)
set_environment("BOB_DOCUMENTATION_SERVER", "/not/set")
+ # avoids conda-build complaints
+ set_environment("NOSE_EVAL_ATTR", "")
+
adj_matrix = compute_adjencence_matrix(gl, package, conda_config,
channels[0], deptypes=deptypes)
diff --git a/bob/devtools/scripts/rebuild.py b/bob/devtools/scripts/rebuild.py
index 17c019ed..7af0ce46 100644
--- a/bob/devtools/scripts/rebuild.py
+++ b/bob/devtools/scripts/rebuild.py
@@ -130,6 +130,14 @@ Examples:
hidden=True,
help="Use this flag to indicate the build will be running on the CI",
)
+@click.option(
+ "-A",
+ "--nose-eval-attr",
+ envvar="NOSE_EVAL_ATTR",
+ default="",
+ help="Use this flag to avoid running certain tests during the build. "
+ "It forwards all settings to ``nosetests --eval-attr=<settings>``",
+)
@verbosity_option()
@bdt.raise_on_error
def rebuild(
@@ -144,6 +152,7 @@ def rebuild(
stable,
dry_run,
ci,
+ nose_eval_attr,
):
"""Tests and rebuilds packages through conda-build with stock
configuration.
@@ -213,6 +222,9 @@ def rebuild(
)
set_environment("BOB_DOCUMENTATION_SERVER", doc_urls)
+ # this is for testing and may limit which tests run
+ set_environ("NOSE_EVAL_ATTR", nose_eval_attr)
+
arch = conda_arch()
for d in recipe_dir:
diff --git a/bob/devtools/scripts/test.py b/bob/devtools/scripts/test.py
index b000ca8a..19eecfaf 100644
--- a/bob/devtools/scripts/test.py
+++ b/bob/devtools/scripts/test.py
@@ -122,6 +122,14 @@ Examples:
hidden=True,
help="Use this flag to indicate the build will be running on the CI",
)
+@click.option(
+ "-A",
+ "--nose-eval-attr",
+ envvar="NOSE_EVAL_ATTR",
+ default="",
+ help="Use this flag to avoid running certain tests during the build. "
+ "It forwards all settings to ``nosetests --eval-attr=<settings>``",
+)
@verbosity_option()
@bdt.raise_on_error
def test(
@@ -135,6 +143,7 @@ def test(
stable,
dry_run,
ci,
+ nose_eval_attr,
):
"""Tests (pre-built) package through conda-build with stock configuration.
@@ -194,6 +203,9 @@ def test(
)
set_environment("BOB_DOCUMENTATION_SERVER", doc_urls)
+ # this is for testing and may limit which tests run
+ set_environ("NOSE_EVAL_ATTR", nose_eval_attr)
+
arch = conda_arch()
for p in package:
logger.info("Testing %s at %s", p, arch)
--
GitLab