From 52757458c2eaa81f9a4c8e3f1f1da5d1e49f030b Mon Sep 17 00:00:00 2001
From: Andre Anjos <andre.dos.anjos@gmail.com>
Date: Mon, 16 Nov 2020 23:10:35 +0100
Subject: [PATCH] [log] Fix excessive logging after loading conda modules
---
bob/devtools/log.py | 19 ++++++++++++-
bob/devtools/scripts/build.py | 47 +++++++++++++++++++++++++--------
bob/devtools/scripts/create.py | 26 +++++++++++++-----
bob/devtools/scripts/rebuild.py | 11 +++++---
bob/devtools/scripts/test.py | 4 ++-
5 files changed, 84 insertions(+), 23 deletions(-)
diff --git a/bob/devtools/log.py b/bob/devtools/log.py
index b43a2a70..a6f70c4d 100644
--- a/bob/devtools/log.py
+++ b/bob/devtools/log.py
@@ -3,9 +3,10 @@
"""Logging utilities."""
-import logging
import os
import sys
+import logging
+import contextlib
import click
import termcolor
@@ -243,3 +244,19 @@ def verbosity_option(**kwargs):
)(f)
return custom_verbosity_option
+
+
+@contextlib.contextmanager
+def root_logger_protection():
+ """Protects the root logger against spurious (conda) manipulation"""
+
+ import copy
+
+ root_logger = logging.getLogger()
+ level = root_logger.level
+ handlers = copy.copy(root_logger.handlers)
+
+ yield
+
+ root_logger.setLevel(level)
+ root_logger.handlers = handlers
diff --git a/bob/devtools/scripts/build.py b/bob/devtools/scripts/build.py
index d9404706..ff42b74a 100644
--- a/bob/devtools/scripts/build.py
+++ b/bob/devtools/scripts/build.py
@@ -27,6 +27,7 @@ from ..constants import MATPLOTLIB_RCDIR
from ..constants import SERVER
from ..log import get_logger
from ..log import verbosity_option
+from ..log import root_logger_protection
from . import bdt
remove_conda_loggers()
@@ -70,7 +71,9 @@ Examples:
help="Version of python to build the environment for",
)
@click.option(
- "-r", "--condarc", help="Use custom conda configuration file instead of our own",
+ "-r",
+ "--condarc",
+ help="Use custom conda configuration file instead of our own",
)
@click.option(
"-m",
@@ -81,7 +84,10 @@ Examples:
help="overwrites the path leading to " "variant configuration file to use",
)
@click.option(
- "-n", "--no-test", is_flag=True, help="Do not test the package, only builds it",
+ "-n",
+ "--no-test",
+ is_flag=True,
+ help="Do not test the package, only builds it",
)
@click.option(
"-a",
@@ -191,7 +197,11 @@ def build(
condarc_options = yaml.load(BASE_CONDARC, Loader=yaml.FullLoader)
channels, upload_channel = get_channels(
- public=(not private), stable=stable, server=server, intranet=ci, group=group,
+ public=(not private),
+ stable=stable,
+ server=server,
+ intranet=ci,
+ group=group,
)
if "channels" not in condarc_options:
@@ -208,7 +218,9 @@ def build(
prefix = get_env_directory(os.environ["CONDA_EXE"], "base")
condarc_options["croot"] = os.path.join(prefix, "conda-bld")
- conda_config = make_conda_config(config, python, append_file, condarc_options)
+ conda_config = make_conda_config(
+ config, python, append_file, condarc_options
+ )
set_environment("MATPLOTLIBRC", MATPLOTLIB_RCDIR)
@@ -216,7 +228,11 @@ def build(
# and derived documentation building via Sphinx)
set_environment("DOCSERVER", server)
doc_urls = get_docserver_setup(
- public=(not private), stable=stable, server=server, intranet=ci, group=group,
+ public=(not private),
+ stable=stable,
+ server=server,
+ intranet=ci,
+ group=group,
)
set_environment("BOB_DOCUMENTATION_SERVER", doc_urls)
@@ -224,7 +240,6 @@ def build(
set_environment("NOSE_EVAL_ATTR", test_mark_expr)
set_environment("PYTEST_ADDOPTS", f"-m '{test_mark_expr}'")
-
arch = conda_arch()
for d in recipe_dir:
@@ -238,21 +253,28 @@ def build(
set_environment("BOB_PACKAGE_VERSION", version)
# pre-renders the recipe - figures out the destination
- metadata = get_rendered_metadata(d, conda_config)
+ with root_logger_protection():
+ metadata = get_rendered_metadata(d, conda_config)
# checks if we should actually build this recipe
if should_skip_build(metadata):
- logger.info("Skipping UNSUPPORTED build of %s for %s", recipe_dir, arch)
+ logger.info(
+ "Skipping UNSUPPORTED build of %s for %s", recipe_dir, arch
+ )
continue
- rendered_recipe = get_parsed_recipe(metadata)
+ with root_logger_protection():
+ rendered_recipe = get_parsed_recipe(metadata)
+
logger.debug("Printing rendered recipe")
logger.debug("\n" + yaml.dump(rendered_recipe))
logger.debug("Finished printing rendered recipe")
path = get_output_path(metadata, conda_config)[0]
# gets the next build number
- build_number, _ = next_build_number(upload_channel, os.path.basename(path))
+ build_number, _ = next_build_number(
+ upload_channel, os.path.basename(path)
+ )
logger.info(
"Building %s-%s-py%s (build: %d) for %s",
@@ -267,7 +289,10 @@ def build(
# set $BOB_BUILD_NUMBER and force conda_build to reparse recipe to
# get it right
set_environment("BOB_BUILD_NUMBER", str(build_number))
- paths = conda_build.api.build(d, config=conda_config, notest=no_test)
+ with root_logger_protection():
+ paths = conda_build.api.build(
+ d, config=conda_config, notest=no_test
+ )
# if you get to this point, the package was successfully rebuilt
# set environment to signal caller we may dispose of it
os.environ["BDT_BUILD"] = ":".join(paths)
diff --git a/bob/devtools/scripts/create.py b/bob/devtools/scripts/create.py
index d97b7fb4..15a38af0 100644
--- a/bob/devtools/scripts/create.py
+++ b/bob/devtools/scripts/create.py
@@ -20,6 +20,7 @@ from ..constants import SERVER
from ..log import echo_normal
from ..log import get_logger
from ..log import verbosity_option
+from ..log import root_logger_protection
from . import bdt
logger = get_logger(__name__)
@@ -95,7 +96,8 @@ Examples:
"--python",
default=("%d.%d" % sys.version_info[:2]),
show_default=True,
- help="Version of python to build the " "environment for [default: %(default)s]",
+ help="Version of python to build the "
+ "environment for [default: %(default)s]",
)
@click.option(
"-o",
@@ -106,7 +108,9 @@ Examples:
show_default=True,
)
@click.option(
- "-r", "--condarc", help="Use custom conda configuration file instead of our own",
+ "-r",
+ "--condarc",
+ help="Use custom conda configuration file instead of our own",
)
@click.option(
"-l",
@@ -262,11 +266,19 @@ def create(
"\n - ".join(condarc_options["channels"]),
)
- conda_config = make_conda_config(config, python, append_file, condarc_options)
- deps = parse_dependencies(recipe_dir, conda_config)
- # when creating a local development environment, remove the always_yes option
+ conda_config = make_conda_config(
+ config, python, append_file, condarc_options
+ )
+ with root_logger_protection():
+ deps = parse_dependencies(recipe_dir, conda_config)
+ # when creating a local development environment, remove the always_yes
+ # option
+
del condarc_options["always_yes"]
- conda_create(conda, name, overwrite, condarc_options, deps, dry_run, use_local)
+ with root_logger_protection():
+ conda_create(
+ conda, name, overwrite, condarc_options, deps, dry_run, use_local
+ )
# part 2: pip-install everything listed in pip-extras
# mix-in stuff from ~/.bdtrc and command-line
@@ -284,4 +296,4 @@ def create(
else:
logger.info(f"Command: {' '.join(cmd)}")
- echo_normal(f">>> Execute on your shell: \"conda activate {name}\"")
+ echo_normal(f'>>> Execute on your shell: "conda activate {name}"')
diff --git a/bob/devtools/scripts/rebuild.py b/bob/devtools/scripts/rebuild.py
index 1297a227..9da0d337 100644
--- a/bob/devtools/scripts/rebuild.py
+++ b/bob/devtools/scripts/rebuild.py
@@ -27,6 +27,7 @@ from ..constants import MATPLOTLIB_RCDIR
from ..constants import SERVER
from ..log import get_logger
from ..log import verbosity_option
+from ..log import root_logger_protection
from . import bdt
logger = get_logger(__name__)
@@ -229,14 +230,17 @@ def rebuild(
set_environment("BOB_PACKAGE_VERSION", version)
# pre-renders the recipe - figures out the destination
- metadata = get_rendered_metadata(d, conda_config)
+ with root_logger_protection():
+ metadata = get_rendered_metadata(d, conda_config)
# checks if we should actually build this recipe
if should_skip_build(metadata):
logger.info("Skipping UNSUPPORTED build of %s for %s", recipe_dir, arch)
continue
- rendered_recipe = get_parsed_recipe(metadata)
+ with root_logger_protection():
+ rendered_recipe = get_parsed_recipe(metadata)
+
path = get_output_path(metadata, conda_config)[0]
# Get the latest build number
@@ -261,7 +265,8 @@ def rebuild(
# rebuild the package or not
logger.info("Testing %s", src)
try:
- result = conda_build.api.test(destpath, config=conda_config)
+ with root_logger_protection():
+ result = conda_build.api.test(destpath, config=conda_config)
should_build = not result
except Exception as error:
logger.exception(error)
diff --git a/bob/devtools/scripts/test.py b/bob/devtools/scripts/test.py
index 6721d40e..6c5c5256 100644
--- a/bob/devtools/scripts/test.py
+++ b/bob/devtools/scripts/test.py
@@ -20,6 +20,7 @@ from ..constants import MATPLOTLIB_RCDIR
from ..constants import SERVER
from ..log import get_logger
from ..log import verbosity_option
+from ..log import root_logger_protection
from . import bdt
remove_conda_loggers()
@@ -202,4 +203,5 @@ def test(
for p in package:
logger.info("Testing %s at %s", p, arch)
if not dry_run:
- conda_build.api.test(p, config=conda_config)
+ with root_logger_protection():
+ conda_build.api.test(p, config=conda_config)
--
GitLab