From 4f636eb2dbb7a65f4614cd3c14301687d305fdde Mon Sep 17 00:00:00 2001
From: Andre Anjos <andre.dos.anjos@gmail.com>
Date: Tue, 26 May 2020 17:39:41 +0200
Subject: [PATCH] [build] Fix base_build() with a proper support for
multi-package-building through conda-build
---
bob/devtools/build.py | 63 ++++++++++-----------------------
bob/devtools/graph.py | 2 +-
bob/devtools/scripts/build.py | 2 +-
bob/devtools/scripts/local.py | 12 +------
bob/devtools/scripts/rebuild.py | 2 +-
5 files changed, 22 insertions(+), 59 deletions(-)
diff --git a/bob/devtools/build.py b/bob/devtools/build.py
index 93ca3562..586027a4 100644
--- a/bob/devtools/build.py
+++ b/bob/devtools/build.py
@@ -215,7 +215,7 @@ def make_conda_config(config, python, append_file, condarc_options):
def get_output_path(metadata, config):
"""Renders the recipe and returns the name of the output file."""
- return conda_build.api.get_output_file_paths(metadata, config=config)[0]
+ return conda_build.api.get_output_file_paths(metadata, config=config)
def get_rendered_metadata(recipe_dir, config):
@@ -565,7 +565,6 @@ def base_build(
group,
recipe_dir,
conda_build_config,
- python_version,
condarc_options,
):
"""Builds a non-beat/non-bob software dependence that doesn't exist on
@@ -590,11 +589,6 @@ def base_build(
our internal webserver. Currently, only "bob" or "beat" will work.
recipe_dir: The directory containing the recipe's ``meta.yaml`` file
conda_build_config: Path to the ``conda_build_config.yaml`` file to use
- python_version: String with the python version to build for, in the format
- ``x.y`` (should be passed even if not building a python package). It
- can also be set to ``noarch``, or ``None``. If set to ``None``, then we
- don't assume there is a python-specific version being built. If set to
- ``noarch``, then it is a python package without a specific build.
condarc_options: Pre-parsed condarc options loaded from the respective YAML
file
@@ -619,54 +613,33 @@ def base_build(
"\n - ".join(condarc_options["channels"]),
)
logger.info("Merging conda configuration files...")
- if python_version not in ("noarch", None):
- conda_config = make_conda_config(
- conda_build_config, python_version, None, condarc_options
- )
- else:
- conda_config = make_conda_config(
- conda_build_config, None, None, condarc_options
- )
+ conda_config = make_conda_config(
+ conda_build_config, None, None, condarc_options
+ )
metadata = get_rendered_metadata(recipe_dir, conda_config)
-
- # handles different cases as explained on the description of
- # ``python_version``
- py_ver = python_version.replace(".", "") if python_version else None
- if py_ver == "noarch":
- py_ver = ""
arch = conda_arch()
# checks we should actually build this recipe
if should_skip_build(metadata):
- if py_ver is None:
- logger.warn(
- 'Skipping UNSUPPORTED build of "%s" on %s', recipe_dir, arch
- )
- elif not py_ver:
- logger.warn(
- 'Skipping UNSUPPORTED build of "%s" for (noarch) python '
- "on %s",
- recipe_dir,
- arch,
- )
- else:
- logger.warn(
- 'Skipping UNSUPPORTED build of "%s" for python-%s ' "on %s",
- recipe_dir,
- python_version,
- arch,
- )
+ logger.warn('Skipping UNSUPPORTED build of "%s" on %s', recipe_dir, arch)
return
- path = get_output_path(metadata, conda_config)
+ paths = get_output_path(metadata, conda_config)
+ urls = [exists_on_channel(channels[0], os.path.basename(k)) for k in paths]
+
+ if all(urls):
+ logger.info("Skipping build for %s as packages with matching "
+ "characteristics exist (%s)", path, ', '.join(urls))
+ return
- url = exists_on_channel(channels[0], os.path.basename(path))
- if url is not None:
- logger.info("Skipping build for %s as it exists (at %s)", path, url)
+ if any(urls):
+ logger.error("One or more packages for %s already exist (%s). "
+ "Change the package build number to trigger a build.",
+ path, ', '.join(urls))
return
- # if you get to this point, just builds the package
+ # if you get to this point, just builds the package(s)
logger.info("Building %s", path)
return conda_build.api.build(recipe_dir, config=conda_config)
@@ -839,7 +812,7 @@ if __name__ == "__main__":
recipe_dir = os.path.join(args.work_dir, "conda")
metadata = get_rendered_metadata(recipe_dir, conda_config)
- path = get_output_path(metadata, conda_config)
+ path = get_output_path(metadata, conda_config)[0]
# asserts we're building at the right location
assert path.startswith(os.path.join(args.conda_root, "conda-bld")), (
diff --git a/bob/devtools/graph.py b/bob/devtools/graph.py
index c7fe0091..a6b7296e 100644
--- a/bob/devtools/graph.py
+++ b/bob/devtools/graph.py
@@ -124,7 +124,7 @@ def compute_adjencence_matrix(
# pre-renders the recipe - figures out the destination
metadata = get_rendered_metadata(recipe_dir, conda_config)
rendered_recipe = get_parsed_recipe(metadata)
- path = get_output_path(metadata, conda_config)
+ path = get_output_path(metadata, conda_config)[0]
# gets the next build number
build_number, _ = next_build_number(
diff --git a/bob/devtools/scripts/build.py b/bob/devtools/scripts/build.py
index 9bc98d45..4601c46b 100644
--- a/bob/devtools/scripts/build.py
+++ b/bob/devtools/scripts/build.py
@@ -266,7 +266,7 @@ def build(
continue
rendered_recipe = get_parsed_recipe(metadata)
- path = get_output_path(metadata, conda_config)
+ path = get_output_path(metadata, conda_config)[0]
# gets the next build number
build_number, _ = next_build_number(channels[0], os.path.basename(path))
diff --git a/bob/devtools/scripts/local.py b/bob/devtools/scripts/local.py
index d7bacd6a..a008757e 100644
--- a/bob/devtools/scripts/local.py
+++ b/bob/devtools/scripts/local.py
@@ -176,13 +176,6 @@ Examples:
"(combine with the verbosity flags - e.g. ``-vvv``) to enable "
"printing to help you understand what will be done",
)
-@click.option(
- "-p",
- "--python",
- multiple=True,
- help='Versions of python in the format "x.y" we should build for. Pass '
- "various times this option to build for multiple python versions",
-)
@click.option(
"-g",
"--group",
@@ -196,7 +189,4 @@ Examples:
def base_build(ctx, order, dry_run, python, group):
"""Run the CI build step locally."""
set_up_environment_variables(python=python, name_space=group)
-
- ctx.invoke(
- ci.base_build, order=order, dry_run=dry_run, group=group, python=python
- )
+ ctx.invoke(ci.base_build, order=order, dry_run=dry_run, group=group)
diff --git a/bob/devtools/scripts/rebuild.py b/bob/devtools/scripts/rebuild.py
index c7bb655c..8b5a3371 100644
--- a/bob/devtools/scripts/rebuild.py
+++ b/bob/devtools/scripts/rebuild.py
@@ -253,7 +253,7 @@ def rebuild(
continue
rendered_recipe = get_parsed_recipe(metadata)
- path = get_output_path(metadata, conda_config)
+ path = get_output_path(metadata, conda_config)[0]
# Get the latest build number
build_number, existing = next_build_number(
--
GitLab