From c0ea1876bcaba99ddc728b5c7ad63cf11e1e1106 Mon Sep 17 00:00:00 2001
From: Andre Anjos <andre.dos.anjos@gmail.com>
Date: Wed, 4 Aug 2021 16:35:22 +0200
Subject: [PATCH] [build] Simplify recipe parsing and environment creation
significantly and fixes multiple environment creation issues; Closes #70
---
bob/devtools/build.py | 68 +++++++++++++++++++++++-----------
bob/devtools/ci.py | 23 ------------
bob/devtools/scripts/ci.py | 2 +-
bob/devtools/scripts/create.py | 11 +-----
4 files changed, 50 insertions(+), 54 deletions(-)
diff --git a/bob/devtools/build.py b/bob/devtools/build.py
index 1ccd8774..52fd2d79 100644
--- a/bob/devtools/build.py
+++ b/bob/devtools/build.py
@@ -227,8 +227,7 @@ def get_parsed_recipe(metadata):
"""Renders the recipe and returns the interpreted YAML file."""
with root_logger_protection():
- output = conda_build.api.output_yaml(metadata[0][0])
- return yaml.load(output, Loader=yaml.FullLoader)
+ return metadata[0][0].get_rendered_recipe_text()
def exists_on_channel(channel_url, basename):
@@ -289,29 +288,56 @@ def remove_pins(deps):
return [ll.split()[0] for ll in deps]
+def uniq(seq, idfun=None):
+ """Very fast, order preserving uniq function."""
+
+ # order preserving
+ if idfun is None:
+
+ def idfun(x):
+ return x
+
+ seen = {}
+ result = []
+ for item in seq:
+ marker = idfun(item)
+ # in old Python versions:
+ # if seen.has_key(marker)
+ # but in new ones:
+ if marker in seen:
+ continue
+ seen[marker] = 1
+ result.append(item)
+ return result
+
+
def parse_dependencies(recipe_dir, config):
metadata = get_rendered_metadata(recipe_dir, config)
recipe = get_parsed_recipe(metadata)
- build_requirements = remove_pins(recipe["requirements"].get("build", []))
- # causes conflicts on macOS
- if "llvm-tools" in build_requirements:
- build_requirements.remove("llvm-tools")
- if "libgfortran4" in build_requirements:
- build_requirements.remove("libgfortran4")
- return (
- build_requirements
- + remove_pins(recipe["requirements"].get("host", []))
- + recipe["requirements"].get("run", [])
- + recipe.get("test", {}).get("requires", [])
- + ["pip"] # required for installing further packages
- + ["bob.buildout"] # required for basic bootstrap of most recipes
- + ["ipython"] # for ipdb
- # Also add anaconda compilers to make sure source installed packages are
- # compiled properly
- + ["clangxx_osx-64" if platform.system() == "Darwin" else "gxx_linux-64"]
- )
- # by last, packages required for local dev
+ requirements = []
+ for section in ("build", "host"):
+ requirements += remove_pins(recipe.get("requirements", {}).get(section, []))
+ # we don't remove pins for the rest of the recipe
+ requirements += recipe.get("requirements", {}).get("run", [])
+ requirements += recipe.get("test", {}).get("requires", [])
+
+ # also add anaconda compilers to make sure source installed packages are
+ # compiled properly
+ if platform.system() == "Darwin":
+ requirements += ["clangxx_osx-64"]
+ else:
+ requirements += ["gxx_linux-64"]
+
+ # further requirements
+ requirements += [
+ "pip", # required for installing further packages
+ "bob.buildout", # required for basic bootstrap of most recipes
+ "ipython", # for ipdb
+ ]
+
+ # remove duplicates without affecting the order
+ return uniq(requirements)
def get_env_directory(conda, name):
diff --git a/bob/devtools/ci.py b/bob/devtools/ci.py
index 186d7675..0206740f 100644
--- a/bob/devtools/ci.py
+++ b/bob/devtools/ci.py
@@ -135,29 +135,6 @@ def read_packages(filename):
return packages
-def uniq(seq, idfun=None):
- """Very fast, order preserving uniq function."""
-
- # order preserving
- if idfun is None:
-
- def idfun(x):
- return x
-
- seen = {}
- result = []
- for item in seq:
- marker = idfun(item)
- # in old Python versions:
- # if seen.has_key(marker)
- # but in new ones:
- if marker in seen:
- continue
- seen[marker] = 1
- result.append(item)
- return result
-
-
def select_build_file(basename, paths, branch):
"""Selects the file to use for a build.
diff --git a/bob/devtools/scripts/ci.py b/bob/devtools/scripts/ci.py
index 9a768696..57e45b96 100644
--- a/bob/devtools/scripts/ci.py
+++ b/bob/devtools/scripts/ci.py
@@ -13,13 +13,13 @@ from click_plugins import with_plugins
from ..build import comment_cleanup
from ..build import load_order_file
+from ..build import uniq
from ..ci import cleanup
from ..ci import is_private
from ..ci import read_packages
from ..ci import select_conda_build_config
from ..ci import select_conda_recipe_append
from ..ci import select_user_condarc
-from ..ci import uniq
from ..constants import BASE_CONDARC
from ..constants import SERVER
from ..deploy import deploy_conda_package
diff --git a/bob/devtools/scripts/create.py b/bob/devtools/scripts/create.py
index 426c1bf0..9d0c4450 100644
--- a/bob/devtools/scripts/create.py
+++ b/bob/devtools/scripts/create.py
@@ -12,6 +12,7 @@ from ..bootstrap import set_environment
from ..build import conda_create
from ..build import make_conda_config
from ..build import parse_dependencies
+from ..build import uniq
from ..config import read_config
from ..constants import BASE_CONDARC
from ..constants import CONDA_BUILD_CONFIG
@@ -25,14 +26,6 @@ from . import bdt
logger = get_logger(__name__)
-def _uniq(seq):
- """Fast order preserving uniq() function for Python lists"""
-
- seen = set()
- seen_add = seen.add
- return [x for x in seq if not (x in seen or seen_add(x))]
-
-
@click.command(
epilog="""
Examples:
@@ -279,7 +272,7 @@ def create(
pip_extras_config = []
if "create" in config:
pip_extras_config = config["create"].get("pip_extras", "").split()
- pip_extras = _uniq(pip_extras_config + list(pip_extras))
+ pip_extras = uniq(pip_extras_config + list(pip_extras))
if pip_extras:
logger.info("Pip-installing: %s", pip_extras)
--
GitLab