diff --git a/bob/devtools/build.py b/bob/devtools/build.py
index afd014d41c7b51f66adb643cd583d81f90856ec0..51331dbc68395b27c3dabfbadc89816420499df0 100644
--- a/bob/devtools/build.py
+++ b/bob/devtools/build.py
@@ -6,6 +6,7 @@
import contextlib
import copy
+import datetime
import distutils.version
import glob
import json
@@ -15,6 +16,7 @@ import platform
import re
import subprocess
import sys
+import tempfile
import conda_build.api
import yaml
@@ -703,6 +705,138 @@ def base_build(
return conda_build.api.build(recipe_dir, config=conda_config)
+def global_pin(
+ bootstrap, server, intranet, group, conda_build_config, condarc_options
+):
+ """
+ Tests that all packages listed in bob/devtools/data/conda_build_config.yaml
+ can be installed in one environment.
+ """
+ # Load the packages and their pins in bob/devtools/data/conda_build_config.yaml
+ # Create a temporary conda-build recipe to test if all packages can be installed
+
+ with open(conda_build_config, "r") as f:
+ content = f.read()
+
+ idx1 = content.find("# AUTOMATIC PARSING START")
+ idx2 = content.find("# AUTOMATIC PARSING END")
+ content = content[idx1:idx2]
+ package_pins = yaml.safe_load(content)
+
+ package_names_map = package_pins.pop("package_names_map")
+
+ packages = [package_names_map.get(p, p) for p in package_pins.keys()]
+
+ with tempfile.TemporaryDirectory() as tmpdir:
+ # Create a conda-build recipe
+ recipe_path = tmpdir + "/meta.yaml"
+ with open(recipe_path, "w") as f:
+ content = """
+package:
+ name: global-pins
+ version: {date}
+
+build:
+ number: 0
+
+requirements:
+ host:
+ - python {{{{ python }}}}
+ - {{{{ compiler('c') }}}}
+ - {{{{ compiler('cxx') }}}}
+{package_list}
+
+ run:
+ - python
+ {{% for package in resolved_packages('host') %}}
+ - {{{{ package }}}}
+ {{% endfor %}}
+
+test:
+ requires:
+ - numpy
+ - ffmpeg
+ - pytorch
+ - torchvision
+ - setuptools
+ commands:
+ # we expect these features from ffmpeg:
+ - ffmpeg -codecs | grep "DEVI.S zlib" # [unix]
+ - ffmpeg -codecs | grep "DEV.LS h264" # [unix]
+""".format(
+ date=datetime.date.today().strftime("%Y.%m.%d"),
+ package_list="\n".join(
+ [
+ " - {p1} {{{{ {p2} }}}}".format(
+ p1=p, p2=p.replace("-", "_").replace(".", "_")
+ )
+ for p in packages
+ ]
+ ),
+ )
+ logger.info(
+ "Writing a conda build recipe with the following content:\n%s",
+ content,
+ )
+ f.write(content)
+
+ # write run_test.py file
+ run_test_path = tmpdir + "/run_test.py"
+ with open(run_test_path, "w") as f:
+ content = """
+import sys
+
+# couple of imports to see if packages are working
+import numpy
+import pkg_resources
+
+
+def test_pytorch():
+ import torch
+ from torchvision.models import DenseNet
+
+ model = DenseNet()
+ t = torch.randn(1, 3, 224, 224)
+ out = model(t)
+ assert out.shape[1] == 1000
+
+
+def _check_package(name, pyname=None):
+ "Checks if a Python package can be `require()`'d"
+
+ pyname = pyname or name
+ print(f"Checking Python setuptools integrity for {name} (pyname: {pyname})")
+ pkg_resources.require(pyname)
+
+
+def test_setuptools_integrity():
+
+ _check_package('pytorch', 'torch')
+ _check_package('torchvision')
+
+
+# test if pytorch installation is sane
+test_pytorch()
+test_setuptools_integrity()
+"""
+ logger.info(
+ "Writing a run_test.py file with the following content:\n%s",
+ content,
+ )
+ f.write(content)
+
+ # run conda build
+ base_build(
+ bootstrap=bootstrap,
+ server=server,
+ intranet=intranet,
+ group=group,
+ recipe_dir=tmpdir,
+ conda_build_config=conda_build_config,
+ condarc_options=condarc_options,
+ )
+
+
if __name__ == "__main__":
import argparse
@@ -824,6 +958,18 @@ if __name__ == "__main__":
if condarc_options.get("conda-build", {}).get("root-dir") is None:
condarc_options["croot"] = os.path.join(prefix, "conda-bld")
+ # Check global pin versions compatibility
+ global_pin(
+ bootstrap=bootstrap,
+ server=server,
+ intranet=not args.internet,
+ group=args.group,
+ conda_build_config=os.path.join(
+ args.work_dir, "bob", "devtools", "data", "conda_build_config.yaml"
+ ),
+ condarc_options=condarc_options,
+ )
+
# builds all dependencies in the 'deps' subdirectory - or at least checks
# these dependencies are already available; these dependencies go directly
# to the public channel once built
diff --git a/bob/devtools/data/conda_build_config.yaml b/bob/devtools/data/conda_build_config.yaml
index 7e8a33e2fd4481ff74114f8656334284aa3b00f5..3872fbaa9c1eb42f030fa1fbcf62cfd461207957 100644
--- a/bob/devtools/data/conda_build_config.yaml
+++ b/bob/devtools/data/conda_build_config.yaml
@@ -69,44 +69,43 @@ zip_keys:
# Here is the version of dependencies are used when building packages (build
# and host requirements). We keep a list of **all of them** here to make sure
-# everything goes as expected in our conda build process. For the version of
-# packages that are used for testing packages, see the recipe of bob-devel.
-# The version here do not necessarily match the versions in bob-devel.
-
-# This version of bob-devel will be used at test time of packages:
-bob_devel:
- - 2021.06.17
-
-# This version of beat-devel will be used at test time of packages. Notice it
-# uses bob-devel and should have a version that is greater or equal its value
-beat_devel:
- - 2021.06.17
-
-# The build time only dependencies (build requirements).
-# Updating these to the latest version all the time is OK and a good idea.
-# These versions should match the versions inside bob-devel as well (if they
-# overlap) so update them in both places.
-cmake:
- - 3.14.0
-make:
- - 4.2.1
-pkg_config:
- - 0.29.2
-
-# The host requirements. Ideally we want to build against the oldest possible version of
-# packages so packages can be installed with a wide range of versions. But the versions
-# here should also be compatible with the pinned versions in bob-devel. For most
-# dependencies, you want to put the exact version of bob-devel in here as well. It is
+# everything goes as expected in our conda build process.
+# Ideally we want to build against the oldest possible version of
+# packages so packages can be installed with a wide range of versions. It is
# best to keep this in sync with:
# https://github.com/AnacondaRecipes/aggregate/blob/master/conda_build_config.yaml The
# names here should not contain dots or dashes. You should replace dots and dashes with
# underlines.
+
+# AUTOMATIC PARSING START
+# DO NOT MODIFY THIS COMMENT
+
+# list all packages with dashes or dots in their names, here:
+package_names_map:
+ click_plugins: click-plugins
+ dask_jobqueue: dask-jobqueue
+ dask_ml: dask-ml
+ docker_py: docker-py
+ pkg_config: pkg-config
+ pytest_cov: pytest-cov
+ python_graphviz: python-graphviz
+ scikit_image: scikit-image
+ scikit_learn: scikit-learn
+ sphinxcontrib_httpdomain: sphinxcontrib-httpdomain
+ sphinxcontrib_mermaid: sphinxcontrib-mermaid
+ sphinxcontrib_programoutput: sphinxcontrib-programoutput
+ zc_buildout: zc.buildout
+ zc_recipe_egg: zc.recipe.egg
+
+
boost:
- 1.73.0
click:
- 8.0.1
click_plugins:
- 1.1.1
+cmake:
+ - 3.14.0
coverage:
- 5.5
dask:
@@ -147,6 +146,8 @@ libpng:
- 1.6.37
libtiff:
- 4.2.0
+make:
+ - 4.2.1
matplotlib:
- 3.3.4
mkl:
@@ -157,7 +158,8 @@ nose:
- 1.3.7
numba:
- 0.53.1
-numpy: # we build against numpy 1.17 but test against newer versions.
+# we build against numpy 1.17 but test against newer versions.
+numpy:
- 1.17
opencv:
- 4.5.0
@@ -165,6 +167,8 @@ pandas:
- 1.2.4
pillow:
- 8.2.0
+pkg_config:
+ - 0.29.2
psutil:
- 5.8.0
psycopg2:
@@ -217,10 +221,10 @@ sphinx_rtd_theme:
- 0.4.3
sphinxcontrib_httpdomain:
- 1.7.0
-sphinxcontrib_programoutput:
- - 0.16
sphinxcontrib_mermaid:
- 0.6.1
+sphinxcontrib_programoutput:
+ - 0.16
sqlalchemy:
- 1.4.15
tabulate:
@@ -241,3 +245,5 @@ zc_buildout:
- 2.13.3
zc_recipe_egg:
- 2.0.7
+
+# AUTOMATIC PARSING END