Remove bob-devel and implement similar tests here

bob/devtools/build.py

@@ -6,6 +6,7 @@
 
 import contextlib
 import copy
+import datetime
 import distutils.version
 import glob
 import json
@@ -15,6 +16,7 @@ import platform
 import re
 import subprocess
 import sys
+import tempfile
 
 import conda_build.api
 import yaml
@@ -703,6 +705,138 @@ def base_build(
         return conda_build.api.build(recipe_dir, config=conda_config)
 
 
+def global_pin(
+    bootstrap, server, intranet, group, conda_build_config, condarc_options
+):
+    """
+    Tests that all packages listed in bob/devtools/data/conda_build_config.yaml
+    can be installed in one environment.
+    """
+    # Load the packages and their pins in bob/devtools/data/conda_build_config.yaml
+    # Create a temporary conda-build recipe to test if all packages can be installed
+
+    with open(conda_build_config, "r") as f:
+        content = f.read()
+
+    idx1 = content.find("# AUTOMATIC PARSING START")
+    idx2 = content.find("# AUTOMATIC PARSING END")
+    content = content[idx1:idx2]
+    package_pins = yaml.safe_load(content)
+
+    package_names_map = package_pins.pop("package_names_map")
+
+    packages = [package_names_map.get(p, p) for p in package_pins.keys()]
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        # Create a conda-build recipe
+        recipe_path = tmpdir + "/meta.yaml"
+        with open(recipe_path, "w") as f:
+            content = """
+package:
+  name: global-pins
+  version: {date}
+
+build:
+  number: 0
+
+requirements:
+  host:
+    - python {{{{ python }}}}
+    - {{{{ compiler('c') }}}}
+    - {{{{ compiler('cxx') }}}}
+{package_list}
+
+  run:
+    - python
+  {{% for package in resolved_packages('host') %}}
+    - {{{{ package }}}}
+  {{% endfor %}}
+
+test:
+  requires:
+    - numpy
+    - ffmpeg
+    - pytorch
+    - torchvision
+    - setuptools
+  commands:
+    # we expect these features from ffmpeg:
+    - ffmpeg -codecs | grep "DEVI.S zlib"  # [unix]
+    - ffmpeg -codecs | grep "DEV.LS h264"  # [unix]
+""".format(
+                date=datetime.date.today().strftime("%Y.%m.%d"),
+                package_list="\n".join(
+                    [
+                        "    - {p1} {{{{ {p2} }}}}".format(
+                            p1=p, p2=p.replace("-", "_").replace(".", "_")
+                        )
+                        for p in packages
+                    ]
+                ),
+            )
+            logger.info(
+                "Writing a conda build recipe with the following content:\n%s",
+                content,
+            )
+            f.write(content)
+
+        # write run_test.py file
+        run_test_path = tmpdir + "/run_test.py"
+        with open(run_test_path, "w") as f:
+            content = """
+import sys
+
+# couple of imports to see if packages are working
+import numpy
+import pkg_resources
+
+
+def test_pytorch():
+    import torch
+    from torchvision.models import DenseNet
+
+    model = DenseNet()
+    t = torch.randn(1, 3, 224, 224)
+    out = model(t)
+    assert out.shape[1] == 1000
+
+
+def _check_package(name, pyname=None):
+    "Checks if a Python package can be `require()`'d"
+
+    pyname = pyname or name
+    print(f"Checking Python setuptools integrity for {name} (pyname: {pyname})")
+    pkg_resources.require(pyname)
+
+
+def test_setuptools_integrity():
+
+    _check_package('pytorch', 'torch')
+    _check_package('torchvision')
+
+
+# test if pytorch installation is sane
+test_pytorch()
+test_setuptools_integrity()
+"""
+            logger.info(
+                "Writing a run_test.py file with the following content:\n%s",
+                content,
+            )
+            f.write(content)
+
+        # run conda build
+        base_build(
+            bootstrap=bootstrap,
+            server=server,
+            intranet=intranet,
+            group=group,
+            recipe_dir=tmpdir,
+            conda_build_config=conda_build_config,
+            condarc_options=condarc_options,
+        )
+
+
 if __name__ == "__main__":
 
     import argparse
@@ -824,6 +958,18 @@ if __name__ == "__main__":
     if condarc_options.get("conda-build", {}).get("root-dir") is None:
         condarc_options["croot"] = os.path.join(prefix, "conda-bld")
 
+    # Check global pin versions compatibility
+    global_pin(
+        bootstrap=bootstrap,
+        server=server,
+        intranet=not args.internet,
+        group=args.group,
+        conda_build_config=os.path.join(
+            args.work_dir, "bob", "devtools", "data", "conda_build_config.yaml"
+        ),
+        condarc_options=condarc_options,
+    )
+
     # builds all dependencies in the 'deps' subdirectory - or at least checks
     # these dependencies are already available; these dependencies go directly
     # to the public channel once built

bob/devtools/data/conda_build_config.yaml

@@ -69,44 +69,43 @@ zip_keys:
 
 # Here is the version of dependencies are used when building packages (build
 # and host requirements). We keep a list of **all of them** here to make sure
-# everything goes as expected in our conda build process. For the version of
-# packages that are used for testing packages, see the recipe of bob-devel.
-# The version here do not necessarily match the versions in bob-devel.
-
-# This version of bob-devel will be used at test time of packages:
-bob_devel:
-  - 2021.06.17
-
-# This version of beat-devel will be used at test time of packages. Notice it
-# uses bob-devel and should have a version that is greater or equal its value
-beat_devel:
-  - 2021.06.17
-
-# The build time only dependencies (build requirements).
-# Updating these to the latest version all the time is OK and a good idea.
-# These versions should match the versions inside bob-devel as well (if they
-# overlap) so update them in both places.
-cmake:
-  - 3.14.0
-make:
-  - 4.2.1
-pkg_config:
-  - 0.29.2
-
-# The host requirements. Ideally we want to build against the oldest possible version of
-# packages so packages can be installed with a wide range of versions. But the versions
-# here should also be compatible with the pinned versions in bob-devel. For most
-# dependencies, you want to put the exact version of bob-devel in here as well. It is
+# everything goes as expected in our conda build process.
+# Ideally we want to build against the oldest possible version of
+# packages so packages can be installed with a wide range of versions. It is
 # best to keep this in sync with:
 # https://github.com/AnacondaRecipes/aggregate/blob/master/conda_build_config.yaml The
 # names here should not contain dots or dashes. You should replace dots and dashes with
 # underlines.
+
+# AUTOMATIC PARSING START
+# DO NOT MODIFY THIS COMMENT
+
+# list all packages with dashes or dots in their names, here:
+package_names_map:
+  click_plugins: click-plugins
+  dask_jobqueue: dask-jobqueue
+  dask_ml: dask-ml
+  docker_py: docker-py
+  pkg_config: pkg-config
+  pytest_cov: pytest-cov
+  python_graphviz: python-graphviz
+  scikit_image: scikit-image
+  scikit_learn: scikit-learn
+  sphinxcontrib_httpdomain: sphinxcontrib-httpdomain
+  sphinxcontrib_mermaid: sphinxcontrib-mermaid
+  sphinxcontrib_programoutput: sphinxcontrib-programoutput
+  zc_buildout: zc.buildout
+  zc_recipe_egg: zc.recipe.egg
+
+
 boost:
   - 1.73.0
 click:
   - 8.0.1
 click_plugins:
   - 1.1.1
+cmake:
+  - 3.14.0
 coverage:
   - 5.5
 dask:
@@ -147,6 +146,8 @@ libpng:
   - 1.6.37
 libtiff:
   - 4.2.0
+make:
+  - 4.2.1
 matplotlib:
   - 3.3.4
 mkl:
@@ -157,7 +158,8 @@ nose:
   - 1.3.7
 numba:
   - 0.53.1
-numpy: # we build against numpy 1.17 but test against newer versions.
+# we build against numpy 1.17 but test against newer versions.
+numpy:
   - 1.17
 opencv:
   - 4.5.0
@@ -165,6 +167,8 @@ pandas:
   - 1.2.4
 pillow:
   - 8.2.0
+pkg_config:
+  - 0.29.2
 psutil:
   - 5.8.0
 psycopg2:
@@ -217,10 +221,10 @@ sphinx_rtd_theme:
   - 0.4.3
 sphinxcontrib_httpdomain:
   - 1.7.0
-sphinxcontrib_programoutput:
-  - 0.16
 sphinxcontrib_mermaid:
   - 0.6.1
+sphinxcontrib_programoutput:
+  - 0.16
 sqlalchemy:
   - 1.4.15
 tabulate:
@@ -241,3 +245,5 @@ zc_buildout:
   - 2.13.3
 zc_recipe_egg:
   - 2.0.7
+
+# AUTOMATIC PARSING END