Add a script to update pins inside bob/devtools/data/conda_build_config.yaml automatically

bob/devtools/build.py

@@ -711,22 +711,7 @@ def base_build(
         return conda_build.api.build(recipe_dir, config=conda_config)
 
 
-def bob_devel(
-    bootstrap,
-    server,
-    intranet,
-    group,
-    conda_build_config,
-    condarc_options,
-    work_dir,
-):
-    """
-    Tests that all packages listed in bob/devtools/data/conda_build_config.yaml
-    can be installed in one environment.
-    """
-    # Load the packages and their pins in bob/devtools/data/conda_build_config.yaml
-    # Create a temporary conda-build recipe to test if all packages can be installed
-
+def load_packages_from_conda_build_config(conda_build_config, condarc_options):
     with open(conda_build_config, "r") as f:
         content = f.read()
 
@@ -746,6 +731,29 @@ def bob_devel(
 
     packages = [package_names_map.get(p, p) for p in package_pins.keys()]
 
+    return packages, package_names_map
+
+
+def bob_devel(
+    bootstrap,
+    server,
+    intranet,
+    group,
+    conda_build_config,
+    condarc_options,
+    work_dir,
+):
+    """
+    Tests that all packages listed in bob/devtools/data/conda_build_config.yaml
+    can be installed in one environment.
+    """
+    # Load the packages and their pins in bob/devtools/data/conda_build_config.yaml
+    # Create a temporary conda-build recipe to test if all packages can be installed
+
+    packages, _ = load_packages_from_conda_build_config(
+        conda_build_config, condarc_options
+    )
+
     recipe_dir = os.path.join(work_dir, "deps", "bob-devel")
     template_yaml = os.path.join(recipe_dir, "meta.template.yaml")
     final_yaml = os.path.join(recipe_dir, "meta.yaml")

bob/devtools/data/conda_build_config.yaml

@@ -322,7 +322,7 @@ click_plugins:
 cmake:
   - 3.21.3
 coverage:
-  - 5.5
+  - 6.0
 dask:
   - 2021.9.1
 dask_jobqueue:
@@ -338,7 +338,7 @@ docker_py:
 docopt:
   - 0.6.2
 ffmpeg:
-  - 4.3
+  - 4.3.2
 flaky:
   - 3.7.0
 font_ttf_dejavu_sans_mono:
@@ -348,7 +348,7 @@ freetype:
 giflib:
   - 5.2.1
 graphviz:
-  - 2.49.0
+  - 2.49.1
 h5py:
   - 3.1.0
 hdf5:
@@ -358,7 +358,7 @@ jinja2:
 jpeg:
   - 9d
 jsonschema:
-  - 3.2.0
+  - 4.0.1
 libblitz:
   - 1.0.2
 libpng:
@@ -377,12 +377,10 @@ nose:
   - 1.3.7
 numba:
   - 0.54.0
-# we build against numpy 1.18 but test against newer versions.
 numpy:
-  # part of a zip_keys: python, python_impl, numpy
   - 1.19.5
 opencv:
-  - 4.5.1
+  - 4.5.2
 pandas:
   - 1.3.3
 pillow:
@@ -394,13 +392,13 @@ psutil:
 psycopg2:
   - 2.9.1
 pybind11:
-  - 2.7.1
+  - 2.8.0
 pytables:
   - 3.6.1
 pytest:
   - 6.2.5
 pytest_cov:
-  - 2.12.1
+  - 3.0.0
 python_graphviz:
   - 0.17
 pytorch:

bob/devtools/scripts/update_pins.py

+"""Updates the pin versions inside bob/devtools/data/conda_build_config.yaml"""
+
+import click
+
+
+@click.command()
+@click.option("--python", required=True, help="Python version to pin, e.g. 3.8")
+def update_pins(python):
+    from subprocess import check_output
+
+    from bob.devtools.build import load_packages_from_conda_build_config
+
+    conda_config_path = "bob/devtools/data/conda_build_config.yaml"
+
+    packages, package_names_map = load_packages_from_conda_build_config(
+        conda_config_path, {"channels": []}
+    )
+    reversed_package_names_map = {v: k for k, v in package_names_map.items()}
+
+    # ask mamba to create an environment with the packages
+    env_text = check_output(
+        [
+            "mamba",
+            "create",
+            "--dry-run",
+            "--override-channels",
+            "-c",
+            "conda-forge",
+            "-n",
+            "temp_env",
+            f"python={python}",
+        ]
+        + packages
+    ).decode("utf-8")
+    print(env_text)
+
+    resolved_packages = []
+    for line in env_text.split("\n"):
+        line = line.strip()
+        if line.startswith("+ "):
+            line = line.split()
+            name, version = line[1], line[2]
+            resolved_packages.append((name, version))
+
+    # write the new pinning
+    with open(conda_config_path, "r") as f:
+        content = f.read()
+
+    START = """
+# AUTOMATIC PARSING START
+# DO NOT MODIFY THIS COMMENT
+
+# list all packages with dashes or dots in their names, here:"""
+    idx1 = content.find(START)
+    idx2 = content.find("# AUTOMATIC PARSING END")
+    pins = "\n".join(
+        f"{reversed_package_names_map.get(name, name)}:\n  - {version}"
+        for name, version in resolved_packages
+        if name in packages
+    )
+    package_names_map_str = "\n".join(
+        f"  {k}: {v}" for k, v in package_names_map.items()
+    )
+
+    new_content = f"""{START}
+package_names_map:
+{package_names_map_str}
+
+
+{pins}
+
+"""
+
+    content = content[:idx1] + new_content + content[idx2:]
+    with open(conda_config_path, "w") as f:
+        f.write(content)
+
+
+if __name__ == "__main__":
+    update_pins()