Merge branch 'install-bob-deps' into 'master'

Add script to install  CI versions of all external Bob dependencies

See merge request !298

bob/devtools/build.py

@@ -716,7 +716,9 @@ def base_build(
         return conda_build.api.build(recipe_dir, config=conda_config)
 
 
-def load_packages_from_conda_build_config(conda_build_config, condarc_options):
+def load_packages_from_conda_build_config(
+    conda_build_config, condarc_options, with_pins=False
+):
     with open(conda_build_config, "r") as f:
         content = f.read()
 
@@ -734,7 +736,15 @@ def load_packages_from_conda_build_config(conda_build_config, condarc_options):
 
     package_names_map = package_pins.pop("package_names_map")
 
-    packages = [package_names_map.get(p, p) for p in package_pins.keys()]
+    if with_pins:
+        # NB : in pins, need to strip the occasional " cuda*" suffix
+        # tensorflow=x.x.x cuda* -> tensorflow=x.x.x
+        packages = [
+            f"{package_names_map.get(p, p)}={str(v[0]).split(' ')[0]}"
+            for p, v in package_pins.items()
+        ]
+    else:
+        packages = [package_names_map.get(p, p) for p in package_pins.keys()]
 
     return packages, package_names_map
 

bob/devtools/scripts/dependencies.py

+"""Create an environment with all external dependencies listed in bob/devtools/data/conda_build_config.yaml"""
+import click
+
+
+@click.command(
+    epilog="""Example:
+
+    Creates an environment called `myenv' based on Python 3.8 and containing all external bob dependencies:
+
+
+    bdt dev dependencies --python 3.8 myenv
+"""
+)
+@click.argument("env_name", nargs=1)
+@click.option("--python", required=True, help="Python version to pin, e.g. 3.8")
+def dependencies(env_name, python):
+    """Creates an environment with all external bob dependencies."""
+    import subprocess
+
+    import pkg_resources
+
+    from bob.devtools.build import load_packages_from_conda_build_config
+
+    conda_config_path = pkg_resources.resource_filename(
+        "bob.devtools", "data/conda_build_config.yaml"
+    )
+
+    packages, _ = load_packages_from_conda_build_config(
+        conda_config_path, {"channels": []}, with_pins=True
+    )
+
+    # ask mamba to create an environment with the packages
+    try:
+        _ = subprocess.run(
+            [
+                "mamba",
+                "create",
+                "--override-channels",
+                "-c",
+                "conda-forge",
+                "-n",
+                env_name,
+                f"python={python}",
+            ]
+            + packages
+            + [
+                "compilers"
+            ],  # Install conda-forge compilers for compiled pacakges
+            check=True,
+        )
+    except subprocess.CalledProcessError as e:
+        print(e.output.decode())
+        raise e
+
+
+if __name__ == "__main__":
+    dependencies()

bob/devtools/scripts/development.py

@@ -118,7 +118,12 @@ bdt dev install -n bobbioface src/bob.bio.base
 bdt dev new -vv bob/bob.newpackage "John Doe" "joe@example.com"
 # edit the conda/meta.yaml and requirements.txt files to add your dependencies
 bdt dev create --python 3.9 bobnewpackage
-bdt install -n bobnewpackage ."""
+bdt install -n bobnewpackage .
+
+\b
+# create an environment with all external bob dependencies
+bdt dev dependencies --python 3.9 my_env
+"""
 )
 def dev():
     """Development scripts"""

doc/development.rst

@@ -36,12 +36,23 @@ assume you have ``bob.devtools`` installed on a conda environment named ``bdt``
    $ bdt dev create -vv dev
    $ conda activate dev
 
+If you know that you plan to develop many packages (or even every Bob package), you can
+also instead create an environment that contains the integrality of external dependencies.
+This avoids the need to run ``bdt dev create`` for many packages. You will need to pick a Python version:
+
+.. code-block:: sh
+
+   $ bdt dev dependencies --python 3.9 dev
+   $ conda activate dev
+
 .. note::
 
    ``bdt`` might try to install the cuda version of deep learning packages. If
    you don't have cuda drivers installed and face errors such as ``nothing
    provides __cuda``, you might need to run: ``export CONDA_OVERRIDE_CUDA=11.6``
    where instead of ``11.6`` you should put the latest version of cuda.
+   You can also use this trick if you actually want to ensure ``bdt`` will
+   install the cuda version of deep learning packages.
 
 * Build the package using pip:
 
@@ -243,6 +254,19 @@ and install ``bob.extension`` as following:
     $ bdt dev install -n dev src/bob.extension
 
 
+If you want to develop many packages, or even all Bob packages at once, you can proceed a bit
+differently. First, create an environment containing all external Bob dependencies.
+
+.. code-block:: sh
+
+    $ bdt dev dependencies --python 3.9 bob_deps
+    $ conda activate bob_deps
+    $ mkdir -pv bob_beta/src
+    $ cd bob_beta/src
+    $ bdt dev checkout --use-ssh bob.extension bob.io.base bob.pipelines # ... checkout all packages you need
+    $ bdt dev install bob.extension bob.io.base bob.pipelines # ... install all packages you need
+
+
 .. _bob.devtools.create_package:
 
 Local development of a new package

setup.py

@@ -112,6 +112,7 @@ setup(
             "create = bob.devtools.scripts.create:create",
             "install = bob.devtools.scripts.development:install",
             "checkout = bob.devtools.scripts.development:checkout",
+            "dependencies = bob.devtools.scripts.dependencies:dependencies",
         ],
     },
     classifiers=[