Merge branch 'pip' into 'master'

Use Pip

See merge request !250

bob/devtools/build.py

@@ -711,22 +711,7 @@ def base_build(
         return conda_build.api.build(recipe_dir, config=conda_config)
 
 
-def bob_devel(
-    bootstrap,
-    server,
-    intranet,
-    group,
-    conda_build_config,
-    condarc_options,
-    work_dir,
-):
-    """
-    Tests that all packages listed in bob/devtools/data/conda_build_config.yaml
-    can be installed in one environment.
-    """
-    # Load the packages and their pins in bob/devtools/data/conda_build_config.yaml
-    # Create a temporary conda-build recipe to test if all packages can be installed
-
+def load_packages_from_conda_build_config(conda_build_config, condarc_options):
     with open(conda_build_config, "r") as f:
         content = f.read()
 
@@ -746,6 +731,29 @@ def bob_devel(
 
     packages = [package_names_map.get(p, p) for p in package_pins.keys()]
 
+    return packages, package_names_map
+
+
+def bob_devel(
+    bootstrap,
+    server,
+    intranet,
+    group,
+    conda_build_config,
+    condarc_options,
+    work_dir,
+):
+    """
+    Tests that all packages listed in bob/devtools/data/conda_build_config.yaml
+    can be installed in one environment.
+    """
+    # Load the packages and their pins in bob/devtools/data/conda_build_config.yaml
+    # Create a temporary conda-build recipe to test if all packages can be installed
+
+    packages, _ = load_packages_from_conda_build_config(
+        conda_build_config, condarc_options
+    )
+
     recipe_dir = os.path.join(work_dir, "deps", "bob-devel")
     template_yaml = os.path.join(recipe_dir, "meta.template.yaml")
     final_yaml = os.path.join(recipe_dir, "meta.yaml")

bob/devtools/data/conda_build_config.yaml

@@ -322,7 +322,7 @@ click_plugins:
 cmake:
   - 3.21.3
 coverage:
-  - 5.5
+  - 6.0
 dask:
   - 2021.9.1
 dask_jobqueue:
@@ -338,7 +338,7 @@ docker_py:
 docopt:
   - 0.6.2
 ffmpeg:
-  - 4.3
+  - 4.3.2
 flaky:
   - 3.7.0
 font_ttf_dejavu_sans_mono:
@@ -348,7 +348,7 @@ freetype:
 giflib:
   - 5.2.1
 graphviz:
-  - 2.49.0
+  - 2.49.1
 h5py:
   - 3.1.0
 hdf5:
@@ -358,7 +358,7 @@ jinja2:
 jpeg:
   - 9d
 jsonschema:
-  - 3.2.0
+  - 4.0.1
 libblitz:
   - 1.0.2
 libpng:
@@ -377,9 +377,7 @@ nose:
   - 1.3.7
 numba:
   - 0.54.0
-# we build against numpy 1.18 but test against newer versions.
 numpy:
-  # part of a zip_keys: python, python_impl, numpy
   - 1.19.5
 opencv:
   - 4.5.1
@@ -387,6 +385,8 @@ pandas:
   - 1.3.3
 pillow:
   - 8.3.2
+pip:
+  - 21.2.4
 pkg_config:
   - 0.29.2
 psutil:
@@ -394,13 +394,13 @@ psutil:
 psycopg2:
   - 2.9.1
 pybind11:
-  - 2.7.1
+  - 2.8.0
 pytables:
   - 3.6.1
 pytest:
   - 6.2.5
 pytest_cov:
-  - 2.12.1
+  - 3.0.0
 python_graphviz:
   - 0.17
 pytorch:

bob/devtools/scripts/update_pins.py

+"""Updates the pin versions inside bob/devtools/data/conda_build_config.yaml"""
+
+import click
+
+
+@click.command(
+    epilog="""Example:
+
+python bob/devtools/scripts/update_pins.py --python 3.8
+
+Force specific version of packages:
+
+python bob/devtools/scripts/update_pins.py --python 3.8 opencv=4.5.1 pytorch=1.9
+"""
+)
+@click.argument("manual_pins", nargs=-1)
+@click.option("--python", required=True, help="Python version to pin, e.g. 3.8")
+def update_pins(manual_pins, python):
+    import subprocess
+
+    from bob.devtools.build import load_packages_from_conda_build_config
+
+    conda_config_path = "bob/devtools/data/conda_build_config.yaml"
+
+    packages, package_names_map = load_packages_from_conda_build_config(
+        conda_config_path, {"channels": []}
+    )
+    reversed_package_names_map = {v: k for k, v in package_names_map.items()}
+
+    # ask mamba to create an environment with the packages
+    try:
+        output = subprocess.run(
+            [
+                "mamba",
+                "create",
+                "--dry-run",
+                "--override-channels",
+                "-c",
+                "conda-forge",
+                "-n",
+                "temp_env",
+                f"python={python}",
+            ]
+            + packages
+            + list(manual_pins),
+            capture_output=True,
+            check=True,
+        )
+    except subprocess.CalledProcessError as e:
+        print(e.output.decode())
+        raise e
+
+    env_text = output.stdout.decode("utf-8")
+    print(env_text)
+
+    resolved_packages = []
+    for line in env_text.split("\n"):
+        line = line.strip()
+        if line.startswith("+ "):
+            line = line.split()
+            name, version = line[1], line[2]
+            resolved_packages.append((name, version))
+
+    # write the new pinning
+    with open(conda_config_path, "r") as f:
+        content = f.read()
+
+    START = """
+# AUTOMATIC PARSING START
+# DO NOT MODIFY THIS COMMENT
+
+# list all packages with dashes or dots in their names, here:"""
+    idx1 = content.find(START)
+    idx2 = content.find("# AUTOMATIC PARSING END")
+    pins = "\n".join(
+        f"{reversed_package_names_map.get(name, name)}:\n  - {version}"
+        for name, version in resolved_packages
+        if name in packages
+    )
+    package_names_map_str = "\n".join(
+        f"  {k}: {v}" for k, v in package_names_map.items()
+    )
+
+    new_content = f"""{START}
+package_names_map:
+{package_names_map_str}
+
+
+{pins}
+
+"""
+
+    content = content[:idx1] + new_content + content[idx2:]
+    with open(conda_config_path, "w") as f:
+        f.write(content)
+
+
+if __name__ == "__main__":
+    update_pins()

bob/devtools/templates/conda/meta.yaml

@@ -12,9 +12,9 @@ build:
   script:
     - cd {{ project_dir }}
     {% if environ.get('BUILD_EGG') %}
-    - python setup.py sdist --formats=zip
+    - "{{ PYTHON }} setup.py sdist --formats=zip"
     {% endif %}
-    - python setup.py install --single-version-externally-managed --record record.txt
+    - "{{ PYTHON }} -m pip install . -vv"
     # installs the documentation source, readme to share/doc so it is available
     # during test time
     - install -d "${PREFIX}/share/doc/{{ name }}"
@@ -25,6 +25,7 @@ requirements:
   host:
     - python {{ python }}
     - setuptools {{ setuptools }}
+    - pip {{ pip }}
     - bob.extension
     # place your other host dependencies here (same as requirements.txt)
     # use the format:
@@ -47,6 +48,7 @@ test:
   imports:
     - {{ name }}
   commands:
+    - pip check
     # runs tests for package only, report only what is in the package
     # creates html and xml reports and place them in specific directories
     - pytest --verbose --cov {{ name }} --cov-report term-missing --cov-report html:{{ project_dir }}/sphinx/coverage --cov-report xml:{{ project_dir }}/coverage.xml --pyargs {{ name }}
@@ -55,6 +57,7 @@ test:
     - conda inspect linkages -p $PREFIX {{ name }}  # [not win]
     - conda inspect objects -p $PREFIX {{ name }}  # [osx]
   requires:
+    - pip {{ pip }}
     - pytest {{ pytest }}
     - pytest-cov {{ pytest_cov }}
     - coverage {{ coverage }}

conda/meta.yaml

@@ -13,9 +13,9 @@ build:
   script:
     - cd {{ environ.get('RECIPE_DIR') + '/..' }}
     {% if environ.get('BUILD_EGG') and not os.path.exists('dist') %}
-    - python setup.py sdist --formats=zip
+    - "{{ PYTHON }} setup.py sdist --formats=zip"
     {% endif %}
-    - {{ PYTHON }} -m pip install --no-deps --ignore-installed .
+    - "{{ PYTHON }} -m pip install . -vv"
     # installs the documentation source, readme to share/doc so it is available
     # during test time
     - install -d "${PREFIX}/share/doc/{{ name }}"
@@ -56,12 +56,14 @@ requirements:
 
 test:
   requires:
+    - pip
     - sphinx_rtd_theme
     - pytest
     - pytest-cov
   imports:
     - {{ name }}
   commands:
+    - pip check
     - bdt -h
     - bdt -?
     - bdt --help