Commit 59688191 authored by Amir MOHAMMADI's avatar Amir MOHAMMADI
Browse files

Add a script to update pins inside bob/devtools/data/conda_build_config.yaml automatically

parent b058cad1
Pipeline #54668 failed with stage
in 13 minutes
......@@ -711,22 +711,7 @@ def base_build(
return conda_build.api.build(recipe_dir, config=conda_config)
def bob_devel(
bootstrap,
server,
intranet,
group,
conda_build_config,
condarc_options,
work_dir,
):
"""
Tests that all packages listed in bob/devtools/data/conda_build_config.yaml
can be installed in one environment.
"""
# Load the packages and their pins in bob/devtools/data/conda_build_config.yaml
# Create a temporary conda-build recipe to test if all packages can be installed
def load_packages_from_conda_build_config(conda_build_config, condarc_options):
with open(conda_build_config, "r") as f:
content = f.read()
......@@ -746,6 +731,29 @@ def bob_devel(
packages = [package_names_map.get(p, p) for p in package_pins.keys()]
return packages, package_names_map
def bob_devel(
bootstrap,
server,
intranet,
group,
conda_build_config,
condarc_options,
work_dir,
):
"""
Tests that all packages listed in bob/devtools/data/conda_build_config.yaml
can be installed in one environment.
"""
# Load the packages and their pins in bob/devtools/data/conda_build_config.yaml
# Create a temporary conda-build recipe to test if all packages can be installed
packages, _ = load_packages_from_conda_build_config(
conda_build_config, condarc_options
)
recipe_dir = os.path.join(work_dir, "deps", "bob-devel")
template_yaml = os.path.join(recipe_dir, "meta.template.yaml")
final_yaml = os.path.join(recipe_dir, "meta.yaml")
......
......@@ -322,7 +322,7 @@ click_plugins:
cmake:
- 3.21.3
coverage:
- 5.5
- 6.0
dask:
- 2021.9.1
dask_jobqueue:
......@@ -338,7 +338,7 @@ docker_py:
docopt:
- 0.6.2
ffmpeg:
- 4.3
- 4.3.2
flaky:
- 3.7.0
font_ttf_dejavu_sans_mono:
......@@ -348,7 +348,7 @@ freetype:
giflib:
- 5.2.1
graphviz:
- 2.49.0
- 2.49.1
h5py:
- 3.1.0
hdf5:
......@@ -358,7 +358,7 @@ jinja2:
jpeg:
- 9d
jsonschema:
- 3.2.0
- 4.0.1
libblitz:
- 1.0.2
libpng:
......@@ -377,12 +377,10 @@ nose:
- 1.3.7
numba:
- 0.54.0
# we build against numpy 1.18 but test against newer versions.
numpy:
# part of a zip_keys: python, python_impl, numpy
- 1.19.5
opencv:
- 4.5.1
- 4.5.2
pandas:
- 1.3.3
pillow:
......@@ -394,13 +392,13 @@ psutil:
psycopg2:
- 2.9.1
pybind11:
- 2.7.1
- 2.8.0
pytables:
- 3.6.1
pytest:
- 6.2.5
pytest_cov:
- 2.12.1
- 3.0.0
python_graphviz:
- 0.17
pytorch:
......
"""Updates the pin versions inside bob/devtools/data/conda_build_config.yaml"""
import click
@click.command()
@click.option("--python", required=True, help="Python version to pin, e.g. 3.8")
def update_pins(python):
from subprocess import check_output
from bob.devtools.build import load_packages_from_conda_build_config
conda_config_path = "bob/devtools/data/conda_build_config.yaml"
packages, package_names_map = load_packages_from_conda_build_config(
conda_config_path, {"channels": []}
)
reversed_package_names_map = {v: k for k, v in package_names_map.items()}
# ask mamba to create an environment with the packages
env_text = check_output(
[
"mamba",
"create",
"--dry-run",
"--override-channels",
"-c",
"conda-forge",
"-n",
"temp_env",
f"python={python}",
]
+ packages
).decode("utf-8")
print(env_text)
resolved_packages = []
for line in env_text.split("\n"):
line = line.strip()
if line.startswith("+ "):
line = line.split()
name, version = line[1], line[2]
resolved_packages.append((name, version))
# write the new pinning
with open(conda_config_path, "r") as f:
content = f.read()
START = """
# AUTOMATIC PARSING START
# DO NOT MODIFY THIS COMMENT
# list all packages with dashes or dots in their names, here:"""
idx1 = content.find(START)
idx2 = content.find("# AUTOMATIC PARSING END")
pins = "\n".join(
f"{reversed_package_names_map.get(name, name)}:\n - {version}"
for name, version in resolved_packages
if name in packages
)
package_names_map_str = "\n".join(
f" {k}: {v}" for k, v in package_names_map.items()
)
new_content = f"""{START}
package_names_map:
{package_names_map_str}
{pins}
"""
content = content[:idx1] + new_content + content[idx2:]
with open(conda_config_path, "w") as f:
f.write(content)
if __name__ == "__main__":
update_pins()
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment