Commit faa882dd authored by Amir MOHAMMADI's avatar Amir MOHAMMADI
Browse files

Merge branch 'pip' into 'master'

Use Pip

See merge request !250
parents b058cad1 aa20560d
Pipeline #54676 passed with stages
in 24 minutes and 6 seconds
......@@ -711,22 +711,7 @@ def base_build(
return conda_build.api.build(recipe_dir, config=conda_config)
def bob_devel(
bootstrap,
server,
intranet,
group,
conda_build_config,
condarc_options,
work_dir,
):
"""
Tests that all packages listed in bob/devtools/data/conda_build_config.yaml
can be installed in one environment.
"""
# Load the packages and their pins in bob/devtools/data/conda_build_config.yaml
# Create a temporary conda-build recipe to test if all packages can be installed
def load_packages_from_conda_build_config(conda_build_config, condarc_options):
with open(conda_build_config, "r") as f:
content = f.read()
......@@ -746,6 +731,29 @@ def bob_devel(
packages = [package_names_map.get(p, p) for p in package_pins.keys()]
return packages, package_names_map
def bob_devel(
bootstrap,
server,
intranet,
group,
conda_build_config,
condarc_options,
work_dir,
):
"""
Tests that all packages listed in bob/devtools/data/conda_build_config.yaml
can be installed in one environment.
"""
# Load the packages and their pins in bob/devtools/data/conda_build_config.yaml
# Create a temporary conda-build recipe to test if all packages can be installed
packages, _ = load_packages_from_conda_build_config(
conda_build_config, condarc_options
)
recipe_dir = os.path.join(work_dir, "deps", "bob-devel")
template_yaml = os.path.join(recipe_dir, "meta.template.yaml")
final_yaml = os.path.join(recipe_dir, "meta.yaml")
......
......@@ -322,7 +322,7 @@ click_plugins:
cmake:
- 3.21.3
coverage:
- 5.5
- 6.0
dask:
- 2021.9.1
dask_jobqueue:
......@@ -338,7 +338,7 @@ docker_py:
docopt:
- 0.6.2
ffmpeg:
- 4.3
- 4.3.2
flaky:
- 3.7.0
font_ttf_dejavu_sans_mono:
......@@ -348,7 +348,7 @@ freetype:
giflib:
- 5.2.1
graphviz:
- 2.49.0
- 2.49.1
h5py:
- 3.1.0
hdf5:
......@@ -358,7 +358,7 @@ jinja2:
jpeg:
- 9d
jsonschema:
- 3.2.0
- 4.0.1
libblitz:
- 1.0.2
libpng:
......@@ -377,9 +377,7 @@ nose:
- 1.3.7
numba:
- 0.54.0
# we build against numpy 1.18 but test against newer versions.
numpy:
# part of a zip_keys: python, python_impl, numpy
- 1.19.5
opencv:
- 4.5.1
......@@ -387,6 +385,8 @@ pandas:
- 1.3.3
pillow:
- 8.3.2
pip:
- 21.2.4
pkg_config:
- 0.29.2
psutil:
......@@ -394,13 +394,13 @@ psutil:
psycopg2:
- 2.9.1
pybind11:
- 2.7.1
- 2.8.0
pytables:
- 3.6.1
pytest:
- 6.2.5
pytest_cov:
- 2.12.1
- 3.0.0
python_graphviz:
- 0.17
pytorch:
......
"""Updates the pin versions inside bob/devtools/data/conda_build_config.yaml"""
import click
@click.command(
epilog="""Example:
python bob/devtools/scripts/update_pins.py --python 3.8
Force specific version of packages:
python bob/devtools/scripts/update_pins.py --python 3.8 opencv=4.5.1 pytorch=1.9
"""
)
@click.argument("manual_pins", nargs=-1)
@click.option("--python", required=True, help="Python version to pin, e.g. 3.8")
def update_pins(manual_pins, python):
import subprocess
from bob.devtools.build import load_packages_from_conda_build_config
conda_config_path = "bob/devtools/data/conda_build_config.yaml"
packages, package_names_map = load_packages_from_conda_build_config(
conda_config_path, {"channels": []}
)
reversed_package_names_map = {v: k for k, v in package_names_map.items()}
# ask mamba to create an environment with the packages
try:
output = subprocess.run(
[
"mamba",
"create",
"--dry-run",
"--override-channels",
"-c",
"conda-forge",
"-n",
"temp_env",
f"python={python}",
]
+ packages
+ list(manual_pins),
capture_output=True,
check=True,
)
except subprocess.CalledProcessError as e:
print(e.output.decode())
raise e
env_text = output.stdout.decode("utf-8")
print(env_text)
resolved_packages = []
for line in env_text.split("\n"):
line = line.strip()
if line.startswith("+ "):
line = line.split()
name, version = line[1], line[2]
resolved_packages.append((name, version))
# write the new pinning
with open(conda_config_path, "r") as f:
content = f.read()
START = """
# AUTOMATIC PARSING START
# DO NOT MODIFY THIS COMMENT
# list all packages with dashes or dots in their names, here:"""
idx1 = content.find(START)
idx2 = content.find("# AUTOMATIC PARSING END")
pins = "\n".join(
f"{reversed_package_names_map.get(name, name)}:\n - {version}"
for name, version in resolved_packages
if name in packages
)
package_names_map_str = "\n".join(
f" {k}: {v}" for k, v in package_names_map.items()
)
new_content = f"""{START}
package_names_map:
{package_names_map_str}
{pins}
"""
content = content[:idx1] + new_content + content[idx2:]
with open(conda_config_path, "w") as f:
f.write(content)
if __name__ == "__main__":
update_pins()
......@@ -12,9 +12,9 @@ build:
script:
- cd {{ project_dir }}
{% if environ.get('BUILD_EGG') %}
- python setup.py sdist --formats=zip
- "{{ PYTHON }} setup.py sdist --formats=zip"
{% endif %}
- python setup.py install --single-version-externally-managed --record record.txt
- "{{ PYTHON }} -m pip install . -vv"
# installs the documentation source, readme to share/doc so it is available
# during test time
- install -d "${PREFIX}/share/doc/{{ name }}"
......@@ -25,6 +25,7 @@ requirements:
host:
- python {{ python }}
- setuptools {{ setuptools }}
- pip {{ pip }}
- bob.extension
# place your other host dependencies here (same as requirements.txt)
# use the format:
......@@ -47,6 +48,7 @@ test:
imports:
- {{ name }}
commands:
- pip check
# runs tests for package only, report only what is in the package
# creates html and xml reports and place them in specific directories
- pytest --verbose --cov {{ name }} --cov-report term-missing --cov-report html:{{ project_dir }}/sphinx/coverage --cov-report xml:{{ project_dir }}/coverage.xml --pyargs {{ name }}
......@@ -55,6 +57,7 @@ test:
- conda inspect linkages -p $PREFIX {{ name }} # [not win]
- conda inspect objects -p $PREFIX {{ name }} # [osx]
requires:
- pip {{ pip }}
- pytest {{ pytest }}
- pytest-cov {{ pytest_cov }}
- coverage {{ coverage }}
......
......@@ -13,9 +13,9 @@ build:
script:
- cd {{ environ.get('RECIPE_DIR') + '/..' }}
{% if environ.get('BUILD_EGG') and not os.path.exists('dist') %}
- python setup.py sdist --formats=zip
- "{{ PYTHON }} setup.py sdist --formats=zip"
{% endif %}
- {{ PYTHON }} -m pip install --no-deps --ignore-installed .
- "{{ PYTHON }} -m pip install . -vv"
# installs the documentation source, readme to share/doc so it is available
# during test time
- install -d "${PREFIX}/share/doc/{{ name }}"
......@@ -56,12 +56,14 @@ requirements:
test:
requires:
- pip
- sphinx_rtd_theme
- pytest
- pytest-cov
imports:
- {{ name }}
commands:
- pip check
- bdt -h
- bdt -?
- bdt --help
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment