Merge branch 'pixi-ci' into 'main'

Full dev environment based on pixi-only

See merge request biosignal/software/mednet!33

.gitlab-ci.yml

@@ -3,26 +3,30 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 
 include:
-  - project: biosignal/software/dev-profile
-    file: /gitlab/python.yml
+  - project: 'biosignal/templates/ci'
+    ref: main
+    file: 'python.yml'
 
 variables:
-  XDG_CONFIG_HOME: $CI_PROJECT_DIR/tests/data
+  OMP_NUM_THREADS: 1  # for pytorch CI tests on docker platform
+  GIT_SUBMODULE_STRATEGY: normal
+  GIT_SUBMODULE_DEPTH: 1
 
-documentation:
-  before_script:
-    # for opencv-python dependence
-    - if [[ "$CI_RUNNER_TAGS" =~ "docker" ]]; then apt-get update && apt-get install -y libgl1-mesa-glx > /dev/null; fi
-    - !reference [.snippets, doc-prepare]
+.snippets:
+  apt-install-libgl:
+      # apt install libgl1-mesa-glx for grad-cam
+      - |
+        if [[ ${CI_RUNNER_TAGS} =~ '"docker"' ]]; then \
+          apt update; \
+          apt install -y libgl1-mesa-glx > /dev/null; \
+        fi
 
 tests:
   before_script:
-    # for opencv-python dependence
-    - if [[ "$CI_RUNNER_TAGS" =~ "docker" ]]; then apt-get update && apt-get install -y libgl1-mesa-glx > /dev/null; fi
-    - !reference [.snippets, test-prepare]
+    - !reference [.snippets, setup-pixi]
+    - !reference [.snippets, apt-install-libgl]
 
-conda-package:
+documentation:
   before_script:
-    # for opencv-python dependence
-    - if [[ "$CI_RUNNER_TAGS" =~ "docker" ]]; then apt-get update && apt-get install -y libgl1-mesa-glx > /dev/null; fi
-    - !reference [.snippets, conda-prepare]
+    - !reference [.snippets, setup-pixi]
+    - !reference [.snippets, apt-install-libgl]

conda/meta.yaml

-# SPDX-FileCopyrightText: 2023 Idiap Research Institute <contact@idiap.ch>
-#
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-{% set data = load_file_data(RECIPE_DIR + '/../pyproject.toml') %}
-
-package:
-  name: {{ data['project']['name'] }}
-  version: {{ environ.get('PACKAGE_VERSION', environ.get('GIT_DESCRIBE_TAG')) }}
-
-source:
-  path: ..
-
-build:
-  noarch: python
-  number: {{ environ.get('NEXT_BUILD_NUMBER', 0) }}
-  run_exports:
-    - {{ pin_subpackage(data['project']['name']) }}
-  script:
-    - "{{ PYTHON }} -m pip install {{ SRC_DIR }} -vv"
-
-requirements:
-  host:
-    - python >=3.10
-    - pip
-    - hatchling
-    - versioningit
-    - clapper {{ clapper }}
-    - click {{ click }}
-    - credible {{ credible }}
-    - grad-cam {{ grad_cam }}
-    - matplotlib {{ matplotlib }}
-    - numpy {{ numpy }}
-    - pillow {{ pillow }}
-    - psutil {{ psutil }}
-    - pytorch {{ pytorch }}
-    - scikit-image {{ scikit_image }}
-    - scikit-learn {{ scikit_learn }}
-    - scipy {{ scipy }}
-    - tabulate {{ tabulate }}
-    - torchvision  {{ torchvision }}
-    - tqdm {{ tqdm }}
-    - tensorboard {{ tensorboard }}
-    - lightning {{ lightning }}
-    - lightning >=2.2.0
-  run:
-    - python >=3.10
-    - versioningit
-    - {{ pin_compatible('clapper') }}
-    - {{ pin_compatible('click') }}
-    - {{ pin_compatible('credible') }}
-    - {{ pin_compatible('grad-cam', max_pin='x.x') }}
-    - {{ pin_compatible('matplotlib') }}
-    - {{ pin_compatible('numpy') }}
-    - {{ pin_compatible('pillow') }}
-    - {{ pin_compatible('psutil') }}
-    - {{ pin_compatible('pytorch') }}
-    - {{ pin_compatible('scikit-image') }}
-    - {{ pin_compatible('scikit-learn') }}
-    - {{ pin_compatible('scipy') }}
-    - {{ pin_compatible('tabulate') }}
-    - {{ pin_compatible('torchvision') }}
-    - {{ pin_compatible('tqdm') }}
-    - {{ pin_compatible('tensorboard') }}
-    - {{ pin_compatible('lightning', max_pin='x.x') }}
-    - lightning >=2.2.0
-
-test:
-  source_files:
-    - tests
-  imports:
-    - {{ data['project']['name'].replace('-','_') }}
-  commands:
-    - pytest -sv tests
-  requires:
-    - pytest {{ pytest }}
-
-about:
-  home: {{ data['project']['urls']['homepage'] }}
-  summary: {{ data['project']['description'] }}
-  license: {{ data['project']['license'] }}
-  license_family: GPL

doc/install.rst

@@ -8,54 +8,65 @@
  Installation
 ==============
 
-We support two installation modes, through pip_, or mamba_ (conda).
+Installation may follow one of three paths: deployment or development for
+CPU-only execution, or a mixed development/deployment with Nvidia CUDA support.
+Choose the relevant tab for details on each of those installation paths.
 
+.. tab:: Deployment
 
-.. tab:: pip
+   Install using pip_, or your preferred Python project management solution (e.g.
+   uv_, rye_ or poetry_).
 
-   Stable, from PyPI:
+   **Stable** release, from PyPI:
 
    .. code:: sh
 
       pip install mednet
+      mednet info
 
-   Latest beta, from GitLab package registry:
+   **Latest** development branch, from its git repository:
 
    .. code:: sh
 
-      pip install --pre --index-url https://gitlab.idiap.ch/api/v4/groups/software/-/packages/pypi/simple --extra-index-url https://pypi.org/simple mednet
+      pip install git+https://gitlab.idiap.ch/biosignal/software/mednet@main
+      mednet info
 
-   .. tip::
-
-      To avoid long command-lines you may configure pip to define the indexes and
-      package search priorities as you like.
 
+.. tab:: Development
 
-.. tab:: mamba/conda
-
-   Stable:
+   Checkout the repository, and then use pixi_ to setup a full development
+   environment:
 
    .. code:: sh
 
-      mamba install -c https://www.idiap.ch/software/biosignal/conda -c conda-forge mednet
+      git clone git@gitlab.idiap.ch:biosignal/software/mednet
+      pixi install --frozen
+      pixi run mednet info
 
-   Latest beta:
+   .. tip::
 
-   .. code:: sh
+      The ``--frozen`` flag will ensure that the latest lock-file available
+      with sources is used.  If you'd like to update the lock-file to the
+      latest set of compatible dependencies, remove that option.
 
-      mamba install -c https://www.idiap.ch/software/biosignal/conda/label/beta -c conda-forge mednet
 
-   .. tip::
+.. tab:: CUDA
 
-      To force-install Nvidia GPU support on Linux machines, execute:
+   Checkout the repository, and then use pixi_ to setup a version of this
+   package that can run on a CUDA-enabled machine:
 
-      .. code:: sh
+   .. code:: sh
 
-         $ mamba install pytorch-gpu
+      git clone git@gitlab.idiap.ch:biosignal/software/mednet
+      cd helpers/cuda
+      pixi install --frozen
+      pixi run mednet info
 
-         # or, to force the Nvidia CUDA version (environments w/o Nvidia setup):
+   .. tip::
 
-         $ CONDA_OVERRIDE_CUDA=12.0 mamba install 'pytorch-gpu=*=cuda118*'
+      The ``--frozen`` flag will ensure that the latest lock-file available
+      with sources is used.  If you'd like to update the lock-file to the
+      latest set of compatible dependencies, remove that option.
 
 
 .. _mednet.setup:

doc/links.rst

@@ -12,7 +12,10 @@
 .. _conda: https://conda.io
 .. _python: http://www.python.org
 .. _pip: https://pip.pypa.io/en/stable/
-.. _mamba: https://mamba.readthedocs.io/en/latest/index.html
+.. _uv: https://github.com/astral-sh/uv
+.. _rye: https://github.com/astral-sh/rye
+.. _poetry: https://python-poetry.org
+.. _pixi: https://pixi.sh
 .. _pytorch: https://pytorch.org
 .. _lightning: https://lightning.ai
 

pixi-cuda.toml → helpers/cuda/pixi.toml

@@ -9,28 +9,18 @@
 # of the mednet environment.  It loosely follows the recipe in the pixi
 # documentation: https://pixi.sh/latest/advanced/channel_priority/#use-case-pytorch-and-nvidia-with-conda-forge
 #
-# USAGE INSTRUCTIONS
-# ------------------
-#
-# To use this file, first link it to `pixi.toml`, and re-run `pixi install`.
-# Having a file named `pixi.toml` on the current directory has precedence over
-# `pyproject.toml`:
-#
-# $ ln -s pixi-cuda.toml pixi.toml
-# $ rm -rf .pixi pixi.lock  # clean-up
-# $ pixi install  # this will install all support libraries
-# $ pixi run build  # this may fail, see next
-#
-# If `pixi run build` does not work (in the case you are building on a machine
-# w/o CUDA support), then do the following:
-#
-# $ uv pip install --no-build-isolation --no-deps --editable ../mednet
+# Why is this separated from the main manifest? Check
+# https://github.com/prefix-dev/pixi/issues/1051.
 
 [project]
 name = "mednet"
 channels = ["nvidia", "conda-forge", "pytorch"]
 platforms = ["linux-64"]
 
+# warning: if you add/remove/pin dependencies, also update
+# the equivalent settings at the root's pyproject.toml file.
+#
+# (c.f.: https://github.com/prefix-dev/pixi/issues/1051)
 [dependencies]
 python = "~=3.12.0"
 clapper = "*"
@@ -45,11 +35,11 @@ psutil = "*"
 tabulate = "*"
 matplotlib = "*"
 pillow = "*"
-pytorch = { version = ">=1.8", channel = "pytorch" }
-torchvision = { version = ">=0.10", channel = "pytorch" }
-lightning = ">=2.2.0"
+pytorch = { version = "~=2.2.2", channel = "pytorch" }
+torchvision = { version = "~=0.17.2", channel = "pytorch" }
+lightning = "~=2.2.1"
 tensorboard = "*"
-grad-cam = ">=1.4.8"
+grad-cam = "~=1.5.0"
 versioningit = "*"
 # special stuff
 cuda = { version = "*", channel = "nvidia" }
@@ -58,10 +48,10 @@ pytorch-cuda = { version = "12.1.*", channel = "pytorch" }
 [host-dependencies]
 hatch = "*"
 versioningit = "*"
-uv = "*"
+pdbpp = "*"
 
 [system-requirements]
 cuda = "12.1"
 
-[tasks]
-build = "uv pip install --no-build-isolation --no-deps --editable ../mednet"
+[pypi-dependencies]
+mednet = { path = "../..", editable = true }

pyproject.toml

@@ -28,6 +28,11 @@ classifiers = [
   "Topic :: Software Development :: Libraries :: Python Modules",
 ]
 
+# warning: if you add/remove/pin dependencies, also update
+# tool.pixi.dependencies in this file, and the equivalent settings at and
+# helpers/cuda/pixi.toml files
+#
+# (c.f.: https://github.com/prefix-dev/pixi/issues/1051)
 dependencies = [
   "clapper",
   "click",
@@ -41,11 +46,11 @@ dependencies = [
   "tabulate",
   "matplotlib",
   "pillow",
-  "torch>=1.8",
-  "torchvision>=0.10",
-  "lightning>=2.2.0",
+  "torch~=2.2.2",
+  "torchvision~=0.17.2",
+  "lightning~=2.2.1",
   "tensorboard",
-  "grad-cam>=1.4.8",
+  "grad-cam~=1.5.0",
   "versioningit",
 ]
 
@@ -72,9 +77,16 @@ test = ["pytest", "pytest-cov"]
 mednet = "mednet.scripts.cli:cli"
 
 [tool.pixi.project]
-channels = ["conda-forge"]
+channels = ["conda-forge", "pytorch"]
 platforms = ["linux-64", "osx-arm64"]
 
+[tool.pixi.system-requirements]
+linux = "4.19.0"
+
+# warning: if you add/remove/pin dependencies, also update project.dependencies
+# in this file, and the equivalent settings at and helpers/cuda/pixi.toml files
+#
+# (c.f.: https://github.com/prefix-dev/pixi/issues/1051)
 [tool.pixi.dependencies]
 clapper = "*"
 click = "*"
@@ -88,18 +100,18 @@ psutil = "*"
 tabulate = "*"
 matplotlib = "*"
 pillow = "*"
-pytorch = ">=1.8"
-torchvision = ">=0.10"
-lightning = ">=2.2.0"
+pytorch = { version = "~=2.2.2", channel = "pytorch" }
+torchvision = { version = "~=0.17.2", channel = "pytorch" }
+lightning = "~=2.2.1"
 tensorboard = "*"
-grad-cam = ">=1.4.8"
+grad-cam = "~=1.5.0"
 versioningit = "*"
 
 [tool.pixi.feature.self.pypi-dependencies]
 mednet = { path = ".", editable = true }
 
-[tool.pixi.feature.dev.pypi-dependencies]
-mednet = { path = ".", editable = true, extras = ["qa", "doc", "test"] }
+[tool.pixi.feature.self.tasks]
+info = "mednet info"
 
 [tool.pixi.feature.py311.dependencies]
 python = "~=3.11.0"
@@ -115,6 +127,7 @@ reuse = "*"
 [tool.pixi.feature.qa.tasks]
 qa-install = "pre-commit install"
 qa = "pre-commit run --all-files"
+qa-ci = "pre-commit run --all-files --show-diff-on-failure --verbose"
 
 [tool.pixi.feature.doc.dependencies]
 sphinx = "*"
@@ -126,7 +139,9 @@ sphinx-inline-tabs = "*"
 sphinx-click = "*"
 
 [tool.pixi.feature.doc.tasks]
-doc = "rm -rf doc/api && rm -rf html && sphinx-build -aEW doc html"
+doc-clean = "rm -rf doc/api && rm -rf html"
+doc = "sphinx-build -aEW doc html"
+doctest = "sphinx-build -aEb doctest doc html/doctest"
 
 [tool.pixi.feature.test.dependencies]
 pytest = "*"
@@ -134,24 +149,42 @@ pytest-cov = "*"
 
 [tool.pixi.feature.test.tasks]
 test = "pytest -sv tests/"
-test-ci = "pytest -sv --cov-report 'html:html/coverage' --cov-report 'xml:coverage.xml' --junitxml 'junit-coverage.xml' --ignore '.profile' tests/"
+test-cov = "pytest -sv --cov-report 'html:html/coverage' tests/"
+test-ci = "cp tests/data/mednet.toml $XDG_CONFIG_HOME && pytest -sv --cov-report 'html:html/coverage' --cov-report 'xml:coverage.xml' --junitxml 'junit-coverage.xml' --ignore '.profile' tests/"
+
+[tool.pixi.feature.build.dependencies]
+hatch = "*"
+versioningit = "*"
+twine = "*"
+
+[tool.pixi.feature.build.tasks]
+build = "hatch build"
+check = "twine check dist/*"
+upload = "twine upload dist/*"
+
+[tool.pixi.feature.dev.dependencies]
+pdbpp = "*"
+uv = "*"
+
+[tool.pixi.feature.dev.tasks]
+uv-update-lock = "uv pip compile -q pyproject.toml --python-platform=linux -o uv.lock"
 
 [tool.pixi.environments]
-default = { features = [ "qa", "doc", "test", "py312", "dev" ], solve-group = "prod-group" }
-previous = { features = ["test", "py311", "self"] }
-prod = { features = ["py312", "self"], solve-group = "prod-group" }
+default = { features = ["qa", "build", "doc", "test", "dev", "py312", "self"] }
+qa-ci = { features = ["qa", "py312"] }
+build-ci = { features = ["build", "py312"] }
+test-ci-311 = { features = ["test", "dev", "py311", "self"] }
 
 [tool.hatch.version]
 source = "versioningit"
 
-[tool.versioningit.next-version]
-method = "smallest"
-
-[tool.versioningit.format]
+# versioningit configuration via tool.hatch.version table
+default-version = "0.0.0+unknown"
+next-version.method = "smallest"
 # Example formatted version: 1.2.4.dev42+ge174a1f
-distance = "{next_version}.dev{distance}+{vcs}{rev}"
+format.distance = "{next_version}.dev{distance}+{vcs}{rev}"
 # Example formatted version: 1.2.4.dev42+ge174a1f.d20230922
-distance-dirty = "{next_version}.dev{distance}+{vcs}{rev}.d{build_date:%Y%m%d}"
+format.distance-dirty = "{next_version}.dev{distance}+{vcs}{rev}.d{build_date:%Y%m%d}"
 
 [tool.hatch.build.targets.sdist]
 include = [

src/mednet/scripts/cli.py

@@ -17,6 +17,7 @@ def cli():
     pass
 
 
+cli.add_command(importlib.import_module("..info", package=__name__).info)
 cli.add_command(importlib.import_module("..config", package=__name__).config)
 cli.add_command(
     importlib.import_module("..database", package=__name__).database,

src/mednet/scripts/info.py

+# SPDX-FileCopyrightText: Copyright © 2024 Idiap Research Institute <contact@idiap.ch>
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import click
+from clapper.click import verbosity_option
+from clapper.logging import setup
+
+logger = setup(__name__.split(".")[0], format="%(levelname)s: %(message)s")
+
+
+@click.command(
+    epilog="""Examples:
+
+1. Provide information about the current installation:
+
+   .. code:: sh
+
+      mednet info
+
+""",
+)
+@verbosity_option(logger=logger, expose_value=False)
+def info(
+    **_,
+) -> None:  # numpydoc ignore=PR01
+    """Provide information about the current installation."""
+
+    import typing
+
+    from ..utils.rc import load_rc
+    from .database import _get_raw_databases
+    from .utils import execution_metadata
+
+    def _echo(left: str, right: str, color: str = "white") -> None:
+        s = [
+            click.style(left, bold=True),
+            click.style(": ", bold=True),
+            click.style(right, fg=color),
+        ]
+        click.echo("".join(s))
+
+    m = execution_metadata()
+    c = load_rc()
+
+    _echo("mednet version", typing.cast(str, m["package-version"]), "green")
+    _echo("platform", typing.cast(str, m["platform"]), "yellow")
+    _echo(
+        "accelerators",
+        ", ".join(typing.cast(list[str], m["accelerators"])),
+        "cyan",
+    )
+
+    if not c.path.exists():
+        _echo("configuration", f"{str(c.path)} [MISSING]", "white")
+    else:
+        _echo("configuration", str(c.path), "white")
+    dbs = _get_raw_databases()
+    click.secho("configured databases:", bold=True)
+    for k, v in dbs.items():
+        if "datadir" not in v:
+            # this database does not have a "datadir"
+            continue
+
+        if v["datadir"] is not None:
+            _echo(f"  - {k} ({v['module']})", f"{v['datadir']}", "green")
+        else:
+            _echo(f"  - {k} ({v['module']})", "NOT installed", "red")
+
+    click.secho("dependencies:", bold=True)
+    python = typing.cast(dict[str, str], m["python"])
+    _echo(
+        "  - python",
+        f"{python['version']} ({python['path']})",
+        "white",
+    )
+    deps = typing.cast(dict[str, str], m["dependencies"])
+    for name, version in deps.items():
+        _echo(f"  - {name}", version, "white")

src/mednet/scripts/utils.py

@@ -81,7 +81,9 @@ def device_properties(
     return retval
 
 
-def execution_metadata() -> dict[str, int | float | str | dict[str, str]]:
+def execution_metadata() -> (
+    dict[str, int | float | str | dict[str, str] | list[str]]
+):
     """Produce metadata concerning the running script, in the form of a
     dictionary.
 
@@ -101,6 +103,7 @@ def execution_metadata() -> dict[str, int | float | str | dict[str, str]]:
         * ``command-line``: the command-line that is being run
         * ``hostname``: machine hostname (e.g. ``localhost``)
         * ``platform``: machine platform (e.g. ``darwin``)
+        * ``accelerator``: acceleration devices available (e.g. ``cuda``)
     """
 
     import importlib.metadata
@@ -118,16 +121,20 @@ def execution_metadata() -> dict[str, int | float | str | dict[str, str]]:
     # current date time, in ISO8610 format
     datetime = __import__("datetime").datetime.now().astimezone().isoformat()
 
-    # collects dependence information
+    # collects dependency information
     package_name = (
         __package__.split(".")[0] if __package__ is not None else "unknown"
     )
     requires = importlib.metadata.requires(package_name) or []
     dependence_names = [re.split(r"(\=|~|!|>|<|;|\s)+", k)[0] for k in requires]
+    installed = {
+        v[0]: k for k, v in importlib.metadata.packages_distributions().items()
+    }
     dependencies = {
         k: importlib.metadata.version(k)  # version number as str
         for k in dependence_names
-        if importlib.util.find_spec(k) is not None  # if is installed
+        if importlib.util.find_spec(k if k not in installed else installed[k])
+        is not None  # if is installed
     }
 
     # checks if the current version corresponds to a dirty (uncommitted) change
@@ -150,10 +157,37 @@ def execution_metadata() -> dict[str, int | float | str | dict[str, str]]:
         logger.debug(f"Error {e}")
         pass
 
+    # checks if any acceleration device is present in the current platform
+    accelerators = [f"cpu ({torch.backends.cpu.get_cpu_capability()})"]
+
+    if torch.cuda.is_available() and torch.backends.cuda.is_built():
+        accelerators.append("cuda")
+
+    if torch.backends.cudnn.is_available():
+        accelerators.append("cudnn")
+
+    if torch.backends.mps.is_available():
+        accelerators.append("mps")
+
+    if torch.backends.mkl.is_available():
+        accelerators.append("mkl")
+
+    if torch.backends.mkldnn.is_available():
+        accelerators.append("mkldnn")
+
+    if torch.backends.openmp.is_available():
+        accelerators.append("openmp")
+
+    python = {
+        "version": ".".join([str(k) for k in sys.version_info[:3]]),
+        "path": sys.executable,
+    }
+
     return {
         "datetime": datetime,
         "package-name": package_name,
         "package-version": current_version,
+        "python": python,
         "dependencies": dependencies,
         "user": __import__("getpass").getuser(),
         "conda-env": os.environ.get("CONDA_DEFAULT_ENV", ""),
@@ -161,6 +195,7 @@ def execution_metadata() -> dict[str, int | float | str | dict[str, str]]:
         "command-line": " ".join(args),
         "hostname": __import__("platform").node(),
         "platform": sys.platform,
+        "accelerators": accelerators,
     }
 
 

tests/test_cli.py

@@ -40,6 +40,26 @@ def _check_help(entry_point):
     assert result.output.startswith("Usage:")
 
 
+def test_info_help():
+    from mednet.scripts.info import info
+
+    _check_help(info)
+
+
+def test_info():
+    from mednet.scripts.info import info
+
+    runner = CliRunner()
+    result = runner.invoke(info)
+    _assert_exit_0(result)
+    assert "platform:" in result.output
+    assert "accelerators:" in result.output
+    assert "version:" in result.output
+    assert "configured databases:" in result.output
+    assert "dependencies:" in result.output
+    assert "python:" in result.output
+
+
 def test_config_help():
     from mednet.scripts.config import config