[tests] Update tests

tests/test_visceral.py → src/mednet/libs/classification/tests/test_visceral.py

@@ -25,7 +25,7 @@ def test_protocol_consistency(
     split: str,
     lenghts: dict[str, int],
 ):
-    from mednet.data.split import make_split
+    from mednet.libs.common.data.split import make_split
 
     database_checkers.check_split(
         make_split("mednet.config.data.visceral", f"{split}.json"),
@@ -37,7 +37,7 @@ def test_protocol_consistency(
 
 @pytest.mark.skip_if_rc_var_not_set("datadir.visceral")
 def test_database_check():
-    from mednet.scripts.database import check
+    from mednet.libs.common.scripts.database import check
 
     runner = CliRunner()
     result = runner.invoke(check, ["visceral"])

src/mednet/libs/common/scripts/upload.py

+# SPDX-FileCopyrightText: Copyright © 2023 Idiap Research Institute <contact@idiap.ch>
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import pathlib
+
+import click
+from clapper.click import ResourceOption, verbosity_option
+from clapper.logging import setup
+
+from .click import ConfigCommand
+
+logger = setup(__name__.split(".")[0], format="%(levelname)s: %(message)s")
+
+
+@click.command(
+    entry_point_group="mednet.config",
+    cls=ConfigCommand,
+    epilog="""Examples:
+
+1. Upload an existing experiment result from a path it resides on (with a default experiment name as {model-name}_{database-name} and a default run name as {date-time}):
+
+   .. code:: sh
+
+      mednet upload --experiment-folder=/path/to/results
+
+2. Upload an existing experiment result with an experiment name:
+
+   .. code:: sh
+
+      mednet upload --experiment-folder=/path/to/results --experiment-name=exp-pasa_mc
+
+3. Upload an existing experiment result with a run name:
+
+   .. code:: sh
+
+      mednet upload --experiment-folder=/path/to/results --run-name=run-1
+
+4. Upload an existing experiment result with defining a size limit of 20MB for each file:
+
+   .. code:: sh
+
+      mednet upload --experiment-folder=/path/to/results --upload-limit-mb=20
+
+""",
+)
+@click.option(
+    "--project-path",
+    "-p",
+    help="Path to the project where to upload model entries",
+    required=True,
+    type=str,
+    default="biosignal/software/mednet",
+    show_default=True,
+    cls=ResourceOption,
+)
+@click.option(
+    "--experiment-folder",
+    "-f",
+    help="Directory in which to upload results from",
+    required=True,
+    type=click.Path(
+        file_okay=False,
+        dir_okay=True,
+        path_type=pathlib.Path,
+    ),
+    default="results",
+    show_default=True,
+    cls=ResourceOption,
+)
+@click.option(
+    "--experiment-name",
+    "-e",
+    help='A string indicating the experiment name (e.g. "exp-pasa-mc" or "exp-densenet-mc-ch")',
+    cls=ResourceOption,
+)
+@click.option(
+    "--run-name",
+    "-r",
+    help='A string indicating the run name (e.g. "run-1")',
+    cls=ResourceOption,
+)
+@click.option(
+    "--upload-limit-mb",
+    "-l",
+    help="Maximim upload size in MB (set to 0 for no limit).",
+    show_default=True,
+    required=True,
+    default=10,
+    type=click.IntRange(min=0),
+    cls=ResourceOption,
+)
+@verbosity_option(logger=logger, cls=ResourceOption, expose_value=False)
+def upload(
+    project_path: str,
+    experiment_folder: pathlib.Path,
+    experiment_name: str,
+    run_name: str,
+    upload_limit_mb: int,
+    **_,  # ignored
+) -> None:  # numpydoc ignore=PR01
+    """Upload results from an experiment folder to GitLab's MLFlow server."""
+
+    import json
+    import os
+    import tempfile
+
+    import mlflow
+    from mednet.libs.common.utils.checkpointer import (
+        get_checkpoint_to_run_inference,
+    )
+    from mednet.libs.common.utils.gitlab import (
+        gitlab_instance_and_token,
+        sanitize_filename,
+        size_in_mb,
+    )
+
+    logger.info(
+        "Retrieving GitLab credentials for access to hosted MLFlow server..."
+    )
+    gitlab, token = gitlab_instance_and_token()
+    project = gitlab.projects.get(project_path)
+    os.environ["MLFLOW_TRACKING_TOKEN"] = token
+    os.environ["MLFLOW_TRACKING_URI"] = (
+        gitlab.api_url + f"/projects/{project.id}/ml/mlflow"
+    )
+
+    # get train files
+    train_folder = experiment_folder / "model"
+    train_meta_file = train_folder / "meta.json"
+    train_log_file = train_folder / "trainlog.pdf"
+    train_model_file = get_checkpoint_to_run_inference(train_folder)
+    train_files = [train_meta_file, train_model_file, train_log_file]
+
+    # get evaluation files
+    evaluation_file = experiment_folder / "evaluation.json"
+    evaluation_meta_file = experiment_folder / "evaluation.meta.json"
+    evaluation_log_file = experiment_folder / "evaluation.pdf"
+    evaluation_files = [
+        evaluation_file,
+        evaluation_meta_file,
+        evaluation_log_file,
+    ]
+
+    # checks for maximum upload limit
+    total_size_mb = sum([size_in_mb(f) for f in train_files + evaluation_files])
+    if upload_limit_mb != 0 and total_size_mb > upload_limit_mb:
+        raise RuntimeError(
+            f"Total size of upload ({total_size_mb:.2f} MB) exceeds "
+            f"permitted maximum ({upload_limit_mb:.2f} MB)."
+        )
+
+    # prepare experiment and run names
+    with train_meta_file.open("r") as meta_file:
+        train_data = json.load(meta_file)
+
+    with evaluation_file.open("r") as meta_file:
+        evaluation_data = json.load(meta_file)
+    evaluation_data = evaluation_data["test"]
+
+    experiment_name = (
+        experiment_name
+        or f"{train_data['model-name']}-{train_data['database-name']}"
+    )
+    run_name = run_name or train_data["datetime"]
+
+    click.secho(
+        f"Uploading entry `{run_name}` to experiment `{experiment_name}` "
+        f"on GitLab project {project_path} (id: {project.id})...",
+        bold=True,
+        fg="green",
+    )
+    exp_meta = mlflow.set_experiment(experiment_name=experiment_name)
+    with mlflow.start_run(run_name=run_name):
+        click.echo("Uploading package metadata...")
+        click.echo(f"  -> `version` ({train_data['package-version']})")
+        mlflow.log_param("package version", train_data["package-version"])
+
+        click.echo("Uploading metrics...")
+
+        for k in [
+            "threshold",
+            "precision",
+            "recall",
+            "f1_score",
+            "average_precision_score",
+            "specificity",
+            "auc_score",
+            "accuracy",
+        ]:
+            click.secho(f"  -> `{k}` ({evaluation_data[k]:.3g})")
+            mlflow.log_metric(k, evaluation_data[k])
+
+        click.echo("Uploading artifacts (files)...")
+
+        with tempfile.TemporaryDirectory() as tmpdir_name:
+            tmpdir = pathlib.Path(tmpdir_name)
+            for f in train_files + evaluation_files:
+                assert f.exists(), f"File `{f}` does not exist - cannot upload!"
+                clean_path = str(sanitize_filename(tmpdir, f))
+                click.secho(f"  -> `{clean_path}` ({size_in_mb(f):.2f} MB)")
+                mlflow.log_artifact(clean_path)
+
+    click.secho(
+        f"Uploaded {total_size_mb:.2f} MB to server.", bold=True, fg="green"
+    )
+    click.secho(
+        f"Visit {gitlab.url}/{project.path_with_namespace}/-/ml/experiments/{exp_meta.experiment_id}",
+        bold=True,
+        fg="blue",
+    )

src/mednet/libs/common/tests/data/iris-test.csv

+5,3,1.6,0.2,Iris-setosa
+5,3.4,1.6,0.4,Iris-setosa
+5.2,3.5,1.5,0.2,Iris-setosa
+5.2,3.4,1.4,0.2,Iris-setosa
+4.7,3.2,1.6,0.2,Iris-setosa
+4.8,3.1,1.6,0.2,Iris-setosa
+5.4,3.4,1.5,0.4,Iris-setosa
+5.2,4.1,1.5,0.1,Iris-setosa
+5.5,4.2,1.4,0.2,Iris-setosa
+4.9,3.1,1.5,0.1,Iris-setosa
+5,3.2,1.2,0.2,Iris-setosa
+5.5,3.5,1.3,0.2,Iris-setosa
+4.9,3.1,1.5,0.1,Iris-setosa
+4.4,3,1.3,0.2,Iris-setosa
+5.1,3.4,1.5,0.2,Iris-setosa
+5,3.5,1.3,0.3,Iris-setosa
+4.5,2.3,1.3,0.3,Iris-setosa
+4.4,3.2,1.3,0.2,Iris-setosa
+5,3.5,1.6,0.6,Iris-setosa
+5.1,3.8,1.9,0.4,Iris-setosa
+4.8,3,1.4,0.3,Iris-setosa
+5.1,3.8,1.6,0.2,Iris-setosa
+4.6,3.2,1.4,0.2,Iris-setosa
+5.3,3.7,1.5,0.2,Iris-setosa
+5,3.3,1.4,0.2,Iris-setosa
+6.6,3,4.4,1.4,Iris-versicolor
+6.8,2.8,4.8,1.4,Iris-versicolor
+6.7,3,5,1.7,Iris-versicolor
+6,2.9,4.5,1.5,Iris-versicolor
+5.7,2.6,3.5,1,Iris-versicolor
+5.5,2.4,3.8,1.1,Iris-versicolor
+5.5,2.4,3.7,1,Iris-versicolor
+5.8,2.7,3.9,1.2,Iris-versicolor
+6,2.7,5.1,1.6,Iris-versicolor
+5.4,3,4.5,1.5,Iris-versicolor
+6,3.4,4.5,1.6,Iris-versicolor
+6.7,3.1,4.7,1.5,Iris-versicolor
+6.3,2.3,4.4,1.3,Iris-versicolor
+5.6,3,4.1,1.3,Iris-versicolor
+5.5,2.5,4,1.3,Iris-versicolor
+5.5,2.6,4.4,1.2,Iris-versicolor
+6.1,3,4.6,1.4,Iris-versicolor
+5.8,2.6,4,1.2,Iris-versicolor
+5,2.3,3.3,1,Iris-versicolor
+5.6,2.7,4.2,1.3,Iris-versicolor
+5.7,3,4.2,1.2,Iris-versicolor
+5.7,2.9,4.2,1.3,Iris-versicolor
+6.2,2.9,4.3,1.3,Iris-versicolor
+5.1,2.5,3,1.1,Iris-versicolor
+5.7,2.8,4.1,1.3,Iris-versicolor
+7.2,3.2,6,1.8,Iris-virginica
+6.2,2.8,4.8,1.8,Iris-virginica
+6.1,3,4.9,1.8,Iris-virginica
+6.4,2.8,5.6,2.1,Iris-virginica
+7.2,3,5.8,1.6,Iris-virginica
+7.4,2.8,6.1,1.9,Iris-virginica
+7.9,3.8,6.4,2,Iris-virginica
+6.4,2.8,5.6,2.2,Iris-virginica
+6.3,2.8,5.1,1.5,Iris-virginica
+6.1,2.6,5.6,1.4,Iris-virginica
+7.7,3,6.1,2.3,Iris-virginica
+6.3,3.4,5.6,2.4,Iris-virginica
+6.4,3.1,5.5,1.8,Iris-virginica
+6,3,4.8,1.8,Iris-virginica
+6.9,3.1,5.4,2.1,Iris-virginica
+6.7,3.1,5.6,2.4,Iris-virginica
+6.9,3.1,5.1,2.3,Iris-virginica
+5.8,2.7,5.1,1.9,Iris-virginica
+6.8,3.2,5.9,2.3,Iris-virginica
+6.7,3.3,5.7,2.5,Iris-virginica
+6.7,3,5.2,2.3,Iris-virginica
+6.3,2.5,5,1.9,Iris-virginica
+6.5,3,5.2,2,Iris-virginica
+6.2,3.4,5.4,2.3,Iris-virginica
+5.9,3,5.1,1.8,Iris-virginica

src/mednet/libs/common/tests/data/iris-train.csv

+5.1,3.5,1.4,0.2,Iris-setosa
+4.9,3,1.4,0.2,Iris-setosa
+4.7,3.2,1.3,0.2,Iris-setosa
+4.6,3.1,1.5,0.2,Iris-setosa
+5,3.6,1.4,0.2,Iris-setosa
+5.4,3.9,1.7,0.4,Iris-setosa
+4.6,3.4,1.4,0.3,Iris-setosa
+5,3.4,1.5,0.2,Iris-setosa
+4.4,2.9,1.4,0.2,Iris-setosa
+4.9,3.1,1.5,0.1,Iris-setosa
+5.4,3.7,1.5,0.2,Iris-setosa
+4.8,3.4,1.6,0.2,Iris-setosa
+4.8,3,1.4,0.1,Iris-setosa
+4.3,3,1.1,0.1,Iris-setosa
+5.8,4,1.2,0.2,Iris-setosa
+5.7,4.4,1.5,0.4,Iris-setosa
+5.4,3.9,1.3,0.4,Iris-setosa
+5.1,3.5,1.4,0.3,Iris-setosa
+5.7,3.8,1.7,0.3,Iris-setosa
+5.1,3.8,1.5,0.3,Iris-setosa
+5.4,3.4,1.7,0.2,Iris-setosa
+5.1,3.7,1.5,0.4,Iris-setosa
+4.6,3.6,1,0.2,Iris-setosa
+5.1,3.3,1.7,0.5,Iris-setosa
+4.8,3.4,1.9,0.2,Iris-setosa
+7,3.2,4.7,1.4,Iris-versicolor
+6.4,3.2,4.5,1.5,Iris-versicolor
+6.9,3.1,4.9,1.5,Iris-versicolor
+5.5,2.3,4,1.3,Iris-versicolor
+6.5,2.8,4.6,1.5,Iris-versicolor
+5.7,2.8,4.5,1.3,Iris-versicolor
+6.3,3.3,4.7,1.6,Iris-versicolor
+4.9,2.4,3.3,1,Iris-versicolor
+6.6,2.9,4.6,1.3,Iris-versicolor
+5.2,2.7,3.9,1.4,Iris-versicolor
+5,2,3.5,1,Iris-versicolor
+5.9,3,4.2,1.5,Iris-versicolor
+6,2.2,4,1,Iris-versicolor
+6.1,2.9,4.7,1.4,Iris-versicolor
+5.6,2.9,3.6,1.3,Iris-versicolor
+6.7,3.1,4.4,1.4,Iris-versicolor
+5.6,3,4.5,1.5,Iris-versicolor
+5.8,2.7,4.1,1,Iris-versicolor
+6.2,2.2,4.5,1.5,Iris-versicolor
+5.6,2.5,3.9,1.1,Iris-versicolor
+5.9,3.2,4.8,1.8,Iris-versicolor
+6.1,2.8,4,1.3,Iris-versicolor
+6.3,2.5,4.9,1.5,Iris-versicolor
+6.1,2.8,4.7,1.2,Iris-versicolor
+6.4,2.9,4.3,1.3,Iris-versicolor
+6.3,3.3,6,2.5,Iris-virginica
+5.8,2.7,5.1,1.9,Iris-virginica
+7.1,3,5.9,2.1,Iris-virginica
+6.3,2.9,5.6,1.8,Iris-virginica
+6.5,3,5.8,2.2,Iris-virginica
+7.6,3,6.6,2.1,Iris-virginica
+4.9,2.5,4.5,1.7,Iris-virginica
+7.3,2.9,6.3,1.8,Iris-virginica
+6.7,2.5,5.8,1.8,Iris-virginica
+7.2,3.6,6.1,2.5,Iris-virginica
+6.5,3.2,5.1,2,Iris-virginica
+6.4,2.7,5.3,1.9,Iris-virginica
+6.8,3,5.5,2.1,Iris-virginica
+5.7,2.5,5,2,Iris-virginica
+5.8,2.8,5.1,2.4,Iris-virginica
+6.4,3.2,5.3,2.3,Iris-virginica
+6.5,3,5.5,1.8,Iris-virginica
+7.7,3.8,6.7,2.2,Iris-virginica
+7.7,2.6,6.9,2.3,Iris-virginica
+6,2.2,5,1.5,Iris-virginica
+6.9,3.2,5.7,2.3,Iris-virginica
+5.6,2.8,4.9,2,Iris-virginica
+7.7,2.8,6.7,2,Iris-virginica
+6.3,2.7,4.9,1.8,Iris-virginica
+6.7,3.3,5.7,2.1,Iris-virginica

src/mednet/libs/common/utils/gitlab.py

+# SPDX-FileCopyrightText: Copyright © 2023 Idiap Research Institute <contact@idiap.ch>
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import configparser
+import logging
+import pathlib
+import shutil
+
+import gitlab
+
+logger = logging.getLogger(__name__)
+
+
+def gitlab_instance_and_token() -> tuple[gitlab.Gitlab, str]:
+    """Return an instance of the Gitlab object for remote operations, and the
+    user token.
+
+    Returns
+    -------
+        Gitlab main object and user token
+    """
+
+    cfg = pathlib.Path("~/.python-gitlab.cfg").expanduser()
+    if cfg.exists():
+        gl = gitlab.Gitlab.from_config("idiap", [str(cfg)])
+        config = configparser.ConfigParser()
+        config.read(cfg)
+        token = config["idiap"]["private_token"]
+
+    else:  # ask the user for a token or use one from the current runner
+        server = "https://gitlab.idiap.ch"
+        token = input(f"{server} (user or project) token: ")
+        gl = gitlab.Gitlab(server, private_token=token, api_version="4")
+
+    # tests authentication with given credential.
+    gl.auth()
+
+    return gl, token
+
+
+def sanitize_filename(tmpdir: pathlib.Path, path: pathlib.Path) -> pathlib.Path:
+    """Sanitize the name of a file to be logged.
+
+    This function sanitizes the basename of a file to be logged on the GitLab
+    MLflow server. It removes unsupported characters (such as ``=``) by
+    creating a copy of the file to be uploaded, with a modified name, on the
+    provided temporary directory.  It then returns the name of such a temporary
+    file.
+
+    If the input file path does not need sanitization, it is returned as is.
+
+    Parameters
+    ----------
+    tmpdir
+        The temporary directory where a copy of the input path, with a
+        sanitized name will be created.
+    path
+        The file that needs its name sanitized.
+
+    Returns
+    -------
+        Path to the temporary folder, and the sanitized copy of the input file
+        in said temporary folder, or the input ``path``, in case its name does not
+        need sanitization.
+    """
+
+    sanitized_filename = path.parts[-1].replace("=", "-")
+    if path.parts[-1] == sanitized_filename:
+        return path
+
+    absolute_sanitized_filename = tmpdir / sanitized_filename
+    logger.info(
+        f"Sanitazing filename `{path}` -> `{absolute_sanitized_filename}`"
+    )
+    shutil.copy2(path, absolute_sanitized_filename)
+    return absolute_sanitized_filename
+
+
+def size_in_mb(path: pathlib.Path) -> float:
+    """Return the size in megabytes of a file.
+
+    Parameters
+    ----------
+    path
+        Input path to calculate file size from.
+
+    Returns
+    -------
+        A floating point number for the size of the object in MB.
+    """
+    return path.stat().st_size / (1024**2)

src/mednet/tests/test_cli.py

+# SPDX-FileCopyrightText: Copyright © 2023 Idiap Research Institute <contact@idiap.ch>
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+"""Tests for our CLI applications."""
+
+import contextlib
+
+from click.testing import CliRunner
+
+
+@contextlib.contextmanager
+def stdout_logging():
+    # copy logging messages to std out
+
+    import io
+    import logging
+
+    buf = io.StringIO()
+    ch = logging.StreamHandler(buf)
+    ch.setFormatter(logging.Formatter("%(message)s"))
+    ch.setLevel(logging.INFO)
+    logger = logging.getLogger("mednet")
+    logger.addHandler(ch)
+    yield buf
+    logger.removeHandler(ch)
+
+
+def _assert_exit_0(result):
+    assert (
+        result.exit_code == 0
+    ), f"Exit code {result.exit_code} != 0 -- Output:\n{result.output}"
+
+
+def _check_help(entry_point):
+    runner = CliRunner()
+    result = runner.invoke(entry_point, ["--help"])
+    _assert_exit_0(result)
+    assert result.output.startswith("Usage:")
+
+
+def test_info_help():
+    from mednet.scripts.info import info
+
+    _check_help(info)
+
+
+def test_info():
+    from mednet.scripts.info import info
+
+    runner = CliRunner()
+    result = runner.invoke(info)
+    _assert_exit_0(result)
+    assert "platform:" in result.output
+    assert "accelerators:" in result.output
+    assert "version:" in result.output
+    assert "configured classification databases:" in result.output
+    assert "configured segmentation databases:" in result.output
+    assert "dependencies:" in result.output
+    assert "python:" in result.output
+
+
+def test_upload_help():
+    from mednet.libs.common.scripts.upload import upload
+
+    _check_help(upload)