[segmentation.scripts] Add upload script

src/mednet/libs/classification/scripts/upload.py

@@ -79,7 +79,7 @@ def upload(
     )
 
     # get train files
-    train_folder = experiment_folder / "model"
+    train_folder = experiment_folder
     train_log_file = train_folder / "trainlog.pdf"
     train_meta_file = train_folder / "train.meta.json"
     train_model_file = get_checkpoint_to_run_inference(train_folder)

src/mednet/libs/segmentation/scripts/cli.py

@@ -44,3 +44,6 @@ segmentation.add_command(dump_annotations.dump_annotations)
 segmentation.add_command(
     importlib.import_module("..experiment", package=__name__).experiment,
 )
+segmentation.add_command(
+    importlib.import_module("..upload", package=__name__).upload,
+)

src/mednet/libs/segmentation/scripts/upload.py

+# SPDX-FileCopyrightText: Copyright © 2023 Idiap Research Institute <contact@idiap.ch>
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import pathlib
+
+import click
+from clapper.click import ResourceOption, verbosity_option
+from clapper.logging import setup
+from mednet.libs.common.scripts.click import ConfigCommand
+from mednet.libs.common.scripts.upload import reusable_options
+
+logger = setup(__name__.split(".")[0], format="%(levelname)s: %(message)s")
+
+
+@click.command(
+    entry_point_group="mednet.config",
+    cls=ConfigCommand,
+    epilog="""Examples:
+
+1. Upload an existing experiment result from a path it resides on (with a default experiment name as {model-name}_{database-name} and a default run name as {date-time}):
+
+   .. code:: sh
+
+      mednet segmentation upload --experiment-folder=/path/to/results
+
+2. Upload an existing experiment result with an experiment name:
+
+   .. code:: sh
+
+      mednet segmentation upload --experiment-folder=/path/to/results --experiment-name=exp-pasa_mc
+
+3. Upload an existing experiment result with a run name:
+
+   .. code:: sh
+
+      mednet segmentation upload --experiment-folder=/path/to/results --run-name=run-1
+
+4. Upload an existing experiment result with defining a size limit of 20MB for each file:
+
+   .. code:: sh
+
+      mednet segmentation upload --experiment-folder=/path/to/results --upload-limit-mb=20
+
+""",
+)
+@reusable_options
+@verbosity_option(logger=logger, cls=ResourceOption, expose_value=False)
+def upload(
+    project_path: str,
+    experiment_folder: pathlib.Path,
+    experiment_name: str,
+    run_name: str,
+    upload_limit_mb: int,
+    **_,  # ignored
+) -> None:  # numpydoc ignore=PR01
+    """Upload results from an experiment folder to GitLab's MLFlow server."""
+
+    import json
+    import os
+    import tempfile
+
+    import mlflow
+    from mednet.libs.common.utils.checkpointer import (
+        get_checkpoint_to_run_inference,
+    )
+    from mednet.libs.common.utils.gitlab import (
+        gitlab_instance_and_token,
+        sanitize_filename,
+        size_in_mb,
+    )
+
+    logger.info("Retrieving GitLab credentials for access to hosted MLFlow server...")
+    gitlab, token = gitlab_instance_and_token()
+    project = gitlab.projects.get(project_path)
+    os.environ["MLFLOW_TRACKING_TOKEN"] = token
+    os.environ["MLFLOW_TRACKING_URI"] = (
+        gitlab.api_url + f"/projects/{project.id}/ml/mlflow"
+    )
+
+    # get train files
+    train_folder = experiment_folder
+    train_log_file = train_folder / "trainlog.pdf"
+    train_meta_file = train_folder / "train.meta.json"
+    train_model_file = get_checkpoint_to_run_inference(train_folder)
+    train_files = [train_meta_file, train_model_file, train_log_file]
+
+    # get evaluation files
+    evaluation_file = experiment_folder / "evaluation.json"
+    evaluation_meta_file = experiment_folder / "evaluation.meta.json"
+    evaluation_meta_file = experiment_folder / "evaluation.rst"
+    evaluation_log_file = experiment_folder / "evaluation.pdf"
+    evaluation_files = [
+        evaluation_file,
+        evaluation_meta_file,
+        evaluation_log_file,
+    ]
+
+    # checks for maximum upload limit
+    total_size_mb = sum([size_in_mb(f) for f in train_files + evaluation_files])
+    if upload_limit_mb != 0 and total_size_mb > upload_limit_mb:
+        raise RuntimeError(
+            f"Total size of upload ({total_size_mb:.2f} MB) exceeds "
+            f"permitted maximum ({upload_limit_mb:.2f} MB)."
+        )
+
+    with train_meta_file.open("r") as meta_file:
+        train_data = json.load(meta_file)
+
+    with evaluation_file.open("r") as meta_file:
+        evaluation_data = json.load(meta_file)
+    evaluation_data = evaluation_data["test"]
+
+    # get lowest validation epoch
+    best_epoch = str(train_model_file).split(".")[0].split("=")[1]
+
+    experiment_name = (
+        experiment_name or f"{train_data['model-name']}-{train_data['database-name']}"
+    )
+    run_name = run_name or train_data["datetime"]
+
+    click.secho(
+        f"Uploading entry `{run_name}` to experiment `{experiment_name}` "
+        f"on GitLab project {project_path} (id: {project.id})...",
+        bold=True,
+        fg="green",
+    )
+    exp_meta = mlflow.set_experiment(experiment_name=experiment_name)
+    with mlflow.start_run(run_name=run_name):
+        click.echo("Uploading package metadata...")
+        click.echo(f"  -> `version` ({train_data['package-version']})")
+        mlflow.log_param("package version", train_data["package-version"])
+
+        click.echo("Uploading metrics...")
+
+        for k in [
+            "epochs",
+            "batch-size",
+        ]:
+            click.secho(f"  -> `{k}` ({train_data[k]})")
+            mlflow.log_param(k, train_data[k])
+
+        click.secho(f"  -> `#accumulations` ({train_data['accumulate-grad-batches']})")
+        mlflow.log_param("#Accumulations", train_data["accumulate-grad-batches"])
+        click.secho(f"  -> `epoch (best)` ({best_epoch})")
+        mlflow.log_param("Epoch (best)", best_epoch)
+
+        for k in [
+            "precision",
+            "recall",
+            "f1",
+            "average_precision_score",
+            "specificity",
+            "auc_score",
+            "accuracy",
+        ]:
+            click.secho(f"  -> `{k}` ({evaluation_data[k]:.3g})")
+            mlflow.log_metric(k, evaluation_data[k])
+
+        click.echo("Uploading artifacts (files)...")
+
+        with tempfile.TemporaryDirectory() as tmpdir_name:
+            tmpdir = pathlib.Path(tmpdir_name)
+            for f in train_files + evaluation_files:
+                assert f.exists(), f"File `{f}` does not exist - cannot upload!"
+                clean_path = str(sanitize_filename(tmpdir, f))
+                click.secho(f"  -> `{clean_path}` ({size_in_mb(f):.2f} MB)")
+                mlflow.log_artifact(clean_path)
+
+    click.secho(f"Uploaded {total_size_mb:.2f} MB to server.", bold=True, fg="green")
+    click.secho(
+        f"Visit {gitlab.url}/{project.path_with_namespace}/-/ml/experiments/{exp_meta.experiment_id}",
+        bold=True,
+        fg="blue",
+    )