diff --git a/doc/api.rst b/doc/api.rst
index 9d0b02c0c9aece7a7647ef95c8ea80827948ad87..186f9eeab1aebde1081615f24fabd9971a70777d 100644
--- a/doc/api.rst
+++ b/doc/api.rst
@@ -97,6 +97,7 @@ Reusable auxiliary functions.
:toctree: api/utils
mednet.utils.checkpointer
+ mednet.utils.gitlab
mednet.utils.rc
mednet.utils.resources
mednet.utils.tensorboard
diff --git a/doc/catalog.json b/doc/catalog.json
index 529a23b480af03f8020f89720ff522d6d9f14d7f..40189913389912fc08ab55351365bc76f7f5da67 100644
--- a/doc/catalog.json
+++ b/doc/catalog.json
@@ -32,5 +32,14 @@
"sources": {
"readthedocs": "tabulate"
}
+ },
+ "python-gitlab": {
+ "versions": {
+ "latest": "https://python-gitlab.readthedocs.io/en/latest/",
+ "stable": "https://python-gitlab.readthedocs.io/en/stable/"
+ },
+ "sources": {
+ "environment": "python-gitlab"
+ }
}
}
diff --git a/doc/conf.py b/doc/conf.py
index f547b0b0938ab095df1a074426d75cb88665d65e..4f6770a65c54efea2b2cbadf0b6df6ba50970088 100644
--- a/doc/conf.py
+++ b/doc/conf.py
@@ -71,7 +71,7 @@ main_doc = "index"
project = "mednet"
package = distribution(project)
-copyright = "%s, Idiap Research Institute" % time.strftime("%Y") # noqa: A001
+copyright = f"{time.strftime("%Y")}, Idiap Research Institute" # noqa
# The short X.Y version.
version = package.version
@@ -126,6 +126,7 @@ auto_intersphinx_packages = [
"tensorboardx",
("clapper", "latest"),
("python", "3"),
+ "python-gitlab",
]
auto_intersphinx_catalog = "catalog.json"
diff --git a/src/mednet/engine/evaluator.py b/src/mednet/engine/evaluator.py
index 42b1308a6714336b9b4f01d392d4500317342043..acbf79b3fcdd9db808a6ee28e6ded77c42d887fb 100644
--- a/src/mednet/engine/evaluator.py
+++ b/src/mednet/engine/evaluator.py
@@ -509,7 +509,7 @@ def _precision_recall_canvas() -> (
y = f_score * x / (2 * x - f_score)
plt.plot(x[y >= 0], y[y >= 0], color="green", alpha=0.1)
tick_locs.append(y[-1])
- tick_labels.append("%.1f" % f_score)
+ tick_labels.append(f"{f_score:.1f}")
axes2.tick_params(axis="y", which="both", pad=0, right=False, left=False)
axes2.set_ylabel("iso-F", color="green", alpha=0.3)
axes2.set_ylim([0.0, 1.0])
diff --git a/src/mednet/scripts/upload.py b/src/mednet/scripts/upload.py
index bed4802f8863ef4bc1d445b56f3c0c2d45fb623d..615b81b25b08d9c8c07ae9e1dbaff1a9753f195e 100644
--- a/src/mednet/scripts/upload.py
+++ b/src/mednet/scripts/upload.py
@@ -3,6 +3,7 @@
# SPDX-License-Identifier: GPL-3.0-or-later
import pathlib
+
import click
from clapper.click import ResourceOption, verbosity_option
from clapper.logging import setup
@@ -12,55 +13,6 @@ from .click import ConfigCommand
logger = setup(__name__.split(".")[0], format="%(levelname)s: %(message)s")
-def _get_gitlab_config():
- """Return an instance of the Gitlab object for remote operations.
-
- Returns
- -------
- Gitlab entry and credential.
- """
- import gitlab
- import configparser
-
- cfg = pathlib.Path("~/.python-gitlab.cfg").expanduser()
- if cfg.exists():
- gl = gitlab.Gitlab.from_config("idiap", [str(cfg)])
- config = configparser.ConfigParser()
- config.read(cfg)
- else: # ask the user for a token or use one from the current runner
- server = "https://gitlab.idiap.ch"
- token = input(f"{server} (user or project) token: ")
- gl = gitlab.Gitlab(server, private_token=token, api_version="4")
- config = {"idiap": {"private_token": token}}
- # test authentication with given credential.
- gl.auth()
-
- return gl, config
-
-
-def _create_temp_copy(source, target):
- """Create a copy of original file in temp folder.
-
- Parameters
- ----------
- source
- Source file.
- target
- Target file.
-
- Returns
- -------
- Path to target file in temp folder.
- """
- import shutil
- import tempfile
-
- temp_dir = pathlib.Path(tempfile.gettempdir())
- target = temp_dir / target
- shutil.copy2(source, target)
- return target
-
-
@click.command(
entry_point_group="mednet.config",
cls=ConfigCommand,
@@ -84,14 +36,24 @@ def _create_temp_copy(source, target):
mednet upload --experiment-folder=/path/to/results --run-name=run-1
-4. Upload an existing experiment result with defining a size limit of 20MB for each file (set 0 for no limit):
+4. Upload an existing experiment result with defining a size limit of 20MB for each file:
.. code:: sh
- mednet upload --experiment-folder=/path/to/results --file-size=20
+ mednet upload --experiment-folder=/path/to/results --upload-limit-mb=20
""",
)
+@click.option(
+ "--project-path",
+ "-p",
+ help="Path to the project where to upload model entries",
+ required=True,
+ type=str,
+ default="biosignal/software/mednet",
+ show_default=True,
+ cls=ResourceOption,
+)
@click.option(
"--experiment-folder",
"-f",
@@ -103,12 +65,13 @@ def _create_temp_copy(source, target):
path_type=pathlib.Path,
),
default="results",
+ show_default=True,
cls=ResourceOption,
)
@click.option(
"--experiment-name",
"-e",
- help='A string indicating the experiment name (e.g. "exp-pasa_mc" or "exp-densenet_mc-ch")',
+ help='A string indicating the experiment name (e.g. "exp-pasa-mc" or "exp-densenet-mc-ch")',
cls=ResourceOption,
)
@click.option(
@@ -118,9 +81,9 @@ def _create_temp_copy(source, target):
cls=ResourceOption,
)
@click.option(
- "--file-limit",
+ "--upload-limit-mb",
"-l",
- help='Limit file size to be uploaded in MB (set 0 for no limit).',
+ help="Maximim upload size in MB (set to 0 for no limit).",
show_default=True,
required=True,
default=10,
@@ -129,84 +92,119 @@ def _create_temp_copy(source, target):
)
@verbosity_option(logger=logger, cls=ResourceOption, expose_value=False)
def upload(
+ project_path: str,
experiment_folder: pathlib.Path,
experiment_name: str,
run_name: str,
- file_limit: int,
+ upload_limit_mb: int,
**_, # ignored
) -> None: # numpydoc ignore=PR01
- """Upload results from an experiment folder."""
- import os
+ """Upload results from an experiment folder to GitLab's MLFlow server."""
+
import json
+ import os
+ import tempfile
+
import mlflow
- logger.info("Getting Gitlab credentials for accessing to MLFlow server...")
- gitlab, config = _get_gitlab_config()
- project = gitlab.projects.get("biosignal/software/mednet")
- os.environ["MLFLOW_TRACKING_TOKEN"] = config["idiap"]["private_token"]
+ from ..utils.checkpointer import get_checkpoint_to_run_inference
+ from ..utils.gitlab import (
+ gitlab_instance_and_token,
+ sanitize_filename,
+ size_in_mb,
+ )
+
+ logger.info(
+ "Retrieving GitLab credentials for access to hosted MLFlow server..."
+ )
+ gitlab, token = gitlab_instance_and_token()
+ project = gitlab.projects.get(project_path)
+ os.environ["MLFLOW_TRACKING_TOKEN"] = token
os.environ["MLFLOW_TRACKING_URI"] = (
gitlab.api_url + f"/projects/{project.id}/ml/mlflow"
)
- # prepare train files
+ # get train files
train_folder = experiment_folder / "model"
train_meta_file = train_folder / "meta.json"
train_log_file = train_folder / "trainlog.pdf"
- train_model_file = [f for f in train_folder.glob("*lowest*")][0]
- train_model_temp_file = train_model_file.parts[-1].replace("=", "_")
- train_model_temp_file = _create_temp_copy(
- train_model_file, train_model_temp_file
- )
- with train_meta_file.open("r") as f:
- train_data = json.load(f)
- train_files = [train_meta_file, train_log_file, train_model_temp_file]
+ train_model_file = get_checkpoint_to_run_inference(train_folder)
+ train_files = [train_meta_file, train_model_file, train_log_file]
- # prepare evaluation files
+ # get evaluation files
evaluation_file = experiment_folder / "evaluation.json"
evaluation_meta_file = experiment_folder / "evaluation.meta.json"
evaluation_log_file = experiment_folder / "evaluation.pdf"
- with evaluation_file.open("r") as f:
- evaluation_data = json.load(f)
- evaluation_data = evaluation_data["test"]
- evaluation_files = [evaluation_file, evaluation_meta_file, evaluation_log_file]
-
- # check for file sizes.
- for f in train_files + evaluation_files:
- file_size = f.stat().st_size / (1024**2)
- if file_limit != 0 and file_size > file_limit:
- raise RuntimeError(
- f"Size of {f} ({file_size:.2f} MB) must be less than or equal to {file_limit} MB."
- )
+ evaluation_files = [
+ evaluation_file,
+ evaluation_meta_file,
+ evaluation_log_file,
+ ]
+
+ # checks for maximum upload limit
+ total_size_mb = sum([size_in_mb(f) for f in train_files + evaluation_files])
+ if upload_limit_mb != 0 and total_size_mb > upload_limit_mb:
+ raise RuntimeError(
+ f"Total size of upload ({total_size_mb:.2f} MB) exceeds "
+ f"permitted maximum ({upload_limit_mb:.2f} MB)."
+ )
# prepare experiment and run names
+ with train_meta_file.open("r") as meta_file:
+ train_data = json.load(meta_file)
+
+ with evaluation_file.open("r") as meta_file:
+ evaluation_data = json.load(meta_file)
+ evaluation_data = evaluation_data["test"]
+
experiment_name = (
experiment_name
- if experiment_name
- else f'{train_data["model-name"]}_{train_data["database-name"]}'
+ or f"{train_data['model-name']}-{train_data['database-name']}"
)
- run_name = run_name if run_name else train_data["datetime"]
+ run_name = run_name or train_data["datetime"]
- logger.info("Setting experiment and run names on the MLFlow server...")
- mlflow.set_experiment(experiment_name=experiment_name)
+ click.secho(
+ f"Uploading entry `{run_name}` to experiment `{experiment_name}` "
+ f"on GitLab project {project_path} (id: {project.id})...",
+ bold=True,
+ fg="green",
+ )
+ exp_meta = mlflow.set_experiment(experiment_name=experiment_name)
with mlflow.start_run(run_name=run_name):
- # upload metrics
- logger.info("Uploading metrics to MLFlow server...")
- mlflow.log_metric("threshold", evaluation_data["threshold"])
- mlflow.log_metric("precision", evaluation_data["precision"])
- mlflow.log_metric("recall", evaluation_data["recall"])
- mlflow.log_metric("f1_score", evaluation_data["f1_score"])
- mlflow.log_metric(
- "average_precision_score", evaluation_data["average_precision_score"]
- )
- mlflow.log_metric("specificity", evaluation_data["specificity"])
- mlflow.log_metric("auc_score", evaluation_data["auc_score"])
- mlflow.log_metric("accuracy", evaluation_data["accuracy"])
- mlflow.log_param("version", train_data["package-version"])
- # upload artifacts
- logger.info("Uploading artifacts to MLFlow server...")
- for f in train_files:
- mlflow.log_artifact(f)
- for f in evaluation_files:
- mlflow.log_artifact(f)
- # delete temporary file as no need it after logging.
- train_model_temp_file.unlink()
+ click.echo("Uploading package metadata...")
+ click.echo(f" -> `version` ({train_data['package-version']})")
+ mlflow.log_param("package version", train_data["package-version"])
+
+ click.echo("Uploading metrics...")
+
+ for k in [
+ "threshold",
+ "precision",
+ "recall",
+ "f1_score",
+ "average_precision_score",
+ "specificity",
+ "auc_score",
+ "accuracy",
+ ]:
+ click.secho(f" -> `{k}` ({evaluation_data[k]:.3g})")
+ mlflow.log_metric(k, evaluation_data[k])
+
+ click.echo("Uploading artifacts (files)...")
+
+ with tempfile.TemporaryDirectory() as tmpdir_name:
+ tmpdir = pathlib.Path(tmpdir_name)
+ for f in train_files + evaluation_files:
+ assert f.exists(), f"File `{f}` does not exist - cannot upload!"
+ clean_path = str(sanitize_filename(tmpdir, f))
+ click.secho(f" -> `{clean_path}` ({size_in_mb(f):.2f} MB)")
+ mlflow.log_artifact(clean_path)
+
+ click.secho(
+ f"Uploaded {total_size_mb:.2f} MB to server.", bold=True, fg="green"
+ )
+ click.secho(
+ f"Visit {gitlab.url}/{project.path_with_namespace}/-/ml/experiments/{exp_meta.experiment_id}",
+ bold=True,
+ fg="blue",
+ )
diff --git a/src/mednet/utils/checkpointer.py b/src/mednet/utils/checkpointer.py
index 27cf9a3673c86bbc5412781751ec470e8d7a644c..4f529de8eb250ed771518ef8a5018c409e6e8003 100644
--- a/src/mednet/utils/checkpointer.py
+++ b/src/mednet/utils/checkpointer.py
@@ -66,19 +66,23 @@ def _get_checkpoint_from_alias(
), f"Template `{str(template)}` does not contain the keyword `{{epoch}}`"
pattern = re.compile(
- template.name.replace("{epoch}", r"epoch(=|_|-)(?P<epoch>\d+)"),
+ template.name.replace(
+ "{epoch}", r"epoch(?P<separator>=|-|_)(?P<epoch>\d+)"
+ ),
)
highest = -1
+ separator = "="
for f in template.parent.iterdir():
match = pattern.match(f.name)
if match is not None:
value = int(match.group("epoch"))
if value > highest:
highest = value
+ separator = match.group("separator")
if highest != -1:
return template.with_name(
- template.name.replace("{epoch}", f"epoch={highest}"),
+ template.name.replace("{epoch}", f"epoch{separator}{highest}"),
)
raise FileNotFoundError(
diff --git a/src/mednet/utils/gitlab.py b/src/mednet/utils/gitlab.py
new file mode 100644
index 0000000000000000000000000000000000000000..d0b111ddcf1f54ec995ffb0544c0199aa3d4f7b4
--- /dev/null
+++ b/src/mednet/utils/gitlab.py
@@ -0,0 +1,92 @@
+# SPDX-FileCopyrightText: Copyright © 2023 Idiap Research Institute <contact@idiap.ch>
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import configparser
+import logging
+import pathlib
+import shutil
+
+import gitlab
+
+logger = logging.getLogger(__name__)
+
+
+def gitlab_instance_and_token() -> tuple[gitlab.Gitlab, str]:
+ """Return an instance of the Gitlab object for remote operations, and the
+ user token.
+
+ Returns
+ -------
+ Gitlab main object and user token
+ """
+
+ cfg = pathlib.Path("~/.python-gitlab.cfg").expanduser()
+ if cfg.exists():
+ gl = gitlab.Gitlab.from_config("idiap", [str(cfg)])
+ config = configparser.ConfigParser()
+ config.read(cfg)
+ token = config["idiap"]["private_token"]
+
+ else: # ask the user for a token or use one from the current runner
+ server = "https://gitlab.idiap.ch"
+ token = input(f"{server} (user or project) token: ")
+ gl = gitlab.Gitlab(server, private_token=token, api_version="4")
+
+ # tests authentication with given credential.
+ gl.auth()
+
+ return gl, token
+
+
+def sanitize_filename(tmpdir: pathlib.Path, path: pathlib.Path) -> pathlib.Path:
+ """Sanitize the name of a file to be logged.
+
+ This function sanitizes the basename of a file to be logged on the GitLab
+ MLflow server. It removes unsupported characters (such as ``=``) by
+ creating a copy of the file to be uploaded, with a modified name, on the
+ provided temporary directory. It then returns the name of such a temporary
+ file.
+
+ If the input file path does not need sanitization, it is returned as is.
+
+ Parameters
+ ----------
+ tmpdir
+ The temporary directory where a copy of the input path, with a
+ sanitized name will be created.
+ path
+ The file that needs its name sanitized.
+
+ Returns
+ -------
+ Path to the temporary folder, and the sanitized copy of the input file
+ in said temporary folder, or the input ``path``, in case its name does not
+ need sanitization.
+ """
+
+ sanitized_filename = path.parts[-1].replace("=", "-")
+ if path.parts[-1] == sanitized_filename:
+ return path
+
+ absolute_sanitized_filename = tmpdir / sanitized_filename
+ logger.info(
+ f"Sanitazing filename `{path}` -> `{absolute_sanitized_filename}`"
+ )
+ shutil.copy2(path, absolute_sanitized_filename)
+ return absolute_sanitized_filename
+
+
+def size_in_mb(path: pathlib.Path) -> float:
+ """Return the size in megabytes of a file.
+
+ Parameters
+ ----------
+ path
+ Input path to calculate file size from.
+
+ Returns
+ -------
+ A floating point number for the size of the object in MB.
+ """
+ return path.stat().st_size / (1024**2)