diff --git a/src/mednet/scripts/evaluate.py b/src/mednet/scripts/evaluate.py
index 047732c008ba40637d783ab41b1ecf649ef96261..c7d144ff672e9790f9be9126d2d01556a1bc5973 100644
--- a/src/mednet/scripts/evaluate.py
+++ b/src/mednet/scripts/evaluate.py
@@ -25,13 +25,13 @@ logger = setup(__name__.split(".")[0], format="%(levelname)s: %(message)s")
.. code:: sh
- mednet evaluate -vv --predictions=path/to/predictions.json --output=evaluation.json
+ mednet evaluate -vv --predictions=path/to/predictions.json
2. Run evaluation on an existing prediction output, tune threshold a priori on the `validation` set:
.. code:: sh
- mednet evaluate -vv --predictions=path/to/predictions.json --output=evaluation.json --threshold=validation
+ mednet evaluate -vv --predictions=path/to/predictions.json --threshold=validation
""",
)
@click.option(
@@ -48,19 +48,18 @@ logger = setup(__name__.split(".")[0], format="%(levelname)s: %(message)s")
cls=ResourceOption,
)
@click.option(
- "--output",
+ "--output-folder",
"-o",
- help="""Path to a JSON file in which to save evaluation results
- (leading directories are created if they do not exist).""",
+ help="Directory in which to store results (created if does not exist)",
required=True,
- default="evaluation.json",
- cls=ResourceOption,
type=click.Path(
- file_okay=True,
- dir_okay=False,
+ file_okay=False,
+ dir_okay=True,
writable=True,
path_type=pathlib.Path,
),
+ default="results",
+ cls=ResourceOption,
)
@click.option(
"--threshold",
@@ -106,7 +105,7 @@ logger = setup(__name__.split(".")[0], format="%(levelname)s: %(message)s")
@verbosity_option(logger=logger, cls=ResourceOption, expose_value=False)
def evaluate(
predictions: pathlib.Path,
- output: pathlib.Path,
+ output_folder: pathlib.Path,
threshold: str | float,
binning: str,
plot: bool,
@@ -128,13 +127,16 @@ def evaluate(
)
from .utils import execution_metadata, save_json_with_backup
+ evaluation_filename = "evaluation.json"
+ evaluation_file = pathlib.Path(output_folder) / evaluation_filename
+
with predictions.open("r") as f:
predict_data = json.load(f)
# register metadata
json_data: dict[str, typing.Any] = execution_metadata()
json_data = {k.replace("_", "-"): v for k, v in json_data.items()}
- save_json_with_backup(output.with_suffix(".meta.json"), json_data)
+ save_json_with_backup(evaluation_file.with_suffix(".meta.json"), json_data)
if threshold in predict_data:
# it is the name of a split
@@ -166,8 +168,8 @@ def evaluate(
)
# records full result analysis to a JSON file
- logger.info(f"Saving evaluation results at `{output}`...")
- with output.open("w") as f:
+ logger.info(f"Saving evaluation results at `{evaluation_file}`...")
+ with evaluation_file.open("w") as f:
json.dump(results, f, indent=2, cls=NumpyJSONEncoder)
# dump evaluation results in RST format to screen and file
@@ -181,7 +183,7 @@ def evaluate(
table = tabulate_results(table_data, fmt="rst")
click.echo(table)
- table_path = output.with_suffix(".rst")
+ table_path = evaluation_file.with_suffix(".rst")
logger.info(
f"Saving evaluation results in table format at `{table_path}`...",
)
@@ -189,7 +191,7 @@ def evaluate(
f.write(table)
# dump evaluation plots in file
- figure_path = output.with_suffix(".pdf")
+ figure_path = evaluation_file.with_suffix(".pdf")
logger.info(f"Saving evaluation figures at `{figure_path}`...")
if plot:
diff --git a/src/mednet/scripts/experiment.py b/src/mednet/scripts/experiment.py
index 3c4f3093fc41941a52ed7a9e29aefec9aa350d31..9f7aff12e53d4b5f26c54c53099f585fcd59e16e 100644
--- a/src/mednet/scripts/experiment.py
+++ b/src/mednet/scripts/experiment.py
@@ -101,11 +101,10 @@ def experiment(
from .train_analysis import train_analysis
logdir = train_output_folder / "logs"
- output_pdf = train_output_folder / "trainlog.pdf"
ctx.invoke(
train_analysis,
logdir=logdir,
- output=output_pdf,
+ output_folder=train_output_folder,
)
logger.info("Ended train analysis")
@@ -139,12 +138,10 @@ def experiment(
from .evaluate import evaluate
- evaluation_output = output_folder / "evaluation.json"
-
ctx.invoke(
evaluate,
predictions=predictions_output,
- output=evaluation_output,
+ output=output_folder,
threshold="validation",
)
diff --git a/src/mednet/scripts/train_analysis.py b/src/mednet/scripts/train_analysis.py
index 46898ed24079375ff5a9fd53274cff5f8a00cb28..902bc48e2bf80e6e75d60dae22400b6249243817 100644
--- a/src/mednet/scripts/train_analysis.py
+++ b/src/mednet/scripts/train_analysis.py
@@ -5,7 +5,7 @@
import pathlib
import click
-from clapper.click import verbosity_option
+from clapper.click import ResourceOption, verbosity_option
from clapper.logging import setup
# avoids X11/graphical desktop requirement when creating plots
@@ -119,18 +119,23 @@ def create_figures(
type=click.Path(dir_okay=True, exists=True, path_type=pathlib.Path),
)
@click.option(
- "--output",
+ "--output-folder",
"-o",
- help="Name of the output file to create (multi-page .pdf)",
+ help="Directory in which to store results (created if does not exist)",
required=True,
- show_default=True,
- default="trainlog.pdf",
- type=click.Path(dir_okay=False, file_okay=True, path_type=pathlib.Path),
+ type=click.Path(
+ file_okay=False,
+ dir_okay=True,
+ writable=True,
+ path_type=pathlib.Path,
+ ),
+ default="results",
+ cls=ResourceOption,
)
@verbosity_option(logger=logger, expose_value=False)
def train_analysis(
logdir: pathlib.Path,
- output: pathlib.Path,
+ output_folder: pathlib.Path,
) -> None: # numpydoc ignore=PR01
"""Create a plot for each metric in the training logs and saves them in a .pdf file."""
import matplotlib.pyplot as plt
@@ -138,11 +143,14 @@ def train_analysis(
from ..utils.tensorboard import scalars_to_dict
+ train_log_filename = "trainlog.pdf"
+ train_log_file = pathlib.Path(output_folder) / train_log_filename
+
data = scalars_to_dict(logdir)
- output.parent.mkdir(parents=True, exist_ok=True)
+ train_log_file.parent.mkdir(parents=True, exist_ok=True)
- with PdfPages(output) as pdf:
+ with PdfPages(train_log_file) as pdf:
for figure in create_figures(data):
pdf.savefig(figure)
plt.close(figure)