evaluate.py

# SPDX-FileCopyrightText: Copyright © 2023 Idiap Research Institute <contact@idiap.ch>
#
# SPDX-License-Identifier: GPL-3.0-or-later

import pathlib
import typing

import click

from clapper.click import ResourceOption, verbosity_option
from clapper.logging import setup

from ...models.typing import SaliencyMapAlgorithm
from ..click import ConfigCommand

# avoids X11/graphical desktop requirement when creating plots
__import__("matplotlib").use("agg")

logger = setup(__name__.split(".")[0], format="%(levelname)s: %(message)s")


@click.command(
    entry_point_group="ptbench.config",
    cls=ConfigCommand,
    epilog="""Examples:

1. Tabulates and generates plots for two saliency map algorithms:

   .. code:: sh

      ptbench saliency evaluate -vv -e gradcam path/to/gradcam-completeness.json path/to/gradcam-interpretability.json -e gradcam++ path/to/gradcam++-completeness.json path/to/gradcam++-interpretability.json
""",
)
@click.option(
    "--entry",
    "-e",
    required=True,
    multiple=True,
    help=f"ENTRY is a triplet containing the algorithm name, the path to the "
    f"scores issued from the completness analysis (``ptbench "
    f"saliency-completness``) and scores issued from the interpretability "
    f"analysis (``ptbench saliency-interpretability``), both in JSON format. "
    f"Paths to score files must exist before the program is called. Valid values "
    f"for saliency map algorithms are "
    f"{'|'.join(typing.get_args(SaliencyMapAlgorithm))}",
    type=(
        click.Choice(
            typing.get_args(SaliencyMapAlgorithm), case_sensitive=False
        ),
        click.Path(
            exists=True,
            file_okay=True,
            dir_okay=False,
            path_type=pathlib.Path,
        ),
        click.Path(
            exists=True,
            file_okay=True,
            dir_okay=False,
            path_type=pathlib.Path,
        ),
    ),
    cls=ResourceOption,
)
@click.option(
    "--output-folder",
    "-o",
    help="Path where to store the analysis result (created if does not exist)",
    required=False,
    default="results",
    type=click.Path(file_okay=False, dir_okay=True, path_type=pathlib.Path),
    cls=ResourceOption,
)
@verbosity_option(logger=logger, expose_value=False)
def evaluate(
    entry,
    output_folder,
    **_,  # ignored
) -> None:
    """Calculates summary statistics for a saliency map algorithm."""
    import json

    from matplotlib.backends.backend_pdf import PdfPages

    from ...engine.saliency.evaluator import run, summary_table

    summary = {
        algo: run(algo, json.load(complet.open()), json.load(interp.open()))
        for algo, complet, interp in entry
    }
    table = summary_table(summary, "rst")
    click.echo(summary)

    if output_folder is not None:
        output_folder.mkdir(parents=True, exist_ok=True)

        table_path = output_folder / "summary.rst"

        logger.info(f"Saving summary table at `{table_path}`...")
        with table_path.open("w") as f:
            f.write(table)

        figure_path = output_folder / "plots.pdf"
        logger.info(f"Saving figures at `{figure_path}`...")

        with PdfPages(figure_path) as pdf:
            for dataset in summary.keys():
                pdf.savefig(summary[dataset]["aopc-combined"]["plot"])
                pdf.savefig(summary[dataset]["proportional-energy"]["plot"])
                pdf.savefig(
                    summary[dataset]["road-weighted-proportional-energy"][
                        "plot"
                    ]
                )