figure.py

'''Runs error analysis on score sets, outputs metrics and plots'''

from __future__ import division, print_function
from abc import ABCMeta, abstractmethod
import math
import sys
import os.path
import numpy
import click
import matplotlib
import matplotlib.pyplot as mpl
from matplotlib.backends.backend_pdf import PdfPages
from tabulate import tabulate
from .. import (far_threshold, plot, utils, ppndf)
import logging

LOGGER = logging.getLogger("bob.measure")


def check_list_value(values, desired_number, name, name2='systems'):
    if values is not None and len(values) != desired_number:
        if len(values) == 1:
            values = values * desired_number
        else:
            raise click.BadParameter(
                '#{} ({}) must be either 1 value or the same as '
                '#{} ({} values)'.format(name, values, name2, desired_number))

    return values


class MeasureBase(object):
    """Base class for metrics and plots.
    This abstract class define the framework to plot or compute metrics from a
    list of (positive, negative) scores tuples.

    Attributes
    ----------
    func_load:
        Function that is used to load the input files
    """
    __metaclass__ = ABCMeta  # for python 2.7 compatibility

    def __init__(self, ctx, scores, evaluation, func_load):
        """
        Parameters
        ----------
        ctx : :py:class:`dict`
            Click context dictionary.

        scores : :any:`list`:
            List of input files (e.g. dev-{1, 2, 3}, {dev,eval}-scores1
            {dev,eval}-scores2)
        eval : :py:class:`bool`
            True if eval data are used
        func_load : Function that is used to load the input files
        """
        self._scores = scores
        self._ctx = ctx
        self.func_load = func_load
        self._legends = ctx.meta.get('legends')
        self._eval = evaluation
        self._min_arg = ctx.meta.get('min_arg', 1)
        if len(scores) < 1 or len(scores) % self._min_arg != 0:
            raise click.BadParameter(
                'Number of argument must be a non-zero multiple of %d' % self._min_arg
            )
        self.n_systems = int(len(scores) / self._min_arg)
        if self._legends is not None and len(self._legends) < self.n_systems:
            raise click.BadParameter("Number of legends must be >= to the "
                                     "number of systems")

    def run(self):
        """ Generate outputs (e.g. metrics, files, pdf plots).
        This function calls abstract methods
        :func:`~bob.measure.script.figure.MeasureBase.init_process` (before
        loop), :py:func:`~bob.measure.script.figure.MeasureBase.compute`
        (in the loop iterating through the different
        systems) and :py:func:`~bob.measure.script.figure.MeasureBase.end_process`
        (after the loop).
        """
        # init matplotlib, log files, ...
        self.init_process()
        # iterates through the different systems and feed `compute`
        # with the dev (and eval) scores of each system
        # Note that more than one dev or eval scores score can be passed to
        # each system
        for idx in range(self.n_systems):
            # load scores for each system: get the corresponding arrays and
            # base-name of files
            input_scores, input_names = self._load_files(
                # Scores are given as followed:
                # SysA-dev SysA-eval ... SysA-XX  SysB-dev SysB-eval ... SysB-XX
                # ------------------------------  ------------------------------
                #   First set of `self._min_arg`     Second set of input files
                #     input files starting at               for SysB
                #    index idx * self._min_arg
                self._scores[idx * self._min_arg:(idx + 1) * self._min_arg]
            )
            self.compute(idx, input_scores, input_names)
        # setup final configuration, plotting properties, ...
        self.end_process()

    # protected functions that need to be overwritten
    def init_process(self):
        """ Called in :py:func:`~bob.measure.script.figure.MeasureBase`.run
        before iterating through the different systems.
        Should reimplemented in derived classes"""
        pass

    # Main computations are done here in the subclasses
    @abstractmethod
    def compute(self, idx, input_scores, input_names):
        """Compute metrics or plots from the given scores provided by
        :py:func:`~bob.measure.script.figure.MeasureBase.run`.
        Should reimplemented in derived classes

        Parameters
        ----------
        idx : :obj:`int`
            index of the system
        input_scores: :any:`list`
            list of scores returned by the loading function
        input_names: :any:`list`
            list of base names for the input file of the system
        """
        pass
        # structure of input is (vuln example):
        # if evaluation is provided
        # [ (dev_licit_neg, dev_licit_pos), (eval_licit_neg, eval_licit_pos),
        #   (dev_spoof_neg, dev_licit_pos), (eval_spoof_neg, eval_licit_pos)]
        # and if only dev:
        # [ (dev_licit_neg, dev_licit_pos), (dev_spoof_neg, dev_licit_pos)]

    # Things to do after the main iterative computations are done
    @abstractmethod
    def end_process(self):
        """ Called in :py:func:`~bob.measure.script.figure.MeasureBase`.run
        after iterating through the different systems.
        Should reimplemented in derived classes"""
        pass

    # common protected functions

    def _load_files(self, filepaths):
        ''' Load the input files and return the base names of the files

        Returns
        -------
            scores: :any:`list`:
                A list that contains the output of
                ``func_load`` for the given files
            basenames: :any:`list`:
                A list of basenames for the given files
        '''
        scores = []
        basenames = []
        for filename in filepaths:
            basenames.append(os.path.basename(filename).split(".")[0])
            scores.append(self.func_load(filename))
        return scores, basenames


class Metrics(MeasureBase):
    ''' Compute metrics from score files

    Attributes
    ----------
    log_file: str
        output stream
    '''

    def __init__(self, ctx, scores, evaluation, func_load,
                 names=('False Positive Rate', 'False Negative Rate',
                        'F1-score', 'Precision', 'Recall')):
        super(Metrics, self).__init__(ctx, scores, evaluation, func_load)
        self.names = names
        self._tablefmt = ctx.meta.get('tablefmt')
        self._criterion = ctx.meta.get('criterion')
        self._open_mode = ctx.meta.get('open_mode')
        self._thres = ctx.meta.get('thres')
        self._decimal = ctx.meta.get('decimal', 2)
        if self._thres is not None:
            if len(self._thres) == 1:
                self._thres = self._thres * self.n_systems
            elif len(self._thres) != self.n_systems:
                raise click.BadParameter(
                    '#thresholds must be the same as #systems (%d)'
                    % len(self.n_systems)
                )
        self._far = ctx.meta.get('far_value')
        self._log = ctx.meta.get('log')
        self.log_file = sys.stdout
        if self._log is not None:
            self.log_file = open(self._log, self._open_mode)

    def get_thres(self, criterion, dev_neg, dev_pos, far):
        return utils.get_thres(criterion, dev_neg, dev_pos, far)

    def _numbers(self, neg, pos, threshold, fta):
        from .. import (farfrr, precision_recall, f_score)
        # fpr and fnr
        fmr, fnmr = farfrr(neg, pos, threshold)
        hter = (fmr + fnmr) / 2.0
        far = fmr * (1 - fta)
        frr = fta + fnmr * (1 - fta)

        ni = neg.shape[0]  # number of impostors
        fm = int(round(fmr * ni))  # number of false accepts
        nc = pos.shape[0]  # number of clients
        fnm = int(round(fnmr * nc))  # number of false rejects

        # precision and recall
        precision, recall = precision_recall(neg, pos, threshold)

        # f_score
        f1_score = f_score(neg, pos, threshold, 1)
        return (fta, fmr, fnmr, hter, far, frr, fm, ni, fnm, nc, precision,
                recall, f1_score)

    def _strings(self, metrics):
        fta_str = "%.1f%%" % (100 * metrics[0])
        fmr_str = "%.1f%% (%d/%d)" % (100 * metrics[1], metrics[6], metrics[7])
        fnmr_str = "%.1f%% (%d/%d)" % (100 * metrics[2], metrics[8], metrics[9])
        far_str = "%.1f%%" % (100 * metrics[4])
        frr_str = "%.1f%%" % (100 * metrics[5])
        hter_str = "%.1f%%" % (100 * metrics[3])
        prec_str = "%.1f" % (metrics[10])
        recall_str = "%.1f" % (metrics[11])
        f1_str = "%.1f" % (metrics[12])

        return (fta_str, fmr_str, fnmr_str, far_str, frr_str, hter_str,
                prec_str, recall_str, f1_str)

    def _get_all_metrics(self, idx, input_scores, input_names):
        ''' Compute all metrics for dev and eval scores'''
        neg_list, pos_list, fta_list = utils.get_fta_list(input_scores)
        dev_neg, dev_pos, dev_fta = neg_list[0], pos_list[0], fta_list[0]
        dev_file = input_names[0]
        if self._eval:
            eval_neg, eval_pos, eval_fta = neg_list[1], pos_list[1], fta_list[1]

        threshold = self.get_thres(self._criterion, dev_neg, dev_pos, self._far) \
            if self._thres is None else self._thres[idx]

        title = self._legends[idx] if self._legends is not None else None
        if self._thres is None:
            far_str = ''
            if self._criterion == 'far' and self._far is not None:
                far_str = str(self._far)
            click.echo("[Min. criterion: %s %s] Threshold on Development set `%s`: %e"
                       % (self._criterion.upper(),
                          far_str, title or dev_file,
                          threshold),
                       file=self.log_file)
        else:
            click.echo("[Min. criterion: user provided] Threshold on "
                       "Development set `%s`: %e"
                       % (dev_file or title, threshold), file=self.log_file)

        res = []
        res.append(self._strings(self._numbers(
            dev_neg, dev_pos, threshold, dev_fta)))

        if self._eval:
            # computes statistics for the eval set based on the threshold a
            # priori
            res.append(self._strings(self._numbers(
                eval_neg, eval_pos, threshold, eval_fta)))
        else:
            res.append(None)

        return res

    def compute(self, idx, input_scores, input_names):
        ''' Compute metrics thresholds and tables (FPR, FNR, precision, recall,
        f1_score) for given system inputs'''
        dev_file = input_names[0]
        title = self._legends[idx] if self._legends is not None else None
        all_metrics = self._get_all_metrics(idx, input_scores, input_names)
        fta_dev = float(all_metrics[0][0].replace('%', ''))
        if fta_dev > 0.0:
            click.echo("NaNs scores (%s) were found in %s" %
                       (all_metrics[0][0], dev_file))
            LOGGER.warn("NaNs scores (%s) were found in %s", all_metrics[0][0],
                        dev_file)
        headers = [' ' or title, 'Development']
        rows = [[self.names[0], all_metrics[0][1]],
                [self.names[1], all_metrics[0][2]],
                [self.names[2], all_metrics[0][6]],
                [self.names[3], all_metrics[0][7]],
                [self.names[4], all_metrics[0][8]]]

        if self._eval:
            eval_file = input_names[1]
            fta_eval = float(all_metrics[1][0].replace('%', ''))
            if fta_eval > 0.0:
                click.echo("NaNs scores (%s) were found in %s" %
                           (all_metrics[1][0], eval_file))
                LOGGER.warn("NaNs scores (%s) were found in %s",
                            all_metrics[1][0], eval_file)
            # computes statistics for the eval set based on the threshold a
            # priori
            headers.append('Evaluation')
            rows[0].append(all_metrics[1][1])
            rows[1].append(all_metrics[1][2])
            rows[2].append(all_metrics[1][6])
            rows[3].append(all_metrics[1][7])
            rows[4].append(all_metrics[1][8])

        click.echo(tabulate(rows, headers, self._tablefmt), file=self.log_file)

    def end_process(self):
        ''' Close log file if needed'''
        if self._log is not None:
            self.log_file.close()


class MultiMetrics(Metrics):
    '''Computes average of metrics based on several protocols (cross
    validation)

    Attributes
    ----------
    log_file : str
        output stream
    names : tuple
        List of names for the metrics.
    '''

    def __init__(self, ctx, scores, evaluation, func_load,
                 names=('NaNs Rate', 'False Positive Rate',
                        'False Negative Rate', 'False Accept Rate',
                        'False Reject Rate', 'Half Total Error Rate')):
        super(MultiMetrics, self).__init__(
            ctx, scores, evaluation, func_load, names=names)

        self.headers = ['Methods'] + list(self.names)
        if self._eval:
            self.headers.insert(1, self.names[5] + ' (dev)')
        self.rows = []

    def _strings(self, metrics):
        ftam, fmrm, fnmrm, hterm, farm, frrm, _, _, _, _ = metrics.mean(axis=0)
        ftas, fmrs, fnmrs, hters, fars, frrs, _, _, _, _ = metrics.std(axis=0)
        fta_str = "%.1f%% (%.1f%%)" % (100 * ftam, 100 * ftas)
        fmr_str = "%.1f%% (%.1f%%)" % (100 * fmrm, 100 * fmrs)
        fnmr_str = "%.1f%% (%.1f%%)" % (100 * fnmrm, 100 * fnmrs)
        far_str = "%.1f%% (%.1f%%)" % (100 * farm, 100 * fars)
        frr_str = "%.1f%% (%.1f%%)" % (100 * frrm, 100 * frrs)
        hter_str = "%.1f%% (%.1f%%)" % (100 * hterm, 100 * hters)

        return fta_str, fmr_str, fnmr_str, far_str, frr_str, hter_str

    def compute(self, idx, input_scores, input_names):
        '''Computes the average of metrics over several protocols.'''
        neg_list, pos_list, fta_list = utils.get_fta_list(input_scores)
        step = 2 if self._eval else 1
        self._dev_metrics = []
        self._thresholds = []
        for i in range(0, len(input_scores), step):
            neg, pos, fta = neg_list[i], pos_list[i], fta_list[i]
            threshold = self.get_thres(self._criterion, neg, pos, self._far) \
                if self._thres is None else self._thres[idx]
            self._thresholds.append(threshold)
            self._dev_metrics.append(self._numbers(neg, pos, threshold, fta))
        self._dev_metrics = numpy.array(self._dev_metrics)

        if self._eval:
            self._eval_metrics = []
            for i in range(1, len(input_scores), step):
                neg, pos, fta = neg_list[i], pos_list[i], fta_list[i]
                threshold = self._thresholds[i // 2]
                self._eval_metrics.append(
                    self._numbers(neg, pos, threshold, fta))
            self._eval_metrics = numpy.array(self._eval_metrics)

        title = self._legends[idx] if self._legends is not None else None

        fta_str, fmr_str, fnmr_str, far_str, frr_str, hter_str = \
            self._strings(self._dev_metrics)

        if self._eval:
            self.rows.append([title, hter_str])
        else:
            self.rows.append([title, fta_str, fmr_str, fnmr_str,
                              far_str, frr_str, hter_str])

        if self._eval:
            # computes statistics for the eval set based on the threshold a
            # priori
            fta_str, fmr_str, fnmr_str, far_str, frr_str, hter_str = \
                self._strings(self._eval_metrics)

            self.rows[-1].extend([fta_str, fmr_str, fnmr_str,
                                  far_str, frr_str, hter_str])

    def end_process(self):
        click.echo(tabulate(self.rows, self.headers,
                            self._tablefmt), file=self.log_file)
        super(MultiMetrics, self).end_process()


class PlotBase(MeasureBase):
    ''' Base class for plots. Regroup several options and code
    shared by the different plots
    '''

    def __init__(self, ctx, scores, evaluation, func_load):
        super(PlotBase, self).__init__(ctx, scores, evaluation, func_load)
        self._output = ctx.meta.get('output')
        self._points = ctx.meta.get('points', 100)
        self._split = ctx.meta.get('split')
        self._axlim = ctx.meta.get('axlim')
        self._disp_legend = ctx.meta.get('disp_legend', True)
        self._legend_loc = ctx.meta.get('legend_loc')
        self._min_dig = None
        if 'min_far_value' in ctx.meta:
            self._min_dig = int(math.log10(ctx.meta['min_far_value']))
        elif self._axlim is not None and self._axlim[0] is not None:
            self._min_dig = int(math.log10(self._axlim[0])
                                if self._axlim[0] != 0 else 0)
        self._clayout = ctx.meta.get('clayout')
        self._far_at = ctx.meta.get('lines_at')
        self._trans_far_val = self._far_at
        if self._far_at is not None:
            self._eval_points = {line: [] for line in self._far_at}
            self._lines_val = []
        self._print_fn = ctx.meta.get('show_fn', True)
        self._x_rotation = ctx.meta.get('x_rotation')
        if 'style' in ctx.meta:
            mpl.style.use(ctx.meta['style'])
        self._nb_figs = 2 if self._eval and self._split else 1
        self._colors = utils.get_colors(self.n_systems)
        self._line_linestyles = ctx.meta.get('line_linestyles', False)
        self._linestyles = utils.get_linestyles(
            self.n_systems, self._line_linestyles)
        self._titles = ctx.meta.get('titles', []) * 2
        # for compatibility
        self._title = ctx.meta.get('title')
        if not self._titles and self._title is not None:
            self._titles = [self._title] * 2

        self._x_label = ctx.meta.get('x_label')
        self._y_label = ctx.meta.get('y_label')
        self._grid_color = 'silver'
        self._pdf_page = None
        self._end_setup_plot = True

    def init_process(self):
        ''' Open pdf and set axis font size if provided '''
        if not hasattr(matplotlib, 'backends'):
            matplotlib.use('pdf')

        self._pdf_page = self._ctx.meta['PdfPages'] if 'PdfPages'in \
            self._ctx.meta else PdfPages(self._output)

        for i in range(self._nb_figs):
            fs = self._ctx.meta.get('figsize')
            fig = mpl.figure(i + 1, figsize=fs)
            fig.set_constrained_layout(self._clayout)
            fig.clear()

    def end_process(self):
        ''' Set title, legend, axis labels, grid colors, save figures, drow
        lines and close pdf if needed '''
        # draw vertical lines
        if self._far_at is not None:
            for (line, line_trans) in zip(self._far_at, self._trans_far_val):
                mpl.figure(1)
                mpl.plot(
                    [line_trans, line_trans], [-100.0, 100.], "--",
                    color='black'
                )
                if self._eval and self._split:
                    mpl.figure(2)
                    x_values = [i for i, _ in self._eval_points[line]]
                    y_values = [j for _, j in self._eval_points[line]]
                    sort_indice = sorted(
                        range(len(x_values)), key=x_values.__getitem__
                    )
                    x_values = [x_values[i] for i in sort_indice]
                    y_values = [y_values[i] for i in sort_indice]
                    mpl.plot(x_values,
                             y_values, '--',
                             color='black')
        # only for plots
        if self._end_setup_plot:
            for i in range(self._nb_figs):
                fig = mpl.figure(i + 1)
                title = '' if not self._titles else self._titles[i]
                mpl.title(title if title.replace(' ', '') else '')
                mpl.xlabel(self._x_label)
                mpl.ylabel(self._y_label)
                mpl.grid(True, color=self._grid_color)
                if self._disp_legend:
                    mpl.legend(loc=self._legend_loc)
                self._set_axis()
                mpl.xticks(rotation=self._x_rotation)
                self._pdf_page.savefig(fig)

        # do not want to close PDF when running evaluate
        if 'PdfPages' in self._ctx.meta and \
           ('closef' not in self._ctx.meta or self._ctx.meta['closef']):
            self._pdf_page.close()

    # common protected functions

    def _label(self, base, name, idx):
        if self._legends is not None and len(self._legends) > idx:
            return self._legends[idx]
        if self.n_systems > 1:
            return base + (" %d (%s)" % (idx + 1, name))
        return base + (" (%s)" % name)

    def _set_axis(self):
        if self._axlim is not None:
            mpl.axis(self._axlim)


class Roc(PlotBase):
    ''' Handles the plotting of ROC'''

    def __init__(self, ctx, scores, evaluation, func_load):
        super(Roc, self).__init__(ctx, scores, evaluation, func_load)
        self._titles = self._titles or ['ROC dev.', 'ROC eval.']
        self._x_label = self._x_label or 'False Positive Rate'
        self._y_label = self._y_label or "1 - False Negative Rate"
        self._semilogx = ctx.meta.get('semilogx', True)
        best_legend = 'lower right' if self._semilogx else 'upper right'
        self._legend_loc = self._legend_loc or best_legend
        # custom defaults
        if self._axlim is None:
            self._axlim = [None, None, -0.05, 1.05]

    def compute(self, idx, input_scores, input_names):
        ''' Plot ROC for dev and eval data using
        :py:func:`bob.measure.plot.roc`'''
        neg_list, pos_list, _ = utils.get_fta_list(input_scores)
        dev_neg, dev_pos = neg_list[0], pos_list[0]
        dev_file = input_names[0]
        if self._eval:
            eval_neg, eval_pos = neg_list[1], pos_list[1]
            eval_file = input_names[1]

        mpl.figure(1)
        if self._eval:
            plot.roc_for_far(
                dev_neg, dev_pos,
                far_values=plot.log_values(self._min_dig or -4),
                CAR=self._semilogx,
                color=self._colors[idx], linestyle=self._linestyles[idx],
                label=self._label('dev', dev_file, idx)
            )
            if self._split:
                mpl.figure(2)

            linestyle = '--' if not self._split else self._linestyles[idx]
            plot.roc_for_far(
                eval_neg, eval_pos, linestyle=linestyle,
                far_values=plot.log_values(self._min_dig or -4),
                CAR=self._semilogx,
                color=self._colors[idx],
                label=self._label('eval.', eval_file, idx)
            )
            if self._far_at is not None:
                from .. import farfrr
                for line in self._far_at:
                    thres_line = far_threshold(dev_neg, dev_pos, line)
                    eval_fmr, eval_fnmr = farfrr(
                        eval_neg, eval_pos, thres_line)
                    eval_fnmr = 1 - eval_fnmr
                    mpl.scatter(eval_fmr, eval_fnmr, c=self._colors[idx], s=30)
                    self._eval_points[line].append((eval_fmr, eval_fnmr))
        else:
            plot.roc_for_far(
                dev_neg, dev_pos,
                far_values=plot.log_values(self._min_dig or -4),
                CAR=self._semilogx,
                color=self._colors[idx], linestyle=self._linestyles[idx],
                label=self._label('dev', dev_file, idx)
            )


class Det(PlotBase):
    ''' Handles the plotting of DET '''

    def __init__(self, ctx, scores, evaluation, func_load):
        super(Det, self).__init__(ctx, scores, evaluation, func_load)
        self._titles = self._titles or ['DET dev.', 'DET eval.']
        self._x_label = self._x_label or 'False Positive Rate (%)'
        self._y_label = self._y_label or 'False Negative Rate (%)'
        self._legend_loc = self._legend_loc or 'upper right'
        if self._far_at is not None:
            self._trans_far_val = [ppndf(float(k)) for k in self._far_at]
        # custom defaults here
        if self._x_rotation is None:
            self._x_rotation = 50

        if self._axlim is None:
            self._axlim = [0.01, 99, 0.01, 99]

        if self._min_dig is not None:
            self._axlim[0] = math.pow(10, self._min_dig) * 100

    def compute(self, idx, input_scores, input_names):
        ''' Plot DET for dev and eval data using
        :py:func:`bob.measure.plot.det`'''
        neg_list, pos_list, _ = utils.get_fta_list(input_scores)
        dev_neg, dev_pos = neg_list[0], pos_list[0]
        dev_file = input_names[0]
        if self._eval:
            eval_neg, eval_pos = neg_list[1], pos_list[1]
            eval_file = input_names[1]

        mpl.figure(1)
        if self._eval and eval_neg is not None:
            plot.det(
                dev_neg, dev_pos, self._points, color=self._colors[idx],
                linestyle=self._linestyles[idx],
                label=self._label('development', dev_file, idx)
            )
            if self._split:
                mpl.figure(2)
            linestyle = '--' if not self._split else self._linestyles[idx]
            plot.det(
                eval_neg, eval_pos, self._points, color=self._colors[idx],
                linestyle=linestyle,
                label=self._label('eval.', eval_file, idx)
            )
            if self._far_at is not None:
                from .. import farfrr
                for line in self._far_at:
                    thres_line = far_threshold(dev_neg, dev_pos, line)
                    eval_fmr, eval_fnmr = farfrr(
                        eval_neg, eval_pos, thres_line)
                    eval_fmr, eval_fnmr = ppndf(eval_fmr), ppndf(eval_fnmr)
                    mpl.scatter(eval_fmr, eval_fnmr, c=self._colors[idx], s=30)
                    self._eval_points[line].append((eval_fmr, eval_fnmr))
        else:
            plot.det(
                dev_neg, dev_pos, self._points, color=self._colors[idx],
                linestyle=self._linestyles[idx],
                label=self._label('development', dev_file, idx)
            )

    def _set_axis(self):
        plot.det_axis(self._axlim)


class Epc(PlotBase):
    ''' Handles the plotting of EPC '''

    def __init__(self, ctx, scores, evaluation, func_load, hter='HTER'):
        super(Epc, self).__init__(ctx, scores, evaluation, func_load)
        if self._min_arg != 2:
            raise click.UsageError("EPC requires dev. and eval. score files")
        self._titles = self._titles or ['EPC'] * 2
        self._x_label = self._x_label or r'$\alpha$'
        self._y_label = self._y_label or hter + ' (%)'
        self._legend_loc = self._legend_loc or 'upper center'
        self._eval = True  # always eval data with EPC
        self._split = False
        self._nb_figs = 1
        self._far_at = None

    def compute(self, idx, input_scores, input_names):
        ''' Plot EPC using :py:func:`bob.measure.plot.epc` '''
        neg_list, pos_list, _ = utils.get_fta_list(input_scores)
        dev_neg, dev_pos = neg_list[0], pos_list[0]
        dev_file = input_names[0]
        if self._eval:
            eval_neg, eval_pos = neg_list[1], pos_list[1]
            eval_file = input_names[1]

        plot.epc(
            dev_neg, dev_pos, eval_neg, eval_pos, self._points,
            color=self._colors[idx], linestyle=self._linestyles[idx],
            label=self._label(
                'curve', dev_file + "_" + eval_file, idx
            )
        )


class Hist(PlotBase):
    ''' Functional base class for histograms'''

    def __init__(self, ctx, scores, evaluation, func_load, nhist_per_system=2):
        super(Hist, self).__init__(ctx, scores, evaluation, func_load)
        self._nbins = ctx.meta.get('n_bins', ['doane'])
        self._nhist_per_system = nhist_per_system
        self._nbins = check_list_value(
            self._nbins, nhist_per_system, 'n_bins',
            'histograms')
        self._thres = ctx.meta.get('thres')
        self._thres = check_list_value(
            self._thres, self.n_systems, 'thresholds')
        self._criterion = ctx.meta.get('criterion')
        # no vertical (threshold) is displayed
        self._no_line = ctx.meta.get('no_line', False)
        # subplot grid
        self._nrows = ctx.meta.get('n_row', 1)
        self._ncols = ctx.meta.get('n_col', 1)
        # do not display dev histo
        self._hide_dev = ctx.meta.get('hide_dev', False)
        if self._hide_dev and not self._eval:
            raise click.BadParameter(
                "You can only use --hide-dev along with --eval")
        # dev hist are displayed next to eval hist
        self._nrows *= 1 if self._hide_dev or not self._eval else 2
        self._nlegends = ctx.meta.get('legends_ncol', 3)
        self._legend_loc = self._legend_loc or 'upper center'
        # number of subplot on one page
        self._step_print = int(self._nrows * self._ncols)
        self._title_base = 'Scores'
        self._y_label = 'Probability density'
        self._x_label = 'Score values'
        self._end_setup_plot = False
        # overide _titles of PlotBase
        self._titles = ctx.meta.get('titles', []) * 2

    def compute(self, idx, input_scores, input_names):
        ''' Draw histograms of negative and positive scores.'''
        dev_neg, dev_pos, eval_neg, eval_pos, threshold = \
            self._get_neg_pos_thres(idx, input_scores, input_names)
        # keep id of the current system
        sys = idx
        # if the id of the current system does not match the id of the plot, 
        # change it
        if not self._hide_dev and self._eval:
            row = int(idx / self._ncols) * 2
            col = idx % self._ncols
            idx = col + self._ncols * row

        if not self._hide_dev or not self._eval:
            self._print_subplot(idx, sys, dev_neg, dev_pos, threshold,
                                not self._no_line, False)

        if self._eval:
            idx += self._ncols if not self._hide_dev else 0
            self._print_subplot(idx, sys, eval_neg, eval_pos, threshold,
                                not self._no_line, True)

    def _print_subplot(self, idx, sys, neg, pos, threshold, draw_line, evaluation):
        ''' print a subplot for the given score and subplot index'''
        n = idx % self._step_print
        col = n % self._ncols
        sub_plot_idx = n + 1
        axis = mpl.subplot(self._nrows, self._ncols, sub_plot_idx)
        self._setup_hist(neg, pos)
        if col == 0:
            axis.set_ylabel(self._y_label)
        # systems per page
        sys_per_page = self._step_print / (1 if self._hide_dev or not
                                           self._eval else 2)
        # rest to be printed
        sys_idx = sys % sys_per_page
        rest_print = self.n_systems - int(sys / sys_per_page) * sys_per_page
        # lower histo only
        is_lower = evaluation or not self._eval
        if is_lower and sys_idx + self._ncols >= min(sys_per_page, rest_print):
            axis.set_xlabel(self._x_label)
        dflt_title = "Eval. scores" if evaluation else "Dev. scores"
        if self.n_systems == 1 and (not self._eval or self._hide_dev):
            dflt_title = " "
        add = self.n_systems if is_lower else 0
        axis.set_title(self._get_title(sys + add, dflt_title))
        label = "%s threshold%s" % (
            '' if self._criterion is None else
            self._criterion.upper(), ' (dev)' if self._eval else ''
        )
        if draw_line:
            self._lines(threshold, label, neg, pos, idx)


        # if it was the last subplot of the page or the last subplot
        # to display, save figure
        if self._step_print == sub_plot_idx or (is_lower and sys ==
                                                self.n_systems - 1):
            # print legend on the page
            self.plot_legends()
            mpl.tight_layout()
            self._pdf_page.savefig(mpl.gcf(), bbox_inches='tight')
            mpl.clf()
            mpl.figure()

    def _get_title(self, idx, dflt=None):
        ''' Get the histo title for the given idx'''
        title = self._titles[idx] if self._titles is not None \
            and idx < len(self._titles) else dflt
        title = title or self._title_base
        title = '' if title is not None and not title.replace(
            ' ', '') else title
        return title or ''

    def plot_legends(self):
        ''' Print legend on current page'''
        lines = []
        labels = []
        for ax in mpl.gcf().get_axes():
            ali, ala = ax.get_legend_handles_labels()
            # avoid duplicates in legend
            for li, la in zip(ali, ala):
                if la not in labels:
                    lines.append(li)
                    labels.append(la)

        if self._disp_legend:
            mpl.gcf().legend(
                lines, labels, loc=self._legend_loc, fancybox=True,
                framealpha=0.5, ncol=self._nlegends,
                bbox_to_anchor=(0.55, 1.1),
            )

    def _get_neg_pos_thres(self, idx, input_scores, input_names):
        ''' Get scores and threshod for the given system at index idx'''
        neg_list, pos_list, _ = utils.get_fta_list(input_scores)
        length = len(neg_list)
        # lists returned by get_fta_list contains all the following items:
        # for bio or measure without eval:
        #   [dev]
        # for vuln with {licit,spoof} with eval:
        #   [dev, eval]
        # for vuln with {licit,spoof} without eval:
        #   [licit_dev, spoof_dev]
        # for vuln with {licit,spoof} with eval:
        #   [licit_dev, licit_eval, spoof_dev, spoof_eval]
        step = 2 if self._eval else 1
        # can have several files for one system
        dev_neg = [neg_list[x] for x in range(0, length, step)]
        dev_pos = [pos_list[x] for x in range(0, length, step)]
        eval_neg = eval_pos = None
        if self._eval:
            eval_neg = [neg_list[x] for x in range(1, length, step)]
            eval_pos = [pos_list[x] for x in range(1, length, step)]

        threshold = utils.get_thres(
            self._criterion, dev_neg[0], dev_pos[0]
        ) if self._thres is None else self._thres[idx]
        return dev_neg, dev_pos, eval_neg, eval_pos, threshold

    def _density_hist(self, scores, n, **kwargs):
        ''' Plots one density histo'''
        n, bins, patches = mpl.hist(
            scores, density=True,
            bins=self._nbins[n],
            **kwargs
        )
        return (n, bins, patches)

    def _lines(self, threshold, label=None, neg=None, pos=None,
               idx=None, **kwargs):
        ''' Plots vertical line at threshold '''
        label = label or 'Threshold'
        kwargs.setdefault('color', 'C3')
        kwargs.setdefault('linestyle', '--')
        kwargs.setdefault('label', label)
        # plot a vertical threshold line
        mpl.axvline(x=threshold, ymin=0, ymax=1, **kwargs)

    def _setup_hist(self, neg, pos):
        ''' This function can be overwritten in derived classes

        Plots all the density histo required in one plot. Here negative and
        positive scores densities.
        '''
        self._density_hist(
            neg[0], n=0,
            label='Negatives', alpha=0.5, color='C3'
        )
        self._density_hist(
            pos[0], n=1,
            label='Positives', alpha=0.5, color='C0'
        )