figure.py 26.8 KB
Newer Older
1 2 3 4
'''Runs error analysis on score sets, outputs metrics and plots'''

import click
import numpy as np
5
import matplotlib.pyplot as mpl
Amir MOHAMMADI's avatar
nit  
Amir MOHAMMADI committed
6
import bob.measure.script.figure as measure_figure
7
from tabulate import tabulate
Amir MOHAMMADI's avatar
nit  
Amir MOHAMMADI committed
8
from bob.measure.utils import get_fta_list
9
from bob.measure import (
10
    far_threshold, eer_threshold, min_hter_threshold, farfrr, epc, ppndf
11
)
12
from bob.measure.plot import (det, det_axis)
13 14 15 16
from . import error_utils

ALL_CRITERIA = ('bpcer20', 'eer', 'min-hter')

17

18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52
def calc_threshold(method, neg, pos):
    """Calculates the threshold based on the given method.
    The scores should be sorted!

    Parameters
    ----------
    method : str
        One of ``bpcer201``, ``eer``, ``min-hter``.
    neg : array_like
        The negative scores. They should be sorted!
    pos : array_like
        The positive scores. They should be sorted!

    Returns
    -------
    float
        The calculated threshold.

    Raises
    ------
    ValueError
        If method is unknown.
    """
    method = method.lower()
    if method == 'bpcer20':
        threshold = far_threshold(neg, pos, 0.05, True)
    elif method == 'eer':
        threshold = eer_threshold(neg, pos, True)
    elif method == 'min-hter':
        threshold = min_hter_threshold(neg, pos, True)
    else:
        raise ValueError("Unknown threshold criteria: {}".format(method))

    return threshold

53

54 55 56 57 58
class Metrics(measure_figure.Metrics):
    def __init__(self, ctx, scores, evaluation, func_load):
        super(Metrics, self).__init__(ctx, scores, evaluation, func_load)

    ''' Compute metrics from score files'''
Amir MOHAMMADI's avatar
nit  
Amir MOHAMMADI committed
59

60
    def compute(self, idx, input_scores, input_names):
61
        ''' Compute metrics for the given criteria'''
62 63 64 65 66 67
        neg_list, pos_list, _ = get_fta_list(input_scores)
        dev_neg, dev_pos = neg_list[0], pos_list[0]
        dev_file = input_names[0]
        if self._eval:
            eval_neg, eval_pos = neg_list[1], pos_list[1]
            eval_file = input_names[1]
68

69
        title = self._legends[idx] if self._legends is not None else None
70
        headers = ['' or title, 'Development %s' % dev_file]
71
        if self._eval:
72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92
            headers.append('Eval. % s' % eval_file)
        for m in ALL_CRITERIA:
            raws = []
            threshold = calc_threshold(m, dev_neg, dev_pos)
            click.echo("\nThreshold of %f selected with the %s criteria" % (
                threshold, m))
            apcer, bpcer = farfrr(dev_neg, dev_pos, threshold)
            raws.append(['BPCER20', '{:>5.1f}%'.format(apcer * 100)])
            raws.append(['EER', '{:>5.1f}%'.format(bpcer * 100)])
            raws.append(['min-HTER', '{:>5.1f}%'.format((apcer + bpcer) * 50)])
            if self._eval and eval_neg is not None:
                apcer, bpcer = farfrr(eval_neg, eval_pos, threshold)
                raws[0].append('{:>5.1f}%'.format(apcer * 100))
                raws[1].append('{:>5.1f}%'.format(bpcer * 100))
                raws[2].append('{:>5.1f}%'.format((apcer + bpcer) * 50))

            click.echo(
                tabulate(raws, headers, self._tablefmt),
                file=self.log_file
            )

93

94 95 96 97 98
class MetricsVuln(measure_figure.Metrics):
    def __init__(self, ctx, scores, evaluation, func_load):
        super(MetricsVuln, self).__init__(ctx, scores, evaluation, func_load)

    ''' Compute metrics from score files'''
Amir MOHAMMADI's avatar
nit  
Amir MOHAMMADI committed
99

100 101 102 103 104 105
    def compute(self, idx, input_scores, input_names):
        ''' Compute metrics for the given criteria'''
        neg_list, pos_list, _ = get_fta_list(input_scores)
        dev_neg, dev_pos = neg_list[0], pos_list[0]
        criter = self._criterion or 'eer'
        threshold = calc_threshold(criter, dev_neg, dev_pos) \
Amir MOHAMMADI's avatar
nit  
Amir MOHAMMADI committed
106
            if self._thres is None else self._thres[idx]
107 108 109 110 111 112
        far, frr = farfrr(neg_list[1], pos_list[1], threshold)
        iapmr, _ = farfrr(neg_list[3], pos_list[1], threshold)
        title = self._legends[idx] if self._legends is not None else None
        headers = ['' or title, '%s (threshold=%.2g)' %
                   (criter.upper(), threshold)]
        rows = []
Amir MOHAMMADI's avatar
nit  
Amir MOHAMMADI committed
113 114 115 116
        rows.append(['FMR (%)', '{:>5.1f}%'.format(100 * far)])
        rows.append(['FMNR (%)', '{:>5.1f}%'.format(frr * 100)])
        rows.append(['HTER (%)', '{:>5.1f}%'.format(50 * (far + frr))])
        rows.append(['IAPMR (%)', '{:>5.1f}%'.format(100 * iapmr)])
117 118 119 120 121 122
        click.echo(
            tabulate(rows, headers, self._tablefmt),
            file=self.log_file
        )


123 124 125 126 127 128
class HistPad(measure_figure.Hist):
    ''' Histograms for PAD '''

    def _setup_hist(self, neg, pos):
        self._title_base = 'PAD'
        self._density_hist(
129
            pos[0], n=0, label='Bona Fide', color='C1'
130 131
        )
        self._density_hist(
132
            neg[0], n=1, label='Presentation attack', alpha=0.4, color='C7',
133
            hatch='\\\\'
134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157
        )


def _iapmr_dot(threshold, iapmr, real_data, **kwargs):
    # plot a dot on threshold versus IAPMR line and show IAPMR as a number
    axlim = mpl.axis()
    mpl.plot(threshold, 100. * iapmr, 'o', color='C3', **kwargs)
    if not real_data:
        mpl.annotate(
            'IAPMR at\noperating point',
            xy=(threshold, 100. * iapmr),
            xycoords='data',
            xytext=(0.85, 0.6),
            textcoords='axes fraction',
            color='black',
            size='large',
            arrowprops=dict(facecolor='black', shrink=0.05, width=2),
            horizontalalignment='center',
            verticalalignment='top',
        )
    else:
        mpl.text(threshold + (threshold - axlim[0]) / 12, 100. * iapmr,
                 '%.1f%%' % (100. * iapmr,), color='C3')

158

159 160 161 162 163 164
def _iapmr_line_plot(scores, n_points=100, **kwargs):
    axlim = mpl.axis()
    step = (axlim[1] - axlim[0]) / float(n_points)
    thres = [(k * step) + axlim[0] for k in range(2, n_points - 1)]
    mix_prob_y = []
    for k in thres:
165
        mix_prob_y.append(100. * error_utils.calc_pass_rate(k, scores))
166 167 168

    mpl.plot(thres, mix_prob_y, label='IAPMR', color='C3', **kwargs)

169

170 171 172 173
def _iapmr_plot(scores, threshold, iapmr, real_data, **kwargs):
    _iapmr_dot(threshold, iapmr, real_data, **kwargs)
    _iapmr_line_plot(scores, n_points=100, **kwargs)

174

175 176 177 178 179 180
class HistVuln(measure_figure.Hist):
    ''' Histograms for vulnerability '''

    def _setup_hist(self, neg, pos):
        self._title_base = 'Vulnerability'
        self._density_hist(
181
            pos[0], n=0, label='Genuine', color='C2'
182 183
        )
        self._density_hist(
184
            neg[0], n=1, label='Zero-effort impostors', alpha=0.8, color='C0'
185 186
        )
        self._density_hist(
187
            neg[1], n=2, label='Presentation attack', alpha=0.4, color='C7',
188
            hatch='\\\\'
189 190
        )

191
    def _lines(self, threshold, label, neg, pos, idx, **kwargs):
192
        if 'iapmr_line' not in self._ctx.meta or self._ctx.meta['iapmr_line']:
Amir MOHAMMADI's avatar
nit  
Amir MOHAMMADI committed
193
            # plot vertical line
194
            super(HistVuln, self)._lines(threshold, label, neg, pos, idx)
195

Amir MOHAMMADI's avatar
nit  
Amir MOHAMMADI committed
196
            # plot iapmr_line
197 198 199 200 201
            iapmr, _ = farfrr(neg[1], pos[0], threshold)
            ax2 = mpl.twinx()
            # we never want grid lines on axis 2
            ax2.grid(False)
            real_data = True if 'real_data' not in self._ctx.meta else \
Amir MOHAMMADI's avatar
nit  
Amir MOHAMMADI committed
202
                self._ctx.meta['real_data']
203
            _iapmr_plot(neg[1], threshold, iapmr, real_data=real_data)
204 205
            n = idx % self._step_print
            col = n % self._ncols
Amir MOHAMMADI's avatar
nit  
Amir MOHAMMADI committed
206 207
            rest_print = self.n_systems - \
                int(idx / self._step_print) * self._step_print
208 209
            if col == self._ncols - 1 or n == rest_print - 1:
                ax2.set_ylabel("IAPMR (%)", color='C3')
210 211 212 213
            ax2.tick_params(axis='y', colors='red')
            ax2.yaxis.label.set_color('red')
            ax2.spines['right'].set_color('red')

214

215 216
class PadPlot(measure_figure.PlotBase):
    '''Base class for PAD plots'''
Amir MOHAMMADI's avatar
nit  
Amir MOHAMMADI committed
217

218 219
    def __init__(self, ctx, scores, evaluation, func_load):
        super(PadPlot, self).__init__(ctx, scores, evaluation, func_load)
220
        mpl.rcParams['figure.constrained_layout.use'] = self._clayout
221 222 223

    def end_process(self):
        '''Close pdf '''
Amir MOHAMMADI's avatar
nit  
Amir MOHAMMADI committed
224
        # do not want to close PDF when running evaluate
225 226 227 228 229
        if 'PdfPages' in self._ctx.meta and \
           ('closef' not in self._ctx.meta or self._ctx.meta['closef']):
            self._pdf_page.close()

    def _plot_legends(self):
Amir MOHAMMADI's avatar
nit  
Amir MOHAMMADI committed
230
        # legends for all axes
231 232 233 234 235 236
        lines = []
        labels = []
        for ax in mpl.gcf().get_axes():
            li, la = ax.get_legend_handles_labels()
            lines += li
            labels += la
237 238 239 240
        if self._disp_legend:
            mpl.gca().legend(lines, labels, loc=self._legend_loc,
                             fancybox=True, framealpha=0.5)

241 242 243

class Epc(PadPlot):
    ''' Handles the plotting of EPC '''
Amir MOHAMMADI's avatar
nit  
Amir MOHAMMADI committed
244

245 246 247
    def __init__(self, ctx, scores, evaluation, func_load):
        super(Epc, self).__init__(ctx, scores, evaluation, func_load)
        self._iapmr = True if 'iapmr' not in self._ctx.meta else \
Amir MOHAMMADI's avatar
nit  
Amir MOHAMMADI committed
248
            self._ctx.meta['iapmr']
249 250 251 252
        self._title = self._title or ('EPC and IAPMR' if self._iapmr else
                                      'EPC')
        self._x_label = self._x_label or r"Weight $\beta$"
        self._y_label = self._y_label or "WER (%)"
Amir MOHAMMADI's avatar
nit  
Amir MOHAMMADI committed
253
        self._eval = True  # always eval data with EPC
254 255 256
        self._split = False
        self._nb_figs = 1

257 258 259 260 261
        if self._min_arg != 4:
            raise click.BadParameter("You must provide 4 scores files:{licit,"
                                     "spoof}/{dev,eval}")

    def compute(self, idx, input_scores, input_names):
262
        ''' Plot EPC for PAD'''
263 264 265 266 267
        licit_dev_neg = input_scores[0][0]
        licit_dev_pos = input_scores[0][1]
        licit_eval_neg = input_scores[1][0]
        licit_eval_pos = input_scores[1][1]
        spoof_eval_neg = input_scores[3][0]
268
        mpl.gcf().clear()
269
        epc_baseline = epc(
270 271 272 273 274 275 276
            licit_dev_neg, licit_dev_pos, licit_eval_neg,
            licit_eval_pos, 100
        )
        mpl.plot(
            epc_baseline[:, 0], [100. * k for k in epc_baseline[:, 1]],
            color='C0',
            label=self._label(
277
                'WER', '%s-%s' % (input_names[0], input_names[1]), idx
278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298
            ),
            linestyle='-'
        )
        mpl.xlabel(self._x_label)
        mpl.ylabel(self._y_label)
        if self._iapmr:
            mix_prob_y = []
            for k in epc_baseline[:, 2]:
                prob_attack = sum(
                    1 for i in spoof_eval_neg if i >= k
                ) / float(spoof_eval_neg.size)
                mix_prob_y.append(100. * prob_attack)

            mpl.gca().set_axisbelow(True)
            prob_ax = mpl.gca().twinx()
            mpl.plot(
                epc_baseline[:, 0],
                mix_prob_y,
                color='C3',
                linestyle='-',
                label=self._label(
299
                    'IAPMR', '%s-%s' % (input_names[0], input_names[1]), idx
300 301 302 303 304 305 306 307 308
                )
            )
            prob_ax.set_yticklabels(prob_ax.get_yticks())
            prob_ax.tick_params(axis='y', colors='red')
            prob_ax.yaxis.label.set_color('red')
            prob_ax.spines['right'].set_color('red')
            ylabels = prob_ax.get_yticks()
            prob_ax.yaxis.set_ticklabels(["%.0f" % val for val in ylabels])
            prob_ax.set_axisbelow(True)
309
        title = self._legends[idx] if self._legends is not None else self._title
310 311
        if title.replace(' ', ''):
            mpl.title(title)
Amir MOHAMMADI's avatar
nit  
Amir MOHAMMADI committed
312
        # legends for all axes
313 314 315 316
        self._plot_legends()
        mpl.xticks(rotation=self._x_rotation)
        self._pdf_page.savefig(mpl.gcf())

317

318 319
class Epsc(PadPlot):
    ''' Handles the plotting of EPSC '''
Amir MOHAMMADI's avatar
nit  
Amir MOHAMMADI committed
320

321 322 323 324
    def __init__(self, ctx, scores, evaluation, func_load,
                 criteria, var_param, fixed_param):
        super(Epsc, self).__init__(ctx, scores, evaluation, func_load)
        self._iapmr = False if 'iapmr' not in self._ctx.meta else \
Amir MOHAMMADI's avatar
nit  
Amir MOHAMMADI committed
325
            self._ctx.meta['iapmr']
326
        self._wer = True if 'wer' not in self._ctx.meta else \
Amir MOHAMMADI's avatar
nit  
Amir MOHAMMADI committed
327
            self._ctx.meta['wer']
328 329 330
        self._criteria = 'eer' if criteria is None else criteria
        self._var_param = "omega" if var_param is None else var_param
        self._fixed_param = 0.5 if fixed_param is None else fixed_param
Amir MOHAMMADI's avatar
nit  
Amir MOHAMMADI committed
331
        self._eval = True  # always eval data with EPC
332 333
        self._split = False
        self._nb_figs = 1
334 335 336 337 338
        self._title = ''

        if self._min_arg != 4:
            raise click.BadParameter("You must provide 4 scores files:{licit,"
                                     "spoof}/{dev,eval}")
339

340
    def compute(self, idx, input_scores, input_names):
341
        ''' Plot EPSC for PAD'''
342 343 344 345 346 347 348 349
        licit_dev_neg = input_scores[0][0]
        licit_dev_pos = input_scores[0][1]
        licit_eval_neg = input_scores[1][0]
        licit_eval_pos = input_scores[1][1]
        spoof_dev_neg = input_scores[2][0]
        spoof_dev_pos = input_scores[2][1]
        spoof_eval_neg = input_scores[3][0]
        spoof_eval_pos = input_scores[3][1]
350
        title = self._legends[idx] if self._legends is not None else None
351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370

        mpl.gcf().clear()
        points = 10

        if self._var_param == 'omega':
            omega, beta, thrs = error_utils.epsc_thresholds(
                licit_dev_neg,
                licit_dev_pos,
                spoof_dev_neg,
                spoof_dev_pos,
                points=points,
                criteria=self._criteria,
                beta=self._fixed_param)
        else:
            omega, beta, thrs = error_utils.epsc_thresholds(
                licit_dev_neg,
                licit_dev_pos,
                spoof_dev_neg,
                spoof_dev_pos,
                points=points,
Amir MOHAMMADI's avatar
nit  
Amir MOHAMMADI committed
371
                criteria=self._criteria,
372 373 374 375 376 377 378
                omega=self._fixed_param
            )

        errors = error_utils.all_error_rates(
            licit_eval_neg, licit_eval_pos, spoof_eval_neg,
            spoof_eval_pos, thrs, omega, beta
        )  # error rates are returned in a list in the
Amir MOHAMMADI's avatar
nit  
Amir MOHAMMADI committed
379
        # following order: frr, far, IAPMR, far_w, wer_w
380 381 382 383

        ax1 = mpl.subplot(
            111
        )  # EPC like curves for FVAS fused scores for weighted error rates
Amir MOHAMMADI's avatar
nit  
Amir MOHAMMADI committed
384
        # between the negatives (impostors and Presentation attacks)
385 386 387 388 389 390 391 392
        if self._wer:
            if self._var_param == 'omega':
                mpl.plot(
                    omega,
                    100. * errors[4].flatten(),
                    color='C0',
                    linestyle='-',
                    label=r"WER$_{\omega,\beta}$")
393
                mpl.xlabel(self._x_label or r"Weight $\omega$")
394 395 396 397 398 399 400
            else:
                mpl.plot(
                    beta,
                    100. * errors[4].flatten(),
                    color='C0',
                    linestyle='-',
                    label=r"WER$_{\omega,\beta}$")
401 402
                mpl.xlabel(self._x_label or r"Weight $\beta$")
            mpl.ylabel(self._y_label or r"WER$_{\omega,\beta}$ (%)")
403 404 405 406 407 408 409 410 411 412 413 414 415

        if self._iapmr:
            axis = mpl.gca()
            if self._wer:
                axis = mpl.twinx()
                axis.grid(False)
            if self._var_param == 'omega':
                mpl.plot(
                    omega,
                    100. * errors[2].flatten(),
                    color='C3',
                    linestyle='-',
                    label='IAPMR')
416
                mpl.xlabel(self._x_label or r"Weight $\omega$")
417 418 419 420 421 422 423
            else:
                mpl.plot(
                    beta,
                    100. * errors[2].flatten(),
                    color='C3',
                    linestyle='-',
                    label='IAPMR')
424 425
                mpl.xlabel(self._x_label or r"Weight $\beta$")
            mpl.ylabel(self._y_label or r"IAPMR  (%)")
426 427 428 429 430 431 432
            if self._wer:
                axis.set_yticklabels(axis.get_yticks())
                axis.tick_params(axis='y', colors='red')
                axis.yaxis.label.set_color('red')
                axis.spines['right'].set_color('red')

        if self._var_param == 'omega':
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
433
            if title is not None and title.replace(' ', ''):
Amir MOHAMMADI's avatar
nit  
Amir MOHAMMADI committed
434
                mpl.title(title or (r"EPSC with $\beta$ = %.2f" %
435
                                    self._fixed_param))
436
        else:
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
437
            if title is not None and title.replace(' ', ''):
Amir MOHAMMADI's avatar
nit  
Amir MOHAMMADI committed
438
                mpl.title(title or (r"EPSC with $\omega$ = %.2f" %
439
                                    self._fixed_param))
440 441 442 443 444 445

        mpl.grid()
        self._plot_legends()
        ax1.set_xticklabels(ax1.get_xticks())
        ax1.set_yticklabels(ax1.get_yticks())
        mpl.xticks(rotation=self._x_rotation)
446
        self._pdf_page.savefig()
447

448

449 450
class Epsc3D(Epsc):
    ''' 3D EPSC plots for PAD'''
Amir MOHAMMADI's avatar
nit  
Amir MOHAMMADI committed
451

452
    def compute(self, idx, input_scores, input_names):
453
        ''' Implements plots'''
454 455 456 457 458 459 460 461
        licit_dev_neg = input_scores[0][0]
        licit_dev_pos = input_scores[0][1]
        licit_eval_neg = input_scores[1][0]
        licit_eval_pos = input_scores[1][1]
        spoof_dev_neg = input_scores[2][0]
        spoof_dev_pos = input_scores[2][1]
        spoof_eval_neg = input_scores[3][0]
        spoof_eval_pos = input_scores[3][1]
462

Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
463
        title = self._legends[idx] if self._legends is not None else "3D EPSC"
464

465 466
        mpl.rcParams.pop('key', None)

467
        mpl.gcf().clear()
468
        mpl.gcf().set_constrained_layout(self._clayout)
469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490

        from mpl_toolkits.mplot3d import Axes3D
        from matplotlib import cm

        points = 10

        omega, beta, thrs = error_utils.epsc_thresholds(
            licit_dev_neg,
            licit_dev_pos,
            spoof_dev_neg,
            spoof_dev_pos,
            points=points,
            criteria=self._criteria)

        errors = error_utils.all_error_rates(
            licit_eval_neg, licit_eval_pos, spoof_eval_neg, spoof_eval_pos,
            thrs, omega, beta
        )
        # error rates are returned in a list as 2D numpy.ndarrays in
        # the following order: frr, far, IAPMR, far_w, wer_wb, hter_wb
        wer_errors = 100 * errors[2 if self._iapmr else 4]

491
        ax1 = mpl.gcf().add_subplot(111, projection='3d')
492 493 494 495 496 497 498 499 500 501 502

        W, B = np.meshgrid(omega, beta)

        ax1.plot_wireframe(
            W, B, wer_errors, cmap=cm.coolwarm, antialiased=False
        )  # surface

        if self._iapmr:
            ax1.azim = -30
            ax1.elev = 50

503 504
        ax1.set_xlabel(self._x_label or r"Weight $\omega$")
        ax1.set_ylabel(self._y_label or r"Weight $\beta$")
505 506 507 508
        ax1.set_zlabel(
            r"WER$_{\omega,\beta}$ (%)" if self._wer else "IAPMR (%)"
        )

509
        if title.replace(' ', ''):
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
510
            mpl.title(title)
511 512 513 514 515

        ax1.set_xticklabels(ax1.get_xticks())
        ax1.set_yticklabels(ax1.get_yticks())
        ax1.set_zticklabels(ax1.get_zticks())

516 517
        self._pdf_page.savefig()

518

519 520
class Det(PadPlot):
    '''DET for PAD'''
Amir MOHAMMADI's avatar
nit  
Amir MOHAMMADI committed
521

522 523 524
    def __init__(self, ctx, scores, evaluation, func_load, criteria, real_data):
        super(Det, self).__init__(ctx, scores, evaluation, func_load)
        self._no_spoof = False if 'no_spoof' not in ctx.meta else\
Amir MOHAMMADI's avatar
nit  
Amir MOHAMMADI committed
525
            ctx.meta['no_spoof']
526 527 528
        self._criteria = criteria
        self._real_data = True if real_data is None else real_data

529
    def compute(self, idx, input_scores, input_names):
530
        ''' Implements plots'''
531 532 533 534 535 536
        licit_dev_neg = input_scores[0][0]
        licit_dev_pos = input_scores[0][1]
        licit_eval_neg = input_scores[1][0]
        licit_eval_pos = input_scores[1][1]
        spoof_eval_neg = input_scores[3][0] if len(input_scores) > 2 else None
        spoof_eval_pos = input_scores[3][1] if len(input_scores) > 2 else None
537 538 539 540 541 542 543

        det(
            licit_eval_neg,
            licit_eval_pos,
            self._points,
            color=self._colors[idx],
            linestyle='-',
544
            label=self._label("licit", input_names[0], idx)
545 546 547 548 549 550 551 552
        )
        if not self._no_spoof and spoof_eval_neg is not None:
            det(
                spoof_eval_neg,
                spoof_eval_pos,
                self._points,
                color=self._colors[idx],
                linestyle='--',
553
                label=self._label("spoof", input_names[3], idx)
554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653
            )

        if self._criteria is None:
            return

        thres_baseline = calc_threshold(
            self._criteria, licit_dev_neg, licit_dev_pos
        )

        axlim = mpl.axis()

        farfrr_licit = farfrr(
            licit_eval_neg, licit_eval_pos,
            thres_baseline)  # calculate test frr @ EER (licit scenario)
        farfrr_spoof = farfrr(
            spoof_eval_neg, spoof_eval_pos,
            thres_baseline)  # calculate test frr @ EER (spoof scenario)
        farfrr_licit_det = [
            ppndf(i) for i in farfrr_licit
        ]
        # find the FAR and FRR values that need to be plotted on normal deviate
        # scale
        farfrr_spoof_det = [
            ppndf(i) for i in farfrr_spoof
        ]
        # find the FAR and FRR values that need to be plotted on normal deviate
        # scale
        if not self._real_data:
            mpl.axhline(
                y=farfrr_licit_det[1],
                xmin=axlim[2],
                xmax=axlim[3],
                color='k',
                linestyle='--',
                label="FRR @ EER")  # vertical FRR threshold
        else:
            mpl.axhline(
                y=farfrr_licit_det[1],
                xmin=axlim[0],
                xmax=axlim[1],
                color='k',
                linestyle='--',
                label="FRR = %.2f%%" %
                (farfrr_licit[1] * 100))  # vertical FRR threshold

        mpl.plot(
            farfrr_licit_det[0],
            farfrr_licit_det[1],
            'o',
            color=self._colors[idx],
            markersize=9)  # FAR point, licit scenario
        mpl.plot(
            farfrr_spoof_det[0],
            farfrr_spoof_det[1],
            'o',
            color=self._colors[idx],
            markersize=9)  # FAR point, spoof scenario

        # annotate the FAR points
        xyannotate_licit = [
            ppndf(0.7 * farfrr_licit[0]),
            ppndf(1.8 * farfrr_licit[1])
        ]
        xyannotate_spoof = [
            ppndf(0.95 * farfrr_spoof[0]),
            ppndf(1.8 * farfrr_licit[1])
        ]

        if not self._real_data:
            mpl.annotate(
                'FMR @\noperating point',
                xy=(farfrr_licit_det[0], farfrr_licit_det[1]),
                xycoords='data',
                xytext=(xyannotate_licit[0], xyannotate_licit[1]),
                color=self._colors[idx])
            mpl.annotate(
                'IAPMR @\noperating point',
                xy=(farfrr_spoof_det[0], farfrr_spoof_det[1]),
                xycoords='data',
                xytext=(xyannotate_spoof[0], xyannotate_spoof[1]),
                color=self._colors[idx])
        else:
            mpl.annotate(
                'FAR=%.2f%%' % (farfrr_licit[0] * 100),
                xy=(farfrr_licit_det[0], farfrr_licit_det[1]),
                xycoords='data',
                xytext=(xyannotate_licit[0], xyannotate_licit[1]),
                color=self._colors[idx],
                size='large')
            mpl.annotate(
                'IAPMR=\n%.2f%%' % (farfrr_spoof[0] * 100),
                xy=(farfrr_spoof_det[0], farfrr_spoof_det[1]),
                xycoords='data',
                xytext=(xyannotate_spoof[0], xyannotate_spoof[1]),
                color=self._colors[idx],
                size='large')

    def end_process(self):
        ''' Set title, legend, axis labels, grid colors, save figures and
        close pdf is needed '''
Amir MOHAMMADI's avatar
nit  
Amir MOHAMMADI committed
654
        # only for plots
655 656 657 658
        add = ''
        if not self._no_spoof:
            add = " and overlaid SPOOF scenario"
        title = self._title if self._title is not None else \
Amir MOHAMMADI's avatar
nit  
Amir MOHAMMADI committed
659
            ('DET: LICIT' + add)
660 661
        if title.replace(' ', ''):
            mpl.title(title)
662 663 664
        mpl.xlabel(self._x_label or "False Acceptance Rate (%)")
        mpl.ylabel(self._y_label or "False Rejection Rate (%)")
        mpl.grid(True, color=self._grid_color)
665 666
        if self._disp_legend:
            mpl.legend(loc=self._legend_loc)
667 668 669 670 671 672 673 674 675 676 677
        self._set_axis()
        fig = mpl.gcf()
        mpl.xticks(rotation=self._x_rotation)
        mpl.tick_params(axis='both', which='major', labelsize=4)
        for tick in mpl.gca().xaxis.get_major_ticks():
            tick.label.set_fontsize(6)
        for tick in mpl.gca().yaxis.get_major_ticks():
            tick.label.set_fontsize(6)

        self._pdf_page.savefig(fig)

Amir MOHAMMADI's avatar
nit  
Amir MOHAMMADI committed
678
        # do not want to close PDF when running evaluate
679
        if 'PdfPages' in self._ctx.meta and \
Amir MOHAMMADI's avatar
nit  
Amir MOHAMMADI committed
680
                ('closef' not in self._ctx.meta or self._ctx.meta['closef']):
681 682 683 684 685 686 687
            self._pdf_page.close()

    def _set_axis(self):
        if self._axlim is not None and None not in self._axlim:
            det_axis(self._axlim)
        else:
            det_axis([0.01, 99, 0.01, 99])
688

689

690 691
class FmrIapmr(PadPlot):
    '''FMR vs IAPMR'''
Amir MOHAMMADI's avatar
nit  
Amir MOHAMMADI committed
692

693 694
    def __init__(self, ctx, scores, evaluation, func_load):
        super(FmrIapmr, self).__init__(ctx, scores, evaluation, func_load)
Amir MOHAMMADI's avatar
nit  
Amir MOHAMMADI committed
695
        self._eval = True  # always eval data with EPC
696 697 698
        self._split = False
        self._nb_figs = 1
        self._semilogx = False if 'semilogx' not in ctx.meta else\
Amir MOHAMMADI's avatar
nit  
Amir MOHAMMADI committed
699
            ctx.meta['semilogx']
700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716
        if self._min_arg != 4:
            raise click.BadParameter("You must provide 4 scores files:{licit,"
                                     "spoof}/{dev,eval}")

    def compute(self, idx, input_scores, input_names):
        ''' Implements plots'''
        licit_eval_neg = input_scores[1][0]
        licit_eval_pos = input_scores[1][1]
        spoof_eval_neg = input_scores[3][0]
        fmr_list = np.linspace(0, 1, 100)
        iapmr_list = []
        for i, fmr in enumerate(fmr_list):
            thr = far_threshold(licit_eval_neg, licit_eval_pos, fmr, True)
            iapmr_list.append(farfrr(spoof_eval_neg, licit_eval_pos, thr)[0])
            # re-calculate fmr since threshold might give a different result
            # for fmr.
            fmr_list[i] = farfrr(licit_eval_neg, licit_eval_pos, thr)[0]
717
        label = self._legends[idx] if self._legends is not None else \
Amir MOHAMMADI's avatar
nit  
Amir MOHAMMADI committed
718
            '(%s/%s)' % (input_names[1], input_names[3])
719 720 721 722 723 724 725 726
        if self._semilogx:
            mpl.semilogx(fmr_list, iapmr_list, label=label)
        else:
            mpl.plot(fmr_list, iapmr_list, label=label)

    def end_process(self):
        ''' Set title, legend, axis labels, grid colors, save figures and
        close pdf is needed '''
Amir MOHAMMADI's avatar
nit  
Amir MOHAMMADI committed
727
        # only for plots
728
        title = self._title if self._title is not None else "FMR vs IAPMR"
729 730
        if title.replace(' ', ''):
            mpl.title(title)
731 732 733
        mpl.xlabel(self._x_label or "False Match Rate (%)")
        mpl.ylabel(self._y_label or "IAPMR (%)")
        mpl.grid(True, color=self._grid_color)
734 735
        if self._disp_legend:
            mpl.legend(loc=self._legend_loc)
736 737 738 739 740 741 742
        self._set_axis()
        fig = mpl.gcf()
        mpl.xticks(rotation=self._x_rotation)
        mpl.tick_params(axis='both', which='major', labelsize=4)

        self._pdf_page.savefig(fig)

Amir MOHAMMADI's avatar
nit  
Amir MOHAMMADI committed
743
        # do not want to close PDF when running evaluate
744
        if 'PdfPages' in self._ctx.meta and \
Amir MOHAMMADI's avatar
nit  
Amir MOHAMMADI committed
745
                ('closef' not in self._ctx.meta or self._ctx.meta['closef']):
746
            self._pdf_page.close()