figure.py 27.3 KB
Newer Older
1
2
3
4
'''Runs error analysis on score sets, outputs metrics and plots'''

import click
import numpy as np
5
import matplotlib.pyplot as mpl
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
6
import bob.measure.script.figure as measure_figure
7
import bob.bio.base.script.figure as bio_figure
8
from tabulate import tabulate
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
9
from bob.measure.utils import get_fta_list
10
from bob.measure import (
11
    far_threshold, eer_threshold, min_hter_threshold, farfrr, epc, ppndf
12
)
13
from bob.measure.plot import (det, det_axis)
14
15
16
17
from . import error_utils

ALL_CRITERIA = ('bpcer20', 'eer', 'min-hter')

18

19
20
21
22
23
24
25
def calc_threshold(method, neg, pos):
    """Calculates the threshold based on the given method.
    The scores should be sorted!

    Parameters
    ----------
    method : str
26
        One of ``bpcer20``, ``eer``, ``min-hter``.
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
    neg : array_like
        The negative scores. They should be sorted!
    pos : array_like
        The positive scores. They should be sorted!

    Returns
    -------
    float
        The calculated threshold.

    Raises
    ------
    ValueError
        If method is unknown.
    """
    method = method.lower()
    if method == 'bpcer20':
        threshold = far_threshold(neg, pos, 0.05, True)
    elif method == 'eer':
        threshold = eer_threshold(neg, pos, True)
    elif method == 'min-hter':
        threshold = min_hter_threshold(neg, pos, True)
    else:
        raise ValueError("Unknown threshold criteria: {}".format(method))

    return threshold

54

55
56
57
58
59
class Metrics(measure_figure.Metrics):
    def __init__(self, ctx, scores, evaluation, func_load):
        super(Metrics, self).__init__(ctx, scores, evaluation, func_load)

    ''' Compute metrics from score files'''
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
60

61
    def compute(self, idx, input_scores, input_names):
62
        ''' Compute metrics for the given criteria'''
63
64
65
66
67
68
        neg_list, pos_list, _ = get_fta_list(input_scores)
        dev_neg, dev_pos = neg_list[0], pos_list[0]
        dev_file = input_names[0]
        if self._eval:
            eval_neg, eval_pos = neg_list[1], pos_list[1]
            eval_file = input_names[1]
69

70
        title = self._legends[idx] if self._legends is not None else None
71
        headers = ['' or title, 'Development %s' % dev_file]
72
        if self._eval:
73
74
75
76
77
78
79
            headers.append('Eval. % s' % eval_file)
        for m in ALL_CRITERIA:
            raws = []
            threshold = calc_threshold(m, dev_neg, dev_pos)
            click.echo("\nThreshold of %f selected with the %s criteria" % (
                threshold, m))
            apcer, bpcer = farfrr(dev_neg, dev_pos, threshold)
80
81
82
            raws.append(['APCER', '{:>5.1f}%'.format(apcer * 100)])
            raws.append(['BP', '{:>5.1f}%'.format(bpcer * 100)])
            raws.append(['ACER', '{:>5.1f}%'.format((apcer + bpcer) * 50)])
83
84
85
86
87
88
89
90
91
92
93
            if self._eval and eval_neg is not None:
                apcer, bpcer = farfrr(eval_neg, eval_pos, threshold)
                raws[0].append('{:>5.1f}%'.format(apcer * 100))
                raws[1].append('{:>5.1f}%'.format(bpcer * 100))
                raws[2].append('{:>5.1f}%'.format((apcer + bpcer) * 50))

            click.echo(
                tabulate(raws, headers, self._tablefmt),
                file=self.log_file
            )

94

95
96
97
98
99
class MetricsVuln(measure_figure.Metrics):
    def __init__(self, ctx, scores, evaluation, func_load):
        super(MetricsVuln, self).__init__(ctx, scores, evaluation, func_load)

    ''' Compute metrics from score files'''
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
100

101
102
103
104
105
106
    def compute(self, idx, input_scores, input_names):
        ''' Compute metrics for the given criteria'''
        neg_list, pos_list, _ = get_fta_list(input_scores)
        dev_neg, dev_pos = neg_list[0], pos_list[0]
        criter = self._criterion or 'eer'
        threshold = calc_threshold(criter, dev_neg, dev_pos) \
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
107
            if self._thres is None else self._thres[idx]
108
109
110
111
112
113
        far, frr = farfrr(neg_list[1], pos_list[1], threshold)
        iapmr, _ = farfrr(neg_list[3], pos_list[1], threshold)
        title = self._legends[idx] if self._legends is not None else None
        headers = ['' or title, '%s (threshold=%.2g)' %
                   (criter.upper(), threshold)]
        rows = []
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
114
        rows.append(['FMR (%)', '{:>5.1f}%'.format(100 * far)])
115
        rows.append(['BPCER (%)', '{:>5.1f}%'.format(frr * 100)])
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
116
117
        rows.append(['HTER (%)', '{:>5.1f}%'.format(50 * (far + frr))])
        rows.append(['IAPMR (%)', '{:>5.1f}%'.format(100 * iapmr)])
118
119
120
121
122
123
        click.echo(
            tabulate(rows, headers, self._tablefmt),
            file=self.log_file
        )


124
125
126
127
128
129
class HistPad(measure_figure.Hist):
    ''' Histograms for PAD '''

    def _setup_hist(self, neg, pos):
        self._title_base = 'PAD'
        self._density_hist(
130
            pos[0], n=0, label='Bona Fide', color='C1'
131
132
        )
        self._density_hist(
133
            neg[0], n=1, label='Presentation attack', alpha=0.4, color='C7',
134
            hatch='\\\\'
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
        )


def _iapmr_dot(threshold, iapmr, real_data, **kwargs):
    # plot a dot on threshold versus IAPMR line and show IAPMR as a number
    axlim = mpl.axis()
    mpl.plot(threshold, 100. * iapmr, 'o', color='C3', **kwargs)
    if not real_data:
        mpl.annotate(
            'IAPMR at\noperating point',
            xy=(threshold, 100. * iapmr),
            xycoords='data',
            xytext=(0.85, 0.6),
            textcoords='axes fraction',
            color='black',
            size='large',
            arrowprops=dict(facecolor='black', shrink=0.05, width=2),
            horizontalalignment='center',
            verticalalignment='top',
        )
    else:
        mpl.text(threshold + (threshold - axlim[0]) / 12, 100. * iapmr,
                 '%.1f%%' % (100. * iapmr,), color='C3')

159

160
161
162
163
164
165
def _iapmr_line_plot(scores, n_points=100, **kwargs):
    axlim = mpl.axis()
    step = (axlim[1] - axlim[0]) / float(n_points)
    thres = [(k * step) + axlim[0] for k in range(2, n_points - 1)]
    mix_prob_y = []
    for k in thres:
166
        mix_prob_y.append(100. * error_utils.calc_pass_rate(k, scores))
167
168
169

    mpl.plot(thres, mix_prob_y, label='IAPMR', color='C3', **kwargs)

170

171
172
173
174
def _iapmr_plot(scores, threshold, iapmr, real_data, **kwargs):
    _iapmr_dot(threshold, iapmr, real_data, **kwargs)
    _iapmr_line_plot(scores, n_points=100, **kwargs)

175

176
177
178
179
180
181
class HistVuln(measure_figure.Hist):
    ''' Histograms for vulnerability '''

    def _setup_hist(self, neg, pos):
        self._title_base = 'Vulnerability'
        self._density_hist(
182
            pos[0], n=0, label='Genuine', color='C2'
183
184
        )
        self._density_hist(
185
            neg[0], n=1, label='Zero-effort impostors', alpha=0.8, color='C0'
186
187
        )
        self._density_hist(
188
            neg[1], n=2, label='Presentation attack', alpha=0.4, color='C7',
189
            hatch='\\\\'
190
191
        )

192
    def _lines(self, threshold, label, neg, pos, idx, **kwargs):
193
        if 'iapmr_line' not in self._ctx.meta or self._ctx.meta['iapmr_line']:
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
194
            # plot vertical line
195
            super(HistVuln, self)._lines(threshold, label, neg, pos, idx)
196

Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
197
            # plot iapmr_line
198
199
200
201
202
            iapmr, _ = farfrr(neg[1], pos[0], threshold)
            ax2 = mpl.twinx()
            # we never want grid lines on axis 2
            ax2.grid(False)
            real_data = True if 'real_data' not in self._ctx.meta else \
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
203
                self._ctx.meta['real_data']
204
            _iapmr_plot(neg[1], threshold, iapmr, real_data=real_data)
205
206
            n = idx % self._step_print
            col = n % self._ncols
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
207
208
            rest_print = self.n_systems - \
                int(idx / self._step_print) * self._step_print
209
210
            if col == self._ncols - 1 or n == rest_print - 1:
                ax2.set_ylabel("IAPMR (%)", color='C3')
211
212
213
214
            ax2.tick_params(axis='y', colors='red')
            ax2.yaxis.label.set_color('red')
            ax2.spines['right'].set_color('red')

215

216
217
class PadPlot(measure_figure.PlotBase):
    '''Base class for PAD plots'''
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
218

219
220
    def __init__(self, ctx, scores, evaluation, func_load):
        super(PadPlot, self).__init__(ctx, scores, evaluation, func_load)
221
        mpl.rcParams['figure.constrained_layout.use'] = self._clayout
222
223
224

    def end_process(self):
        '''Close pdf '''
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
225
        # do not want to close PDF when running evaluate
226
227
228
229
230
        if 'PdfPages' in self._ctx.meta and \
           ('closef' not in self._ctx.meta or self._ctx.meta['closef']):
            self._pdf_page.close()

    def _plot_legends(self):
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
231
        # legends for all axes
232
233
234
235
236
237
        lines = []
        labels = []
        for ax in mpl.gcf().get_axes():
            li, la = ax.get_legend_handles_labels()
            lines += li
            labels += la
238
239
240
241
        if self._disp_legend:
            mpl.gca().legend(lines, labels, loc=self._legend_loc,
                             fancybox=True, framealpha=0.5)

242
243
244

class Epc(PadPlot):
    ''' Handles the plotting of EPC '''
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
245

246
247
248
    def __init__(self, ctx, scores, evaluation, func_load):
        super(Epc, self).__init__(ctx, scores, evaluation, func_load)
        self._iapmr = True if 'iapmr' not in self._ctx.meta else \
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
249
            self._ctx.meta['iapmr']
250
251
252
253
        self._title = self._title or ('EPC and IAPMR' if self._iapmr else
                                      'EPC')
        self._x_label = self._x_label or r"Weight $\beta$"
        self._y_label = self._y_label or "WER (%)"
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
254
        self._eval = True  # always eval data with EPC
255
256
257
        self._split = False
        self._nb_figs = 1

258
259
260
261
262
        if self._min_arg != 4:
            raise click.BadParameter("You must provide 4 scores files:{licit,"
                                     "spoof}/{dev,eval}")

    def compute(self, idx, input_scores, input_names):
263
        ''' Plot EPC for PAD'''
264
265
266
267
268
        licit_dev_neg = input_scores[0][0]
        licit_dev_pos = input_scores[0][1]
        licit_eval_neg = input_scores[1][0]
        licit_eval_pos = input_scores[1][1]
        spoof_eval_neg = input_scores[3][0]
269
        mpl.gcf().clear()
270
        epc_baseline = epc(
271
272
273
274
275
276
277
            licit_dev_neg, licit_dev_pos, licit_eval_neg,
            licit_eval_pos, 100
        )
        mpl.plot(
            epc_baseline[:, 0], [100. * k for k in epc_baseline[:, 1]],
            color='C0',
            label=self._label(
278
                'WER', '%s-%s' % (input_names[0], input_names[1]), idx
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
            ),
            linestyle='-'
        )
        mpl.xlabel(self._x_label)
        mpl.ylabel(self._y_label)
        if self._iapmr:
            mix_prob_y = []
            for k in epc_baseline[:, 2]:
                prob_attack = sum(
                    1 for i in spoof_eval_neg if i >= k
                ) / float(spoof_eval_neg.size)
                mix_prob_y.append(100. * prob_attack)

            mpl.gca().set_axisbelow(True)
            prob_ax = mpl.gca().twinx()
            mpl.plot(
                epc_baseline[:, 0],
                mix_prob_y,
                color='C3',
                linestyle='-',
                label=self._label(
300
                    'IAPMR', '%s-%s' % (input_names[0], input_names[1]), idx
301
302
303
                )
            )
            prob_ax.set_yticklabels(prob_ax.get_yticks())
304
305
306
            prob_ax.tick_params(axis='y', colors='C3')
            prob_ax.yaxis.label.set_color('C3')
            prob_ax.spines['right'].set_color('C3')
307
308
            ylabels = prob_ax.get_yticks()
            prob_ax.yaxis.set_ticklabels(["%.0f" % val for val in ylabels])
309
            prob_ax.set_ylabel('IAPMR', color='C3')
310
            prob_ax.set_axisbelow(True)
311
        title = self._legends[idx] if self._legends is not None else self._title
312
313
        if title.replace(' ', ''):
            mpl.title(title)
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
314
        # legends for all axes
315
316
317
318
        self._plot_legends()
        mpl.xticks(rotation=self._x_rotation)
        self._pdf_page.savefig(mpl.gcf())

319

320
321
class Epsc(PadPlot):
    ''' Handles the plotting of EPSC '''
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
322

323
324
325
326
    def __init__(self, ctx, scores, evaluation, func_load,
                 criteria, var_param, fixed_param):
        super(Epsc, self).__init__(ctx, scores, evaluation, func_load)
        self._iapmr = False if 'iapmr' not in self._ctx.meta else \
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
327
            self._ctx.meta['iapmr']
328
        self._wer = True if 'wer' not in self._ctx.meta else \
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
329
            self._ctx.meta['wer']
330
331
332
        self._criteria = 'eer' if criteria is None else criteria
        self._var_param = "omega" if var_param is None else var_param
        self._fixed_param = 0.5 if fixed_param is None else fixed_param
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
333
        self._eval = True  # always eval data with EPC
334
335
        self._split = False
        self._nb_figs = 1
336
337
338
339
340
        self._title = ''

        if self._min_arg != 4:
            raise click.BadParameter("You must provide 4 scores files:{licit,"
                                     "spoof}/{dev,eval}")
341

342
    def compute(self, idx, input_scores, input_names):
343
        ''' Plot EPSC for PAD'''
344
345
346
347
348
349
350
351
        licit_dev_neg = input_scores[0][0]
        licit_dev_pos = input_scores[0][1]
        licit_eval_neg = input_scores[1][0]
        licit_eval_pos = input_scores[1][1]
        spoof_dev_neg = input_scores[2][0]
        spoof_dev_pos = input_scores[2][1]
        spoof_eval_neg = input_scores[3][0]
        spoof_eval_pos = input_scores[3][1]
352
        title = self._legends[idx] if self._legends is not None else None
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372

        mpl.gcf().clear()
        points = 10

        if self._var_param == 'omega':
            omega, beta, thrs = error_utils.epsc_thresholds(
                licit_dev_neg,
                licit_dev_pos,
                spoof_dev_neg,
                spoof_dev_pos,
                points=points,
                criteria=self._criteria,
                beta=self._fixed_param)
        else:
            omega, beta, thrs = error_utils.epsc_thresholds(
                licit_dev_neg,
                licit_dev_pos,
                spoof_dev_neg,
                spoof_dev_pos,
                points=points,
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
373
                criteria=self._criteria,
374
375
376
377
378
379
380
                omega=self._fixed_param
            )

        errors = error_utils.all_error_rates(
            licit_eval_neg, licit_eval_pos, spoof_eval_neg,
            spoof_eval_pos, thrs, omega, beta
        )  # error rates are returned in a list in the
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
381
        # following order: frr, far, IAPMR, far_w, wer_w
382
383
384
385

        ax1 = mpl.subplot(
            111
        )  # EPC like curves for FVAS fused scores for weighted error rates
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
386
        # between the negatives (impostors and Presentation attacks)
387
388
389
390
391
392
393
394
        if self._wer:
            if self._var_param == 'omega':
                mpl.plot(
                    omega,
                    100. * errors[4].flatten(),
                    color='C0',
                    linestyle='-',
                    label=r"WER$_{\omega,\beta}$")
395
                mpl.xlabel(self._x_label or r"Weight $\omega$")
396
397
398
399
400
401
402
            else:
                mpl.plot(
                    beta,
                    100. * errors[4].flatten(),
                    color='C0',
                    linestyle='-',
                    label=r"WER$_{\omega,\beta}$")
403
404
                mpl.xlabel(self._x_label or r"Weight $\beta$")
            mpl.ylabel(self._y_label or r"WER$_{\omega,\beta}$ (%)")
405
406
407
408
409
410
411
412
413
414
415
416
417

        if self._iapmr:
            axis = mpl.gca()
            if self._wer:
                axis = mpl.twinx()
                axis.grid(False)
            if self._var_param == 'omega':
                mpl.plot(
                    omega,
                    100. * errors[2].flatten(),
                    color='C3',
                    linestyle='-',
                    label='IAPMR')
418
                mpl.xlabel(self._x_label or r"Weight $\omega$")
419
420
421
422
423
424
425
            else:
                mpl.plot(
                    beta,
                    100. * errors[2].flatten(),
                    color='C3',
                    linestyle='-',
                    label='IAPMR')
426
427
                mpl.xlabel(self._x_label or r"Weight $\beta$")
            mpl.ylabel(self._y_label or r"IAPMR  (%)")
428
429
430
431
432
433
434
            if self._wer:
                axis.set_yticklabels(axis.get_yticks())
                axis.tick_params(axis='y', colors='red')
                axis.yaxis.label.set_color('red')
                axis.spines['right'].set_color('red')

        if self._var_param == 'omega':
Theophile GENTILHOMME's avatar
Bug fix    
Theophile GENTILHOMME committed
435
            if title is not None and title.replace(' ', ''):
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
436
                mpl.title(title or (r"EPSC with $\beta$ = %.2f" %
437
                                    self._fixed_param))
438
        else:
Theophile GENTILHOMME's avatar
Bug fix    
Theophile GENTILHOMME committed
439
            if title is not None and title.replace(' ', ''):
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
440
                mpl.title(title or (r"EPSC with $\omega$ = %.2f" %
441
                                    self._fixed_param))
442
443
444
445
446
447

        mpl.grid()
        self._plot_legends()
        ax1.set_xticklabels(ax1.get_xticks())
        ax1.set_yticklabels(ax1.get_yticks())
        mpl.xticks(rotation=self._x_rotation)
448
        self._pdf_page.savefig()
449

450

451
452
class Epsc3D(Epsc):
    ''' 3D EPSC plots for PAD'''
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
453

454
    def compute(self, idx, input_scores, input_names):
455
        ''' Implements plots'''
456
457
458
459
460
461
462
463
        licit_dev_neg = input_scores[0][0]
        licit_dev_pos = input_scores[0][1]
        licit_eval_neg = input_scores[1][0]
        licit_eval_pos = input_scores[1][1]
        spoof_dev_neg = input_scores[2][0]
        spoof_dev_pos = input_scores[2][1]
        spoof_eval_neg = input_scores[3][0]
        spoof_eval_pos = input_scores[3][1]
464

Theophile GENTILHOMME's avatar
Bug fix    
Theophile GENTILHOMME committed
465
        title = self._legends[idx] if self._legends is not None else "3D EPSC"
466

467
468
        mpl.rcParams.pop('key', None)

469
        mpl.gcf().clear()
470
        mpl.gcf().set_constrained_layout(self._clayout)
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492

        from mpl_toolkits.mplot3d import Axes3D
        from matplotlib import cm

        points = 10

        omega, beta, thrs = error_utils.epsc_thresholds(
            licit_dev_neg,
            licit_dev_pos,
            spoof_dev_neg,
            spoof_dev_pos,
            points=points,
            criteria=self._criteria)

        errors = error_utils.all_error_rates(
            licit_eval_neg, licit_eval_pos, spoof_eval_neg, spoof_eval_pos,
            thrs, omega, beta
        )
        # error rates are returned in a list as 2D numpy.ndarrays in
        # the following order: frr, far, IAPMR, far_w, wer_wb, hter_wb
        wer_errors = 100 * errors[2 if self._iapmr else 4]

493
        ax1 = mpl.gcf().add_subplot(111, projection='3d')
494
495
496
497
498
499
500
501
502
503
504

        W, B = np.meshgrid(omega, beta)

        ax1.plot_wireframe(
            W, B, wer_errors, cmap=cm.coolwarm, antialiased=False
        )  # surface

        if self._iapmr:
            ax1.azim = -30
            ax1.elev = 50

505
506
        ax1.set_xlabel(self._x_label or r"Weight $\omega$")
        ax1.set_ylabel(self._y_label or r"Weight $\beta$")
507
508
509
510
        ax1.set_zlabel(
            r"WER$_{\omega,\beta}$ (%)" if self._wer else "IAPMR (%)"
        )

511
        if title.replace(' ', ''):
Theophile GENTILHOMME's avatar
Bug fix    
Theophile GENTILHOMME committed
512
            mpl.title(title)
513
514
515
516
517

        ax1.set_xticklabels(ax1.get_xticks())
        ax1.set_yticklabels(ax1.get_yticks())
        ax1.set_zticklabels(ax1.get_zticks())

518
519
        self._pdf_page.savefig()

520

521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
class Roc(bio_figure.Roc):
    '''ROC for PARD'''
    def __init__(self, ctx, scores, evaluation, func_load):
        super(Roc, self).__init__(ctx, scores, evaluation, func_load)
        self._x_label = ctx.meta.get('x_label') or 'APCER'
        self._y_label = ctx.meta.get('y_label') or '1 - BPCER'

class DetPad(bio_figure.Det):
    def __init__(self, ctx, scores, evaluation, func_load):
        super(DetPad, self).__init__(ctx, scores, evaluation, func_load)
        self._x_label = ctx.meta.get('x_label') or 'APCER'
        self._y_label = ctx.meta.get('y_label') or 'BPCER'



536
class Det(PadPlot):
537
    '''DET for VULN'''
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
538

539
540
    def __init__(self, ctx, scores, evaluation, func_load, criteria, real_data,
                no_spoof):
541
        super(Det, self).__init__(ctx, scores, evaluation, func_load)
542
543
        self._no_spoof = no_spoof
        self._criteria = criteria or 'eer'
544
        self._real_data = True if real_data is None else real_data
545
546
        self._x_label = self._x_label or "APCER"
        self._y_label = self._y_label or "BPCER"
547

548
    def compute(self, idx, input_scores, input_names):
549
        ''' Implements plots'''
550
551
552
553
554
555
        licit_dev_neg = input_scores[0][0]
        licit_dev_pos = input_scores[0][1]
        licit_eval_neg = input_scores[1][0]
        licit_eval_pos = input_scores[1][1]
        spoof_eval_neg = input_scores[3][0] if len(input_scores) > 2 else None
        spoof_eval_pos = input_scores[3][1] if len(input_scores) > 2 else None
556
557
558
559
        det(
            licit_eval_neg,
            licit_eval_pos,
            self._points,
560
            color='C0',
561
            linestyle='-',
562
            label=self._label("licit", input_names[0], idx)
563
564
565
566
567
568
        )
        if not self._no_spoof and spoof_eval_neg is not None:
            det(
                spoof_eval_neg,
                spoof_eval_pos,
                self._points,
569
570
                color='C3',
                linestyle=':',
571
                label=self._label("spoof", input_names[3], idx)
572
573
            )

574
        if self._criteria is None or self._no_spoof:
575
576
577
578
579
580
581
582
583
584
            return

        thres_baseline = calc_threshold(
            self._criteria, licit_dev_neg, licit_dev_pos
        )

        axlim = mpl.axis()

        farfrr_licit = farfrr(
            licit_eval_neg, licit_eval_pos,
585
586
            thres_baseline
        )  # calculate test frr @ EER (licit scenario)
587
588
        farfrr_spoof = farfrr(
            spoof_eval_neg, spoof_eval_pos,
589
590
            thres_baseline
        )  # calculate test frr @ EER (spoof scenario)
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
        farfrr_licit_det = [
            ppndf(i) for i in farfrr_licit
        ]
        # find the FAR and FRR values that need to be plotted on normal deviate
        # scale
        farfrr_spoof_det = [
            ppndf(i) for i in farfrr_spoof
        ]
        # find the FAR and FRR values that need to be plotted on normal deviate
        # scale
        if not self._real_data:
            mpl.axhline(
                y=farfrr_licit_det[1],
                xmin=axlim[2],
                xmax=axlim[3],
                color='k',
                linestyle='--',
608
                label="BPCER @ EER")
609
610
611
612
613
614
615
        else:
            mpl.axhline(
                y=farfrr_licit_det[1],
                xmin=axlim[0],
                xmax=axlim[1],
                color='k',
                linestyle='--',
616
                label="BPCER = %.2f%%" %
617
                (farfrr_licit[1] * 100))
618
619
620
621
622

        mpl.plot(
            farfrr_licit_det[0],
            farfrr_licit_det[1],
            'o',
623
624
625
            color='C0',
            markersize=9
        )  # FAR point, licit scenario
626
627
628
629
        mpl.plot(
            farfrr_spoof_det[0],
            farfrr_spoof_det[1],
            'o',
630
631
632
            color='C3',
            markersize=9
        )  # FAR point, spoof scenario
633
634
635

        # annotate the FAR points
        xyannotate_licit = [
636
637
            0.15 + farfrr_licit_det[0],
            farfrr_licit_det[1] - 0.15,
638
639
        ]
        xyannotate_spoof = [
640
641
            0.15 + farfrr_spoof_det[0],
            farfrr_spoof_det[1] - 0.15,
642
643
644
645
        ]

        if not self._real_data:
            mpl.annotate(
646
                'FMR @ operating point',
647
648
                xy=(farfrr_licit_det[0], farfrr_licit_det[1]),
                xycoords='data',
649
                xytext=(xyannotate_licit[0], xyannotate_licit[1]))
650
            mpl.annotate(
651
                'IAPMR @ operating point',
652
653
                xy=(farfrr_spoof_det[0], farfrr_spoof_det[1]),
                xycoords='data',
654
                xytext=(xyannotate_spoof[0], xyannotate_spoof[1]))
655
656
        else:
            mpl.annotate(
657
                'APCER=%.2f%%' % (farfrr_licit[0] * 100),
658
659
660
                xy=(farfrr_licit_det[0], farfrr_licit_det[1]),
                xycoords='data',
                xytext=(xyannotate_licit[0], xyannotate_licit[1]),
661
                color='C0',
662
663
                size='large')
            mpl.annotate(
664
                'IAPMR=%.2f%%' % (farfrr_spoof[0] * 100),
665
666
667
                xy=(farfrr_spoof_det[0], farfrr_spoof_det[1]),
                xycoords='data',
                xytext=(xyannotate_spoof[0], xyannotate_spoof[1]),
668
                color='C3',
669
670
671
672
673
                size='large')

    def end_process(self):
        ''' Set title, legend, axis labels, grid colors, save figures and
        close pdf is needed '''
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
674
        # only for plots
675
676
677
678
        add = ''
        if not self._no_spoof:
            add = " and overlaid SPOOF scenario"
        title = self._title if self._title is not None else \
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
679
            ('DET: LICIT' + add)
680
681
        if title.replace(' ', ''):
            mpl.title(title)
682
683
684
        mpl.xlabel(self._x_label or "False Acceptance Rate (%)")
        mpl.ylabel(self._y_label or "False Rejection Rate (%)")
        mpl.grid(True, color=self._grid_color)
685
686
        if self._disp_legend:
            mpl.legend(loc=self._legend_loc)
687
688
689
690
691
692
693
694
695
696
697
        self._set_axis()
        fig = mpl.gcf()
        mpl.xticks(rotation=self._x_rotation)
        mpl.tick_params(axis='both', which='major', labelsize=4)
        for tick in mpl.gca().xaxis.get_major_ticks():
            tick.label.set_fontsize(6)
        for tick in mpl.gca().yaxis.get_major_ticks():
            tick.label.set_fontsize(6)

        self._pdf_page.savefig(fig)

Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
698
        # do not want to close PDF when running evaluate
699
        if 'PdfPages' in self._ctx.meta and \
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
700
                ('closef' not in self._ctx.meta or self._ctx.meta['closef']):
701
702
703
704
705
706
707
            self._pdf_page.close()

    def _set_axis(self):
        if self._axlim is not None and None not in self._axlim:
            det_axis(self._axlim)
        else:
            det_axis([0.01, 99, 0.01, 99])
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
708

709

Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
710
711
class FmrIapmr(PadPlot):
    '''FMR vs IAPMR'''
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
712

Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
713
714
    def __init__(self, ctx, scores, evaluation, func_load):
        super(FmrIapmr, self).__init__(ctx, scores, evaluation, func_load)
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
715
        self._eval = True  # always eval data with EPC
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
716
717
718
        self._split = False
        self._nb_figs = 1
        self._semilogx = False if 'semilogx' not in ctx.meta else\
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
719
            ctx.meta['semilogx']
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
        if self._min_arg != 4:
            raise click.BadParameter("You must provide 4 scores files:{licit,"
                                     "spoof}/{dev,eval}")

    def compute(self, idx, input_scores, input_names):
        ''' Implements plots'''
        licit_eval_neg = input_scores[1][0]
        licit_eval_pos = input_scores[1][1]
        spoof_eval_neg = input_scores[3][0]
        fmr_list = np.linspace(0, 1, 100)
        iapmr_list = []
        for i, fmr in enumerate(fmr_list):
            thr = far_threshold(licit_eval_neg, licit_eval_pos, fmr, True)
            iapmr_list.append(farfrr(spoof_eval_neg, licit_eval_pos, thr)[0])
            # re-calculate fmr since threshold might give a different result
            # for fmr.
            fmr_list[i] = farfrr(licit_eval_neg, licit_eval_pos, thr)[0]
737
        label = self._legends[idx] if self._legends is not None else \
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
738
            '(%s/%s)' % (input_names[1], input_names[3])
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
739
740
741
742
743
744
745
746
        if self._semilogx:
            mpl.semilogx(fmr_list, iapmr_list, label=label)
        else:
            mpl.plot(fmr_list, iapmr_list, label=label)

    def end_process(self):
        ''' Set title, legend, axis labels, grid colors, save figures and
        close pdf is needed '''
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
747
        # only for plots
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
748
        title = self._title if self._title is not None else "FMR vs IAPMR"
749
750
        if title.replace(' ', ''):
            mpl.title(title)
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
751
752
753
        mpl.xlabel(self._x_label or "False Match Rate (%)")
        mpl.ylabel(self._y_label or "IAPMR (%)")
        mpl.grid(True, color=self._grid_color)
754
755
        if self._disp_legend:
            mpl.legend(loc=self._legend_loc)
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
756
757
758
759
760
761
762
        self._set_axis()
        fig = mpl.gcf()
        mpl.xticks(rotation=self._x_rotation)
        mpl.tick_params(axis='both', which='major', labelsize=4)

        self._pdf_page.savefig(fig)

Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
763
        # do not want to close PDF when running evaluate
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
764
        if 'PdfPages' in self._ctx.meta and \
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
765
                ('closef' not in self._ctx.meta or self._ctx.meta['closef']):
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
766
            self._pdf_page.close()