figure.py 26.6 KB
Newer Older
1
2
3
4
'''Runs error analysis on score sets, outputs metrics and plots'''

import click
import numpy as np
5
import matplotlib.pyplot as mpl
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
6
import bob.measure.script.figure as measure_figure
7
from tabulate import tabulate
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
8
from bob.measure.utils import get_fta_list
9
from bob.measure import (
10
    far_threshold, eer_threshold, min_hter_threshold, farfrr, epc, ppndf
11
)
12
from bob.measure.plot import (det, det_axis)
13
14
15
16
from . import error_utils

ALL_CRITERIA = ('bpcer20', 'eer', 'min-hter')

17

18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
def calc_threshold(method, neg, pos):
    """Calculates the threshold based on the given method.
    The scores should be sorted!

    Parameters
    ----------
    method : str
        One of ``bpcer201``, ``eer``, ``min-hter``.
    neg : array_like
        The negative scores. They should be sorted!
    pos : array_like
        The positive scores. They should be sorted!

    Returns
    -------
    float
        The calculated threshold.

    Raises
    ------
    ValueError
        If method is unknown.
    """
    method = method.lower()
    if method == 'bpcer20':
        threshold = far_threshold(neg, pos, 0.05, True)
    elif method == 'eer':
        threshold = eer_threshold(neg, pos, True)
    elif method == 'min-hter':
        threshold = min_hter_threshold(neg, pos, True)
    else:
        raise ValueError("Unknown threshold criteria: {}".format(method))

    return threshold

53

54
55
56
57
58
class Metrics(measure_figure.Metrics):
    def __init__(self, ctx, scores, evaluation, func_load):
        super(Metrics, self).__init__(ctx, scores, evaluation, func_load)

    ''' Compute metrics from score files'''
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
59

60
    def compute(self, idx, input_scores, input_names):
61
        ''' Compute metrics for the given criteria'''
62
63
64
65
66
67
        neg_list, pos_list, _ = get_fta_list(input_scores)
        dev_neg, dev_pos = neg_list[0], pos_list[0]
        dev_file = input_names[0]
        if self._eval:
            eval_neg, eval_pos = neg_list[1], pos_list[1]
            eval_file = input_names[1]
68

69
        title = self._legends[idx] if self._legends is not None else None
70
        headers = ['' or title, 'Development %s' % dev_file]
71
        if self._eval:
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
            headers.append('Eval. % s' % eval_file)
        for m in ALL_CRITERIA:
            raws = []
            threshold = calc_threshold(m, dev_neg, dev_pos)
            click.echo("\nThreshold of %f selected with the %s criteria" % (
                threshold, m))
            apcer, bpcer = farfrr(dev_neg, dev_pos, threshold)
            raws.append(['BPCER20', '{:>5.1f}%'.format(apcer * 100)])
            raws.append(['EER', '{:>5.1f}%'.format(bpcer * 100)])
            raws.append(['min-HTER', '{:>5.1f}%'.format((apcer + bpcer) * 50)])
            if self._eval and eval_neg is not None:
                apcer, bpcer = farfrr(eval_neg, eval_pos, threshold)
                raws[0].append('{:>5.1f}%'.format(apcer * 100))
                raws[1].append('{:>5.1f}%'.format(bpcer * 100))
                raws[2].append('{:>5.1f}%'.format((apcer + bpcer) * 50))

            click.echo(
                tabulate(raws, headers, self._tablefmt),
                file=self.log_file
            )

93

94
95
96
97
98
class MetricsVuln(measure_figure.Metrics):
    def __init__(self, ctx, scores, evaluation, func_load):
        super(MetricsVuln, self).__init__(ctx, scores, evaluation, func_load)

    ''' Compute metrics from score files'''
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
99

100
101
102
103
104
105
    def compute(self, idx, input_scores, input_names):
        ''' Compute metrics for the given criteria'''
        neg_list, pos_list, _ = get_fta_list(input_scores)
        dev_neg, dev_pos = neg_list[0], pos_list[0]
        criter = self._criterion or 'eer'
        threshold = calc_threshold(criter, dev_neg, dev_pos) \
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
106
            if self._thres is None else self._thres[idx]
107
108
109
110
111
112
        far, frr = farfrr(neg_list[1], pos_list[1], threshold)
        iapmr, _ = farfrr(neg_list[3], pos_list[1], threshold)
        title = self._legends[idx] if self._legends is not None else None
        headers = ['' or title, '%s (threshold=%.2g)' %
                   (criter.upper(), threshold)]
        rows = []
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
113
114
115
116
        rows.append(['FMR (%)', '{:>5.1f}%'.format(100 * far)])
        rows.append(['FMNR (%)', '{:>5.1f}%'.format(frr * 100)])
        rows.append(['HTER (%)', '{:>5.1f}%'.format(50 * (far + frr))])
        rows.append(['IAPMR (%)', '{:>5.1f}%'.format(100 * iapmr)])
117
118
119
120
121
122
        click.echo(
            tabulate(rows, headers, self._tablefmt),
            file=self.log_file
        )


123
124
125
126
127
128
class HistPad(measure_figure.Hist):
    ''' Histograms for PAD '''

    def _setup_hist(self, neg, pos):
        self._title_base = 'PAD'
        self._density_hist(
129
            pos[0], n=0, label='Bona Fide', color='C1'
130
131
        )
        self._density_hist(
132
            neg[0], n=1, label='Presentation attack', alpha=0.4, color='C7',
133
            hatch='\\\\'
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
        )


def _iapmr_dot(threshold, iapmr, real_data, **kwargs):
    # plot a dot on threshold versus IAPMR line and show IAPMR as a number
    axlim = mpl.axis()
    mpl.plot(threshold, 100. * iapmr, 'o', color='C3', **kwargs)
    if not real_data:
        mpl.annotate(
            'IAPMR at\noperating point',
            xy=(threshold, 100. * iapmr),
            xycoords='data',
            xytext=(0.85, 0.6),
            textcoords='axes fraction',
            color='black',
            size='large',
            arrowprops=dict(facecolor='black', shrink=0.05, width=2),
            horizontalalignment='center',
            verticalalignment='top',
        )
    else:
        mpl.text(threshold + (threshold - axlim[0]) / 12, 100. * iapmr,
                 '%.1f%%' % (100. * iapmr,), color='C3')

158

159
160
161
162
163
164
def _iapmr_line_plot(scores, n_points=100, **kwargs):
    axlim = mpl.axis()
    step = (axlim[1] - axlim[0]) / float(n_points)
    thres = [(k * step) + axlim[0] for k in range(2, n_points - 1)]
    mix_prob_y = []
    for k in thres:
165
        mix_prob_y.append(100. * error_utils.calc_pass_rate(k, scores))
166
167
168

    mpl.plot(thres, mix_prob_y, label='IAPMR', color='C3', **kwargs)

169

170
171
172
173
def _iapmr_plot(scores, threshold, iapmr, real_data, **kwargs):
    _iapmr_dot(threshold, iapmr, real_data, **kwargs)
    _iapmr_line_plot(scores, n_points=100, **kwargs)

174

175
176
177
178
179
180
class HistVuln(measure_figure.Hist):
    ''' Histograms for vulnerability '''

    def _setup_hist(self, neg, pos):
        self._title_base = 'Vulnerability'
        self._density_hist(
181
            pos[0], n=0, label='Genuine', color='C2'
182
183
        )
        self._density_hist(
184
            neg[0], n=1, label='Zero-effort impostors', alpha=0.8, color='C0'
185
186
        )
        self._density_hist(
187
            neg[1], n=2, label='Presentation attack', alpha=0.4, color='C7',
188
            hatch='\\\\'
189
190
        )

191
    def _lines(self, threshold, label, neg, pos, idx, **kwargs):
192
        if 'iapmr_line' not in self._ctx.meta or self._ctx.meta['iapmr_line']:
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
193
            # plot vertical line
194
            super(HistVuln, self)._lines(threshold, label, neg, pos, idx)
195

Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
196
            # plot iapmr_line
197
198
199
200
201
            iapmr, _ = farfrr(neg[1], pos[0], threshold)
            ax2 = mpl.twinx()
            # we never want grid lines on axis 2
            ax2.grid(False)
            real_data = True if 'real_data' not in self._ctx.meta else \
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
202
                self._ctx.meta['real_data']
203
            _iapmr_plot(neg[1], threshold, iapmr, real_data=real_data)
204
205
            n = idx % self._step_print
            col = n % self._ncols
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
206
207
            rest_print = self.n_systems - \
                int(idx / self._step_print) * self._step_print
208
209
            if col == self._ncols - 1 or n == rest_print - 1:
                ax2.set_ylabel("IAPMR (%)", color='C3')
210
211
212
213
            ax2.tick_params(axis='y', colors='red')
            ax2.yaxis.label.set_color('red')
            ax2.spines['right'].set_color('red')

214

215
216
class PadPlot(measure_figure.PlotBase):
    '''Base class for PAD plots'''
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
217

218
219
    def __init__(self, ctx, scores, evaluation, func_load):
        super(PadPlot, self).__init__(ctx, scores, evaluation, func_load)
220
        mpl.rcParams['figure.constrained_layout.use'] = self._clayout
221
222
223

    def end_process(self):
        '''Close pdf '''
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
224
        # do not want to close PDF when running evaluate
225
226
227
228
229
        if 'PdfPages' in self._ctx.meta and \
           ('closef' not in self._ctx.meta or self._ctx.meta['closef']):
            self._pdf_page.close()

    def _plot_legends(self):
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
230
        # legends for all axes
231
232
233
234
235
236
        lines = []
        labels = []
        for ax in mpl.gcf().get_axes():
            li, la = ax.get_legend_handles_labels()
            lines += li
            labels += la
237
238
239
240
        if self._disp_legend:
            mpl.gca().legend(lines, labels, loc=self._legend_loc,
                             fancybox=True, framealpha=0.5)

241
242
243

class Epc(PadPlot):
    ''' Handles the plotting of EPC '''
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
244

245
246
247
    def __init__(self, ctx, scores, evaluation, func_load):
        super(Epc, self).__init__(ctx, scores, evaluation, func_load)
        self._iapmr = True if 'iapmr' not in self._ctx.meta else \
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
248
            self._ctx.meta['iapmr']
249
250
251
252
        self._title = self._title or ('EPC and IAPMR' if self._iapmr else
                                      'EPC')
        self._x_label = self._x_label or r"Weight $\beta$"
        self._y_label = self._y_label or "WER (%)"
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
253
        self._eval = True  # always eval data with EPC
254
255
256
        self._split = False
        self._nb_figs = 1

257
258
259
260
261
        if self._min_arg != 4:
            raise click.BadParameter("You must provide 4 scores files:{licit,"
                                     "spoof}/{dev,eval}")

    def compute(self, idx, input_scores, input_names):
262
        ''' Plot EPC for PAD'''
263
264
265
266
267
        licit_dev_neg = input_scores[0][0]
        licit_dev_pos = input_scores[0][1]
        licit_eval_neg = input_scores[1][0]
        licit_eval_pos = input_scores[1][1]
        spoof_eval_neg = input_scores[3][0]
268
        mpl.gcf().clear()
269
        epc_baseline = epc(
270
271
272
273
274
275
276
            licit_dev_neg, licit_dev_pos, licit_eval_neg,
            licit_eval_pos, 100
        )
        mpl.plot(
            epc_baseline[:, 0], [100. * k for k in epc_baseline[:, 1]],
            color='C0',
            label=self._label(
277
                'WER', '%s-%s' % (input_names[0], input_names[1]), idx
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
            ),
            linestyle='-'
        )
        mpl.xlabel(self._x_label)
        mpl.ylabel(self._y_label)
        if self._iapmr:
            mix_prob_y = []
            for k in epc_baseline[:, 2]:
                prob_attack = sum(
                    1 for i in spoof_eval_neg if i >= k
                ) / float(spoof_eval_neg.size)
                mix_prob_y.append(100. * prob_attack)

            mpl.gca().set_axisbelow(True)
            prob_ax = mpl.gca().twinx()
            mpl.plot(
                epc_baseline[:, 0],
                mix_prob_y,
                color='C3',
                linestyle='-',
                label=self._label(
299
                    'IAPMR', '%s-%s' % (input_names[0], input_names[1]), idx
300
301
302
303
304
305
306
307
308
                )
            )
            prob_ax.set_yticklabels(prob_ax.get_yticks())
            prob_ax.tick_params(axis='y', colors='red')
            prob_ax.yaxis.label.set_color('red')
            prob_ax.spines['right'].set_color('red')
            ylabels = prob_ax.get_yticks()
            prob_ax.yaxis.set_ticklabels(["%.0f" % val for val in ylabels])
            prob_ax.set_axisbelow(True)
309
        title = self._legends[idx] if self._legends is not None else self._title
310
311
        if title.replace(' ', ''):
            mpl.title(title)
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
312
        # legends for all axes
313
314
315
316
        self._plot_legends()
        mpl.xticks(rotation=self._x_rotation)
        self._pdf_page.savefig(mpl.gcf())

317

318
319
class Epsc(PadPlot):
    ''' Handles the plotting of EPSC '''
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
320

321
322
323
324
    def __init__(self, ctx, scores, evaluation, func_load,
                 criteria, var_param, fixed_param):
        super(Epsc, self).__init__(ctx, scores, evaluation, func_load)
        self._iapmr = False if 'iapmr' not in self._ctx.meta else \
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
325
            self._ctx.meta['iapmr']
326
        self._wer = True if 'wer' not in self._ctx.meta else \
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
327
            self._ctx.meta['wer']
328
329
330
        self._criteria = 'eer' if criteria is None else criteria
        self._var_param = "omega" if var_param is None else var_param
        self._fixed_param = 0.5 if fixed_param is None else fixed_param
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
331
        self._eval = True  # always eval data with EPC
332
333
        self._split = False
        self._nb_figs = 1
334
335
336
337
338
        self._title = ''

        if self._min_arg != 4:
            raise click.BadParameter("You must provide 4 scores files:{licit,"
                                     "spoof}/{dev,eval}")
339

340
    def compute(self, idx, input_scores, input_names):
341
        ''' Plot EPSC for PAD'''
342
343
344
345
346
347
348
349
        licit_dev_neg = input_scores[0][0]
        licit_dev_pos = input_scores[0][1]
        licit_eval_neg = input_scores[1][0]
        licit_eval_pos = input_scores[1][1]
        spoof_dev_neg = input_scores[2][0]
        spoof_dev_pos = input_scores[2][1]
        spoof_eval_neg = input_scores[3][0]
        spoof_eval_pos = input_scores[3][1]
350
        title = self._legends[idx] if self._legends is not None else None
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370

        mpl.gcf().clear()
        points = 10

        if self._var_param == 'omega':
            omega, beta, thrs = error_utils.epsc_thresholds(
                licit_dev_neg,
                licit_dev_pos,
                spoof_dev_neg,
                spoof_dev_pos,
                points=points,
                criteria=self._criteria,
                beta=self._fixed_param)
        else:
            omega, beta, thrs = error_utils.epsc_thresholds(
                licit_dev_neg,
                licit_dev_pos,
                spoof_dev_neg,
                spoof_dev_pos,
                points=points,
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
371
                criteria=self._criteria,
372
373
374
375
376
377
378
                omega=self._fixed_param
            )

        errors = error_utils.all_error_rates(
            licit_eval_neg, licit_eval_pos, spoof_eval_neg,
            spoof_eval_pos, thrs, omega, beta
        )  # error rates are returned in a list in the
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
379
        # following order: frr, far, IAPMR, far_w, wer_w
380
381
382
383

        ax1 = mpl.subplot(
            111
        )  # EPC like curves for FVAS fused scores for weighted error rates
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
384
        # between the negatives (impostors and Presentation attacks)
385
386
387
388
389
390
391
392
        if self._wer:
            if self._var_param == 'omega':
                mpl.plot(
                    omega,
                    100. * errors[4].flatten(),
                    color='C0',
                    linestyle='-',
                    label=r"WER$_{\omega,\beta}$")
393
                mpl.xlabel(self._x_label or r"Weight $\omega$")
394
395
396
397
398
399
400
            else:
                mpl.plot(
                    beta,
                    100. * errors[4].flatten(),
                    color='C0',
                    linestyle='-',
                    label=r"WER$_{\omega,\beta}$")
401
402
                mpl.xlabel(self._x_label or r"Weight $\beta$")
            mpl.ylabel(self._y_label or r"WER$_{\omega,\beta}$ (%)")
403
404
405
406
407
408
409
410
411
412
413
414
415

        if self._iapmr:
            axis = mpl.gca()
            if self._wer:
                axis = mpl.twinx()
                axis.grid(False)
            if self._var_param == 'omega':
                mpl.plot(
                    omega,
                    100. * errors[2].flatten(),
                    color='C3',
                    linestyle='-',
                    label='IAPMR')
416
                mpl.xlabel(self._x_label or r"Weight $\omega$")
417
418
419
420
421
422
423
            else:
                mpl.plot(
                    beta,
                    100. * errors[2].flatten(),
                    color='C3',
                    linestyle='-',
                    label='IAPMR')
424
425
                mpl.xlabel(self._x_label or r"Weight $\beta$")
            mpl.ylabel(self._y_label or r"IAPMR  (%)")
426
427
428
429
430
431
432
            if self._wer:
                axis.set_yticklabels(axis.get_yticks())
                axis.tick_params(axis='y', colors='red')
                axis.yaxis.label.set_color('red')
                axis.spines['right'].set_color('red')

        if self._var_param == 'omega':
Theophile GENTILHOMME's avatar
Bug fix    
Theophile GENTILHOMME committed
433
            if title is not None and title.replace(' ', ''):
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
434
                mpl.title(title or (r"EPSC with $\beta$ = %.2f" %
435
                                    self._fixed_param))
436
        else:
Theophile GENTILHOMME's avatar
Bug fix    
Theophile GENTILHOMME committed
437
            if title is not None and title.replace(' ', ''):
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
438
                mpl.title(title or (r"EPSC with $\omega$ = %.2f" %
439
                                    self._fixed_param))
440
441
442
443
444
445

        mpl.grid()
        self._plot_legends()
        ax1.set_xticklabels(ax1.get_xticks())
        ax1.set_yticklabels(ax1.get_yticks())
        mpl.xticks(rotation=self._x_rotation)
446
        self._pdf_page.savefig()
447

448

449
450
class Epsc3D(Epsc):
    ''' 3D EPSC plots for PAD'''
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
451

452
    def compute(self, idx, input_scores, input_names):
453
        ''' Implements plots'''
454
455
456
457
458
459
460
461
        licit_dev_neg = input_scores[0][0]
        licit_dev_pos = input_scores[0][1]
        licit_eval_neg = input_scores[1][0]
        licit_eval_pos = input_scores[1][1]
        spoof_dev_neg = input_scores[2][0]
        spoof_dev_pos = input_scores[2][1]
        spoof_eval_neg = input_scores[3][0]
        spoof_eval_pos = input_scores[3][1]
462

Theophile GENTILHOMME's avatar
Bug fix    
Theophile GENTILHOMME committed
463
        title = self._legends[idx] if self._legends is not None else "3D EPSC"
464

465
466
        mpl.rcParams.pop('key', None)

467
        mpl.gcf().clear()
468
        mpl.gcf().set_constrained_layout(self._clayout)
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490

        from mpl_toolkits.mplot3d import Axes3D
        from matplotlib import cm

        points = 10

        omega, beta, thrs = error_utils.epsc_thresholds(
            licit_dev_neg,
            licit_dev_pos,
            spoof_dev_neg,
            spoof_dev_pos,
            points=points,
            criteria=self._criteria)

        errors = error_utils.all_error_rates(
            licit_eval_neg, licit_eval_pos, spoof_eval_neg, spoof_eval_pos,
            thrs, omega, beta
        )
        # error rates are returned in a list as 2D numpy.ndarrays in
        # the following order: frr, far, IAPMR, far_w, wer_wb, hter_wb
        wer_errors = 100 * errors[2 if self._iapmr else 4]

491
        ax1 = mpl.gcf().add_subplot(111, projection='3d')
492
493
494
495
496
497
498
499
500
501
502

        W, B = np.meshgrid(omega, beta)

        ax1.plot_wireframe(
            W, B, wer_errors, cmap=cm.coolwarm, antialiased=False
        )  # surface

        if self._iapmr:
            ax1.azim = -30
            ax1.elev = 50

503
504
        ax1.set_xlabel(self._x_label or r"Weight $\omega$")
        ax1.set_ylabel(self._y_label or r"Weight $\beta$")
505
506
507
508
        ax1.set_zlabel(
            r"WER$_{\omega,\beta}$ (%)" if self._wer else "IAPMR (%)"
        )

509
        if title.replace(' ', ''):
Theophile GENTILHOMME's avatar
Bug fix    
Theophile GENTILHOMME committed
510
            mpl.title(title)
511
512
513
514
515

        ax1.set_xticklabels(ax1.get_xticks())
        ax1.set_yticklabels(ax1.get_yticks())
        ax1.set_zticklabels(ax1.get_zticks())

516
517
        self._pdf_page.savefig()

518

519
520
class Det(PadPlot):
    '''DET for PAD'''
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
521

522
523
    def __init__(self, ctx, scores, evaluation, func_load, criteria, real_data,
                no_spoof):
524
        super(Det, self).__init__(ctx, scores, evaluation, func_load)
525
526
        self._no_spoof = no_spoof
        self._criteria = criteria or 'eer'
527
528
        self._real_data = True if real_data is None else real_data

529
    def compute(self, idx, input_scores, input_names):
530
        ''' Implements plots'''
531
532
533
534
535
536
        licit_dev_neg = input_scores[0][0]
        licit_dev_pos = input_scores[0][1]
        licit_eval_neg = input_scores[1][0]
        licit_eval_pos = input_scores[1][1]
        spoof_eval_neg = input_scores[3][0] if len(input_scores) > 2 else None
        spoof_eval_pos = input_scores[3][1] if len(input_scores) > 2 else None
537
538
539
540
        det(
            licit_eval_neg,
            licit_eval_pos,
            self._points,
541
            color='C0',
542
            linestyle='-',
543
            label=self._label("licit", input_names[0], idx)
544
545
546
547
548
549
        )
        if not self._no_spoof and spoof_eval_neg is not None:
            det(
                spoof_eval_neg,
                spoof_eval_pos,
                self._points,
550
551
                color='C3',
                linestyle=':',
552
                label=self._label("spoof", input_names[3], idx)
553
554
            )

555
        if self._criteria is None or self._no_spoof:
556
557
558
559
560
561
562
563
564
565
            return

        thres_baseline = calc_threshold(
            self._criteria, licit_dev_neg, licit_dev_pos
        )

        axlim = mpl.axis()

        farfrr_licit = farfrr(
            licit_eval_neg, licit_eval_pos,
566
567
            thres_baseline
        )  # calculate test frr @ EER (licit scenario)
568
569
        farfrr_spoof = farfrr(
            spoof_eval_neg, spoof_eval_pos,
570
571
            thres_baseline
        )  # calculate test frr @ EER (spoof scenario)
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
        farfrr_licit_det = [
            ppndf(i) for i in farfrr_licit
        ]
        # find the FAR and FRR values that need to be plotted on normal deviate
        # scale
        farfrr_spoof_det = [
            ppndf(i) for i in farfrr_spoof
        ]
        # find the FAR and FRR values that need to be plotted on normal deviate
        # scale
        if not self._real_data:
            mpl.axhline(
                y=farfrr_licit_det[1],
                xmin=axlim[2],
                xmax=axlim[3],
                color='k',
                linestyle='--',
589
                label="FRR @ EER")
590
591
592
593
594
595
596
597
        else:
            mpl.axhline(
                y=farfrr_licit_det[1],
                xmin=axlim[0],
                xmax=axlim[1],
                color='k',
                linestyle='--',
                label="FRR = %.2f%%" %
598
                (farfrr_licit[1] * 100))
599
600
601
602
603

        mpl.plot(
            farfrr_licit_det[0],
            farfrr_licit_det[1],
            'o',
604
605
606
            color='C0',
            markersize=9
        )  # FAR point, licit scenario
607
608
609
610
        mpl.plot(
            farfrr_spoof_det[0],
            farfrr_spoof_det[1],
            'o',
611
612
613
            color='C3',
            markersize=9
        )  # FAR point, spoof scenario
614
615
616

        # annotate the FAR points
        xyannotate_licit = [
617
618
            0.6*farfrr_licit_det[0],
            0.6*farfrr_licit_det[1],
619
620
        ]
        xyannotate_spoof = [
621
622
            0.6*farfrr_spoof_det[0],
            0.6*farfrr_spoof_det[1],
623
624
625
626
        ]

        if not self._real_data:
            mpl.annotate(
627
                'FMR @ operating point',
628
629
630
631
632
                xy=(farfrr_licit_det[0], farfrr_licit_det[1]),
                xycoords='data',
                xytext=(xyannotate_licit[0], xyannotate_licit[1]),
                color=self._colors[idx])
            mpl.annotate(
633
                'IAPMR @ operating point',
634
635
636
637
638
639
640
641
642
643
                xy=(farfrr_spoof_det[0], farfrr_spoof_det[1]),
                xycoords='data',
                xytext=(xyannotate_spoof[0], xyannotate_spoof[1]),
                color=self._colors[idx])
        else:
            mpl.annotate(
                'FAR=%.2f%%' % (farfrr_licit[0] * 100),
                xy=(farfrr_licit_det[0], farfrr_licit_det[1]),
                xycoords='data',
                xytext=(xyannotate_licit[0], xyannotate_licit[1]),
644
                color='C0',
645
646
                size='large')
            mpl.annotate(
647
                'IAPMR=%.2f%%' % (farfrr_spoof[0] * 100),
648
649
650
                xy=(farfrr_spoof_det[0], farfrr_spoof_det[1]),
                xycoords='data',
                xytext=(xyannotate_spoof[0], xyannotate_spoof[1]),
651
                color='C3',
652
653
654
655
656
                size='large')

    def end_process(self):
        ''' Set title, legend, axis labels, grid colors, save figures and
        close pdf is needed '''
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
657
        # only for plots
658
659
660
661
        add = ''
        if not self._no_spoof:
            add = " and overlaid SPOOF scenario"
        title = self._title if self._title is not None else \
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
662
            ('DET: LICIT' + add)
663
664
        if title.replace(' ', ''):
            mpl.title(title)
665
666
667
        mpl.xlabel(self._x_label or "False Acceptance Rate (%)")
        mpl.ylabel(self._y_label or "False Rejection Rate (%)")
        mpl.grid(True, color=self._grid_color)
668
669
        if self._disp_legend:
            mpl.legend(loc=self._legend_loc)
670
671
672
673
674
675
676
677
678
679
680
        self._set_axis()
        fig = mpl.gcf()
        mpl.xticks(rotation=self._x_rotation)
        mpl.tick_params(axis='both', which='major', labelsize=4)
        for tick in mpl.gca().xaxis.get_major_ticks():
            tick.label.set_fontsize(6)
        for tick in mpl.gca().yaxis.get_major_ticks():
            tick.label.set_fontsize(6)

        self._pdf_page.savefig(fig)

Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
681
        # do not want to close PDF when running evaluate
682
        if 'PdfPages' in self._ctx.meta and \
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
683
                ('closef' not in self._ctx.meta or self._ctx.meta['closef']):
684
685
686
687
688
689
690
            self._pdf_page.close()

    def _set_axis(self):
        if self._axlim is not None and None not in self._axlim:
            det_axis(self._axlim)
        else:
            det_axis([0.01, 99, 0.01, 99])
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
691

692

Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
693
694
class FmrIapmr(PadPlot):
    '''FMR vs IAPMR'''
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
695

Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
696
697
    def __init__(self, ctx, scores, evaluation, func_load):
        super(FmrIapmr, self).__init__(ctx, scores, evaluation, func_load)
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
698
        self._eval = True  # always eval data with EPC
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
699
700
701
        self._split = False
        self._nb_figs = 1
        self._semilogx = False if 'semilogx' not in ctx.meta else\
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
702
            ctx.meta['semilogx']
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
        if self._min_arg != 4:
            raise click.BadParameter("You must provide 4 scores files:{licit,"
                                     "spoof}/{dev,eval}")

    def compute(self, idx, input_scores, input_names):
        ''' Implements plots'''
        licit_eval_neg = input_scores[1][0]
        licit_eval_pos = input_scores[1][1]
        spoof_eval_neg = input_scores[3][0]
        fmr_list = np.linspace(0, 1, 100)
        iapmr_list = []
        for i, fmr in enumerate(fmr_list):
            thr = far_threshold(licit_eval_neg, licit_eval_pos, fmr, True)
            iapmr_list.append(farfrr(spoof_eval_neg, licit_eval_pos, thr)[0])
            # re-calculate fmr since threshold might give a different result
            # for fmr.
            fmr_list[i] = farfrr(licit_eval_neg, licit_eval_pos, thr)[0]
720
        label = self._legends[idx] if self._legends is not None else \
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
721
            '(%s/%s)' % (input_names[1], input_names[3])
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
722
723
724
725
726
727
728
729
        if self._semilogx:
            mpl.semilogx(fmr_list, iapmr_list, label=label)
        else:
            mpl.plot(fmr_list, iapmr_list, label=label)

    def end_process(self):
        ''' Set title, legend, axis labels, grid colors, save figures and
        close pdf is needed '''
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
730
        # only for plots
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
731
        title = self._title if self._title is not None else "FMR vs IAPMR"
732
733
        if title.replace(' ', ''):
            mpl.title(title)
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
734
735
736
        mpl.xlabel(self._x_label or "False Match Rate (%)")
        mpl.ylabel(self._y_label or "IAPMR (%)")
        mpl.grid(True, color=self._grid_color)
737
738
        if self._disp_legend:
            mpl.legend(loc=self._legend_loc)
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
739
740
741
742
743
744
745
        self._set_axis()
        fig = mpl.gcf()
        mpl.xticks(rotation=self._x_rotation)
        mpl.tick_params(axis='both', which='major', labelsize=4)

        self._pdf_page.savefig(fig)

Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
746
        # do not want to close PDF when running evaluate
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
747
        if 'PdfPages' in self._ctx.meta and \
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
748
                ('closef' not in self._ctx.meta or self._ctx.meta['closef']):
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
749
            self._pdf_page.close()