figure.py 28.7 KB
Newer Older
1
2
3
4
'''Runs error analysis on score sets, outputs metrics and plots'''

import click
import numpy as np
5
import matplotlib.pyplot as mpl
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
6
import bob.measure.script.figure as measure_figure
7
import bob.bio.base.script.figure as bio_figure
8
from tabulate import tabulate
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
9
from bob.measure.utils import get_fta_list
10
from bob.measure import (
11
    far_threshold, eer_threshold, min_hter_threshold, farfrr, epc, ppndf
12
)
13
from bob.measure.plot import (det, det_axis, roc_for_far, log_values)
14
15
16
17
from . import error_utils

ALL_CRITERIA = ('bpcer20', 'eer', 'min-hter')

18

19
20
21
22
23
24
25
def calc_threshold(method, neg, pos):
    """Calculates the threshold based on the given method.
    The scores should be sorted!

    Parameters
    ----------
    method : str
26
        One of ``bpcer20``, ``eer``, ``min-hter``.
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
    neg : array_like
        The negative scores. They should be sorted!
    pos : array_like
        The positive scores. They should be sorted!

    Returns
    -------
    float
        The calculated threshold.

    Raises
    ------
    ValueError
        If method is unknown.
    """
    method = method.lower()
    if method == 'bpcer20':
        threshold = far_threshold(neg, pos, 0.05, True)
    elif method == 'eer':
        threshold = eer_threshold(neg, pos, True)
    elif method == 'min-hter':
        threshold = min_hter_threshold(neg, pos, True)
    else:
        raise ValueError("Unknown threshold criteria: {}".format(method))

    return threshold

54

55
56
57
58
59
class Metrics(measure_figure.Metrics):
    def __init__(self, ctx, scores, evaluation, func_load):
        super(Metrics, self).__init__(ctx, scores, evaluation, func_load)

    ''' Compute metrics from score files'''
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
60

61
    def compute(self, idx, input_scores, input_names):
62
        ''' Compute metrics for the given criteria'''
63
64
65
66
67
68
        neg_list, pos_list, _ = get_fta_list(input_scores)
        dev_neg, dev_pos = neg_list[0], pos_list[0]
        dev_file = input_names[0]
        if self._eval:
            eval_neg, eval_pos = neg_list[1], pos_list[1]
            eval_file = input_names[1]
69

70
        title = self._legends[idx] if self._legends is not None else None
71
        headers = ['' or title, 'Development %s' % dev_file]
72
        if self._eval:
73
74
75
76
77
78
79
            headers.append('Eval. % s' % eval_file)
        for m in ALL_CRITERIA:
            raws = []
            threshold = calc_threshold(m, dev_neg, dev_pos)
            click.echo("\nThreshold of %f selected with the %s criteria" % (
                threshold, m))
            apcer, bpcer = farfrr(dev_neg, dev_pos, threshold)
80
            raws.append(['APCER', '{:>5.1f}%'.format(apcer * 100)])
81
            raws.append(['BPCER', '{:>5.1f}%'.format(bpcer * 100)])
82
            raws.append(['ACER', '{:>5.1f}%'.format((apcer + bpcer) * 50)])
83
84
85
86
87
88
89
90
91
92
93
            if self._eval and eval_neg is not None:
                apcer, bpcer = farfrr(eval_neg, eval_pos, threshold)
                raws[0].append('{:>5.1f}%'.format(apcer * 100))
                raws[1].append('{:>5.1f}%'.format(bpcer * 100))
                raws[2].append('{:>5.1f}%'.format((apcer + bpcer) * 50))

            click.echo(
                tabulate(raws, headers, self._tablefmt),
                file=self.log_file
            )

94

95
96
97
98
99
class MetricsVuln(measure_figure.Metrics):
    def __init__(self, ctx, scores, evaluation, func_load):
        super(MetricsVuln, self).__init__(ctx, scores, evaluation, func_load)

    ''' Compute metrics from score files'''
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
100

101
102
103
104
105
106
    def compute(self, idx, input_scores, input_names):
        ''' Compute metrics for the given criteria'''
        neg_list, pos_list, _ = get_fta_list(input_scores)
        dev_neg, dev_pos = neg_list[0], pos_list[0]
        criter = self._criterion or 'eer'
        threshold = calc_threshold(criter, dev_neg, dev_pos) \
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
107
            if self._thres is None else self._thres[idx]
108
109
110
111
112
113
        far, frr = farfrr(neg_list[1], pos_list[1], threshold)
        iapmr, _ = farfrr(neg_list[3], pos_list[1], threshold)
        title = self._legends[idx] if self._legends is not None else None
        headers = ['' or title, '%s (threshold=%.2g)' %
                   (criter.upper(), threshold)]
        rows = []
114
        rows.append(['APCER (%)', '{:>5.1f}%'.format(100 * far)])
115
        rows.append(['BPCER (%)', '{:>5.1f}%'.format(frr * 100)])
116
        rows.append(['ACER (%)', '{:>5.1f}%'.format(50 * (far + frr))])
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
117
        rows.append(['IAPMR (%)', '{:>5.1f}%'.format(100 * iapmr)])
118
119
120
121
122
123
        click.echo(
            tabulate(rows, headers, self._tablefmt),
            file=self.log_file
        )


124
125
126
127
128
129
class HistPad(measure_figure.Hist):
    ''' Histograms for PAD '''

    def _setup_hist(self, neg, pos):
        self._title_base = 'PAD'
        self._density_hist(
130
            pos[0], n=0, label='Bona Fide', color='C1'
131
132
        )
        self._density_hist(
133
            neg[0], n=1, label='Presentation attack', alpha=0.4, color='C7',
134
            hatch='\\\\'
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
        )


def _iapmr_dot(threshold, iapmr, real_data, **kwargs):
    # plot a dot on threshold versus IAPMR line and show IAPMR as a number
    axlim = mpl.axis()
    mpl.plot(threshold, 100. * iapmr, 'o', color='C3', **kwargs)
    if not real_data:
        mpl.annotate(
            'IAPMR at\noperating point',
            xy=(threshold, 100. * iapmr),
            xycoords='data',
            xytext=(0.85, 0.6),
            textcoords='axes fraction',
            color='black',
            size='large',
            arrowprops=dict(facecolor='black', shrink=0.05, width=2),
            horizontalalignment='center',
            verticalalignment='top',
        )
    else:
        mpl.text(threshold + (threshold - axlim[0]) / 12, 100. * iapmr,
                 '%.1f%%' % (100. * iapmr,), color='C3')

159

160
161
162
163
164
165
def _iapmr_line_plot(scores, n_points=100, **kwargs):
    axlim = mpl.axis()
    step = (axlim[1] - axlim[0]) / float(n_points)
    thres = [(k * step) + axlim[0] for k in range(2, n_points - 1)]
    mix_prob_y = []
    for k in thres:
166
        mix_prob_y.append(100. * error_utils.calc_pass_rate(k, scores))
167
168
169

    mpl.plot(thres, mix_prob_y, label='IAPMR', color='C3', **kwargs)

170

171
172
173
174
def _iapmr_plot(scores, threshold, iapmr, real_data, **kwargs):
    _iapmr_dot(threshold, iapmr, real_data, **kwargs)
    _iapmr_line_plot(scores, n_points=100, **kwargs)

175

176
177
178
179
180
181
class HistVuln(measure_figure.Hist):
    ''' Histograms for vulnerability '''

    def _setup_hist(self, neg, pos):
        self._title_base = 'Vulnerability'
        self._density_hist(
182
            pos[0], n=0, label='Genuine', color='C2'
183
184
        )
        self._density_hist(
185
            neg[0], n=1, label='Zero-effort impostors', alpha=0.8, color='C0'
186
187
        )
        self._density_hist(
188
            neg[1], n=2, label='Presentation attack', alpha=0.4, color='C7',
189
            hatch='\\\\'
190
191
        )

192
    def _lines(self, threshold, label, neg, pos, idx, **kwargs):
193
        if 'iapmr_line' not in self._ctx.meta or self._ctx.meta['iapmr_line']:
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
194
            # plot vertical line
195
            super(HistVuln, self)._lines(threshold, label, neg, pos, idx)
196

Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
197
            # plot iapmr_line
198
199
200
201
202
            iapmr, _ = farfrr(neg[1], pos[0], threshold)
            ax2 = mpl.twinx()
            # we never want grid lines on axis 2
            ax2.grid(False)
            real_data = True if 'real_data' not in self._ctx.meta else \
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
203
                self._ctx.meta['real_data']
204
            _iapmr_plot(neg[1], threshold, iapmr, real_data=real_data)
205
206
            n = idx % self._step_print
            col = n % self._ncols
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
207
208
            rest_print = self.n_systems - \
                int(idx / self._step_print) * self._step_print
209
210
            if col == self._ncols - 1 or n == rest_print - 1:
                ax2.set_ylabel("IAPMR (%)", color='C3')
211
212
213
214
            ax2.tick_params(axis='y', colors='red')
            ax2.yaxis.label.set_color('red')
            ax2.spines['right'].set_color('red')

215

216
217
class PadPlot(measure_figure.PlotBase):
    '''Base class for PAD plots'''
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
218

219
220
    def __init__(self, ctx, scores, evaluation, func_load):
        super(PadPlot, self).__init__(ctx, scores, evaluation, func_load)
221
        mpl.rcParams['figure.constrained_layout.use'] = self._clayout
222
223
224

    def end_process(self):
        '''Close pdf '''
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
225
        # do not want to close PDF when running evaluate
226
227
228
229
230
        if 'PdfPages' in self._ctx.meta and \
           ('closef' not in self._ctx.meta or self._ctx.meta['closef']):
            self._pdf_page.close()

    def _plot_legends(self):
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
231
        # legends for all axes
232
233
234
235
236
237
        lines = []
        labels = []
        for ax in mpl.gcf().get_axes():
            li, la = ax.get_legend_handles_labels()
            lines += li
            labels += la
238
239
240
241
        if self._disp_legend:
            mpl.gca().legend(lines, labels, loc=self._legend_loc,
                             fancybox=True, framealpha=0.5)

242
243
244

class Epc(PadPlot):
    ''' Handles the plotting of EPC '''
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
245

246
247
248
    def __init__(self, ctx, scores, evaluation, func_load):
        super(Epc, self).__init__(ctx, scores, evaluation, func_load)
        self._iapmr = True if 'iapmr' not in self._ctx.meta else \
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
249
            self._ctx.meta['iapmr']
250
251
252
253
        self._title = self._title or ('EPC and IAPMR' if self._iapmr else
                                      'EPC')
        self._x_label = self._x_label or r"Weight $\beta$"
        self._y_label = self._y_label or "WER (%)"
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
254
        self._eval = True  # always eval data with EPC
255
256
257
        self._split = False
        self._nb_figs = 1

258
259
260
261
262
        if self._min_arg != 4:
            raise click.BadParameter("You must provide 4 scores files:{licit,"
                                     "spoof}/{dev,eval}")

    def compute(self, idx, input_scores, input_names):
263
        ''' Plot EPC for PAD'''
264
265
266
267
268
        licit_dev_neg = input_scores[0][0]
        licit_dev_pos = input_scores[0][1]
        licit_eval_neg = input_scores[1][0]
        licit_eval_pos = input_scores[1][1]
        spoof_eval_neg = input_scores[3][0]
269
        mpl.gcf().clear()
270
        epc_baseline = epc(
271
272
273
274
275
276
277
            licit_dev_neg, licit_dev_pos, licit_eval_neg,
            licit_eval_pos, 100
        )
        mpl.plot(
            epc_baseline[:, 0], [100. * k for k in epc_baseline[:, 1]],
            color='C0',
            label=self._label(
278
                'WER', '%s-%s' % (input_names[0], input_names[1]), idx
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
            ),
            linestyle='-'
        )
        mpl.xlabel(self._x_label)
        mpl.ylabel(self._y_label)
        if self._iapmr:
            mix_prob_y = []
            for k in epc_baseline[:, 2]:
                prob_attack = sum(
                    1 for i in spoof_eval_neg if i >= k
                ) / float(spoof_eval_neg.size)
                mix_prob_y.append(100. * prob_attack)

            mpl.gca().set_axisbelow(True)
            prob_ax = mpl.gca().twinx()
            mpl.plot(
                epc_baseline[:, 0],
                mix_prob_y,
                color='C3',
                linestyle='-',
                label=self._label(
300
                    'IAPMR', '%s-%s' % (input_names[0], input_names[1]), idx
301
302
303
                )
            )
            prob_ax.set_yticklabels(prob_ax.get_yticks())
304
305
306
            prob_ax.tick_params(axis='y', colors='C3')
            prob_ax.yaxis.label.set_color('C3')
            prob_ax.spines['right'].set_color('C3')
307
308
            ylabels = prob_ax.get_yticks()
            prob_ax.yaxis.set_ticklabels(["%.0f" % val for val in ylabels])
309
            prob_ax.set_ylabel('IAPMR', color='C3')
310
            prob_ax.set_axisbelow(True)
311
        title = self._legends[idx] if self._legends is not None else self._title
312
313
        if title.replace(' ', ''):
            mpl.title(title)
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
314
        # legends for all axes
315
316
317
318
        self._plot_legends()
        mpl.xticks(rotation=self._x_rotation)
        self._pdf_page.savefig(mpl.gcf())

319

320
321
class Epsc(PadPlot):
    ''' Handles the plotting of EPSC '''
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
322

323
324
325
326
    def __init__(self, ctx, scores, evaluation, func_load,
                 criteria, var_param, fixed_param):
        super(Epsc, self).__init__(ctx, scores, evaluation, func_load)
        self._iapmr = False if 'iapmr' not in self._ctx.meta else \
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
327
            self._ctx.meta['iapmr']
328
        self._wer = True if 'wer' not in self._ctx.meta else \
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
329
            self._ctx.meta['wer']
330
331
332
        self._criteria = 'eer' if criteria is None else criteria
        self._var_param = "omega" if var_param is None else var_param
        self._fixed_param = 0.5 if fixed_param is None else fixed_param
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
333
        self._eval = True  # always eval data with EPC
334
335
        self._split = False
        self._nb_figs = 1
336
        self._title = ''
337
        self._sampling = ctx.meta.get('sampling', 5)
338
339
340
341

        if self._min_arg != 4:
            raise click.BadParameter("You must provide 4 scores files:{licit,"
                                     "spoof}/{dev,eval}")
342

343
    def compute(self, idx, input_scores, input_names):
344
        ''' Plot EPSC for PAD'''
345
346
347
348
349
350
351
352
        licit_dev_neg = input_scores[0][0]
        licit_dev_pos = input_scores[0][1]
        licit_eval_neg = input_scores[1][0]
        licit_eval_pos = input_scores[1][1]
        spoof_dev_neg = input_scores[2][0]
        spoof_dev_pos = input_scores[2][1]
        spoof_eval_neg = input_scores[3][0]
        spoof_eval_pos = input_scores[3][1]
353
        title = self._legends[idx] if self._legends is not None else None
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373

        mpl.gcf().clear()
        points = 10

        if self._var_param == 'omega':
            omega, beta, thrs = error_utils.epsc_thresholds(
                licit_dev_neg,
                licit_dev_pos,
                spoof_dev_neg,
                spoof_dev_pos,
                points=points,
                criteria=self._criteria,
                beta=self._fixed_param)
        else:
            omega, beta, thrs = error_utils.epsc_thresholds(
                licit_dev_neg,
                licit_dev_pos,
                spoof_dev_neg,
                spoof_dev_pos,
                points=points,
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
374
                criteria=self._criteria,
375
376
377
378
379
380
381
                omega=self._fixed_param
            )

        errors = error_utils.all_error_rates(
            licit_eval_neg, licit_eval_pos, spoof_eval_neg,
            spoof_eval_pos, thrs, omega, beta
        )  # error rates are returned in a list in the
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
382
        # following order: frr, far, IAPMR, far_w, wer_w
383
384
385
386

        ax1 = mpl.subplot(
            111
        )  # EPC like curves for FVAS fused scores for weighted error rates
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
387
        # between the negatives (impostors and Presentation attacks)
388
389
390
391
392
393
394
395
        if self._wer:
            if self._var_param == 'omega':
                mpl.plot(
                    omega,
                    100. * errors[4].flatten(),
                    color='C0',
                    linestyle='-',
                    label=r"WER$_{\omega,\beta}$")
396
                mpl.xlabel(self._x_label or r"Weight $\omega$")
397
398
399
400
401
402
403
            else:
                mpl.plot(
                    beta,
                    100. * errors[4].flatten(),
                    color='C0',
                    linestyle='-',
                    label=r"WER$_{\omega,\beta}$")
404
405
                mpl.xlabel(self._x_label or r"Weight $\beta$")
            mpl.ylabel(self._y_label or r"WER$_{\omega,\beta}$ (%)")
406
407
408
409
410
411
412
413
414
415
416
417
418

        if self._iapmr:
            axis = mpl.gca()
            if self._wer:
                axis = mpl.twinx()
                axis.grid(False)
            if self._var_param == 'omega':
                mpl.plot(
                    omega,
                    100. * errors[2].flatten(),
                    color='C3',
                    linestyle='-',
                    label='IAPMR')
419
                mpl.xlabel(self._x_label or r"Weight $\omega$")
420
421
422
423
424
425
426
            else:
                mpl.plot(
                    beta,
                    100. * errors[2].flatten(),
                    color='C3',
                    linestyle='-',
                    label='IAPMR')
427
428
                mpl.xlabel(self._x_label or r"Weight $\beta$")
            mpl.ylabel(self._y_label or r"IAPMR  (%)")
429
430
431
432
433
434
435
            if self._wer:
                axis.set_yticklabels(axis.get_yticks())
                axis.tick_params(axis='y', colors='red')
                axis.yaxis.label.set_color('red')
                axis.spines['right'].set_color('red')

        if self._var_param == 'omega':
Theophile GENTILHOMME's avatar
Bug fix    
Theophile GENTILHOMME committed
436
            if title is not None and title.replace(' ', ''):
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
437
                mpl.title(title or (r"EPSC with $\beta$ = %.2f" %
438
                                    self._fixed_param))
439
        else:
Theophile GENTILHOMME's avatar
Bug fix    
Theophile GENTILHOMME committed
440
            if title is not None and title.replace(' ', ''):
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
441
                mpl.title(title or (r"EPSC with $\omega$ = %.2f" %
442
                                    self._fixed_param))
443
444
445
446
447
448

        mpl.grid()
        self._plot_legends()
        ax1.set_xticklabels(ax1.get_xticks())
        ax1.set_yticklabels(ax1.get_yticks())
        mpl.xticks(rotation=self._x_rotation)
449
        self._pdf_page.savefig()
450

451

452
453
class Epsc3D(Epsc):
    ''' 3D EPSC plots for PAD'''
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
454

455
    def compute(self, idx, input_scores, input_names):
456
        ''' Implements plots'''
457
458
459
460
461
462
463
464
        licit_dev_neg = input_scores[0][0]
        licit_dev_pos = input_scores[0][1]
        licit_eval_neg = input_scores[1][0]
        licit_eval_pos = input_scores[1][1]
        spoof_dev_neg = input_scores[2][0]
        spoof_dev_pos = input_scores[2][1]
        spoof_eval_neg = input_scores[3][0]
        spoof_eval_pos = input_scores[3][1]
465

Theophile GENTILHOMME's avatar
Bug fix    
Theophile GENTILHOMME committed
466
        title = self._legends[idx] if self._legends is not None else "3D EPSC"
467

468
469
        mpl.rcParams.pop('key', None)

470
        mpl.gcf().clear()
471
        mpl.gcf().set_constrained_layout(self._clayout)
472
473
474
475

        from mpl_toolkits.mplot3d import Axes3D
        from matplotlib import cm

476
        points = self._sampling or 5
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493

        omega, beta, thrs = error_utils.epsc_thresholds(
            licit_dev_neg,
            licit_dev_pos,
            spoof_dev_neg,
            spoof_dev_pos,
            points=points,
            criteria=self._criteria)

        errors = error_utils.all_error_rates(
            licit_eval_neg, licit_eval_pos, spoof_eval_neg, spoof_eval_pos,
            thrs, omega, beta
        )
        # error rates are returned in a list as 2D numpy.ndarrays in
        # the following order: frr, far, IAPMR, far_w, wer_wb, hter_wb
        wer_errors = 100 * errors[2 if self._iapmr else 4]

494
        ax1 = mpl.gcf().add_subplot(111, projection='3d')
495
496
497
498
499
500
501
502
503
504
505

        W, B = np.meshgrid(omega, beta)

        ax1.plot_wireframe(
            W, B, wer_errors, cmap=cm.coolwarm, antialiased=False
        )  # surface

        if self._iapmr:
            ax1.azim = -30
            ax1.elev = 50

506
507
        ax1.set_xlabel(self._x_label or r"Weight $\omega$")
        ax1.set_ylabel(self._y_label or r"Weight $\beta$")
508
509
510
511
        ax1.set_zlabel(
            r"WER$_{\omega,\beta}$ (%)" if self._wer else "IAPMR (%)"
        )

512
        if title.replace(' ', ''):
Theophile GENTILHOMME's avatar
Bug fix    
Theophile GENTILHOMME committed
513
            mpl.title(title)
514
515
516
517
518

        ax1.set_xticklabels(ax1.get_xticks())
        ax1.set_yticklabels(ax1.get_yticks())
        ax1.set_zticklabels(ax1.get_zticks())

519
520
        self._pdf_page.savefig()

521

522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
class Roc(bio_figure.Roc):
    '''ROC for PARD'''
    def __init__(self, ctx, scores, evaluation, func_load):
        super(Roc, self).__init__(ctx, scores, evaluation, func_load)
        self._x_label = ctx.meta.get('x_label') or 'APCER'
        self._y_label = ctx.meta.get('y_label') or '1 - BPCER'

class DetPad(bio_figure.Det):
    def __init__(self, ctx, scores, evaluation, func_load):
        super(DetPad, self).__init__(ctx, scores, evaluation, func_load)
        self._x_label = ctx.meta.get('x_label') or 'APCER'
        self._y_label = ctx.meta.get('y_label') or 'BPCER'



537
538
class BaseDetRoc(PadPlot):
    '''Base for DET and ROC'''
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
539

540
541
    def __init__(self, ctx, scores, evaluation, func_load, criteria, real_data,
                no_spoof):
542
        super(BaseDetRoc, self).__init__(ctx, scores, evaluation, func_load)
543
544
        self._no_spoof = no_spoof
        self._criteria = criteria or 'eer'
545
        self._real_data = True if real_data is None else real_data
546

547

548
    def compute(self, idx, input_scores, input_names):
549
        ''' Implements plots'''
550
551
552
553
554
555
        licit_dev_neg = input_scores[0][0]
        licit_dev_pos = input_scores[0][1]
        licit_eval_neg = input_scores[1][0]
        licit_eval_pos = input_scores[1][1]
        spoof_eval_neg = input_scores[3][0] if len(input_scores) > 2 else None
        spoof_eval_pos = input_scores[3][1] if len(input_scores) > 2 else None
556
        self._plot(
557
558
559
            licit_eval_neg,
            licit_eval_pos,
            self._points,
560
            color='C0',
561
            linestyle='-',
562
            label=self._label("licit", input_names[0], idx)
563
564
        )
        if not self._no_spoof and spoof_eval_neg is not None:
565
            self._plot(
566
567
568
                spoof_eval_neg,
                spoof_eval_pos,
                self._points,
569
570
                color='C3',
                linestyle=':',
571
                label=self._label("spoof", input_names[3], idx)
572
573
            )

574
        if self._criteria is None or self._no_spoof:
575
576
577
578
579
580
581
582
            return

        thres_baseline = calc_threshold(
            self._criteria, licit_dev_neg, licit_dev_pos
        )

        axlim = mpl.axis()

583
        farfrr_licit, farfrr_licit_det = self._get_farfrr(
584
            licit_eval_neg, licit_eval_pos,
585
            thres_baseline
586
587
588
589
590
591
        )
        if farfrr_licit is None:
            return
        farfrr_spoof, farfrr_spoof_det = self._get_farfrr(
            spoof_eval_neg, spoof_eval_pos, thres_baseline
        )
592
593
594
595
596
597
598
        if not self._real_data:
            mpl.axhline(
                y=farfrr_licit_det[1],
                xmin=axlim[2],
                xmax=axlim[3],
                color='k',
                linestyle='--',
599
                label="%s @ EER" % self._y_label)
600
601
602
603
604
605
606
        else:
            mpl.axhline(
                y=farfrr_licit_det[1],
                xmin=axlim[0],
                xmax=axlim[1],
                color='k',
                linestyle='--',
607
608
                label="%s = %.2f%%" %
                (self._y_label, farfrr_licit[1] * 100))
609
610
611
612
613

        mpl.plot(
            farfrr_licit_det[0],
            farfrr_licit_det[1],
            'o',
614
615
            color='C0',
        )  # FAR point, licit scenario
616
617
618
619
        mpl.plot(
            farfrr_spoof_det[0],
            farfrr_spoof_det[1],
            'o',
620
621
            color='C3',
        )  # FAR point, spoof scenario
622
623
624

        # annotate the FAR points
        xyannotate_licit = [
625
626
            0.15 + farfrr_licit_det[0],
            farfrr_licit_det[1] - 0.15,
627
628
        ]
        xyannotate_spoof = [
629
630
            0.15 + farfrr_spoof_det[0],
            farfrr_spoof_det[1] - 0.15,
631
632
633
634
        ]

        if not self._real_data:
            mpl.annotate(
635
                '%s @ operating point' % self._y_label,
636
637
                xy=(farfrr_licit_det[0], farfrr_licit_det[1]),
                xycoords='data',
638
                xytext=(xyannotate_licit[0], xyannotate_licit[1]))
639
            mpl.annotate(
640
                'IAPMR @ operating point',
641
642
                xy=(farfrr_spoof_det[0], farfrr_spoof_det[1]),
                xycoords='data',
643
                xytext=(xyannotate_spoof[0], xyannotate_spoof[1]))
644
645
        else:
            mpl.annotate(
646
                'APCER=%.2f%%' % (farfrr_licit[0] * 100),
647
648
649
                xy=(farfrr_licit_det[0], farfrr_licit_det[1]),
                xycoords='data',
                xytext=(xyannotate_licit[0], xyannotate_licit[1]),
650
                color='C0',
651
                size='small')
652
            mpl.annotate(
653
                'IAPMR=%.2f%%' % (farfrr_spoof[0] * 100),
654
655
656
                xy=(farfrr_spoof_det[0], farfrr_spoof_det[1]),
                xycoords='data',
                xytext=(xyannotate_spoof[0], xyannotate_spoof[1]),
657
                color='C3',
658
                size='small')
659
660
661
662

    def end_process(self):
        ''' Set title, legend, axis labels, grid colors, save figures and
        close pdf is needed '''
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
663
        # only for plots
664
665
666
667
668

        if self._title.replace(' ', ''):
            mpl.title(self._title)
        mpl.xlabel(self._x_label)
        mpl.ylabel(self._y_label)
669
        mpl.grid(True, color=self._grid_color)
670
671
        if self._disp_legend:
            mpl.legend(loc=self._legend_loc)
672
673
674
675
676
677
678
679
680
681
682
        self._set_axis()
        fig = mpl.gcf()
        mpl.xticks(rotation=self._x_rotation)
        mpl.tick_params(axis='both', which='major', labelsize=4)
        for tick in mpl.gca().xaxis.get_major_ticks():
            tick.label.set_fontsize(6)
        for tick in mpl.gca().yaxis.get_major_ticks():
            tick.label.set_fontsize(6)

        self._pdf_page.savefig(fig)

Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
683
        # do not want to close PDF when running evaluate
684
        if 'PdfPages' in self._ctx.meta and \
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
685
                ('closef' not in self._ctx.meta or self._ctx.meta['closef']):
686
687
            self._pdf_page.close()

688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
    def _get_farfrr(self, x, y, thres):
        return None, None

    def _plot(self, x, y, points, **kwargs):
        pass

class Det(BaseDetRoc):
    '''Base for DET and ROC'''

    def __init__(self, ctx, scores, evaluation, func_load, criteria, real_data,
                no_spoof):
        super(Det, self).__init__(ctx, scores, evaluation, func_load,
                                  criteria, real_data, no_spoof)
        self._x_label = self._x_label or "APCER"
        self._y_label = self._y_label or "BPCER"
        add = ''
        if not self._no_spoof:
            add = " and overlaid SPOOF scenario"
        self._title = self._title or ('DET: LICIT' + add)


709
710
711
712
713
    def _set_axis(self):
        if self._axlim is not None and None not in self._axlim:
            det_axis(self._axlim)
        else:
            det_axis([0.01, 99, 0.01, 99])
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
714

715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
    def _get_farfrr(self, x, y, thres):
        # calculate test frr @ EER (licit scenario)
        points = farfrr(x, y, thres)
        return points, [ppndf(i) for i in points]


    def _plot(self, x, y, points, **kwargs):
        det(
            x, y, points,
            color=kwargs.get('color'),
            linestyle=kwargs.get('linestyle'),
            label=kwargs.get('label')
        )

class RocVuln(BaseDetRoc):
    '''ROC for vuln'''

    def __init__(self, ctx, scores, evaluation, func_load, criteria, real_data,
                no_spoof):
        super(RocVuln, self).__init__(ctx, scores, evaluation, func_load,
                                      criteria, real_data, no_spoof)
        self._x_label = self._x_label or "APCER"
        self._y_label = self._y_label or "1 - BPCER"
        self._semilogx = ctx.meta.get('semilogx', True)
        add = ''
        if not self._no_spoof:
            add = " and overlaid SPOOF scenario"
        self._title = self._title or ('ROC: LICIT' + add)


    def _plot(self, x, y, points, **kwargs):
        roc_for_far(
            x, y,
            far_values=log_values(self._min_dig or -4),
            CAR=self._semilogx,
            color=kwargs.get('color'), linestyle=kwargs.get('linestyle'),
            label=kwargs.get('label')
        )

754

Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
755
756
class FmrIapmr(PadPlot):
    '''FMR vs IAPMR'''
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
757

Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
758
759
    def __init__(self, ctx, scores, evaluation, func_load):
        super(FmrIapmr, self).__init__(ctx, scores, evaluation, func_load)
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
760
        self._eval = True  # always eval data with EPC
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
761
762
763
        self._split = False
        self._nb_figs = 1
        self._semilogx = False if 'semilogx' not in ctx.meta else\
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
764
            ctx.meta['semilogx']
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
        if self._min_arg != 4:
            raise click.BadParameter("You must provide 4 scores files:{licit,"
                                     "spoof}/{dev,eval}")

    def compute(self, idx, input_scores, input_names):
        ''' Implements plots'''
        licit_eval_neg = input_scores[1][0]
        licit_eval_pos = input_scores[1][1]
        spoof_eval_neg = input_scores[3][0]
        fmr_list = np.linspace(0, 1, 100)
        iapmr_list = []
        for i, fmr in enumerate(fmr_list):
            thr = far_threshold(licit_eval_neg, licit_eval_pos, fmr, True)
            iapmr_list.append(farfrr(spoof_eval_neg, licit_eval_pos, thr)[0])
            # re-calculate fmr since threshold might give a different result
            # for fmr.
            fmr_list[i] = farfrr(licit_eval_neg, licit_eval_pos, thr)[0]
782
        label = self._legends[idx] if self._legends is not None else \
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
783
            '(%s/%s)' % (input_names[1], input_names[3])
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
784
785
786
787
788
789
790
791
        if self._semilogx:
            mpl.semilogx(fmr_list, iapmr_list, label=label)
        else:
            mpl.plot(fmr_list, iapmr_list, label=label)

    def end_process(self):
        ''' Set title, legend, axis labels, grid colors, save figures and
        close pdf is needed '''
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
792
        # only for plots
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
793
        title = self._title if self._title is not None else "FMR vs IAPMR"
794
795
        if title.replace(' ', ''):
            mpl.title(title)
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
796
797
798
        mpl.xlabel(self._x_label or "False Match Rate (%)")
        mpl.ylabel(self._y_label or "IAPMR (%)")
        mpl.grid(True, color=self._grid_color)
799
800
        if self._disp_legend:
            mpl.legend(loc=self._legend_loc)
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
801
802
803
804
805
806
807
        self._set_axis()
        fig = mpl.gcf()
        mpl.xticks(rotation=self._x_rotation)
        mpl.tick_params(axis='both', which='major', labelsize=4)

        self._pdf_page.savefig(fig)

Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
808
        # do not want to close PDF when running evaluate
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
809
        if 'PdfPages' in self._ctx.meta and \
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
810
                ('closef' not in self._ctx.meta or self._ctx.meta['closef']):
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
811
            self._pdf_page.close()