vuln_figure.py 25.3 KB
Newer Older
1
2
'''Runs error analysis on score sets, outputs metrics and plots'''

3
import math
4
5
import click
import numpy as np
6
import matplotlib.pyplot as mpl
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
7
import bob.measure.script.figure as measure_figure
8
import bob.bio.base.script.figure as bio_figure
9
from tabulate import tabulate
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
10
from bob.measure.utils import get_fta_list
11
from bob.measure import (
12
13
    frr_threshold, far_threshold, eer_threshold, min_hter_threshold, farfrr,
    epc, ppndf, min_weighted_error_rate_threshold
14
)
15
from bob.measure.plot import (det, det_axis, roc_for_far, log_values, epc)
16
17
from . import error_utils

18

19
20
21
22
class Metrics(measure_figure.Metrics):
    def __init__(self, ctx, scores, evaluation, func_load):
        super(Metrics, self).__init__(ctx, scores, evaluation, func_load)

23
    ''' Compute metrics from score files'''
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
24

25
26
    def compute(self, idx, input_scores, input_names):
        ''' Compute metrics for the given criteria'''
27
        # extract pos and negative and remove NaNs
28
29
30
        neg_list, pos_list, _ = get_fta_list(input_scores)
        dev_neg, dev_pos = neg_list[0], pos_list[0]
        criter = self._criterion or 'eer'
31
        threshold = error_utils.calc_threshold(criter, dev_neg, dev_pos) \
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
32
            if self._thres is None else self._thres[idx]
33
34
35
36
37
38
        far, frr = farfrr(neg_list[1], pos_list[1], threshold)
        iapmr, _ = farfrr(neg_list[3], pos_list[1], threshold)
        title = self._legends[idx] if self._legends is not None else None
        headers = ['' or title, '%s (threshold=%.2g)' %
                   (criter.upper(), threshold)]
        rows = []
39
40
41
        rows.append(['FMR (%)', '{:>5.1f}%'.format(100 * far)])
        rows.append(['FNMR (%)', '{:>5.1f}%'.format(frr * 100)])
        rows.append(['HTER (%)', '{:>5.1f}%'.format(50 * (far + frr))])
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
42
        rows.append(['IAPMR (%)', '{:>5.1f}%'.format(100 * iapmr)])
43
44
45
46
47
48
        click.echo(
            tabulate(rows, headers, self._tablefmt),
            file=self.log_file
        )


49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
def _iapmr_dot(threshold, iapmr, real_data, **kwargs):
    # plot a dot on threshold versus IAPMR line and show IAPMR as a number
    axlim = mpl.axis()
    mpl.plot(threshold, 100. * iapmr, 'o', color='C3', **kwargs)
    if not real_data:
        mpl.annotate(
            'IAPMR at\noperating point',
            xy=(threshold, 100. * iapmr),
            xycoords='data',
            xytext=(0.85, 0.6),
            textcoords='axes fraction',
            color='black',
            size='large',
            arrowprops=dict(facecolor='black', shrink=0.05, width=2),
            horizontalalignment='center',
            verticalalignment='top',
        )
    else:
        mpl.text(threshold + (threshold - axlim[0]) / 12, 100. * iapmr,
                 '%.1f%%' % (100. * iapmr,), color='C3')

70

71
72
73
74
75
76
def _iapmr_line_plot(scores, n_points=100, **kwargs):
    axlim = mpl.axis()
    step = (axlim[1] - axlim[0]) / float(n_points)
    thres = [(k * step) + axlim[0] for k in range(2, n_points - 1)]
    mix_prob_y = []
    for k in thres:
77
        mix_prob_y.append(100. * error_utils.calc_pass_rate(k, scores))
78
79
80

    mpl.plot(thres, mix_prob_y, label='IAPMR', color='C3', **kwargs)

81

82
83
84
85
def _iapmr_plot(scores, threshold, iapmr, real_data, **kwargs):
    _iapmr_dot(threshold, iapmr, real_data, **kwargs)
    _iapmr_line_plot(scores, n_points=100, **kwargs)

86

87
88
89
class HistVuln(measure_figure.Hist):
    ''' Histograms for vulnerability '''

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
90
91
92
93
    def __init__(self, ctx, scores, evaluation, func_load):
        super(HistVuln, self).__init__(
            ctx, scores, evaluation, func_load, nhist_per_system=3)

94
95
96
    def _setup_hist(self, neg, pos):
        self._title_base = 'Vulnerability'
        self._density_hist(
97
            pos[0], n=0, label='Genuine', color='C2'
98
99
        )
        self._density_hist(
100
            neg[0], n=1, label='Zero-effort impostors', alpha=0.8, color='C0'
101
102
        )
        self._density_hist(
103
            neg[1], n=2, label='Presentation attack', alpha=0.4, color='C7',
104
            hatch='\\\\'
105
106
        )

107
    def _lines(self, threshold, label, neg, pos, idx, **kwargs):
108
        if 'iapmr_line' not in self._ctx.meta or self._ctx.meta['iapmr_line']:
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
109
            # plot vertical line
110
            super(HistVuln, self)._lines(threshold, label, neg, pos, idx)
111

Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
112
            # plot iapmr_line
113
114
115
116
117
            iapmr, _ = farfrr(neg[1], pos[0], threshold)
            ax2 = mpl.twinx()
            # we never want grid lines on axis 2
            ax2.grid(False)
            real_data = True if 'real_data' not in self._ctx.meta else \
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
118
                self._ctx.meta['real_data']
119
            _iapmr_plot(neg[1], threshold, iapmr, real_data=real_data)
120
121
            n = idx % self._step_print
            col = n % self._ncols
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
122
123
            rest_print = self.n_systems - \
                int(idx / self._step_print) * self._step_print
124
125
            if col == self._ncols - 1 or n == rest_print - 1:
                ax2.set_ylabel("IAPMR (%)", color='C3')
126
127
128
129
            ax2.tick_params(axis='y', colors='red')
            ax2.yaxis.label.set_color('red')
            ax2.spines['right'].set_color('red')

130

131
132
class PadPlot(measure_figure.PlotBase):
    '''Base class for PAD plots'''
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
133

134
135
    def __init__(self, ctx, scores, evaluation, func_load):
        super(PadPlot, self).__init__(ctx, scores, evaluation, func_load)
136
        mpl.rcParams['figure.constrained_layout.use'] = self._clayout
137
138
139

    def end_process(self):
        '''Close pdf '''
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
140
        # do not want to close PDF when running evaluate
141
142
143
144
145
        if 'PdfPages' in self._ctx.meta and \
           ('closef' not in self._ctx.meta or self._ctx.meta['closef']):
            self._pdf_page.close()

    def _plot_legends(self):
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
146
        # legends for all axes
147
148
149
150
151
152
        lines = []
        labels = []
        for ax in mpl.gcf().get_axes():
            li, la = ax.get_legend_handles_labels()
            lines += li
            labels += la
153
154
155
156
        if self._disp_legend:
            mpl.gca().legend(lines, labels, loc=self._legend_loc,
                             fancybox=True, framealpha=0.5)

157
158
159

class Epc(PadPlot):
    ''' Handles the plotting of EPC '''
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
160

161
162
163
    def __init__(self, ctx, scores, evaluation, func_load):
        super(Epc, self).__init__(ctx, scores, evaluation, func_load)
        self._iapmr = True if 'iapmr' not in self._ctx.meta else \
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
164
            self._ctx.meta['iapmr']
165
166
167
168
        self._title = self._title or ('EPC and IAPMR' if self._iapmr else
                                      'EPC')
        self._x_label = self._x_label or r"Weight $\beta$"
        self._y_label = self._y_label or "WER (%)"
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
169
        self._eval = True  # always eval data with EPC
170
171
172
        self._split = False
        self._nb_figs = 1

173
174
175
176
177
        if self._min_arg != 4:
            raise click.BadParameter("You must provide 4 scores files:{licit,"
                                     "spoof}/{dev,eval}")

    def compute(self, idx, input_scores, input_names):
178
        ''' Plot EPC for PAD'''
179
180
181
182
183
184
        # extract pos and negative and remove NaNs
        neg_list, pos_list, _ = get_fta_list(input_scores)
        licit_dev_neg, licit_dev_pos = neg_list[0], pos_list[0]
        licit_eval_neg, licit_eval_pos = neg_list[1], pos_list[1]
        spoof_eval_neg = neg_list[3]

185
        mpl.gcf().clear()
186
187
188
189
190

        epc(
            licit_dev_neg, licit_dev_pos, licit_eval_neg, licit_eval_pos,
            self._points,
            color='C0', linestyle=self._linestyles[idx],
191
            label=self._label(
192
                'WER', '%s-%s' % (input_names[0], input_names[1]), idx
193
194
195
196
197
            ),
        )
        mpl.xlabel(self._x_label)
        mpl.ylabel(self._y_label)
        if self._iapmr:
198
            axlim = mpl.axis()
199
200
            mpl.gca().set_axisbelow(True)
            prob_ax = mpl.gca().twinx()
201
202
203
204
205
206
207
208
209
210
            step = 1.0 / float(self._points)
            thres = [float(k * step) for k in range(self._points)]
            apply_thres = [min_weighted_error_rate_threshold(
                licit_dev_neg, licit_dev_pos, t) for t in thres]
            mix_prob_y = []
            for k in apply_thres:
                mix_prob_y.append(
                    100. * error_utils.calc_pass_rate(k, spoof_eval_neg)
                )

211
            mpl.plot(
212
                thres, mix_prob_y, label=self._label(
213
                    'IAPMR', '%s-%s' % (input_names[0], input_names[1]), idx
214
                ), color='C3'
215
            )
216

217
            prob_ax.set_yticklabels(prob_ax.get_yticks())
218
219
220
            prob_ax.tick_params(axis='y', colors='C3')
            prob_ax.yaxis.label.set_color('C3')
            prob_ax.spines['right'].set_color('C3')
221
222
            ylabels = prob_ax.get_yticks()
            prob_ax.yaxis.set_ticklabels(["%.0f" % val for val in ylabels])
223
            prob_ax.set_ylabel('IAPMR', color='C3')
224
            prob_ax.set_axisbelow(True)
225
226


227
        title = self._legends[idx] if self._legends is not None else self._title
228
229
        if title.replace(' ', ''):
            mpl.title(title)
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
230
        # legends for all axes
231
        mpl.grid()
232
233
234
235
        self._plot_legends()
        mpl.xticks(rotation=self._x_rotation)
        self._pdf_page.savefig(mpl.gcf())

236

237
238
class Epsc(PadPlot):
    ''' Handles the plotting of EPSC '''
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
239

240
241
242
243
    def __init__(self, ctx, scores, evaluation, func_load,
                 criteria, var_param, fixed_param):
        super(Epsc, self).__init__(ctx, scores, evaluation, func_load)
        self._iapmr = False if 'iapmr' not in self._ctx.meta else \
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
244
            self._ctx.meta['iapmr']
245
        self._wer = True if 'wer' not in self._ctx.meta else \
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
246
            self._ctx.meta['wer']
247
248
249
        self._criteria = 'eer' if criteria is None else criteria
        self._var_param = "omega" if var_param is None else var_param
        self._fixed_param = 0.5 if fixed_param is None else fixed_param
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
250
        self._eval = True  # always eval data with EPC
251
252
        self._split = False
        self._nb_figs = 1
253
        self._title = ''
254
        self._sampling = ctx.meta.get('sampling', 5)
255
256
257
258

        if self._min_arg != 4:
            raise click.BadParameter("You must provide 4 scores files:{licit,"
                                     "spoof}/{dev,eval}")
259

260
    def compute(self, idx, input_scores, input_names):
261
        ''' Plot EPSC for PAD'''
262
263
264
265
266
267
268
269
        licit_dev_neg = input_scores[0][0]
        licit_dev_pos = input_scores[0][1]
        licit_eval_neg = input_scores[1][0]
        licit_eval_pos = input_scores[1][1]
        spoof_dev_neg = input_scores[2][0]
        spoof_dev_pos = input_scores[2][1]
        spoof_eval_neg = input_scores[3][0]
        spoof_eval_pos = input_scores[3][1]
270
        title = self._legends[idx] if self._legends is not None else None
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290

        mpl.gcf().clear()
        points = 10

        if self._var_param == 'omega':
            omega, beta, thrs = error_utils.epsc_thresholds(
                licit_dev_neg,
                licit_dev_pos,
                spoof_dev_neg,
                spoof_dev_pos,
                points=points,
                criteria=self._criteria,
                beta=self._fixed_param)
        else:
            omega, beta, thrs = error_utils.epsc_thresholds(
                licit_dev_neg,
                licit_dev_pos,
                spoof_dev_neg,
                spoof_dev_pos,
                points=points,
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
291
                criteria=self._criteria,
292
293
294
295
296
297
298
                omega=self._fixed_param
            )

        errors = error_utils.all_error_rates(
            licit_eval_neg, licit_eval_pos, spoof_eval_neg,
            spoof_eval_pos, thrs, omega, beta
        )  # error rates are returned in a list in the
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
299
        # following order: frr, far, IAPMR, far_w, wer_w
300
301
302
303

        ax1 = mpl.subplot(
            111
        )  # EPC like curves for FVAS fused scores for weighted error rates
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
304
        # between the negatives (impostors and Presentation attacks)
305
306
307
308
309
310
311
312
        if self._wer:
            if self._var_param == 'omega':
                mpl.plot(
                    omega,
                    100. * errors[4].flatten(),
                    color='C0',
                    linestyle='-',
                    label=r"WER$_{\omega,\beta}$")
313
                mpl.xlabel(self._x_label or r"Weight $\omega$")
314
315
316
317
318
319
320
            else:
                mpl.plot(
                    beta,
                    100. * errors[4].flatten(),
                    color='C0',
                    linestyle='-',
                    label=r"WER$_{\omega,\beta}$")
321
322
                mpl.xlabel(self._x_label or r"Weight $\beta$")
            mpl.ylabel(self._y_label or r"WER$_{\omega,\beta}$ (%)")
323
324
325
326
327
328
329
330
331
332
333
334
335

        if self._iapmr:
            axis = mpl.gca()
            if self._wer:
                axis = mpl.twinx()
                axis.grid(False)
            if self._var_param == 'omega':
                mpl.plot(
                    omega,
                    100. * errors[2].flatten(),
                    color='C3',
                    linestyle='-',
                    label='IAPMR')
336
                mpl.xlabel(self._x_label or r"Weight $\omega$")
337
338
339
340
341
342
343
            else:
                mpl.plot(
                    beta,
                    100. * errors[2].flatten(),
                    color='C3',
                    linestyle='-',
                    label='IAPMR')
344
345
                mpl.xlabel(self._x_label or r"Weight $\beta$")
            mpl.ylabel(self._y_label or r"IAPMR  (%)")
346
347
348
349
350
351
352
            if self._wer:
                axis.set_yticklabels(axis.get_yticks())
                axis.tick_params(axis='y', colors='red')
                axis.yaxis.label.set_color('red')
                axis.spines['right'].set_color('red')

        if self._var_param == 'omega':
Theophile GENTILHOMME's avatar
Bug fix    
Theophile GENTILHOMME committed
353
            if title is not None and title.replace(' ', ''):
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
354
                mpl.title(title or (r"EPSC with $\beta$ = %.2f" %
355
                                    self._fixed_param))
356
        else:
Theophile GENTILHOMME's avatar
Bug fix    
Theophile GENTILHOMME committed
357
            if title is not None and title.replace(' ', ''):
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
358
                mpl.title(title or (r"EPSC with $\omega$ = %.2f" %
359
                                    self._fixed_param))
360
361
362
363
364
365

        mpl.grid()
        self._plot_legends()
        ax1.set_xticklabels(ax1.get_xticks())
        ax1.set_yticklabels(ax1.get_yticks())
        mpl.xticks(rotation=self._x_rotation)
366
        self._pdf_page.savefig()
367

368

369
370
class Epsc3D(Epsc):
    ''' 3D EPSC plots for PAD'''
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
371

372
    def compute(self, idx, input_scores, input_names):
373
        ''' Implements plots'''
374
375
376
377
378
379
380
381
        licit_dev_neg = input_scores[0][0]
        licit_dev_pos = input_scores[0][1]
        licit_eval_neg = input_scores[1][0]
        licit_eval_pos = input_scores[1][1]
        spoof_dev_neg = input_scores[2][0]
        spoof_dev_pos = input_scores[2][1]
        spoof_eval_neg = input_scores[3][0]
        spoof_eval_pos = input_scores[3][1]
382

Theophile GENTILHOMME's avatar
Bug fix    
Theophile GENTILHOMME committed
383
        title = self._legends[idx] if self._legends is not None else "3D EPSC"
384

385
386
        mpl.rcParams.pop('key', None)

387
        mpl.gcf().clear()
388
        mpl.gcf().set_constrained_layout(self._clayout)
389
390
391
392

        from mpl_toolkits.mplot3d import Axes3D
        from matplotlib import cm

393
        points = self._sampling or 5
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410

        omega, beta, thrs = error_utils.epsc_thresholds(
            licit_dev_neg,
            licit_dev_pos,
            spoof_dev_neg,
            spoof_dev_pos,
            points=points,
            criteria=self._criteria)

        errors = error_utils.all_error_rates(
            licit_eval_neg, licit_eval_pos, spoof_eval_neg, spoof_eval_pos,
            thrs, omega, beta
        )
        # error rates are returned in a list as 2D numpy.ndarrays in
        # the following order: frr, far, IAPMR, far_w, wer_wb, hter_wb
        wer_errors = 100 * errors[2 if self._iapmr else 4]

411
        ax1 = mpl.gcf().add_subplot(111, projection='3d')
412
413
414
415
416
417
418
419
420
421
422

        W, B = np.meshgrid(omega, beta)

        ax1.plot_wireframe(
            W, B, wer_errors, cmap=cm.coolwarm, antialiased=False
        )  # surface

        if self._iapmr:
            ax1.azim = -30
            ax1.elev = 50

423
424
        ax1.set_xlabel(self._x_label or r"Weight $\omega$")
        ax1.set_ylabel(self._y_label or r"Weight $\beta$")
425
426
427
428
        ax1.set_zlabel(
            r"WER$_{\omega,\beta}$ (%)" if self._wer else "IAPMR (%)"
        )

429
        if title.replace(' ', ''):
Theophile GENTILHOMME's avatar
Bug fix    
Theophile GENTILHOMME committed
430
            mpl.title(title)
431
432
433
434
435

        ax1.set_xticklabels(ax1.get_xticks())
        ax1.set_yticklabels(ax1.get_yticks())
        ax1.set_zticklabels(ax1.get_zticks())

436
437
        self._pdf_page.savefig()

438

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
439
class BaseVulnDetRoc(PadPlot):
440
    '''Base for DET and ROC'''
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
441

442
    def __init__(self, ctx, scores, evaluation, func_load, real_data,
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
443
                 no_spoof):
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
444
445
        super(BaseVulnDetRoc, self).__init__(
            ctx, scores, evaluation, func_load)
446
        self._no_spoof = no_spoof
447
        self._hlines_at = ctx.meta.get('hlines_at', [])
448
        self._real_data = True if real_data is None else real_data
449
        self._legend_loc = None
450

451
    def compute(self, idx, input_scores, input_names):
452
        ''' Implements plots'''
453
454
455
456
        licit_neg = input_scores[0][0]
        licit_pos = input_scores[0][1]
        spoof_neg = input_scores[1][0]
        spoof_pos = input_scores[1][1]
457
        self._plot(
458
459
            licit_neg,
            licit_pos,
460
            self._points,
461
            color='C0',
462
            linestyle='-',
463
            label=self._label("licit", input_names[0], idx)
464
        )
465
        if not self._no_spoof and spoof_neg is not None:
466
467
468
469
470
471
472
            ax1 = mpl.gca()
            ax2 = ax1.twiny()
            ax2.set_xlabel('IAPMR', color='C3')
            ax2.set_xticklabels(ax2.get_xticks())
            ax2.tick_params(axis='x', colors='C3')
            ax2.xaxis.label.set_color('C3')
            ax2.spines['top'].set_color('C3')
473
474
475
            ax2.spines['bottom'].set_color('C0')
            ax1.xaxis.label.set_color('C0')
            ax1.tick_params(axis='x', colors='C0')
476
            self._plot(
477
478
                spoof_neg,
                spoof_pos,
479
                self._points,
480
481
                color='C3',
                linestyle=':',
482
                label=self._label("spoof", input_names[1], idx)
483
            )
484
            mpl.sca(ax1)
485

486
        if self._hlines_at is None:
487
488
            return

489
490
        for line in self._hlines_at:
            thres_baseline = frr_threshold(licit_neg, licit_pos, line)
491

492
            axlim = mpl.axis()
493

494
495
496
497
498
499
            farfrr_licit, farfrr_licit_det = self._get_farfrr(
                licit_neg, licit_pos,
                thres_baseline
            )
            if farfrr_licit is None:
                return
500

501
502
503
504
            farfrr_spoof, farfrr_spoof_det = self._get_farfrr(
                spoof_neg, spoof_pos,
                frr_threshold(spoof_neg, spoof_pos, farfrr_licit[1])
            )
505

506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
            if not self._real_data:
                mpl.axhline(
                    y=farfrr_licit_det[1],
                    xmin=axlim[2],
                    xmax=axlim[3],
                    color='k',
                    linestyle='--',
                    label="%s @ EER" % self._y_label)
            else:
                mpl.axhline(
                    y=farfrr_licit_det[1],
                    xmin=axlim[0],
                    xmax=axlim[1],
                    color='k',
                    linestyle='--',
                    label="%s = %.2f%%" %
522
523
524
525
526
527
528
529
                    ('FMNR', farfrr_licit[1] * 100))

            if not self._real_data:
                label_licit = '%s @ operating point' % self._y_label
                label_spoof = 'IAPMR @ operating point'
            else:
                label_licit = 'FMR=%.2f%%' % (farfrr_licit[0] * 100)
                label_spoof = 'IAPMR=%.2f%%' % (farfrr_spoof[0] * 100)
530
531
532
533
534

            mpl.plot(
                farfrr_licit_det[0],
                farfrr_licit_det[1],
                'o',
535
                color='C0',
536
                label=label_licit
537
538
539
540
541
            )  # FAR point, licit scenario
            mpl.plot(
                farfrr_spoof_det[0],
                farfrr_spoof_det[1],
                'o',
542
                color='C3',
543
                label=label_spoof
544
545
            )  # FAR point, spoof scenario

546
547
548
549

    def end_process(self):
        ''' Set title, legend, axis labels, grid colors, save figures and
        close pdf is needed '''
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
550
        # only for plots
551
552

        if self._title.replace(' ', ''):
553
            mpl.title(self._title, y=1.15)
554
555
        mpl.xlabel(self._x_label)
        mpl.ylabel(self._y_label)
556
        mpl.grid(True, color=self._grid_color)
557
558
559
560
561
562
563
564
565
566
567
        lines = []
        labels = []
        for ax in mpl.gcf().get_axes():
            li, la = ax.get_legend_handles_labels()
            lines += li
            labels += la
            mpl.sca(ax)
            self._set_axis()
            fig = mpl.gcf()
            mpl.xticks(rotation=self._x_rotation)
            mpl.tick_params(axis='both', which='major', labelsize=6)
568
        if self._disp_legend:
569
570
571
572
            mpl.gca().legend(
                lines, labels, loc=self._legend_loc, fancybox=True,
                framealpha=0.5
            )
573
574
        self._pdf_page.savefig(fig)

Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
575
        # do not want to close PDF when running evaluate
576
        if 'PdfPages' in self._ctx.meta and \
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
577
                ('closef' not in self._ctx.meta or self._ctx.meta['closef']):
578
579
            self._pdf_page.close()

580
581
582
583
584
585
    def _get_farfrr(self, x, y, thres):
        return None, None

    def _plot(self, x, y, points, **kwargs):
        pass

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
586

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
587
class DetVuln(BaseVulnDetRoc):
588
    '''DET for vuln'''
589

590
    def __init__(self, ctx, scores, evaluation, func_load, real_data,
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
591
                 no_spoof):
592
        super(DetVuln, self).__init__(ctx, scores, evaluation, func_load,
593
                                  real_data, no_spoof)
594
595
        self._x_label = self._x_label or "FMR"
        self._y_label = self._y_label or "FNMR"
596
597
598
599
        add = ''
        if not self._no_spoof:
            add = " and overlaid SPOOF scenario"
        self._title = self._title or ('DET: LICIT' + add)
600
        self._legend_loc = self._legend_loc or 'upper right'
601

602
603
604
605
606
    def _set_axis(self):
        if self._axlim is not None and None not in self._axlim:
            det_axis(self._axlim)
        else:
            det_axis([0.01, 99, 0.01, 99])
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
607

608
609
610
611
612
613
614
615
616
617
618
619
    def _get_farfrr(self, x, y, thres):
        points = farfrr(x, y, thres)
        return points, [ppndf(i) for i in points]

    def _plot(self, x, y, points, **kwargs):
        det(
            x, y, points,
            color=kwargs.get('color'),
            linestyle=kwargs.get('linestyle'),
            label=kwargs.get('label')
        )

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
620

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
621
class RocVuln(BaseVulnDetRoc):
622
623
    '''ROC for vuln'''

624
    def __init__(self, ctx, scores, evaluation, func_load, real_data, no_spoof):
625
        super(RocVuln, self).__init__(ctx, scores, evaluation, func_load,
626
                                      real_data, no_spoof)
627
628
        self._x_label = self._x_label or "FMR"
        self._y_label = self._y_label or "1 - FNMR"
629
630
631
632
633
        self._semilogx = ctx.meta.get('semilogx', True)
        add = ''
        if not self._no_spoof:
            add = " and overlaid SPOOF scenario"
        self._title = self._title or ('ROC: LICIT' + add)
634
635
        best_legend = 'lower right' if self._semilogx else 'upper right'
        self._legend_loc = self._legend_loc or best_legend
636
637
638
639
640
641
642
643
644
645

    def _plot(self, x, y, points, **kwargs):
        roc_for_far(
            x, y,
            far_values=log_values(self._min_dig or -4),
            CAR=self._semilogx,
            color=kwargs.get('color'), linestyle=kwargs.get('linestyle'),
            label=kwargs.get('label')
        )

646
647
648
649
    def _get_farfrr(self, x, y, thres):
        points = farfrr(x, y, thres)
        points2 = (points[0], 1 - points[1])
        return points, points2
650

651

Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
652
653
class FmrIapmr(PadPlot):
    '''FMR vs IAPMR'''
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
654

Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
655
656
    def __init__(self, ctx, scores, evaluation, func_load):
        super(FmrIapmr, self).__init__(ctx, scores, evaluation, func_load)
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
657
        self._eval = True  # always eval data with EPC
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
658
659
660
        self._split = False
        self._nb_figs = 1
        self._semilogx = False if 'semilogx' not in ctx.meta else\
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
661
            ctx.meta['semilogx']
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
        if self._min_arg != 4:
            raise click.BadParameter("You must provide 4 scores files:{licit,"
                                     "spoof}/{dev,eval}")

    def compute(self, idx, input_scores, input_names):
        ''' Implements plots'''
        licit_eval_neg = input_scores[1][0]
        licit_eval_pos = input_scores[1][1]
        spoof_eval_neg = input_scores[3][0]
        fmr_list = np.linspace(0, 1, 100)
        iapmr_list = []
        for i, fmr in enumerate(fmr_list):
            thr = far_threshold(licit_eval_neg, licit_eval_pos, fmr, True)
            iapmr_list.append(farfrr(spoof_eval_neg, licit_eval_pos, thr)[0])
            # re-calculate fmr since threshold might give a different result
            # for fmr.
            fmr_list[i] = farfrr(licit_eval_neg, licit_eval_pos, thr)[0]
679
        label = self._legends[idx] if self._legends is not None else \
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
680
            '(%s/%s)' % (input_names[1], input_names[3])
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
681
682
683
684
685
686
687
688
        if self._semilogx:
            mpl.semilogx(fmr_list, iapmr_list, label=label)
        else:
            mpl.plot(fmr_list, iapmr_list, label=label)

    def end_process(self):
        ''' Set title, legend, axis labels, grid colors, save figures and
        close pdf is needed '''
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
689
        # only for plots
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
690
        title = self._title if self._title is not None else "FMR vs IAPMR"
691
692
        if title.replace(' ', ''):
            mpl.title(title)
693
        mpl.xlabel(self._x_label or "FMR (%)")
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
694
695
        mpl.ylabel(self._y_label or "IAPMR (%)")
        mpl.grid(True, color=self._grid_color)
696
697
        if self._disp_legend:
            mpl.legend(loc=self._legend_loc)
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
698
699
700
701
702
703
704
        self._set_axis()
        fig = mpl.gcf()
        mpl.xticks(rotation=self._x_rotation)
        mpl.tick_params(axis='both', which='major', labelsize=4)

        self._pdf_page.savefig(fig)

Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
705
        # do not want to close PDF when running evaluate
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
706
        if 'PdfPages' in self._ctx.meta and \
Amir MOHAMMADI's avatar
nit    
Amir MOHAMMADI committed
707
                ('closef' not in self._ctx.meta or self._ctx.meta['closef']):
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
708
            self._pdf_page.close()