figure.py 26.8 KB
 1 2 3 4 '''Runs error analysis on score sets, outputs metrics and plots''' import click import numpy as np 5 import matplotlib.pyplot as mpl Amir MOHAMMADI committed May 08, 2018 6 import bob.measure.script.figure as measure_figure 7 from tabulate import tabulate Amir MOHAMMADI committed May 08, 2018 8 from bob.measure.utils import get_fta_list 9 from bob.measure import ( 10 far_threshold, eer_threshold, min_hter_threshold, farfrr, epc, ppndf 11 ) 12 from bob.measure.plot import (det, det_axis) 13 14 15 16 from . import error_utils ALL_CRITERIA = ('bpcer20', 'eer', 'min-hter') Theophile GENTILHOMME committed May 04, 2018 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 def calc_threshold(method, neg, pos): """Calculates the threshold based on the given method. The scores should be sorted! Parameters ---------- method : str One of bpcer201, eer, min-hter. neg : array_like The negative scores. They should be sorted! pos : array_like The positive scores. They should be sorted! Returns ------- float The calculated threshold. Raises ------ ValueError If method is unknown. """ method = method.lower() if method == 'bpcer20': threshold = far_threshold(neg, pos, 0.05, True) elif method == 'eer': threshold = eer_threshold(neg, pos, True) elif method == 'min-hter': threshold = min_hter_threshold(neg, pos, True) else: raise ValueError("Unknown threshold criteria: {}".format(method)) return threshold Theophile GENTILHOMME committed May 04, 2018 53 54 55 56 57 58 class Metrics(measure_figure.Metrics): def __init__(self, ctx, scores, evaluation, func_load): super(Metrics, self).__init__(ctx, scores, evaluation, func_load) ''' Compute metrics from score files''' Amir MOHAMMADI committed May 08, 2018 59 Theophile GENTILHOMME committed Apr 23, 2018 60 def compute(self, idx, input_scores, input_names): 61 ''' Compute metrics for the given criteria''' Theophile GENTILHOMME committed Apr 23, 2018 62 63 64 65 66 67 neg_list, pos_list, _ = get_fta_list(input_scores) dev_neg, dev_pos = neg_list[0], pos_list[0] dev_file = input_names[0] if self._eval: eval_neg, eval_pos = neg_list[1], pos_list[1] eval_file = input_names[1] 68 Theophile GENTILHOMME committed Apr 30, 2018 69 title = self._legends[idx] if self._legends is not None else None 70 headers = ['' or title, 'Development %s' % dev_file] Theophile GENTILHOMME committed Apr 23, 2018 71 if self._eval: 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 headers.append('Eval. % s' % eval_file) for m in ALL_CRITERIA: raws = [] threshold = calc_threshold(m, dev_neg, dev_pos) click.echo("\nThreshold of %f selected with the %s criteria" % ( threshold, m)) apcer, bpcer = farfrr(dev_neg, dev_pos, threshold) raws.append(['BPCER20', '{:>5.1f}%'.format(apcer * 100)]) raws.append(['EER', '{:>5.1f}%'.format(bpcer * 100)]) raws.append(['min-HTER', '{:>5.1f}%'.format((apcer + bpcer) * 50)]) if self._eval and eval_neg is not None: apcer, bpcer = farfrr(eval_neg, eval_pos, threshold) raws[0].append('{:>5.1f}%'.format(apcer * 100)) raws[1].append('{:>5.1f}%'.format(bpcer * 100)) raws[2].append('{:>5.1f}%'.format((apcer + bpcer) * 50)) click.echo( tabulate(raws, headers, self._tablefmt), file=self.log_file ) Theophile GENTILHOMME committed May 04, 2018 93 Theophile GENTILHOMME committed May 04, 2018 94 95 96 97 98 class MetricsVuln(measure_figure.Metrics): def __init__(self, ctx, scores, evaluation, func_load): super(MetricsVuln, self).__init__(ctx, scores, evaluation, func_load) ''' Compute metrics from score files''' Amir MOHAMMADI committed May 08, 2018 99 Theophile GENTILHOMME committed May 04, 2018 100 101 102 103 104 105 def compute(self, idx, input_scores, input_names): ''' Compute metrics for the given criteria''' neg_list, pos_list, _ = get_fta_list(input_scores) dev_neg, dev_pos = neg_list[0], pos_list[0] criter = self._criterion or 'eer' threshold = calc_threshold(criter, dev_neg, dev_pos) \ Amir MOHAMMADI committed May 08, 2018 106 if self._thres is None else self._thres[idx] Theophile GENTILHOMME committed May 04, 2018 107 108 109 110 111 112 far, frr = farfrr(neg_list[1], pos_list[1], threshold) iapmr, _ = farfrr(neg_list[3], pos_list[1], threshold) title = self._legends[idx] if self._legends is not None else None headers = ['' or title, '%s (threshold=%.2g)' % (criter.upper(), threshold)] rows = [] Amir MOHAMMADI committed May 08, 2018 113 114 115 116 rows.append(['FMR (%)', '{:>5.1f}%'.format(100 * far)]) rows.append(['FMNR (%)', '{:>5.1f}%'.format(frr * 100)]) rows.append(['HTER (%)', '{:>5.1f}%'.format(50 * (far + frr))]) rows.append(['IAPMR (%)', '{:>5.1f}%'.format(100 * iapmr)]) Theophile GENTILHOMME committed May 04, 2018 117 118 119 120 121 122 click.echo( tabulate(rows, headers, self._tablefmt), file=self.log_file ) 123 124 125 126 127 128 class HistPad(measure_figure.Hist): ''' Histograms for PAD ''' def _setup_hist(self, neg, pos): self._title_base = 'PAD' self._density_hist( Theophile GENTILHOMME committed Apr 26, 2018 129 pos[0], n=0, label='Bona Fide', color='C1' 130 131 ) self._density_hist( Theophile GENTILHOMME committed Apr 26, 2018 132 neg[0], n=1, label='Presentation attack', alpha=0.4, color='C7', Theophile GENTILHOMME committed Apr 23, 2018 133 hatch='\\\\' 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 ) def _iapmr_dot(threshold, iapmr, real_data, **kwargs): # plot a dot on threshold versus IAPMR line and show IAPMR as a number axlim = mpl.axis() mpl.plot(threshold, 100. * iapmr, 'o', color='C3', **kwargs) if not real_data: mpl.annotate( 'IAPMR at\noperating point', xy=(threshold, 100. * iapmr), xycoords='data', xytext=(0.85, 0.6), textcoords='axes fraction', color='black', size='large', arrowprops=dict(facecolor='black', shrink=0.05, width=2), horizontalalignment='center', verticalalignment='top', ) else: mpl.text(threshold + (threshold - axlim[0]) / 12, 100. * iapmr, '%.1f%%' % (100. * iapmr,), color='C3') Theophile GENTILHOMME committed May 04, 2018 158 159 160 161 162 163 164 def _iapmr_line_plot(scores, n_points=100, **kwargs): axlim = mpl.axis() step = (axlim[1] - axlim[0]) / float(n_points) thres = [(k * step) + axlim[0] for k in range(2, n_points - 1)] mix_prob_y = [] for k in thres: Theophile GENTILHOMME committed May 04, 2018 165 mix_prob_y.append(100. * error_utils.calc_pass_rate(k, scores)) 166 167 168 mpl.plot(thres, mix_prob_y, label='IAPMR', color='C3', **kwargs) Theophile GENTILHOMME committed May 04, 2018 169 170 171 172 173 def _iapmr_plot(scores, threshold, iapmr, real_data, **kwargs): _iapmr_dot(threshold, iapmr, real_data, **kwargs) _iapmr_line_plot(scores, n_points=100, **kwargs) Theophile GENTILHOMME committed May 04, 2018 174 175 176 177 178 179 180 class HistVuln(measure_figure.Hist): ''' Histograms for vulnerability ''' def _setup_hist(self, neg, pos): self._title_base = 'Vulnerability' self._density_hist( Amir MOHAMMADI committed May 03, 2018 181 pos[0], n=0, label='Genuine', color='C2' 182 183 ) self._density_hist( Theophile GENTILHOMME committed Apr 26, 2018 184 neg[0], n=1, label='Zero-effort impostors', alpha=0.8, color='C0' 185 186 ) self._density_hist( Theophile GENTILHOMME committed Apr 26, 2018 187 neg[1], n=2, label='Presentation attack', alpha=0.4, color='C7', Theophile GENTILHOMME committed Apr 23, 2018 188 hatch='\\\\' 189 190 ) Theophile GENTILHOMME committed May 03, 2018 191 def _lines(self, threshold, label, neg, pos, idx, **kwargs): 192 if 'iapmr_line' not in self._ctx.meta or self._ctx.meta['iapmr_line']: Amir MOHAMMADI committed May 08, 2018 193 # plot vertical line Theophile GENTILHOMME committed May 03, 2018 194 super(HistVuln, self)._lines(threshold, label, neg, pos, idx) 195 Amir MOHAMMADI committed May 08, 2018 196 # plot iapmr_line 197 198 199 200 201 iapmr, _ = farfrr(neg[1], pos[0], threshold) ax2 = mpl.twinx() # we never want grid lines on axis 2 ax2.grid(False) real_data = True if 'real_data' not in self._ctx.meta else \ Amir MOHAMMADI committed May 08, 2018 202 self._ctx.meta['real_data'] 203 _iapmr_plot(neg[1], threshold, iapmr, real_data=real_data) Theophile GENTILHOMME committed May 03, 2018 204 205 n = idx % self._step_print col = n % self._ncols Amir MOHAMMADI committed May 08, 2018 206 207 rest_print = self.n_systems - \ int(idx / self._step_print) * self._step_print Theophile GENTILHOMME committed May 03, 2018 208 209 if col == self._ncols - 1 or n == rest_print - 1: ax2.set_ylabel("IAPMR (%)", color='C3') 210 211 212 213 ax2.tick_params(axis='y', colors='red') ax2.yaxis.label.set_color('red') ax2.spines['right'].set_color('red') Theophile GENTILHOMME committed May 04, 2018 214 215 216 class PadPlot(measure_figure.PlotBase): '''Base class for PAD plots''' Amir MOHAMMADI committed May 08, 2018 217 218 219 def __init__(self, ctx, scores, evaluation, func_load): super(PadPlot, self).__init__(ctx, scores, evaluation, func_load) 220 mpl.rcParams['figure.constrained_layout.use'] = self._clayout 221 222 223 def end_process(self): '''Close pdf ''' Amir MOHAMMADI committed May 08, 2018 224 # do not want to close PDF when running evaluate 225 226 227 228 229 if 'PdfPages' in self._ctx.meta and \ ('closef' not in self._ctx.meta or self._ctx.meta['closef']): self._pdf_page.close() def _plot_legends(self): Amir MOHAMMADI committed May 08, 2018 230 # legends for all axes 231 232 233 234 235 236 lines = [] labels = [] for ax in mpl.gcf().get_axes(): li, la = ax.get_legend_handles_labels() lines += li labels += la Theophile GENTILHOMME committed May 04, 2018 237 238 239 240 if self._disp_legend: mpl.gca().legend(lines, labels, loc=self._legend_loc, fancybox=True, framealpha=0.5) 241 242 243 class Epc(PadPlot): ''' Handles the plotting of EPC ''' Amir MOHAMMADI committed May 08, 2018 244 245 246 247 def __init__(self, ctx, scores, evaluation, func_load): super(Epc, self).__init__(ctx, scores, evaluation, func_load) self._iapmr = True if 'iapmr' not in self._ctx.meta else \ Amir MOHAMMADI committed May 08, 2018 248 self._ctx.meta['iapmr'] Theophile GENTILHOMME committed Apr 24, 2018 249 250 251 252 self._title = self._title or ('EPC and IAPMR' if self._iapmr else 'EPC') self._x_label = self._x_label or r"Weight $\beta$" self._y_label = self._y_label or "WER (%)" Amir MOHAMMADI committed May 08, 2018 253 self._eval = True # always eval data with EPC 254 255 256 self._split = False self._nb_figs = 1 Theophile GENTILHOMME committed Apr 23, 2018 257 258 259 260 261 if self._min_arg != 4: raise click.BadParameter("You must provide 4 scores files:{licit," "spoof}/{dev,eval}") def compute(self, idx, input_scores, input_names): 262 ''' Plot EPC for PAD''' Theophile GENTILHOMME committed Apr 23, 2018 263 264 265 266 267 licit_dev_neg = input_scores[0][0] licit_dev_pos = input_scores[0][1] licit_eval_neg = input_scores[1][0] licit_eval_pos = input_scores[1][1] spoof_eval_neg = input_scores[3][0] 268 mpl.gcf().clear() Theophile GENTILHOMME committed Apr 13, 2018 269 epc_baseline = epc( 270 271 272 273 274 275 276 licit_dev_neg, licit_dev_pos, licit_eval_neg, licit_eval_pos, 100 ) mpl.plot( epc_baseline[:, 0], [100. * k for k in epc_baseline[:, 1]], color='C0', label=self._label( Theophile GENTILHOMME committed Apr 23, 2018 277 'WER', '%s-%s' % (input_names[0], input_names[1]), idx 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 ), linestyle='-' ) mpl.xlabel(self._x_label) mpl.ylabel(self._y_label) if self._iapmr: mix_prob_y = [] for k in epc_baseline[:, 2]: prob_attack = sum( 1 for i in spoof_eval_neg if i >= k ) / float(spoof_eval_neg.size) mix_prob_y.append(100. * prob_attack) mpl.gca().set_axisbelow(True) prob_ax = mpl.gca().twinx() mpl.plot( epc_baseline[:, 0], mix_prob_y, color='C3', linestyle='-', label=self._label( Theophile GENTILHOMME committed Apr 23, 2018 299 'IAPMR', '%s-%s' % (input_names[0], input_names[1]), idx 300 301 302 303 304 305 306 307 308 ) ) prob_ax.set_yticklabels(prob_ax.get_yticks()) prob_ax.tick_params(axis='y', colors='red') prob_ax.yaxis.label.set_color('red') prob_ax.spines['right'].set_color('red') ylabels = prob_ax.get_yticks() prob_ax.yaxis.set_ticklabels(["%.0f" % val for val in ylabels]) prob_ax.set_axisbelow(True) Theophile GENTILHOMME committed Apr 30, 2018 309 title = self._legends[idx] if self._legends is not None else self._title Theophile GENTILHOMME committed May 04, 2018 310 311 if title.replace(' ', ''): mpl.title(title) Amir MOHAMMADI committed May 08, 2018 312 # legends for all axes 313 314 315 316 self._plot_legends() mpl.xticks(rotation=self._x_rotation) self._pdf_page.savefig(mpl.gcf()) Theophile GENTILHOMME committed May 04, 2018 317 318 319 class Epsc(PadPlot): ''' Handles the plotting of EPSC ''' Amir MOHAMMADI committed May 08, 2018 320 321 322 323 324 def __init__(self, ctx, scores, evaluation, func_load, criteria, var_param, fixed_param): super(Epsc, self).__init__(ctx, scores, evaluation, func_load) self._iapmr = False if 'iapmr' not in self._ctx.meta else \ Amir MOHAMMADI committed May 08, 2018 325 self._ctx.meta['iapmr'] 326 self._wer = True if 'wer' not in self._ctx.meta else \ Amir MOHAMMADI committed May 08, 2018 327 self._ctx.meta['wer'] 328 329 330 self._criteria = 'eer' if criteria is None else criteria self._var_param = "omega" if var_param is None else var_param self._fixed_param = 0.5 if fixed_param is None else fixed_param Amir MOHAMMADI committed May 08, 2018 331 self._eval = True # always eval data with EPC 332 333 self._split = False self._nb_figs = 1 Theophile GENTILHOMME committed Apr 23, 2018 334 335 336 337 338 self._title = '' if self._min_arg != 4: raise click.BadParameter("You must provide 4 scores files:{licit," "spoof}/{dev,eval}") 339 Theophile GENTILHOMME committed Apr 23, 2018 340 def compute(self, idx, input_scores, input_names): 341 ''' Plot EPSC for PAD''' Theophile GENTILHOMME committed Apr 23, 2018 342 343 344 345 346 347 348 349 licit_dev_neg = input_scores[0][0] licit_dev_pos = input_scores[0][1] licit_eval_neg = input_scores[1][0] licit_eval_pos = input_scores[1][1] spoof_dev_neg = input_scores[2][0] spoof_dev_pos = input_scores[2][1] spoof_eval_neg = input_scores[3][0] spoof_eval_pos = input_scores[3][1] Theophile GENTILHOMME committed Apr 30, 2018 350 title = self._legends[idx] if self._legends is not None else None 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 mpl.gcf().clear() points = 10 if self._var_param == 'omega': omega, beta, thrs = error_utils.epsc_thresholds( licit_dev_neg, licit_dev_pos, spoof_dev_neg, spoof_dev_pos, points=points, criteria=self._criteria, beta=self._fixed_param) else: omega, beta, thrs = error_utils.epsc_thresholds( licit_dev_neg, licit_dev_pos, spoof_dev_neg, spoof_dev_pos, points=points, Amir MOHAMMADI committed May 08, 2018 371 criteria=self._criteria, 372 373 374 375 376 377 378 omega=self._fixed_param ) errors = error_utils.all_error_rates( licit_eval_neg, licit_eval_pos, spoof_eval_neg, spoof_eval_pos, thrs, omega, beta ) # error rates are returned in a list in the Amir MOHAMMADI committed May 08, 2018 379 # following order: frr, far, IAPMR, far_w, wer_w 380 381 382 383 ax1 = mpl.subplot( 111 ) # EPC like curves for FVAS fused scores for weighted error rates Amir MOHAMMADI committed May 08, 2018 384 # between the negatives (impostors and Presentation attacks) 385 386 387 388 389 390 391 392 if self._wer: if self._var_param == 'omega': mpl.plot( omega, 100. * errors[4].flatten(), color='C0', linestyle='-', label=r"WER$_{\omega,\beta}$") Theophile GENTILHOMME committed Apr 30, 2018 393 mpl.xlabel(self._x_label or r"Weight $\omega$") 394 395 396 397 398 399 400 else: mpl.plot( beta, 100. * errors[4].flatten(), color='C0', linestyle='-', label=r"WER$_{\omega,\beta}$") Theophile GENTILHOMME committed Apr 30, 2018 401 402 mpl.xlabel(self._x_label or r"Weight $\beta$") mpl.ylabel(self._y_label or r"WER$_{\omega,\beta}$ (%)") 403 404 405 406 407 408 409 410 411 412 413 414 415 if self._iapmr: axis = mpl.gca() if self._wer: axis = mpl.twinx() axis.grid(False) if self._var_param == 'omega': mpl.plot( omega, 100. * errors[2].flatten(), color='C3', linestyle='-', label='IAPMR') Theophile GENTILHOMME committed Apr 30, 2018 416 mpl.xlabel(self._x_label or r"Weight $\omega$") 417 418 419 420 421 422 423 else: mpl.plot( beta, 100. * errors[2].flatten(), color='C3', linestyle='-', label='IAPMR') Theophile GENTILHOMME committed Apr 30, 2018 424 425 mpl.xlabel(self._x_label or r"Weight $\beta$") mpl.ylabel(self._y_label or r"IAPMR (%)") 426 427 428 429 430 431 432 if self._wer: axis.set_yticklabels(axis.get_yticks()) axis.tick_params(axis='y', colors='red') axis.yaxis.label.set_color('red') axis.spines['right'].set_color('red') if self._var_param == 'omega': Theophile GENTILHOMME committed May 04, 2018 433 if title is not None and title.replace(' ', ''): Amir MOHAMMADI committed May 08, 2018 434 mpl.title(title or (r"EPSC with $\beta$ = %.2f" % Theophile GENTILHOMME committed May 04, 2018 435 self._fixed_param)) 436 else: Theophile GENTILHOMME committed May 04, 2018 437 if title is not None and title.replace(' ', ''): Amir MOHAMMADI committed May 08, 2018 438 mpl.title(title or (r"EPSC with $\omega$ = %.2f" % Theophile GENTILHOMME committed May 04, 2018 439 self._fixed_param)) 440 441 442 443 444 445 mpl.grid() self._plot_legends() ax1.set_xticklabels(ax1.get_xticks()) ax1.set_yticklabels(ax1.get_yticks()) mpl.xticks(rotation=self._x_rotation) 446 self._pdf_page.savefig() 447 Theophile GENTILHOMME committed May 04, 2018 448 449 450 class Epsc3D(Epsc): ''' 3D EPSC plots for PAD''' Amir MOHAMMADI committed May 08, 2018 451 Theophile GENTILHOMME committed Apr 23, 2018 452 def compute(self, idx, input_scores, input_names): 453 ''' Implements plots''' Theophile GENTILHOMME committed Apr 23, 2018 454 455 456 457 458 459 460 461 licit_dev_neg = input_scores[0][0] licit_dev_pos = input_scores[0][1] licit_eval_neg = input_scores[1][0] licit_eval_pos = input_scores[1][1] spoof_dev_neg = input_scores[2][0] spoof_dev_pos = input_scores[2][1] spoof_eval_neg = input_scores[3][0] spoof_eval_pos = input_scores[3][1] 462 Theophile GENTILHOMME committed May 04, 2018 463 title = self._legends[idx] if self._legends is not None else "3D EPSC" 464 465 466 mpl.rcParams.pop('key', None) 467 mpl.gcf().clear() 468 mpl.gcf().set_constrained_layout(self._clayout) 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 from mpl_toolkits.mplot3d import Axes3D from matplotlib import cm points = 10 omega, beta, thrs = error_utils.epsc_thresholds( licit_dev_neg, licit_dev_pos, spoof_dev_neg, spoof_dev_pos, points=points, criteria=self._criteria) errors = error_utils.all_error_rates( licit_eval_neg, licit_eval_pos, spoof_eval_neg, spoof_eval_pos, thrs, omega, beta ) # error rates are returned in a list as 2D numpy.ndarrays in # the following order: frr, far, IAPMR, far_w, wer_wb, hter_wb wer_errors = 100 * errors[2 if self._iapmr else 4] 491 ax1 = mpl.gcf().add_subplot(111, projection='3d') 492 493 494 495 496 497 498 499 500 501 502 W, B = np.meshgrid(omega, beta) ax1.plot_wireframe( W, B, wer_errors, cmap=cm.coolwarm, antialiased=False ) # surface if self._iapmr: ax1.azim = -30 ax1.elev = 50 Theophile GENTILHOMME committed Apr 30, 2018 503 504 ax1.set_xlabel(self._x_label or r"Weight $\omega$") ax1.set_ylabel(self._y_label or r"Weight $\beta$") 505 506 507 508 ax1.set_zlabel( r"WER$_{\omega,\beta}$ (%)" if self._wer else "IAPMR (%)" ) Theophile GENTILHOMME committed May 04, 2018 509 if title.replace(' ', ''): Theophile GENTILHOMME committed May 04, 2018 510 mpl.title(title) 511 512 513 514 515 ax1.set_xticklabels(ax1.get_xticks()) ax1.set_yticklabels(ax1.get_yticks()) ax1.set_zticklabels(ax1.get_zticks()) 516 517 self._pdf_page.savefig() Theophile GENTILHOMME committed May 04, 2018 518 519 520 class Det(PadPlot): '''DET for PAD''' Amir MOHAMMADI committed May 08, 2018 521 522 523 524 def __init__(self, ctx, scores, evaluation, func_load, criteria, real_data): super(Det, self).__init__(ctx, scores, evaluation, func_load) self._no_spoof = False if 'no_spoof' not in ctx.meta else\ Amir MOHAMMADI committed May 08, 2018 525 ctx.meta['no_spoof'] 526 527 528 self._criteria = criteria self._real_data = True if real_data is None else real_data Theophile GENTILHOMME committed Apr 23, 2018 529 def compute(self, idx, input_scores, input_names): 530 ''' Implements plots''' Theophile GENTILHOMME committed Apr 23, 2018 531 532 533 534 535 536 licit_dev_neg = input_scores[0][0] licit_dev_pos = input_scores[0][1] licit_eval_neg = input_scores[1][0] licit_eval_pos = input_scores[1][1] spoof_eval_neg = input_scores[3][0] if len(input_scores) > 2 else None spoof_eval_pos = input_scores[3][1] if len(input_scores) > 2 else None 537 538 539 540 541 542 543 det( licit_eval_neg, licit_eval_pos, self._points, color=self._colors[idx], linestyle='-', Theophile GENTILHOMME committed Apr 23, 2018 544 label=self._label("licit", input_names[0], idx) 545 546 547 548 549 550 551 552 ) if not self._no_spoof and spoof_eval_neg is not None: det( spoof_eval_neg, spoof_eval_pos, self._points, color=self._colors[idx], linestyle='--', Theophile GENTILHOMME committed Apr 23, 2018 553 label=self._label("spoof", input_names[3], idx) 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 ) if self._criteria is None: return thres_baseline = calc_threshold( self._criteria, licit_dev_neg, licit_dev_pos ) axlim = mpl.axis() farfrr_licit = farfrr( licit_eval_neg, licit_eval_pos, thres_baseline) # calculate test frr @ EER (licit scenario) farfrr_spoof = farfrr( spoof_eval_neg, spoof_eval_pos, thres_baseline) # calculate test frr @ EER (spoof scenario) farfrr_licit_det = [ ppndf(i) for i in farfrr_licit ] # find the FAR and FRR values that need to be plotted on normal deviate # scale farfrr_spoof_det = [ ppndf(i) for i in farfrr_spoof ] # find the FAR and FRR values that need to be plotted on normal deviate # scale if not self._real_data: mpl.axhline( y=farfrr_licit_det[1], xmin=axlim[2], xmax=axlim[3], color='k', linestyle='--', label="FRR @ EER") # vertical FRR threshold else: mpl.axhline( y=farfrr_licit_det[1], xmin=axlim[0], xmax=axlim[1], color='k', linestyle='--', label="FRR = %.2f%%" % (farfrr_licit[1] * 100)) # vertical FRR threshold mpl.plot( farfrr_licit_det[0], farfrr_licit_det[1], 'o', color=self._colors[idx], markersize=9) # FAR point, licit scenario mpl.plot( farfrr_spoof_det[0], farfrr_spoof_det[1], 'o', color=self._colors[idx], markersize=9) # FAR point, spoof scenario # annotate the FAR points xyannotate_licit = [ ppndf(0.7 * farfrr_licit[0]), ppndf(1.8 * farfrr_licit[1]) ] xyannotate_spoof = [ ppndf(0.95 * farfrr_spoof[0]), ppndf(1.8 * farfrr_licit[1]) ] if not self._real_data: mpl.annotate( 'FMR @\noperating point', xy=(farfrr_licit_det[0], farfrr_licit_det[1]), xycoords='data', xytext=(xyannotate_licit[0], xyannotate_licit[1]), color=self._colors[idx]) mpl.annotate( 'IAPMR @\noperating point', xy=(farfrr_spoof_det[0], farfrr_spoof_det[1]), xycoords='data', xytext=(xyannotate_spoof[0], xyannotate_spoof[1]), color=self._colors[idx]) else: mpl.annotate( 'FAR=%.2f%%' % (farfrr_licit[0] * 100), xy=(farfrr_licit_det[0], farfrr_licit_det[1]), xycoords='data', xytext=(xyannotate_licit[0], xyannotate_licit[1]), color=self._colors[idx], size='large') mpl.annotate( 'IAPMR=\n%.2f%%' % (farfrr_spoof[0] * 100), xy=(farfrr_spoof_det[0], farfrr_spoof_det[1]), xycoords='data', xytext=(xyannotate_spoof[0], xyannotate_spoof[1]), color=self._colors[idx], size='large') def end_process(self): ''' Set title, legend, axis labels, grid colors, save figures and close pdf is needed ''' Amir MOHAMMADI committed May 08, 2018 654 # only for plots 655 656 657 658 add = '' if not self._no_spoof: add = " and overlaid SPOOF scenario" title = self._title if self._title is not None else \ Amir MOHAMMADI committed May 08, 2018 659 ('DET: LICIT' + add) Theophile GENTILHOMME committed May 04, 2018 660 661 if title.replace(' ', ''): mpl.title(title) 662 663 664 mpl.xlabel(self._x_label or "False Acceptance Rate (%)") mpl.ylabel(self._y_label or "False Rejection Rate (%)") mpl.grid(True, color=self._grid_color) Theophile GENTILHOMME committed May 04, 2018 665 666 if self._disp_legend: mpl.legend(loc=self._legend_loc) 667 668 669 670 671 672 673 674 675 676 677 self._set_axis() fig = mpl.gcf() mpl.xticks(rotation=self._x_rotation) mpl.tick_params(axis='both', which='major', labelsize=4) for tick in mpl.gca().xaxis.get_major_ticks(): tick.label.set_fontsize(6) for tick in mpl.gca().yaxis.get_major_ticks(): tick.label.set_fontsize(6) self._pdf_page.savefig(fig) Amir MOHAMMADI committed May 08, 2018 678 # do not want to close PDF when running evaluate 679 if 'PdfPages' in self._ctx.meta and \ Amir MOHAMMADI committed May 08, 2018 680 ('closef' not in self._ctx.meta or self._ctx.meta['closef']): 681 682 683 684 685 686 687 self._pdf_page.close() def _set_axis(self): if self._axlim is not None and None not in self._axlim: det_axis(self._axlim) else: det_axis([0.01, 99, 0.01, 99]) Theophile GENTILHOMME committed Apr 24, 2018 688 Theophile GENTILHOMME committed May 04, 2018 689 Theophile GENTILHOMME committed Apr 24, 2018 690 691 class FmrIapmr(PadPlot): '''FMR vs IAPMR''' Amir MOHAMMADI committed May 08, 2018 692 Theophile GENTILHOMME committed Apr 24, 2018 693 694 def __init__(self, ctx, scores, evaluation, func_load): super(FmrIapmr, self).__init__(ctx, scores, evaluation, func_load) Amir MOHAMMADI committed May 08, 2018 695 self._eval = True # always eval data with EPC Theophile GENTILHOMME committed Apr 24, 2018 696 697 698 self._split = False self._nb_figs = 1 self._semilogx = False if 'semilogx' not in ctx.meta else\ Amir MOHAMMADI committed May 08, 2018 699 ctx.meta['semilogx'] Theophile GENTILHOMME committed Apr 24, 2018 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 if self._min_arg != 4: raise click.BadParameter("You must provide 4 scores files:{licit," "spoof}/{dev,eval}") def compute(self, idx, input_scores, input_names): ''' Implements plots''' licit_eval_neg = input_scores[1][0] licit_eval_pos = input_scores[1][1] spoof_eval_neg = input_scores[3][0] fmr_list = np.linspace(0, 1, 100) iapmr_list = [] for i, fmr in enumerate(fmr_list): thr = far_threshold(licit_eval_neg, licit_eval_pos, fmr, True) iapmr_list.append(farfrr(spoof_eval_neg, licit_eval_pos, thr)[0]) # re-calculate fmr since threshold might give a different result # for fmr. fmr_list[i] = farfrr(licit_eval_neg, licit_eval_pos, thr)[0] Theophile GENTILHOMME committed Apr 30, 2018 717 label = self._legends[idx] if self._legends is not None else \ Amir MOHAMMADI committed May 08, 2018 718 '(%s/%s)' % (input_names[1], input_names[3]) Theophile GENTILHOMME committed Apr 24, 2018 719 720 721 722 723 724 725 726 if self._semilogx: mpl.semilogx(fmr_list, iapmr_list, label=label) else: mpl.plot(fmr_list, iapmr_list, label=label) def end_process(self): ''' Set title, legend, axis labels, grid colors, save figures and close pdf is needed ''' Amir MOHAMMADI committed May 08, 2018 727 # only for plots Theophile GENTILHOMME committed Apr 24, 2018 728 title = self._title if self._title is not None else "FMR vs IAPMR" Theophile GENTILHOMME committed May 04, 2018 729 730 if title.replace(' ', ''): mpl.title(title) Theophile GENTILHOMME committed Apr 24, 2018 731 732 733 mpl.xlabel(self._x_label or "False Match Rate (%)") mpl.ylabel(self._y_label or "IAPMR (%)") mpl.grid(True, color=self._grid_color) Theophile GENTILHOMME committed May 04, 2018 734 735 if self._disp_legend: mpl.legend(loc=self._legend_loc) Theophile GENTILHOMME committed Apr 24, 2018 736 737 738 739 740 741 742 self._set_axis() fig = mpl.gcf() mpl.xticks(rotation=self._x_rotation) mpl.tick_params(axis='both', which='major', labelsize=4) self._pdf_page.savefig(fig) Amir MOHAMMADI committed May 08, 2018 743 # do not want to close PDF when running evaluate Theophile GENTILHOMME committed Apr 24, 2018 744 if 'PdfPages' in self._ctx.meta and \ Amir MOHAMMADI committed May 08, 2018 745 ('closef' not in self._ctx.meta or self._ctx.meta['closef']): Theophile GENTILHOMME committed Apr 24, 2018 746 self._pdf_page.close()