figure.py 27.3 KB
 1 2 3 4 '''Runs error analysis on score sets, outputs metrics and plots''' import click import numpy as np  5 import matplotlib.pyplot as mpl  Amir MOHAMMADI committed May 08, 2018 6 import bob.measure.script.figure as measure_figure  Theophile GENTILHOMME committed May 25, 2018 7 import bob.bio.base.script.figure as bio_figure  8 from tabulate import tabulate  Amir MOHAMMADI committed May 08, 2018 9 from bob.measure.utils import get_fta_list  10 from bob.measure import (  11  far_threshold, eer_threshold, min_hter_threshold, farfrr, epc, ppndf  12 )  13 from bob.measure.plot import (det, det_axis)  14 15 16 17 from . import error_utils ALL_CRITERIA = ('bpcer20', 'eer', 'min-hter')  Theophile GENTILHOMME committed May 04, 2018 18   19 20 21 22 23 24 25 def calc_threshold(method, neg, pos): """Calculates the threshold based on the given method. The scores should be sorted! Parameters ---------- method : str  Theophile GENTILHOMME committed May 25, 2018 26  One of bpcer20, eer, min-hter.  27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53  neg : array_like The negative scores. They should be sorted! pos : array_like The positive scores. They should be sorted! Returns ------- float The calculated threshold. Raises ------ ValueError If method is unknown. """ method = method.lower() if method == 'bpcer20': threshold = far_threshold(neg, pos, 0.05, True) elif method == 'eer': threshold = eer_threshold(neg, pos, True) elif method == 'min-hter': threshold = min_hter_threshold(neg, pos, True) else: raise ValueError("Unknown threshold criteria: {}".format(method)) return threshold  Theophile GENTILHOMME committed May 04, 2018 54   55 56 57 58 59 class Metrics(measure_figure.Metrics): def __init__(self, ctx, scores, evaluation, func_load): super(Metrics, self).__init__(ctx, scores, evaluation, func_load) ''' Compute metrics from score files'''  Amir MOHAMMADI committed May 08, 2018 60   Theophile GENTILHOMME committed Apr 23, 2018 61  def compute(self, idx, input_scores, input_names):  62  ''' Compute metrics for the given criteria'''  Theophile GENTILHOMME committed Apr 23, 2018 63 64 65 66 67 68  neg_list, pos_list, _ = get_fta_list(input_scores) dev_neg, dev_pos = neg_list[0], pos_list[0] dev_file = input_names[0] if self._eval: eval_neg, eval_pos = neg_list[1], pos_list[1] eval_file = input_names[1]  69   Theophile GENTILHOMME committed Apr 30, 2018 70  title = self._legends[idx] if self._legends is not None else None  71  headers = ['' or title, 'Development %s' % dev_file]  Theophile GENTILHOMME committed Apr 23, 2018 72  if self._eval:  73 74 75 76 77 78 79  headers.append('Eval. % s' % eval_file) for m in ALL_CRITERIA: raws = [] threshold = calc_threshold(m, dev_neg, dev_pos) click.echo("\nThreshold of %f selected with the %s criteria" % ( threshold, m)) apcer, bpcer = farfrr(dev_neg, dev_pos, threshold)  Theophile GENTILHOMME committed May 25, 2018 80 81 82  raws.append(['APCER', '{:>5.1f}%'.format(apcer * 100)]) raws.append(['BP', '{:>5.1f}%'.format(bpcer * 100)]) raws.append(['ACER', '{:>5.1f}%'.format((apcer + bpcer) * 50)])  83 84 85 86 87 88 89 90 91 92 93  if self._eval and eval_neg is not None: apcer, bpcer = farfrr(eval_neg, eval_pos, threshold) raws[0].append('{:>5.1f}%'.format(apcer * 100)) raws[1].append('{:>5.1f}%'.format(bpcer * 100)) raws[2].append('{:>5.1f}%'.format((apcer + bpcer) * 50)) click.echo( tabulate(raws, headers, self._tablefmt), file=self.log_file )  Theophile GENTILHOMME committed May 04, 2018 94   Theophile GENTILHOMME committed May 04, 2018 95 96 97 98 99 class MetricsVuln(measure_figure.Metrics): def __init__(self, ctx, scores, evaluation, func_load): super(MetricsVuln, self).__init__(ctx, scores, evaluation, func_load) ''' Compute metrics from score files'''  Amir MOHAMMADI committed May 08, 2018 100   Theophile GENTILHOMME committed May 04, 2018 101 102 103 104 105 106  def compute(self, idx, input_scores, input_names): ''' Compute metrics for the given criteria''' neg_list, pos_list, _ = get_fta_list(input_scores) dev_neg, dev_pos = neg_list[0], pos_list[0] criter = self._criterion or 'eer' threshold = calc_threshold(criter, dev_neg, dev_pos) \  Amir MOHAMMADI committed May 08, 2018 107  if self._thres is None else self._thres[idx]  Theophile GENTILHOMME committed May 04, 2018 108 109 110 111 112 113  far, frr = farfrr(neg_list[1], pos_list[1], threshold) iapmr, _ = farfrr(neg_list[3], pos_list[1], threshold) title = self._legends[idx] if self._legends is not None else None headers = ['' or title, '%s (threshold=%.2g)' % (criter.upper(), threshold)] rows = []  Amir MOHAMMADI committed May 08, 2018 114  rows.append(['FMR (%)', '{:>5.1f}%'.format(100 * far)])  Theophile GENTILHOMME committed May 25, 2018 115  rows.append(['BPCER (%)', '{:>5.1f}%'.format(frr * 100)])  Amir MOHAMMADI committed May 08, 2018 116 117  rows.append(['HTER (%)', '{:>5.1f}%'.format(50 * (far + frr))]) rows.append(['IAPMR (%)', '{:>5.1f}%'.format(100 * iapmr)])  Theophile GENTILHOMME committed May 04, 2018 118 119 120 121 122 123  click.echo( tabulate(rows, headers, self._tablefmt), file=self.log_file )  124 125 126 127 128 129 class HistPad(measure_figure.Hist): ''' Histograms for PAD ''' def _setup_hist(self, neg, pos): self._title_base = 'PAD' self._density_hist(  Theophile GENTILHOMME committed Apr 26, 2018 130  pos[0], n=0, label='Bona Fide', color='C1'  131 132  ) self._density_hist(  Theophile GENTILHOMME committed Apr 26, 2018 133  neg[0], n=1, label='Presentation attack', alpha=0.4, color='C7',  Theophile GENTILHOMME committed Apr 23, 2018 134  hatch='\\\\'  135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158  ) def _iapmr_dot(threshold, iapmr, real_data, **kwargs): # plot a dot on threshold versus IAPMR line and show IAPMR as a number axlim = mpl.axis() mpl.plot(threshold, 100. * iapmr, 'o', color='C3', **kwargs) if not real_data: mpl.annotate( 'IAPMR at\noperating point', xy=(threshold, 100. * iapmr), xycoords='data', xytext=(0.85, 0.6), textcoords='axes fraction', color='black', size='large', arrowprops=dict(facecolor='black', shrink=0.05, width=2), horizontalalignment='center', verticalalignment='top', ) else: mpl.text(threshold + (threshold - axlim[0]) / 12, 100. * iapmr, '%.1f%%' % (100. * iapmr,), color='C3')  Theophile GENTILHOMME committed May 04, 2018 159   160 161 162 163 164 165 def _iapmr_line_plot(scores, n_points=100, **kwargs): axlim = mpl.axis() step = (axlim[1] - axlim[0]) / float(n_points) thres = [(k * step) + axlim[0] for k in range(2, n_points - 1)] mix_prob_y = [] for k in thres:  Theophile GENTILHOMME committed May 04, 2018 166  mix_prob_y.append(100. * error_utils.calc_pass_rate(k, scores))  167 168 169  mpl.plot(thres, mix_prob_y, label='IAPMR', color='C3', **kwargs)  Theophile GENTILHOMME committed May 04, 2018 170   171 172 173 174 def _iapmr_plot(scores, threshold, iapmr, real_data, **kwargs): _iapmr_dot(threshold, iapmr, real_data, **kwargs) _iapmr_line_plot(scores, n_points=100, **kwargs)  Theophile GENTILHOMME committed May 04, 2018 175   176 177 178 179 180 181 class HistVuln(measure_figure.Hist): ''' Histograms for vulnerability ''' def _setup_hist(self, neg, pos): self._title_base = 'Vulnerability' self._density_hist(  Amir MOHAMMADI committed May 03, 2018 182  pos[0], n=0, label='Genuine', color='C2'  183 184  ) self._density_hist(  Theophile GENTILHOMME committed Apr 26, 2018 185  neg[0], n=1, label='Zero-effort impostors', alpha=0.8, color='C0'  186 187  ) self._density_hist(  Theophile GENTILHOMME committed Apr 26, 2018 188  neg[1], n=2, label='Presentation attack', alpha=0.4, color='C7',  Theophile GENTILHOMME committed Apr 23, 2018 189  hatch='\\\\'  190 191  )  Theophile GENTILHOMME committed May 03, 2018 192  def _lines(self, threshold, label, neg, pos, idx, **kwargs):  193  if 'iapmr_line' not in self._ctx.meta or self._ctx.meta['iapmr_line']:  Amir MOHAMMADI committed May 08, 2018 194  # plot vertical line  Theophile GENTILHOMME committed May 03, 2018 195  super(HistVuln, self)._lines(threshold, label, neg, pos, idx)  196   Amir MOHAMMADI committed May 08, 2018 197  # plot iapmr_line  198 199 200 201 202  iapmr, _ = farfrr(neg[1], pos[0], threshold) ax2 = mpl.twinx() # we never want grid lines on axis 2 ax2.grid(False) real_data = True if 'real_data' not in self._ctx.meta else \  Amir MOHAMMADI committed May 08, 2018 203  self._ctx.meta['real_data']  204  _iapmr_plot(neg[1], threshold, iapmr, real_data=real_data)  Theophile GENTILHOMME committed May 03, 2018 205 206  n = idx % self._step_print col = n % self._ncols  Amir MOHAMMADI committed May 08, 2018 207 208  rest_print = self.n_systems - \ int(idx / self._step_print) * self._step_print  Theophile GENTILHOMME committed May 03, 2018 209 210  if col == self._ncols - 1 or n == rest_print - 1: ax2.set_ylabel("IAPMR (%)", color='C3')  211 212 213 214  ax2.tick_params(axis='y', colors='red') ax2.yaxis.label.set_color('red') ax2.spines['right'].set_color('red')  Theophile GENTILHOMME committed May 04, 2018 215   216 217 class PadPlot(measure_figure.PlotBase): '''Base class for PAD plots'''  Amir MOHAMMADI committed May 08, 2018 218   219 220  def __init__(self, ctx, scores, evaluation, func_load): super(PadPlot, self).__init__(ctx, scores, evaluation, func_load)  221  mpl.rcParams['figure.constrained_layout.use'] = self._clayout  222 223 224  def end_process(self): '''Close pdf '''  Amir MOHAMMADI committed May 08, 2018 225  # do not want to close PDF when running evaluate  226 227 228 229 230  if 'PdfPages' in self._ctx.meta and \ ('closef' not in self._ctx.meta or self._ctx.meta['closef']): self._pdf_page.close() def _plot_legends(self):  Amir MOHAMMADI committed May 08, 2018 231  # legends for all axes  232 233 234 235 236 237  lines = [] labels = [] for ax in mpl.gcf().get_axes(): li, la = ax.get_legend_handles_labels() lines += li labels += la  Theophile GENTILHOMME committed May 04, 2018 238 239 240 241  if self._disp_legend: mpl.gca().legend(lines, labels, loc=self._legend_loc, fancybox=True, framealpha=0.5)  242 243 244  class Epc(PadPlot): ''' Handles the plotting of EPC '''  Amir MOHAMMADI committed May 08, 2018 245   246 247 248  def __init__(self, ctx, scores, evaluation, func_load): super(Epc, self).__init__(ctx, scores, evaluation, func_load) self._iapmr = True if 'iapmr' not in self._ctx.meta else \  Amir MOHAMMADI committed May 08, 2018 249  self._ctx.meta['iapmr']  Theophile GENTILHOMME committed Apr 24, 2018 250 251 252 253  self._title = self._title or ('EPC and IAPMR' if self._iapmr else 'EPC') self._x_label = self._x_label or r"Weight $\beta$" self._y_label = self._y_label or "WER (%)"  Amir MOHAMMADI committed May 08, 2018 254  self._eval = True # always eval data with EPC  255 256 257  self._split = False self._nb_figs = 1  Theophile GENTILHOMME committed Apr 23, 2018 258 259 260 261 262  if self._min_arg != 4: raise click.BadParameter("You must provide 4 scores files:{licit," "spoof}/{dev,eval}") def compute(self, idx, input_scores, input_names):  263  ''' Plot EPC for PAD'''  Theophile GENTILHOMME committed Apr 23, 2018 264 265 266 267 268  licit_dev_neg = input_scores[0][0] licit_dev_pos = input_scores[0][1] licit_eval_neg = input_scores[1][0] licit_eval_pos = input_scores[1][1] spoof_eval_neg = input_scores[3][0]  269  mpl.gcf().clear()  Theophile GENTILHOMME committed Apr 13, 2018 270  epc_baseline = epc(  271 272 273 274 275 276 277  licit_dev_neg, licit_dev_pos, licit_eval_neg, licit_eval_pos, 100 ) mpl.plot( epc_baseline[:, 0], [100. * k for k in epc_baseline[:, 1]], color='C0', label=self._label(  Theophile GENTILHOMME committed Apr 23, 2018 278  'WER', '%s-%s' % (input_names[0], input_names[1]), idx  279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299  ), linestyle='-' ) mpl.xlabel(self._x_label) mpl.ylabel(self._y_label) if self._iapmr: mix_prob_y = [] for k in epc_baseline[:, 2]: prob_attack = sum( 1 for i in spoof_eval_neg if i >= k ) / float(spoof_eval_neg.size) mix_prob_y.append(100. * prob_attack) mpl.gca().set_axisbelow(True) prob_ax = mpl.gca().twinx() mpl.plot( epc_baseline[:, 0], mix_prob_y, color='C3', linestyle='-', label=self._label(  Theophile GENTILHOMME committed Apr 23, 2018 300  'IAPMR', '%s-%s' % (input_names[0], input_names[1]), idx  301 302 303  ) ) prob_ax.set_yticklabels(prob_ax.get_yticks())  Theophile GENTILHOMME committed May 25, 2018 304 305 306  prob_ax.tick_params(axis='y', colors='C3') prob_ax.yaxis.label.set_color('C3') prob_ax.spines['right'].set_color('C3')  307 308  ylabels = prob_ax.get_yticks() prob_ax.yaxis.set_ticklabels(["%.0f" % val for val in ylabels])  Theophile GENTILHOMME committed May 25, 2018 309  prob_ax.set_ylabel('IAPMR', color='C3')  310  prob_ax.set_axisbelow(True)  Theophile GENTILHOMME committed Apr 30, 2018 311  title = self._legends[idx] if self._legends is not None else self._title  Theophile GENTILHOMME committed May 04, 2018 312 313  if title.replace(' ', ''): mpl.title(title)  Amir MOHAMMADI committed May 08, 2018 314  # legends for all axes  315 316 317 318  self._plot_legends() mpl.xticks(rotation=self._x_rotation) self._pdf_page.savefig(mpl.gcf())  Theophile GENTILHOMME committed May 04, 2018 319   320 321 class Epsc(PadPlot): ''' Handles the plotting of EPSC '''  Amir MOHAMMADI committed May 08, 2018 322   323 324 325 326  def __init__(self, ctx, scores, evaluation, func_load, criteria, var_param, fixed_param): super(Epsc, self).__init__(ctx, scores, evaluation, func_load) self._iapmr = False if 'iapmr' not in self._ctx.meta else \  Amir MOHAMMADI committed May 08, 2018 327  self._ctx.meta['iapmr']  328  self._wer = True if 'wer' not in self._ctx.meta else \  Amir MOHAMMADI committed May 08, 2018 329  self._ctx.meta['wer']  330 331 332  self._criteria = 'eer' if criteria is None else criteria self._var_param = "omega" if var_param is None else var_param self._fixed_param = 0.5 if fixed_param is None else fixed_param  Amir MOHAMMADI committed May 08, 2018 333  self._eval = True # always eval data with EPC  334 335  self._split = False self._nb_figs = 1  Theophile GENTILHOMME committed Apr 23, 2018 336 337 338 339 340  self._title = '' if self._min_arg != 4: raise click.BadParameter("You must provide 4 scores files:{licit," "spoof}/{dev,eval}")  341   Theophile GENTILHOMME committed Apr 23, 2018 342  def compute(self, idx, input_scores, input_names):  343  ''' Plot EPSC for PAD'''  Theophile GENTILHOMME committed Apr 23, 2018 344 345 346 347 348 349 350 351  licit_dev_neg = input_scores[0][0] licit_dev_pos = input_scores[0][1] licit_eval_neg = input_scores[1][0] licit_eval_pos = input_scores[1][1] spoof_dev_neg = input_scores[2][0] spoof_dev_pos = input_scores[2][1] spoof_eval_neg = input_scores[3][0] spoof_eval_pos = input_scores[3][1]  Theophile GENTILHOMME committed Apr 30, 2018 352  title = self._legends[idx] if self._legends is not None else None  353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372  mpl.gcf().clear() points = 10 if self._var_param == 'omega': omega, beta, thrs = error_utils.epsc_thresholds( licit_dev_neg, licit_dev_pos, spoof_dev_neg, spoof_dev_pos, points=points, criteria=self._criteria, beta=self._fixed_param) else: omega, beta, thrs = error_utils.epsc_thresholds( licit_dev_neg, licit_dev_pos, spoof_dev_neg, spoof_dev_pos, points=points,  Amir MOHAMMADI committed May 08, 2018 373  criteria=self._criteria,  374 375 376 377 378 379 380  omega=self._fixed_param ) errors = error_utils.all_error_rates( licit_eval_neg, licit_eval_pos, spoof_eval_neg, spoof_eval_pos, thrs, omega, beta ) # error rates are returned in a list in the  Amir MOHAMMADI committed May 08, 2018 381  # following order: frr, far, IAPMR, far_w, wer_w  382 383 384 385  ax1 = mpl.subplot( 111 ) # EPC like curves for FVAS fused scores for weighted error rates  Amir MOHAMMADI committed May 08, 2018 386  # between the negatives (impostors and Presentation attacks)  387 388 389 390 391 392 393 394  if self._wer: if self._var_param == 'omega': mpl.plot( omega, 100. * errors[4].flatten(), color='C0', linestyle='-', label=r"WER$_{\omega,\beta}$")  Theophile GENTILHOMME committed Apr 30, 2018 395  mpl.xlabel(self._x_label or r"Weight $\omega$")  396 397 398 399 400 401 402  else: mpl.plot( beta, 100. * errors[4].flatten(), color='C0', linestyle='-', label=r"WER$_{\omega,\beta}$")  Theophile GENTILHOMME committed Apr 30, 2018 403 404  mpl.xlabel(self._x_label or r"Weight $\beta$") mpl.ylabel(self._y_label or r"WER$_{\omega,\beta}$ (%)")  405 406 407 408 409 410 411 412 413 414 415 416 417  if self._iapmr: axis = mpl.gca() if self._wer: axis = mpl.twinx() axis.grid(False) if self._var_param == 'omega': mpl.plot( omega, 100. * errors[2].flatten(), color='C3', linestyle='-', label='IAPMR')  Theophile GENTILHOMME committed Apr 30, 2018 418  mpl.xlabel(self._x_label or r"Weight $\omega$")  419 420 421 422 423 424 425  else: mpl.plot( beta, 100. * errors[2].flatten(), color='C3', linestyle='-', label='IAPMR')  Theophile GENTILHOMME committed Apr 30, 2018 426 427  mpl.xlabel(self._x_label or r"Weight $\beta$") mpl.ylabel(self._y_label or r"IAPMR (%)")  428 429 430 431 432 433 434  if self._wer: axis.set_yticklabels(axis.get_yticks()) axis.tick_params(axis='y', colors='red') axis.yaxis.label.set_color('red') axis.spines['right'].set_color('red') if self._var_param == 'omega':  Theophile GENTILHOMME committed May 04, 2018 435  if title is not None and title.replace(' ', ''):  Amir MOHAMMADI committed May 08, 2018 436  mpl.title(title or (r"EPSC with $\beta$ = %.2f" %  Theophile GENTILHOMME committed May 04, 2018 437  self._fixed_param))  438  else:  Theophile GENTILHOMME committed May 04, 2018 439  if title is not None and title.replace(' ', ''):  Amir MOHAMMADI committed May 08, 2018 440  mpl.title(title or (r"EPSC with $\omega$ = %.2f" %  Theophile GENTILHOMME committed May 04, 2018 441  self._fixed_param))  442 443 444 445 446 447  mpl.grid() self._plot_legends() ax1.set_xticklabels(ax1.get_xticks()) ax1.set_yticklabels(ax1.get_yticks()) mpl.xticks(rotation=self._x_rotation)  448  self._pdf_page.savefig()  449   Theophile GENTILHOMME committed May 04, 2018 450   451 452 class Epsc3D(Epsc): ''' 3D EPSC plots for PAD'''  Amir MOHAMMADI committed May 08, 2018 453   Theophile GENTILHOMME committed Apr 23, 2018 454  def compute(self, idx, input_scores, input_names):  455  ''' Implements plots'''  Theophile GENTILHOMME committed Apr 23, 2018 456 457 458 459 460 461 462 463  licit_dev_neg = input_scores[0][0] licit_dev_pos = input_scores[0][1] licit_eval_neg = input_scores[1][0] licit_eval_pos = input_scores[1][1] spoof_dev_neg = input_scores[2][0] spoof_dev_pos = input_scores[2][1] spoof_eval_neg = input_scores[3][0] spoof_eval_pos = input_scores[3][1]  464   Theophile GENTILHOMME committed May 04, 2018 465  title = self._legends[idx] if self._legends is not None else "3D EPSC"  466   467 468  mpl.rcParams.pop('key', None)  469  mpl.gcf().clear()  470  mpl.gcf().set_constrained_layout(self._clayout)  471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492  from mpl_toolkits.mplot3d import Axes3D from matplotlib import cm points = 10 omega, beta, thrs = error_utils.epsc_thresholds( licit_dev_neg, licit_dev_pos, spoof_dev_neg, spoof_dev_pos, points=points, criteria=self._criteria) errors = error_utils.all_error_rates( licit_eval_neg, licit_eval_pos, spoof_eval_neg, spoof_eval_pos, thrs, omega, beta ) # error rates are returned in a list as 2D numpy.ndarrays in # the following order: frr, far, IAPMR, far_w, wer_wb, hter_wb wer_errors = 100 * errors[2 if self._iapmr else 4]  493  ax1 = mpl.gcf().add_subplot(111, projection='3d')  494 495 496 497 498 499 500 501 502 503 504  W, B = np.meshgrid(omega, beta) ax1.plot_wireframe( W, B, wer_errors, cmap=cm.coolwarm, antialiased=False ) # surface if self._iapmr: ax1.azim = -30 ax1.elev = 50  Theophile GENTILHOMME committed Apr 30, 2018 505 506  ax1.set_xlabel(self._x_label or r"Weight $\omega$") ax1.set_ylabel(self._y_label or r"Weight $\beta$")  507 508 509 510  ax1.set_zlabel( r"WER$_{\omega,\beta}$ (%)" if self._wer else "IAPMR (%)" )  Theophile GENTILHOMME committed May 04, 2018 511  if title.replace(' ', ''):  Theophile GENTILHOMME committed May 04, 2018 512  mpl.title(title)  513 514 515 516 517  ax1.set_xticklabels(ax1.get_xticks()) ax1.set_yticklabels(ax1.get_yticks()) ax1.set_zticklabels(ax1.get_zticks())  518 519  self._pdf_page.savefig()  Theophile GENTILHOMME committed May 04, 2018 520   Theophile GENTILHOMME committed May 25, 2018 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 class Roc(bio_figure.Roc): '''ROC for PARD''' def __init__(self, ctx, scores, evaluation, func_load): super(Roc, self).__init__(ctx, scores, evaluation, func_load) self._x_label = ctx.meta.get('x_label') or 'APCER' self._y_label = ctx.meta.get('y_label') or '1 - BPCER' class DetPad(bio_figure.Det): def __init__(self, ctx, scores, evaluation, func_load): super(DetPad, self).__init__(ctx, scores, evaluation, func_load) self._x_label = ctx.meta.get('x_label') or 'APCER' self._y_label = ctx.meta.get('y_label') or 'BPCER'  536 class Det(PadPlot):  Theophile GENTILHOMME committed May 25, 2018 537  '''DET for VULN'''  Amir MOHAMMADI committed May 08, 2018 538   Theophile GENTILHOMME committed May 14, 2018 539 540  def __init__(self, ctx, scores, evaluation, func_load, criteria, real_data, no_spoof):  541  super(Det, self).__init__(ctx, scores, evaluation, func_load)  Theophile GENTILHOMME committed May 14, 2018 542 543  self._no_spoof = no_spoof self._criteria = criteria or 'eer'  544  self._real_data = True if real_data is None else real_data  Theophile GENTILHOMME committed May 25, 2018 545 546  self._x_label = self._x_label or "APCER" self._y_label = self._y_label or "BPCER"  547   Theophile GENTILHOMME committed Apr 23, 2018 548  def compute(self, idx, input_scores, input_names):  549  ''' Implements plots'''  Theophile GENTILHOMME committed Apr 23, 2018 550 551 552 553 554 555  licit_dev_neg = input_scores[0][0] licit_dev_pos = input_scores[0][1] licit_eval_neg = input_scores[1][0] licit_eval_pos = input_scores[1][1] spoof_eval_neg = input_scores[3][0] if len(input_scores) > 2 else None spoof_eval_pos = input_scores[3][1] if len(input_scores) > 2 else None  556 557 558 559  det( licit_eval_neg, licit_eval_pos, self._points,  Theophile GENTILHOMME committed May 14, 2018 560  color='C0',  561  linestyle='-',  Theophile GENTILHOMME committed Apr 23, 2018 562  label=self._label("licit", input_names[0], idx)  563 564 565 566 567 568  ) if not self._no_spoof and spoof_eval_neg is not None: det( spoof_eval_neg, spoof_eval_pos, self._points,  Theophile GENTILHOMME committed May 14, 2018 569 570  color='C3', linestyle=':',  Theophile GENTILHOMME committed Apr 23, 2018 571  label=self._label("spoof", input_names[3], idx)  572 573  )  Theophile GENTILHOMME committed May 14, 2018 574  if self._criteria is None or self._no_spoof:  575 576 577 578 579 580 581 582 583 584  return thres_baseline = calc_threshold( self._criteria, licit_dev_neg, licit_dev_pos ) axlim = mpl.axis() farfrr_licit = farfrr( licit_eval_neg, licit_eval_pos,  Theophile GENTILHOMME committed May 14, 2018 585 586  thres_baseline ) # calculate test frr @ EER (licit scenario)  587 588  farfrr_spoof = farfrr( spoof_eval_neg, spoof_eval_pos,  Theophile GENTILHOMME committed May 14, 2018 589 590  thres_baseline ) # calculate test frr @ EER (spoof scenario)  591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607  farfrr_licit_det = [ ppndf(i) for i in farfrr_licit ] # find the FAR and FRR values that need to be plotted on normal deviate # scale farfrr_spoof_det = [ ppndf(i) for i in farfrr_spoof ] # find the FAR and FRR values that need to be plotted on normal deviate # scale if not self._real_data: mpl.axhline( y=farfrr_licit_det[1], xmin=axlim[2], xmax=axlim[3], color='k', linestyle='--',  Theophile GENTILHOMME committed May 25, 2018 608  label="BPCER @ EER")  609 610 611 612 613 614 615  else: mpl.axhline( y=farfrr_licit_det[1], xmin=axlim[0], xmax=axlim[1], color='k', linestyle='--',  Theophile GENTILHOMME committed May 25, 2018 616  label="BPCER = %.2f%%" %  Theophile GENTILHOMME committed May 14, 2018 617  (farfrr_licit[1] * 100))  618 619 620 621 622  mpl.plot( farfrr_licit_det[0], farfrr_licit_det[1], 'o',  Theophile GENTILHOMME committed May 14, 2018 623 624 625  color='C0', markersize=9 ) # FAR point, licit scenario  626 627 628 629  mpl.plot( farfrr_spoof_det[0], farfrr_spoof_det[1], 'o',  Theophile GENTILHOMME committed May 14, 2018 630 631 632  color='C3', markersize=9 ) # FAR point, spoof scenario  633 634 635  # annotate the FAR points xyannotate_licit = [  Theophile GENTILHOMME committed May 25, 2018 636 637  0.15 + farfrr_licit_det[0], farfrr_licit_det[1] - 0.15,  638 639  ] xyannotate_spoof = [  Theophile GENTILHOMME committed May 25, 2018 640 641  0.15 + farfrr_spoof_det[0], farfrr_spoof_det[1] - 0.15,  642 643 644 645  ] if not self._real_data: mpl.annotate(  Theophile GENTILHOMME committed May 14, 2018 646  'FMR @ operating point',  647 648  xy=(farfrr_licit_det[0], farfrr_licit_det[1]), xycoords='data',  Theophile GENTILHOMME committed May 25, 2018 649  xytext=(xyannotate_licit[0], xyannotate_licit[1]))  650  mpl.annotate(  Theophile GENTILHOMME committed May 14, 2018 651  'IAPMR @ operating point',  652 653  xy=(farfrr_spoof_det[0], farfrr_spoof_det[1]), xycoords='data',  Theophile GENTILHOMME committed May 25, 2018 654  xytext=(xyannotate_spoof[0], xyannotate_spoof[1]))  655 656  else: mpl.annotate(  Theophile GENTILHOMME committed May 25, 2018 657  'APCER=%.2f%%' % (farfrr_licit[0] * 100),  658 659 660  xy=(farfrr_licit_det[0], farfrr_licit_det[1]), xycoords='data', xytext=(xyannotate_licit[0], xyannotate_licit[1]),  Theophile GENTILHOMME committed May 14, 2018 661  color='C0',  662 663  size='large') mpl.annotate(  Theophile GENTILHOMME committed May 14, 2018 664  'IAPMR=%.2f%%' % (farfrr_spoof[0] * 100),  665 666 667  xy=(farfrr_spoof_det[0], farfrr_spoof_det[1]), xycoords='data', xytext=(xyannotate_spoof[0], xyannotate_spoof[1]),  Theophile GENTILHOMME committed May 14, 2018 668  color='C3',  669 670 671 672 673  size='large') def end_process(self): ''' Set title, legend, axis labels, grid colors, save figures and close pdf is needed '''  Amir MOHAMMADI committed May 08, 2018 674  # only for plots  675 676 677 678  add = '' if not self._no_spoof: add = " and overlaid SPOOF scenario" title = self._title if self._title is not None else \  Amir MOHAMMADI committed May 08, 2018 679  ('DET: LICIT' + add)  Theophile GENTILHOMME committed May 04, 2018 680 681  if title.replace(' ', ''): mpl.title(title)  682 683 684  mpl.xlabel(self._x_label or "False Acceptance Rate (%)") mpl.ylabel(self._y_label or "False Rejection Rate (%)") mpl.grid(True, color=self._grid_color)  Theophile GENTILHOMME committed May 04, 2018 685 686  if self._disp_legend: mpl.legend(loc=self._legend_loc)  687 688 689 690 691 692 693 694 695 696 697  self._set_axis() fig = mpl.gcf() mpl.xticks(rotation=self._x_rotation) mpl.tick_params(axis='both', which='major', labelsize=4) for tick in mpl.gca().xaxis.get_major_ticks(): tick.label.set_fontsize(6) for tick in mpl.gca().yaxis.get_major_ticks(): tick.label.set_fontsize(6) self._pdf_page.savefig(fig)  Amir MOHAMMADI committed May 08, 2018 698  # do not want to close PDF when running evaluate  699  if 'PdfPages' in self._ctx.meta and \  Amir MOHAMMADI committed May 08, 2018 700  ('closef' not in self._ctx.meta or self._ctx.meta['closef']):  701 702 703 704 705 706 707  self._pdf_page.close() def _set_axis(self): if self._axlim is not None and None not in self._axlim: det_axis(self._axlim) else: det_axis([0.01, 99, 0.01, 99])  Theophile GENTILHOMME committed Apr 24, 2018 708   Theophile GENTILHOMME committed May 04, 2018 709   Theophile GENTILHOMME committed Apr 24, 2018 710 711 class FmrIapmr(PadPlot): '''FMR vs IAPMR'''  Amir MOHAMMADI committed May 08, 2018 712   Theophile GENTILHOMME committed Apr 24, 2018 713 714  def __init__(self, ctx, scores, evaluation, func_load): super(FmrIapmr, self).__init__(ctx, scores, evaluation, func_load)  Amir MOHAMMADI committed May 08, 2018 715  self._eval = True # always eval data with EPC  Theophile GENTILHOMME committed Apr 24, 2018 716 717 718  self._split = False self._nb_figs = 1 self._semilogx = False if 'semilogx' not in ctx.meta else\  Amir MOHAMMADI committed May 08, 2018 719  ctx.meta['semilogx']  Theophile GENTILHOMME committed Apr 24, 2018 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736  if self._min_arg != 4: raise click.BadParameter("You must provide 4 scores files:{licit," "spoof}/{dev,eval}") def compute(self, idx, input_scores, input_names): ''' Implements plots''' licit_eval_neg = input_scores[1][0] licit_eval_pos = input_scores[1][1] spoof_eval_neg = input_scores[3][0] fmr_list = np.linspace(0, 1, 100) iapmr_list = [] for i, fmr in enumerate(fmr_list): thr = far_threshold(licit_eval_neg, licit_eval_pos, fmr, True) iapmr_list.append(farfrr(spoof_eval_neg, licit_eval_pos, thr)[0]) # re-calculate fmr since threshold might give a different result # for fmr. fmr_list[i] = farfrr(licit_eval_neg, licit_eval_pos, thr)[0]  Theophile GENTILHOMME committed Apr 30, 2018 737  label = self._legends[idx] if self._legends is not None else \  Amir MOHAMMADI committed May 08, 2018 738  '(%s/%s)' % (input_names[1], input_names[3])  Theophile GENTILHOMME committed Apr 24, 2018 739 740 741 742 743 744 745 746  if self._semilogx: mpl.semilogx(fmr_list, iapmr_list, label=label) else: mpl.plot(fmr_list, iapmr_list, label=label) def end_process(self): ''' Set title, legend, axis labels, grid colors, save figures and close pdf is needed '''  Amir MOHAMMADI committed May 08, 2018 747  # only for plots  Theophile GENTILHOMME committed Apr 24, 2018 748  title = self._title if self._title is not None else "FMR vs IAPMR"  Theophile GENTILHOMME committed May 04, 2018 749 750  if title.replace(' ', ''): mpl.title(title)  Theophile GENTILHOMME committed Apr 24, 2018 751 752 753  mpl.xlabel(self._x_label or "False Match Rate (%)") mpl.ylabel(self._y_label or "IAPMR (%)") mpl.grid(True, color=self._grid_color)  Theophile GENTILHOMME committed May 04, 2018 754 755  if self._disp_legend: mpl.legend(loc=self._legend_loc)  Theophile GENTILHOMME committed Apr 24, 2018 756 757 758 759 760 761 762  self._set_axis() fig = mpl.gcf() mpl.xticks(rotation=self._x_rotation) mpl.tick_params(axis='both', which='major', labelsize=4) self._pdf_page.savefig(fig)  Amir MOHAMMADI committed May 08, 2018 763  # do not want to close PDF when running evaluate  Theophile GENTILHOMME committed Apr 24, 2018 764  if 'PdfPages' in self._ctx.meta and \  Amir MOHAMMADI committed May 08, 2018 765  ('closef' not in self._ctx.meta or self._ctx.meta['closef']):  Theophile GENTILHOMME committed Apr 24, 2018 766  self._pdf_page.close()