figure.py 28.7 KB
 1 2 3 4 '''Runs error analysis on score sets, outputs metrics and plots''' import click import numpy as np  5 import matplotlib.pyplot as mpl  Amir MOHAMMADI committed May 08, 2018 6 import bob.measure.script.figure as measure_figure  Theophile GENTILHOMME committed May 25, 2018 7 import bob.bio.base.script.figure as bio_figure  8 from tabulate import tabulate  Amir MOHAMMADI committed May 08, 2018 9 from bob.measure.utils import get_fta_list  10 from bob.measure import (  11  far_threshold, eer_threshold, min_hter_threshold, farfrr, epc, ppndf  12 )  Theophile GENTILHOMME committed May 25, 2018 13 from bob.measure.plot import (det, det_axis, roc_for_far, log_values)  14 15 16 17 from . import error_utils ALL_CRITERIA = ('bpcer20', 'eer', 'min-hter')  Theophile GENTILHOMME committed May 04, 2018 18   19 20 21 22 23 24 25 def calc_threshold(method, neg, pos): """Calculates the threshold based on the given method. The scores should be sorted! Parameters ---------- method : str  Theophile GENTILHOMME committed May 25, 2018 26  One of bpcer20, eer, min-hter.  27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53  neg : array_like The negative scores. They should be sorted! pos : array_like The positive scores. They should be sorted! Returns ------- float The calculated threshold. Raises ------ ValueError If method is unknown. """ method = method.lower() if method == 'bpcer20': threshold = far_threshold(neg, pos, 0.05, True) elif method == 'eer': threshold = eer_threshold(neg, pos, True) elif method == 'min-hter': threshold = min_hter_threshold(neg, pos, True) else: raise ValueError("Unknown threshold criteria: {}".format(method)) return threshold  Theophile GENTILHOMME committed May 04, 2018 54   55 56 57 58 59 class Metrics(measure_figure.Metrics): def __init__(self, ctx, scores, evaluation, func_load): super(Metrics, self).__init__(ctx, scores, evaluation, func_load) ''' Compute metrics from score files'''  Amir MOHAMMADI committed May 08, 2018 60   Theophile GENTILHOMME committed Apr 23, 2018 61  def compute(self, idx, input_scores, input_names):  62  ''' Compute metrics for the given criteria'''  Theophile GENTILHOMME committed Apr 23, 2018 63 64 65 66 67 68  neg_list, pos_list, _ = get_fta_list(input_scores) dev_neg, dev_pos = neg_list[0], pos_list[0] dev_file = input_names[0] if self._eval: eval_neg, eval_pos = neg_list[1], pos_list[1] eval_file = input_names[1]  69   Theophile GENTILHOMME committed Apr 30, 2018 70  title = self._legends[idx] if self._legends is not None else None  71  headers = ['' or title, 'Development %s' % dev_file]  Theophile GENTILHOMME committed Apr 23, 2018 72  if self._eval:  73 74 75 76 77 78 79  headers.append('Eval. % s' % eval_file) for m in ALL_CRITERIA: raws = [] threshold = calc_threshold(m, dev_neg, dev_pos) click.echo("\nThreshold of %f selected with the %s criteria" % ( threshold, m)) apcer, bpcer = farfrr(dev_neg, dev_pos, threshold)  Theophile GENTILHOMME committed May 25, 2018 80  raws.append(['APCER', '{:>5.1f}%'.format(apcer * 100)])  Theophile GENTILHOMME committed May 29, 2018 81  raws.append(['BPCER', '{:>5.1f}%'.format(bpcer * 100)])  Theophile GENTILHOMME committed May 25, 2018 82  raws.append(['ACER', '{:>5.1f}%'.format((apcer + bpcer) * 50)])  83 84 85 86 87 88 89 90 91 92 93  if self._eval and eval_neg is not None: apcer, bpcer = farfrr(eval_neg, eval_pos, threshold) raws[0].append('{:>5.1f}%'.format(apcer * 100)) raws[1].append('{:>5.1f}%'.format(bpcer * 100)) raws[2].append('{:>5.1f}%'.format((apcer + bpcer) * 50)) click.echo( tabulate(raws, headers, self._tablefmt), file=self.log_file )  Theophile GENTILHOMME committed May 04, 2018 94   Theophile GENTILHOMME committed May 04, 2018 95 96 97 98 99 class MetricsVuln(measure_figure.Metrics): def __init__(self, ctx, scores, evaluation, func_load): super(MetricsVuln, self).__init__(ctx, scores, evaluation, func_load) ''' Compute metrics from score files'''  Amir MOHAMMADI committed May 08, 2018 100   Theophile GENTILHOMME committed May 04, 2018 101 102 103 104 105 106  def compute(self, idx, input_scores, input_names): ''' Compute metrics for the given criteria''' neg_list, pos_list, _ = get_fta_list(input_scores) dev_neg, dev_pos = neg_list[0], pos_list[0] criter = self._criterion or 'eer' threshold = calc_threshold(criter, dev_neg, dev_pos) \  Amir MOHAMMADI committed May 08, 2018 107  if self._thres is None else self._thres[idx]  Theophile GENTILHOMME committed May 04, 2018 108 109 110 111 112 113  far, frr = farfrr(neg_list[1], pos_list[1], threshold) iapmr, _ = farfrr(neg_list[3], pos_list[1], threshold) title = self._legends[idx] if self._legends is not None else None headers = ['' or title, '%s (threshold=%.2g)' % (criter.upper(), threshold)] rows = []  Theophile GENTILHOMME committed May 29, 2018 114  rows.append(['APCER (%)', '{:>5.1f}%'.format(100 * far)])  Theophile GENTILHOMME committed May 25, 2018 115  rows.append(['BPCER (%)', '{:>5.1f}%'.format(frr * 100)])  Theophile GENTILHOMME committed May 29, 2018 116  rows.append(['ACER (%)', '{:>5.1f}%'.format(50 * (far + frr))])  Amir MOHAMMADI committed May 08, 2018 117  rows.append(['IAPMR (%)', '{:>5.1f}%'.format(100 * iapmr)])  Theophile GENTILHOMME committed May 04, 2018 118 119 120 121 122 123  click.echo( tabulate(rows, headers, self._tablefmt), file=self.log_file )  124 125 126 127 128 129 class HistPad(measure_figure.Hist): ''' Histograms for PAD ''' def _setup_hist(self, neg, pos): self._title_base = 'PAD' self._density_hist(  Theophile GENTILHOMME committed Apr 26, 2018 130  pos[0], n=0, label='Bona Fide', color='C1'  131 132  ) self._density_hist(  Theophile GENTILHOMME committed Apr 26, 2018 133  neg[0], n=1, label='Presentation attack', alpha=0.4, color='C7',  Theophile GENTILHOMME committed Apr 23, 2018 134  hatch='\\\\'  135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158  ) def _iapmr_dot(threshold, iapmr, real_data, **kwargs): # plot a dot on threshold versus IAPMR line and show IAPMR as a number axlim = mpl.axis() mpl.plot(threshold, 100. * iapmr, 'o', color='C3', **kwargs) if not real_data: mpl.annotate( 'IAPMR at\noperating point', xy=(threshold, 100. * iapmr), xycoords='data', xytext=(0.85, 0.6), textcoords='axes fraction', color='black', size='large', arrowprops=dict(facecolor='black', shrink=0.05, width=2), horizontalalignment='center', verticalalignment='top', ) else: mpl.text(threshold + (threshold - axlim[0]) / 12, 100. * iapmr, '%.1f%%' % (100. * iapmr,), color='C3')  Theophile GENTILHOMME committed May 04, 2018 159   160 161 162 163 164 165 def _iapmr_line_plot(scores, n_points=100, **kwargs): axlim = mpl.axis() step = (axlim[1] - axlim[0]) / float(n_points) thres = [(k * step) + axlim[0] for k in range(2, n_points - 1)] mix_prob_y = [] for k in thres:  Theophile GENTILHOMME committed May 04, 2018 166  mix_prob_y.append(100. * error_utils.calc_pass_rate(k, scores))  167 168 169  mpl.plot(thres, mix_prob_y, label='IAPMR', color='C3', **kwargs)  Theophile GENTILHOMME committed May 04, 2018 170   171 172 173 174 def _iapmr_plot(scores, threshold, iapmr, real_data, **kwargs): _iapmr_dot(threshold, iapmr, real_data, **kwargs) _iapmr_line_plot(scores, n_points=100, **kwargs)  Theophile GENTILHOMME committed May 04, 2018 175   176 177 178 179 180 181 class HistVuln(measure_figure.Hist): ''' Histograms for vulnerability ''' def _setup_hist(self, neg, pos): self._title_base = 'Vulnerability' self._density_hist(  Amir MOHAMMADI committed May 03, 2018 182  pos[0], n=0, label='Genuine', color='C2'  183 184  ) self._density_hist(  Theophile GENTILHOMME committed Apr 26, 2018 185  neg[0], n=1, label='Zero-effort impostors', alpha=0.8, color='C0'  186 187  ) self._density_hist(  Theophile GENTILHOMME committed Apr 26, 2018 188  neg[1], n=2, label='Presentation attack', alpha=0.4, color='C7',  Theophile GENTILHOMME committed Apr 23, 2018 189  hatch='\\\\'  190 191  )  Theophile GENTILHOMME committed May 03, 2018 192  def _lines(self, threshold, label, neg, pos, idx, **kwargs):  193  if 'iapmr_line' not in self._ctx.meta or self._ctx.meta['iapmr_line']:  Amir MOHAMMADI committed May 08, 2018 194  # plot vertical line  Theophile GENTILHOMME committed May 03, 2018 195  super(HistVuln, self)._lines(threshold, label, neg, pos, idx)  196   Amir MOHAMMADI committed May 08, 2018 197  # plot iapmr_line  198 199 200 201 202  iapmr, _ = farfrr(neg[1], pos[0], threshold) ax2 = mpl.twinx() # we never want grid lines on axis 2 ax2.grid(False) real_data = True if 'real_data' not in self._ctx.meta else \  Amir MOHAMMADI committed May 08, 2018 203  self._ctx.meta['real_data']  204  _iapmr_plot(neg[1], threshold, iapmr, real_data=real_data)  Theophile GENTILHOMME committed May 03, 2018 205 206  n = idx % self._step_print col = n % self._ncols  Amir MOHAMMADI committed May 08, 2018 207 208  rest_print = self.n_systems - \ int(idx / self._step_print) * self._step_print  Theophile GENTILHOMME committed May 03, 2018 209 210  if col == self._ncols - 1 or n == rest_print - 1: ax2.set_ylabel("IAPMR (%)", color='C3')  211 212 213 214  ax2.tick_params(axis='y', colors='red') ax2.yaxis.label.set_color('red') ax2.spines['right'].set_color('red')  Theophile GENTILHOMME committed May 04, 2018 215   216 217 class PadPlot(measure_figure.PlotBase): '''Base class for PAD plots'''  Amir MOHAMMADI committed May 08, 2018 218   219 220  def __init__(self, ctx, scores, evaluation, func_load): super(PadPlot, self).__init__(ctx, scores, evaluation, func_load)  221  mpl.rcParams['figure.constrained_layout.use'] = self._clayout  222 223 224  def end_process(self): '''Close pdf '''  Amir MOHAMMADI committed May 08, 2018 225  # do not want to close PDF when running evaluate  226 227 228 229 230  if 'PdfPages' in self._ctx.meta and \ ('closef' not in self._ctx.meta or self._ctx.meta['closef']): self._pdf_page.close() def _plot_legends(self):  Amir MOHAMMADI committed May 08, 2018 231  # legends for all axes  232 233 234 235 236 237  lines = [] labels = [] for ax in mpl.gcf().get_axes(): li, la = ax.get_legend_handles_labels() lines += li labels += la  Theophile GENTILHOMME committed May 04, 2018 238 239 240 241  if self._disp_legend: mpl.gca().legend(lines, labels, loc=self._legend_loc, fancybox=True, framealpha=0.5)  242 243 244  class Epc(PadPlot): ''' Handles the plotting of EPC '''  Amir MOHAMMADI committed May 08, 2018 245   246 247 248  def __init__(self, ctx, scores, evaluation, func_load): super(Epc, self).__init__(ctx, scores, evaluation, func_load) self._iapmr = True if 'iapmr' not in self._ctx.meta else \  Amir MOHAMMADI committed May 08, 2018 249  self._ctx.meta['iapmr']  Theophile GENTILHOMME committed Apr 24, 2018 250 251 252 253  self._title = self._title or ('EPC and IAPMR' if self._iapmr else 'EPC') self._x_label = self._x_label or r"Weight $\beta$" self._y_label = self._y_label or "WER (%)"  Amir MOHAMMADI committed May 08, 2018 254  self._eval = True # always eval data with EPC  255 256 257  self._split = False self._nb_figs = 1  Theophile GENTILHOMME committed Apr 23, 2018 258 259 260 261 262  if self._min_arg != 4: raise click.BadParameter("You must provide 4 scores files:{licit," "spoof}/{dev,eval}") def compute(self, idx, input_scores, input_names):  263  ''' Plot EPC for PAD'''  Theophile GENTILHOMME committed Apr 23, 2018 264 265 266 267 268  licit_dev_neg = input_scores[0][0] licit_dev_pos = input_scores[0][1] licit_eval_neg = input_scores[1][0] licit_eval_pos = input_scores[1][1] spoof_eval_neg = input_scores[3][0]  269  mpl.gcf().clear()  Theophile GENTILHOMME committed Apr 13, 2018 270  epc_baseline = epc(  271 272 273 274 275 276 277  licit_dev_neg, licit_dev_pos, licit_eval_neg, licit_eval_pos, 100 ) mpl.plot( epc_baseline[:, 0], [100. * k for k in epc_baseline[:, 1]], color='C0', label=self._label(  Theophile GENTILHOMME committed Apr 23, 2018 278  'WER', '%s-%s' % (input_names[0], input_names[1]), idx  279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299  ), linestyle='-' ) mpl.xlabel(self._x_label) mpl.ylabel(self._y_label) if self._iapmr: mix_prob_y = [] for k in epc_baseline[:, 2]: prob_attack = sum( 1 for i in spoof_eval_neg if i >= k ) / float(spoof_eval_neg.size) mix_prob_y.append(100. * prob_attack) mpl.gca().set_axisbelow(True) prob_ax = mpl.gca().twinx() mpl.plot( epc_baseline[:, 0], mix_prob_y, color='C3', linestyle='-', label=self._label(  Theophile GENTILHOMME committed Apr 23, 2018 300  'IAPMR', '%s-%s' % (input_names[0], input_names[1]), idx  301 302 303  ) ) prob_ax.set_yticklabels(prob_ax.get_yticks())  Theophile GENTILHOMME committed May 25, 2018 304 305 306  prob_ax.tick_params(axis='y', colors='C3') prob_ax.yaxis.label.set_color('C3') prob_ax.spines['right'].set_color('C3')  307 308  ylabels = prob_ax.get_yticks() prob_ax.yaxis.set_ticklabels(["%.0f" % val for val in ylabels])  Theophile GENTILHOMME committed May 25, 2018 309  prob_ax.set_ylabel('IAPMR', color='C3')  310  prob_ax.set_axisbelow(True)  Theophile GENTILHOMME committed Apr 30, 2018 311  title = self._legends[idx] if self._legends is not None else self._title  Theophile GENTILHOMME committed May 04, 2018 312 313  if title.replace(' ', ''): mpl.title(title)  Amir MOHAMMADI committed May 08, 2018 314  # legends for all axes  315 316 317 318  self._plot_legends() mpl.xticks(rotation=self._x_rotation) self._pdf_page.savefig(mpl.gcf())  Theophile GENTILHOMME committed May 04, 2018 319   320 321 class Epsc(PadPlot): ''' Handles the plotting of EPSC '''  Amir MOHAMMADI committed May 08, 2018 322   323 324 325 326  def __init__(self, ctx, scores, evaluation, func_load, criteria, var_param, fixed_param): super(Epsc, self).__init__(ctx, scores, evaluation, func_load) self._iapmr = False if 'iapmr' not in self._ctx.meta else \  Amir MOHAMMADI committed May 08, 2018 327  self._ctx.meta['iapmr']  328  self._wer = True if 'wer' not in self._ctx.meta else \  Amir MOHAMMADI committed May 08, 2018 329  self._ctx.meta['wer']  330 331 332  self._criteria = 'eer' if criteria is None else criteria self._var_param = "omega" if var_param is None else var_param self._fixed_param = 0.5 if fixed_param is None else fixed_param  Amir MOHAMMADI committed May 08, 2018 333  self._eval = True # always eval data with EPC  334 335  self._split = False self._nb_figs = 1  Theophile GENTILHOMME committed Apr 23, 2018 336  self._title = ''  Theophile GENTILHOMME committed May 29, 2018 337  self._sampling = ctx.meta.get('sampling', 5)  Theophile GENTILHOMME committed Apr 23, 2018 338 339 340 341  if self._min_arg != 4: raise click.BadParameter("You must provide 4 scores files:{licit," "spoof}/{dev,eval}")  342   Theophile GENTILHOMME committed Apr 23, 2018 343  def compute(self, idx, input_scores, input_names):  344  ''' Plot EPSC for PAD'''  Theophile GENTILHOMME committed Apr 23, 2018 345 346 347 348 349 350 351 352  licit_dev_neg = input_scores[0][0] licit_dev_pos = input_scores[0][1] licit_eval_neg = input_scores[1][0] licit_eval_pos = input_scores[1][1] spoof_dev_neg = input_scores[2][0] spoof_dev_pos = input_scores[2][1] spoof_eval_neg = input_scores[3][0] spoof_eval_pos = input_scores[3][1]  Theophile GENTILHOMME committed Apr 30, 2018 353  title = self._legends[idx] if self._legends is not None else None  354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373  mpl.gcf().clear() points = 10 if self._var_param == 'omega': omega, beta, thrs = error_utils.epsc_thresholds( licit_dev_neg, licit_dev_pos, spoof_dev_neg, spoof_dev_pos, points=points, criteria=self._criteria, beta=self._fixed_param) else: omega, beta, thrs = error_utils.epsc_thresholds( licit_dev_neg, licit_dev_pos, spoof_dev_neg, spoof_dev_pos, points=points,  Amir MOHAMMADI committed May 08, 2018 374  criteria=self._criteria,  375 376 377 378 379 380 381  omega=self._fixed_param ) errors = error_utils.all_error_rates( licit_eval_neg, licit_eval_pos, spoof_eval_neg, spoof_eval_pos, thrs, omega, beta ) # error rates are returned in a list in the  Amir MOHAMMADI committed May 08, 2018 382  # following order: frr, far, IAPMR, far_w, wer_w  383 384 385 386  ax1 = mpl.subplot( 111 ) # EPC like curves for FVAS fused scores for weighted error rates  Amir MOHAMMADI committed May 08, 2018 387  # between the negatives (impostors and Presentation attacks)  388 389 390 391 392 393 394 395  if self._wer: if self._var_param == 'omega': mpl.plot( omega, 100. * errors[4].flatten(), color='C0', linestyle='-', label=r"WER$_{\omega,\beta}$")  Theophile GENTILHOMME committed Apr 30, 2018 396  mpl.xlabel(self._x_label or r"Weight $\omega$")  397 398 399 400 401 402 403  else: mpl.plot( beta, 100. * errors[4].flatten(), color='C0', linestyle='-', label=r"WER$_{\omega,\beta}$")  Theophile GENTILHOMME committed Apr 30, 2018 404 405  mpl.xlabel(self._x_label or r"Weight $\beta$") mpl.ylabel(self._y_label or r"WER$_{\omega,\beta}$ (%)")  406 407 408 409 410 411 412 413 414 415 416 417 418  if self._iapmr: axis = mpl.gca() if self._wer: axis = mpl.twinx() axis.grid(False) if self._var_param == 'omega': mpl.plot( omega, 100. * errors[2].flatten(), color='C3', linestyle='-', label='IAPMR')  Theophile GENTILHOMME committed Apr 30, 2018 419  mpl.xlabel(self._x_label or r"Weight $\omega$")  420 421 422 423 424 425 426  else: mpl.plot( beta, 100. * errors[2].flatten(), color='C3', linestyle='-', label='IAPMR')  Theophile GENTILHOMME committed Apr 30, 2018 427 428  mpl.xlabel(self._x_label or r"Weight $\beta$") mpl.ylabel(self._y_label or r"IAPMR (%)")  429 430 431 432 433 434 435  if self._wer: axis.set_yticklabels(axis.get_yticks()) axis.tick_params(axis='y', colors='red') axis.yaxis.label.set_color('red') axis.spines['right'].set_color('red') if self._var_param == 'omega':  Theophile GENTILHOMME committed May 04, 2018 436  if title is not None and title.replace(' ', ''):  Amir MOHAMMADI committed May 08, 2018 437  mpl.title(title or (r"EPSC with $\beta$ = %.2f" %  Theophile GENTILHOMME committed May 04, 2018 438  self._fixed_param))  439  else:  Theophile GENTILHOMME committed May 04, 2018 440  if title is not None and title.replace(' ', ''):  Amir MOHAMMADI committed May 08, 2018 441  mpl.title(title or (r"EPSC with $\omega$ = %.2f" %  Theophile GENTILHOMME committed May 04, 2018 442  self._fixed_param))  443 444 445 446 447 448  mpl.grid() self._plot_legends() ax1.set_xticklabels(ax1.get_xticks()) ax1.set_yticklabels(ax1.get_yticks()) mpl.xticks(rotation=self._x_rotation)  449  self._pdf_page.savefig()  450   Theophile GENTILHOMME committed May 04, 2018 451   452 453 class Epsc3D(Epsc): ''' 3D EPSC plots for PAD'''  Amir MOHAMMADI committed May 08, 2018 454   Theophile GENTILHOMME committed Apr 23, 2018 455  def compute(self, idx, input_scores, input_names):  456  ''' Implements plots'''  Theophile GENTILHOMME committed Apr 23, 2018 457 458 459 460 461 462 463 464  licit_dev_neg = input_scores[0][0] licit_dev_pos = input_scores[0][1] licit_eval_neg = input_scores[1][0] licit_eval_pos = input_scores[1][1] spoof_dev_neg = input_scores[2][0] spoof_dev_pos = input_scores[2][1] spoof_eval_neg = input_scores[3][0] spoof_eval_pos = input_scores[3][1]  465   Theophile GENTILHOMME committed May 04, 2018 466  title = self._legends[idx] if self._legends is not None else "3D EPSC"  467   468 469  mpl.rcParams.pop('key', None)  470  mpl.gcf().clear()  471  mpl.gcf().set_constrained_layout(self._clayout)  472 473 474 475  from mpl_toolkits.mplot3d import Axes3D from matplotlib import cm  Theophile GENTILHOMME committed May 29, 2018 476  points = self._sampling or 5  477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493  omega, beta, thrs = error_utils.epsc_thresholds( licit_dev_neg, licit_dev_pos, spoof_dev_neg, spoof_dev_pos, points=points, criteria=self._criteria) errors = error_utils.all_error_rates( licit_eval_neg, licit_eval_pos, spoof_eval_neg, spoof_eval_pos, thrs, omega, beta ) # error rates are returned in a list as 2D numpy.ndarrays in # the following order: frr, far, IAPMR, far_w, wer_wb, hter_wb wer_errors = 100 * errors[2 if self._iapmr else 4]  494  ax1 = mpl.gcf().add_subplot(111, projection='3d')  495 496 497 498 499 500 501 502 503 504 505  W, B = np.meshgrid(omega, beta) ax1.plot_wireframe( W, B, wer_errors, cmap=cm.coolwarm, antialiased=False ) # surface if self._iapmr: ax1.azim = -30 ax1.elev = 50  Theophile GENTILHOMME committed Apr 30, 2018 506 507  ax1.set_xlabel(self._x_label or r"Weight $\omega$") ax1.set_ylabel(self._y_label or r"Weight $\beta$")  508 509 510 511  ax1.set_zlabel( r"WER$_{\omega,\beta}$ (%)" if self._wer else "IAPMR (%)" )  Theophile GENTILHOMME committed May 04, 2018 512  if title.replace(' ', ''):  Theophile GENTILHOMME committed May 04, 2018 513  mpl.title(title)  514 515 516 517 518  ax1.set_xticklabels(ax1.get_xticks()) ax1.set_yticklabels(ax1.get_yticks()) ax1.set_zticklabels(ax1.get_zticks())  519 520  self._pdf_page.savefig()  Theophile GENTILHOMME committed May 04, 2018 521   Theophile GENTILHOMME committed May 25, 2018 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 class Roc(bio_figure.Roc): '''ROC for PARD''' def __init__(self, ctx, scores, evaluation, func_load): super(Roc, self).__init__(ctx, scores, evaluation, func_load) self._x_label = ctx.meta.get('x_label') or 'APCER' self._y_label = ctx.meta.get('y_label') or '1 - BPCER' class DetPad(bio_figure.Det): def __init__(self, ctx, scores, evaluation, func_load): super(DetPad, self).__init__(ctx, scores, evaluation, func_load) self._x_label = ctx.meta.get('x_label') or 'APCER' self._y_label = ctx.meta.get('y_label') or 'BPCER'  Theophile GENTILHOMME committed May 25, 2018 537 538 class BaseDetRoc(PadPlot): '''Base for DET and ROC'''  Amir MOHAMMADI committed May 08, 2018 539   Theophile GENTILHOMME committed May 14, 2018 540 541  def __init__(self, ctx, scores, evaluation, func_load, criteria, real_data, no_spoof):  Theophile GENTILHOMME committed May 25, 2018 542  super(BaseDetRoc, self).__init__(ctx, scores, evaluation, func_load)  Theophile GENTILHOMME committed May 14, 2018 543 544  self._no_spoof = no_spoof self._criteria = criteria or 'eer'  545  self._real_data = True if real_data is None else real_data  Theophile GENTILHOMME committed May 25, 2018 546   547   Theophile GENTILHOMME committed Apr 23, 2018 548  def compute(self, idx, input_scores, input_names):  549  ''' Implements plots'''  Theophile GENTILHOMME committed Apr 23, 2018 550 551 552 553 554 555  licit_dev_neg = input_scores[0][0] licit_dev_pos = input_scores[0][1] licit_eval_neg = input_scores[1][0] licit_eval_pos = input_scores[1][1] spoof_eval_neg = input_scores[3][0] if len(input_scores) > 2 else None spoof_eval_pos = input_scores[3][1] if len(input_scores) > 2 else None  Theophile GENTILHOMME committed May 25, 2018 556  self._plot(  557 558 559  licit_eval_neg, licit_eval_pos, self._points,  Theophile GENTILHOMME committed May 14, 2018 560  color='C0',  561  linestyle='-',  Theophile GENTILHOMME committed Apr 23, 2018 562  label=self._label("licit", input_names[0], idx)  563 564  ) if not self._no_spoof and spoof_eval_neg is not None:  Theophile GENTILHOMME committed May 25, 2018 565  self._plot(  566 567 568  spoof_eval_neg, spoof_eval_pos, self._points,  Theophile GENTILHOMME committed May 14, 2018 569 570  color='C3', linestyle=':',  Theophile GENTILHOMME committed Apr 23, 2018 571  label=self._label("spoof", input_names[3], idx)  572 573  )  Theophile GENTILHOMME committed May 14, 2018 574  if self._criteria is None or self._no_spoof:  575 576 577 578 579 580 581 582  return thres_baseline = calc_threshold( self._criteria, licit_dev_neg, licit_dev_pos ) axlim = mpl.axis()  Theophile GENTILHOMME committed May 25, 2018 583  farfrr_licit, farfrr_licit_det = self._get_farfrr(  584  licit_eval_neg, licit_eval_pos,  Theophile GENTILHOMME committed May 14, 2018 585  thres_baseline  Theophile GENTILHOMME committed May 25, 2018 586 587 588 589 590 591  ) if farfrr_licit is None: return farfrr_spoof, farfrr_spoof_det = self._get_farfrr( spoof_eval_neg, spoof_eval_pos, thres_baseline )  592 593 594 595 596 597 598  if not self._real_data: mpl.axhline( y=farfrr_licit_det[1], xmin=axlim[2], xmax=axlim[3], color='k', linestyle='--',  Theophile GENTILHOMME committed May 25, 2018 599  label="%s @ EER" % self._y_label)  600 601 602 603 604 605 606  else: mpl.axhline( y=farfrr_licit_det[1], xmin=axlim[0], xmax=axlim[1], color='k', linestyle='--',  Theophile GENTILHOMME committed May 25, 2018 607 608  label="%s = %.2f%%" % (self._y_label, farfrr_licit[1] * 100))  609 610 611 612 613  mpl.plot( farfrr_licit_det[0], farfrr_licit_det[1], 'o',  Theophile GENTILHOMME committed May 14, 2018 614 615  color='C0', ) # FAR point, licit scenario  616 617 618 619  mpl.plot( farfrr_spoof_det[0], farfrr_spoof_det[1], 'o',  Theophile GENTILHOMME committed May 14, 2018 620 621  color='C3', ) # FAR point, spoof scenario  622 623 624  # annotate the FAR points xyannotate_licit = [  Theophile GENTILHOMME committed May 25, 2018 625 626  0.15 + farfrr_licit_det[0], farfrr_licit_det[1] - 0.15,  627 628  ] xyannotate_spoof = [  Theophile GENTILHOMME committed May 25, 2018 629 630  0.15 + farfrr_spoof_det[0], farfrr_spoof_det[1] - 0.15,  631 632 633 634  ] if not self._real_data: mpl.annotate(  Theophile GENTILHOMME committed May 25, 2018 635  '%s @ operating point' % self._y_label,  636 637  xy=(farfrr_licit_det[0], farfrr_licit_det[1]), xycoords='data',  Theophile GENTILHOMME committed May 25, 2018 638  xytext=(xyannotate_licit[0], xyannotate_licit[1]))  639  mpl.annotate(  Theophile GENTILHOMME committed May 14, 2018 640  'IAPMR @ operating point',  641 642  xy=(farfrr_spoof_det[0], farfrr_spoof_det[1]), xycoords='data',  Theophile GENTILHOMME committed May 25, 2018 643  xytext=(xyannotate_spoof[0], xyannotate_spoof[1]))  644 645  else: mpl.annotate(  Theophile GENTILHOMME committed May 25, 2018 646  'APCER=%.2f%%' % (farfrr_licit[0] * 100),  647 648 649  xy=(farfrr_licit_det[0], farfrr_licit_det[1]), xycoords='data', xytext=(xyannotate_licit[0], xyannotate_licit[1]),  Theophile GENTILHOMME committed May 14, 2018 650  color='C0',  Theophile GENTILHOMME committed May 25, 2018 651  size='small')  652  mpl.annotate(  Theophile GENTILHOMME committed May 14, 2018 653  'IAPMR=%.2f%%' % (farfrr_spoof[0] * 100),  654 655 656  xy=(farfrr_spoof_det[0], farfrr_spoof_det[1]), xycoords='data', xytext=(xyannotate_spoof[0], xyannotate_spoof[1]),  Theophile GENTILHOMME committed May 14, 2018 657  color='C3',  Theophile GENTILHOMME committed May 25, 2018 658  size='small')  659 660 661 662  def end_process(self): ''' Set title, legend, axis labels, grid colors, save figures and close pdf is needed '''  Amir MOHAMMADI committed May 08, 2018 663  # only for plots  Theophile GENTILHOMME committed May 25, 2018 664 665 666 667 668  if self._title.replace(' ', ''): mpl.title(self._title) mpl.xlabel(self._x_label) mpl.ylabel(self._y_label)  669  mpl.grid(True, color=self._grid_color)  Theophile GENTILHOMME committed May 04, 2018 670 671  if self._disp_legend: mpl.legend(loc=self._legend_loc)  672 673 674 675 676 677 678 679 680 681 682  self._set_axis() fig = mpl.gcf() mpl.xticks(rotation=self._x_rotation) mpl.tick_params(axis='both', which='major', labelsize=4) for tick in mpl.gca().xaxis.get_major_ticks(): tick.label.set_fontsize(6) for tick in mpl.gca().yaxis.get_major_ticks(): tick.label.set_fontsize(6) self._pdf_page.savefig(fig)  Amir MOHAMMADI committed May 08, 2018 683  # do not want to close PDF when running evaluate  684  if 'PdfPages' in self._ctx.meta and \  Amir MOHAMMADI committed May 08, 2018 685  ('closef' not in self._ctx.meta or self._ctx.meta['closef']):  686 687  self._pdf_page.close()  Theophile GENTILHOMME committed May 25, 2018 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708  def _get_farfrr(self, x, y, thres): return None, None def _plot(self, x, y, points, **kwargs): pass class Det(BaseDetRoc): '''Base for DET and ROC''' def __init__(self, ctx, scores, evaluation, func_load, criteria, real_data, no_spoof): super(Det, self).__init__(ctx, scores, evaluation, func_load, criteria, real_data, no_spoof) self._x_label = self._x_label or "APCER" self._y_label = self._y_label or "BPCER" add = '' if not self._no_spoof: add = " and overlaid SPOOF scenario" self._title = self._title or ('DET: LICIT' + add)  709 710 711 712 713  def _set_axis(self): if self._axlim is not None and None not in self._axlim: det_axis(self._axlim) else: det_axis([0.01, 99, 0.01, 99])  Theophile GENTILHOMME committed Apr 24, 2018 714   Theophile GENTILHOMME committed May 25, 2018 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753  def _get_farfrr(self, x, y, thres): # calculate test frr @ EER (licit scenario) points = farfrr(x, y, thres) return points, [ppndf(i) for i in points] def _plot(self, x, y, points, **kwargs): det( x, y, points, color=kwargs.get('color'), linestyle=kwargs.get('linestyle'), label=kwargs.get('label') ) class RocVuln(BaseDetRoc): '''ROC for vuln''' def __init__(self, ctx, scores, evaluation, func_load, criteria, real_data, no_spoof): super(RocVuln, self).__init__(ctx, scores, evaluation, func_load, criteria, real_data, no_spoof) self._x_label = self._x_label or "APCER" self._y_label = self._y_label or "1 - BPCER" self._semilogx = ctx.meta.get('semilogx', True) add = '' if not self._no_spoof: add = " and overlaid SPOOF scenario" self._title = self._title or ('ROC: LICIT' + add) def _plot(self, x, y, points, **kwargs): roc_for_far( x, y, far_values=log_values(self._min_dig or -4), CAR=self._semilogx, color=kwargs.get('color'), linestyle=kwargs.get('linestyle'), label=kwargs.get('label') )  Theophile GENTILHOMME committed May 04, 2018 754   Theophile GENTILHOMME committed Apr 24, 2018 755 756 class FmrIapmr(PadPlot): '''FMR vs IAPMR'''  Amir MOHAMMADI committed May 08, 2018 757   Theophile GENTILHOMME committed Apr 24, 2018 758 759  def __init__(self, ctx, scores, evaluation, func_load): super(FmrIapmr, self).__init__(ctx, scores, evaluation, func_load)  Amir MOHAMMADI committed May 08, 2018 760  self._eval = True # always eval data with EPC  Theophile GENTILHOMME committed Apr 24, 2018 761 762 763  self._split = False self._nb_figs = 1 self._semilogx = False if 'semilogx' not in ctx.meta else\  Amir MOHAMMADI committed May 08, 2018 764  ctx.meta['semilogx']  Theophile GENTILHOMME committed Apr 24, 2018 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781  if self._min_arg != 4: raise click.BadParameter("You must provide 4 scores files:{licit," "spoof}/{dev,eval}") def compute(self, idx, input_scores, input_names): ''' Implements plots''' licit_eval_neg = input_scores[1][0] licit_eval_pos = input_scores[1][1] spoof_eval_neg = input_scores[3][0] fmr_list = np.linspace(0, 1, 100) iapmr_list = [] for i, fmr in enumerate(fmr_list): thr = far_threshold(licit_eval_neg, licit_eval_pos, fmr, True) iapmr_list.append(farfrr(spoof_eval_neg, licit_eval_pos, thr)[0]) # re-calculate fmr since threshold might give a different result # for fmr. fmr_list[i] = farfrr(licit_eval_neg, licit_eval_pos, thr)[0]  Theophile GENTILHOMME committed Apr 30, 2018 782  label = self._legends[idx] if self._legends is not None else \  Amir MOHAMMADI committed May 08, 2018 783  '(%s/%s)' % (input_names[1], input_names[3])  Theophile GENTILHOMME committed Apr 24, 2018 784 785 786 787 788 789 790 791  if self._semilogx: mpl.semilogx(fmr_list, iapmr_list, label=label) else: mpl.plot(fmr_list, iapmr_list, label=label) def end_process(self): ''' Set title, legend, axis labels, grid colors, save figures and close pdf is needed '''  Amir MOHAMMADI committed May 08, 2018 792  # only for plots  Theophile GENTILHOMME committed Apr 24, 2018 793  title = self._title if self._title is not None else "FMR vs IAPMR"  Theophile GENTILHOMME committed May 04, 2018 794 795  if title.replace(' ', ''): mpl.title(title)  Theophile GENTILHOMME committed Apr 24, 2018 796 797 798  mpl.xlabel(self._x_label or "False Match Rate (%)") mpl.ylabel(self._y_label or "IAPMR (%)") mpl.grid(True, color=self._grid_color)  Theophile GENTILHOMME committed May 04, 2018 799 800  if self._disp_legend: mpl.legend(loc=self._legend_loc)  Theophile GENTILHOMME committed Apr 24, 2018 801 802 803 804 805 806 807  self._set_axis() fig = mpl.gcf() mpl.xticks(rotation=self._x_rotation) mpl.tick_params(axis='both', which='major', labelsize=4) self._pdf_page.savefig(fig)  Amir MOHAMMADI committed May 08, 2018 808  # do not want to close PDF when running evaluate  Theophile GENTILHOMME committed Apr 24, 2018 809  if 'PdfPages' in self._ctx.meta and \  Amir MOHAMMADI committed May 08, 2018 810  ('closef' not in self._ctx.meta or self._ctx.meta['closef']):  Theophile GENTILHOMME committed Apr 24, 2018 811  self._pdf_page.close()