figure.py 26.6 KB
 1 2 3 4 '''Runs error analysis on score sets, outputs metrics and plots''' import click import numpy as np  5 import matplotlib.pyplot as mpl  Amir MOHAMMADI committed May 08, 2018 6 import bob.measure.script.figure as measure_figure  7 from tabulate import tabulate  Amir MOHAMMADI committed May 08, 2018 8 from bob.measure.utils import get_fta_list  9 from bob.measure import (  10  far_threshold, eer_threshold, min_hter_threshold, farfrr, epc, ppndf  11 )  12 from bob.measure.plot import (det, det_axis)  13 14 15 16 from . import error_utils ALL_CRITERIA = ('bpcer20', 'eer', 'min-hter')  Theophile GENTILHOMME committed May 04, 2018 17   18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 def calc_threshold(method, neg, pos): """Calculates the threshold based on the given method. The scores should be sorted! Parameters ---------- method : str One of bpcer201, eer, min-hter. neg : array_like The negative scores. They should be sorted! pos : array_like The positive scores. They should be sorted! Returns ------- float The calculated threshold. Raises ------ ValueError If method is unknown. """ method = method.lower() if method == 'bpcer20': threshold = far_threshold(neg, pos, 0.05, True) elif method == 'eer': threshold = eer_threshold(neg, pos, True) elif method == 'min-hter': threshold = min_hter_threshold(neg, pos, True) else: raise ValueError("Unknown threshold criteria: {}".format(method)) return threshold  Theophile GENTILHOMME committed May 04, 2018 53   54 55 56 57 58 class Metrics(measure_figure.Metrics): def __init__(self, ctx, scores, evaluation, func_load): super(Metrics, self).__init__(ctx, scores, evaluation, func_load) ''' Compute metrics from score files'''  Amir MOHAMMADI committed May 08, 2018 59   Theophile GENTILHOMME committed Apr 23, 2018 60  def compute(self, idx, input_scores, input_names):  61  ''' Compute metrics for the given criteria'''  Theophile GENTILHOMME committed Apr 23, 2018 62 63 64 65 66 67  neg_list, pos_list, _ = get_fta_list(input_scores) dev_neg, dev_pos = neg_list[0], pos_list[0] dev_file = input_names[0] if self._eval: eval_neg, eval_pos = neg_list[1], pos_list[1] eval_file = input_names[1]  68   Theophile GENTILHOMME committed Apr 30, 2018 69  title = self._legends[idx] if self._legends is not None else None  70  headers = ['' or title, 'Development %s' % dev_file]  Theophile GENTILHOMME committed Apr 23, 2018 71  if self._eval:  72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92  headers.append('Eval. % s' % eval_file) for m in ALL_CRITERIA: raws = [] threshold = calc_threshold(m, dev_neg, dev_pos) click.echo("\nThreshold of %f selected with the %s criteria" % ( threshold, m)) apcer, bpcer = farfrr(dev_neg, dev_pos, threshold) raws.append(['BPCER20', '{:>5.1f}%'.format(apcer * 100)]) raws.append(['EER', '{:>5.1f}%'.format(bpcer * 100)]) raws.append(['min-HTER', '{:>5.1f}%'.format((apcer + bpcer) * 50)]) if self._eval and eval_neg is not None: apcer, bpcer = farfrr(eval_neg, eval_pos, threshold) raws[0].append('{:>5.1f}%'.format(apcer * 100)) raws[1].append('{:>5.1f}%'.format(bpcer * 100)) raws[2].append('{:>5.1f}%'.format((apcer + bpcer) * 50)) click.echo( tabulate(raws, headers, self._tablefmt), file=self.log_file )  Theophile GENTILHOMME committed May 04, 2018 93   Theophile GENTILHOMME committed May 04, 2018 94 95 96 97 98 class MetricsVuln(measure_figure.Metrics): def __init__(self, ctx, scores, evaluation, func_load): super(MetricsVuln, self).__init__(ctx, scores, evaluation, func_load) ''' Compute metrics from score files'''  Amir MOHAMMADI committed May 08, 2018 99   Theophile GENTILHOMME committed May 04, 2018 100 101 102 103 104 105  def compute(self, idx, input_scores, input_names): ''' Compute metrics for the given criteria''' neg_list, pos_list, _ = get_fta_list(input_scores) dev_neg, dev_pos = neg_list[0], pos_list[0] criter = self._criterion or 'eer' threshold = calc_threshold(criter, dev_neg, dev_pos) \  Amir MOHAMMADI committed May 08, 2018 106  if self._thres is None else self._thres[idx]  Theophile GENTILHOMME committed May 04, 2018 107 108 109 110 111 112  far, frr = farfrr(neg_list[1], pos_list[1], threshold) iapmr, _ = farfrr(neg_list[3], pos_list[1], threshold) title = self._legends[idx] if self._legends is not None else None headers = ['' or title, '%s (threshold=%.2g)' % (criter.upper(), threshold)] rows = []  Amir MOHAMMADI committed May 08, 2018 113 114 115 116  rows.append(['FMR (%)', '{:>5.1f}%'.format(100 * far)]) rows.append(['FMNR (%)', '{:>5.1f}%'.format(frr * 100)]) rows.append(['HTER (%)', '{:>5.1f}%'.format(50 * (far + frr))]) rows.append(['IAPMR (%)', '{:>5.1f}%'.format(100 * iapmr)])  Theophile GENTILHOMME committed May 04, 2018 117 118 119 120 121 122  click.echo( tabulate(rows, headers, self._tablefmt), file=self.log_file )  123 124 125 126 127 128 class HistPad(measure_figure.Hist): ''' Histograms for PAD ''' def _setup_hist(self, neg, pos): self._title_base = 'PAD' self._density_hist(  Theophile GENTILHOMME committed Apr 26, 2018 129  pos[0], n=0, label='Bona Fide', color='C1'  130 131  ) self._density_hist(  Theophile GENTILHOMME committed Apr 26, 2018 132  neg[0], n=1, label='Presentation attack', alpha=0.4, color='C7',  Theophile GENTILHOMME committed Apr 23, 2018 133  hatch='\\\\'  134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157  ) def _iapmr_dot(threshold, iapmr, real_data, **kwargs): # plot a dot on threshold versus IAPMR line and show IAPMR as a number axlim = mpl.axis() mpl.plot(threshold, 100. * iapmr, 'o', color='C3', **kwargs) if not real_data: mpl.annotate( 'IAPMR at\noperating point', xy=(threshold, 100. * iapmr), xycoords='data', xytext=(0.85, 0.6), textcoords='axes fraction', color='black', size='large', arrowprops=dict(facecolor='black', shrink=0.05, width=2), horizontalalignment='center', verticalalignment='top', ) else: mpl.text(threshold + (threshold - axlim[0]) / 12, 100. * iapmr, '%.1f%%' % (100. * iapmr,), color='C3')  Theophile GENTILHOMME committed May 04, 2018 158   159 160 161 162 163 164 def _iapmr_line_plot(scores, n_points=100, **kwargs): axlim = mpl.axis() step = (axlim[1] - axlim[0]) / float(n_points) thres = [(k * step) + axlim[0] for k in range(2, n_points - 1)] mix_prob_y = [] for k in thres:  Theophile GENTILHOMME committed May 04, 2018 165  mix_prob_y.append(100. * error_utils.calc_pass_rate(k, scores))  166 167 168  mpl.plot(thres, mix_prob_y, label='IAPMR', color='C3', **kwargs)  Theophile GENTILHOMME committed May 04, 2018 169   170 171 172 173 def _iapmr_plot(scores, threshold, iapmr, real_data, **kwargs): _iapmr_dot(threshold, iapmr, real_data, **kwargs) _iapmr_line_plot(scores, n_points=100, **kwargs)  Theophile GENTILHOMME committed May 04, 2018 174   175 176 177 178 179 180 class HistVuln(measure_figure.Hist): ''' Histograms for vulnerability ''' def _setup_hist(self, neg, pos): self._title_base = 'Vulnerability' self._density_hist(  Amir MOHAMMADI committed May 03, 2018 181  pos[0], n=0, label='Genuine', color='C2'  182 183  ) self._density_hist(  Theophile GENTILHOMME committed Apr 26, 2018 184  neg[0], n=1, label='Zero-effort impostors', alpha=0.8, color='C0'  185 186  ) self._density_hist(  Theophile GENTILHOMME committed Apr 26, 2018 187  neg[1], n=2, label='Presentation attack', alpha=0.4, color='C7',  Theophile GENTILHOMME committed Apr 23, 2018 188  hatch='\\\\'  189 190  )  Theophile GENTILHOMME committed May 03, 2018 191  def _lines(self, threshold, label, neg, pos, idx, **kwargs):  192  if 'iapmr_line' not in self._ctx.meta or self._ctx.meta['iapmr_line']:  Amir MOHAMMADI committed May 08, 2018 193  # plot vertical line  Theophile GENTILHOMME committed May 03, 2018 194  super(HistVuln, self)._lines(threshold, label, neg, pos, idx)  195   Amir MOHAMMADI committed May 08, 2018 196  # plot iapmr_line  197 198 199 200 201  iapmr, _ = farfrr(neg[1], pos[0], threshold) ax2 = mpl.twinx() # we never want grid lines on axis 2 ax2.grid(False) real_data = True if 'real_data' not in self._ctx.meta else \  Amir MOHAMMADI committed May 08, 2018 202  self._ctx.meta['real_data']  203  _iapmr_plot(neg[1], threshold, iapmr, real_data=real_data)  Theophile GENTILHOMME committed May 03, 2018 204 205  n = idx % self._step_print col = n % self._ncols  Amir MOHAMMADI committed May 08, 2018 206 207  rest_print = self.n_systems - \ int(idx / self._step_print) * self._step_print  Theophile GENTILHOMME committed May 03, 2018 208 209  if col == self._ncols - 1 or n == rest_print - 1: ax2.set_ylabel("IAPMR (%)", color='C3')  210 211 212 213  ax2.tick_params(axis='y', colors='red') ax2.yaxis.label.set_color('red') ax2.spines['right'].set_color('red')  Theophile GENTILHOMME committed May 04, 2018 214   215 216 class PadPlot(measure_figure.PlotBase): '''Base class for PAD plots'''  Amir MOHAMMADI committed May 08, 2018 217   218 219  def __init__(self, ctx, scores, evaluation, func_load): super(PadPlot, self).__init__(ctx, scores, evaluation, func_load)  220  mpl.rcParams['figure.constrained_layout.use'] = self._clayout  221 222 223  def end_process(self): '''Close pdf '''  Amir MOHAMMADI committed May 08, 2018 224  # do not want to close PDF when running evaluate  225 226 227 228 229  if 'PdfPages' in self._ctx.meta and \ ('closef' not in self._ctx.meta or self._ctx.meta['closef']): self._pdf_page.close() def _plot_legends(self):  Amir MOHAMMADI committed May 08, 2018 230  # legends for all axes  231 232 233 234 235 236  lines = [] labels = [] for ax in mpl.gcf().get_axes(): li, la = ax.get_legend_handles_labels() lines += li labels += la  Theophile GENTILHOMME committed May 04, 2018 237 238 239 240  if self._disp_legend: mpl.gca().legend(lines, labels, loc=self._legend_loc, fancybox=True, framealpha=0.5)  241 242 243  class Epc(PadPlot): ''' Handles the plotting of EPC '''  Amir MOHAMMADI committed May 08, 2018 244   245 246 247  def __init__(self, ctx, scores, evaluation, func_load): super(Epc, self).__init__(ctx, scores, evaluation, func_load) self._iapmr = True if 'iapmr' not in self._ctx.meta else \  Amir MOHAMMADI committed May 08, 2018 248  self._ctx.meta['iapmr']  Theophile GENTILHOMME committed Apr 24, 2018 249 250 251 252  self._title = self._title or ('EPC and IAPMR' if self._iapmr else 'EPC') self._x_label = self._x_label or r"Weight $\beta$" self._y_label = self._y_label or "WER (%)"  Amir MOHAMMADI committed May 08, 2018 253  self._eval = True # always eval data with EPC  254 255 256  self._split = False self._nb_figs = 1  Theophile GENTILHOMME committed Apr 23, 2018 257 258 259 260 261  if self._min_arg != 4: raise click.BadParameter("You must provide 4 scores files:{licit," "spoof}/{dev,eval}") def compute(self, idx, input_scores, input_names):  262  ''' Plot EPC for PAD'''  Theophile GENTILHOMME committed Apr 23, 2018 263 264 265 266 267  licit_dev_neg = input_scores[0][0] licit_dev_pos = input_scores[0][1] licit_eval_neg = input_scores[1][0] licit_eval_pos = input_scores[1][1] spoof_eval_neg = input_scores[3][0]  268  mpl.gcf().clear()  Theophile GENTILHOMME committed Apr 13, 2018 269  epc_baseline = epc(  270 271 272 273 274 275 276  licit_dev_neg, licit_dev_pos, licit_eval_neg, licit_eval_pos, 100 ) mpl.plot( epc_baseline[:, 0], [100. * k for k in epc_baseline[:, 1]], color='C0', label=self._label(  Theophile GENTILHOMME committed Apr 23, 2018 277  'WER', '%s-%s' % (input_names[0], input_names[1]), idx  278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298  ), linestyle='-' ) mpl.xlabel(self._x_label) mpl.ylabel(self._y_label) if self._iapmr: mix_prob_y = [] for k in epc_baseline[:, 2]: prob_attack = sum( 1 for i in spoof_eval_neg if i >= k ) / float(spoof_eval_neg.size) mix_prob_y.append(100. * prob_attack) mpl.gca().set_axisbelow(True) prob_ax = mpl.gca().twinx() mpl.plot( epc_baseline[:, 0], mix_prob_y, color='C3', linestyle='-', label=self._label(  Theophile GENTILHOMME committed Apr 23, 2018 299  'IAPMR', '%s-%s' % (input_names[0], input_names[1]), idx  300 301 302 303 304 305 306 307 308  ) ) prob_ax.set_yticklabels(prob_ax.get_yticks()) prob_ax.tick_params(axis='y', colors='red') prob_ax.yaxis.label.set_color('red') prob_ax.spines['right'].set_color('red') ylabels = prob_ax.get_yticks() prob_ax.yaxis.set_ticklabels(["%.0f" % val for val in ylabels]) prob_ax.set_axisbelow(True)  Theophile GENTILHOMME committed Apr 30, 2018 309  title = self._legends[idx] if self._legends is not None else self._title  Theophile GENTILHOMME committed May 04, 2018 310 311  if title.replace(' ', ''): mpl.title(title)  Amir MOHAMMADI committed May 08, 2018 312  # legends for all axes  313 314 315 316  self._plot_legends() mpl.xticks(rotation=self._x_rotation) self._pdf_page.savefig(mpl.gcf())  Theophile GENTILHOMME committed May 04, 2018 317   318 319 class Epsc(PadPlot): ''' Handles the plotting of EPSC '''  Amir MOHAMMADI committed May 08, 2018 320   321 322 323 324  def __init__(self, ctx, scores, evaluation, func_load, criteria, var_param, fixed_param): super(Epsc, self).__init__(ctx, scores, evaluation, func_load) self._iapmr = False if 'iapmr' not in self._ctx.meta else \  Amir MOHAMMADI committed May 08, 2018 325  self._ctx.meta['iapmr']  326  self._wer = True if 'wer' not in self._ctx.meta else \  Amir MOHAMMADI committed May 08, 2018 327  self._ctx.meta['wer']  328 329 330  self._criteria = 'eer' if criteria is None else criteria self._var_param = "omega" if var_param is None else var_param self._fixed_param = 0.5 if fixed_param is None else fixed_param  Amir MOHAMMADI committed May 08, 2018 331  self._eval = True # always eval data with EPC  332 333  self._split = False self._nb_figs = 1  Theophile GENTILHOMME committed Apr 23, 2018 334 335 336 337 338  self._title = '' if self._min_arg != 4: raise click.BadParameter("You must provide 4 scores files:{licit," "spoof}/{dev,eval}")  339   Theophile GENTILHOMME committed Apr 23, 2018 340  def compute(self, idx, input_scores, input_names):  341  ''' Plot EPSC for PAD'''  Theophile GENTILHOMME committed Apr 23, 2018 342 343 344 345 346 347 348 349  licit_dev_neg = input_scores[0][0] licit_dev_pos = input_scores[0][1] licit_eval_neg = input_scores[1][0] licit_eval_pos = input_scores[1][1] spoof_dev_neg = input_scores[2][0] spoof_dev_pos = input_scores[2][1] spoof_eval_neg = input_scores[3][0] spoof_eval_pos = input_scores[3][1]  Theophile GENTILHOMME committed Apr 30, 2018 350  title = self._legends[idx] if self._legends is not None else None  351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370  mpl.gcf().clear() points = 10 if self._var_param == 'omega': omega, beta, thrs = error_utils.epsc_thresholds( licit_dev_neg, licit_dev_pos, spoof_dev_neg, spoof_dev_pos, points=points, criteria=self._criteria, beta=self._fixed_param) else: omega, beta, thrs = error_utils.epsc_thresholds( licit_dev_neg, licit_dev_pos, spoof_dev_neg, spoof_dev_pos, points=points,  Amir MOHAMMADI committed May 08, 2018 371  criteria=self._criteria,  372 373 374 375 376 377 378  omega=self._fixed_param ) errors = error_utils.all_error_rates( licit_eval_neg, licit_eval_pos, spoof_eval_neg, spoof_eval_pos, thrs, omega, beta ) # error rates are returned in a list in the  Amir MOHAMMADI committed May 08, 2018 379  # following order: frr, far, IAPMR, far_w, wer_w  380 381 382 383  ax1 = mpl.subplot( 111 ) # EPC like curves for FVAS fused scores for weighted error rates  Amir MOHAMMADI committed May 08, 2018 384  # between the negatives (impostors and Presentation attacks)  385 386 387 388 389 390 391 392  if self._wer: if self._var_param == 'omega': mpl.plot( omega, 100. * errors[4].flatten(), color='C0', linestyle='-', label=r"WER$_{\omega,\beta}$")  Theophile GENTILHOMME committed Apr 30, 2018 393  mpl.xlabel(self._x_label or r"Weight $\omega$")  394 395 396 397 398 399 400  else: mpl.plot( beta, 100. * errors[4].flatten(), color='C0', linestyle='-', label=r"WER$_{\omega,\beta}$")  Theophile GENTILHOMME committed Apr 30, 2018 401 402  mpl.xlabel(self._x_label or r"Weight $\beta$") mpl.ylabel(self._y_label or r"WER$_{\omega,\beta}$ (%)")  403 404 405 406 407 408 409 410 411 412 413 414 415  if self._iapmr: axis = mpl.gca() if self._wer: axis = mpl.twinx() axis.grid(False) if self._var_param == 'omega': mpl.plot( omega, 100. * errors[2].flatten(), color='C3', linestyle='-', label='IAPMR')  Theophile GENTILHOMME committed Apr 30, 2018 416  mpl.xlabel(self._x_label or r"Weight $\omega$")  417 418 419 420 421 422 423  else: mpl.plot( beta, 100. * errors[2].flatten(), color='C3', linestyle='-', label='IAPMR')  Theophile GENTILHOMME committed Apr 30, 2018 424 425  mpl.xlabel(self._x_label or r"Weight $\beta$") mpl.ylabel(self._y_label or r"IAPMR (%)")  426 427 428 429 430 431 432  if self._wer: axis.set_yticklabels(axis.get_yticks()) axis.tick_params(axis='y', colors='red') axis.yaxis.label.set_color('red') axis.spines['right'].set_color('red') if self._var_param == 'omega':  Theophile GENTILHOMME committed May 04, 2018 433  if title is not None and title.replace(' ', ''):  Amir MOHAMMADI committed May 08, 2018 434  mpl.title(title or (r"EPSC with $\beta$ = %.2f" %  Theophile GENTILHOMME committed May 04, 2018 435  self._fixed_param))  436  else:  Theophile GENTILHOMME committed May 04, 2018 437  if title is not None and title.replace(' ', ''):  Amir MOHAMMADI committed May 08, 2018 438  mpl.title(title or (r"EPSC with $\omega$ = %.2f" %  Theophile GENTILHOMME committed May 04, 2018 439  self._fixed_param))  440 441 442 443 444 445  mpl.grid() self._plot_legends() ax1.set_xticklabels(ax1.get_xticks()) ax1.set_yticklabels(ax1.get_yticks()) mpl.xticks(rotation=self._x_rotation)  446  self._pdf_page.savefig()  447   Theophile GENTILHOMME committed May 04, 2018 448   449 450 class Epsc3D(Epsc): ''' 3D EPSC plots for PAD'''  Amir MOHAMMADI committed May 08, 2018 451   Theophile GENTILHOMME committed Apr 23, 2018 452  def compute(self, idx, input_scores, input_names):  453  ''' Implements plots'''  Theophile GENTILHOMME committed Apr 23, 2018 454 455 456 457 458 459 460 461  licit_dev_neg = input_scores[0][0] licit_dev_pos = input_scores[0][1] licit_eval_neg = input_scores[1][0] licit_eval_pos = input_scores[1][1] spoof_dev_neg = input_scores[2][0] spoof_dev_pos = input_scores[2][1] spoof_eval_neg = input_scores[3][0] spoof_eval_pos = input_scores[3][1]  462   Theophile GENTILHOMME committed May 04, 2018 463  title = self._legends[idx] if self._legends is not None else "3D EPSC"  464   465 466  mpl.rcParams.pop('key', None)  467  mpl.gcf().clear()  468  mpl.gcf().set_constrained_layout(self._clayout)  469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490  from mpl_toolkits.mplot3d import Axes3D from matplotlib import cm points = 10 omega, beta, thrs = error_utils.epsc_thresholds( licit_dev_neg, licit_dev_pos, spoof_dev_neg, spoof_dev_pos, points=points, criteria=self._criteria) errors = error_utils.all_error_rates( licit_eval_neg, licit_eval_pos, spoof_eval_neg, spoof_eval_pos, thrs, omega, beta ) # error rates are returned in a list as 2D numpy.ndarrays in # the following order: frr, far, IAPMR, far_w, wer_wb, hter_wb wer_errors = 100 * errors[2 if self._iapmr else 4]  491  ax1 = mpl.gcf().add_subplot(111, projection='3d')  492 493 494 495 496 497 498 499 500 501 502  W, B = np.meshgrid(omega, beta) ax1.plot_wireframe( W, B, wer_errors, cmap=cm.coolwarm, antialiased=False ) # surface if self._iapmr: ax1.azim = -30 ax1.elev = 50  Theophile GENTILHOMME committed Apr 30, 2018 503 504  ax1.set_xlabel(self._x_label or r"Weight $\omega$") ax1.set_ylabel(self._y_label or r"Weight $\beta$")  505 506 507 508  ax1.set_zlabel( r"WER$_{\omega,\beta}$ (%)" if self._wer else "IAPMR (%)" )  Theophile GENTILHOMME committed May 04, 2018 509  if title.replace(' ', ''):  Theophile GENTILHOMME committed May 04, 2018 510  mpl.title(title)  511 512 513 514 515  ax1.set_xticklabels(ax1.get_xticks()) ax1.set_yticklabels(ax1.get_yticks()) ax1.set_zticklabels(ax1.get_zticks())  516 517  self._pdf_page.savefig()  Theophile GENTILHOMME committed May 04, 2018 518   519 520 class Det(PadPlot): '''DET for PAD'''  Amir MOHAMMADI committed May 08, 2018 521   Theophile GENTILHOMME committed May 14, 2018 522 523  def __init__(self, ctx, scores, evaluation, func_load, criteria, real_data, no_spoof):  524  super(Det, self).__init__(ctx, scores, evaluation, func_load)  Theophile GENTILHOMME committed May 14, 2018 525 526  self._no_spoof = no_spoof self._criteria = criteria or 'eer'  527 528  self._real_data = True if real_data is None else real_data  Theophile GENTILHOMME committed Apr 23, 2018 529  def compute(self, idx, input_scores, input_names):  530  ''' Implements plots'''  Theophile GENTILHOMME committed Apr 23, 2018 531 532 533 534 535 536  licit_dev_neg = input_scores[0][0] licit_dev_pos = input_scores[0][1] licit_eval_neg = input_scores[1][0] licit_eval_pos = input_scores[1][1] spoof_eval_neg = input_scores[3][0] if len(input_scores) > 2 else None spoof_eval_pos = input_scores[3][1] if len(input_scores) > 2 else None  537 538 539 540  det( licit_eval_neg, licit_eval_pos, self._points,  Theophile GENTILHOMME committed May 14, 2018 541  color='C0',  542  linestyle='-',  Theophile GENTILHOMME committed Apr 23, 2018 543  label=self._label("licit", input_names[0], idx)  544 545 546 547 548 549  ) if not self._no_spoof and spoof_eval_neg is not None: det( spoof_eval_neg, spoof_eval_pos, self._points,  Theophile GENTILHOMME committed May 14, 2018 550 551  color='C3', linestyle=':',  Theophile GENTILHOMME committed Apr 23, 2018 552  label=self._label("spoof", input_names[3], idx)  553 554  )  Theophile GENTILHOMME committed May 14, 2018 555  if self._criteria is None or self._no_spoof:  556 557 558 559 560 561 562 563 564 565  return thres_baseline = calc_threshold( self._criteria, licit_dev_neg, licit_dev_pos ) axlim = mpl.axis() farfrr_licit = farfrr( licit_eval_neg, licit_eval_pos,  Theophile GENTILHOMME committed May 14, 2018 566 567  thres_baseline ) # calculate test frr @ EER (licit scenario)  568 569  farfrr_spoof = farfrr( spoof_eval_neg, spoof_eval_pos,  Theophile GENTILHOMME committed May 14, 2018 570 571  thres_baseline ) # calculate test frr @ EER (spoof scenario)  572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588  farfrr_licit_det = [ ppndf(i) for i in farfrr_licit ] # find the FAR and FRR values that need to be plotted on normal deviate # scale farfrr_spoof_det = [ ppndf(i) for i in farfrr_spoof ] # find the FAR and FRR values that need to be plotted on normal deviate # scale if not self._real_data: mpl.axhline( y=farfrr_licit_det[1], xmin=axlim[2], xmax=axlim[3], color='k', linestyle='--',  Theophile GENTILHOMME committed May 14, 2018 589  label="FRR @ EER")  590 591 592 593 594 595 596 597  else: mpl.axhline( y=farfrr_licit_det[1], xmin=axlim[0], xmax=axlim[1], color='k', linestyle='--', label="FRR = %.2f%%" %  Theophile GENTILHOMME committed May 14, 2018 598  (farfrr_licit[1] * 100))  599 600 601 602 603  mpl.plot( farfrr_licit_det[0], farfrr_licit_det[1], 'o',  Theophile GENTILHOMME committed May 14, 2018 604 605 606  color='C0', markersize=9 ) # FAR point, licit scenario  607 608 609 610  mpl.plot( farfrr_spoof_det[0], farfrr_spoof_det[1], 'o',  Theophile GENTILHOMME committed May 14, 2018 611 612 613  color='C3', markersize=9 ) # FAR point, spoof scenario  614 615 616  # annotate the FAR points xyannotate_licit = [  Theophile GENTILHOMME committed May 14, 2018 617 618  0.6*farfrr_licit_det[0], 0.6*farfrr_licit_det[1],  619 620  ] xyannotate_spoof = [  Theophile GENTILHOMME committed May 14, 2018 621 622  0.6*farfrr_spoof_det[0], 0.6*farfrr_spoof_det[1],  623 624 625 626  ] if not self._real_data: mpl.annotate(  Theophile GENTILHOMME committed May 14, 2018 627  'FMR @ operating point',  628 629 630 631 632  xy=(farfrr_licit_det[0], farfrr_licit_det[1]), xycoords='data', xytext=(xyannotate_licit[0], xyannotate_licit[1]), color=self._colors[idx]) mpl.annotate(  Theophile GENTILHOMME committed May 14, 2018 633  'IAPMR @ operating point',  634 635 636 637 638 639 640 641 642 643  xy=(farfrr_spoof_det[0], farfrr_spoof_det[1]), xycoords='data', xytext=(xyannotate_spoof[0], xyannotate_spoof[1]), color=self._colors[idx]) else: mpl.annotate( 'FAR=%.2f%%' % (farfrr_licit[0] * 100), xy=(farfrr_licit_det[0], farfrr_licit_det[1]), xycoords='data', xytext=(xyannotate_licit[0], xyannotate_licit[1]),  Theophile GENTILHOMME committed May 14, 2018 644  color='C0',  645 646  size='large') mpl.annotate(  Theophile GENTILHOMME committed May 14, 2018 647  'IAPMR=%.2f%%' % (farfrr_spoof[0] * 100),  648 649 650  xy=(farfrr_spoof_det[0], farfrr_spoof_det[1]), xycoords='data', xytext=(xyannotate_spoof[0], xyannotate_spoof[1]),  Theophile GENTILHOMME committed May 14, 2018 651  color='C3',  652 653 654 655 656  size='large') def end_process(self): ''' Set title, legend, axis labels, grid colors, save figures and close pdf is needed '''  Amir MOHAMMADI committed May 08, 2018 657  # only for plots  658 659 660 661  add = '' if not self._no_spoof: add = " and overlaid SPOOF scenario" title = self._title if self._title is not None else \  Amir MOHAMMADI committed May 08, 2018 662  ('DET: LICIT' + add)  Theophile GENTILHOMME committed May 04, 2018 663 664  if title.replace(' ', ''): mpl.title(title)  665 666 667  mpl.xlabel(self._x_label or "False Acceptance Rate (%)") mpl.ylabel(self._y_label or "False Rejection Rate (%)") mpl.grid(True, color=self._grid_color)  Theophile GENTILHOMME committed May 04, 2018 668 669  if self._disp_legend: mpl.legend(loc=self._legend_loc)  670 671 672 673 674 675 676 677 678 679 680  self._set_axis() fig = mpl.gcf() mpl.xticks(rotation=self._x_rotation) mpl.tick_params(axis='both', which='major', labelsize=4) for tick in mpl.gca().xaxis.get_major_ticks(): tick.label.set_fontsize(6) for tick in mpl.gca().yaxis.get_major_ticks(): tick.label.set_fontsize(6) self._pdf_page.savefig(fig)  Amir MOHAMMADI committed May 08, 2018 681  # do not want to close PDF when running evaluate  682  if 'PdfPages' in self._ctx.meta and \  Amir MOHAMMADI committed May 08, 2018 683  ('closef' not in self._ctx.meta or self._ctx.meta['closef']):  684 685 686 687 688 689 690  self._pdf_page.close() def _set_axis(self): if self._axlim is not None and None not in self._axlim: det_axis(self._axlim) else: det_axis([0.01, 99, 0.01, 99])  Theophile GENTILHOMME committed Apr 24, 2018 691   Theophile GENTILHOMME committed May 04, 2018 692   Theophile GENTILHOMME committed Apr 24, 2018 693 694 class FmrIapmr(PadPlot): '''FMR vs IAPMR'''  Amir MOHAMMADI committed May 08, 2018 695   Theophile GENTILHOMME committed Apr 24, 2018 696 697  def __init__(self, ctx, scores, evaluation, func_load): super(FmrIapmr, self).__init__(ctx, scores, evaluation, func_load)  Amir MOHAMMADI committed May 08, 2018 698  self._eval = True # always eval data with EPC  Theophile GENTILHOMME committed Apr 24, 2018 699 700 701  self._split = False self._nb_figs = 1 self._semilogx = False if 'semilogx' not in ctx.meta else\  Amir MOHAMMADI committed May 08, 2018 702  ctx.meta['semilogx']  Theophile GENTILHOMME committed Apr 24, 2018 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719  if self._min_arg != 4: raise click.BadParameter("You must provide 4 scores files:{licit," "spoof}/{dev,eval}") def compute(self, idx, input_scores, input_names): ''' Implements plots''' licit_eval_neg = input_scores[1][0] licit_eval_pos = input_scores[1][1] spoof_eval_neg = input_scores[3][0] fmr_list = np.linspace(0, 1, 100) iapmr_list = [] for i, fmr in enumerate(fmr_list): thr = far_threshold(licit_eval_neg, licit_eval_pos, fmr, True) iapmr_list.append(farfrr(spoof_eval_neg, licit_eval_pos, thr)[0]) # re-calculate fmr since threshold might give a different result # for fmr. fmr_list[i] = farfrr(licit_eval_neg, licit_eval_pos, thr)[0]  Theophile GENTILHOMME committed Apr 30, 2018 720  label = self._legends[idx] if self._legends is not None else \  Amir MOHAMMADI committed May 08, 2018 721  '(%s/%s)' % (input_names[1], input_names[3])  Theophile GENTILHOMME committed Apr 24, 2018 722 723 724 725 726 727 728 729  if self._semilogx: mpl.semilogx(fmr_list, iapmr_list, label=label) else: mpl.plot(fmr_list, iapmr_list, label=label) def end_process(self): ''' Set title, legend, axis labels, grid colors, save figures and close pdf is needed '''  Amir MOHAMMADI committed May 08, 2018 730  # only for plots  Theophile GENTILHOMME committed Apr 24, 2018 731  title = self._title if self._title is not None else "FMR vs IAPMR"  Theophile GENTILHOMME committed May 04, 2018 732 733  if title.replace(' ', ''): mpl.title(title)  Theophile GENTILHOMME committed Apr 24, 2018 734 735 736  mpl.xlabel(self._x_label or "False Match Rate (%)") mpl.ylabel(self._y_label or "IAPMR (%)") mpl.grid(True, color=self._grid_color)  Theophile GENTILHOMME committed May 04, 2018 737 738  if self._disp_legend: mpl.legend(loc=self._legend_loc)  Theophile GENTILHOMME committed Apr 24, 2018 739 740 741 742 743 744 745  self._set_axis() fig = mpl.gcf() mpl.xticks(rotation=self._x_rotation) mpl.tick_params(axis='both', which='major', labelsize=4) self._pdf_page.savefig(fig)  Amir MOHAMMADI committed May 08, 2018 746  # do not want to close PDF when running evaluate  Theophile GENTILHOMME committed Apr 24, 2018 747  if 'PdfPages' in self._ctx.meta and \  Amir MOHAMMADI committed May 08, 2018 748  ('closef' not in self._ctx.meta or self._ctx.meta['closef']):  Theophile GENTILHOMME committed Apr 24, 2018 749  self._pdf_page.close()