vuln_commands.py 16.1 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13
"""The main entry for bob.pad and its(click-based) scripts.
"""

import os
import logging
import numpy
import click
import pkg_resources
from click_plugins import with_plugins
from click.types import FLOAT
from bob.measure.script import common_options
from bob.extension.scripts.click_helper import (verbosity_option,
                                                open_file_mode_option,
14
                                               bool_option,
15
                                               AliasedGroup, list_float_option)
16 17 18
from bob.core import random
from bob.io.base import create_directories_safe
from bob.bio.base.score import load
19
from . import vuln_figure as figure
20 21 22 23 24 25

NUM_GENUINE_ACCESS = 5000
NUM_ZEIMPOSTORS = 5000
NUM_PA = 5000


26 27 28 29 30 31 32 33 34
def hlines_at_option(dflt=' ', **kwargs):
    '''Get option to draw const FNMRlines'''
    return list_float_option(
        name='hlines-at', short_name='hla',
        desc='If given, draw horizontal lines at the given axis positions. '
        'Your values must be separated with a comma (,) without space. '
        'This option works in ROC and DET curves.',
        nitems=None, dflt=dflt, **kwargs
    )
35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79


def gen_score_distr(mean_gen, mean_zei, mean_pa, sigma_gen=1, sigma_zei=1,
                    sigma_pa=1):
  mt = random.mt19937()  # initialise the random number generator

  genuine_generator = random.normal(numpy.float32, mean_gen, sigma_gen)
  zei_generator = random.normal(numpy.float32, mean_zei, sigma_zei)
  pa_generator = random.normal(numpy.float32, mean_pa, sigma_pa)

  genuine_scores = [genuine_generator(mt) for i in range(NUM_GENUINE_ACCESS)]
  zei_scores = [zei_generator(mt) for i in range(NUM_ZEIMPOSTORS)]
  pa_scores = [pa_generator(mt) for i in range(NUM_PA)]

  return genuine_scores, zei_scores, pa_scores



def write_scores_to_file(neg, pos, filename, attack=False):
  """Writes score distributions into 4-column score files. For the format of
    the 4-column score files, please refer to Bob's documentation.

  Parameters
  ----------
  neg : array_like
      Scores for negative samples.
  pos : array_like
      Scores for positive samples.
  filename : str
      The path to write the score to.
  """
  create_directories_safe(os.path.dirname(filename))
  with open(filename, 'wt') as f:
      for i in pos:
          f.write('x x foo %f\n' % i)
      for i in neg:
          if attack:
              f.write('x attack foo %f\n' % i)
          else:
              f.write('x y foo %f\n' % i)



@click.command()
@click.argument('outdir')
80 81
@click.option('--mean-gen', default=7, type=FLOAT, show_default=True)
@click.option('--mean-zei', default=3, type=FLOAT, show_default=True)
82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98
@click.option('--mean-pa', default=5, type=FLOAT, show_default=True)
@verbosity_option()
def gen(outdir, mean_gen, mean_zei, mean_pa):
  """Generate random scores.
  Generates random scores for three types of verification attempts:
  genuine users, zero-effort impostors and spoofing attacks and writes them
  into 4-column score files for so called licit and spoof scenario. The
  scores are generated using Gaussian distribution whose mean is an input
  parameter. The generated scores can be used as hypothetical datasets.
  """
  # Generate the data
  genuine_dev, zei_dev, pa_dev = gen_score_distr(
      mean_gen, mean_zei, mean_pa)
  genuine_eval, zei_eval, pa_eval = gen_score_distr(
      mean_gen, mean_zei, mean_pa)

  # Write the data into files
99
  write_scores_to_file(zei_dev, genuine_dev,
100
                       os.path.join(outdir, 'licit', 'scores-dev'))
101
  write_scores_to_file(zei_eval, genuine_eval,
102
                       os.path.join(outdir, 'licit', 'scores-eval'))
103
  write_scores_to_file(pa_dev, genuine_dev,
104 105
                       os.path.join(outdir, 'spoof', 'scores-dev'),
                       attack=True)
106
  write_scores_to_file(pa_eval, genuine_eval,
107 108 109 110 111
                       os.path.join(outdir, 'spoof', 'scores-eval'),
                       attack=True)



112
@click.command()
113
@common_options.scores_argument(min_arg=2, nargs=-1)
114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130
@common_options.output_plot_file_option(default_out='vuln_roc.pdf')
@common_options.legends_option()
@common_options.no_legend_option()
@common_options.legend_loc_option(dflt='upper-right')
@common_options.title_option()
@common_options.const_layout_option()
@common_options.style_option()
@common_options.figsize_option(dflt=None)
@common_options.min_far_option()
@common_options.axes_val_option()
@verbosity_option()
@common_options.x_rotation_option(dflt=45)
@common_options.x_label_option()
@common_options.y_label_option()
@click.option('--real-data/--no-real-data', default=True, show_default=True,
              help='If False, will annotate the plots hypothetically, instead '
              'of with real data values of the calculated error rates.')
131
@hlines_at_option()
132
@click.pass_context
133
def roc(ctx, scores, real_data, **kwargs):
134 135
  """Plot ROC

136
  You need to provide 2 scores
137
  files for each vulnerability system in this order:
138 139

  \b
140 141
  * licit scores
  * spoof scores
142 143

  Examples:
144
      $ bob vuln roc -v licit-scores spoof-scores
145

146
      $ bob vuln roc -v scores-{licit,spoof}
147
  """
148
  process = figure.RocVuln(ctx, scores, True, load.split, real_data, False)
149 150 151
  process.run()


152
@click.command()
153
@common_options.scores_argument(min_arg=2, nargs=-1)
154 155 156 157 158 159 160
@common_options.output_plot_file_option(default_out='vuln_det.pdf')
@common_options.legends_option()
@common_options.no_legend_option()
@common_options.legend_loc_option(dflt='upper-right')
@common_options.title_option()
@common_options.const_layout_option()
@common_options.style_option()
161
@common_options.figsize_option(dflt=None)
162 163 164 165 166 167 168 169
@verbosity_option()
@common_options.axes_val_option(dflt='0.01,95,0.01,95')
@common_options.x_rotation_option(dflt=45)
@common_options.x_label_option()
@common_options.y_label_option()
@click.option('--real-data/--no-real-data', default=True, show_default=True,
              help='If False, will annotate the plots hypothetically, instead '
              'of with real data values of the calculated error rates.')
170
@hlines_at_option()
171
@click.pass_context
172
def det(ctx, scores, real_data, **kwargs):
173 174
  """Plot DET

175 176 177

  You need to provide 2 scores
  files for each vulnerability system in this order:
178 179

  \b
180 181
  * licit scores
  * spoof scores
182 183

  Examples:
184
      $ bob vuln det -v licit-scores spoof-scores
185

186
      $ bob vuln det -v scores-{licit,spoof}
187
  """
188
  process = figure.DetVuln(ctx, scores, True, load.split, real_data, False)
189 190 191 192 193 194 195 196 197 198 199 200 201 202
  process.run()



@click.command()
@common_options.scores_argument(min_arg=2, force_eval=True, nargs=-1)
@common_options.output_plot_file_option(default_out='vuln_epc.pdf')
@common_options.legends_option()
@common_options.no_legend_option()
@common_options.legend_loc_option()
@common_options.title_option()
@common_options.const_layout_option()
@common_options.x_label_option()
@common_options.y_label_option()
203
@common_options.figsize_option(dflt=None)
204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226
@common_options.style_option()
@common_options.bool_option(
    'iapmr', 'I', 'Whether to plot the IAPMR related lines or not.', True
)
@common_options.style_option()
@verbosity_option()
@click.pass_context
def epc(ctx, scores, **kwargs):
  """Plot EPC (expected performance curve):

  You need to provide 4 score
  files for each biometric system in this order:

  \b
  * licit development scores
  * licit evaluation scores
  * spoof development scores
  * spoof evaluation scores

  See :ref:`bob.pad.base.vulnerability` in the documentation for a guide on
  vulnerability analysis.

  Examples:
227
      $ bob vuln epc -v dev-scores eval-scores
228

229
      $ bob vuln epc -v -o my_epc.pdf dev-scores1 eval-scores1
230

231
      $ bob vuln epc -v {licit,spoof}/scores-{dev,eval}
232 233 234 235 236 237 238 239 240 241 242 243 244 245 246
  """
  process = figure.Epc(ctx, scores, True, load.split)
  process.run()



@click.command()
@common_options.scores_argument(min_arg=2, force_eval=True, nargs=-1)
@common_options.output_plot_file_option(default_out='vuln_epsc.pdf')
@common_options.legends_option()
@common_options.no_legend_option()
@common_options.legend_loc_option()
@common_options.const_layout_option()
@common_options.x_label_option()
@common_options.y_label_option()
247
@common_options.figsize_option(dflt=None)
248 249 250 251 252 253 254 255 256 257 258 259
@common_options.style_option()
@common_options.bool_option(
    'wer', 'w', 'Whether to plot the WER related lines or not.', True
)
@common_options.bool_option(
    'three-d', 'D', 'If true, generate 3D plots', False
)
@common_options.bool_option(
    'iapmr', 'I', 'Whether to plot the IAPMR related lines or not.', False
)
@click.option('-c', '--criteria', default="eer", show_default=True,
              help='Criteria for threshold selection',
260
              type=click.Choice(('eer', 'min-hter')))
261 262 263 264 265 266
@click.option('-vp', '--var-param', default="omega", show_default=True,
              help='Name of the varying parameter',
              type=click.Choice(('omega', 'beta')))
@click.option('-fp', '--fixed-param', default=0.5, show_default=True,
              help='Value of the fixed parameter',
              type=click.FLOAT)
267 268
@click.option('-s', '--sampling', default=5, show_default=True,
              help='Sampling of the EPSC 3D surface', type=click.INT)
269 270
@verbosity_option()
@click.pass_context
271 272
def epsc(ctx, scores, criteria, var_param, fixed_param, three_d, sampling,
         **kwargs):
273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290
    """Plot EPSC (expected performance spoofing curve):

    You need to provide 4 score
    files for each biometric system in this order:

    \b
    * licit development scores
    * licit evaluation scores
    * spoof development scores
    * spoof evaluation scores

    See :ref:`bob.pad.base.vulnerability` in the documentation for a guide on
    vulnerability analysis.

    Note that when using 3D plots with option ``--three-d``, you cannot plot
    both WER and IAPMR on the same figure (which is possible in 2D).

    Examples:
291
        $ bob vuln epsc -v -o my_epsc.pdf dev-scores1 eval-scores1
292

293
        $ bob vuln epsc -v -D {licit,spoof}/scores-{dev,eval}
294 295 296 297
    """
    if three_d:
        if (ctx.meta['wer'] and ctx.meta['iapmr']):
            raise click.BadParameter('Cannot plot both WER and IAPMR in 3D')
298
        ctx.meta['sampling'] = sampling
299 300 301 302 303 304 305 306 307 308 309 310 311 312
        process = figure.Epsc3D(
            ctx, scores, True, load.split,
            criteria, var_param, fixed_param
        )
    else:
        process = figure.Epsc(
            ctx, scores, True, load.split,
            criteria, var_param, fixed_param
        )
    process.run()



@click.command()
313
@common_options.scores_argument(nargs=-1, min_arg=2)
314 315 316 317 318 319 320 321 322 323 324 325 326
@common_options.output_plot_file_option(default_out='vuln_hist.pdf')
@common_options.n_bins_option()
@common_options.criterion_option()
@common_options.thresholds_option()
@common_options.print_filenames_option(dflt=False)
@bool_option(
    'iapmr-line', 'I', 'Whether to plot the IAPMR related lines or not.', True
)
@bool_option(
    'real-data', 'R',
    'If False, will annotate the plots hypothetically, instead '
    'of with real data values of the calculated error rates.', True
)
327
@common_options.titles_option()
328
@common_options.const_layout_option()
329 330 331 332
@common_options.figsize_option(dflt=None)
@common_options.subplot_option()
@common_options.legend_ncols_option()
@common_options.style_option()
333
@common_options.hide_dev_option()
334
@common_options.eval_option()
335 336
@verbosity_option()
@click.pass_context
337
def hist(ctx, scores, evaluation, **kwargs):
338 339
  '''Vulnerability analysis distributions.

340 341 342
  Plots the histogram of score distributions. You need to provide 2 or 4 score
  files for each biometric system in this order.
  When evaluation scores are provided, you must use the ``--eval`` option.
343 344 345

  \b
  * licit development scores
346
  * (optional) licit evaluation scores
347
  * spoof development scores
348
  * (optional) spoof evaluation scores
349 350 351 352 353 354 355

  See :ref:`bob.pad.base.vulnerability` in the documentation for a guide on
  vulnerability analysis.


  By default, when eval-scores are given, only eval-scores histograms are
  displayed with threshold line
356
  computed from dev-scores.
357 358 359

  Examples:

360
      $ bob vuln vuln_hist -e -v licit/scores-dev licit/scores-eval \
361 362
                          spoof/scores-dev spoof/scores-eval

363
      $ bob vuln vuln_hist -e -v {licit,spoof}/scores-{dev,eval}
364
  '''
365
  process = figure.HistVuln(ctx, scores, evaluation, load.split)
366 367 368 369 370 371 372
  process.run()



@click.command(context_settings=dict(token_normalize_func=lambda x: x.lower()))
@common_options.scores_argument(min_arg=2, force_eval=True, nargs=-1)
@common_options.table_option()
373
@common_options.criterion_option(lcriteria=['eer', 'min-hter'])
374 375 376 377 378 379 380 381 382
@common_options.thresholds_option()
@open_file_mode_option()
@common_options.output_log_metric_option()
@common_options.legends_option()
@verbosity_option()
@click.pass_context
def metrics(ctx, scores, **kwargs):
  """Generate table of metrics for vulnerability PAD

383 384
  You need to provide 4 scores
  files for each vuln system in this order:
385 386 387 388 389 390 391 392 393

  \b
  * licit development scores
  * licit evaluation scores
  * spoof development scores
  * spoof evaluation scores


  Examples:
394
      $ bob vuln vuln_metrics -v {licit,spoof}/scores-{dev,eval}
395
  """
396
  process = figure.Metrics(ctx, scores, True, load.split)
397 398 399 400 401 402 403 404 405 406 407 408 409
  process.run()



@click.command()
@common_options.scores_argument(min_arg=2, force_eval=True, nargs=-1)
@common_options.output_plot_file_option(default_out='fmr_iapmr.pdf')
@common_options.legends_option()
@common_options.no_legend_option()
@common_options.legend_loc_option()
@common_options.title_option()
@common_options.const_layout_option()
@common_options.style_option()
410
@common_options.figsize_option(dflt=None)
411 412 413 414 415 416 417 418 419 420
@verbosity_option()
@common_options.axes_val_option()
@common_options.x_rotation_option()
@common_options.x_label_option()
@common_options.y_label_option()
@common_options.semilogx_option()
@click.pass_context
def fmr_iapmr(ctx, scores, **kwargs):
    """Plot FMR vs IAPMR

421 422
    You need to provide 4 scores
    files for each vuln system in this order:
423 424 425 426

    \b
    * licit development scores
    * licit evaluation scores
427 428
    * spoof development scores
    * spoof evaluation scores
429 430

    Examples:
431
        $ bob vuln fmr_iapmr -v dev-scores eval-scores
432

433
        $ bob vuln fmr_iapmr -v {licit,spoof}/scores-{dev,eval}
434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449
    """
    process = figure.FmrIapmr(ctx, scores, True, load.split)
    process.run()



@click.command()
@common_options.scores_argument(min_arg=2, force_eval=True, nargs=-1)
@common_options.legends_option()
@common_options.sep_dev_eval_option()
@common_options.table_option()
@common_options.output_log_metric_option()
@common_options.output_plot_file_option(default_out='vuln_eval.pdf')
@common_options.points_curve_option()
@common_options.lines_at_option()
@common_options.const_layout_option()
450
@common_options.figsize_option(dflt=None)
451 452 453 454 455 456 457 458
@common_options.style_option()
@common_options.linestyles_option()
@verbosity_option()
@click.pass_context
def evaluate(ctx, scores, **kwargs):
  '''Runs error analysis on score sets for vulnerability studies

  \b
459
  1. Computes bob vuln vuln_metrics
460 461 462 463 464 465 466 467 468 469 470 471 472
  2. Plots EPC, EPSC, vulnerability histograms, fmr vs IAPMR to a multi-page
     PDF file


  You need to provide 4 score files for each biometric system in this order:

  \b
  * licit development scores
  * licit evaluation scores
  * spoof development scores
  * spoof evaluation scores

  Examples:
473
      $ bob vuln evaluate -o my_epsc.pdf dev-scores1 eval-scores1
474

475
      $ bob vuln evaluate -D {licit,spoof}/scores-{dev,eval}
476 477 478 479 480 481 482 483 484 485 486 487 488 489
  '''
  # first time erase if existing file
  click.echo("Computing vuln metrics...")
  ctx.invoke(metrics, scores=scores, evaluation=True)
  if 'log' in ctx.meta and ctx.meta['log'] is not None:
      click.echo("[metrics] => %s" % ctx.meta['log'])

  # avoid closing pdf file before all figures are plotted
  ctx.meta['closef'] = False
  click.echo("Computing histograms...")
  ctx.meta['criterion'] = 'eer'  # no criterion passed in evaluate
  ctx.forward(hist)  # use class defaults plot settings
  click.echo("Computing DET...")
  ctx.forward(det)  # use class defaults plot settings
490 491
  click.echo("Computing ROC...")
  ctx.forward(roc)  # use class defaults plot settings
492 493 494 495 496 497 498 499 500
  click.echo("Computing EPC...")
  ctx.forward(epc)  # use class defaults plot settings
  click.echo("Computing EPSC...")
  ctx.forward(epsc)  # use class defaults plot settings
  click.echo("Computing FMR vs IAPMR...")
  ctx.meta['closef'] = True
  ctx.forward(fmr_iapmr)  # use class defaults plot settings
  click.echo("Vuln successfully completed!")
  click.echo("[plots] => %s" % (ctx.meta['output']))