vuln_commands.py 16.5 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76
"""The main entry for bob.pad and its(click-based) scripts.
"""

import os
import logging
import numpy
import click
import pkg_resources
from click_plugins import with_plugins
from click.types import FLOAT
from bob.measure.script import common_options
from bob.extension.scripts.click_helper import (verbosity_option,
                                                open_file_mode_option,
                                               bool_option)
from bob.core import random
from bob.io.base import create_directories_safe
from bob.bio.base.score import load
from . import figure

NUM_GENUINE_ACCESS = 5000
NUM_ZEIMPOSTORS = 5000
NUM_PA = 5000



@with_plugins(pkg_resources.iter_entry_points('bob.vuln.cli'))
@click.group()
def vuln():
  """Presentation Vulnerability related commands."""
  pass



def gen_score_distr(mean_gen, mean_zei, mean_pa, sigma_gen=1, sigma_zei=1,
                    sigma_pa=1):
  mt = random.mt19937()  # initialise the random number generator

  genuine_generator = random.normal(numpy.float32, mean_gen, sigma_gen)
  zei_generator = random.normal(numpy.float32, mean_zei, sigma_zei)
  pa_generator = random.normal(numpy.float32, mean_pa, sigma_pa)

  genuine_scores = [genuine_generator(mt) for i in range(NUM_GENUINE_ACCESS)]
  zei_scores = [zei_generator(mt) for i in range(NUM_ZEIMPOSTORS)]
  pa_scores = [pa_generator(mt) for i in range(NUM_PA)]

  return genuine_scores, zei_scores, pa_scores



def write_scores_to_file(neg, pos, filename, attack=False):
  """Writes score distributions into 4-column score files. For the format of
    the 4-column score files, please refer to Bob's documentation.

  Parameters
  ----------
  neg : array_like
      Scores for negative samples.
  pos : array_like
      Scores for positive samples.
  filename : str
      The path to write the score to.
  """
  create_directories_safe(os.path.dirname(filename))
  with open(filename, 'wt') as f:
      for i in pos:
          f.write('x x foo %f\n' % i)
      for i in neg:
          if attack:
              f.write('x attack foo %f\n' % i)
          else:
              f.write('x y foo %f\n' % i)



@click.command()
@click.argument('outdir')
77 78
@click.option('--mean-gen', default=7, type=FLOAT, show_default=True)
@click.option('--mean-zei', default=3, type=FLOAT, show_default=True)
79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95
@click.option('--mean-pa', default=5, type=FLOAT, show_default=True)
@verbosity_option()
def gen(outdir, mean_gen, mean_zei, mean_pa):
  """Generate random scores.
  Generates random scores for three types of verification attempts:
  genuine users, zero-effort impostors and spoofing attacks and writes them
  into 4-column score files for so called licit and spoof scenario. The
  scores are generated using Gaussian distribution whose mean is an input
  parameter. The generated scores can be used as hypothetical datasets.
  """
  # Generate the data
  genuine_dev, zei_dev, pa_dev = gen_score_distr(
      mean_gen, mean_zei, mean_pa)
  genuine_eval, zei_eval, pa_eval = gen_score_distr(
      mean_gen, mean_zei, mean_pa)

  # Write the data into files
96
  write_scores_to_file(zei_dev, genuine_dev,
97
                       os.path.join(outdir, 'licit', 'scores-dev'))
98
  write_scores_to_file(zei_eval, genuine_eval,
99
                       os.path.join(outdir, 'licit', 'scores-eval'))
100
  write_scores_to_file(pa_dev, genuine_dev,
101 102
                       os.path.join(outdir, 'spoof', 'scores-dev'),
                       attack=True)
103
  write_scores_to_file(pa_eval, genuine_eval,
104 105 106 107 108
                       os.path.join(outdir, 'spoof', 'scores-eval'),
                       attack=True)



109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135
@click.command()
@common_options.scores_argument(min_arg=2, force_eval=True, nargs=-1)
@common_options.output_plot_file_option(default_out='vuln_roc.pdf')
@common_options.legends_option()
@common_options.no_legend_option()
@common_options.legend_loc_option(dflt='upper-right')
@common_options.title_option()
@common_options.const_layout_option()
@common_options.style_option()
@common_options.figsize_option(dflt=None)
@common_options.min_far_option()
@common_options.axes_val_option()
@verbosity_option()
@common_options.x_rotation_option(dflt=45)
@common_options.x_label_option()
@common_options.y_label_option()
@click.option('-c', '--criteria', default=None, show_default=True,
              help='Criteria for threshold selection',
              type=click.Choice(('eer', 'min-hter', 'bpcer20')))
@click.option('--real-data/--no-real-data', default=True, show_default=True,
              help='If False, will annotate the plots hypothetically, instead '
              'of with real data values of the calculated error rates.')
@click.pass_context
def roc(ctx, scores, criteria, real_data, **kwargs):
  """Plot ROC

  You need to provide 4 scores
136
  files for each vulnerability system in this order:
137 138 139 140 141 142 143 144

  \b
  * licit development scores
  * licit evaluation scores
  * spoof development scores
  * spoof evaluation scores

  Examples:
145
      $ bob vuln roc -v dev-scores eval-scores
146

147
      $ bob vuln roc -v {licit,spoof}/scores-{dev,eval}
148 149 150 151 152 153
  """
  process = figure.RocVuln(ctx, scores, True, load.split, criteria, real_data,
                       False)
  process.run()


154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179
@click.command()
@common_options.scores_argument(min_arg=2, force_eval=True, nargs=-1)
@common_options.output_plot_file_option(default_out='vuln_det.pdf')
@common_options.legends_option()
@common_options.no_legend_option()
@common_options.legend_loc_option(dflt='upper-right')
@common_options.title_option()
@common_options.const_layout_option()
@common_options.style_option()
@common_options.figsize_option()
@verbosity_option()
@common_options.axes_val_option(dflt='0.01,95,0.01,95')
@common_options.x_rotation_option(dflt=45)
@common_options.x_label_option()
@common_options.y_label_option()
@click.option('-c', '--criteria', default=None, show_default=True,
              help='Criteria for threshold selection',
              type=click.Choice(('eer', 'min-hter', 'bpcer20')))
@click.option('--real-data/--no-real-data', default=True, show_default=True,
              help='If False, will annotate the plots hypothetically, instead '
              'of with real data values of the calculated error rates.')
@click.pass_context
def det(ctx, scores, criteria, real_data, **kwargs):
  """Plot DET

  You need to provide 4 scores
180
  files for each vuln system in this order:
181 182 183 184 185 186 187 188

  \b
  * licit development scores
  * licit evaluation scores
  * spoof development scores
  * spoof evaluation scores

  Examples:
189
      $ bob vuln det -v dev-scores eval-scores
190

191
      $ bob vuln det -v {licit,spoof}/scores-{dev,eval}
192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232
  """
  process = figure.Det(ctx, scores, True, load.split, criteria, real_data,
                       False)
  process.run()



@click.command()
@common_options.scores_argument(min_arg=2, force_eval=True, nargs=-1)
@common_options.output_plot_file_option(default_out='vuln_epc.pdf')
@common_options.legends_option()
@common_options.no_legend_option()
@common_options.legend_loc_option()
@common_options.title_option()
@common_options.const_layout_option()
@common_options.x_label_option()
@common_options.y_label_option()
@common_options.figsize_option()
@common_options.style_option()
@common_options.bool_option(
    'iapmr', 'I', 'Whether to plot the IAPMR related lines or not.', True
)
@common_options.style_option()
@verbosity_option()
@click.pass_context
def epc(ctx, scores, **kwargs):
  """Plot EPC (expected performance curve):

  You need to provide 4 score
  files for each biometric system in this order:

  \b
  * licit development scores
  * licit evaluation scores
  * spoof development scores
  * spoof evaluation scores

  See :ref:`bob.pad.base.vulnerability` in the documentation for a guide on
  vulnerability analysis.

  Examples:
233
      $ bob vuln epc -v dev-scores eval-scores
234

235
      $ bob vuln epc -v -o my_epc.pdf dev-scores1 eval-scores1
236

237
      $ bob vuln epc -v {licit,spoof}/scores-{dev,eval}
238 239 240 241 242 243 244 245 246 247 248 249 250 251 252
  """
  process = figure.Epc(ctx, scores, True, load.split)
  process.run()



@click.command()
@common_options.scores_argument(min_arg=2, force_eval=True, nargs=-1)
@common_options.output_plot_file_option(default_out='vuln_epsc.pdf')
@common_options.legends_option()
@common_options.no_legend_option()
@common_options.legend_loc_option()
@common_options.const_layout_option()
@common_options.x_label_option()
@common_options.y_label_option()
253
@common_options.figsize_option(dflt=None)
254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272
@common_options.style_option()
@common_options.bool_option(
    'wer', 'w', 'Whether to plot the WER related lines or not.', True
)
@common_options.bool_option(
    'three-d', 'D', 'If true, generate 3D plots', False
)
@common_options.bool_option(
    'iapmr', 'I', 'Whether to plot the IAPMR related lines or not.', False
)
@click.option('-c', '--criteria', default="eer", show_default=True,
              help='Criteria for threshold selection',
              type=click.Choice(('eer', 'min-hter', 'bpcer20')))
@click.option('-vp', '--var-param', default="omega", show_default=True,
              help='Name of the varying parameter',
              type=click.Choice(('omega', 'beta')))
@click.option('-fp', '--fixed-param', default=0.5, show_default=True,
              help='Value of the fixed parameter',
              type=click.FLOAT)
273 274
@click.option('-s', '--sampling', default=5, show_default=True,
              help='Sampling of the EPSC 3D surface', type=click.INT)
275 276
@verbosity_option()
@click.pass_context
277 278
def epsc(ctx, scores, criteria, var_param, fixed_param, three_d, sampling,
         **kwargs):
279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296
    """Plot EPSC (expected performance spoofing curve):

    You need to provide 4 score
    files for each biometric system in this order:

    \b
    * licit development scores
    * licit evaluation scores
    * spoof development scores
    * spoof evaluation scores

    See :ref:`bob.pad.base.vulnerability` in the documentation for a guide on
    vulnerability analysis.

    Note that when using 3D plots with option ``--three-d``, you cannot plot
    both WER and IAPMR on the same figure (which is possible in 2D).

    Examples:
297
        $ bob vuln epsc -v -o my_epsc.pdf dev-scores1 eval-scores1
298

299
        $ bob vuln epsc -v -D {licit,spoof}/scores-{dev,eval}
300 301 302 303
    """
    if three_d:
        if (ctx.meta['wer'] and ctx.meta['iapmr']):
            raise click.BadParameter('Cannot plot both WER and IAPMR in 3D')
304
        ctx.meta['sampling'] = sampling
305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335
        process = figure.Epsc3D(
            ctx, scores, True, load.split,
            criteria, var_param, fixed_param
        )
    else:
        process = figure.Epsc(
            ctx, scores, True, load.split,
            criteria, var_param, fixed_param
        )
    process.run()



@click.command()
@common_options.scores_argument(nargs=-1, min_arg=2)
@common_options.title_option()
@common_options.output_plot_file_option(default_out='vuln_hist.pdf')
@common_options.eval_option()
@common_options.n_bins_option()
@common_options.criterion_option()
@common_options.thresholds_option()
@common_options.print_filenames_option(dflt=False)
@bool_option(
    'iapmr-line', 'I', 'Whether to plot the IAPMR related lines or not.', True
)
@bool_option(
    'real-data', 'R',
    'If False, will annotate the plots hypothetically, instead '
    'of with real data values of the calculated error rates.', True
)
@common_options.legends_option()
336
@common_options.const_layout_option()
337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369
@common_options.figsize_option(dflt=None)
@common_options.subplot_option()
@common_options.legend_ncols_option()
@common_options.style_option()
@verbosity_option()
@click.pass_context
def hist(ctx, scores, evaluation, **kwargs):
  '''Vulnerability analysis distributions.

  Plots the histogram of score distributions. You need to provide 4 score
  files for each biometric system in this order:

  \b
  * licit development scores
  * licit evaluation scores
  * spoof development scores
  * spoof evaluation scores

  See :ref:`bob.pad.base.vulnerability` in the documentation for a guide on
  vulnerability analysis.

  You need to provide one or more development score file(s) for each
  experiment. You can also provide eval files along with dev files. If only
  dev-scores are used set the flag `--no-evaluation` is required in that
  case.

  By default, when eval-scores are given, only eval-scores histograms are
  displayed with threshold line
  computed from dev-scores. If you want to display dev-scores distributions
  as well, use ``--show-dev`` option.

  Examples:

370
      $ bob vuln vuln_hist -v licit/scores-dev licit/scores-eval \
371 372
                          spoof/scores-dev spoof/scores-eval

373
      $ bob vuln vuln_hist -v {licit,spoof}/scores-{dev,eval}
374 375 376 377 378 379 380 381 382 383
  '''
  process = figure.HistVuln(ctx, scores, evaluation, load.split)
  process.run()



@click.command(context_settings=dict(token_normalize_func=lambda x: x.lower()))
@common_options.scores_argument(min_arg=2, force_eval=True, nargs=-1)
@common_options.eval_option()
@common_options.table_option()
384
@common_options.criterion_option(lcriteria=['bpcer20', 'eer', 'min-hter'])
385 386 387 388 389 390 391 392 393
@common_options.thresholds_option()
@open_file_mode_option()
@common_options.output_log_metric_option()
@common_options.legends_option()
@verbosity_option()
@click.pass_context
def metrics(ctx, scores, **kwargs):
  """Generate table of metrics for vulnerability PAD

394 395
  You need to provide 4 scores
  files for each vuln system in this order:
396 397 398 399 400 401 402 403 404

  \b
  * licit development scores
  * licit evaluation scores
  * spoof development scores
  * spoof evaluation scores


  Examples:
405
      $ bob vuln vuln_metrics -v {licit,spoof}/scores-{dev,eval}
406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431
  """
  process = figure.MetricsVuln(ctx, scores, True, load.split)
  process.run()



@click.command()
@common_options.scores_argument(min_arg=2, force_eval=True, nargs=-1)
@common_options.output_plot_file_option(default_out='fmr_iapmr.pdf')
@common_options.legends_option()
@common_options.no_legend_option()
@common_options.legend_loc_option()
@common_options.title_option()
@common_options.const_layout_option()
@common_options.style_option()
@common_options.figsize_option()
@verbosity_option()
@common_options.axes_val_option()
@common_options.x_rotation_option()
@common_options.x_label_option()
@common_options.y_label_option()
@common_options.semilogx_option()
@click.pass_context
def fmr_iapmr(ctx, scores, **kwargs):
    """Plot FMR vs IAPMR

432 433
    You need to provide 4 scores
    files for each vuln system in this order:
434 435 436 437

    \b
    * licit development scores
    * licit evaluation scores
438 439
    * spoof development scores
    * spoof evaluation scores
440 441

    Examples:
442
        $ bob vuln fmr_iapmr -v dev-scores eval-scores
443

444
        $ bob vuln fmr_iapmr -v {licit,spoof}/scores-{dev,eval}
445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460
    """
    process = figure.FmrIapmr(ctx, scores, True, load.split)
    process.run()



@click.command()
@common_options.scores_argument(min_arg=2, force_eval=True, nargs=-1)
@common_options.legends_option()
@common_options.sep_dev_eval_option()
@common_options.table_option()
@common_options.output_log_metric_option()
@common_options.output_plot_file_option(default_out='vuln_eval.pdf')
@common_options.points_curve_option()
@common_options.lines_at_option()
@common_options.const_layout_option()
461
@common_options.figsize_option(dflt=None)
462 463 464 465 466 467 468 469
@common_options.style_option()
@common_options.linestyles_option()
@verbosity_option()
@click.pass_context
def evaluate(ctx, scores, **kwargs):
  '''Runs error analysis on score sets for vulnerability studies

  \b
470
  1. Computes bob vuln vuln_metrics
471 472 473 474 475 476 477 478 479 480 481 482 483
  2. Plots EPC, EPSC, vulnerability histograms, fmr vs IAPMR to a multi-page
     PDF file


  You need to provide 4 score files for each biometric system in this order:

  \b
  * licit development scores
  * licit evaluation scores
  * spoof development scores
  * spoof evaluation scores

  Examples:
484
      $ bob vuln evaluate -o my_epsc.pdf dev-scores1 eval-scores1
485

486
      $ bob vuln evaluate -D {licit,spoof}/scores-{dev,eval}
487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509
  '''
  # first time erase if existing file
  click.echo("Computing vuln metrics...")
  ctx.invoke(metrics, scores=scores, evaluation=True)
  if 'log' in ctx.meta and ctx.meta['log'] is not None:
      click.echo("[metrics] => %s" % ctx.meta['log'])

  # avoid closing pdf file before all figures are plotted
  ctx.meta['closef'] = False
  click.echo("Computing histograms...")
  ctx.meta['criterion'] = 'eer'  # no criterion passed in evaluate
  ctx.forward(hist)  # use class defaults plot settings
  click.echo("Computing DET...")
  ctx.forward(det)  # use class defaults plot settings
  click.echo("Computing EPC...")
  ctx.forward(epc)  # use class defaults plot settings
  click.echo("Computing EPSC...")
  ctx.forward(epsc)  # use class defaults plot settings
  click.echo("Computing FMR vs IAPMR...")
  ctx.meta['closef'] = True
  ctx.forward(fmr_iapmr)  # use class defaults plot settings
  click.echo("Vuln successfully completed!")
  click.echo("[plots] => %s" % (ctx.meta['output']))