vuln_commands.py 12.6 KB
Newer Older
1 2 3 4 5 6 7 8
"""The main entry for bob.pad and its(click-based) scripts.
"""

import os
import numpy
import click
from click.types import FLOAT
from bob.measure.script import common_options
9 10
from bob.extension.scripts.click_helper import (
    verbosity_option, bool_option, list_float_option)
11 12 13
from bob.core import random
from bob.io.base import create_directories_safe
from bob.bio.base.score import load
14
from . import vuln_figure as figure
15 16 17 18 19 20

NUM_GENUINE_ACCESS = 5000
NUM_ZEIMPOSTORS = 5000
NUM_PA = 5000


21 22 23 24 25 26 27 28 29
def fnmr_at_option(dflt=' ', **kwargs):
  '''Get option to draw const FNMR lines'''
  return list_float_option(
      name='fnmr', short_name='fnmr',
      desc='If given, draw horizontal lines at the given FNMR position. '
      'Your values must be separated with a comma (,) without space. '
      'This option works in ROC and DET curves.',
      nitems=None, dflt=dflt, **kwargs
  )
30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61


def gen_score_distr(mean_gen, mean_zei, mean_pa, sigma_gen=1, sigma_zei=1,
                    sigma_pa=1):
  mt = random.mt19937()  # initialise the random number generator

  genuine_generator = random.normal(numpy.float32, mean_gen, sigma_gen)
  zei_generator = random.normal(numpy.float32, mean_zei, sigma_zei)
  pa_generator = random.normal(numpy.float32, mean_pa, sigma_pa)

  genuine_scores = [genuine_generator(mt) for i in range(NUM_GENUINE_ACCESS)]
  zei_scores = [zei_generator(mt) for i in range(NUM_ZEIMPOSTORS)]
  pa_scores = [pa_generator(mt) for i in range(NUM_PA)]

  return genuine_scores, zei_scores, pa_scores


def write_scores_to_file(neg, pos, filename, attack=False):
  """Writes score distributions into 4-column score files. For the format of
    the 4-column score files, please refer to Bob's documentation.

  Parameters
  ----------
  neg : array_like
      Scores for negative samples.
  pos : array_like
      Scores for positive samples.
  filename : str
      The path to write the score to.
  """
  create_directories_safe(os.path.dirname(filename))
  with open(filename, 'wt') as f:
62 63 64 65 66 67 68
    for i in pos:
      f.write('x x foo %f\n' % i)
    for i in neg:
      if attack:
        f.write('x attack foo %f\n' % i)
      else:
        f.write('x y foo %f\n' % i)
69 70 71 72


@click.command()
@click.argument('outdir')
73 74
@click.option('--mean-gen', default=7, type=FLOAT, show_default=True)
@click.option('--mean-zei', default=3, type=FLOAT, show_default=True)
75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91
@click.option('--mean-pa', default=5, type=FLOAT, show_default=True)
@verbosity_option()
def gen(outdir, mean_gen, mean_zei, mean_pa):
  """Generate random scores.
  Generates random scores for three types of verification attempts:
  genuine users, zero-effort impostors and spoofing attacks and writes them
  into 4-column score files for so called licit and spoof scenario. The
  scores are generated using Gaussian distribution whose mean is an input
  parameter. The generated scores can be used as hypothetical datasets.
  """
  # Generate the data
  genuine_dev, zei_dev, pa_dev = gen_score_distr(
      mean_gen, mean_zei, mean_pa)
  genuine_eval, zei_eval, pa_eval = gen_score_distr(
      mean_gen, mean_zei, mean_pa)

  # Write the data into files
92
  write_scores_to_file(zei_dev, genuine_dev,
93
                       os.path.join(outdir, 'licit', 'scores-dev'))
94
  write_scores_to_file(zei_eval, genuine_eval,
95
                       os.path.join(outdir, 'licit', 'scores-eval'))
96
  write_scores_to_file(pa_dev, genuine_dev,
97 98
                       os.path.join(outdir, 'spoof', 'scores-dev'),
                       attack=True)
99
  write_scores_to_file(pa_eval, genuine_eval,
100 101 102 103
                       os.path.join(outdir, 'spoof', 'scores-eval'),
                       attack=True)


104
@click.command()
105
@common_options.scores_argument(min_arg=2, nargs=-1)
106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122
@common_options.output_plot_file_option(default_out='vuln_roc.pdf')
@common_options.legends_option()
@common_options.no_legend_option()
@common_options.legend_loc_option(dflt='upper-right')
@common_options.title_option()
@common_options.const_layout_option()
@common_options.style_option()
@common_options.figsize_option(dflt=None)
@common_options.min_far_option()
@common_options.axes_val_option()
@verbosity_option()
@common_options.x_rotation_option(dflt=45)
@common_options.x_label_option()
@common_options.y_label_option()
@click.option('--real-data/--no-real-data', default=True, show_default=True,
              help='If False, will annotate the plots hypothetically, instead '
              'of with real data values of the calculated error rates.')
123
@fnmr_at_option()
124
@click.pass_context
125
def roc(ctx, scores, real_data, **kwargs):
126 127
  """Plot ROC

128
  You need to provide 2 scores
129
  files for each vulnerability system in this order:
130 131

  \b
132 133
  * licit scores
  * spoof scores
134 135

  Examples:
136
      $ bob vuln roc -v licit-scores spoof-scores
137

138
      $ bob vuln roc -v scores-{licit,spoof}
139
  """
140
  process = figure.RocVuln(ctx, scores, True, load.split, real_data, False)
141 142 143
  process.run()


144
@click.command()
145
@common_options.scores_argument(min_arg=2, nargs=-1)
146 147 148 149 150 151 152
@common_options.output_plot_file_option(default_out='vuln_det.pdf')
@common_options.legends_option()
@common_options.no_legend_option()
@common_options.legend_loc_option(dflt='upper-right')
@common_options.title_option()
@common_options.const_layout_option()
@common_options.style_option()
153
@common_options.figsize_option(dflt=None)
154 155 156 157 158 159 160 161
@verbosity_option()
@common_options.axes_val_option(dflt='0.01,95,0.01,95')
@common_options.x_rotation_option(dflt=45)
@common_options.x_label_option()
@common_options.y_label_option()
@click.option('--real-data/--no-real-data', default=True, show_default=True,
              help='If False, will annotate the plots hypothetically, instead '
              'of with real data values of the calculated error rates.')
162
@fnmr_at_option()
163
@click.pass_context
164
def det(ctx, scores, real_data, **kwargs):
165 166
  """Plot DET

167 168 169

  You need to provide 2 scores
  files for each vulnerability system in this order:
170 171

  \b
172 173
  * licit scores
  * spoof scores
174 175

  Examples:
176
      $ bob vuln det -v licit-scores spoof-scores
177

178
      $ bob vuln det -v scores-{licit,spoof}
179
  """
180
  process = figure.DetVuln(ctx, scores, True, load.split, real_data, False)
181 182 183 184 185 186 187 188 189 190 191 192 193
  process.run()


@click.command()
@common_options.scores_argument(min_arg=2, force_eval=True, nargs=-1)
@common_options.output_plot_file_option(default_out='vuln_epc.pdf')
@common_options.legends_option()
@common_options.no_legend_option()
@common_options.legend_loc_option()
@common_options.title_option()
@common_options.const_layout_option()
@common_options.x_label_option()
@common_options.y_label_option()
194
@common_options.figsize_option(dflt=None)
195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217
@common_options.style_option()
@common_options.bool_option(
    'iapmr', 'I', 'Whether to plot the IAPMR related lines or not.', True
)
@common_options.style_option()
@verbosity_option()
@click.pass_context
def epc(ctx, scores, **kwargs):
  """Plot EPC (expected performance curve):

  You need to provide 4 score
  files for each biometric system in this order:

  \b
  * licit development scores
  * licit evaluation scores
  * spoof development scores
  * spoof evaluation scores

  See :ref:`bob.pad.base.vulnerability` in the documentation for a guide on
  vulnerability analysis.

  Examples:
218
      $ bob vuln epc -v dev-scores eval-scores
219

220
      $ bob vuln epc -v -o my_epc.pdf dev-scores1 eval-scores1
221

222
      $ bob vuln epc -v {licit,spoof}/scores-{dev,eval}
223 224 225 226 227 228 229 230 231 232 233 234 235 236
  """
  process = figure.Epc(ctx, scores, True, load.split)
  process.run()


@click.command()
@common_options.scores_argument(min_arg=2, force_eval=True, nargs=-1)
@common_options.output_plot_file_option(default_out='vuln_epsc.pdf')
@common_options.legends_option()
@common_options.no_legend_option()
@common_options.legend_loc_option()
@common_options.const_layout_option()
@common_options.x_label_option()
@common_options.y_label_option()
237
@common_options.figsize_option(dflt=None)
238 239 240 241 242 243 244 245 246 247 248 249
@common_options.style_option()
@common_options.bool_option(
    'wer', 'w', 'Whether to plot the WER related lines or not.', True
)
@common_options.bool_option(
    'three-d', 'D', 'If true, generate 3D plots', False
)
@common_options.bool_option(
    'iapmr', 'I', 'Whether to plot the IAPMR related lines or not.', False
)
@click.option('-c', '--criteria', default="eer", show_default=True,
              help='Criteria for threshold selection',
250
              type=click.Choice(('eer', 'min-hter')))
251 252 253 254 255 256
@click.option('-vp', '--var-param', default="omega", show_default=True,
              help='Name of the varying parameter',
              type=click.Choice(('omega', 'beta')))
@click.option('-fp', '--fixed-param', default=0.5, show_default=True,
              help='Value of the fixed parameter',
              type=click.FLOAT)
257 258
@click.option('-s', '--sampling', default=5, show_default=True,
              help='Sampling of the EPSC 3D surface', type=click.INT)
259 260
@verbosity_option()
@click.pass_context
261 262
def epsc(ctx, scores, criteria, var_param, fixed_param, three_d, sampling,
         **kwargs):
263
  """Plot EPSC (expected performance spoofing curve):
264

265 266
  You need to provide 4 score
  files for each biometric system in this order:
267

268 269 270 271 272
  \b
  * licit development scores
  * licit evaluation scores
  * spoof development scores
  * spoof evaluation scores
273

274 275
  See :ref:`bob.pad.base.vulnerability` in the documentation for a guide on
  vulnerability analysis.
276

277 278
  Note that when using 3D plots with option ``--three-d``, you cannot plot
  both WER and IAPMR on the same figure (which is possible in 2D).
279

280 281
  Examples:
      $ bob vuln epsc -v -o my_epsc.pdf dev-scores1 eval-scores1
282

283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298
      $ bob vuln epsc -v -D {licit,spoof}/scores-{dev,eval}
  """
  if three_d:
    if (ctx.meta['wer'] and ctx.meta['iapmr']):
      raise click.BadParameter('Cannot plot both WER and IAPMR in 3D')
    ctx.meta['sampling'] = sampling
    process = figure.Epsc3D(
        ctx, scores, True, load.split,
        criteria, var_param, fixed_param
    )
  else:
    process = figure.Epsc(
        ctx, scores, True, load.split,
        criteria, var_param, fixed_param
    )
  process.run()
299 300 301


@click.command()
302
@common_options.scores_argument(nargs=-1, min_arg=2)
303 304 305 306 307 308 309 310 311 312 313 314 315
@common_options.output_plot_file_option(default_out='vuln_hist.pdf')
@common_options.n_bins_option()
@common_options.criterion_option()
@common_options.thresholds_option()
@common_options.print_filenames_option(dflt=False)
@bool_option(
    'iapmr-line', 'I', 'Whether to plot the IAPMR related lines or not.', True
)
@bool_option(
    'real-data', 'R',
    'If False, will annotate the plots hypothetically, instead '
    'of with real data values of the calculated error rates.', True
)
316
@common_options.titles_option()
317
@common_options.const_layout_option()
318 319 320 321
@common_options.figsize_option(dflt=None)
@common_options.subplot_option()
@common_options.legend_ncols_option()
@common_options.style_option()
322
@common_options.hide_dev_option()
323
@common_options.eval_option()
324 325
@verbosity_option()
@click.pass_context
326
def hist(ctx, scores, evaluation, **kwargs):
327 328
  '''Vulnerability analysis distributions.

329 330 331
  Plots the histogram of score distributions. You need to provide 2 or 4 score
  files for each biometric system in this order.
  When evaluation scores are provided, you must use the ``--eval`` option.
332 333 334

  \b
  * licit development scores
335
  * (optional) licit evaluation scores
336
  * spoof development scores
337
  * (optional) spoof evaluation scores
338 339 340 341 342 343 344

  See :ref:`bob.pad.base.vulnerability` in the documentation for a guide on
  vulnerability analysis.


  By default, when eval-scores are given, only eval-scores histograms are
  displayed with threshold line
345
  computed from dev-scores.
346 347 348

  Examples:

349
      $ bob vuln hist -e -v licit/scores-dev licit/scores-eval \
350 351
                          spoof/scores-dev spoof/scores-eval

352
      $ bob vuln hist -e -v {licit,spoof}/scores-{dev,eval}
353
  '''
354
  process = figure.HistVuln(ctx, scores, evaluation, load.split)
355 356 357 358 359 360 361 362 363 364 365 366
  process.run()


@click.command()
@common_options.scores_argument(min_arg=2, force_eval=True, nargs=-1)
@common_options.output_plot_file_option(default_out='fmr_iapmr.pdf')
@common_options.legends_option()
@common_options.no_legend_option()
@common_options.legend_loc_option()
@common_options.title_option()
@common_options.const_layout_option()
@common_options.style_option()
367
@common_options.figsize_option(dflt=None)
368 369 370 371 372 373 374 375
@verbosity_option()
@common_options.axes_val_option()
@common_options.x_rotation_option()
@common_options.x_label_option()
@common_options.y_label_option()
@common_options.semilogx_option()
@click.pass_context
def fmr_iapmr(ctx, scores, **kwargs):
376
  """Plot FMR vs IAPMR
377

378 379
  You need to provide 4 scores
  files for each vuln system in this order:
380 381 382 383 384 385 386 387

  \b
  * licit development scores
  * licit evaluation scores
  * spoof development scores
  * spoof evaluation scores

  Examples:
388
      $ bob vuln fmr_iapmr -v dev-scores eval-scores
389

390 391 392 393
      $ bob vuln fmr_iapmr -v {licit,spoof}/scores-{dev,eval}
  """
  process = figure.FmrIapmr(ctx, scores, True, load.split)
  process.run()