vuln_commands.py 12.9 KB
Newer Older
1 2 3 4
"""The main entry for bob.pad and its(click-based) scripts.
"""

import os
5
import logging
6 7 8 9
import numpy
import click
from click.types import FLOAT
from bob.measure.script import common_options
10
from bob.extension.scripts.click_helper import (
11
    verbosity_option, bool_option, list_float_option
12
)
13 14 15
from bob.core import random
from bob.io.base import create_directories_safe
from bob.bio.base.score import load
16
from . import vuln_figure as figure
17

18
LOGGER = logging.getLogger(__name__)
19 20 21 22 23
NUM_GENUINE_ACCESS = 5000
NUM_ZEIMPOSTORS = 5000
NUM_PA = 5000


24 25 26 27 28 29 30 31 32
def fnmr_at_option(dflt=' ', **kwargs):
  '''Get option to draw const FNMR lines'''
  return list_float_option(
      name='fnmr', short_name='fnmr',
      desc='If given, draw horizontal lines at the given FNMR position. '
      'Your values must be separated with a comma (,) without space. '
      'This option works in ROC and DET curves.',
      nitems=None, dflt=dflt, **kwargs
  )
33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64


def gen_score_distr(mean_gen, mean_zei, mean_pa, sigma_gen=1, sigma_zei=1,
                    sigma_pa=1):
  mt = random.mt19937()  # initialise the random number generator

  genuine_generator = random.normal(numpy.float32, mean_gen, sigma_gen)
  zei_generator = random.normal(numpy.float32, mean_zei, sigma_zei)
  pa_generator = random.normal(numpy.float32, mean_pa, sigma_pa)

  genuine_scores = [genuine_generator(mt) for i in range(NUM_GENUINE_ACCESS)]
  zei_scores = [zei_generator(mt) for i in range(NUM_ZEIMPOSTORS)]
  pa_scores = [pa_generator(mt) for i in range(NUM_PA)]

  return genuine_scores, zei_scores, pa_scores


def write_scores_to_file(neg, pos, filename, attack=False):
  """Writes score distributions into 4-column score files. For the format of
    the 4-column score files, please refer to Bob's documentation.

  Parameters
  ----------
  neg : array_like
      Scores for negative samples.
  pos : array_like
      Scores for positive samples.
  filename : str
      The path to write the score to.
  """
  create_directories_safe(os.path.dirname(filename))
  with open(filename, 'wt') as f:
65 66 67 68 69 70 71
    for i in pos:
      f.write('x x foo %f\n' % i)
    for i in neg:
      if attack:
        f.write('x attack foo %f\n' % i)
      else:
        f.write('x y foo %f\n' % i)
72 73


74
@click.command()
75
@click.argument('outdir')
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
76 77 78
@click.option('-mg', '--mean-gen', default=7, type=FLOAT, show_default=True)
@click.option('-mz', '--mean-zei', default=3, type=FLOAT, show_default=True)
@click.option('-mp', '--mean-pa', default=5, type=FLOAT, show_default=True)
79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94
@verbosity_option()
def gen(outdir, mean_gen, mean_zei, mean_pa):
  """Generate random scores.
  Generates random scores for three types of verification attempts:
  genuine users, zero-effort impostors and spoofing attacks and writes them
  into 4-column score files for so called licit and spoof scenario. The
  scores are generated using Gaussian distribution whose mean is an input
  parameter. The generated scores can be used as hypothetical datasets.
  """
  # Generate the data
  genuine_dev, zei_dev, pa_dev = gen_score_distr(
      mean_gen, mean_zei, mean_pa)
  genuine_eval, zei_eval, pa_eval = gen_score_distr(
      mean_gen, mean_zei, mean_pa)

  # Write the data into files
95
  write_scores_to_file(zei_dev, genuine_dev,
96
                       os.path.join(outdir, 'licit', 'scores-dev'))
97
  write_scores_to_file(zei_eval, genuine_eval,
98
                       os.path.join(outdir, 'licit', 'scores-eval'))
99
  write_scores_to_file(pa_dev, genuine_dev,
100 101
                       os.path.join(outdir, 'spoof', 'scores-dev'),
                       attack=True)
102
  write_scores_to_file(pa_eval, genuine_eval,
103 104 105 106
                       os.path.join(outdir, 'spoof', 'scores-eval'),
                       attack=True)


107
@click.command()
108
@common_options.scores_argument(min_arg=2, nargs=-1)
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
109
@common_options.output_plot_file_option(default_out='roc.pdf')
110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125
@common_options.legends_option()
@common_options.no_legend_option()
@common_options.legend_loc_option(dflt='upper-right')
@common_options.title_option()
@common_options.const_layout_option()
@common_options.style_option()
@common_options.figsize_option(dflt=None)
@common_options.min_far_option()
@common_options.axes_val_option()
@verbosity_option()
@common_options.x_rotation_option(dflt=45)
@common_options.x_label_option()
@common_options.y_label_option()
@click.option('--real-data/--no-real-data', default=True, show_default=True,
              help='If False, will annotate the plots hypothetically, instead '
              'of with real data values of the calculated error rates.')
126
@fnmr_at_option()
127
@click.pass_context
128
def roc(ctx, scores, real_data, **kwargs):
129 130
  """Plot ROC

131
  You need to provide 2 scores
132
  files for each vulnerability system in this order:
133 134

  \b
135 136
  * licit scores
  * spoof scores
137 138

  Examples:
139
      $ bob vuln roc -v licit-scores spoof-scores
140

141
      $ bob vuln roc -v scores-{licit,spoof}
142
  """
143
  process = figure.RocVuln(ctx, scores, True, load.split, real_data, False)
144 145 146
  process.run()


147
@click.command()
148
@common_options.scores_argument(min_arg=2, nargs=-1)
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
149
@common_options.output_plot_file_option(default_out='det.pdf')
150 151 152 153 154 155
@common_options.legends_option()
@common_options.no_legend_option()
@common_options.legend_loc_option(dflt='upper-right')
@common_options.title_option()
@common_options.const_layout_option()
@common_options.style_option()
156
@common_options.figsize_option(dflt=None)
157 158 159 160 161 162 163 164
@verbosity_option()
@common_options.axes_val_option(dflt='0.01,95,0.01,95')
@common_options.x_rotation_option(dflt=45)
@common_options.x_label_option()
@common_options.y_label_option()
@click.option('--real-data/--no-real-data', default=True, show_default=True,
              help='If False, will annotate the plots hypothetically, instead '
              'of with real data values of the calculated error rates.')
165
@fnmr_at_option()
166
@click.pass_context
167
def det(ctx, scores, real_data, **kwargs):
168 169
  """Plot DET

170 171 172

  You need to provide 2 scores
  files for each vulnerability system in this order:
173 174

  \b
175 176
  * licit scores
  * spoof scores
177 178

  Examples:
179
      $ bob vuln det -v licit-scores spoof-scores
180

181
      $ bob vuln det -v scores-{licit,spoof}
182
  """
183
  process = figure.DetVuln(ctx, scores, True, load.split, real_data, False)
184 185 186
  process.run()


187
@click.command()
188
@common_options.scores_argument(min_arg=2, force_eval=True, nargs=-1)
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
189
@common_options.output_plot_file_option(default_out='epc.pdf')
190 191 192 193 194 195 196
@common_options.legends_option()
@common_options.no_legend_option()
@common_options.legend_loc_option()
@common_options.title_option()
@common_options.const_layout_option()
@common_options.x_label_option()
@common_options.y_label_option()
197
@common_options.figsize_option(dflt=None)
198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220
@common_options.style_option()
@common_options.bool_option(
    'iapmr', 'I', 'Whether to plot the IAPMR related lines or not.', True
)
@common_options.style_option()
@verbosity_option()
@click.pass_context
def epc(ctx, scores, **kwargs):
  """Plot EPC (expected performance curve):

  You need to provide 4 score
  files for each biometric system in this order:

  \b
  * licit development scores
  * licit evaluation scores
  * spoof development scores
  * spoof evaluation scores

  See :ref:`bob.pad.base.vulnerability` in the documentation for a guide on
  vulnerability analysis.

  Examples:
221
      $ bob vuln epc -v dev-scores eval-scores
222

223
      $ bob vuln epc -v -o my_epc.pdf dev-scores1 eval-scores1
224

225
      $ bob vuln epc -v {licit,spoof}/scores-{dev,eval}
226 227 228 229 230
  """
  process = figure.Epc(ctx, scores, True, load.split)
  process.run()


231
@click.command()
232
@common_options.scores_argument(min_arg=2, force_eval=True, nargs=-1)
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
233
@common_options.output_plot_file_option(default_out='epsc.pdf')
234
@common_options.titles_option()
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
235
@common_options.legends_option()
236
@common_options.no_legend_option()
Theophile GENTILHOMME's avatar
Theophile GENTILHOMME committed
237
@common_options.legend_ncols_option()
238 239 240
@common_options.const_layout_option()
@common_options.x_label_option()
@common_options.y_label_option()
241
@common_options.figsize_option(dflt='5,3')
242 243 244 245 246
@common_options.style_option()
@common_options.bool_option(
    'wer', 'w', 'Whether to plot the WER related lines or not.', True
)
@common_options.bool_option(
247 248
    'three-d', 'D', 'If true, generate 3D plots. You need to turn off '
    'wer or iapmr when using this option.', False
249 250
)
@common_options.bool_option(
251
    'iapmr', 'I', 'Whether to plot the IAPMR related lines or not.', True
252 253 254
)
@click.option('-c', '--criteria', default="eer", show_default=True,
              help='Criteria for threshold selection',
255
              type=click.Choice(('eer', 'min-hter')))
256 257 258
@click.option('-vp', '--var-param', default="omega", show_default=True,
              help='Name of the varying parameter',
              type=click.Choice(('omega', 'beta')))
259 260
@list_float_option(name='fixed-params', short_name='fp', dflt='0.5',
                   desc='Values of the fixed parameter, separated by commas')
261 262
@click.option('-s', '--sampling', default=5, show_default=True,
              help='Sampling of the EPSC 3D surface', type=click.INT)
263 264
@verbosity_option()
@click.pass_context
265
def epsc(ctx, scores, criteria, var_param, three_d, sampling,
266
         **kwargs):
267
  """Plot EPSC (expected performance spoofing curve):
268

269 270
  You need to provide 4 score
  files for each biometric system in this order:
271

272 273 274 275 276
  \b
  * licit development scores
  * licit evaluation scores
  * spoof development scores
  * spoof evaluation scores
277

278 279
  See :ref:`bob.pad.base.vulnerability` in the documentation for a guide on
  vulnerability analysis.
280

281 282
  Note that when using 3D plots with option ``--three-d``, you cannot plot
  both WER and IAPMR on the same figure (which is possible in 2D).
283

284 285
  Examples:
      $ bob vuln epsc -v -o my_epsc.pdf dev-scores1 eval-scores1
286

287 288
      $ bob vuln epsc -v -D {licit,spoof}/scores-{dev,eval}
  """
289
  fixed_params = ctx.meta.get('fixed_params', [0.5])
290 291
  if three_d:
    if (ctx.meta['wer'] and ctx.meta['iapmr']):
292 293
      LOGGER.info('Cannot plot both WER and IAPMR in 3D. Will turn IAPMR off.')
      ctx.meta['iapmr'] = False
294 295 296
    ctx.meta['sampling'] = sampling
    process = figure.Epsc3D(
        ctx, scores, True, load.split,
297
        criteria, var_param, fixed_params
298 299 300 301
    )
  else:
    process = figure.Epsc(
        ctx, scores, True, load.split,
302
        criteria, var_param, fixed_params
303 304
    )
  process.run()
305 306


307
@click.command()
308
@common_options.scores_argument(nargs=-1, min_arg=2)
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
309
@common_options.output_plot_file_option(default_out='hist.pdf')
310 311 312 313 314 315 316 317 318 319 320 321
@common_options.n_bins_option()
@common_options.criterion_option()
@common_options.thresholds_option()
@common_options.print_filenames_option(dflt=False)
@bool_option(
    'iapmr-line', 'I', 'Whether to plot the IAPMR related lines or not.', True
)
@bool_option(
    'real-data', 'R',
    'If False, will annotate the plots hypothetically, instead '
    'of with real data values of the calculated error rates.', True
)
322
@common_options.titles_option()
323
@common_options.const_layout_option()
324 325 326 327
@common_options.figsize_option(dflt=None)
@common_options.subplot_option()
@common_options.legend_ncols_option()
@common_options.style_option()
328
@common_options.hide_dev_option()
329
@common_options.eval_option()
330 331
@verbosity_option()
@click.pass_context
332
def hist(ctx, scores, evaluation, **kwargs):
333 334
  '''Vulnerability analysis distributions.

335 336 337
  Plots the histogram of score distributions. You need to provide 2 or 4 score
  files for each biometric system in this order.
  When evaluation scores are provided, you must use the ``--eval`` option.
338 339 340

  \b
  * licit development scores
341
  * (optional) licit evaluation scores
342
  * spoof development scores
343
  * (optional) spoof evaluation scores
344 345 346 347 348 349 350

  See :ref:`bob.pad.base.vulnerability` in the documentation for a guide on
  vulnerability analysis.


  By default, when eval-scores are given, only eval-scores histograms are
  displayed with threshold line
351
  computed from dev-scores.
352 353 354

  Examples:

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
355 356
      $ bob vuln hist -v licit/scores-dev spoof/scores-dev

357
      $ bob vuln hist -e -v licit/scores-dev licit/scores-eval \
358 359
                          spoof/scores-dev spoof/scores-eval

360
      $ bob vuln hist -e -v {licit,spoof}/scores-{dev,eval}
361
  '''
362
  process = figure.HistVuln(ctx, scores, evaluation, load.split)
363 364 365
  process.run()


366
@click.command()
367 368 369 370 371 372 373 374
@common_options.scores_argument(min_arg=2, force_eval=True, nargs=-1)
@common_options.output_plot_file_option(default_out='fmr_iapmr.pdf')
@common_options.legends_option()
@common_options.no_legend_option()
@common_options.legend_loc_option()
@common_options.title_option()
@common_options.const_layout_option()
@common_options.style_option()
375
@common_options.figsize_option()
376 377 378 379 380 381 382 383
@verbosity_option()
@common_options.axes_val_option()
@common_options.x_rotation_option()
@common_options.x_label_option()
@common_options.y_label_option()
@common_options.semilogx_option()
@click.pass_context
def fmr_iapmr(ctx, scores, **kwargs):
384
  """Plot FMR vs IAPMR
385

386 387
  You need to provide 4 scores
  files for each vuln system in this order:
388 389 390 391 392 393 394 395

  \b
  * licit development scores
  * licit evaluation scores
  * spoof development scores
  * spoof evaluation scores

  Examples:
396
      $ bob vuln fmr_iapmr -v dev-scores eval-scores
397

398 399 400 401
      $ bob vuln fmr_iapmr -v {licit,spoof}/scores-{dev,eval}
  """
  process = figure.FmrIapmr(ctx, scores, True, load.split)
  process.run()