baselines.py 13.2 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
#!../bin/python
from __future__ import print_function

import subprocess
import os
import sys
import argparse

import bob.bio.base

import bob.core
logger = bob.core.log.setup("bob.bio.face")

# This is the default set of algorithms that can be run using this script.
all_databases = bob.bio.base.resource_keys('database')
# check, which databases can actually be assessed
available_databases = []

for database in all_databases:
  try:
    bob.bio.base.load_resource(database, 'database')
    available_databases.append(database)
  except:
    pass

# collect all algorithms that we provide baselines for
all_algorithms = ['eigenface', 'lda', 'gabor-graph', 'lgbphs', 'plda', 'bic']

try:
  # try if GMM-based algorithms are available
  bob.bio.base.load_resource('gmm', 'algorithm')
  bob.bio.base.load_resource('isv', 'algorithm')
  bob.bio.base.load_resource('ivector', 'algorithm')
  all_algorithms += ['gmm', 'isv', 'ivector']
except:
  print("Could not load the GMM-based algorithms. Did you specify bob.bio.gmm in your config file?")

try:
  # try if the CSU extension is enabled
  bob.bio.base.load_resource('lrpca', 'algorithm')
  bob.bio.base.load_resource('lda-ir', 'algorithm')
  all_algorithms += ['lrpca', 'lda-ir']
except:
  print("Could not load the algorithms from the CSU resources. Did you specify bob.bio.csu in your config file?")


def command_line_arguments(command_line_parameters):
  """Defines the command line parameters that are accepted."""

  # create parser
  parser = argparse.ArgumentParser(description='Execute baseline algorithms with default parameters', formatter_class=argparse.ArgumentDefaultsHelpFormatter)

  # add parameters
  # - the algorithm to execute
  parser.add_argument('-a', '--algorithms', choices = all_algorithms, default = ('eigenface',), nargs = '+', help = 'Select one (or more) algorithms that you want to execute.')
  parser.add_argument('--all', action = 'store_true', help = 'Select all algorithms.')
  # - the database to choose
  parser.add_argument('-d', '--database', choices = available_databases, default = 'atnt', help = 'The database on which the baseline algorithm is executed.')
  # - the database to choose
  parser.add_argument('-b', '--baseline-directory', default = 'baselines', help = 'The sub-directory, where the baseline results are stored.')
  # - the directory to write
  parser.add_argument('-f', '--directory', help = 'The directory to write the data of the experiment into. If not specified, the default directories of the verify.py script are used (see ./bin/verify.py --help).')

  # - use the Idiap grid -- option is only useful if you are at Idiap
  parser.add_argument('-g', '--grid', action = 'store_true', help = 'Execute the algorithm in the SGE grid.')
  # - run in parallel on the local machine
  parser.add_argument('-l', '--parallel', type=int, help = 'Run the algorithms in parallel on the local machine, using the given number of parallel threads')
  # - perform ZT-normalization
  parser.add_argument('-z', '--zt-norm', action = 'store_true', help = 'Compute the ZT norm for the files (might not be availabe for all databases).')

  # - just print?
  parser.add_argument('-q', '--dry-run', action = 'store_true', help = 'Just print the commands, but do not execute them.')

  # - evaluate the algorithm (after it has finished)
  parser.add_argument('-e', '--evaluate', nargs='+', choices = ('EER', 'HTER', 'ROC', 'DET', 'CMC', 'RR'), help = 'Evaluate the results of the algorithms (instead of running them) using the given evaluation techniques.')

  # - other parameters that are passed to the underlying script
  parser.add_argument('parameters', nargs = argparse.REMAINDER, help = 'Parameters directly passed to the ./bin/verify.py script.')

  bob.core.log.add_command_line_option(parser)
  args = parser.parse_args(command_line_parameters)
  if args.all:
    args.algorithms = all_algorithms

  bob.core.log.set_verbosity_level(logger, args.verbose)

  return args


# In these functions, some default experiments are prepared.
# An experiment consists of three configuration files:
# - The features to be extracted
# - The algorithm to be run
# - The grid configuration that it requires (only used when the --grid option is chosen)

CONFIGURATIONS = {
  'eigenface' : dict(
    preprocessor = ('face-crop-eyes', 'base'),
    extractor    = 'linearize',
    algorithm    = 'pca',
  ),

  'lda': dict(
    preprocessor = ('face-crop-eyes', 'base'),
    extractor    = 'eigenface',
    algorithm    = 'lda',
  ),

  'plda': dict(
    preprocessor = ('face-crop-eyes', 'base'),
    extractor    = 'linearize',
    algorithm    = 'pca+plda',
    grid         = 'demanding'
  ),

  'gabor-graph': dict(
    preprocessor = ('inorm-lbp-crop', 'inorm-lbp'),
    extractor    = 'grid-graph',
    algorithm    = 'gabor-jet',
  ),

  'lgbphs': dict(
    preprocessor = ('tan-triggs-crop', 'tan-triggs'),
    extractor    = 'lgbphs',
    algorithm    = 'lgbphs',
  ),

  'bic': dict(
    preprocessor = ('face-crop-eyes', 'base'),
    extractor    = 'grid-graph',
    algorithm    = 'bic-jets',
    grid         = 'demanding'
  ),

  'gmm': dict(
    preprocessor = ('tan-triggs-crop', 'tan-triggs'),
    extractor    = 'dct-blocks',
    algorithm    = 'gmm',
    grid         = 'demanding',
    script       = './bin/verify_gmm.py'
  ),

  'isv': dict(
    preprocessor = ('tan-triggs-crop', 'tan-triggs'),
    extractor    = 'dct-blocks',
    algorithm    = 'isv',
    grid         = 'demanding',
    script       = './bin/verify_isv.py'
  ),

  'ivector': dict(
    preprocessor = ('tan-triggs-crop', 'tan-triggs'),
    extractor    = 'dct-blocks',
    algorithm    = 'ivector',
    grid         = 'demanding',
    script       = './bin/verify_ivector.py'
  ),

  'lrpca': dict(
    preprocessor = ('lrpca', None),
    extractor    = 'lrpca',
    algorithm    = 'lrpca'
  ),

  'lda-ir': dict(
    preprocessor = ('lda-ir', None),
    extractor    = 'lda-ir',
    algorithm    = 'lda-ir'
  )
}

def main(command_line_parameters = None):

  # Collect command line arguments
  args = command_line_arguments(command_line_parameters)

  # Check the database configuration file
  has_eyes = args.database != 'atnt'
  has_zt_norm = args.database in ('banca', 'mobio', 'multipie', 'scface')
  has_eval = args.database in ('banca', 'mobio', 'multipie', 'scface', 'xm2vts')

  if not args.evaluate:

    # execution of the job is requested
    for algorithm in args.algorithms:
      logger.info("Executing algorithm '%s'", algorithm)

      # get the setup for the desired algorithm
      import copy
      setup = copy.deepcopy(CONFIGURATIONS[algorithm])
      if 'grid' not in setup: setup['grid'] = 'grid'
      if 'script' not in setup or (not args.grid and args.parallel is None): setup['script'] = './bin/verify.py'

      # select the preprocessor
      setup['preprocessor'] = setup['preprocessor'][0 if has_eyes else 1]
      if setup['preprocessor'] is None:
        logger.warn("Skipping algorithm '%s' since no preprocessor is found that matches the given databases' '%s' configuration", algorithm, args.database)


      # this is the default sub-directory that is used
      sub_directory = os.path.join(args.baseline_directory, algorithm)

      # create the command to the faceverify script
      command = [
          setup['script'],
          '--database', args.database,
          '--preprocessor', setup['preprocessor'],
          '--extractor', setup['extractor'],
          '--algorithm', setup['algorithm'],
          '--sub-directory', sub_directory
      ]

      # add grid argument, if available
      if args.grid:
        command += ['--grid', setup['grid'], '--stop-on-failure']

      if args.parallel is not None:
        command += ['--grid', 'bob.bio.base.grid.Grid("local", number_of_parallel_processes=%d)' % args.parallel, '--run-local-scheduler', '--stop-on-failure']

      # compute ZT-norm if the database provides this setup
      if has_zt_norm and args.zt_norm:
        command += ['--zt-norm']

      # compute results for both 'dev' and 'eval' group if the database provides these
      if has_eval:
        command += ['--groups', 'dev', 'eval']

      # set the directories, if desired; we set both directories to be identical.
      if args.directory is not None:
        command += ['--temp-directory', os.path.join(args.directory, args.database), '--result-directory', os.path.join(args.directory, args.database)]

      # set the verbosity level
      if args.verbose:
        command += ['-' + 'v'*args.verbose]

      # add the command line arguments that were specified on command line
      if args.parameters:
        command += args.parameters[1:]

      # print the command so that it can easily be re-issued
      logger.info("Executing command:\n%s", bob.bio.base.tools.command_line(command))

#      import ipdb; ipdb.set_trace()
      # run the command
      if not args.dry_run:
        subprocess.call(command)

  else:
    # call the evaluate script with the desired parameters

    # get the base directory of the results
    is_idiap = os.path.isdir("/idiap")
    if args.directory is None:
      args.directory = "/idiap/user/%s/%s" % (os.environ["USER"], args.database) if is_idiap else "results"
    if not os.path.exists(args.directory):
      if not args.dry_run:
        raise IOError("The result directory '%s' cannot be found. Please specify the --directory as it was specified during execution of the algorithms." % args.directory)

    # get the result directory of the database
    result_dir = os.path.join(args.directory, args.baseline_directory)
    if not os.path.exists(result_dir):
      if not args.dry_run:
        raise IOError("The result directory '%s' for the desired database cannot be found. Did you already run the experiments for this database?" % result_dir)

    # iterate over the algorithms and collect the result files
    result_dev = []
    result_eval = []
    result_zt_dev = []
    result_zt_eval = []
    legends = []

    # evaluate the results
    for algorithm in args.algorithms:
      if not os.path.exists(os.path.join(result_dir, algorithm)):
        logger.warn("Skipping algorithm '%s' since the results cannot be found.", algorithm)
        continue
      protocols = [d for d in os.listdir(os.path.join(result_dir, algorithm)) if os.path.isdir(os.path.join(result_dir, algorithm, d))]
      if not len(protocols):
        logger.warn("Skipping algorithm '%s' since the results cannot be found.", algorithm)
        continue
      if len(protocols) > 1:
282
283
284
285
286
287
288
289
290
291
        # load the default protocol of the database
        protocol = bob.bio.base.load_resource(args.database, "database").protocol
        if protocol not in protocols:
          protocol = protocols[0]
          logger.warn("There are several protocols found in directory '%s'. Here, we use protocol '%s'.", os.path.join(result_dir, algorithm), protocols[0])
      else:
        protocol = protocols[0]

      nonorm_sub_dir = os.path.join(algorithm, protocol, 'nonorm')
      ztnorm_sub_dir = os.path.join(algorithm, protocol, 'ztnorm')
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351

      # collect the resulting files
      if os.path.exists(os.path.join(result_dir, nonorm_sub_dir, 'scores-dev')):
        result_dev.append(os.path.join(nonorm_sub_dir, 'scores-dev'))
        legends.append(algorithm)

        if has_eval and os.path.exists(os.path.join(result_dir, nonorm_sub_dir, 'scores-eval')):
          result_eval.append(os.path.join(nonorm_sub_dir, 'scores-eval'))

        if has_zt_norm:
          if os.path.exists(os.path.join(result_dir, ztnorm_sub_dir, 'scores-dev')):
            result_zt_dev.append(os.path.join(ztnorm_sub_dir, 'scores-dev'))
          if has_eval and os.path.exists(os.path.join(result_dir, ztnorm_sub_dir, 'scores-eval')):
            result_zt_eval.append(os.path.join(ztnorm_sub_dir, 'scores-eval'))

    # check if we have found some results
    if not result_dev:
      logger.warn("No result files were detected -- skipping evaluation.")
      return

    # call the evaluate script
    base_command = ['./bin/evaluate.py', '--directory', result_dir, '--legends'] + legends
    if 'EER' in args.evaluate:
      base_command += ['--criterion', 'EER']
    elif 'HTER' in args.evaluate:
      base_command += ['--criterion', 'HTER']
    if 'ROC' in args.evaluate:
      base_command += ['--roc', 'ROCxxx.pdf']
    if 'DET' in args.evaluate:
      base_command += ['--det', 'DETxxx.pdf']
    if 'CMC' in args.evaluate:
      base_command += ['--cmc', 'CMCxxx.pdf']
    if 'RR' in args.evaluate:
      base_command += ['--rr']
    if args.verbose:
      base_command += ['-' + 'v'*args.verbose]

    # first, run the nonorm evaluation
    if result_zt_dev:
      command = [cmd.replace('xxx','_dev') for cmd in base_command]
    else:
      command = [cmd.replace('xxx','') for cmd in base_command]
    command += ['--dev-files'] + result_dev
    if result_eval:
      command += ['--eval-files'] + result_eval

    logger.info("Executing command:\n%s", bob.bio.base.tools.command_line(command))
    if not args.dry_run:
      subprocess.call(command)

    # now, also run the ZT norm evaluation, if available
    if result_zt_dev:
      command = [cmd.replace('xxx','_eval') for cmd in base_command]
      command += ['--dev-files'] + result_zt_dev
      if result_zt_eval:
        command += ['--eval-files'] + result_zt_eval

      logger.info("Executing command:\n%s", bob.bio.base.tools.command_line(command))
      if not args.dry_run:
        subprocess.call(command)