DatabaseBob.py 19.8 KB
Newer Older
1
from .Database import Database, DatabaseZT
2
import os
3

Manuel Günther's avatar
Manuel Günther committed
4
5
import bob.db.verification.utils

6
class DatabaseBob (Database):
Manuel Günther's avatar
Manuel Günther committed
7
8
  """This class can be used whenever you have a database that follows the Bob verification database interface, which is defined in :py:class:`bob.db.verification.utils.Database`

9
  **Parameters:**
Manuel Günther's avatar
Manuel Günther committed
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33

  database : derivative of :py:class:`bob.db.verification.utils.Database`
    The database instance (such as a :py:class:`bob.db.atnt.Database`) that provides the actual interface, see :ref:`verification_databases` for a list.

  all_files_options : dict
    Dictionary of options passed to the :py:meth:`bob.db.verification.utils.Database.objects` database query when retrieving all data.

  extractor_training_options : dict
    Dictionary of options passed to the :py:meth:`bob.db.verification.utils.Database.objects` database query used to retrieve the files for the extractor training.

  projector_training_options : dict
    Dictionary of options passed to the :py:meth:`bob.db.verification.utils.Database.objects` database query used to retrieve the files for the projector training.

  enroller_training_options : dict
    Dictionary of options passed to the :py:meth:`bob.db.verification.utils.Database.objects` database query used to retrieve the files for the enroller training.

  check_original_files_for_existence : bool
    Enables to test for the original data files when querying the database.

  kwargs : ``key=value`` pairs
    The arguments of the :py:class:`Database` base class constructor.

    .. note:: Usually, the ``name``, ``protocol``, ``training_depends_on_protocol`` and ``models_depend_on_protocol`` keyword parameters of the base class constructor need to be specified.
  """
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50

  def __init__(
      self,
      database,  # The bob database that is used
      all_files_options = {}, # additional options for the database query that can be used to extract all files
      extractor_training_options = {}, # additional options for the database query that can be used to extract the training files for the extractor training
      projector_training_options = {}, # additional options for the database query that can be used to extract the training files for the extractor training
      enroller_training_options = {},  # additional options for the database query that can be used to extract the training files for the extractor training
      check_original_files_for_existence = False,
      **kwargs  # The default parameters of the base class
  ):

    Database.__init__(
        self,
        **kwargs
    )

Manuel Günther's avatar
Manuel Günther committed
51
52
    assert isinstance(database, bob.db.verification.utils.Database), "Only databases derived from bob.db.verification.utils.Database are supported by this interface. Please implement your own bob.bio.base.database.Database interface."

53
54
    self.database = database
    self.original_directory = database.original_directory
55
56
57
58
    try:
      self.annotation_directory = database.annotation_directory
    except AttributeError:
      pass
59
60
61
62
63
64
65
66
67
68
69

    self.all_files_options = all_files_options
    self.extractor_training_options = extractor_training_options
    self.projector_training_options = projector_training_options
    self.enroller_training_options = enroller_training_options
    self.check_existence = check_original_files_for_existence

    self._kwargs = kwargs


  def __str__(self):
Manuel Günther's avatar
Manuel Günther committed
70
71
72
73
74
75
76
77
78
    """__str__() -> info

    This function returns all parameters of this class (and its derived class).

    **Returns:**

    info : str
      A string containing the full information of all parameters of this (and the derived) class.
    """
79
80
81
82
83
84
85
86
87
88
    params = ", ".join(["%s=%s" % (key, value) for key, value in self._kwargs.items()])
    params += ", original_directory=%s, original_extension=%s" % (self.original_directory, self.original_extension)
    if self.all_files_options: params += ", all_files_options=%s"%self.all_files_options
    if self.extractor_training_options: params += ", extractor_training_options=%s"%self.extractor_training_options
    if self.projector_training_options: params += ", projector_training_options=%s"%self.projector_training_options
    if self.enroller_training_options: params += ", enroller_training_options=%s"%self.enroller_training_options

    return "%s(%s)" % (str(self.__class__), params)


89
  def replace_directories(self, replacements = None):
Manuel Günther's avatar
Manuel Günther committed
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
    """This helper function replaces the ``original_directory`` and the ``annotation_directory`` of the database with the directories read from the given replacement file.

    This function is provided for convenience, so that the database configuration files do not need to be modified.
    Instead, this function uses the given dictionary of replacements to change the original directory and the original extension (if given).

    The given ``replacements`` can be of type ``dict``, including all replacements, or a file name (as a ``str``), in which case the file is read.
    The structure of the file should be:

    .. code-block:: text

       # Comments starting with # and empty lines are ignored

       [YOUR_..._DATA_DIRECTORY] = /path/to/your/data
       [YOUR_..._ANNOTATION_DIRECTORY] = /path/to/your/annotations

    If no annotation files are available (e.g. when they are stored inside the ``database``), the annotation directory can be left out.

107
    **Parameters:**
Manuel Günther's avatar
Manuel Günther committed
108
109
110
111
112

    replacements : dict or str
      A dictionary with replacements, or a name of a file to read the dictionary from.
      If the file name does not exist, no directories are replaced.
    """
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
    if replacements is None:
      return
    if isinstance(replacements, str):
      if not os.path.exists(replacements):
        return
      # Open the database replacement file and reads its content
      with open(replacements) as f:
        replacements = {}
        for line in f:
          if line.strip() and not line.startswith("#"):
            splits = line.split("=")
            assert len(splits) == 2
            replacements[splits[0].strip()] = splits[1].strip()

    assert isinstance(replacements, dict)

    if self.original_directory in replacements:
      self.original_directory = replacements[self.original_directory]
      self.database.original_directory = replacements[self.database.original_directory]

    try:
      if self.annotation_directory in replacements:
        self.annotation_directory = replacements[self.annotation_directory]
        self.database.annotation_directory = replacements[self.database.annotation_directory]
    except AttributeError:
      pass


141
142
  def uses_probe_file_sets(self):
    """Defines if, for the current protocol, the database uses several probe files to generate a score."""
Manuel Günther's avatar
Manuel Günther committed
143
    return self.database.provides_file_set_for_protocol(self.protocol)
144
145
146


  def all_files(self, groups = None):
Manuel Günther's avatar
Manuel Günther committed
147
148
149
150
151
    """all_files(groups=None) -> files

    Returns all files of the database, respecting the current protocol.
    The files can be limited using the ``all_files_options`` in the constructor.

152
    **Parameters:**
Manuel Günther's avatar
Manuel Günther committed
153
154
155
156
157
158
159
160
161
162
163

    groups : some of ``('world', 'dev', 'eval')`` or ``None``
      The groups to get the data for.
      If ``None``, data for all groups is returned.

    **Returns:**

    files : [:py:class:`bob.db.verification.utils.File`]
      The sorted and unique list of all files of the database.
    """
    return self.sort(self.database.objects(protocol = self.protocol, groups = groups, **self.all_files_options))
164
165
166


  def training_files(self, step = None, arrange_by_client = False):
Manuel Günther's avatar
Manuel Günther committed
167
168
169
170
171
    """training_files(step = None, arrange_by_client = False) -> files

    Returns all training files for the given step, and arranges them by client, if desired, respecting the current protocol.
    The files for the steps can be limited using the ``..._training_options`` defined in the constructor.

172
    **Parameters:**
Manuel Günther's avatar
Manuel Günther committed
173
174
175
176
177
178

    step : one of ``('train_extractor', 'train_projector', 'train_enroller')`` or ``None``
      The step for which the training data should be returned.

    arrange_by_client : bool
      Should the training files be arranged by client?
179
180
      If set to ``True``, training files will be returned in [[:py:class:`bob.db.verification.utils.File`]], where each sub-list contains the files of a single client.
      Otherwise, all files will be stored in a simple [:py:class:`bob.db.verification.utils.File`].
Manuel Günther's avatar
Manuel Günther committed
181
182
183
184
185
186

    **Returns:**

    files : [:py:class:`bob.db.verification.utils.File`] or [[:py:class:`bob.db.verification.utils.File`]]
      The (arranged) list of files used for the training of the given step.
    """
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
    if step is None:
      training_options = self.all_files_options
    elif step == 'train_extractor':
      training_options = self.extractor_training_options
    elif step == 'train_projector':
      training_options = self.projector_training_options
    elif step == 'train_enroller':
      training_options = self.enroller_training_options
    else:
      raise ValueError("The given step '%s' must be one of ('train_extractor', 'train_projector', 'train_enroller')" % step)

    files = self.sort(self.database.objects(protocol = self.protocol, groups = 'world', **training_options))
    if arrange_by_client:
      return self.arrange_by_client(files)
    else:
      return files

  def test_files(self, groups = ['dev']):
Manuel Günther's avatar
Manuel Günther committed
205
206
207
208
209
    """test_files(groups = ['dev']) -> files

    Returns all test files (i.e., files used for enrollment and probing) for the given groups, respecting the current protocol.
    The files for the steps can be limited using the ``all_files_options`` defined in the constructor.

210
    **Parameters:**
Manuel Günther's avatar
Manuel Günther committed
211
212
213
214
215
216
217
218
219

    groups : some of ``('dev', 'eval')``
      The groups to get the data for.

    **Returns:**

    files : [:py:class:`bob.db.verification.utils.File`]
      The sorted and unique list of test files of the database.
    """
220
221
222
    return self.sort(self.database.test_files(protocol = self.protocol, groups = groups, **self.all_files_options))

  def model_ids(self, group = 'dev'):
Manuel Günther's avatar
Manuel Günther committed
223
224
225
226
    """model_ids(group = 'dev') -> ids

    Returns a list of model ids for the given group, respecting the current protocol.

227
    **Parameters:**
Manuel Günther's avatar
Manuel Günther committed
228
229
230
231
232
233
234
235
236
237

    group : one of ``('dev', 'eval')``
      The group to get the model ids for.

    **Returns:**

    ids : [int] or [str]
      The list of (unique) model ids for the given group.
    """
    return sorted(self.database.model_ids(protocol = self.protocol, groups = group))
238
239
240


  def client_id_from_model_id(self, model_id, group = 'dev'):
Manuel Günther's avatar
Manuel Günther committed
241
242
243
244
    """client_id_from_model_id(model_id, group = 'dev') -> client_id

    Uses :py:meth:`bob.db.verification.utils.Database.get_client_id_from_model_id` to retrieve the client id for the given model id.

245
    **Parameters:**
Manuel Günther's avatar
Manuel Günther committed
246
247
248
249
250
251
252
253
254
255
256
257
258

    model_id : int or str
      A unique ID that identifies the model for the client.

    group : one of ``('dev', 'eval')``
      The group to get the client ids for.

    **Returns:**

    client_id : [int] or [str]
      A unique ID that identifies the client, to which the model belongs.
    """
    return self.database.get_client_id_from_model_id(model_id)
259
260
261


  def enroll_files(self, model_id, group = 'dev'):
Manuel Günther's avatar
Manuel Günther committed
262
263
264
265
    """enroll_files(model_id, group = 'dev') -> files

    Returns a list of File objects that should be used to enroll the model with the given model id from the given group, respecting the current protocol.

266
    **Parameters:**
Manuel Günther's avatar
Manuel Günther committed
267
268
269
270
271
272
273
274
275
276
277
278
279

    model_id : int or str
      A unique ID that identifies the model.

    group : one of ``('dev', 'eval')``
      The group to get the enrollment files for.

    **Returns:**

    files : [:py:class:`bob.db.verification.utils.File`]
      The list of files used for to enroll the model with the given model id.
    """
    return self.sort(self.database.objects(protocol = self.protocol, groups = group, model_ids = (model_id,), purposes = 'enroll', **self.all_files_options))
280
281
282


  def probe_files(self, model_id = None, group = 'dev'):
Manuel Günther's avatar
Manuel Günther committed
283
284
285
286
287
288
    """probe_files(model_id = None, group = 'dev') -> files

    Returns a list of probe File objects, respecting the current protocol.
    If a ``model_id`` is specified, only the probe files that should be compared with the given model id are returned (for most databases, these are all probe files of the given group).
    Otherwise, all probe files of the given group are returned.

289
    **Parameters:**
Manuel Günther's avatar
Manuel Günther committed
290
291
292
293
294
295
296
297
298
299
300
301
302

    model_id : int or str or ``None``
      A unique ID that identifies the model.

    group : one of ``('dev', 'eval')``
      The group to get the enrollment files for.

    **Returns:**

    files : [:py:class:`bob.db.verification.utils.File`]
      The list of files used for to probe the model with the given model id.
    """
    if model_id is not None:
303
304
305
306
307
308
309
      files = self.database.objects(protocol = self.protocol, groups = group, model_ids = (model_id,), purposes = 'probe', **self.all_files_options)
    else:
      files = self.database.objects(protocol = self.protocol, groups = group, purposes = 'probe', **self.all_files_options)
    return self.sort(files)


  def probe_file_sets(self, model_id = None, group = 'dev'):
Manuel Günther's avatar
Manuel Günther committed
310
311
312
313
314
315
    """probe_file_sets(model_id = None, group = 'dev') -> files

    Returns a list of probe FileSet objects, respecting the current protocol.
    If a ``model_id`` is specified, only the probe files that should be compared with the given model id are returned (for most databases, these are all probe files of the given group).
    Otherwise, all probe files of the given group are returned.

316
    **Parameters:**
Manuel Günther's avatar
Manuel Günther committed
317
318
319
320
321
322
323
324
325
326
327

    model_id : int or str or ``None``
      A unique ID that identifies the model.

    group : one of ``('dev', 'eval')``
      The group to get the enrollment files for.

    **Returns:**

    files : [:py:class:`FileSet`] or something similar
      The list of file sets used to probe the model with the given model id."""
328
329
330
331
332
333
334
335
    if model_id:
      file_sets = self.database.object_sets(protocol = self.protocol, groups = group, model_ids = (model_id,), purposes = 'probe', **self.all_files_options)
    else:
      file_sets = self.database.object_sets(protocol = self.protocol, groups = group, purposes = 'probe', **self.all_files_options)
    return self.sort(file_sets)


  def annotations(self, file):
Manuel Günther's avatar
Manuel Günther committed
336
337
338
339
    """annotations(file) -> annots

    Returns the annotations for the given File object, if available.

340
    **Parameters:**
Manuel Günther's avatar
Manuel Günther committed
341
342
343
344
345
346
347
348
349

    file : :py:class:`bob.db.verification.utils.File`
      The file for which annotations should be returned.

    **Returns:**

    annots : dict or None
      The annotations for the file, if available.
    """
350
351
352
353
    return self.database.annotations(file)


  def original_file_names(self, files):
Manuel Günther's avatar
Manuel Günther committed
354
355
356
357
    """original_file_names(files) -> paths

    Returns the full path of the original data of the given File objects, as returned by :py:meth:`bob.db.verification.utils.Database.original_file_names`.

358
    **Parameters:**
Manuel Günther's avatar
Manuel Günther committed
359
360
361
362
363
364
365
366
367
368

    files : [:py:class:`bob.db.verification.utils.File`]
      The list of file object to retrieve the original data file names for.

    **Returns:**

    paths : [str] or [[str]]
      The paths extracted for the files, in the same order.
      If this database provides file sets, a list of lists of file names is returned, one sub-list for each file set.
    """
369
370
371
372
373
    return self.database.original_file_names(files, self.check_existence)



class DatabaseBobZT (DatabaseBob, DatabaseZT):
Manuel Günther's avatar
Manuel Günther committed
374
375
  """This class can be used whenever you have a database that follows the Bob ZT-norm verification database interface, which is defined in :py:class:`bob.db.verification.utils.ZTDatabase`.

376
  **Parameters:**
Manuel Günther's avatar
Manuel Günther committed
377
378
379
380
381

  database : derivative of :py:class:`bob.db.verification.utils.ZTDatabase`
    The database instance (such as a :py:class:`bob.db.mobio.Database`) that provides the actual interface, see :ref:`verification_databases` for a list.

  z_probe_options : dict
382
    Dictionary of options passed to the :py:meth:`bob.db.verification.utils.ZTDatabase.z_probe_files` database query when retrieving files for Z-probing.
Manuel Günther's avatar
Manuel Günther committed
383
384
385
386
387
388

  kwargs : ``key=value`` pairs
    The arguments of the :py:class:`DatabaseBob` base class constructor.

    .. note:: Usually, the ``name``, ``protocol``, ``training_depends_on_protocol`` and ``models_depend_on_protocol`` keyword parameters of the :py:class:`Database` base class constructor need to be specified.
  """
389
390
391

  def __init__(
      self,
Manuel Günther's avatar
Manuel Günther committed
392
      database,
393
394
395
      z_probe_options = {}, # Limit the z-probes
      **kwargs
  ):
Manuel Günther's avatar
Manuel Günther committed
396
#    assert isinstance(database, bob.db.verification.utils.ZTDatabase) // fails in tests
397
    # call base class constructor, passing all the parameters to it
Manuel Günther's avatar
Manuel Günther committed
398
    DatabaseBob.__init__(self, database = database, z_probe_options = z_probe_options, **kwargs)
399

Manuel Günther's avatar
Manuel Günther committed
400
    self.z_probe_options = z_probe_options
401
402
403


  def all_files(self, groups = ['dev']):
Manuel Günther's avatar
Manuel Günther committed
404
405
406
407
408
    """all_files(groups=None) -> files

    Returns all files of the database, including those for ZT norm, respecting the current protocol.
    The files can be limited using the ``all_files_options`` and the the ``z_probe_options`` in the constructor.

409
    **Parameters:**
Manuel Günther's avatar
Manuel Günther committed
410
411
412
413
414
415
416
417
418
419
420

    groups : some of ``('world', 'dev', 'eval')`` or ``None``
      The groups to get the data for.
      If ``None``, data for all groups is returned.

    **Returns:**

    files : [:py:class:`bob.db.verification.utils.File`]
      The sorted and unique list of all files of the database.
    """
    files = self.database.objects(protocol = self.protocol, groups = groups, **self.all_files_options)
421
422
423
424

    # add all files that belong to the ZT-norm
    for group in groups:
      if group == 'world': continue
Manuel Günther's avatar
Manuel Günther committed
425
426
      files += self.database.tobjects(protocol = self.protocol, groups = group, model_ids = None)
      files += self.database.zobjects(protocol = self.protocol, groups = group, **self.z_probe_options)
427
428
429
430
    return self.sort(files)


  def t_model_ids(self, group = 'dev'):
Manuel Günther's avatar
Manuel Günther committed
431
    """t_model_ids(group = 'dev') -> ids
432

Manuel Günther's avatar
Manuel Günther committed
433
    Returns a list of model ids of T-Norm models for the given group, respecting the current protocol.
434

435
    **Parameters:**
Manuel Günther's avatar
Manuel Günther committed
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452

    group : one of ``('dev', 'eval')``
      The group to get the model ids for.

    **Returns:**

    ids : [int] or [str]
      The list of (unique) model ids for T-Norm models of the given group.
    """
    return sorted(self.database.t_model_ids(protocol = self.protocol, groups = group))


  def t_enroll_files(self, t_model_id, group = 'dev'):
    """t_enroll_files(t_model_id, group = 'dev') -> files

    Returns a list of File objects that should be used to enroll the T-Norm model with the given model id from the given group, respecting the current protocol.

453
    **Parameters:**
Manuel Günther's avatar
Manuel Günther committed
454
455
456
457
458
459
460
461
462
463
464
465
466

    t_model_id : int or str
      A unique ID that identifies the model.

    group : one of ``('dev', 'eval')``
      The group to get the enrollment files for.

    **Returns:**

    files : [:py:class:`bob.db.verification.utils.File`]
      The sorted list of files used for to enroll the model with the given model id.
    """
    return self.sort(self.database.t_enroll_files(protocol = self.protocol, groups = group, model_id = t_model_id))
467
468
469


  def z_probe_files(self, group = 'dev'):
Manuel Günther's avatar
Manuel Günther committed
470
471
472
    """z_probe_files(group = 'dev') -> files

    Returns a list of probe files used to compute the Z-Norm, respecting the current protocol.
473
    The Z-probe files can be limited using the ``z_probe_options`` in the query to :py:meth:`bob.db.verification.utils.ZTDatabase.z_probe_files`
Manuel Günther's avatar
Manuel Günther committed
474

475
    **Parameters:**
Manuel Günther's avatar
Manuel Günther committed
476
477
478
479
480
481
482
483
484
485

    group : one of ``('dev', 'eval')``
      The group to get the Z-norm probe files for.

    **Returns:**

    files : [:py:class:`bob.db.verification.utils.File`]
      The unique list of files used to compute the Z-norm.
    """
    files = self.database.z_probe_files(protocol = self.protocol, groups = group, **self.z_probe_options)
486
487
488
489
    return self.sort(files)


  def z_probe_file_sets(self, group = 'dev'):
Manuel Günther's avatar
Manuel Günther committed
490
491
492
493
494
    """z_probe_file_sets(group = 'dev') -> files

    Returns a list of probe FileSet objects used to compute the Z-Norm.
    The Z-probe files can be limited using the ``z_probe_options`` in the query to

495
    **Parameters:**
Manuel Günther's avatar
Manuel Günther committed
496
497
498
499
500
501
502
503
504

    group : one of ``('dev', 'eval')``
      The group to get the Z-norm probe files for.

    **Returns:**

    files : [:py:class:`FileSet`] or similar
      The unique list of file sets used to compute the Z-norm.
    """
505
    file_sets = self.database.z_probe_file_sets(protocol = self.protocol, groups = group, **self.z_probe_options)
506
    return self.sort(file_sets)