IVector.py 12.6 KB
Newer Older
Manuel Günther's avatar
Manuel Günther committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Laurent El Shafey <Laurent.El-Shafey@idiap.ch>

import bob.core
import bob.io.base
import bob.learn.linear
import bob.learn.em

import numpy

from .GMM import GMM
from bob.bio.base.algorithm import Algorithm

import logging
logger = logging.getLogger("bob.bio.gmm")

class IVector (GMM):
  """Tool for extracting I-Vectors"""

  def __init__(
      self,
      # IVector training
      subspace_dimension_of_t,       # T subspace dimension
      tv_training_iterations = 25,   # Number of EM iterations for the JFA training
      update_sigma = True,
Elie KHOURY's avatar
Elie KHOURY committed
27
      use_whitening = True,
28
29
30
      use_lda = False,
      use_wccn = False,
      use_plda = False,
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
31
32
      lda_dim = None,
      lda_strip_to_rank=True,
33
34
35
      plda_dim_F  = 50,
      plda_dim_G = 50,
      plda_training_iterations = 50,
Manuel Günther's avatar
Manuel Günther committed
36
37
38
39
40
41
42
43
44
45
46
47
48
      # parameters of the GMM
      **kwargs
  ):
    """Initializes the local GMM tool with the given file selector object"""
    # call base class constructor with its set of parameters
    GMM.__init__(self, **kwargs)

    # call tool constructor to overwrite what was set before
    Algorithm.__init__(
        self,
        performs_projection = True,
        use_projected_features_for_enrollment = True,
        requires_enroller_training = False, # not needed anymore because it's done while training the projector
49
        split_training_features_by_client = True,
Manuel Günther's avatar
Manuel Günther committed
50
51
52
53

        subspace_dimension_of_t = subspace_dimension_of_t,
        tv_training_iterations = tv_training_iterations,
        update_sigma = update_sigma,
Elie KHOURY's avatar
Elie KHOURY committed
54
        use_whitening = use_whitening,
55
56
57
58
        use_lda = use_lda,
        use_wccn = use_wccn,
        use_plda = use_plda,
        lda_dim = lda_dim,
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
59
        lda_strip_to_rank = lda_strip_to_rank,
60
61
62
        plda_dim_F  = plda_dim_F,
        plda_dim_G = plda_dim_G,
        plda_training_iterations = plda_training_iterations,
Manuel Günther's avatar
Manuel Günther committed
63
64
65
66
67
68
69

        multiple_model_scoring = None,
        multiple_probe_scoring = None,
        **kwargs
    )

    self.update_sigma = update_sigma
Elie KHOURY's avatar
Elie KHOURY committed
70
    self.use_whitening = use_whitening
71
72
73
    self.use_lda = use_lda
    self.use_wccn = use_wccn
    self.use_plda = use_plda
Manuel Günther's avatar
Manuel Günther committed
74
75
    self.subspace_dimension_of_t = subspace_dimension_of_t
    self.tv_training_iterations = tv_training_iterations
76

Manuel Günther's avatar
Manuel Günther committed
77
78
    self.ivector_trainer = bob.learn.em.IVectorTrainer(update_sigma=update_sigma)
    self.whitening_trainer = bob.learn.linear.WhiteningTrainer()
79

80
    self.lda_dim = lda_dim
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
81
    self.lda_trainer = bob.learn.linear.FisherLDATrainer(strip_to_rank=lda_strip_to_rank)
82
83
84
85
86
    self.wccn_trainer = bob.learn.linear.WCCNTrainer()
    self.plda_trainer = bob.learn.em.PLDATrainer()
    self.plda_dim_F  = plda_dim_F
    self.plda_dim_G = plda_dim_G
    self.plda_training_iterations = plda_training_iterations
87

88
89
90


  def _check_ivector(self, feature):
Manuel Günther's avatar
Manuel Günther committed
91
    """Checks that the features are appropriate"""
92
    if not isinstance(feature, numpy.ndarray) or feature.ndim != 1 or feature.dtype != numpy.float64:
Manuel Günther's avatar
Manuel Günther committed
93
94
95
96
      raise ValueError("The given feature is not appropriate")

  def train_ivector(self, training_stats):
    logger.info("  -> Training IVector enroller")
97
    self.tv = bob.learn.em.IVectorMachine(self.ubm, self.subspace_dimension_of_t, self.variance_threshold)
Manuel Günther's avatar
Manuel Günther committed
98

99
100
101
    # Reseting the pseudo random number generator so we can have the same initialization for serial and parallel execution. 
    self.rng = bob.core.random.mt19937(self.init_seed)

Manuel Günther's avatar
Manuel Günther committed
102
103
104
    # train IVector model
    bob.learn.em.train(self.ivector_trainer, self.tv, training_stats, self.tv_training_iterations, rng=self.rng)

105
106

  def train_whitener(self, training_features):
107
    logger.info("  -> Training Whitening")
Manuel Günther's avatar
Manuel Günther committed
108
109
110
111
112
113
    ivectors_matrix = numpy.vstack(training_features)
    # create a Linear Machine
    self.whitener = bob.learn.linear.Machine(ivectors_matrix.shape[1],ivectors_matrix.shape[1])
    # create the whitening trainer
    self.whitening_trainer.train(ivectors_matrix, self.whitener)

114
115
116
  def train_lda(self, training_features):
    logger.info("  -> Training LDA projector")
    self.lda, __eig_vals = self.lda_trainer.train(training_features)
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
117

118
    # resize the machine if desired
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
119
120
121
122
123
124
    # You can only clip if the rank is higher than LDA_DIM 
    if self.lda_dim is not None:
      if len(__eig_vals) < self.lda_dim:
        logger.warning("  -> You are resizing the LDA matrix to a value above its rank"
                       "(from {0} to {1}). Be aware that this may lead you to imprecise eigenvectors.".\
                        format(len(__eig_vals), self.lda_dim))
125
      self.lda.resize(self.lda.shape[0], self.lda_dim)
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
126
       
127
128
129
130
131
132
133
134
135
136
137
138
139

  def train_wccn(self, training_features):
    logger.info("  -> Training WCCN projector")
    self.wccn = self.wccn_trainer.train(training_features)

  def train_plda(self, training_features):
    logger.info("  -> Training PLDA projector")
    self.plda_trainer.init_f_method = 'BETWEEN_SCATTER'
    self.plda_trainer.init_g_method = 'WITHIN_SCATTER'
    self.plda_trainer.init_sigma_method = 'VARIANCE_DATA'
    variance_flooring = 1e-5
    training_features = [numpy.vstack(client) for client in training_features]
    input_dim = training_features[0].shape[1]
140
141
142
143

    # Reseting the pseudo random number generator so we can have the same initialization for serial and parallel execution. 
    self.rng = bob.core.random.mt19937(self.init_seed)
    
144
145
146
    self.plda_base = bob.learn.em.PLDABase(input_dim, self.plda_dim_F, self.plda_dim_G, variance_flooring)
    bob.learn.em.train(self.plda_trainer, self.plda_base, training_features, self.plda_training_iterations, rng=self.rng)

147

Manuel Günther's avatar
Manuel Günther committed
148
149
  def train_projector(self, train_features, projector_file):
    """Train Projector and Enroller at the same time"""
150

151
    [self._check_feature(feature) for client in train_features for feature in client]
Manuel Günther's avatar
Manuel Günther committed
152
153

    # train UBM
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
154
    data = numpy.vstack(feature for client in train_features for feature in client)
Manuel Günther's avatar
Manuel Günther committed
155
156
157
    self.train_ubm(data)
    del data

158
    # project training data
Manuel Günther's avatar
Manuel Günther committed
159
    logger.info("  -> Projecting training data")
160
161
162
    train_gmm_stats = [[self.project_ubm(feature) for feature in client] for client in train_features]
    train_gmm_stats_flatten = [stats for client in train_gmm_stats for stats in client]

Manuel Günther's avatar
Manuel Günther committed
163
    # train IVector
164
165
    logger.info("  -> Projecting training data")
    self.train_ivector(train_gmm_stats_flatten)
Manuel Günther's avatar
Manuel Günther committed
166
167

    # project training i-vectors
168
169
170
    train_ivectors = [[self.project_ivector(stats) for stats in client] for client in train_gmm_stats]
    train_ivectors_flatten = [stats for client in train_ivectors for stats in client]

Elie KHOURY's avatar
Elie KHOURY committed
171
172
173
174
175
    if self.use_whitening:
      # Train Whitening
      self.train_whitener(train_ivectors_flatten)
      # whitening and length-normalizing i-vectors
      train_ivectors = [[self.project_whitening(ivec) for ivec in client] for client in train_ivectors]
176

177
178
179
    if self.use_lda:
      self.train_lda(train_ivectors)
      train_ivectors = [[self.project_lda(ivec) for ivec in client] for client in train_ivectors]
180

181
182
183
    if self.use_wccn:
      self.train_wccn(train_ivectors)
      train_ivectors = [[self.project_wccn(ivec) for ivec in client] for client in train_ivectors]
184

185
186
    if self.use_plda:
      self.train_plda(train_ivectors)
Manuel Günther's avatar
Manuel Günther committed
187
188
189
190

    # save
    self.save_projector(projector_file)

191

Manuel Günther's avatar
Manuel Günther committed
192
193
194
195
196
197
198
199
200
201
202
203
  def save_projector(self, projector_file):
    # Save the IVector base AND the UBM AND the whitening into the same file
    hdf5file = bob.io.base.HDF5File(projector_file, "w")
    hdf5file.create_group('Projector')
    hdf5file.cd('Projector')
    self.save_ubm(hdf5file)

    hdf5file.cd('/')
    hdf5file.create_group('Enroller')
    hdf5file.cd('Enroller')
    self.tv.save(hdf5file)

Elie KHOURY's avatar
Elie KHOURY committed
204
205
206
207
208
    if self.use_whitening:
      hdf5file.cd('/')
      hdf5file.create_group('Whitener')
      hdf5file.cd('Whitener')
      self.whitener.save(hdf5file)
209

210
211
212
213
214
215
216
217
218
219
220
    if self.use_lda:
      hdf5file.cd('/')
      hdf5file.create_group('LDA')
      hdf5file.cd('LDA')
      self.lda.save(hdf5file)

    if self.use_wccn:
      hdf5file.cd('/')
      hdf5file.create_group('WCCN')
      hdf5file.cd('WCCN')
      self.wccn.save(hdf5file)
221

222
223
224
225
226
    if self.use_plda:
      hdf5file.cd('/')
      hdf5file.create_group('PLDA')
      hdf5file.cd('PLDA')
      self.plda_base.save(hdf5file)
227

Manuel Günther's avatar
Manuel Günther committed
228
229
230
231
232
233
234

  def load_tv(self, tv_file):
    hdf5file = bob.io.base.HDF5File(tv_file)
    self.tv = bob.learn.em.IVectorMachine(hdf5file)
    # add UBM model from base class
    self.tv.ubm = self.ubm

235
  def load_whitener(self, whitening_file):
Manuel Günther's avatar
Manuel Günther committed
236
237
238
    hdf5file = bob.io.base.HDF5File(whitening_file)
    self.whitener = bob.learn.linear.Machine(hdf5file)

239
240
241
242
243
244
245
  def load_lda(self, lda_file):
    hdf5file = bob.io.base.HDF5File(lda_file)
    self.lda = bob.learn.linear.Machine(hdf5file)

  def load_wccn(self, wccn_file):
    hdf5file = bob.io.base.HDF5File(wccn_file)
    self.wccn = bob.learn.linear.Machine(hdf5file)
246

247
248
249
250
  def load_plda(self, plda_file):
    hdf5file = bob.io.base.HDF5File(plda_file)
    self.plda_base = bob.learn.em.PLDABase(hdf5file)
    self.plda_machine = bob.learn.em.PLDAMachine(self.plda_base)
251

Manuel Günther's avatar
Manuel Günther committed
252
253
254
255
256
257
258
259
260
261
262
263
  def load_projector(self, projector_file):
    """Load the GMM and the ISV model from the same HDF5 file"""
    hdf5file = bob.io.base.HDF5File(projector_file)

    # Load Projector
    hdf5file.cd('/Projector')
    self.load_ubm(hdf5file)

    # Load Enroller
    hdf5file.cd('/Enroller')
    self.load_tv(hdf5file)

Elie KHOURY's avatar
Elie KHOURY committed
264
265
266
267
    if self.use_whitening:
      # Load Whitening
      hdf5file.cd('/Whitener')
      self.load_whitener(hdf5file)
268

269
270
271
272
    if self.use_lda:
      # Load LDA
      hdf5file.cd('/LDA')
      self.load_lda(hdf5file)
273
274

    if self.use_wccn:
275
276
277
278
      # Load WCCN
      hdf5file.cd('/WCCN')
      self.load_wccn(hdf5file)

279
    if self.use_plda:
280
281
282
     # Load PLDA
      hdf5file.cd('/PLDA')
      self.load_plda(hdf5file)
Manuel Günther's avatar
Manuel Günther committed
283
284


285
  def project_ivector(self, gmm_stats):
Manuel Günther's avatar
Manuel Günther committed
286
287
288
289
290
291
    return self.tv.project(gmm_stats)

  def project_whitening(self, ivector):
    whitened = self.whitener.forward(ivector)
    return whitened / numpy.linalg.norm(whitened)

292
293
294
295
296
297
298
299
300
301
  def project_lda(self, ivector):
    out_ivector = numpy.ndarray(self.lda.shape[1], numpy.float64)
    self.lda(ivector, out_ivector)
    return out_ivector

  def project_wccn(self, ivector):
    out_ivector = numpy.ndarray(self.wccn.shape[1], numpy.float64)
    self.wccn(ivector, out_ivector)
    return out_ivector

Manuel Günther's avatar
Manuel Günther committed
302
303
304
305
306
307
308
309
  #######################################################
  ############## IVector projection #####################
  def project(self, feature_array):
    """Computes GMM statistics against a UBM, then corresponding Ux vector"""
    self._check_feature(feature_array)
    # project UBM
    projected_ubm = self.project_ubm(feature_array)
    # project I-Vector
310
    ivector = self.project_ivector(projected_ubm)
Manuel Günther's avatar
Manuel Günther committed
311
    # whiten I-Vector
Elie KHOURY's avatar
Elie KHOURY committed
312
313
    if self.use_whitening:
      ivector = self.project_whitening(ivector)
314
315
316
317
318
319
320
    # LDA projection
    if self.use_lda:
      ivector = self.project_lda(ivector)
    # WCCN projection
    if self.use_wccn:
      ivector = self.project_wccn(ivector)
    return ivector
Manuel Günther's avatar
Manuel Günther committed
321
322

  #######################################################
323
  ################## Read / Write I-Vectors ####################
Manuel Günther's avatar
Manuel Günther committed
324
325
326
327
328
329
330
331
332
333
334
335
336
  def write_feature(self, data, feature_file):
    """Saves the feature, which is the (whitened) I-Vector."""
    bob.bio.base.save(data, feature_file)

  def read_feature(self, feature_file):
    """Read the type of features that we require, namely i-vectors (stored as simple numpy arrays)"""
    return bob.bio.base.load(feature_file)


  #######################################################
  ################## Model  Enrollment ###################
  def enroll(self, enroll_features):
    """Performs IVector enrollment"""
337
338
339
340
341
342
343
344
    [self._check_ivector(feature) for feature in enroll_features]
    average_ivector = numpy.mean(numpy.vstack(enroll_features), axis=0)
    if self.use_plda:
      average_ivector = average_ivector.reshape(1,-1)
      self.plda_trainer.enroll(self.plda_machine, average_ivector)
      return self.plda_machine
    else:
      return average_ivector
Manuel Günther's avatar
Manuel Günther committed
345
346
347
348
349
350


  ######################################################
  ################ Feature comparison ##################
  def read_model(self, model_file):
    """Reads the whitened i-vector that holds the model"""
351
352
353
354
    if self.use_plda:
      return bob.learn.em.PLDAMachine(bob.io.base.HDF5File(str(model_file)), self.plda_base)
    else:
      return bob.bio.base.load(model_file)
Manuel Günther's avatar
Manuel Günther committed
355
356
357
358


  def score(self, model, probe):
    """Computes the score for the given model and the given probe."""
359
360
361
362
363
364
    self._check_ivector(probe)
    if self.use_plda:
      return model.log_likelihood_ratio(probe)
    else:
      self._check_ivector(model)
      return numpy.dot(model/numpy.linalg.norm(model), probe/numpy.linalg.norm(probe))
Manuel Günther's avatar
Manuel Günther committed
365
366
367
368
369
370


  def score_for_multiple_probes(self, model, probes):
    """This function computes the score between the given model and several given probe files."""
    probe = numpy.mean(numpy.vstack(probes), axis=0)
    return self.score(model, probe)