Propagated --allow-missing-files to the UBM training

propagating allow missing files

propagating allow missing files

bob/bio/gmm/script/verify_gmm.py

@@ -158,6 +158,7 @@ def execute(args):
         algorithm,
         args.extractor,
         args.limit_training_data,
+        allow_missing_files = args.allow_missing_files,
         force = args.force)
 
   # train the feature projector
@@ -166,6 +167,7 @@ def execute(args):
         algorithm,
         args.extractor,
         args.iteration,
+        allow_missing_files = args.allow_missing_files,
         indices = base_tools.indices(fs.training_list('extracted', 'train_projector'), args.grid.number_of_projection_jobs),
         force = args.force)
 
@@ -183,6 +185,7 @@ def execute(args):
         algorithm,
         args.extractor,
         args.limit_training_data,
+        allow_missing_files = args.allow_missing_files,
         force = args.force)
 
   # train the feature projector
@@ -191,6 +194,7 @@ def execute(args):
         algorithm,
         args.extractor,
         args.iteration,
+        allow_missing_files = args.allow_missing_files,
         indices = base_tools.indices(fs.training_list('extracted', 'train_projector'), args.grid.number_of_projection_jobs),
         force = args.force)
 

bob/bio/gmm/tools/__init__.py

+import logging
+logger = logging.getLogger("bob.bio.gmm")
+
+def check_allow_missing_files(allow_missing_files, filename):
+  """
+  Just a snniped that checks the allow missing files flag
+  """
+  if not os.path.exists(filename):
+    if allow_missing_files:
+      logger.debug("... Cannot find the file %s; skipping", filename)
+    else:
+      raise RuntimeError("Cannot find the file '%s' " %  filename)
+
 from .utils import *
 from .command_line import *
 from .gmm import *
 from .isv import *
 from .ivector import *
 
+
 # gets sphinx autodoc done right - don't remove it
 __all__ = [_ for _ in dir() if not _.startswith('_')]

bob/bio/gmm/tools/gmm.py

@@ -12,7 +12,7 @@ from bob.bio.base import utils, tools
 from .utils import read_feature
 
 
-def kmeans_initialize(algorithm, extractor, limit_data = None, force = False):
+def kmeans_initialize(algorithm, extractor, limit_data = None, force = False, allow_missing_files = False):
   """Initializes the K-Means training (non-parallel)."""
   fs = FileSelector.instance()
 
@@ -27,7 +27,7 @@ def kmeans_initialize(algorithm, extractor, limit_data = None, force = False):
 
     # read the features
     reader = functools.partial(read_feature, extractor)
-    data = utils.vstack_features(reader, training_list)
+    data = utils.vstack_features(reader, training_list, allow_missing_files=allow_missing_files)
 
     # Perform KMeans initialization
     kmeans_machine = bob.learn.em.KMeansMachine(algorithm.gaussians, data.shape[1])
@@ -38,7 +38,7 @@ def kmeans_initialize(algorithm, extractor, limit_data = None, force = False):
     logger.info("UBM training: saved initial KMeans machine to '%s'", output_file)
 
 
-def kmeans_estep(algorithm, extractor, iteration, indices, force=False):
+def kmeans_estep(algorithm, extractor, iteration, indices, force=False, allow_missing_files = False):
   """Performs a single E-step of the K-Means algorithm (parallel)"""
   if indices[0] >= indices[1]:
     return
@@ -62,7 +62,8 @@ def kmeans_estep(algorithm, extractor, iteration, indices, force=False):
     reader = functools.partial(read_feature, extractor)
     data = utils.vstack_features(
         reader,
-        (training_list[index] for index in range(indices[0], indices[1])))
+        (training_list[index] for index in range(indices[0], indices[1])),
+        allow_missing_files=allow_missing_files)
 
     # Performs the E-step
     trainer = algorithm.kmeans_trainer
@@ -162,7 +163,7 @@ def kmeans_mstep(algorithm, iteration, number_of_parallel_jobs, force=False, cle
 
 
 
-def gmm_initialize(algorithm, extractor, limit_data = None, force = False):
+def gmm_initialize(algorithm, extractor, limit_data = None, force = False, allow_missing_files = False):
   """Initializes the GMM calculation with the result of the K-Means algorithm (non-parallel).
   This might require a lot of memory."""
   fs = FileSelector.instance()
@@ -178,7 +179,7 @@ def gmm_initialize(algorithm, extractor, limit_data = None, force = False):
 
     # read the features
     reader = functools.partial(read_feature, extractor)
-    data = utils.vstack_features(reader, training_list)
+    data = utils.vstack_features(reader, training_list, allow_missing_files=allow_missing_files)
 
     # get means and variances of kmeans result
     kmeans_machine = bob.learn.em.KMeansMachine(bob.io.base.HDF5File(fs.kmeans_file))
@@ -199,7 +200,7 @@ def gmm_initialize(algorithm, extractor, limit_data = None, force = False):
     logger.info("UBM Training: Wrote GMM file '%s'", output_file)
 
 
-def gmm_estep(algorithm, extractor, iteration, indices, force=False):
+def gmm_estep(algorithm, extractor, iteration, indices, force=False, allow_missing_files = False):
   """Performs a single E-step of the GMM training (parallel)."""
   if indices[0] >= indices[1]:
     return
@@ -221,7 +222,8 @@ def gmm_estep(algorithm, extractor, iteration, indices, force=False):
     reader = functools.partial(read_feature, extractor)
     data = utils.vstack_features(
         reader,
-        (training_list[index] for index in range(indices[0], indices[1])))
+        (training_list[index] for index in range(indices[0], indices[1]))
+        , allow_missing_files=allow_missing_files)
 
     trainer = algorithm.ubm_trainer
     trainer.initialize(gmm_machine, None)
@@ -294,7 +296,7 @@ def gmm_mstep(algorithm, iteration, number_of_parallel_jobs, force=False, clean=
     shutil.rmtree(old_dir)
 
 
-def gmm_project(algorithm, extractor, indices, force=False):
+def gmm_project(algorithm, extractor, indices, force=False, allow_missing_files = False):
   """Performs GMM projection"""
   fs = FileSelector.instance()
 
@@ -311,8 +313,9 @@ def gmm_project(algorithm, extractor, indices, force=False):
     projected_file = projected_files[i]
 
     if not utils.check_file(projected_file, force):
+
       # load feature
-      feature = read_feature(extractor, feature_file)
+      feature = read_feature(extractor, feature_file, allow_missing_files=allow_missing_files)
       # project feature
       projected = algorithm.project_ubm(feature)
       # write it

bob/bio/gmm/tools/isv.py

@@ -6,8 +6,9 @@ import os
 
 from bob.bio.base.tools.FileSelector import FileSelector
 from bob.bio.base import utils, tools
+from bob.bio.gmm.tools import check_allow_missing_files
 
-def train_isv(algorithm, force=False):
+def train_isv(algorithm, force=False, allow_missing_files=False):
   """Finally, the UBM is used to train the ISV projector/enroller."""
   fs = FileSelector.instance()
 
@@ -19,7 +20,15 @@ def train_isv(algorithm, force=False):
 
     # read training data
     training_list = fs.training_list('projected_gmm', 'train_projector', arrange_by_client = True)
-    train_gmm_stats = [[algorithm.read_gmm_stats(filename) for filename in client_files] for client_files in training_list]
+
+    train_gmm_stats = []
+    for client_files in training_list:
+      client_stats = []
+      for filename in client_files:      
+        check_allow_missing_files(allow_missing_files, filename)
+        client_stats.append(algorithm.read_gmm_stats(filename))
+      train_gmm_stats.append(client_stats)
+    #train_gmm_stats = [[algorithm.read_gmm_stats(filename) for filename in client_files] for client_files in training_list]
 
     # perform ISV training
     logger.info("ISV training: training ISV with %d clients", len(train_gmm_stats))

bob/bio/gmm/tools/ivector.py

@@ -8,9 +8,9 @@ import shutil
 from bob.bio.base.tools.FileSelector import FileSelector
 from bob.bio.base import utils, tools
 
+from . import check_allow_missing_files
 
-
-def ivector_estep(algorithm, iteration, indices, force=False):
+def ivector_estep(algorithm, iteration, indices, force=False, allow_missing_files = False):
   """Performs a single E-step of the IVector algorithm (parallel)"""
   fs = FileSelector.instance()
   stats_file = fs.ivector_stats_file(iteration, indices[0], indices[1])
@@ -38,7 +38,13 @@ def ivector_estep(algorithm, iteration, indices, force=False):
 
     # Load data
     training_list = fs.training_list('projected_gmm', 'train_projector')
-    data = [algorithm.read_gmm_stats(training_list[i]) for i in range(indices[0], indices[1])]
+    
+    data = []
+    for i in range(indices[0], indices[1]):
+      filename = training_list[i]
+      check_allow_missing_files(allow_missing_files, filename)
+      data.append(algorithm.read_gmm_stats(filename))
+    #data = [algorithm.read_gmm_stats(training_list[i]) for i in range(indices[0], indices[1])]
 
     # Perform the E-step
     trainer.e_step(tv, data)
@@ -134,7 +140,7 @@ def ivector_mstep(algorithm, iteration, number_of_parallel_jobs, force=False, cl
     shutil.rmtree(old_dir)
 
 
-def ivector_project(algorithm, indices, force=False):
+def ivector_project(algorithm, indices, force=False, allow_missing_files=False):
   """Performs IVector projection"""
   # read UBM and TV into the IVector class
   fs = FileSelector.instance()
@@ -150,6 +156,9 @@ def ivector_project(algorithm, indices, force=False):
     gmm_stats_file = gmm_stats_files[i]
     ivector_file = ivector_files[i]
     if not utils.check_file(ivector_file, force):
+
+      check_allow_missing_files(allow_missing_files, gmm_stats_file)
+      
       # load feature
       feature = algorithm.read_gmm_stats(gmm_stats_file)
       # project feature
@@ -159,7 +168,7 @@ def ivector_project(algorithm, indices, force=False):
       bob.bio.base.save(projected, ivector_file)
 
 
-def train_whitener(algorithm, force=False):
+def train_whitener(algorithm, force=False, allow_missing_files=False):
   """Train the feature projector with the extracted features of the world group."""
   fs = FileSelector.instance()
 
@@ -167,7 +176,14 @@ def train_whitener(algorithm, force=False):
     logger.info("- Whitening projector '%s' already exists.", fs.whitener_file)
   else:
     train_files = fs.training_list('projected_ivector', 'train_projector')
-    train_features = [bob.bio.base.load(f) for f in train_files]
+    
+    train_features = []
+    for f in train_files:
+      check_allow_missing_files(allow_missing_files, f)
+      train_features.append(bob.bio.base.load(f))
+    
+    #train_features = [bob.bio.base.load(f) for f in train_files]
+      
     # perform training
     algorithm.train_whitener(train_features)
     bob.io.base.create_directories_safe(os.path.dirname(fs.whitener_file))
@@ -188,6 +204,8 @@ def whitening_project(algorithm, indices, force=False):
     ivector_file = ivector_files[i]
     whitened_file = whitened_files[i]
     if not utils.check_file(whitened_file, force):
+    
+      check_allow_missing_files(allow_missing_files, ivector_file)
       # load feature
       ivector = algorithm.read_feature(ivector_file)
       # project feature
@@ -204,7 +222,16 @@ def train_lda(algorithm, force=False):
     logger.info("- LDA projector '%s' already exists.", fs.lda_file)
   else:
     train_files = fs.training_list('whitened', 'train_projector', arrange_by_client = True)
-    train_features = [[bob.bio.base.load(filename) for filename in client_files] for client_files in train_files]
+    #train_features = [[bob.bio.base.load(filename) for filename in client_files] for client_files in train_files]
+    
+    train_features = []
+    for client_files in train_files:
+      client_features = []
+      for filename in client_files:      
+        check_allow_missing_files(allow_missing_files, filename)
+        client_features.append(bob.bio.base.load(filename))
+      train_features.append(client_features)
+    
     # perform training
     algorithm.train_lda(train_features)
     bob.io.base.create_directories_safe(os.path.dirname(fs.lda_file))
@@ -224,6 +251,8 @@ def lda_project(algorithm, indices, force=False):
     ivector_file = whitened_files[i]
     lda_projected_file = lda_projected_files[i]
     if not utils.check_file(lda_projected_file, force):
+      check_allow_missing_files(allow_missing_files, ivector_file)
+    
       # load feature
       ivector = algorithm.read_feature(ivector_file)
       # project feature
@@ -244,7 +273,16 @@ def train_wccn(algorithm, force=False):
     else:
       input_label = 'whitened'
     train_files = fs.training_list(input_label, 'train_projector', arrange_by_client = True)
-    train_features = [[bob.bio.base.load(filename) for filename in client_files] for client_files in train_files]
+    #train_features = [[bob.bio.base.load(filename) for filename in client_files] for client_files in train_files]
+    
+    train_features = []
+    for client_files in train_files:
+      client_features = []
+      for filename in client_files:      
+        check_allow_missing_files(allow_missing_files, filename)
+        client_features.append(bob.bio.base.load(filename))
+      train_features.append(client_features)    
+    
     # perform training
     algorithm.train_wccn(train_features)
     bob.io.base.create_directories_safe(os.path.dirname(fs.wccn_file))
@@ -267,7 +305,9 @@ def wccn_project(algorithm, indices, force=False):
   for i in range(indices[0], indices[1]):
     ivector_file = input_files[i]
     wccn_projected_file = wccn_projected_files[i]
-    if not utils.check_file(wccn_projected_file, force):
+    if not utils.check_file(wccn_projected_file, force):  
+      check_allow_missing_files(allow_missing_files, ivector_file)
+    
       # load feature
       ivector = algorithm.read_feature(ivector_file)
       # project feature
@@ -290,7 +330,16 @@ def train_plda(algorithm, force=False):
     else:
       input_label = 'whitened'
     train_files = fs.training_list(input_label, 'train_projector', arrange_by_client = True)
-    train_features = [[bob.bio.base.load(filename) for filename in client_files] for client_files in train_files]
+    #train_features = [[bob.bio.base.load(filename) for filename in client_files] for client_files in train_files]
+    
+    train_features = []
+    for client_files in train_files:
+      client_features = []
+      for filename in client_files:      
+        check_allow_missing_files(allow_missing_files, filename)
+        client_features.append(bob.bio.base.load(filename))
+      train_features.append(client_features)     
+
     # perform training
     algorithm.train_plda(train_features)
     bob.io.base.create_directories_safe(os.path.dirname(fs.plda_file))

bob/bio/gmm/tools/utils.py

 import bob.bio.base
 import numpy
+import os
 
 def add_jobs(args, submitter, local_job_adder):
   """Adds all (desired) jobs of the tool chain to the grid, or to the local list to be executed."""
@@ -63,7 +64,14 @@ def base(algorithm):
   """Returns the base algorithm, if it is a video extension, otherwise returns the algorithm itself"""
   return algorithm.algorithm if is_video_extension(algorithm) else algorithm
 
-def read_feature(extractor, feature_file):
+def read_feature(extractor, feature_file, allow_missing_files = False):
+
+  if not os.path.exists(feature_file):
+    if allow_missing_files:
+      logger.debug("... Cannot find preprocessed data file %s; skipping", feature_file)
+    else:
+      raise RuntimeError("Cannot find file '%s' " %  feature_file)
+
   feature = extractor.read_feature(feature_file)
   try:
     import bob.bio.video