Added the reconstruction error option to VideoSparseCoding preprocessor

bob/pad/face/extractor/VideoHistOfSparseCodes.py

@@ -102,6 +102,21 @@ class VideoHistOfSparseCodes(Extractor, object):
         return frame_container
 
 
+    #==========================================================================
+    def reduce_features_number(self, list_of_arrays):
+        """
+        Reduce the number of features.
+        """
+
+        return_list = []
+
+        for item in list_of_arrays:
+
+            return_list.append( item[1][32:] )
+
+        return return_list
+
+
     #==========================================================================
     def __call__(self, frames):
         """
@@ -119,7 +134,9 @@ class VideoHistOfSparseCodes(Extractor, object):
             Histograms of sparse codes stored in the FrameContainer.
         """
 
-        histograms = self.comp_hist_of_sparse_codes(frames, self.method)
+#        histograms = self.comp_hist_of_sparse_codes(frames, self.method)
+
+        histograms = self.reduce_features_number(frames)
 
         frame_container = self.convert_sparse_codes_to_frame_container(histograms)
 

bob/pad/face/preprocessor/VideoSparseCoding.py

@@ -77,6 +77,11 @@ class VideoSparseCoding(Preprocessor, object):
         "mean" and "hist". This argument is valid only if ``extract_histograms_flag``
         is set to ``True``.
         Default: "hist".
+
+    ``comp_reconstruct_err_flag`` : :py:class:`bool`
+        If this flag is set to ``True`` resulting feature vector will be a
+        reconstruction error, not a histogram.
+        Default: ``False``.
     """
 
     #==========================================================================
@@ -89,6 +94,7 @@ class VideoSparseCoding(Preprocessor, object):
                  frame_step = 1,
                  extract_histograms_flag = False,
                  method = "hist",
+                 comp_reconstruct_err_flag = False,
                  **kwargs):
 
         super(VideoSparseCoding, self).__init__(block_size = block_size,
@@ -98,6 +104,7 @@ class VideoSparseCoding(Preprocessor, object):
                                                 dictionary_file_names = dictionary_file_names,
                                                 frame_step = frame_step,
                                                 extract_histograms_flag = extract_histograms_flag,
+                                                comp_reconstruct_err_flag = comp_reconstruct_err_flag,
                                                 method = method)
 
         self.block_size = block_size
@@ -108,6 +115,7 @@ class VideoSparseCoding(Preprocessor, object):
         self.frame_step = frame_step
         self.extract_histograms_flag = extract_histograms_flag
         self.method = method
+        self.comp_reconstruct_err_flag = comp_reconstruct_err_flag
 
         self.video_preprocessor = bob.bio.video.preprocessor.Wrapper()
 
@@ -799,6 +807,164 @@ class VideoSparseCoding(Preprocessor, object):
         return frame_container
 
 
+    #==========================================================================
+    def compute_patches_mean_squared_errors(self, sparse_codes, original_data, dictionary):
+        """
+        This function computes mean squared errors (MSE) for each feature (column)
+        in the reconstructed array of vectorized patches. The patches are reconstructed
+        given array of sparse codes and a dictionary.
+
+        **Parameters:**
+
+        ``sparse_codes`` : 2D :py:class:`numpy.ndarray`
+            An array of sparse codes. Each row contains a sparse code encoding a
+            vectorized patch. The dimensionality of the array:
+            (``n_samples`` x ``n_words_in_dictionary``).
+
+        ``original_data`` : 2D :py:class:`numpy.ndarray`
+            An array with original vectorized patches.
+            The dimensionality of the array:
+            (``n_samples`` x ``n_features_in_patch``).
+
+        ``dictionary`` : 2D :py:class:`numpy.ndarray`
+            A dictionary with vectorized visual words.
+            The dimensionality of the array:
+            (``n_words_in_dictionary`` x ``n_features_in_patch``).
+
+        **Returns:**
+
+        ``squared_errors`` : 1D :py:class:`numpy.ndarray`
+            MSE for each feature across all patches/samples.
+            The dimensionality of the array:
+            (``n_features_in_patch``, ).
+        """
+
+        recovered_data = np.dot(sparse_codes, dictionary)
+
+        n_samples = recovered_data.shape[0]
+
+        squared_error = 1.0 / n_samples * np.sum((recovered_data - original_data) ** 2, axis=0)
+
+        return squared_error
+
+
+    #==========================================================================
+    def compute_mse_for_all_patches_types(self, sparse_codes_list, original_data_list, dictionary_list):
+        """
+        This function computes mean squared errors (MSE) for all types of patches:
+        frontal, horizontal, and vertical. In this case the function
+        ``compute_patches_mean_squared_errors`` is called in a loop for all
+        values in the input lists.
+
+        **Parameters:**
+
+        ``sparse_codes_list`` : [2D :py:class:`numpy.ndarray`]
+            A list with arrays of sparse codes. Each row in the arrays contains a
+            sparse code encoding a vectorized patch of particular type.
+            The dimensionality of the each array:
+            (``n_samples`` x ``n_words_in_dictionary``).
+
+        ``original_data_list`` : [2D :py:class:`numpy.ndarray`]
+            A list of arrays with original vectorized patches of various types.
+            The dimensionality of the arrays might be different for various types
+            of the patches:
+            (``n_samples`` x ``n_features_in_patch_of_particular_type``).
+
+        ``dictionary_list`` : [2D :py:class:`numpy.ndarray`]
+            A list of dictionaries with vectorized visual words of various types.
+            The dimensionality of the arrays might be different for various types
+            of the patches:
+            (``n_words_in_dictionary`` x ``n_features_in_patch_of_particular_type``).
+
+        **Returns:**
+
+        ``squared_errors`` : 2D :py:class:`numpy.ndarray`
+            First row:
+            MSE of features for various types of patches concatenated into a single
+            vector.
+            Second row:
+            The same as above but MSE are sorted for each type of patches.
+            The dimensionality of the array:
+            (2 x ``n_features_in_patch_of_all_types``).
+        """
+
+        squared_errors = []
+
+        squared_errors_sorted = []
+
+        for sparse_codes, original_data, dictionary in zip(sparse_codes_list, original_data_list, dictionary_list):
+
+            squared_error = self.compute_patches_mean_squared_errors(sparse_codes, original_data, dictionary)
+
+            squared_error_sorted = np.sort(squared_error)
+
+            squared_errors.append(squared_error)
+
+            squared_errors_sorted.append(squared_error_sorted)
+
+        squared_errors = np.hstack(squared_errors)
+
+        squared_errors_sorted = np.hstack(squared_errors_sorted)
+
+        squared_errors = np.vstack([squared_errors, squared_errors_sorted])
+
+        return squared_errors
+
+
+    #==========================================================================
+    def compute_mse_for_all_stacks(self, video_codes_list, patches_list, dictionary_list):
+        """
+        Call ``compute_mse_for_all_patches_types`` for data coming from all stacks
+        of facial images.
+
+        **Parameters:**
+
+        ``video_codes_list`` : [ [2D :py:class:`numpy.ndarray`] ]
+            A list with ``frontal_video_codes``, ``horizontal_video_codes``, and
+            ``vertical_video_codes`` as returned by ``get_sparse_codes_for_list_of_patches``
+            method of this class.
+
+        ``patches_list`` : [ [2D :py:class:`numpy.ndarray`] ]
+            A list with ``frontal_patches``, ``horizontal_patches``, and
+            ``vertical_patches`` as returned by ``extract_patches_from_blocks``
+            method of this class.
+
+        ``dictionary_list`` : [2D :py:class:`numpy.ndarray`]
+            A list of dictionaries with vectorized visual words of various types.
+            The dimensionality of the arrays might be different for various types
+            of the patches:
+            (``n_words_in_dictionary`` x ``n_features_in_patch_of_particular_type``).
+
+        **Returns:**
+
+        ``squared_errors_list`` : [2D :py:class:`numpy.ndarray`]
+            A list of ``squared_errors`` as returned by ``compute_mse_for_all_patches_types``
+            method of this class.
+        """
+
+        fcs = video_codes_list[0]
+        hcs = video_codes_list[1]
+        vcs = video_codes_list[2]
+
+        fps = patches_list[0]
+        hps = patches_list[1]
+        vps = patches_list[2]
+
+        squared_errors_list = []
+
+        for fc, hc, vc, fp, hp, vp in zip(fcs, hcs, vcs, fps, hps, vps):
+
+            sparse_codes_list = [fc, hc, vc]
+
+            original_data_list = [fp, hp, vp]
+
+            squared_errors = self.compute_mse_for_all_patches_types(sparse_codes_list, original_data_list, dictionary_list)
+
+            squared_errors_list.append(squared_errors)
+
+        return squared_errors_list
+
+
     #==========================================================================
     def __call__(self, frames, annotations):
         """
@@ -859,18 +1025,37 @@ class VideoSparseCoding(Preprocessor, object):
         # Download the dictionaries:
         dictionary_frontal, dictionary_horizontal, dictionary_vertical = self.load_the_dictionaries(self.dictionary_file_names)
 
+        # Select subset of patches if ``frame_step`` > 1:
+        frontal_patches_subset = frontal_patches[::self.frame_step]
+        horizontal_patches_subset = horizontal_patches[::self.frame_step]
+        vertical_patches_subset = vertical_patches[::self.frame_step]
+
         # Compute sparse codes for all patches of all types:
-        frontal_video_codes = self.get_sparse_codes_for_list_of_patches(frontal_patches[::self.frame_step], dictionary_frontal)
-        horizontal_video_codes = self.get_sparse_codes_for_list_of_patches(horizontal_patches[::self.frame_step], dictionary_horizontal)
-        vertical_video_codes = self.get_sparse_codes_for_list_of_patches(vertical_patches[::self.frame_step], dictionary_vertical)
+        frontal_video_codes = self.get_sparse_codes_for_list_of_patches(frontal_patches_subset, dictionary_frontal)
+        horizontal_video_codes = self.get_sparse_codes_for_list_of_patches(horizontal_patches_subset, dictionary_horizontal)
+        vertical_video_codes = self.get_sparse_codes_for_list_of_patches(vertical_patches_subset, dictionary_vertical)
+
+        if self.comp_reconstruct_err_flag:
+
+            video_codes_list = [frontal_video_codes, horizontal_video_codes, vertical_video_codes]
+
+            patches_list = [frontal_patches_subset, horizontal_patches_subset, vertical_patches_subset]
+
+            dictionary_list = [dictionary_frontal, dictionary_horizontal, dictionary_vertical]
+
+            squared_errors_list = self.compute_mse_for_all_stacks(video_codes_list, patches_list, dictionary_list)
+
+            frame_container = self.convert_arrays_to_frame_container(squared_errors_list)
+
+        else:
 
-        frame_container = self.convert_sparse_codes_to_frame_container([frontal_video_codes, horizontal_video_codes, vertical_video_codes])
+            frame_container = self.convert_sparse_codes_to_frame_container([frontal_video_codes, horizontal_video_codes, vertical_video_codes])
 
-        if self.extract_histograms_flag: # in this case histograms will be extracted in the preprocessor , no feature extraction is needed then
+            if self.extract_histograms_flag: # in this case histograms will be extracted in the preprocessor , no feature extraction is needed then
 
-            histograms = self.comp_hist_of_sparse_codes(frame_container, self.method)
+                histograms = self.comp_hist_of_sparse_codes(frame_container, self.method)
 
-            frame_container = self.convert_arrays_to_frame_container(histograms)
+                frame_container = self.convert_arrays_to_frame_container(histograms)
 
         return frame_container