From 75451c07ad03b2757c3301e79fe45df473eb866c Mon Sep 17 00:00:00 2001
From: Tiago Freitas Pereira <tiagofrepereira@gmail.com>
Date: Wed, 24 Jan 2018 15:19:44 +0100
Subject: [PATCH] Appended the metadata during preprocessing
---
bob/bio/base/test/dummy/preprocessor.py | 11 ++++++++++-
bob/bio/base/test/test_scripts.py | 2 +-
bob/bio/base/tools/preprocessor.py | 13 +++++++++----
setup.py | 1 +
4 files changed, 21 insertions(+), 6 deletions(-)
diff --git a/bob/bio/base/test/dummy/preprocessor.py b/bob/bio/base/test/dummy/preprocessor.py
index 89376c31..506f0ef8 100644
--- a/bob/bio/base/test/dummy/preprocessor.py
+++ b/bob/bio/base/test/dummy/preprocessor.py
@@ -16,5 +16,14 @@ class DummyPreprocessor (Preprocessor):
return data
-
preprocessor = DummyPreprocessor()
+
+
+class DummyPreprocessorMetadata (DummyPreprocessor):
+
+ def __call__(self, data, annotation, metadata=None):
+ """Does nothing, simply converts the data type of the data, ignoring any annotation."""
+ assert metadata is not None
+ return super(DummyPreprocessorMetadata, self).__call__(data, annotation)
+
+preprocessor_metadata = DummyPreprocessorMetadata()
diff --git a/bob/bio/base/test/test_scripts.py b/bob/bio/base/test/test_scripts.py
index fcf369b7..60948798 100644
--- a/bob/bio/base/test/test_scripts.py
+++ b/bob/bio/base/test/test_scripts.py
@@ -148,7 +148,7 @@ def test_verify_resources_metadata():
# define dummy parameters
parameters = [
'-d', 'dummy',
- '-p', 'dummy',
+ '-p', 'dummy_metadata',
'-e', 'dummy_metadata',
'-a', 'dummy',
'--zt-norm',
diff --git a/bob/bio/base/tools/preprocessor.py b/bob/bio/base/tools/preprocessor.py
index 83eafab8..b4140dfa 100644
--- a/bob/bio/base/tools/preprocessor.py
+++ b/bob/bio/base/tools/preprocessor.py
@@ -2,6 +2,7 @@ import bob.io.base
import os
import logging
+import inspect
logger = logging.getLogger("bob.bio.base")
from .FileSelector import FileSelector
@@ -45,6 +46,7 @@ def preprocess(preprocessor, groups = None, indices = None, allow_missing_files
data_files = fs.original_data_list(groups=groups)
original_directory, original_extension = fs.original_directory_and_extension()
preprocessed_data_files = fs.preprocessed_data_list(groups=groups)
+ metadata = fs.original_data_list(groups=groups)
# select a subset of keys to iterate
if indices is not None:
@@ -58,7 +60,7 @@ def preprocess(preprocessor, groups = None, indices = None, allow_missing_files
# read annotation files
annotation_list = fs.annotation_list(groups=groups)
- # iterate over the selected files
+ # iterate over the selected files
for i in index_range:
preprocessed_data_file = preprocessed_data_files[i]
file_object = data_files[i]
@@ -78,7 +80,11 @@ def preprocess(preprocessor, groups = None, indices = None, allow_missing_files
annotations = fs.get_annotations(annotation_list[i])
# call the preprocessor
- preprocessed_data = preprocessor(data, annotations)
+ if "metadata" in inspect.getargspec(preprocessor.__call__).args:
+ preprocessed_data = preprocessor(data, annotations, metadata=metadata[i])
+ else:
+ preprocessed_data = preprocessor(data, annotations)
+
if preprocessed_data is None:
if allow_missing_files:
logger.debug("... Processing original data file '%s' was not successful", file_name)
@@ -90,8 +96,7 @@ def preprocess(preprocessor, groups = None, indices = None, allow_missing_files
preprocessor.write_data(preprocessed_data, preprocessed_data_file)
else:
- logger.debug("... Skipping original data file '%s' since preprocessed data '%s' exists", file_name, preprocessed_data_file)
-
+ logger.debug("... Skipping original data file '%s' since preprocessed data '%s' exists", file_name, preprocessed_data_file)
def read_preprocessed_data(file_names, preprocessor, split_by_client = False, allow_missing_files = False):
diff --git a/setup.py b/setup.py
index 91b66c18..4ddf3c93 100644
--- a/setup.py
+++ b/setup.py
@@ -95,6 +95,7 @@ setup(
'bob.bio.preprocessor': [
'dummy = bob.bio.base.test.dummy.preprocessor:preprocessor', # for test purposes only
'filename = bob.bio.base.config.preprocessor.filename:preprocessor',
+ 'dummy_metadata = bob.bio.base.test.dummy.preprocessor:preprocessor_metadata',
],
'bob.bio.extractor': [
--
GitLab