Commit 8129781d authored by Tiago de Freitas Pereira's avatar Tiago de Freitas Pereira
Browse files

Created a dummy db that dump some fake missing files

Removed dummy-missingfiles database

Created a mechanism to return None at certain probability
parent 700d7a48
Pipeline #13426 passed with stages
in 11 minutes and 59 seconds
from import Preprocessor
import numpy
class DummyPreprocessor (Preprocessor):
def __init__(self, return_none=False, **kwargs):
def __init__(self, return_none=False, probability_of_none=1, **kwargs):
self.return_none = return_none
self.probability_of_none = probability_of_none
def __call__(self, data, annotation):
"""Does nothing, simply converts the data type of the data, ignoring any annotation."""
if self.return_none:
return None
return numpy.random.choice([None, data], p=[self.probability_of_none, 1-self.probability_of_none])
return data
......@@ -68,6 +68,15 @@ def preprocess(preprocessor, groups = None, indices = None, allow_missing_files
if not utils.check_file(preprocessed_data_file, force,
logger.debug("... Processing original data file '%s'", file_name)
# Maybe we have missing file in the databse
if not os.path.exists(file_name):
if allow_missing_files:
logger.debug("... Original data file is missing '%s' and will be skipped", file_name)
raise RuntimeError("Original data file is missing '%s' " % file_name)
data = preprocessor.read_original_data(file_object, original_directory, original_extension)
# create output directory before reading the data file (is sometimes required, when relative directories are specified, especially, including a .. somewhere)
......@@ -89,7 +89,7 @@ setup(
'': [
'dummy =', # for test purposes only
'dummy =', # for test purposes only
'': [
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment