Commit 072055bf authored by Tiago de Freitas Pereira's avatar Tiago de Freitas Pereira
Browse files

Added pickle test

parent 9cd66139
......@@ -15,6 +15,10 @@ import bob.io.base
from .legacy import DatabaseConnector
from .blocks import SampleLoader
from bob.pipelines.sample.sample import SampleSet, DelayedSample, Sample
from bob.pipelines.utils import is_picklable
import logging
logger = logging.getLogger("bob.bio.base")
class DatabaseConnectorAnnotated(DatabaseConnector):
......@@ -293,19 +297,28 @@ class SampleLoaderAnnotated(SampleLoader):
# because we are checkpointing, we return a DelayedSample
# instead of normal (preloaded) sample. This allows the next
# phase to avoid loading it would it be unnecessary (e.g. next
# phase is already check-pointed)
#reader = bob.io.base.load
# phase is already check-pointed)
reader = (
getattr(func, "read_data")
if hasattr(func, "read_data")
else getattr(func, "read_feature")
)
reader = reader.__func__ # The reader object might not be picklable
samples.append(
DelayedSample(
functools.partial(reader, None, candidate), parent=s
)
if is_picklable(reader):
samples.append(
DelayedSample(
functools.partial(reader, candidate), parent=s
)
)
)
else:
logger.warning(f"The method {reader} is not picklable. Shiping its unbounded method to `DelayedSample`.")
reader = reader.__func__ # The reader object might not be picklable
samples.append(
DelayedSample(
functools.partial(reader, None, candidate), parent=s
)
)
else:
# if checkpointing is not required, load the data and preprocess it
# as we would normally do
......
......@@ -7,8 +7,11 @@ import numpy
import os
import bob.io.base
from bob.pipelines.sample.sample import DelayedSample, SampleSet, Sample
from bob.pipelines.utils import is_picklable
"""Re-usable blocks for legacy bob.bio.base algorithms"""
import logging
logger = logging.getLogger("bob.bio.base")
class SampleLoader:
......@@ -107,10 +110,21 @@ class SampleLoader:
if hasattr(func, "read_data")
else getattr(func, "read_feature")
)
reader = reader.__func__ # The reader object might not be picklable
samples.append(
DelayedSample(functools.partial(reader, None, candidate), parent=s)
)
if is_picklable(reader):
samples.append(
DelayedSample(
functools.partial(reader, candidate), parent=s
)
)
else:
logger.warning(f"The method {func} is not picklable. Shiping its unbounded method to `DelayedSample`.")
reader = reader.__func__ # The reader object might not be picklable
samples.append(
DelayedSample(
functools.partial(reader, None, candidate), parent=s
)
)
else:
# if checkpointing is not required, load the data and preprocess it
# as we would normally do
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment