Skip to content
Snippets Groups Projects
Commit e90bb4d8 authored by Tiago de Freitas Pereira's avatar Tiago de Freitas Pereira Committed by Amir MOHAMMADI
Browse files

Added pickle test

parent e4dda73d
No related branches found
No related tags found
1 merge request!180[dask] Preparing bob.bio.base for dask pipelines
......@@ -15,6 +15,10 @@ import bob.io.base
from .legacy import DatabaseConnector
from .blocks import SampleLoader
from bob.pipelines.sample.sample import SampleSet, DelayedSample, Sample
from bob.pipelines.utils import is_picklable
import logging
logger = logging.getLogger("bob.bio.base")
class DatabaseConnectorAnnotated(DatabaseConnector):
......@@ -293,19 +297,28 @@ class SampleLoaderAnnotated(SampleLoader):
# because we are checkpointing, we return a DelayedSample
# instead of normal (preloaded) sample. This allows the next
# phase to avoid loading it would it be unnecessary (e.g. next
# phase is already check-pointed)
#reader = bob.io.base.load
# phase is already check-pointed)
reader = (
getattr(func, "read_data")
if hasattr(func, "read_data")
else getattr(func, "read_feature")
)
reader = reader.__func__ # The reader object might not be picklable
samples.append(
DelayedSample(
functools.partial(reader, None, candidate), parent=s
)
if is_picklable(reader):
samples.append(
DelayedSample(
functools.partial(reader, candidate), parent=s
)
)
)
else:
logger.warning(f"The method {reader} is not picklable. Shiping its unbounded method to `DelayedSample`.")
reader = reader.__func__ # The reader object might not be picklable
samples.append(
DelayedSample(
functools.partial(reader, None, candidate), parent=s
)
)
else:
# if checkpointing is not required, load the data and preprocess it
# as we would normally do
......
......@@ -7,8 +7,11 @@ import numpy
import os
import bob.io.base
from bob.pipelines.sample.sample import DelayedSample, SampleSet, Sample
from bob.pipelines.utils import is_picklable
"""Re-usable blocks for legacy bob.bio.base algorithms"""
import logging
logger = logging.getLogger("bob.bio.base")
class SampleLoader:
......@@ -107,10 +110,21 @@ class SampleLoader:
if hasattr(func, "read_data")
else getattr(func, "read_feature")
)
reader = reader.__func__ # The reader object might not be picklable
samples.append(
DelayedSample(functools.partial(reader, None, candidate), parent=s)
)
if is_picklable(reader):
samples.append(
DelayedSample(
functools.partial(reader, candidate), parent=s
)
)
else:
logger.warning(f"The method {func} is not picklable. Shiping its unbounded method to `DelayedSample`.")
reader = reader.__func__ # The reader object might not be picklable
samples.append(
DelayedSample(
functools.partial(reader, None, candidate), parent=s
)
)
else:
# if checkpointing is not required, load the data and preprocess it
# as we would normally do
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment