From 072055bfbb805767d3f26f03911e9e0a64e12abd Mon Sep 17 00:00:00 2001
From: Tiago Freitas Pereira <tiagofrepereira@gmail.com>
Date: Wed, 26 Feb 2020 13:19:22 +0100
Subject: [PATCH] Added pickle test
---
.../vanilla_biometrics/annotated_legacy.py | 29 ++++++++++++++-----
.../pipelines/vanilla_biometrics/blocks.py | 22 +++++++++++---
2 files changed, 39 insertions(+), 12 deletions(-)
diff --git a/bob/bio/base/pipelines/vanilla_biometrics/annotated_legacy.py b/bob/bio/base/pipelines/vanilla_biometrics/annotated_legacy.py
index 25392a25..91f8d487 100644
--- a/bob/bio/base/pipelines/vanilla_biometrics/annotated_legacy.py
+++ b/bob/bio/base/pipelines/vanilla_biometrics/annotated_legacy.py
@@ -15,6 +15,10 @@ import bob.io.base
from .legacy import DatabaseConnector
from .blocks import SampleLoader
from bob.pipelines.sample.sample import SampleSet, DelayedSample, Sample
+from bob.pipelines.utils import is_picklable
+
+import logging
+logger = logging.getLogger("bob.bio.base")
class DatabaseConnectorAnnotated(DatabaseConnector):
@@ -293,19 +297,28 @@ class SampleLoaderAnnotated(SampleLoader):
# because we are checkpointing, we return a DelayedSample
# instead of normal (preloaded) sample. This allows the next
# phase to avoid loading it would it be unnecessary (e.g. next
- # phase is already check-pointed)
- #reader = bob.io.base.load
+ # phase is already check-pointed)
reader = (
getattr(func, "read_data")
if hasattr(func, "read_data")
else getattr(func, "read_feature")
- )
- reader = reader.__func__ # The reader object might not be picklable
- samples.append(
- DelayedSample(
- functools.partial(reader, None, candidate), parent=s
+ )
+ if is_picklable(reader):
+ samples.append(
+ DelayedSample(
+ functools.partial(reader, candidate), parent=s
+ )
)
- )
+ else:
+ logger.warning(f"The method {reader} is not picklable. Shiping its unbounded method to `DelayedSample`.")
+ reader = reader.__func__ # The reader object might not be picklable
+
+ samples.append(
+ DelayedSample(
+ functools.partial(reader, None, candidate), parent=s
+ )
+ )
+
else:
# if checkpointing is not required, load the data and preprocess it
# as we would normally do
diff --git a/bob/bio/base/pipelines/vanilla_biometrics/blocks.py b/bob/bio/base/pipelines/vanilla_biometrics/blocks.py
index 0e0c9b7f..88390c13 100644
--- a/bob/bio/base/pipelines/vanilla_biometrics/blocks.py
+++ b/bob/bio/base/pipelines/vanilla_biometrics/blocks.py
@@ -7,8 +7,11 @@ import numpy
import os
import bob.io.base
from bob.pipelines.sample.sample import DelayedSample, SampleSet, Sample
+from bob.pipelines.utils import is_picklable
"""Re-usable blocks for legacy bob.bio.base algorithms"""
+import logging
+logger = logging.getLogger("bob.bio.base")
class SampleLoader:
@@ -107,10 +110,21 @@ class SampleLoader:
if hasattr(func, "read_data")
else getattr(func, "read_feature")
)
- reader = reader.__func__ # The reader object might not be picklable
- samples.append(
- DelayedSample(functools.partial(reader, None, candidate), parent=s)
- )
+ if is_picklable(reader):
+ samples.append(
+ DelayedSample(
+ functools.partial(reader, candidate), parent=s
+ )
+ )
+ else:
+ logger.warning(f"The method {func} is not picklable. Shiping its unbounded method to `DelayedSample`.")
+ reader = reader.__func__ # The reader object might not be picklable
+
+ samples.append(
+ DelayedSample(
+ functools.partial(reader, None, candidate), parent=s
+ )
+ )
else:
# if checkpointing is not required, load the data and preprocess it
# as we would normally do
--
GitLab