Merge branch 'datasets' into 'master'

[datasets] remove fieldnames from CSVToSamples

See merge request !95

bob/pipelines/datasets.py

@@ -53,22 +53,18 @@ class CSVToSamples(FileListToSamples):
         self,
         list_file,
         transformer=None,
-        fieldnames=None,
         dict_reader_kwargs=None,
         **kwargs,
     ):
         list_file = _maybe_open_file(list_file, newline="")
         super().__init__(list_file=list_file, transformer=transformer, **kwargs)
-        self.fieldnames = fieldnames
         self.dict_reader_kwargs = dict_reader_kwargs
 
     @property
     def rows(self):
         self.list_file.seek(0)
         kw = self.dict_reader_kwargs or {}
-        reader = csv.DictReader(
-            self.list_file, fieldnames=self.fieldnames, **kw
-        )
+        reader = csv.DictReader(self.list_file, **kw)
         return reader
 
 

bob/pipelines/sample.py

@@ -132,27 +132,27 @@ class DelayedSample(Sample):
         self.__running_init__ = True
         # Merge parent's and param's delayed_attributes
         parent_attr = getattr(parent, "_delayed_attributes", None)
-        self._delayed_attributes = (
-            None if parent_attr is None else parent_attr.copy()
-        )
-        if (
-            self._delayed_attributes is not None
-            and delayed_attributes is not None
-        ):
-            self._delayed_attributes.update(delayed_attributes)
-        elif self._delayed_attributes is None:
-            self._delayed_attributes = delayed_attributes
+        self._delayed_attributes = None
+        if parent_attr is not None:
+            self._delayed_attributes = parent_attr.copy()
+
+        if delayed_attributes is not None:
+            if self._delayed_attributes is None:
+                self._delayed_attributes = delayed_attributes.copy()
+            else:
+                self._delayed_attributes.update(delayed_attributes)
+
         # Inherit attributes from parent, without calling delayed_attributes
         for key in getattr(parent, "__dict__", []):
-            if (
-                not key.startswith("_")
-                and key not in SAMPLE_DATA_ATTRS
-                and (
-                    self._delayed_attributes is None
-                    or key not in self._delayed_attributes
-                )
-            ):
-                setattr(self, key, getattr(parent, key))
+            if key.startswith("_"):
+                continue
+            if key in SAMPLE_DATA_ATTRS:
+                continue
+            if self._delayed_attributes is not None:
+                if key in self._delayed_attributes:
+                    continue
+            setattr(self, key, getattr(parent, key))
+
         # Create the delayed attributes, but leave their values as None for now.
         if self._delayed_attributes is not None:
             kwargs.update({k: None for k in self._delayed_attributes})

bob/pipelines/sample_loaders.py

@@ -166,9 +166,8 @@ class AnnotationsLoader(TransformerMixin, BaseEstimator):
             )
 
             annotated_samples.append(
-                DelayedSample(
-                    x._load,
-                    parent=x,
+                DelayedSample.from_sample(
+                    x,
                     delayed_attributes=dict(
                         annotations=lambda: read_annotation_file(
                             annotation_file, self.annotation_type

bob/pipelines/tests/test_samples.py

@@ -82,28 +82,28 @@ def test_delayed_samples():
     def load_annot_variant():
         return "annotation_variant"
 
-    delayed_attr_read = False
-
-    def load_check():
-        nonlocal delayed_attr_read
-        delayed_attr_read = True
-        return "delayed_attr_data"
-
     delayed_sample = DelayedSample(
         load_data, delayed_attributes=dict(annot=load_annot)
     )
-    assert delayed_sample.data == 0, delayed_sample.data
-    assert delayed_sample.annot == "annotation", delayed_sample.annot
+    assert delayed_sample.data == 0
+    assert delayed_sample.annot == "annotation"
 
     child_sample = Sample(1, parent=delayed_sample)
-    assert child_sample.data == 1, child_sample.data
-    assert child_sample.annot == "annotation", child_sample.annot
+    assert child_sample.data == 1
+    assert child_sample.annot == "annotation"
     assert child_sample.__dict__ == {
         "data": 1,
         "annot": "annotation",
-    }, child_sample.__dict__
+    }
 
     # Overwriting and adding delayed_attributes to the child
+    delayed_attr_read = False
+
+    def load_check():
+        nonlocal delayed_attr_read
+        delayed_attr_read = True
+        return "delayed_attr_data"
+
     new_delayed_attr = {
         "annot": load_annot_variant,  # Override parent's annot
         "new_annot": load_annot,  # Add the new_annot attribute
@@ -114,30 +114,29 @@ def test_delayed_samples():
     )
 
     assert delayed_sample._delayed_attributes == dict(annot=load_annot)
-    assert child_sample.data == 0, child_sample.data
-    assert child_sample.annot == "annotation_variant", child_sample.annot
-    assert child_sample.new_annot == "annotation", child_sample.new_annot
+    assert child_sample.data == 0
+    assert child_sample.annot == "annotation_variant"
+    assert child_sample.new_annot == "annotation"
     assert not delayed_attr_read, "delayed attribute has been read early"
-    assert (
-        child_sample.read_check == "delayed_attr_data"
-    ), child_sample.read_check
+    assert child_sample.read_check == "delayed_attr_data"
     assert delayed_attr_read, "delayed attribute should have been read by now"
 
     delayed_sample.annot = "changed"
-    assert delayed_sample.annot == "changed", delayed_sample.annot
+    assert delayed_sample.annot == "changed"
 
     # test DelayedSample.from_smaple
 
     # check if a non-delayed sample is correctly converted to a delayed sample
     non_delayed_sample = Sample(1)
     delayed_sample = DelayedSample.from_sample(non_delayed_sample, annot="test")
-    assert delayed_sample.data == 1, delayed_sample.data
-    assert delayed_sample.annot == "test", delayed_sample.annot
+    assert delayed_sample.data == 1
+    assert delayed_sample.annot == "test"
 
-    # check if converting a delayed sample will not load the data
+    # check if converting a delayed sample will not load the data and delayed attributes
     raise_error = True
 
     def never_load():
+        nonlocal raise_error
         if raise_error:
             raise ValueError("never_load should not be called")
         return 0
@@ -149,5 +148,5 @@ def test_delayed_samples():
     child_sample = DelayedSample.from_sample(delayed_sample)
 
     raise_error = False
-    assert child_sample.data == 0, child_sample.data
-    assert child_sample.annot == 0, child_sample.annot
+    assert child_sample.data == 0
+    assert child_sample.annot == 0