Skip to content
Snippets Groups Projects
Commit 3a4fae54 authored by Amir MOHAMMADI's avatar Amir MOHAMMADI
Browse files

Merge branch 'datasets' into 'master'

[datasets] remove fieldnames from CSVToSamples

See merge request !95
parents 9b4716e5 2a70da22
Branches
No related tags found
1 merge request!95[datasets] remove fieldnames from CSVToSamples
Pipeline #62048 passed
...@@ -53,22 +53,18 @@ class CSVToSamples(FileListToSamples): ...@@ -53,22 +53,18 @@ class CSVToSamples(FileListToSamples):
self, self,
list_file, list_file,
transformer=None, transformer=None,
fieldnames=None,
dict_reader_kwargs=None, dict_reader_kwargs=None,
**kwargs, **kwargs,
): ):
list_file = _maybe_open_file(list_file, newline="") list_file = _maybe_open_file(list_file, newline="")
super().__init__(list_file=list_file, transformer=transformer, **kwargs) super().__init__(list_file=list_file, transformer=transformer, **kwargs)
self.fieldnames = fieldnames
self.dict_reader_kwargs = dict_reader_kwargs self.dict_reader_kwargs = dict_reader_kwargs
@property @property
def rows(self): def rows(self):
self.list_file.seek(0) self.list_file.seek(0)
kw = self.dict_reader_kwargs or {} kw = self.dict_reader_kwargs or {}
reader = csv.DictReader( reader = csv.DictReader(self.list_file, **kw)
self.list_file, fieldnames=self.fieldnames, **kw
)
return reader return reader
......
...@@ -132,27 +132,27 @@ class DelayedSample(Sample): ...@@ -132,27 +132,27 @@ class DelayedSample(Sample):
self.__running_init__ = True self.__running_init__ = True
# Merge parent's and param's delayed_attributes # Merge parent's and param's delayed_attributes
parent_attr = getattr(parent, "_delayed_attributes", None) parent_attr = getattr(parent, "_delayed_attributes", None)
self._delayed_attributes = ( self._delayed_attributes = None
None if parent_attr is None else parent_attr.copy() if parent_attr is not None:
) self._delayed_attributes = parent_attr.copy()
if (
self._delayed_attributes is not None if delayed_attributes is not None:
and delayed_attributes is not None if self._delayed_attributes is None:
): self._delayed_attributes = delayed_attributes.copy()
self._delayed_attributes.update(delayed_attributes) else:
elif self._delayed_attributes is None: self._delayed_attributes.update(delayed_attributes)
self._delayed_attributes = delayed_attributes
# Inherit attributes from parent, without calling delayed_attributes # Inherit attributes from parent, without calling delayed_attributes
for key in getattr(parent, "__dict__", []): for key in getattr(parent, "__dict__", []):
if ( if key.startswith("_"):
not key.startswith("_") continue
and key not in SAMPLE_DATA_ATTRS if key in SAMPLE_DATA_ATTRS:
and ( continue
self._delayed_attributes is None if self._delayed_attributes is not None:
or key not in self._delayed_attributes if key in self._delayed_attributes:
) continue
): setattr(self, key, getattr(parent, key))
setattr(self, key, getattr(parent, key))
# Create the delayed attributes, but leave their values as None for now. # Create the delayed attributes, but leave their values as None for now.
if self._delayed_attributes is not None: if self._delayed_attributes is not None:
kwargs.update({k: None for k in self._delayed_attributes}) kwargs.update({k: None for k in self._delayed_attributes})
......
...@@ -166,9 +166,8 @@ class AnnotationsLoader(TransformerMixin, BaseEstimator): ...@@ -166,9 +166,8 @@ class AnnotationsLoader(TransformerMixin, BaseEstimator):
) )
annotated_samples.append( annotated_samples.append(
DelayedSample( DelayedSample.from_sample(
x._load, x,
parent=x,
delayed_attributes=dict( delayed_attributes=dict(
annotations=lambda: read_annotation_file( annotations=lambda: read_annotation_file(
annotation_file, self.annotation_type annotation_file, self.annotation_type
......
...@@ -82,28 +82,28 @@ def test_delayed_samples(): ...@@ -82,28 +82,28 @@ def test_delayed_samples():
def load_annot_variant(): def load_annot_variant():
return "annotation_variant" return "annotation_variant"
delayed_attr_read = False
def load_check():
nonlocal delayed_attr_read
delayed_attr_read = True
return "delayed_attr_data"
delayed_sample = DelayedSample( delayed_sample = DelayedSample(
load_data, delayed_attributes=dict(annot=load_annot) load_data, delayed_attributes=dict(annot=load_annot)
) )
assert delayed_sample.data == 0, delayed_sample.data assert delayed_sample.data == 0
assert delayed_sample.annot == "annotation", delayed_sample.annot assert delayed_sample.annot == "annotation"
child_sample = Sample(1, parent=delayed_sample) child_sample = Sample(1, parent=delayed_sample)
assert child_sample.data == 1, child_sample.data assert child_sample.data == 1
assert child_sample.annot == "annotation", child_sample.annot assert child_sample.annot == "annotation"
assert child_sample.__dict__ == { assert child_sample.__dict__ == {
"data": 1, "data": 1,
"annot": "annotation", "annot": "annotation",
}, child_sample.__dict__ }
# Overwriting and adding delayed_attributes to the child # Overwriting and adding delayed_attributes to the child
delayed_attr_read = False
def load_check():
nonlocal delayed_attr_read
delayed_attr_read = True
return "delayed_attr_data"
new_delayed_attr = { new_delayed_attr = {
"annot": load_annot_variant, # Override parent's annot "annot": load_annot_variant, # Override parent's annot
"new_annot": load_annot, # Add the new_annot attribute "new_annot": load_annot, # Add the new_annot attribute
...@@ -114,30 +114,29 @@ def test_delayed_samples(): ...@@ -114,30 +114,29 @@ def test_delayed_samples():
) )
assert delayed_sample._delayed_attributes == dict(annot=load_annot) assert delayed_sample._delayed_attributes == dict(annot=load_annot)
assert child_sample.data == 0, child_sample.data assert child_sample.data == 0
assert child_sample.annot == "annotation_variant", child_sample.annot assert child_sample.annot == "annotation_variant"
assert child_sample.new_annot == "annotation", child_sample.new_annot assert child_sample.new_annot == "annotation"
assert not delayed_attr_read, "delayed attribute has been read early" assert not delayed_attr_read, "delayed attribute has been read early"
assert ( assert child_sample.read_check == "delayed_attr_data"
child_sample.read_check == "delayed_attr_data"
), child_sample.read_check
assert delayed_attr_read, "delayed attribute should have been read by now" assert delayed_attr_read, "delayed attribute should have been read by now"
delayed_sample.annot = "changed" delayed_sample.annot = "changed"
assert delayed_sample.annot == "changed", delayed_sample.annot assert delayed_sample.annot == "changed"
# test DelayedSample.from_smaple # test DelayedSample.from_smaple
# check if a non-delayed sample is correctly converted to a delayed sample # check if a non-delayed sample is correctly converted to a delayed sample
non_delayed_sample = Sample(1) non_delayed_sample = Sample(1)
delayed_sample = DelayedSample.from_sample(non_delayed_sample, annot="test") delayed_sample = DelayedSample.from_sample(non_delayed_sample, annot="test")
assert delayed_sample.data == 1, delayed_sample.data assert delayed_sample.data == 1
assert delayed_sample.annot == "test", delayed_sample.annot assert delayed_sample.annot == "test"
# check if converting a delayed sample will not load the data # check if converting a delayed sample will not load the data and delayed attributes
raise_error = True raise_error = True
def never_load(): def never_load():
nonlocal raise_error
if raise_error: if raise_error:
raise ValueError("never_load should not be called") raise ValueError("never_load should not be called")
return 0 return 0
...@@ -149,5 +148,5 @@ def test_delayed_samples(): ...@@ -149,5 +148,5 @@ def test_delayed_samples():
child_sample = DelayedSample.from_sample(delayed_sample) child_sample = DelayedSample.from_sample(delayed_sample)
raise_error = False raise_error = False
assert child_sample.data == 0, child_sample.data assert child_sample.data == 0
assert child_sample.annot == 0, child_sample.annot assert child_sample.annot == 0
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment