Skip to content
Snippets Groups Projects
Commit d647dc76 authored by Tiago de Freitas Pereira's avatar Tiago de Freitas Pereira
Browse files

Merge branch 'legacydb' into 'master'

Solving possible race condition issues with certain datasets

See merge request !256
parents 86647b4d 6e862353
No related branches found
No related tags found
1 merge request!256Solving possible race condition issues with certain datasets
Pipeline #52227 failed
...@@ -57,16 +57,14 @@ def get_temp_directory(sub_dir): ...@@ -57,16 +57,14 @@ def get_temp_directory(sub_dir):
return os.path.join(tempfile.TemporaryDirectory().name, sub_dir) return os.path.join(tempfile.TemporaryDirectory().name, sub_dir)
def _biofile_to_delayed_sample(biofile, database): def _biofile_to_delayed_sample(biofile, database, purpose="probe"):
return DelayedSample( return DelayedSample(
load=functools.partial( load=functools.partial(
biofile.load, biofile.load, database.original_directory, database.original_extension,
database.original_directory,
database.original_extension,
), ),
reference_id=str(biofile.client_id), reference_id=str(biofile.client_id),
key=biofile.path, key=f"{biofile.path}{purpose}",
path=biofile.path, path=f"{biofile.path}{purpose}",
delayed_attributes=dict( delayed_attributes=dict(
annotations=functools.partial(database.annotations, biofile) annotations=functools.partial(database.annotations, biofile)
), ),
...@@ -107,6 +105,10 @@ class DatabaseConnector(Database): ...@@ -107,6 +105,10 @@ class DatabaseConnector(Database):
memory_demanding: bool memory_demanding: bool
Sinalizes that a database has some memory demanding components. Sinalizes that a database has some memory demanding components.
It might be useful for future processing It might be useful for future processing
append_purpose: bool
If True, `sample.key` will be appended with the purpose of the sample (world, probe, or bio-ref).
""" """
def __init__( def __init__(
...@@ -116,6 +118,7 @@ class DatabaseConnector(Database): ...@@ -116,6 +118,7 @@ class DatabaseConnector(Database):
annotation_type="eyes-center", annotation_type="eyes-center",
fixed_positions=None, fixed_positions=None,
memory_demanding=False, memory_demanding=False,
append_purpose=False,
**kwargs, **kwargs,
): ):
super().__init__( super().__init__(
...@@ -127,6 +130,7 @@ class DatabaseConnector(Database): ...@@ -127,6 +130,7 @@ class DatabaseConnector(Database):
memory_demanding=memory_demanding, memory_demanding=memory_demanding,
**kwargs, **kwargs,
) )
self.append_purpose = append_purpose
self.database = database self.database = database
def background_model_samples(self): def background_model_samples(self):
...@@ -140,7 +144,12 @@ class DatabaseConnector(Database): ...@@ -140,7 +144,12 @@ class DatabaseConnector(Database):
model training. model training.
""" """
objects = self.database.training_files() objects = self.database.training_files()
return [_biofile_to_delayed_sample(k, self.database) for k in objects] return [
_biofile_to_delayed_sample(
k, self.database, purpose="world" if self.append_purpose else ""
)
for k in objects
]
def references(self, group="dev"): def references(self, group="dev"):
"""Returns references to enroll biometric references """Returns references to enroll biometric references
...@@ -167,7 +176,14 @@ class DatabaseConnector(Database): ...@@ -167,7 +176,14 @@ class DatabaseConnector(Database):
retval.append( retval.append(
SampleSet( SampleSet(
[_biofile_to_delayed_sample(k, self.database) for k in objects], [
_biofile_to_delayed_sample(
k,
self.database,
purpose="bio-ref" if self.append_purpose else "",
)
for k in objects
],
key=str(m), key=str(m),
path=str(m), path=str(m),
reference_id=(str(m)), reference_id=(str(m)),
...@@ -204,7 +220,13 @@ class DatabaseConnector(Database): ...@@ -204,7 +220,13 @@ class DatabaseConnector(Database):
for o in objects: for o in objects:
if o.id not in probes: if o.id not in probes:
probes[o.id] = SampleSet( probes[o.id] = SampleSet(
[_biofile_to_delayed_sample(o, self.database)], [
_biofile_to_delayed_sample(
o,
self.database,
purpose="probe" if self.append_purpose else "",
)
],
key=str(o.path), key=str(o.path),
path=o.path, path=o.path,
reference_id=str(m), reference_id=str(m),
...@@ -248,7 +270,12 @@ class DatabaseConnector(Database): ...@@ -248,7 +270,12 @@ class DatabaseConnector(Database):
high_level_names=["train", "dev", "eval"], high_level_names=["train", "dev", "eval"],
) )
objects = self.database.all_files(groups=low_level_groups) objects = self.database.all_files(groups=low_level_groups)
return [_biofile_to_delayed_sample(k, self.database) for k in objects] return [
_biofile_to_delayed_sample(
k, self.database, "all" if self.append_purpose else ""
)
for k in objects
]
def groups(self): def groups(self):
grps = self.database.groups() grps = self.database.groups()
...@@ -288,12 +315,7 @@ class BioAlgorithmLegacy(BioAlgorithm): ...@@ -288,12 +315,7 @@ class BioAlgorithmLegacy(BioAlgorithm):
""" """
def __init__( def __init__(
self, self, instance, base_dir, force=False, projector_file=None, **kwargs,
instance,
base_dir,
force=False,
projector_file=None,
**kwargs,
): ):
super().__init__(**kwargs) super().__init__(**kwargs)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment