Skip to content

GitLab

Commit bd592d5a authored by Tiago de Freitas Pereira's avatar Tiago de Freitas Pereira
Browse files

[db] implementing MEDS database in the new interface

parent a8a26357
Hide whitespace changes
Inline Side-by-side
bob/bio/face/database/__init__.py
View file @ bd592d5a
......@@ -12,13 +12,13 @@ from .multipie import MultipieBioDatabase
from .ijbc import IJBCBioDatabase
from .replaymobile import ReplayMobileBioDatabase
from .fargo import FargoBioDatabase
from .meds import MEDSDatabase
# gets sphinx autodoc done right - don't remove it
def __appropriate__(*args):
"""Says object was actually declared here, and not in the import module.
"""Says object was actually declared here, and not in the import module.
Fixing sphinx warnings of not being able to find classes, when path is shortened.
Parameters:
......@@ -28,20 +28,21 @@ def __appropriate__(*args):
<https://github.com/sphinx-doc/sphinx/issues/3048>`
"""
for obj in args:
obj.__module__ = __name__
for obj in args:
obj.__module__ = __name__
__appropriate__(
FaceBioFile,
MobioBioDatabase,
ReplayBioDatabase,
AtntBioDatabase,
GBUBioDatabase,
ARFaceBioDatabase,
LFWBioDatabase,
MultipieBioDatabase,
IJBCBioDatabase,
ReplayMobileBioDatabase,
FargoBioDatabase
FaceBioFile,
MobioBioDatabase,
ReplayBioDatabase,
AtntBioDatabase,
GBUBioDatabase,
ARFaceBioDatabase,
LFWBioDatabase,
MultipieBioDatabase,
IJBCBioDatabase,
ReplayMobileBioDatabase,
FargoBioDatabase,
)
__all__ = [_ for _ in dir() if not _.startswith('_')]
__all__ = [_ for _ in dir() if not _.startswith("_")]
bob/bio/face/database/meds.py 0 → 100644
View file @ bd592d5a
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
"""
MEDS database implementation
"""
from bob.bio.base.database import CSVDatasetDevEval, CSVToSampleLoader
from bob.extension import rc
import bob.io.base
from bob.bio.face.database.sample_loaders import EyesAnnotationsLoader
# TODO: POINT TO THE `.bob/meds``
dataset_protocol_path = "/idiap/user/tpereira/gitlab/bob/bob.nightlies/meds"
class MEDSDatabase(CSVDatasetDevEval):
def __init__(
self,
protocol,
dataset_protocol_path=dataset_protocol_path,
csv_to_sample_loader=CSVToSampleLoader(
data_loader=bob.io.base.load,
metadata_loader=EyesAnnotationsLoader(),
dataset_original_directory=rc["bob.db.meds.directory"],
extension=".jpg",
),
):
# TODO: IMPLEMENT THE DOWNLOAD MECHANISM
super().__init__(dataset_protocol_path, protocol, csv_to_sample_loader)
bob/bio/face/database/sample_loaders.py
View file @ bd592d5a
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
""" Sample loader"""
""" Sample and Metatada loader"""
from bob.bio.base.database import CSVToSampleLoader
from bob.pipelines import Sample, DelayedSample, SampleSet
import functools
import os
class CSVToSampleLoaderEyesAnnotations(CSVToSampleLoader):
class EyesAnnotationsLoader:
"""
Convert CSV files in the format below to either a list of
:any:`bob.pipelines.DelayedSample` or :any:`bob.pipelines.SampleSet`
Convert leye_x, leye_y, reye_x, reye_y attributes to `annotations = (leye, reye)`
"""
def convert_row_to_sample(self, row, header):
path = row[0]
subject = row[1]
kwargs = dict([[h, r] for h, r in zip(header[2:], row[2:])])
annotations = {
"leye": (kwargs["leye_x"], kwargs["leye_y"]),
"reye": (kwargs["reye_x"], kwargs["reye_y"]),
def __call__(self, row, header=None):
def find_attribute(attribute):
for i, a in enumerate(header):
if a == attribute:
return i
else:
ValueError(f"Attribute not found in the dataset: {a}")
eyes = {
"leye": (row[find_attribute("leye_x")], row[find_attribute("leye_y")]),
"reye": (row[find_attribute("reye_x")], row[find_attribute("reye_y")]),
}
kwargs.pop("leye_x")
kwargs.pop("leye_y")
kwargs.pop("reye_x")
kwargs.pop("reye_y")
return DelayedSample(
functools.partial(
self.data_loader,
os.path.join(self.dataset_original_directory, path + self.extension),
),
key=path,
subject=subject,
annotations=annotations,
**kwargs,
)
"""
class CSVToSampleLoaderEyesAnnotations(CSVToSampleLoader):
def __call__(self, filename):
import ipdb
ipdb.set_trace()
samples = super(CSVToSampleLoaderEyesAnnotations, self).__call__(filename)
def generate_annotations(sample):
Convert leye_x, leye_y, reye_x, reye_y attributes to
`annotations = (leye, reye)`
check_keys = [
a in (sample.__dict__.keys())
for a in ["leye_x", "leye_y", "reye_x", "reye_y"]
]
if not check_keys:
raise ValueError(
"Sample needs to contain the following annotations: 'leye_x', 'leye_y', 'reye_x', 'reye_y'"
)
annotations = {
"leye": (sample.leye_x, sample.leye_y),
"reye": (sample.reye_x, sample.reye_y),
}
# Changing the state of samples for efficiency
# We might have a gigantic amount of datasets
sample.__dict__.pop("leye_x")
sample.__dict__.pop("leye_y")
sample.__dict__.pop("reye_x")
sample.__dict__.pop("reye_y")
sample.annotations = annotations
for sample in samples:
generate_annotations(sample)
annotation = {"annotations": eyes}
return samples
"""
return annotation
bob/bio/face/test/test_databases.py
View file @ bd592d5a
......@@ -22,16 +22,23 @@ from nose.plugins.skip import SkipTest
import bob.bio.base
from bob.bio.base.test.utils import db_available
from bob.bio.base.test.test_database_implementations import check_database, check_database_zt
from bob.bio.base.test.test_database_implementations import (
check_database,
check_database_zt,
)
import bob.core
logger = bob.core.log.setup("bob.bio.face")
def _check_annotations(database, topleft=False, required=True, limit_files=None, framed=False):
def _check_annotations(
database, topleft=False, required=True, limit_files=None, framed=False
):
database_legacy = database.database
files = database_legacy.all_files()
if limit_files is not None:
import random
files = random.sample(files, limit_files)
found_none = False
......@@ -45,118 +52,168 @@ def _check_annotations(database, topleft=False, required=True, limit_files=None,
# take one of the frames
annotations = annotations[list(annotations.keys())[0]]
if topleft:
assert 'topleft' in annotations
assert 'bottomright' in annotations
assert "topleft" in annotations
assert "bottomright" in annotations
else:
assert 'reye' in annotations
assert 'leye' in annotations
assert "reye" in annotations
assert "leye" in annotations
else:
found_none = True
if found_none:
logger.warn("Some annotations were None for {}".format(database_legacy.name))
@db_available('arface')
@db_available("arface")
def test_arface():
database = bob.bio.base.load_resource(
'arface', 'database', preferred_package='bob.bio.face')
"arface", "database", preferred_package="bob.bio.face"
)
try:
check_database(database, groups=('dev', 'eval'))
check_database(database, groups=("dev", "eval"))
except IOError as e:
raise SkipTest(
"The database could not queried; probably the db.sql3 file is missing. Here is the error: '%s'" % e)
"The database could not queried; probably the db.sql3 file is missing. Here is the error: '%s'"
% e
)
try:
_check_annotations(database)
except IOError as e:
raise SkipTest(
"The annotations could not be queried; probably the annotation files are missing. Here is the error: '%s'" % e)
"The annotations could not be queried; probably the annotation files are missing. Here is the error: '%s'"
% e
)
@db_available('atnt')
@db_available("atnt")
def test_atnt():
database = bob.bio.base.load_resource(
'atnt', 'database', preferred_package='bob.bio.face')
"atnt", "database", preferred_package="bob.bio.face"
)
try:
check_database(database)
except IOError as e:
raise SkipTest(
"The database could not queried; probably the db.sql3 file is missing. Here is the error: '%s'" % e)
"The database could not queried; probably the db.sql3 file is missing. Here is the error: '%s'"
% e
)
@db_available('gbu')
@db_available("gbu")
def test_gbu():
database = bob.bio.base.load_resource(
'gbu', 'database', preferred_package='bob.bio.face')
"gbu", "database", preferred_package="bob.bio.face"
)
try:
check_database(database, models_depend=True)
check_database(database, protocol='Bad', models_depend=True)
check_database(database, protocol='Ugly', models_depend=True)
check_database(database, protocol="Bad", models_depend=True)
check_database(database, protocol="Ugly", models_depend=True)
except IOError as e:
raise SkipTest(
"The database could not queried; probably the db.sql3 file is missing. Here is the error: '%s'" % e)
"The database could not queried; probably the db.sql3 file is missing. Here is the error: '%s'"
% e
)
try:
_check_annotations(database, limit_files=1000)
except IOError as e:
raise SkipTest(
"The annotations could not be queried; probably the annotation files are missing. Here is the error: '%s'" % e)
"The annotations could not be queried; probably the annotation files are missing. Here is the error: '%s'"
% e
)
@db_available('lfw')
@db_available("lfw")
def test_lfw():
database = bob.bio.base.load_resource(
'lfw-restricted', 'database', preferred_package='bob.bio.face')
"lfw-restricted", "database", preferred_package="bob.bio.face"
)
try:
check_database(database, training_depends=True, models_depend=True)
check_database(database, groups=('dev', 'eval'),
protocol='fold1', training_depends=True, models_depend=True)
check_database(bob.bio.base.load_resource('lfw-unrestricted', 'database', preferred_package='bob.bio.face'),
training_depends=True, models_depend=True)
check_database(
database,
groups=("dev", "eval"),
protocol="fold1",
training_depends=True,
models_depend=True,
)
check_database(
bob.bio.base.load_resource(
"lfw-unrestricted", "database", preferred_package="bob.bio.face"
),
training_depends=True,
models_depend=True,
)
except IOError as e:
raise SkipTest(
"The database could not queried; probably the db.sql3 file is missing. Here is the error: '%s'" % e)
"The database could not queried; probably the db.sql3 file is missing. Here is the error: '%s'"
% e
)
try:
_check_annotations(database, limit_files=1000)
except IOError as e:
raise SkipTest(
"The annotations could not be queried; probably the annotation files are missing. Here is the error: '%s'" % e)
"The annotations could not be queried; probably the annotation files are missing. Here is the error: '%s'"
% e
)
@db_available('mobio')
@db_available("mobio")
def test_mobio():
database = bob.bio.base.load_resource(
'mobio-image', 'database', preferred_package='bob.bio.face')
"mobio-image", "database", preferred_package="bob.bio.face"
)
try:
check_database_zt(database, models_depend=True)
check_database_zt(database, protocol='female', models_depend=True)
check_database_zt(bob.bio.base.load_resource('mobio-male', 'database', preferred_package='bob.bio.face'),
models_depend=True)
check_database_zt(bob.bio.base.load_resource('mobio-female', 'database', preferred_package='bob.bio.face'),
models_depend=True)
check_database_zt(database, protocol="female", models_depend=True)
check_database_zt(
bob.bio.base.load_resource(
"mobio-male", "database", preferred_package="bob.bio.face"
),
models_depend=True,
)
check_database_zt(
bob.bio.base.load_resource(
"mobio-female", "database", preferred_package="bob.bio.face"
),
models_depend=True,
)
except IOError as e:
raise SkipTest(
"The database could not be queried; probably the db.sql3 file is missing. Here is the error: '%s'" % e)
"The database could not be queried; probably the db.sql3 file is missing. Here is the error: '%s'"
% e
)
try:
_check_annotations(database, limit_files=1000)
except IOError as e:
raise SkipTest(
"The annotations could not be queried; probably the annotation files are missing. Here is the error: '%s'" % e)
"The annotations could not be queried; probably the annotation files are missing. Here is the error: '%s'"
% e
)
@db_available('multipie')
@db_available("multipie")
def test_multipie():
database = bob.bio.base.load_resource(
'multipie', 'database', preferred_package='bob.bio.face')
"multipie", "database", preferred_package="bob.bio.face"
)
try:
check_database_zt(database, training_depends=True)
check_database_zt(bob.bio.base.load_resource('multipie-pose', 'database', preferred_package='bob.bio.face'),
training_depends=True)
check_database_zt(
bob.bio.base.load_resource(
"multipie-pose", "database", preferred_package="bob.bio.face"
),
training_depends=True,
)
except IOError as e:
raise SkipTest(
"The database could not queried; probably the db.sql3 file is missing. Here is the error: '%s'" % e)
"The database could not queried; probably the db.sql3 file is missing. Here is the error: '%s'"
% e
)
except ValueError as e:
raise SkipTest(
"The database could not queried; probably the protocol is missing inside the db.sql3 file. Here is the error: '%s'" % e)
"The database could not queried; probably the protocol is missing inside the db.sql3 file. Here is the error: '%s'"
% e
)
try:
if database.database.annotation_directory is None:
......@@ -164,94 +221,130 @@ def test_multipie():
_check_annotations(database)
except IOError as e:
raise SkipTest(
"The annotations could not be queried; probably the annotation files are missing. Here is the error: '%s'" % e)
"The annotations could not be queried; probably the annotation files are missing. Here is the error: '%s'"
% e
)
@db_available('replay')
@db_available("replay")
def test_replay_licit():
database = bob.bio.base.load_resource(
'replay-img-licit', 'database', preferred_package='bob.bio.face')
"replay-img-licit", "database", preferred_package="bob.bio.face"
)
try:
check_database(database, groups=('dev', 'eval'))
check_database(database, groups=("dev", "eval"))
except IOError as e:
raise SkipTest(
"The database could not be queried; probably the db.sql3 file is missing. Here is the error: '%s'" % e)
"The database could not be queried; probably the db.sql3 file is missing. Here is the error: '%s'"
% e
)
try:
_check_annotations(database, topleft=True)
except IOError as e:
raise SkipTest(
"The annotations could not be queried; probably the annotation files are missing. Here is the error: '%s'" % e)
"The annotations could not be queried; probably the annotation files are missing. Here is the error: '%s'"
% e
)
@db_available('replay')
@db_available("replay")
def test_replay_spoof():
database = bob.bio.base.load_resource(
'replay-img-spoof', 'database', preferred_package='bob.bio.face')
"replay-img-spoof", "database", preferred_package="bob.bio.face"
)
try:
check_database(database, groups=('dev', 'eval'))
check_database(database, groups=("dev", "eval"))
except IOError as e:
raise SkipTest(
"The database could not be queried; probably the db.sql3 file is missing. Here is the error: '%s'" % e)
"The database could not be queried; probably the db.sql3 file is missing. Here is the error: '%s'"
% e
)
try:
_check_annotations(database, topleft=True)
except IOError as e:
raise SkipTest(
"The annotations could not be queried; probably the annotation files are missing. Here is the error: '%s'" % e)
"The annotations could not be queried; probably the annotation files are missing. Here is the error: '%s'"
% e
)
@db_available('replaymobile')
@db_available("replaymobile")
def test_replaymobile_licit():
database = bob.bio.base.load_resource(
'replaymobile-img-licit', 'database', preferred_package='bob.bio.face')
"replaymobile-img-licit", "database", preferred_package="bob.bio.face"
)
try:
check_database(database, groups=('dev', 'eval'))
check_database(database, groups=("dev", "eval"))
except IOError as e:
raise SkipTest(
"The database could not be queried; probably the db.sql3 file is missing. Here is the error: '%s'" % e)
"The database could not be queried; probably the db.sql3 file is missing. Here is the error: '%s'"
% e
)
try:
_check_annotations(database, topleft=True, limit_files=20)
except IOError as e:
raise SkipTest(
"The annotations could not be queried; probably the annotation files are missing. Here is the error: '%s'" % e)
"The annotations could not be queried; probably the annotation files are missing. Here is the error: '%s'"
% e
)
@db_available('replaymobile')
@db_available("replaymobile")
def test_replaymobile_spoof():
database = bob.bio.base.load_resource(
'replaymobile-img-spoof', 'database', preferred_package='bob.bio.face')
"replaymobile-img-spoof", "database", preferred_package="bob.bio.face"
)
try:
check_database(database, groups=('dev', 'eval'))
check_database(database, groups=("dev", "eval"))
except IOError as e:
raise SkipTest(
"The database could not be queried; probably the db.sql3 file is missing. Here is the error: '%s'" % e)
"The database could not be queried; probably the db.sql3 file is missing. Here is the error: '%s'"
% e
)
try:
_check_annotations(database, topleft=True, limit_files=20)
except IOError as e:
raise SkipTest(
"The annotations could not be queried; probably the annotation files are missing. Here is the error: '%s'" % e)
"The annotations could not be queried; probably the annotation files are missing. Here is the error: '%s'"
% e
)
@db_available('ijbc')
@db_available("ijbc")
def test_ijbc():
database = bob.bio.base.load_resource(
'ijbc-11', 'database', preferred_package='bob.bio.face')
"ijbc-11", "database", preferred_package="bob.bio.face"
)
try:
check_database(database, models_depend=True, training_depends=True)
except IOError as e:
raise SkipTest(
"The database could not queried; Here is the error: '%s'" % e)
raise SkipTest("The database could not queried; Here is the error: '%s'" % e)
try:
_check_annotations(database, topleft=True, limit_files=1000)
except IOError as e:
raise SkipTest(
"The annotations could not be queried; probably the annotation files are missing. Here is the error: '%s'" % e)
"The annotations could not be queried; probably the annotation files are missing. Here is the error: '%s'"
% e
)
@db_available('fargo')
@db_available("fargo")
def test_fargo():
database = bob.bio.base.load_resource(
'fargo', 'database', preferred_package='bob.bio.face')
"fargo", "database", preferred_package="bob.bio.face"
)
try:
check_database(database)
except IOError as e:
raise SkipTest(
"The database could not queried; Here is the error: '%s'" % e)
raise SkipTest("The database could not queried; Here is the error: '%s'" % e)
def test_meds():
from bob.bio.face.database import MEDSDatabase
database = MEDSDatabase("verification_fold1")
assert len(database.background_model_samples()) == 234
assert len(database.references()) == 223 // 2
assert len(database.probes()) == 313
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment