Merge branch 'ijbc' into 'master'

Ported IJBC database

See merge request !116

bob/bio/face/config/database/ijbc.py

 #!/usr/bin/env python
 
-from bob.bio.face.database import IJBCBioDatabase
-from bob.bio.base.pipelines.vanilla_biometrics import DatabaseConnector
+from bob.bio.face.database import IJBCDatabase
 from bob.extension import rc
 
-
-ijbc_directory = rc["bob.db.ijbc.directory"]
-
-database = DatabaseConnector(
-    IJBCBioDatabase(original_directory=ijbc_directory, protocol="1:1"),
-    annotation_type = "eyes-center",
-    fixed_positions = None,
-)
-
-#ijbc_covariates = DatabaseConnector(
-#    IJBCBioDatabase(
-#        original_directory=ijbc_directory, protocol="Covariates"
-#    )
-#)
+database = IJBCDatabase()

bob/bio/face/database/__init__.py

@@ -9,7 +9,7 @@ from .gbu import GBUBioDatabase
 from .arface import ARFaceBioDatabase
 from .lfw import LFWBioDatabase
 from .multipie import MultipieDatabase
-from .ijbc import IJBCBioDatabase
+from .ijbc import IJBCDatabase
 from .replaymobile import ReplayMobileBioDatabase
 from .fargo import FargoBioDatabase
 from .meds import MEDSDatabase

bob/bio/face/database/ijbc.py

 #!/usr/bin/env python
 # vim: set fileencoding=utf-8 :
 # Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
-# Sat 20 Aug 15:43:10 CEST 2016
+# Sat 20 Aug 15:43:10 CEST 2020
 
-"""
-  IJBC database implementation of bob.bio.base.database.BioDatabase interface.
-  It is an extension of the database interface, which directly talks to IJBC database, for
-  verification experiments (good to use in bob.bio.base framework).
-"""
-
-from .database import FaceBioFile
-from bob.bio.base.database import BioDatabase, BioFileSet
+from bob.pipelines.utils import hash_string
+from bob.extension.download import get_file, find_element_in_tarball
+import pickle
 import os
-import bob.io.image
-
-
-class IJBCBioFile(FaceBioFile):
-    def __init__(self, f):
-        super(IJBCBioFile, self).__init__(
-            client_id=f.client_id, path=f.path, file_id=f.id)
-        self.f = f
-
-    def load(self, directory, extension=None, add_client_id=False):
-        return bob.io.image.load(self.make_path(directory, self.f.extension, add_client_id))
-
-    def make_path(self, directory, extension, add_client_id=True):
-        if add_client_id:
-            # add client ID to the path, so that a unique path is generated
-            # (there might be several identities in each physical file)
-            path = "%s-%s%s" % (self.path, self.client_id, extension or '')
-        else:
-            # do not add the client ID to be able to obtain the original image file
-            path = "%s%s" % (self.path,  extension or '')
-
-        return str(os.path.join(directory or '', path))
-
-
-class IJBCBioFileSet(BioFileSet):
-    def __init__(self, template):
-        super(IJBCBioFileSet, self).__init__(file_set_id=template.id, files=[
-            IJBCBioFile(f) for f in template.files], path=template.path)
-
-
-class IJBCBioDatabase(BioDatabase):
-    """
-      IJBC database implementation of :py:class:`bob.bio.base.database.BioDatabase` interface.
-      It is an extension of an SQL-based database interface, which directly talks to IJBC database, for
-      verification experiments (good to use in bob.bio.base framework).
-    """
-
-    def __init__(
-        self,
-        original_directory=None,
-        annotation_directory=None,
-        original_extension=None,
-        **kwargs
-    ):
-        from bob.db.ijbc.query import Database as LowLevelDatabase
-        self._db = LowLevelDatabase(
-            original_directory)
-
-        # call base class constructors to open a session to the database
-        super(IJBCBioDatabase, self).__init__(
-            name='ijbc',
-            models_depend_on_protocol=True,
-            training_depends_on_protocol=True,
-            original_directory=original_directory,
-            annotation_directory=annotation_directory,
-            original_extension=original_extension,
-            **kwargs)
-
-    @property
-    def original_directory(self):
-        return self._db.original_directory
-
-    @original_directory.setter
-    def original_directory(self, value):
-        self._db.original_directory = value
-
-    @property
-    def annotation_directory(self):
-        return self._db.annotation_directory
-
-    @annotation_directory.setter
-    def annotation_directory(self, value):
-        self._db.annotation_directory = value
-
-    def uses_probe_file_sets(self):
-        return True
-
-    def model_ids_with_protocol(self, groups=None, protocol="1:1", **kwargs):
-        return self._db.model_ids(groups=groups, protocol=protocol)
-
-    def objects(self, groups=None, protocol="1:1", purposes=None, model_ids=None, **kwargs):
-        return [IJBCBioFile(f) for f in self._db.objects(groups=groups, protocol=protocol, purposes=purposes, model_ids=model_ids, **kwargs)]
-
-    def object_sets(self, groups=None, protocol="1:1", purposes=None, model_ids=None):
-        return [IJBCBioFileSet(t) for t in self._db.object_sets(groups=groups, protocol=protocol, purposes=purposes, model_ids=model_ids)]
-
-    def annotations(self, biofile):
-        return self._db.annotations(biofile.f)
 
-    def client_id_from_model_id(self, model_id, group='dev'):
-        return self._db.get_client_id_from_model_id(self.protocol, model_id)
 
-    def original_file_names(self, files):
-        return [f.make_path(self.original_directory, f.f.extension, False) for f in files]
+def load_ijbc_sample(original_path, extension=[".jpg", ".png"]):
+    for e in extension:
+        path = original_path + e
+        if os.path.exists(path):
+            return path
+    else:
+        return ""
+
+
+class IJBCDatabase:
+    def __init__(self, pkl_directory=None):
+        self.annotation_type = "bounding-box"
+        self.fixed_positions = None
+        self.allow_scoring_with_all_biometric_references = False
+        self.hash_fn = hash_string
+        self.memory_demanding = True
+
+        if pkl_directory is None:
+            urls = IJBCDatabase.urls()
+            pkl_directory = get_file(
+                "ijbc.tar.gz", urls, file_hash="4b25d7f10595eb9f97f328a2d448d957"
+            )
+
+        self.pkl_directory = pkl_directory
+
+    def _assert_group(self, group):
+        assert (
+            group == "dev"
+        ), "The IJBC database only has a `dev` group. Received : {}".format(group)
+
+    def references(self, group="dev"):
+        self._assert_group(group)
+        return pickle.loads(
+            find_element_in_tarball(self.pkl_directory, "db_references.pickle", True)
+        )
+
+    def probes(self, group="dev"):
+        self._assert_group(group)
+        return pickle.loads(
+            find_element_in_tarball(self.pkl_directory, "db_probes.pickle", True)
+        )
+
+    def background_model_samples(self):
+        import cloudpickle
+
+        return cloudpickle.loads(
+            find_element_in_tarball(
+                self.pkl_directory, "db_background_model_samples.pickle", True
+            )
+        )
+
+    @staticmethod
+    def urls():
+        return [
+            "https://www.idiap.ch/software/bob/databases/latest/ijbc.tar.gz",
+            "http://www.idiap.ch/software/bob/databases/latest/ijbc.tar.gz",
+        ]

bob/bio/face/test/test_databases.py

@@ -313,22 +313,24 @@ def test_replaymobile_spoof():
         )
 
 
-@db_available("ijbc")
 def test_ijbc():
-    database = bob.bio.base.load_resource(
-        "ijbc-11", "database", preferred_package="bob.bio.face"
-    )
-    try:
-        check_database(database, models_depend=True, training_depends=True)
-    except IOError as e:
-        pytest.skip("The database could not queried; Here is the error: '%s'" % e)
+    from bob.bio.face.database import IJBCDatabase
+
+    # Getting the absolute path
+    urls = IJBCDatabase.urls()
+    filename = get_file("ijbc.tar.gz", urls)
+
+    # Removing the file before the test
     try:
-        _check_annotations(database, topleft=True, limit_files=1000)
-    except IOError as e:
-        pytest.skip(
-            "The annotations could not be queried; probably the annotation files are missing. Here is the error: '%s'"
-            % e
-        )
+        os.remove(filename)
+    except Exception:
+        pass
+
+    database = IJBCDatabase()
+
+    assert len(database.background_model_samples()) == 140732
+    assert len(database.references()) == 3531
+    assert len(database.probes()) == 19593
 
 
 @db_available("fargo")