Merge branch 'issue-8-remove-database-configuration' into 'master'

Issue 8 remove database configuration



See merge request !1

bob/bio/video/__init__.py

@@ -2,7 +2,7 @@ from .utils import *
 from . import preprocessor
 from . import extractor
 from . import algorithm
-
+from . import database
 from . import test
 
 def get_config():

bob/bio/video/config/database/mobio.py

+#!/usr/bin/env python
+
+from bob.bio.video.database import MobioBioDatabase
+
+mobio_video_directory = "[YOUR_MOBIO_VIDEO_DIRECTORY]"
+
+database = MobioBioDatabase(
+    original_directory=mobio_video_directory,
+    original_extension=".mp4",
+    protocol='male',
+    models_depend_on_protocol=True,
+    all_files_options={'subworld': 'twothirds-subsampled'},
+    extractor_training_options={'subworld': 'twothirds-subsampled'},
+    projector_training_options={'subworld': 'twothirds-subsampled'},
+    enroller_training_options={'subworld': 'twothirds-subsampled'},
+)
+
+

bob/bio/video/config/database/youtube.py

+#!/usr/bin/env python
+
+from bob.bio.video.database import YoutubeBioDatabase
+
+youtube_directory = "[YOUR_YOUTUBE_DIRECTORY]"
+
+database = YoutubeBioDatabase(
+    original_directory=youtube_directory,
+    protocol='fold1',
+    models_depend_on_protocol=True,
+    training_depends_on_protocol=True,
+
+    all_files_options={'subworld': 'fivefolds'},
+    extractor_training_options={'subworld': 'fivefolds'},
+    projector_training_options={'subworld': 'fivefolds'},
+    enroller_training_options={'subworld': 'fivefolds'},
+)

bob/bio/video/database/__init__.py

+from .database import VideoBioFile
+from .mobio import MobioBioDatabase
+from .youtube import YoutubeBioDatabase
+
+
+# gets sphinx autodoc done right - don't remove it
+__all__ = [_ for _ in dir() if not _.startswith('_')]

bob/bio/video/database/database.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+# Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
+# Wed 20 July 14:43:22 CEST 2016
+
+"""
+  Verification API for bob.db.voxforge
+"""
+
+from bob.bio.base.database.file import BioFile
+from bob.bio.video.utils.FrameSelector import FrameSelector
+
+
+class VideoBioFile(BioFile):
+    def __init__(self, client_id, path, file_id):
+        """
+        Initializes this File object with an File equivalent for
+        VoxForge database.
+        """
+        super(VideoBioFile, self).__init__(client_id=client_id, path=path, file_id=file_id)
+
+    def load(self, directory=None, extension='.avi'):
+        return FrameSelector()(self.make_path(directory, extension))
+
+
+

bob/bio/video/database/mobio.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+# Amir Mohammadi <amir.mohammadi@idiap.ch>
+# Wed 13 Jul 16:43:22 CEST 2016
+
+"""
+  MOBIO database implementation of bob.bio.base.database.ZTDatabase interface.
+  It is an extension of an SQL-based database interface, which directly talks to Mobio database, for
+  verification experiments (good to use in bob.bio.base framework).
+"""
+
+from .database import VideoBioFile
+from bob.bio.base.database import ZTBioDatabase, BioFile
+
+
+class MobioBioDatabase(ZTBioDatabase):
+    """
+    Implements verification API for querying Mobio database.
+    """
+
+    def __init__(
+            self,
+            **kwargs
+    ):
+        # call base class constructors to open a session to the database
+        super(MobioBioDatabase, self).__init__(name='mobio',
+                                               **kwargs)
+
+        from bob.db.mobio.query import Database as LowLevelDatabase
+        self.__db = LowLevelDatabase()
+
+    def model_ids_with_protocol(self, groups=None, protocol=None, gender=None):
+        return self.__db.model_ids(groups=groups, protocol=protocol, gender=gender)
+
+    def tmodel_ids_with_protocol(self, protocol=None, groups=None, **kwargs):
+        return self.__db.tmodel_ids(protocol=protocol, groups=groups, **kwargs)
+
+    def objects(self, groups=None, protocol=None, purposes=None, model_ids=None, **kwargs):
+        retval = self.__db.objects(groups=groups, protocol=protocol, purposes=purposes, model_ids=model_ids, **kwargs)
+        return [VideoBioFile(f) for f in retval]
+
+    def tobjects(self, groups=None, protocol=None, model_ids=None, **kwargs):
+        retval = self.__db.tobjects(groups=groups, protocol=protocol, model_ids=model_ids, **kwargs)
+        return [VideoBioFile(f) for f in retval]
+
+    def zobjects(self, groups=None, protocol=None, **kwargs):
+        retval = self.__db.zobjects(groups=groups, protocol=protocol, **kwargs)
+        return [VideoBioFile(f) for f in retval]

bob/bio/video/database/youtube.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+# Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
+# Sat 20 Aug 15:43:10 CEST 2016
+
+"""
+  YOUTUBE database implementation of bob.bio.base.database.ZTDatabase interface.
+  It is an extension of an SQL-based database interface, which directly talks to YOUTUBE database, for
+  verification experiments (good to use in bob.bio.base framework).
+"""
+
+
+from .database import VideoBioFile
+from bob.bio.base.database import ZTBioDatabase, BioFile
+from bob.bio.video.utils import FrameContainer
+import os
+import bob.io.base
+
+
+class YoutubeBioFile(VideoBioFile):
+    def load(self, directory=None, extension='.jpg'):
+        files = sorted(os.listdir(self.make_path(directory, '')))
+        fc = FrameContainer()
+
+        for f in files:
+            if extension == os.path.splitext(f)[1]:
+                file_name = os.path.join(self.make_path(directory, ''), f)
+                fc.add(os.path.basename(file_name), bob.io.base.load(file_name))
+        return fc
+
+
+class YoutubeBioDatabase(ZTBioDatabase):
+    """
+    Implements verification API for querying Youtube database.
+    """
+
+    def __init__(
+            self,
+            **kwargs
+    ):
+        # call base class constructors to open a session to the database
+        super(YoutubeBioDatabase, self).__init__(name='youtube',
+                                                 **kwargs)
+
+        from bob.db.youtube.query import Database as LowLevelDatabase
+        self.__db = LowLevelDatabase()
+
+    def model_ids_with_protocol(self, groups=None, protocol=None, **kwargs):
+        return self.__db.model_ids(groups=groups, protocol=protocol)
+
+    def tmodel_ids_with_protocol(self, protocol=None, groups=None, **kwargs):
+        return self.__db.tmodel_ids(protocol=protocol, groups=groups, **kwargs)
+
+    def objects(self, groups=None, protocol=None, purposes=None, model_ids=None, **kwargs):
+        retval = self.__db.objects(groups=groups, protocol=protocol, purposes=purposes, model_ids=model_ids, **kwargs)
+        return [YoutubeBioFile(client_id=f.client_id, path=f.path, file_id=f.id) for f in retval]
+
+    def tobjects(self, groups=None, protocol=None, model_ids=None, **kwargs):
+        retval = self.__db.tobjects(groups=groups, protocol=protocol, model_ids=model_ids, **kwargs)
+        return [YoutubeBioFile(client_id=f.client_id, path=f.path, file_id=f.id) for f in retval]
+
+    def zobjects(self, groups=None, protocol=None, **kwargs):
+        retval = self.__db.zobjects(groups=groups, protocol=protocol, **kwargs)
+        return [YoutubeBioFile(client_id=f.client_id, path=f.path, file_id=f.id) for f in retval]

bob/bio/video/test/dummy/database.py

-from bob.bio.db import ZTBioDatabase
+from bob.bio.base.database import ZTBioDatabase
 from bob.bio.base.test.utils import atnt_database_directory
+from bob.bio.base.database.file import BioFile
+from bob.bio.video.database import VideoBioFile
+from bob.bio.video.utils import FrameContainer
+import bob.io.base
+import os
+
+
+class DummyBioFile(VideoBioFile):
+    def load(self, directory=None, extension='.pgm'):
+        file_name = self.make_path(directory, extension)
+        fc = FrameContainer()
+        fc.add(os.path.basename(file_name), bob.io.base.load(file_name))
+        return fc
 
 
 class DummyDatabase(ZTBioDatabase):
@@ -17,11 +30,14 @@ class DummyDatabase(ZTBioDatabase):
         import bob.db.atnt
         self.__db = bob.db.atnt.Database()
 
+    def _make_bio(self, files):
+      return [DummyBioFile(client_id=f.client_id, path=f.path, file_id=f.id) for f in files]
+
     def model_ids_with_protocol(self, groups=None, protocol=None, **kwargs):
         return self.__db.model_ids(groups, protocol)
 
     def objects(self, groups=None, protocol=None, purposes=None, model_ids=None, **kwargs):
-        return self.__db.objects(model_ids, groups, purposes, protocol, **kwargs)
+        return self._make_bio(self.__db.objects(model_ids, groups, purposes, protocol, **kwargs))
 
     def tobjects(self, groups=None, protocol=None, model_ids=None, **kwargs):
         return []
@@ -38,10 +54,6 @@ class DummyDatabase(ZTBioDatabase):
     def z_probe_files(self, group='dev'):
         return self.probe_files(None, group)
 
-    # override all_files to return a one-element lists of files
-    def all_files(self, groups):
-        return [[n] for n in super(DummyDatabase, self).all_files(groups)]
-
     def file_names(self, files, directory, extension):
         if isinstance(files[0], list):
             files = list(list(zip(*files))[0])

bob/bio/video/test/test_preprocessor.py

@@ -9,70 +9,86 @@ import bob.io.video
 import bob.bio.base
 import bob.bio.video
 import bob.db.base
-
 import pkg_resources
+from bob.bio.video.test.dummy.database import DummyBioFile
 
 regenerate_refs = False
 
 def test_annotations():
-  # use annotations to grep
-  image_files = [pkg_resources.resource_filename("bob.bio.face.test", "data/testimage.jpg")]
-  annotations = {os.path.basename(image_files[0]) : bob.db.base.read_annotation_file(pkg_resources.resource_filename("bob.bio.face.test", "data/testimage.pos"), 'named')}
+    original_path = pkg_resources.resource_filename("bob.bio.face.test", "")
+    image_files = DummyBioFile(client_id=1, file_id=1, path="data/testimage")
+
+    # use annotations to grep
+    annotations = {os.path.basename(image_files.make_path(original_path, ".jpg")): bob.db.base.read_annotation_file(
+        pkg_resources.resource_filename("bob.bio.face.test", "data/testimage.pos"), 'named')}
 
-  # video preprocessor using a face crop preprocessor
-  frame_selector = bob.bio.video.FrameSelector(selection_style="all")
-  preprocessor = bob.bio.video.preprocessor.Wrapper('face-crop-eyes', frame_selector, compressed_io=False)
+    # video preprocessor using a face crop preprocessor
+    frame_selector = bob.bio.video.FrameSelector(selection_style="all")
+    preprocessor = bob.bio.video.preprocessor.Wrapper('face-crop-eyes', frame_selector, compressed_io=False)
 
-  # read original data
-  original = preprocessor.read_original_data(image_files)
-  assert isinstance(original, bob.bio.video.FrameContainer)
-  assert len(original) == 1
-  assert original[0][0] == os.path.basename(image_files[0])
+    # read original data
+    original = preprocessor.read_original_data(image_files, original_path, ".jpg")
 
-  # preprocess data including annotations
-  preprocessed = preprocessor(original, annotations)
-  assert isinstance(preprocessed, bob.bio.video.FrameContainer)
-  assert len(preprocessed) == 1
-  assert preprocessed[0][0] == os.path.basename(image_files[0])
-  assert preprocessed[0][2] is None
-  assert numpy.allclose(preprocessed[0][1], bob.io.base.load(pkg_resources.resource_filename("bob.bio.face.test", "data/cropped.hdf5")))
+    assert isinstance(original, bob.bio.video.FrameContainer)
+    assert len(original) == 1
+    assert original[0][0] == os.path.basename(image_files.make_path(original_path, ".jpg"))
 
+    # preprocess data including annotations
+    preprocessed = preprocessor(original, annotations)
+    assert isinstance(preprocessed, bob.bio.video.FrameContainer)
+    assert len(preprocessed) == 1
+    assert preprocessed[0][0] == os.path.basename(image_files.make_path(original_path, ".jpg"))
+    assert preprocessed[0][2] is None
+    assert numpy.allclose(preprocessed[0][1],
+                          bob.io.base.load(pkg_resources.resource_filename("bob.bio.face.test", "data/cropped.hdf5")))
 
 def test_detect():
-  # load test video
-  video_file = pkg_resources.resource_filename("bob.bio.video.test", "data/testvideo.avi")
-  frame_selector = bob.bio.video.FrameSelector(max_number_of_frames=3, selection_style="spread")
 
-  preprocessor = bob.bio.video.preprocessor.Wrapper('face-detect', frame_selector, compressed_io=False)
-  video = preprocessor.read_original_data(video_file)
-  assert isinstance(video, bob.bio.video.FrameContainer)
+    def read_original_data(video_object, original_path, original_extension):
+        return frame_selector(video_object.make_path(original_path, original_extension))
+
+    # load test video
+    original_path = pkg_resources.resource_filename("bob.bio.video.test", "")
+    video_object = bob.bio.video.database.VideoBioFile(client_id=1, file_id=1, path="data/testvideo")
 
-  preprocessed_video = preprocessor(video)
-  assert isinstance(preprocessed_video, bob.bio.video.FrameContainer)
+    frame_selector = bob.bio.video.FrameSelector(max_number_of_frames=3, selection_style="spread")
+    preprocessor = bob.bio.video.preprocessor.Wrapper('face-detect', frame_selector, compressed_io=False,
+                                                      read_original_data=read_original_data)
 
-  reference_file = pkg_resources.resource_filename("bob.bio.video.test", "data/preprocessed.hdf5")
-  if regenerate_refs:
-    preprocessed_video.save(bob.io.base.HDF5File(reference_file, 'w'))
-  reference_data = bob.bio.video.FrameContainer(bob.io.base.HDF5File(reference_file, 'r'))
+    video = preprocessor.read_original_data(video_object, original_path,".avi")
+    assert isinstance(video, bob.bio.video.FrameContainer)
 
-  assert preprocessed_video.is_similar_to(reference_data)
+    preprocessed_video = preprocessor(video)
+    assert isinstance(preprocessed_video, bob.bio.video.FrameContainer)
+
+    reference_file = pkg_resources.resource_filename("bob.bio.video.test", "data/preprocessed.hdf5")
+    if regenerate_refs:
+        preprocessed_video.save(bob.io.base.HDF5File(reference_file, 'w'))
+    reference_data = bob.bio.video.FrameContainer(bob.io.base.HDF5File(reference_file, 'r'))
+
+    assert preprocessed_video.is_similar_to(reference_data)
 
 
 def test_flandmark():
 
-  video_file = pkg_resources.resource_filename("bob.bio.video.test", "data/testvideo.avi")
-  frame_selector = bob.bio.video.FrameSelector(max_number_of_frames=3, selection_style="spread")
+    def read_original_data(video_object, original_path, original_extension):
+        return frame_selector(video_object.make_path(original_path, original_extension))
+
+    original_path = pkg_resources.resource_filename("bob.bio.video.test", "")
+    video_object = bob.bio.video.database.VideoBioFile(client_id=1, file_id=1, path="data/testvideo")
+    frame_selector = bob.bio.video.FrameSelector(max_number_of_frames=3, selection_style="spread")
 
-  preprocessor = bob.bio.video.preprocessor.Wrapper('landmark-detect', frame_selector, compressed_io=False)
-  video = preprocessor.read_original_data(video_file)
-  assert isinstance(video, bob.bio.video.FrameContainer)
+    preprocessor = bob.bio.video.preprocessor.Wrapper('landmark-detect', frame_selector, compressed_io=False,
+                                                      read_original_data=read_original_data)
+    video = preprocessor.read_original_data(video_object, original_path, ".avi")
+    assert isinstance(video, bob.bio.video.FrameContainer)
 
-  preprocessed_video = preprocessor(video)
-  assert isinstance(preprocessed_video, bob.bio.video.FrameContainer)
+    preprocessed_video = preprocessor(video)
+    assert isinstance(preprocessed_video, bob.bio.video.FrameContainer)
 
-  reference_file = pkg_resources.resource_filename("bob.bio.video.test", "data/preprocessed-flandmark.hdf5")
-  if regenerate_refs:
-    preprocessed_video.save(bob.io.base.HDF5File(reference_file, 'w'))
-  reference_data = bob.bio.video.FrameContainer(bob.io.base.HDF5File(reference_file, 'r'))
+    reference_file = pkg_resources.resource_filename("bob.bio.video.test", "data/preprocessed-flandmark.hdf5")
+    if regenerate_refs:
+        preprocessed_video.save(bob.io.base.HDF5File(reference_file, 'w'))
+    reference_data = bob.bio.video.FrameContainer(bob.io.base.HDF5File(reference_file, 'r'))
 
-  assert preprocessed_video.is_similar_to(reference_data)
+    assert preprocessed_video.is_similar_to(reference_data)

requirements.txt

@@ -17,7 +17,6 @@ bob.learn.em
 bob.measure
 bob.db.base
 bob.bio.base
-bob.bio.db
 bob.bio.face
 bob.learn.boosting
 bob.ip.facedetect

setup.py

@@ -65,10 +65,12 @@ setup(
     #
     # In this simple example we will create a single program that will print
     # the version of bob.
-    entry_points = {
+    entry_points={
 
       'bob.bio.database': [
-        'dummy-video       = bob.bio.video.test.dummy.database:database', # for test purposes only
+        'dummy-video       = bob.bio.video.test.dummy.database:database',# for test purposes only
+        'mobio             = bob.bio.video.config.database.mobio:database',
+        'youtube             = bob.bio.video.config.database.youtube:database',
       ],
 
     },

test-requirements.txt

 bob.db.atnt
+gridtk
\ No newline at end of file

[Amir MOHAMMADI]

Could you please remove bob/bio/video/config/database/.DS_Store @tiago.pereira

[Tiago de Freitas Pereira]

yes

[Amir MOHAMMADI]

Removing this custom all_files was ok? @tiago.pereira

[Tiago de Freitas Pereira]

Yes, the default behaviour is fine