Commit 554419d6 authored by Tiago de Freitas Pereira's avatar Tiago de Freitas Pereira
Browse files

Merge branch 'issue-8-remove-database-configuration' into 'master'

Issue 8 remove database configuration

This merge request sediments what we have discussed along the week.
Basically I removed bob.bio.db package and move the BioDatabase and BioFile classes to bob.bio.base.
I will branch the bob.nightlies to better do the integration tests.

I will leave this merge request open, just to not forget

See merge request !37
parents 28a640f6 fafbda49
Pipeline #3718 passed with stages
in 55 minutes and 38 seconds
from .utils import *
from . import database
from . import preprocessor
from . import extractor
from . import algorithm
......
from .file import BioFile
from .file import BioFileSet
from .database import BioDatabase
from .database import ZTBioDatabase
# gets sphinx autodoc done right - don't remove it
__all__ = [_ for _ in dir() if not _.startswith('_')]
This diff is collapsed.
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
import bob.db.base
class BioFile(bob.db.base.File):
"""A simple base class that defines basic properties of File object for the use in verification experiments"""
def __init__(self, client_id, path, file_id=None):
"""**Constructor Documentation**
Initialize the File object with the minimum required data.
Parameters:
client_id : various type
The id of the client this file belongs to.
Its type depends on your implementation.
If you use an SQL database, this should be an SQL type like Integer or String.
For path and file_id, please refer to :py:class:`bob.db.base.File` constructor
"""
bob.db.base.File.__init__(self, path, file_id)
# just copy the information
self.client_id = client_id
"""The id of the client, to which this file belongs to."""
class BioFileSet(BioFile):
"""This class defines the minimum interface of a set of database files that needs to be exported.
Use this class, whenever the database provides several files that belong to the same probe.
Each file set has an id, and a list of associated files, which are of type :py:class:`BioFile` of the same client.
The file set id can be anything hashable, but needs to be unique all over the database.
**Parameters:**
file_set_id : str or int
A unique ID that identifies the file set.
files : [:py:class:`BioFile`]
A non-empty list of BioFile objects that should be stored inside this file.
All files of that list need to have the same client ID.
"""
def __init__(self, file_set_id, files, path=None):
# don't accept empty file lists
assert len(files), "Cannot create an empty BioFileSet"
# call base class constructor
BioFile.__init__(self, files[0].client_id, "+".join(f.path for f in files) if path is None else path, file_set_id)
# check that all files come from the same client
assert all(f.client_id == self.client_id for f in files)
# The list of files contained in this set
self.files = files
"""The list of :py:class:`BioFile` objects stored in this file set"""
def __lt__(self, other):
"""Defines an order between file sets by using the order of the file set ids."""
# compare two BioFile set objects by comparing their IDs
return self.id < other.id
......@@ -10,11 +10,12 @@ from .Preprocessor import Preprocessor
class Filename (Preprocessor):
"""This preprocessor is simply passing over the file name, in order to be used in an extractor that loads the data from file.
The file name that will be returned by the :py:meth:`read_data` function will contain the path of the :py:class:`bob.bio.db.BioFile`, but it might contain more paths (such as the ``--preprocessed-directory`` passed on command line).
The file name that will be returned by the :py:meth:`read_data` function will contain the path of the :py:class:`bob.bio.base.database.BioFile`, but it might contain more paths (such as the ``--preprocessed-directory`` passed on command line).
"""
def __init__(self):
Preprocessor.__init__(self, writes_data=False)
# call base class constructor, using a custom ``read_original_data`` that does nothing and always returns None
Preprocessor.__init__(self, writes_data=False, read_original_data = lambda x,y,z: None)
# The call function (i.e. the operator() in C++ terms)
......@@ -40,28 +41,6 @@ class Filename (Preprocessor):
return 1
############################################################
### Special functions that might be overwritten on need
############################################################
def read_original_data(self, original_file_name):
"""read_original_data(original_file_name) -> data
This function does **not** read the original image..
**Parameters:**
``original_file_name`` : any
ignored
**Returns:**
``data`` : ``None``
throughout.
"""
pass
def write_data(self, data, data_file):
"""Does **not** write any data.
......
......@@ -3,10 +3,6 @@
# @author: Manuel Guenther <Manuel.Guenther@idiap.ch>
# @date: Tue Oct 2 12:12:39 CEST 2012
import bob.io.base
import os
from .. import utils
class Preprocessor:
......@@ -18,14 +14,19 @@ class Preprocessor:
writes_data : bool
Select, if the preprocessor actually writes preprocessed images, or if it is simply returning values.
read_original_data: callable
This function is used to read the original data from file.
It takes three inputs: A :py:class:`bob.bio.base.database.BioFile` (or one of its derivatives), the original directory (as ``str``) and the original extension (as ``str``).
kwargs : ``key=value`` pairs
A list of keyword arguments to be written in the :py:meth:`__str__` function.
"""
def __init__(self, writes_data = True, **kwargs):
def __init__(self, writes_data = True, read_original_data = lambda biofile,directory,extension : biofile.load(directory,extension), **kwargs):
# Each class needs to have a constructor taking
# all the parameters that are required for the preprocessing as arguments
self.writes_data = writes_data
self.read_original_data = read_original_data
self._kwargs = kwargs
pass
......@@ -70,25 +71,6 @@ class Preprocessor:
### Special functions that might be overwritten on need
############################################################
def read_original_data(self, original_file_name):
"""read_original_data(original_file_name) -> data
Reads the *original* data (usually something like an image) from file.
In this base class implementation, it uses :py:func:`bob.io.base.load` to do that.
If you have different format, please overwrite this function.
**Parameters:**
original_file_name : str
The file name to read the original data from.
**Returns:**
data : object (usually :py:class:`numpy.ndarray`)
The original data read from file.
"""
return bob.io.base.load(original_file_name)
def write_data(self, data, data_file):
"""Writes the given *preprocessed* data to a file with the given name.
......
......@@ -6,6 +6,7 @@ import bob.core
logger = bob.core.log.setup("bob.bio.base")
import bob.bio.base
from bob.bio.base.database.file import BioFile
import bob.db.base
import numpy
......@@ -46,7 +47,7 @@ def main(command_line_parameters=None):
preprocessor = bob.bio.base.load_resource(' '.join(args.preprocessor), "preprocessor")
logger.debug("Loading input data from file '%s'%s", args.input_file, " and '%s'" % args.annotation_file if args.annotation_file is not None else "")
data = preprocessor.read_original_data(args.input_file)
data = preprocessor.read_original_data(BioFile(1, args.input_file, 2), "", "")
annotations = bob.db.base.annotations.read_annotation_file(args.annotation_file, 'named') if args.annotation_file is not None else None
logger.info("Preprocessing data")
......
from bob.bio.db import ZTBioDatabase
from bob.bio.base.database import ZTBioDatabase
from bob.bio.base.database.file import BioFile
from bob.bio.base.test.utils import atnt_database_directory
......@@ -17,11 +18,14 @@ class DummyDatabase(ZTBioDatabase):
import bob.db.atnt
self.__db = bob.db.atnt.Database()
def _make_bio(self, files):
return [BioFile(client_id=f.client_id, path=f.path, file_id=f.id) for f in files]
def model_ids_with_protocol(self, groups=None, protocol=None, **kwargs):
return self.__db.model_ids(groups, protocol)
def objects(self, groups=None, protocol=None, purposes=None, model_ids=None, **kwargs):
return self.__db.objects(model_ids, groups, purposes, protocol, **kwargs)
return self._make_bio(self.__db.objects(model_ids, groups, purposes, protocol, **kwargs))
def tobjects(self, groups=None, protocol=None, model_ids=None, **kwargs):
return []
......
from bob.bio.db import ZTBioDatabase, BioFileSet, BioFile
from bob.bio.base.database import ZTBioDatabase, BioFileSet, BioFile
from bob.bio.base.test.utils import atnt_database_directory
class DummyDatabase(ZTBioDatabase):
def __init__(self):
......@@ -20,25 +19,21 @@ class DummyDatabase(ZTBioDatabase):
def uses_probe_file_sets(self):
return True
def _make_bio(self, files):
return [BioFile(client_id=f.client_id, path=f.path, file_id=f.id) for f in files]
def probe_file_sets(self, model_id=None, group='dev'):
"""Returns the list of probe File objects (for the given model id, if given)."""
# import ipdb; ipdb.set_trace()
files = self.arrange_by_client(self.sort(self.objects(protocol=None, groups=group, purposes='probe')))
# arrange files by clients
file_sets = []
for client_files in files:
# convert into our File objects (so that they are tested as well)
our_files = [BioFile(f.client_id, f.path, f.id) for f in client_files]
# generate file set for each client
file_set = BioFileSet(our_files[0].client_id, our_files)
file_sets.append(file_set)
file_sets = [BioFileSet(client_files[0].client_id, client_files) for client_files in files]
return file_sets
def model_ids_with_protocol(self, groups=None, protocol=None, **kwargs):
return self.__db.model_ids(groups, protocol)
def objects(self, groups=None, protocol=None, purposes=None, model_ids=None, **kwargs):
return self.__db.objects(model_ids, groups, purposes, protocol, **kwargs)
return self._make_bio(self.__db.objects(model_ids, groups, purposes, protocol, **kwargs))
def tobjects(self, groups=None, protocol=None, model_ids=None, **kwargs):
return []
......
import bob.bio.base
from . import utils
def test_filename():
# load extractor
preprocessor = bob.bio.base.load_resource("filename", "preprocessor", preferred_package = 'bob.bio.base')
......@@ -9,7 +7,7 @@ def test_filename():
assert isinstance(preprocessor, bob.bio.base.preprocessor.Filename)
# try to load the original image
assert preprocessor.read_original_data("/any/path") is None
assert preprocessor.read_original_data(bob.bio.base.database.file.BioFile(1,"2",3), "/any/path", ".any.extension") is None
# try to process
assert preprocessor(None, None) == 1
......
......@@ -8,7 +8,7 @@ logger = bob.core.log.setup("bob.bio.base")
from .. import utils
from . import FileSelector
from bob.bio.db import BioDatabase
from bob.bio.base.database import BioDatabase
"""Execute biometric recognition algorithms on a certain biometric database.
"""
......
......@@ -61,18 +61,12 @@ def preprocess(preprocessor, groups = None, indices = None, allow_missing_files
for i in index_range:
preprocessed_data_file = preprocessed_data_files[i]
file_object = data_files[i]
if isinstance(file_object, list):
file_name = [f.make_path(original_directory, original_extension) for f in file_object]
else:
file_name = file_object.make_path(original_directory, original_extension)
file_name = file_object.make_path(original_directory, original_extension)
# check for existence
if not utils.check_file(preprocessed_data_file, force, 1000):
logger.debug("... Processing original data file '%s'", file_name)
if hasattr(file_object, 'load'):
data = file_object.load(original_directory, original_extension)
else:
data = preprocessor.read_original_data(file_name)
data = preprocessor.read_original_data(file_object, original_directory, original_extension)
# create output directory before reading the data file (is sometimes required, when relative directories are specified, especially, including a .. somewhere)
bob.io.base.create_directories_safe(os.path.dirname(preprocessed_data_file))
......
......@@ -25,7 +25,6 @@ develop = src/bob.extension
src/bob.learn.linear
src/bob.learn.em
src/bob.db.atnt
src/bob.bio.db
.
; options for bob.buildout
......@@ -47,7 +46,6 @@ bob.learn.activation = git git@gitlab.idiap.ch:bob/bob.learn.activation
bob.learn.linear = git git@gitlab.idiap.ch:bob/bob.learn.linear
bob.learn.em = git git@gitlab.idiap.ch:bob/bob.learn.em
bob.db.atnt = git git@gitlab.idiap.ch:bob/bob.db.atnt
bob.bio.db = git git@gitlab.idiap.ch:bob/bob.bio.db
[scripts]
recipe = bob.buildout:scripts
......
......@@ -218,13 +218,13 @@ Verification Database Interface
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
For most of the data sets, we rely on the database interfaces from Bob_.
Particularly, all databases that are derived from the :py:class:`bob.bio.db.BioDatabase` (click :ref:`here <verification_databases>` for a list of implemented databases) are supported by a special derivation of the databases from above.
Particularly, all databases that are derived from the :py:class:`bob.bio.base.database.BioDatabase` (click :ref:`here <verification_databases>` for a list of implemented databases) are supported by a special derivation of the databases from above.
For these databases, the special :py:class:`bob.bio.base.database.DatabaseBob` interface is provided, which takes the Bob_ database as parameter.
Several such databases are defined in the according packages, i.e., :ref:`bob.bio.spear <bob.bio.spear>`, :ref:`bob.bio.face <bob.bio.face>` and :ref:`bob.bio.video <bob.bio.video>`.
For Bob_'s ZT-norm databases, we provide the :py:class:`bob.bio.base.database.DatabaseBobZT` interface.
Additionally, a generic database interface, which is derived from :py:class:`bob.bio.base.database.DatabaseBobZT`, is the :py:class:`bob.bio.base.database.DatabaseFileList`.
This database interfaces with the :py:class:`bob.db.verification.filelist.Database`, which is a generic database based on file lists, implementing the :py:class:`bob.bio.db.BioDatabase` interface.
This database interfaces with the :py:class:`bob.db.verification.filelist.Database`, which is a generic database based on file lists, implementing the :py:class:`bob.bio.base.database.BioDatabase` interface.
Defining your own Database
~~~~~~~~~~~~~~~~~~~~~~~~~~
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment