Commit c466cc81 authored by Manuel Günther's avatar Manuel Günther
Browse files

Based database on the novel xbob.db.verification.utils interface; some clean-up and some bug-fixes.

parent 96d11329
......@@ -25,7 +25,8 @@ setup(
install_requires=[
'setuptools',
'bob', # base signal proc./machine learning library
],
'xbob.db.verification.utils' # defines a set of utilities for face verification databases like this one.
],
namespace_packages = [
'xbob',
......
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Laurent El Shafey <laurent.el-shafey@idiap.ch>
"""Checks for installed files.
"""
import os
import sys
# Driver API
# ==========
def checkfiles(args):
"""Checks existence of files based on your criteria"""
from .query import Database
db = Database()
r = db.objects(
protocol=args.protocol,
purposes=args.purposes,
#model_ids=args.model_ids,
groups=args.groups,
)
# go through all files, check if they are available on the filesystem
good = []
bad = []
for f in r:
if os.path.exists(f.make_path(args.directory, args.extension)):
good.append(f)
else:
bad.append(f)
# report
output = sys.stdout
if args.selftest:
from bob.db.utils import null
output = null()
if bad:
for f in bad:
output.write('Cannot find file "%s"\n' % (f.make_path(args.directory, args.extension),))
output.write('%d files (out of %d) were not found at "%s"\n' % \
(len(bad), len(r), args.directory))
return 0
def add_command(subparsers):
"""Add specific subcommands that the action "checkfiles" can use"""
from argparse import SUPPRESS
parser = subparsers.add_parser('checkfiles', help=checkfiles.__doc__)
from .query import Database
db = Database()
if db.is_valid():
protocols = db.protocol_names()
purposes = db.purposes()
groups = db.groups()
else:
protocols, purposes, groups = (), (), ()
parser.add_argument('-d', '--directory', dest="directory", default='', help="if given, this path will be prepended to every entry returned (defaults to '%(default)s')")
parser.add_argument('-e', '--extension', dest="extension", default='', help="if given, this extension will be appended to every entry returned (defaults to '%(default)s')")
parser.add_argument('-p', '--protocol', dest="protocol", default='', help="if given, limits the check to a particular subset of the data that corresponds to the given protocol (defaults to '%(default)s')", choices=db.protocols)
parser.add_argument('-u', '--purposes', dest="purposes", default='', help="if given, this value will limit the output files to those designed for the given purposes. (defaults to '%(default)s')", choices=purposes)
parser.add_argument('-g', '--groups', dest="groups", default='', help="if given, this value will limit the output files to those belonging to a particular protocolar group. (defaults to '%(default)s')", choices=groups)
# TODO: model_ids
parser.add_argument('-c', '--classes', dest="classes", default='', help="if given, this value will limit the output files to those belonging to the given classes. (defaults to '%(default)s')", choices=('client', 'impostor', ''))
parser.add_argument('--self-test', dest="selftest", default=False,
action='store_true', help=SUPPRESS)
parser.set_defaults(func=checkfiles) #action
......@@ -9,39 +9,68 @@ import os
import sys
from bob.db.driver import Interface as BaseInterface
def reverse(args):
"""Returns a list of file database identifiers given the path stems"""
# Driver API
# ==========
def dumplist(args):
"""Dumps lists of files based on your criteria"""
from .query import Database
db = Database()
r = db.objects(
protocol=args.protocol,
purposes=args.purpose,
model_ids=args.client,
groups=args.group,
classes=args.sclass
)
output = sys.stdout
if args.selftest:
from bob.db.utils import null
output = null()
r = db.reverse(args.path)
for id in r: output.write('%d\n' % id)
if not r: return 1
for f in r:
output.write('%s\n' % (f.make_path(args.directory, args.extension),))
return 0
def reverse_command(subparsers):
"""Adds the specific options for the reverse command"""
from argparse import SUPPRESS
def checkfiles(args):
"""Checks existence of files based on your criteria"""
parser = subparsers.add_parser('reverse', help=reverse.__doc__)
from .query import Database
db = Database()
parser.add_argument('path', nargs='+', type=str, help="one or more path stems to look up. If you provide more than one, files which cannot be reversed will be omitted from the output.")
parser.add_argument('--self-test', dest="selftest", default=False,
action='store_true', help=SUPPRESS)
r = db.objects()
parser.set_defaults(func=reverse) #action
# go through all files, check if they are available on the filesystem
good = []
bad = []
for f in r:
if os.path.exists(f.make_path(args.directory, args.extension)):
good.append(f)
else:
bad.append(f)
def path(args):
"""Returns a list of fully formed paths or stems given some file id"""
# report
output = sys.stdout
if args.selftest:
from bob.db.utils import null
output = null()
if bad:
for f in bad:
output.write('Cannot find file "%s"\n' % (f.make_path(args.directory, args.extension),))
output.write('%d files (out of %d) were not found at "%s"\n' % \
(len(bad), len(r), args.directory))
return 0
def reverse(args):
"""Returns a list of file database identifiers given the path stems"""
from .query import Database
db = Database()
......@@ -51,39 +80,41 @@ def path(args):
from bob.db.utils import null
output = null()
r = db.paths(args.id, prefix=args.directory, suffix=args.extension)
for path in r: output.write('%s\n' % path)
r = db.reverse(args.path)
for id in r: output.write('%d\n' % id)
if not r: return 1
return 0
def path_command(subparsers):
"""Adds the specific options for the path command"""
def path(args):
"""Returns a list of fully formed paths or stems given some file id"""
from argparse import SUPPRESS
from .query import Database
db = Database()
parser = subparsers.add_parser('path', help=path.__doc__)
output = sys.stdout
if args.selftest:
from bob.db.utils import null
output = null()
parser.add_argument('-d', '--directory', dest="directory", default='', help="if given, this path will be prepended to every entry returned (defaults to '%(default)s')")
parser.add_argument('-e', '--extension', dest="extension", default='', help="if given, this extension will be appended to every entry returned (defaults to '%(default)s')")
parser.add_argument('id', nargs='+', type=int, help="one or more file ids to look up. If you provide more than one, files which cannot be found will be omitted from the output. If you provide a single id to lookup, an error message will be printed if the id does not exist in the database. The exit status will be non-zero in such case.")
parser.add_argument('--self-test', dest="selftest", default=False,
action='store_true', help=SUPPRESS)
r = db.paths(args.id, prefix=args.directory, suffix=args.extension)
for path in r: output.write('%s\n' % path)
parser.set_defaults(func=path) #action
if not r: return 1
return 0
class Interface(BaseInterface):
def name(self):
return 'biosecure'
def version(self):
import pkg_resources # part of setuptools
return pkg_resources.require('xbob.db.%s' % self.name())[0].version
def files(self):
from pkg_resources import resource_filename
......@@ -96,24 +127,48 @@ class Interface(BaseInterface):
def add_commands(self, parser):
from . import __doc__ as docs
subparsers = self.setup_parser(parser,
"Biosecure database", docs)
# example: get the "create" action from a submodule
# the "create" action from a submodule
from .create import add_command as create_command
create_command(subparsers)
# example: get the "dumplist" action from a submodule
from .dumplist import add_command as dumplist_command
dumplist_command(subparsers)
# example: get the "checkfiles" action from a submodule
from .checkfiles import add_command as checkfiles_command
checkfiles_command(subparsers)
from .query import Database
import argparse
db = Database()
# the "dumplist" action
parser = subparsers.add_parser('dumplist', help=dumplist.__doc__)
parser.add_argument('-d', '--directory', help="if given, this path will be prepended to every entry returned.")
parser.add_argument('-e', '--extension', help="if given, this extension will be appended to every entry returned.")
parser.add_argument('-p', '--protocol', help="if given, limits the dump to a particular subset of the data that corresponds to the given protocol", choices=db.protocol_names() if db.is_valid() else ())
parser.add_argument('-u', '--purpose', help="if given, this value will limit the output files to those designed for the given purposes.", choices=db.purposes() if db.is_valid() else ())
parser.add_argument('-g', '--group', help="if given, this value will limit the output files to those belonging to a particular protocolar group.", choices=db.groups() if db.is_valid() else ())
parser.add_argument('-C', '--client', type=int, help="if given, this value will limit the output files to those belonging to the given client.", choices=db.model_ids() if db.is_valid() else ())
parser.add_argument('-c', '--class', dest="sclass", help="if given, this value will limit the output files to those belonging to the given classes.", choices=('client', 'impostor'))
parser.add_argument('--self-test', dest="selftest", action='store_true', help=argparse.SUPPRESS)
parser.set_defaults(func=dumplist) #action
# the "checkfiles" action
parser = subparsers.add_parser('checkfiles', help=checkfiles.__doc__)
parser.add_argument('-d', '--directory', help="if given, this path will be prepended to every entry returned.")
parser.add_argument('-e', '--extension', help="if given, this extension will be appended to every entry returned.")
parser.add_argument('--self-test', dest="selftest", action='store_true', help=argparse.SUPPRESS)
parser.set_defaults(func=checkfiles) #action
# adds the "reverse" command
reverse_command(subparsers)
parser = subparsers.add_parser('reverse', help=reverse.__doc__)
parser.add_argument('path', nargs='+', help="one or more path stems to look up. If you provide more than one, files which cannot be reversed will be omitted from the output.")
parser.add_argument('--self-test', dest="selftest", action='store_true', help=argparse.SUPPRESS)
parser.set_defaults(func=reverse) #action
# adds the "path" command
path_command(subparsers)
parser = subparsers.add_parser('path', help=path.__doc__)
parser.add_argument('-d', '--directory', help="if given, this path will be prepended to every entry returned.")
parser.add_argument('-e', '--extension', help="if given, this extension will be appended to every entry returned.")
parser.add_argument('id', nargs='+', type=int, help="one or more file ids to look up. If you provide more than one, files which cannot be found will be omitted from the output. If you provide a single id to lookup, an error message will be printed if the id does not exist in the database. The exit status will be non-zero in such case.")
parser.add_argument('--self-test', dest="selftest", action='store_true', help=argparse.SUPPRESS)
parser.set_defaults(func=path) #action
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Laurent El Shafey <laurent.el-shafey@idiap.ch>
"""Dumps lists of files.
"""
import os
import sys
# Driver API
# ==========
def dumplist(args):
"""Dumps lists of files based on your criteria"""
from .query import Database
db = Database()
r = db.objects(
protocol=args.protocol,
purposes=args.purposes,
#model_ids=args.model_ids,
groups=args.groups,
)
output = sys.stdout
if args.selftest:
from bob.db.utils import null
output = null()
for f in r:
output.write('%s\n' % (f.make_path(args.directory, args.extension),))
return 0
def add_command(subparsers):
"""Add specific subcommands that the action "dumplist" can use"""
from argparse import SUPPRESS
parser = subparsers.add_parser('dumplist', help=dumplist.__doc__)
from .query import Database
db = Database()
if db.is_valid():
protocols = db.protocol_names()
purposes = db.purposes()
groups = db.groups()
else:
protocols, purposes, groups = (), (), ()
parser.add_argument('-d', '--directory', dest="directory", default='', help="if given, this path will be prepended to every entry returned (defaults to '%(default)s')")
parser.add_argument('-e', '--extension', dest="extension", default='', help="if given, this extension will be appended to every entry returned (defaults to '%(default)s')")
parser.add_argument('-p', '--protocol', dest="protocol", default='', help="if given, limits the dump to a particular subset of the data that corresponds to the given protocol (defaults to '%(default)s')", choices=protocols)
parser.add_argument('-u', '--purposes', dest="purposes", default='', help="if given, this value will limit the output files to those designed for the given purposes. (defaults to '%(default)s')", choices=purposes)
parser.add_argument('-g', '--groups', dest="groups", default='', help="if given, this value will limit the output files to those belonging to a particular protocolar group. (defaults to '%(default)s')", choices=groups)
# TODO: model_ids
parser.add_argument('-c', '--classes', dest="classes", default='', help="if given, this value will limit the output files to those belonging to the given classes. (defaults to '%(default)s')", choices=('client', 'impostor', ''))
parser.add_argument('--self-test', dest="selftest", default=False,
action='store_true', help=SUPPRESS)
parser.set_defaults(func=dumplist) #action
......@@ -12,6 +12,8 @@ from bob.db.sqlalchemy_migration import Enum, relationship
from sqlalchemy.orm import backref
from sqlalchemy.ext.declarative import declarative_base
import xbob.db.verification.utils
Base = declarative_base()
protocolPurpose_file_association = Table('protocolPurpose_file_association', Base.metadata,
......@@ -38,7 +40,7 @@ class Client(Base):
__tablename__ = 'client'
class File(Base):
class File(Base, xbob.db.verification.utils.File):
"""Generic file container"""
__tablename__ = 'file'
......@@ -59,65 +61,20 @@ class File(Base):
# for Python
client = relationship("Client", backref=backref("files", order_by=id))
def __init__(self, client_id, path, session_id, camera, shot_id):
self.client_id = client_id
self.path = path
# call base class constructor
xbob.db.verification.utils.File.__init__(self, client_id = client_id, path = path)
self.session_id = session_id
self.camera = camera
self.shot_id = shot_id
def __repr__(self):
return "File('%s')" % self.path
def make_path(self, directory=None, extension=None):
"""Wraps the current path so that a complete path is formed
Keyword parameters:
directory
An optional directory name that will be prefixed to the returned result.
extension
An optional extension that will be suffixed to the returned filename. The
extension normally includes the leading ``.`` character as in ``.jpg`` or
``.hdf5``.
Returns a string containing the newly generated file path.
"""
if not directory: directory = ''
if not extension: extension = ''
return os.path.join(directory, self.path + extension)
def save(self, data, directory=None, extension='.hdf5'):
"""Saves the input data at the specified location and using the given
extension.
Keyword parameters:
data
The data blob to be saved (normally a :py:class:`numpy.ndarray`).
directory
If not empty or None, this directory is prefixed to the final file
destination
extension
The extension of the filename - this will control the type of output and
the codec for saving the input blob.
"""
path = self.make_path(directory, extension)
bob.utils.makedirs_safe(os.path.dirname(path))
bob.io.save(data, path)
class Protocol(Base):
"""Biosecure protocols"""
__tablename__ = 'protocol'
# Unique identifier for this protocol object
id = Column(Integer, primary_key=True)
# Name of the protocol associated with this object
......@@ -133,7 +90,7 @@ class ProtocolPurpose(Base):
"""Biosecure protocol purposes"""
__tablename__ = 'protocolPurpose'
# Unique identifier for this protocol purpose object
id = Column(Integer, primary_key=True)
# Id of the protocol associated with this protocol purpose object
......
......@@ -11,11 +11,11 @@ from bob.db import utils
from .models import *
from .driver import Interface
INFO = Interface()
import xbob.db.verification.utils
SQLITE_FILE = INFO.files()[0]
SQLITE_FILE = Interface().files()[0]
class Database(object):
class Database(xbob.db.verification.utils.SQLiteDatabase):
"""The dataset class opens and maintains a connection opened to the Database.
It provides many different ways to probe for the characteristics of the data
......@@ -23,37 +23,8 @@ class Database(object):
"""
def __init__(self):
# opens a session to the database - keep it open until the end
self.connect()
def connect(self):
"""Tries connecting or re-connecting to the database"""
if not os.path.exists(SQLITE_FILE):
self.session = None
else:
self.session = utils.session_try_readonly(INFO.type(), SQLITE_FILE)
def is_valid(self):
"""Returns if a valid session has been opened for reading the database"""
return self.session is not None
def assert_validity(self):
"""Raise a RuntimeError if the database backend is not available"""
if not self.is_valid():
raise RuntimeError, "Database '%s' cannot be found at expected location '%s'. Create it and then try re-connecting using Database.connect()" % (INFO.name(), SQLITE_FILE)
def __check_validity__(self, l, obj, valid, default):
"""Checks validity of user input data against a set of valid values"""
if not l: return default
elif not isinstance(l, (tuple,list)):
return self.__check_validity__((l,), obj, valid, default)
for k in l:
if k not in valid:
raise RuntimeError, 'Invalid %s "%s". Valid values are %s, or lists/tuples of those' % (obj, k, valid)
return l
# call base class constructors
xbob.db.verification.utils.SQLiteDatabase.__init__(self, SQLITE_FILE)
def groups(self):
"""Returns the names of all registered groups"""
......@@ -74,14 +45,10 @@ class Database(object):
Returns: A list containing all the clients which have the given properties.
"""
self.assert_validity()
VALID_PROTOCOLS = self.protocol_names()
VALID_GROUPS = self.groups()
protocol = self.__check_validity__(protocol, "protocol", VALID_PROTOCOLS, VALID_PROTOCOLS)
groups = self.__check_validity__(groups, "group", VALID_GROUPS, VALID_GROUPS)
protocol = self.check_parameters_for_validity(protocol, "protocol", self.protocol_names())
groups = self.check_parameters_for_validity(groups, "group", self.groups())
# List of the clients
q = self.session.query(Client).filter(Client.sgroup.in_(groups)).\
q = self.query(Client).filter(Client.sgroup.in_(groups)).\
order_by(Client.id)
return list(q)
......@@ -101,22 +68,36 @@ class Database(object):
return self.clients(protocol, groups)
def model_ids(self, protocol=None, groups=None):
"""Returns a list of model ids for the specific query by the user.
Keyword Parameters:
protocol
The protocol to consider ('ca0', 'caf', 'wc')
groups
The groups to which the subjects attached to the models belong ('dev', 'eval', 'world')
Returns: A list containing the ids of all models belonging to the given group.
"""
return [client.id for client in self.clients(protocol, groups)]
def has_client_id(self, id):
"""Returns True if we have a client with a certain integer identifier"""
self.assert_validity()
return self.session.query(Client).filter(Client.id==id).count() != 0
return self.query(Client).filter(Client.id==id).count() != 0
def client(self, id):
"""Returns the client object in the database given a certain id. Raises
an error if that does not exist."""
self.assert_validity()
return self.session.query(Client).filter(Client.id==id).one()
return self.query(Client).filter(Client.id==id).one()
def get_client_id_from_model_id(self, model_id):
"""Returns the client_id attached to the given model_id
Keyword Parameters:
model_id
......@@ -126,7 +107,7 @@ class Database(object):
"""
return model_id
def objects(self, protocol=None, purposes=None, model_ids=None, groups=None,
def objects(self, protocol=None, purposes=None, model_ids=None, groups=None,
classes=None):
"""Returns a set of filenames for the specific query by the user.
WARNING: Files used as impostor access for several different models are
......@@ -139,39 +120,32 @@ class Database(object):
purposes
The purposes required to be retrieved ('enrol', 'probe', 'train') or a tuple
with several of them. If 'None' is given (this is the default), it is
with several of them. If 'None' is given (this is the default), it is
considered the same as a tuple with all possible values. This field is
ignored for the data from the "world" group.
model_ids
Only retrieves the files for the provided list of model ids (claimed
client id). The model ids are string. If 'None' is given (this is
Only retrieves the files for the provided list of model ids (claimed
client id). The model ids are string. If 'None' is given (this is
the default), no filter over the model_ids is performed.
groups
One of the groups ('dev', 'eval', 'world') or a tuple with several of them.
If 'None' is given (this is the default), it is considered the same as a
One of the groups ('dev', 'eval', 'world') or a tuple with several of them.
If 'None' is given (this is the default), it is considered the same as a
tuple with all possible values.
classes
The classes (types of accesses) to be retrieved ('client', 'impostor')
or a tuple with several of them. If 'None' is given (this is the
The classes (types of accesses) to be retrieved ('client', 'impostor')
or a tuple with several of them. If 'None' is given (this is the
default), it is considered the same as a tuple with all possible values.
Returns: A list of files which have the given properties.
"""
self.assert_validity()