Commit 5f394119 authored by Manuel Günther's avatar Manuel Günther

Based database on the novel xbob.db.verification.utils interface; some cleaned up.

parent 43416491
......@@ -25,6 +25,7 @@ setup(
install_requires=[
'setuptools',
'bob', # base signal proc./machine learning library
'xbob.db.verification.utils' # defines a set of utilities for face verification databases like this one.
],
namespace_packages = [
......
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Laurent El Shafey <laurent.el-shafey@idiap.ch>
"""Checks for installed files.
"""
import os
import sys
# Driver API
# ==========
def checkfiles(args):
"""Checks existence of files based on your criteria"""
from .query import Database
db = Database()
r = db.objects(
protocol=args.protocol,
purposes=args.purposes,
#model_ids=args.model_ids,
groups=args.groups,
languages=args.languages
)
# go through all files, check if they are available on the filesystem
good = []
bad = []
for f in r:
if os.path.exists(f.make_path(args.directory, args.extension)):
good.append(f)
else:
bad.append(f)
# report
output = sys.stdout
if args.selftest:
from bob.db.utils import null
output = null()
if bad:
for f in bad:
output.write('Cannot find file "%s"\n' % (f.make_path(args.directory, args.extension),))
output.write('%d files (out of %d) were not found at "%s"\n' % \
(len(bad), len(r), args.directory))
return 0
def add_command(subparsers):
"""Add specific subcommands that the action "checkfiles" can use"""
from argparse import SUPPRESS
parser = subparsers.add_parser('checkfiles', help=checkfiles.__doc__)
from .query import Database
db = Database()
parser.add_argument('-d', '--directory', dest="directory", default='', help="if given, this path will be prepended to every entry returned (defaults to '%(default)s')")
parser.add_argument('-e', '--extension', dest="extension", default='', help="if given, this extension will be appended to every entry returned (defaults to '%(default)s')")
parser.add_argument('-p', '--protocol', dest="protocol", default='', help="if given, limits the check to a particular subset of the data that corresponds to the given protocol (defaults to '%(default)s')", choices=db.protocol_names())
parser.add_argument('-u', '--purposes', dest="purposes", default='', help="if given, this value will limit the output files to those designed for the given purposes. (defaults to '%(default)s')", choices=db.purposes())
# TODO: model_ids
parser.add_argument('-g', '--groups', dest="groups", default='', help="if given, this value will limit the output files to those belonging to a particular protocolar group. (defaults to '%(default)s')", choices=db.groups())
parser.add_argument('-l', '--languages', dest="languages", default='', help="if given, this value will limit the output files to those belonging to the given languages. (defaults to '%(default)s')", choices=db.languages())
parser.add_argument('-c', '--classes', dest="classes", default='', help="if given, this value will limit the output files to those belonging to the given classes. (defaults to '%(default)s')", choices=('client', 'impostor', ''))
parser.add_argument('--self-test', dest="selftest", default=False,
action='store_true', help=SUPPRESS)
parser.set_defaults(func=checkfiles) #action
......@@ -12,12 +12,12 @@ from .models import *
def nodot(item):
"""Can be used to ignore hidden files, starting with the . character."""
return item[0] != '.'
return item[0] != '.'
def add_files(session, imagedir, verbose):
"""Add files (and clients) to the BANCA database."""
def add_file(session, filename, client_dict, verbose):
def add_file(session, subdir, filename, client_dict, verbose):
"""Parse a single filename and add it to the list.
Also add a client entry if not already in the database."""
......@@ -28,15 +28,16 @@ def add_files(session, imagedir, verbose):
session.add(Client(int(v[0]), v[1], v[2], v[5]))
client_dict[v[0]] = True
session_id = int(v[3].split('s')[1])
if verbose: print "Adding file '%s'..." %(os.path.basename(filename).split('.')[0], )
session.add(File(int(v[0]), os.path.basename(filename).split('.')[0], v[4], v[6], session_id))
base_path = os.path.join(subdir,os.path.basename(filename).split('.')[0])
if verbose: print "Adding file '%s'..." %(base_path, )
session.add(File(int(v[0]), base_path, v[4], v[6], session_id))
subdir_list = filter(nodot, os.listdir(imagedir))
client_dict = {}
for subdir in subdir_list:
file_list = filter(nodot, os.listdir(os.path.join(imagedir, subdir)))
for filename in file_list:
add_file(session, os.path.join(imagedir, filename), client_dict, verbose)
add_file(session, subdir, os.path.join(imagedir, filename), client_dict, verbose)
def add_subworlds(session, verbose):
"""Adds splits in the world set, based on the client ids"""
......
......@@ -9,6 +9,64 @@ import os
import sys
from bob.db.driver import Interface as BaseInterface
def dumplist(args):
"""Dumps lists of files based on your criteria"""
from .query import Database
db = Database()
r = db.objects(
protocol=args.protocol,
purposes=args.purposes,
model_ids=(args.client,),
groups=args.groups,
languages=args.languages
)
output = sys.stdout
if args.selftest:
from bob.db.utils import null
output = null()
for f in r:
output.write('%s\n' % (f.make_path(args.directory, args.extension),))
return 0
def checkfiles(args):
"""Checks existence of files based on your criteria"""
from .query import Database
db = Database()
r = db.objects()
# go through all files, check if they are available on the filesystem
good = []
bad = []
for f in r:
if os.path.exists(f.make_path(args.directory, args.extension)):
good.append(f)
else:
bad.append(f)
# report
output = sys.stdout
if args.selftest:
from bob.db.utils import null
output = null()
if bad:
for f in bad:
output.write('Cannot find file "%s"\n' % (f.make_path(args.directory, args.extension),))
output.write('%d files (out of %d) were not found at "%s"\n' % \
(len(bad), len(r), args.directory))
return 0
def reverse(args):
"""Returns a list of file database identifiers given the path stems"""
......@@ -27,18 +85,6 @@ def reverse(args):
return 0
def reverse_command(subparsers):
"""Adds the specific options for the reverse command"""
from argparse import SUPPRESS
parser = subparsers.add_parser('reverse', help=reverse.__doc__)
parser.add_argument('path', nargs='+', type=str, help="one or more path stems to look up. If you provide more than one, files which cannot be reversed will be omitted from the output.")
parser.add_argument('--self-test', dest="selftest", default=False,
action='store_true', help=SUPPRESS)
parser.set_defaults(func=reverse) #action
def path(args):
"""Returns a list of fully formed paths or stems given some file id"""
......@@ -58,30 +104,16 @@ def path(args):
return 0
def path_command(subparsers):
"""Adds the specific options for the path command"""
from argparse import SUPPRESS
parser = subparsers.add_parser('path', help=path.__doc__)
parser.add_argument('-d', '--directory', dest="directory", default='', help="if given, this path will be prepended to every entry returned (defaults to '%(default)s')")
parser.add_argument('-e', '--extension', dest="extension", default='', help="if given, this extension will be appended to every entry returned (defaults to '%(default)s')")
parser.add_argument('id', nargs='+', type=int, help="one or more file ids to look up. If you provide more than one, files which cannot be found will be omitted from the output. If you provide a single id to lookup, an error message will be printed if the id does not exist in the database. The exit status will be non-zero in such case.")
parser.add_argument('--self-test', dest="selftest", default=False,
action='store_true', help=SUPPRESS)
parser.set_defaults(func=path) #action
class Interface(BaseInterface):
def name(self):
return 'banca'
def version(self):
import pkg_resources # part of setuptools
return pkg_resources.require('xbob.db.%s' % self.name())[0].version
def files(self):
from pkg_resources import resource_filename
......@@ -94,24 +126,49 @@ class Interface(BaseInterface):
def add_commands(self, parser):
from . import __doc__ as docs
subparsers = self.setup_parser(parser,
"BANCA database", docs)
# example: get the "create" action from a submodule
from .create import add_command as create_command
create_command(subparsers)
# example: get the "dumplist" action from a submodule
from .dumplist import add_command as dumplist_command
dumplist_command(subparsers)
# example: get the "checkfiles" action from a submodule
from .checkfiles import add_command as checkfiles_command
checkfiles_command(subparsers)
# example: get the "dumplist" action from a submodule
from .query import Database
import argparse
db = Database()
parser = subparsers.add_parser('dumplist', help=dumplist.__doc__)
parser.add_argument('-d', '--directory', dest="directory", default='', help="if given, this path will be prepended to every entry returned.")
parser.add_argument('-e', '--extension', dest="extension", default='', help="if given, this extension will be appended to every entry returned.")
parser.add_argument('-p', '--protocol', dest="protocol", default='', help="if given, limits the dump to a particular subset of the data that corresponds to the given protocol.", choices=db.protocol_names() if db.is_valid() else ())
parser.add_argument('-u', '--purposes', dest="purposes", default='', help="if given, this value will limit the output files to those designed for the given purposes.", choices=db.purposes() if db.is_valid() else ())
parser.add_argument('-C', '--client', dest="client", default=None, type=int, help="if given, limits the dump to a particular client", choices=db.clients() if db.is_valid() else ())
parser.add_argument('-g', '--groups', dest="groups", default='', help="if given, this value will limit the output files to those belonging to a particular protocolar group.", choices=db.groups() if db.is_valid() else ())
parser.add_argument('-l', '--languages', dest="languages", default='', help="if given, this value will limit the output files to those belonging to the given languages.", choices=db.languages() if db.is_valid() else ())
parser.add_argument('-c', '--classes', dest="classes", default='', help="if given, this value will limit the output files to those belonging to the given classes.", choices=('client', 'impostor', ''))
parser.add_argument('--self-test', dest="selftest", action='store_true', help=argparse.SUPPRESS)
parser.set_defaults(func=dumplist) #action
# the "checkfiles" action
parser = subparsers.add_parser('checkfiles', help=checkfiles.__doc__)
parser.add_argument('-d', '--directory', dest="directory", default='', help="if given, this path will be prepended to every entry returned.")
parser.add_argument('-e', '--extension', dest="extension", default='', help="if given, this extension will be appended to every entry returned.")
parser.add_argument('--self-test', dest="selftest", action='store_true', help=argparse.SUPPRESS)
parser.set_defaults(func=checkfiles) #action
# adds the "reverse" command
reverse_command(subparsers)
parser = subparsers.add_parser('reverse', help=reverse.__doc__)
parser.add_argument('path', nargs='+', type=str, help="one or more path stems to look up. If you provide more than one, files which cannot be reversed will be omitted from the output.")
parser.add_argument('--self-test', dest="selftest", action='store_true', help=argparse.SUPPRESS)
parser.set_defaults(func=reverse) #action
# adds the "path" command
path_command(subparsers)
parser = subparsers.add_parser('path', help=path.__doc__)
parser.add_argument('-d', '--directory', dest="directory", default='', help="if given, this path will be prepended to every entry returned.")
parser.add_argument('-e', '--extension', dest="extension", default='', help="if given, this extension will be appended to every entry returned.")
parser.add_argument('id', nargs='+', type=int, help="one or more file ids to look up. If you provide more than one, files which cannot be found will be omitted from the output. If you provide a single id to lookup, an error message will be printed if the id does not exist in the database. The exit status will be non-zero in such case.")
parser.add_argument('--self-test', dest="selftest", action='store_true', help=argparse.SUPPRESS)
parser.set_defaults(func=path) #action
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Laurent El Shafey <laurent.el-shafey@idiap.ch>
"""Dumps lists of files.
"""
import os
import sys
# Driver API
# ==========
def dumplist(args):
"""Dumps lists of files based on your criteria"""
from .query import Database
db = Database()
r = db.objects(
protocol=args.protocol,
purposes=args.purposes,
model_ids=(args.client,),
groups=args.groups,
languages=args.languages
)
output = sys.stdout
if args.selftest:
from bob.db.utils import null
output = null()
for f in r:
output.write('%s\n' % (f.make_path(args.directory, args.extension),))
return 0
def add_command(subparsers):
"""Add specific subcommands that the action "dumplist" can use"""
from argparse import SUPPRESS
parser = subparsers.add_parser('dumplist', help=dumplist.__doc__)
from .query import Database
db = Database()
if not db.is_valid():
clients = tuple()
else:
clients = [client.id for client in db.clients()]
parser.add_argument('-d', '--directory', dest="directory", default='', help="if given, this path will be prepended to every entry returned (defaults to '%(default)s')")
parser.add_argument('-e', '--extension', dest="extension", default='', help="if given, this extension will be appended to every entry returned (defaults to '%(default)s')")
parser.add_argument('-p', '--protocol', dest="protocol", default='', help="if given, limits the dump to a particular subset of the data that corresponds to the given protocol (defaults to '%(default)s')", choices=db.protocol_names())
parser.add_argument('-u', '--purposes', dest="purposes", default='', help="if given, this value will limit the output files to those designed for the given purposes. (defaults to '%(default)s')", choices=db.purposes())
parser.add_argument('-C', '--client', dest="client", default=None, type=int, help="if given, limits the dump to a particular client (defaults to '%(default)s')", choices=clients)
parser.add_argument('-g', '--groups', dest="groups", default='', help="if given, this value will limit the output files to those belonging to a particular protocolar group. (defaults to '%(default)s')", choices=db.groups())
parser.add_argument('-l', '--languages', dest="languages", default='', help="if given, this value will limit the output files to those belonging to the given languages. (defaults to '%(default)s')", choices=db.languages())
parser.add_argument('-c', '--classes', dest="classes", default='', help="if given, this value will limit the output files to those belonging to the given classes. (defaults to '%(default)s')", choices=('client', 'impostor', ''))
parser.add_argument('--self-test', dest="selftest", default=False,
action='store_true', help=SUPPRESS)
parser.set_defaults(func=dumplist) #action
......@@ -12,6 +12,8 @@ from bob.db.sqlalchemy_migration import Enum, relationship
from sqlalchemy.orm import backref
from sqlalchemy.ext.declarative import declarative_base
import xbob.db.verification.utils
Base = declarative_base()
subworld_client_association = Table('subworld_client_association', Base.metadata,
......@@ -68,7 +70,7 @@ class Subworld(Base):
def __repr__(self):
return "Subworld('%s')" % (self.name)
class File(Base):
class File(Base, xbob.db.verification.utils.File):
"""Generic file container"""
__tablename__ = 'file'
......@@ -90,57 +92,13 @@ class File(Base):
real_client = relationship("Client", backref=backref("files", order_by=id))
def __init__(self, client_id, path, claimed_id, shot_id, session_id):
self.client_id = client_id
self.path = path
# call base class constructor
xbob.db.verification.utils.File.__init__(self, client_id = client_id, path = path)
self.claimed_id = claimed_id
self.shot_id = shot_id
self.session_id = session_id
def __repr__(self):
return "File('%s')" % self.path
def make_path(self, directory=None, extension=None):
"""Wraps the current path so that a complete path is formed
Keyword parameters:
directory
An optional directory name that will be prefixed to the returned result.
extension
An optional extension that will be suffixed to the returned filename. The
extension normally includes the leading ``.`` character as in ``.jpg`` or
``.hdf5``.
Returns a string containing the newly generated file path.
"""
if not directory: directory = ''
if not extension: extension = ''
return os.path.join(directory, self.path + extension)
def save(self, data, directory=None, extension='.hdf5'):
"""Saves the input data at the specified location and using the given
extension.
Keyword parameters:
data
The data blob to be saved (normally a :py:class:`numpy.ndarray`).
directory
If not empty or None, this directory is prefixed to the final file
destination
extension
The extension of the filename - this will control the type of output and
the codec for saving the input blob.
"""
path = self.make_path(directory, extension)
bob.utils.makedirs_safe(os.path.dirname(path))
bob.io.save(data, path)
class Protocol(Base):
"""BANCA protocols"""
......
This diff is collapsed.
......@@ -21,24 +21,68 @@
import os, sys
import unittest
from .query import Database
import xbob.db.banca
class BancaDatabaseTest(unittest.TestCase):
"""Performs various tests on the BANCA database."""
def test01_manage_dumplist_1(self):
def test01_clients(self):
# test whether the correct number of clients is returned
db = xbob.db.banca.Database()
self.assertEqual(len(db.clients()), 82)
self.assertEqual(len(db.clients(groups='world')), 30)
self.assertEqual(len(db.clients(groups='dev')), 26)
self.assertEqual(len(db.clients(groups='eval')), 26)
self.assertEqual(len(db.tclients(groups='dev')), 26)
self.assertEqual(len(db.tclients(groups='eval')), 26)
self.assertEqual(len(db.clients(genders='f')), 41)
self.assertEqual(len(db.clients(genders='m')), 41)
def test02_objects(self):
# tests if the right number of File objects is returned
db = xbob.db.banca.Database()
self.assertEqual(len(db.objects()), 6540)
self.assertEqual(len(db.objects(groups='world')), 300)
self.assertEqual(len(db.objects(groups='dev')), 3120)
self.assertEqual(len(db.objects(groups='eval')), 3120)
# test for the different protocols
for protocol in db.protocols():
# assure that the number of enroll files is independent from the protocol
for group in ('dev', 'eval'):
self.assertEqual(len(db.objects(groups=group, purposes='enrol')), 390)
for model_id in db.model_ids(groups=group):
self.assertEqual(len(db.objects(groups=group, purposes='enrol', model_ids=model_id)), 15)
for model_id in db.tmodel_ids(groups=group):
self.assertEqual(len(db.tobjects(groups=group, model_ids=model_id)), 15)
# check the number of probe files
for group in ('dev', 'eval'):
self.assertEqual(len(db.objects(groups=group, purposes='probe')), 2730)
for model_id in db.model_ids(groups=group):
self.assertEqual(len(db.objects(groups=group, purposes='probe', model_ids=model_id)), 105)
for model_id in db.tmodel_ids(groups=group):
self.assertEqual(len(db.zobjects(groups=group, model_ids=model_id)), 105)
def test03_manage_dumplist_1(self):
from bob.db.script.dbmanage import main
self.assertEqual(main('banca dumplist --self-test'.split()), 0)
def test02_manage_dumplist_2(self):
def test04_manage_dumplist_2(self):
from bob.db.script.dbmanage import main
self.assertEqual(main('banca dumplist --protocol=P --classes=client --groups=dev --purposes=enrol --self-test'.split()), 0)
def test03_manage_checkfiles(self):
def test05_manage_checkfiles(self):
from bob.db.script.dbmanage import main
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment