Commit 42b07ac9 authored by André Anjos's avatar André Anjos 💬

Initial version

parents
*~
*.swp
*.pyc
bin
eggs
parts
.installed.cfg
.mr.developer.cfg
*.egg-info
src
develop-eggs
This diff is collapsed.
This diff is collapsed.
; vim: set fileencoding=utf-8 :
; Andre Anjos <andre.anjos@idiap.ch>
; Mon 16 Apr 08:29:18 2012 CEST
[buildout]
parts = python
develop = .
eggs = bob.db.replay
[python]
recipe = zc.recipe.egg
interpreter = python
eggs = ${buildout:eggs}
; vim: set fileencoding=utf-8 :
; Andre Anjos <andre.anjos@idiap.ch>
; Mon 16 Apr 08:29:18 2012 CEST
; Example buildout recipe using a local (off-root) Bob installation
[buildout]
parts = bob python tests
develop = .
eggs = bob
bob.db.replay
[bob]
recipe = local.bob.recipe:config
; Set here the private installation directory for Bob
install-directory = /scratch/aanjos/bob-master/build/newdb/release
[python]
recipe = zc.recipe.egg
interpreter = python
eggs = ${buildout:eggs}
[tests]
recipe = pbp.recipe.noserunner
eggs = ${buildout:eggs}
pbp.recipe.noserunner
script = tests.py
========================
Replay Attack Database
========================
This package contains the access API and descriptions for the `Replay Attack
Database <http://www.idiap.ch/dataset/replayattack/>`_. The actual raw data for
the database should be downloaded from the given URL, this package contains
only the `Bob <http://idiap.ch/software/bob/>`_ accessor methods to use the DB
directly from python, with our certified protocols.
You would normally not install this package unless you are maintaining it. What
you would do instead is to tie it in at the package you need to **use** it.
There are a few ways to achieve this:
1. You can add this package as a requirement at the ``setup.py`` for your own
`satellite package
<https://github.com/idiap/bob/wiki/Virtual-Work-Environments-with-Buildout>`_
or to your Buildout ``.cfg`` file, if you prefer it that way. With this
method, this package gets automatically downloaded and installed on your
working environment, or
2. You can manually download and install this package using commands like
``easy_install`` or ``pip``.
The package is available in two different distribution formats:
a. You can download it from `PyPI <http://pypi.python.org/pypi>`_, or
b. You can download it in its source form from `its git repository
<http://github.com/bioidiap/bob.db.replay.git>`_. When you download the
version at the git repository, you will need to run a command to recreate the
backend SQLite file required for its operation. This means that the database
raw files must be installed somewhere in this case. With option ``a`` you can
run in `dummy` mode and only download the raw data files for the database
once you are happy with your setup.
You can mix and match points 1/2 and a/b above based on your requirements. Here
are some examples:
Modify your setup.py and download from PyPI
===========================================
That is the easiest. Edit your ``setup.py`` in your satellite package and add
the following entry in the ``install_requires`` section (note: ``...`` means
`whatever extra stuff you may have in-between`, don't put that on your
script)::
install_requires=[
...
"bob.db.replay >= 1.1",
],
Proceed normally with your ``boostrap/buildout`` steps and you should be all
set. That means you can now import ``bob.db.replay`` into your scripts.
Modify your buildout.cfg and download from git
==============================================
You will need to add a dependence to `mr.developer
<http://pypi.python.org/pypi/mr.developer/>`_ to be able to install from our
git repositories. Your ``buildout.cfg`` file should contain the following
lines::
[buildout]
...
extensions = mr.developer
auto-checkout = *
eggs = bob
...
bob.db.replay
[sources]
bob.db.replay = git https://github.com/bioidiap/bob.db.replay.git
...
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Andre Anjos <andre.dos.anjos@gmail.com>
# Wed 18 May 09:28:44 2011
"""The Idiap Replay attack database consists of Photo and Video attacks to
different identities under different illumination conditions.
"""
import os
def dbname():
'''Returns the database name'''
return 'replay'
def version():
'''Returns the current version number defined in setup.py (DRY)'''
import pkg_resources # part of setuptools
return pkg_resources.require('bob.db.replay')[0].version
def location():
'''Returns the directory that contains the data'''
return os.path.dirname(os.path.realpath(__file__))
def files():
'''Returns a python iterable with all auxiliary files needed'''
return ('db.sql3',)
def type():
'''Returns the type of auxiliary files you have for this database
If you return 'sqlite', then we append special actions such as 'dbshell'
on 'bob_dbmanage.py' automatically for you. Otherwise, we don't.
If you use auxiliary text files, just return 'text'. We may provide
special services for those types in the future.
'''
return 'sqlite'
# these are required for the dbmanage.py driver
from .query import Database
from .commands import add_commands
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Andre Anjos <andre.anjos@idiap.ch>
# Tue 20 Mar 19:20:22 2012 CET
"""Checks for installed files.
"""
import os
import sys
# Driver API
# ==========
def checkfiles(args):
"""Checks existence files based on your criteria"""
from .query import Database
db = Database()
r = db.files(
directory=args.directory,
extension=args.extension,
protocol=args.protocol,
support=args.support,
groups=args.group,
cls=args.cls,
light=args.light,
)
# go through all files, check if they are available on the filesystem
good = {}
bad = {}
for id, f in r.items():
if os.path.exists(f): good[id] = f
else: bad[id] = f
# report
output = sys.stdout
if args.selftest:
from bob.db.utils import null
output = null()
if bad:
for id, f in bad.items():
output.write('Cannot find file "%s"\n' % (f,))
output.write('%d files (out of %d) were not found at "%s"\n' % \
(len(bad), len(r), args.directory))
return 0
def add_command(subparsers):
"""Add specific subcommands that the action "checkfiles" can use"""
from argparse import SUPPRESS
parser = subparsers.add_parser('checkfiles', help=checkfiles.__doc__)
from .query import Database
db = Database()
if not db.is_valid():
protocols = ('waiting','for','database','creation')
else:
protocols = db.protocols()
parser.add_argument('-d', '--directory', dest="directory", default='', help="if given, this path will be prepended to every entry checked (defaults to '%(default)s')")
parser.add_argument('-e', '--extension', dest="extension", default='', help="if given, this extension will be appended to every entry checked (defaults to '%(default)s')")
parser.add_argument('-c', '--class', dest="cls", default='', help="if given, limits the check to a particular subset of the data that corresponds to the given class (defaults to '%(default)s')", choices=('real', 'attack', 'enroll'))
parser.add_argument('-g', '--group', dest="group", default='', help="if given, this value will limit the check to those files belonging to a particular protocolar group. (defaults to '%(default)s')", choices=db.groups())
parser.add_argument('-s', '--support', dest="support", default='', help="if given, this value will limit the check to those files using this type of attack support. (defaults to '%(default)s')", choices=db.attack_supports())
parser.add_argument('-x', '--protocol', dest="protocol", default='', help="if given, this value will limit the check to those files for a given protocol. (defaults to '%(default)s')", choices=protocols)
parser.add_argument('-l', '--light', dest="light", default='', help="if given, this value will limit the check to those files shot under a given lighting. (defaults to '%(default)s')", choices=db.lights())
parser.add_argument('--self-test', dest="selftest", default=False,
action='store_true', help=SUPPRESS)
parser.set_defaults(func=checkfiles) #action
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Andre Anjos <andre.anjos@idiap.ch>
# Tue 28 Jun 2011 15:20:09 CEST
"""Commands this database can respond to.
"""
import os
import sys
def reverse(args):
"""Returns a list of file database identifiers given the path stems"""
from .query import Database
db = Database()
output = sys.stdout
if args.selftest:
from bob.db.utils import null
output = null()
r = db.reverse(args.path)
for id in r: output.write('%d\n' % id)
if not r: return 1
return 0
def reverse_command(subparsers):
"""Adds the specific options for the reverse command"""
from argparse import SUPPRESS
parser = subparsers.add_parser('reverse', help=reverse.__doc__)
parser.add_argument('path', nargs='+', type=str, help="one or more path stems to look up. If you provide more than one, files which cannot be reversed will be omitted from the output.")
parser.add_argument('--self-test', dest="selftest", default=False,
action='store_true', help=SUPPRESS)
parser.set_defaults(func=reverse) #action
def path(args):
"""Returns a list of fully formed paths or stems given some file id"""
from .query import Database
db = Database()
output = sys.stdout
if args.selftest:
from bob.db.utils import null
output = null()
r = db.paths(args.id, prefix=args.directory, suffix=args.extension)
for path in r: output.write('%s\n' % path)
if not r: return 1
return 0
def path_command(subparsers):
"""Adds the specific options for the path command"""
from argparse import SUPPRESS
parser = subparsers.add_parser('path', help=path.__doc__)
parser.add_argument('-d', '--directory', dest="directory", default='', help="if given, this path will be prepended to every entry returned (defaults to '%(default)s')")
parser.add_argument('-e', '--extension', dest="extension", default='', help="if given, this extension will be appended to every entry returned (defaults to '%(default)s')")
parser.add_argument('id', nargs='+', type=int, help="one or more file ids to look up. If you provide more than one, files which cannot be found will be omitted from the output. If you provide a single id to lookup, an error message will be printed if the id does not exist in the database. The exit status will be non-zero in such case.")
parser.add_argument('--self-test', dest="selftest", default=False,
action='store_true', help=SUPPRESS)
parser.set_defaults(func=path) #action
def add_commands(parser):
"""Adds my subset of options and arguments to the top-level parser. For
details on syntax, please consult:
http://docs.python.org/dev/library/argparse.html
The strategy assumed here is that each command will have its own set of
options that are relevant to that command. So, we just scan such commands and
attach the options from those.
"""
from . import dbname, location, version, type, files
from bob.db.utils import standard_commands
from . import __doc__ as dbdoc
from argparse import RawDescriptionHelpFormatter
# creates a top-level parser for this database
myname = dbname()
top_level = parser.add_parser(myname,
formatter_class=RawDescriptionHelpFormatter,
help="Photo/Video Replay attack database", description=dbdoc)
top_level.set_defaults(dbname=myname)
top_level.set_defaults(location=location())
top_level.set_defaults(version=version())
top_level.set_defaults(type=type())
top_level.set_defaults(files=files())
# declare it has subparsers for each of the supported commands
subparsers = top_level.add_subparsers(title="subcommands")
# attach standard commands
standard_commands(subparsers, type(), files())
# get the "create" action from a submodule
from .create import add_command as create_command
create_command(subparsers)
# get the "dumplist" action from a submodule
from .dumplist import add_command as dumplist_command
dumplist_command(subparsers)
# get the "checkfiles" action from a submodule
from .checkfiles import add_command as checkfiles_command
checkfiles_command(subparsers)
# adds the "reverse" command
reverse_command(subparsers)
# adds the "path" command
path_command(subparsers)
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Andre Anjos <andre.dos.anjos@gmail.com>
# Thu 12 May 08:19:50 2011
"""This script creates the Replay-Attack database in a single pass.
"""
import os
import fnmatch
from .models import *
def add_clients(session, protodir, verbose):
"""Add clients to the replay attack database."""
for client in open(os.path.join(protodir, 'clients.txt'), 'rt'):
s = client.strip().split(' ', 2)
if not s: continue #empty line
id = int(s[0])
set = s[1]
if verbose: print "Adding client %d on '%s' set..." % (id, set)
session.add(Client(id, set))
def add_real_lists(session, protodir, verbose):
"""Adds all RCD filelists"""
def add_real_list(session, filename):
"""Adds an RCD filelist and materializes RealAccess'es."""
def parse_real_filename(f):
"""Parses the RCD filename and break it in the relevant chunks."""
v = os.path.splitext(os.path.basename(f))[0].split('_')
client_id = int(v[0].replace('client',''))
path = os.path.splitext(f)[0] #keep only the filename stem
purpose = v[3]
light = v[4]
if len(v) == 6: take = int(v[5]) #authentication session
else: take = 1 #enrollment session
return [client_id, path, light], [purpose, take]
for fname in open(filename, 'rt'):
s = fname.strip()
if not s: continue #emtpy line
filefields, realfields = parse_real_filename(s)
filefields[0] = session.query(Client).filter(Client.id == filefields[0]).one()
file = File(*filefields)
session.add(file)
realfields.insert(0, file)
session.add(RealAccess(*realfields))
add_real_list(session, os.path.join(protodir, 'real-train.txt'))
add_real_list(session, os.path.join(protodir, 'real-devel.txt'))
add_real_list(session, os.path.join(protodir, 'real-test.txt'))
add_real_list(session, os.path.join(protodir, 'recognition-train.txt'))
add_real_list(session, os.path.join(protodir, 'recognition-devel.txt'))
add_real_list(session, os.path.join(protodir, 'recognition-test.txt'))
def add_attack_lists(session, protodir, verbose):
"""Adds all RAD filelists"""
def add_attack_list(session, filename):
"""Adds an RAD filelist and materializes Attacks."""
def parse_attack_filename(f):
"""Parses the RAD filename and break it in the relevant chunks."""
v = os.path.splitext(os.path.basename(f))[0].split('_')
attack_device = v[1] #print, mobile or highdef
client_id = int(v[2].replace('client',''))
path = os.path.splitext(f)[0] #keep only the filename stem
sample_device = v[4] #highdef or mobile
sample_type = v[5] #photo or video
light = v[6]
attack_support = f.split('/')[-2]
return [client_id, path, light], [attack_support, attack_device, sample_type, sample_device]
for fname in open(filename, 'rt'):
s = fname.strip()
if not s: continue #emtpy line
filefields, attackfields = parse_attack_filename(s)
filefields[0] = session.query(Client).filter(Client.id == filefields[0]).one()
file = File(*filefields)
session.add(file)
attackfields.insert(0, file)
session.add(Attack(*attackfields))
add_attack_list(session,os.path.join(protodir, 'attack-grandtest-allsupports-train.txt'))
add_attack_list(session,os.path.join(protodir, 'attack-grandtest-allsupports-devel.txt'))
add_attack_list(session,os.path.join(protodir, 'attack-grandtest-allsupports-test.txt'))
def define_protocols(session, protodir, verbose):
"""Defines all available protocols"""
#figures out which protocols to use
valid = {}
for fname in fnmatch.filter(os.listdir(protodir), 'attack-*-allsupports-train.txt'):
s = fname.split('-', 4)
consider = True
files = {}
for grp in ('train', 'devel', 'test'):
# check attack file
attack = os.path.join(protodir, 'attack-%s-allsupports-%s.txt' % (s[1], grp))
if not os.path.exists(attack):
if verbose:
print "Not considering protocol %s as attack list '%s' was not found" % (s[1], attack)
consider = False
# check real file
real = os.path.join(protodir, 'real-%s-allsupports-%s.txt' % (s[1], grp))
if not os.path.exists(real):
alt_real = os.path.join(protodir, 'real-%s.txt' % (grp,))
if not os.path.exists(alt_real):
if verbose:
print "Not considering protocol %s as real list '%s' or '%s' were not found" % (s[1], real, alt_real)
consider = False
else:
real = alt_real
if consider: files[grp] = (attack, real)
if consider: valid[s[1]] = files
for protocol, groups in valid.iteritems():
if verbose: print "Creating protocol '%s'..." % protocol
# create protocol on the protocol table
obj = Protocol(name=protocol)
for grp, flist in groups.iteritems():
counter = 0
for fname in open(flist[0], 'rt'):
s = os.path.splitext(fname.strip())[0]
q = session.query(Attack).join(File).filter(File.path == s).one()
q.protocols.append(obj)
counter += 1
if verbose: print " -> %5s/%-6s: %d files" % (grp, "attack", counter)
counter = 0
for fname in open(flist[1], 'rt'):
s = os.path.splitext(fname.strip())[0]
q = session.query(RealAccess).join(File).filter(File.path == s).one()
q.protocols.append(obj)
counter += 1
if verbose: print " -> %5s/%-6s: %d files" % (grp, "real", counter)
session.add(obj)
def create_tables(args):
"""Creates all necessary tables (only to be used at the first time)"""
from bob.db.utils import connection_string
from sqlalchemy import create_engine
engine = create_engine(connection_string(args.type, args.location,
args.files[0]), echo=args.verbose)
Client.metadata.create_all(engine)
RealAccess.metadata.create_all(engine)
Attack.metadata.create_all(engine)
Protocol.metadata.create_all(engine)
# Driver API
# ==========
def create(args):
"""Creates or re-creates this database"""
from bob.db.utils import session
dbfile = os.path.join(args.location, args.files[0])
args.verbose = 0 if args.verbose is None else sum(args.verbose)
if args.recreate:
if args.verbose and os.path.exists(dbfile):
print('unlinking %s...' % dbfile)
if os.path.exists(dbfile): os.unlink(dbfile)
if not os.path.exists(os.path.dirname(dbfile)):
os.makedirs(os.path.dirname(dbfile))
# the real work...
create_tables(args)
s = session(args.type, args.location, args.files[0], echo=(args.verbose >= 2))
add_clients(s, args.protodir, args.verbose)
add_real_lists(s, args.protodir, args.verbose)
add_attack_lists(s, args.protodir, args.verbose)
define_protocols(s, args.protodir, args.verbose)
s.commit()
s.close()
return 0
def add_command(subparsers):
"""Add specific subcommands that the action "create" can use"""
parser = subparsers.add_parser('create', help=create.__doc__)
parser.add_argument('-R', '--recreate', action='store_true', default=False,
help="If set, I'll first erase the current database")
parser.add_argument('-v', '--verbose', action='append_const', const=1,
help="Do SQL operations in a verbose way")
parser.add_argument('-D', '--protodir', action='store',
default='/idiap/group/replay/database/protocols/replayattack-database/protocols',
metavar='DIR',
help="Change the relative path to the directory containing the protocol definitions for replay attacks (defaults to %(default)s)")
parser.set_defaults(func=create) #action
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Andre Anjos <andre.dos.anjos@gmail.com>
# Thu 12 May 14:02:28 2011
"""Dumps lists of files.
"""
import os
import sys
# Driver API
# ==========
def dumplist(args):
"""Dumps lists of files based on your criteria"""
from .query import Database
db = Database()
r = db.files(
directory=args.directory,
extension=args.extension,
protocol=args.protocol,
support=args.support,
groups=args.group,
cls=args.cls,
light=args.light,
)
output = sys.stdout
if args.selftest:
from bob.db.utils import null
output = null()
for id, f in r.items():
output.write('%s\n' % (f,))
return 0
def add_command(subparsers):
"""Add specific subcommands that the action "dumplist" can use"""
from argparse import SUPPRESS
parser = subparsers.add_parser('dumplist', help=dumplist.__doc__)
from .query import Database
db = Database()
if not db.is_valid():
protocols = ('waiting','for','database','creation')
else:
protocols = db.protocols()
parser.add_argument('-d', '--directory', dest="directory", default='', help="if given, this path will be prepended to every entry returned (defaults to '%(default)s')")
parser.add_argument('-e', '--extension', dest="extension", default='', help="if given, this extension will be appended to every entry returned (defaults to '%(default)s')")
parser.add_argument('-c', '--class', dest="cls", default='', help="if given, limits the dump to a particular subset of the data that corresponds to the given class (defaults to '%(default)s')", choices=('real', 'attack', 'enroll'))
parser.add_argument('-g', '--group', dest="group", default='', help="if given, this value will limit the output files to those belonging to a particular protocolar group. (defaults to '%(default)s')", choices=db.groups())
parser.add_argument('-s', '--support', dest="support", default='', help="if given, this value will limit the output files to those using this type of attack support. (defaults to '%(default)s')", choices=db.attack_supports())
parser.add_argument('-x', '--protocol', dest="protocol", default='', help="if given, this value will limit the output files to those for a given protocol. (defaults to '%(default)s')", choices=protocols)
parser.add_argument('-l', '--light', dest="light", default='', help="if given, this value will limit the output files to those shot under a given lighting. (defaults to '%(default)s')", choices=db.lights())
parser.add_argument('--self-test', dest="selftest", default=False,
action='store_true', help=SUPPRESS)
parser.set_defaults(func=dumplist) #action
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Andre Anjos <andre.dos.anjos@gmail.com>
# Wed 11 May 18:52:38 2011
"""Table models and functionality for the Replay Attack DB.
"""
from sqlalchemy import Table, Column, Integer, String, ForeignKey
from bob.db.sqlalchemy_migration import Enum, relationship
from sqlalchemy.orm import backref
from sqlalchemy.ext.declarative import declarative_base
Base = declarative_base()
class Client(Base):
__tablename__ = 'client'
set_choices = ('train', 'devel', 'test')
id = Column(Integer, primary_key=True)
set = Column(Enum(*set_choices))
def __init__(self, id, set):
self.id = id
self.set = set
def __repr__(self):
return "<Client('%s', '%s')>" % (self.id, self.set)
class File(Base):
__tablename__ = 'file'
light_choices = ('controlled', 'adverse')
id = Column(Integer, primary_key=True)
client_id = Column(Integer, ForeignKey('client.id')) # for SQL
path = Column(String(100), unique=True)
light = Column(Enum(*light_choices))
# for Python
client = relationship(Client, backref=backref('files', order_by=id))
def __init__(self, client, path, light):
self.client = client
self.path = path