Commit b41e6df4 authored by Artur Costa Pazo's avatar Artur Costa Pazo
Browse files

Initial commit

parents
*~
*.swp
*.pyc
*.so
*.dylib
bin
eggs
parts
.installed.cfg
.mr.developer.cfg
*.egg-info
develop-eggs
sphinx
dist
.nfs*
.gdb_history
build
*.egg
src/
#db.sql3
doc/html
language: python
matrix:
include:
- python: 2.6
- python: 2.7
env:
- secure: G+01O0hR4OdUjV0rmzydDlVhrJOcVDBByecGnORLT9LV1qSd3IU04de7AfzkZhnhl58UxmmQsxd+XqPaCJoU/uTWKa0GW0BCJSxmLxx8cXco3Y8yV03zbBRJC8rbX7O+jrlx+6Js+CFpTfm+MlOXeeGAbky5gr1KZQD2pXqr+0w=
- secure: XUeglE6tOR0dckxT/2WPQivRY3n3tsWYnFbR1oQk17p+7o5/S5rXYxR7PvL9E8k4pprqFf54IGMVe41F5Q8k/5ZanAVRBKSnvod9or1INmqXgsMDS9yUjO0jTHnrazSn4jNIUhWd/7+1VuxdIPxUSBfVOr+9ow8IrQorQqc7eOc=
- BOB_DOCUMENTATION_SERVER=https://www.idiap.ch/software/bob/docs/latest/bioidiap/%s/master
- BOB_UPLOAD_WHEEL="--universal"
- python: 3.3
- python: 3.4
before_install:
- sudo add-apt-repository -y ppa:biometrics/bob
- sudo apt-get update -qq
- sudo apt-get install -qq --force-yes libboost-all-dev libblitz1-dev libhdf5-serial-dev libatlas-dev libatlas-base-dev liblapack-dev libsvm-dev texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended
- pip install --find-links https://www.idiap.ch/software/bob/wheels/travis/ --use-wheel sphinx nose numpy matplotlib coverage
- pip install --find-links https://www.idiap.ch/software/bob/wheels/travis/ --use-wheel --pre -r requirements.txt coveralls
install:
- python bootstrap-buildout.py
- ./bin/buildout buildout:develop=. buildout:extensions=bob.buildout buildout:auto-checkout=
script:
- ./bin/python -c 'import pkg_resources; from bob.db.replay import get_config; print(get_config())'
- ./bin/bob_dbmanage.py replay download
- ./bin/coverage run --source=bob.db.replay ./bin/nosetests -sv
- ./bin/sphinx-build -b doctest doc sphinx
- ./bin/sphinx-build -b html doc sphinx
after_success:
- coveralls
- wget https://raw.githubusercontent.com/bioidiap/bob.extension/master/scripts/upload-{sphinx,wheel}.sh
- chmod a+x upload-sphinx.sh upload-wheel.sh
- ./upload-sphinx.sh
- ./upload-wheel.sh
include README.rst bootstrap-buildout.py buildout.cfg version.txt requirements.txt
recursive-include doc *.py *.rst
recursive-include bob *.sql3
.. vim: set fileencoding=utf-8 :
.. Manuel Guenther <manuel.guenther@idiap.ch>
.. Fri Oct 31 14:18:57 CET 2014
.. image:: http://img.shields.io/badge/docs-stable-yellow.png
:target: http://pythonhosted.org/bob.db.replay/index.html
.. image:: http://img.shields.io/badge/docs-latest-orange.png
:target: https://www.idiap.ch/software/bob/docs/latest/bioidiap/bob.db.replay/master/index.html
.. image:: https://travis-ci.org/bioidiap/bob.db.replay.svg?branch=master
:target: https://travis-ci.org/bioidiap/bob.db.replay
.. image:: https://coveralls.io/repos/bioidiap/bob.db.replay/badge.png
:target: https://coveralls.io/r/bioidiap/bob.db.replay
.. image:: https://img.shields.io/badge/github-master-0000c0.png
:target: https://github.com/bioidiap/bob.db.replay/tree/master
.. image:: http://img.shields.io/pypi/v/bob.db.replay.png
:target: https://pypi.python.org/pypi/bob.db.replay
.. image:: http://img.shields.io/pypi/dm/bob.db.replay.png
:target: https://pypi.python.org/pypi/bob.db.replay
.. image:: https://img.shields.io/badge/original-data--files-a000a0.png
:target: http://www.idiap.ch/dataset/replayattack
===================================================
Replay Mobile Database Database Interface for Bob
===================================================
This package contains the access API and descriptions for the `Replay Mobile`_ Database.
The actual raw data for the database should be downloaded from the original URL.
This package only contains the Bob_ accessor methods to use the database directly from Python, with our certified protocols.
Installation
------------
To install this package -- alone or together with other `Packages of Bob <https://github.com/idiap/bob/wiki/Packages>`_ -- please read the `Installation Instructions <https://github.com/idiap/bob/wiki/Installation>`_.
For Bob_ to be able to work properly, some dependent packages are required to be installed.
Please make sure that you have read the `Dependencies <https://github.com/idiap/bob/wiki/Dependencies>`_ for your operating system.
Documentation
-------------
For further documentation on this package, please read the `Stable Version <http://pythonhosted.org/bob.db.replay/index.html>`_ or the `Latest Version <https://www.idiap.ch/software/bob/docs/latest/bioidiap/bob.db.replay/master/index.html>`_ of the documentation.
For a list of tutorials on this or the other packages ob Bob_, or information on submitting issues, asking questions and starting discussions, please visit its website.
.. _bob: https://www.idiap.ch/software/bob
.. _replay mobile: http://www.idiap.ch/dataset/replaymobile
#see http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
__import__('pkg_resources').declare_namespace(__name__)
#see http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
__import__('pkg_resources').declare_namespace(__name__)
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Andre Anjos <andre.dos.anjos@gmail.com>
# Wed 18 May 09:28:44 2011
"""The Replay-Mobile Database accessors for Bob
"""
from .query import Database
from .models import Client, File, Protocol, RealAccess, Attack
from . import spoofing
def get_config():
"""Returns a string containing the configuration information.
"""
import bob.extension
return bob.extension.get_config(__name__)
# gets sphinx autodoc done right - don't remove it
__all__ = [_ for _ in dir() if not _.startswith('_')]
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Andre Anjos <andre.anjos@idiap.ch>
# Tue 20 Mar 19:20:22 2012 CET
"""Checks for installed files.
"""
import os
import sys
# Driver API
# ==========
def checkfiles(args):
"""Checks existence files based on your criteria"""
from .query import Database
db = Database()
r = db.objects(
protocol=args.protocol,
support=args.support,
groups=args.group,
cls=args.cls,
light=args.light,
clients=args.client,
)
# go through all files, check if they are available on the filesystem
good = []
bad = []
for f in r:
if os.path.exists(f.make_path(args.directory, args.extension)):
good.append(f)
else:
bad.append(f)
# report
output = sys.stdout
if args.selftest:
from bob.db.base.utils import null
output = null()
if bad:
for f in bad:
output.write('Cannot find file "%s"\n' % (f.make_path(args.directory, args.extension),))
output.write('%d files (out of %d) were not found at "%s"\n' % \
(len(bad), len(r), args.directory))
return 0
def add_command(subparsers):
"""Add specific subcommands that the action "checkfiles" can use"""
from argparse import SUPPRESS
parser = subparsers.add_parser('checkfiles', help=checkfiles.__doc__)
from .query import Database
db = Database()
if not db.is_valid():
protocols = ('waiting','for','database','creation')
clients = tuple()
else:
protocols = [k.name for k in db.protocols()]
clients = [k.id for k in db.clients()]
parser.add_argument('-d', '--directory', dest="directory", default='', help="if given, this path will be prepended to every entry checked (defaults to '%(default)s')")
parser.add_argument('-e', '--extension', dest="extension", default='', help="if given, this extension will be appended to every entry checked (defaults to '%(default)s')")
parser.add_argument('-c', '--class', dest="cls", default='', help="if given, limits the check to a particular subset of the data that corresponds to the given class (defaults to '%(default)s')", choices=('real', 'attack', 'enroll'))
parser.add_argument('-g', '--group', dest="group", default='', help="if given, this value will limit the check to those files belonging to a particular protocolar group. (defaults to '%(default)s')", choices=db.groups())
parser.add_argument('-s', '--support', dest="support", default='', help="if given, this value will limit the check to those files using this type of attack support. (defaults to '%(default)s')", choices=db.attack_supports())
parser.add_argument('-x', '--protocol', dest="protocol", default='', help="if given, this value will limit the check to those files for a given protocol. (defaults to '%(default)s')", choices=protocols)
parser.add_argument('-l', '--light', dest="light", default='', help="if given, this value will limit the check to those files shot under a given lighting. (defaults to '%(default)s')", choices=db.lights())
parser.add_argument('-C', '--client', dest="client", default=None, type=int, help="if given, limits the dump to a particular client (defaults to '%(default)s')", choices=clients)
parser.add_argument('--self-test', dest="selftest", default=False,
action='store_true', help=SUPPRESS)
parser.set_defaults(func=checkfiles) #action
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Artur Costa Pazo <artur.costa-pazo@idiap.ch>
# Fri Nov 13 11:11:11 2015 +0200
#
# Copyright (C) 2015 Idiap Research Institute, Martigny, Switzerland
"""This script creates the Replay-Attack database in a single pass.
"""
import os
import fnmatch
from .models import *
def add_clients(session, protodir, verbose):
"""Add clients to the replay attack database."""
for client in open(os.path.join(protodir, 'clients.txt'), 'rt'):
s = client.strip().split(' ', 2)
if not s: continue #empty line
id = int(s[0])
set = s[1]
if verbose: print("Adding client %d on '%s' set..." % (id, set))
session.add(Client(id, set))
def add_real_lists(session, protodir, verbose):
"""Adds all RCD filelists"""
def add_real_list(session, filename):
"""Adds an RCD filelist and materializes RealAccess'es."""
def parse_real_filename(f):
"""Parses the RCD filename and break it in the relevant chunks."""
v = os.path.splitext(os.path.basename(f))[0].split('_')
client_id = int(v[0].replace('client',''))
path = os.path.splitext(f)[0] #keep only the filename stem
propuse =v[2]
device = v[3]
light = v[4]
return [client_id, path, light,device], [propuse,device]
for fname in open(filename, 'rt'):
s = fname.strip()
if not s: continue #emtpy line
filefields, realfields = parse_real_filename(s)
filefields[0] = session.query(Client).filter(Client.id == filefields[0]).one()
file = File(*filefields)
session.add(file)
realfields.insert(0, file)
session.add(RealAccess(*realfields))
add_real_list(session, os.path.join(protodir, 'alldevices/real-alldevices-train.txt'))
add_real_list(session, os.path.join(protodir, 'alldevices/real-alldevices-devel.txt'))
add_real_list(session, os.path.join(protodir, 'alldevices/real-alldevices-test.txt'))
add_real_list(session, os.path.join(protodir, 'alldevices/enroll-alldevices-train.txt'))
add_real_list(session, os.path.join(protodir, 'alldevices/enroll-alldevices-devel.txt'))
add_real_list(session, os.path.join(protodir, 'alldevices/enroll-alldevices-test.txt'))
def add_attack_lists(session, protodir, verbose):
"""Adds all RAD filelists"""
def add_attack_list(session, filename):
"""Adds an RAD filelist and materializes Attacks."""
def parse_attack_filename(f):
"""Parses the RAD filename and break it in the relevant chunks."""
v = os.path.splitext(os.path.basename(f))[0].split('_')
client_id = int(v[1].replace('client',''))
path = os.path.splitext(f)[0] #keep only the filename stem
light = v[7]
#attack_support = f.split('/')[-2]
attack_support = v[4] #fixed, hand
attack_device = v[3] #print, mattescreen
sample_type = v[6] #photo or video
sample_device = v[5] #tablet or mobile
return [client_id, path, light ,sample_device], [attack_support, attack_device, sample_type, sample_device]
for fname in open(filename, 'rt'):
s = fname.strip()
if not s: continue #emtpy line
filefields, attackfields = parse_attack_filename(s)
filefields[0] = session.query(Client).filter(Client.id == filefields[0]).one()
file = File(*filefields)
session.add(file)
attackfields.insert(0, file)
session.add(Attack(*attackfields))
add_attack_list(session,os.path.join(protodir, 'alldevices/attack-grandtest-allsupports-alldevices-train.txt'))
add_attack_list(session,os.path.join(protodir, 'alldevices/attack-grandtest-allsupports-alldevices-devel.txt'))
add_attack_list(session,os.path.join(protodir, 'alldevices/attack-grandtest-allsupports-alldevices-test.txt'))
def define_protocols(session, protodir, verbose):
"""Defines all available protocols"""
#figures out which protocols to use
valid = {}
# Three grandtest protocols
#files_protocol = ['alldevices/attack-grandtest-allsupports-alldevices-train.txt', 'mobile/attack-mobilegrandtest-allsupports-mobile-train' ,'tablet/attack-tabletgrandtest-allsupports-tablet-train' ]
files_protocol = ['alldevices/attack-grandtest-allsupports-alldevices-train.txt', 'alldevices/attack-print-allsupports-alldevices-train' ,'alldevices/attack-mattescreen-fixed-alldevices-alltype-train']
for fname in files_protocol:
s = fname.split('-', 5)
folder = fname.split('/', 1)[0]
consider = True
files = {}
for grp in ('train', 'devel', 'test'):
# check attack file
if len(s)<6:
attack = os.path.join(protodir, '%s/attack-%s-%s-%s-%s.txt' % (folder,s[1],s[2],s[3], grp))
else :
attack = os.path.join(protodir, '%s/attack-%s-%s-%s-alltype-%s.txt' % (folder,s[1],s[2],s[3], grp))
if not os.path.exists(attack):
if verbose:
print("Not considering protocol %s as attack list '%s' was not found" % (s[1], attack))
consider = False
# check real file
real = os.path.join(protodir, '%s/real-%s-%s.txt' % (folder,s[3],grp))
if not os.path.exists(real):
alt_real = os.path.join(protodir, '%s/real-%s-%s.txt' % (folder,s[3],grp))
if not os.path.exists(alt_real):
if verbose:
print("Not considering protocol %s as real list '%s' or '%s' were not found" % (s[1], real, alt_real))
consider = False
else:
real = alt_real
if consider: files[grp] = (attack, real)
if consider: valid[s[1]] = files
for protocol, groups in valid.items():
if verbose: print("Creating protocol '%s'..." % protocol)
# create protocol on the protocol table
obj = Protocol(name=protocol)
for grp, flist in groups.items():
#print grp
#print flist
counter = 0
for fname in open(flist[0], 'rt'):
s = os.path.splitext(fname.strip())[0]
q = session.query(Attack).join(File).filter(File.path == s).one()
q.protocols.append(obj)
counter += 1
if verbose: print(" -> %5s/%-6s: %d files" % (grp, "attack", counter))
counter = 0
for fname in open(flist[1], 'rt'):
s = os.path.splitext(fname.strip())[0]
q = session.query(RealAccess).join(File).filter(File.path == s).one()
q.protocols.append(obj)
counter += 1
if verbose: print(" -> %5s/%-6s: %d files" % (grp, "real", counter))
session.add(obj)
def create_tables(args):
"""Creates all necessary tables (only to be used at the first time)"""
from bob.db.base.utils import create_engine_try_nolock
engine = create_engine_try_nolock(args.type, args.files[0], echo=(args.verbose >= 2))
Client.metadata.create_all(engine)
RealAccess.metadata.create_all(engine)
Attack.metadata.create_all(engine)
Protocol.metadata.create_all(engine)
# Driver API
# ==========
def create(args):
"""Creates or re-creates this database"""
from bob.db.base.utils import session_try_nolock
dbfile = args.files[0]
if args.recreate:
if args.verbose and os.path.exists(dbfile):
print(('unlinking %s...' % dbfile))
if os.path.exists(dbfile): os.unlink(dbfile)
if not os.path.exists(os.path.dirname(dbfile)):
os.makedirs(os.path.dirname(dbfile))
# the real work...
create_tables(args)
s = session_try_nolock(args.type, args.files[0], echo=(args.verbose >= 2))
add_clients(s, args.protodir, args.verbose)
add_real_lists(s, args.protodir, args.verbose)
add_attack_lists(s, args.protodir, args.verbose)
define_protocols(s, args.protodir, args.verbose)
s.commit()
s.close()
return 0
def add_command(subparsers):
"""Add specific subcommands that the action "create" can use"""
parser = subparsers.add_parser('create', help=create.__doc__)
parser.add_argument('-R', '--recreate', action='store_true', default=False,
help="If set, I'll first erase the current database")
parser.add_argument('-v', '--verbose', action='count', default=0,
help="Do SQL operations in a verbose way")
parser.add_argument('-D', '--protodir', action='store',
default='/idiap/group/biometric/databases/replay-mobile/database/protocols',
metavar='DIR',
help="Change the relative path to the directory containing the protocol definitions for replay attacks (defaults to %(default)s)")
parser.set_defaults(func=create) #action
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Andre Anjos <andre.dos.anjos@gmail.com>
# Wed 15 Aug 11:26:11 2012
"""Bob Database Driver entry-point for the Replay Mobile Database
"""
import os
import sys
from bob.db.base.driver import Interface as BaseInterface
def reverse(args):
"""Returns a list of file database identifiers given the path stems"""
from .query import Database
db = Database()
output = sys.stdout
if args.selftest:
from bob.db.base.utils import null
output = null()
r = db.reverse(args.path)
for id in r: output.write('%d\n' % id)
if not r: return 1
return 0
def reverse_command(subparsers):
"""Adds the specific options for the reverse command"""
from argparse import SUPPRESS
parser = subparsers.add_parser('reverse', help=reverse.__doc__)
parser.add_argument('path', nargs='+', type=str, help="one or more path stems to look up. If you provide more than one, files which cannot be reversed will be omitted from the output.")
parser.add_argument('--self-test', dest="selftest", default=False,
action='store_true', help=SUPPRESS)
parser.set_defaults(func=reverse) #action
def path(args):
"""Returns a list of fully formed paths or stems given some file id"""
from .query import Database
db = Database()
output = sys.stdout
if args.selftest:
from bob.db.base.utils import null
output = null()
r = db.paths(args.id, prefix=args.directory, suffix=args.extension)
for path in r: output.write('%s\n' % path)
if not r: return 1
return 0
def path_command(subparsers):
"""Adds the specific options for the path command"""
from argparse import SUPPRESS
parser = subparsers.add_parser('path', help=path.__doc__)
parser.add_argument('-d', '--directory', dest="directory", default='', help="if given, this path will be prepended to every entry returned (defaults to '%(default)s')")
parser.add_argument('-e', '--extension', dest="extension", default='', help="if given, this extension will be appended to every entry returned (defaults to '%(default)s')")
parser.add_argument('id', nargs='+', type=int, help="one or more file ids to look up. If you provide more than one, files which cannot be found will be omitted from the output. If you provide a single id to lookup, an error message will be printed if the id does not exist in the database. The exit status will be non-zero in such case.")
parser.add_argument('--self-test', dest="selftest", default=False,
action='store_true', help=SUPPRESS)
parser.set_defaults(func=path) #action
class Interface(BaseInterface):
def name(self):
return 'replaymobile'
def version(self):
import pkg_resources # part of setuptools
return pkg_resources.require('bob.db.%s' % self.name())[0].version
def files(self):
from pkg_resources import resource_filename
raw_files = ('db.sql3',)
return [resource_filename(__name__, k) for k in raw_files]
def type(self):
return 'sqlite'
def add_commands(self, parser):
from . import __doc__ as docs
subparsers = self.setup_parser(parser,
"Photo/Video Replay mobile database", docs)
# get the "create" action from a submodule
from .create import add_command as create_command
create_command(subparsers)
# get the "dumplist" action from a submodule
from .dumplist import add_command as dumplist_command
dumplist_command(subparsers)
# get the "checkfiles" action from a submodule
from .checkfiles import add_command as checkfiles_command
checkfiles_command(subparsers)
# adds the "reverse" command
reverse_command(subparsers)
# adds the "path" command
path_command(subparsers)
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Andre Anjos <andre.dos.anjos@gmail.com>
# Thu 12 May 14:02:28 2011
"""Dumps lists of files.
"""
import os
import sys
# Driver API
# ==========
def dumplist(args):
"""Dumps lists of files based on your criteria"""
from .query import Database
db = Database()
r = db.objects(
protocol=args.protocol,
support=args.support,
groups=args.group,
cls=args.cls,
light=args.light,
clients=args.client,
)
output = sys.stdout
if args.selftest:
from bob.db.base.utils import null
output = null()
for f in r:
output.write('%s\n' % (f.make_path(args.directory, args.extension),))
return 0