Commit cba3d260 authored by Amir Mohammadi's avatar Amir Mohammadi
Browse files

Code cleanup

parent ad16f16f
......@@ -9,6 +9,7 @@
from .query import Database
from .models import Client, File, Protocol, RealAccess, Attack
def get_config():
"""Returns a string containing the configuration information.
"""
......
......@@ -12,6 +12,7 @@ import sys
# Driver API
# ==========
def checkfiles(args):
"""Checks existence files based on your criteria"""
......@@ -19,13 +20,13 @@ def checkfiles(args):
db = Database()
r = db.objects(
protocol=args.protocol,
support=args.support,
protocol=args.protocol,
support=args.support,
groups=args.group,
cls=args.cls,
light=args.light,
clients=args.client,
)
)
# go through all files, check if they are available on the filesystem
good = []
......@@ -33,7 +34,7 @@ def checkfiles(args):
for f in r:
if os.path.exists(f.make_path(args.directory, args.extension)):
good.append(f)
else:
else:
bad.append(f)
# report
......@@ -45,11 +46,12 @@ def checkfiles(args):
if bad:
for f in bad:
output.write('Cannot find file "%s"\n' % (f.make_path(args.directory, args.extension),))
output.write('%d files (out of %d) were not found at "%s"\n' % \
(len(bad), len(r), args.directory))
output.write('%d files (out of %d) were not found at "%s"\n' %
(len(bad), len(r), args.directory))
return 0
def add_command(subparsers):
"""Add specific subcommands that the action "checkfiles" can use"""
......@@ -62,7 +64,7 @@ def add_command(subparsers):
db = Database()
if not db.is_valid():
protocols = ('waiting','for','database','creation')
protocols = ('waiting', 'for', 'database', 'creation')
clients = tuple()
else:
protocols = [k.name for k in db.protocols()]
......@@ -77,6 +79,6 @@ def add_command(subparsers):
parser.add_argument('-l', '--light', dest="light", default='', help="if given, this value will limit the check to those files shot under a given lighting. (defaults to '%(default)s')", choices=db.lights())
parser.add_argument('-C', '--client', dest="client", default=None, type=int, help="if given, limits the dump to a particular client (defaults to '%(default)s')", choices=clients)
parser.add_argument('--self-test', dest="selftest", default=False,
action='store_true', help=SUPPRESS)
action='store_true', help=SUPPRESS)
parser.set_defaults(func=checkfiles) #action
parser.set_defaults(func=checkfiles) # action
......@@ -11,17 +11,21 @@ import fnmatch
from .models import *
def add_clients(session, protodir, verbose):
"""Add clients to the replay attack database."""
for client in open(os.path.join(protodir, 'clients.txt'), 'rt'):
s = client.strip().split(' ', 2)
if not s: continue #empty line
if not s:
continue # empty line
id = int(s[0])
set = s[1]
if verbose: print("Adding client %d on '%s' set..." % (id, set))
if verbose:
print("Adding client %d on '%s' set..." % (id, set))
session.add(Client(id, set))
def add_real_lists(session, protodir, verbose):
"""Adds all RCD filelists"""
......@@ -32,17 +36,20 @@ def add_real_lists(session, protodir, verbose):
"""Parses the RCD filename and break it in the relevant chunks."""
v = os.path.splitext(os.path.basename(f))[0].split('_')
client_id = int(v[0].replace('client',''))
path = os.path.splitext(f)[0] #keep only the filename stem
client_id = int(v[0].replace('client', ''))
path = os.path.splitext(f)[0] # keep only the filename stem
purpose = v[3]
light = v[4]
if len(v) == 6: take = int(v[5]) #authentication session
else: take = 1 #enrollment session
if len(v) == 6:
take = int(v[5]) # authentication session
else:
take = 1 # enrollment session
return [client_id, path, light], [purpose, take]
for fname in open(filename, 'rt'):
s = fname.strip()
if not s: continue #emtpy line
if not s:
continue # emtpy line
filefields, realfields = parse_real_filename(s)
filefields[0] = session.query(Client).filter(Client.id == filefields[0]).one()
file = File(*filefields)
......@@ -57,6 +64,7 @@ def add_real_lists(session, protodir, verbose):
add_real_list(session, os.path.join(protodir, 'recognition-devel.txt'))
add_real_list(session, os.path.join(protodir, 'recognition-test.txt'))
def add_attack_lists(session, protodir, verbose):
"""Adds all RAD filelists"""
......@@ -67,18 +75,19 @@ def add_attack_lists(session, protodir, verbose):
"""Parses the RAD filename and break it in the relevant chunks."""
v = os.path.splitext(os.path.basename(f))[0].split('_')
attack_device = v[1] #print, mobile or highdef
client_id = int(v[2].replace('client',''))
path = os.path.splitext(f)[0] #keep only the filename stem
sample_device = v[4] #highdef or mobile
sample_type = v[5] #photo or video
attack_device = v[1] # print, mobile or highdef
client_id = int(v[2].replace('client', ''))
path = os.path.splitext(f)[0] # keep only the filename stem
sample_device = v[4] # highdef or mobile
sample_type = v[5] # photo or video
light = v[6]
attack_support = f.split('/')[-2]
return [client_id, path, light], [attack_support, attack_device, sample_type, sample_device]
for fname in open(filename, 'rt'):
s = fname.strip()
if not s: continue #emtpy line
if not s:
continue # emtpy line
filefields, attackfields = parse_attack_filename(s)
filefields[0] = session.query(Client).filter(Client.id == filefields[0]).one()
file = File(*filefields)
......@@ -86,14 +95,15 @@ def add_attack_lists(session, protodir, verbose):
attackfields.insert(0, file)
session.add(Attack(*attackfields))
add_attack_list(session,os.path.join(protodir, 'attack-grandtest-allsupports-train.txt'))
add_attack_list(session,os.path.join(protodir, 'attack-grandtest-allsupports-devel.txt'))
add_attack_list(session,os.path.join(protodir, 'attack-grandtest-allsupports-test.txt'))
add_attack_list(session, os.path.join(protodir, 'attack-grandtest-allsupports-train.txt'))
add_attack_list(session, os.path.join(protodir, 'attack-grandtest-allsupports-devel.txt'))
add_attack_list(session, os.path.join(protodir, 'attack-grandtest-allsupports-test.txt'))
def define_protocols(session, protodir, verbose):
"""Defines all available protocols"""
#figures out which protocols to use
# figures out which protocols to use
valid = {}
for fname in fnmatch.filter(os.listdir(protodir), 'attack-*-allsupports-train.txt'):
......@@ -122,12 +132,15 @@ def define_protocols(session, protodir, verbose):
else:
real = alt_real
if consider: files[grp] = (attack, real)
if consider:
files[grp] = (attack, real)
if consider: valid[s[1]] = files
if consider:
valid[s[1]] = files
for protocol, groups in valid.items():
if verbose: print("Creating protocol '%s'..." % protocol)
if verbose:
print("Creating protocol '%s'..." % protocol)
# create protocol on the protocol table
obj = Protocol(name=protocol)
......@@ -140,7 +153,8 @@ def define_protocols(session, protodir, verbose):
q = session.query(Attack).join(File).filter(File.path == s).one()
q.protocols.append(obj)
counter += 1
if verbose: print(" -> %5s/%-6s: %d files" % (grp, "attack", counter))
if verbose:
print(" -> %5s/%-6s: %d files" % (grp, "attack", counter))
counter = 0
for fname in open(flist[1], 'rt'):
......@@ -148,10 +162,12 @@ def define_protocols(session, protodir, verbose):
q = session.query(RealAccess).join(File).filter(File.path == s).one()
q.protocols.append(obj)
counter += 1
if verbose: print(" -> %5s/%-6s: %d files" % (grp, "real", counter))
if verbose:
print(" -> %5s/%-6s: %d files" % (grp, "real", counter))
session.add(obj)
def create_tables(args):
"""Creates all necessary tables (only to be used at the first time)"""
......@@ -166,6 +182,7 @@ def create_tables(args):
# Driver API
# ==========
def create(args):
"""Creates or re-creates this database"""
......@@ -176,7 +193,8 @@ def create(args):
if args.recreate:
if args.verbose and os.path.exists(dbfile):
print(('unlinking %s...' % dbfile))
if os.path.exists(dbfile): os.unlink(dbfile)
if os.path.exists(dbfile):
os.unlink(dbfile)
if not os.path.exists(os.path.dirname(dbfile)):
os.makedirs(os.path.dirname(dbfile))
......@@ -193,18 +211,19 @@ def create(args):
return 0
def add_command(subparsers):
"""Add specific subcommands that the action "create" can use"""
parser = subparsers.add_parser('create', help=create.__doc__)
parser.add_argument('-R', '--recreate', action='store_true', default=False,
help="If set, I'll first erase the current database")
help="If set, I'll first erase the current database")
parser.add_argument('-v', '--verbose', action='count', default=0,
help="Do SQL operations in a verbose way")
help="Do SQL operations in a verbose way")
parser.add_argument('-D', '--protodir', action='store',
default='/idiap/group/replay/database/protocols/replayattack-database/protocols',
metavar='DIR',
help="Change the relative path to the directory containing the protocol definitions for replay attacks (defaults to %(default)s)")
default='/idiap/group/replay/database/protocols/replayattack-database/protocols',
metavar='DIR',
help="Change the relative path to the directory containing the protocol definitions for replay attacks (defaults to %(default)s)")
parser.set_defaults(func=create) #action
parser.set_defaults(func=create) # action
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Andre Anjos <andre.dos.anjos@gmail.com>
# Wed 15 Aug 11:26:11 2012
# Wed 15 Aug 11:26:11 2012
"""Bob Database Driver entry-point for the Replay Attack Database
"""
import os
import sys
from bob.db.base.driver import Interface as BaseInterface
def reverse(args):
"""Returns a list of file database identifiers given the path stems"""
......@@ -22,12 +22,15 @@ def reverse(args):
output = null()
r = db.reverse(args.path)
for id in r: output.write('%d\n' % id)
if not r: return 1
for id in r:
output.write('%d\n' % id)
if not r:
return 1
return 0
def reverse_command(subparsers):
"""Adds the specific options for the reverse command"""
......@@ -37,9 +40,10 @@ def reverse_command(subparsers):
parser.add_argument('path', nargs='+', type=str, help="one or more path stems to look up. If you provide more than one, files which cannot be reversed will be omitted from the output.")
parser.add_argument('--self-test', dest="selftest", default=False,
action='store_true', help=SUPPRESS)
action='store_true', help=SUPPRESS)
parser.set_defaults(func=reverse) # action
parser.set_defaults(func=reverse) #action
def path(args):
"""Returns a list of fully formed paths or stems given some file id"""
......@@ -53,12 +57,15 @@ def path(args):
output = null()
r = db.paths(args.id, prefix=args.directory, suffix=args.extension)
for path in r: output.write('%s\n' % path)
for path in r:
output.write('%s\n' % path)
if not r: return 1
if not r:
return 1
return 0
def path_command(subparsers):
"""Adds the specific options for the path command"""
......@@ -70,9 +77,10 @@ def path_command(subparsers):
parser.add_argument('-e', '--extension', dest="extension", default='', help="if given, this extension will be appended to every entry returned (defaults to '%(default)s')")
parser.add_argument('id', nargs='+', type=int, help="one or more file ids to look up. If you provide more than one, files which cannot be found will be omitted from the output. If you provide a single id to lookup, an error message will be printed if the id does not exist in the database. The exit status will be non-zero in such case.")
parser.add_argument('--self-test', dest="selftest", default=False,
action='store_true', help=SUPPRESS)
action='store_true', help=SUPPRESS)
parser.set_defaults(func=path) # action
parser.set_defaults(func=path) #action
class Interface(BaseInterface):
......@@ -95,9 +103,9 @@ class Interface(BaseInterface):
def add_commands(self, parser):
from . import __doc__ as docs
subparsers = self.setup_parser(parser,
"Photo/Video Replay attack database", docs)
subparsers = self.setup_parser(parser,
"Photo/Video Replay attack database", docs)
# get the "create" action from a submodule
from .create import add_command as create_command
......
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Andre Anjos <andre.dos.anjos@gmail.com>
# Thu 12 May 14:02:28 2011
# Thu 12 May 14:02:28 2011
"""Dumps lists of files.
"""
import os
import sys
# Driver API
# ==========
def dumplist(args):
"""Dumps lists of files based on your criteria"""
......@@ -19,13 +19,13 @@ def dumplist(args):
db = Database()
r = db.objects(
protocol=args.protocol,
support=args.support,
groups=args.group,
protocol=args.protocol,
support=args.support,
groups=args.group,
cls=args.cls,
light=args.light,
clients=args.client,
)
)
output = sys.stdout
if args.selftest:
......@@ -37,6 +37,7 @@ def dumplist(args):
return 0
def add_command(subparsers):
"""Add specific subcommands that the action "dumplist" can use"""
......@@ -49,7 +50,7 @@ def add_command(subparsers):
db = Database()
if not db.is_valid():
protocols = ('waiting','for','database','creation')
protocols = ('waiting', 'for', 'database', 'creation')
clients = tuple()
else:
protocols = [k.name for k in db.protocols()]
......@@ -64,6 +65,6 @@ def add_command(subparsers):
parser.add_argument('-l', '--light', dest="light", default='', help="if given, this value will limit the output files to those shot under a given lighting. (defaults to '%(default)s')", choices=db.lights())
parser.add_argument('-C', '--client', dest="client", default=None, type=int, help="if given, limits the dump to a particular client (defaults to '%(default)s')", choices=clients)
parser.add_argument('--self-test', dest="selftest", default=False,
action='store_true', help=SUPPRESS)
action='store_true', help=SUPPRESS)
parser.set_defaults(func=dumplist) #action
parser.set_defaults(func=dumplist) # action
......@@ -6,7 +6,6 @@
"""Table models and functionality for the Replay Attack DB.
"""
import sqlalchemy
import os
from sqlalchemy import Table, Column, Integer, String, ForeignKey
from bob.db.base.sqlalchemy_migration import Enum, relationship
......@@ -18,6 +17,7 @@ import bob
Base = declarative_base()
class Client(Base):
"""Database clients, marked by an integer identifier and the set they belong
to"""
......@@ -40,6 +40,7 @@ class Client(Base):
def __repr__(self):
return "Client('%s', '%s')" % (self.id, self.set)
class File(Base):
"""Generic file container"""
......@@ -51,7 +52,7 @@ class File(Base):
id = Column(Integer, primary_key=True)
"""Key identifier for files"""
client_id = Column(Integer, ForeignKey('client.id')) # for SQL
client_id = Column(Integer, ForeignKey('client.id')) # for SQL
"""The client identifier to which this file is bound to"""
path = Column(String(100), unique=True)
......@@ -88,8 +89,10 @@ class File(Base):
Returns a string containing the newly generated file path.
"""
if not directory: directory = ''
if not extension: extension = ''
if not directory:
directory = ''
if not extension:
extension = ''
return str(os.path.join(directory, self.path + extension))
......@@ -117,7 +120,8 @@ class File(Base):
Returns a string containing the face file path.
"""
if not directory: directory = ''
if not directory:
directory = ''
directory = os.path.join(directory, 'face-locations')
return self.make_path(directory, '.face')
......@@ -207,15 +211,16 @@ class File(Base):
# Intermediate mapping from RealAccess's to Protocol's
realaccesses_protocols = Table('realaccesses_protocols', Base.metadata,
Column('realaccess_id', Integer, ForeignKey('realaccess.id')),
Column('protocol_id', Integer, ForeignKey('protocol.id')),
)
Column('realaccess_id', Integer, ForeignKey('realaccess.id')),
Column('protocol_id', Integer, ForeignKey('protocol.id')),
)
# Intermediate mapping from Attack's to Protocol's
attacks_protocols = Table('attacks_protocols', Base.metadata,
Column('attack_id', Integer, ForeignKey('attack.id')),
Column('protocol_id', Integer, ForeignKey('protocol.id')),
)
Column('attack_id', Integer, ForeignKey('attack.id')),
Column('protocol_id', Integer, ForeignKey('protocol.id')),
)
class Protocol(Base):
"""Replay attack protocol"""
......@@ -234,6 +239,7 @@ class Protocol(Base):
def __repr__(self):
return "Protocol('%s')" % (self.name,)
class RealAccess(Base):
"""Defines Real-Accesses (licit attempts to authenticate)"""
......@@ -245,7 +251,7 @@ class RealAccess(Base):
id = Column(Integer, primary_key=True)
"""Unique identifier for this real-access object"""
file_id = Column(Integer, ForeignKey('file.id')) # for SQL
file_id = Column(Integer, ForeignKey('file.id')) # for SQL
"""The file identifier the current real-access is bound to"""
purpose = Column(Enum(*purpose_choices))
......@@ -259,7 +265,7 @@ class RealAccess(Base):
"""A direct link to the :py:class:`.File` object this real-access belongs to"""
protocols = relationship("Protocol", secondary=realaccesses_protocols,
backref='realaccesses')
backref='realaccesses')
"""A direct link to the protocols this file is linked to"""
def __init__(self, file, purpose, take):
......@@ -270,6 +276,7 @@ class RealAccess(Base):
def __repr__(self):
return "RealAccess('%s')" % (self.file.path)
class Attack(Base):
"""Defines Spoofing Attacks (illicit attempts to authenticate)"""
......@@ -290,7 +297,7 @@ class Attack(Base):
id = Column(Integer, primary_key=True)
"""Unique identifier for this attack"""
file_id = Column(Integer, ForeignKey('file.id')) # for SQL
file_id = Column(Integer, ForeignKey('file.id')) # for SQL
"""The file identifier this attack is linked to"""
attack_support = Column(Enum(*attack_support_choices))
......@@ -310,7 +317,7 @@ class Attack(Base):
"""A direct link to the :py:class:`.File` object bound to this attack"""
protocols = relationship("Protocol", secondary=attacks_protocols,
backref='attacks')
backref='attacks')
"""A direct link to the protocols this file is linked to"""
def __init__(self, file, attack_support, attack_device, sample_type, sample_device):
......
......@@ -8,7 +8,6 @@ replay attack database in the most obvious ways.
"""
import os
import logging
from bob.db.base import utils, Database
from .models import *
from .driver import Interface
......@@ -17,6 +16,7 @@ INFO = Interface()
SQLITE_FILE = INFO.files()[0]
class Database(Database):
"""The dataset class opens and maintains a connection opened to the Database.
......@@ -63,8 +63,8 @@ class Database(Database):
raise RuntimeError("Database '%s' cannot be found at expected location '%s'. Create it and then try re-connecting using Database.connect()" % (INFO.name(), SQLITE_FILE))
def objects(self, support=Attack.attack_support_choices,
protocol='grandtest', groups=Client.set_choices, cls=('attack', 'real'),
light=File.light_choices, clients=None):
protocol='grandtest', groups=Client.set_choices, cls=('attack', 'real'),
light=File.light_choices, clients=None):
"""Returns a list of unique :py:class:`.File` objects for the specific
query by the user.
......@@ -107,7 +107,8 @@ class Database(Database):
def check_validity(l, obj, valid, default):
"""Checks validity of user input data against a set of valid values"""
if not l: return default
if not l:
return default
elif not isinstance(l, (tuple, list)):
return check_validity((l,), obj, valid, default)
for k in l:
......@@ -128,7 +129,8 @@ class Database(Database):
cls = check_validity(cls, "class", VALID_CLASSES, ('real', 'attack'))
# check protocol validity
if not protocol: protocol = 'grandtest' #default
if not protocol:
protocol = 'grandtest' # default
VALID_PROTOCOLS = [k.name for k in self.protocols()]
protocol = check_validity(protocol, "protocol", VALID_PROTOCOLS, ('grandtest',))
......@@ -143,23 +145,28 @@ class Database(Database):
# now query the database
retval = []
from sqlalchemy.sql.expression import or_
# real-accesses are simpler to query
if 'enroll' in cls:
q = self.session.query(File).join(RealAccess).join(Client)
if groups: q = q.filter(Client.set.in_(groups))
if clients: q = q.filter(Client.id.in_(clients))
if light: q = q.filter(File.light.in_(light))
q = q.filter(RealAccess.purpose=='enroll')
if groups:
q = q.filter(Client.set.in_(groups))
if clients:
q = q.filter(Client.id.in_(clients))
if light:
q = q.filter(File.light.in_(light))
q = q.filter(RealAccess.purpose == 'enroll')
q = q.order_by(Client.id)
retval += list(q)
# real-accesses are simpler to query
if 'real' in cls:
q = self.session.query(File).join(RealAccess).join((Protocol, RealAccess.protocols)).join(Client)
if groups: q = q.filter(Client.set.in_(groups))
if clients: q = q.filter(Client.id.in_(clients))
if light: q = q.filter(File.light.in_(light))
if groups:
q = q.filter(Client.set.in_(groups))
if clients:
q = q.filter(Client.id.in_(clients))
if light:
q = q.filter(File.light.in_(light))
q = q.filter(Protocol.name.in_(protocol))
q = q.order_by(Client.id)
retval += list(q)
......@@ -167,10 +174,14 @@ class Database(Database):
# attacks will have to be filtered a little bit more
if 'attack' in cls: