Commit 68dacd38 authored by Amir Mohammadi's avatar Amir Mohammadi
Browse files

pep-8 code clean-up

parent 6e90bcb1
Pipeline #4592 failed with stage
in 7 minutes and 41 seconds
......@@ -7,6 +7,7 @@
from .query import Database
from .models import Client, File, Protocol, RealAccess, Attack
def get_config():
"""Returns a string containing the configuration information.
"""
......
......@@ -10,6 +10,7 @@ import sys
# Driver API
# ==========
def checkfiles(args):
"""Checks existence files based on your criteria"""
......@@ -17,13 +18,13 @@ def checkfiles(args):
db = Database()
r = db.objects(
protocol=args.protocol,
support=args.support,
protocol=args.protocol,
support=args.support,
groups=args.group,
cls=args.cls,
light=args.light,
clients=args.client,
)
)
# go through all files, check if they are available on the filesystem
good = []
......@@ -31,7 +32,7 @@ def checkfiles(args):
for f in r:
if os.path.exists(f.make_path(args.directory, args.extension)):
good.append(f)
else:
else:
bad.append(f)
# report
......@@ -43,11 +44,12 @@ def checkfiles(args):
if bad:
for f in bad:
output.write('Cannot find file "%s"\n' % (f.make_path(args.directory, args.extension),))
output.write('%d files (out of %d) were not found at "%s"\n' % \
(len(bad), len(r), args.directory))
output.write('%d files (out of %d) were not found at "%s"\n' %
(len(bad), len(r), args.directory))
return 0
def add_command(subparsers):
"""Add specific subcommands that the action "checkfiles" can use"""
......@@ -60,7 +62,7 @@ def add_command(subparsers):
db = Database()
if not db.is_valid():
protocols = ('waiting','for','database','creation')
protocols = ('waiting', 'for', 'database', 'creation')
clients = tuple()
else:
protocols = [k.name for k in db.protocols()]
......@@ -75,6 +77,6 @@ def add_command(subparsers):
parser.add_argument('-l', '--light', dest="light", default='', help="if given, this value will limit the check to those files shot under a given lighting. (defaults to '%(default)s')", choices=db.lights())
parser.add_argument('-C', '--client', dest="client", default=None, type=int, help="if given, limits the dump to a particular client (defaults to '%(default)s')", choices=clients)
parser.add_argument('--self-test', dest="selftest", default=False,
action='store_true', help=SUPPRESS)
action='store_true', help=SUPPRESS)
parser.set_defaults(func=checkfiles) #action
parser.set_defaults(func=checkfiles) # action
......@@ -5,21 +5,24 @@
"""
import os
import fnmatch
from .models import *
def add_clients(session, protodir, verbose):
"""Add clients to the replay attack database."""
for client in open(os.path.join(protodir, 'clients.txt'), 'rt'):
s = client.strip().split(' ', 2)
if not s: continue #empty line
if not s:
continue # empty line
id = int(s[0])
set = s[1]
if verbose: print("Adding client %d on '%s' set..." % (id, set))
if verbose:
print("Adding client %d on '%s' set..." % (id, set))
session.add(Client(id, set))
def add_real_lists(session, protodir, verbose):
"""Adds all RCD filelists"""
......@@ -30,16 +33,17 @@ def add_real_lists(session, protodir, verbose):
"""Parses the RCD filename and break it in the relevant chunks."""
v = os.path.splitext(os.path.basename(f))[0].split('_')
client_id = int(v[0].replace('client',''))
path = os.path.splitext(f)[0] #keep only the filename stem
propuse =v[2]
client_id = int(v[0].replace('client', ''))
path = os.path.splitext(f)[0] # keep only the filename stem
propuse = v[2]
device = v[3]
light = v[4]
return [client_id, path, light,device], [propuse,device]
return [client_id, path, light, device], [propuse, device]
for fname in open(filename, 'rt'):
s = fname.strip()
if not s: continue #emtpy line
if not s:
continue # emtpy line
filefields, realfields = parse_real_filename(s)
filefields[0] = session.query(Client).filter(Client.id == filefields[0]).one()
file = File(*filefields)
......@@ -47,7 +51,6 @@ def add_real_lists(session, protodir, verbose):
realfields.insert(0, file)
session.add(RealAccess(*realfields))
add_real_list(session, os.path.join(protodir, 'alldevices/real-alldevices-train.txt'))
add_real_list(session, os.path.join(protodir, 'alldevices/real-alldevices-devel.txt'))
add_real_list(session, os.path.join(protodir, 'alldevices/real-alldevices-test.txt'))
......@@ -67,21 +70,22 @@ def add_attack_lists(session, protodir, verbose):
"""Parses the RAD filename and break it in the relevant chunks."""
v = os.path.splitext(os.path.basename(f))[0].split('_')
client_id = int(v[1].replace('client',''))
path = os.path.splitext(f)[0] #keep only the filename stem
client_id = int(v[1].replace('client', ''))
path = os.path.splitext(f)[0] # keep only the filename stem
light = v[7]
#attack_support = f.split('/')[-2]
# attack_support = f.split('/')[-2]
attack_support = v[4] #fixed, hand
attack_device = v[3] #print, mattescreen
sample_type = v[6] #photo or video
sample_device = v[5] #tablet or mobile
attack_support = v[4] # fixed, hand
attack_device = v[3] # print, mattescreen
sample_type = v[6] # photo or video
sample_device = v[5] # tablet or mobile
return [client_id, path, light ,sample_device], [attack_support, attack_device, sample_type, sample_device]
return [client_id, path, light, sample_device], [attack_support, attack_device, sample_type, sample_device]
for fname in open(filename, 'rt'):
s = fname.strip()
if not s: continue #emtpy line
if not s:
continue # emtpy line
filefields, attackfields = parse_attack_filename(s)
filefields[0] = session.query(Client).filter(Client.id == filefields[0]).one()
file = File(*filefields)
......@@ -89,42 +93,43 @@ def add_attack_lists(session, protodir, verbose):
attackfields.insert(0, file)
session.add(Attack(*attackfields))
add_attack_list(session,os.path.join(protodir, 'alldevices/attack-grandtest-allsupports-alldevices-train.txt'))
add_attack_list(session,os.path.join(protodir, 'alldevices/attack-grandtest-allsupports-alldevices-devel.txt'))
add_attack_list(session,os.path.join(protodir, 'alldevices/attack-grandtest-allsupports-alldevices-test.txt'))
add_attack_list(session, os.path.join(protodir, 'alldevices/attack-grandtest-allsupports-alldevices-train.txt'))
add_attack_list(session, os.path.join(protodir, 'alldevices/attack-grandtest-allsupports-alldevices-devel.txt'))
add_attack_list(session, os.path.join(protodir, 'alldevices/attack-grandtest-allsupports-alldevices-test.txt'))
def define_protocols(session, protodir, verbose):
"""Defines all available protocols"""
#figures out which protocols to use
# figures out which protocols to use
valid = {}
# Three grandtest protocols
#files_protocol = ['alldevices/attack-grandtest-allsupports-alldevices-train.txt', 'mobile/attack-mobilegrandtest-allsupports-mobile-train' ,'tablet/attack-tabletgrandtest-allsupports-tablet-train' ]
# Three grandtest protocols
# files_protocol = ['alldevices/attack-grandtest-allsupports-alldevices-train.txt', 'mobile/attack-mobilegrandtest-allsupports-mobile-train' ,'tablet/attack-tabletgrandtest-allsupports-tablet-train' ]
files_protocol = ['alldevices/attack-grandtest-allsupports-alldevices-train.txt', 'alldevices/attack-print-allsupports-alldevices-train' ,'alldevices/attack-mattescreen-fixed-alldevices-alltype-train']
files_protocol = ['alldevices/attack-grandtest-allsupports-alldevices-train.txt', 'alldevices/attack-print-allsupports-alldevices-train', 'alldevices/attack-mattescreen-fixed-alldevices-alltype-train']
for fname in files_protocol:
s = fname.split('-', 5)
folder = fname.split('/', 1)[0]
consider = True
files = {}
for grp in ('train', 'devel', 'test'):
# check attack file
if len(s)<6:
attack = os.path.join(protodir, '%s/attack-%s-%s-%s-%s.txt' % (folder,s[1],s[2],s[3], grp))
else :
attack = os.path.join(protodir, '%s/attack-%s-%s-%s-alltype-%s.txt' % (folder,s[1],s[2],s[3], grp))
if len(s) < 6:
attack = os.path.join(protodir, '%s/attack-%s-%s-%s-%s.txt' % (folder, s[1], s[2], s[3], grp))
else:
attack = os.path.join(protodir, '%s/attack-%s-%s-%s-alltype-%s.txt' % (folder, s[1], s[2], s[3], grp))
if not os.path.exists(attack):
if verbose:
print("Not considering protocol %s as attack list '%s' was not found" % (s[1], attack))
consider = False
# check real file
real = os.path.join(protodir, '%s/real-%s-%s.txt' % (folder,s[3],grp))
real = os.path.join(protodir, '%s/real-%s-%s.txt' % (folder, s[3], grp))
if not os.path.exists(real):
alt_real = os.path.join(protodir, '%s/real-%s-%s.txt' % (folder,s[3],grp))
alt_real = os.path.join(protodir, '%s/real-%s-%s.txt' % (folder, s[3], grp))
if not os.path.exists(alt_real):
if verbose:
print("Not considering protocol %s as real list '%s' or '%s' were not found" % (s[1], real, alt_real))
......@@ -132,26 +137,30 @@ def define_protocols(session, protodir, verbose):
else:
real = alt_real
if consider: files[grp] = (attack, real)
if consider:
files[grp] = (attack, real)
if consider: valid[s[1]] = files
if consider:
valid[s[1]] = files
for protocol, groups in valid.items():
if verbose: print("Creating protocol '%s'..." % protocol)
if verbose:
print("Creating protocol '%s'..." % protocol)
# create protocol on the protocol table
obj = Protocol(name=protocol)
for grp, flist in groups.items():
#print grp
#print flist
# print grp
# print flist
counter = 0
for fname in open(flist[0], 'rt'):
s = os.path.splitext(fname.strip())[0]
q = session.query(Attack).join(File).filter(File.path == s).one()
q.protocols.append(obj)
counter += 1
if verbose: print(" -> %5s/%-6s: %d files" % (grp, "attack", counter))
if verbose:
print(" -> %5s/%-6s: %d files" % (grp, "attack", counter))
counter = 0
for fname in open(flist[1], 'rt'):
......@@ -159,10 +168,12 @@ def define_protocols(session, protodir, verbose):
q = session.query(RealAccess).join(File).filter(File.path == s).one()
q.protocols.append(obj)
counter += 1
if verbose: print(" -> %5s/%-6s: %d files" % (grp, "real", counter))
if verbose:
print(" -> %5s/%-6s: %d files" % (grp, "real", counter))
session.add(obj)
def create_tables(args):
"""Creates all necessary tables (only to be used at the first time)"""
......@@ -177,6 +188,7 @@ def create_tables(args):
# Driver API
# ==========
def create(args):
"""Creates or re-creates this database"""
......@@ -187,7 +199,8 @@ def create(args):
if args.recreate:
if args.verbose and os.path.exists(dbfile):
print(('unlinking %s...' % dbfile))
if os.path.exists(dbfile): os.unlink(dbfile)
if os.path.exists(dbfile):
os.unlink(dbfile)
if not os.path.exists(os.path.dirname(dbfile)):
os.makedirs(os.path.dirname(dbfile))
......@@ -204,19 +217,20 @@ def create(args):
return 0
def add_command(subparsers):
"""Add specific subcommands that the action "create" can use"""
parser = subparsers.add_parser('create', help=create.__doc__)
parser.add_argument('-R', '--recreate', action='store_true', default=False,
help="If set, I'll first erase the current database")
help="If set, I'll first erase the current database")
parser.add_argument('-v', '--verbose', action='count', default=0,
help="Do SQL operations in a verbose way")
help="Do SQL operations in a verbose way")
parser.add_argument('-D', '--protodir', action='store',
default='/idiap/group/biometric/databases/replay-mobile/database/protocols',
# default='/idiap/user/sbhatta/work/replay-mobile/database/protocols',
metavar='DIR',
help="Change the relative path to the directory containing the protocol definitions for replay attacks (defaults to %(default)s)")
default='/idiap/group/biometric/databases/replay-mobile/database/protocols',
# default='/idiap/user/sbhatta/work/replay-mobile/database/protocols',
metavar='DIR',
help="Change the relative path to the directory containing the protocol definitions for replay attacks (defaults to %(default)s)")
parser.set_defaults(func=create) #action
parser.set_defaults(func=create) # action
......@@ -4,10 +4,10 @@
"""Bob Database Driver entry-point for the Replay Mobile Database
"""
import os
import sys
from bob.db.base.driver import Interface as BaseInterface
def reverse(args):
"""Returns a list of file database identifiers given the path stems"""
......@@ -20,12 +20,15 @@ def reverse(args):
output = null()
r = db.reverse(args.path)
for id in r: output.write('%d\n' % id)
if not r: return 1
for id in r:
output.write('%d\n' % id)
if not r:
return 1
return 0
def reverse_command(subparsers):
"""Adds the specific options for the reverse command"""
......@@ -35,9 +38,10 @@ def reverse_command(subparsers):
parser.add_argument('path', nargs='+', type=str, help="one or more path stems to look up. If you provide more than one, files which cannot be reversed will be omitted from the output.")
parser.add_argument('--self-test', dest="selftest", default=False,
action='store_true', help=SUPPRESS)
action='store_true', help=SUPPRESS)
parser.set_defaults(func=reverse) # action
parser.set_defaults(func=reverse) #action
def path(args):
"""Returns a list of fully formed paths or stems given some file id"""
......@@ -51,12 +55,15 @@ def path(args):
output = null()
r = db.paths(args.id, prefix=args.directory, suffix=args.extension)
for path in r: output.write('%s\n' % path)
for path in r:
output.write('%s\n' % path)
if not r: return 1
if not r:
return 1
return 0
def path_command(subparsers):
"""Adds the specific options for the path command"""
......@@ -68,9 +75,10 @@ def path_command(subparsers):
parser.add_argument('-e', '--extension', dest="extension", default='', help="if given, this extension will be appended to every entry returned (defaults to '%(default)s')")
parser.add_argument('id', nargs='+', type=int, help="one or more file ids to look up. If you provide more than one, files which cannot be found will be omitted from the output. If you provide a single id to lookup, an error message will be printed if the id does not exist in the database. The exit status will be non-zero in such case.")
parser.add_argument('--self-test', dest="selftest", default=False,
action='store_true', help=SUPPRESS)
action='store_true', help=SUPPRESS)
parser.set_defaults(func=path) # action
parser.set_defaults(func=path) #action
class Interface(BaseInterface):
......@@ -93,9 +101,9 @@ class Interface(BaseInterface):
def add_commands(self, parser):
from . import __doc__ as docs
subparsers = self.setup_parser(parser,
"Photo/Video Replay mobile database", docs)
subparsers = self.setup_parser(parser,
"Photo/Video Replay mobile database", docs)
# get the "create" action from a submodule
from .create import add_command as create_command
......
......@@ -4,12 +4,12 @@
"""Dumps lists of files.
"""
import os
import sys
# Driver API
# ==========
def dumplist(args):
"""Dumps lists of files based on your criteria"""
......@@ -17,13 +17,13 @@ def dumplist(args):
db = Database()
r = db.objects(
protocol=args.protocol,
support=args.support,
groups=args.group,
protocol=args.protocol,
support=args.support,
groups=args.group,
cls=args.cls,
light=args.light,
clients=args.client,
)
)
output = sys.stdout
if args.selftest:
......@@ -35,6 +35,7 @@ def dumplist(args):
return 0
def add_command(subparsers):
"""Add specific subcommands that the action "dumplist" can use"""
......@@ -47,7 +48,7 @@ def add_command(subparsers):
db = Database()
if not db.is_valid():
protocols = ('waiting','for','database','creation')
protocols = ('waiting', 'for', 'database', 'creation')
clients = tuple()
else:
protocols = [k.name for k in db.protocols()]
......@@ -63,6 +64,6 @@ def add_command(subparsers):
parser.add_argument('-l', '--light', dest="light", default='', help="if given, this value will limit the output files to those shot under a given lighting. (defaults to '%(default)s')", choices=db.lights())
parser.add_argument('-C', '--client', dest="client", default=None, type=int, help="if given, limits the dump to a particular client (defaults to '%(default)s')", choices=clients)
parser.add_argument('--self-test', dest="selftest", default=False,
action='store_true', help=SUPPRESS)
action='store_true', help=SUPPRESS)
parser.set_defaults(func=dumplist) #action
parser.set_defaults(func=dumplist) # action
......@@ -4,7 +4,6 @@
"""Table models and functionality for the Replay Mobile DB.
"""
import sqlalchemy
import os
from sqlalchemy import Table, Column, Integer, String, ForeignKey
from bob.db.base.sqlalchemy_migration import Enum, relationship
......@@ -16,6 +15,7 @@ import bob
Base = declarative_base()
class Client(Base):
"""Database clients, marked by an integer identifier and the set they belong
to"""
......@@ -38,21 +38,22 @@ class Client(Base):
def __repr__(self):
return "Client('%s', '%s')" % (self.id, self.set)
class File(Base):
"""Generic file container"""
__tablename__ = 'file'
light_choices = ('lighton','lightoff','controlled', 'adverse','direct','lateral','diffuse')
light_choices = ('lighton', 'lightoff', 'controlled', 'adverse', 'direct', 'lateral', 'diffuse')
"""List of illumination conditions for data taking"""
device_choices = ('mobile','tablet')
device_choices = ('mobile', 'tablet')
"""List of devices"""
id = Column(Integer, primary_key=True)
"""Key identifier for files"""
client_id = Column(Integer, ForeignKey('client.id')) # for SQL
client_id = Column(Integer, ForeignKey('client.id')) # for SQL
"""The client identifier to which this file is bound to"""
path = Column(String(100), unique=True)
......@@ -68,7 +69,7 @@ class File(Base):
client = relationship(Client, backref=backref('files', order_by=id))
"""A direct link to the client object that this file belongs to"""
def __init__(self, client, path, light,device):
def __init__(self, client, path, light, device):
self.client = client
self.path = path
self.light = light
......@@ -93,8 +94,10 @@ class File(Base):
Returns a string containing the newly generated file path.
"""
if not directory: directory = ''
if not extension: extension = ''
if not directory:
directory = ''
if not extension:
extension = ''
return str(os.path.join(directory, self.path + extension))
......@@ -122,8 +125,9 @@ class File(Base):
Returns a string containing the face file path.
"""
if not directory: directory = ''
#directory = os.path.join(directory, 'face-locations')
if not directory:
directory = ''
# directory = os.path.join(directory, 'face-locations')
directory = os.path.join(directory, 'faceloc/rect/')
return self.make_path(directory, '.face')
......@@ -162,14 +166,14 @@ class File(Base):
def is_mobile(self):
"""True if the video file is originally recorded with mobile device, False otherwise """
value = False
if self.device=='mobile':
if self.device == 'mobile':
value = True
return value
def is_tablet(self):
"""True if the video file is originally recorded rotated by 270 degrees, False otherwise """
value = False
if self.device=='tablet':
if self.device == 'tablet':
value = True
return value
......@@ -185,7 +189,7 @@ class File(Base):
raise RuntimeError("%s is not an attack" % self)
return self.attack[0]
#def load(self, directory=None, extension='.hdf5'):
# def load(self, directory=None, extension='.hdf5'):
def load(self, directory=None, extension=None):
"""Loads the data at the specified location and using the given extension.
......@@ -215,18 +219,18 @@ class File(Base):
video = bob.io.video.reader(vfilename)
vin = video.load()
else:
vin = bob.io.base.load(self.make_path(directory, extension))
vin = bob.io.base.load(self.make_path(directory, extension))
vin = numpy.rollaxis(vin, 3, 2)
if not self.is_tablet():
print("flipping mobile video")
vin = vin[:, :, ::-1,:]
vin = vin[:, :, ::-1, :]
# if self.is_rotated():
# vin = vin[:, :, ::-1,:]
# vin = vin[:, :, ::-1,:]
return vin
#return bob.io.base.load(self.make_path(directory, extension))
# return bob.io.base.load(self.make_path(directory, extension))
def save(self, data, directory=None, extension='.hdf5'):
"""Saves the input data at the specified location and using the given
......@@ -250,17 +254,19 @@ class File(Base):
bob.io.base.create_directories_safe(os.path.dirname(path))
bob.io.base.save(data, path)
# Intermediate mapping from RealAccess's to Protocol's
realaccesses_protocols = Table('realaccesses_protocols', Base.metadata,
Column('realaccess_id', Integer, ForeignKey('realaccess.id')),
Column('protocol_id', Integer, ForeignKey('protocol.id')),
)
Column('realaccess_id', Integer, ForeignKey('realaccess.id')),
Column('protocol_id', Integer, ForeignKey('protocol.id')),
)
# Intermediate mapping from Attack's to Protocol's
attacks_protocols = Table('attacks_protocols', Base.metadata,
Column('attack_id', Integer, ForeignKey('attack.id')),
Column('protocol_id', Integer, ForeignKey('protocol.id')),
)
Column('attack_id', Integer, ForeignKey('attack.id')),
Column('protocol_id', Integer, ForeignKey('protocol.id')),
)
class Protocol(Base):
"""Replay mobile protocol"""
......@@ -279,6 +285,7 @@ class Protocol(Base):
def __repr__(self):
return "Protocol('%s')" % (self.name,)