Commit 541858b1 authored by Amir MOHAMMADI's avatar Amir MOHAMMADI

A filelist database

parent c8890f7c
include README.rst LICENSE version.txt requirements.txt
include README.rst buildout.cfg LICENSE version.txt requirements.txt
recursive-include doc *.py *.rst *.ico *.png
include bob/db/swan/db.sql3
recursive-include bob *.lst
#!/usr/bin/env python
from bob.db.swan import Database
swan_raw_data_directory = "[YOUR_SWAN_DIRECTORY]"
database = Database(
original_directory=swan_raw_data_directory,
original_extension=".mp4",
)
This diff is collapsed.
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
"""Commands the SWAN database can respond to.
"""Commands the Swan database can respond to.
"""
import os
import sys
from bob.db.base.driver import Interface as BaseInterface
......@@ -17,11 +15,8 @@ def dumplist(args):
db = Database()
r = db.objects(
protocol=args.protocol,
purposes=args.purpose,
model_ids=args.client,
groups=args.group,
classes=args.sclass
)
output = sys.stdout
......@@ -30,7 +25,8 @@ def dumplist(args):
output = null()
for f in r:
output.write('%s\n' % (f.make_path(args.directory, args.extension),))
output.write('%s\n' % f.make_path(
directory=args.directory, extension=args.extension))
return 0
......@@ -61,55 +57,13 @@ def checkfiles(args):
if bad:
for f in bad:
output.write('Cannot find file "%s"\n' %
(f.make_path(args.directory, args.extension),))
f.make_path(args.directory, args.extension))
output.write('%d files (out of %d) were not found at "%s"\n' %
(len(bad), len(r), args.directory))
return 0
def reverse(args):
"""Returns a list of file database identifiers given the path stems"""
from .query import Database
db = Database()
output = sys.stdout
if args.selftest:
from bob.db.base.utils import null
output = null()
r = db.reverse(args.path)
for f in r:
output.write('%d\n' % f.id)
if not r:
return 1
return 0
def path(args):
"""Returns a list of fully formed paths or stems given some file id"""
from .query import Database
db = Database()
output = sys.stdout
if args.selftest:
from bob.db.base.utils import null
output = null()
r = db.paths(args.id, prefix=args.directory, suffix=args.extension)
for path in r:
output.write('%s\n' % path)
if not r:
return 1
return 0
class Interface(BaseInterface):
def name(self):
......@@ -120,124 +74,43 @@ class Interface(BaseInterface):
return pkg_resources.require('bob.db.%s' % self.name())[0].version
def files(self):
from pkg_resources import resource_filename
raw_files = ('db.sql3',)
return [resource_filename(__name__, k) for k in raw_files]
return ()
def type(self):
return 'sqlite'
return 'text'
def add_commands(self, parser):
from . import __doc__ as docs
subparsers = self.setup_parser(parser,
"The SWAN database", docs)
"Swan database", docs)
import argparse
from .query import Database
db = Database()
# example: get the "create" action from a submodule
from .create import add_command as create_command
create_command(subparsers)
# the "dumplist" action
parser = subparsers.add_parser('dumplist', help=dumplist.__doc__)
parser.add_argument('-d', '--directory', default='',
help="if given, this path will be prepended to every entry returned.")
parser.add_argument('-e', '--extension', default='',
help="if given, this extension will be appended to every entry returned.")
parser.add_argument(
'-d', '--directory', help="if given, this path will be prepended "
"to every entry returned.")
parser.add_argument(
'-e', '--extension', help="if given, this extension will be "
"appended to every entry returned.")
parser.add_argument(
'-p', '--protocol', help="if given, limits the check to a "
"particular subset of the data that corresponds to the given "
"protocol.",
choices=list(db.protocol_names()).extend(['male', 'female']) if
db.is_valid() else ())
parser.add_argument(
'-u',
'--purpose',
help="if given, this value will limit the output files to those "
"designed for the given purposes.",
choices=db.purposes() if db.is_valid() else ())
parser.add_argument(
'-C',
'--client',
type=int,
help="if given, limits the dump to a particular client.",
choices=db.model_ids() if db.is_valid() else ())
parser.add_argument(
'-g',
'--group',
help="if given, this value will limit the output files to those "
"belonging to a particular protocolar group.",
choices=db.groups() if db.is_valid() else ())
parser.add_argument(
'-c',
'--class',
dest="sclass",
help="if given, this value will limit the output files to those "
"belonging to the given classes.",
choices=(
'client',
'impostor'))
parser.add_argument(
'--self-test', dest="selftest",
action='store_true', help=argparse.SUPPRESS)
'-u', '--purpose', help="if given, this value will limit the output files to those designed for the given purposes.", choices=('enroll', 'probe', ''))
parser.add_argument('-g', '--group', help="if given, this value will limit the output files to those belonging to a particular protocolar group.",
choices=('dev', 'eval', 'world', ''))
parser.add_argument('--self-test', dest="selftest",
action='store_true', help=argparse.SUPPRESS)
parser.set_defaults(func=dumplist) # action
# the "checkfiles" action
parser = subparsers.add_parser('checkfiles', help=checkfiles.__doc__)
parser.add_argument(
'-d',
'--directory',
help="if given, this path will be prepended to every entry "
"returned.")
parser.add_argument(
'-e',
'--extension',
help="if given, this extension will be appended to every entry "
"returned.")
parser.add_argument('--self-test', dest="selftest",
action='store_true', help=argparse.SUPPRESS)
parser.set_defaults(func=checkfiles) # action
# adds the "reverse" command
parser = subparsers.add_parser('reverse', help=reverse.__doc__)
parser.add_argument(
'path',
nargs='+',
help="one or more path stems to look up. If you provide more than "
"one, files which cannot be reversed will be omitted from the "
"output.")
parser.add_argument('-l', '--list-directory', required=True,
help="The directory which contains the file lists.")
parser.add_argument('-d', '--directory', dest="directory", default='',
help="if given, this path will be prepended to every entry returned.")
parser.add_argument('-e', '--extension', dest="extension", default='',
help="if given, this extension will be appended to every entry returned.")
parser.add_argument('--self-test', dest="selftest",
action='store_true', help=argparse.SUPPRESS)
parser.set_defaults(func=reverse) # action
# adds the "path" command
parser = subparsers.add_parser('path', help=path.__doc__)
parser.add_argument(
'-d',
'--directory',
help="if given, this path will be prepended to every entry "
"returned.")
parser.add_argument(
'-e',
'--extension',
help="if given, this extension will be appended to every entry "
"returned.")
parser.add_argument(
'id',
nargs='+',
type=int,
help="one or more file ids to look up. If you provide more than "
"one, files which cannot be found will be omitted from the output."
" If you provide a single id to lookup, an error message will be "
"printed if the id does not exist in the database. The exit status"
" will be non-zero in such case.")
parser.add_argument('--self-test', dest="selftest",
action='store_true', help=argparse.SUPPRESS)
parser.set_defaults(func=path) # action
parser.set_defaults(func=checkfiles) # action
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
"""Table models and functionality for the SWAN database.
"""
import bob.db.base.utils
from sqlalchemy import Table, Column, Integer, String, ForeignKey
from bob.db.base.sqlalchemy_migration import Enum, relationship
from sqlalchemy.orm import backref
from sqlalchemy.ext.declarative import declarative_base
import bob.db.base
Base = declarative_base()
protocolPurpose_file_association = Table(
'protocolPurpose_file_association', Base.metadata,
Column('protocolPurpose_id', Integer, ForeignKey(
'protocolPurpose.id')),
Column('file_id', Integer, ForeignKey('file.id')))
class Client(Base):
"""Database clients, marked by an integer identifier and the group they
belong to"""
__tablename__ = 'client'
# Key identifier for the client
id = Column(Integer, primary_key=True)
# the id of the client in the database
orig_id = Column(Integer)
# Gender to which the client belongs to
gender_choices = ('female', 'male')
gender = Column(Enum(*gender_choices))
# Group to which the client belongs to
group_choices = ('world', 'dev', 'eval')
sgroup = Column(Enum(*group_choices)) # do NOT use group (SQL keyword)
# Institute to which the client belongs to
institute_choices = ('IDIAP', 'MPH-FRA', 'MPH-IND', 'NTNU', 'UIO')
institute = Column(Enum(*institute_choices))
def __init__(self, orig_id, group, gender, institute):
self.orig_id = orig_id
self.sgroup = group
self.gender = gender
self.institute = institute
def __repr__(self):
return "Client(id={}, orig_id={}, group={}, gender={}, institute={})"\
.format(self.id, self.orig_id, self.sgroup, self.gender,
self.institute)
class File(Base, bob.db.base.File):
"""Generic file container"""
__tablename__ = 'file'
# Key identifier for the file
id = Column(Integer, primary_key=True)
# Unique path to this file inside the database
path = Column(String(100), unique=True)
# Identifier of the session
session = Column(Integer)
# Identifier of the device
device_choices = ('iPhone', 'iPad')
device = Column(Enum(*device_choices))
modality_choices = ('face', 'voice', 'eye', 'finger')
modality = Column(Enum(*modality_choices))
camera_choices = ('rear', 'front')
camera = Column(Enum(*camera_choices))
recording_choices = ('video', 'photo')
recording = Column(Enum(*recording_choices))
nrecording = Column(Integer)
# Key identifier of the client associated with this file
client_id = Column(Integer, ForeignKey('client.id')) # for SQL
# For Python: A direct link to the client object that this file belongs to
client = relationship("Client", backref=backref("files", order_by=id))
def __init__(self, path, client, session, device, modality,
camera, recording, nrecording):
# call base class constructor
bob.db.base.File.__init__(self, path=path)
# fill the remaining bits of the file information
self.client = client
self.session = session
self.device = device
self.modality = modality
self.camera = camera
self.recording = recording
self.nrecording = nrecording
class Protocol(Base):
"""SWAN protocols"""
__tablename__ = 'protocol'
# Unique identifier for this protocol object
id = Column(Integer, primary_key=True)
# Name of the protocol associated with this object
name = Column(String(20), unique=True)
def __init__(self, name):
self.name = name
def __repr__(self):
return "Protocol({})".format(self.name)
class ProtocolPurpose(Base):
"""SWAN protocol purposes"""
__tablename__ = 'protocolPurpose'
# Unique identifier for this protocol purpose object
id = Column(Integer, primary_key=True)
# Id of the protocol associated with this protocol purpose object
protocol_id = Column(Integer, ForeignKey('protocol.id')) # for SQL
# Group associated with this protocol purpose object
group_choices = Client.group_choices
sgroup = Column(Enum(*group_choices))
# Purpose associated with this protocol purpose object
purpose_choices = ('train', 'enroll', 'probe')
purpose = Column(Enum(*purpose_choices))
# For Python: A direct link to the Protocol object that this
# ProtocolPurpose belongs to
protocol = relationship(
"Protocol", backref=backref("purposes", order_by=id))
# For Python: A direct link to the File objects associated with this
# ProtcolPurpose
files = relationship("File", secondary=protocolPurpose_file_association,
backref=backref("protocol_purposes", order_by=id))
def __init__(self, protocol_id, group, purpose):
self.protocol_id = protocol_id
self.sgroup = group
self.purpose = purpose
def __repr__(self):
return "ProtocolPurpose({}, {}, {})".format(
self.protocol.name, self.sgroup, self.purpose)
This diff is collapsed.
setuptools
-egit+git@gitlab.idiap.ch:bob/bob.extension.git#egg=bob.extension
bob.extension
-egit+git@gitlab.idiap.ch:bob/bob.blitz.git#egg=bob.blitz
bob.blitz
-egit+git@gitlab.idiap.ch:bob/bob.core.git#egg=bob.core
bob.core
-egit+git@gitlab.idiap.ch:bob/bob.io.base.git#egg=bob.io.base
bob.io.base
-egit+git@gitlab.idiap.ch:bob/bob.io.image.git#egg=bob.io.image
bob.io.image
-egit+git@gitlab.idiap.ch:bob/bob.io.video.git#egg=bob.io.video
bob.io.video
-egit+git@gitlab.idiap.ch:bob/bob.db.base.git#egg=bob.db.base
bob.db.base
-egit+git@gitlab.idiap.ch:bob/bob.bio.base.git#egg=bob.bio.base
numpy
scipy
bob.bio.base
-egit+git@gitlab.idiap.ch:bob/bob.bio.spear.git#egg=bob.bio.spear
bob.bio.spear
......@@ -5,7 +5,8 @@ from setuptools import setup, dist
dist.Distribution(dict(setup_requires=['bob.extension']))
# load the requirements.txt for additional requirements
from bob.extension.utils import find_packages
from bob.extension.utils import find_packages, load_requirements
install_requires = load_requirements()
setup(
name='bob.db.swan',
......@@ -27,16 +28,12 @@ setup(
packages=find_packages('bob'),
include_package_data=True,
install_requires=['bob.extension',
'bob.blitz',
'bob.io.base',
'bob.io.image',
'bob.io.video',
'bob.db.base'],
install_requires=install_requires,
entry_points={
'bob.db': ['swan = bob.db.swan.driver:Interface'],
'bob.bio.database': ['swan = bob.db.swan.config:database'],
},
classifiers=[
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment