A filelist database

MANIFEST.in

-include README.rst LICENSE version.txt requirements.txt
+include README.rst buildout.cfg LICENSE version.txt requirements.txt
 recursive-include doc *.py *.rst *.ico *.png
-include bob/db/swan/db.sql3
+recursive-include bob *.lst

bob/db/swan/config.py

+#!/usr/bin/env python
+
+from bob.db.swan import Database
+
+swan_raw_data_directory = "[YOUR_SWAN_DIRECTORY]"
+
+database = Database(
+    original_directory=swan_raw_data_directory,
+    original_extension=".mp4",
+)

bob/db/swan/driver.py

 #!/usr/bin/env python
-# vim: set fileencoding=utf-8 :
 
-"""Commands the SWAN database can respond to.
+"""Commands the Swan database can respond to.
 """
 
 import os
 import sys
-
 from bob.db.base.driver import Interface as BaseInterface
 
 
@@ -17,11 +15,8 @@ def dumplist(args):
     db = Database()
 
     r = db.objects(
-        protocol=args.protocol,
         purposes=args.purpose,
-        model_ids=args.client,
         groups=args.group,
-        classes=args.sclass
     )
 
     output = sys.stdout
@@ -30,7 +25,8 @@ def dumplist(args):
         output = null()
 
     for f in r:
-        output.write('%s\n' % (f.make_path(args.directory, args.extension),))
+        output.write('%s\n' % f.make_path(
+            directory=args.directory, extension=args.extension))
 
     return 0
 
@@ -61,55 +57,13 @@ def checkfiles(args):
     if bad:
         for f in bad:
             output.write('Cannot find file "%s"\n' %
-                         (f.make_path(args.directory, args.extension),))
+                         f.make_path(args.directory, args.extension))
         output.write('%d files (out of %d) were not found at "%s"\n' %
                      (len(bad), len(r), args.directory))
 
     return 0
 
 
-def reverse(args):
-    """Returns a list of file database identifiers given the path stems"""
-
-    from .query import Database
-    db = Database()
-
-    output = sys.stdout
-    if args.selftest:
-        from bob.db.base.utils import null
-        output = null()
-
-    r = db.reverse(args.path)
-    for f in r:
-        output.write('%d\n' % f.id)
-
-    if not r:
-        return 1
-
-    return 0
-
-
-def path(args):
-    """Returns a list of fully formed paths or stems given some file id"""
-
-    from .query import Database
-    db = Database()
-
-    output = sys.stdout
-    if args.selftest:
-        from bob.db.base.utils import null
-        output = null()
-
-    r = db.paths(args.id, prefix=args.directory, suffix=args.extension)
-    for path in r:
-        output.write('%s\n' % path)
-
-    if not r:
-        return 1
-
-    return 0
-
-
 class Interface(BaseInterface):
 
     def name(self):
@@ -120,124 +74,43 @@ class Interface(BaseInterface):
         return pkg_resources.require('bob.db.%s' % self.name())[0].version
 
     def files(self):
-
-        from pkg_resources import resource_filename
-        raw_files = ('db.sql3',)
-        return [resource_filename(__name__, k) for k in raw_files]
+        return ()
 
     def type(self):
-        return 'sqlite'
+        return 'text'
 
     def add_commands(self, parser):
 
         from . import __doc__ as docs
 
         subparsers = self.setup_parser(parser,
-                                       "The SWAN database", docs)
+                                       "Swan database", docs)
 
         import argparse
-        from .query import Database
-        db = Database()
-
-        # example: get the "create" action from a submodule
-        from .create import add_command as create_command
-        create_command(subparsers)
 
         # the "dumplist" action
         parser = subparsers.add_parser('dumplist', help=dumplist.__doc__)
+        parser.add_argument('-d', '--directory', default='',
+                            help="if given, this path will be prepended to every entry returned.")
+        parser.add_argument('-e', '--extension', default='',
+                            help="if given, this extension will be appended to every entry returned.")
         parser.add_argument(
-            '-d', '--directory', help="if given, this path will be prepended "
-            "to every entry returned.")
-        parser.add_argument(
-            '-e', '--extension', help="if given, this extension will be "
-            "appended to every entry returned.")
-        parser.add_argument(
-            '-p', '--protocol', help="if given, limits the check to a "
-            "particular subset of the data that corresponds to the given "
-            "protocol.",
-            choices=list(db.protocol_names()).extend(['male', 'female']) if
-            db.is_valid() else ())
-        parser.add_argument(
-            '-u',
-            '--purpose',
-            help="if given, this value will limit the output files to those "
-            "designed for the given purposes.",
-            choices=db.purposes() if db.is_valid() else ())
-        parser.add_argument(
-            '-C',
-            '--client',
-            type=int,
-            help="if given, limits the dump to a particular client.",
-            choices=db.model_ids() if db.is_valid() else ())
-        parser.add_argument(
-            '-g',
-            '--group',
-            help="if given, this value will limit the output files to those "
-            "belonging to a particular protocolar group.",
-            choices=db.groups() if db.is_valid() else ())
-        parser.add_argument(
-            '-c',
-            '--class',
-            dest="sclass",
-            help="if given, this value will limit the output files to those "
-            "belonging to the given classes.",
-            choices=(
-                'client',
-                'impostor'))
-        parser.add_argument(
-            '--self-test', dest="selftest",
-            action='store_true', help=argparse.SUPPRESS)
+            '-u', '--purpose', help="if given, this value will limit the output files to those designed for the given purposes.", choices=('enroll', 'probe', ''))
+        parser.add_argument('-g', '--group', help="if given, this value will limit the output files to those belonging to a particular protocolar group.",
+                            choices=('dev', 'eval', 'world', ''))
+        parser.add_argument('--self-test', dest="selftest",
+                            action='store_true', help=argparse.SUPPRESS)
         parser.set_defaults(func=dumplist)  # action
 
         # the "checkfiles" action
         parser = subparsers.add_parser('checkfiles', help=checkfiles.__doc__)
-        parser.add_argument(
-            '-d',
-            '--directory',
-            help="if given, this path will be prepended to every entry "
-            "returned.")
-        parser.add_argument(
-            '-e',
-            '--extension',
-            help="if given, this extension will be appended to every entry "
-            "returned.")
-        parser.add_argument('--self-test', dest="selftest",
-                            action='store_true', help=argparse.SUPPRESS)
-        parser.set_defaults(func=checkfiles)  # action
-
-        # adds the "reverse" command
-        parser = subparsers.add_parser('reverse', help=reverse.__doc__)
-        parser.add_argument(
-            'path',
-            nargs='+',
-            help="one or more path stems to look up. If you provide more than "
-            "one, files which cannot be reversed will be omitted from the "
-            "output.")
+        parser.add_argument('-l', '--list-directory', required=True,
+                            help="The directory which contains the file lists.")
+        parser.add_argument('-d', '--directory', dest="directory", default='',
+                            help="if given, this path will be prepended to every entry returned.")
+        parser.add_argument('-e', '--extension', dest="extension", default='',
+                            help="if given, this extension will be appended to every entry returned.")
         parser.add_argument('--self-test', dest="selftest",
                             action='store_true', help=argparse.SUPPRESS)
-        parser.set_defaults(func=reverse)  # action
 
-        # adds the "path" command
-        parser = subparsers.add_parser('path', help=path.__doc__)
-        parser.add_argument(
-            '-d',
-            '--directory',
-            help="if given, this path will be prepended to every entry "
-            "returned.")
-        parser.add_argument(
-            '-e',
-            '--extension',
-            help="if given, this extension will be appended to every entry "
-            "returned.")
-        parser.add_argument(
-            'id',
-            nargs='+',
-            type=int,
-            help="one or more file ids to look up. If you provide more than "
-            "one, files which cannot be found will be omitted from the output."
-            " If you provide a single id to lookup, an error message will be "
-            "printed if the id does not exist in the database. The exit status"
-            " will be non-zero in such case.")
-        parser.add_argument('--self-test', dest="selftest",
-                            action='store_true', help=argparse.SUPPRESS)
-        parser.set_defaults(func=path)  # action
+        parser.set_defaults(func=checkfiles)  # action

bob/db/swan/models.py

-#!/usr/bin/env python
-# vim: set fileencoding=utf-8 :
-
-"""Table models and functionality for the SWAN database.
-"""
-
-import bob.db.base.utils
-from sqlalchemy import Table, Column, Integer, String, ForeignKey
-from bob.db.base.sqlalchemy_migration import Enum, relationship
-from sqlalchemy.orm import backref
-from sqlalchemy.ext.declarative import declarative_base
-
-import bob.db.base
-
-Base = declarative_base()
-
-protocolPurpose_file_association = Table(
-    'protocolPurpose_file_association', Base.metadata,
-    Column('protocolPurpose_id', Integer, ForeignKey(
-        'protocolPurpose.id')),
-    Column('file_id', Integer, ForeignKey('file.id')))
-
-
-class Client(Base):
-    """Database clients, marked by an integer identifier and the group they
-    belong to"""
-
-    __tablename__ = 'client'
-
-    # Key identifier for the client
-    id = Column(Integer, primary_key=True)
-    # the id of the client in the database
-    orig_id = Column(Integer)
-    # Gender to which the client belongs to
-    gender_choices = ('female', 'male')
-    gender = Column(Enum(*gender_choices))
-    # Group to which the client belongs to
-    group_choices = ('world', 'dev', 'eval')
-    sgroup = Column(Enum(*group_choices))  # do NOT use group (SQL keyword)
-    # Institute to which the client belongs to
-    institute_choices = ('IDIAP', 'MPH-FRA', 'MPH-IND', 'NTNU', 'UIO')
-    institute = Column(Enum(*institute_choices))
-
-    def __init__(self, orig_id, group, gender, institute):
-        self.orig_id = orig_id
-        self.sgroup = group
-        self.gender = gender
-        self.institute = institute
-
-    def __repr__(self):
-        return "Client(id={}, orig_id={}, group={}, gender={}, institute={})"\
-            .format(self.id, self.orig_id, self.sgroup, self.gender,
-                    self.institute)
-
-
-class File(Base, bob.db.base.File):
-    """Generic file container"""
-
-    __tablename__ = 'file'
-
-    # Key identifier for the file
-    id = Column(Integer, primary_key=True)
-    # Unique path to this file inside the database
-    path = Column(String(100), unique=True)
-    # Identifier of the session
-    session = Column(Integer)
-    # Identifier of the device
-    device_choices = ('iPhone', 'iPad')
-    device = Column(Enum(*device_choices))
-
-    modality_choices = ('face', 'voice', 'eye', 'finger')
-    modality = Column(Enum(*modality_choices))
-
-    camera_choices = ('rear', 'front')
-    camera = Column(Enum(*camera_choices))
-
-    recording_choices = ('video', 'photo')
-    recording = Column(Enum(*recording_choices))
-
-    nrecording = Column(Integer)
-
-    # Key identifier of the client associated with this file
-    client_id = Column(Integer, ForeignKey('client.id'))  # for SQL
-    # For Python: A direct link to the client object that this file belongs to
-    client = relationship("Client", backref=backref("files", order_by=id))
-
-    def __init__(self, path, client, session, device, modality,
-                 camera, recording, nrecording):
-        # call base class constructor
-        bob.db.base.File.__init__(self, path=path)
-
-        # fill the remaining bits of the file information
-        self.client = client
-        self.session = session
-        self.device = device
-        self.modality = modality
-        self.camera = camera
-        self.recording = recording
-        self.nrecording = nrecording
-
-
-class Protocol(Base):
-    """SWAN protocols"""
-
-    __tablename__ = 'protocol'
-
-    # Unique identifier for this protocol object
-    id = Column(Integer, primary_key=True)
-    # Name of the protocol associated with this object
-    name = Column(String(20), unique=True)
-
-    def __init__(self, name):
-        self.name = name
-
-    def __repr__(self):
-        return "Protocol({})".format(self.name)
-
-
-class ProtocolPurpose(Base):
-    """SWAN protocol purposes"""
-
-    __tablename__ = 'protocolPurpose'
-
-    # Unique identifier for this protocol purpose object
-    id = Column(Integer, primary_key=True)
-    # Id of the protocol associated with this protocol purpose object
-    protocol_id = Column(Integer, ForeignKey('protocol.id'))  # for SQL
-    # Group associated with this protocol purpose object
-    group_choices = Client.group_choices
-    sgroup = Column(Enum(*group_choices))
-    # Purpose associated with this protocol purpose object
-    purpose_choices = ('train', 'enroll', 'probe')
-    purpose = Column(Enum(*purpose_choices))
-
-    # For Python: A direct link to the Protocol object that this
-    # ProtocolPurpose belongs to
-    protocol = relationship(
-        "Protocol", backref=backref("purposes", order_by=id))
-    # For Python: A direct link to the File objects associated with this
-    # ProtcolPurpose
-    files = relationship("File", secondary=protocolPurpose_file_association,
-                         backref=backref("protocol_purposes", order_by=id))
-
-    def __init__(self, protocol_id, group, purpose):
-        self.protocol_id = protocol_id
-        self.sgroup = group
-        self.purpose = purpose
-
-    def __repr__(self):
-        return "ProtocolPurpose({}, {}, {})".format(
-            self.protocol.name, self.sgroup, self.purpose)

requirements.txt

 setuptools
--egit+git@gitlab.idiap.ch:bob/bob.extension.git#egg=bob.extension
-bob.extension
--egit+git@gitlab.idiap.ch:bob/bob.blitz.git#egg=bob.blitz
-bob.blitz
--egit+git@gitlab.idiap.ch:bob/bob.core.git#egg=bob.core
-bob.core
--egit+git@gitlab.idiap.ch:bob/bob.io.base.git#egg=bob.io.base
-bob.io.base
--egit+git@gitlab.idiap.ch:bob/bob.io.image.git#egg=bob.io.image
-bob.io.image
--egit+git@gitlab.idiap.ch:bob/bob.io.video.git#egg=bob.io.video
-bob.io.video
--egit+git@gitlab.idiap.ch:bob/bob.db.base.git#egg=bob.db.base
-bob.db.base
--egit+git@gitlab.idiap.ch:bob/bob.bio.base.git#egg=bob.bio.base
+numpy
+scipy
 bob.bio.base
--egit+git@gitlab.idiap.ch:bob/bob.bio.spear.git#egg=bob.bio.spear
 bob.bio.spear

setup.py

@@ -5,7 +5,8 @@ from setuptools import setup, dist
 dist.Distribution(dict(setup_requires=['bob.extension']))
 
 # load the requirements.txt for additional requirements
-from bob.extension.utils import find_packages
+from bob.extension.utils import find_packages, load_requirements
+install_requires = load_requirements()
 
 setup(
     name='bob.db.swan',
@@ -27,16 +28,12 @@ setup(
     packages=find_packages('bob'),
     include_package_data=True,
 
-    install_requires=['bob.extension',
-                      'bob.blitz',
-                      'bob.io.base',
-                      'bob.io.image',
-                      'bob.io.video',
-                      'bob.db.base'],
+    install_requires=install_requires,
 
     entry_points={
 
         'bob.db': ['swan = bob.db.swan.driver:Interface'],
+        'bob.bio.database': ['swan = bob.db.swan.config:database'],
     },
 
     classifiers=[