Initial commit

.gitignore

+*~
+*.swp
+*.pyc
+bin
+eggs
+parts
+.installed.cfg
+.mr.developer.cfg
+*.egg-info
+src
+develop-eggs
+sphinx
+dist
+conda

.gitlab-ci.yml

+# This build file heavily uses template features from YAML so it is generic
+# enough for any Bob project. Don't modify it unless you know what you're
+# doing.
+
+
+# Definition of our build pipeline
+stages:
+  - build
+  - test
+  - docs
+  - wheels
+  - deploy
+
+
+# ---------
+# Templates
+# ---------
+
+# Template for the build stage
+# Needs to run on all supported architectures, platforms and python versions
+.build_template: &build_job
+  stage: build
+  before_script:
+    - git clean -ffdx
+    - mkdir _ci
+    - curl --silent "https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/install.sh" > _ci/install.sh
+    - chmod 755 _ci/install.sh
+    - ./_ci/install.sh _ci #updates
+    - ./_ci/before_build.sh
+  script:
+    - ./_ci/build.sh
+  after_script:
+    - ./_ci/after_build.sh
+  artifacts:
+    expire_in: 1 week
+    paths:
+      - _ci/
+      - dist/
+      - sphinx/
+
+
+# Template for the test stage - re-installs from uploaded wheels
+# Needs to run on all supported architectures, platforms and python versions
+.test_template: &test_job
+  stage: test
+  before_script:
+    - ./_ci/install.sh _ci #updates
+    - ./_ci/before_test.sh
+  script:
+    - ./_ci/test.sh
+  after_script:
+    - ./_ci/after_test.sh
+
+
+# Template for the wheel uploading stage
+# Needs to run against one supported architecture, platform and python version
+.wheels_template: &wheels_job
+  stage: wheels
+  environment: intranet
+  only:
+    - master
+    - /^v\d+\.\d+\.\d+([abc]\d*)?$/  # PEP-440 compliant version (tags)
+  before_script:
+    - ./_ci/install.sh _ci #updates
+    - ./_ci/before_wheels.sh
+  script:
+    - ./_ci/wheels.sh
+  after_script:
+    - ./_ci/after_wheels.sh
+
+
+# Template for (latest) documentation upload stage
+# Only one real job needs to do this
+.docs_template: &docs_job
+  stage: docs
+  environment: intranet
+  only:
+    - master
+  before_script:
+    - ./_ci/install.sh _ci #updates
+    - ./_ci/before_docs.sh
+  script:
+    - ./_ci/docs.sh
+  after_script:
+    - ./_ci/after_docs.sh
+
+
+# Template for the deployment stage - re-installs from uploaded wheels
+# Needs to run on a single architecture only
+# Will deploy your package to PyPI and other required services
+# Only runs for tags
+.deploy_template: &deploy_job
+  stage: deploy
+  environment: internet
+  only:
+    - /^v\d+\.\d+\.\d+([abc]\d*)?$/  # PEP-440 compliant version (tags)
+  except:
+    - branches
+  before_script:
+    - ./_ci/install.sh _ci #updates
+    - ./_ci/before_deploy.sh
+  script:
+    - ./_ci/deploy.sh
+  after_script:
+    - ./_ci/after_deploy.sh
+
+
+# -------------
+# Build Targets
+# -------------
+
+# Linux + Python 2.7: Builds, tests, uploads wheel and deploys (if needed)
+build_linux_27:
+  <<: *build_job
+  variables: &linux_27_build_variables
+    PYTHON_VERSION: "2.7"
+    WHEEL_TAG: "py27"
+  tags:
+    - conda-linux
+
+test_linux_27:
+  <<: *test_job
+  variables: *linux_27_build_variables
+  dependencies:
+    - build_linux_27
+  tags:
+    - conda-linux
+
+wheels_linux_27:
+  <<: *wheels_job
+  variables: *linux_27_build_variables
+  dependencies:
+    - build_linux_27
+  tags:
+    - conda-linux
+
+deploy_linux_27:
+  <<: *deploy_job
+  variables: *linux_27_build_variables
+  dependencies:
+    - build_linux_27
+  tags:
+    - conda-linux
+
+
+# Linux + Python 3.5: Builds, tests and uploads wheel
+build_linux_35:
+  <<: *build_job
+  variables: &linux_35_build_variables
+    PYTHON_VERSION: "3.5"
+    WHEEL_TAG: "py3"
+  tags:
+    - conda-linux
+
+test_linux_35:
+  <<: *test_job
+  variables: *linux_35_build_variables
+  dependencies:
+    - build_linux_35
+  tags:
+    - conda-linux
+
+wheels_linux_35:
+  <<: *wheels_job
+  variables: *linux_35_build_variables
+  dependencies:
+    - build_linux_35
+  tags:
+    - conda-linux
+
+docs_linux_35:
+  <<: *docs_job
+  variables: *linux_35_build_variables
+  dependencies:
+    - build_linux_35
+  tags:
+    - conda-linux
+
+
+# Linux + Python 3.6: Builds and tests
+build_linux_36:
+  <<: *build_job
+  variables: &linux_36_build_variables
+    PYTHON_VERSION: "3.6"
+    WHEEL_TAG: "py3"
+  tags:
+    - conda-linux
+
+test_linux_36:
+  <<: *test_job
+  variables: *linux_36_build_variables
+  dependencies:
+    - build_linux_36
+  tags:
+    - conda-linux
+
+
+# Mac OSX + Python 2.7: Builds and tests
+build_macosx_27:
+  <<: *build_job
+  variables: &macosx_27_build_variables
+    PYTHON_VERSION: "2.7"
+    WHEEL_TAG: "py27"
+  tags:
+    - conda-macosx
+
+test_macosx_27:
+  <<: *test_job
+  variables: *macosx_27_build_variables
+  dependencies:
+    - build_macosx_27
+  tags:
+    - conda-macosx
+
+
+# Mac OSX + Python 3.5: Builds and tests
+build_macosx_35:
+  <<: *build_job
+  variables: &macosx_35_build_variables
+    PYTHON_VERSION: "3.5"
+    WHEEL_TAG: "py3"
+  tags:
+    - conda-macosx
+
+test_macosx_35:
+  <<: *test_job
+  variables: *macosx_35_build_variables
+  dependencies:
+    - build_macosx_35
+  tags:
+    - conda-macosx
+
+
+# Mac OSX + Python 3.6: Builds and tests
+build_macosx_36:
+  <<: *build_job
+  variables: &macosx_36_build_variables
+    PYTHON_VERSION: "3.6"
+    WHEEL_TAG: "py3"
+  tags:
+    - conda-macosx
+
+test_macosx_36:
+  <<: *test_job
+  variables: *macosx_36_build_variables
+  dependencies:
+    - build_macosx_36
+  tags:
+    - conda-macosx

LICENSE

+Copyright (c) 2017 Idiap Research Institute, http://www.idiap.ch/
+Written by Amir Mohammadi <amir.mohammadi@idiap.ch>
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors
+may be used to endorse or promote products derived from this software without
+specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file

MANIFEST.in

+include README.rst bootstrap-buildout.py buildout.cfg COPYING version.txt requirements.txt
+recursive-include doc *.py *.rst *.ico *.png

README.rst

+.. vim: set fileencoding=utf-8 :
+.. Fri May  5 11:35:14 CEST 2017
+
+.. image:: http://img.shields.io/badge/docs-stable-yellow.svg
+   :target: http://pythonhosted.org/bob.db.swan/index.html
+.. image:: http://img.shields.io/badge/docs-latest-orange.svg
+   :target: https://www.idiap.ch/software/bob/docs/latest/bob/bob.db.swan/master/index.html
+.. image:: https://gitlab.idiap.ch/bob/bob.db.swan/badges/master/build.svg
+   :target: https://gitlab.idiap.ch/bob/bob.db.swan/commits/master
+.. image:: https://gitlab.idiap.ch/bob/bob.db.swan/badges/master/coverage.svg
+   :target: https://gitlab.idiap.ch/bob/bob.db.swan/commits/master
+.. image:: https://img.shields.io/badge/gitlab-project-0000c0.svg
+   :target: https://gitlab.idiap.ch/bob/bob.db.swan
+.. image:: http://img.shields.io/pypi/v/bob.db.swan.svg
+   :target: https://pypi.python.org/pypi/bob.db.swan
+.. image:: http://img.shields.io/pypi/dm/bob.db.swan.svg
+   :target: https://pypi.python.org/pypi/bob.db.swan
+
+
+==================================
+ SWAN Database Access API for Bob
+==================================
+
+This package is part of the signal-processing and machine learning toolbox
+Bob_. It contains an interface for the evaluation protocol of the `SWAN`_
+database. This package does not contain the original `SWAN`_ data files, which
+need to be obtained through the link above.
+
+
+Installation
+------------
+
+Follow our `installation`_ instructions. Then, to install this pacakge, run::
+
+  $ conda install bob.db.swan
+
+
+Contact
+-------
+
+For questions or reporting issues to this software package, contact our
+development `mailing list`_.
+
+
+.. Place your references here:
+.. _bob: https://www.idiap.ch/software/bob
+.. _installation: https://www.idiap.ch/software/bob/install
+.. _mailing list: https://www.idiap.ch/software/bob/discuss
+.. _swan: http://www.idiap.ch/dataset/swan

bob/__init__.py

+# see https://docs.python.org/3/library/pkgutil.html
+from pkgutil import extend_path
+__path__ = extend_path(__path__, __name__)

bob/db/__init__.py

+# see https://docs.python.org/3/library/pkgutil.html
+from pkgutil import extend_path
+__path__ = extend_path(__path__, __name__)

bob/db/swan/__init__.py

+
+def get_config():
+    """Returns a string containing the configuration information.
+    """
+
+    import bob.extension
+    return bob.extension.get_config(__name__)
+
+
+# gets sphinx autodoc done right - don't remove it
+__all__ = [_ for _ in dir() if not _.startswith('_')]

bob/db/swan/create.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+"""This script creates the SWAN database in a single pass.
+"""
+
+import os
+
+from .models import Client, File, Protocol, ProtocolPurpose, Base
+
+
+def nodot(item):
+    """Can be used to ignore hidden files, starting with the . character."""
+    return item[0] != '.'
+
+
+def add_clients_and_files(sql_session, datadir, extensions, verbose):
+    """Add files to the SWAN database."""
+
+    def add_file(session, datadir, location, client_id_dir, session_device, basename, verbose):
+        """Parse a single filename and add it to the list.
+             Also add a client entry if not already in the database."""
+        v = os.path.splitext(basename)[0].split('_')
+        bname = os.path.splitext(basename)[0]
+        full_bname = os.path.join(
+            location, client_id_dir, session_device, bname)
+
+        gender = ''
+        if v[0][0] == 'm':
+            gender = 'male'
+        if v[0][0] == 'f':
+            gender = 'female'
+        institute = int(v[0][1])
+        institute_dir = ''
+        if institute == 0:
+            institute = 'idiap'
+            institute_dir = 'idiap'
+        elif institute == 1:
+            institute = 'manchester'
+            institute_dir = 'uman'
+        elif institute == 2:
+            institute = 'surrey'
+            institute_dir = 'unis'
+        elif institute == 3:
+            institute = 'oulu'
+            institute_dir = 'uoulu'
+        elif institute == 4:
+            institute = 'brno'
+            institute_dir = 'but'
+        elif institute == 5:
+            institute = 'avignon'
+            institute_dir = 'lia'
+        if institute_dir != location:
+            error_msg = "File: %s -- Find location %s in directory of location %s!" % (
+                full_bname, location, institute_dir)
+            raise RuntimeError(error_msg)
+        client_id = v[0][1:4]
+        if v[0][0:4] != client_id_dir:
+            error_msg = "File: %s -- Find identity %s in directory of identity %s!" % (
+                full_bname, v[0][0:4], client_id)
+            raise RuntimeError(error_msg)
+        if not (client_id in client_dict):
+            if (institute == 'surrey' or institute == 'avignon'):
+                group = 'world'
+            elif (institute == 'manchester' or institute == 'oulu'):
+                group = 'dev'
+            elif (institute == 'idiap' or institute == 'brno'):
+                group = 'eval'
+            if verbose > 1:
+                print("  Adding client %d..." % int(client_id))
+            session.add(Client(int(client_id), group, gender, institute))
+            client_dict[client_id] = True
+
+        w = session_device.split('_')
+        session_id_from_dir = int(w[0])
+        device_from_dir = w[1]
+
+        session_id = int(v[1])
+        speech_type = v[2][0]
+        shot_id = v[2][1:3]
+        environment = v[3][0]
+        device = v[3][1]
+        if(device == '0'):
+            device = 'mobile'
+        elif(device == '1'):
+            device = 'laptop'
+        if device != device_from_dir:
+            error_msg = "File: %s -- Find device %s in directory of device %s!" % (
+                full_bname, device, device_from_dir)
+            raise RuntimeError(error_msg)
+        if session_id != session_id_from_dir:
+            error_msg = "File: %s -- Find session_id %d in directory of session_id %d!" % (
+                full_bname, session_id, session_id_from_dir)
+            raise RuntimeError(error_msg)
+        channel = int(v[4][0])
+
+        if verbose > 1:
+            print("    Adding file '%s'..." % full_bname)
+        session.add(File(int(client_id), full_bname, session_id,
+                         speech_type, shot_id, environment, device, channel))
+
+    client_dict = {}
+    if verbose:
+        print("Adding clients and files ...")
+    for location in filter(nodot, os.listdir(datadir)):
+        location_dir = os.path.join(datadir, location)
+        if os.path.isdir(location_dir):
+            for client_id in filter(nodot, os.listdir(location_dir)):
+                client_dir = os.path.join(location_dir, client_id)
+                if os.path.isdir(client_dir):
+                    for session_device in filter(nodot, os.listdir(client_dir)):
+                        session_device_dir = os.path.join(
+                            client_dir, session_device)
+                        if os.path.isdir(session_device_dir):
+                            for filename in filter(nodot, os.listdir(session_device_dir)):
+                                for ext in extensions:
+                                    if filename.endswith(ext):
+                                        add_file(sql_session, datadir, location, client_id, session_device, os.path.basename(
+                                            filename), verbose)
+
+
+def add_protocols(session, verbose):
+    """Adds protocols"""
+
+    # 1. DEFINITIONS
+    # Numbers in the lists correspond to session identifiers
+    protocol_definitions = {}
+
+    # Split male and female clients: list of (client_id, first_session_id) #
+    # few exceptions with 2 as first session
+    clients_male = [(1, 1), (2, 1), (4, 1), (8, 1), (11, 1), (12, 1), (15, 1), (16, 1), (17, 1), (19, 2),
+                    (21, 1), (23, 1), (24, 1), (25, 1), (26,
+                                                         1), (28, 1), (29, 1), (30, 1), (31, 1), (33, 1),
+                    (34, 1), (103, 1), (104, 1), (106, 1), (107,
+                                                            1), (108, 1), (109, 1), (111, 1), (112, 1), (114, 1),
+                    (115, 1), (116, 1), (117, 1), (119, 1), (120,
+                                                             1), (301, 1), (304, 1), (305, 1), (308, 1), (310, 1),
+                    (313, 1), (314, 1), (315, 1), (317, 1), (319,
+                                                             1), (416, 1), (417, 1), (418, 1), (419, 1), (420, 1),
+                    (421, 1), (422, 1), (423, 1), (424, 1), (425,
+                                                             1), (426, 1), (427, 1), (428, 1), (429, 1), (430, 1),
+                    (431, 1), (432, 1)]
+    clients_female = [(7, 2), (9, 1), (10, 1), (22, 1), (32, 1), (118, 1), (122, 1), (123, 1), (125, 1), (126, 1),
+                      (127, 1), (128, 1), (129, 1), (130, 1), (131,
+                                                               1), (133, 1), (302, 1), (303, 1), (306, 1), (307, 1),
+                      (309, 1), (311, 1), (320, 1), (401, 1), (402,
+                                                               1), (403, 1), (404, 1), (405, 2), (406, 1), (407, 1),
+                      (408, 1), (409, 1), (410, 1), (411, 1), (412, 1), (413, 1), (415, 1), (433, 1)]
+    train_mobile = ['mobile']
+    train_all = None
+    enroll_laptop = [['laptop'], ['p']]
+    enroll_mobile = [['mobile'], ['p']]
+    enroll_laptop_mobile = [['laptop', 'mobile'], ['p']]
+    probe = [['mobile'], ['r', 'f']]
+    gender_male = 'male'
+    gender_female = 'female'
+    protocol_definitions['mobile0-male'] = [clients_male,
+                                            train_mobile, enroll_mobile, probe, gender_male]
+    protocol_definitions['mobile0-female'] = [clients_female,
+                                              train_mobile, enroll_mobile, probe, gender_female]
+    protocol_definitions['mobile1-male'] = [clients_male,
+                                            train_all, enroll_mobile, probe, gender_male]
+    protocol_definitions['mobile1-female'] = [clients_female,
+                                              train_all, enroll_mobile, probe, gender_female]
+    protocol_definitions['laptop1-male'] = [clients_male,
+                                            train_all, enroll_laptop, probe, gender_male]
+    protocol_definitions['laptop1-female'] = [clients_female,
+                                              train_all, enroll_laptop, probe, gender_female]
+    protocol_definitions['laptop_mobile1-male'] = [clients_male,
+                                                   train_all, enroll_laptop_mobile, probe, gender_male]
+    protocol_definitions['laptop_mobile1-female'] = [clients_female,
+                                                     train_all, enroll_laptop_mobile, probe, gender_female]
+
+    # 2. ADDITIONS TO THE SQL DATABASE
+    protocolPurpose_list = [('world', 'train'), ('dev', 'enroll'),
+                            ('dev', 'probe'), ('eval', 'enroll'), ('eval', 'probe')]
+    for proto in protocol_definitions:
+        p = Protocol(proto, protocol_definitions[proto][4])
+        # Add protocol
+        if verbose:
+            print("Adding protocol '%s'..." % (proto))
+        session.add(p)
+        session.flush()
+        session.refresh(p)
+
+        # Add protocol purposes
+        for key in range(len(protocolPurpose_list)):
+            purpose = protocolPurpose_list[key]
+            pu = ProtocolPurpose(p.id, purpose[0], purpose[1])
+            if verbose > 1:
+                print("  Adding protocol purpose ('%s','%s')..." %
+                      (purpose[0], purpose[1]))
+            session.add(pu)
+            session.flush()
+            session.refresh(pu)
+
+            # Add files attached with this protocol purpose
+            client_group = ""
+            device_list = []
+            speech_list = []
+            if(key == 0):
+                client_group = "world"
+            elif(key == 1 or key == 2):
+                client_group = "dev"
+            elif(key == 3 or key == 4):
+                client_group = "eval"
+            if(key == 0):
+                world_list = True
+                session_list_in = False
+                device_list = protocol_definitions[proto][1]
+            if(key == 1 or key == 3):
+                world_list = False
+                session_list_in = True
+                device_list = protocol_definitions[proto][2][0]
+                speech_list = protocol_definitions[proto][2][1]
+            elif(key == 2 or key == 4):
+                world_list = False
+                session_list_in = False
+                device_list = protocol_definitions[proto][3][0]
+                speech_list = protocol_definitions[proto][3][1]
+
+            # Adds 'protocol' files
+            # World set
+            if world_list:
+                q = session.query(File).join(Client).filter(
+                    Client.sgroup == 'world').order_by(File.id)
+                if device_list:
+                    q = q.filter(File.device.in_(device_list))
+                for k in q:
+                    if verbose > 1:
+                        print("    Adding protocol file '%s'..." % (k.path))
+                    pu.files.append(k)
+            # Dev/eval set
+            else:
+                for client in protocol_definitions[proto][0]:
+                    cid = client[0]  # client id
+                    sid = client[1]  # session id
+                    q = session.query(File).join(Client).\
+                        filter(Client.sgroup == client_group).filter(
+                            Client.id == cid)
+                    if session_list_in:
+                        q = q.filter(File.session_id == sid)
+                    else:
+                        q = q.filter(File.session_id != sid)
+                    if device_list:
+                        q = q.filter(File.device.in_(device_list))
+                    if speech_list:
+                        q = q.filter(File.speech_type.in_(speech_list))
+                    q = q.order_by(File.id)
+                    for k in q:
+                        if verbose > 1:
+                            print("    Adding protocol file '%s'..." %
+                                  (k.path))
+                        pu.files.append(k)
+
+        # Add protocol
+        speech_type = ['p', 'l', 'r', 'f']
+        mobile_only = False
+        if 'mobile0' in proto:
+            mobile_only = True
+        add_tmodels(session, p.id, mobile_only, speech_type, verbose)
+
+
+def create_tables(args):
+    """Creates all necessary tables (only to be used at the first time)"""
+
+    from bob.db.base.utils import create_engine_try_nolock
+
+    engine = create_engine_try_nolock(
+        args.type, args.files[0], echo=(args.verbose > 2))
+    Base.metadata.create_all(engine)
+
+# Driver API
+# ==========
+
+
+def create(args):
+    """Creates or re-creates this database"""
+
+    from bob.db.base.utils import session_try_nolock
+
+    dbfile = args.files[0]
+
+    if args.recreate:
+        if args.verbose and os.path.exists(dbfile):
+            print('unlinking %s...' % dbfile)
+        if os.path.exists(dbfile):
+            os.unlink(dbfile)
+
+    if not os.path.exists(os.path.dirname(dbfile)):
+        os.makedirs(os.path.dirname(dbfile))
+
+    # the real work...
+    create_tables(args)
+    s = session_try_nolock(args.type, args.files[0], echo=(args.verbose > 2))
+    add_files(s, args.datadir, args.extensions, args.verbose)
+    add_protocols(s, args.verbose)
+    s.commit()
+    s.close()
+
+
+def add_command(subparsers):
+    """Add specific subcommands that the action "create" can use"""
+
+    parser = subparsers.add_parser('create', help=create.__doc__)
+
+    parser.add_argument('-R', '--recreate', action='store_true',
+                        help="If set, I'll first erase the current database")
+    parser.add_argument('-v', '--verbose', action='count',
+                        help="Do SQL operations in a verbose way?")
+    parser.add_argument('-D', '--datadir', metavar='DIR', default=None,
+                        help="Change the relative path to the directory "
+                        "containing the data of the SWAN database.")
+    parser.add_argument('-E', '--extensions', type=str, nargs='+',
+                        default=['.mp4'], help="Change the extension of the "
+                        "SWAN files used to create the database.")
+    parser.set_defaults(func=create)  # action

bob/db/swan/driver.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+"""Commands the SWAN database can respond to.
+"""
+
+import os
+import sys
+
+from bob.db.base.driver import Interface as BaseInterface
+
+
+def dumplist(args):
+    """Dumps lists of files based on your criteria"""
+
+    from .query import Database
+    db = Database()
+
+    r = db.objects(
+        protocol=args.protocol,
+        purposes=args.purpose,
+        model_ids=args.client,
+        groups=args.group,
+        classes=args.sclass
+    )
+
+    output = sys.stdout
+    if args.selftest:
+        from bob.db.base.utils import null
+        output = null()
+
+    for f in r:
+        output.write('%s\n' % (f.make_path(args.directory, args.extension),))
+
+    return 0
+
+
+def checkfiles(args):
+    """Checks existence of files based on your criteria"""
+
+    from .query import Database
+    db = Database()
+
+    r = db.objects()
+
+    # go through all files, check if they are available on the filesystem
+    good = []
+    bad = []
+    for f in r:
+        if os.path.exists(f.make_path(args.directory, args.extension)):
+            good.append(f)
+        else:
+            bad.append(f)
+
+    # report
+    output = sys.stdout
+    if args.selftest:
+        from bob.db.base.utils import null
+        output = null()
+
+    if bad:
+        for f in bad:
+            output.write('Cannot find file "%s"\n' %
+                         (f.make_path(args.directory, args.extension),))
+        output.write('%d files (out of %d) were not found at "%s"\n' %
+                     (len(bad), len(r), args.directory))
+
+    return 0
+
+
+def reverse(args):
+    """Returns a list of file database identifiers given the path stems"""
+
+    from .query import Database
+    db = Database()
+
+    output = sys.stdout
+    if args.selftest:
+        from bob.db.base.utils import null
+        output = null()
+
+    r = db.reverse(args.path)
+    for f in r:
+        output.write('%d\n' % f.id)
+
+    if not r:
+        return 1
+
+    return 0
+
+
+def path(args):
+    """Returns a list of fully formed paths or stems given some file id"""
+
+    from .query import Database
+    db = Database()
+
+    output = sys.stdout
+    if args.selftest:
+        from bob.db.base.utils import null
+        output = null()
+
+    r = db.paths(args.id, prefix=args.directory, suffix=args.extension)
+    for path in r:
+        output.write('%s\n' % path)
+
+    if not r:
+        return 1
+
+    return 0
+
+
+class Interface(BaseInterface):
+
+    def name(self):
+        return 'swan'
+
+    def version(self):
+        import pkg_resources  # part of setuptools
+        return pkg_resources.require('bob.db.%s' % self.name())[0].version
+
+    def files(self):
+
+        from pkg_resources import resource_filename
+        raw_files = ('db.sql3',)
+        return [resource_filename(__name__, k) for k in raw_files]
+
+    def type(self):
+        return 'sqlite'
+
+    def add_commands(self, parser):
+
+        from . import __doc__ as docs
+
+        subparsers = self.setup_parser(parser,
+                                       "The SWAN database", docs)
+
+        import argparse
+        from .query import Database
+        db = Database()
+
+        # example: get the "create" action from a submodule
+        from .create import add_command as create_command
+        create_command(subparsers)
+
+        # the "dumplist" action
+        parser = subparsers.add_parser('dumplist', help=dumplist.__doc__)
+        parser.add_argument(
+            '-d', '--directory', help="if given, this path will be prepended "
+            "to every entry returned.")
+        parser.add_argument(
+            '-e', '--extension', help="if given, this extension will be "
+            "appended to every entry returned.")
+        parser.add_argument(
+            '-p', '--protocol', help="if given, limits the check to a "
+            "particular subset of the data that corresponds to the given "
+            "protocol.",
+            choices=list(db.protocol_names()).extend(['male', 'female']) if
+            db.is_valid() else ())
+        parser.add_argument(
+            '-u',
+            '--purpose',
+            help="if given, this value will limit the output files to those "
+            "designed for the given purposes.",
+            choices=db.purposes() if db.is_valid() else ())
+        parser.add_argument(
+            '-C',
+            '--client',
+            type=int,
+            help="if given, limits the dump to a particular client.",
+            choices=db.model_ids() if db.is_valid() else ())
+        parser.add_argument(
+            '-g',
+            '--group',
+            help="if given, this value will limit the output files to those "
+            "belonging to a particular protocolar group.",
+            choices=db.groups() if db.is_valid() else ())
+        parser.add_argument(
+            '-c',
+            '--class',
+            dest="sclass",
+            help="if given, this value will limit the output files to those "
+            "belonging to the given classes.",
+            choices=(
+                'client',
+                'impostor'))
+        parser.add_argument(
+            '--self-test', dest="selftest",
+            action='store_true', help=argparse.SUPPRESS)
+        parser.set_defaults(func=dumplist)  # action
+
+        # the "checkfiles" action
+        parser = subparsers.add_parser('checkfiles', help=checkfiles.__doc__)
+        parser.add_argument(
+            '-d',
+            '--directory',
+            help="if given, this path will be prepended to every entry "
+            "returned.")
+        parser.add_argument(
+            '-e',
+            '--extension',
+            help="if given, this extension will be appended to every entry "
+            "returned.")
+        parser.add_argument('--self-test', dest="selftest",
+                            action='store_true', help=argparse.SUPPRESS)
+        parser.set_defaults(func=checkfiles)  # action
+
+        # adds the "reverse" command
+        parser = subparsers.add_parser('reverse', help=reverse.__doc__)
+        parser.add_argument(
+            'path',
+            nargs='+',
+            help="one or more path stems to look up. If you provide more than "
+            "one, files which cannot be reversed will be omitted from the "
+            "output.")
+        parser.add_argument('--self-test', dest="selftest",
+                            action='store_true', help=argparse.SUPPRESS)
+        parser.set_defaults(func=reverse)  # action
+
+        # adds the "path" command
+        parser = subparsers.add_parser('path', help=path.__doc__)
+        parser.add_argument(
+            '-d',
+            '--directory',
+            help="if given, this path will be prepended to every entry "
+            "returned.")
+        parser.add_argument(
+            '-e',
+            '--extension',
+            help="if given, this extension will be appended to every entry "
+            "returned.")
+        parser.add_argument(
+            'id',
+            nargs='+',
+            type=int,
+            help="one or more file ids to look up. If you provide more than "
+            "one, files which cannot be found will be omitted from the output."
+            " If you provide a single id to lookup, an error message will be "
+            "printed if the id does not exist in the database. The exit status"
+            " will be non-zero in such case.")
+        parser.add_argument('--self-test', dest="selftest",
+                            action='store_true', help=argparse.SUPPRESS)
+        parser.set_defaults(func=path)  # action

bob/db/swan/models.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+"""Table models and functionality for the SWAN database.
+"""
+
+import bob.db.base.utils
+from sqlalchemy import Table, Column, Integer, String, ForeignKey
+from bob.db.base.sqlalchemy_migration import Enum, relationship
+from sqlalchemy.orm import backref
+from sqlalchemy.ext.declarative import declarative_base
+
+import bob.db.base
+
+Base = declarative_base()
+
+protocolPurpose_file_association = Table(
+    'protocolPurpose_file_association', Base.metadata,
+    Column('protocolPurpose_id', Integer, ForeignKey(
+        'protocolPurpose.id')),
+    Column('file_id', Integer, ForeignKey('file.id')))
+
+
+class Client(Base):
+    """Database clients, marked by an integer identifier and the group they
+    belong to"""
+
+    __tablename__ = 'client'
+
+    # Key identifier for the client
+    id = Column(Integer, primary_key=True)
+    # the id of the client in the database
+    orig_id = Column(Integer)
+    # Gender to which the client belongs to
+    gender_choices = ('female', 'male')
+    gender = Column(Enum(*gender_choices))
+    # Group to which the client belongs to
+    group_choices = ('world', 'dev', 'eval')
+    sgroup = Column(Enum(*group_choices))  # do NOT use group (SQL keyword)
+    # Institute to which the client belongs to
+    institute_choices = ('IDIAP', 'MPH-FRA', 'MPH-IND', 'NTNU', 'UIO')
+    institute = Column(Enum(*institute_choices))
+
+    def __init__(self, orig_id, group, gender, institute):
+        self.orig_id = orig_id
+        self.sgroup = group
+        self.gender = gender
+        self.institute = institute
+
+    def __repr__(self):
+        return "Client(id={}, orig_id={}, group={}, institute={})".format(
+            self.id, self.sgroup, self.gender, self.institute)
+
+
+class File(Base, bob.db.base.File):
+    """Generic file container"""
+
+    __tablename__ = 'file'
+
+    # Key identifier for the file
+    id = Column(Integer, primary_key=True)
+    # Key identifier of the client associated with this file
+    client_id = Column(Integer, ForeignKey('client.id'))  # for SQL
+    # Unique path to this file inside the database
+    path = Column(String(100), unique=True)
+    # Identifier of the session
+    session_id = Column(Integer)
+    # Identifier of the device
+    device_choices = ('mobile', 'tablet')
+    device = Column(Enum(*device_choices))
+
+    # For Python: A direct link to the client object that this file belongs to
+    client = relationship("Client", backref=backref("files", order_by=id))
+
+    def __init__(self, client_id, path, session_id, device):
+        # call base class constructor
+        bob.db.base.File.__init__(self, path=path)
+
+        # fill the remaining bits of the file information
+        self.client_id = client_id
+        self.device = device
+
+
+class Protocol(Base):
+    """SWAN protocols"""
+
+    __tablename__ = 'protocol'
+
+    # Unique identifier for this protocol object
+    id = Column(Integer, primary_key=True)
+    # Name of the protocol associated with this object
+    name = Column(String(20), unique=True)
+
+    def __init__(self, name):
+        self.name = name
+
+    def __repr__(self):
+        return "Protocol({})".format(self.name)
+
+
+class ProtocolPurpose(Base):
+    """SWAN protocol purposes"""
+
+    __tablename__ = 'protocolPurpose'
+
+    # Unique identifier for this protocol purpose object
+    id = Column(Integer, primary_key=True)
+    # Id of the protocol associated with this protocol purpose object
+    protocol_id = Column(Integer, ForeignKey('protocol.id'))  # for SQL
+    # Group associated with this protocol purpose object
+    group_choices = Client.group_choices
+    sgroup = Column(Enum(*group_choices))
+    # Purpose associated with this protocol purpose object
+    purpose_choices = ('train', 'enroll', 'probe')
+    purpose = Column(Enum(*purpose_choices))
+
+    # For Python: A direct link to the Protocol object that this
+    # ProtocolPurpose belongs to
+    protocol = relationship(
+        "Protocol", backref=backref("purposes", order_by=id))
+    # For Python: A direct link to the File objects associated with this
+    # ProtcolPurpose
+    files = relationship("File", secondary=protocolPurpose_file_association,
+                         backref=backref("protocol_purposes", order_by=id))
+
+    def __init__(self, protocol_id, group, purpose):
+        self.protocol_id = protocol_id
+        self.sgroup = group
+        self.purpose = purpose
+
+    def __repr__(self):
+        return "ProtocolPurpose({}, {}, {})".format(
+            self.protocol.name, self.sgroup, self.purpose)

bob/db/swan/query.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+"""This module provides the Dataset interface allowing the user to query the
+MOBIO database in the most obvious ways.
+"""
+
+import six
+from .models import *
+from .driver import Interface
+from sqlalchemy import and_, not_
+
+import bob.db.base
+
+SQLITE_FILE = Interface().files()[0]
+
+
+class Database(bob.db.base.SQLiteDatabase):
+    """The dataset class opens and maintains a connection opened to the Database.
+
+    It provides many different ways to probe for the characteristics of the data
+    and for the data itself inside the database.
+    """
+
+    def __init__(self, original_directory=None, original_extension=None, annotation_directory=None, annotation_extension='.pos'):
+        # call base class constructors to open a session to the database
+        super(Database, self).__init__(SQLITE_FILE, File)
+
+        self.annotation_directory = annotation_directory
+        self.annotation_extension = annotation_extension
+
+    def groups(self, protocol=None):
+        """Returns the names of all registered groups"""
+
+        return ProtocolPurpose.group_choices
+
+    def genders(self):
+        """Returns the list of genders"""
+
+        return Client.gender_choices
+
+    def subworld_names(self):
+        """Returns all registered subworld names"""
+
+        self.assert_validity()
+        l = self.subworlds()
+        retval = [str(k.name) for k in l]
+        return retval
+
+    def subworlds(self):
+        """Returns the list of subworlds"""
+
+        return list(self.query(Subworld))
+
+    def has_subworld(self, name):
+        """Tells if a certain subworld is available"""
+
+        self.assert_validity()
+        return self.query(Subworld).filter(Subworld.name == name).count() != 0
+
+    def _replace_protocol_alias(self, protocol):
+        if protocol == 'male':
+            return 'mobile0-male'
+        elif protocol == 'female':
+            return 'mobile0-female'
+        else:
+            return protocol
+
+    def _replace_protocols_alias(self, protocol):
+        # print(protocol)
+        if protocol:
+            from six import string_types
+            if isinstance(protocol, string_types):
+                # print([self._replace_protocol_alias(protocol)])
+                return [self._replace_protocol_alias(protocol)]
+            else:
+                # print(list(set(self._replace_protocol_alias(k) for k in protocols)))
+                return list(set(self._replace_protocol_alias(k) for k in protocols))
+        else:
+            return None
+
+    def clients(self, protocol=None, groups=None, subworld=None, gender=None):
+        """Returns a list of Clients for the specific query by the user.
+
+        Keyword Parameters:
+
+        protocol
+            One of the MOBIO protocols ('mobile0-male', 'mobile0-female', 'mobile1-male', 'mobile1-female', \
+                'laptop1-male', 'laptop1-female', 'laptop_mobile1-male', 'laptop_mobile1-female')
+            'male'and 'female' are aliases for 'mobile0-male' and 'mobile0-female', respectively.
+
+        groups
+            The groups to which the clients belong ('dev', 'eval', 'world')
+            Please note that world data are protocol/gender independent
+
+        subworld
+            Specify a split of the world data ('onethird', 'twothirds', 'twothirds-subsampled')
+            In order to be considered, 'world' should be in groups and only one
+            split should be specified.
+
+        gender
+            The gender to consider ('male', 'female')
+
+        Returns: A list containing all the clients which have the given properties.
+        """
+
+        protocol = self._replace_protocols_alias(protocol)
+        protocol = self.check_parameters_for_validity(
+            protocol, "protocol", self.protocol_names(), [])
+        groups = self.check_parameters_for_validity(
+            groups, "group", self.groups(), self.groups())
+        subworld = self.check_parameters_for_validity(
+            subworld, "subworld", self.subworld_names(), [])
+        gender = self.check_parameters_for_validity(
+            gender, "gender", self.genders(), [])
+
+        # List of the clients
+        retval = []
+        if 'world' in groups:
+            q = self.query(Client).filter(Client.sgroup == 'world')
+            if subworld:
+                q = q.join((Subworld, Client.subworld)).filter(
+                    Subworld.name.in_(subworld))
+            if gender:
+                q = q.filter(Client.gender.in_(gender))
+            q = q.order_by(Client.id)
+            retval += list(q)
+
+        dev_eval = []
+        if 'dev' in groups:
+            dev_eval.append('dev')
+        if 'eval' in groups:
+            dev_eval.append('eval')
+        if dev_eval:
+            protocol_gender = None
+            if protocol:
+                q = self.query(Protocol).filter(
+                    Protocol.name.in_(protocol)).one()
+                protocol_gender = [q.gender]
+            q = self.query(Client).filter(Client.sgroup.in_(dev_eval))
+            if protocol_gender:
+                q = q.filter(Client.gender.in_(protocol_gender))
+            if gender:
+                q = q.filter(Client.gender.in_(gender))
+            q = q.order_by(Client.id)
+            retval += list(q)
+
+        return retval
+
+    def has_client_id(self, id):
+        """Returns True if we have a client with a certain integer identifier"""
+
+        return self.query(Client).filter(Client.id == id).count() != 0
+
+    def client(self, id):
+        """Returns the Client object in the database given a certain id. Raises
+        an error if that does not exist."""
+
+        return self.query(Client).filter(Client.id == id).one()
+
+    def tclients(self, protocol=None, groups=None, subworld='onethird', gender=None):
+        """Returns a set of T-Norm clients for the specific query by the user.
+
+        Keyword Parameters:
+
+        protocol
+            One of the MOBIO protocols ('mobile0-male', 'mobile0-female', 'mobile1-male', 'mobile1-female', \
+                'laptop1-male', 'laptop1-female', 'laptop_mobile1-male', 'laptop_mobile1-female')
+            'male'and 'female' are aliases for 'mobile0-male' and 'mobile0-female', respectively.
+
+        groups
+            Ignored.
+            For the MOBIO database, this has no impact as the T-Norm clients are coming from
+            the 'world' set, and are hence the same for both the 'dev' and 'eval' sets.
+
+        subworld
+            Specify a split of the world data ('onethird', 'twothirds', 'twothirds-subsampled')
+            Please note that 'onethird' is the default value.
+
+        gender
+            The gender to consider ('male', 'female')
+
+        Returns: A list containing all the T-norm clients belonging to the given group.
+        """
+
+        return self.clients(protocol, 'world', subworld, gender)
+
+    def zclients(self, protocol=None, groups=None, subworld='onethird', gender=None):
+        """Returns a set of Z-Norm clients for the specific query by the user.
+
+        Keyword Parameters:
+
+        protocol
+            One of the MOBIO protocols ('mobile0-male', 'mobile0-female', 'mobile1-male', 'mobile1-female', \
+                'laptop1-male', 'laptop1-female', 'laptop_mobile1-male', 'laptop_mobile1-female')
+            'male'and 'female' are aliases for 'mobile0-male' and 'mobile0-female', respectively.
+
+        groups
+            Ignored.
+            For the MOBIO database, this has no impact as the Z-Norm clients are coming from
+            the 'world' set, and are hence the same for both the 'dev' and 'eval' sets.
+
+        subworld
+            Specify a split of the world data ('onethird', 'twothirds', 'twothirds-subsampled')
+            Please note that 'onethird' is the default value.
+
+        gender
+            The gender to consider ('male', 'female')
+
+        Returns: A list containing all the Z-norm clients belonging to the given group.
+        """
+
+        return self.clients(protocol, 'world', subworld, gender)
+
+    def models(self, protocol=None, groups=None, subworld=None, gender=None):
+        """Returns a set of models for the specific query by the user.
+
+        Keyword Parameters:
+
+        protocol
+            One of the MOBIO protocols ('mobile0-male', 'mobile0-female', 'mobile1-male', 'mobile1-female', \
+                'laptop1-male', 'laptop1-female', 'laptop_mobile1-male', 'laptop_mobile1-female')
+            'male'and 'female' are aliases for 'mobile0-male' and 'mobile0-female', respectively.
+
+        groups
+            The groups to which the subjects attached to the models belong ('dev', 'eval', 'world')
+            Please note that world data are protocol/gender independent
+
+        subworld
+            Specify a split of the world data ('onethird', 'twothirds', 'twothirds-subsampled')
+            In order to be considered, 'world' should be in groups and only one
+            split should be specified.
+
+        gender
+            The gender to consider ('male', 'female')
+
+        Returns: A list containing all the models belonging to the given group.
+        """
+
+        return self.clients(protocol, groups, subworld, gender)
+
+    def model_ids(self, protocol=None, groups=None, subworld=None, gender=None):
+        """Returns a set of models ids for the specific query by the user.
+
+        Keyword Parameters:
+
+        protocol
+            One of the MOBIO protocols ('mobile0-male', 'mobile0-female', 'mobile1-male', 'mobile1-female', \
+                'laptop1-male', 'laptop1-female', 'laptop_mobile1-male', 'laptop_mobile1-female')
+            'male'and 'female' are aliases for 'mobile0-male' and 'mobile0-female', respectively.
+
+        groups
+            The groups to which the subjects attached to the models belong ('dev', 'eval', 'world')
+            Please note that world data are protocol/gender independent
+
+        subworld
+            Specify a split of the world data ('onethird', 'twothirds', 'twothirds-subsampled')
+            In order to be considered, 'world' should be in groups and only one
+            split should be specified.
+
+        gender
+            The gender to consider ('male', 'female')
+
+        Returns: A list containing the ids of all models belonging to the given group.
+        """
+
+        return [client.id for client in self.clients(protocol, groups, subworld, gender)]
+
+    def tmodels(self, protocol=None, groups=None, subworld='onethird', gender=None):
+        """Returns a set of T-Norm models for the specific query by the user.
+
+        Keyword Parameters:
+
+        protocol
+            One of the MOBIO protocols ('mobile0-male', 'mobile0-female', 'mobile1-male', 'mobile1-female', \
+                'laptop1-male', 'laptop1-female', 'laptop_mobile1-male', 'laptop_mobile1-female')
+            'male'and 'female' are aliases for 'mobile0-male' and 'mobile0-female', respectively.
+
+        groups
+            Ignored.
+            For the MOBIO database, this has no impact as the T-Norm clients are coming from
+            the 'world' set, and are hence the same for both the 'dev' and 'eval' sets.
+
+        subworld
+            Specify a split of the world data ('onethird', 'twothirds', 'twothirds-subsampled')
+            Please note that 'onethird' is the default value.
+
+        gender
+            The gender to consider ('male', 'female')
+
+        Returns: A list containing all the T-norm models belonging to the given group.
+        """
+
+        protocol = self._replace_protocols_alias(protocol)
+        protocol = self.check_parameters_for_validity(
+            protocol, "protocol", self.protocol_names())
+        subworld = self.check_parameters_for_validity(
+            subworld, "subworld", self.subworld_names(), [])
+        gender = self.check_parameters_for_validity(
+            gender, "gender", self.genders(), [])
+
+        # List of the clients
+        q = self.query(TModel).join(Client).join(
+            Protocol).filter(Protocol.name.in_(protocol))
+        if subworld:
+            q = q.join((Subworld, Client.subworld)).filter(
+                Subworld.name.in_(subworld))
+        if gender:
+            q = q.filter(Client.gender.in_(gender))
+        q = q.order_by(TModel.id)
+        return list(q)
+
+    def tmodel_ids(self, protocol=None, groups=None, subworld='onethird', gender=None):
+        """Returns a list of ids of T-Norm models for the specific query by the user.
+
+        Keyword Parameters:
+
+        protocol
+            One of the MOBIO protocols ('mobile0-male', 'mobile0-female', 'mobile1-male', 'mobile1-female', \
+                'laptop1-male', 'laptop1-female', 'laptop_mobile1-male', 'laptop_mobile1-female')
+            'male'and 'female' are aliases for 'mobile0-male' and 'mobile0-female', respectively.
+
+        groups
+            Ignored.
+            For the MOBIO database, this has no impact as the T-Norm clients are coming from
+            the 'world' set, and are hence the same for both the 'dev' and 'eval' sets.
+
+        subworld
+            Specify a split of the world data ('onethird', 'twothirds', 'twothirds-subsampled')
+            Please note that 'onethird' is the default value.
+
+        gender
+            The gender to consider ('male', 'female')
+
+        Returns: A list containing the ids of all T-norm models belonging to the given group.
+        """
+        return [tmodel.mid for tmodel in self.tmodels(protocol, groups, subworld, gender)]
+
+    def get_client_id_from_model_id(self, model_id, **kwargs):
+        """Returns the client_id attached to the given model_id
+
+        Keyword Parameters:
+
+        model_id
+            The model_id to consider
+
+        Returns: The client_id attached to the given model_id
+        """
+        return model_id
+
+    def objects(self, protocol=None, purposes=None, model_ids=None,
+                groups=None, classes=None, subworld=None, gender=None, device=None):
+        """Returns a set of Files for the specific query by the user.
+
+        Keyword Parameters:
+
+        protocol
+            One of the MOBIO protocols ('mobile0-male', 'mobile0-female', 'mobile1-male', 'mobile1-female', \
+                'laptop1-male', 'laptop1-female', 'laptop_mobile1-male', 'laptop_mobile1-female')
+            'male'and 'female' are aliases for 'mobile0-male' and 'mobile0-female', respectively.
+
+        purposes
+            The purposes required to be retrieved ('enroll', 'probe') or a tuple
+            with several of them. If 'None' is given (this is the default), it is
+            considered the same as a tuple with all possible values. This field is
+            ignored for the data from the "world" group.
+
+        model_ids
+            Only retrieves the files for the provided list of model ids (claimed
+            client id).  If 'None' is given (this is the default), no filter over
+            the model_ids is performed.
+
+        groups
+            One of the groups ('dev', 'eval', 'world') or a tuple with several of them.
+            If 'None' is given (this is the default), it is considered the same as a
+            tuple with all possible values.
+
+        classes
+            The classes (types of accesses) to be retrieved ('client', 'impostor')
+            or a tuple with several of them. If 'None' is given (this is the
+            default), it is considered the same as a tuple with all possible values.
+
+        subworld
+            Specify a split of the world data ('onethird', 'twothirds', 'twothirds-subsampled')
+            In order to be considered, "world" should be in groups and only one
+            split should be specified.
+
+        gender
+            The gender to consider ('male', 'female')
+
+        device
+            The device to consider ('laptop', 'mobile')
+
+        Returns: A set of Files with the given properties.
+        """
+
+        protocol = self._replace_protocols_alias(protocol)
+        protocol = self.check_parameters_for_validity(
+            protocol, "protocol", self.protocol_names())
+        purposes = self.check_parameters_for_validity(
+            purposes, "purpose", self.purposes())
+        groups = self.check_parameters_for_validity(
+            groups, "group", self.groups())
+        classes = self.check_parameters_for_validity(
+            classes, "class", ('client', 'impostor'))
+        subworld = self.check_parameters_for_validity(
+            subworld, "subworld", self.subworld_names(), [])
+        gender = self.check_parameters_for_validity(
+            gender, "gender", self.genders(), [])
+        device = self.check_parameters_for_validity(
+            device, "device", File.device_choices, [])
+
+        import collections
+        if(model_ids is None):
+            model_ids = ()
+        elif not isinstance(model_ids, collections.Iterable):
+            model_ids = (model_ids,)
+
+        # Now query the database
+        retval = []
+        if 'world' in groups and 'train' in purposes:
+            q = self.query(File).join(Client).filter(Client.sgroup == 'world').join((ProtocolPurpose, File.protocol_purposes)).join(Protocol).\
+                filter(and_(Protocol.name.in_(protocol),
+                            ProtocolPurpose.sgroup == 'world'))
+            if subworld:
+                q = q.join((Subworld, File.subworld)).filter(
+                    Subworld.name.in_(subworld))
+            if gender:
+                q = q.filter(Client.gender.in_(gender))
+            if device:
+                q = q.filter(File.device.in_(device))
+            if model_ids:
+                q = q.filter(File.client_id.in_(model_ids))
+            q = q.order_by(File.client_id, File.session_id,
+                           File.speech_type, File.shot_id, File.device)
+            retval += list(q)
+
+        if ('dev' in groups or 'eval' in groups):
+            if('enroll' in purposes):
+                q = self.query(File).join(Client).join((ProtocolPurpose, File.protocol_purposes)).join(Protocol).\
+                    filter(and_(Protocol.name.in_(protocol), ProtocolPurpose.sgroup.in_(
+                        groups), ProtocolPurpose.purpose == 'enroll'))
+                if gender:
+                    q = q.filter(Client.gender.in_(gender))
+                if device:
+                    q = q.filter(File.device.in_(device))
+                if model_ids:
+                    q = q.filter(Client.id.in_(model_ids))
+                q = q.order_by(File.client_id, File.session_id,
+                               File.speech_type, File.shot_id, File.device)
+                retval += list(q)
+
+            if('probe' in purposes):
+                if('client' in classes):
+                    q = self.query(File).join(Client).join((ProtocolPurpose, File.protocol_purposes)).join(Protocol).\
+                        filter(and_(Protocol.name.in_(protocol), ProtocolPurpose.sgroup.in_(
+                            groups), ProtocolPurpose.purpose == 'probe'))
+                    if gender:
+                        q = q.filter(Client.gender.in_(gender))
+                    if device:
+                        q = q.filter(File.device.in_(device))
+                    if model_ids:
+                        q = q.filter(Client.id.in_(model_ids))
+                    q = q.order_by(File.client_id, File.session_id,
+                                   File.speech_type, File.shot_id, File.device)
+                    retval += list(q)
+
+                if('impostor' in classes):
+                    q = self.query(File).join(Client).join((ProtocolPurpose, File.protocol_purposes)).join(Protocol).\
+                        filter(and_(Protocol.name.in_(protocol), ProtocolPurpose.sgroup.in_(
+                            groups), ProtocolPurpose.purpose == 'probe'))
+                    if gender:
+                        q = q.filter(Client.gender.in_(gender))
+                    if device:
+                        q = q.filter(File.device.in_(device))
+                    if len(model_ids) == 1:
+                        q = q.filter(not_(File.client_id.in_(model_ids)))
+                    q = q.order_by(File.client_id, File.session_id,
+                                   File.speech_type, File.shot_id, File.device)
+                    retval += list(q)
+
+        return list(set(retval))  # To remove duplicates
+
+    def tobjects(self, protocol=None, model_ids=None, groups=None, subworld='onethird', gender=None, speech_type=None, device=None):
+        """Returns a set of filenames for enrolling T-norm models for score
+             normalization.
+
+        Keyword Parameters:
+
+        protocol
+            One of the MOBIO protocols ('mobile0-male', 'mobile0-female', 'mobile1-male', 'mobile1-female', \
+                'laptop1-male', 'laptop1-female', 'laptop_mobile1-male', 'laptop_mobile1-female')
+            'male'and 'female' are aliases for 'mobile0-male' and 'mobile0-female', respectively.
+
+        model_ids
+            Only retrieves the files for the provided list of model ids.
+            If 'None' is given (this is the default), no filter over
+            the model_ids is performed.
+
+        groups
+            Ignored.
+            For the MOBIO database, this has no impact as the T-Norm clients are coming from
+            the 'world' set, and are hence the same for both the 'dev' and 'eval' sets.
+
+        subworld
+            Specify a split of the world data ('onethird', 'twothirds', 'twothirds-subsampled')
+            Please note that 'onethird' is the default value.
+
+        gender
+            The gender to consider ('male', 'female')
+
+        speech_type
+            The speech type to consider ('p', 'l', 'r', 'f')
+
+        device
+            The device choice to consider ('mobile', 'laptop')
+
+        Returns: A set of Files with the given properties.
+        """
+
+        protocol = self._replace_protocols_alias(protocol)
+        protocol = self.check_parameters_for_validity(
+            protocol, "protocol", self.protocol_names())
+        subworld = self.check_parameters_for_validity(
+            subworld, "subworld", self.subworld_names(), [])
+        gender = self.check_parameters_for_validity(
+            gender, "gender", self.genders(), [])
+
+        if(model_ids is None):
+            model_ids = ()
+        elif isinstance(model_ids, six.string_types):
+            model_ids = (model_ids,)
+
+        # Now query the database
+        q = self.query(File, Protocol).filter(
+            Protocol.name.in_(protocol)).join(Client)
+        if subworld:
+            q = q.join((Subworld, File.subworld)).filter(
+                Subworld.name.in_(subworld))
+        q = q.join((TModel, File.tmodels)).filter(
+            TModel.protocol_id == Protocol.id)
+        if model_ids:
+            q = q.filter(TModel.mid.in_(model_ids))
+        if gender:
+            q = q.filter(Client.gender.in_(gender))
+        if speech_type:
+            q = q.filter(File.speech_type.in_(speech_type))
+        if device:
+            q = q.filter(File.device.in_(device))
+        q = q.order_by(File.client_id, File.session_id,
+                       File.speech_type, File.shot_id, File.device)
+        retval = [v[0] for v in q]
+        return list(retval)
+
+    def zobjects(self, protocol=None, model_ids=None, groups=None, subworld='onethird', gender=None, speech_type=['r', 'f'], device=['mobile']):
+        """Returns a set of Files to perform Z-norm score normalization.
+
+        Keyword Parameters:
+
+        protocol
+            One of the MOBIO protocols ('mobile0-male', 'mobile0-female', 'mobile1-male', 'mobile1-female', \
+                'laptop1-male', 'laptop1-female', 'laptop_mobile1-male', 'laptop_mobile1-female')
+            'male'and 'female' are aliases for 'mobile0-male' and 'mobile0-female', respectively.
+
+        model_ids
+            Only retrieves the files for the provided list of model ids (claimed
+            client id).  If 'None' is given (this is the default), no filter over
+            the model_ids is performed.
+
+        groups
+            Ignored.
+            For the MOBIO database, this has no impact as the Z-Norm clients are coming from
+            the 'world' set, and are hence the same for both the 'dev' and 'eval' sets.
+
+        subworld
+            Specify a split of the world data ('onethird', 'twothirds', 'twothirds-subsampled')
+            Please note that 'onethird' is the default value.
+
+        gender
+            The gender to consider ('male', 'female')
+
+        speech_type
+            The speech type to consider ('p', 'l', 'r', 'f')
+
+        device
+            The device choice to consider ('mobile', 'laptop')
+
+        Returns: A set of Files with the given properties.
+        """
+
+        protocol = self._replace_protocols_alias(protocol)
+        protocol = self.check_parameters_for_validity(
+            protocol, "protocol", self.protocol_names())
+        groups = self.check_parameters_for_validity(
+            groups, "group", self.groups())
+        subworld = self.check_parameters_for_validity(
+            subworld, "subworld", self.subworld_names(), [])
+        gender = self.check_parameters_for_validity(
+            gender, "gender", self.genders(), [])
+        speech_type = self.check_parameters_for_validity(
+            speech_type, "speech_type", File.speech_type_choices)
+        device = self.check_parameters_for_validity(
+            device, "device", File.device_choices)
+
+        import collections
+        if(model_ids is None):
+            model_ids = ()
+        elif not isinstance(model_ids, collections.Iterable):
+            model_ids = (model_ids,)
+
+        # Now query the database
+        q = self.query(File).join(Client).filter(Client.sgroup == 'world').join((ProtocolPurpose, File.protocol_purposes)).join(Protocol).\
+            filter(and_(Protocol.name.in_(protocol),
+                        ProtocolPurpose.sgroup == 'world'))
+        if subworld:
+            q = q.join((Subworld, File.subworld)).filter(
+                Subworld.name.in_(subworld))
+        if gender:
+            q = q.filter(Client.gender.in_(gender))
+        if speech_type:
+            q = q.filter(File.speech_type.in_(speech_type))
+        if device:
+            q = q.filter(File.device.in_(device))
+        if model_ids:
+            q = q.filter(File.client_id.in_(model_ids))
+        q = q.order_by(File.client_id, File.session_id,
+                       File.speech_type, File.shot_id, File.device)
+        return list(q)
+
+    def annotations(self, file):
+        """Reads the annotations for the given file id from file and returns them in a dictionary.
+
+        If you don't have a copy of the annotation files, you can download them under http://www.idiap.ch/resource/biometric.
+
+        Keyword parameters:
+
+        file
+            The ``File`` object for which the annotations should be read.
+
+        Return value
+            The annotations as a dictionary: {'reye':(re_y,re_x), 'leye':(le_y,le_x)}
+        """
+        if self.annotation_directory is None:
+            return None
+
+        self.assert_validity()
+        annotation_file = file.make_path(
+            self.annotation_directory, self.annotation_extension)
+
+        # return the annotations as read from file
+        return bob.db.base.read_annotation_file(annotation_file, 'eyecenter')
+
+    def protocol_names(self):
+        """Returns all registered protocol names"""
+
+        l = self.protocols()
+        retval = [str(k.name) for k in l]
+        return retval
+
+    def protocols(self):
+        """Returns all registered protocols"""
+
+        return list(self.query(Protocol))
+
+    def has_protocol(self, name):
+        """Tells if a certain protocol is available"""
+
+        return self.query(Protocol).filter(Protocol.name == self._replace_protocol_alias(name)).count() != 0
+
+    def protocol(self, name):
+        """Returns the protocol object in the database given a certain name. Raises
+        an error if that does not exist."""
+
+        return self.query(Protocol).filter(Protocol.name == self._replace_protocol_alias(name)).one()
+
+    def protocol_purposes(self):
+        """Returns all registered protocol purposes"""
+
+        return list(self.query(ProtocolPurpose))
+
+    def purposes(self):
+        """Returns the list of allowed purposes"""
+
+        return ProtocolPurpose.purpose_choices

bob/db/swan/script/generate_filelist.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+"""Prints the version of bob and exits
+"""
+
+def main():
+  """Main routine, called by the script that gets the configuration of bob.blitz"""
+
+  import bob.blitz
+  print (bob.blitz.get_config())
+  return 0
+

bob/db/swan/test.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+"""Test Units
+"""

buildout.cfg

+; vim: set fileencoding=utf-8 :
+; Fri Dec 23 13:58:08 CET 2016
+
+[buildout]
+parts = scripts
+develop = .
+eggs = bob.db.swan
+extensions = bob.buildout
+newest = false
+verbose = true
+
+[scripts]
+recipe = bob.buildout:scripts
+dependent-scripts = true

doc/conf.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+import os
+import sys
+import glob
+import pkg_resources
+
+
+# -- General configuration -----------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+needs_sphinx = '1.3'
+
+# Add any Sphinx extension module names here, as strings. They can be extensions
+# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
+extensions = [
+    'sphinx.ext.todo',
+    'sphinx.ext.coverage',
+    'sphinx.ext.ifconfig',
+    'sphinx.ext.autodoc',
+    'sphinx.ext.autosummary',
+    'sphinx.ext.doctest',
+    'sphinx.ext.graphviz',
+    'sphinx.ext.intersphinx',
+    'sphinx.ext.napoleon',
+    'sphinx.ext.viewcode',
+    #'matplotlib.sphinxext.plot_directive'
+    ]
+
+import sphinx
+if sphinx.__version__ >= "1.4.1":
+    extensions.append('sphinx.ext.imgmath')
+    imgmath_image_format = 'svg'
+else:
+    extensions.append('sphinx.ext.pngmath')
+
+# Be picky about warnings
+nitpicky = True
+
+# Ignores stuff we can't easily resolve on other project's sphinx manuals
+nitpick_ignore = []
+
+# Allows the user to override warnings from a separate file
+if os.path.exists('nitpick-exceptions.txt'):
+    for line in open('nitpick-exceptions.txt'):
+        if line.strip() == "" or line.startswith("#"):
+            continue
+        dtype, target = line.split(None, 1)
+        target = target.strip()
+        try: # python 2.x
+            target = unicode(target)
+        except NameError:
+            pass
+        nitpick_ignore.append((dtype, target))
+
+# Always includes todos
+todo_include_todos = True
+
+# Generates auto-summary automatically
+autosummary_generate = True
+
+# Create numbers on figures with captions
+numfig = True
+
+# If we are on OSX, the 'dvipng' path maybe different
+dvipng_osx = '/opt/local/libexec/texlive/binaries/dvipng'
+if os.path.exists(dvipng_osx): pngmath_dvipng = dvipng_osx
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix of source filenames.
+source_suffix = '.rst'
+
+# The encoding of source files.
+#source_encoding = 'utf-8-sig'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'bob.db.swan'
+import time
+copyright = u'%s, Idiap Research Institute' % time.strftime('%Y')
+
+# Grab the setup entry
+distribution = pkg_resources.require(project)[0]
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = distribution.version
+# The full version, including alpha/beta/rc tags.
+release = distribution.version
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#today = ''
+# Else, today_fmt is used as the format for a strftime call.
+#today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+exclude_patterns = ['links.rst']
+
+# The reST default role (used for this markup: `text`) to use for all documents.
+#default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# A list of ignored prefixes for module index sorting.
+#modindex_common_prefix = []
+
+# Some variables which are useful for generated material
+project_variable = project.replace('.', '_')
+short_description = u'Example for using Bob inside a buildout project'
+owner = [u'Idiap Research Institute']
+
+
+# -- Options for HTML output ---------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+import sphinx_rtd_theme
+html_theme = 'sphinx_rtd_theme'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
+
+# The name for this set of Sphinx documents.  If None, it defaults to
+# "<project> v<release> documentation".
+#html_title = None
+
+# A shorter title for the navigation bar.  Default is the same as html_title.
+#html_short_title = project_variable
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+html_logo = 'img/logo.png'
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+html_favicon = 'img/favicon.ico'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+#html_static_path = ['_static']
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+#html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+#html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+#html_additional_pages = {}
+
+# If false, no module index is generated.
+#html_domain_indices = True
+
+# If false, no index is generated.
+#html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+#html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+#html_show_sourcelink = True
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+#html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+#html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it.  The value of this option must be the
+# base URL from which the finished HTML is served.
+#html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+#html_file_suffix = None
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = project_variable + u'_doc'
+
+
+# -- Post configuration --------------------------------------------------------
+
+# Included after all input documents
+rst_epilog = """
+.. |project| replace:: Bob
+.. |version| replace:: %s
+.. |current-year| date:: %%Y
+""" % (version,)
+
+# Default processing flags for sphinx
+autoclass_content = 'class'
+autodoc_member_order = 'bysource'
+autodoc_default_flags = [
+  'members',
+  'undoc-members',
+  'inherited-members',
+  'show-inheritance',
+  ]
+
+# For inter-documentation mapping:
+from bob.extension.utils import link_documentation, load_requirements
+sphinx_requirements = "extra-intersphinx.txt"
+if os.path.exists(sphinx_requirements):
+  intersphinx_mapping = link_documentation(
+      additional_packages=['python','numpy'] + \
+          load_requirements(sphinx_requirements)
+          )
+else:
+  intersphinx_mapping = link_documentation()
+
+
+# We want to remove all private (i.e. _. or __.__) members
+# that are not in the list of accepted functions
+accepted_private_functions = ['__array__']
+
+def member_function_test(app, what, name, obj, skip, options):
+  # test if we have a private function
+  if len(name) > 1 and name[0] == '_':
+    # test if this private function should be allowed
+    if name not in accepted_private_functions:
+      # omit privat functions that are not in the list of accepted private functions
+      return skip
+    else:
+      # test if the method is documented
+      if not hasattr(obj, '__doc__') or not obj.__doc__:
+        return skip
+  return False
+
+def setup(app):
+  app.connect('autodoc-skip-member', member_function_test)

doc/index.rst

+.. vim: set fileencoding=utf-8 :
+.. Andre Anjos <andre.anjos@idiap.ch>
+.. Mon 13 Aug 2012 12:36:40 CEST
+
+.. _bob.db.swan:
+
+=====================
+ Bob Example Project
+=====================
+
+
+Package Documentation
+---------------------
+
+.. automodule:: bob.db.swan
+
+