Skip to content
Snippets Groups Projects
Commit ce71659d authored by Amir MOHAMMADI's avatar Amir MOHAMMADI
Browse files

Initial commit

parents
No related branches found
No related tags found
No related merge requests found
Pipeline #
Showing with 2054 additions and 0 deletions
*~
*.swp
*.pyc
bin
eggs
parts
.installed.cfg
.mr.developer.cfg
*.egg-info
src
develop-eggs
sphinx
dist
conda
# This build file heavily uses template features from YAML so it is generic
# enough for any Bob project. Don't modify it unless you know what you're
# doing.
# Definition of our build pipeline
stages:
- build
- test
- docs
- wheels
- deploy
# ---------
# Templates
# ---------
# Template for the build stage
# Needs to run on all supported architectures, platforms and python versions
.build_template: &build_job
stage: build
before_script:
- git clean -ffdx
- mkdir _ci
- curl --silent "https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/install.sh" > _ci/install.sh
- chmod 755 _ci/install.sh
- ./_ci/install.sh _ci #updates
- ./_ci/before_build.sh
script:
- ./_ci/build.sh
after_script:
- ./_ci/after_build.sh
artifacts:
expire_in: 1 week
paths:
- _ci/
- dist/
- sphinx/
# Template for the test stage - re-installs from uploaded wheels
# Needs to run on all supported architectures, platforms and python versions
.test_template: &test_job
stage: test
before_script:
- ./_ci/install.sh _ci #updates
- ./_ci/before_test.sh
script:
- ./_ci/test.sh
after_script:
- ./_ci/after_test.sh
# Template for the wheel uploading stage
# Needs to run against one supported architecture, platform and python version
.wheels_template: &wheels_job
stage: wheels
environment: intranet
only:
- master
- /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags)
before_script:
- ./_ci/install.sh _ci #updates
- ./_ci/before_wheels.sh
script:
- ./_ci/wheels.sh
after_script:
- ./_ci/after_wheels.sh
# Template for (latest) documentation upload stage
# Only one real job needs to do this
.docs_template: &docs_job
stage: docs
environment: intranet
only:
- master
before_script:
- ./_ci/install.sh _ci #updates
- ./_ci/before_docs.sh
script:
- ./_ci/docs.sh
after_script:
- ./_ci/after_docs.sh
# Template for the deployment stage - re-installs from uploaded wheels
# Needs to run on a single architecture only
# Will deploy your package to PyPI and other required services
# Only runs for tags
.deploy_template: &deploy_job
stage: deploy
environment: internet
only:
- /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags)
except:
- branches
before_script:
- ./_ci/install.sh _ci #updates
- ./_ci/before_deploy.sh
script:
- ./_ci/deploy.sh
after_script:
- ./_ci/after_deploy.sh
# -------------
# Build Targets
# -------------
# Linux + Python 2.7: Builds, tests, uploads wheel and deploys (if needed)
build_linux_27:
<<: *build_job
variables: &linux_27_build_variables
PYTHON_VERSION: "2.7"
WHEEL_TAG: "py27"
tags:
- conda-linux
test_linux_27:
<<: *test_job
variables: *linux_27_build_variables
dependencies:
- build_linux_27
tags:
- conda-linux
wheels_linux_27:
<<: *wheels_job
variables: *linux_27_build_variables
dependencies:
- build_linux_27
tags:
- conda-linux
deploy_linux_27:
<<: *deploy_job
variables: *linux_27_build_variables
dependencies:
- build_linux_27
tags:
- conda-linux
# Linux + Python 3.5: Builds, tests and uploads wheel
build_linux_35:
<<: *build_job
variables: &linux_35_build_variables
PYTHON_VERSION: "3.5"
WHEEL_TAG: "py3"
tags:
- conda-linux
test_linux_35:
<<: *test_job
variables: *linux_35_build_variables
dependencies:
- build_linux_35
tags:
- conda-linux
wheels_linux_35:
<<: *wheels_job
variables: *linux_35_build_variables
dependencies:
- build_linux_35
tags:
- conda-linux
docs_linux_35:
<<: *docs_job
variables: *linux_35_build_variables
dependencies:
- build_linux_35
tags:
- conda-linux
# Linux + Python 3.6: Builds and tests
build_linux_36:
<<: *build_job
variables: &linux_36_build_variables
PYTHON_VERSION: "3.6"
WHEEL_TAG: "py3"
tags:
- conda-linux
test_linux_36:
<<: *test_job
variables: *linux_36_build_variables
dependencies:
- build_linux_36
tags:
- conda-linux
# Mac OSX + Python 2.7: Builds and tests
build_macosx_27:
<<: *build_job
variables: &macosx_27_build_variables
PYTHON_VERSION: "2.7"
WHEEL_TAG: "py27"
tags:
- conda-macosx
test_macosx_27:
<<: *test_job
variables: *macosx_27_build_variables
dependencies:
- build_macosx_27
tags:
- conda-macosx
# Mac OSX + Python 3.5: Builds and tests
build_macosx_35:
<<: *build_job
variables: &macosx_35_build_variables
PYTHON_VERSION: "3.5"
WHEEL_TAG: "py3"
tags:
- conda-macosx
test_macosx_35:
<<: *test_job
variables: *macosx_35_build_variables
dependencies:
- build_macosx_35
tags:
- conda-macosx
# Mac OSX + Python 3.6: Builds and tests
build_macosx_36:
<<: *build_job
variables: &macosx_36_build_variables
PYTHON_VERSION: "3.6"
WHEEL_TAG: "py3"
tags:
- conda-macosx
test_macosx_36:
<<: *test_job
variables: *macosx_36_build_variables
dependencies:
- build_macosx_36
tags:
- conda-macosx
LICENSE 0 → 100644
Copyright (c) 2017 Idiap Research Institute, http://www.idiap.ch/
Written by Amir Mohammadi <amir.mohammadi@idiap.ch>
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
3. Neither the name of the copyright holder nor the names of its contributors
may be used to endorse or promote products derived from this software without
specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
include README.rst bootstrap-buildout.py buildout.cfg COPYING version.txt requirements.txt
recursive-include doc *.py *.rst *.ico *.png
.. vim: set fileencoding=utf-8 :
.. Fri May 5 11:35:14 CEST 2017
.. image:: http://img.shields.io/badge/docs-stable-yellow.svg
:target: http://pythonhosted.org/bob.db.swan/index.html
.. image:: http://img.shields.io/badge/docs-latest-orange.svg
:target: https://www.idiap.ch/software/bob/docs/latest/bob/bob.db.swan/master/index.html
.. image:: https://gitlab.idiap.ch/bob/bob.db.swan/badges/master/build.svg
:target: https://gitlab.idiap.ch/bob/bob.db.swan/commits/master
.. image:: https://gitlab.idiap.ch/bob/bob.db.swan/badges/master/coverage.svg
:target: https://gitlab.idiap.ch/bob/bob.db.swan/commits/master
.. image:: https://img.shields.io/badge/gitlab-project-0000c0.svg
:target: https://gitlab.idiap.ch/bob/bob.db.swan
.. image:: http://img.shields.io/pypi/v/bob.db.swan.svg
:target: https://pypi.python.org/pypi/bob.db.swan
.. image:: http://img.shields.io/pypi/dm/bob.db.swan.svg
:target: https://pypi.python.org/pypi/bob.db.swan
==================================
SWAN Database Access API for Bob
==================================
This package is part of the signal-processing and machine learning toolbox
Bob_. It contains an interface for the evaluation protocol of the `SWAN`_
database. This package does not contain the original `SWAN`_ data files, which
need to be obtained through the link above.
Installation
------------
Follow our `installation`_ instructions. Then, to install this pacakge, run::
$ conda install bob.db.swan
Contact
-------
For questions or reporting issues to this software package, contact our
development `mailing list`_.
.. Place your references here:
.. _bob: https://www.idiap.ch/software/bob
.. _installation: https://www.idiap.ch/software/bob/install
.. _mailing list: https://www.idiap.ch/software/bob/discuss
.. _swan: http://www.idiap.ch/dataset/swan
# see https://docs.python.org/3/library/pkgutil.html
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
# see https://docs.python.org/3/library/pkgutil.html
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
def get_config():
"""Returns a string containing the configuration information.
"""
import bob.extension
return bob.extension.get_config(__name__)
# gets sphinx autodoc done right - don't remove it
__all__ = [_ for _ in dir() if not _.startswith('_')]
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
"""This script creates the SWAN database in a single pass.
"""
import os
from .models import Client, File, Protocol, ProtocolPurpose, Base
def nodot(item):
"""Can be used to ignore hidden files, starting with the . character."""
return item[0] != '.'
def add_clients_and_files(sql_session, datadir, extensions, verbose):
"""Add files to the SWAN database."""
def add_file(session, datadir, location, client_id_dir, session_device, basename, verbose):
"""Parse a single filename and add it to the list.
Also add a client entry if not already in the database."""
v = os.path.splitext(basename)[0].split('_')
bname = os.path.splitext(basename)[0]
full_bname = os.path.join(
location, client_id_dir, session_device, bname)
gender = ''
if v[0][0] == 'm':
gender = 'male'
if v[0][0] == 'f':
gender = 'female'
institute = int(v[0][1])
institute_dir = ''
if institute == 0:
institute = 'idiap'
institute_dir = 'idiap'
elif institute == 1:
institute = 'manchester'
institute_dir = 'uman'
elif institute == 2:
institute = 'surrey'
institute_dir = 'unis'
elif institute == 3:
institute = 'oulu'
institute_dir = 'uoulu'
elif institute == 4:
institute = 'brno'
institute_dir = 'but'
elif institute == 5:
institute = 'avignon'
institute_dir = 'lia'
if institute_dir != location:
error_msg = "File: %s -- Find location %s in directory of location %s!" % (
full_bname, location, institute_dir)
raise RuntimeError(error_msg)
client_id = v[0][1:4]
if v[0][0:4] != client_id_dir:
error_msg = "File: %s -- Find identity %s in directory of identity %s!" % (
full_bname, v[0][0:4], client_id)
raise RuntimeError(error_msg)
if not (client_id in client_dict):
if (institute == 'surrey' or institute == 'avignon'):
group = 'world'
elif (institute == 'manchester' or institute == 'oulu'):
group = 'dev'
elif (institute == 'idiap' or institute == 'brno'):
group = 'eval'
if verbose > 1:
print(" Adding client %d..." % int(client_id))
session.add(Client(int(client_id), group, gender, institute))
client_dict[client_id] = True
w = session_device.split('_')
session_id_from_dir = int(w[0])
device_from_dir = w[1]
session_id = int(v[1])
speech_type = v[2][0]
shot_id = v[2][1:3]
environment = v[3][0]
device = v[3][1]
if(device == '0'):
device = 'mobile'
elif(device == '1'):
device = 'laptop'
if device != device_from_dir:
error_msg = "File: %s -- Find device %s in directory of device %s!" % (
full_bname, device, device_from_dir)
raise RuntimeError(error_msg)
if session_id != session_id_from_dir:
error_msg = "File: %s -- Find session_id %d in directory of session_id %d!" % (
full_bname, session_id, session_id_from_dir)
raise RuntimeError(error_msg)
channel = int(v[4][0])
if verbose > 1:
print(" Adding file '%s'..." % full_bname)
session.add(File(int(client_id), full_bname, session_id,
speech_type, shot_id, environment, device, channel))
client_dict = {}
if verbose:
print("Adding clients and files ...")
for location in filter(nodot, os.listdir(datadir)):
location_dir = os.path.join(datadir, location)
if os.path.isdir(location_dir):
for client_id in filter(nodot, os.listdir(location_dir)):
client_dir = os.path.join(location_dir, client_id)
if os.path.isdir(client_dir):
for session_device in filter(nodot, os.listdir(client_dir)):
session_device_dir = os.path.join(
client_dir, session_device)
if os.path.isdir(session_device_dir):
for filename in filter(nodot, os.listdir(session_device_dir)):
for ext in extensions:
if filename.endswith(ext):
add_file(sql_session, datadir, location, client_id, session_device, os.path.basename(
filename), verbose)
def add_protocols(session, verbose):
"""Adds protocols"""
# 1. DEFINITIONS
# Numbers in the lists correspond to session identifiers
protocol_definitions = {}
# Split male and female clients: list of (client_id, first_session_id) #
# few exceptions with 2 as first session
clients_male = [(1, 1), (2, 1), (4, 1), (8, 1), (11, 1), (12, 1), (15, 1), (16, 1), (17, 1), (19, 2),
(21, 1), (23, 1), (24, 1), (25, 1), (26,
1), (28, 1), (29, 1), (30, 1), (31, 1), (33, 1),
(34, 1), (103, 1), (104, 1), (106, 1), (107,
1), (108, 1), (109, 1), (111, 1), (112, 1), (114, 1),
(115, 1), (116, 1), (117, 1), (119, 1), (120,
1), (301, 1), (304, 1), (305, 1), (308, 1), (310, 1),
(313, 1), (314, 1), (315, 1), (317, 1), (319,
1), (416, 1), (417, 1), (418, 1), (419, 1), (420, 1),
(421, 1), (422, 1), (423, 1), (424, 1), (425,
1), (426, 1), (427, 1), (428, 1), (429, 1), (430, 1),
(431, 1), (432, 1)]
clients_female = [(7, 2), (9, 1), (10, 1), (22, 1), (32, 1), (118, 1), (122, 1), (123, 1), (125, 1), (126, 1),
(127, 1), (128, 1), (129, 1), (130, 1), (131,
1), (133, 1), (302, 1), (303, 1), (306, 1), (307, 1),
(309, 1), (311, 1), (320, 1), (401, 1), (402,
1), (403, 1), (404, 1), (405, 2), (406, 1), (407, 1),
(408, 1), (409, 1), (410, 1), (411, 1), (412, 1), (413, 1), (415, 1), (433, 1)]
train_mobile = ['mobile']
train_all = None
enroll_laptop = [['laptop'], ['p']]
enroll_mobile = [['mobile'], ['p']]
enroll_laptop_mobile = [['laptop', 'mobile'], ['p']]
probe = [['mobile'], ['r', 'f']]
gender_male = 'male'
gender_female = 'female'
protocol_definitions['mobile0-male'] = [clients_male,
train_mobile, enroll_mobile, probe, gender_male]
protocol_definitions['mobile0-female'] = [clients_female,
train_mobile, enroll_mobile, probe, gender_female]
protocol_definitions['mobile1-male'] = [clients_male,
train_all, enroll_mobile, probe, gender_male]
protocol_definitions['mobile1-female'] = [clients_female,
train_all, enroll_mobile, probe, gender_female]
protocol_definitions['laptop1-male'] = [clients_male,
train_all, enroll_laptop, probe, gender_male]
protocol_definitions['laptop1-female'] = [clients_female,
train_all, enroll_laptop, probe, gender_female]
protocol_definitions['laptop_mobile1-male'] = [clients_male,
train_all, enroll_laptop_mobile, probe, gender_male]
protocol_definitions['laptop_mobile1-female'] = [clients_female,
train_all, enroll_laptop_mobile, probe, gender_female]
# 2. ADDITIONS TO THE SQL DATABASE
protocolPurpose_list = [('world', 'train'), ('dev', 'enroll'),
('dev', 'probe'), ('eval', 'enroll'), ('eval', 'probe')]
for proto in protocol_definitions:
p = Protocol(proto, protocol_definitions[proto][4])
# Add protocol
if verbose:
print("Adding protocol '%s'..." % (proto))
session.add(p)
session.flush()
session.refresh(p)
# Add protocol purposes
for key in range(len(protocolPurpose_list)):
purpose = protocolPurpose_list[key]
pu = ProtocolPurpose(p.id, purpose[0], purpose[1])
if verbose > 1:
print(" Adding protocol purpose ('%s','%s')..." %
(purpose[0], purpose[1]))
session.add(pu)
session.flush()
session.refresh(pu)
# Add files attached with this protocol purpose
client_group = ""
device_list = []
speech_list = []
if(key == 0):
client_group = "world"
elif(key == 1 or key == 2):
client_group = "dev"
elif(key == 3 or key == 4):
client_group = "eval"
if(key == 0):
world_list = True
session_list_in = False
device_list = protocol_definitions[proto][1]
if(key == 1 or key == 3):
world_list = False
session_list_in = True
device_list = protocol_definitions[proto][2][0]
speech_list = protocol_definitions[proto][2][1]
elif(key == 2 or key == 4):
world_list = False
session_list_in = False
device_list = protocol_definitions[proto][3][0]
speech_list = protocol_definitions[proto][3][1]
# Adds 'protocol' files
# World set
if world_list:
q = session.query(File).join(Client).filter(
Client.sgroup == 'world').order_by(File.id)
if device_list:
q = q.filter(File.device.in_(device_list))
for k in q:
if verbose > 1:
print(" Adding protocol file '%s'..." % (k.path))
pu.files.append(k)
# Dev/eval set
else:
for client in protocol_definitions[proto][0]:
cid = client[0] # client id
sid = client[1] # session id
q = session.query(File).join(Client).\
filter(Client.sgroup == client_group).filter(
Client.id == cid)
if session_list_in:
q = q.filter(File.session_id == sid)
else:
q = q.filter(File.session_id != sid)
if device_list:
q = q.filter(File.device.in_(device_list))
if speech_list:
q = q.filter(File.speech_type.in_(speech_list))
q = q.order_by(File.id)
for k in q:
if verbose > 1:
print(" Adding protocol file '%s'..." %
(k.path))
pu.files.append(k)
# Add protocol
speech_type = ['p', 'l', 'r', 'f']
mobile_only = False
if 'mobile0' in proto:
mobile_only = True
add_tmodels(session, p.id, mobile_only, speech_type, verbose)
def create_tables(args):
"""Creates all necessary tables (only to be used at the first time)"""
from bob.db.base.utils import create_engine_try_nolock
engine = create_engine_try_nolock(
args.type, args.files[0], echo=(args.verbose > 2))
Base.metadata.create_all(engine)
# Driver API
# ==========
def create(args):
"""Creates or re-creates this database"""
from bob.db.base.utils import session_try_nolock
dbfile = args.files[0]
if args.recreate:
if args.verbose and os.path.exists(dbfile):
print('unlinking %s...' % dbfile)
if os.path.exists(dbfile):
os.unlink(dbfile)
if not os.path.exists(os.path.dirname(dbfile)):
os.makedirs(os.path.dirname(dbfile))
# the real work...
create_tables(args)
s = session_try_nolock(args.type, args.files[0], echo=(args.verbose > 2))
add_files(s, args.datadir, args.extensions, args.verbose)
add_protocols(s, args.verbose)
s.commit()
s.close()
def add_command(subparsers):
"""Add specific subcommands that the action "create" can use"""
parser = subparsers.add_parser('create', help=create.__doc__)
parser.add_argument('-R', '--recreate', action='store_true',
help="If set, I'll first erase the current database")
parser.add_argument('-v', '--verbose', action='count',
help="Do SQL operations in a verbose way?")
parser.add_argument('-D', '--datadir', metavar='DIR', default=None,
help="Change the relative path to the directory "
"containing the data of the SWAN database.")
parser.add_argument('-E', '--extensions', type=str, nargs='+',
default=['.mp4'], help="Change the extension of the "
"SWAN files used to create the database.")
parser.set_defaults(func=create) # action
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
"""Commands the SWAN database can respond to.
"""
import os
import sys
from bob.db.base.driver import Interface as BaseInterface
def dumplist(args):
"""Dumps lists of files based on your criteria"""
from .query import Database
db = Database()
r = db.objects(
protocol=args.protocol,
purposes=args.purpose,
model_ids=args.client,
groups=args.group,
classes=args.sclass
)
output = sys.stdout
if args.selftest:
from bob.db.base.utils import null
output = null()
for f in r:
output.write('%s\n' % (f.make_path(args.directory, args.extension),))
return 0
def checkfiles(args):
"""Checks existence of files based on your criteria"""
from .query import Database
db = Database()
r = db.objects()
# go through all files, check if they are available on the filesystem
good = []
bad = []
for f in r:
if os.path.exists(f.make_path(args.directory, args.extension)):
good.append(f)
else:
bad.append(f)
# report
output = sys.stdout
if args.selftest:
from bob.db.base.utils import null
output = null()
if bad:
for f in bad:
output.write('Cannot find file "%s"\n' %
(f.make_path(args.directory, args.extension),))
output.write('%d files (out of %d) were not found at "%s"\n' %
(len(bad), len(r), args.directory))
return 0
def reverse(args):
"""Returns a list of file database identifiers given the path stems"""
from .query import Database
db = Database()
output = sys.stdout
if args.selftest:
from bob.db.base.utils import null
output = null()
r = db.reverse(args.path)
for f in r:
output.write('%d\n' % f.id)
if not r:
return 1
return 0
def path(args):
"""Returns a list of fully formed paths or stems given some file id"""
from .query import Database
db = Database()
output = sys.stdout
if args.selftest:
from bob.db.base.utils import null
output = null()
r = db.paths(args.id, prefix=args.directory, suffix=args.extension)
for path in r:
output.write('%s\n' % path)
if not r:
return 1
return 0
class Interface(BaseInterface):
def name(self):
return 'swan'
def version(self):
import pkg_resources # part of setuptools
return pkg_resources.require('bob.db.%s' % self.name())[0].version
def files(self):
from pkg_resources import resource_filename
raw_files = ('db.sql3',)
return [resource_filename(__name__, k) for k in raw_files]
def type(self):
return 'sqlite'
def add_commands(self, parser):
from . import __doc__ as docs
subparsers = self.setup_parser(parser,
"The SWAN database", docs)
import argparse
from .query import Database
db = Database()
# example: get the "create" action from a submodule
from .create import add_command as create_command
create_command(subparsers)
# the "dumplist" action
parser = subparsers.add_parser('dumplist', help=dumplist.__doc__)
parser.add_argument(
'-d', '--directory', help="if given, this path will be prepended "
"to every entry returned.")
parser.add_argument(
'-e', '--extension', help="if given, this extension will be "
"appended to every entry returned.")
parser.add_argument(
'-p', '--protocol', help="if given, limits the check to a "
"particular subset of the data that corresponds to the given "
"protocol.",
choices=list(db.protocol_names()).extend(['male', 'female']) if
db.is_valid() else ())
parser.add_argument(
'-u',
'--purpose',
help="if given, this value will limit the output files to those "
"designed for the given purposes.",
choices=db.purposes() if db.is_valid() else ())
parser.add_argument(
'-C',
'--client',
type=int,
help="if given, limits the dump to a particular client.",
choices=db.model_ids() if db.is_valid() else ())
parser.add_argument(
'-g',
'--group',
help="if given, this value will limit the output files to those "
"belonging to a particular protocolar group.",
choices=db.groups() if db.is_valid() else ())
parser.add_argument(
'-c',
'--class',
dest="sclass",
help="if given, this value will limit the output files to those "
"belonging to the given classes.",
choices=(
'client',
'impostor'))
parser.add_argument(
'--self-test', dest="selftest",
action='store_true', help=argparse.SUPPRESS)
parser.set_defaults(func=dumplist) # action
# the "checkfiles" action
parser = subparsers.add_parser('checkfiles', help=checkfiles.__doc__)
parser.add_argument(
'-d',
'--directory',
help="if given, this path will be prepended to every entry "
"returned.")
parser.add_argument(
'-e',
'--extension',
help="if given, this extension will be appended to every entry "
"returned.")
parser.add_argument('--self-test', dest="selftest",
action='store_true', help=argparse.SUPPRESS)
parser.set_defaults(func=checkfiles) # action
# adds the "reverse" command
parser = subparsers.add_parser('reverse', help=reverse.__doc__)
parser.add_argument(
'path',
nargs='+',
help="one or more path stems to look up. If you provide more than "
"one, files which cannot be reversed will be omitted from the "
"output.")
parser.add_argument('--self-test', dest="selftest",
action='store_true', help=argparse.SUPPRESS)
parser.set_defaults(func=reverse) # action
# adds the "path" command
parser = subparsers.add_parser('path', help=path.__doc__)
parser.add_argument(
'-d',
'--directory',
help="if given, this path will be prepended to every entry "
"returned.")
parser.add_argument(
'-e',
'--extension',
help="if given, this extension will be appended to every entry "
"returned.")
parser.add_argument(
'id',
nargs='+',
type=int,
help="one or more file ids to look up. If you provide more than "
"one, files which cannot be found will be omitted from the output."
" If you provide a single id to lookup, an error message will be "
"printed if the id does not exist in the database. The exit status"
" will be non-zero in such case.")
parser.add_argument('--self-test', dest="selftest",
action='store_true', help=argparse.SUPPRESS)
parser.set_defaults(func=path) # action
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
"""Table models and functionality for the SWAN database.
"""
import bob.db.base.utils
from sqlalchemy import Table, Column, Integer, String, ForeignKey
from bob.db.base.sqlalchemy_migration import Enum, relationship
from sqlalchemy.orm import backref
from sqlalchemy.ext.declarative import declarative_base
import bob.db.base
Base = declarative_base()
protocolPurpose_file_association = Table(
'protocolPurpose_file_association', Base.metadata,
Column('protocolPurpose_id', Integer, ForeignKey(
'protocolPurpose.id')),
Column('file_id', Integer, ForeignKey('file.id')))
class Client(Base):
"""Database clients, marked by an integer identifier and the group they
belong to"""
__tablename__ = 'client'
# Key identifier for the client
id = Column(Integer, primary_key=True)
# the id of the client in the database
orig_id = Column(Integer)
# Gender to which the client belongs to
gender_choices = ('female', 'male')
gender = Column(Enum(*gender_choices))
# Group to which the client belongs to
group_choices = ('world', 'dev', 'eval')
sgroup = Column(Enum(*group_choices)) # do NOT use group (SQL keyword)
# Institute to which the client belongs to
institute_choices = ('IDIAP', 'MPH-FRA', 'MPH-IND', 'NTNU', 'UIO')
institute = Column(Enum(*institute_choices))
def __init__(self, orig_id, group, gender, institute):
self.orig_id = orig_id
self.sgroup = group
self.gender = gender
self.institute = institute
def __repr__(self):
return "Client(id={}, orig_id={}, group={}, institute={})".format(
self.id, self.sgroup, self.gender, self.institute)
class File(Base, bob.db.base.File):
"""Generic file container"""
__tablename__ = 'file'
# Key identifier for the file
id = Column(Integer, primary_key=True)
# Key identifier of the client associated with this file
client_id = Column(Integer, ForeignKey('client.id')) # for SQL
# Unique path to this file inside the database
path = Column(String(100), unique=True)
# Identifier of the session
session_id = Column(Integer)
# Identifier of the device
device_choices = ('mobile', 'tablet')
device = Column(Enum(*device_choices))
# For Python: A direct link to the client object that this file belongs to
client = relationship("Client", backref=backref("files", order_by=id))
def __init__(self, client_id, path, session_id, device):
# call base class constructor
bob.db.base.File.__init__(self, path=path)
# fill the remaining bits of the file information
self.client_id = client_id
self.device = device
class Protocol(Base):
"""SWAN protocols"""
__tablename__ = 'protocol'
# Unique identifier for this protocol object
id = Column(Integer, primary_key=True)
# Name of the protocol associated with this object
name = Column(String(20), unique=True)
def __init__(self, name):
self.name = name
def __repr__(self):
return "Protocol({})".format(self.name)
class ProtocolPurpose(Base):
"""SWAN protocol purposes"""
__tablename__ = 'protocolPurpose'
# Unique identifier for this protocol purpose object
id = Column(Integer, primary_key=True)
# Id of the protocol associated with this protocol purpose object
protocol_id = Column(Integer, ForeignKey('protocol.id')) # for SQL
# Group associated with this protocol purpose object
group_choices = Client.group_choices
sgroup = Column(Enum(*group_choices))
# Purpose associated with this protocol purpose object
purpose_choices = ('train', 'enroll', 'probe')
purpose = Column(Enum(*purpose_choices))
# For Python: A direct link to the Protocol object that this
# ProtocolPurpose belongs to
protocol = relationship(
"Protocol", backref=backref("purposes", order_by=id))
# For Python: A direct link to the File objects associated with this
# ProtcolPurpose
files = relationship("File", secondary=protocolPurpose_file_association,
backref=backref("protocol_purposes", order_by=id))
def __init__(self, protocol_id, group, purpose):
self.protocol_id = protocol_id
self.sgroup = group
self.purpose = purpose
def __repr__(self):
return "ProtocolPurpose({}, {}, {})".format(
self.protocol.name, self.sgroup, self.purpose)
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
"""This module provides the Dataset interface allowing the user to query the
MOBIO database in the most obvious ways.
"""
import six
from .models import *
from .driver import Interface
from sqlalchemy import and_, not_
import bob.db.base
SQLITE_FILE = Interface().files()[0]
class Database(bob.db.base.SQLiteDatabase):
"""The dataset class opens and maintains a connection opened to the Database.
It provides many different ways to probe for the characteristics of the data
and for the data itself inside the database.
"""
def __init__(self, original_directory=None, original_extension=None, annotation_directory=None, annotation_extension='.pos'):
# call base class constructors to open a session to the database
super(Database, self).__init__(SQLITE_FILE, File)
self.annotation_directory = annotation_directory
self.annotation_extension = annotation_extension
def groups(self, protocol=None):
"""Returns the names of all registered groups"""
return ProtocolPurpose.group_choices
def genders(self):
"""Returns the list of genders"""
return Client.gender_choices
def subworld_names(self):
"""Returns all registered subworld names"""
self.assert_validity()
l = self.subworlds()
retval = [str(k.name) for k in l]
return retval
def subworlds(self):
"""Returns the list of subworlds"""
return list(self.query(Subworld))
def has_subworld(self, name):
"""Tells if a certain subworld is available"""
self.assert_validity()
return self.query(Subworld).filter(Subworld.name == name).count() != 0
def _replace_protocol_alias(self, protocol):
if protocol == 'male':
return 'mobile0-male'
elif protocol == 'female':
return 'mobile0-female'
else:
return protocol
def _replace_protocols_alias(self, protocol):
# print(protocol)
if protocol:
from six import string_types
if isinstance(protocol, string_types):
# print([self._replace_protocol_alias(protocol)])
return [self._replace_protocol_alias(protocol)]
else:
# print(list(set(self._replace_protocol_alias(k) for k in protocols)))
return list(set(self._replace_protocol_alias(k) for k in protocols))
else:
return None
def clients(self, protocol=None, groups=None, subworld=None, gender=None):
"""Returns a list of Clients for the specific query by the user.
Keyword Parameters:
protocol
One of the MOBIO protocols ('mobile0-male', 'mobile0-female', 'mobile1-male', 'mobile1-female', \
'laptop1-male', 'laptop1-female', 'laptop_mobile1-male', 'laptop_mobile1-female')
'male'and 'female' are aliases for 'mobile0-male' and 'mobile0-female', respectively.
groups
The groups to which the clients belong ('dev', 'eval', 'world')
Please note that world data are protocol/gender independent
subworld
Specify a split of the world data ('onethird', 'twothirds', 'twothirds-subsampled')
In order to be considered, 'world' should be in groups and only one
split should be specified.
gender
The gender to consider ('male', 'female')
Returns: A list containing all the clients which have the given properties.
"""
protocol = self._replace_protocols_alias(protocol)
protocol = self.check_parameters_for_validity(
protocol, "protocol", self.protocol_names(), [])
groups = self.check_parameters_for_validity(
groups, "group", self.groups(), self.groups())
subworld = self.check_parameters_for_validity(
subworld, "subworld", self.subworld_names(), [])
gender = self.check_parameters_for_validity(
gender, "gender", self.genders(), [])
# List of the clients
retval = []
if 'world' in groups:
q = self.query(Client).filter(Client.sgroup == 'world')
if subworld:
q = q.join((Subworld, Client.subworld)).filter(
Subworld.name.in_(subworld))
if gender:
q = q.filter(Client.gender.in_(gender))
q = q.order_by(Client.id)
retval += list(q)
dev_eval = []
if 'dev' in groups:
dev_eval.append('dev')
if 'eval' in groups:
dev_eval.append('eval')
if dev_eval:
protocol_gender = None
if protocol:
q = self.query(Protocol).filter(
Protocol.name.in_(protocol)).one()
protocol_gender = [q.gender]
q = self.query(Client).filter(Client.sgroup.in_(dev_eval))
if protocol_gender:
q = q.filter(Client.gender.in_(protocol_gender))
if gender:
q = q.filter(Client.gender.in_(gender))
q = q.order_by(Client.id)
retval += list(q)
return retval
def has_client_id(self, id):
"""Returns True if we have a client with a certain integer identifier"""
return self.query(Client).filter(Client.id == id).count() != 0
def client(self, id):
"""Returns the Client object in the database given a certain id. Raises
an error if that does not exist."""
return self.query(Client).filter(Client.id == id).one()
def tclients(self, protocol=None, groups=None, subworld='onethird', gender=None):
"""Returns a set of T-Norm clients for the specific query by the user.
Keyword Parameters:
protocol
One of the MOBIO protocols ('mobile0-male', 'mobile0-female', 'mobile1-male', 'mobile1-female', \
'laptop1-male', 'laptop1-female', 'laptop_mobile1-male', 'laptop_mobile1-female')
'male'and 'female' are aliases for 'mobile0-male' and 'mobile0-female', respectively.
groups
Ignored.
For the MOBIO database, this has no impact as the T-Norm clients are coming from
the 'world' set, and are hence the same for both the 'dev' and 'eval' sets.
subworld
Specify a split of the world data ('onethird', 'twothirds', 'twothirds-subsampled')
Please note that 'onethird' is the default value.
gender
The gender to consider ('male', 'female')
Returns: A list containing all the T-norm clients belonging to the given group.
"""
return self.clients(protocol, 'world', subworld, gender)
def zclients(self, protocol=None, groups=None, subworld='onethird', gender=None):
"""Returns a set of Z-Norm clients for the specific query by the user.
Keyword Parameters:
protocol
One of the MOBIO protocols ('mobile0-male', 'mobile0-female', 'mobile1-male', 'mobile1-female', \
'laptop1-male', 'laptop1-female', 'laptop_mobile1-male', 'laptop_mobile1-female')
'male'and 'female' are aliases for 'mobile0-male' and 'mobile0-female', respectively.
groups
Ignored.
For the MOBIO database, this has no impact as the Z-Norm clients are coming from
the 'world' set, and are hence the same for both the 'dev' and 'eval' sets.
subworld
Specify a split of the world data ('onethird', 'twothirds', 'twothirds-subsampled')
Please note that 'onethird' is the default value.
gender
The gender to consider ('male', 'female')
Returns: A list containing all the Z-norm clients belonging to the given group.
"""
return self.clients(protocol, 'world', subworld, gender)
def models(self, protocol=None, groups=None, subworld=None, gender=None):
"""Returns a set of models for the specific query by the user.
Keyword Parameters:
protocol
One of the MOBIO protocols ('mobile0-male', 'mobile0-female', 'mobile1-male', 'mobile1-female', \
'laptop1-male', 'laptop1-female', 'laptop_mobile1-male', 'laptop_mobile1-female')
'male'and 'female' are aliases for 'mobile0-male' and 'mobile0-female', respectively.
groups
The groups to which the subjects attached to the models belong ('dev', 'eval', 'world')
Please note that world data are protocol/gender independent
subworld
Specify a split of the world data ('onethird', 'twothirds', 'twothirds-subsampled')
In order to be considered, 'world' should be in groups and only one
split should be specified.
gender
The gender to consider ('male', 'female')
Returns: A list containing all the models belonging to the given group.
"""
return self.clients(protocol, groups, subworld, gender)
def model_ids(self, protocol=None, groups=None, subworld=None, gender=None):
"""Returns a set of models ids for the specific query by the user.
Keyword Parameters:
protocol
One of the MOBIO protocols ('mobile0-male', 'mobile0-female', 'mobile1-male', 'mobile1-female', \
'laptop1-male', 'laptop1-female', 'laptop_mobile1-male', 'laptop_mobile1-female')
'male'and 'female' are aliases for 'mobile0-male' and 'mobile0-female', respectively.
groups
The groups to which the subjects attached to the models belong ('dev', 'eval', 'world')
Please note that world data are protocol/gender independent
subworld
Specify a split of the world data ('onethird', 'twothirds', 'twothirds-subsampled')
In order to be considered, 'world' should be in groups and only one
split should be specified.
gender
The gender to consider ('male', 'female')
Returns: A list containing the ids of all models belonging to the given group.
"""
return [client.id for client in self.clients(protocol, groups, subworld, gender)]
def tmodels(self, protocol=None, groups=None, subworld='onethird', gender=None):
"""Returns a set of T-Norm models for the specific query by the user.
Keyword Parameters:
protocol
One of the MOBIO protocols ('mobile0-male', 'mobile0-female', 'mobile1-male', 'mobile1-female', \
'laptop1-male', 'laptop1-female', 'laptop_mobile1-male', 'laptop_mobile1-female')
'male'and 'female' are aliases for 'mobile0-male' and 'mobile0-female', respectively.
groups
Ignored.
For the MOBIO database, this has no impact as the T-Norm clients are coming from
the 'world' set, and are hence the same for both the 'dev' and 'eval' sets.
subworld
Specify a split of the world data ('onethird', 'twothirds', 'twothirds-subsampled')
Please note that 'onethird' is the default value.
gender
The gender to consider ('male', 'female')
Returns: A list containing all the T-norm models belonging to the given group.
"""
protocol = self._replace_protocols_alias(protocol)
protocol = self.check_parameters_for_validity(
protocol, "protocol", self.protocol_names())
subworld = self.check_parameters_for_validity(
subworld, "subworld", self.subworld_names(), [])
gender = self.check_parameters_for_validity(
gender, "gender", self.genders(), [])
# List of the clients
q = self.query(TModel).join(Client).join(
Protocol).filter(Protocol.name.in_(protocol))
if subworld:
q = q.join((Subworld, Client.subworld)).filter(
Subworld.name.in_(subworld))
if gender:
q = q.filter(Client.gender.in_(gender))
q = q.order_by(TModel.id)
return list(q)
def tmodel_ids(self, protocol=None, groups=None, subworld='onethird', gender=None):
"""Returns a list of ids of T-Norm models for the specific query by the user.
Keyword Parameters:
protocol
One of the MOBIO protocols ('mobile0-male', 'mobile0-female', 'mobile1-male', 'mobile1-female', \
'laptop1-male', 'laptop1-female', 'laptop_mobile1-male', 'laptop_mobile1-female')
'male'and 'female' are aliases for 'mobile0-male' and 'mobile0-female', respectively.
groups
Ignored.
For the MOBIO database, this has no impact as the T-Norm clients are coming from
the 'world' set, and are hence the same for both the 'dev' and 'eval' sets.
subworld
Specify a split of the world data ('onethird', 'twothirds', 'twothirds-subsampled')
Please note that 'onethird' is the default value.
gender
The gender to consider ('male', 'female')
Returns: A list containing the ids of all T-norm models belonging to the given group.
"""
return [tmodel.mid for tmodel in self.tmodels(protocol, groups, subworld, gender)]
def get_client_id_from_model_id(self, model_id, **kwargs):
"""Returns the client_id attached to the given model_id
Keyword Parameters:
model_id
The model_id to consider
Returns: The client_id attached to the given model_id
"""
return model_id
def objects(self, protocol=None, purposes=None, model_ids=None,
groups=None, classes=None, subworld=None, gender=None, device=None):
"""Returns a set of Files for the specific query by the user.
Keyword Parameters:
protocol
One of the MOBIO protocols ('mobile0-male', 'mobile0-female', 'mobile1-male', 'mobile1-female', \
'laptop1-male', 'laptop1-female', 'laptop_mobile1-male', 'laptop_mobile1-female')
'male'and 'female' are aliases for 'mobile0-male' and 'mobile0-female', respectively.
purposes
The purposes required to be retrieved ('enroll', 'probe') or a tuple
with several of them. If 'None' is given (this is the default), it is
considered the same as a tuple with all possible values. This field is
ignored for the data from the "world" group.
model_ids
Only retrieves the files for the provided list of model ids (claimed
client id). If 'None' is given (this is the default), no filter over
the model_ids is performed.
groups
One of the groups ('dev', 'eval', 'world') or a tuple with several of them.
If 'None' is given (this is the default), it is considered the same as a
tuple with all possible values.
classes
The classes (types of accesses) to be retrieved ('client', 'impostor')
or a tuple with several of them. If 'None' is given (this is the
default), it is considered the same as a tuple with all possible values.
subworld
Specify a split of the world data ('onethird', 'twothirds', 'twothirds-subsampled')
In order to be considered, "world" should be in groups and only one
split should be specified.
gender
The gender to consider ('male', 'female')
device
The device to consider ('laptop', 'mobile')
Returns: A set of Files with the given properties.
"""
protocol = self._replace_protocols_alias(protocol)
protocol = self.check_parameters_for_validity(
protocol, "protocol", self.protocol_names())
purposes = self.check_parameters_for_validity(
purposes, "purpose", self.purposes())
groups = self.check_parameters_for_validity(
groups, "group", self.groups())
classes = self.check_parameters_for_validity(
classes, "class", ('client', 'impostor'))
subworld = self.check_parameters_for_validity(
subworld, "subworld", self.subworld_names(), [])
gender = self.check_parameters_for_validity(
gender, "gender", self.genders(), [])
device = self.check_parameters_for_validity(
device, "device", File.device_choices, [])
import collections
if(model_ids is None):
model_ids = ()
elif not isinstance(model_ids, collections.Iterable):
model_ids = (model_ids,)
# Now query the database
retval = []
if 'world' in groups and 'train' in purposes:
q = self.query(File).join(Client).filter(Client.sgroup == 'world').join((ProtocolPurpose, File.protocol_purposes)).join(Protocol).\
filter(and_(Protocol.name.in_(protocol),
ProtocolPurpose.sgroup == 'world'))
if subworld:
q = q.join((Subworld, File.subworld)).filter(
Subworld.name.in_(subworld))
if gender:
q = q.filter(Client.gender.in_(gender))
if device:
q = q.filter(File.device.in_(device))
if model_ids:
q = q.filter(File.client_id.in_(model_ids))
q = q.order_by(File.client_id, File.session_id,
File.speech_type, File.shot_id, File.device)
retval += list(q)
if ('dev' in groups or 'eval' in groups):
if('enroll' in purposes):
q = self.query(File).join(Client).join((ProtocolPurpose, File.protocol_purposes)).join(Protocol).\
filter(and_(Protocol.name.in_(protocol), ProtocolPurpose.sgroup.in_(
groups), ProtocolPurpose.purpose == 'enroll'))
if gender:
q = q.filter(Client.gender.in_(gender))
if device:
q = q.filter(File.device.in_(device))
if model_ids:
q = q.filter(Client.id.in_(model_ids))
q = q.order_by(File.client_id, File.session_id,
File.speech_type, File.shot_id, File.device)
retval += list(q)
if('probe' in purposes):
if('client' in classes):
q = self.query(File).join(Client).join((ProtocolPurpose, File.protocol_purposes)).join(Protocol).\
filter(and_(Protocol.name.in_(protocol), ProtocolPurpose.sgroup.in_(
groups), ProtocolPurpose.purpose == 'probe'))
if gender:
q = q.filter(Client.gender.in_(gender))
if device:
q = q.filter(File.device.in_(device))
if model_ids:
q = q.filter(Client.id.in_(model_ids))
q = q.order_by(File.client_id, File.session_id,
File.speech_type, File.shot_id, File.device)
retval += list(q)
if('impostor' in classes):
q = self.query(File).join(Client).join((ProtocolPurpose, File.protocol_purposes)).join(Protocol).\
filter(and_(Protocol.name.in_(protocol), ProtocolPurpose.sgroup.in_(
groups), ProtocolPurpose.purpose == 'probe'))
if gender:
q = q.filter(Client.gender.in_(gender))
if device:
q = q.filter(File.device.in_(device))
if len(model_ids) == 1:
q = q.filter(not_(File.client_id.in_(model_ids)))
q = q.order_by(File.client_id, File.session_id,
File.speech_type, File.shot_id, File.device)
retval += list(q)
return list(set(retval)) # To remove duplicates
def tobjects(self, protocol=None, model_ids=None, groups=None, subworld='onethird', gender=None, speech_type=None, device=None):
"""Returns a set of filenames for enrolling T-norm models for score
normalization.
Keyword Parameters:
protocol
One of the MOBIO protocols ('mobile0-male', 'mobile0-female', 'mobile1-male', 'mobile1-female', \
'laptop1-male', 'laptop1-female', 'laptop_mobile1-male', 'laptop_mobile1-female')
'male'and 'female' are aliases for 'mobile0-male' and 'mobile0-female', respectively.
model_ids
Only retrieves the files for the provided list of model ids.
If 'None' is given (this is the default), no filter over
the model_ids is performed.
groups
Ignored.
For the MOBIO database, this has no impact as the T-Norm clients are coming from
the 'world' set, and are hence the same for both the 'dev' and 'eval' sets.
subworld
Specify a split of the world data ('onethird', 'twothirds', 'twothirds-subsampled')
Please note that 'onethird' is the default value.
gender
The gender to consider ('male', 'female')
speech_type
The speech type to consider ('p', 'l', 'r', 'f')
device
The device choice to consider ('mobile', 'laptop')
Returns: A set of Files with the given properties.
"""
protocol = self._replace_protocols_alias(protocol)
protocol = self.check_parameters_for_validity(
protocol, "protocol", self.protocol_names())
subworld = self.check_parameters_for_validity(
subworld, "subworld", self.subworld_names(), [])
gender = self.check_parameters_for_validity(
gender, "gender", self.genders(), [])
if(model_ids is None):
model_ids = ()
elif isinstance(model_ids, six.string_types):
model_ids = (model_ids,)
# Now query the database
q = self.query(File, Protocol).filter(
Protocol.name.in_(protocol)).join(Client)
if subworld:
q = q.join((Subworld, File.subworld)).filter(
Subworld.name.in_(subworld))
q = q.join((TModel, File.tmodels)).filter(
TModel.protocol_id == Protocol.id)
if model_ids:
q = q.filter(TModel.mid.in_(model_ids))
if gender:
q = q.filter(Client.gender.in_(gender))
if speech_type:
q = q.filter(File.speech_type.in_(speech_type))
if device:
q = q.filter(File.device.in_(device))
q = q.order_by(File.client_id, File.session_id,
File.speech_type, File.shot_id, File.device)
retval = [v[0] for v in q]
return list(retval)
def zobjects(self, protocol=None, model_ids=None, groups=None, subworld='onethird', gender=None, speech_type=['r', 'f'], device=['mobile']):
"""Returns a set of Files to perform Z-norm score normalization.
Keyword Parameters:
protocol
One of the MOBIO protocols ('mobile0-male', 'mobile0-female', 'mobile1-male', 'mobile1-female', \
'laptop1-male', 'laptop1-female', 'laptop_mobile1-male', 'laptop_mobile1-female')
'male'and 'female' are aliases for 'mobile0-male' and 'mobile0-female', respectively.
model_ids
Only retrieves the files for the provided list of model ids (claimed
client id). If 'None' is given (this is the default), no filter over
the model_ids is performed.
groups
Ignored.
For the MOBIO database, this has no impact as the Z-Norm clients are coming from
the 'world' set, and are hence the same for both the 'dev' and 'eval' sets.
subworld
Specify a split of the world data ('onethird', 'twothirds', 'twothirds-subsampled')
Please note that 'onethird' is the default value.
gender
The gender to consider ('male', 'female')
speech_type
The speech type to consider ('p', 'l', 'r', 'f')
device
The device choice to consider ('mobile', 'laptop')
Returns: A set of Files with the given properties.
"""
protocol = self._replace_protocols_alias(protocol)
protocol = self.check_parameters_for_validity(
protocol, "protocol", self.protocol_names())
groups = self.check_parameters_for_validity(
groups, "group", self.groups())
subworld = self.check_parameters_for_validity(
subworld, "subworld", self.subworld_names(), [])
gender = self.check_parameters_for_validity(
gender, "gender", self.genders(), [])
speech_type = self.check_parameters_for_validity(
speech_type, "speech_type", File.speech_type_choices)
device = self.check_parameters_for_validity(
device, "device", File.device_choices)
import collections
if(model_ids is None):
model_ids = ()
elif not isinstance(model_ids, collections.Iterable):
model_ids = (model_ids,)
# Now query the database
q = self.query(File).join(Client).filter(Client.sgroup == 'world').join((ProtocolPurpose, File.protocol_purposes)).join(Protocol).\
filter(and_(Protocol.name.in_(protocol),
ProtocolPurpose.sgroup == 'world'))
if subworld:
q = q.join((Subworld, File.subworld)).filter(
Subworld.name.in_(subworld))
if gender:
q = q.filter(Client.gender.in_(gender))
if speech_type:
q = q.filter(File.speech_type.in_(speech_type))
if device:
q = q.filter(File.device.in_(device))
if model_ids:
q = q.filter(File.client_id.in_(model_ids))
q = q.order_by(File.client_id, File.session_id,
File.speech_type, File.shot_id, File.device)
return list(q)
def annotations(self, file):
"""Reads the annotations for the given file id from file and returns them in a dictionary.
If you don't have a copy of the annotation files, you can download them under http://www.idiap.ch/resource/biometric.
Keyword parameters:
file
The ``File`` object for which the annotations should be read.
Return value
The annotations as a dictionary: {'reye':(re_y,re_x), 'leye':(le_y,le_x)}
"""
if self.annotation_directory is None:
return None
self.assert_validity()
annotation_file = file.make_path(
self.annotation_directory, self.annotation_extension)
# return the annotations as read from file
return bob.db.base.read_annotation_file(annotation_file, 'eyecenter')
def protocol_names(self):
"""Returns all registered protocol names"""
l = self.protocols()
retval = [str(k.name) for k in l]
return retval
def protocols(self):
"""Returns all registered protocols"""
return list(self.query(Protocol))
def has_protocol(self, name):
"""Tells if a certain protocol is available"""
return self.query(Protocol).filter(Protocol.name == self._replace_protocol_alias(name)).count() != 0
def protocol(self, name):
"""Returns the protocol object in the database given a certain name. Raises
an error if that does not exist."""
return self.query(Protocol).filter(Protocol.name == self._replace_protocol_alias(name)).one()
def protocol_purposes(self):
"""Returns all registered protocol purposes"""
return list(self.query(ProtocolPurpose))
def purposes(self):
"""Returns the list of allowed purposes"""
return ProtocolPurpose.purpose_choices
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
"""Prints the version of bob and exits
"""
def main():
"""Main routine, called by the script that gets the configuration of bob.blitz"""
import bob.blitz
print (bob.blitz.get_config())
return 0
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
"""Test Units
"""
; vim: set fileencoding=utf-8 :
; Fri Dec 23 13:58:08 CET 2016
[buildout]
parts = scripts
develop = .
eggs = bob.db.swan
extensions = bob.buildout
newest = false
verbose = true
[scripts]
recipe = bob.buildout:scripts
dependent-scripts = true
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
import os
import sys
import glob
import pkg_resources
# -- General configuration -----------------------------------------------------
# If your documentation needs a minimal Sphinx version, state it here.
needs_sphinx = '1.3'
# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = [
'sphinx.ext.todo',
'sphinx.ext.coverage',
'sphinx.ext.ifconfig',
'sphinx.ext.autodoc',
'sphinx.ext.autosummary',
'sphinx.ext.doctest',
'sphinx.ext.graphviz',
'sphinx.ext.intersphinx',
'sphinx.ext.napoleon',
'sphinx.ext.viewcode',
#'matplotlib.sphinxext.plot_directive'
]
import sphinx
if sphinx.__version__ >= "1.4.1":
extensions.append('sphinx.ext.imgmath')
imgmath_image_format = 'svg'
else:
extensions.append('sphinx.ext.pngmath')
# Be picky about warnings
nitpicky = True
# Ignores stuff we can't easily resolve on other project's sphinx manuals
nitpick_ignore = []
# Allows the user to override warnings from a separate file
if os.path.exists('nitpick-exceptions.txt'):
for line in open('nitpick-exceptions.txt'):
if line.strip() == "" or line.startswith("#"):
continue
dtype, target = line.split(None, 1)
target = target.strip()
try: # python 2.x
target = unicode(target)
except NameError:
pass
nitpick_ignore.append((dtype, target))
# Always includes todos
todo_include_todos = True
# Generates auto-summary automatically
autosummary_generate = True
# Create numbers on figures with captions
numfig = True
# If we are on OSX, the 'dvipng' path maybe different
dvipng_osx = '/opt/local/libexec/texlive/binaries/dvipng'
if os.path.exists(dvipng_osx): pngmath_dvipng = dvipng_osx
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
# The suffix of source filenames.
source_suffix = '.rst'
# The encoding of source files.
#source_encoding = 'utf-8-sig'
# The master toctree document.
master_doc = 'index'
# General information about the project.
project = u'bob.db.swan'
import time
copyright = u'%s, Idiap Research Institute' % time.strftime('%Y')
# Grab the setup entry
distribution = pkg_resources.require(project)[0]
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = distribution.version
# The full version, including alpha/beta/rc tags.
release = distribution.version
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None
# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['links.rst']
# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None
# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True
# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True
# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'
# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []
# Some variables which are useful for generated material
project_variable = project.replace('.', '_')
short_description = u'Example for using Bob inside a buildout project'
owner = [u'Idiap Research Institute']
# -- Options for HTML output ---------------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
import sphinx_rtd_theme
html_theme = 'sphinx_rtd_theme'
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}
# Add any paths that contain custom themes here, relative to this directory.
html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
# The name for this set of Sphinx documents. If None, it defaults to
# "<project> v<release> documentation".
#html_title = None
# A shorter title for the navigation bar. Default is the same as html_title.
#html_short_title = project_variable
# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
html_logo = 'img/logo.png'
# The name of an image file (within the static path) to use as favicon of the
# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
html_favicon = 'img/favicon.ico'
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
#html_static_path = ['_static']
# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'
# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True
# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}
# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}
# If false, no module index is generated.
#html_domain_indices = True
# If false, no index is generated.
#html_use_index = True
# If true, the index is split into individual pages for each letter.
#html_split_index = False
# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True
# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True
# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True
# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it. The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''
# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None
# Output file base name for HTML help builder.
htmlhelp_basename = project_variable + u'_doc'
# -- Post configuration --------------------------------------------------------
# Included after all input documents
rst_epilog = """
.. |project| replace:: Bob
.. |version| replace:: %s
.. |current-year| date:: %%Y
""" % (version,)
# Default processing flags for sphinx
autoclass_content = 'class'
autodoc_member_order = 'bysource'
autodoc_default_flags = [
'members',
'undoc-members',
'inherited-members',
'show-inheritance',
]
# For inter-documentation mapping:
from bob.extension.utils import link_documentation, load_requirements
sphinx_requirements = "extra-intersphinx.txt"
if os.path.exists(sphinx_requirements):
intersphinx_mapping = link_documentation(
additional_packages=['python','numpy'] + \
load_requirements(sphinx_requirements)
)
else:
intersphinx_mapping = link_documentation()
# We want to remove all private (i.e. _. or __.__) members
# that are not in the list of accepted functions
accepted_private_functions = ['__array__']
def member_function_test(app, what, name, obj, skip, options):
# test if we have a private function
if len(name) > 1 and name[0] == '_':
# test if this private function should be allowed
if name not in accepted_private_functions:
# omit privat functions that are not in the list of accepted private functions
return skip
else:
# test if the method is documented
if not hasattr(obj, '__doc__') or not obj.__doc__:
return skip
return False
def setup(app):
app.connect('autodoc-skip-member', member_function_test)
doc/img/favicon.ico

4.19 KiB

doc/img/logo.png

6.12 KiB

.. vim: set fileencoding=utf-8 :
.. Andre Anjos <andre.anjos@idiap.ch>
.. Mon 13 Aug 2012 12:36:40 CEST
.. _bob.db.swan:
=====================
Bob Example Project
=====================
Package Documentation
---------------------
.. automodule:: bob.db.swan
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment