Commit ce71659d authored by Amir MOHAMMADI's avatar Amir MOHAMMADI
Browse files

Initial commit

parents
Pipeline #8854 failed with stages
in 30 minutes and 31 seconds
*~
*.swp
*.pyc
bin
eggs
parts
.installed.cfg
.mr.developer.cfg
*.egg-info
src
develop-eggs
sphinx
dist
conda
# This build file heavily uses template features from YAML so it is generic
# enough for any Bob project. Don't modify it unless you know what you're
# doing.
# Definition of our build pipeline
stages:
- build
- test
- docs
- wheels
- deploy
# ---------
# Templates
# ---------
# Template for the build stage
# Needs to run on all supported architectures, platforms and python versions
.build_template: &build_job
stage: build
before_script:
- git clean -ffdx
- mkdir _ci
- curl --silent "https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/install.sh" > _ci/install.sh
- chmod 755 _ci/install.sh
- ./_ci/install.sh _ci #updates
- ./_ci/before_build.sh
script:
- ./_ci/build.sh
after_script:
- ./_ci/after_build.sh
artifacts:
expire_in: 1 week
paths:
- _ci/
- dist/
- sphinx/
# Template for the test stage - re-installs from uploaded wheels
# Needs to run on all supported architectures, platforms and python versions
.test_template: &test_job
stage: test
before_script:
- ./_ci/install.sh _ci #updates
- ./_ci/before_test.sh
script:
- ./_ci/test.sh
after_script:
- ./_ci/after_test.sh
# Template for the wheel uploading stage
# Needs to run against one supported architecture, platform and python version
.wheels_template: &wheels_job
stage: wheels
environment: intranet
only:
- master
- /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags)
before_script:
- ./_ci/install.sh _ci #updates
- ./_ci/before_wheels.sh
script:
- ./_ci/wheels.sh
after_script:
- ./_ci/after_wheels.sh
# Template for (latest) documentation upload stage
# Only one real job needs to do this
.docs_template: &docs_job
stage: docs
environment: intranet
only:
- master
before_script:
- ./_ci/install.sh _ci #updates
- ./_ci/before_docs.sh
script:
- ./_ci/docs.sh
after_script:
- ./_ci/after_docs.sh
# Template for the deployment stage - re-installs from uploaded wheels
# Needs to run on a single architecture only
# Will deploy your package to PyPI and other required services
# Only runs for tags
.deploy_template: &deploy_job
stage: deploy
environment: internet
only:
- /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags)
except:
- branches
before_script:
- ./_ci/install.sh _ci #updates
- ./_ci/before_deploy.sh
script:
- ./_ci/deploy.sh
after_script:
- ./_ci/after_deploy.sh
# -------------
# Build Targets
# -------------
# Linux + Python 2.7: Builds, tests, uploads wheel and deploys (if needed)
build_linux_27:
<<: *build_job
variables: &linux_27_build_variables
PYTHON_VERSION: "2.7"
WHEEL_TAG: "py27"
tags:
- conda-linux
test_linux_27:
<<: *test_job
variables: *linux_27_build_variables
dependencies:
- build_linux_27
tags:
- conda-linux
wheels_linux_27:
<<: *wheels_job
variables: *linux_27_build_variables
dependencies:
- build_linux_27
tags:
- conda-linux
deploy_linux_27:
<<: *deploy_job
variables: *linux_27_build_variables
dependencies:
- build_linux_27
tags:
- conda-linux
# Linux + Python 3.5: Builds, tests and uploads wheel
build_linux_35:
<<: *build_job
variables: &linux_35_build_variables
PYTHON_VERSION: "3.5"
WHEEL_TAG: "py3"
tags:
- conda-linux
test_linux_35:
<<: *test_job
variables: *linux_35_build_variables
dependencies:
- build_linux_35
tags:
- conda-linux
wheels_linux_35:
<<: *wheels_job
variables: *linux_35_build_variables
dependencies:
- build_linux_35
tags:
- conda-linux
docs_linux_35:
<<: *docs_job
variables: *linux_35_build_variables
dependencies:
- build_linux_35
tags:
- conda-linux
# Linux + Python 3.6: Builds and tests
build_linux_36:
<<: *build_job
variables: &linux_36_build_variables
PYTHON_VERSION: "3.6"
WHEEL_TAG: "py3"
tags:
- conda-linux
test_linux_36:
<<: *test_job
variables: *linux_36_build_variables
dependencies:
- build_linux_36
tags:
- conda-linux
# Mac OSX + Python 2.7: Builds and tests
build_macosx_27:
<<: *build_job
variables: &macosx_27_build_variables
PYTHON_VERSION: "2.7"
WHEEL_TAG: "py27"
tags:
- conda-macosx
test_macosx_27:
<<: *test_job
variables: *macosx_27_build_variables
dependencies:
- build_macosx_27
tags:
- conda-macosx
# Mac OSX + Python 3.5: Builds and tests
build_macosx_35:
<<: *build_job
variables: &macosx_35_build_variables
PYTHON_VERSION: "3.5"
WHEEL_TAG: "py3"
tags:
- conda-macosx
test_macosx_35:
<<: *test_job
variables: *macosx_35_build_variables
dependencies:
- build_macosx_35
tags:
- conda-macosx
# Mac OSX + Python 3.6: Builds and tests
build_macosx_36:
<<: *build_job
variables: &macosx_36_build_variables
PYTHON_VERSION: "3.6"
WHEEL_TAG: "py3"
tags:
- conda-macosx
test_macosx_36:
<<: *test_job
variables: *macosx_36_build_variables
dependencies:
- build_macosx_36
tags:
- conda-macosx
Copyright (c) 2017 Idiap Research Institute, http://www.idiap.ch/
Written by Amir Mohammadi <amir.mohammadi@idiap.ch>
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
3. Neither the name of the copyright holder nor the names of its contributors
may be used to endorse or promote products derived from this software without
specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
include README.rst bootstrap-buildout.py buildout.cfg COPYING version.txt requirements.txt
recursive-include doc *.py *.rst *.ico *.png
.. vim: set fileencoding=utf-8 :
.. Fri May 5 11:35:14 CEST 2017
.. image:: http://img.shields.io/badge/docs-stable-yellow.svg
:target: http://pythonhosted.org/bob.db.swan/index.html
.. image:: http://img.shields.io/badge/docs-latest-orange.svg
:target: https://www.idiap.ch/software/bob/docs/latest/bob/bob.db.swan/master/index.html
.. image:: https://gitlab.idiap.ch/bob/bob.db.swan/badges/master/build.svg
:target: https://gitlab.idiap.ch/bob/bob.db.swan/commits/master
.. image:: https://gitlab.idiap.ch/bob/bob.db.swan/badges/master/coverage.svg
:target: https://gitlab.idiap.ch/bob/bob.db.swan/commits/master
.. image:: https://img.shields.io/badge/gitlab-project-0000c0.svg
:target: https://gitlab.idiap.ch/bob/bob.db.swan
.. image:: http://img.shields.io/pypi/v/bob.db.swan.svg
:target: https://pypi.python.org/pypi/bob.db.swan
.. image:: http://img.shields.io/pypi/dm/bob.db.swan.svg
:target: https://pypi.python.org/pypi/bob.db.swan
==================================
SWAN Database Access API for Bob
==================================
This package is part of the signal-processing and machine learning toolbox
Bob_. It contains an interface for the evaluation protocol of the `SWAN`_
database. This package does not contain the original `SWAN`_ data files, which
need to be obtained through the link above.
Installation
------------
Follow our `installation`_ instructions. Then, to install this pacakge, run::
$ conda install bob.db.swan
Contact
-------
For questions or reporting issues to this software package, contact our
development `mailing list`_.
.. Place your references here:
.. _bob: https://www.idiap.ch/software/bob
.. _installation: https://www.idiap.ch/software/bob/install
.. _mailing list: https://www.idiap.ch/software/bob/discuss
.. _swan: http://www.idiap.ch/dataset/swan
# see https://docs.python.org/3/library/pkgutil.html
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
# see https://docs.python.org/3/library/pkgutil.html
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
def get_config():
"""Returns a string containing the configuration information.
"""
import bob.extension
return bob.extension.get_config(__name__)
# gets sphinx autodoc done right - don't remove it
__all__ = [_ for _ in dir() if not _.startswith('_')]
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
"""This script creates the SWAN database in a single pass.
"""
import os
from .models import Client, File, Protocol, ProtocolPurpose, Base
def nodot(item):
"""Can be used to ignore hidden files, starting with the . character."""
return item[0] != '.'
def add_clients_and_files(sql_session, datadir, extensions, verbose):
"""Add files to the SWAN database."""
def add_file(session, datadir, location, client_id_dir, session_device, basename, verbose):
"""Parse a single filename and add it to the list.
Also add a client entry if not already in the database."""
v = os.path.splitext(basename)[0].split('_')
bname = os.path.splitext(basename)[0]
full_bname = os.path.join(
location, client_id_dir, session_device, bname)
gender = ''
if v[0][0] == 'm':
gender = 'male'
if v[0][0] == 'f':
gender = 'female'
institute = int(v[0][1])
institute_dir = ''
if institute == 0:
institute = 'idiap'
institute_dir = 'idiap'
elif institute == 1:
institute = 'manchester'
institute_dir = 'uman'
elif institute == 2:
institute = 'surrey'
institute_dir = 'unis'
elif institute == 3:
institute = 'oulu'
institute_dir = 'uoulu'
elif institute == 4:
institute = 'brno'
institute_dir = 'but'
elif institute == 5:
institute = 'avignon'
institute_dir = 'lia'
if institute_dir != location:
error_msg = "File: %s -- Find location %s in directory of location %s!" % (
full_bname, location, institute_dir)
raise RuntimeError(error_msg)
client_id = v[0][1:4]
if v[0][0:4] != client_id_dir:
error_msg = "File: %s -- Find identity %s in directory of identity %s!" % (
full_bname, v[0][0:4], client_id)
raise RuntimeError(error_msg)
if not (client_id in client_dict):
if (institute == 'surrey' or institute == 'avignon'):
group = 'world'
elif (institute == 'manchester' or institute == 'oulu'):
group = 'dev'
elif (institute == 'idiap' or institute == 'brno'):
group = 'eval'
if verbose > 1:
print(" Adding client %d..." % int(client_id))
session.add(Client(int(client_id), group, gender, institute))
client_dict[client_id] = True
w = session_device.split('_')
session_id_from_dir = int(w[0])
device_from_dir = w[1]
session_id = int(v[1])
speech_type = v[2][0]
shot_id = v[2][1:3]
environment = v[3][0]
device = v[3][1]
if(device == '0'):
device = 'mobile'
elif(device == '1'):
device = 'laptop'
if device != device_from_dir:
error_msg = "File: %s -- Find device %s in directory of device %s!" % (
full_bname, device, device_from_dir)
raise RuntimeError(error_msg)
if session_id != session_id_from_dir:
error_msg = "File: %s -- Find session_id %d in directory of session_id %d!" % (
full_bname, session_id, session_id_from_dir)
raise RuntimeError(error_msg)
channel = int(v[4][0])
if verbose > 1:
print(" Adding file '%s'..." % full_bname)
session.add(File(int(client_id), full_bname, session_id,
speech_type, shot_id, environment, device, channel))
client_dict = {}
if verbose:
print("Adding clients and files ...")
for location in filter(nodot, os.listdir(datadir)):
location_dir = os.path.join(datadir, location)
if os.path.isdir(location_dir):
for client_id in filter(nodot, os.listdir(location_dir)):
client_dir = os.path.join(location_dir, client_id)
if os.path.isdir(client_dir):
for session_device in filter(nodot, os.listdir(client_dir)):
session_device_dir = os.path.join(
client_dir, session_device)
if os.path.isdir(session_device_dir):
for filename in filter(nodot, os.listdir(session_device_dir)):
for ext in extensions:
if filename.endswith(ext):
add_file(sql_session, datadir, location, client_id, session_device, os.path.basename(
filename), verbose)
def add_protocols(session, verbose):
"""Adds protocols"""
# 1. DEFINITIONS
# Numbers in the lists correspond to session identifiers
protocol_definitions = {}
# Split male and female clients: list of (client_id, first_session_id) #
# few exceptions with 2 as first session
clients_male = [(1, 1), (2, 1), (4, 1), (8, 1), (11, 1), (12, 1), (15, 1), (16, 1), (17, 1), (19, 2),
(21, 1), (23, 1), (24, 1), (25, 1), (26,
1), (28, 1), (29, 1), (30, 1), (31, 1), (33, 1),
(34, 1), (103, 1), (104, 1), (106, 1), (107,
1), (108, 1), (109, 1), (111, 1), (112, 1), (114, 1),
(115, 1), (116, 1), (117, 1), (119, 1), (120,
1), (301, 1), (304, 1), (305, 1), (308, 1), (310, 1),
(313, 1), (314, 1), (315, 1), (317, 1), (319,
1), (416, 1), (417, 1), (418, 1), (419, 1), (420, 1),
(421, 1), (422, 1), (423, 1), (424, 1), (425,
1), (426, 1), (427, 1), (428, 1), (429, 1), (430, 1),
(431, 1), (432, 1)]
clients_female = [(7, 2), (9, 1), (10, 1), (22, 1), (32, 1), (118, 1), (122, 1), (123, 1), (125, 1), (126, 1),
(127, 1), (128, 1), (129, 1), (130, 1), (131,
1), (133, 1), (302, 1), (303, 1), (306, 1), (307, 1),
(309, 1), (311, 1), (320, 1), (401, 1), (402,
1), (403, 1), (404, 1), (405, 2), (406, 1), (407, 1),
(408, 1), (409, 1), (410, 1), (411, 1), (412, 1), (413, 1), (415, 1), (433, 1)]
train_mobile = ['mobile']
train_all = None
enroll_laptop = [['laptop'], ['p']]
enroll_mobile = [['mobile'], ['p']]
enroll_laptop_mobile = [['laptop', 'mobile'], ['p']]
probe = [['mobile'], ['r', 'f']]
gender_male = 'male'
gender_female = 'female'
protocol_definitions['mobile0-male'] = [clients_male,
train_mobile, enroll_mobile, probe, gender_male]
protocol_definitions['mobile0-female'] = [clients_female,
train_mobile, enroll_mobile, probe, gender_female]
protocol_definitions['mobile1-male'] = [clients_male,
train_all, enroll_mobile, probe, gender_male]
protocol_definitions['mobile1-female'] = [clients_female,
train_all, enroll_mobile, probe, gender_female]
protocol_definitions['laptop1-male'] = [clients_male,
train_all, enroll_laptop, probe, gender_male]
protocol_definitions['laptop1-female'] = [clients_female,
train_all, enroll_laptop, probe, gender_female]
protocol_definitions['laptop_mobile1-male'] = [clients_male,
train_all, enroll_laptop_mobile, probe, gender_male]
protocol_definitions['laptop_mobile1-female'] = [clients_female,
train_all, enroll_laptop_mobile, probe, gender_female]
# 2. ADDITIONS TO THE SQL DATABASE
protocolPurpose_list = [('world', 'train'), ('dev', 'enroll'),
('dev', 'probe'), ('eval', 'enroll'), ('eval', 'probe')]
for proto in protocol_definitions:
p = Protocol(proto, protocol_definitions[proto][4])
# Add protocol
if verbose:
print("Adding protocol '%s'..." % (proto))
session.add(p)
session.flush()
session.refresh(p)
# Add protocol purposes
for key in range(len(protocolPurpose_list)):
purpose = protocolPurpose_list[key]
pu = ProtocolPurpose(p.id, purpose[0], purpose[1])
if verbose > 1:
print(" Adding protocol purpose ('%s','%s')..." %
(purpose[0], purpose[1]))
session.add(pu)
session.flush()
session.refresh(pu)
# Add files attached with this protocol purpose
client_group = ""
device_list = []
speech_list = []
if(key == 0):
client_group = "world"
elif(key == 1 or key == 2):
client_group = "dev"
elif(key == 3 or key == 4):
client_group = "eval"
if(key == 0):
world_list = True
session_list_in = False
device_list = protocol_definitions[proto][1]
if(key == 1 or key == 3):
world_list = False
session_list_in = True
device_list = protocol_definitions[proto][2][0]
speech_list = protocol_definitions[proto][2][1]
elif(key == 2 or key == 4):
world_list = False
session_list_in = False
device_list = protocol_definitions[proto][3][0]
speech_list = protocol_definitions[proto][3][1]
# Adds 'protocol' files
# World set
if world_list:
q = session.query(File).join(Client).filter(
Client.sgroup == 'world').order_by(File.id)
if device_list:
q = q.filter(File.device.in_(device_list))
for k in q:
if verbose > 1:
print(" Adding protocol file '%s'..." % (k.path))
pu.files.append(k)
# Dev/eval set
else:
for client in protocol_definitions[proto][0]:
cid = client[0] # client id
sid = client[1] # session id
q = session.query(File).join(Client).\
filter(Client.sgroup == client_group).filter(
Client.id == cid)
if session_list_in:
q = q.filter(File.session_id == sid)
else:
q = q.filter(File.session_id != sid)
if device_list:
q = q.filter(File.device.in_(device_list))
if speech_list:
q = q.filter(File.speech_type.in_(speech_list))
q = q.order_by(File.id)
for k in q:
if verbose > 1:
print(" Adding protocol file '%s'..." %
(k.path))
pu.files.append(k)
# Add protocol