Commit b52563a9 authored by Manuel Günther's avatar Manuel Günther
Browse files

First working version of the JANUS database interface, including tests.

parents
*~
*.swp
*.pyc
*.so
*.dylib
bin
eggs
parts
.installed.cfg
.mr.developer.cfg
*.egg-info
develop-eggs
sphinx
dist
.nfs*
.gdb_history
build
*.egg
src/
db.sql3
language: python
matrix:
include:
- python: 2.6
- python: 2.7
env:
- secure: VD6bVhngzXrNa6cXb802ciinSGk/tMVSWyvAE3bDA5HVDYeDp33TeaCZEGtUuecUUa4+cCfZpDLMEddm/LC4tVbCx+G3I5333ldq499NYvA6USuwBMz+ZnXVL9heY98DP7m42cX6NMF47Sb/u/CtXn0X3iTSNdMYh0f1RkV9ZlA=
- secure: h6sD2IEGEKd+cR8iPVh+AqDOlEabLnxDY75/TYZfI80SsmdDNn7CzFHWlokIcLk5qrj2IT45FFAHgaR/UiJJ46+ScP31+nd7GdhtbQs9g/Hytqr/VEoyMXXOSsMnXaaELlL4TpWcdkTOT/Osmq4HDFeS2VNNfPCrmVYDw+6H7K0=
- BOB_DOCUMENTATION_SERVER=https://www.idiap.ch/software/bob/docs/latest/bioidiap/%s/master
- BOB_UPLOAD_WHEEL="--universal"
- python: 3.3
- python: 3.4
before_install:
- sudo add-apt-repository -y ppa:biometrics/bob
- sudo apt-get update -qq
- sudo apt-get install -qq --force-yes libboost-all-dev libblitz1-dev libhdf5-serial-dev libatlas-dev libatlas-base-dev liblapack-dev texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended
- pip install --find-links https://www.idiap.ch/software/bob/wheels/travis/ --use-wheel sphinx nose numpy coverage
- pip install --find-links https://www.idiap.ch/software/bob/wheels/travis/ --use-wheel --pre -r requirements.txt coveralls
install:
- python bootstrap-buildout.py
- ./bin/buildout buildout:develop=. buildout:extensions=bob.buildout buildout:auto-checkout=
script:
- ./bin/python -c 'import pkg_resources; from bob.db.caspeal import get_config; print(get_config())'
- ./bin/bob_dbmanage.py caspeal download
- ./bin/coverage run --source=bob.db.caspeal ./bin/nosetests -sv
- ./bin/sphinx-build -b doctest doc sphinx
- ./bin/sphinx-build -b html doc sphinx
after_success:
- coveralls
- wget https://raw.githubusercontent.com/bioidiap/bob.extension/master/scripts/upload-{sphinx,wheel}.sh
- chmod a+x upload-sphinx.sh upload-wheel.sh
- ./upload-sphinx.sh
- ./upload-wheel.sh
include README.rst bootstrap-buildout.py buildout.cfg version.txt requirements.txt
recursive-include doc *.py *.rst
recursive-include bob *.sql3
.. vim: set fileencoding=utf-8 :
.. Manuel Gunther <mgunther@vast.uccs.edu>
.. Tue Sep 8 15:05:38 MDT 2015
.. image:: http://img.shields.io/badge/docs-stable-yellow.png
:target: http://pythonhosted.org/bob.db.janus/index.html
.. image:: http://img.shields.io/badge/docs-latest-orange.png
:target: https://www.idiap.ch/software/bob/docs/latest/bioidiap/bob.db.janus/master/index.html
.. image:: https://travis-ci.org/bioidiap/bob.db.janus.svg?branch=master
:target: https://travis-ci.org/bioidiap/bob.db.janus
.. image:: https://coveralls.io/repos/bioidiap/bob.db.janus/badge.png
:target: https://coveralls.io/r/bioidiap/bob.db.janus
.. image:: https://img.shields.io/badge/github-master-0000c0.png
:target: https://github.com/bioidiap/bob.db.janus/tree/master
.. image:: http://img.shields.io/pypi/v/bob.db.janus.png
:target: https://pypi.python.org/pypi/bob.db.janus
.. image:: http://img.shields.io/pypi/dm/bob.db.janus.png
:target: https://pypi.python.org/pypi/bob.db.janus
.. image:: https://img.shields.io/badge/original-data--files-a000a0.png
:target: http://www.jdl.ac.cn/peal
==================================
JANUS Database Interface for Bob
==================================
This package contains an interface for the evaluation protocols of the JANUS V1 database.
Installation
------------
To install this package -- alone or together with other `Packages of Bob <https://github.com/idiap/bob/wiki/Packages>`_ -- please read the `Installation Instructions <https://github.com/idiap/bob/wiki/Installation>`_.
For Bob_ to be able to work properly, some dependent packages are required to be installed.
Please make sure that you have read the `Dependencies <https://github.com/idiap/bob/wiki/Dependencies>`_ for your operating system.
Documentation
-------------
For further documentation on this package, please read the `Stable Version <http://pythonhosted.org/bob.db.janus/index.html>`_ or the `Latest Version <https://www.idiap.ch/software/bob/docs/latest/bioidiap/bob.db.janus/master/index.html>`_ of the documentation.
For a list of tutorials on this or the other packages ob Bob_, or information on submitting issues, asking questions and starting discussions, please visit its website.
.. _bob: https://www.idiap.ch/software/bob
#see http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
__import__('pkg_resources').declare_namespace(__name__)
#see http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
__import__('pkg_resources').declare_namespace(__name__)
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Manuel Guenther <Manuel.Guenther@idiap.ch>
# @date: Mon Dec 10 14:29:51 CET 2012
#
# Copyright (C) 2011-2012 Idiap Research Institute, Martigny, Switzerland
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""This is the Bob database entry for the CAS-PEAL database.
"""
from .query import Database
from .models import File, Annotation, Template
def get_config():
"""Returns a string containing the configuration information.
"""
import bob.extension
return bob.extension.get_config(__name__)
# gets sphinx autodoc done right - don't remove it
__all__ = [_ for _ in dir() if not _.startswith('_')]
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Manuel Guenther <Manuel.Guenther@idiap.ch>
# @date: Mon Dec 10 14:29:51 CET 2012
#
# Copyright (C) 2011-2012 Idiap Research Institute, Martigny, Switzerland
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""This script creates the CAS-PEAL database in a single pass.
"""
from __future__ import print_function
import os
from .models import *
def _update(session, field):
"""Add, updates and returns the given field for in the current session"""
session.add(field)
session.flush()
session.refresh(field)
return field
def add_files(session, directory, add_splits, verbose):
"""Adds all files of the JANUS database"""
clients = set()
# templates = {}
files = {}
# filenames = ['gallery.csv', 'probe.csv']
# if add_splits:
# filenames += ["split{s}/{p}_{s}.csv".format(s=s, p=p) for s in range(1,11) for p in ("train", "gallery", "probe")]
filename = os.path.join(directory, 'protocol', 'metadata.csv')
if verbose:
print("Reading files from", filename)
with open(os.path.join(directory, 'protocol', filename)) as f:
# ignore the first line
_ = f.readline()
# read the rest of the lines
for line in f:
splits = line.rstrip().split(',')
assert len(splits) == 24, splits
# first, create a Client, if not yet there
subject_id = int(splits[1])
if subject_id not in clients:
if verbose > 1: print(". Adding client", subject_id)
client = _update(session, Client(subject_id))
clients.add(subject_id)
# now, create a File
path = splits[2]
# unique file id is the path id and the sighting_id
path_id = "%s-%s" % (path, splits[4])
if verbose > 2: print("... Adding file", path)
if path_id not in files:
file = _update(session, File(path, subject_id, int(splits[3]), int(splits[4]), int(splits[5]) if splits[5] else None))
files[path_id] = file
# create annotations
if verbose > 2: print("... Adding annotation")
annotation = Annotation(file.id, [float(a) if a else None for a in splits[6:17]], [int(a) if a else None for a in splits[17:]])
session.add(annotation)
# finally, return the files that we have created
return files
def add_protocols(session, files, directory, add_splits, verbose):
"""Adds the protocols for the JANUS database"""
def _read_file(filename):
"""Reads the given file and yields the template id, the subject id and path_id (path + sighting_id)"""
with open(os.path.join(directory, 'protocol', filename)) as f:
# skip the first line
_ = f.readline()
for line in f:
splits = line.rstrip().split(',')
assert len(splits) == 24, splits
# we only carte about the template id.
yield int(splits[0]), int(splits[1]), "%s-%s" % (splits[2], splits[4])
# create and add protocol
protocol = _update(session, Protocol("NoTrain"))
if verbose: print("Adding Protocol", protocol.name)
# read gallery and probe files
for purpose in ("gallery", "probe"):
# create protocol purpose association entry
pp = _update(session, ProtocolPurpose(purpose, protocol.id))
filename = '%s.csv' % purpose
templates = {}
for template_id, subject_id, path_id in _read_file(filename):
# create template with given IDs for the given protocol purpose
if template_id not in templates:
if verbose > 1: print(". Adding template", template_id)
template = _update(session, Template(template_id, subject_id, pp.id))
templates[template_id] = template
else:
template = templates[template_id]
# add files
assert path_id in files
template.files.append(files[path_id])
if add_splits:
# Now, add the training splits protocols
for split in [str(i) for i in range(1,11)]:
# create and add protocol
protocol = _update(session, Protocol("split%s" % split))
if verbose: print("Adding Protocol", protocol.name)
# read gallery and probe files
for purpose in ("train", "gallery", "probe"):
# create protocol purpose association entry
pp = _update(session, ProtocolPurpose(purpose, protocol.id))
if verbose: print("Adding Protocol purpose", pp.protocol.name, pp.purpose)
filename = os.path.join("split%s" % split, "%s_%s.csv" % (purpose, split))
templates = {}
for template_id, subject_id, path_id in _read_file(filename):
# create template with given IDs for the given protocol purpose
if template_id not in templates:
if verbose > 1: print(". Adding template", template_id)
template = _update(session, Template(template_id, subject_id, pp.id))
templates[template_id] = template
else:
template = templates[template_id]
# add files
assert path_id in files
template.files.append(files[path_id])
def create_tables(args):
"""Creates all necessary tables (only to be used at the first time)"""
from bob.db.base.utils import create_engine_try_nolock
engine = create_engine_try_nolock(args.type, args.files[0], echo=(args.verbose > 2))
File.metadata.create_all(engine)
Annotation.metadata.create_all(engine)
Template.metadata.create_all(engine)
Protocol.metadata.create_all(engine)
# Driver API
# ==========
def create(args):
"""Creates or re-creates this database"""
from bob.db.base.utils import session_try_nolock
dbfile = args.files[0]
if args.recreate:
if args.verbose and os.path.exists(dbfile):
print('unlinking %s...' % dbfile)
if os.path.exists(dbfile): os.unlink(dbfile)
if not os.path.exists(os.path.dirname(dbfile)):
os.makedirs(os.path.dirname(dbfile))
# the real work...
create_tables(args)
session = session_try_nolock(args.type, args.files[0], echo=(args.verbose > 2))
templates = add_files(session, args.directory, not args.no_splits, args.verbose)
add_protocols(session, templates, args.directory, not args.no_splits, args.verbose)
session.commit()
session.close()
def add_command(subparsers):
"""Add specific subcommands that the action "create" can use"""
parser = subparsers.add_parser('create', help=create.__doc__)
parser.add_argument('-R', '--recreate', action='store_true', help='If set, I\'ll first erase the current database')
parser.add_argument('-v', '--verbose', action='count', help='Do SQL operations in a verbose way?')
parser.add_argument('-D', '--directory', metavar='DIR', default='/home/mgunther/databases/janus', help='The path to the JANUS database')
parser.add_argument('-n', '--no-splits', action='store_true', help='If selected, the 10 splits of the protocols are disabled.')
parser.set_defaults(func=create) #action
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Manuel Guenther <Manuel.Guenther@idiap.ch>
# @date: Mon Dec 10 14:29:51 CET 2012
#
# Copyright (C) 2011-2012 Idiap Research Institute, Martigny, Switzerland
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""Commands this database can respond to.
"""
import os
import sys
from bob.db.base.driver import Interface as BaseInterface
def dumplist(args):
"""Dumps lists of files based on your criteria"""
from .query import Database
db = Database()
r = db.objects(
groups=args.group,
protocol=args.protocol,
purposes=args.purpose,
model_ids=args.template_id
)
output = sys.stdout
if args.selftest:
from bob.db.base.utils import null
output = null()
for f in r:
output.write('%s\n' % (f.make_path(args.directory, args.extension, args.add_sighting_id),))
return 0
def checkfiles(args):
"""Checks existence of files based on your criteria"""
from .query import Database
db = Database()
r = db.objects()
# go through all files, check if they are available on the filesystem
good = {}
bad = {}
for f in r:
if os.path.exists(f.make_path(args.directory,args.extension,args.add_sighting_id)): good[f.id] = f
else: bad[f.id] = f
# report
output = sys.stdout
if args.selftest:
from bob.db.base.utils import null
output = null()
if bad:
for id, f in bad.items():
output.write('Cannot find file "%s"\n' % f.make_path(args.directory,args.extension,args.add_sighting_id))
output.write('%d files (out of %d) were not found at "%s"\n' % \
(len(bad), len(r), args.directory))
return 0
def reverse(args):
"""Returns a list of file database identifiers given the path stems"""
from .query import Database
db = Database()
output = sys.stdout
if args.selftest:
from bob.db.base.utils import null
output = null()
r = db.reverse(args.path)
for f in r: output.write('%s\n' % f.id)
if not r: return 1
return 0
def path(args):
"""Returns a list of fully formed paths or stems given some file id"""
from .query import Database
db = Database()
output = sys.stdout
if args.selftest:
from bob.db.base.utils import null
output = null()
r = db.paths(args.id, prefix=args.directory, suffix=args.extension)
for path in r: output.write('%s\n' % path)
if not r: return 1
return 0
class Interface(BaseInterface):
def name(self):
return 'janus'
def version(self):
import pkg_resources # part of setuptools
return pkg_resources.require('bob.db.%s' % self.name())[0].version
def files(self):
from pkg_resources import resource_filename
raw_files = ('db.sql3',)
return [resource_filename(__name__, k) for k in raw_files]
def type(self):
return 'sqlite'
def add_commands(self, parser):
from . import __doc__ as docs
import argparse
subparsers = self.setup_parser(parser,
"JANUS database", docs)
# example: get the "create" action from a submodule
from .create import add_command as create_command
create_command(subparsers)
from .query import Database
db = Database()
from .models import ProtocolPurpose
# the "dumplist" action
parser = subparsers.add_parser('dumplist', help=dumplist.__doc__)
parser.add_argument('-d', '--directory', help="if given, this path will be prepended to every entry returned.")
parser.add_argument('-e', '--extension', help="if given, this extension will be appended to every entry returned.")
parser.add_argument('-a', '--add-sighting-id', action='store_true', help="if selected, unique path names will be created by adding the sighting id.")
parser.add_argument('-g', '--group', help="if given, this value will limit the output files to those belonging to a particular group.", choices = ProtocolPurpose.group_choices)
parser.add_argument('-p', '--protocol', default='NoTrain', help="limits the dump to a particular subset of the data that corresponds to the given protocol.", choices = db.protocol_names() if db.is_valid() else ())
parser.add_argument('-u', '--purpose', help="if given, this value will limit the output files to those designed for the given purposes.", choices=ProtocolPurpose.purpose_choices)
parser.add_argument('-m', '--template-id', type=int, help="if given, this value will limit the output files to those of the given template id.", choices=db.template_ids() if db.is_valid() else ())
parser.add_argument('--self-test', dest="selftest", action='store_true', help=argparse.SUPPRESS)
parser.set_defaults(func=dumplist) #action
# the "checkfiles" action
parser = subparsers.add_parser('checkfiles', help=checkfiles.__doc__)
parser.add_argument('-d', '--directory', help="if given, this path will be prepended to every entry returned.")
parser.add_argument('-e', '--extension', help="if given, this extension will be appended to every entry returned.")
parser.add_argument('-a', '--add-sighting-id', action='store_true', help="if selected, unique path names will be created by adding the sighting id.")
parser.add_argument('--self-test', dest="selftest", action='store_true', help=argparse.SUPPRESS)
parser.set_defaults(func=checkfiles) #action
# adds the "reverse" command
parser = subparsers.add_parser('reverse', help=reverse.__doc__)
parser.add_argument('path', nargs='+', help="one or more path stems to look up. If you provide more than one, files which cannot be reversed will be omitted from the output.")
parser.add_argument('--self-test', dest="selftest", action='store_true', help=argparse.SUPPRESS)
parser.set_defaults(func=reverse) #action
# adds the "path" command
parser = subparsers.add_parser('path', help=path.__doc__)
parser.add_argument('-d', '--directory', help="if given, this path will be prepended to every entry returned.")
parser.add_argument('-e', '--extension', help="if given, this extension will be appended to every entry returned.")
parser.add_argument('id', type=int, nargs='+', help="one or more file ids to look up. If you provide more than one, files which cannot be found will be omitted from the output. If you provide a single id to lookup, an error message will be printed if the id does not exist in the database. The exit status will be non-zero in such case.")
parser.add_argument('--self-test', dest="selftest", action='store_true', help=argparse.SUPPRESS)
parser.set_defaults(func=path) #action
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Manuel Guenther <Manuel.Guenther@idiap.ch>
# @date: Mon Dec 10 14:29:51 CET 2012
#
# Copyright (C) 2011-2012 Idiap Research Institute, Martigny, Switzerland
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""Table models and functionality for the CAS-PEAL database.
"""
import sqlalchemy
from sqlalchemy import Table, Column, Integer, Float, String, Boolean, ForeignKey, or_, and_, not_
from bob.db.base.sqlalchemy_migration import Enum, relationship
from sqlalchemy.orm import backref
from sqlalchemy.ext.declarative import declarative_base
import os
import bob.db.verification.utils
Base = declarative_base()
class Client(Base):
"""The clients of the JANUS database"""
__tablename__ = 'client'
id = Column(Integer, primary_key=True)
def __init__(self, subject_id):
self.id = subject_id
class Annotation(Base):
"""Annotations of the CAS-PEAL database consists only of the left and right eye positions.
There is exactly one annotation for each file."""
__tablename__ = 'annotation'
id = Column(Integer, primary_key=True)
file_id = Column(Integer, ForeignKey('file.id'))
tl_x = Column(Float) # top-left position
tl_y = Column(Float)
size_x = Column(Float) # size of face
size_y = Column(Float)
le_x = Column(Float) # left eye
le_y = Column(Float)
re_x = Column(Float) # right eye
re_y = Column(Float)
n_x = Column(Float) # nose base
n_y = Column(Float)
yaw = Column(Float) # face yaw
forehead = Column(Integer) # forehead visible
eyes = Column(Integer) # eyes visible
nm = Column(Integer) # nose mouth visible
indoor = Column(Integer) # indoor
gender = Column(Integer) # gender
skin = Column(Integer) # skin tone
age = Column(Integer) # age (not the real age, but some age indicator)
def __init__(self, file_id, annotations, extras):
self.file_id = file_id
assert len(annotations) == 11
assert len(extras) == 7
# assert that at least the face bounding box is labeled
assert not any(a is None for a in annotations[:4])
self.tl_x = annotations[0]
self.tl_y = annotations[1]
self.size_x = annotations[2]
self.size_y = annotations[3]
# self.br_x = self.tl_x + annotations[2] if annotations[2] is not None else None
# self.br_y = self.tl_y + annotations[3] if annotations[3] is not None else None
self.re_x = annotations[4]
self.re_y = annotations[5]