Commit dd90ce97 authored by Pedro TOME's avatar Pedro TOME
Browse files

first commit

parents
*~
*.swp
*.pyc
bin
eggs
parts
.installed.cfg
.mr.developer.cfg
*.egg-info
src
develop-eggs
built-docs
dist
sphinx
db.sql3
This diff is collapsed.
include README.rst bootstrap.py buildout.cfg COPYING
recursive-include docs *.py *.rst
recursive-include xbob *.sql3
================
UTFVP Database
================
This package contains the access API and descriptions for the `UTFVP
Database`_. The actual raw data for the database should be downloaded from the original URL. This API is only
compatible with the images from the UTFVP English database. This package only
contains the `Bob <http://www.idiap.ch/software/bob/>`_ accessor methods to use
the DB directly from python, with our certified protocols.
You would normally not install this package unless you are maintaining it. What
you would do instead is to tie it in at the package you need to **use** it.
There are a few ways to achieve this:
1. You can add this package as a requirement at the ``setup.py`` for your own
`satellite package
<https://github.com/idiap/bob/wiki/Virtual-Work-Environments-with-Buildout>`_
or to your Buildout ``.cfg`` file, if you prefer it that way. With this
method, this package gets automatically downloaded and installed on your
working environment, or
2. You can manually download and install this package using commands like
``easy_install`` or ``pip``.
The package is available in two different distribution formats:
1. You can download it from `PyPI <http://pypi.python.org/pypi>`_, or
2. You can download it in its source form from `its git repository
<https://github.com/bioidiap/xbob.db.utfvp>`_. When you download the
version at the git repository, you will need to run a command to recreate
the backend SQLite file required for its operation. This means that the
database raw files must be installed somewhere in this case. With option
``a`` you can run in `dummy` mode and only download the raw data files for
the database once you are happy with your setup.
You can mix and match points 1/2 and a/b above based on your requirements. Here
are some examples:
Modify your setup.py and download from PyPI
===========================================
That is the easiest. Edit your ``setup.py`` in your satellite package and add
the following entry in the ``install_requires`` section (note: ``...`` means
`whatever extra stuff you may have in-between`, don't put that on your
script)::
install_requires=[
...
"bob==1.2.2", # base signal proc./machine learning library
"xbob.db.utfvp",
],
Proceed normally with your ``boostrap/buildout`` steps and you should be all
set. That means you can now import the ``xbob.db.utfvp`` namespace into your scripts.
Modify your buildout.cfg and download from git
==============================================
You will need to add a dependence to `mr.developer
<http://pypi.python.org/pypi/mr.developer/>`_ to be able to install from our
git repositories. Your ``buildout.cfg`` file should contain the following
lines::
[buildout]
...
extensions = mr.developer
auto-checkout = *
eggs = bob
...
xbob.db.utfvp
[sources]
xbob.db.utfvp = git https://github.com/bioidiap/xbob.db.utfvp.git
...
##############################################################################
#
# Copyright (c) 2006 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Bootstrap a buildout-based project
Simply run this script in a directory containing a buildout.cfg.
The script accepts buildout command-line options, so you can
use the -c option to specify an alternate configuration file.
"""
import os
import shutil
import sys
import tempfile
from optparse import OptionParser
tmpeggs = tempfile.mkdtemp()
usage = '''\
[DESIRED PYTHON FOR BUILDOUT] bootstrap.py [options]
Bootstraps a buildout-based project.
Simply run this script in a directory containing a buildout.cfg, using the
Python that you want bin/buildout to use.
Note that by using --find-links to point to local resources, you can keep
this script from going over the network.
'''
parser = OptionParser(usage=usage)
parser.add_option("-v", "--version", help="use a specific zc.buildout version")
parser.add_option("-t", "--accept-buildout-test-releases",
dest='accept_buildout_test_releases',
action="store_true", default=False,
help=("Normally, if you do not specify a --version, the "
"bootstrap script and buildout gets the newest "
"*final* versions of zc.buildout and its recipes and "
"extensions for you. If you use this flag, "
"bootstrap and buildout will get the newest releases "
"even if they are alphas or betas."))
parser.add_option("-c", "--config-file",
help=("Specify the path to the buildout configuration "
"file to be used."))
parser.add_option("-f", "--find-links",
help=("Specify a URL to search for buildout releases"))
options, args = parser.parse_args()
######################################################################
# load/install setuptools
to_reload = False
try:
import pkg_resources
import setuptools
except ImportError:
ez = {}
try:
from urllib.request import urlopen
except ImportError:
from urllib2 import urlopen
# XXX use a more permanent ez_setup.py URL when available.
exec(urlopen('https://bitbucket.org/pypa/setuptools/raw/0.7.2/ez_setup.py'
).read(), ez)
setup_args = dict(to_dir=tmpeggs, download_delay=0)
ez['use_setuptools'](**setup_args)
if to_reload:
reload(pkg_resources)
import pkg_resources
# This does not (always?) update the default working set. We will
# do it.
for path in sys.path:
if path not in pkg_resources.working_set.entries:
pkg_resources.working_set.add_entry(path)
######################################################################
# Try to best guess the version of buildout given setuptools
if options.version is None:
try:
from distutils.version import LooseVersion
package = pkg_resources.require('setuptools')[0]
v = LooseVersion(package.version)
if v < LooseVersion('0.7'):
options.version = '2.1.1'
except:
pass
######################################################################
# Install buildout
ws = pkg_resources.working_set
cmd = [sys.executable, '-c',
'from setuptools.command.easy_install import main; main()',
'-mZqNxd', tmpeggs]
find_links = os.environ.get(
'bootstrap-testing-find-links',
options.find_links or
('http://downloads.buildout.org/'
if options.accept_buildout_test_releases else None)
)
if find_links:
cmd.extend(['-f', find_links])
setuptools_path = ws.find(
pkg_resources.Requirement.parse('setuptools')).location
requirement = 'zc.buildout'
version = options.version
if version is None and not options.accept_buildout_test_releases:
# Figure out the most recent final version of zc.buildout.
import setuptools.package_index
_final_parts = '*final-', '*final'
def _final_version(parsed_version):
for part in parsed_version:
if (part[:1] == '*') and (part not in _final_parts):
return False
return True
index = setuptools.package_index.PackageIndex(
search_path=[setuptools_path])
if find_links:
index.add_find_links((find_links,))
req = pkg_resources.Requirement.parse(requirement)
if index.obtain(req) is not None:
best = []
bestv = None
for dist in index[req.project_name]:
distv = dist.parsed_version
if _final_version(distv):
if bestv is None or distv > bestv:
best = [dist]
bestv = distv
elif distv == bestv:
best.append(dist)
if best:
best.sort()
version = best[-1].version
if version:
requirement = '=='.join((requirement, version))
cmd.append(requirement)
import subprocess
if subprocess.call(cmd, env=dict(os.environ, PYTHONPATH=setuptools_path)) != 0:
raise Exception(
"Failed to execute command:\n%s",
repr(cmd)[1:-1])
######################################################################
# Import and run buildout
ws.add_entry(tmpeggs)
ws.require(requirement)
import zc.buildout.buildout
if not [a for a in args if '=' not in a]:
args.append('bootstrap')
# if -c was provided, we push it back into args for buildout' main function
if options.config_file is not None:
args[0:0] = ['-c', options.config_file]
zc.buildout.buildout.main(args)
shutil.rmtree(tmpeggs)
; vim: set fileencoding=utf-8 :
[buildout]
parts = scripts
develop = .
eggs = xbob.db.utfvp
newest = false
[scripts]
recipe = xbob.buildout:scripts
dependent-scripts = true
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Pedro Tome <Pedro.Tome@idiap.ch>
#
# Copyright (C) 2014 Idiap Research Institute, Martigny, Switzerland
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from setuptools import setup, find_packages
# The only thing we do in this file is to call the setup() function with all
# parameters that define our package.
setup(
name='xbob.db.utfvp',
version='1.0.0',
description='UTFVP Database Access API for Bob',
url='https://pypi.python.org/pypi/xbob.db.utfvp',
license='GPLv3',
author='',
author_email='',
keywords='vein recognition, bob, xbob, xbob.db, UTFVP',
long_description=open('README.rst').read(),
# This line is required for any distutils based packaging.
packages=find_packages(),
include_package_data=True,
zip_safe=False,
install_requires=[
'setuptools',
'six', # py2/3 compatibility library
'bob==1.2.2', # base signal proc./machine learning library
'xbob.db.verification.utils>=1.0.0' # defines a set of utilities for face verification databases like this one.
],
namespace_packages = [
'xbob',
'xbob.db',
],
entry_points = {
# bob database declaration
'bob.db': [
'utfvp = xbob.db.utfvp.driver:Interface',
],
# bob unittest declaration
'bob.test': [
'utfvp = xbob.db.utfvp.test:UTFVPDatabaseTest',
],
},
classifiers = [
'Development Status :: 3 - Beta',
'Environment :: Console',
'Intended Audience :: Developers',
'Intended Audience :: Education',
'Intended Audience :: Science/Research',
'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
'Natural Language :: English',
'Programming Language :: Python',
'Programming Language :: Python :: 3',
'Topic :: Scientific/Engineering :: Artificial Intelligence',
'Topic :: Database :: Front-Ends',
],
)
#see http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
__import__('pkg_resources').declare_namespace(__name__)
#see http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
__import__('pkg_resources').declare_namespace(__name__)
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Laurent El Shafey <laurent.el-shafey@idiap.ch>
#
# Copyright (C) 2014 Idiap Research Institute, Martigny, Switzerland
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""The UTFVP Database for finger verification
"""
from .query import Database
from .models import Client, File, Protocol, Model, ProtocolPurpose
__all__ = dir()
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Laurent El Shafey <Laurent.El-Shafey@idiap.ch>
# Wed Feb 19 10:53:53 CET 2014
#
# Copyright (C) 2014 Idiap Research Institute, Martigny, Switzerland
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""This script creates the UTFVP database in a single pass.
"""
import os
from .models import *
def nodot(item):
"""Can be used to ignore hidden files, starting with the . character."""
return item[0] != '.'
def add_files(session, imagedir, verbose):
"""Add files (and clients) to the UTFVP database."""
def add_file(session, subdir, filename, client_dict, model_dict, file_dict, verbose):
"""Parse a single filename and add it to the list.
Also add a client entry if not already in the database."""
v = os.path.splitext(os.path.basename(filename))[0].split('_')
subclient_id = int(v[0])
finger_id = int(v[1])
client_id = "%d_%d" % (subclient_id, finger_id)
if not (client_id in client_dict):
c = Client(client_id, subclient_id)
session.add(c)
session.flush()
session.refresh(c)
client_dict[client_id] = True
session_id = int(v[2])
base_path = os.path.join(subdir, os.path.basename(filename).split('.')[0])
sgroup = 'dev'
if subclient_id <= 36:
if subclient_id < 19:
if finger_id == ((subclient_id-1) % 6) + 1:
sgroup = 'world'
elif subclient_id > 19:
if finger_id == ((subclient_id-2) % 6) + 1:
sgroup = 'world'
if verbose>1: print(" Adding file '%s'..." %(base_path, ))
cfile = File(client_id, base_path, sgroup, finger_id, session_id)
session.add(cfile)
session.flush()
session.refresh(cfile)
file_dict[sgroup][cfile.id] = cfile
if sgroup == 'dev':
model_id = "%d_%d_%d" % (subclient_id, finger_id, session_id)
if verbose>1: print(" Adding Model '%s'..." %(model_id, ))
model = Model(model_id, client_id, cfile.id)
session.add(model)
session.flush()
session.refresh(model)
model_dict[model_id] = model
return [client_dict, model_dict, file_dict]
if verbose: print("Adding files...")
subdir_list = list(filter(nodot, os.listdir(imagedir)))
client_dict = {}
model_dict = {}
file_dict = {}
file_dict['world'] = {}
file_dict['dev'] = {}
for subdir in subdir_list:
file_list = list(filter(nodot, os.listdir(os.path.join(imagedir, subdir))))
for filename in file_list:
filename_, extension = os.path.splitext(filename)
if extension == '.png':
client_dict, model_dict, file_dict = add_file(session, subdir, os.path.join(imagedir, filename), client_dict, model_dict, file_dict, verbose)
return [client_dict, model_dict, file_dict]
def add_protocols(session, client_dict, model_dict, file_dict, verbose):
"""Adds protocols"""
# 2. ADDITIONS TO THE SQL DATABASE
protocol_list = ['master', 'paper']
protocolPurpose_list = [('world', 'train'), ('dev', 'enrol'), ('dev', 'probe')]
for proto in protocol_list:
p = Protocol(proto)
# Add protocol
if verbose: print("Adding protocol %s..." % (proto))
session.add(p)
session.flush()
session.refresh(p)
for purpose in protocolPurpose_list:
pu = ProtocolPurpose(p.id, purpose[0], purpose[1])
if verbose>1: print(" Adding protocol purpose ('%s', '%s','%s')..." % (p.name, purpose[0], purpose[1]))
session.add(pu)
session.flush()
session.refresh(pu)
cfile_dict = file_dict[purpose[0]]
for f_id, f_file in cfile_dict.iteritems():
if f_file.client.subclient_id == 19 and proto == 'master':
continue
if verbose>1: print(" Adding file ('%s') to protocol purpose ('%s', '%s','%s')..." % (f_file.path, p.name, purpose[0], purpose[1]))
pu.files.append(f_file)
for m_id, model in model_dict.iteritems():
if model.client.subclient_id == 19 and proto == 'master':
continue
p.models.append(model)
def create_tables(args):
"""Creates all necessary tables (only to be used at the first time)"""
from bob.db.utils import create_engine_try_nolock
engine = create_engine_try_nolock(args.type, args.files[0], echo=(args.verbose > 2))
Base.metadata.create_all(engine)
# Driver API
# ==========
def create(args):
"""Creates or re-creates this database"""
from bob.db.utils import session_try_nolock
dbfile = args.files[0]
if args.recreate:
if args.verbose and os.path.exists(dbfile):
print('unlinking %s...' % dbfile)
if os.path.exists(dbfile): os.unlink(dbfile)
if not os.path.exists(os.path.dirname(dbfile)):
os.makedirs(os.path.dirname(dbfile))
# the real work...
create_tables(args)
s = session_try_nolock(args.type, args.files[0], echo=(args.verbose > 2))
client_dict, model_dict, file_dict = add_files(s, args.imagedir, args.verbose)
add_protocols(s, client_dict, model_dict, file_dict, args.verbose)
s.commit()
s.close()
def add_command(subparsers):
"""Add specific subcommands that the action "create" can use"""
parser = subparsers.add_parser('create', help=create.__doc__)
parser.add_argument('-R', '--recreate', action='store_true', help="If set, I'll first erase the current database")
parser.add_argument('-v', '--verbose', action='count', help='Do SQL operations in a verbose way')
parser.add_argument('-D', '--imagedir', metavar='DIR', default='/idiap/resource/database/UTFVP/data', help="Change the relative path to the directory containing the images of the UTFVP database (defaults to %(default)s)")
parser.set_defaults(func=create) #action
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Laurent El Shafey <laurent.el-shafey@idiap.ch>
# Wed Feb 19 12:31:20 CET 2014
#
# Copyright (C) 2014 Idiap Research Institute, Martigny, Switzerland
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""Bob Database Driver entry-point for the UTFVP
"""
import os
import sys
from bob.db.driver import Interface as BaseInterface
def dumplist(args):
"""Dumps lists of files based on your criteria"""
from .query import Database
db = Database()
r = db.objects(
protocol=args.protocol,
purposes=args.purpose,
model_ids=(args.models,),
groups=args.group,
classes=args.sclass
)
output = sys.stdout
if args.selftest:
from bob.db.utils import null
output = null()
for f in r:
output.write('%s\n' % (f.make_path(args.directory, args.extension),))
return 0
def checkfiles(args):