Commit 3bc670b8 authored by André Anjos's avatar André Anjos 💬
Browse files

Initial commit

parents
*~
*.swp
*.pyc
*.so
*.dylib
bin
eggs
parts
.installed.cfg
.mr.developer.cfg
*.egg-info
develop-eggs
sphinx
dist
.nfs*
.gdb_history
build
*.egg
src/
db.sql3
# This build file heavily uses template features from YAML so it is generic
# enough for any Bob project. Don't modify it unless you know what you're
# doing.
# Definition of our build pipeline
stages:
- build
- test
- docs
- wheels
- deploy
# ---------
# Templates
# ---------
# Template for the build stage
# Needs to run on all supported architectures, platforms and python versions
.build_template: &build_job
stage: build
before_script:
- git clean -ffdx
- mkdir _ci
- curl --silent "https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/install.sh" > _ci/install.sh
- chmod 755 _ci/install.sh
- ./_ci/install.sh _ci #updates
- ./_ci/before_build.sh
script:
- ./_ci/build.sh
after_script:
- ./_ci/after_build.sh
artifacts:
expire_in: 1 week
paths:
- _ci/
- dist/
- sphinx/
# Template for the test stage - re-installs from uploaded wheels
# Needs to run on all supported architectures, platforms and python versions
.test_template: &test_job
stage: test
before_script:
- ./_ci/install.sh _ci #updates
- ./_ci/before_test.sh
script:
- ./_ci/test.sh
after_script:
- ./_ci/after_test.sh
# Template for the wheel uploading stage
# Needs to run against one supported architecture, platform and python version
.wheels_template: &wheels_job
stage: wheels
environment: intranet
only:
- master
- /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags)
before_script:
- ./_ci/install.sh _ci #updates
- ./_ci/before_wheels.sh
script:
- ./_ci/wheels.sh
after_script:
- ./_ci/after_wheels.sh
# Template for (latest) documentation upload stage
# Only one real job needs to do this
.docs_template: &docs_job
stage: docs
environment: intranet
only:
- master
before_script:
- ./_ci/install.sh _ci #updates
- ./_ci/before_docs.sh
script:
- ./_ci/docs.sh
after_script:
- ./_ci/after_docs.sh
# Template for the deployment stage - re-installs from uploaded wheels
# Needs to run on a single architecture only
# Will deploy your package to PyPI and other required services
# Only runs for tags
.deploy_template: &deploy_job
stage: deploy
environment: internet
only:
- /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags)
except:
- branches
before_script:
- ./_ci/install.sh _ci #updates
- ./_ci/before_deploy.sh
script:
- ./_ci/deploy.sh
after_script:
- ./_ci/after_deploy.sh
# -------------
# Build Targets
# -------------
# Linux + Python 2.7: Builds, tests, uploads wheel and deploys (if needed)
build_linux_27:
<<: *build_job
variables: &linux_27_build_variables
PYTHON_VERSION: "2.7"
WHEEL_TAG: "py27"
tags:
- conda-linux
test_linux_27:
<<: *test_job
variables: *linux_27_build_variables
dependencies:
- build_linux_27
tags:
- conda-linux
wheels_linux_27:
<<: *wheels_job
variables: *linux_27_build_variables
dependencies:
- build_linux_27
tags:
- conda-linux
deploy_linux_27:
<<: *deploy_job
variables: *linux_27_build_variables
dependencies:
- build_linux_27
tags:
- conda-linux
# Linux + Python 3.4: Builds and tests
build_linux_34:
<<: *build_job
variables: &linux_34_build_variables
PYTHON_VERSION: "3.4"
WHEEL_TAG: "py3"
tags:
- conda-linux
test_linux_34:
<<: *test_job
variables: *linux_34_build_variables
dependencies:
- build_linux_34
tags:
- conda-linux
# Linux + Python 3.5: Builds, tests and uploads wheel
build_linux_35:
<<: *build_job
variables: &linux_35_build_variables
PYTHON_VERSION: "3.5"
WHEEL_TAG: "py3"
tags:
- conda-linux
test_linux_35:
<<: *test_job
variables: *linux_35_build_variables
dependencies:
- build_linux_35
tags:
- conda-linux
wheels_linux_35:
<<: *wheels_job
variables: *linux_35_build_variables
dependencies:
- build_linux_35
tags:
- conda-linux
docs_linux_35:
<<: *docs_job
variables: *linux_35_build_variables
dependencies:
- build_linux_35
tags:
- conda-linux
# Mac OSX + Python 2.7: Builds and tests
build_macosx_27:
<<: *build_job
variables: &macosx_27_build_variables
PYTHON_VERSION: "2.7"
WHEEL_TAG: "py27"
tags:
- conda-macosx
test_macosx_27:
<<: *test_job
variables: *macosx_27_build_variables
dependencies:
- build_macosx_27
tags:
- conda-macosx
# Mac OSX + Python 3.4: Builds and tests
build_macosx_34:
<<: *build_job
variables: &macosx_34_build_variables
PYTHON_VERSION: "3.4"
WHEEL_TAG: "py3"
tags:
- conda-macosx
test_macosx_34:
<<: *test_job
variables: *macosx_34_build_variables
dependencies:
- build_macosx_34
tags:
- conda-macosx
# Mac OSX + Python 3.5: Builds and tests
build_macosx_35:
<<: *build_job
variables: &macosx_35_build_variables
PYTHON_VERSION: "3.5"
WHEEL_TAG: "py3"
tags:
- conda-macosx
test_macosx_35:
<<: *test_job
variables: *macosx_35_build_variables
dependencies:
- build_macosx_35
tags:
- conda-macosx
Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/
Written by Andre Anjos <andre.anjos@idiap.ch>
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
3. Neither the name of the copyright holder nor the names of its contributors
may be used to endorse or promote products derived from this software without
specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
include README.rst bootstrap-buildout.py buildout.cfg develop.cfg LICENSE version.txt requirements.txt
recursive-include doc *.py *.rst
recursive-include bob *.sql3
.. vim: set fileencoding=utf-8 :
.. Fri 02 Dec 2016 11:41:17 CET
.. image:: http://img.shields.io/badge/docs-stable-yellow.png
:target: http://pythonhosted.org/bob.db.3dfv/index.html
.. image:: http://img.shields.io/badge/docs-latest-orange.png
:target: https://www.idiap.ch/software/bob/docs/latest/bob/bob.db.3dfv/master/index.html
.. image:: https://gitlab.idiap.ch/bob/bob.db.3dfv/badges/master/build.svg
:target: https://gitlab.idiap.ch/bob/bob.db.3dfv/commits/master
.. image:: https://img.shields.io/badge/gitlab-project-0000c0.svg
:target: https://gitlab.idiap.ch/bob/bob.db.3dfv
.. image:: http://img.shields.io/pypi/v/bob.db.3dfv.png
:target: https://pypi.python.org/pypi/bob.db.3dfv
.. image:: http://img.shields.io/pypi/dm/bob.db.3dfv.png
:target: https://pypi.python.org/pypi/bob.db.3dfv
==========================================
3D Fingervein Database Interface for Bob
==========================================
This package is part of the signal-processing and machine learning toolbox
Bob_. It contains an interface for the evaluation protocols of the `3D
Fingervein Database`_. Notice this package does not contain the raw data files
from this dataset, which need to be obtained through the link above.
Installation
------------
Follow our `installation`_ instructions. Then, using the Python interpreter
provided by the distribution, bootstrap and buildout this package::
$ python bootstrap-buildout.py
$ ./bin/buildout
Contact
-------
For questions or reporting issues to this software package, contact our
development `mailing list`_.
.. Place your references here:
.. _bob: https://www.idiap.ch/software/bob
.. _installation: https://gitlab.idiap.ch/bob/bob/wikis/Installation
.. _mailing list: https://groups.google.com/forum/?fromgroups#!forum/bob-devel
.. _3d fingervein database: https://www.idiap.ch/dataset/3d-fingervein
# see https://docs.python.org/3/library/pkgutil.html
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
"""The VERA Database for finger verification
"""
from .query import Database
from .models import Client, Finger, File, Protocol, Subset
def get_config():
"""Returns a string containing the configuration information.
"""
import bob.extension
return bob.extension.get_config(__name__)
# gets sphinx autodoc done right - don't remove it
__all__ = [_ for _ in dir() if not _.startswith('_')]
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
"""This script creates the 3D Fingervein SQL database in a single pass.
"""
import os
import re
import csv
import pkg_resources
from .models import *
def add_clients(session, verbose):
"""Create client entries at the database"""
metadata = pkg_resources.resource_filename(__name__, os.path.join('data',
'metadata.csv'))
with open(metadata, 'rt') as f:
header = None
for row in csv.reader(f):
if header is None:
header = row
continue
id_, age, gender, skin, occ = row
id_ = int(id_)
age = int(age)
# create client
client = Client(id_, gender, age, skin, occ)
session.add(client)
if verbose:
print("Created %s" % client)
FILENAME_RE = re.compile(r'^(?P<id>\d{3})\-(?P<age>\d{3})\-(?P<gender>[fm])(?P<skin>[1-6x])(?P<occ>[0-9x])(?P<side>[lr])(?P<finger>[timlr])(?P<session>[1-3])(?P<attempt>[1-5])(?P<snap>[1-5])(?P<cam>[1-3])\.png$')
def try_get_metadata(path):
'''Returns the metadata from a path or ``None`` if no match occurs'''
m = FILENAME_RE.match(os.path.basename(path))
if m is None: return m
return {
'id': int(m.group('id')),
'age': int(m.group('age')),
'gender': m.group('gender'),
'skin': m.group('skin'),
'occ': m.group('occ'),
'side': m.group('side'),
'finger': m.group('finger'),
'session': int(m.group('session')),
'attempt': int(m.group('attempt')),
'snap': int(m.group('snap')),
'cam': int(m.group('cam')),
}
def add_files(basedir, db_session, verbose):
"""Create file entries at the database"""
dir_re = re.compile(r'^(?P<id>\d{3})$')
counter = 0
for path, dirs, files in os.walk(basedir):
for f in files:
info = try_get_metadata(f)
src_filename = os.path.join(path, f)
if info is None:
print("Skipping file `%s' (no match)" % src_filename)
continue
else: #get a match, copy with new name
counter += 1
# checks if the finger with the specifications is there/unique
try:
finger = db_session.query(Finger).join(Client).filter(
Client.id=info['id'],
Finger.side=info['side'],
Finger.name=info['name'],
).one()
except RuntimeError as e:
# creates the missing finger
client = db_session.query(Client).filter(Client.id=info['id'])
finger = Finger(client, info['side'], info['name'])
db_session.add(finger)
if verbose:
print("Created %s" % finger)
# associates file to finger
file_ = File(finger, info['session'], info['attempt'], info['snap'],
info['cam'])
db_session.add(file_)
if verbose:
print("Created %s" % file_)
def retrieve_file(session, ref):
"""Retrieves the given File object from a full path"""
info = try_get_metadata(ref)
return session.query(File).join(Finger,Client).filter(
Client.id=info['id'],
Finger.name=info['finger'],
Finger.side=info['side'],
File.session=info['session'],
File.attempt=info['attempt'],
File.snapshot=info['snap'],
File.camera=info['cam'],
).one()
def add_protocols(session, verbose):
"""Create protocol entries at the database"""
protocol_dir = pkg_resources.resource_filename(__name__, os.path.join('data',
'protocols'))
for name in os.listdir(protocol_dir):
protocol = Protocol(name)
session.add(protocol)
if verbose:
print("Created %s" % protocol)
# training data
train_filename = os.path.join(protocol_dir, name, 'train.txt')
with open(train_filename, 'rt') as f:
for row in f:
filename_ref, finger_ref = row.split()
file_ = retrieve_file(session, filename_ref)
protocol.training_set.append(file_)
if verbose:
print("Added %s to %s" % (file_, subset))
# enrollment data
models_filename = os.path.join(protocol_dir, name, 'dev-models.txt')
subset = Subset(protocol, 'dev', 'enroll')
session.add(subset)
if verbose:
print("Created %s" % subset)
with open(models_filename, 'rt') as f:
for row in f:
filename_ref, model_ref = row.split()
file_ = retrieve_file(session, filename_ref)
model = session.query(Model).filter(Model.name=model_ref)
if model.count() == 0:
model = Model(model_ref, 'dev', file_.finger, protocol)
if verbose:
print("Created model %s" % (model,))
model.files.append(file_)
if verbose:
print("Added %s to %s" % (file_, model))
# probing data
probes_filename = os.path.join(protocol_dir, name, 'dev-probes.txt')
with open(probes_filename, 'rt') as f:
for row in f:
filename_ref, model_ref = row.split()
file_ = retrieve_file(session, filename_ref)
probe = Probe('dev', protocol, file_)
session.add(probe)
if verbose:
print("Created %s" % probe)
def create_tables(args):
"""Creates all necessary tables (only to be used at the first time)"""
from bob.db.base.utils import create_engine_try_nolock
echo = args.verbose > 2 if args.verbose else False
engine = create_engine_try_nolock(args.type, args.files[0], echo=echo)
Base.metadata.create_all(engine)
def create(args):
"""Creates or re-creates this database"""
from bob.db.base.utils import session_try_nolock
dbfile = args.files[0]
if args.recreate:
if args.verbose and os.path.exists(dbfile):
print('unlinking %s...' % dbfile)
if os.path.exists(dbfile): os.unlink(dbfile)
if not os.path.exists(os.path.dirname(dbfile)):
os.makedirs(os.path.dirname(dbfile))
# the real work...
create_tables(args)
echo = args.verbose > 2 if args.verbose else False
s = session_try_nolock(args.type, args.files[0], echo=echo)
add_clients(s, args.verbose)
add_fingers(s, args.verbose)
add_files(s, args.verbose)
add_protocols(s, args.verbose)
s.commit()
s.close()
def add_command(subparsers):
"""Add specific subcommands that the action "create" can use"""
parser = subparsers.add_parser('create', help=create.__doc__)
parser.add_argument('-R', '--recreate', action='store_true',
help="If set, I'll first erase the current database")
parser.add_argument('-v', '--verbose', action='count',
help='Do SQL operations in a verbose way')
parser.set_defaults(func=create) #action
Id,Age,Gender,Skin Color,Occupation
1,48,f,2,x
2,24,f,3,x
3,54,f,2,x
4,49,f,3,x
5,48,m,3,x
6,18,m,3,x
7,36,m,1,x
8,69,f,2,x
9,69,m,2,x
10,26,m,x,x
11,26,m,3,2
12,45,m,3,x
13,24,m,2,x
14,24,f,3,x
15,72,m,1,x
16,25,f,2,x
17,65,f,x,x
18,50,m,x,x
19,64,f,1,x
21,40,m,4,1
22,58,f,1,1
23,39,m,2,1
24,68,m,2,2
25,72,f,2,9
26,68,f,2,8
27,49,f,5,3
28,27,m,3,2
29,25,m,3,0
30,72,f,2,1
31,73,f,2,2
32,53,f,2,2