Commit 8ca8a457 authored by Pavel KORSHUNOV's avatar Pavel KORSHUNOV

Merge branch 'move-pad-db-definition-here' into 'master'

moved pad DB interface here, tests are passing

Moving support for PAD DB interfaces here, so that `bob.pad.db`packages becomes deprecated.

See merge request !1
parents a215ab22 a454e82c
Pipeline #3872 failed with stages
in 42 minutes and 13 seconds
# This build file is defined in two parts: 1) a generic set of instructions you
# probably **don't** need to change and 2) a part you may have to tune to your
# project. It heavily uses template features from YAML to help you in only
# changing a minimal part of it and avoid code duplication to a maximum while
# still providing a nice pipeline display on your package.
# 1) Generic instructions (only change if you know what you're doing)
# -------------------------------------------------------------------
# Definition of our build pipeline
stages:
- build
- test
- docs
- wheels
# Global variables
variables:
CONDA_PREFIX: env
# Template for the build stage
# Needs to run on all supported architectures, platforms and python versions
.build_template: &build_job
stage: build
before_script:
- git clean -ffdx
- curl --silent https://gitlab.idiap.ch/bob/bob/snippets/7/raw | tr -d '\r' > bootstrap-conda.sh
- chmod 755 ./bootstrap-conda.sh
- ./bootstrap-conda.sh ${CONDA_FOLDER} ${PYTHON_VER} ${CONDA_PREFIX}
variables: &build_variables
BOB_DOCUMENTATION_SERVER: "http://www.idiap.ch/software/bob/docs/latest/bob/%s/master/"
script:
- ./bin/buildout
- ./bin/sphinx-build doc sphinx
- ./bin/python setup.py bdist_wheel --python-tag ${WHEEL_TAG}
after_script:
- rm -rf ${CONDA_PREFIX}
artifacts:
expire_in: 1 day
paths:
- bootstrap-conda.sh
- dist/
- sphinx/
# Template for building on a Linux machine
.build_linux_template: &linux_build_job
<<: *build_job
variables: &linux_build_variables
<<: *build_variables
CONDA_FOLDER: "/local/conda"
CFLAGS: "-D_GLIBCXX_USE_CXX11_ABI=0 -coverage"
CXXFLAGS: "-D_GLIBCXX_USE_CXX11_ABI=0 -coverage"
# Template for building on a Mac OSX machine
.build_mac_template: &macosx_build_job
<<: *build_job
variables: &macosx_build_variables
<<: *build_variables
CONDA_FOLDER: "/opt/conda"
MACOSX_DEPLOYMENT_TARGET: "10.9"
CFLAGS: "-pthread -coverage"
CXXFLAGS: "-pthread -coverage"
LDFLAGS: "-lpthread"
# Template for the test stage - re-install from uploaded wheels
# Needs to run on all supported architectures, platforms and python versions
.test_template: &test_job
stage: test
before_script:
- ./bootstrap-conda.sh ${CONDA_FOLDER} ${PYTHON_VER} ${CONDA_PREFIX}
- source ${CONDA_FOLDER}/bin/activate ${CONDA_PREFIX}
- pip install --use-wheel --no-index --pre dist/*.whl
script:
- cd ${CONDA_PREFIX}
- python -c "from ${CI_PROJECT_NAME} import get_config; print(get_config())"
- if [ -x ./bin/bob_dbmanage.py ]; then ./bin/bob_dbmanage.py all download --force; fi
- coverage run --source=${CI_PROJECT_NAME} ./bin/nosetests -sv ${CI_PROJECT_NAME}
- coverage report
- sphinx-build -b doctest ../doc ../sphinx
after_script:
- rm -rf ${CONDA_PREFIX}
# Template for the wheel uploading stage
# Needs to run against one combination of python 2.x and 3.x if it is a python
# only package, otherwise, needs to run in both pythons to all supported
# architectures (Linux and Mac OSX 64-bit)
.wheels_template: &wheels_job
stage: wheels
only:
- master
- tags
before_script:
- curl --silent https://gitlab.idiap.ch/bob/bob/snippets/8/raw | tr -d '\r' > upload-wheel.sh
- chmod 755 upload-wheel.sh
script:
- ./upload-wheel.sh
# Template for (latest) documentation upload stage
# Only one real job needs to do this
.docs_template: &docs_job
stage: docs
only:
- master
before_script:
- curl --silent https://gitlab.idiap.ch/bob/bob/snippets/9/raw | tr -d '\r' > upload-sphinx.sh
- chmod 755 upload-sphinx.sh
script:
- ./upload-sphinx.sh
# 2) Package specific instructions (you may tune this if needed)
# --------------------------------------------------------------
# Linux + Python 2.7: Builds, tests, uploads wheel
build_linux_27:
<<: *linux_build_job
variables: &linux_27_build_variables
<<: *linux_build_variables
PYTHON_VER: "2.7"
WHEEL_TAG: "py27"
tags:
- conda-linux
test_linux_27:
<<: *test_job
variables: *linux_27_build_variables
dependencies:
- build_linux_27
tags:
- conda-linux
wheels_linux_27:
<<: *wheels_job
dependencies:
- build_linux_27
tags:
- conda-linux
# Linux + Python 3.4: Builds and tests
build_linux_34:
<<: *linux_build_job
variables: &linux_34_build_variables
<<: *linux_build_variables
PYTHON_VER: "3.4"
WHEEL_TAG: "py3"
tags:
- conda-linux
test_linux_34:
<<: *test_job
variables: *linux_34_build_variables
dependencies:
- build_linux_34
tags:
- conda-linux
# Linux + Python 3.5: Builds, tests, uploads wheel
build_linux_35:
<<: *linux_build_job
variables: &linux_35_build_variables
<<: *linux_build_variables
PYTHON_VER: "3.5"
WHEEL_TAG: "py3"
tags:
- conda-linux
test_linux_35:
<<: *test_job
variables: *linux_35_build_variables
dependencies:
- build_linux_35
tags:
- conda-linux
wheels_linux_35:
<<: *wheels_job
dependencies:
- build_linux_35
tags:
- conda-linux
docs_linux_35:
<<: *docs_job
dependencies:
- build_linux_35
tags:
- conda-linux
# Mac OSX + Python 2.7: Builds and tests
build_macosx_27:
<<: *macosx_build_job
variables: &macosx_27_build_variables
<<: *macosx_build_variables
PYTHON_VER: "2.7"
WHEEL_TAG: "py27"
tags:
- conda-macosx
test_macosx_27:
<<: *test_job
variables: *macosx_27_build_variables
dependencies:
- build_macosx_27
tags:
- conda-macosx
# Mac OSX + Python 3.4: Builds and tests
build_macosx_34:
<<: *macosx_build_job
variables: &macosx_34_build_variables
<<: *macosx_build_variables
PYTHON_VER: "3.4"
WHEEL_TAG: "py3"
tags:
- conda-macosx
test_macosx_34:
<<: *test_job
variables: *macosx_34_build_variables
dependencies:
- build_macosx_34
tags:
- conda-macosx
# Mac OSX + Python 3.5: Builds and tests
build_macosx_35:
<<: *macosx_build_job
variables: &macosx_35_build_variables
<<: *macosx_build_variables
PYTHON_VER: "3.5"
WHEEL_TAG: "py3"
tags:
- conda-macosx
test_macosx_35:
<<: *test_job
variables: *macosx_35_build_variables
dependencies:
- build_macosx_35
tags:
- conda-macosx
\ No newline at end of file
include README.rst bootstrap-buildout.py buildout.cfg COPYING version.txt requirements.txt
include README.rst bootstrap-buildout.py buildout.cfg develop.cfg version.txt requirements.txt
recursive-include doc *.py *.rst
recursive-include bob/bio/base/test/data *-dev*
recursive-include bob *.txt *.hdf5
recursive-include bob *.sql3
recursive-include bob/pad/base/test/data scores-* *.sql3
from . import database
from . import algorithm
from . import tools
#from . import grid # only one file, not complete directory
from . import script
from . import test
......
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Pavel Korshunov <pavel.korshunov@idiap.ch>
# @date: Wed 19 Aug 13:43:21 2015
#
# Copyright (C) 2011-2012 Idiap Research Institute, Martigny, Switzerland
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from bob.bio.base.database.Database import Database
import os
import antispoofing.utils.db
class DatabaseBobSpoof(Database):
"""This class can be used whenever you have a database that follows the Bob
antispoofing database interface, which is defined in :py:class:`antispoofing.utils.db.Database`
**Parameters:**
database : derivative of :py:class:`antispoofing.utils.db.Database`
The database instance that provides the actual interface, see :ref:`antispoofing_databases` for a list.
all_files_options : dict
Dictionary of options passed to the :py:meth:`antispoofing.utils.db.Database.objects` database query when retrieving all data.
check_original_files_for_existence : bool
Enables to test for the original data files when querying the database.
kwargs : ``key=value`` pairs
The arguments of the :py:class:`Database` base class constructor.
.. note:: Usually, the ``name``, ``protocol`` keyword parameters of the base class constructor need to be specified.
"""
def __init__(
self,
database, # The bob database that is used
all_files_options={}, # additional options for the database query that can be used to extract all files
original_directory=None, # the directory where the data files are located
check_original_files_for_existence=False,
**kwargs # The default parameters of the base class
):
Database.__init__(
self,
**kwargs
)
assert isinstance(database, antispoofing.utils.db.Database), \
"Only databases derived from antispoofing.utils.db.Database are supported by this interface. " \
"Please implement your own bob.bio.base.database.Database interface for anti-spoofing experiments."
self.database = database
if original_directory is None:
self.original_directory = database.original_directory
else:
self.original_directory = original_directory
self.all_files_options = all_files_options
self.check_existence = check_original_files_for_existence
self._kwargs = kwargs
def set_protocol(self, protocol):
"""
Sets the protocol for the database. The protocol can be specified via command line to spoof.py
script with option -P
:param protocol: name of the protocol
:return: None
"""
self.protocol = protocol
self.database.set_kwargs({'protocol': protocol})
def __str__(self):
"""__str__() -> info
This function returns all parameters of this class (and its derived class).
**Returns:**
info : str
A string containing the full information of all parameters of this (and the derived) class.
"""
params = ", ".join(["%s=%s" % (key, value) for key, value in self._kwargs.items()])
params += ", original_directory=%s" % (self.original_directory)
if self.all_files_options: params += ", all_files_options=%s" % self.all_files_options
return "%s(%s)" % (str(self.__class__), params)
def replace_directories(self, replacements=None):
"""This helper function replaces the ``original_directory`` of the database with
the directory read from the given replacement file.
This function is provided for convenience, so that the database
configuration files do not need to be modified.
Instead, this function uses the given dictionary of replacements to change the original directory.
The given ``replacements`` can be of type ``dict``, including all replacements,
or a file name (as a ``str``), in which case the file is read.
The structure of the file should be:
.. code-block:: text
# Comments starting with # and empty lines are ignored
original/path/to/data = /path/to/your/data
**Parameters:**
replacements : dict or str
A dictionary with replacements, or a name of a file to read the dictionary from.
If the file name does not exist, no directories are replaced.
"""
if replacements is None:
return
if isinstance(replacements, str):
if not os.path.exists(replacements):
return
# Open the database replacement file and reads its content
with open(replacements) as f:
replacements = {}
for line in f:
if line.strip() and not line.startswith("#"):
splits = line.split("=")
assert len(splits) == 2
replacements[splits[0].strip()] = splits[1].strip()
assert isinstance(replacements, dict)
if self.original_directory in replacements:
self.original_directory = replacements[self.original_directory]
self.database.original_directory = self.original_directory
def all_files(self, groups=('train', 'dev', 'eval')):
"""all_files(groups=('train', 'dev', 'eval')) -> files
Returns all files of the database, respecting the current protocol.
**Parameters:**
groups : some of ``('train', 'dev', 'eval')`` or ``None``
The groups to get the data for.
If ``None``, data for all groups is returned.
**Returns:**
files : [:py:class:`antispoofing.utils.db.File`]
The sorted and unique list of all files of the database.
"""
realset = []
attackset = []
if 'train' in groups:
real, attack = self.database.get_train_data()
realset += real
attackset += attack
if 'dev' in groups:
real, attack = self.database.get_devel_data()
realset += real
attackset += attack
if 'eval' in groups:
real, attack = self.database.get_test_data()
realset += real
attackset += attack
return [realset, attackset]
def training_files(self, step=None, arrange_by_client=False):
"""training_files(step = None, arrange_by_client = False) -> files
Returns all training File objects
This function needs to be implemented in derived class implementations.
**Parameters:**
The parameters are not applicable in this version of anti-spoofing experiments
**Returns:**
files : [:py:class:`File`] or [[:py:class:`File`]]
The (arranged) list of files used for the training.
"""
return self.database.get_train_data()
def original_file_names(self, files):
"""original_file_names(files) -> paths
Returns the full paths of the real and attack data of the given File objects.
**Parameters:**
files : [[:py:class:`antispoofing.utils.db.File`], [:py:class:`antispoofing.utils.db.File`]]
The list of lists ([real, attack]]) of file object to retrieve the original data file names for.
**Returns:**
paths : [str]
The paths extracted for the concatenated real+attack files, in the preserved order.
"""
realfiles = files[0]
attackfiles = files[1]
realpaths = [file.make_path(directory=self.original_directory, extension=self.original_extension) for file in
realfiles]
attackpaths = [file.make_path(directory=self.original_directory, extension=self.original_extension) for file in
attackfiles]
return realpaths + attackpaths
#from .utils import File, FileSet
# from bob.bio.base.database.Database import Database
from .DatabaseBobSpoof import DatabaseBobSpoof
from .database import PadDatabase
from .file import PadFile
# gets sphinx autodoc done right - don't remove it
__all__ = [_ for _ in dir() if not _.startswith('_')]
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Manuel Guenther <Manuel.Guenther@idiap.ch>
# @author: Pavel Korshunov <pavel.korshunov@idiap.ch>
# @date: Tue May 17 12:09:22 CET 2016
#
import abc
import bob.bio.base.database
class PadDatabase(bob.bio.base.database.BioDatabase):
def __init__(
self,
name,
all_files_options={}, # additional options for the database query that can be used to extract all files
check_original_files_for_existence=False,
original_directory=None,
original_extension=None,
protocol='Default',
**kwargs # The rest of the default parameters of the base class
):
"""This class represents the basic API for database access.
Please use this class as a base class for your database access classes.
Do not forget to call the constructor of this base class in your derived class.
**Parameters:**
name : str
A unique name for the database.
all_files_options : dict
Dictionary of options passed to the second-level database query when retrieving all data.
check_original_files_for_existence : bool
Enables to test for the original data files when querying the database.
original_directory : str
The directory where the original data of the database are stored.
original_extension : str
The file name extension of the original data.
protocol : str or ``None``
The name of the protocol that defines the default experimental setup for this database.
kwargs : ``key=value`` pairs
The arguments of the :py:class:`bob.bio.base.BioDatabase` base class constructor.
"""
super(PadDatabase, self).__init__(name=name, all_files_options=all_files_options, check_original_files_for_existence=check_original_files_for_existence, original_directory=original_directory, original_extension=original_extension, protocol=protocol, **kwargs)
def original_file_names(self, files):
"""original_file_names(files) -> paths
Returns the full paths of the real and attack data of the given PadFile objects.
**Parameters:**
files : [[:py:class:`bob.pad.db.PadFile`], [:py:class:`bob.pad.db.PadFile`]
The list of lists ([real, attack]) of file object to retrieve the original data file names for.
**Returns:**
paths : [str] or [[str]]
The paths extracted for the concatenated real+attack files, in the preserved order.
"""
assert self.original_directory is not None
assert self.original_extension is not None
realfiles = files[0]
attackfiles = files[1]
realpaths = [file.make_path(directory=self.original_directory, extension=self.original_extension) for file in
realfiles]
attackpaths = [file.make_path(directory=self.original_directory, extension=self.original_extension) for file in
attackfiles]
return realpaths + attackpaths
def model_ids_with_protocol(self, groups=None, protocol=None, **kwargs):
"""model_ids_with_protocol(groups = None, protocol = None, **kwargs) -> ids
Client-based PAD is not implemented.
"""
return []
def annotations(self, file):
"""
Annotations are not supported by PAD interface
"""
return None
@abc.abstractmethod
def objects(self, groups=None, protocol=None, purposes=None, model_ids=None, **kwargs):
"""This function returns lists of File objects, which fulfill the given restrictions.
Keyword parameters:
groups : str or [str]
The groups of which the clients should be returned.
Usually, groups are one or more elements of ('train', 'dev', 'eval')
protocol
The protocol for which the clients should be retrieved.
The protocol is dependent on your database.
If you do not have protocols defined, just ignore this field.
purposes : str or [str]
The purposes for which File objects should be retrieved.
Usually it is either 'real' or 'attack'.
model_ids : [various type]
This parameter is not suported in PAD databases yet
"""
raise NotImplementedError("This function must be implemented in your derived class.")
#################################################################
######### Methods to provide common functionality ###############
#################################################################
def all_files(self, groups=('train', 'dev', 'eval')):
"""all_files(groups=('train', 'dev', 'eval')) -> files
Returns all files of the database, respecting the current protocol.
The files can be limited using the ``all_files_options`` in the constructor.
**Parameters:**
groups : some of ``('train', 'dev', 'eval')`` or ``None``
The groups to get the data for.
**Returns:**
files : [:py:class:`File`]
The sorted and unique list of all files of the database.
"""
realset = self.sort(self.objects(protocol=self.protocol, groups=groups, purposes='real', **self.all_files_options))
attackset = self.sort(self.objects(protocol=self.protocol, groups=groups, purposes='attack', **self.all_files_options))
return [realset, attackset]
def training_files(self, step=None, arrange_by_client=False):
"""training_files(step = None, arrange_by_client = False) -> files
Returns all training File objects
This function needs to be implemented in derived class implementations.
**Parameters:**
The parameters are not applicable in this version of anti-spoofing experiments
**Returns:**
files : [:py:class:`File`] or [[:py:class:`File`]]
The (arranged) list of files used for the training.
"""
return self.all_files(groups=('train',))
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Pavel Korshunov <pavel.korshunov@idiap.ch>
# @date: Wed May 18 10:09:22 CET 2016
#
import bob.bio.base.database
class PadFile(bob.bio.base.database.BioFile):
"""A simple base class that defines basic properties of File object for the use in PAD experiments"""
def __init__(self, client_id, path, attack_type=None, file_id=None):
"""**Constructor Documentation**
Initialize the File object with the minimum required data.
Parameters:
attack_type : a string type
In cased of a spoofed data, this parameter should indicate what kind of spoofed attack it is.
The default None value is interpreted that the PadFile is a genuine or real sample.
For client_id, path and file_id, please refer to :py:class:`bob.bio.base.BioFile` constructor
"""
super(PadFile, self).__init__(client_id, path, file_id)
if attack_type is not None:
assert isinstance(attack_type, str)
# just copy the information
self.attack_type = attack_type
"""The attack type of the sample, None if it is a genuine sample."""
from . import database
from . import database_sql
from . import preprocessor
from . import extractor
from . import algorithm
......@@ -21,8 +21,8 @@ import os
import sys
import six
from bob.pad.db import PadFile
from bob.pad.db import PadDatabase
from bob.pad.base.database import PadFile
from bob.pad.base.database import PadDatabase
import bob.io.base
from bob.db.base.driver import Interface as BaseInterface
......@@ -35,6 +35,11 @@ dummy_train_list = ['train_real', 'train_attack']
dummy_devel_list = ['dev_real', 'dev_attack']
dummy_test_list = ['eval_real', 'eval_attack']
dummy_data = {'train_real': 1.0, 'train_attack': 2.0,
'dev_real': 3.0, 'dev_attack': 4.0,
'eval_real': 5.0, 'eval_attack': 6.0}
class TestFile(PadFile):
def __init__(self, path, id):
attack_type = None
......@@ -42,6 +47,27 @@ class TestFile(PadFile):
attack_type = "attack"
PadFile.__init__(self, client_id=1, path=path, file_id=id, attack_type=attack_type)
def load(self, directory=None, extension='.hdf5'):