Commit 9aec1594 authored by Pavel KORSHUNOV's avatar Pavel KORSHUNOV

Merge branch 'filelist' into 'master'

Adding support for Filelist-based databases

See merge request !9
parents 10d6c6cf ba652edc
Pipeline #8988 passed with stages
in 29 minutes and 52 seconds
......@@ -2,4 +2,5 @@ include README.rst bootstrap-buildout.py buildout.cfg develop.cfg version.txt re
recursive-include doc *.py *.rst
recursive-include bob *.txt *.hdf5
recursive-include bob *.sql3
recursive-include bob/pad/base/test/data scores-* *.sql3
recursive-include bob/pad/base/test/data scores-* *.sql3 *.wav
recursive-include bob/pad/base/test/data/example_filelist *.lst
from .database import PadDatabase
from .file import PadFile
from .database import PadDatabase
from .filelist.query import FileListPadDatabase
from .filelist.models import Client
from . import filelist
# to fix sphinx warnings of not able to find classes, when path is shortened
PadDatabase.__module__ = "bob.pad.base.database"
PadFile.__module__ = "bob.pad.base.database"
# gets sphinx autodoc done right - don't remove it
def __appropriate__(*args):
"""Says object was actually declared here, and not in the import module.
Fixing sphinx warnings of not being able to find classes, when path is shortened.
Parameters:
*args: An iterable of objects to modify
Resolves `Sphinx referencing issues
<https://github.com/sphinx-doc/sphinx/issues/3048>`
"""
for obj in args:
obj.__module__ = __name__
__appropriate__(
PadFile,
PadDatabase,
FileListPadDatabase,
Client,
)
__all__ = [_ for _ in dir() if not _.startswith('_')]
......@@ -21,11 +21,8 @@ class PadDatabase(BioDatabase):
name : str
A unique name for the database.
all_files_options : dict
Dictionary of options passed to the second-level database query when retrieving all data.
check_original_files_for_existence : bool
Enables to test for the original data files when querying the database.
protocol : str or ``None``
The name of the protocol that defines the default experimental setup for this database.
original_directory : str
The directory where the original data of the database are stored.
......@@ -33,9 +30,6 @@ class PadDatabase(BioDatabase):
original_extension : str
The file name extension of the original data.
protocol : str or ``None``
The name of the protocol that defines the default experimental setup for this database.
kwargs : ``key=value`` pairs
The arguments of the :py:class:`bob.bio.base.database.BioDatabase` base class constructor.
......@@ -44,14 +38,18 @@ class PadDatabase(BioDatabase):
def __init__(
self,
name,
all_files_options={}, # additional options for the database query that can be used to extract all files
check_original_files_for_existence=False,
protocol='Default',
original_directory=None,
original_extension=None,
protocol='Default',
**kwargs # The rest of the default parameters of the base class
):
super(PadDatabase, self).__init__(name=name, all_files_options=all_files_options, check_original_files_for_existence=check_original_files_for_existence, original_directory=original_directory, original_extension=original_extension, protocol=protocol, **kwargs)
super(PadDatabase, self).__init__(
name=name,
protocol=protocol,
original_directory=original_directory,
original_extension=original_extension,
**kwargs)
def original_file_names(self, files):
"""original_file_names(files) -> paths
......@@ -124,7 +122,7 @@ class PadDatabase(BioDatabase):
Usually it is either 'real' or 'attack'.
model_ids : [various type]
This parameter is not suported in PAD databases yet
This parameter is not supported in PAD databases yet
"""
raise NotImplementedError("This function must be implemented in your derived class.")
......
......@@ -4,10 +4,9 @@
# @date: Wed May 18 10:09:22 CET 2016
#
from bob.bio.base.database import BioFile
import bob.bio.base.database
class PadFile(BioFile):
class PadFile(bob.bio.base.database.BioFile):
"""A simple base class that defines basic properties of File object for the use in PAD experiments"""
def __init__(self, client_id, path, attack_type=None, file_id=None):
......@@ -29,5 +28,5 @@ class PadFile(BioFile):
assert isinstance(attack_type, str)
# just copy the information
# The attack type of the sample, None if it is a genuine sample.
self.attack_type = attack_type
"""The attack type of the sample, None if it is a genuine sample."""
from .models import ListReader, Client, FileListFile
from .query import FileListPadDatabase
from .driver import Interface
# gets sphinx autodoc done right - don't remove it
def __appropriate__(*args):
"""Says object was actually declared here, and not in the import module.
Fixing sphinx warnings of not being able to find classes, when path is shortened.
Parameters:
*args: An iterable of objects to modify
Resolves `Sphinx referencing issues
<https://github.com/sphinx-doc/sphinx/issues/3048>`
"""
for obj in args:
obj.__module__ = __name__
__appropriate__(
ListReader,
Client,
FileListFile,
FileListPadDatabase,
Interface,
)
__all__ = [_ for _ in dir() if not _.startswith('_')]
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Laurent El Shafey <laurent.el-shafey@idiap.ch>
#
# Copyright (C) 2011-2013 Idiap Research Institute, Martigny, Switzerland
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""Commands the PAD Filelists database can respond to.
"""
import os
import sys
from bob.db.base.driver import Interface as BaseInterface
def dumplist(args):
"""Dumps lists of files based on your criteria"""
from .query import FileListPadDatabase
db = FileListPadDatabase(args.list_directory, 'pad_filelist')
r = db.objects(
purposes=args.purpose,
groups=args.group,
protocol=args.protocol
)
output = sys.stdout
if args.selftest:
from bob.db.base.utils import null
output = null()
for f in r:
output.write('%s\n' % f.make_path(directory=args.directory, extension=args.extension))
return 0
def checkfiles(args):
"""Checks existence of files based on your criteria"""
from .query import FileListPadDatabase
db = FileListPadDatabase(args.list_directory, 'pad_filelist')
r = db.objects(protocol=args.protocol)
# go through all files, check if they are available on the filesystem
good = []
bad = []
for f in r:
if os.path.exists(f.make_path(args.directory, args.extension)):
good.append(f)
else:
bad.append(f)
# report
output = sys.stdout
if args.selftest:
from bob.db.base.utils import null
output = null()
if bad:
for f in bad:
output.write('Cannot find file "%s"\n' % f.make_path(args.directory, args.extension))
output.write('%d files (out of %d) were not found at "%s"\n' % (len(bad), len(r), args.directory))
return 0
class Interface(BaseInterface):
def name(self):
return 'pad_filelist'
def version(self):
import pkg_resources # part of setuptools
return pkg_resources.require('bob.pad.base')[0].version
def files(self):
return ()
def type(self):
return 'text'
def add_commands(self, parser):
from . import __doc__ as docs
subparsers = self.setup_parser(parser,
"Presentation Attack Detection File Lists database", docs)
import argparse
# the "dumplist" action
parser = subparsers.add_parser('dumplist', help=dumplist.__doc__)
parser.add_argument('-l', '--list-directory', required=True,
help="The directory which contains the file lists.")
parser.add_argument('-d', '--directory', default='',
help="if given, this path will be prepended to every entry returned.")
parser.add_argument('-e', '--extension', default='',
help="if given, this extension will be appended to every entry returned.")
parser.add_argument('-u', '--purpose',
help="if given, this value will limit the output files to those designed "
"for the given purposes.",
choices=('real', 'attack', ''))
parser.add_argument('-g', '--group',
help="if given, this value will limit the output files to those belonging to a "
"particular protocolar group.",
choices=('dev', 'eval', 'train', ''))
parser.add_argument('-p', '--protocol', default=None,
help="If set, the protocol is appended to the directory that contains the file lists.")
parser.add_argument('--self-test', dest="selftest", action='store_true', help=argparse.SUPPRESS)
parser.set_defaults(func=dumplist) # action
# the "checkfiles" action
parser = subparsers.add_parser('checkfiles', help=checkfiles.__doc__)
parser.add_argument('-l', '--list-directory', required=True,
help="The directory which contains the file lists.")
parser.add_argument('-d', '--directory', dest="directory", default='',
help="if given, this path will be prepended to every entry returned.")
parser.add_argument('-e', '--extension', dest="extension", default='',
help="if given, this extension will be appended to every entry returned.")
parser.add_argument('-p', '--protocol', default=None,
help="If set, the protocol is appended to the directory that contains the file lists.")
parser.add_argument('--self-test', dest="selftest", action='store_true', help=argparse.SUPPRESS)
parser.set_defaults(func=checkfiles) # action
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Pavel Korshunov <pavel.korshunov@idiap.ch>
# @date: Thu Nov 17 16:09:22 CET 2016
#
# Copyright (C) 2011-2013 Idiap Research Institute, Martigny, Switzerland
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
This file defines simple Client and File interfaces that are comparable with other bob.db databases.
"""
import os
import fileinput
import re
from bob.pad.base.database import PadFile
class Client(object):
"""
The clients of this database contain ONLY client ids. Nothing special.
"""
def __init__(self, client_id):
self.id = client_id
"""The ID of the client, which is stored as a :py:class:`str` object."""
class FileListFile(PadFile):
"""
Initialize the File object with the minimum required data.
**Parameters**
path : str
The path of this file, relative to the basic directory.
Please do not specify any file extensions.
This path will be used as an underlying file_id, as it is assumed to be unique
client_id : various type
The id of the client, this file belongs to.
The type of it is dependent on your implementation.
If you use an SQL database, this should be an SQL type like Integer or String.
"""
def __init__(self, file_name, client_id, attack_type=None):
super(FileListFile, self).__init__(client_id=client_id, path=file_name, attack_type=attack_type, file_id=file_name)
#############################################################################
### internal access functions for the file lists; do not export!
#############################################################################
class ListReader(object):
def __init__(self, store_lists):
self.m_read_lists = {}
self.m_store_lists = store_lists
def _read_multi_column_list(self, list_file):
rows = []
if not os.path.isfile(list_file):
raise RuntimeError('File %s does not exist.' % (list_file,))
try:
for line in fileinput.input(list_file):
parsed_line = re.findall('[\w/(-.)]+', line)
if len(parsed_line):
# perform some sanity checks
if len(parsed_line) not in (2, 3):
raise IOError("The read line '%s' from file '%s' could not be parsed successfully!" %
(line.rstrip(), list_file))
if len(rows) and len(rows[0]) != len(parsed_line):
raise IOError("The parsed line '%s' from file '%s' has a different number of elements "
"than the first parsed line '%s'!" % (parsed_line, list_file, rows[0]))
# append the read line
rows.append(parsed_line)
fileinput.close()
except IOError as e:
raise RuntimeError("Error reading the file '%s' : '%s'." % (list_file, e))
# return the read list as a vector of columns
return rows
def _read_column_list(self, list_file, column_count):
# read the list
rows = self._read_multi_column_list(list_file)
# extract the file from the first two columns
file_list = []
for row in rows:
if column_count == 2:
assert len(row) == 2
# we expect: filename client_id
file_list.append(FileListFile(file_name=row[0], client_id=row[1]))
elif column_count == 3:
assert len(row) == 3
# we expect: filename, model_id, client_id
file_list.append(FileListFile(file_name=row[0], client_id=row[1], attack_type=row[2]))
else:
raise ValueError("The given column count %d cannot be interpreted. This is a BUG, please "
"report to the author." % column_count)
return file_list
def read_list(self, list_file, group, type=None):
"""Reads the list of Files from the given list file (if not done yet) and returns it."""
if group not in self.m_read_lists:
self.m_read_lists[group] = {}
if type not in self.m_read_lists[group]:
if type == 'for_real':
files_list = self._read_column_list(list_file, 2)
elif type == 'for_attack':
files_list = self._read_column_list(list_file, 3)
else:
raise ValueError("The given type must be one of %s, but not '%s'" % (('for_real', 'for_attack'), type))
if self.m_store_lists:
self.m_read_lists[group][type] = files_list
return files_list
return self.m_read_lists[group][type]
This diff is collapsed.
from bob.pad.base.database import FileListPadDatabase
import pkg_resources
database = FileListPadDatabase(
name='test_filelist',
protocol=None,
filelists_directory=pkg_resources.resource_filename('bob.pad.base.test', 'data/example_filelist'),
original_directory=pkg_resources.resource_filename('bob.pad.base.test', 'data'),
original_extension=".wav",
train_subdir='.',
dev_subdir='.',
eval_subdir='.',
real_filename='for_real.lst',
attack_filename='for_attack.lst',
keep_read_lists_in_memory=True,
check_original_files_for_existence=True,
training_depends_on_protocol=False,
models_depend_on_protocol=False
)
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Pavel Korshunov <pavel.korshunov@idiap.ch>
# @date: Thu Nov 17 16:09:22 CET 2016
#
# Copyright (C) 2011-2013 Idiap Research Institute, Martigny, Switzerland
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
Tests for the PAD Filelist database.
"""
import os
import bob.io.base.test_utils
from bob.pad.base.database import FileListPadDatabase
example_dir = os.path.realpath(bob.io.base.test_utils.datafile('.', __name__, 'data/example_filelist'))
def test_query():
db = FileListPadDatabase(example_dir, 'test_padfilelist')
assert len(db.groups()) == 3 # 3 groups (dev, eval, train)
print(db.client_ids())
# 5 client ids for real data of train, dev and eval sets (ignore all ids that are in attacks only)
assert len(db.client_ids()) == 5
assert len(db.client_ids(groups='train')) == 2 # 2 client ids for train
assert len(db.client_ids(groups='dev')) == 2 # 2 client ids for dev
assert len(db.client_ids(groups='eval')) == 1 # 2 client ids for eval
assert len(db.objects(groups='train')) == 3 # 3 samples in the train set
assert len(db.objects(groups='dev', purposes='real')) == 2 # 2 samples of real data in the dev set
assert len(db.objects(groups='dev', purposes='attack')) == 1 # 1 attack in the dev set
def test_query_protocol():
db = FileListPadDatabase(os.path.dirname(example_dir), 'test_padfilelist')
p = 'example_filelist'
assert len(db.groups(protocol=p)) == 3 # 3 groups (dev, eval, train)
assert len(db.client_ids(protocol=p)) == 5 # 6 client ids for train, dev and eval
assert len(db.client_ids(groups='train', protocol=p)) == 2 # 2 client ids for train
assert len(db.client_ids(groups='dev', protocol=p)) == 2 # 2 client ids for dev
assert len(db.client_ids(groups='eval', protocol=p)) == 1 # 2 client ids for eval
assert len(db.objects(groups='train', protocol=p)) == 3 # 3 samples in the train set
assert len(db.objects(groups='dev', purposes='real', protocol=p)) == 2 # 2 samples of real data in the dev set
assert len(db.objects(groups='dev', purposes='attack', protocol=p)) == 1 # 1 attack in the dev set
def test_driver_api():
from bob.db.base.script.dbmanage import main
assert main(('pad_filelist dumplist --list-directory=%s --self-test' % example_dir).split()) == 0
assert main(('pad_filelist dumplist --list-directory=%s --purpose=real --group=dev --self-test' %
example_dir).split()) == 0
assert main(('pad_filelist checkfiles --list-directory=%s --self-test' % example_dir).split()) == 0
......@@ -16,7 +16,6 @@ import tempfile
import numpy
import bob.io.base.test_utils
import bob.io.image
import bob.bio.base
import bob.pad.base
from bob.bio.base.test import utils
......
......@@ -3,4 +3,5 @@ numpy
bob.bio.spear
gridtk
bob.db.base
bob.db.avspoof
\ No newline at end of file
bob.db.avspoof
bob.bio.base
\ No newline at end of file
......@@ -109,6 +109,7 @@ setup(
# bob database declaration
'bob.db': [
'dummy = bob.pad.base.test.dummy.database:Interface', # driver for bobdb_manage
'pad_filelist = bob.pad.base.database.filelist.driver:Interface',
],
'bob.pad.database': [
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment