Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
bob
bob.pad.base
Commits
749c6799
Commit
749c6799
authored
Sep 20, 2016
by
Pavel KORSHUNOV
Browse files
Update the existing files to new DB interface
parent
3312b848
Pipeline
#3822
failed with stages
in 38 minutes and 56 seconds
Changes
11
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
MANIFEST.in
View file @
749c6799
include README.rst bootstrap-buildout.py buildout.cfg
COPYING
version.txt requirements.txt
include README.rst bootstrap-buildout.py buildout.cfg
develop.cfg
version.txt requirements.txt
recursive-include doc *.py *.rst
recursive-include bob/bio/base/test/data *-dev*
recursive-include bob *.txt *.hdf5
recursive-include bob *.sql3
bob/pad/base/__init__.py
View file @
749c6799
from
.
import
database
from
.
import
algorithm
from
.
import
tools
#from . import grid # only one file, not complete directory
from
.
import
script
from
.
import
test
...
...
bob/pad/base/database/DatabaseBobSpoof.py
deleted
100644 → 0
View file @
3312b848
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Pavel Korshunov <pavel.korshunov@idiap.ch>
# @date: Wed 19 Aug 13:43:21 2015
#
# Copyright (C) 2011-2012 Idiap Research Institute, Martigny, Switzerland
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from
bob.bio.base.database.Database
import
Database
import
os
import
antispoofing.utils.db
class
DatabaseBobSpoof
(
Database
):
"""This class can be used whenever you have a database that follows the Bob
antispoofing database interface, which is defined in :py:class:`antispoofing.utils.db.Database`
**Parameters:**
database : derivative of :py:class:`antispoofing.utils.db.Database`
The database instance that provides the actual interface, see :ref:`antispoofing_databases` for a list.
all_files_options : dict
Dictionary of options passed to the :py:meth:`antispoofing.utils.db.Database.objects` database query when retrieving all data.
check_original_files_for_existence : bool
Enables to test for the original data files when querying the database.
kwargs : ``key=value`` pairs
The arguments of the :py:class:`Database` base class constructor.
.. note:: Usually, the ``name``, ``protocol`` keyword parameters of the base class constructor need to be specified.
"""
def
__init__
(
self
,
database
,
# The bob database that is used
all_files_options
=
{},
# additional options for the database query that can be used to extract all files
original_directory
=
None
,
# the directory where the data files are located
check_original_files_for_existence
=
False
,
**
kwargs
# The default parameters of the base class
):
Database
.
__init__
(
self
,
**
kwargs
)
assert
isinstance
(
database
,
antispoofing
.
utils
.
db
.
Database
),
\
"Only databases derived from antispoofing.utils.db.Database are supported by this interface. "
\
"Please implement your own bob.bio.base.database.Database interface for anti-spoofing experiments."
self
.
database
=
database
if
original_directory
is
None
:
self
.
original_directory
=
database
.
original_directory
else
:
self
.
original_directory
=
original_directory
self
.
all_files_options
=
all_files_options
self
.
check_existence
=
check_original_files_for_existence
self
.
_kwargs
=
kwargs
def
set_protocol
(
self
,
protocol
):
"""
Sets the protocol for the database. The protocol can be specified via command line to spoof.py
script with option -P
:param protocol: name of the protocol
:return: None
"""
self
.
protocol
=
protocol
self
.
database
.
set_kwargs
({
'protocol'
:
protocol
})
def
__str__
(
self
):
"""__str__() -> info
This function returns all parameters of this class (and its derived class).
**Returns:**
info : str
A string containing the full information of all parameters of this (and the derived) class.
"""
params
=
", "
.
join
([
"%s=%s"
%
(
key
,
value
)
for
key
,
value
in
self
.
_kwargs
.
items
()])
params
+=
", original_directory=%s"
%
(
self
.
original_directory
)
if
self
.
all_files_options
:
params
+=
", all_files_options=%s"
%
self
.
all_files_options
return
"%s(%s)"
%
(
str
(
self
.
__class__
),
params
)
def
replace_directories
(
self
,
replacements
=
None
):
"""This helper function replaces the ``original_directory`` of the database with
the directory read from the given replacement file.
This function is provided for convenience, so that the database
configuration files do not need to be modified.
Instead, this function uses the given dictionary of replacements to change the original directory.
The given ``replacements`` can be of type ``dict``, including all replacements,
or a file name (as a ``str``), in which case the file is read.
The structure of the file should be:
.. code-block:: text
# Comments starting with # and empty lines are ignored
original/path/to/data = /path/to/your/data
**Parameters:**
replacements : dict or str
A dictionary with replacements, or a name of a file to read the dictionary from.
If the file name does not exist, no directories are replaced.
"""
if
replacements
is
None
:
return
if
isinstance
(
replacements
,
str
):
if
not
os
.
path
.
exists
(
replacements
):
return
# Open the database replacement file and reads its content
with
open
(
replacements
)
as
f
:
replacements
=
{}
for
line
in
f
:
if
line
.
strip
()
and
not
line
.
startswith
(
"#"
):
splits
=
line
.
split
(
"="
)
assert
len
(
splits
)
==
2
replacements
[
splits
[
0
].
strip
()]
=
splits
[
1
].
strip
()
assert
isinstance
(
replacements
,
dict
)
if
self
.
original_directory
in
replacements
:
self
.
original_directory
=
replacements
[
self
.
original_directory
]
self
.
database
.
original_directory
=
self
.
original_directory
def
all_files
(
self
,
groups
=
(
'train'
,
'dev'
,
'eval'
)):
"""all_files(groups=('train', 'dev', 'eval')) -> files
Returns all files of the database, respecting the current protocol.
**Parameters:**
groups : some of ``('train', 'dev', 'eval')`` or ``None``
The groups to get the data for.
If ``None``, data for all groups is returned.
**Returns:**
files : [:py:class:`antispoofing.utils.db.File`]
The sorted and unique list of all files of the database.
"""
realset
=
[]
attackset
=
[]
if
'train'
in
groups
:
real
,
attack
=
self
.
database
.
get_train_data
()
realset
+=
real
attackset
+=
attack
if
'dev'
in
groups
:
real
,
attack
=
self
.
database
.
get_devel_data
()
realset
+=
real
attackset
+=
attack
if
'eval'
in
groups
:
real
,
attack
=
self
.
database
.
get_test_data
()
realset
+=
real
attackset
+=
attack
return
[
realset
,
attackset
]
def
training_files
(
self
,
step
=
None
,
arrange_by_client
=
False
):
"""training_files(step = None, arrange_by_client = False) -> files
Returns all training File objects
This function needs to be implemented in derived class implementations.
**Parameters:**
The parameters are not applicable in this version of anti-spoofing experiments
**Returns:**
files : [:py:class:`File`] or [[:py:class:`File`]]
The (arranged) list of files used for the training.
"""
return
self
.
database
.
get_train_data
()
def
original_file_names
(
self
,
files
):
"""original_file_names(files) -> paths
Returns the full paths of the real and attack data of the given File objects.
**Parameters:**
files : [[:py:class:`antispoofing.utils.db.File`], [:py:class:`antispoofing.utils.db.File`]]
The list of lists ([real, attack]]) of file object to retrieve the original data file names for.
**Returns:**
paths : [str]
The paths extracted for the concatenated real+attack files, in the preserved order.
"""
realfiles
=
files
[
0
]
attackfiles
=
files
[
1
]
realpaths
=
[
file
.
make_path
(
directory
=
self
.
original_directory
,
extension
=
self
.
original_extension
)
for
file
in
realfiles
]
attackpaths
=
[
file
.
make_path
(
directory
=
self
.
original_directory
,
extension
=
self
.
original_extension
)
for
file
in
attackfiles
]
return
realpaths
+
attackpaths
bob/pad/base/test/dummy/__init__.py
View file @
749c6799
from
.
import
database
from
.
import
database_sql
from
.
import
preprocessor
from
.
import
extractor
from
.
import
algorithm
bob/pad/base/test/dummy/database.py
View file @
749c6799
...
...
@@ -21,8 +21,8 @@ import os
import
sys
import
six
from
bob.pad.
d
b
import
PadFile
from
bob.pad.
d
b
import
PadDatabase
from
bob.pad.b
ase.database
import
PadFile
from
bob.pad.b
ase.database
import
PadDatabase
import
bob.io.base
from
bob.db.base.driver
import
Interface
as
BaseInterface
...
...
@@ -35,6 +35,11 @@ dummy_train_list = ['train_real', 'train_attack']
dummy_devel_list
=
[
'dev_real'
,
'dev_attack'
]
dummy_test_list
=
[
'eval_real'
,
'eval_attack'
]
dummy_data
=
{
'train_real'
:
1.0
,
'train_attack'
:
2.0
,
'dev_real'
:
3.0
,
'dev_attack'
:
4.0
,
'eval_real'
:
5.0
,
'eval_attack'
:
6.0
}
class
TestFile
(
PadFile
):
def
__init__
(
self
,
path
,
id
):
attack_type
=
None
...
...
@@ -42,6 +47,27 @@ class TestFile(PadFile):
attack_type
=
"attack"
PadFile
.
__init__
(
self
,
client_id
=
1
,
path
=
path
,
file_id
=
id
,
attack_type
=
attack_type
)
def
load
(
self
,
directory
=
None
,
extension
=
'.hdf5'
):
"""Loads the data at the specified location and using the given extension.
Override it if you need to load differently.
Keyword Parameters:
data
The data blob to be saved (normally a :py:class:`numpy.ndarray`).
directory
[optional] If not empty or None, this directory is prefixed to the final
file destination
extension
[optional] The extension of the filename - this will control the type of
output and the codec for saving the input blob.
"""
# get the path
path
=
self
.
make_path
(
directory
or
''
,
extension
or
''
)
return
dummy_data
[
os
.
path
.
basename
(
path
)]
def
dumplist
(
args
):
"""Dumps lists of files based on your criteria"""
...
...
bob/pad/base/test/dummy/preprocessor.py
View file @
749c6799
...
...
@@ -33,8 +33,5 @@ class DummyPreprocessor(Preprocessor):
"""Does nothing, simply converts the data type of the data, ignoring any annotation."""
return
data
def
read_original_data
(
self
,
original_file_name
):
return
dummy_data
[
os
.
path
.
basename
(
original_file_name
)]
preprocessor
=
DummyPreprocessor
()
bob/pad/base/tools/FileSelector.py
View file @
749c6799
...
...
@@ -107,9 +107,18 @@ class FileSelector:
# List of files that will be used for all files
def
original_data_list
(
self
,
groups
=
None
):
"""Returns the t
uple of
list
s
of original (real
,
a
ttack) data that can be used for preprocessing
."""
"""Returns the t
he joint
list of original (real a
nd attack) file names
."""
return
self
.
database
.
original_file_names
(
self
.
database
.
all_files
(
groups
=
groups
))
def
original_data_list_files
(
self
,
groups
=
None
):
"""Returns the joint list of original (real and attack) data files that can be used for preprocessing."""
files
=
self
.
database
.
all_files
(
groups
=
groups
)
if
len
(
files
)
!=
2
:
fileset
=
files
else
:
fileset
=
files
[
0
]
+
files
[
1
]
return
fileset
,
self
.
database
.
original_directory
,
self
.
database
.
original_extension
def
preprocessed_data_list
(
self
,
groups
=
None
):
"""Returns the tuple of lists (real, attacks) of preprocessed data files."""
return
self
.
get_paths
(
self
.
database
.
all_files
(
groups
=
groups
),
"preprocessed"
)
...
...
bob/pad/base/tools/command_line.py
View file @
749c6799
...
...
@@ -26,6 +26,8 @@ import bob.core
logger
=
bob
.
core
.
log
.
setup
(
"bob.pad.base"
)
from
bob.pad.base.database
import
PadDatabase
from
bob.bio.base
import
utils
from
.
import
FileSelector
from
..
import
database
...
...
@@ -264,7 +266,7 @@ def initialize(parsers, command_line_parameters=None, skips=[]):
projector_sub_dir
=
extractor_sub_dir
# Database directories, which should be automatically replaced
if
isinstance
(
args
.
database
,
database
.
DatabaseBobSpoof
):
if
isinstance
(
args
.
database
,
PadDatabase
):
args
.
database
.
replace_directories
(
args
.
database_directories_file
)
# initialize the file selector
...
...
bob/pad/base/tools/preprocessor.py
View file @
749c6799
...
...
@@ -55,8 +55,10 @@ def preprocess(preprocessor, groups=None, indices=None, force=False):
fs
=
FileSelector
.
instance
()
# get the file lists
data_files
=
fs
.
original_data_list
(
groups
=
groups
)
data_files
,
original_directory
,
original_extension
=
fs
.
original_data_list
_files
(
groups
=
groups
)
preprocessed_data_files
=
fs
.
preprocessed_data_list
(
groups
=
groups
)
print
(
"len of data files: %s"
%
(
str
(
len
(
data_files
))))
print
(
"len of preprocessed data files (paths): %s"
%
(
str
(
len
(
preprocessed_data_files
))))
# select a subset of keys to iterate
if
indices
is
not
None
:
...
...
@@ -71,18 +73,23 @@ def preprocess(preprocessor, groups=None, indices=None, force=False):
# iterate over the selected files
for
i
in
index_range
:
preprocessed_data_file
=
str
(
preprocessed_data_files
[
i
])
file_object
=
data_files
[
i
]
file_name
=
file_object
.
make_path
(
original_directory
,
original_extension
)
# check for existence
if
not
utils
.
check_file
(
preprocessed_data_file
,
force
,
1000
):
file_name
=
data_files
[
i
]
data
=
preprocessor
.
read_original_data
(
file_name
)
logger
.
info
(
"... Processing original data file '%s'"
,
file_name
)
data
=
preprocessor
.
read_original_data
(
file_object
,
original_directory
,
original_extension
)
# create output directory before reading the data file (is sometimes required, when relative directories are specified, especially, including a .. somewhere)
bob
.
io
.
base
.
create_directories_safe
(
os
.
path
.
dirname
(
preprocessed_data_file
))
# call the preprocessor
logger
.
info
(
"- Preprocessor: processing file: %s"
,
file_name
)
preprocessed_data
=
preprocessor
(
data
,
None
)
if
preprocessed_data
is
None
:
logger
.
error
(
"Preprocessing of file '%s' was not successful"
,
file_name
)
continue
# write the data
bob
.
io
.
base
.
create_directories_safe
(
os
.
path
.
dirname
(
preprocessed_data_file
))
preprocessor
.
write_data
(
preprocessed_data
,
preprocessed_data_file
)
...
...
buildout.cfg
View file @
749c6799
; vim: set fileencoding=utf-8 :
; Pavel Korshunov <Pavel.Korshunov@idiap.ch>
; Wed 19 Aug 13:43:22 2015
; Tue 16 Aug 15:00:20 CEST 2016
[buildout]
parts = scripts
develop = .
eggs = bob.pad.base
gridtk
extensions = bob.buildout
mr.developer
auto-checkout = *
develop = src/bob.db.base
src/bob.bio.base
src/bob.bio.db
src/bob.pad.db
.
; options for bob.buildout
debug = true
verbose = true
newest = false
[sources]
bob.db.base = git branch=refactoring_2016 git@github.com:bioidiap/bob.db.base.git
bob.bio.base = git https://github.com/bioidiap/bob.bio.base
bob.bio.db = git git@gitlab.idiap.ch:biometric/bob.bio.db.git
bob.pad.db = git git@gitlab.idiap.ch:biometric/bob.pad.db.git
verbose = true
[scripts]
recipe = bob.buildout:scripts
...
...
requirements.txt
View file @
749c6799
setuptools
bob.extension
bob.io.base
bob.db.base
bob.bio.base
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment