Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
bob
bob.pad.base
Commits
0f669bce
Commit
0f669bce
authored
May 16, 2017
by
Pavel KORSHUNOV
Browse files
Merge branch 'config-support' into 'master'
Support for configuration files See merge request
!13
parents
ee0abff6
de6231d8
Pipeline
#9279
passed with stages
in 9 minutes and 45 seconds
Changes
5
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
bob/pad/base/database/filelist/query.py
View file @
0f669bce
...
...
@@ -30,7 +30,7 @@ from .. import PadFile
from
..
import
PadDatabase
from
bob.bio.base.database
import
FileListBioDatabase
class
FileListPadDatabase
(
FileListBioDatabase
,
Pad
Database
):
class
FileListPadDatabase
(
PadDatabase
,
FileListBio
Database
):
"""This class provides a user-friendly interface to databases that are given as file lists.
Keyword parameters:
...
...
@@ -160,7 +160,7 @@ class FileListPadDatabase(FileListBioDatabase, PadDatabase):
return
[
self
.
pad_file_class
(
client_id
=
f
.
client_id
,
path
=
f
.
path
,
attack_type
=
f
.
attack_type
,
file_id
=
f
.
id
)
for
f
in
files
]
def
groups
(
self
,
protocol
=
None
):
def
groups
(
self
,
protocol
=
None
,
add_world
=
False
,
add_subworld
=
False
):
"""This function returns the list of groups for this database.
protocol : str or ``None``
...
...
@@ -272,7 +272,7 @@ class FileListPadDatabase(FileListBioDatabase, PadDatabase):
return
self
.
_make_pad
(
retval
)
def
annotations
(
self
,
file
):
return
super
(
FileList
Pad
Database
,
self
)
.
annotations
(
file
)
return
FileList
Bio
Database
.
annotations
(
self
,
file
)
def
tobjects
(
self
,
groups
=
None
,
protocol
=
None
,
model_ids
=
None
,
**
kwargs
):
pass
...
...
bob/pad/base/test/dummy/database.py
View file @
0f669bce
...
...
@@ -34,8 +34,7 @@ class TestFile(PadFile):
attack_type
=
None
if
"attack"
in
path
:
attack_type
=
"attack"
PadFile
.
__init__
(
self
,
client_id
=
1
,
path
=
path
,
file_id
=
id
,
attack_type
=
attack_type
)
super
(
TestFile
,
self
).
__init__
(
client_id
=
1
,
path
=
path
,
file_id
=
id
,
attack_type
=
attack_type
)
def
load
(
self
,
directory
=
None
,
extension
=
'.hdf5'
):
"""Loads the data at the specified location and using the given extension.
...
...
@@ -100,9 +99,11 @@ class Interface(BaseInterface):
dumpparser
=
subparsers
.
add_parser
(
'dumplist'
,
help
=
""
)
dumpparser
.
add_argument
(
'-d'
,
'--directory'
,
dest
=
"directory"
,
default
=
''
,
help
=
"if given, this path will be prepended to every entry returned (defaults to '%(default)s')"
)
help
=
"if given, this path will be prepended to every entry returned "
"(defaults to '%(default)s')"
)
dumpparser
.
add_argument
(
'-e'
,
'--extension'
,
dest
=
"extension"
,
default
=
''
,
help
=
"if given, this extension will be appended to every entry returned (defaults to '%(default)s')"
)
help
=
"if given, this extension will be appended to every entry returned "
"(defaults to '%(default)s')"
)
dumpparser
.
add_argument
(
'--self-test'
,
dest
=
"selftest"
,
default
=
False
,
action
=
'store_true'
,
help
=
SUPPRESS
)
...
...
@@ -199,4 +200,4 @@ class TestDatabase(PadDatabase):
return
None
database
=
TestDatabase
(
original_directory
=
data_dir
,
original_extension
=
''
)
database
=
TestDatabase
()
\ No newline at end of file
bob/pad/base/test/dummy/database_sql.py
View file @
0f669bce
...
...
@@ -46,8 +46,10 @@ def create_database():
class
TestDatabaseSql
(
bob
.
pad
.
base
.
database
.
PadDatabase
,
bob
.
db
.
base
.
SQLiteBaseDatabase
):
def
__init__
(
self
):
bob
.
pad
.
base
.
database
.
PadDatabase
.
__init__
(
self
,
'pad_test'
,
original_directory
=
"original/directory"
,
original_extension
=
".orig"
)
bob
.
pad
.
base
.
database
.
PadDatabase
.
__init__
(
self
,
'pad_test'
,
original_directory
=
"original/directory"
,
original_extension
=
".orig"
)
bob
.
db
.
base
.
SQLiteBaseDatabase
.
__init__
(
self
,
dbfile
,
TestFileSql
)
def
groups
(
self
,
protocol
=
None
):
...
...
@@ -59,4 +61,4 @@ class TestDatabaseSql (bob.pad.base.database.PadDatabase, bob.db.base.SQLiteBase
def
annotations
(
self
,
file
):
return
None
database
=
TestDatabaseSql
()
database
=
TestDatabaseSql
()
\ No newline at end of file
bob/pad/base/tools/command_line.py
View file @
0f669bce
...
...
@@ -5,6 +5,10 @@
# @date: Wed 19 Aug 13:43:21 2015
#
"""
Execute PAD algorithms on a database with presentation attacks.
"""
import
argparse
import
os
...
...
@@ -16,12 +20,9 @@ logger = bob.core.log.setup("bob.pad.base")
from
bob.pad.base.database
import
PadDatabase
from
bob.bio.base
import
utils
from
.
import
FileSelector
from
..
import
database
"""Execute spoofmetric recognition algorithms on a certain spoofmetric database.
"""
from
bob.bio.base
import
tools
def
is_idiap
():
...
...
@@ -62,36 +63,8 @@ def command_line_parser(description=__doc__, exclude_resources_from=[]):
#######################################################################################
############## options that are required to be specified #######################
config_group
=
parser
.
add_argument_group
(
'
\n
Parameters defining the experiment. Most of these parameters can be a registered resource, a '
'configuration file, or even a string that defines a newly created object'
)
config_group
.
add_argument
(
'-d'
,
'--database'
,
metavar
=
'x'
,
nargs
=
'+'
,
required
=
True
,
help
=
'Database and the protocol; registered databases are: %s'
%
utils
.
resource_keys
(
'database'
,
exclude_resources_from
,
package_prefix
=
'bob.pad.'
))
config_group
.
add_argument
(
'-p'
,
'--preprocessor'
,
metavar
=
'x'
,
nargs
=
'+'
,
required
=
True
,
help
=
'Data preprocessing; registered preprocessors are: %s'
%
utils
.
resource_keys
(
'preprocessor'
,
exclude_resources_from
,
package_prefix
=
'bob.pad.'
))
config_group
.
add_argument
(
'-e'
,
'--extractor'
,
metavar
=
'x'
,
nargs
=
'+'
,
required
=
True
,
help
=
'Feature extraction; registered feature extractors are: %s'
%
utils
.
resource_keys
(
'extractor'
,
exclude_resources_from
,
package_prefix
=
'bob.pad.'
))
config_group
.
add_argument
(
'-a'
,
'--algorithm'
,
metavar
=
'x'
,
nargs
=
'+'
,
required
=
True
,
help
=
'Anti-spoofing registered algorithms are: %s'
%
utils
.
resource_keys
(
'algorithm'
,
exclude_resources_from
,
package_prefix
=
'bob.pad.'
))
config_group
.
add_argument
(
'-g'
,
'--grid'
,
metavar
=
'x'
,
nargs
=
'+'
,
help
=
'Configuration for the grid setup; if not specified, the commands are '
'executed sequentially on the local machine.'
)
config_group
.
add_argument
(
'--imports'
,
metavar
=
'LIB'
,
nargs
=
'+'
,
default
=
[
'bob.pad.base'
],
help
=
'If one of your configuration files is an actual command, please specify the '
'lists of required libraries (imports) to execute this command'
)
config_group
.
add_argument
(
'-s'
,
'--sub-directory'
,
metavar
=
'DIR'
,
required
=
True
,
help
=
'The sub-directory where the files of the current experiment should be stored. '
'Please specify a directory name with a name describing your experiment.'
)
config_group
.
add_argument
(
'--groups'
,
metavar
=
'GROUP'
,
nargs
=
'+'
,
default
=
[
'dev'
],
help
=
"The groups (i.e., 'train', 'dev', 'eval') for which the models and scores "
"should be generated; by default, only the 'dev' group is evaluated"
)
config_group
.
add_argument
(
'-P'
,
'--protocol'
,
metavar
=
'PROTOCOL'
,
help
=
'Overwrite the protocol that is stored in the database by the given one '
'(might not by applicable for all databases).'
)
config_group
=
tools
.
command_line_config_group
(
parser
,
package_prefix
=
'bob.pad.'
,
exclude_resources_from
=
exclude_resources_from
)
#######################################################################################
############## options to modify default directories or file names ####################
...
...
@@ -170,6 +143,12 @@ def command_line_parser(description=__doc__, exclude_resources_from=[]):
flag_group
.
add_argument
(
'-A'
,
'--allow-missing-files'
,
action
=
'store_true'
,
help
=
"If given, missing files will not stop the processing; this is helpful if not "
"all files of the database can be processed; missing scores will be NaN."
)
flag_group
.
add_argument
(
'-r'
,
'--parallel'
,
type
=
int
,
help
=
'This flag is a shortcut for running the commands on the local machine with '
'the given amount of parallel threads; equivalent to --grid '
'bob.bio.base.grid.Grid("local", number_of_parallel_threads=X) '
'--run-local-scheduler --stop-on-failure.'
)
flag_group
.
add_argument
(
'-t'
,
'--environment'
,
dest
=
'env'
,
nargs
=
'*'
,
default
=
[],
help
=
'Passes specific environment variables to the job.'
)
...
...
@@ -213,61 +192,50 @@ def initialize(parsers, command_line_parameters=None, skips=[]):
args : namespace
A namespace of arguments as read from the command line.
.. note::
The database, preprocessor, extractor, algorithm and grid (if specified) are actual
instances of the according classes.
.. note:: The database, preprocessor, extractor, algorithm and grid (if specified) are actual
instances of the according classes.
"""
# execute-only
if
skips
is
not
None
:
#######################################################################################
################# options for skipping parts of the toolchain #########################
skip_group
=
parsers
[
'main'
].
add_argument_group
(
'
\n
Flags that allow to skip certain parts of the experiments. This does only make sense when the '
'generated files are already there (e.g. when reusing parts of other experiments)'
)
for
skip
in
skips
:
skip_group
.
add_argument
(
'--skip-%s'
%
skip
,
action
=
'store_true'
,
help
=
'Skip the %s step.'
%
skip
)
skip_group
.
add_argument
(
'-o'
,
'--execute-only'
,
nargs
=
'+'
,
choices
=
skips
,
help
=
'If specified, executes only the given parts of the tool chain.'
)
args
=
parsers
[
'main'
].
parse_args
(
command_line_parameters
)
# evaluate skips
if
skips
is
not
None
and
args
.
execute_only
is
not
None
:
for
skip
in
skips
:
if
skip
not
in
args
.
execute_only
:
exec
(
"args.skip_%s = True"
%
(
skip
.
replace
(
"-"
,
"_"
)))
# logging
bob
.
core
.
log
.
set_verbosity_level
(
logger
,
args
.
verbose
)
# timer
if
args
.
timer
is
not
None
and
not
len
(
args
.
timer
):
args
.
timer
=
(
'real'
,
'system'
,
'user'
)
# load configuration resources
args
.
database
=
utils
.
load_resource
(
' '
.
join
(
args
.
database
),
'database'
,
imports
=
args
.
imports
,
package_prefix
=
'bob.pad.'
)
args
.
preprocessor
=
utils
.
load_resource
(
' '
.
join
(
args
.
preprocessor
),
'preprocessor'
,
imports
=
args
.
imports
,
package_prefix
=
'bob.pad.'
)
args
.
extractor
=
utils
.
load_resource
(
' '
.
join
(
args
.
extractor
),
'extractor'
,
imports
=
args
.
imports
,
package_prefix
=
'bob.pad.'
)
args
.
algorithm
=
utils
.
load_resource
(
' '
.
join
(
args
.
algorithm
),
'algorithm'
,
imports
=
args
.
imports
,
package_prefix
=
'bob.pad.'
)
if
args
.
grid
is
not
None
:
args
.
grid
=
utils
.
load_resource
(
' '
.
join
(
args
.
grid
),
'grid'
,
imports
=
args
.
imports
,
package_prefix
=
'bob.pad.'
)
# set base directories
if
args
.
temp_directory
is
None
:
args
.
temp_directory
=
"/idiap/temp/%s/%s"
%
(
os
.
environ
[
"USER"
],
args
.
database
.
name
)
if
is_idiap
()
else
"temp"
if
args
.
result_directory
is
None
:
args
.
result_directory
=
"/idiap/user/%s/%s"
%
(
os
.
environ
[
"USER"
],
args
.
database
.
name
)
if
is_idiap
()
else
"results"
args
.
temp_directory
=
os
.
path
.
join
(
args
.
temp_directory
,
args
.
sub_directory
)
args
.
result_directory
=
os
.
path
.
join
(
args
.
result_directory
,
args
.
sub_directory
)
args
.
grid_log_directory
=
os
.
path
.
join
(
args
.
temp_directory
,
args
.
grid_log_directory
)
args
=
tools
.
command_line_skip_group
(
parsers
,
command_line_parameters
,
skips
)
args_dictionary
=
{
'required'
:
[
'database'
,
'preprocessor'
,
'extractor'
,
'algorithm'
,
'sub_directory'
],
'common'
:
[
'protocol'
,
'grid'
,
'parallel'
,
'verbose'
,
'groups'
,
'temp_directory'
,
'result_directory'
,
'allow_missing_files'
,
'dry_run'
,
'force'
],
'optional'
:
[
'preprocessed_directory'
,
'extracted_directory'
,
'projected_directory'
,
'extractor_file'
,
'projector_file'
]
}
keywords
=
(
"protocol"
,
"groups"
,
"parallel"
,
"preferred_package"
,
"temp_directory"
,
"result_directory"
,
"extractor_file"
,
"projector_file"
,
"gridtk_database_file"
,
"experiment_info_file"
,
"database_directories_file"
,
"preprocessed_directory"
,
"extracted_directory"
,
"projected_directory"
,
"score_directories"
,
"grid_log_directory"
,
"verbose"
,
"dry_run"
,
"force"
,
"write_compressed_score_files"
,
"stop_on_failure"
,
"run_local_scheduler"
,
"external_dependencies"
,
"timer"
,
"nice"
,
"delete_jobs_finished_with_status"
,
"allow_missing_files"
,
"env"
,
)
args
=
tools
.
parse_config_file
(
parsers
,
args
,
args_dictionary
,
keywords
,
skips
)
args
=
tools
.
set_extra_flags
(
args
)
# protocol command line override
if
args
.
protocol
is
not
None
:
...
...
develop.cfg
View file @
0f669bce
...
...
@@ -5,7 +5,8 @@
[buildout]
parts = scripts
eggs = bob.pad.base
bob.extension
bob.db.base
bob.bio.base
gridtk
extensions = bob.buildout
...
...
@@ -13,7 +14,6 @@ extensions = bob.buildout
auto-checkout = *
develop = src/bob.db.base
src/bob.bio.base
src/bob.extension
.
; options for bob.buildout
...
...
@@ -24,7 +24,6 @@ newest = false
[sources]
bob.db.base = git git@gitlab.idiap.ch:bob/bob.db.base.git
bob.bio.base = git git@gitlab.idiap.ch:bob/bob.bio.base.git
bob.extension = git git@gitlab.idiap.ch:bob/bob.extension.git
[scripts]
recipe = bob.buildout:scripts
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment