Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
bob
bob.bio.base
Commits
77951f3b
Commit
77951f3b
authored
Oct 07, 2020
by
Tiago de Freitas Pereira
Browse files
Moved back the Current FileList Interface
parent
9f6da96e
Pipeline
#43922
failed with stage
in 9 minutes and 44 seconds
Changes
22
Pipelines
1
Expand all
Hide whitespace changes
Inline
Side-by-side
bob/bio/base/database/__init__.py
View file @
77951f3b
...
...
@@ -3,6 +3,8 @@ from .file import BioFile
from
.file
import
BioFileSet
from
.database
import
BioDatabase
from
.database
import
ZTBioDatabase
from
.filelist
import
FileListBioDatabase
from
.
import
filelist
# gets sphinx autodoc done right - don't remove it
...
...
@@ -26,7 +28,8 @@ __appropriate__(
BioFileSet
,
BioDatabase
,
ZTBioDatabase
,
CSVDatasetDevEval
,
CSVToSampleLoader
CSVDatasetDevEval
,
CSVToSampleLoader
,
CSVDatasetCrossValidation
)
__all__
=
[
_
for
_
in
dir
()
if
not
_
.
startswith
(
'_'
)]
bob/bio/base/database/filelist/__init__.py
0 → 100644
View file @
77951f3b
from
.models
import
FileListFile
from
.query
import
FileListBioDatabase
from
.driver
import
Interface
# gets sphinx autodoc done right - don't remove it
def
__appropriate__
(
*
args
):
"""Says object was actually declared here, and not in the import module.
Fixing sphinx warnings of not being able to find classes, when path is shortened.
Parameters:
*args: An iterable of objects to modify
Resolves `Sphinx referencing issues
<https://github.com/sphinx-doc/sphinx/issues/3048>`
"""
for
obj
in
args
:
obj
.
__module__
=
__name__
__appropriate__
(
FileListFile
,
FileListBioDatabase
,
Interface
,
)
__all__
=
[
_
for
_
in
dir
()
if
not
_
.
startswith
(
'_'
)]
bob/bio/base/database/filelist/driver.py
0 → 100644
View file @
77951f3b
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Laurent El Shafey <laurent.el-shafey@idiap.ch>
#
# Copyright (C) 2011-2013 Idiap Research Institute, Martigny, Switzerland
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""Commands the Verification Filelists database can respond to.
"""
import
os
import
sys
from
bob.db.base.driver
import
Interface
as
BaseInterface
def
dumplist
(
args
):
"""Dumps lists of files based on your criteria"""
from
.query
import
FileListBioDatabase
db
=
FileListBioDatabase
(
args
.
list_directory
,
'bio_filelist'
,
use_dense_probe_file_list
=
False
)
r
=
db
.
objects
(
purposes
=
args
.
purpose
,
groups
=
args
.
group
,
classes
=
args
.
sclass
,
protocol
=
args
.
protocol
)
output
=
sys
.
stdout
if
args
.
selftest
:
from
bob.db.base.utils
import
null
output
=
null
()
for
f
in
r
:
output
.
write
(
'%s
\n
'
%
f
.
make_path
(
directory
=
args
.
directory
,
extension
=
args
.
extension
))
return
0
def
checkfiles
(
args
):
"""Checks existence of files based on your criteria"""
from
.query
import
FileListBioDatabase
db
=
FileListBioDatabase
(
args
.
list_directory
,
'bio_filelist'
,
use_dense_probe_file_list
=
False
)
r
=
db
.
objects
(
protocol
=
args
.
protocol
)
# go through all files, check if they are available on the filesystem
good
=
[]
bad
=
[]
for
f
in
r
:
if
os
.
path
.
exists
(
f
.
make_path
(
args
.
directory
,
args
.
extension
)):
good
.
append
(
f
)
else
:
bad
.
append
(
f
)
# report
output
=
sys
.
stdout
if
args
.
selftest
:
from
bob.db.base.utils
import
null
output
=
null
()
if
bad
:
for
f
in
bad
:
output
.
write
(
'Cannot find file "%s"
\n
'
%
f
.
make_path
(
args
.
directory
,
args
.
extension
))
output
.
write
(
'%d files (out of %d) were not found at "%s"
\n
'
%
(
len
(
bad
),
len
(
r
),
args
.
directory
))
return
0
class
Interface
(
BaseInterface
):
def
name
(
self
):
return
'bio_filelist'
def
version
(
self
):
import
pkg_resources
# part of setuptools
return
pkg_resources
.
require
(
'bob.bio.base'
)[
0
].
version
def
files
(
self
):
return
()
def
type
(
self
):
return
'text'
def
add_commands
(
self
,
parser
):
from
.
import
__doc__
as
docs
subparsers
=
self
.
setup_parser
(
parser
,
"Face Verification File Lists database"
,
docs
)
import
argparse
# the "dumplist" action
parser
=
subparsers
.
add_parser
(
'dumplist'
,
help
=
dumplist
.
__doc__
)
parser
.
add_argument
(
'-l'
,
'--list-directory'
,
required
=
True
,
help
=
"The directory which contains the file lists."
)
parser
.
add_argument
(
'-d'
,
'--directory'
,
default
=
''
,
help
=
"if given, this path will be prepended to every entry returned."
)
parser
.
add_argument
(
'-e'
,
'--extension'
,
default
=
''
,
help
=
"if given, this extension will be appended to every entry returned."
)
parser
.
add_argument
(
'-u'
,
'--purpose'
,
help
=
"if given, this value will limit the output files to those designed for the given purposes."
,
choices
=
(
'enroll'
,
'probe'
,
''
))
parser
.
add_argument
(
'-g'
,
'--group'
,
help
=
"if given, this value will limit the output files to those belonging to a particular protocolar group."
,
choices
=
(
'dev'
,
'eval'
,
'world'
,
'optional_world_1'
,
'optional_world_2'
,
''
))
parser
.
add_argument
(
'-c'
,
'--class'
,
dest
=
"sclass"
,
help
=
"if given, this value will limit the output files to those belonging to the given classes."
,
choices
=
(
'client'
,
'impostor'
,
''
))
parser
.
add_argument
(
'-p'
,
'--protocol'
,
default
=
None
,
help
=
"If set, the protocol is appended to the directory that contains the file lists."
)
parser
.
add_argument
(
'--self-test'
,
dest
=
"selftest"
,
action
=
'store_true'
,
help
=
argparse
.
SUPPRESS
)
parser
.
set_defaults
(
func
=
dumplist
)
# action
# the "checkfiles" action
parser
=
subparsers
.
add_parser
(
'checkfiles'
,
help
=
checkfiles
.
__doc__
)
parser
.
add_argument
(
'-l'
,
'--list-directory'
,
required
=
True
,
help
=
"The directory which contains the file lists."
)
parser
.
add_argument
(
'-d'
,
'--directory'
,
dest
=
"directory"
,
default
=
''
,
help
=
"if given, this path will be prepended to every entry returned."
)
parser
.
add_argument
(
'-e'
,
'--extension'
,
dest
=
"extension"
,
default
=
''
,
help
=
"if given, this extension will be appended to every entry returned."
)
parser
.
add_argument
(
'-p'
,
'--protocol'
,
default
=
None
,
help
=
"If set, the protocol is appended to the directory that contains the file lists."
)
parser
.
add_argument
(
'--self-test'
,
dest
=
"selftest"
,
action
=
'store_true'
,
help
=
argparse
.
SUPPRESS
)
parser
.
set_defaults
(
func
=
checkfiles
)
# action
bob/bio/base/database/filelist/models.py
0 → 100644
View file @
77951f3b
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Manuel Guenther <Manuel.Guenther@idiap.ch>
# @date: Wed Oct 24 10:47:43 CEST 2012
#
# Copyright (C) 2011-2013 Idiap Research Institute, Martigny, Switzerland
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
This file defines a simple interface that are comparable with other bob.db databases.
"""
import
os
import
fileinput
import
re
class
FileListFile
(
object
):
"""
Initialize the File object with the minimum required data.
If the ``model_id`` is not specified, ``model_id`` and ``client_id`` are identical.
If the ``claimed_id`` is not specified, it is expected to be the ``client_id``.
Parameters
----------
client_id : various type
The id of the client, this file belongs to.
The type of it is dependent on your implementation.
If you use an SQL database, this should be an SQL type like Integer or String.
path : str
The path of this file, relative to the basic directory.
If you use an SQL database, this should be the SQL type String.
Please do not specify any file extensions.
file_id : various type
The id of the file.
The type of it is dependent on your implementation.
If you use an SQL database, this should be an SQL type like Integer or String.
If you are using an automatically determined file id, you can skip selecting the file id.
"""
def
__init__
(
self
,
file_name
,
client_id
,
model_id
=
None
,
claimed_id
=
None
):
# super(FileListFile, self).__init__(client_id=client_id, path=file_name, file_id=file_name)
super
(
FileListFile
,
self
).
__init__
()
self
.
client_id
=
client_id
self
.
path
=
file_name
self
.
id
=
file_name
# Note: in case of probe files, model ids are considered to be the ids of the model for the given probe file.
# Hence, there might be several probe files with the same file id, but different model ids.
# Therefore, please DO NOT USE the model_id outside of this class (or the according database queries).
# when the model id is not specified, we use the client id instead
self
.
_model_id
=
client_id
if
model_id
is
None
else
model_id
# when the claimed id is not specified, we use the client id instead
self
.
claimed_id
=
client_id
if
claimed_id
is
None
else
claimed_id
#############################################################################
# internal access functions for the file lists; do not export!
#############################################################################
class
ListReader
(
object
):
def
__init__
(
self
,
store_lists
):
self
.
m_read_lists
=
{}
self
.
m_model_dicts
=
{}
self
.
m_store_lists
=
store_lists
def
_read_multi_column_list
(
self
,
list_file
):
rows
=
[]
if
not
os
.
path
.
isfile
(
list_file
):
raise
RuntimeError
(
'File %s does not exist.'
%
(
list_file
,))
try
:
for
line
in
fileinput
.
input
(
list_file
):
if
line
.
strip
().
startswith
(
'#'
):
continue
parsed_line
=
re
.
findall
(
'[\w/(-.)]+'
,
line
)
if
len
(
parsed_line
):
# perform some sanity checks
if
len
(
parsed_line
)
not
in
(
2
,
3
,
4
):
raise
IOError
(
"The read line '%s' from file '%s' could not be parsed successfully!"
%
(
line
.
rstrip
(),
list_file
))
if
len
(
rows
)
and
len
(
rows
[
0
])
!=
len
(
parsed_line
):
raise
IOError
(
"The parsed line '%s' from file '%s' has a different number of elements than the first parsed line '%s'!"
%
(
parsed_line
,
list_file
,
rows
[
0
]))
# append the read line
rows
.
append
(
parsed_line
)
fileinput
.
close
()
except
IOError
as
e
:
raise
RuntimeError
(
"Error reading the file '%s' : '%s'."
%
(
list_file
,
e
))
# return the read list as a vector of columns
return
rows
def
_read_column_list
(
self
,
list_file
,
column_count
):
# read the list
rows
=
self
.
_read_multi_column_list
(
list_file
)
# extract the file from the first two columns
file_list
=
[]
for
row
in
rows
:
if
column_count
==
2
:
assert
len
(
row
)
==
2
# we expect: filename client_id
file_list
.
append
(
FileListFile
(
file_name
=
row
[
0
],
client_id
=
row
[
1
]))
elif
column_count
==
3
:
assert
len
(
row
)
in
(
2
,
3
)
# we expect: filename, model_id, client_id
file_list
.
append
(
FileListFile
(
file_name
=
row
[
0
],
client_id
=
row
[
2
]
if
len
(
row
)
>
2
else
row
[
1
],
model_id
=
row
[
1
]))
elif
column_count
==
4
:
assert
len
(
row
)
in
(
3
,
4
)
# we expect: filename, model_id, claimed_id, client_id
file_list
.
append
(
FileListFile
(
file_name
=
row
[
0
],
client_id
=
row
[
3
]
if
len
(
row
)
>
3
else
row
[
1
],
model_id
=
row
[
1
],
claimed_id
=
row
[
2
]))
else
:
raise
ValueError
(
"The given column count %d cannot be interpreted. This is a BUG, please report to the author."
%
column_count
)
return
file_list
def
_create_model_dictionary
(
self
,
files
):
# remember model ids
retval
=
{}
for
file
in
files
:
if
file
.
_model_id
not
in
retval
:
retval
[
file
.
_model_id
]
=
file
.
client_id
else
:
if
retval
[
file
.
_model_id
]
!=
file
.
client_id
:
raise
ValueError
(
"The read model id '%s' is associated to two different client ids '%s' and '%s'!"
%
(
file
.
_model_id
,
file
.
client_id
,
retval
[
file
.
_model_id
]))
return
retval
def
read_list
(
self
,
list_file
,
group
,
type
=
None
):
"""Reads the list of Files from the given list file (if not done yet) and returns it."""
if
group
in
(
'world'
,
'optional_world_1'
,
'optional_world_2'
):
if
group
not
in
self
.
m_read_lists
:
# read the world list into memory
list
=
self
.
_read_column_list
(
list_file
,
2
)
if
self
.
m_store_lists
:
self
.
m_read_lists
[
group
]
=
list
return
list
# just return the previously read list
return
self
.
m_read_lists
[
group
]
else
:
if
group
not
in
self
.
m_read_lists
:
self
.
m_read_lists
[
group
]
=
{}
if
type
not
in
self
.
m_read_lists
[
group
]:
if
type
in
(
'for_models'
,
'for_tnorm'
):
list
=
self
.
_read_column_list
(
list_file
,
3
)
elif
type
==
'for_scores'
:
list
=
self
.
_read_column_list
(
list_file
,
4
)
elif
type
in
(
'for_probes'
,
'for_znorm'
):
list
=
self
.
_read_column_list
(
list_file
,
2
)
else
:
raise
ValueError
(
"The given type must be one of %s, but not '%s'"
%
(
(
'for_models'
,
'for_scores'
,
'for_probes'
,
'for_tnorm'
,
'for_znorm'
),
type
))
if
self
.
m_store_lists
:
self
.
m_read_lists
[
group
][
type
]
=
list
return
list
return
self
.
m_read_lists
[
group
][
type
]
def
read_models
(
self
,
list_file
,
group
,
type
=
None
):
"""Generates a dictionary from model_ids to client_ids for the given list file, if not done yet, and returns it"""
assert
group
in
(
'dev'
,
'eval'
,
'world'
,
'optional_world_1'
,
'optional_world_2'
)
assert
type
in
(
'for_models'
,
'for_tnorm'
)
if
group
not
in
self
.
m_model_dicts
:
self
.
m_model_dicts
[
group
]
=
{}
if
type
not
in
self
.
m_model_dicts
[
group
]:
dict
=
self
.
_create_model_dictionary
(
self
.
read_list
(
list_file
,
group
,
type
))
if
self
.
m_store_lists
:
self
.
m_model_dicts
[
group
][
type
]
=
dict
return
dict
return
self
.
m_model_dicts
[
group
][
type
]
bob/bio/base/database/filelist/query.py
0 → 100644
View file @
77951f3b
This diff is collapsed.
Click to expand it.
bob/bio/base/test/data/example_filelist/data/model4_session1_sample2.pos
0 → 100644
View file @
77951f3b
key1 10 20
key2 30 40
bob/bio/base/test/data/example_filelist/dev/for_models.lst
0 → 100644
View file @
77951f3b
data/model3_session1_sample1 3 3
data/model3_session1_sample2 3 3
data/model3_session1_sample3 3 3
data/model3_session2_sample1 3 3
data/model4_session1_sample1 4 4
data/model4_session1_sample2 4 4
data/model4_session1_sample3 4 4
data/model4_session2_sample1 4 4
bob/bio/base/test/data/example_filelist/dev/for_probes.lst
0 → 100644
View file @
77951f3b
data/model3_session3_sample1 3
data/model3_session3_sample2 3
data/model3_session3_sample3 3
data/model3_session4_sample1 3
data/model4_session3_sample1 4
data/model4_session3_sample2 4
data/model4_session3_sample1 4
data/model4_session3_sample2 4
data/model4_session3_sample3 4
data/model4_session4_sample1 4
bob/bio/base/test/data/example_filelist/dev/for_scores.lst
0 → 100644
View file @
77951f3b
data/model3_session3_sample1 3 3 3
data/model3_session3_sample2 3 3 3
data/model3_session3_sample3 3 3 3
data/model3_session4_sample1 3 3 3
data/model4_session3_sample1 3 3 4
data/model4_session3_sample2 3 3 4
data/model4_session3_sample1 4 4 4
data/model4_session3_sample2 4 4 4
data/model4_session3_sample3 4 4 4
data/model4_session4_sample1 4 4 4
data/model3_session3_sample1 4 4 3
data/model3_session3_sample2 4 4 3
bob/bio/base/test/data/example_filelist/dev/for_tnorm.lst
0 → 100644
View file @
77951f3b
data/model7_session1_sample1 7 7
data/model7_session1_sample2 7 7
data/model7_session1_sample3 7 7
data/model7_session2_sample1 7 7
data/model8_session1_sample1 8 8
data/model8_session1_sample2 8 8
data/model8_session1_sample3 8 8
data/model8_session2_sample1 8 8
bob/bio/base/test/data/example_filelist/dev/for_znorm.lst
0 → 100644
View file @
77951f3b
data/model9_session1_sample1 9
data/model9_session1_sample2 9
data/model9_session1_sample3 9
data/model9_session2_sample1 9
data/model10_session1_sample1 10
data/model10_session1_sample2 10
data/model10_session1_sample3 10
data/model10_session2_sample1 10
bob/bio/base/test/data/example_filelist/eval/for_models.lst
0 → 100644
View file @
77951f3b
data/model5_session1_sample1 5 5
data/model5_session1_sample2 5 5
data/model5_session1_sample3 5 5
data/model5_session2_sample1 5 5
data/model6_session1_sample1 6 6
data/model6_session1_sample2 6 6
data/model6_session1_sample3 6 6
data/model6_session2_sample1 6 6
bob/bio/base/test/data/example_filelist/eval/for_probes.lst
0 → 100644
View file @
77951f3b
data/model5_session3_sample1 5
data/model5_session3_sample2 5
data/model5_session3_sample3 5
data/model5_session4_sample1 5
data/model6_session3_sample1 6
data/model6_session3_sample2 6
data/model6_session3_sample3 6
data/model6_session4_sample1 6
bob/bio/base/test/data/example_filelist/eval/for_scores.lst
0 → 100644
View file @
77951f3b
data/model5_session3_sample1 5 5 5
data/model5_session3_sample2 5 5 5
data/model5_session3_sample3 5 5 5
data/model5_session4_sample1 5 5 5
data/model6_session3_sample1 6 6 6
data/model6_session3_sample2 6 6 6
data/model6_session3_sample3 6 6 6
data/model6_session4_sample1 6 6 6
bob/bio/base/test/data/example_filelist/eval/for_tnorm.lst
0 → 100644
View file @
77951f3b
data/model7_session1_sample1 7 7
data/model7_session1_sample2 7 7
data/model7_session1_sample3 7 7
data/model7_session2_sample1 7 7
data/model8_session1_sample1 8 8
data/model8_session1_sample2 8 8
data/model8_session1_sample3 8 8
data/model8_session2_sample1 8 8
bob/bio/base/test/data/example_filelist/eval/for_znorm.lst
0 → 100644
View file @
77951f3b
data/model9_session1_sample1 9
data/model9_session1_sample2 9
data/model9_session1_sample3 9
data/model9_session2_sample1 9
data/model10_session1_sample1 10
data/model10_session1_sample2 10
data/model10_session1_sample3 10
data/model10_session2_sample1 10
bob/bio/base/test/data/example_filelist/norm/train_optional_world_1.lst
0 → 100644
View file @
77951f3b
data/model11_session1_sample1 1
data/model11_session1_sample2 1
data/model11_session1_sample3 1
data/model11_session2_sample1 1
data/model12_session1_sample1 2
data/model12_session1_sample2 2
data/model12_session1_sample3 2
data/model12_session2_sample1 2
bob/bio/base/test/data/example_filelist/norm/train_optional_world_2.lst
0 → 100644
View file @
77951f3b
data/model13_session1_sample1 1
data/model13_session1_sample2 1
data/model13_session1_sample3 1
data/model13_session2_sample1 1
data/model14_session1_sample1 2
data/model14_session1_sample2 2
data/model14_session1_sample3 2
data/model14_session2_sample1 2
bob/bio/base/test/data/example_filelist/norm/train_world.lst
0 → 100644
View file @
77951f3b
data/model1_session1_sample1 1
data/model1_session1_sample2 1
data/model1_session1_sample3 1
data/model1_session2_sample1 1
data/model2_session1_sample1 2
data/model2_session1_sample2 2
data/model2_session1_sample3 2
data/model2_session2_sample1 2
bob/bio/base/test/data/example_filelist2/dev/for_models.lst
0 → 100644
View file @
77951f3b
# Model samples
#Modelsamples
#data/model3_session1_sample1 3 3
data/model3_session1_sample1 3 3
data/model3_session1_sample2 3 3
data/model3_session1_sample3 3 3
data/model3_session2_sample1 3 3
data/model4_session1_sample1 4 4
data/model4_session1_sample2 4 4
data/model4_session1_sample3 4 4
data/model4_session2_sample1 4 4
data/model5_session1_sample1 5 5
data/model5_session1_sample2 5 5
data/model5_session1_sample3 5 5
data/model5_session2_sample1 5 5
Prev
1
2
Next
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment