Skip to content

bob.pad.base
bob.pad.base
Commit c1227609 authored by Amir MOHAMMADI's avatar Amir MOHAMMADI
Browse files
Options

Implement a better file csv interface

parent 932cdf39
Pipeline #48601 passed with stage
    in 7 minutes and 2 seconds
    Expand all Hide whitespace changes
    Inline Side-by-side
    Showing with 52 additions and 1092 deletions
    bob/pad/base/database/__init__.py
    View file @ c1227609
    from .file import PadFile
    from .database import PadDatabase
    from .filelist.query import FileListPadDatabase
    from .filelist.models import Client
    from .csv_dataset import FileListPadDatabase
    from .PadBioFileDB import HighBioDatabase, HighPadDatabase
    from .csv_dataset import CSVPADDataset, CSVToSampleLoader, LSTToSampleLoader
    # gets sphinx autodoc done right - don't remove it
    def __appropriate__(*args):
    ......@@ -25,11 +23,7 @@ __appropriate__(
    PadFile,
    PadDatabase,
    FileListPadDatabase,
    Client,
    HighBioDatabase,
    HighPadDatabase,
    CSVPADDataset,
    CSVToSampleLoader,
    LSTToSampleLoader,
    )
    __all__ = [_ for _ in dir() if not _.startswith("_")]
    bob/pad/base/database/csv_dataset.py
    View file @ c1227609
    This diff is collapsed.
    bob/pad/base/database/filelist/__init__.py deleted 100644 → 0
    View file @ 932cdf39
    from .models import ListReader, Client, FileListFile
    from .query import FileListPadDatabase
    from .driver import Interface
    # gets sphinx autodoc done right - don't remove it
    def __appropriate__(*args):
    """Says object was actually declared here, and not in the import module.
    Fixing sphinx warnings of not being able to find classes, when path is shortened.
    Parameters:
    *args: An iterable of objects to modify
    Resolves `Sphinx referencing issues
    <https://github.com/sphinx-doc/sphinx/issues/3048>`
    """
    for obj in args:
    obj.__module__ = __name__
    __appropriate__(
    ListReader,
    Client,
    FileListFile,
    FileListPadDatabase,
    Interface,
    )
    __all__ = [_ for _ in dir() if not _.startswith('_')]
    bob/pad/base/database/filelist/driver.py deleted 100644 → 0
    View file @ 932cdf39
    #!/usr/bin/env python
    # vim: set fileencoding=utf-8 :
    # Laurent El Shafey <laurent.el-shafey@idiap.ch>
    #
    # Copyright (C) 2011-2013 Idiap Research Institute, Martigny, Switzerland
    #
    # This program is free software: you can redistribute it and/or modify
    # it under the terms of the GNU General Public License as published by
    # the Free Software Foundation, version 3 of the License.
    #
    # This program is distributed in the hope that it will be useful,
    # but WITHOUT ANY WARRANTY; without even the implied warranty of
    # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    # GNU General Public License for more details.
    #
    # You should have received a copy of the GNU General Public License
    # along with this program. If not, see <http://www.gnu.org/licenses/>.
    """Commands the PAD Filelists database can respond to.
    """
    import os
    import sys
    from bob.db.base.driver import Interface as BaseInterface
    def clients(args):
    """Dumps lists of client IDs based on your criteria"""
    from .query import FileListPadDatabase
    db = FileListPadDatabase(args.list_directory, 'pad_filelist')
    client_ids = db.client_ids(
    protocol=args.protocol,
    groups = args.group
    )
    output = sys.stdout
    if args.selftest:
    from bob.db.base.utils import null
    output = null()
    for client in client_ids:
    output.write('%s\n' % client)
    return 0
    def dumplist(args):
    """Dumps lists of files based on your criteria"""
    from .query import FileListPadDatabase
    db = FileListPadDatabase(args.list_directory, 'pad_filelist')
    file_objects = db.objects(
    purposes=args.purpose,
    groups=args.group,
    protocol=args.protocol
    )
    output = sys.stdout
    if args.selftest:
    from bob.db.base.utils import null
    output = null()
    for file_obj in file_objects:
    output.write('%s\n' % file_obj.make_path(directory=args.directory, extension=args.extension))
    return 0
    def checkfiles(args):
    """Checks existence of files based on your criteria"""
    from .query import FileListPadDatabase
    db = FileListPadDatabase(args.list_directory, 'pad_filelist')
    file_objects = db.objects(protocol=args.protocol)
    # go through all files, check if they are available on the filesystem
    good = []
    bad = []
    for file_obj in file_objects:
    if os.path.exists(file_obj.make_path(args.directory, args.extension)):
    good.append(file_obj)
    else:
    bad.append(file_obj)
    # report
    output = sys.stdout
    if args.selftest:
    from bob.db.base.utils import null
    output = null()
    if bad:
    for file_obj in bad:
    output.write('Cannot find file "%s"\n' % file_obj.make_path(args.directory, args.extension))
    output.write('%d files (out of %d) were not found at "%s"\n' % (len(bad), len(file_objects), args.directory))
    return 0
    class Interface(BaseInterface):
    def name(self):
    return 'pad_filelist'
    def version(self):
    import pkg_resources # part of setuptools
    return pkg_resources.require('bob.pad.base')[0].version
    def files(self):
    return ()
    def type(self):
    return 'text'
    def add_commands(self, parser):
    from . import __doc__ as docs
    subparsers = self.setup_parser(parser,
    "Presentation Attack Detection File Lists database", docs)
    import argparse
    # the "clients" action
    parser = subparsers.add_parser('clients', help=dumplist.__doc__)
    parser.add_argument('-l', '--list-directory', required=True,
    help="The directory which contains the file lists.")
    parser.add_argument('-g', '--group',
    help="if given, this value will limit the output files to those belonging to a "
    "particular group.",
    choices=('dev', 'eval', 'train', ''))
    parser.add_argument('-p', '--protocol', default=None,
    help="If set, the protocol is appended to the directory that contains the file lists.")
    parser.add_argument('--self-test', dest="selftest", action='store_true', help=argparse.SUPPRESS)
    parser.set_defaults(func=clients) # action
    # the "dumplist" action
    parser = subparsers.add_parser('dumplist', help=dumplist.__doc__)
    parser.add_argument('-l', '--list-directory', required=True,
    help="The directory which contains the file lists.")
    parser.add_argument('-d', '--directory', default='',
    help="if given, this path will be prepended to every entry returned.")
    parser.add_argument('-e', '--extension', default='',
    help="if given, this extension will be appended to every entry returned.")
    parser.add_argument('-u', '--purpose',
    help="if given, this value will limit the output files to those designed "
    "for the given purposes.",
    choices=('real', 'attack', ''))
    parser.add_argument('-g', '--group',
    help="if given, this value will limit the output files to those belonging to a "
    "particular group.",
    choices=('dev', 'eval', 'train', ''))
    parser.add_argument('-p', '--protocol', default=None,
    help="If set, the protocol is appended to the directory that contains the file lists.")
    parser.add_argument('--self-test', dest="selftest", action='store_true', help=argparse.SUPPRESS)
    parser.set_defaults(func=dumplist) # action
    # the "checkfiles" action
    parser = subparsers.add_parser('checkfiles', help=checkfiles.__doc__)
    parser.add_argument('-l', '--list-directory', required=True,
    help="The directory which contains the file lists.")
    parser.add_argument('-d', '--directory', dest="directory", default='',
    help="if given, this path will be prepended to every entry returned.")
    parser.add_argument('-e', '--extension', dest="extension", default='',
    help="if given, this extension will be appended to every entry returned.")
    parser.add_argument('-p', '--protocol', default=None,
    help="If set, the protocol is appended to the directory that contains the file lists.")
    parser.add_argument('--self-test', dest="selftest", action='store_true', help=argparse.SUPPRESS)
    parser.set_defaults(func=checkfiles) # action
    bob/pad/base/database/filelist/models.py deleted 100644 → 0
    View file @ 932cdf39
    #!/usr/bin/env python
    # vim: set fileencoding=utf-8 :
    # @author: Pavel Korshunov <pavel.korshunov@idiap.ch>
    # @date: Thu Nov 17 16:09:22 CET 2016
    #
    # Copyright (C) 2011-2013 Idiap Research Institute, Martigny, Switzerland
    #
    # This program is free software: you can redistribute it and/or modify
    # it under the terms of the GNU General Public License as published by
    # the Free Software Foundation, version 3 of the License.
    #
    # This program is distributed in the hope that it will be useful,
    # but WITHOUT ANY WARRANTY; without even the implied warranty of
    # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    # GNU General Public License for more details.
    #
    # You should have received a copy of the GNU General Public License
    # along with this program. If not, see <http://www.gnu.org/licenses/>.
    """
    This file defines simple Client and File interfaces that are comparable with other bob.db databases.
    """
    import os
    import fileinput
    import re
    from bob.pad.base.database import PadFile
    class Client(object):
    """
    The clients of this database contain ONLY client ids. Nothing special.
    """
    def __init__(self, client_id):
    self.id = client_id
    """The ID of the client, which is stored as a :py:class:`str` object."""
    class FileListFile(PadFile):
    """
    Initialize the File object with the minimum required data.
    **Parameters**
    path : str
    The path of this file, relative to the basic directory.
    Please do not specify any file extensions.
    This path will be used as an underlying file_id, as it is assumed to be unique
    client_id : various type
    The id of the client, this file belongs to.
    The type of it is dependent on your implementation.
    If you use an SQL database, this should be an SQL type like Integer or String.
    """
    def __init__(self, file_name, client_id, attack_type=None):
    super(FileListFile, self).__init__(client_id=client_id, path=file_name, attack_type=attack_type, file_id=file_name)
    #############################################################################
    ### internal access functions for the file lists; do not export!
    #############################################################################
    class ListReader(object):
    def __init__(self, store_lists):
    self.m_read_lists = {}
    self.m_store_lists = store_lists
    def _read_multi_column_list(self, list_file):
    rows = []
    if not os.path.isfile(list_file):
    raise RuntimeError('File %s does not exist.' % (list_file,))
    try:
    for line in fileinput.input(list_file):
    parsed_line = re.findall('[\w/(-.)]+', line)
    if len(parsed_line):
    # perform some sanity checks
    if len(parsed_line) not in (2, 3):
    raise IOError("The read line '%s' from file '%s' could not be parsed successfully!" %
    (line.rstrip(), list_file))
    if len(rows) and len(rows[0]) != len(parsed_line):
    raise IOError("The parsed line '%s' from file '%s' has a different number of elements "
    "than the first parsed line '%s'!" % (parsed_line, list_file, rows[0]))
    # append the read line
    rows.append(parsed_line)
    fileinput.close()
    except IOError as e:
    raise RuntimeError("Error reading the file '%s' : '%s'." % (list_file, e))
    # return the read list as a vector of columns
    return rows
    def _read_column_list(self, list_file, column_count):
    # read the list
    rows = self._read_multi_column_list(list_file)
    # extract the file from the first two columns
    file_list = []
    for row in rows:
    if column_count == 2:
    assert len(row) == 2
    # we expect: filename client_id
    file_list.append(FileListFile(file_name=row[0], client_id=row[1]))
    elif column_count == 3:
    assert len(row) == 3
    # we expect: filename, model_id, client_id
    file_list.append(FileListFile(file_name=row[0], client_id=row[1], attack_type=row[2]))
    else:
    raise ValueError("The given column count %d cannot be interpreted. This is a BUG, please "
    "report to the author." % column_count)
    return file_list
    def read_list(self, list_file, group, type=None):
    """Reads the list of Files from the given list file (if not done yet) and returns it."""
    if group not in self.m_read_lists:
    self.m_read_lists[group] = {}
    if type not in self.m_read_lists[group]:
    if type == 'for_real':
    files_list = self._read_column_list(list_file, 2)
    elif type == 'for_attack':
    files_list = self._read_column_list(list_file, 3)
    else:
    raise ValueError("The given type must be one of %s, but not '%s'" % (('for_real', 'for_attack'), type))
    if self.m_store_lists:
    self.m_read_lists[group][type] = files_list
    return files_list
    return self.m_read_lists[group][type]
    bob/pad/base/database/filelist/query.py deleted 100644 → 0
    View file @ 932cdf39
    This diff is collapsed.
    bob/pad/base/test/test_filelist.py deleted 100644 → 0
    View file @ 932cdf39
    #!/usr/bin/env python
    # vim: set fileencoding=utf-8 :
    # @author: Pavel Korshunov <pavel.korshunov@idiap.ch>
    # @date: Thu Nov 17 16:09:22 CET 2016
    #
    # Copyright (C) 2011-2013 Idiap Research Institute, Martigny, Switzerland
    #
    # This program is free software: you can redistribute it and/or modify
    # it under the terms of the GNU General Public License as published by
    # the Free Software Foundation, version 3 of the License.
    #
    # This program is distributed in the hope that it will be useful,
    # but WITHOUT ANY WARRANTY; without even the implied warranty of
    # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    # GNU General Public License for more details.
    #
    # You should have received a copy of the GNU General Public License
    # along with this program. If not, see <http://www.gnu.org/licenses/>.
    """
    Tests for the PAD Filelist database.
    """
    import os
    import bob.io.base.test_utils
    from bob.pad.base.database import FileListPadDatabase, CSVPADDataset, LSTToSampleLoader
    example_dir = os.path.realpath(
    bob.io.base.test_utils.datafile(".", __name__, "data/example_filelist")
    )
    csv_example_dir = os.path.realpath(
    bob.io.base.test_utils.datafile(".", __name__, "data/csv_dataset")
    )
    csv_example_tarball = os.path.realpath(
    bob.io.base.test_utils.datafile(".", __name__, "data/csv_dataset.tar.gz")
    )
    def test_query():
    db = FileListPadDatabase(example_dir, "test_padfilelist")
    assert len(db.groups()) == 3 # 3 groups (dev, eval, train)
    print(db.client_ids())
    # 5 client ids for real data of train, dev and eval sets (ignore all ids that are in attacks only)
    assert len(db.client_ids()) == 5
    assert len(db.client_ids(groups="train")) == 2 # 2 client ids for train
    assert len(db.client_ids(groups="dev")) == 2 # 2 client ids for dev
    assert len(db.client_ids(groups="eval")) == 1 # 2 client ids for eval
    assert len(db.objects(groups="train")) == 3 # 3 samples in the train set
    assert (
    len(db.objects(groups="dev", purposes="real")) == 2
    ) # 2 samples of real data in the dev set
    assert (
    len(db.objects(groups="dev", purposes="attack")) == 1
    ) # 1 attack in the dev set
    def test_query_protocol():
    db = FileListPadDatabase(os.path.dirname(example_dir), "test_padfilelist")
    p = "example_filelist"
    assert len(db.groups(protocol=p)) == 3 # 3 groups (dev, eval, train)
    assert len(db.client_ids(protocol=p)) == 5 # 6 client ids for train, dev and eval
    assert len(db.client_ids(groups="train", protocol=p)) == 2 # 2 client ids for train
    assert len(db.client_ids(groups="dev", protocol=p)) == 2 # 2 client ids for dev
    assert len(db.client_ids(groups="eval", protocol=p)) == 1 # 2 client ids for eval
    assert (
    len(db.objects(groups="train", protocol=p)) == 3
    ) # 3 samples in the train set
    assert (
    len(db.objects(groups="dev", purposes="real", protocol=p)) == 2
    ) # 2 samples of real data in the dev set
    assert (
    len(db.objects(groups="dev", purposes="attack", protocol=p)) == 1
    ) # 1 attack in the dev set
    def test_driver_api():
    from bob.db.base.script.dbmanage import main
    assert (
    main(
    (
    "pad_filelist clients --list-directory=%s --self-test" % example_dir
    ).split()
    )
    == 0
    )
    assert (
    main(
    (
    "pad_filelist dumplist --list-directory=%s --self-test" % example_dir
    ).split()
    )
    == 0
    )
    assert (
    main(
    (
    "pad_filelist dumplist --list-directory=%s --purpose=real --group=dev --self-test"
    % example_dir
    ).split()
    )
    == 0
    )
    assert (
    main(
    (
    "pad_filelist checkfiles --list-directory=%s --self-test" % example_dir
    ).split()
    )
    == 0
    )
    def test_csv_dataset():
    def run(path):
    dataset = CSVPADDataset(path, "protocol1")
    # Train
    assert len(dataset.fit_samples()) == 5
    # 2 out of 5 are bonafides
    assert sum([s.is_bonafide for s in dataset.fit_samples()]) == 2
    # DEV
    assert len(dataset.predict_samples()) == 5
    # 2 out of 5 are bonafides
    assert sum([s.is_bonafide for s in dataset.predict_samples()]) == 2
    # EVAL
    assert len(dataset.predict_samples(group="eval")) == 7
    # 3 out of 5 are bonafides
    assert sum([s.is_bonafide for s in dataset.predict_samples(group="eval")]) == 3
    run(csv_example_dir)
    run(csv_example_tarball)
    def test_csv_dataset_lst():
    dataset = CSVPADDataset(
    example_dir,
    "",
    csv_to_sample_loader=LSTToSampleLoader(
    data_loader=bob.io.base.load,
    metadata_loader=None,
    dataset_original_directory="",
    extension="",
    ),
    )
    # Train
    assert len(dataset.fit_samples()) == 3
    # 2 out of 3 are bonafides
    assert sum([s.is_bonafide for s in dataset.fit_samples()]) == 2
    # DEV
    assert len(dataset.predict_samples()) == 3
    # 2 out of 3 are bonafides
    assert sum([s.is_bonafide for s in dataset.predict_samples()]) == 2
    # EVAL
    assert len(dataset.predict_samples(group="eval")) == 2
    # 1 out of 2 are bonafides
    assert sum([s.is_bonafide for s in dataset.predict_samples(group="eval")]) == 1
    Markdown is supported
    0% or
    You are about to add 0 people to the discussion. Proceed with caution.
    Finish editing this message first!
    Please register or to comment