Commit 60676f84 authored by André Anjos's avatar André Anjos 💬
Browse files

Merge branch 'issue_4' into 'master'

Issue 4

Closes #4

See merge request !3
parents 3bfb3a7a 6059a383
Pipeline #6101 passed with stages
in 5 minutes and 40 seconds
......@@ -18,3 +18,4 @@ build
*.egg
src/
db.sql3
bob/db/ijba/data/
include README.rst bootstrap-buildout.py buildout.cfg develop.cfg version.txt requirements.txt
recursive-include doc *.py *.rst
recursive-include bob *.csv
recursive-include doc *.py *.rst *.png *.ico
recursive-include bob *.txt *.csv
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Manuel Gunther <mgunther@vast.uccs.edu>
# @date: Fri Sep 11 14:53:52 MDT 2015
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""This is the Bob database entry for the JANUS database.
"""
from .query import Database
from .reader import get_templates
from .reader import File, Template, get_templates, read_annotations
#from .models import Client, File, Annotation, Template, Protocol, Comparisons, Protocol_Template_Association
#, File_Template_Association
def get_config():
"""Returns a string containing the configuration information.
......@@ -32,4 +16,24 @@ def get_config():
# gets sphinx autodoc done right - don't remove it
def __appropriate__(*args):
"""Says object was actually declared here, an not on the import module.
Parameters:
*args: An iterable of objects to modify
Resolves `Sphinx referencing issues
<https://github.com/sphinx-doc/sphinx/issues/3048>`
"""
for obj in args: obj.__module__ = __name__
__appropriate__(
Database,
File,
Template,
get_templates,
read_annotations,
)
__all__ = [_ for _ in dir() if not _.startswith('_')]
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Manuel Gunther <mgunther@vast.uccs.edu>
# @date: Fri Sep 11 14:53:52 MDT 2015
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""Commands this database can respond to.
"""
import os
import sys
import pkg_resources
from bob.db.base.driver import Interface as BaseInterface
......@@ -40,15 +27,15 @@ def checkfiles(args):
bad = {}
# go through all files, check if they are available on the filesystem
for f in r:
found = False
for e in args.extension:
if os.path.exists(f.make_path(args.directory,e)):
if os.path.exists(f.make_path(args.directory,e)):
good[f.id] = f.make_path(args.directory,e)
found = True
break
if not found:
if not found:
bad[f.id] = f.make_path(args.directory,args.extension[0])
# report
......@@ -87,98 +74,38 @@ def path(args):
return 0
def upload_text(arguments):
"""For our raw database: uploads the bob/db/ijba/data database file to a server."""
# get the file name of the target db
assert len(arguments.files) == 1
assert os.path.basename(arguments.files[0]) == 'data'
source_file = "./bob/db/ijba/data"
target_file = os.path.join(arguments.destination, arguments.name + ".tar.bz2")
if os.path.exists(source_file):
print ("Compressing file '%s' to '%s'" %(source_file, target_file))
import tarfile, stat
f = tarfile.open(target_file, 'w:bz2')
f.add(source_file, os.path.basename(source_file))
f.close()
os.chmod(target_file, stat.S_IRUSR|stat.S_IWUSR | stat.S_IRGRP|stat.S_IWGRP | stat.S_IROTH)
else:
print ("WARNING! Database file '%s' is not available.'" % (source_file))
def download_text(arguments):
"""For our raw database: Downloads the data dir from a server."""
# get the file name of the target db
assert len(arguments.files) == 1
assert os.path.basename(arguments.files[0]) == 'data'
target_file = arguments.files[0]
if os.path.exists(target_file) and not arguments.force:
print ("Skipping download of file '%s' since it exists already." % target_file)
else:
# get URL of database file
source_url = os.path.join(arguments.source, arguments.name + ".tar.bz2")
# download
import sys, tempfile, tarfile
if sys.version_info[0] <= 2:
import urllib2 as urllib
else:
import urllib.request as urllib
try:
print ("Extracting url '%s' to '%s'" %(source_url, target_file))
u = urllib.urlopen(source_url)
f = tempfile.NamedTemporaryFile(suffix = ".tar.bz2")
open(f.name, 'wb').write(u.read())
t = tarfile.open(fileobj=f, mode = 'r:bz2')
t.extractall(os.path.dirname(target_file))
t.close()
f.close()
except Exception as e:
print ("Error while downloading: '%s'" % e)
class Interface(BaseInterface):
def name(self):
return 'ijba'
def version(self):
import pkg_resources # part of setuptools
return pkg_resources.require('bob.db.%s' % self.name())[0].version
def files(self):
from pkg_resources import resource_filename
raw_files = ('data',)
return [resource_filename(__name__, k) for k in raw_files]
#return ()
basedir = pkg_resources.resource_filename(__name__, '')
filelist = os.path.join(basedir, 'files.txt')
return [os.path.join(basedir, k.strip()) for k in \
open(filelist, 'rt').readlines() if k.strip()]
def type(self):
return 'text'
def add_commands(self, parser):
from . import __doc__ as docs
import argparse
subparsers = self.setup_parser(parser,
"IJBA database", docs)
subparsers = self.setup_parser(parser, "IJBA database", docs)
from .query import Database
db = Database()
#Setting and reseting the download/upload commands
from bob.db.base.driver import upload_command, download_command
parser = upload_command(subparsers)
parser.set_defaults(func=upload_text)
parser = download_command(subparsers)
parser.set_defaults(func=download_text)
# the "checkfiles" action
parser = subparsers.add_parser('checkfiles', help=checkfiles.__doc__)
parser.add_argument('-d', '--directory', help="if given, this path will be prepended to every entry returned.")
......@@ -186,7 +113,6 @@ class Interface(BaseInterface):
parser.add_argument('--self-test', dest="selftest", action='store_true', help=argparse.SUPPRESS)
parser.set_defaults(func=checkfiles) #action
# adds the "path" command
parser = subparsers.add_parser('path', help=path.__doc__)
parser.add_argument('-d', '--directory', help="if given, this path will be prepended to every entry returned.")
......
data/IJB-A_11_sets/split1/train_1.csv
data/IJB-A_11_sets/split1/verify_comparisons_1.csv
data/IJB-A_11_sets/split1/verify_metadata_1.csv
data/IJB-A_11_sets/split10/train_10.csv
data/IJB-A_11_sets/split10/verify_comparisons_10.csv
data/IJB-A_11_sets/split10/verify_metadata_10.csv
data/IJB-A_11_sets/split2/train_2.csv
data/IJB-A_11_sets/split2/verify_comparisons_2.csv
data/IJB-A_11_sets/split2/verify_metadata_2.csv
data/IJB-A_11_sets/split3/train_3.csv
data/IJB-A_11_sets/split3/verify_comparisons_3.csv
data/IJB-A_11_sets/split3/verify_metadata_3.csv
data/IJB-A_11_sets/split4/train_4.csv
data/IJB-A_11_sets/split4/verify_comparisons_4.csv
data/IJB-A_11_sets/split4/verify_metadata_4.csv
data/IJB-A_11_sets/split5/train_5.csv
data/IJB-A_11_sets/split5/verify_comparisons_5.csv
data/IJB-A_11_sets/split5/verify_metadata_5.csv
data/IJB-A_11_sets/split6/train_6.csv
data/IJB-A_11_sets/split6/verify_comparisons_6.csv
data/IJB-A_11_sets/split6/verify_metadata_6.csv
data/IJB-A_11_sets/split7/train_7.csv
data/IJB-A_11_sets/split7/verify_comparisons_7.csv
data/IJB-A_11_sets/split7/verify_metadata_7.csv
data/IJB-A_11_sets/split8/train_8.csv
data/IJB-A_11_sets/split8/verify_comparisons_8.csv
data/IJB-A_11_sets/split8/verify_metadata_8.csv
data/IJB-A_11_sets/split9/train_9.csv
data/IJB-A_11_sets/split9/verify_comparisons_9.csv
data/IJB-A_11_sets/split9/verify_metadata_9.csv
data/IJB-A_1N_sets/split1/search_gallery_1.csv
data/IJB-A_1N_sets/split1/search_probe_1.csv
data/IJB-A_1N_sets/split1/train_1.csv
data/IJB-A_1N_sets/split10/search_gallery_10.csv
data/IJB-A_1N_sets/split10/search_probe_10.csv
data/IJB-A_1N_sets/split10/train_10.csv
data/IJB-A_1N_sets/split2/search_gallery_2.csv
data/IJB-A_1N_sets/split2/search_probe_2.csv
data/IJB-A_1N_sets/split2/train_2.csv
data/IJB-A_1N_sets/split3/search_gallery_3.csv
data/IJB-A_1N_sets/split3/search_probe_3.csv
data/IJB-A_1N_sets/split3/train_3.csv
data/IJB-A_1N_sets/split4/search_gallery_4.csv
data/IJB-A_1N_sets/split4/search_probe_4.csv
data/IJB-A_1N_sets/split4/train_4.csv
data/IJB-A_1N_sets/split5/search_gallery_5.csv
data/IJB-A_1N_sets/split5/search_probe_5.csv
data/IJB-A_1N_sets/split5/train_5.csv
data/IJB-A_1N_sets/split6/search_gallery_6.csv
data/IJB-A_1N_sets/split6/search_probe_6.csv
data/IJB-A_1N_sets/split6/train_6.csv
data/IJB-A_1N_sets/split7/search_gallery_7.csv
data/IJB-A_1N_sets/split7/search_probe_7.csv
data/IJB-A_1N_sets/split7/train_7.csv
data/IJB-A_1N_sets/split8/search_gallery_8.csv
data/IJB-A_1N_sets/split8/search_probe_8.csv
data/IJB-A_1N_sets/split8/train_8.csv
data/IJB-A_1N_sets/split9/search_gallery_9.csv
data/IJB-A_1N_sets/split9/search_probe_9.csv
data/IJB-A_1N_sets/split9/train_9.csv
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Manuel Gunther <mgunther@vast.uccs.edu>
# @date: Fri Sep 11 14:53:52 MDT 2015
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""This module provides the Database interface allowing the user to query the JANUS database.
"""
......@@ -34,29 +20,30 @@ class Database(bob.db.base.Database):
It provides many different ways to probe for the characteristics of the data
and for the data itself inside the database.
"""
"""
def __init__(self, original_directory = None, annotations_directory=None, original_extension=None):
# call base class constructor
self.original_directory = original_directory
self.original_extension = original_extension
#Creating our data structure to deal with the db files
self.memory_db = {}
self.templates = {} #Dictionary with the templates in a unique list
if(annotations_directory is None):#Get the default location
annotations_directory = bob.db.ijba.driver.Interface().files()[0]
import pkg_resources
annotations_directory = pkg_resources.resource_filename(__name__, 'data')
self.annotations_directory = annotations_directory
def _solve_comparisons(self, protocol):
"""
Given a protocol, try to solve the filename verify_comparisons_[n].csv where n is the split number
"""
relative_dir = "IJB-A_11_sets"
#Getting the split
......@@ -65,23 +52,23 @@ class Database(bob.db.base.Database):
if(split in protocol):
relative_dir = os.path.join(self.annotations_directory,relative_dir,split,"verify_comparisons_{0}.csv".format(i))
break
return relative_dir
def _solve_filename(self, protocol, purpose):
"""
Given a protocol and the purpose, try to solve the filename
"""
relative_dir = ""
#Getting the recognition task
if("search" in protocol):
relative_dir = "IJB-A_1N_sets"
else:
relative_dir = "IJB-A_11_sets"
#Getting the split
for i in range(1,10):
split = "split{0}".format(i)
......@@ -89,7 +76,7 @@ class Database(bob.db.base.Database):
relative_dir = os.path.join(relative_dir,split)
split_number = i
break
#Getting the file
if purpose=="train":
return os.path.join(self.annotations_directory, relative_dir,"train_{0}.csv".format(split_number))
......@@ -99,7 +86,7 @@ class Database(bob.db.base.Database):
return os.path.join(self.annotations_directory,relative_dir,"search_gallery_{0}.csv".format(split_number))
else:
return os.path.join(self.annotations_directory,relative_dir,"search_probe_{0}.csv".format(split_number))
else:
#comparison
return os.path.join(self.annotations_directory, relative_dir,"verify_metadata_{0}.csv".format(split_number))
......@@ -111,7 +98,7 @@ class Database(bob.db.base.Database):
"""
if not protocol in self.memory_db:
self.memory_db[protocol] = {}
self.memory_db[protocol] = {}
#Training set is the same for both major protocols (search and comparison)
if purpose=="train":
......@@ -197,15 +184,15 @@ class Database(bob.db.base.Database):
if "search" in protocol:
objects = self.objects(groups=groups, protocol=protocol)
else:
objects = []
for g in groups:
for g in groups:
if g == "world":
objects.extend(self.objects(groups=g, protocol=protocol))
else:
self._load_data(protocol, "dev", "")
objects.extend([o for t in self.memory_db[protocol]['comparison-templates'] for o in self.memory_db[protocol]['comparison-templates'][t].files ])
ids = list(set([o.client_id for o in objects ]))
return ids
......@@ -230,8 +217,8 @@ class Database(bob.db.base.Database):
protocol = self.check_parameter_for_validity(protocol, "protocol", self.protocol_names())
groups = self.check_parameters_for_validity(groups, "group", self.groups())
purposes = self.check_parameters_for_validity(purposes, "purpose", ["enroll","probe"])
ids = []
ids = []
if "search" in protocol:
for p in purposes:
self._load_data(protocol, "dev", p)
......@@ -239,7 +226,7 @@ class Database(bob.db.base.Database):
else:
self._load_data(protocol, "dev", "")
for p in purposes:
if p == "enroll":
for c in self.memory_db[protocol]['comparisons']:
ids.append(c)
......@@ -262,7 +249,7 @@ class Database(bob.db.base.Database):
This function returns a list of actual template_ids.
The according templates might differ between the protocols.
"""
"""
return self.model_ids(protocol)
......@@ -317,30 +304,30 @@ class Database(bob.db.base.Database):
objects.extend([o for t in self.memory_db[protocol]['enroll'] for o in self.memory_db[protocol]['enroll'][t].files])
else:
objects.extend([o for t in model_ids for o in self.memory_db[protocol]['enroll'][t].files])
if 'probe' in purposes:
self._load_data(protocol, "dev", "probe")
#The probes for the search are the same for all users
#The probes for the search are the same for all users
objects.extend([o for t in self.memory_db[protocol]['probe'] for o in self.memory_db[protocol]['probe'][t].files])
#Dealing with comparisons
else:
self._load_data(protocol, "dev", "")
if 'enroll' in purposes:
if model_ids is None:
for c in self.memory_db[protocol]['comparisons']:
objects.extend(self.memory_db[protocol]['comparison-templates'][c].files)
else:
for m in model_ids:
objects.extend(self.memory_db[protocol]['comparison-templates'][m].files)
if 'probe' in purposes:
if(model_ids is None):
for t in self.memory_db[protocol]['comparison-templates']:
......@@ -384,8 +371,8 @@ class Database(bob.db.base.Database):
# check that every parameter is as expected
#groups = self.check_parameters_for_validity(groups, "group", ["dev","world"])
purposes = self.check_parameters_for_validity(purposes, "purpose", ["enroll","probe"])
protocol = self.check_parameter_for_validity(protocol, "protocol", self.protocol_names())
purposes = self.check_parameters_for_validity(purposes, "purpose", ["enroll","probe"])
protocol = self.check_parameter_for_validity(protocol, "protocol", self.protocol_names())
templates = []
self._load_data(protocol, "dev", "enroll")
......@@ -400,17 +387,19 @@ class Database(bob.db.base.Database):
else:
for t in template_ids:
templates.append(self.memory_db[protocol]['comparison-templates'][t])
else:
else:
templates.extend(self.memory_db[protocol][p][m])
return templates
def annotations(self, file):
"""Returns the annotations for the given :py:class:`File` object as a dictionary, see :py:class:`Annotation` for details."""
# return annotations as obtained from the __call__ command of the Annotation class
"""Returns the annotations for the given :py:class:`File` object as a
dictionary, see :py:func:`read_annotations` for details.
"""
return file.annotations
......@@ -423,7 +412,7 @@ class Database(bob.db.base.Database):
"""Returns all possible protocols."""
protocol_choices = ['search_split%d' % d for d in range(1,11)]
protocol_choices += ['compare_split%d' % d for d in range(1,11)]
protocol_choices += ['compare_split%d' % d for d in range(1,11)]
return protocol_choices
......@@ -444,9 +433,9 @@ class Database(bob.db.base.Database):
self._load_data(p, "dev", "probe")
else:
self._load_data(p, "dev", "")
return self.templates[model_id].client_id
def original_file_name(self, file, check_existence = True):
......
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
# @date: Thu 18 Feb 2016 15:23:45 CET
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
This script has some sort of utilitary functions that parses the original database files
......@@ -29,9 +15,9 @@ import bob.db.base
class File(bob.db.base.File):
"""
IJBA File class
Diferent from its ascendent class, this one has the client ID as input
"""
def __init__(self, client_id, path, file_id = None):
"""**Constructor Documentation**
......@@ -58,20 +44,33 @@ class File(bob.db.base.File):
"""
super(File, self).__init__(path, file_id)
self.client_id = client_id
class Template():
"""A ``Template`` contains a list of :py:class:`File` objects belonging to the same subject (there might be several templates per subject).
class Template:
"""A ``Template`` contains a list of :py:class:`File` objects belonging to
the same subject (there might be several templates per subject).
These are listed in the ``self.files`` field.
A ``Template`` can serve for training, model enrollment, or for probing.
Each template belongs specifically to a certain protocol, as the template_id in the original file lists might differ for different protocols.
The according :py:class:`ProtocolPurpose` can be obtained using the ``self.protocol_purpose`` after creation of the database.
Note that the ``template_id`` corresponds to the template_id of the file lists, while the ``id`` is only used as a un
ique key for querying the database.
For convenience, the template also contains a ``path``, which is a concatenation of the first :py:attr:`File.media_id
` of the first file, and the ``self.template_id``, making it unique (at least per protocol).
Each template belongs specifically to a certain protocol, as the template_id
in the original file lists might differ for different protocols.
The protocol purpose can be obtained using ``self.protocol_purpose`` after
creation of the database.
Note that the ``template_id`` corresponds to the template_id of the file
lists, while the ``id`` is only used as a unique key for querying the
database.
For convenience, the template also contains a ``path``, which is a
concatenation of the ``File.media_id`` of the first file, and the
``self.template_id``, making it unique (at least per protocol).
"""
def __init__(self, template_id, subject_id, files):
self.id = template_id
self.client_id = subject_id
......@@ -82,13 +81,13 @@ ique key for querying the database.
def read_file(filename):
"""Reads the given file and yields the template id, the subject id and path_id (path + sighting_id)"""
with open(filename) as f:
# skip the first line
_ = f.readline()
for line in f:
splits = line.rstrip().split(',')