Commit ba3aa520 authored by Manuel Günther's avatar Manuel Günther
Browse files

Updated and fixed documentation handling

parent f434c631
Pipeline #5793 failed with stages
in 4 minutes and 37 seconds
......@@ -17,4 +17,4 @@ dist
build
*.egg
src/
db.sql3
bob/db/ijba/data
......@@ -15,11 +15,11 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""This is the Bob database entry for the JANUS database.
"""This is the Bob database entry for the IJB-A database.
"""
from .query import Database
from .reader import get_templates
from .reader import get_templates, Template, File
#from .models import Client, File, Annotation, Template, Protocol, Comparisons, Protocol_Template_Association
#, File_Template_Association
......@@ -31,5 +31,25 @@ def get_config():
return bob.extension.get_config(__name__)
# gets sphinx autodoc done right - don't remove it
def __appropriate__(*args):
"""Says object was actually declared here, and not in the import module.
Fixing sphinx warnings of not being able to find classes, when path is shortened.
Parameters:
*args: An iterable of objects to modify
Resolves `Sphinx referencing issues
<https://github.com/sphinx-doc/sphinx/issues/3048>`
"""
for obj in args: obj.__module__ = __name__
__appropriate__(
Database,
get_templates,
Template,
File,
)
# gets sphinx autodoc done right - don't remove it
__all__ = [_ for _ in dir() if not _.startswith('_')]
......@@ -15,7 +15,7 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""This module provides the Database interface allowing the user to query the JANUS database.
"""This module provides the Database interface allowing the user to query the IJB-A database.
"""
import os
......@@ -34,29 +34,29 @@ class Database(bob.db.base.Database):
It provides many different ways to probe for the characteristics of the data
and for the data itself inside the database.
"""
"""
def __init__(self, original_directory = None, annotations_directory=None, original_extension=None):
# call base class constructor
self.original_directory = original_directory
self.original_extension = original_extension
#Creating our data structure to deal with the db files
self.memory_db = {}
self.templates = {} #Dictionary with the templates in a unique list
if(annotations_directory is None):#Get the default location
annotations_directory = bob.db.ijba.driver.Interface().files()[0]
self.annotations_directory = annotations_directory
def _solve_comparisons(self, protocol):
"""
Given a protocol, try to solve the filename verify_comparisons_[n].csv where n is the split number
"""
relative_dir = "IJB-A_11_sets"
#Getting the split
......@@ -65,23 +65,23 @@ class Database(bob.db.base.Database):
if(split in protocol):
relative_dir = os.path.join(self.annotations_directory,relative_dir,split,"verify_comparisons_{0}.csv".format(i))
break
return relative_dir
def _solve_filename(self, protocol, purpose):
"""
Given a protocol and the purpose, try to solve the filename
"""
relative_dir = ""
#Getting the recognition task
if("search" in protocol):
relative_dir = "IJB-A_1N_sets"
else:
relative_dir = "IJB-A_11_sets"
#Getting the split
for i in range(1,10):
split = "split{0}".format(i)
......@@ -89,7 +89,7 @@ class Database(bob.db.base.Database):
relative_dir = os.path.join(relative_dir,split)
split_number = i
break
#Getting the file
if purpose=="train":
return os.path.join(self.annotations_directory, relative_dir,"train_{0}.csv".format(split_number))
......@@ -99,7 +99,7 @@ class Database(bob.db.base.Database):
return os.path.join(self.annotations_directory,relative_dir,"search_gallery_{0}.csv".format(split_number))
else:
return os.path.join(self.annotations_directory,relative_dir,"search_probe_{0}.csv".format(split_number))
else:
#comparison
return os.path.join(self.annotations_directory, relative_dir,"verify_metadata_{0}.csv".format(split_number))
......@@ -111,7 +111,7 @@ class Database(bob.db.base.Database):
"""
if not protocol in self.memory_db:
self.memory_db[protocol] = {}
self.memory_db[protocol] = {}
#Training set is the same for both major protocols (search and comparison)
if purpose=="train":
......@@ -197,15 +197,15 @@ class Database(bob.db.base.Database):
if "search" in protocol:
objects = self.objects(groups=groups, protocol=protocol)
else:
objects = []
for g in groups:
for g in groups:
if g == "world":
objects.extend(self.objects(groups=g, protocol=protocol))
else:
self._load_data(protocol, "dev", "")
objects.extend([o for t in self.memory_db[protocol]['comparison-templates'] for o in self.memory_db[protocol]['comparison-templates'][t].files ])
ids = list(set([o.client_id for o in objects ]))
return ids
......@@ -230,8 +230,8 @@ class Database(bob.db.base.Database):
protocol = self.check_parameter_for_validity(protocol, "protocol", self.protocol_names())
groups = self.check_parameters_for_validity(groups, "group", self.groups())
purposes = self.check_parameters_for_validity(purposes, "purpose", ["enroll","probe"])
ids = []
ids = []
if "search" in protocol:
for p in purposes:
self._load_data(protocol, "dev", p)
......@@ -239,7 +239,7 @@ class Database(bob.db.base.Database):
else:
self._load_data(protocol, "dev", "")
for p in purposes:
if p == "enroll":
for c in self.memory_db[protocol]['comparisons']:
ids.append(c)
......@@ -258,11 +258,11 @@ class Database(bob.db.base.Database):
def template_ids(self, protocol='search_split1'):
"""Returns a list of valid template ids, where :py:class:`Template`'s can be used both for model enrollment or probing.
"""Returns a list of valid template ids, where :py:class:`bob.db.ijba.Template`'s can be used both for model enrollment or probing.
This function returns a list of actual template_ids.
The according templates might differ between the protocols.
"""
"""
return self.model_ids(protocol)
......@@ -317,30 +317,30 @@ class Database(bob.db.base.Database):
objects.extend([o for t in self.memory_db[protocol]['enroll'] for o in self.memory_db[protocol]['enroll'][t].files])
else:
objects.extend([o for t in model_ids for o in self.memory_db[protocol]['enroll'][t].files])
if 'probe' in purposes:
self._load_data(protocol, "dev", "probe")
#The probes for the search are the same for all users
#The probes for the search are the same for all users
objects.extend([o for t in self.memory_db[protocol]['probe'] for o in self.memory_db[protocol]['probe'][t].files])
#Dealing with comparisons
else:
self._load_data(protocol, "dev", "")
if 'enroll' in purposes:
if model_ids is None:
for c in self.memory_db[protocol]['comparisons']:
objects.extend(self.memory_db[protocol]['comparison-templates'][c].files)
else:
for m in model_ids:
objects.extend(self.memory_db[protocol]['comparison-templates'][m].files)
if 'probe' in purposes:
if(model_ids is None):
for t in self.memory_db[protocol]['comparison-templates']:
......@@ -358,7 +358,7 @@ class Database(bob.db.base.Database):
def object_sets(self, groups='dev', protocol='search_split1', purposes='probe', model_ids=None, media_ids=None, frames=None):
"""Using the specified restrictions, this function returns a list of :py:class:`Template` objects.
"""Using the specified restrictions, this function returns a list of :py:class:`bob.db.ijba.Template` objects.
Keyword Parameters:
......@@ -384,8 +384,8 @@ class Database(bob.db.base.Database):
# check that every parameter is as expected
#groups = self.check_parameters_for_validity(groups, "group", ["dev","world"])
purposes = self.check_parameters_for_validity(purposes, "purpose", ["enroll","probe"])
protocol = self.check_parameter_for_validity(protocol, "protocol", self.protocol_names())
purposes = self.check_parameters_for_validity(purposes, "purpose", ["enroll","probe"])
protocol = self.check_parameter_for_validity(protocol, "protocol", self.protocol_names())
templates = []
self._load_data(protocol, "dev", "enroll")
......@@ -400,16 +400,20 @@ class Database(bob.db.base.Database):
else:
for t in template_ids:
templates.append(self.memory_db[protocol]['comparison-templates'][t])
else:
else:
templates.extend(self.memory_db[protocol][p][m])
return templates
def annotations(self, file):
"""Returns the annotations for the given :py:class:`File` object as a dictionary, see :py:class:`Annotation` for details."""
"""Returns the annotations for the given :py:class:`bob.db.ijba.File` object as a dictionary.
.. todo::
Explain the annotations in more detail.
"""
# return annotations as obtained from the __call__ command of the Annotation class
return file.annotations
......@@ -423,7 +427,7 @@ class Database(bob.db.base.Database):
"""Returns all possible protocols."""
protocol_choices = ['search_split%d' % d for d in range(1,11)]
protocol_choices += ['compare_split%d' % d for d in range(1,11)]
protocol_choices += ['compare_split%d' % d for d in range(1,11)]
return protocol_choices
......@@ -444,14 +448,14 @@ class Database(bob.db.base.Database):
self._load_data(p, "dev", "probe")
else:
self._load_data(p, "dev", "")
return self.templates[model_id].client_id
def original_file_name(self, file, check_existence = True):
"""Returns the original image file name with the correct file name extension.
To be able to call this function, the ``original_directory`` must have been specified in the :py:class:`Database` constructor.
To be able to call this function, the ``original_directory`` must have been specified in the :py:class:`bob.db.ijba.Database` constructor.
Keyword parameters:
......
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
# @date: Thu 18 Feb 2016 15:23:45 CET
# @date: Thu 18 Feb 2016 15:23:45 CET
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
......@@ -29,11 +29,11 @@ import bob.db.base
class File(bob.db.base.File):
"""
IJBA File class
Diferent from its ascendent class, this one has the client ID as input
"""
def __init__(self, client_id, path, file_id = None):
def __init__(self, client_id, path, file_id, annotations, extension, media_id):
"""**Constructor Documentation**
Initialize the File object with the minimum required data.
......@@ -55,10 +55,22 @@ class File(bob.db.base.File):
The type of it is dependent on your implementation.
If you use an SQL database, this should be an SQL type like Integer or String.
If you are using an automatically determined file id, you can skip selecting the file id.
annotations: dict
Dictionary of hand-labeled annotations.
extension: str
The filename extension of this file.
media_id: str
The media id (e.g., the video name for frames).
"""
super(File, self).__init__(path, file_id)
self.client_id = client_id
self.annotations = annotations
self.extension = extension
self.media_id = media_id
class Template():
......@@ -66,11 +78,8 @@ class Template():
These are listed in the ``self.files`` field.
A ``Template`` can serve for training, model enrollment, or for probing.
Each template belongs specifically to a certain protocol, as the template_id in the original file lists might differ for different protocols.
The according :py:class:`ProtocolPurpose` can be obtained using the ``self.protocol_purpose`` after creation of the database.
Note that the ``template_id`` corresponds to the template_id of the file lists, while the ``id`` is only used as a un
ique key for querying the database.
For convenience, the template also contains a ``path``, which is a concatenation of the first :py:attr:`File.media_id
` of the first file, and the ``self.template_id``, making it unique (at least per protocol).
Note that the ``template_id`` corresponds to the template_id of the file lists, while the ``id`` is only used as a unique key for querying the database.
For convenience, the template also contains a ``path``, which is a concatenation of the first :py:attr:`bob.db.ijba.File.media_id` of the first file, and the ``self.template_id``, making it unique (at least per protocol).
"""
def __init__(self, template_id, subject_id, files):
self.id = template_id
......@@ -82,13 +91,13 @@ ique key for querying the database.
def read_file(filename):
"""Reads the given file and yields the template id, the subject id and path_id (path + sighting_id)"""
with open(filename) as f:
# skip the first line
_ = f.readline()
for line in f:
splits = line.rstrip().split(',')
assert len(splits) == 25
......@@ -97,16 +106,13 @@ def read_file(filename):
client_id = int(splits[1])
path,extension = os.path.splitext(splits[2])
media_id = splits[3]
sighting_id = splits[4]
file_id = "%s-%s" % (path, sighting_id)
annotations = read_annotations(splits[6:])
#Creating the file object and binding the annotations directly to the object
file_obj = File(client_id, path, file_id)
annotations = read_annotations(splits[6:])
file_obj.annotations = annotations
file_obj.extension = extension
file_obj.media_id = splits[3]
file_obj = File(client_id, path, file_id, annotations, extension, media_id)
yield template_id, client_id, file_obj
......@@ -121,7 +127,7 @@ def get_comparisons(filename):
for line in lines:
splits = line.rstrip().split(',')
splits = line.rstrip().split(',')
assert len(splits) == 2
template_A = int(splits[0])
......@@ -131,33 +137,33 @@ def get_comparisons(filename):
template_comparisons[template_A] = [template_B]
else:
template_comparisons[template_A].append(template_B)
return template_comparisons
def get_templates(filename, verbose=True):
"""
Given a IJBA file, get a dictionary with all their templates with their respective files in the following format:
templates['template_01'] = [file_01, file_02, file_03]
templates['template_02'] = [file_01, file_02, file_03]
.
.
.
"""
"""
templates = {}
for template_id, client_id, file_obj in read_file(filename):
# create template with given IDs
# create template with given IDs
if template_id not in templates:
templates[template_id] = Template(template_id,client_id,[file_obj])
else:
templates[template_id].files.append(file_obj)
return templates
......@@ -181,7 +187,7 @@ def read_annotations(raw_annotations):
n_x = float(raw_annotations[8]) if raw_annotations[8]!='' else None
n_y = float(raw_annotations[9]) if raw_annotations[9]!='' else None
yaw = float(raw_annotations[10]) if raw_annotations[10]!='' else None
forehead = raw_annotations[17-6]
eyes = raw_annotations[18-6]
nm = raw_annotations[19-6]
......@@ -208,5 +214,3 @@ def read_annotations(raw_annotations):
if yaw is not None: annotations['yaw'] = yaw
return annotations
py:class bob.db.base.file.File
py:class bob.db.base.database.SQLiteDatabase
py:class bob.db.base.database.Database
py:class sqlalchemy.ext.declarative.api.Base
py:class File
py:class Annotation
py:class Template
py:class bob.db.ijba.Template
# Not available in Python 2.7, but ok in Python 3.x
py:exc ValueError
# The attribute is not separately documented
py:attr bob.db.ijba.File.media_id
......@@ -3,7 +3,7 @@
.. @date: Fri Sep 11 14:53:52 MDT 2015
================
JANUS Database
IJB-A Database
================
.. automodule:: bob.db.ijba
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment