Commit 81fcf187 authored by Philip ABBET's avatar Philip ABBET

Add atnt/4 (api change: beat.backend.python v1.4.1)

parent df15c5e8
{
"description": "The AT&T Database of Faces",
"root_folder": "/idiap/group/biometric/databases/orl",
"protocols": [
{
"name": "idiap",
"template": "simple_face_recognition",
"sets": [
{
"name": "train",
"template": "train",
"view": "Train",
"outputs": {
"file_id": "{{ system_user.username }}/uint64/1",
"client_id": "{{ system_user.username }}/uint64/1",
"image": "{{ system_user.username }}/array_2d_uint8/1"
}
},
{
"name": "templates",
"template": "templates",
"view": "Templates",
"outputs": {
"file_id": "{{ system_user.username }}/uint64/1",
"client_id": "{{ system_user.username }}/uint64/1",
"template_id": "{{ system_user.username }}/uint64/1",
"image": "{{ system_user.username }}/array_2d_uint8/1"
}
},
{
"name": "probes",
"template": "probes",
"view": "Probes",
"outputs": {
"file_id": "{{ system_user.username }}/uint64/1",
"probe_id": "{{ system_user.username }}/uint64/1",
"client_id": "{{ system_user.username }}/uint64/1",
"template_ids": "{{ system_user.username }}/array_1d_uint64/1",
"image": "{{ system_user.username }}/array_2d_uint8/1"
}
}
]
},
{
"name": "idiap_test_eyepos",
"template": "advanced_face_recognition",
"sets": [
{
"name": "train",
"template": "train",
"view": "Train",
"outputs": {
"file_id": "{{ system_user.username }}/uint64/1",
"client_id": "{{ system_user.username }}/uint64/1",
"image": "{{ system_user.username }}/array_3d_uint8/1",
"eye_centers": "{{ system_user.username }}/eye_positions/1"
}
},
{
"name": "dev_templates",
"template": "templates",
"view": "Templates",
"parameters": {
"group": "dev"
},
"outputs": {
"file_id": "{{ system_user.username }}/uint64/1",
"client_id": "{{ system_user.username }}/uint64/1",
"template_id": "{{ system_user.username }}/uint64/1",
"image": "{{ system_user.username }}/array_3d_uint8/1",
"eye_centers": "{{ system_user.username }}/eye_positions/1"
}
},
{
"name": "dev_probes",
"template": "probes",
"view": "Probes",
"parameters": {
"group": "dev"
},
"outputs": {
"file_id": "{{ system_user.username }}/uint64/1",
"probe_id": "{{ system_user.username }}/uint64/1",
"client_id": "{{ system_user.username }}/uint64/1",
"template_ids": "{{ system_user.username }}/array_1d_uint64/1",
"image": "{{ system_user.username }}/array_3d_uint8/1",
"eye_centers": "{{ system_user.username }}/eye_positions/1"
}
},
{
"name": "test_templates",
"template": "templates",
"view": "Templates",
"parameters": {
"group": "eval"
},
"outputs": {
"file_id": "{{ system_user.username }}/uint64/1",
"client_id": "{{ system_user.username }}/uint64/1",
"template_id": "{{ system_user.username }}/uint64/1",
"image": "{{ system_user.username }}/array_3d_uint8/1",
"eye_centers": "{{ system_user.username }}/eye_positions/1"
}
},
{
"name": "test_probes",
"template": "probes",
"view": "Probes",
"parameters": {
"group": "eval"
},
"outputs": {
"file_id": "{{ system_user.username }}/uint64/1",
"probe_id": "{{ system_user.username }}/uint64/1",
"client_id": "{{ system_user.username }}/uint64/1",
"template_ids": "{{ system_user.username }}/array_1d_uint64/1",
"image": "{{ system_user.username }}/array_3d_uint8/1",
"eye_centers": "{{ system_user.username }}/eye_positions/1"
}
}
]
}
]
}
<
###############################################################################
# #
# Copyright (c) 2017 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# This file is part of the beat.examples module of the BEAT platform. #
# #
# Commercial License Usage #
# Licensees holding valid commercial BEAT licenses may use this file in #
# accordance with the terms contained in a written agreement between you #
# and Idiap. For further information contact tto@idiap.ch #
# #
# Alternatively, this file may be used under the terms of the GNU Affero #
# Public License version 3 as published by the Free Software and appearing #
# in the file LICENSE.AGPL included in the packaging of this file. #
# The BEAT platform is distributed in the hope that it will be useful, but #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
# or FITNESS FOR A PARTICULAR PURPOSE. #
# #
# You should have received a copy of the GNU Affero Public License along #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
# #
###############################################################################
import numpy as np
import bob.io.base
import bob.db.atnt
#----------------------------------------------------------
def get_client_end_index(objs, client_id, client_start_index,
start_index, end_index):
client_end_index = client_start_index
while client_end_index + 1 <= end_index:
obj = objs[client_end_index + 1 - start_index]
if isinstance(obj, tuple):
obj = obj[1]
if obj.client_id != client_id:
return client_end_index
client_end_index += 1
return end_index
#----------------------------------------------------------
def get_template_end_index(objs, template_id, template_start_index,
start_index, end_index):
template_end_index = template_start_index
while template_end_index + 1 <= end_index:
id = objs[template_end_index + 1 - start_index][0]
if id != template_id:
return template_end_index
template_end_index += 1
return end_index
#----------------------------------------------------------
class Train:
"""Outputs:
- image: "{{ system_user.username }}/array_2d_uint8/1"
- file_id: "{{ system_user.username }}/uint64/1"
- client_id: "{{ system_user.username }}/uint64/1"
- (optional) eye_centers: "{{ system_user.username }}/eye_positions/1"
One "file_id" is associated with a given "image".
One "eye_centers" is associated with a given "image".
Several "image" are associated with a given "client_id".
--------------- --------------- --------------- --------------- --------------- ---------------
| image | | image | | image | | image | | image | | image |
--------------- --------------- --------------- --------------- --------------- ---------------
--------------- --------------- --------------- --------------- --------------- ---------------
| file_id | | file_id | | file_id | | file_id | | file_id | | file_id |
--------------- --------------- --------------- --------------- --------------- ---------------
--------------- --------------- --------------- --------------- --------------- ---------------
| eye_centers | | eye_centers | | eye_centers | | eye_centers | | eye_centers | | eye_centers |
--------------- --------------- --------------- --------------- --------------- ---------------
----------------------------------------------- -----------------------------------------------
| client_id | | client_id |
----------------------------------------------- -----------------------------------------------
"""
def setup(self, root_folder, outputs, parameters, force_start_index=None,
force_end_index=None):
# Initialisations
self.root_folder = root_folder
self.outputs = outputs
# Open the database and load the objects to provide via the outputs
self.db = bob.db.atnt.Database()
self.objs = sorted(self.db.objects(groups='world', purposes=None),
key=lambda x: (x.client_id, x.id))
# Determine the range of indices that must be provided
self.start_index = force_start_index if force_start_index is not None else 0
self.end_index = force_end_index if force_end_index is not None else len(self.objs) - 1
self.objs = self.objs[self.start_index : self.end_index + 1]
self.next_index = self.start_index
return True
def done(self, last_data_index):
return last_data_index >= self.end_index
def next(self):
obj = self.objs[self.next_index - self.start_index]
# Output: client_id (only provide data when the client_id change)
if self.outputs['client_id'].isConnected() and \
self.outputs['client_id'].last_written_data_index < self.next_index:
client_end_index = get_client_end_index(self.objs, obj.client_id,
self.next_index,
self.start_index,
self.end_index)
self.outputs['client_id'].write(
{
'value': np.uint64(obj.client_id)
},
client_end_index
)
# Output: file_id (provide data at each iteration)
if self.outputs['file_id'].isConnected():
self.outputs['file_id'].write(
{
'value': np.uint64(obj.id)
},
self.next_index
)
# Output: image (provide data at each iteration)
if self.outputs['image'].isConnected():
self.outputs['image'].write(
{
'value': bob.io.base.load(obj.make_path(self.root_folder, '.pgm'))
},
self.next_index
)
# Optional output: eye_centers (provide data at each iteration)
if (self.outputs['eye_centers'] is not None) and self.outputs['eye_centers'].isConnected():
self.outputs['eye_centers'].write({
'left': {
'y': np.int32(48),
'x': np.int32(63),
},
'right': {
'y': np.int32(48),
'x': np.int32(27),
}
},
self.next_index
)
# Determine the next data index that must be provided
self.next_index = 1 + min([ x.last_written_data_index for x in self.outputs
if x.isConnected() ]
)
return True
#----------------------------------------------------------
class Templates:
"""Outputs:
- image: "{{ system_user.username }}/array_2d_uint8/1"
- file_id: "{{ system_user.username }}/uint64/1"
- client_id: "{{ system_user.username }}/uint64/1"
- template_id: "{{ system_user.username }}/uint64/1"
- (optional) eye_centers: "{{ system_user.username }}/eye_positions/1"
One "file_id" is associated with a given "image".
One "eye_centers" is associated with a given "image".
Several "image" are associated with a given "client_id".
Several "client_id" are associated with a given "template_id".
--------------- --------------- --------------- --------------- --------------- ---------------
| image | | image | | image | | image | | image | | image |
--------------- --------------- --------------- --------------- --------------- ---------------
--------------- --------------- --------------- --------------- --------------- ---------------
| file_id | | file_id | | file_id | | file_id | | file_id | | file_id |
--------------- --------------- --------------- --------------- --------------- ---------------
--------------- --------------- --------------- --------------- --------------- ---------------
| eye_centers | | eye_centers | | eye_centers | | eye_centers | | eye_centers | | eye_centers |
--------------- --------------- --------------- --------------- --------------- ---------------
----------------------------------------------- -----------------------------------------------
| client_id | | client_id |
----------------------------------------------- -----------------------------------------------
-----------------------------------------------------------------------------------------------
| template_id |
-----------------------------------------------------------------------------------------------
Note: for this particular database, there is only one "client_id"
per "template_id".
"""
def setup(self, root_folder, outputs, parameters, force_start_index=None,
force_end_index=None):
# Initialisations
self.root_folder = root_folder
self.outputs = outputs
# Open the database and load the objects to provide via the outputs
self.db = bob.db.atnt.Database()
template_ids = sorted(self.db.model_ids(groups='dev'), key=lambda x: int(x))
self.objs = []
for template_id in template_ids:
objs = sorted(self.db.objects(groups='dev', purposes='enroll',
model_ids=[template_id]),
key=lambda x: (x.client_id, x.id))
self.objs.extend([ (template_id, obj) for obj in objs ])
# Determine the range of indices that must be provided
self.start_index = force_start_index if force_start_index is not None else 0
self.end_index = force_end_index if force_end_index is not None else len(self.objs) - 1
self.objs = self.objs[self.start_index : self.end_index + 1]
self.next_index = self.start_index
return True
def done(self, last_data_index):
return last_data_index >= self.end_index
def next(self):
(template_id, obj) = self.objs[self.next_index - self.start_index]
# Output: template_id (only provide data when the template_id change)
if self.outputs['template_id'].isConnected() and \
self.outputs['template_id'].last_written_data_index < self.next_index:
template_end_index = get_template_end_index(self.objs, template_id,
self.next_index,
self.start_index,
self.end_index)
self.outputs['template_id'].write(
{
'value': np.uint64(template_id)
},
template_end_index
)
# Output: client_id (only provide data when the client_id change)
if self.outputs['client_id'].isConnected() and \
self.outputs['client_id'].last_written_data_index < self.next_index:
client_end_index = get_client_end_index(self.objs, obj.client_id,
self.next_index,
self.start_index,
self.end_index)
self.outputs['client_id'].write(
{
'value': np.uint64(obj.client_id)
},
client_end_index
)
# Output: file_id (provide data at each iteration)
if self.outputs['file_id'].isConnected():
self.outputs['file_id'].write(
{
'value': np.uint64(obj.id)
},
self.next_index
)
# Output: image (provide data at each iteration)
if self.outputs['image'].isConnected():
self.outputs['image'].write(
{
'value': bob.io.base.load(obj.make_path(self.root_folder, '.pgm'))
},
self.next_index
)
# Optional output: eye_centers (provide data at each iteration)
if (self.outputs['eye_centers'] is not None) and self.outputs['eye_centers'].isConnected():
self.outputs['eye_centers'].write({
'left': {
'y': np.int32(48),
'x': np.int32(63),
},
'right': {
'y': np.int32(48),
'x': np.int32(27),
}
},
self.next_index
)
# Determine the next data index that must be provided
self.next_index = 1 + min([ x.last_written_data_index for x in self.outputs
if x.isConnected() ]
)
return True
#----------------------------------------------------------
class Probes:
"""Outputs:
- image: "{{ system_user.username }}/array_2d_uint8/1"
- file_id: "{{ system_user.username }}/uint64/1"
- client_id: "{{ system_user.username }}/uint64/1"
- probe_id: "{{ system_user.username }}/uint64/1"
- template_ids: "{{ system_user.username }}/array_1d_uint64/1"
- (optional) eye_centers: "{{ system_user.username }}/eye_positions/1"
One "file_id" is associated with a given "image".
One "eye_centers" is associated with a given "image".
One "probe_id" is associated with a given "image".
Several "image" are associated with a given "client_id".
Several "client_id" are associated with a given "template_ids".
--------------- --------------- --------------- --------------- --------------- ---------------
| image | | image | | image | | image | | image | | image |
--------------- --------------- --------------- --------------- --------------- ---------------
--------------- --------------- --------------- --------------- --------------- ---------------
| file_id | | file_id | | file_id | | file_id | | file_id | | file_id |
--------------- --------------- --------------- --------------- --------------- ---------------
--------------- --------------- --------------- --------------- --------------- ---------------
| eye_centers | | eye_centers | | eye_centers | | eye_centers | | eye_centers | | eye_centers |
--------------- --------------- --------------- --------------- --------------- ---------------
--------------- --------------- --------------- --------------- --------------- ---------------
| probe_id | | probe_id | | probe_id | | probe_id | | probe_id | | probe_id |
--------------- --------------- --------------- --------------- --------------- ---------------
----------------------------------------------- -----------------------------------------------
| client_id | | client_id |
----------------------------------------------- -----------------------------------------------
-----------------------------------------------------------------------------------------------
| template_ids |
-----------------------------------------------------------------------------------------------
"""
def setup(self, root_folder, outputs, parameters, force_start_index=None,
force_end_index=None):
# Initialisations
self.root_folder = root_folder
self.outputs = outputs
# Open the database and load the objects to provide via the outputs
self.db = bob.db.atnt.Database()
self.objs = sorted(self.db.objects(groups='dev', purposes='probe'),
key=lambda x: (x.client_id, x.id))
# Determine the range of indices that must be provided
self.start_index = force_start_index if force_start_index is not None else 0
self.end_index = force_end_index if force_end_index is not None else len(self.objs) - 1
self.objs = self.objs[self.start_index : self.end_index + 1]
self.next_index = self.start_index
return True
def done(self, last_data_index):
return last_data_index >= self.end_index
def next(self):
obj = self.objs[self.next_index - self.start_index]
# Output: template_ids (only provide data when the template_ids change)
if self.outputs['template_ids'].isConnected() and \
self.outputs['template_ids'].last_written_data_index < self.next_index:
self.outputs['template_ids'].write(
{
'value': np.array(sorted(self.db.model_ids(groups='dev'),
key=lambda x: int(x)),
dtype='uint64')
},
self.end_index
)
# Output: client_id (only provide data when the client_id change)
if self.outputs['client_id'].isConnected() and \
self.outputs['client_id'].last_written_data_index < self.next_index:
client_end_index = get_client_end_index(self.objs, obj.client_id,
self.next_index,
self.start_index,
self.end_index)
self.outputs['client_id'].write(
{
'value': np.uint64(obj.client_id)
},
client_end_index
)
# Output: probe_id (provide data at each iteration)
if self.outputs['probe_id'].isConnected():
self.outputs['probe_id'].write(
{
'value': np.uint64(obj.id)
},
self.next_index
)
# Output: file_id (provide data at each iteration)
if self.outputs['file_id'].isConnected():
self.outputs['file_id'].write(
{
'value': np.uint64(obj.id)
},
self.next_index
)
# Output: image (provide data at each iteration)
if self.outputs['image'].isConnected():
self.outputs['image'].write(
{
'value': bob.io.base.load(obj.make_path(self.root_folder, '.pgm'))
},
self.next_index
)
# Optional output: eye_centers (provide data at each iteration)
if (self.outputs['eye_centers'] is not None) and self.outputs['eye_centers'].isConnected():
self.outputs['eye_centers'].write({
'left': {
'y': np.int32(48),
'x': np.int32(63),
},
'right': {
'y': np.int32(48),
'x': np.int32(27),
}
},
self.next_index
)
# Determine the next data index that must be provided
self.next_index = 1 + min([ x.last_written_data_index for x in self.outputs
if x.isConnected() ]
)
return True
#----------------------------------------------------------
# Test the behavior of the views (on fake data)
if __name__ == '__main__':
# Install a mock load method for the images
def mock_load(root_folder):
return np.ndarray((92, 112), dtype=np.uint8)
bob.io.base.load = mock_load
# Mock output class
class MockOutput:
def __init__(self, name, connected):
self.name = name
self.connected = connected
self.last_written_data_index = -1
self.written_data = []
def write(self, data, end_data_index):
self.written_data.append(( self.last_written_data_index + 1, end_data_index, data ))
self.last_written_data_index = end_data_index
def isConnected(self):
return self.connected
# Tester utility class
from beat.backend.python.outputs import OutputList
import itertools
class Tester:
def __init__(self, name, view_class, outputs_declaration):
self.name = name
self.view_class = view_class
self.outputs_declaration = {}
self.determine_increments(outputs_declaration)
for L in range(0, len(self.outputs_declaration) + 1):
for subset in itertools.combinations(self.outputs_declaration.keys(), L):
self.run(subset)
def determine_increments(self, outputs_declaration):
outputs = OutputList()
for name in outputs_declaration:
outputs.add(MockOutput(name, True))
parameters = dict()
view = self.view_class()
view.setup('', outputs, parameters)
view.next()
print "View '%s', increments found:" % self.name
for output in outputs:
self.outputs_declaration[output.name] = output.last_written_data_index + 1
print ' - %s: %d' % (output.name, output.last_written_data_index + 1)
def run(self, connected_outputs):
if len(connected_outputs) == 0:
return
print "Testing '%s', with %d output(s): %s" % (self.name, len(connected_outputs),
', '.join(connected_outputs))
connected_outputs = dict([ x for x in self.outputs_declaration.items()
if x[0] in connected_outputs ])
not_connected_outputs = dict([ x for x in self.outputs_declaration.items()
if x[0] not in connected_outputs ])
outputs = OutputList()
for name in self.outputs_declaration.keys():
outputs.add(MockOutput(name, name in connected_outputs))
parameters = dict()
view = self.view_class()
view.setup('', outputs, parameters)
biggest_index_increment = max(connected_outputs.values())
biggest_index_output = [ x[0] for x in connected_outputs.items()
if x[1] == biggest_index_increment ][0]
lowest_index_increment = min(connected_outputs.values())
lowest_index_output = [ x[0] for x in connected_outputs.items()
if x[1] == lowest_index_increment ][0]
next_expected_indices = {}
for name, increment in connected_outputs.items():
next_expected_indices[name] = 0