Commit 78825360 authored by Amir MOHAMMADI's avatar Amir MOHAMMADI

convert databases to v2 format

parent f1ca7458
.. vim: set fileencoding=utf-8 :
======================
BEAT Tutorial Prefix
======================
=====================
A Hands-On Tutorial
=====================
This package contains the prefix folder for BEAT tutorials. Run any
BEAT commands relating to the prefix in the top-level folder of this
project, next to the `prefix` folder.
This repository contains a hands-on BEAT tutorial.
To get started, run the commands below to install BEAT::
$ conda create -n beat-tutorial --override-channels \
-c https://www.idiap.ch/software/beat/conda \
-c defaults \
python=3 beat.editor scikit-learn matplotlib ipdb jupyterlab
$ conda activate beat-tutorial
$ conda config --env --add channels defaults
$ conda config --env --add channels https://www.idiap.ch/software/beat/conda
It is best that you create a separate conda environment for this tutorial
using the instructions provided above.
Then, launch jupyter lab and follow the instructions in ``tutorial.ipynb``::
$ jupyter lab
Support
......
{
"root_folder": "/remote/dataset/att_faces",
"description": "The AT&T Database of Faces",
"protocols": [
{
"name": "idiap",
"template": "idiap/1",
"views": {
"train": {
"view": "Train",
"parameters": {}
},
"templates": {
"view": "Templates",
"parameters": {}
},
"probes": {
"view": "Probes",
"parameters": {}
}
}
},
{
"name": "idiap_test_eyepos",
"template": "idiap_test_eyepos/1",
"views": {
"train": {
"view": "TrainEyePositions",
"parameters": {}
},
"dev_templates": {
"view": "TemplatesEyePositions",
"parameters": {
"group": "dev"
}
},
"dev_probes": {
"view": "ProbesEyePositions",
"parameters": {
"group": "dev"
}
},
"test_templates": {
"view": "TemplatesEyePositions",
"parameters": {
"group": "eval"
}
},
"test_probes": {
"view": "ProbesEyePositions",
"parameters": {
"group": "eval"
}
}
}
}
],
"schema_version": 2
}
\ No newline at end of file
This diff is collapsed.
The AT&T Database of Faces
\ No newline at end of file
{
"description": "The AT&T Database of Faces",
"protocols": [
{
"name": "idiap",
"template": "idiap/1",
"views": {
"train": {
"view": "Train",
"parameters": {}
},
"templates": {
"view": "Templates",
"parameters": {}
},
"probes": {
"view": "Probes",
"parameters": {}
}
}
}
],
"root_folder": "/home/jae/code/atnt_db",
"schema_version": 2
}
\ No newline at end of file
This diff is collapsed.
The AT&T Database of Faces
\ No newline at end of file
{
"description": "",
"protocols": [
{
"name": "Main",
"template": "Main/1",
"views": {
"training": {
"view": "Training",
"parameters": {}
},
"testing": {
"view": "Testing",
"parameters": {}
}
}
}
],
"root_folder": "/not/needed",
"schema_version": 2
}
\ No newline at end of file
#!/usr/bin/env python
# You may import any python packages that will be available in the environment you will run this database in
# Environments can change based on the experiment's settings
from collections import namedtuple
import numpy as np
from beat.backend.python.database import View
import bob.db.iris
class Training(View):
"""Outputs:
- measurements: 1D float array (four values)
- species: text label
"""
# build the data for your view
# split the raw data into (homogenous) bits and return a keyed iterable
# (something with `.keys()` available to it, like a dict)
# the key names must be the same as the output names for sets that use this view
# root_folder: the path to the root folder of the database's files (not always applicable)
# parameters: parameters passed to the view, defined in the metadata
def index(self, root_folder, parameters):
Entry = namedtuple('Entry', ['measurements', 'species'])
db = bob.db.iris.data()
objs = list()
for species in db.keys():
for measurement in db[species]:
objs.append(Entry(measurement, species))
return objs
# returns a value at a specific index in the iterable for this view
# output: the specific output value requested
# index: the current index of the iterable
def get(self, output, index):
# to get the current object referenced by the given index:
# obj = self.objs[index]
# note that this object is a named tuple, with fields equivalent to your keys from
# the objects returned from the index function
obj = self.objs[index]
if output == 'measurements':
return {
'value': obj.measurements
}
else:
return {
'text': obj.species
}
class Testing(View):
"""Outputs:
- measurements: 1D float array (four values)
- species: text label
"""
# build the data for your view
# split the raw data into (homogenous) bits and return a keyed iterable
# (something with `.keys()` available to it, like a dict)
# the key names must be the same as the output names for sets that use this view
# root_folder: the path to the root folder of the database's files (not always applicable)
# parameters: parameters passed to the view, defined in the metadata
def index(self, root_folder, parameters):
Entry = namedtuple('Entry', ['measurements', 'species'])
db = bob.db.iris.data()
objs = list()
for species in db.keys():
for measurement in db[species]:
objs.append(Entry(measurement , species))
return objs
# returns a value at a specific index in the iterable for this view
# output: the specific output value requested
# index: the current index of the iterable
def get(self, output, index):
# to get the current object referenced by the given index:
# obj = self.objs[index]
# note that this object is a named tuple, with fields equivalent to your keys from
# the objects returned from the index function
obj = self.objs[index]
if output == 'measurements':
return {
'value': obj.measurements
}
else:
return {
'text': obj.species
}
{
"description": "The MNIST Database of Handwritten Digits",
"protocols": [
{
"name": "idiap",
"template": "idiap/1",
"views": {
"train": {
"view": "View",
"parameters": {
"group": "train"
}
},
"test": {
"view": "View",
"parameters": {
"group": "test"
}
}
}
}
],
"root_folder": "/remote/dataset/mnist",
"schema_version": 2
}
\ No newline at end of file
###############################################################################
# #
# Copyright (c) 2017 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# This file is part of the beat.examples module of the BEAT platform. #
# #
# Commercial License Usage #
# Licensees holding valid commercial BEAT licenses may use this file in #
# accordance with the terms contained in a written agreement between you #
# and Idiap. For further information contact tto@idiap.ch #
# #
# Alternatively, this file may be used under the terms of the GNU Affero #
# Public License version 3 as published by the Free Software and appearing #
# in the file LICENSE.AGPL included in the packaging of this file. #
# The BEAT platform is distributed in the hope that it will be useful, but #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
# or FITNESS FOR A PARTICULAR PURPOSE. #
# #
# You should have received a copy of the GNU Affero Public License along #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
# #
###############################################################################
import numpy as np
import bob.db.mnist
#----------------------------------------------------------
def get_label_end_index(objs, label, label_start_index,
start_index, end_index):
label_end_index = label_start_index
while label_end_index + 1 <= end_index:
label_ = objs[label_end_index + 1 - start_index][1]
if label_ != label:
return label_end_index
label_end_index += 1
return end_index
#----------------------------------------------------------
class View:
"""Outputs:
- image: "{{ system_user.username }}/array_2d_uint8/1"
- id: "{{ system_user.username }}/uint64/1"
- class_id: "{{ system_user.username }}/uint64/1"
One "id" is associated with a given "image".
Several "image" are associated with a given "class_id".
--------------- --------------- --------------- --------------- --------------- ---------------
| image | | image | | image | | image | | image | | image |
--------------- --------------- --------------- --------------- --------------- ---------------
--------------- --------------- --------------- --------------- --------------- ---------------
| id | | id | | id | | id | | id | | id |
--------------- --------------- --------------- --------------- --------------- ---------------
----------------------------------------------- -----------------------------------------------
| class_id | | class_id |
----------------------------------------------- -----------------------------------------------
"""
def setup(self, root_folder, outputs, parameters, force_start_index=None,
force_end_index=None):
# Initialisations
self.root_folder = root_folder
self.outputs = outputs
self.parameters = parameters
# Open the database and load the objects to provide via the outputs
self.db = bob.db.mnist.Database(data_dir=self.root_folder)
features, labels = self.db.data(groups=parameters['group'])
self.objs = sorted([ (features[i], labels[i]) for i in range(len(features)) ],
key=lambda x: x[1])
# Determine the range of indices that must be provided
self.start_index = force_start_index if force_start_index is not None else 0
self.end_index = force_end_index if force_end_index is not None else len(self.objs) - 1
self.objs = self.objs[self.start_index : self.end_index + 1]
self.next_index = self.start_index
return True
def done(self, last_data_index):
return last_data_index >= self.end_index
def next(self):
features, label = self.objs[self.next_index - self.start_index]
# Output: class_id (only provide data when the class_id change)
if self.outputs['class_id'].isConnected() and \
self.outputs['class_id'].last_written_data_index < self.next_index:
label_end_index = get_label_end_index(self.objs, label,
self.next_index,
self.start_index,
self.end_index)
self.outputs['class_id'].write(
{
'value': np.uint64(label)
},
label_end_index
)
# Output: id (provide data at each iteration)
if self.outputs['id'].isConnected():
self.outputs['id'].write(
{
'value': np.uint64(self.next_index)
},
self.next_index
)
# Output: image (provide data at each iteration)
if self.outputs['image'].isConnected():
self.outputs['image'].write(
{
'value': features.reshape((28, 28))
},
self.next_index
)
# Determine the next data index that must be provided
self.next_index = 1 + min([ x.last_written_data_index for x in self.outputs
if x.isConnected() ]
)
return True
#----------------------------------------------------------
def setup_tests():
pass
#----------------------------------------------------------
# Test the behavior of the views (on fake data)
if __name__ == '__main__':
setup_tests()
# Note: This database can't be tested without the actual data, since
# the actual files are needed by this implementation
from beat.backend.python.database import DatabaseTester
DatabaseTester('View', View,
[
'class_id',
'id',
'image',
],
parameters=dict(
group='train',
),
)
The MNIST Database of Handwritten Digits
\ No newline at end of file
{
"description": "The MOBIO Database of Faces",
"protocols": [
{
"name": "male",
"template": "male/1",
"views": {
"train": {
"view": "Train",
"parameters": {
"annotations": "../IMAGE_ANNOTATIONS",
"protocol": "male"
}
},
"dev_templates": {
"view": "Templates",
"parameters": {
"annotations": "../IMAGE_ANNOTATIONS",
"group": "dev",
"protocol": "male"
}
},
"dev_probes": {
"view": "Probes",
"parameters": {
"annotations": "../IMAGE_ANNOTATIONS",
"group": "dev",
"protocol": "male"
}
},
"test_templates": {
"view": "Templates",
"parameters": {
"annotations": "../IMAGE_ANNOTATIONS",
"group": "eval",
"protocol": "male"
}
},
"test_probes": {
"view": "Probes",
"parameters": {
"annotations": "../IMAGE_ANNOTATIONS",
"group": "eval",
"protocol": "male"
}
}
}
},
{
"name": "female",
"template": "female/1",
"views": {
"train": {
"view": "Train",
"parameters": {
"annotations": "../IMAGE_ANNOTATIONS",
"protocol": "female"
}
},
"dev_templates": {
"view": "Templates",
"parameters": {
"annotations": "../IMAGE_ANNOTATIONS",
"group": "dev",
"protocol": "female"
}
},
"dev_probes": {
"view": "Probes",
"parameters": {
"annotations": "../IMAGE_ANNOTATIONS",
"group": "dev",
"protocol": "female"
}
},
"test_templates": {
"view": "Templates",
"parameters": {
"annotations": "../IMAGE_ANNOTATIONS",
"group": "eval",
"protocol": "female"
}
},
"test_probes": {
"view": "Probes",
"parameters": {
"annotations": "../IMAGE_ANNOTATIONS",
"group": "eval",
"protocol": "female"
}
}
}
}
],
"root_folder": "/remote/dataset/mobio/IMAGES_PNG",
"schema_version": 2
}
\ No newline at end of file
This diff is collapsed.
The MOBIO Database of Faces
\ No newline at end of file
{
"value8": "int8",
"value16": "int16",
"value32": "int32",
"value64": "int64"
}
{
"schema_version": 1,
"sets": [
{
"name": "training",
"outputs": {
"measurements": "system/array_1d_floats/1",
"species": "system/text/1"
}
},
{
"name": "testing",
"outputs": {
"measurements": "system/array_1d_floats/1",
"species": "system/text/1"
}
}
]
}
\ No newline at end of file
{
"schema_version": 1,
"sets": [
{
"name": "train",
"outputs": {
"client_id": "system/uint64/1",
"eye_centers": "system/eye_positions/1",
"file_id": "system/uint64/1",
"image": "system/array_3d_uint8/1"
}
},
{
"name": "dev_templates",
"outputs": {
"client_id": "system/uint64/1",
"eye_centers": "system/eye_positions/1",
"file_id": "system/uint64/1",
"image": "system/array_3d_uint8/1",
"template_id": "system/uint64/1"
}
},
{
"name": "dev_probes",
"outputs": {
"client_id": "system/uint64/1",
"eye_centers": "system/eye_positions/1",
"file_id": "system/uint64/1",
"image": "system/array_3d_uint8/1",
"probe_id": "system/uint64/1",
"template_ids": "system/array_1d_uint64/1"
}
},
{
"name": "test_templates",
"outputs": {
"client_id": "system/uint64/1",
"eye_centers": "system/eye_positions/1",
"file_id": "system/uint64/1",
"image": "system/array_3d_uint8/1",
"template_id": "system/uint64/1"
}
},
{
"name": "test_probes",
"outputs": {
"client_id": "system/uint64/1",
"eye_centers": "system/eye_positions/1",
"file_id": "system/uint64/1",
"image": "system/array_3d_uint8/1",
"probe_id": "system/uint64/1",
"template_ids": "system/array_1d_uint64/1"
}
}
]
}
\ No newline at end of file
{
"schema_version": 1,
"sets": [
{
"outputs": {
"image": "system/array_2d_uint8/1",
"file_id": "system/uint64/1",
"client_id": "system/uint64/1"
},
"name": "train"
},
{
"outputs": {
"image": "system/array_2d_uint8/1",
"file_id": "system/uint64/1",
"client_id": "system/uint64/1",
"template_id": "system/uint64/1"
},
"name": "templates"
},
{
"outputs": {
"probe_id": "system/uint64/1",
"image": "system/array_2d_uint8/1",
"file_id": "system/uint64/1",
"client_id": "system/uint64/1",
"template_ids": "system/array_1d_uint64/1"
},
"name": "probes"
}
]
}
\ No newline at end of file
{
"schema_version": 1,
"sets": [
{
"outputs": {
"image": "system/array_3d_uint8/1",
"eye_centers": "system/eye_positions/1",
"file_id": "system/uint64/1",
"client_id": "system/uint64/1"
},
"name": "train"
},
{
"outputs": {
"image": "system/array_3d_uint8/1",
"eye_centers": "system/eye_positions/1",
"file_id": "system/uint64/1",
"client_id": "system/uint64/1",
"template_id": "system/uint64/1"
},
"name": "dev_templates"
},
{
"outputs": {
"image": "system/array_3d_uint8/1",
"probe_id": "system/uint64/1",
"file_id": "system/uint64/1",
"client_id": "system/uint64/1",
"eye_centers": "system/eye_positions/1",
"template_ids": "system/array_1d_uint64/1"
},
"name": "dev_probes"
},
{
"outputs": {
"image": "system/array_3d_uint8/1",
"eye_centers": "system/eye_positions/1",
"file_id": "system/uint64/1",
"client_id": "system/uint64/1",
"template_id": "system/uint64/1"
},
"name": "test_templates"
},
{