-
Tiago de Freitas Pereira authoredTiago de Freitas Pereira authored
ExperimentAnalizer.py 5.26 KiB
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
# @date: Tue 09 Aug 2016 15:33 CEST
"""
Neural net work error rates analizer
"""
import numpy
import bob.measure
from tensorflow.core.framework import summary_pb2
from scipy.spatial.distance import cosine
from bob.learn.tensorflow.datashuffler import Memory, Disk
class ExperimentAnalizer:
"""
Analizer.
I don't know if this is the best way to do, but what this class do is the following.
As an enrollment sample, averare all the TRAINING samples for one particular class.
The probing is done with the validation set
"""
def __init__(self, convergence_threshold=0.01, convergence_reference='eer'):
"""
Use the CNN as feature extractor for a n-class classification
** Parameters **
data_shuffler:
graph:
session:
convergence_threshold:
convergence_reference: References to analize the convergence. Possible values are `eer`, `far10`, `far10`
"""
self.data_shuffler = None
self.network = None
self.session = None
# Statistics
self.eer = []
self.far10 = []
self.far100 = []
self.far1000 = []
def __call__(self, data_shuffler, network, session):
#if self.data_shuffler is None:
# self.data_shuffler = data_shuffler
# self.network = network
# self.session = session
# Getting the base class. Recipe extracted from
# http://stackoverflow.com/questions/5516263/creating-an-object-from-a-base-class-object-in-python/5516330#5516330
if isinstance(data_shuffler, Memory):
base_data_shuffler = object.__new__(Memory)
base_data_shuffler.__dict__ = data_shuffler.__dict__.copy()
else:
base_data_shuffler = object.__new__(Disk)
base_data_shuffler.__dict__ = data_shuffler.__dict__.copy()
# Extracting features for enrollment
enroll_data, enroll_labels = base_data_shuffler.get_batch()
enroll_features = network(enroll_data, session=session)
del enroll_data
# Extracting features for probing
probe_data, probe_labels = base_data_shuffler.get_batch()
probe_features = network(probe_data, session=session)
del probe_data
# Creating models
models = []
for i in range(len(base_data_shuffler.possible_labels)):
indexes_model = numpy.where(enroll_labels == data_shuffler.possible_labels[i])[0]
models.append(numpy.mean(enroll_features[indexes_model, :], axis=0))
# Probing
positive_scores = numpy.zeros(shape=0)
negative_scores = numpy.zeros(shape=0)
for i in range(len(base_data_shuffler.possible_labels)):
#for i in self.data_shuffler.possible_labels:
# Positive scoring
indexes = probe_labels == base_data_shuffler.possible_labels[i]
positive_data = probe_features[indexes, :]
p = [cosine(models[i], positive_data[j]) for j in range(positive_data.shape[0])]
positive_scores = numpy.hstack((positive_scores, p))
# negative scoring
indexes = probe_labels != base_data_shuffler.possible_labels[i]
negative_data = probe_features[indexes, :]
n = [cosine(models[i], negative_data[j]) for j in range(negative_data.shape[0])]
negative_scores = numpy.hstack((negative_scores, n))
return self.__compute_tensorflow_summary((-1)*negative_scores, (-1) * positive_scores)
def __compute_tensorflow_summary(self, negative_scores, positive_scores):
"""
Compute some stats with the scores, such as:
- EER
- FAR 10
- FAR 100
- FAR 1000
- RANK 1
- RANK 10
**Parameters**
negative_scores:
positive_scores:
"""
summaries = []
# Compute EER
threshold = bob.measure.eer_threshold(negative_scores, positive_scores)
far, frr = bob.measure.farfrr(negative_scores, positive_scores, threshold)
eer = (far + frr) / 2.
summaries.append(summary_pb2.Summary.Value(tag="EER", simple_value=eer))
self.eer.append(eer)
# Computing FAR 10
threshold = bob.measure.far_threshold(negative_scores, positive_scores, far_value=0.1)
far, frr = bob.measure.farfrr(negative_scores, positive_scores, threshold)
summaries.append(summary_pb2.Summary.Value(tag="FAR 10", simple_value=frr))
self.far10.append(frr)
# Computing FAR 100
threshold = bob.measure.far_threshold(negative_scores, positive_scores, far_value=0.01)
far, frr = bob.measure.farfrr(negative_scores, positive_scores, threshold)
summaries.append(summary_pb2.Summary.Value(tag="FAR 100", simple_value=frr))
self.far100.append(frr)
# Computing FAR 1000
threshold = bob.measure.far_threshold(negative_scores, positive_scores, far_value=0.001)
far, frr = bob.measure.farfrr(negative_scores, positive_scores, threshold)
summaries.append(summary_pb2.Summary.Value(tag="FAR 1000", simple_value=frr))
self.far1000.append(frr)
return summary_pb2.Summary(value=summaries)