ExperimentAnalizer.py 4.74 KB
Newer Older
1 2 3 4 5 6 7 8 9 10
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
# @date: Tue 09 Aug 2016 15:33 CEST

"""
Neural net work error rates analizer
"""
import numpy
import bob.measure
11
from tensorflow.core.framework import summary_pb2
12 13
from scipy.spatial.distance import cosine

14 15

class ExperimentAnalizer:
16 17 18 19 20 21 22 23 24 25
    """
    Analizer.

    I don't know if this is the best way to do, but what this class do is the following.

    As an enrollment sample, averare all the TRAINING samples for one particular class.
    The probing is done with the validation set

    """

26
    def __init__(self, convergence_threshold=0.01, convergence_reference='eer'):
27 28 29 30 31 32 33
        """
        Use the CNN as feature extractor for a n-class classification

        ** Parameters **

          data_shuffler:
          graph:
34 35 36 37
          session:
          convergence_threshold:
          convergence_reference: References to analize the convergence. Possible values are `eer`, `far10`, `far10`

38 39 40

        """

41 42 43
        self.data_shuffler = None
        self.network = None
        self.session = None
44 45 46 47 48 49 50

        # Statistics
        self.eer = []
        self.far10 = []
        self.far100 = []
        self.far1000 = []

51 52 53 54 55 56
    def __call__(self, data_shuffler, network, session):

        if self.data_shuffler is None:
            self.data_shuffler = data_shuffler
            self.network = network
            self.session = session
57

58
        # Extracting features for enrollment
59
        enroll_data, enroll_labels = self.data_shuffler.get_batch()
60
        enroll_features = self.network(enroll_data, session=self.session)
61 62 63
        del enroll_data

        # Extracting features for probing
64
        probe_data, probe_labels = self.data_shuffler.get_batch()
65
        probe_features = self.network(probe_data, session=self.session)
66
        del probe_data
67 68 69

        # Creating models
        models = []
70 71
        for i in range(len(self.data_shuffler.possible_labels)):
            indexes_model = numpy.where(enroll_labels == self.data_shuffler.possible_labels[i])[0]
72 73 74 75 76
            models.append(numpy.mean(enroll_features[indexes_model, :], axis=0))

        # Probing
        positive_scores = numpy.zeros(shape=0)
        negative_scores = numpy.zeros(shape=0)
77 78
        for i in range(len(self.data_shuffler.possible_labels)):
            #for i in self.data_shuffler.possible_labels:
79
            # Positive scoring
80
            indexes = probe_labels == self.data_shuffler.possible_labels[i]
81 82 83 84 85
            positive_data = probe_features[indexes, :]
            p = [cosine(models[i], positive_data[j]) for j in range(positive_data.shape[0])]
            positive_scores = numpy.hstack((positive_scores, p))

            # negative scoring
86
            indexes = probe_labels != self.data_shuffler.possible_labels[i]
87 88 89 90
            negative_data = probe_features[indexes, :]
            n = [cosine(models[i], negative_data[j]) for j in range(negative_data.shape[0])]
            negative_scores = numpy.hstack((negative_scores, n))

91
        return self.__compute_tensorflow_summary((-1)*negative_scores, (-1) * positive_scores)
92

93
    def __compute_tensorflow_summary(self, negative_scores, positive_scores):
94 95 96 97 98 99 100 101 102 103 104 105 106 107
        """
        Compute some stats with the scores, such as:
          - EER
          - FAR 10
          - FAR 100
          - FAR 1000
          - RANK 1
          - RANK 10

        **Parameters**
          negative_scores:
          positive_scores:
        """

108 109
        summaries = []

110 111 112 113
        # Compute EER
        threshold = bob.measure.eer_threshold(negative_scores, positive_scores)
        far, frr = bob.measure.farfrr(negative_scores, positive_scores, threshold)
        eer = (far + frr) / 2.
114
        summaries.append(summary_pb2.Summary.Value(tag="EER", simple_value=eer))
115
        self.eer.append(eer)
116 117 118 119

        # Computing FAR 10
        threshold = bob.measure.far_threshold(negative_scores, positive_scores, far_value=0.1)
        far, frr = bob.measure.farfrr(negative_scores, positive_scores, threshold)
120
        summaries.append(summary_pb2.Summary.Value(tag="FAR 10", simple_value=frr))
121 122 123 124 125
        self.far10.append(frr)

        # Computing FAR 100
        threshold = bob.measure.far_threshold(negative_scores, positive_scores, far_value=0.01)
        far, frr = bob.measure.farfrr(negative_scores, positive_scores, threshold)
126
        summaries.append(summary_pb2.Summary.Value(tag="FAR 100", simple_value=frr))
127 128 129 130 131
        self.far100.append(frr)

        # Computing FAR 1000
        threshold = bob.measure.far_threshold(negative_scores, positive_scores, far_value=0.001)
        far, frr = bob.measure.farfrr(negative_scores, positive_scores, threshold)
132
        summaries.append(summary_pb2.Summary.Value(tag="FAR 1000", simple_value=frr))
133 134
        self.far1000.append(frr)

135
        return summary_pb2.Summary(value=summaries)