ExperimentAnalizer.py 5.08 KB
Newer Older
1 2 3 4 5 6 7 8 9 10
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
# @date: Tue 09 Aug 2016 15:33 CEST

"""
Neural net work error rates analizer
"""
import numpy
import bob.measure
11
from tensorflow.core.framework import summary_pb2
12 13
from scipy.spatial.distance import cosine

Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
14 15
from bob.learn.tensorflow.datashuffler import Memory, Disk

16 17

class ExperimentAnalizer:
18 19 20 21 22 23 24 25 26 27
    """
    Analizer.

    I don't know if this is the best way to do, but what this class do is the following.

    As an enrollment sample, averare all the TRAINING samples for one particular class.
    The probing is done with the validation set

    """

28
    def __init__(self, convergence_threshold=0.01, convergence_reference='eer'):
29 30 31 32 33
        """
        Use the CNN as feature extractor for a n-class classification

        ** Parameters **

Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
34
        convergence_threshold:
35

Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
36
        convergence_reference:
37 38 39

        """

40 41 42
        self.data_shuffler = None
        self.network = None
        self.session = None
43 44 45 46 47 48 49

        # Statistics
        self.eer = []
        self.far10 = []
        self.far100 = []
        self.far1000 = []

50 51
    def __call__(self, data_shuffler, network, session):

Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
52 53 54 55
        #if self.data_shuffler is None:
        #    self.data_shuffler = data_shuffler
        #    self.network = network
        #    self.session = session
56

Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
57 58 59 60 61 62 63 64 65
        # Getting the base class. Recipe extracted from
        # http://stackoverflow.com/questions/5516263/creating-an-object-from-a-base-class-object-in-python/5516330#5516330
        if isinstance(data_shuffler, Memory):
            base_data_shuffler = object.__new__(Memory)
            base_data_shuffler.__dict__ = data_shuffler.__dict__.copy()
        else:
            base_data_shuffler = object.__new__(Disk)
            base_data_shuffler.__dict__ = data_shuffler.__dict__.copy()

66
        # Extracting features for enrollment
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
67
        enroll_data, enroll_labels = base_data_shuffler.get_batch()
68
        enroll_features = network(enroll_data)
69 70 71
        del enroll_data

        # Extracting features for probing
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
72
        probe_data, probe_labels = base_data_shuffler.get_batch()
73
        probe_features = network(probe_data)
74
        del probe_data
75 76 77

        # Creating models
        models = []
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
78
        for i in range(len(base_data_shuffler.possible_labels)):
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
79
            indexes_model = numpy.where(enroll_labels == data_shuffler.possible_labels[i])[0]
80 81 82 83 84
            models.append(numpy.mean(enroll_features[indexes_model, :], axis=0))

        # Probing
        positive_scores = numpy.zeros(shape=0)
        negative_scores = numpy.zeros(shape=0)
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
85
        for i in range(len(base_data_shuffler.possible_labels)):
86
            #for i in self.data_shuffler.possible_labels:
87
            # Positive scoring
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
88
            indexes = probe_labels == base_data_shuffler.possible_labels[i]
89 90 91 92 93
            positive_data = probe_features[indexes, :]
            p = [cosine(models[i], positive_data[j]) for j in range(positive_data.shape[0])]
            positive_scores = numpy.hstack((positive_scores, p))

            # negative scoring
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
94
            indexes = probe_labels != base_data_shuffler.possible_labels[i]
95 96 97 98
            negative_data = probe_features[indexes, :]
            n = [cosine(models[i], negative_data[j]) for j in range(negative_data.shape[0])]
            negative_scores = numpy.hstack((negative_scores, n))

99
        return self.__compute_tensorflow_summary((-1)*negative_scores, (-1) * positive_scores)
100

101
    def __compute_tensorflow_summary(self, negative_scores, positive_scores):
102 103 104 105 106 107 108 109 110 111
        """
        Compute some stats with the scores, such as:
          - EER
          - FAR 10
          - FAR 100
          - FAR 1000
          - RANK 1
          - RANK 10

        **Parameters**
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
112 113 114 115

        negative_scores:

        positive_scores:
116 117
        """

118 119
        summaries = []

120 121 122 123
        # Compute EER
        threshold = bob.measure.eer_threshold(negative_scores, positive_scores)
        far, frr = bob.measure.farfrr(negative_scores, positive_scores, threshold)
        eer = (far + frr) / 2.
124
        summaries.append(summary_pb2.Summary.Value(tag="EER", simple_value=eer))
125
        self.eer.append(eer)
126 127 128 129

        # Computing FAR 10
        threshold = bob.measure.far_threshold(negative_scores, positive_scores, far_value=0.1)
        far, frr = bob.measure.farfrr(negative_scores, positive_scores, threshold)
130
        summaries.append(summary_pb2.Summary.Value(tag="FAR 10", simple_value=frr))
131 132 133 134 135
        self.far10.append(frr)

        # Computing FAR 100
        threshold = bob.measure.far_threshold(negative_scores, positive_scores, far_value=0.01)
        far, frr = bob.measure.farfrr(negative_scores, positive_scores, threshold)
136
        summaries.append(summary_pb2.Summary.Value(tag="FAR 100", simple_value=frr))
137 138 139 140 141
        self.far100.append(frr)

        # Computing FAR 1000
        threshold = bob.measure.far_threshold(negative_scores, positive_scores, far_value=0.001)
        far, frr = bob.measure.farfrr(negative_scores, positive_scores, threshold)
142
        summaries.append(summary_pb2.Summary.Value(tag="FAR 1000", simple_value=frr))
143 144
        self.far1000.append(frr)

145
        return summary_pb2.Summary(value=summaries)