ExperimentAnalizer.py 5.28 KB
Newer Older
1 2 3 4 5 6 7 8 9 10
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
# @date: Tue 09 Aug 2016 15:33 CEST

"""
Neural net work error rates analizer
"""
import numpy
import bob.measure
11
from tensorflow.core.framework import summary_pb2
12 13
from scipy.spatial.distance import cosine

Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
14 15
from bob.learn.tensorflow.datashuffler import Memory, Disk

16 17

class ExperimentAnalizer:
18 19 20 21 22 23 24 25 26 27
    """
    Analizer.

    I don't know if this is the best way to do, but what this class do is the following.

    As an enrollment sample, averare all the TRAINING samples for one particular class.
    The probing is done with the validation set

    """

28
    def __init__(self, convergence_threshold=0.01, convergence_reference='eer'):
29 30 31 32 33 34 35
        """
        Use the CNN as feature extractor for a n-class classification

        ** Parameters **

          data_shuffler:
          graph:
36 37 38 39
          session:
          convergence_threshold:
          convergence_reference: References to analize the convergence. Possible values are `eer`, `far10`, `far10`

40 41 42

        """

43 44 45
        self.data_shuffler = None
        self.network = None
        self.session = None
46 47 48 49 50 51 52

        # Statistics
        self.eer = []
        self.far10 = []
        self.far100 = []
        self.far1000 = []

53 54 55 56 57 58
    def __call__(self, data_shuffler, network, session):

        if self.data_shuffler is None:
            self.data_shuffler = data_shuffler
            self.network = network
            self.session = session
59

Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
60 61 62 63 64 65 66 67 68
        # Getting the base class. Recipe extracted from
        # http://stackoverflow.com/questions/5516263/creating-an-object-from-a-base-class-object-in-python/5516330#5516330
        if isinstance(data_shuffler, Memory):
            base_data_shuffler = object.__new__(Memory)
            base_data_shuffler.__dict__ = data_shuffler.__dict__.copy()
        else:
            base_data_shuffler = object.__new__(Disk)
            base_data_shuffler.__dict__ = data_shuffler.__dict__.copy()

69
        # Extracting features for enrollment
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
70
        enroll_data, enroll_labels = base_data_shuffler.get_batch()
71
        enroll_features = self.network(enroll_data, session=self.session)
72 73 74
        del enroll_data

        # Extracting features for probing
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
75
        probe_data, probe_labels = base_data_shuffler.get_batch()
76
        probe_features = self.network(probe_data, session=self.session)
77
        del probe_data
78 79 80

        # Creating models
        models = []
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
81
        for i in range(len(base_data_shuffler.possible_labels)):
82
            indexes_model = numpy.where(enroll_labels == self.data_shuffler.possible_labels[i])[0]
83 84 85 86 87
            models.append(numpy.mean(enroll_features[indexes_model, :], axis=0))

        # Probing
        positive_scores = numpy.zeros(shape=0)
        negative_scores = numpy.zeros(shape=0)
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
88
        for i in range(len(base_data_shuffler.possible_labels)):
89
            #for i in self.data_shuffler.possible_labels:
90
            # Positive scoring
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
91
            indexes = probe_labels == base_data_shuffler.possible_labels[i]
92 93 94 95 96
            positive_data = probe_features[indexes, :]
            p = [cosine(models[i], positive_data[j]) for j in range(positive_data.shape[0])]
            positive_scores = numpy.hstack((positive_scores, p))

            # negative scoring
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
97
            indexes = probe_labels != base_data_shuffler.possible_labels[i]
98 99 100 101
            negative_data = probe_features[indexes, :]
            n = [cosine(models[i], negative_data[j]) for j in range(negative_data.shape[0])]
            negative_scores = numpy.hstack((negative_scores, n))

102
        return self.__compute_tensorflow_summary((-1)*negative_scores, (-1) * positive_scores)
103

104
    def __compute_tensorflow_summary(self, negative_scores, positive_scores):
105 106 107 108 109 110 111 112 113 114 115 116 117 118
        """
        Compute some stats with the scores, such as:
          - EER
          - FAR 10
          - FAR 100
          - FAR 1000
          - RANK 1
          - RANK 10

        **Parameters**
          negative_scores:
          positive_scores:
        """

119 120
        summaries = []

121 122 123 124
        # Compute EER
        threshold = bob.measure.eer_threshold(negative_scores, positive_scores)
        far, frr = bob.measure.farfrr(negative_scores, positive_scores, threshold)
        eer = (far + frr) / 2.
125
        summaries.append(summary_pb2.Summary.Value(tag="EER", simple_value=eer))
126
        self.eer.append(eer)
127 128 129 130

        # Computing FAR 10
        threshold = bob.measure.far_threshold(negative_scores, positive_scores, far_value=0.1)
        far, frr = bob.measure.farfrr(negative_scores, positive_scores, threshold)
131
        summaries.append(summary_pb2.Summary.Value(tag="FAR 10", simple_value=frr))
132 133 134 135 136
        self.far10.append(frr)

        # Computing FAR 100
        threshold = bob.measure.far_threshold(negative_scores, positive_scores, far_value=0.01)
        far, frr = bob.measure.farfrr(negative_scores, positive_scores, threshold)
137
        summaries.append(summary_pb2.Summary.Value(tag="FAR 100", simple_value=frr))
138 139 140 141 142
        self.far100.append(frr)

        # Computing FAR 1000
        threshold = bob.measure.far_threshold(negative_scores, positive_scores, far_value=0.001)
        far, frr = bob.measure.farfrr(negative_scores, positive_scores, threshold)
143
        summaries.append(summary_pb2.Summary.Value(tag="FAR 1000", simple_value=frr))
144 145
        self.far1000.append(frr)

146
        return summary_pb2.Summary(value=summaries)