ExperimentAnalizer.py 4.6 KB
Newer Older
1 2 3 4 5 6 7 8 9 10
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
# @date: Tue 09 Aug 2016 15:33 CEST

"""
Neural net work error rates analizer
"""
import numpy
import bob.measure
11
from tensorflow.core.framework import summary_pb2
12 13
from scipy.spatial.distance import cosine

14 15

class ExperimentAnalizer:
16 17 18 19 20 21 22 23 24 25
    """
    Analizer.

    I don't know if this is the best way to do, but what this class do is the following.

    As an enrollment sample, averare all the TRAINING samples for one particular class.
    The probing is done with the validation set

    """

26
    def __init__(self, data_shuffler, machine, session, convergence_threshold=0.01, convergence_reference='eer'):
27 28 29 30 31 32 33
        """
        Use the CNN as feature extractor for a n-class classification

        ** Parameters **

          data_shuffler:
          graph:
34 35 36 37
          session:
          convergence_threshold:
          convergence_reference: References to analize the convergence. Possible values are `eer`, `far10`, `far10`

38 39 40 41

        """

        self.data_shuffler = data_shuffler
42
        self.machine = machine
43 44 45 46 47 48 49 50 51 52
        self.session = session

        # Statistics
        self.eer = []
        self.far10 = []
        self.far100 = []
        self.far1000 = []

    def __call__(self):

53
        # Extracting features for enrollment
54
        enroll_data, enroll_labels = self.data_shuffler.get_batch()
55
        enroll_features = self.machine(enroll_data, session=self.session)
56 57 58
        del enroll_data

        # Extracting features for probing
59
        probe_data, probe_labels = self.data_shuffler.get_batch()
60
        probe_features = self.machine(probe_data, session=self.session)
61
        del probe_data
62 63 64

        # Creating models
        models = []
65 66
        for i in range(len(self.data_shuffler.possible_labels)):
            indexes_model = numpy.where(enroll_labels == self.data_shuffler.possible_labels[i])[0]
67 68 69 70 71
            models.append(numpy.mean(enroll_features[indexes_model, :], axis=0))

        # Probing
        positive_scores = numpy.zeros(shape=0)
        negative_scores = numpy.zeros(shape=0)
72 73
        for i in range(len(self.data_shuffler.possible_labels)):
            #for i in self.data_shuffler.possible_labels:
74
            # Positive scoring
75
            indexes = probe_labels == self.data_shuffler.possible_labels[i]
76 77 78 79 80
            positive_data = probe_features[indexes, :]
            p = [cosine(models[i], positive_data[j]) for j in range(positive_data.shape[0])]
            positive_scores = numpy.hstack((positive_scores, p))

            # negative scoring
81
            indexes = probe_labels != self.data_shuffler.possible_labels[i]
82 83 84 85
            negative_data = probe_features[indexes, :]
            n = [cosine(models[i], negative_data[j]) for j in range(negative_data.shape[0])]
            negative_scores = numpy.hstack((negative_scores, n))

86
        return self.__compute_tensorflow_summary((-1)*negative_scores, (-1) * positive_scores)
87

88
    def __compute_tensorflow_summary(self, negative_scores, positive_scores):
89 90 91 92 93 94 95 96 97 98 99 100 101 102
        """
        Compute some stats with the scores, such as:
          - EER
          - FAR 10
          - FAR 100
          - FAR 1000
          - RANK 1
          - RANK 10

        **Parameters**
          negative_scores:
          positive_scores:
        """

103 104
        summaries = []

105 106 107 108
        # Compute EER
        threshold = bob.measure.eer_threshold(negative_scores, positive_scores)
        far, frr = bob.measure.farfrr(negative_scores, positive_scores, threshold)
        eer = (far + frr) / 2.
109
        summaries.append(summary_pb2.Summary.Value(tag="EER", simple_value=eer))
110
        self.eer.append(eer)
111 112 113 114

        # Computing FAR 10
        threshold = bob.measure.far_threshold(negative_scores, positive_scores, far_value=0.1)
        far, frr = bob.measure.farfrr(negative_scores, positive_scores, threshold)
115
        summaries.append(summary_pb2.Summary.Value(tag="FAR 10", simple_value=frr))
116 117 118 119 120
        self.far10.append(frr)

        # Computing FAR 100
        threshold = bob.measure.far_threshold(negative_scores, positive_scores, far_value=0.01)
        far, frr = bob.measure.farfrr(negative_scores, positive_scores, threshold)
121
        summaries.append(summary_pb2.Summary.Value(tag="FAR 100", simple_value=frr))
122 123 124 125 126
        self.far100.append(frr)

        # Computing FAR 1000
        threshold = bob.measure.far_threshold(negative_scores, positive_scores, far_value=0.001)
        far, frr = bob.measure.farfrr(negative_scores, positive_scores, threshold)
127
        summaries.append(summary_pb2.Summary.Value(tag="FAR 1000", simple_value=frr))
128 129
        self.far1000.append(frr)

130
        return summary_pb2.Summary(value=summaries)