test_cnn.py 8.55 KB
Newer Older
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
1
2
3
4
5
6
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
# @date: Thu 13 Oct 2016 13:35 CEST

import numpy
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
7
from bob.learn.tensorflow.datashuffler import Memory, SiameseMemory, TripletMemory, ImageAugmentation, ScaleFactor
Tiago Pereira's avatar
Tiago Pereira committed
8
from bob.learn.tensorflow.network import Chopra
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
9
from bob.learn.tensorflow.loss import BaseLoss, ContrastiveLoss, TripletLoss
10
from bob.learn.tensorflow.trainers import Trainer, SiameseTrainer, TripletTrainer, constant
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
11
from .test_cnn_scratch import validate_network
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
12
from bob.learn.tensorflow.network import Embedding
13

Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
14
from bob.learn.tensorflow.utils import load_mnist
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
15
16
17
18
19
20
21
22
23
24
import tensorflow as tf
import bob.io.base
import shutil
from scipy.spatial.distance import cosine
import bob.measure

"""
Some unit tests for the datashuffler
"""

Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
25
batch_size = 32
Tiago Pereira's avatar
Tiago Pereira committed
26
validation_batch_size = 32
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
27
iterations = 300
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
28
29
30
seed = 10


Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
31
def dummy_experiment(data_s, architecture):
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
32
33
34
35
36
37
38
39
40
    """
    Create a dummy experiment and return the EER
    """

    data_shuffler = object.__new__(Memory)
    data_shuffler.__dict__ = data_s.__dict__.copy()

    # Extracting features for enrollment
    enroll_data, enroll_labels = data_shuffler.get_batch()
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
41
    enroll_features = architecture(enroll_data)
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
42
43
44
45
    del enroll_data

    # Extracting features for probing
    probe_data, probe_labels = data_shuffler.get_batch()
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
46
    probe_features = architecture(probe_data)
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
47
48
49
50
51
52
53
54
55
56
57
    del probe_data

    # Creating models
    models = []
    for i in range(len(data_shuffler.possible_labels)):
        indexes_model = numpy.where(enroll_labels == data_shuffler.possible_labels[i])[0]
        models.append(numpy.mean(enroll_features[indexes_model, :], axis=0))

    # Probing
    positive_scores = numpy.zeros(shape=0)
    negative_scores = numpy.zeros(shape=0)
58

Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
    for i in range(len(data_shuffler.possible_labels)):
        # Positive scoring
        indexes = probe_labels == data_shuffler.possible_labels[i]
        positive_data = probe_features[indexes, :]
        p = [cosine(models[i], positive_data[j]) for j in range(positive_data.shape[0])]
        positive_scores = numpy.hstack((positive_scores, p))

        # negative scoring
        indexes = probe_labels != data_shuffler.possible_labels[i]
        negative_data = probe_features[indexes, :]
        n = [cosine(models[i], negative_data[j]) for j in range(negative_data.shape[0])]
        negative_scores = numpy.hstack((negative_scores, n))

    threshold = bob.measure.eer_threshold((-1) * negative_scores, (-1) * positive_scores)
    far, frr = bob.measure.farfrr((-1) * negative_scores, (-1) * positive_scores, threshold)

    return (far + frr) / 2.


def test_cnn_trainer():
79

Tiago Pereira's avatar
Tiago Pereira committed
80
    # Loading data
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
81
82
83
84
85
    train_data, train_labels, validation_data, validation_labels = load_mnist()
    train_data = numpy.reshape(train_data, (train_data.shape[0], 28, 28, 1))
    validation_data = numpy.reshape(validation_data, (validation_data.shape[0], 28, 28, 1))

    # Creating datashufflers
86
    data_augmentation = ImageAugmentation()
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
87
    train_data_shuffler = Memory(train_data, train_labels,
Tiago Pereira's avatar
Tiago Pereira committed
88
                                 input_shape=[None, 28, 28, 1],
89
                                 batch_size=batch_size,
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
90
91
                                 data_augmentation=data_augmentation,
                                 normalizer=ScaleFactor())
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
92

Tiago Pereira's avatar
Tiago Pereira committed
93
94
95
96
97
98
    validation_data_shuffler = Memory(validation_data, validation_labels,
                                      input_shape=[None, 28, 28, 1],
                                      batch_size=batch_size,
                                      data_augmentation=data_augmentation,
                                      normalizer=ScaleFactor())

99
100
101
102
    directory = "./temp/cnn"

    # Loss for the softmax
    loss = BaseLoss(tf.nn.sparse_softmax_cross_entropy_with_logits, tf.reduce_mean)
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
103

Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
104
105
106
    # Preparing the architecture
    architecture = Chopra(seed=seed,
                          fc1_output=10)
Tiago Pereira's avatar
Tiago Pereira committed
107
108
109
    input_pl = train_data_shuffler("data", from_queue=True)
    graph = architecture(input_pl)
    embedding = Embedding(train_data_shuffler("data", from_queue=False), graph)
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
110

111
    # One graph trainer
Tiago Pereira's avatar
Tiago Pereira committed
112
    trainer = Trainer(train_data_shuffler,
113
114
                      iterations=iterations,
                      analizer=None,
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
115
116
                      temp_dir=directory
                      )
Tiago Pereira's avatar
Tiago Pereira committed
117
118
119
120
121
122
123
124
125
    trainer.create_network_from_scratch(graph=graph,
                                        loss=loss,
                                        learning_rate=constant(0.01, name="regular_lr"),
                                        optimizer=tf.train.GradientDescentOptimizer(0.01),
                                        )
    trainer.train()
    #trainer.train(validation_data_shuffler)

    # Using embedding to compute the accuracy
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
126
    accuracy = validate_network(embedding, validation_data, validation_labels)
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
127
128
    # At least 80% of accuracy
    assert accuracy > 80.
129
    shutil.rmtree(directory)
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
130
    del trainer
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
131
    del graph
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146


def test_siamesecnn_trainer():
    train_data, train_labels, validation_data, validation_labels = load_mnist()
    train_data = numpy.reshape(train_data, (train_data.shape[0], 28, 28, 1))
    validation_data = numpy.reshape(validation_data, (validation_data.shape[0], 28, 28, 1))

    # Creating datashufflers
    train_data_shuffler = SiameseMemory(train_data, train_labels,
                                        input_shape=[28, 28, 1],
                                        batch_size=batch_size)
    validation_data_shuffler = SiameseMemory(validation_data, validation_labels,
                                             input_shape=[28, 28, 1],
                                             batch_size=validation_batch_size)

147
    directory = "./temp/siamesecnn"
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
148

149
150
    # Preparing the architecture
    architecture = Chopra(seed=seed, fc1_output=10)
151
152
153
154
155
156
157
158
    inputs = {}
    inputs['left'] = tf.placeholder(tf.float32, shape=[None, 28, 28, 1], name="input_left")
    inputs['right'] = tf.placeholder(tf.float32, shape=[None, 28, 28, 1], name="input_right")
    inputs['label'] = tf.placeholder(tf.int64, shape=[None], name="label")

    graph = {}
    graph['left'] = architecture(inputs['left'])
    graph['right'] = architecture(inputs['right'])
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
159

160
161
    # Loss for the Siamese
    loss = ContrastiveLoss(contrastive_margin=4.)
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
162

163
    # One graph trainer
164
165
    trainer = SiameseTrainer(inputs=inputs,
                             graph=graph,
166
167
168
169
170
                             loss=loss,
                             iterations=iterations,
                             prefetch=False,
                             analizer=None,
                             learning_rate=constant(0.05, name="siamese_lr"),
171
                             optimizer=tf.train.AdamOptimizer(name="adam_siamese"),
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
172
173
                             temp_dir=directory
                             )
174
    trainer.train(train_data_shuffler)
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
175

Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
176
    eer = dummy_experiment(validation_data_shuffler, architecture)
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
177

Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
178
179
180
    # At least 80% of accuracy
    assert eer < 0.25
    shutil.rmtree(directory)
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
181

Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
182
183
    del architecture
    del trainer  # Just to clean tf.variables
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198


def test_tripletcnn_trainer():
    train_data, train_labels, validation_data, validation_labels = load_mnist()
    train_data = numpy.reshape(train_data, (train_data.shape[0], 28, 28, 1))
    validation_data = numpy.reshape(validation_data, (validation_data.shape[0], 28, 28, 1))

    # Creating datashufflers
    train_data_shuffler = TripletMemory(train_data, train_labels,
                                        input_shape=[28, 28, 1],
                                        batch_size=batch_size)
    validation_data_shuffler = TripletMemory(validation_data, validation_labels,
                                             input_shape=[28, 28, 1],
                                             batch_size=validation_batch_size)

199
    directory = "./temp/tripletcnn"
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
200

201
202
    # Preparing the architecture
    architecture = Chopra(seed=seed, fc1_output=10)
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
203

204
205
    # Loss for the Siamese
    loss = TripletLoss(margin=4.)
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
206

207
208
209
210
211
212
213
    # One graph trainer
    trainer = TripletTrainer(architecture=architecture,
                             loss=loss,
                             iterations=iterations,
                             prefetch=False,
                             analizer=None,
                             learning_rate=constant(0.05, name="triplet_lr"),
214
                             optimizer=tf.train.AdamOptimizer(name="adam_triplet"),
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
215
216
                             temp_dir=directory
                             )
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
217

218
219
    trainer.train(train_data_shuffler)

Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
220
221
    # Testing
    eer = dummy_experiment(validation_data_shuffler, architecture)
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
222

Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
223
224
225
    # At least 80% of accuracy
    assert eer < 0.25
    shutil.rmtree(directory)
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
226

Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
227
228
    del architecture
    del trainer  # Just to clean tf.variables