test_cnn_scratch.py 5.8 KB
Newer Older
1
2
3
4
5
6
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
# @date: Thu 13 Oct 2016 13:35 CEST

import numpy
7
from bob.learn.tensorflow.datashuffler import Memory, ImageAugmentation, ScaleFactor, Linear, TFRecord
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
8
from bob.learn.tensorflow.network import Embedding
9
from bob.learn.tensorflow.loss import BaseLoss
Tiago Pereira's avatar
Tiago Pereira committed
10
from bob.learn.tensorflow.trainers import Trainer, constant
11
from bob.learn.tensorflow.utils import load_mnist
12
13
import tensorflow as tf
import shutil
14
import os
15
16
17
18
19
20
21

"""
Some unit tests that create networks on the fly
"""

batch_size = 16
validation_batch_size = 400
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
22
iterations = 300
23
seed = 10
24
directory = "./temp/cnn_scratch"
25

Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
26
27
slim = tf.contrib.slim

28

29
def scratch_network(train_data_shuffler, reuse=False):
Tiago Pereira's avatar
Tiago Pereira committed
30
31

    inputs = train_data_shuffler("data", from_queue=False)
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
32

Tiago Pereira's avatar
Tiago Pereira committed
33
    # Creating a random network
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
34
    initializer = tf.contrib.layers.xavier_initializer(seed=seed)
Tiago Pereira's avatar
Tiago Pereira committed
35
    graph = slim.conv2d(inputs, 10, [3, 3], activation_fn=tf.nn.relu, stride=1, scope='conv1',
36
                        weights_initializer=initializer, reuse=reuse)
Tiago Pereira's avatar
Tiago Pereira committed
37
38
39
    graph = slim.max_pool2d(graph, [4, 4], scope='pool1')
    graph = slim.flatten(graph, scope='flatten1')
    graph = slim.fully_connected(graph, 10, activation_fn=None, scope='fc1',
40
                                 weights_initializer=initializer, reuse=reuse)
Tiago Pereira's avatar
Tiago Pereira committed
41

Tiago Pereira's avatar
Tiago Pereira committed
42
    return graph
43
44


45
def validate_network(embedding, validation_data, validation_labels, input_shape=[None, 28, 28, 1], normalizer=ScaleFactor()):
46
47
    # Testing
    validation_data_shuffler = Memory(validation_data, validation_labels,
48
                                      input_shape=input_shape,
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
49
                                      batch_size=validation_batch_size,
50
                                      normalizer=normalizer)
51
52

    [data, labels] = validation_data_shuffler.get_batch()
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
53
54
    predictions = embedding(data)
    accuracy = 100. * numpy.sum(numpy.argmax(predictions, axis=1) == labels) / predictions.shape[0]
55

56
57
58
59
    return accuracy


def test_cnn_trainer_scratch():
60
    tf.reset_default_graph()
61

62
63
64
65
66
67
    train_data, train_labels, validation_data, validation_labels = load_mnist()
    train_data = numpy.reshape(train_data, (train_data.shape[0], 28, 28, 1))

    # Creating datashufflers
    data_augmentation = ImageAugmentation()
    train_data_shuffler = Memory(train_data, train_labels,
Tiago Pereira's avatar
Tiago Pereira committed
68
                                 input_shape=[None, 28, 28, 1],
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
69
70
71
                                 batch_size=batch_size,
                                 data_augmentation=data_augmentation,
                                 normalizer=ScaleFactor())
72

Tiago Pereira's avatar
Tiago Pereira committed
73
    validation_data = numpy.reshape(validation_data, (validation_data.shape[0], 28, 28, 1))
74
    # Create scratch network
Tiago Pereira's avatar
Tiago Pereira committed
75
    graph = scratch_network(train_data_shuffler)
Tiago Pereira's avatar
Tiago Pereira committed
76
77

    # Setting the placeholders
Tiago Pereira's avatar
Tiago Pereira committed
78
    embedding = Embedding(train_data_shuffler("data", from_queue=False), graph)
79
80
81
82
83

    # Loss for the softmax
    loss = BaseLoss(tf.nn.sparse_softmax_cross_entropy_with_logits, tf.reduce_mean)

    # One graph trainer
Tiago Pereira's avatar
Tiago Pereira committed
84
    trainer = Trainer(train_data_shuffler,
85
86
                      iterations=iterations,
                      analizer=None,
Tiago Pereira's avatar
Tiago Pereira committed
87
                      temp_dir=directory)
88

Tiago Pereira's avatar
Tiago Pereira committed
89
    trainer.create_network_from_scratch(graph=graph,
Tiago Pereira's avatar
Tiago Pereira committed
90
91
92
93
                                        loss=loss,
                                        learning_rate=constant(0.01, name="regular_lr"),
                                        optimizer=tf.train.GradientDescentOptimizer(0.01),
                                        )
94

Tiago Pereira's avatar
Tiago Pereira committed
95
    trainer.train()
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
96
    accuracy = validate_network(embedding, validation_data, validation_labels)
Tiago Pereira's avatar
Tiago Pereira committed
97
    assert accuracy > 70
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
98
    shutil.rmtree(directory)
99
    del trainer
100
101
102
103
104
    
    
    
    
def test_cnn_trainer_scratch_tfrecord():
105
106
107
108
109
110
111
112
    train_data, train_labels, validation_data, validation_labels = load_mnist()
    train_data = train_data.astype("float32") *  0.00390625

    def _bytes_feature(value):
        return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))

    def _int64_feature(value):
        return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
113

114

115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
    def create_tf_record(tfrecords_filename):
        writer = tf.python_io.TFRecordWriter(tfrecords_filename)

        for i in range(train_data.shape[0]):
            img = train_data[i]
            img_raw = img.tostring()
            
            feature = {'train/image': _bytes_feature(img_raw),
                       'train/label': _int64_feature(train_labels[i])
                      }
            
            example = tf.train.Example(features=tf.train.Features(feature=feature))
            writer.write(example.SerializeToString())
        writer.close()

    tf.reset_default_graph()
    
    # Creating the tf record
    tfrecords_filename = "mnist_train.tfrecords"
    create_tf_record(tfrecords_filename)   
    filename_queue = tf.train.string_input_producer([tfrecords_filename], num_epochs=1, name="input")
136

137
    # Creating the CNN using the TFRecord as input
138
139
140
141
142
143
144
145
146
147
148
149
150
151
    train_data_shuffler  = TFRecord(filename_queue=filename_queue,
                                    batch_size=batch_size)
    graph = scratch_network(train_data_shuffler)

    # Setting the placeholders
    # Loss for the softmax
    loss = BaseLoss(tf.nn.sparse_softmax_cross_entropy_with_logits, tf.reduce_mean)

    # One graph trainer
    trainer = Trainer(train_data_shuffler,
                      iterations=iterations,
                      analizer=None,
                      temp_dir=directory)

152
    learning_rate = constant(0.01, name="regular_lr")
153
154
    trainer.create_network_from_scratch(graph=graph,
                                        loss=loss,
155
156
                                        learning_rate=learning_rate,
                                        optimizer=tf.train.GradientDescentOptimizer(learning_rate),
157
158
159
                                        )

    trainer.train()
160
161
    os.remove(tfrecords_filename)
    assert True
162
163