Migrated MLP to tf-slim

bob/learn/tensorflow/network/MLP.py

@@ -8,13 +8,9 @@ Class that creates the lenet architecture
 """
 
 import tensorflow as tf
-from .SequenceNetwork import SequenceNetwork
-from ..layers import Conv2D, FullyConnected, MaxPooling
-from bob.learn.tensorflow.initialization import Xavier
-from bob.learn.tensorflow.initialization import Constant
 
 
-class MLP(SequenceNetwork):
+class MLP(object):
     """An MLP is a representation of a Multi-Layer Perceptron.
 
     This implementation is feed-forward and fully-connected.
@@ -35,39 +31,44 @@ class MLP(SequenceNetwork):
 
         output_activation: Activation of the output layer.  If you set to `None`, the activation will be linear
 
-        weights_initialization: How you will initialize the neurons.
-                                See :py:mod:`bob.learn.tensorflow.initialization`.
+        seed: 
 
-        bias_initialization:   How you will initialize the biases.
-                               See :py:mod:`bob.learn.tensorflow.initialization`.
-
-        use_gpu: If ``True`` uses the GPU in the computation
-
-        seed = 10
+        device:
     """
     def __init__(self,
                  output_shape,
                  hidden_layers=[10],
                  hidden_activation=tf.nn.tanh,
                  output_activation=None,
-                 weights_initialization=Xavier(),
-                 bias_initialization=Constant(),
-                 use_gpu=False):
-        super(MLP, self).__init__(use_gpu=use_gpu)
-
-        if (not (isinstance(hidden_layers, list) or isinstance(hidden_layers, tuple))) or len(hidden_layers) == 0:
-            raise ValueError("Invalid input for hidden_layers: {0} ".format(hidden_layers))
-
-        for i in range(len(hidden_layers)):
-            l = hidden_layers[i]
-            self.add(FullyConnected(name="mlp_fc{0}".format(i),
-                                    output_dim=l,
-                                    activation=hidden_activation,
-                                    weights_initialization=weights_initialization,
-                                    bias_initialization=bias_initialization))
-
-        self.add(FullyConnected(name="mlp_fc_output",
-                                output_dim=output_shape,
-                                activation=output_activation,
-                                weights_initialization=weights_initialization,
-                                bias_initialization=bias_initialization))
+                 seed=10,
+                 device="/cpu:0"):
+
+        self.output_shape = output_shape
+        self.hidden_layers = hidden_layers
+        self.hidden_activation = hidden_activation
+        self.output_activation = output_activation
+        self.seed = seed
+        self.device = device
+
+    def __call__(self, inputs):
+        slim = tf.contrib.slim
+        initializer = tf.contrib.layers.xavier_initializer(uniform=False, dtype=tf.float32, seed=self.seed)
+
+        #if (not (isinstance(hidden_layers, list) or isinstance(hidden_layers, tuple))) or len(hidden_layers) == 0:
+        #    raise ValueError("Invalid input for hidden_layers: {0} ".format(hidden_layers))
+
+        graph = inputs
+        for i in range(len(self.hidden_layers)):
+
+            weights = self.hidden_layers[i]
+            graph = slim.fully_connected(graph, weights,
+                                         weights_initializer=initializer,
+                                         activation_fn=self.hidden_activation,
+                                         scope='fc_{0}'.format(i))
+
+        graph = slim.fully_connected(graph, self.output_shape,
+                                     weights_initializer=initializer,
+                                     activation_fn=self.output_activation,
+                                     scope='fc_output')
+
+        return graph

bob/learn/tensorflow/test/test_dnn.py

@@ -4,8 +4,8 @@
 # @date: Thu 13 Oct 2016 13:35 CEST
 
 import numpy
-from bob.learn.tensorflow.datashuffler import Memory
-from bob.learn.tensorflow.network import MLP
+from bob.learn.tensorflow.datashuffler import Memory, ScaleFactor
+from bob.learn.tensorflow.network import MLP, Embedding
 from bob.learn.tensorflow.loss import BaseLoss
 from bob.learn.tensorflow.trainers import Trainer, constant
 from bob.learn.tensorflow.utils import load_mnist
@@ -22,15 +22,16 @@ iterations = 200
 seed = 10
 
 
-def validate_network(validation_data, validation_labels, network):
+def validate_network(embedding, validation_data, validation_labels):
     # Testing
     validation_data_shuffler = Memory(validation_data, validation_labels,
-                                      input_shape=[784],
-                                      batch_size=validation_batch_size)
+                                      input_shape=[None, 28*28],
+                                      batch_size=validation_batch_size,
+                                      normalizer=ScaleFactor())
 
     [data, labels] = validation_data_shuffler.get_batch()
-    predictions = network.predict(data)
-    accuracy = 100. * numpy.sum(predictions == labels) / predictions.shape[0]
+    predictions = embedding(data)
+    accuracy = 100. * numpy.sum(numpy.argmax(predictions, axis=1) == labels) / predictions.shape[0]
 
     return accuracy
 
@@ -40,30 +41,37 @@ def test_dnn_trainer():
 
     # Creating datashufflers
     train_data_shuffler = Memory(train_data, train_labels,
-                                 input_shape=[784],
-                                 batch_size=batch_size)
+                                 input_shape=[None, 784],
+                                 batch_size=batch_size,
+                                 normalizer=ScaleFactor())
 
     directory = "./temp/dnn"
 
     # Preparing the architecture
     architecture = MLP(10, hidden_layers=[20, 40])
 
+    input_pl = train_data_shuffler("data", from_queue=False)
+    graph = architecture(input_pl)
+
     # Loss for the softmax
     loss = BaseLoss(tf.nn.sparse_softmax_cross_entropy_with_logits, tf.reduce_mean)
 
     # One graph trainer
-    trainer = Trainer(architecture=architecture,
-                      loss=loss,
+    trainer = Trainer(train_data_shuffler,
                       iterations=iterations,
                       analizer=None,
-                      prefetch=False,
-                      learning_rate=constant(0.05, name="dnn_lr"),
-                      optimizer=tf.train.AdamOptimizer(name="adam_dnn"),
                       temp_dir=directory
                       )
-    trainer.train(train_data_shuffler)
 
-    accuracy = validate_network(validation_data, validation_labels, architecture)
+    trainer.create_network_from_scratch(graph=graph,
+                                        loss=loss,
+                                        learning_rate=constant(0.01, name="regular_lr"),
+                                        optimizer=tf.train.GradientDescentOptimizer(0.01),
+                                        )
+
+    trainer.train()
+    embedding = Embedding(train_data_shuffler("data", from_queue=False), graph)
+    accuracy = validate_network(embedding, validation_data, validation_labels)
 
     # At least 50% of accuracy for the DNN
     assert accuracy > 50.