Removed the hdf5 backend and tests with batch normalization

bob/learn/tensorflow/analyzers/SoftmaxAnalizer.py

@@ -18,37 +18,14 @@ class SoftmaxAnalizer(object):
     def __init__(self):
         """
         Softmax analizer
-
-        ** Parameters **
-
-          data_shuffler:
-          graph:
-          session:
-          convergence_threshold:
-          convergence_reference: References to analize the convergence. Possible values are `eer`, `far10`, `far10`
-
-
         """
-
-        self.data_shuffler = None
-        self.network = None
-        self.session = None
+        pass
 
     def __call__(self, data_shuffler, network, session):
+        data, labels = data_shuffler.get_batch()
 
-        if self.data_shuffler is None:
-            self.data_shuffler = data_shuffler
-            self.network = network
-            self.session = session
-
-        # Creating the graph
-        feature_batch, label_batch = self.data_shuffler.get_placeholders(name="validation_accuracy")
-        data, labels = self.data_shuffler.get_batch()
-        graph = self.network.compute_graph(feature_batch)
-
-        predictions = numpy.argmax(self.session.run(graph, feed_dict={feature_batch: data[:]}), 1)
+        predictions = numpy.argmax(session.run(network.inference_graph, feed_dict={network.inference_placeholder: data[:]}), 1)
         accuracy = 100. * numpy.sum(predictions == labels) / predictions.shape[0]
 
-        summaries = []
-        summaries.append(summary_pb2.Summary.Value(tag="accuracy_validation", simple_value=float(accuracy)))
+        summaries = [(summary_pb2.Summary.Value(tag="accuracy_validation", simple_value=float(accuracy)))]
         return summary_pb2.Summary(value=summaries)

bob/learn/tensorflow/datashuffler/Base.py

@@ -58,7 +58,7 @@ class Base(object):
         self.label_placeholder = None
 
         self.data_augmentation = data_augmentation
-        self.deployment_shape = [-1] + list(input_shape)
+        self.deployment_shape = [None] + list(input_shape)
 
     def get_placeholders(self, name=""):
         """

bob/learn/tensorflow/initialization/Initialization.py

@@ -5,6 +5,7 @@
 
 import logging
 logger = logging.getLogger("bob.learn.tensorflow")
+import tensorflow as tf
 
 
 class Initialization(object):
@@ -25,6 +26,10 @@ class Initialization(object):
 
         self.seed = seed
         self.use_gpu = use_gpu
+        tf.set_random_seed(seed)
+
+    def variable_exist(self, var):
+        return var in [v.name.split("/")[0] for v in tf.all_variables()]
 
     def __call__(self, shape, name, scope):
         NotImplementedError("Please implement this function in derived classes")

bob/learn/tensorflow/initialization/Xavier.py

@@ -39,6 +39,18 @@ class Xavier(Initialization):
 
         initializer = tf.truncated_normal(shape, stddev=stddev, seed=self.seed)
 
+        reuse = self.variable_exist(scope)
+        """
+        with tf.variable_scope(scope, reuse=reuse):
+            if self.use_gpu:
+                with tf.device("/gpu:0"):
+                    return tf.get_variable(name, initializer=initializer, dtype=tf.float32)
+            else:
+                with tf.device("/cpu"):
+                    return tf.get_variable(name, initializer=initializer, dtype=tf.float32)
+
+        """
+
         try:
             with tf.variable_scope(scope):
                 if self.use_gpu:

bob/learn/tensorflow/layers/Conv2D.py

@@ -68,8 +68,7 @@ class Conv2D(Layer):
 
             self.b = self.bias_initialization(shape=[self.filters],
                                               name="b_" + str(self.name) + "bias",
-                                              scope="b_" + str(self.name)
-                                             )
+                                              scope="b_" + str(self.name))
 
     def get_graph(self, training_phase=True):
 

bob/learn/tensorflow/layers/Layer.py

@@ -43,6 +43,8 @@ class Layer(object):
         # Batch normalization variables
         self.beta = None
         self.gamma = None
+        self.batch_mean = None
+        self.batch_var = None
 
     def create_variables(self, input_layer):
         NotImplementedError("Please implement this function in derived classes")
@@ -50,6 +52,9 @@ class Layer(object):
     def get_graph(self, training_phase=True):
         NotImplementedError("Please implement this function in derived classes")
 
+    def variable_exist(self, var):
+        return var in [v.name.split("/")[0] for v in tf.all_variables()]
+
     def batch_normalize(self, x, phase_train):
         """
         Batch normalization on convolutional maps.
@@ -66,32 +71,42 @@ class Layer(object):
         from tensorflow.python.ops import control_flow_ops
 
         name = "batch_norm_" + str(self.name)
-        #with tf.variable_scope(name):
-        phase_train = tf.convert_to_tensor(phase_train, dtype=tf.bool)
-        n_out = int(x.get_shape()[-1])
-        self.beta = tf.get_variable(name + '_beta',
-                                    initializer=tf.constant(0.0, shape=[n_out], dtype=x.dtype),
-                                    trainable=True,
-                                    dtype=x.dtype)
-        self.gamma = tf.get_variable(name + '_gamma',
-                                     initializer=tf.constant(1.0, shape=[n_out], dtype=x.dtype),
-                                     trainable=True,
-                                     dtype=x.dtype)
-
-        if len(x.get_shape()) == 2:
-            batch_mean, batch_var = tf.nn.moments(x, [0], name='moments_{0}'.format(name))
-        else:
-            batch_mean, batch_var = tf.nn.moments(x, range(len(x.get_shape())-1), name='moments_{0}'.format(name))
-
-        ema = tf.train.ExponentialMovingAverage(decay=0.9)
-
-        def mean_var_with_update():
-            ema_apply_op = ema.apply([batch_mean, batch_var])
-            with tf.control_dependencies([ema_apply_op]):
-                return tf.identity(batch_mean), tf.identity(batch_var)
-
-        mean, var = control_flow_ops.cond(phase_train,
-                                          mean_var_with_update,
-                                          lambda: (ema.average(batch_mean), ema.average(batch_var)))
-        normed = tf.nn.batch_normalization(x, mean, var, self.beta, self.gamma, 1e-3)
+        reuse = self.variable_exist(name)
+
+        #if reuse:
+            #import ipdb; ipdb.set_trace();
+
+        with tf.variable_scope(name, reuse=reuse):
+
+            phase_train = tf.convert_to_tensor(phase_train, dtype=tf.bool)
+            n_out = int(x.get_shape()[-1])
+
+            self.beta = tf.get_variable(name + '_beta',
+                                        initializer=tf.constant(0.0, shape=[n_out], dtype=x.dtype),
+                                        trainable=True,
+                                        dtype=x.dtype)
+
+            self.gamma = tf.get_variable(name + '_gamma',
+                                         initializer=tf.constant(1.0, shape=[n_out], dtype=x.dtype),
+                                         trainable=True,
+                                         dtype=x.dtype)
+
+            if len(x.get_shape()) == 2:
+                self.batch_mean, self.batch_var = tf.nn.moments(x, [0], name='moments_{0}'.format(name))
+            else:
+                self.batch_mean, self.batch_var = tf.nn.moments(x, range(len(x.get_shape())-1), name='moments_{0}'.format(name))
+
+            ema = tf.train.ExponentialMovingAverage(decay=0.9)
+
+            def mean_var_with_update():
+                ema_apply_op = ema.apply([self.batch_mean, self.batch_var])
+                with tf.control_dependencies([ema_apply_op]):
+                    return tf.identity(self.batch_mean), tf.identity(self.batch_var)
+
+            mean, var = control_flow_ops.cond(phase_train,
+                                                        mean_var_with_update,
+                                                        lambda: (ema.average(self.batch_mean), ema.average(self.batch_var)),
+                                                        name=name + "mean_var")
+
+            normed = tf.nn.batch_normalization(x, mean, var, self.beta, self.gamma, 1e-3)
         return normed

bob/learn/tensorflow/network/Chopra.py

@@ -75,28 +75,30 @@ class Chopra(SequenceNetwork):
                  default_feature_layer="fc1",
 
                  seed=10,
-                 use_gpu=False):
+                 use_gpu=False,
+                 batch_norm=False):
 
         super(Chopra, self).__init__(default_feature_layer=default_feature_layer,
                                      use_gpu=use_gpu)
 
         self.add(Conv2D(name="conv1", kernel_size=conv1_kernel_size,
                         filters=conv1_output,
-                        activation=None,
+                        activation=tf.nn.relu,
                         weights_initialization=Xavier(seed=seed, use_gpu=self.use_gpu),
-                        bias_initialization=Constant(use_gpu=self.use_gpu)
+                        bias_initialization=Constant(use_gpu=self.use_gpu),
+                        batch_norm=batch_norm
                         ))
-        self.add(MaxPooling(name="pooling1", shape=pooling1_size, activation=tf.nn.tanh))
+        self.add(MaxPooling(name="pooling1", shape=pooling1_size, activation=tf.nn.relu, batch_norm=False))
 
         self.add(Conv2D(name="conv2", kernel_size=conv2_kernel_size,
                         filters=conv2_output,
-                        activation=None,
+                        activation=tf.nn.relu,
                         weights_initialization=Xavier(seed=seed,  use_gpu=self.use_gpu),
-                        bias_initialization=Constant(use_gpu=self.use_gpu)
-                        ))
-        self.add(MaxPooling(name="pooling2", shape=pooling2_size, activation=tf.nn.tanh))
+                        bias_initialization=Constant(use_gpu=self.use_gpu),
+                        batch_norm=batch_norm))
+        self.add(MaxPooling(name="pooling2", shape=pooling2_size, activation=tf.nn.relu, batch_norm=False))
 
         self.add(FullyConnected(name="fc1", output_dim=fc1_output,
                                 activation=None,
                                 weights_initialization=Xavier(seed=seed, use_gpu=self.use_gpu),
-                                bias_initialization=Constant(use_gpu=self.use_gpu)))
+                                bias_initialization=Constant(use_gpu=self.use_gpu), batch_norm=False))

bob/learn/tensorflow/network/SequenceNetwork.py

@@ -38,6 +38,10 @@ class SequenceNetwork(six.with_metaclass(abc.ABCMeta, object)):
         self.pickle_architecture = None# The trainer triggers this
         self.deployment_shape = None# The trainer triggers this
 
+        # Inference graph
+        self.inference_graph = None
+        self.inference_placeholder = None
+
     def add(self, layer):
         """
         Add a :py:class:`bob.learn.tensorflow.layers.Layer` in the sequence network
@@ -77,17 +81,23 @@ class SequenceNetwork(six.with_metaclass(abc.ABCMeta, object)):
 
         return input_offset
 
-    def compute_projection_graph(self, placeholder):
+    def compute_inference_graph(self, feature_layer=None):
         """Generate a graph for feature extraction
 
         **Parameters**
 
         placeholder: tensorflow placeholder as input data
         """
-        return self.compute_graph(placeholder)
+        if feature_layer is None:
+            feature_layer = self.default_feature_layer
+
+        self.inference_graph = self.compute_graph(self.inference_placeholder, feature_layer, training=False)
+
+    def compute_inference_placeholder(self, data_shape):
+        self.inference_placeholder = tf.placeholder(tf.float32, shape=data_shape, name="feature")
 
     def __call__(self, data, session=None, feature_layer=None):
-        """Run a graph
+        """Run a graph and compute the embeddings
 
         **Parameters**
 
@@ -103,16 +113,16 @@ class SequenceNetwork(six.with_metaclass(abc.ABCMeta, object)):
             session = tf.Session()
 
         # Feeding the placeholder
-        feature_placeholder = tf.placeholder(tf.float32, shape=data.shape, name="feature")
-        feed_dict = {feature_placeholder: data}
+        if self.inference_placeholder is None:
+            self.compute_inference_placeholder(data.shape[1:])
+        feed_dict = {self.inference_placeholder: data}
 
-        if feature_layer is None:
-            feature_layer = self.default_feature_layer
+        if self.inference_graph is None:
+            self.compute_inference_graph(self.inference_placeholder, feature_layer)
 
-        feature = session.run([self.compute_graph(feature_placeholder, feature_layer, training=False)], feed_dict=feed_dict)[0]
-        del feature_placeholder
+        embedding = session.run([self.inference_graph], feed_dict=feed_dict)[0]
 
-        return feature
+        return embedding
 
     def dump_variables(self):
         """
@@ -129,6 +139,8 @@ class SequenceNetwork(six.with_metaclass(abc.ABCMeta, object)):
                 if self.sequence_net[k].batch_norm:
                     variables[self.sequence_net[k].beta.name] = self.sequence_net[k].beta
                     variables[self.sequence_net[k].gamma.name] = self.sequence_net[k].gamma
+                    #variables[self.sequence_net[k].mean.name] = self.sequence_net[k].mean
+                    #variables[self.sequence_net[k].var.name] = self.sequence_net[k].var
 
         return variables
 
@@ -189,7 +201,7 @@ class SequenceNetwork(six.with_metaclass(abc.ABCMeta, object)):
 
         return samples_per_sample
 
-    def save(self, hdf5):
+    def save_hdf5(self, hdf5):
         """
         Save the state of the network in HDF5 format
 
@@ -250,7 +262,7 @@ class SequenceNetwork(six.with_metaclass(abc.ABCMeta, object)):
 
         hdf5.cd("..")
 
-    def load(self, hdf5, shape=None, session=None, batch=1, use_gpu=False):
+    def load_hdf5(self, hdf5, shape=None, session=None, batch=1, use_gpu=False):
         """
         Load the network from scratch.
         This will build the graphs
@@ -287,25 +299,15 @@ class SequenceNetwork(six.with_metaclass(abc.ABCMeta, object)):
         tf.initialize_all_variables().run(session=session)
         self.load_variables_only(hdf5, session)
 
-    def save_original(self, session, saver, path):
+    def save(self, session, saver, path):
+        open(path+"_sequence_net.pickle", 'w').write(self.pickle_architecture)
         return saver.save(session, path)
 
-    def load_original(self, session, path):
+    def load(self, session, path):
+        self.sequence_net = pickle.loads(open(path+"_sequence_net.pickle").read())
         saver = tf.train.import_meta_graph(path + ".meta")
         saver.restore(session, path)
+        self.inference_graph = tf.get_collection("inference_graph")[0]
+        self.inference_placeholder = tf.get_collection("inference_placeholder")[0]
 
-        #if session is None:
-        #    session = tf.Session()
-            #tf.initialize_all_variables().run(session=session)
-
-        # Loading variables
-        #place_holder = tf.placeholder(tf.float32, shape=shape, name="load")
-        #self.compute_graph(place_holder)
-        #tf.initialize_all_variables().run(session=session)
-
-        #if self.saver is None:
-            #variables = self.dump_variables()
-            #variables['input_divide'] = self.input_divide
-            #variables['input_subtract'] = self.input_subtract
-            #self.saver = tf.train.Saver(variables)
-        #self.saver.restore(session, path)
+        return saver

bob/learn/tensorflow/script/train_mnist.py

@@ -59,21 +59,19 @@ def main():
                                                   input_shape=[28, 28, 1],
                                                   batch_size=VALIDATION_BATCH_SIZE)
 
-
-
     # Preparing the architecture
     cnn = True
     if cnn:
-        architecture = Chopra(seed=SEED, fc1_output=10)
-        #architecture = Lenet(seed=SEED)
-        #architecture = Dummy(seed=SEED)
+        architecture = Chopra(seed=SEED, fc1_output=10, batch_norm=False)
         loss = BaseLoss(tf.nn.sparse_softmax_cross_entropy_with_logits, tf.reduce_mean)
-        #trainer = Trainer(architecture=architecture,
-        #                  loss=loss,
-        #                  iterations=ITERATIONS,
-        #                  analizer=ExperimentAnalizer(),
-        #                  prefetch=False, temp_dir="./temp/cnn")
-        #trainer.train(train_data_shuffler, validation_data_shuffler)
+        trainer = Trainer(architecture=architecture,
+                          loss=loss,
+                          iterations=ITERATIONS,
+                          prefetch=False, temp_dir="./temp/cnn/no-batch-norm-all-relu")
+
+        #prefetch = False, temp_dir = "./temp/cnn/batch-norm-2convs-all-relu")
+
+        trainer.train(train_data_shuffler, validation_data_shuffler)
         #trainer.train(train_data_shuffler)
     else:
         mlp = MLP(10, hidden_layers=[15, 20])
@@ -82,16 +80,16 @@ def main():
         trainer.train(train_data_shuffler, validation_data_shuffler)
 
     # Loading
-    test_data_shuffler = Memory(validation_data, validation_labels,
-                                input_shape=[28, 28, 1],
-                                batch_size=400)
+    #test_data_shuffler = Memory(validation_data, validation_labels,
+    #                            input_shape=[28, 28, 1],
+    #                            batch_size=400)
 
-    with tf.Session() as session:
-        new_net = Chopra(seed=SEED, fc1_output=10)
-        new_net.load(bob.io.base.HDF5File("./temp/cnn/model.hdf5"), shape=[400, 28, 28, 1], session=session)
+    #with tf.Session() as session:
+        #new_net = Chopra(seed=SEED, fc1_output=10)
+        #new_net.load(bob.io.base.HDF5File("./temp/cnn/model.hdf5"), shape=[400, 28, 28, 1], session=session)
 
-        [data, labels] = test_data_shuffler.get_batch()
-        print new_net(data, session)
+        #[data, labels] = test_data_shuffler.get_batch()
+        #print new_net(data, session)
 
 
 

bob/learn/tensorflow/script/train_siamese_casia_webface.py

@@ -23,7 +23,7 @@ import tensorflow as tf
 from .. import util
 SEED = 10
 from bob.learn.tensorflow.datashuffler import TripletDisk, TripletWithSelectionDisk, TripletWithFastSelectionDisk
-from bob.learn.tensorflow.network import Lenet, MLP, LenetDropout, VGG, Chopra, Dummy, FaceNet, FaceNetSimple
+from bob.learn.tensorflow.network import Lenet, MLP, LenetDropout, VGG, Chopra, Dummy, FaceNet, FaceNetSimple, VGG16
 from bob.learn.tensorflow.trainers import SiameseTrainer, TripletTrainer, constant
 from bob.learn.tensorflow.loss import ContrastiveLoss, TripletLoss
 import numpy
@@ -56,10 +56,14 @@ def main():
         extension=".hdf5")
                         for o in train_objects]
 
-    train_data_shuffler = TripletWithFastSelectionDisk(train_file_names, train_labels,
-                                                       input_shape=[224, 224, 3],
-                                                       batch_size=BATCH_SIZE,
-                                                       total_identities=16)
+    #train_data_shuffler = TripletWithFastSelectionDisk(train_file_names, train_labels,
+    #                                                   input_shape=[224, 224, 3],
+    #                                                   batch_size=BATCH_SIZE,
+    #                                                   total_identities=16)
+
+    train_data_shuffler = TripletDisk(train_file_names, train_labels,
+                                           input_shape=[224, 224, 3],
+                                           batch_size=VALIDATION_BATCH_SIZE)
 
 
     # Preparing train set
@@ -77,7 +81,8 @@ def main():
                                            batch_size=VALIDATION_BATCH_SIZE)
     # Preparing the architecture
     # LENET PAPER CHOPRA
-    architecture = FaceNetSimple(seed=SEED, use_gpu=USE_GPU)
+    #architecture = FaceNetSimple(seed=SEED, use_gpu=USE_GPU)
+    architecture = VGG16(seed=SEED, use_gpu=USE_GPU)
 
     optimizer = tf.train.GradientDescentOptimizer(0.05)
     loss = TripletLoss(margin=0.2)

bob/learn/tensorflow/test/test_cnn.py

@@ -86,29 +86,30 @@ def test_cnn_trainer():
                                  batch_size=batch_size,
                                  data_augmentation=data_augmentation)
 
-    with tf.Session() as session:
-        directory = "./temp/cnn"
+    directory = "./temp/cnn"
+
+    # Preparing the architecture
+    architecture = Chopra(seed=seed, fc1_output=10)
 
-        # Preparing the architecture
-        architecture = Chopra(seed=seed, fc1_output=10)
+    # Loss for the softmax
+    loss = BaseLoss(tf.nn.sparse_softmax_cross_entropy_with_logits, tf.reduce_mean)
 
-        # Loss for the softmax
-        loss = BaseLoss(tf.nn.sparse_softmax_cross_entropy_with_logits, tf.reduce_mean)
+    # One graph trainer
+    trainer = Trainer(architecture=architecture,
+                      loss=loss,
+                      iterations=iterations,
+                      analizer=None,
+                      prefetch=False,
+                      temp_dir=directory)
+    trainer.train(train_data_shuffler)
+    del trainer #Just to clean tf.variables
 
-        # One graph trainer
-        trainer = Trainer(architecture=architecture,
-                          loss=loss,
-                          iterations=iterations,
-                          analizer=None,
-                          prefetch=False,
-                          temp_dir=directory)
-        trainer.train(train_data_shuffler)
+    with tf.Session() as session:
 
         # Testing
-        validation_shape = [400, 28, 28, 1]
         chopra = Chopra(seed=seed, fc1_output=10)
-        chopra.load(bob.io.base.HDF5File(os.path.join(directory, "model.hdf5")),
-                    shape=validation_shape, session=session)
+        chopra.load(session, os.path.join(directory, "model.ckp"))
+
         validation_data_shuffler = Memory(validation_data, validation_labels,
                                           input_shape=[28, 28, 1],
                                           batch_size=validation_batch_size)
@@ -135,33 +136,32 @@ def test_siamesecnn_trainer():
                                              input_shape=[28, 28, 1],
                                              batch_size=validation_batch_size)
 
-    with tf.Session() as session:
-        directory = "./temp/siamesecnn"
+    directory = "./temp/siamesecnn"
 
-        # Preparing the architecture
-        architecture = Chopra(seed=seed, fc1_output=10)
+    # Preparing the architecture
+    architecture = Chopra(seed=seed, fc1_output=10)
 
-        # Loss for the Siamese
-        loss = ContrastiveLoss(contrastive_margin=4.)
+    # Loss for the Siamese
+    loss = ContrastiveLoss(contrastive_margin=4.)
 
-        # One graph trainer
-        trainer = SiameseTrainer(architecture=architecture,
-                                 loss=loss,
-                                 iterations=iterations,
-                                 prefetch=False,
-                                 analizer=None,
-                                 learning_rate=constant(0.05, name="siamese_lr"),
-                                 temp_dir=directory)
+    # One graph trainer
+    trainer = SiameseTrainer(architecture=architecture,
+                             loss=loss,
+                             iterations=iterations,
+                             prefetch=False,
+                             analizer=None,
+                             learning_rate=constant(0.05, name="siamese_lr"),
+                             temp_dir=directory)
 
-        trainer.train(train_data_shuffler)
+    trainer.train(train_data_shuffler)
+    del trainer  # Just to clean tf.variables
 
+    with tf.Session() as session:
         # Testing
-        validation_shape = [400, 28, 28, 1]
         chopra = Chopra(seed=seed, fc1_output=10)
-        chopra.load(bob.io.base.HDF5File(os.path.join(directory, "model.hdf5")),
-                    shape=validation_shape, session=session)
+        chopra.load(session, os.path.join(directory, "model.ckp"))
 
-        eer = dummy_experiment(validation_data_shuffler, architecture, session)
+        eer = dummy_experiment(validation_data_shuffler, chopra, session)
 
         # At least 80% of accuracy
         assert eer < 0.25
@@ -181,33 +181,33 @@ def test_tripletcnn_trainer():
                                              input_shape=[28, 28, 1],
                                              batch_size=validation_batch_size)
 
-    with tf.Session() as session:
-        directory = "./temp/tripletcnn"
+    directory = "./temp/tripletcnn"
 
-        # Preparing the architecture
-        architecture = Chopra(seed=seed, fc1_output=10)
+    # Preparing the architecture
+    architecture = Chopra(seed=seed, fc1_output=10)
 
-        # Loss for the Siamese
-        loss = TripletLoss(margin=4.)
+    # Loss for the Siamese
+    loss = TripletLoss(margin=4.)
 
-        # One graph trainer
-        trainer = TripletTrainer(architecture=architecture,
-                                 loss=loss,
-                                 iterations=iterations,
-                                 prefetch=False,
-                                 analizer=None,
-                                 learning_rate=constant(0.05, name="triplet_lr"),
-                                 temp_dir=directory)
+    # One graph trainer
+    trainer = TripletTrainer(architecture=architecture,
+                             loss=loss,
+                             iterations=iterations,
+                             prefetch=False,
+                             analizer=None,
+                             learning_rate=constant(0.05, name="triplet_lr"),
+                             temp_dir=directory)
 
-        trainer.train(train_data_shuffler)
+    trainer.train(train_data_shuffler)
+    del trainer  # Just to clean tf.variables
+
+    with tf.Session() as session: