Still dumping code.

bob/learn/tensorflow/DataShuffler.py

@@ -4,7 +4,7 @@
 # @date: Wed 11 May 2016 09:39:36 CEST 
 
 import numpy
-
+import tensorflow as tf
 
 def scale_mean_norm(data, scale=0.00390625):
     mean = numpy.mean(data)
@@ -14,7 +14,7 @@ def scale_mean_norm(data, scale=0.00390625):
 
 
 class DataShuffler(object):
-    def __init__(self, perc_train=0.9, scale=True):
+    def __init__(self, data, labels, perc_train=0.9, scale=True, train_batch_size=1, validation_batch_size=1):
         """
          Some base functions for neural networks
 
@@ -25,38 +25,51 @@ class DataShuffler(object):
         self.perc_train = perc_train
         self.scale = True
         self.scale_value = 0.00390625
+        self.train_batch_size = train_batch_size
+        self.validation_batch_size = validation_batch_size
+        self.data = data
+        self.labels =labels
+
+        self.n_samples = self.data.shape[0]
+        self.width = self.data.shape[1]
+        self.height = self.data.shape[2]
+        self.channels = self.data.shape[3]
+        self.start_shuffler()
+
+    def get_placeholders(self, name=""):
+        data = tf.placeholder(tf.float32, shape=(self.train_batch_size, self.width,
+                                                 self.height, self.channels), name=name)
 
+        labels = tf.placeholder(tf.int64, shape=self.train_batch_size)
 
-    def start_shuffler(self, data, labels):
+        return data, labels
+
+    def start_shuffler(self):
         """
          Some base functions for neural networks
 
          **Parameters**
            data:
         """
-        scale_value = 0.00390625
-
-        total_samples = data.shape[0]
 
-        indexes = numpy.array(range(total_samples))
+        indexes = numpy.array(range(self.n_samples))
         numpy.random.shuffle(indexes)
 
         # Spliting train and validation
-        train_samples = int(round(total_samples * self.perc_train))
-        validation_samples = total_samples - train_samples
-        data = numpy.reshape(data, (data.shape[0], 28, 28, 1))
+        train_samples = int(round(self.n_samples * self.perc_train))
+        validation_samples = self.n_samples - train_samples
 
-        self.train_data = data[indexes[0:train_samples], :, :, :]
-        self.train_labels = labels[indexes[0:train_samples]]
+        self.train_data = self.data[indexes[0:train_samples], :, :, :]
+        self.train_labels = self.labels[indexes[0:train_samples]]
 
-        self.validation_data = data[indexes[train_samples:train_samples + validation_samples], :, :, :]
-        self.validation_labels = labels[indexes[train_samples:train_samples + validation_samples]]
+        self.validation_data = self.data[indexes[train_samples:train_samples + validation_samples], :, :, :]
+        self.validation_labels = self.labels[indexes[train_samples:train_samples + validation_samples]]
         self.total_labels = 10
 
         if self.scale:
             # data = scale_minmax_norm(data,lower_bound = -1, upper_bound = 1)
             self.train_data, self.mean = scale_mean_norm(self.train_data)
-            self.validation_data = (self.validation_data - self.mean) * scale_value
+            self.validation_data = (self.validation_data - self.mean) * self.scale_value
 
     def get_batch(self, n_samples, train_dataset=True):
 

bob/learn/tensorflow/layers/Conv2D.py

@@ -3,7 +3,7 @@
 # @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
 # @date: Wed 11 May 2016 17:38 CEST
 
-import tensoflow as tf
+import tensorflow as tf
 from bob.learn.tensorflow.util import *
 from .Layer import Layer
 
@@ -14,7 +14,7 @@ class Conv2D(Layer):
     2D Convolution
     """
 
-    def __init__(self, input, activation=None,
+    def __init__(self, name, activation=None,
                  kernel_size=3,
                  filters=8,
                  initialization='xavier',
@@ -33,20 +33,31 @@ class Conv2D(Layer):
         use_gpu: Store data in the GPU
         seed: Seed for the Random number generation
         """
-        super(Conv2D, self).__init__(input, initialization='xavier', use_gpu=False, seed=10)
-        self.activation = activation
-
-        self.W = create_weight_variables([kernel_size, kernel_size, 1, filters],
-                                         seed=seed, name="conv", use_gpu=use_gpu)
-
-        if activation is not None:
-            self.b = create_bias_variables([filters], name="bias", use_gpu=self.use_gpu)
+        super(Conv2D, self).__init__(name, activation=activation, initialization='xavier',
+                                     use_gpu=use_gpu, seed=seed)
+        self.kernel_size = kernel_size
+        self.filters = filters
+        self.initialization = initialization
+        self.W = None
+        self.b = None
+
+    def create_variables(self, input):
+        self.input = input
+        if self.W is None:
+            self.W = create_weight_variables([self.kernel_size, self.kernel_size, 1, self.filters],
+                                             seed=self.seed, name=str(self.name), use_gpu=self.use_gpu)
+            if self.activation is not None:
+                self.b = create_bias_variables([self.filters], name=str(self.name) + "bias", use_gpu=self.use_gpu)
 
     def get_graph(self):
-        with tf.name_scope('conv'):
-            conv = tf.nn.conv2d(self.input, self.W, strides=[1, 1, 1, 1], padding='SAME')
+        with tf.name_scope(str(self.name)):
+            conv2d = tf.nn.conv2d(self.input, self.W, strides=[1, 1, 1, 1], padding='SAME')
 
-        with tf.name_scope('activation'):
-            non_linearity = tf.nn.tanh(tf.nn.bias_add(conv, self.b))
+        if self.activation is not None:
+            with tf.name_scope(str(self.name) + 'activation'):
+                non_linear_conv2d = tf.nn.tanh(tf.nn.bias_add(conv2d, self.b))
+                self.output = non_linear_conv2d
+        else:
+            self.output = conv2d
 
-        return non_linearity
+        return self.output

bob/learn/tensorflow/layers/FullyConnected.py

@@ -3,9 +3,10 @@
 # @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
 # @date: Wed 11 May 2016 17:38 CEST
 
-import tensoflow as tf
+import tensorflow as tf
 from bob.learn.tensorflow.util import *
 from .Layer import Layer
+from operator import mul
 
 
 class FullyConnected(Layer):
@@ -14,7 +15,7 @@ class FullyConnected(Layer):
     2D Convolution
     """
 
-    def __init__(self, input, activation=None,
+    def __init__(self, name, output_dim, activation=None,
                  initialization='xavier',
                  use_gpu=False,
                  seed=10
@@ -29,23 +30,36 @@ class FullyConnected(Layer):
         use_gpu: Store data in the GPU
         seed: Seed for the Random number generation
         """
-        super(FullyConnected, self).__init__(input, initialization='xavier', use_gpu=False, seed=10)
-        self.activation = activation
+        super(FullyConnected, self).__init__(name, activation=activation,
+                                             initialization=initialization, use_gpu=use_gpu, seed=seed)
+        self.output_dim = output_dim
+        self.W = None
+        self.b = None
 
+    def create_variables(self, input):
+        self.input = input
+        if self.W is None:
+            input_dim = reduce(mul, self.input.get_shape().as_list())
 
-        if len(input.get_shape())==4:
-            self.W = create_weight_variables([kernel_size, kernel_size, 1, filters],
-                                         seed=seed, name="conv", use_gpu=use_gpu)
-
-        if activation is not None:
-            self.b = create_bias_variables([filters], name="bias", use_gpu=self.use_gpu)
+            self.W = create_weight_variables([input_dim, self.output_dim],
+                                             seed=self.seed, name=str(self.name), use_gpu=self.use_gpu)
+            if self.activation is not None:
+                self.b = create_bias_variables([self.output_dim], name=str(self.name)+"_bias", use_gpu=self.use_gpu)
 
     def get_graph(self):
         with tf.name_scope('fc'):
-            conv = tf.nn.conv2d(self.input, self.W, strides=[1, 1, 1, 1], padding='SAME')
+
+            if len(self.input.get_shape()) == 4:
+                shape = self.input.get_shape().as_list()
+                fc = tf.reshape(self.input, [shape[0], shape[1] * shape[2] * shape[3]])
 
         if self.activation is not None:
             with tf.name_scope('activation'):
-                non_linearity = tf.nn.tanh(tf.nn.bias_add(conv, self.b))
+                non_linear_fc = tf.nn.tanh(tf.matmul(fc, self.W) + self.b)
+                self.output = non_linear_fc
+        else:
+            self.output = fc
+
+        return self.output
+
 
-        return non_linearity

bob/learn/tensorflow/layers/Layer.py

@@ -3,7 +3,7 @@
 # @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
 # @date: Wed 11 May 2016 17:38 CEST
 
-import tensoflow as tf
+import tensorflow as tf
 
 
 class Layer(object):
@@ -12,7 +12,7 @@ class Layer(object):
     Layer base class
     """
 
-    def __init__(self, input, initialization='xavier', use_gpu=False, seed=10):
+    def __init__(self, name, activation=None, initialization='xavier', use_gpu=False, seed=10):
         """
         Base constructor
 
@@ -20,10 +20,23 @@ class Layer(object):
         input: Layer input
 
         """
-        self.input = input
+        self.name = name
         self.initialization = initialization
         self.use_gpu = use_gpu
         self.seed = seed
 
+        self.input = None
+        self.activation = None
+        self.output = None
+
+    def create_variables(self, input):
+        NotImplementedError("Please implement this function in derived classes")
+
     def get_graph(self):
         NotImplementedError("Please implement this function in derived classes")
+
+    def get_shape(self):
+        if self.output is None:
+            NotImplementedError("This class was not implemented properly")
+        else:
+            return self.output.get_shape()

bob/learn/tensorflow/layers/MaxPooling.py

@@ -3,18 +3,25 @@
 # @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
 # @date: Wed 11 May 2016 17:38 CEST
 
-import tensoflow as tf
+import tensorflow as tf
 from bob.learn.tensorflow.util import *
 from .Layer import Layer
 
 
 class MaxPooling(Layer):
 
-    def __init__(self, input, use_gpu=False):
+    def __init__(self, name, use_gpu=False):
         """
         Constructor
         """
-        super(MaxPooling, self).__init__(input, use_gpu=False)
+        super(MaxPooling, self).__init__(name, use_gpu=False)
+
+    def create_variables(self, input):
+        self.input = input
+        return
 
     def get_graph(self):
-        tf.nn.max_pool(self.input, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
+        with tf.name_scope(str(self.name)):
+            self.output = tf.nn.max_pool(self.input, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
+
+        return self.output

bob/learn/tensorflow/layers/__init__.py

@@ -2,6 +2,10 @@
 from pkgutil import extend_path
 __path__ = extend_path(__path__, __name__)
 
-from DataShuffler import *
+#from DataShuffler import *
+from .Layer import Layer
+from .Conv2D import Conv2D
+from .FullyConnected import FullyConnected
+from .MaxPooling import MaxPooling
 
 

bob/learn/tensorflow/loss/ContrastiveLoss.py

@@ -6,7 +6,7 @@
 import logging
 logger = logging.getLogger("bob.learn.tensorflow")
 import tensorflow as tf
-from bob.learn.tensorflow.loss.Baseloss as Baseloss
+from .Baseloss import Baseloss
 
 
 class ContrastiveLoss(BaseLoss):

bob/learn/tensorflow/network/Lenet.py

@@ -9,10 +9,11 @@ Class that creates the lenet architecture
 
 from ..util import *
 import tensorflow as tf
-from bob.learn.tensorflow.network.BaseArchitecture import BaseArchitecture
+from .SequenceNetwork import SequenceNetwork
+from ..layers import Conv2D, FullyConnected, MaxPooling
 
 
-class Lenet(BaseArchitecture):
+class Lenet(SequenceNetwork):
 
     def __init__(self,
                  conv1_kernel_size=5,
@@ -40,80 +41,11 @@ class Lenet(BaseArchitecture):
 
             seed = 10
         """
-
-        self.conv1_kernel_size = conv1_kernel_size,
-        self.conv1_output = conv1_output,
-
-        self.conv2_kernel_size = conv2_kernel_size,
-        self.conv2_output = conv2_output,
-
-        self.fc1_output = fc1_output,
-        self.n_classes = n_classes,
-
-        super(Lenet, self).__init__(seed=seed, use_gpu=use_gpu)
-
-    def create_variables(self):
-
-        # First convolutional
-        self.W_conv1 = create_weight_variables([self.conv1_kernel_size, self.conv1_kernel_size, 1, self.conv1_output],
-                                               seed=self.seed, name="W_conv1", use_gpu=self.use_gpu)
-        self.b_conv1 = create_bias_variables([self.conv1_output], name="bias_conv1", use_gpu=self.use_gpu)
-
-        # Second convolutional
-        self.W_conv2 = create_weight_variables([self.conv2_kernel_size, self.conv2_kernel_size, self.conv1_output,
-                                                self.conv2_output], seed=self.seed, name="W_conv2",
-                                               use_gpu=self.use_gpu)
-        self.b_conv2 = create_bias_variables([self.conv2_output], name="bias_conv2", use_gpu=self.use_gpu)
-
-        # First fc
-        self.W_fc1 = create_weight_variables([(28 // 4) * (28 // 4) * self.conv2_output, self.fc1_output],
-                                             seed=self.seed, name="W_fc1", use_gpu=self.use_gpu)
-        self.b_fc1 = create_bias_variables([self.fc1_output], name="bias_fc1", use_gpu=self.use_gpu)
-
-        # Second FC fc
-        self.W_fc2 = create_weight_variables([self.fc1_output, self.n_classes], seed=self.seed,
-                                             name="W_fc2", use_gpu=self.use_gpu)
-        self.b_fc2 = create_bias_variables([self.n_classes], name="bias_fc2", use_gpu=self.use_gpu)
-
-        self.seed = self.seed
-
-    def create_graph(self, data):
-
-        # Creating the architecture
-        # First convolutional
-        with tf.name_scope('conv_1') as scope:
-            conv1 = create_conv2d(data, self.W_conv1)
-
-        with tf.name_scope('tanh_1') as scope:
-            tanh_1 = create_tanh(conv1, self.b_conv1)
-
-        # Pooling
-        with tf.name_scope('pool_1') as scope:
-            pool1 = create_max_pool(tanh_1)
-
-        # Second convolutional
-        with tf.name_scope('conv_2') as scope:
-            conv2 = create_conv2d(pool1, self.W_conv2)
-
-        with tf.name_scope('tanh_2') as scope:
-            tanh_2 = create_tanh(conv2, self.b_conv2)
-
-        # Pooling
-        with tf.name_scope('pool_2') as scope:
-            pool2 = create_max_pool(tanh_2)
-
-        #if train:
-            #pool2 = tf.nn.dropout(pool2, 0.5, seed=self.seed)
-
-        # Reshaping all the convolved images to 2D to feed the FC layers
-        # FC1
-        with tf.name_scope('fc_1') as scope:
-            pool_shape = pool2.get_shape().as_list()
-            reshape = tf.reshape(pool2, [pool_shape[0], pool_shape[1] * pool_shape[2] * pool_shape[3]])
-            fc1 = tf.nn.tanh(tf.matmul(reshape, self.W_fc1) + self.b_fc1)
-
-        # FC2
-        with tf.name_scope('fc_2') as scope:
-            fc2 = tf.matmul(fc1, self.W_fc2) + self.b_fc2
-
-        return fc2
+        super(Lenet, self).__init__()
+
+        self.add(Conv2D(name="conv1", kernel_size=conv1_kernel_size, filters=conv1_output))
+        self.add(MaxPooling(name="pooling1"))
+        self.add(Conv2D(name="conv2", kernel_size=conv2_kernel_size, filters=conv2_output))
+        self.add(MaxPooling(name="pooling2"))
+        self.add(FullyConnected(name="fc1", output_dim=fc1_output, activation=True))
+        self.add(FullyConnected(name="fc1", output_dim=n_classes, activation=False))

bob/learn/tensorflow/network/BaseArchitecture.py → bob/learn/tensorflow/network/SequenceNetwork.py

 #!/usr/bin/env python
 # vim: set fileencoding=utf-8 :
 # @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
-# @date: Wed 11 May 2016 09:39:36 CEST 
+# @date: Thu 11 Aug 2016 09:39:36 CEST
 
 """
 Class that creates the lenet architecture
@@ -12,35 +12,44 @@ import tensorflow as tf
 import abc
 import six
 
+from collections import OrderedDict
+from bob.learn.tensorflow.layers import *
 
-class BaseArchitecture(six.with_metaclass(abc.ABCMeta, object)):
+
+class SequenceNetwork(six.with_metaclass(abc.ABCMeta, object)):
     """
     Base class to create architectures using TensorFlow
     """
 
-    def __init__(self, seed=10, use_gpu=False):
+    def __init__(self):
         """
         Base constructor
-        """
-        self.seed = seed
-        self.use_gpu = use_gpu
-        self.create_variables()
 
-    def create_variables(self):
-        """
-        Create the Tensor flow variables
+        **Parameters**
+        input: Place Holder
         """
-        raise NotImplementedError("Please implement this function in derived classes")
 
-    def create_graph(self, data):
-        """
-        Create the Tensor flow variables
+        self.sequence_net = OrderedDict()
+
+    def add(self, layer):
+        if not isinstance(layer, Layer):
+            raise ValueError("Input `layer` must be an instance of `bob.learn.tensorflow.layers.Layer`")
+        self.sequence_net[layer.name] = layer
+
+    def compute_graph(self, input_data):
+
+
+        input_offset = input_data
+        for k in self.sequence_net.keys():
+            print k
+            import ipdb;
+            ipdb.set_trace();
+
+            current_layer = self.sequence_net[k]
+            current_layer.create_variables(input_offset)
+            input_offset = current_layer.get_graph
+
+        return input_offset
 
-        **Parameters**
-            data: Tensorflow Placeholder
 
-        **Returns**
-            Network output
-        """
 
-        raise NotImplementedError("Please implement this function in derived classes")

bob/learn/tensorflow/network/__init__.py

@@ -2,4 +2,6 @@
 from pkgutil import extend_path
 __path__ = extend_path(__path__, __name__)
 
+from .SequenceNetwork import SequenceNetwork
+from .Lenet import Lenet
 

bob/learn/tensorflow/script/train_mnist.py

@@ -22,8 +22,9 @@ import tensorflow as tf
 from .. import util
 SEED = 10
 from ..DataShuffler import *
-from ..lenet import Lenet
-import sys
+from bob.learn.tensorflow.network import Lenet
+from bob.learn.tensorflow.trainers import Trainer
+import numpy
 
 def main():
     args = docopt(__doc__, version='Mnist training with TensorFlow')
@@ -36,59 +37,14 @@ def main():
 
     # Loading data
     data, labels = util.load_mnist(data_dir="./src/bob.db.mnist/bob/db/mnist/")
+    data = numpy.reshape(data, (data.shape[0], 28, 28, 1))
     data_shuffler = DataShuffler(data, labels)
 
-    # Defining place holders for train and validation
-    train_data_node = tf.placeholder(tf.float32, shape=(BATCH_SIZE, 28, 28, 1))
-    train_labels_node = tf.placeholder(tf.int64, shape=BATCH_SIZE)
-    validation_data_node = tf.placeholder(tf.float32, shape=(data_shuffler.validation_data.shape[0], 28, 28, 1))
+    # Preparing the architecture
+    lenet = Lenet()
 
-    # Creating the architecture for train and validation
-    lenet_architecture = Lenet(seed=SEED, use_gpu=USE_GPU)
-    lenet_train = lenet_architecture.create_lenet(train_data_node)
-    lenet_validation = lenet_architecture.create_lenet(validation_data_node, train=False)
+    loss = tf.nn.sparse_softmax_cross_entropy_with_logits
+    trainer = Trainer(architecture=lenet, loss=loss)
+    trainer.train(data_shuffler)
 
-    # Simple loss
-    loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(lenet_train, train_labels_node))
 
-    #regularizer = (tf.nn.l2_loss(W_fc1) + tf.nn.l2_loss(b_fc1) +
-    #                tf.nn.l2_loss(W_fc2) + tf.nn.l2_loss(b_fc2))
-    #loss += 5e-4 * regularizer
-
-    # Learning rate
-    batch = tf.Variable(0)
-    learning_rate = tf.train.exponential_decay(
-        0.001, # Learning rate
-        batch * BATCH_SIZE,
-        data_shuffler.train_data.shape[0],
-        0.95 # Decay step
-    )
-
-    optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=batch)
-    train_prediction = tf.nn.softmax(lenet_train)
-    validation_prediction = tf.nn.softmax(lenet_validation)
-
-    print("Initializing !!")
-    # Training
-    with tf.Session() as session:
-        tf.initialize_all_variables().run()
-        for step in range(ITERATIONS):
-
-            train_data, train_labels = data_shuffler.get_batch(BATCH_SIZE)
-
-            feed_dict = {train_data_node: train_data,
-                         train_labels_node: train_labels}
-
-            _, l, lr, predictions = session.run([optimizer, loss, learning_rate, train_prediction], feed_dict=feed_dict)
-            if step % VALIDATION_TEST == 0:
-                validation_data, validation_labels = data_shuffler.get_batch(data_shuffler.validation_data.shape[0],
-                                                                             train_dataset=False)
-
-                accuracy = util.evaluate_softmax(validation_data, validation_labels, session, validation_prediction, validation_data_node)
-                print("Step {0}. Loss = {1}, Lr={2}, Accuracy validation = {3}".format(step, l, lr, accuracy))
-                sys.stdout.flush()
-                
-    print("Step {0}. Loss = {1}, Lr={2}, Accuracy validation = {3}".format(step, l, lr, accuracy))
-
-
-    print("End !!")

bob/learn/tensorflow/trainers/BaseTrainer.py

-#!/usr/bin/env python
-# vim: set fileencoding=utf-8 :
-# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
-# @date: Tue 09 Aug 2016 15:25:22 CEST
-
-import logging
-logger = logging.getLogger("bob.learn.tensorflow")
-from ..DataShuffler import DataShuffler
-
-
-class BaseTrainer(object):
-
-    def __init__(self,
-                 data=None,
-                 labels=None,
-                 network=None,
-                 width=28,
-                 height=28,