Scratch

.gitignore

+*~
+*.swp
+*.pyc
+bin
+eggs
+parts
+.installed.cfg
+.mr.developer.cfg
+*.egg-info
+src
+develop-eggs
+sphinx
+dist

MANIFEST.in

+include README.rst bootstrap-buildout.py buildout.cfg COPYING version.txt requirements.txt
+recursive-include doc *.py *.rst

README.rst

+.. vim: set fileencoding=utf-8 :
+.. Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
+.. Thu 30 Jan 08:46:53 2014 CET
+
+======================
+ Tensorflow Examples
+======================
+
+This example package implements simple CNN trainings with tensorflow using the MNIST database.
+It is implemented two CNNs:
+
+* Lenet using softmax
+* Siamease network (Lenet as a base)
+
+Installation
+------------
+
+With tensorflow installed in your environment just run::
+
+  $ python bootstrap.py
+  $ ./bin/buildout
+
+Documentation
+-------------
+
+For the time being has just one script that trains the MNIST using the Lenet architecture using the softmax
+cross entropy loss. Run the command bellow to have some help::
+
+  $ ./bin/train_mnist.py --help # Lenet using softmax
+  $ ./bin/train_mnist_siamese.py --help # Lenet using siamese net
+
+
+It is just the classical stuff.
+I haven't added any regularization (L2, L1, dropout, etc )

bob/__init__.py

+# see https://docs.python.org/3/library/pkgutil.html
+from pkgutil import extend_path
+__path__ = extend_path(__path__, __name__)
\ No newline at end of file

bob/learn/__init__.py

+# see https://docs.python.org/3/library/pkgutil.html
+from pkgutil import extend_path
+__path__ = extend_path(__path__, __name__)
\ No newline at end of file

bob/learn/tensorflow/DataShuffler.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
+# @date: Wed 11 May 2016 09:39:36 CEST 
+
+import numpy
+
+
+def scale_mean_norm(data, scale=0.00390625):
+    mean = numpy.mean(data)
+    data = (data - mean) * scale
+
+    return data, mean
+
+
+class DataShuffler(object):
+    def __init__(self, perc_train=0.9, scale=True):
+        """
+         Some base functions for neural networks
+
+         **Parameters**
+           data:
+        """
+
+        self.perc_train = perc_train
+        self.scale = True
+        self.scale_value = 0.00390625
+
+
+    def start_shuffler(self, data, labels):
+        """
+         Some base functions for neural networks
+
+         **Parameters**
+           data:
+        """
+        scale_value = 0.00390625
+
+        total_samples = data.shape[0]
+
+        indexes = numpy.array(range(total_samples))
+        numpy.random.shuffle(indexes)
+
+        # Spliting train and validation
+        train_samples = int(round(total_samples * self.perc_train))
+        validation_samples = total_samples - train_samples
+        data = numpy.reshape(data, (data.shape[0], 28, 28, 1))
+
+        self.train_data = data[indexes[0:train_samples], :, :, :]
+        self.train_labels = labels[indexes[0:train_samples]]
+
+        self.validation_data = data[indexes[train_samples:train_samples + validation_samples], :, :, :]
+        self.validation_labels = labels[indexes[train_samples:train_samples + validation_samples]]
+        self.total_labels = 10
+
+        if self.scale:
+            # data = scale_minmax_norm(data,lower_bound = -1, upper_bound = 1)
+            self.train_data, self.mean = scale_mean_norm(self.train_data)
+            self.validation_data = (self.validation_data - self.mean) * scale_value
+
+    def get_batch(self, n_samples, train_dataset=True):
+
+        if train_dataset:
+            data = self.train_data
+            label = self.train_labels
+        else:
+            data = self.validation_data
+            label = self.validation_labels
+
+        # Shuffling samples
+        indexes = numpy.array(range(data.shape[0]))
+        numpy.random.shuffle(indexes)
+
+        selected_data = data[indexes[0:n_samples], :, :, :]
+        selected_labels = label[indexes[0:n_samples]]
+
+        return selected_data.astype("float32"), selected_labels
+
+    def get_pair(self, n_pair=1, is_target_set_train=True, zero_one_labels=True):
+        """
+        Get a random pair of samples
+
+        **Parameters**
+            is_target_set_train: Defining the target set to get the batch
+
+        **Return**
+        """
+
+        def get_genuine_or_not(input_data, input_labels, genuine=True):
+
+            if genuine:
+                # TODO: THIS KEY SELECTION NEEDS TO BE MORE EFFICIENT
+
+                # Getting a client
+                index = numpy.random.randint(self.total_labels)
+
+                # Getting the indexes of the data from a particular client
+                indexes = numpy.where(input_labels == index)[0]
+                numpy.random.shuffle(indexes)
+
+                # Picking a pair
+                data = input_data[indexes[0], :, :, :]
+                data_p = input_data[indexes[1], :, :, :]
+
+            else:
+                # Picking a pair from different clients
+                index = numpy.random.choice(self.total_labels, 2, replace=False)
+
+                # Getting the indexes of the two clients
+                indexes = numpy.where(input_labels == index[0])[0]
+                indexes_p = numpy.where(input_labels == index[1])[0]
+                numpy.random.shuffle(indexes)
+                numpy.random.shuffle(indexes_p)
+
+                # Picking a pair
+                data = input_data[indexes[0], :, :, :]
+                data_p = input_data[indexes_p[0], :, :, :]
+
+            return data, data_p
+
+        if is_target_set_train:
+            target_data = self.train_data
+            target_labels = self.train_labels
+        else:
+            target_data = self.validation_data
+            target_labels = self.validation_labels
+
+        total_data = n_pair * 2
+        c = target_data.shape[3]
+        w = target_data.shape[1]
+        h = target_data.shape[2]
+
+        data = numpy.zeros(shape=(total_data, w, h, c), dtype='float32')
+        data_p = numpy.zeros(shape=(total_data, w, h, c), dtype='float32')
+        labels_siamese = numpy.zeros(shape=total_data, dtype='float32')
+
+        genuine = True
+        for i in range(total_data):
+            data[i, :, :, :], data_p[i, :, :, :] = get_genuine_or_not(target_data, target_labels, genuine=genuine)
+            if zero_one_labels:
+                labels_siamese[i] = not genuine
+            else:
+                labels_siamese[i] = -1 if genuine else +1
+            genuine = not genuine
+
+        return data, data_p, labels_siamese
+
+    def get_triplet(self, n_labels, n_triplets=1, is_target_set_train=True):
+        """
+        Get a triplet
+
+        **Parameters**
+            is_target_set_train: Defining the target set to get the batch
+
+        **Return**
+        """
+
+        def get_one_triplet(input_data, input_labels):
+
+            # Getting a pair of clients
+            index = numpy.random.choice(n_labels, 2, replace=False)
+            label_positive = index[0]
+            label_negative = index[1]
+
+            # Getting the indexes of the data from a particular client
+            indexes = numpy.where(input_labels == index[0])[0]
+            numpy.random.shuffle(indexes)
+
+            # Picking a positive pair
+            data_anchor = input_data[indexes[0], :, :, :]
+            data_positive = input_data[indexes[1], :, :, :]
+
+            # Picking a negative sample
+            indexes = numpy.where(input_labels == index[1])[0]
+            numpy.random.shuffle(indexes)
+            data_negative = input_data[indexes[0], :, :, :]
+
+            return data_anchor, data_positive, data_negative, label_positive, label_positive, label_negative
+
+        if is_target_set_train:
+            target_data = self.train_data
+            target_labels = self.train_labels
+        else:
+            target_data = self.validation_data
+            target_labels = self.validation_labels
+
+        c = target_data.shape[3]
+        w = target_data.shape[1]
+        h = target_data.shape[2]
+
+        data_a = numpy.zeros(shape=(n_triplets, w, h, c), dtype='float32')
+        data_p = numpy.zeros(shape=(n_triplets, w, h, c), dtype='float32')
+        data_n = numpy.zeros(shape=(n_triplets, w, h, c), dtype='float32')
+        labels_a = numpy.zeros(shape=n_triplets, dtype='float32')
+        labels_p = numpy.zeros(shape=n_triplets, dtype='float32')
+        labels_n = numpy.zeros(shape=n_triplets, dtype='float32')
+
+        for i in range(n_triplets):
+            data_a[i, :, :, :], data_p[i, :, :, :], data_n[i, :, :, :], \
+            labels_a[i], labels_p[i], labels_n[i] = \
+                get_one_triplet(target_data, target_labels)
+
+        return data_a, data_p, data_n, labels_a, labels_p, labels_n

bob/learn/tensorflow/__init__.py

+# see https://docs.python.org/3/library/pkgutil.html
+from pkgutil import extend_path
+__path__ = extend_path(__path__, __name__)
+
+from DataShuffler import *
+
+

bob/learn/tensorflow/layers/Layers.py

+# see https://docs.python.org/3/library/pkgutil.html
+from pkgutil import extend_path
+__path__ = extend_path(__path__, __name__)
+
+from DataShuffler import *
+
+

bob/learn/tensorflow/layers/__init__.py

+# see https://docs.python.org/3/library/pkgutil.html
+from pkgutil import extend_path
+__path__ = extend_path(__path__, __name__)
+
+from DataShuffler import *
+
+

bob/learn/tensorflow/loss/BaseLoss.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
+# @date: Tue 09 Aug 2016 16:38 CEST
+
+import logging
+logger = logging.getLogger("bob.learn.tensorflow")
+
+
+class BaseLoss(object):
+    """
+    Base loss function.
+
+    One exam
+    """
+
+    def __init__(self, loss):
+        self.loss = loss
+
+    def __call__(self):
+        return self.loss

bob/learn/tensorflow/loss/ContrastiveLoss.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
+# @date: Wed 10 Aug 2016 16:38 CEST
+
+import logging
+logger = logging.getLogger("bob.learn.tensorflow")
+import tensorflow as tf
+from bob.learn.tensorflow.loss.Baseloss as Baseloss
+
+
+class ContrastiveLoss(BaseLoss):
+    """
+    Compute the contrastive loss as in
+
+    http://yann.lecun.com/exdb/publis/pdf/hadsell-chopra-lecun-06.pdf
+
+    L = 0.5 * (Y) * D^2 + 0.5 * (1-Y) * {max(0, margin - D)}^2
+
+    OR MAYBE THAT
+
+    L = 0.5 * (1-Y) * D^2 + 0.5 * (Y) * {max(0, margin - D)}^2
+
+    **Parameters**
+     left_feature: First element of the pair
+     right_feature: Second element of the pair
+     label: Label of the pair (0 or 1)
+     margin: Contrastive margin
+
+    """
+
+    def __init__(self, label, left_feature, right_feature, margin):
+
+        with tf.name_scope("contrastive_loss"):
+            label = tf.to_float(label)
+            one = tf.constant(1.0)
+
+            d = bob.learn.tensorflow.util.compute_euclidean_distance(left_feature, right_feature)
+            # first_part = tf.mul(one - label, tf.square(d))  # (Y-1)*(d^2)
+            # first_part = tf.mul(label, tf.square(d))  # (Y-1)*(d^2)
+            between_class = tf.exp(tf.mul(one - label, tf.square(d)))  # (1-Y)*(d^2)
+            max_part = tf.square(tf.maximum(margin - d, 0))
+
+            within_class = tf.mul(label, max_part)  # (Y) * max((margin - d)^2, 0)
+            # second_part = tf.mul(one-label, max_part)  # (Y) * max((margin - d)^2, 0)
+
+            self.loss = 0.5 * tf.reduce_mean(within_class + between_class)
+
+            self.within_class = tf.reduce_mean(within_class), tf.reduce_mean(between_class)

bob/learn/tensorflow/loss/__init__.py

+# see https://docs.python.org/3/library/pkgutil.html
+from pkgutil import extend_path
+__path__ = extend_path(__path__, __name__)
+
+from .BaseLoss import BaseLoss
+
+
+
+
+

bob/learn/tensorflow/network/BaseArchitecture.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
+# @date: Wed 11 May 2016 09:39:36 CEST 
+
+"""
+Class that creates the lenet architecture
+"""
+
+from ..util import *
+import tensorflow as tf
+import abc
+import six
+
+
+class BaseArchitecture(six.with_metaclass(abc.ABCMeta, object)):
+    """
+    Base class to create architectures using TensorFlow
+    """
+
+    def __init__(self, seed=10, use_gpu=False):
+        """
+        Base constructor
+        """
+        self.seed = seed
+        self.use_gpu = use_gpu
+        self.create_variables()
+
+    def create_variables(self):
+        """
+        Create the Tensor flow variables
+        """
+        raise NotImplementedError("Please implement this function in derived classes")
+
+    def create_graph(self, data):
+        """
+        Create the Tensor flow variables
+
+        **Parameters**
+            data: Tensorflow Placeholder
+
+        **Returns**
+            Network output
+        """
+
+        raise NotImplementedError("Please implement this function in derived classes")

bob/learn/tensorflow/network/Lenet.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
+# @date: Wed 11 May 2016 09:39:36 CEST 
+
+"""
+Class that creates the lenet architecture
+"""
+
+from ..util import *
+import tensorflow as tf
+from bob.learn.tensorflow.network.BaseArchitecture import BaseArchitecture
+
+
+class Lenet(BaseArchitecture):
+
+    def __init__(self,
+                 conv1_kernel_size=5,
+                 conv1_output=16,
+
+                 conv2_kernel_size=5,
+                 conv2_output=32,
+
+                 fc1_output=400,
+                 n_classes=10,
+
+                 seed=10, use_gpu = False):
+        """
+        Create all the necessary variables for this CNN
+
+        **Parameters**
+            conv1_kernel_size=5,
+            conv1_output=32,
+
+            conv2_kernel_size=5,
+            conv2_output=64,
+
+            fc1_output=400,
+            n_classes=10
+
+            seed = 10
+        """
+
+        self.conv1_kernel_size = conv1_kernel_size,
+        self.conv1_output = conv1_output,
+
+        self.conv2_kernel_size = conv2_kernel_size,
+        self.conv2_output = conv2_output,
+
+        self.fc1_output = fc1_output,
+        self.n_classes = n_classes,
+
+        super(Lenet, self).__init__(seed=seed, use_gpu=use_gpu)
+
+    def create_variables(self):
+
+        # First convolutional
+        self.W_conv1 = create_weight_variables([self.conv1_kernel_size, self.conv1_kernel_size, 1, self.conv1_output],
+                                               seed=self.seed, name="W_conv1", use_gpu=self.use_gpu)
+        self.b_conv1 = create_bias_variables([self.conv1_output], name="bias_conv1", use_gpu=self.use_gpu)
+
+        # Second convolutional
+        self.W_conv2 = create_weight_variables([self.conv2_kernel_size, self.conv2_kernel_size, self.conv1_output,
+                                                self.conv2_output], seed=self.seed, name="W_conv2",
+                                               use_gpu=self.use_gpu)
+        self.b_conv2 = create_bias_variables([self.conv2_output], name="bias_conv2", use_gpu=self.use_gpu)
+
+        # First fc
+        self.W_fc1 = create_weight_variables([(28 // 4) * (28 // 4) * self.conv2_output, self.fc1_output],
+                                             seed=self.seed, name="W_fc1", use_gpu=self.use_gpu)
+        self.b_fc1 = create_bias_variables([self.fc1_output], name="bias_fc1", use_gpu=self.use_gpu)
+
+        # Second FC fc
+        self.W_fc2 = create_weight_variables([self.fc1_output, self.n_classes], seed=self.seed,
+                                             name="W_fc2", use_gpu=self.use_gpu)
+        self.b_fc2 = create_bias_variables([self.n_classes], name="bias_fc2", use_gpu=self.use_gpu)
+
+        self.seed = self.seed
+
+    def create_graph(self, data):
+
+        # Creating the architecture
+        # First convolutional
+        with tf.name_scope('conv_1') as scope:
+            conv1 = create_conv2d(data, self.W_conv1)
+
+        with tf.name_scope('tanh_1') as scope:
+            tanh_1 = create_tanh(conv1, self.b_conv1)
+
+        # Pooling
+        with tf.name_scope('pool_1') as scope:
+            pool1 = create_max_pool(tanh_1)
+
+        # Second convolutional
+        with tf.name_scope('conv_2') as scope:
+            conv2 = create_conv2d(pool1, self.W_conv2)
+
+        with tf.name_scope('tanh_2') as scope:
+            tanh_2 = create_tanh(conv2, self.b_conv2)
+
+        # Pooling
+        with tf.name_scope('pool_2') as scope:
+            pool2 = create_max_pool(tanh_2)
+
+        #if train:
+            #pool2 = tf.nn.dropout(pool2, 0.5, seed=self.seed)
+
+        # Reshaping all the convolved images to 2D to feed the FC layers
+        # FC1
+        with tf.name_scope('fc_1') as scope:
+            pool_shape = pool2.get_shape().as_list()
+            reshape = tf.reshape(pool2, [pool_shape[0], pool_shape[1] * pool_shape[2] * pool_shape[3]])
+            fc1 = tf.nn.tanh(tf.matmul(reshape, self.W_fc1) + self.b_fc1)
+
+        # FC2
+        with tf.name_scope('fc_2') as scope:
+            fc2 = tf.matmul(fc1, self.W_fc2) + self.b_fc2
+
+        return fc2

bob/learn/tensorflow/network/__init__.py

+# see https://docs.python.org/3/library/pkgutil.html
+from pkgutil import extend_path
+__path__ = extend_path(__path__, __name__)
+
+

bob/learn/tensorflow/script/__init__.py

+# gets sphinx autodoc done right - don't remove it
+__all__ = [_ for _ in dir() if not _.startswith('_')]

bob/learn/tensorflow/script/copy of train_mnist_siamese.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
+# @date: Wed 11 May 2016 09:39:36 CEST 
+
+
+"""
+Simple script that trains MNIST with LENET using Tensor flow
+
+Usage:
+  train_mnist.py [--batch-size=<arg> --iterations=<arg> --validation-interval=<arg> --use-gpu]
+  train_mnist.py -h | --help
+Options:
+  -h --help     Show this screen.
+  --batch-size=<arg>  [default: 1]
+  --iterations=<arg>  [default: 30000]
+  --validation-interval=<arg>  [default: 100]
+"""
+
+from docopt import docopt
+import tensorflow as tf
+from .. import util
+from ..DataShuffler import *
+from ..lenet import Lenet
+from matplotlib.backends.backend_pdf import PdfPages
+import sys
+
+SEED = 10
+from ..DataShuffler import *
+
+
+def compute_euclidean_distance(x, y):
+    """
+    Computes the euclidean distance between two tensorflow variables
+    """
+
+    #d = tf.square(tf.sub(x, y))
+    #d = tf.sqrt(tf.reduce_sum(d)) # What about the axis ???
+    d = tf.sqrt(tf.reduce_sum(tf.square(tf.sub(x, y)), 1))
+    return d
+
+
+def compute_contrastive_loss(left_feature, right_feature, label, margin):
+
+    """
+    Compute the contrastive loss as in
+
+    http://yann.lecun.com/exdb/publis/pdf/hadsell-chopra-lecun-06.pdf
+
+    L = 0.5 * (Y) * D^2 + 0.5 * (1-Y) * {max(0, margin - D)}^2
+
+    **Parameters**
+     left_feature: First element of the pair
+     right_feature: Second element of the pair
+     label: Label of the pair (0 or 1)
+     margin: Contrastive margin
+
+    **Returns**
+     Return the loss operation
+
+    """
+
+    #label = tf.to_float(label)
+    #one = tf.constant(1.0)
+    #zero = tf.constant(0.0)
+    #half = tf.constant(0.5)
+    #m = tf.constant(margin)
+
+    #d = compute_euclidean_distance(left_feature, right_feature)
+    #first_part = tf.mul(label, tf.square(d))# (Y)*(d^2)
+
+    #max_part = tf.square(tf.maximum(m-d, zero))
+    #second_part = tf.mul(one-label, max_part)  # (1-Y) * max(margin - d, 0)
+    #loss = half * tf.reduce_sum(first_part + second_part)
+
+    #return loss
+
+
+    # Stack overflow "fix"
+
+    label = tf.to_float(label)
+    one = tf.constant(1.0)
+
+    d = compute_euclidean_distance(left_feature, right_feature)
+    first_part = tf.mul(one - label, tf.square(d))  # (Y-1)*(d^2)
+
+    max_part = tf.square(tf.maximum(margin - d, 0))
+    second_part = tf.mul(label, max_part)  # (Y) * max((margin - d)^2, 0)
+
+    loss = 0.5 * tf.reduce_mean(first_part + second_part)
+
+    return loss
+
+
+def main():
+    args = docopt(__doc__, version='Mnist training with TensorFlow')
+
+    BATCH_SIZE = int(args['--batch-size'])
+    ITERATIONS = int(args['--iterations'])
+    VALIDATION_TEST = int(args['--validation-interval'])
+    perc_train = 0.9
+    CONTRASTIVE_MARGIN = 0.1
+    USE_GPU = args['--use-gpu']
+
+
+    #print("Load data")
+    #sys.stdout.flush()
+
+    data, labels = util.load_mnist(data_dir="./src/bob.db.mnist/bob/db/mnist/")
+    data_shuffler = DataShuffler(data, labels, scale=True)
+
+    #print("A")
+    #sys.stdout.flush()
+
+    # Siamease place holders
+    train_left_data = tf.placeholder(tf.float32, shape=(BATCH_SIZE*2, 28, 28, 1), name="left")
+    train_right_data = tf.placeholder(tf.float32, shape=(BATCH_SIZE * 2, 28, 28, 1), name="right")
+    labels_data = tf.placeholder(tf.int32, shape=BATCH_SIZE*2)
+
+    validation_data = tf.placeholder(tf.float32, shape=(data_shuffler.validation_data.shape[0], 28, 28, 1))
+
+    #print("B")
+    #sys.stdout.flush()
+
+
+    # Creating the architecture
+    lenet_architecture = Lenet(seed=SEED, use_gpu=USE_GPU)
+    lenet_train_left = lenet_architecture.create_lenet(train_left_data)
+    lenet_train_right = lenet_architecture.create_lenet(train_right_data)
+    lenet_validation = lenet_architecture.create_lenet(validation_data, train=False)
+
+    #print("C")
+    #sys.stdout.flush()
+
+
+    # Defining the constrastive loss
+    #left_output = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(lenet_train_left, labels_data))
+    #right_output = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(lenet_train_right, labels_data))
+    #loss = compute_contrastive_loss(tf.nn.softmax(lenet_train_left), tf.nn.softmax(lenet_train_right), labels_data, CONTRASTIVE_MARGIN)
+    loss = compute_contrastive_loss(lenet_train_left, lenet_train_right, labels_data, CONTRASTIVE_MARGIN)
+
+    #print("D")
+    #sys.stdout.flush()