Scratch

parents
*~
*.swp
*.pyc
bin
eggs
parts
.installed.cfg
.mr.developer.cfg
*.egg-info
src
develop-eggs
sphinx
dist
This diff is collapsed.
include README.rst bootstrap-buildout.py buildout.cfg COPYING version.txt requirements.txt
recursive-include doc *.py *.rst
.. vim: set fileencoding=utf-8 :
.. Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
.. Thu 30 Jan 08:46:53 2014 CET
======================
Tensorflow Examples
======================
This example package implements simple CNN trainings with tensorflow using the MNIST database.
It is implemented two CNNs:
* Lenet using softmax
* Siamease network (Lenet as a base)
Installation
------------
With tensorflow installed in your environment just run::
$ python bootstrap.py
$ ./bin/buildout
Documentation
-------------
For the time being has just one script that trains the MNIST using the Lenet architecture using the softmax
cross entropy loss. Run the command bellow to have some help::
$ ./bin/train_mnist.py --help # Lenet using softmax
$ ./bin/train_mnist_siamese.py --help # Lenet using siamese net
It is just the classical stuff.
I haven't added any regularization (L2, L1, dropout, etc )
# see https://docs.python.org/3/library/pkgutil.html
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
\ No newline at end of file
# see https://docs.python.org/3/library/pkgutil.html
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
\ No newline at end of file
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
# @date: Wed 11 May 2016 09:39:36 CEST
import numpy
def scale_mean_norm(data, scale=0.00390625):
mean = numpy.mean(data)
data = (data - mean) * scale
return data, mean
class DataShuffler(object):
def __init__(self, perc_train=0.9, scale=True):
"""
Some base functions for neural networks
**Parameters**
data:
"""
self.perc_train = perc_train
self.scale = True
self.scale_value = 0.00390625
def start_shuffler(self, data, labels):
"""
Some base functions for neural networks
**Parameters**
data:
"""
scale_value = 0.00390625
total_samples = data.shape[0]
indexes = numpy.array(range(total_samples))
numpy.random.shuffle(indexes)
# Spliting train and validation
train_samples = int(round(total_samples * self.perc_train))
validation_samples = total_samples - train_samples
data = numpy.reshape(data, (data.shape[0], 28, 28, 1))
self.train_data = data[indexes[0:train_samples], :, :, :]
self.train_labels = labels[indexes[0:train_samples]]
self.validation_data = data[indexes[train_samples:train_samples + validation_samples], :, :, :]
self.validation_labels = labels[indexes[train_samples:train_samples + validation_samples]]
self.total_labels = 10
if self.scale:
# data = scale_minmax_norm(data,lower_bound = -1, upper_bound = 1)
self.train_data, self.mean = scale_mean_norm(self.train_data)
self.validation_data = (self.validation_data - self.mean) * scale_value
def get_batch(self, n_samples, train_dataset=True):
if train_dataset:
data = self.train_data
label = self.train_labels
else:
data = self.validation_data
label = self.validation_labels
# Shuffling samples
indexes = numpy.array(range(data.shape[0]))
numpy.random.shuffle(indexes)
selected_data = data[indexes[0:n_samples], :, :, :]
selected_labels = label[indexes[0:n_samples]]
return selected_data.astype("float32"), selected_labels
def get_pair(self, n_pair=1, is_target_set_train=True, zero_one_labels=True):
"""
Get a random pair of samples
**Parameters**
is_target_set_train: Defining the target set to get the batch
**Return**
"""
def get_genuine_or_not(input_data, input_labels, genuine=True):
if genuine:
# TODO: THIS KEY SELECTION NEEDS TO BE MORE EFFICIENT
# Getting a client
index = numpy.random.randint(self.total_labels)
# Getting the indexes of the data from a particular client
indexes = numpy.where(input_labels == index)[0]
numpy.random.shuffle(indexes)
# Picking a pair
data = input_data[indexes[0], :, :, :]
data_p = input_data[indexes[1], :, :, :]
else:
# Picking a pair from different clients
index = numpy.random.choice(self.total_labels, 2, replace=False)
# Getting the indexes of the two clients
indexes = numpy.where(input_labels == index[0])[0]
indexes_p = numpy.where(input_labels == index[1])[0]
numpy.random.shuffle(indexes)
numpy.random.shuffle(indexes_p)
# Picking a pair
data = input_data[indexes[0], :, :, :]
data_p = input_data[indexes_p[0], :, :, :]
return data, data_p
if is_target_set_train:
target_data = self.train_data
target_labels = self.train_labels
else:
target_data = self.validation_data
target_labels = self.validation_labels
total_data = n_pair * 2
c = target_data.shape[3]
w = target_data.shape[1]
h = target_data.shape[2]
data = numpy.zeros(shape=(total_data, w, h, c), dtype='float32')
data_p = numpy.zeros(shape=(total_data, w, h, c), dtype='float32')
labels_siamese = numpy.zeros(shape=total_data, dtype='float32')
genuine = True
for i in range(total_data):
data[i, :, :, :], data_p[i, :, :, :] = get_genuine_or_not(target_data, target_labels, genuine=genuine)
if zero_one_labels:
labels_siamese[i] = not genuine
else:
labels_siamese[i] = -1 if genuine else +1
genuine = not genuine
return data, data_p, labels_siamese
def get_triplet(self, n_labels, n_triplets=1, is_target_set_train=True):
"""
Get a triplet
**Parameters**
is_target_set_train: Defining the target set to get the batch
**Return**
"""
def get_one_triplet(input_data, input_labels):
# Getting a pair of clients
index = numpy.random.choice(n_labels, 2, replace=False)
label_positive = index[0]
label_negative = index[1]
# Getting the indexes of the data from a particular client
indexes = numpy.where(input_labels == index[0])[0]
numpy.random.shuffle(indexes)
# Picking a positive pair
data_anchor = input_data[indexes[0], :, :, :]
data_positive = input_data[indexes[1], :, :, :]
# Picking a negative sample
indexes = numpy.where(input_labels == index[1])[0]
numpy.random.shuffle(indexes)
data_negative = input_data[indexes[0], :, :, :]
return data_anchor, data_positive, data_negative, label_positive, label_positive, label_negative
if is_target_set_train:
target_data = self.train_data
target_labels = self.train_labels
else:
target_data = self.validation_data
target_labels = self.validation_labels
c = target_data.shape[3]
w = target_data.shape[1]
h = target_data.shape[2]
data_a = numpy.zeros(shape=(n_triplets, w, h, c), dtype='float32')
data_p = numpy.zeros(shape=(n_triplets, w, h, c), dtype='float32')
data_n = numpy.zeros(shape=(n_triplets, w, h, c), dtype='float32')
labels_a = numpy.zeros(shape=n_triplets, dtype='float32')
labels_p = numpy.zeros(shape=n_triplets, dtype='float32')
labels_n = numpy.zeros(shape=n_triplets, dtype='float32')
for i in range(n_triplets):
data_a[i, :, :, :], data_p[i, :, :, :], data_n[i, :, :, :], \
labels_a[i], labels_p[i], labels_n[i] = \
get_one_triplet(target_data, target_labels)
return data_a, data_p, data_n, labels_a, labels_p, labels_n
# see https://docs.python.org/3/library/pkgutil.html
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
from DataShuffler import *
# see https://docs.python.org/3/library/pkgutil.html
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
from DataShuffler import *
# see https://docs.python.org/3/library/pkgutil.html
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
from DataShuffler import *
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
# @date: Tue 09 Aug 2016 16:38 CEST
import logging
logger = logging.getLogger("bob.learn.tensorflow")
class BaseLoss(object):
"""
Base loss function.
One exam
"""
def __init__(self, loss):
self.loss = loss
def __call__(self):
return self.loss
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
# @date: Wed 10 Aug 2016 16:38 CEST
import logging
logger = logging.getLogger("bob.learn.tensorflow")
import tensorflow as tf
from bob.learn.tensorflow.loss.Baseloss as Baseloss
class ContrastiveLoss(BaseLoss):
"""
Compute the contrastive loss as in
http://yann.lecun.com/exdb/publis/pdf/hadsell-chopra-lecun-06.pdf
L = 0.5 * (Y) * D^2 + 0.5 * (1-Y) * {max(0, margin - D)}^2
OR MAYBE THAT
L = 0.5 * (1-Y) * D^2 + 0.5 * (Y) * {max(0, margin - D)}^2
**Parameters**
left_feature: First element of the pair
right_feature: Second element of the pair
label: Label of the pair (0 or 1)
margin: Contrastive margin
"""
def __init__(self, label, left_feature, right_feature, margin):
with tf.name_scope("contrastive_loss"):
label = tf.to_float(label)
one = tf.constant(1.0)
d = bob.learn.tensorflow.util.compute_euclidean_distance(left_feature, right_feature)
# first_part = tf.mul(one - label, tf.square(d)) # (Y-1)*(d^2)
# first_part = tf.mul(label, tf.square(d)) # (Y-1)*(d^2)
between_class = tf.exp(tf.mul(one - label, tf.square(d))) # (1-Y)*(d^2)
max_part = tf.square(tf.maximum(margin - d, 0))
within_class = tf.mul(label, max_part) # (Y) * max((margin - d)^2, 0)
# second_part = tf.mul(one-label, max_part) # (Y) * max((margin - d)^2, 0)
self.loss = 0.5 * tf.reduce_mean(within_class + between_class)
self.within_class = tf.reduce_mean(within_class), tf.reduce_mean(between_class)
# see https://docs.python.org/3/library/pkgutil.html
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
from .BaseLoss import BaseLoss
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
# @date: Wed 11 May 2016 09:39:36 CEST
"""
Class that creates the lenet architecture
"""
from ..util import *
import tensorflow as tf
import abc
import six
class BaseArchitecture(six.with_metaclass(abc.ABCMeta, object)):
"""
Base class to create architectures using TensorFlow
"""
def __init__(self, seed=10, use_gpu=False):
"""
Base constructor
"""
self.seed = seed
self.use_gpu = use_gpu
self.create_variables()
def create_variables(self):
"""
Create the Tensor flow variables
"""
raise NotImplementedError("Please implement this function in derived classes")
def create_graph(self, data):
"""
Create the Tensor flow variables
**Parameters**
data: Tensorflow Placeholder
**Returns**
Network output
"""
raise NotImplementedError("Please implement this function in derived classes")
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
# @date: Wed 11 May 2016 09:39:36 CEST
"""
Class that creates the lenet architecture
"""
from ..util import *
import tensorflow as tf
from bob.learn.tensorflow.network.BaseArchitecture import BaseArchitecture
class Lenet(BaseArchitecture):
def __init__(self,
conv1_kernel_size=5,
conv1_output=16,
conv2_kernel_size=5,
conv2_output=32,
fc1_output=400,
n_classes=10,
seed=10, use_gpu = False):
"""
Create all the necessary variables for this CNN
**Parameters**
conv1_kernel_size=5,
conv1_output=32,
conv2_kernel_size=5,
conv2_output=64,
fc1_output=400,
n_classes=10
seed = 10
"""
self.conv1_kernel_size = conv1_kernel_size,
self.conv1_output = conv1_output,
self.conv2_kernel_size = conv2_kernel_size,
self.conv2_output = conv2_output,
self.fc1_output = fc1_output,
self.n_classes = n_classes,
super(Lenet, self).__init__(seed=seed, use_gpu=use_gpu)
def create_variables(self):
# First convolutional
self.W_conv1 = create_weight_variables([self.conv1_kernel_size, self.conv1_kernel_size, 1, self.conv1_output],
seed=self.seed, name="W_conv1", use_gpu=self.use_gpu)
self.b_conv1 = create_bias_variables([self.conv1_output], name="bias_conv1", use_gpu=self.use_gpu)
# Second convolutional
self.W_conv2 = create_weight_variables([self.conv2_kernel_size, self.conv2_kernel_size, self.conv1_output,
self.conv2_output], seed=self.seed, name="W_conv2",
use_gpu=self.use_gpu)
self.b_conv2 = create_bias_variables([self.conv2_output], name="bias_conv2", use_gpu=self.use_gpu)
# First fc
self.W_fc1 = create_weight_variables([(28 // 4) * (28 // 4) * self.conv2_output, self.fc1_output],
seed=self.seed, name="W_fc1", use_gpu=self.use_gpu)
self.b_fc1 = create_bias_variables([self.fc1_output], name="bias_fc1", use_gpu=self.use_gpu)
# Second FC fc
self.W_fc2 = create_weight_variables([self.fc1_output, self.n_classes], seed=self.seed,
name="W_fc2", use_gpu=self.use_gpu)
self.b_fc2 = create_bias_variables([self.n_classes], name="bias_fc2", use_gpu=self.use_gpu)
self.seed = self.seed
def create_graph(self, data):
# Creating the architecture
# First convolutional
with tf.name_scope('conv_1') as scope:
conv1 = create_conv2d(data, self.W_conv1)
with tf.name_scope('tanh_1') as scope:
tanh_1 = create_tanh(conv1, self.b_conv1)
# Pooling
with tf.name_scope('pool_1') as scope:
pool1 = create_max_pool(tanh_1)
# Second convolutional
with tf.name_scope('conv_2') as scope:
conv2 = create_conv2d(pool1, self.W_conv2)
with tf.name_scope('tanh_2') as scope:
tanh_2 = create_tanh(conv2, self.b_conv2)
# Pooling
with tf.name_scope('pool_2') as scope:
pool2 = create_max_pool(tanh_2)
#if train:
#pool2 = tf.nn.dropout(pool2, 0.5, seed=self.seed)
# Reshaping all the convolved images to 2D to feed the FC layers
# FC1
with tf.name_scope('fc_1') as scope:
pool_shape = pool2.get_shape().as_list()
reshape = tf.reshape(pool2, [pool_shape[0], pool_shape[1] * pool_shape[2] * pool_shape[3]])
fc1 = tf.nn.tanh(tf.matmul(reshape, self.W_fc1) + self.b_fc1)
# FC2
with tf.name_scope('fc_2') as scope:
fc2 = tf.matmul(fc1, self.W_fc2) + self.b_fc2
return fc2
# see https://docs.python.org/3/library/pkgutil.html
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
# gets sphinx autodoc done right - don't remove it
__all__ = [_ for _ in dir() if not _.startswith('_')]
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
# @date: Wed 11 May 2016 09:39:36 CEST
"""
Simple script that trains MNIST with LENET using Tensor flow
Usage:
train_mnist.py [--batch-size=<arg> --iterations=<arg> --validation-interval=<arg> --use-gpu]
train_mnist.py -h | --help
Options:
-h --help Show this screen.
--batch-size=<arg> [default: 1]
--iterations=<arg> [default: 30000]
--validation-interval=<arg> [default: 100]
"""
from docopt import docopt
import tensorflow as tf
from .. import util
from ..DataShuffler import *
from ..lenet import Lenet
from matplotlib.backends.backend_pdf import PdfPages
import sys
SEED = 10
from ..DataShuffler import *
def compute_euclidean_distance(x, y):
"""
Computes the euclidean distance between two tensorflow variables
"""
#d = tf.square(tf.sub(x, y))
#d = tf.sqrt(tf.reduce_sum(d)) # What about the axis ???
d = tf.sqrt(tf.reduce_sum(tf.square(tf.sub(x, y)), 1))
return d
def compute_contrastive_loss(left_feature, right_feature, label, margin):
"""
Compute the contrastive loss as in
http://yann.lecun.com/exdb/publis/pdf/hadsell-chopra-lecun-06.pdf
L = 0.5 * (Y) * D^2 + 0.5 * (1-Y) * {max(0, margin - D)}^2
**Parameters**
left_feature: First element of the pair
right_feature: Second element of the pair
label: Label of the pair (0 or 1)
margin: Contrastive margin
**Returns**
Return the loss operation
"""
#label = tf.to_float(label)
#one = tf.constant(1.0)
#zero = tf.constant(0.0)
#half = tf.constant(0.5)
#m = tf.constant(margin)
#d = compute_euclidean_distance(left_feature, right_feature)
#first_part = tf.mul(label, tf.square(d))# (Y)*(d^2)
#max_part = tf.square(tf.maximum(m-d, zero))
#second_part = tf.mul(one-label, max_part) # (1-Y) * max(margin - d, 0)
#loss = half * tf.reduce_sum(first_part + second_part)
#return loss
# Stack overflow "fix"
label = tf.to_float(label)
one = tf.constant(1.0)
d = compute_euclidean_distance(left_feature, right_feature)
first_part = tf.mul(one - label, tf.square(d)) # (Y-1)*(d^2)
max_part = tf.square(tf.maximum(margin - d, 0))
second_part = tf.mul(label, max_part) # (Y) * max((margin - d)^2, 0)
loss = 0.5 * tf.reduce_mean(first_part + second_part)
return loss
def main():
args = docopt(__doc__, version='Mnist training with TensorFlow')
BATCH_SIZE = int(args['--batch-size'])
ITERATIONS = int(args['--iterations'])
VALIDATION_TEST = int(args['--validation-interval'])
perc_train = 0.9
CONTRASTIVE_MARGIN = 0.1
USE_GPU = args['--use-gpu']
#print("Load data")
#sys.stdout.flush()
data, labels = util.load_mnist(data_dir="./src/bob.db.mnist/bob/db/mnist/")
data_shuffler = DataShuffler(data, labels, scale=True)
#print("A")
#sys.stdout.flush()
# Siamease place holders
train_left_data = tf.placeholder(tf.float32, shape=(BATCH_SIZE*2, 28, 28, 1), name="left")
train_right_data = tf.placeholder(tf.float32, shape=(BATCH_SIZE * 2, 28, 28, 1), name="right")
labels_data = tf.placeholder(tf.int32, shape=BATCH_SIZE*2)
validation_data = tf.placeholder(tf.float32, shape=(data_shuffler.validation_data.shape[0], 28, 28, 1))
#print("B")
#sys.stdout.flush()
# Creating the architecture
lenet_architecture = Lenet(seed=SEED, use_gpu=USE_GPU)
lenet_train_left = lenet_architecture.create_lenet(train_left_data)
lenet_train_right = lenet_architecture.create_lenet(train_right_data)
lenet_validation = lenet_architecture.create_lenet(validation_data, train=False)
#print("C")
#sys.stdout.flush()
# Defining the constrastive loss
#left_output = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(lenet_train_left, labels_data))
#right_output = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(lenet_train_right, labels_data))
#loss = compute_contrastive_loss(tf.nn.softmax(lenet_train_left), tf.nn.softmax(lenet_train_right), labels_data, CONTRASTIVE_MARGIN)
loss = compute_contrastive_loss(lenet_train_left, lenet_train_right, labels_data, CONTRASTIVE_MARGIN)
#print("D")
#sys.stdout.flush()
#regularizer = (tf.nn.l2_loss(lenet_architecture.W_fc1) + tf.nn.l2_loss(lenet_architecture.b_fc1) +
# tf.nn.l2_loss(lenet_architecture.W_fc2) + tf.nn.l2_loss(lenet_architecture.b_fc2))
#loss += 5e-4 * regularizer
# Defining training parameters
batch = tf.Variable(0)
learning_rate = tf.train.exponential_decay(