Still dumping code.

parent 50eb11a9
...@@ -4,7 +4,7 @@ ...@@ -4,7 +4,7 @@
# @date: Wed 11 May 2016 09:39:36 CEST # @date: Wed 11 May 2016 09:39:36 CEST
import numpy import numpy
import tensorflow as tf
def scale_mean_norm(data, scale=0.00390625): def scale_mean_norm(data, scale=0.00390625):
mean = numpy.mean(data) mean = numpy.mean(data)
...@@ -14,7 +14,7 @@ def scale_mean_norm(data, scale=0.00390625): ...@@ -14,7 +14,7 @@ def scale_mean_norm(data, scale=0.00390625):
class DataShuffler(object): class DataShuffler(object):
def __init__(self, perc_train=0.9, scale=True): def __init__(self, data, labels, perc_train=0.9, scale=True, train_batch_size=1, validation_batch_size=1):
""" """
Some base functions for neural networks Some base functions for neural networks
...@@ -25,38 +25,51 @@ class DataShuffler(object): ...@@ -25,38 +25,51 @@ class DataShuffler(object):
self.perc_train = perc_train self.perc_train = perc_train
self.scale = True self.scale = True
self.scale_value = 0.00390625 self.scale_value = 0.00390625
self.train_batch_size = train_batch_size
self.validation_batch_size = validation_batch_size
self.data = data
self.labels =labels
self.n_samples = self.data.shape[0]
self.width = self.data.shape[1]
self.height = self.data.shape[2]
self.channels = self.data.shape[3]
self.start_shuffler()
def get_placeholders(self, name=""):
data = tf.placeholder(tf.float32, shape=(self.train_batch_size, self.width,
self.height, self.channels), name=name)
labels = tf.placeholder(tf.int64, shape=self.train_batch_size)
def start_shuffler(self, data, labels): return data, labels
def start_shuffler(self):
""" """
Some base functions for neural networks Some base functions for neural networks
**Parameters** **Parameters**
data: data:
""" """
scale_value = 0.00390625
total_samples = data.shape[0]
indexes = numpy.array(range(total_samples)) indexes = numpy.array(range(self.n_samples))
numpy.random.shuffle(indexes) numpy.random.shuffle(indexes)
# Spliting train and validation # Spliting train and validation
train_samples = int(round(total_samples * self.perc_train)) train_samples = int(round(self.n_samples * self.perc_train))
validation_samples = total_samples - train_samples validation_samples = self.n_samples - train_samples
data = numpy.reshape(data, (data.shape[0], 28, 28, 1))
self.train_data = data[indexes[0:train_samples], :, :, :] self.train_data = self.data[indexes[0:train_samples], :, :, :]
self.train_labels = labels[indexes[0:train_samples]] self.train_labels = self.labels[indexes[0:train_samples]]
self.validation_data = data[indexes[train_samples:train_samples + validation_samples], :, :, :] self.validation_data = self.data[indexes[train_samples:train_samples + validation_samples], :, :, :]
self.validation_labels = labels[indexes[train_samples:train_samples + validation_samples]] self.validation_labels = self.labels[indexes[train_samples:train_samples + validation_samples]]
self.total_labels = 10 self.total_labels = 10
if self.scale: if self.scale:
# data = scale_minmax_norm(data,lower_bound = -1, upper_bound = 1) # data = scale_minmax_norm(data,lower_bound = -1, upper_bound = 1)
self.train_data, self.mean = scale_mean_norm(self.train_data) self.train_data, self.mean = scale_mean_norm(self.train_data)
self.validation_data = (self.validation_data - self.mean) * scale_value self.validation_data = (self.validation_data - self.mean) * self.scale_value
def get_batch(self, n_samples, train_dataset=True): def get_batch(self, n_samples, train_dataset=True):
......
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch> # @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
# @date: Wed 11 May 2016 17:38 CEST # @date: Wed 11 May 2016 17:38 CEST
import tensoflow as tf import tensorflow as tf
from bob.learn.tensorflow.util import * from bob.learn.tensorflow.util import *
from .Layer import Layer from .Layer import Layer
...@@ -14,7 +14,7 @@ class Conv2D(Layer): ...@@ -14,7 +14,7 @@ class Conv2D(Layer):
2D Convolution 2D Convolution
""" """
def __init__(self, input, activation=None, def __init__(self, name, activation=None,
kernel_size=3, kernel_size=3,
filters=8, filters=8,
initialization='xavier', initialization='xavier',
...@@ -33,20 +33,31 @@ class Conv2D(Layer): ...@@ -33,20 +33,31 @@ class Conv2D(Layer):
use_gpu: Store data in the GPU use_gpu: Store data in the GPU
seed: Seed for the Random number generation seed: Seed for the Random number generation
""" """
super(Conv2D, self).__init__(input, initialization='xavier', use_gpu=False, seed=10) super(Conv2D, self).__init__(name, activation=activation, initialization='xavier',
self.activation = activation use_gpu=use_gpu, seed=seed)
self.kernel_size = kernel_size
self.W = create_weight_variables([kernel_size, kernel_size, 1, filters], self.filters = filters
seed=seed, name="conv", use_gpu=use_gpu) self.initialization = initialization
self.W = None
if activation is not None: self.b = None
self.b = create_bias_variables([filters], name="bias", use_gpu=self.use_gpu)
def create_variables(self, input):
self.input = input
if self.W is None:
self.W = create_weight_variables([self.kernel_size, self.kernel_size, 1, self.filters],
seed=self.seed, name=str(self.name), use_gpu=self.use_gpu)
if self.activation is not None:
self.b = create_bias_variables([self.filters], name=str(self.name) + "bias", use_gpu=self.use_gpu)
def get_graph(self): def get_graph(self):
with tf.name_scope('conv'): with tf.name_scope(str(self.name)):
conv = tf.nn.conv2d(self.input, self.W, strides=[1, 1, 1, 1], padding='SAME') conv2d = tf.nn.conv2d(self.input, self.W, strides=[1, 1, 1, 1], padding='SAME')
with tf.name_scope('activation'): if self.activation is not None:
non_linearity = tf.nn.tanh(tf.nn.bias_add(conv, self.b)) with tf.name_scope(str(self.name) + 'activation'):
non_linear_conv2d = tf.nn.tanh(tf.nn.bias_add(conv2d, self.b))
self.output = non_linear_conv2d
else:
self.output = conv2d
return non_linearity return self.output
...@@ -3,9 +3,10 @@ ...@@ -3,9 +3,10 @@
# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch> # @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
# @date: Wed 11 May 2016 17:38 CEST # @date: Wed 11 May 2016 17:38 CEST
import tensoflow as tf import tensorflow as tf
from bob.learn.tensorflow.util import * from bob.learn.tensorflow.util import *
from .Layer import Layer from .Layer import Layer
from operator import mul
class FullyConnected(Layer): class FullyConnected(Layer):
...@@ -14,7 +15,7 @@ class FullyConnected(Layer): ...@@ -14,7 +15,7 @@ class FullyConnected(Layer):
2D Convolution 2D Convolution
""" """
def __init__(self, input, activation=None, def __init__(self, name, output_dim, activation=None,
initialization='xavier', initialization='xavier',
use_gpu=False, use_gpu=False,
seed=10 seed=10
...@@ -29,23 +30,36 @@ class FullyConnected(Layer): ...@@ -29,23 +30,36 @@ class FullyConnected(Layer):
use_gpu: Store data in the GPU use_gpu: Store data in the GPU
seed: Seed for the Random number generation seed: Seed for the Random number generation
""" """
super(FullyConnected, self).__init__(input, initialization='xavier', use_gpu=False, seed=10) super(FullyConnected, self).__init__(name, activation=activation,
self.activation = activation initialization=initialization, use_gpu=use_gpu, seed=seed)
self.output_dim = output_dim
self.W = None
self.b = None
def create_variables(self, input):
self.input = input
if self.W is None:
input_dim = reduce(mul, self.input.get_shape().as_list())
if len(input.get_shape())==4: self.W = create_weight_variables([input_dim, self.output_dim],
self.W = create_weight_variables([kernel_size, kernel_size, 1, filters], seed=self.seed, name=str(self.name), use_gpu=self.use_gpu)
seed=seed, name="conv", use_gpu=use_gpu) if self.activation is not None:
self.b = create_bias_variables([self.output_dim], name=str(self.name)+"_bias", use_gpu=self.use_gpu)
if activation is not None:
self.b = create_bias_variables([filters], name="bias", use_gpu=self.use_gpu)
def get_graph(self): def get_graph(self):
with tf.name_scope('fc'): with tf.name_scope('fc'):
conv = tf.nn.conv2d(self.input, self.W, strides=[1, 1, 1, 1], padding='SAME')
if len(self.input.get_shape()) == 4:
shape = self.input.get_shape().as_list()
fc = tf.reshape(self.input, [shape[0], shape[1] * shape[2] * shape[3]])
if self.activation is not None: if self.activation is not None:
with tf.name_scope('activation'): with tf.name_scope('activation'):
non_linearity = tf.nn.tanh(tf.nn.bias_add(conv, self.b)) non_linear_fc = tf.nn.tanh(tf.matmul(fc, self.W) + self.b)
self.output = non_linear_fc
else:
self.output = fc
return self.output
return non_linearity
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch> # @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
# @date: Wed 11 May 2016 17:38 CEST # @date: Wed 11 May 2016 17:38 CEST
import tensoflow as tf import tensorflow as tf
class Layer(object): class Layer(object):
...@@ -12,7 +12,7 @@ class Layer(object): ...@@ -12,7 +12,7 @@ class Layer(object):
Layer base class Layer base class
""" """
def __init__(self, input, initialization='xavier', use_gpu=False, seed=10): def __init__(self, name, activation=None, initialization='xavier', use_gpu=False, seed=10):
""" """
Base constructor Base constructor
...@@ -20,10 +20,23 @@ class Layer(object): ...@@ -20,10 +20,23 @@ class Layer(object):
input: Layer input input: Layer input
""" """
self.input = input self.name = name
self.initialization = initialization self.initialization = initialization
self.use_gpu = use_gpu self.use_gpu = use_gpu
self.seed = seed self.seed = seed
self.input = None
self.activation = None
self.output = None
def create_variables(self, input):
NotImplementedError("Please implement this function in derived classes")
def get_graph(self): def get_graph(self):
NotImplementedError("Please implement this function in derived classes") NotImplementedError("Please implement this function in derived classes")
def get_shape(self):
if self.output is None:
NotImplementedError("This class was not implemented properly")
else:
return self.output.get_shape()
...@@ -3,18 +3,25 @@ ...@@ -3,18 +3,25 @@
# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch> # @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
# @date: Wed 11 May 2016 17:38 CEST # @date: Wed 11 May 2016 17:38 CEST
import tensoflow as tf import tensorflow as tf
from bob.learn.tensorflow.util import * from bob.learn.tensorflow.util import *
from .Layer import Layer from .Layer import Layer
class MaxPooling(Layer): class MaxPooling(Layer):
def __init__(self, input, use_gpu=False): def __init__(self, name, use_gpu=False):
""" """
Constructor Constructor
""" """
super(MaxPooling, self).__init__(input, use_gpu=False) super(MaxPooling, self).__init__(name, use_gpu=False)
def create_variables(self, input):
self.input = input
return
def get_graph(self): def get_graph(self):
tf.nn.max_pool(self.input, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME') with tf.name_scope(str(self.name)):
self.output = tf.nn.max_pool(self.input, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
return self.output
...@@ -2,6 +2,10 @@ ...@@ -2,6 +2,10 @@
from pkgutil import extend_path from pkgutil import extend_path
__path__ = extend_path(__path__, __name__) __path__ = extend_path(__path__, __name__)
from DataShuffler import * #from DataShuffler import *
from .Layer import Layer
from .Conv2D import Conv2D
from .FullyConnected import FullyConnected
from .MaxPooling import MaxPooling
...@@ -6,7 +6,7 @@ ...@@ -6,7 +6,7 @@
import logging import logging
logger = logging.getLogger("bob.learn.tensorflow") logger = logging.getLogger("bob.learn.tensorflow")
import tensorflow as tf import tensorflow as tf
from bob.learn.tensorflow.loss.Baseloss as Baseloss from .Baseloss import Baseloss
class ContrastiveLoss(BaseLoss): class ContrastiveLoss(BaseLoss):
......
...@@ -9,10 +9,11 @@ Class that creates the lenet architecture ...@@ -9,10 +9,11 @@ Class that creates the lenet architecture
from ..util import * from ..util import *
import tensorflow as tf import tensorflow as tf
from bob.learn.tensorflow.network.BaseArchitecture import BaseArchitecture from .SequenceNetwork import SequenceNetwork
from ..layers import Conv2D, FullyConnected, MaxPooling
class Lenet(BaseArchitecture): class Lenet(SequenceNetwork):
def __init__(self, def __init__(self,
conv1_kernel_size=5, conv1_kernel_size=5,
...@@ -40,80 +41,11 @@ class Lenet(BaseArchitecture): ...@@ -40,80 +41,11 @@ class Lenet(BaseArchitecture):
seed = 10 seed = 10
""" """
super(Lenet, self).__init__()
self.conv1_kernel_size = conv1_kernel_size,
self.conv1_output = conv1_output, self.add(Conv2D(name="conv1", kernel_size=conv1_kernel_size, filters=conv1_output))
self.add(MaxPooling(name="pooling1"))
self.conv2_kernel_size = conv2_kernel_size, self.add(Conv2D(name="conv2", kernel_size=conv2_kernel_size, filters=conv2_output))
self.conv2_output = conv2_output, self.add(MaxPooling(name="pooling2"))
self.add(FullyConnected(name="fc1", output_dim=fc1_output, activation=True))
self.fc1_output = fc1_output, self.add(FullyConnected(name="fc1", output_dim=n_classes, activation=False))
self.n_classes = n_classes,
super(Lenet, self).__init__(seed=seed, use_gpu=use_gpu)
def create_variables(self):
# First convolutional
self.W_conv1 = create_weight_variables([self.conv1_kernel_size, self.conv1_kernel_size, 1, self.conv1_output],
seed=self.seed, name="W_conv1", use_gpu=self.use_gpu)
self.b_conv1 = create_bias_variables([self.conv1_output], name="bias_conv1", use_gpu=self.use_gpu)
# Second convolutional
self.W_conv2 = create_weight_variables([self.conv2_kernel_size, self.conv2_kernel_size, self.conv1_output,
self.conv2_output], seed=self.seed, name="W_conv2",
use_gpu=self.use_gpu)
self.b_conv2 = create_bias_variables([self.conv2_output], name="bias_conv2", use_gpu=self.use_gpu)
# First fc
self.W_fc1 = create_weight_variables([(28 // 4) * (28 // 4) * self.conv2_output, self.fc1_output],
seed=self.seed, name="W_fc1", use_gpu=self.use_gpu)
self.b_fc1 = create_bias_variables([self.fc1_output], name="bias_fc1", use_gpu=self.use_gpu)
# Second FC fc
self.W_fc2 = create_weight_variables([self.fc1_output, self.n_classes], seed=self.seed,
name="W_fc2", use_gpu=self.use_gpu)
self.b_fc2 = create_bias_variables([self.n_classes], name="bias_fc2", use_gpu=self.use_gpu)
self.seed = self.seed
def create_graph(self, data):
# Creating the architecture
# First convolutional
with tf.name_scope('conv_1') as scope:
conv1 = create_conv2d(data, self.W_conv1)
with tf.name_scope('tanh_1') as scope:
tanh_1 = create_tanh(conv1, self.b_conv1)
# Pooling
with tf.name_scope('pool_1') as scope:
pool1 = create_max_pool(tanh_1)
# Second convolutional
with tf.name_scope('conv_2') as scope:
conv2 = create_conv2d(pool1, self.W_conv2)
with tf.name_scope('tanh_2') as scope:
tanh_2 = create_tanh(conv2, self.b_conv2)
# Pooling
with tf.name_scope('pool_2') as scope:
pool2 = create_max_pool(tanh_2)
#if train:
#pool2 = tf.nn.dropout(pool2, 0.5, seed=self.seed)
# Reshaping all the convolved images to 2D to feed the FC layers
# FC1
with tf.name_scope('fc_1') as scope:
pool_shape = pool2.get_shape().as_list()
reshape = tf.reshape(pool2, [pool_shape[0], pool_shape[1] * pool_shape[2] * pool_shape[3]])
fc1 = tf.nn.tanh(tf.matmul(reshape, self.W_fc1) + self.b_fc1)
# FC2
with tf.name_scope('fc_2') as scope:
fc2 = tf.matmul(fc1, self.W_fc2) + self.b_fc2
return fc2
#!/usr/bin/env python #!/usr/bin/env python
# vim: set fileencoding=utf-8 : # vim: set fileencoding=utf-8 :
# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch> # @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
# @date: Wed 11 May 2016 09:39:36 CEST # @date: Thu 11 Aug 2016 09:39:36 CEST
""" """
Class that creates the lenet architecture Class that creates the lenet architecture
...@@ -12,35 +12,44 @@ import tensorflow as tf ...@@ -12,35 +12,44 @@ import tensorflow as tf
import abc import abc
import six import six
from collections import OrderedDict
from bob.learn.tensorflow.layers import *
class BaseArchitecture(six.with_metaclass(abc.ABCMeta, object)):
class SequenceNetwork(six.with_metaclass(abc.ABCMeta, object)):
""" """
Base class to create architectures using TensorFlow Base class to create architectures using TensorFlow
""" """
def __init__(self, seed=10, use_gpu=False): def __init__(self):
""" """
Base constructor Base constructor
"""
self.seed = seed
self.use_gpu = use_gpu
self.create_variables()
def create_variables(self): **Parameters**
""" input: Place Holder
Create the Tensor flow variables
""" """
raise NotImplementedError("Please implement this function in derived classes")
def create_graph(self, data): self.sequence_net = OrderedDict()
"""
Create the Tensor flow variables def add(self, layer):
if not isinstance(layer, Layer):
raise ValueError("Input `layer` must be an instance of `bob.learn.tensorflow.layers.Layer`")
self.sequence_net[layer.name] = layer
def compute_graph(self, input_data):
input_offset = input_data
for k in self.sequence_net.keys():
print k
import ipdb;
ipdb.set_trace();
current_layer = self.sequence_net[k]
current_layer.create_variables(input_offset)
input_offset = current_layer.get_graph
return input_offset
**Parameters**
data: Tensorflow Placeholder
**Returns**
Network output
"""
raise NotImplementedError("Please implement this function in derived classes")
...@@ -2,4 +2,6 @@ ...@@ -2,4 +2,6 @@
from pkgutil import extend_path from pkgutil import extend_path
__path__ = extend_path(__path__, __name__) __path__ = extend_path(__path__, __name__)
from .SequenceNetwork import SequenceNetwork
from .Lenet import Lenet
...@@ -22,8 +22,9 @@ import tensorflow as tf ...@@ -22,8 +22,9 @@ import tensorflow as tf
from .. import util from .. import util
SEED = 10 SEED = 10
from ..DataShuffler import * from ..DataShuffler import *
from ..lenet import Lenet from bob.learn.tensorflow.network import Lenet
import sys from bob.learn.tensorflow.trainers import Trainer
import numpy
def main(): def main():
args = docopt(__doc__, version='Mnist training with TensorFlow') args = docopt(__doc__, version='Mnist training with TensorFlow')
...@@ -36,59 +37,14 @@ def main(): ...@@ -36,59 +37,14 @@ def main():
# Loading data # Loading data
data, labels = util.load_mnist(data_dir="./src/bob.db.mnist/bob/db/mnist/") data, labels = util.load_mnist(data_dir="./src/bob.db.mnist/bob/db/mnist/")
data = numpy.reshape(data, (data.shape[0], 28, 28, 1))
data_shuffler = DataShuffler(data, labels) data_shuffler = DataShuffler(data, labels)
# Defining place holders for train and validation # Preparing the architecture
train_data_node = tf.placeholder(tf.float32, shape=(BATCH_SIZE, 28, 28, 1)) lenet = Lenet()
train_labels_node = tf.placeholder(tf.int64, shape=BATCH_SIZE)
validation_data_node = tf.placeholder(tf.float32, shape=(data_shuffler.validation_data.shape[0], 28, 28, 1))
# Creating the architecture for train and validation loss = tf.nn.sparse_softmax_cross_entropy_with_logits
lenet_architecture = Lenet(seed=SEED, use_gpu=USE_GPU) trainer = Trainer(architecture=lenet, loss=loss)
lenet_train = lenet_architecture.create_lenet(train_data_node) trainer.train(data_shuffler)
lenet_validation = lenet_architecture.create_lenet(validation_data_node, train=False)
# Simple loss
loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(lenet_train, train_labels_node))
#regularizer = (tf.nn.l2_loss(W_fc1) + tf.nn.l2_loss(b_fc1) +
# tf.nn.l2_loss(W_fc2) + tf.nn.l2_loss(b_fc2))
#loss += 5e-4 * regularizer
# Learning rate
batch = tf.Variable(0)
learning_rate = tf.train.exponential_decay(
0.001, # Learning rate
batch * BATCH_SIZE,
data_shuffler.train_data.shape[0],
0.95 # Decay step
)
optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=batch)
train_prediction = tf.nn.softmax(lenet_train)
validation_prediction = tf.nn.softmax(lenet_validation)
print("Initializing !!")
# Training
with tf.Session() as session:
tf.initialize_all_variables().run()
for step in range(ITERATIONS):
train_data, train_labels = data_shuffler.get_batch(BATCH_SIZE)
feed_dict = {train_data_node: train_data,
train_labels_node: train_labels}
_, l, lr, predictions = session.run([optimizer, loss, learning_rate, train_prediction], feed_dict=feed_dict)
if step % VALIDATION_TEST == 0:
validation_data, validation_labels = data_shuffler.get_batch(data_shuffler.validation_data.shape[0],
train_dataset=False)
accuracy = util.evaluate_softmax(validation_data, validation_labels, session, validation_prediction, validation_data_node)
print("Step {0}. Loss = {1}, Lr={2}, Accuracy validation = {3}".format(step, l, lr, accuracy))
sys.stdout.flush()