Skip to content

bob.learn.tensorflow
bob.learn.tensorflow
Commit 7bd53d0e authored by Tiago de Freitas Pereira's avatar Tiago de Freitas Pereira
Browse files
Options

Still dumping code.

parent 50eb11a9
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 304 additions and 586 deletions
bob/learn/tensorflow/DataShuffler.py
View file @ 7bd53d0e
......@@ -4,7 +4,7 @@
# @date: Wed 11 May 2016 09:39:36 CEST
import numpy
import tensorflow as tf
def scale_mean_norm(data, scale=0.00390625):
mean = numpy.mean(data)
......@@ -14,7 +14,7 @@ def scale_mean_norm(data, scale=0.00390625):
class DataShuffler(object):
def __init__(self, perc_train=0.9, scale=True):
def __init__(self, data, labels, perc_train=0.9, scale=True, train_batch_size=1, validation_batch_size=1):
"""
Some base functions for neural networks
......@@ -25,38 +25,51 @@ class DataShuffler(object):
self.perc_train = perc_train
self.scale = True
self.scale_value = 0.00390625
self.train_batch_size = train_batch_size
self.validation_batch_size = validation_batch_size
self.data = data
self.labels =labels
self.n_samples = self.data.shape[0]
self.width = self.data.shape[1]
self.height = self.data.shape[2]
self.channels = self.data.shape[3]
self.start_shuffler()
def get_placeholders(self, name=""):
data = tf.placeholder(tf.float32, shape=(self.train_batch_size, self.width,
self.height, self.channels), name=name)
labels = tf.placeholder(tf.int64, shape=self.train_batch_size)
def start_shuffler(self, data, labels):
return data, labels
def start_shuffler(self):
"""
Some base functions for neural networks
**Parameters**
data:
"""
scale_value = 0.00390625
total_samples = data.shape[0]
indexes = numpy.array(range(total_samples))
indexes = numpy.array(range(self.n_samples))
numpy.random.shuffle(indexes)
# Spliting train and validation
train_samples = int(round(total_samples * self.perc_train))
validation_samples = total_samples - train_samples
data = numpy.reshape(data, (data.shape[0], 28, 28, 1))
train_samples = int(round(self.n_samples * self.perc_train))
validation_samples = self.n_samples - train_samples
self.train_data = data[indexes[0:train_samples], :, :, :]
self.train_labels = labels[indexes[0:train_samples]]
self.train_data = self.data[indexes[0:train_samples], :, :, :]
self.train_labels = self.labels[indexes[0:train_samples]]
self.validation_data = data[indexes[train_samples:train_samples + validation_samples], :, :, :]
self.validation_labels = labels[indexes[train_samples:train_samples + validation_samples]]
self.validation_data = self.data[indexes[train_samples:train_samples + validation_samples], :, :, :]
self.validation_labels = self.labels[indexes[train_samples:train_samples + validation_samples]]
self.total_labels = 10
if self.scale:
# data = scale_minmax_norm(data,lower_bound = -1, upper_bound = 1)
self.train_data, self.mean = scale_mean_norm(self.train_data)
self.validation_data = (self.validation_data - self.mean) * scale_value
self.validation_data = (self.validation_data - self.mean) * self.scale_value
def get_batch(self, n_samples, train_dataset=True):
......
bob/learn/tensorflow/layers/Conv2D.py
View file @ 7bd53d0e
......@@ -3,7 +3,7 @@
# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
# @date: Wed 11 May 2016 17:38 CEST
import tensoflow as tf
import tensorflow as tf
from bob.learn.tensorflow.util import *
from .Layer import Layer
......@@ -14,7 +14,7 @@ class Conv2D(Layer):
2D Convolution
"""
def __init__(self, input, activation=None,
def __init__(self, name, activation=None,
kernel_size=3,
filters=8,
initialization='xavier',
......@@ -33,20 +33,31 @@ class Conv2D(Layer):
use_gpu: Store data in the GPU
seed: Seed for the Random number generation
"""
super(Conv2D, self).__init__(input, initialization='xavier', use_gpu=False, seed=10)
self.activation = activation
self.W = create_weight_variables([kernel_size, kernel_size, 1, filters],
seed=seed, name="conv", use_gpu=use_gpu)
if activation is not None:
self.b = create_bias_variables([filters], name="bias", use_gpu=self.use_gpu)
super(Conv2D, self).__init__(name, activation=activation, initialization='xavier',
use_gpu=use_gpu, seed=seed)
self.kernel_size = kernel_size
self.filters = filters
self.initialization = initialization
self.W = None
self.b = None
def create_variables(self, input):
self.input = input
if self.W is None:
self.W = create_weight_variables([self.kernel_size, self.kernel_size, 1, self.filters],
seed=self.seed, name=str(self.name), use_gpu=self.use_gpu)
if self.activation is not None:
self.b = create_bias_variables([self.filters], name=str(self.name) + "bias", use_gpu=self.use_gpu)
def get_graph(self):
with tf.name_scope('conv'):
conv = tf.nn.conv2d(self.input, self.W, strides=[1, 1, 1, 1], padding='SAME')
with tf.name_scope(str(self.name)):
conv2d = tf.nn.conv2d(self.input, self.W, strides=[1, 1, 1, 1], padding='SAME')
with tf.name_scope('activation'):
non_linearity = tf.nn.tanh(tf.nn.bias_add(conv, self.b))
if self.activation is not None:
with tf.name_scope(str(self.name) + 'activation'):
non_linear_conv2d = tf.nn.tanh(tf.nn.bias_add(conv2d, self.b))
self.output = non_linear_conv2d
else:
self.output = conv2d
return non_linearity
return self.output
bob/learn/tensorflow/layers/FullyConnected.py
View file @ 7bd53d0e
......@@ -3,9 +3,10 @@
# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
# @date: Wed 11 May 2016 17:38 CEST
import tensoflow as tf
import tensorflow as tf
from bob.learn.tensorflow.util import *
from .Layer import Layer
from operator import mul
class FullyConnected(Layer):
......@@ -14,7 +15,7 @@ class FullyConnected(Layer):
2D Convolution
"""
def __init__(self, input, activation=None,
def __init__(self, name, output_dim, activation=None,
initialization='xavier',
use_gpu=False,
seed=10
......@@ -29,23 +30,36 @@ class FullyConnected(Layer):
use_gpu: Store data in the GPU
seed: Seed for the Random number generation
"""
super(FullyConnected, self).__init__(input, initialization='xavier', use_gpu=False, seed=10)
self.activation = activation
super(FullyConnected, self).__init__(name, activation=activation,
initialization=initialization, use_gpu=use_gpu, seed=seed)
self.output_dim = output_dim
self.W = None
self.b = None
def create_variables(self, input):
self.input = input
if self.W is None:
input_dim = reduce(mul, self.input.get_shape().as_list())
if len(input.get_shape())==4:
self.W = create_weight_variables([kernel_size, kernel_size, 1, filters],
seed=seed, name="conv", use_gpu=use_gpu)
if activation is not None:
self.b = create_bias_variables([filters], name="bias", use_gpu=self.use_gpu)
self.W = create_weight_variables([input_dim, self.output_dim],
seed=self.seed, name=str(self.name), use_gpu=self.use_gpu)
if self.activation is not None:
self.b = create_bias_variables([self.output_dim], name=str(self.name)+"_bias", use_gpu=self.use_gpu)
def get_graph(self):
with tf.name_scope('fc'):
conv = tf.nn.conv2d(self.input, self.W, strides=[1, 1, 1, 1], padding='SAME')
if len(self.input.get_shape()) == 4:
shape = self.input.get_shape().as_list()
fc = tf.reshape(self.input, [shape[0], shape[1] * shape[2] * shape[3]])
if self.activation is not None:
with tf.name_scope('activation'):
non_linearity = tf.nn.tanh(tf.nn.bias_add(conv, self.b))
non_linear_fc = tf.nn.tanh(tf.matmul(fc, self.W) + self.b)
self.output = non_linear_fc
else:
self.output = fc
return self.output
return non_linearity
bob/learn/tensorflow/layers/Layer.py
View file @ 7bd53d0e
......@@ -3,7 +3,7 @@
# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
# @date: Wed 11 May 2016 17:38 CEST
import tensoflow as tf
import tensorflow as tf
class Layer(object):
......@@ -12,7 +12,7 @@ class Layer(object):
Layer base class
"""
def __init__(self, input, initialization='xavier', use_gpu=False, seed=10):
def __init__(self, name, activation=None, initialization='xavier', use_gpu=False, seed=10):
"""
Base constructor
......@@ -20,10 +20,23 @@ class Layer(object):
input: Layer input
"""
self.input = input
self.name = name
self.initialization = initialization
self.use_gpu = use_gpu
self.seed = seed
self.input = None
self.activation = None
self.output = None
def create_variables(self, input):
NotImplementedError("Please implement this function in derived classes")
def get_graph(self):
NotImplementedError("Please implement this function in derived classes")
def get_shape(self):
if self.output is None:
NotImplementedError("This class was not implemented properly")
else:
return self.output.get_shape()
bob/learn/tensorflow/layers/MaxPooling.py
View file @ 7bd53d0e
......@@ -3,18 +3,25 @@
# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
# @date: Wed 11 May 2016 17:38 CEST
import tensoflow as tf
import tensorflow as tf
from bob.learn.tensorflow.util import *
from .Layer import Layer
class MaxPooling(Layer):
def __init__(self, input, use_gpu=False):
def __init__(self, name, use_gpu=False):
"""
Constructor
"""
super(MaxPooling, self).__init__(input, use_gpu=False)
super(MaxPooling, self).__init__(name, use_gpu=False)
def create_variables(self, input):
self.input = input
return
def get_graph(self):
tf.nn.max_pool(self.input, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
with tf.name_scope(str(self.name)):
self.output = tf.nn.max_pool(self.input, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
return self.output
bob/learn/tensorflow/layers/__init__.py
View file @ 7bd53d0e
......@@ -2,6 +2,10 @@
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
from DataShuffler import *
#from DataShuffler import *
from .Layer import Layer
from .Conv2D import Conv2D
from .FullyConnected import FullyConnected
from .MaxPooling import MaxPooling
bob/learn/tensorflow/loss/ContrastiveLoss.py
View file @ 7bd53d0e
......@@ -6,7 +6,7 @@
import logging
logger = logging.getLogger("bob.learn.tensorflow")
import tensorflow as tf
from bob.learn.tensorflow.loss.Baseloss as Baseloss
from .Baseloss import Baseloss
class ContrastiveLoss(BaseLoss):
......
bob/learn/tensorflow/network/Lenet.py
View file @ 7bd53d0e
......@@ -9,10 +9,11 @@ Class that creates the lenet architecture
from ..util import *
import tensorflow as tf
from bob.learn.tensorflow.network.BaseArchitecture import BaseArchitecture
from .SequenceNetwork import SequenceNetwork
from ..layers import Conv2D, FullyConnected, MaxPooling
class Lenet(BaseArchitecture):
class Lenet(SequenceNetwork):
def __init__(self,
conv1_kernel_size=5,
......@@ -40,80 +41,11 @@ class Lenet(BaseArchitecture):
seed = 10
"""
self.conv1_kernel_size = conv1_kernel_size,
self.conv1_output = conv1_output,
self.conv2_kernel_size = conv2_kernel_size,
self.conv2_output = conv2_output,
self.fc1_output = fc1_output,
self.n_classes = n_classes,
super(Lenet, self).__init__(seed=seed, use_gpu=use_gpu)
def create_variables(self):
# First convolutional
self.W_conv1 = create_weight_variables([self.conv1_kernel_size, self.conv1_kernel_size, 1, self.conv1_output],
seed=self.seed, name="W_conv1", use_gpu=self.use_gpu)
self.b_conv1 = create_bias_variables([self.conv1_output], name="bias_conv1", use_gpu=self.use_gpu)
# Second convolutional
self.W_conv2 = create_weight_variables([self.conv2_kernel_size, self.conv2_kernel_size, self.conv1_output,
self.conv2_output], seed=self.seed, name="W_conv2",
use_gpu=self.use_gpu)
self.b_conv2 = create_bias_variables([self.conv2_output], name="bias_conv2", use_gpu=self.use_gpu)
# First fc
self.W_fc1 = create_weight_variables([(28 // 4) * (28 // 4) * self.conv2_output, self.fc1_output],
seed=self.seed, name="W_fc1", use_gpu=self.use_gpu)
self.b_fc1 = create_bias_variables([self.fc1_output], name="bias_fc1", use_gpu=self.use_gpu)
# Second FC fc
self.W_fc2 = create_weight_variables([self.fc1_output, self.n_classes], seed=self.seed,
name="W_fc2", use_gpu=self.use_gpu)
self.b_fc2 = create_bias_variables([self.n_classes], name="bias_fc2", use_gpu=self.use_gpu)
self.seed = self.seed
def create_graph(self, data):
# Creating the architecture
# First convolutional
with tf.name_scope('conv_1') as scope:
conv1 = create_conv2d(data, self.W_conv1)
with tf.name_scope('tanh_1') as scope:
tanh_1 = create_tanh(conv1, self.b_conv1)
# Pooling
with tf.name_scope('pool_1') as scope:
pool1 = create_max_pool(tanh_1)
# Second convolutional
with tf.name_scope('conv_2') as scope:
conv2 = create_conv2d(pool1, self.W_conv2)
with tf.name_scope('tanh_2') as scope:
tanh_2 = create_tanh(conv2, self.b_conv2)
# Pooling
with tf.name_scope('pool_2') as scope:
pool2 = create_max_pool(tanh_2)
#if train:
#pool2 = tf.nn.dropout(pool2, 0.5, seed=self.seed)
# Reshaping all the convolved images to 2D to feed the FC layers
# FC1
with tf.name_scope('fc_1') as scope:
pool_shape = pool2.get_shape().as_list()
reshape = tf.reshape(pool2, [pool_shape[0], pool_shape[1] * pool_shape[2] * pool_shape[3]])
fc1 = tf.nn.tanh(tf.matmul(reshape, self.W_fc1) + self.b_fc1)
# FC2
with tf.name_scope('fc_2') as scope:
fc2 = tf.matmul(fc1, self.W_fc2) + self.b_fc2
return fc2
super(Lenet, self).__init__()
self.add(Conv2D(name="conv1", kernel_size=conv1_kernel_size, filters=conv1_output))
self.add(MaxPooling(name="pooling1"))
self.add(Conv2D(name="conv2", kernel_size=conv2_kernel_size, filters=conv2_output))
self.add(MaxPooling(name="pooling2"))
self.add(FullyConnected(name="fc1", output_dim=fc1_output, activation=True))
self.add(FullyConnected(name="fc1", output_dim=n_classes, activation=False))
bob/learn/tensorflow/network/BaseArchitecture.py → bob/learn/tensorflow/network/SequenceNetwork.py
View file @ 7bd53d0e
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
# @date: Wed 11 May 2016 09:39:36 CEST
# @date: Thu 11 Aug 2016 09:39:36 CEST
"""
Class that creates the lenet architecture
......@@ -12,35 +12,44 @@ import tensorflow as tf
import abc
import six
from collections import OrderedDict
from bob.learn.tensorflow.layers import *
class BaseArchitecture(six.with_metaclass(abc.ABCMeta, object)):
class SequenceNetwork(six.with_metaclass(abc.ABCMeta, object)):
"""
Base class to create architectures using TensorFlow
"""
def __init__(self, seed=10, use_gpu=False):
def __init__(self):
"""
Base constructor
"""
self.seed = seed
self.use_gpu = use_gpu
self.create_variables()
def create_variables(self):
"""
Create the Tensor flow variables
**Parameters**
input: Place Holder
"""
raise NotImplementedError("Please implement this function in derived classes")
def create_graph(self, data):
"""
Create the Tensor flow variables
self.sequence_net = OrderedDict()
def add(self, layer):
if not isinstance(layer, Layer):
raise ValueError("Input `layer` must be an instance of `bob.learn.tensorflow.layers.Layer`")
self.sequence_net[layer.name] = layer
def compute_graph(self, input_data):
input_offset = input_data
for k in self.sequence_net.keys():
print k
import ipdb;
ipdb.set_trace();
current_layer = self.sequence_net[k]
current_layer.create_variables(input_offset)
input_offset = current_layer.get_graph
return input_offset
**Parameters**
data: Tensorflow Placeholder
**Returns**
Network output
"""
raise NotImplementedError("Please implement this function in derived classes")
bob/learn/tensorflow/network/__init__.py
View file @ 7bd53d0e
......@@ -2,4 +2,6 @@
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
from .SequenceNetwork import SequenceNetwork
from .Lenet import Lenet
bob/learn/tensorflow/script/train_mnist.py
View file @ 7bd53d0e
......@@ -22,8 +22,9 @@ import tensorflow as tf
from .. import util
SEED = 10
from ..DataShuffler import *
from ..lenet import Lenet
import sys
from bob.learn.tensorflow.network import Lenet
from bob.learn.tensorflow.trainers import Trainer
import numpy
def main():
args = docopt(__doc__, version='Mnist training with TensorFlow')
......@@ -36,59 +37,14 @@ def main():
# Loading data
data, labels = util.load_mnist(data_dir="./src/bob.db.mnist/bob/db/mnist/")
data = numpy.reshape(data, (data.shape[0], 28, 28, 1))
data_shuffler = DataShuffler(data, labels)
# Defining place holders for train and validation
train_data_node = tf.placeholder(tf.float32, shape=(BATCH_SIZE, 28, 28, 1))
train_labels_node = tf.placeholder(tf.int64, shape=BATCH_SIZE)
validation_data_node = tf.placeholder(tf.float32, shape=(data_shuffler.validation_data.shape[0], 28, 28, 1))
# Preparing the architecture
lenet = Lenet()
# Creating the architecture for train and validation
lenet_architecture = Lenet(seed=SEED, use_gpu=USE_GPU)
lenet_train = lenet_architecture.create_lenet(train_data_node)
lenet_validation = lenet_architecture.create_lenet(validation_data_node, train=False)
loss = tf.nn.sparse_softmax_cross_entropy_with_logits
trainer = Trainer(architecture=lenet, loss=loss)
trainer.train(data_shuffler)
# Simple loss
loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(lenet_train, train_labels_node))
#regularizer = (tf.nn.l2_loss(W_fc1) + tf.nn.l2_loss(b_fc1) +
# tf.nn.l2_loss(W_fc2) + tf.nn.l2_loss(b_fc2))
#loss += 5e-4 * regularizer
# Learning rate
batch = tf.Variable(0)
learning_rate = tf.train.exponential_decay(
0.001, # Learning rate
batch * BATCH_SIZE,
data_shuffler.train_data.shape[0],
0.95 # Decay step
)
optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=batch)
train_prediction = tf.nn.softmax(lenet_train)
validation_prediction = tf.nn.softmax(lenet_validation)
print("Initializing !!")
# Training
with tf.Session() as session:
tf.initialize_all_variables().run()
for step in range(ITERATIONS):
train_data, train_labels = data_shuffler.get_batch(BATCH_SIZE)
feed_dict = {train_data_node: train_data,
train_labels_node: train_labels}
_, l, lr, predictions = session.run([optimizer, loss, learning_rate, train_prediction], feed_dict=feed_dict)
if step % VALIDATION_TEST == 0:
validation_data, validation_labels = data_shuffler.get_batch(data_shuffler.validation_data.shape[0],
train_dataset=False)
accuracy = util.evaluate_softmax(validation_data, validation_labels, session, validation_prediction, validation_data_node)
print("Step {0}. Loss = {1}, Lr={2}, Accuracy validation = {3}".format(step, l, lr, accuracy))
sys.stdout.flush()
print("Step {0}. Loss = {1}, Lr={2}, Accuracy validation = {3}".format(step, l, lr, accuracy))
print("End !!")
bob/learn/tensorflow/trainers/BaseTrainer.py deleted 100644 → 0
View file @ 50eb11a9
This diff is collapsed.
bob/learn/tensorflow/trainers/Trainer.py 0 → 100644
View file @ 7bd53d0e
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
# @date: Tue 09 Aug 2016 15:25:22 CEST
import logging
logger = logging.getLogger("bob.learn.tensorflow")
from ..DataShuffler import DataShuffler
import tensorflow as tf
from ..network import SequenceNetwork
class Trainer(object):
def __init__(self,
architecture=None,
use_gpu=False,
loss=None,
###### training options ##########
convergence_threshold = 0.01,
iterations=5000,
base_lr=0.00001,
momentum=0.9,
weight_decay=0.0005,
# The learning rate policy
snapshot=100):
self.loss = loss
self.loss_instance = None
self.optimizer = None
self.architecture = architecture
self.use_gpu = use_gpu
self.iterations = iterations
self.snapshot = snapshot
self.base_lr = base_lr
self.momentum = momentum
self.weight_decay = weight_decay
self.convergence_threshold = convergence_threshold
def train(self, data_shuffler):
"""
Do the loop forward --> backward --|
^--------------------|
"""
train_placeholder_data, train_placeholder_labels = data_shuffler.get_placeholders(name="train")
validation_placeholder_data, validation_placeholder_labels = data_shuffler.get_placeholders(name="validation")
# Creating the architecture for train and validation
if not isinstance(self.architecture, SequenceNetwork):
raise ValueError("The variable `architecture` must be an instance of "
"`bob.learn.tensorflow.network.SequenceNetwork`")
train_graph = self.architecture.compute_graph(train_placeholder_data)
loss_instance = tf.reduce_mean(self.loss(train_graph))
batch = tf.Variable(0)
learning_rate = tf.train.exponential_decay(
self.base_lr, # Learning rate
batch * self.train_batch_size,
data_shuffler.train_data.shape[0],
self.weight_decay # Decay step
)
optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(self.loss_instance,
global_step=batch)
train_prediction = tf.nn.softmax(train_graph)
print("Initializing !!")
# Training
with tf.Session() as session:
tf.initialize_all_variables().run()
for step in range(self.iterations):
train_data, train_labels = data_shuffler.get_batch(self.train_batch_size)
feed_dict = {train_placeholder_data: train_data,
train_placeholder_labels: train_labels}
_, l, lr, predictions = session.run([self.optimizer, self.loss_instance,
self.learning_rate, train_prediction], feed_dict=feed_dict)
if step % self.snapshot == 0:
validation_data, validation_labels = data_shuffler.get_batch(data_shuffler.validation_data.shape[0],
train_dataset=False)
feed_dict = {validation_placeholder_data: validation_data,
validation_placeholder_labels: validation_labels}
l, predictions = session.run([self.loss_instance, train_prediction], feed_dict=feed_dict)
print("Step {0}. Loss = {1}, Lr={2}".format(step, l, predictions))
#accuracy = util.evaluate_softmax(validation_data, validation_labels, session, validation_prediction,
# validation_data_node)
#print("Step {0}. Loss = {1}, Lr={2}, Accuracy validation = {3}".format(step, l, lr, accuracy))
#sys.stdout.flush()
bob/learn/tensorflow/trainers/__init__.py 0 → 100644
View file @ 7bd53d0e
# see https://docs.python.org/3/library/pkgutil.html
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
from ..DataShuffler import DataShuffler
from .Trainer import Trainer
buildout.cfg
View file @ 7bd53d0e
......@@ -4,7 +4,7 @@
[buildout]
parts = scripts
eggs = tensorflow.examples