Still dumping code.

parent 50eb11a9
......@@ -4,7 +4,7 @@
# @date: Wed 11 May 2016 09:39:36 CEST
import numpy
import tensorflow as tf
def scale_mean_norm(data, scale=0.00390625):
mean = numpy.mean(data)
......@@ -14,7 +14,7 @@ def scale_mean_norm(data, scale=0.00390625):
class DataShuffler(object):
def __init__(self, perc_train=0.9, scale=True):
def __init__(self, data, labels, perc_train=0.9, scale=True, train_batch_size=1, validation_batch_size=1):
"""
Some base functions for neural networks
......@@ -25,38 +25,51 @@ class DataShuffler(object):
self.perc_train = perc_train
self.scale = True
self.scale_value = 0.00390625
self.train_batch_size = train_batch_size
self.validation_batch_size = validation_batch_size
self.data = data
self.labels =labels
self.n_samples = self.data.shape[0]
self.width = self.data.shape[1]
self.height = self.data.shape[2]
self.channels = self.data.shape[3]
self.start_shuffler()
def get_placeholders(self, name=""):
data = tf.placeholder(tf.float32, shape=(self.train_batch_size, self.width,
self.height, self.channels), name=name)
labels = tf.placeholder(tf.int64, shape=self.train_batch_size)
def start_shuffler(self, data, labels):
return data, labels
def start_shuffler(self):
"""
Some base functions for neural networks
**Parameters**
data:
"""
scale_value = 0.00390625
total_samples = data.shape[0]
indexes = numpy.array(range(total_samples))
indexes = numpy.array(range(self.n_samples))
numpy.random.shuffle(indexes)
# Spliting train and validation
train_samples = int(round(total_samples * self.perc_train))
validation_samples = total_samples - train_samples
data = numpy.reshape(data, (data.shape[0], 28, 28, 1))
train_samples = int(round(self.n_samples * self.perc_train))
validation_samples = self.n_samples - train_samples
self.train_data = data[indexes[0:train_samples], :, :, :]
self.train_labels = labels[indexes[0:train_samples]]
self.train_data = self.data[indexes[0:train_samples], :, :, :]
self.train_labels = self.labels[indexes[0:train_samples]]
self.validation_data = data[indexes[train_samples:train_samples + validation_samples], :, :, :]
self.validation_labels = labels[indexes[train_samples:train_samples + validation_samples]]
self.validation_data = self.data[indexes[train_samples:train_samples + validation_samples], :, :, :]
self.validation_labels = self.labels[indexes[train_samples:train_samples + validation_samples]]
self.total_labels = 10
if self.scale:
# data = scale_minmax_norm(data,lower_bound = -1, upper_bound = 1)
self.train_data, self.mean = scale_mean_norm(self.train_data)
self.validation_data = (self.validation_data - self.mean) * scale_value
self.validation_data = (self.validation_data - self.mean) * self.scale_value
def get_batch(self, n_samples, train_dataset=True):
......
......@@ -3,7 +3,7 @@
# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
# @date: Wed 11 May 2016 17:38 CEST
import tensoflow as tf
import tensorflow as tf
from bob.learn.tensorflow.util import *
from .Layer import Layer
......@@ -14,7 +14,7 @@ class Conv2D(Layer):
2D Convolution
"""
def __init__(self, input, activation=None,
def __init__(self, name, activation=None,
kernel_size=3,
filters=8,
initialization='xavier',
......@@ -33,20 +33,31 @@ class Conv2D(Layer):
use_gpu: Store data in the GPU
seed: Seed for the Random number generation
"""
super(Conv2D, self).__init__(input, initialization='xavier', use_gpu=False, seed=10)
self.activation = activation
self.W = create_weight_variables([kernel_size, kernel_size, 1, filters],
seed=seed, name="conv", use_gpu=use_gpu)
if activation is not None:
self.b = create_bias_variables([filters], name="bias", use_gpu=self.use_gpu)
super(Conv2D, self).__init__(name, activation=activation, initialization='xavier',
use_gpu=use_gpu, seed=seed)
self.kernel_size = kernel_size
self.filters = filters
self.initialization = initialization
self.W = None
self.b = None
def create_variables(self, input):
self.input = input
if self.W is None:
self.W = create_weight_variables([self.kernel_size, self.kernel_size, 1, self.filters],
seed=self.seed, name=str(self.name), use_gpu=self.use_gpu)
if self.activation is not None:
self.b = create_bias_variables([self.filters], name=str(self.name) + "bias", use_gpu=self.use_gpu)
def get_graph(self):
with tf.name_scope('conv'):
conv = tf.nn.conv2d(self.input, self.W, strides=[1, 1, 1, 1], padding='SAME')
with tf.name_scope(str(self.name)):
conv2d = tf.nn.conv2d(self.input, self.W, strides=[1, 1, 1, 1], padding='SAME')
with tf.name_scope('activation'):
non_linearity = tf.nn.tanh(tf.nn.bias_add(conv, self.b))
if self.activation is not None:
with tf.name_scope(str(self.name) + 'activation'):
non_linear_conv2d = tf.nn.tanh(tf.nn.bias_add(conv2d, self.b))
self.output = non_linear_conv2d
else:
self.output = conv2d
return non_linearity
return self.output
......@@ -3,9 +3,10 @@
# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
# @date: Wed 11 May 2016 17:38 CEST
import tensoflow as tf
import tensorflow as tf
from bob.learn.tensorflow.util import *
from .Layer import Layer
from operator import mul
class FullyConnected(Layer):
......@@ -14,7 +15,7 @@ class FullyConnected(Layer):
2D Convolution
"""
def __init__(self, input, activation=None,
def __init__(self, name, output_dim, activation=None,
initialization='xavier',
use_gpu=False,
seed=10
......@@ -29,23 +30,36 @@ class FullyConnected(Layer):
use_gpu: Store data in the GPU
seed: Seed for the Random number generation
"""
super(FullyConnected, self).__init__(input, initialization='xavier', use_gpu=False, seed=10)
self.activation = activation
super(FullyConnected, self).__init__(name, activation=activation,
initialization=initialization, use_gpu=use_gpu, seed=seed)
self.output_dim = output_dim
self.W = None
self.b = None
def create_variables(self, input):
self.input = input
if self.W is None:
input_dim = reduce(mul, self.input.get_shape().as_list())
if len(input.get_shape())==4:
self.W = create_weight_variables([kernel_size, kernel_size, 1, filters],
seed=seed, name="conv", use_gpu=use_gpu)
if activation is not None:
self.b = create_bias_variables([filters], name="bias", use_gpu=self.use_gpu)
self.W = create_weight_variables([input_dim, self.output_dim],
seed=self.seed, name=str(self.name), use_gpu=self.use_gpu)
if self.activation is not None:
self.b = create_bias_variables([self.output_dim], name=str(self.name)+"_bias", use_gpu=self.use_gpu)
def get_graph(self):
with tf.name_scope('fc'):
conv = tf.nn.conv2d(self.input, self.W, strides=[1, 1, 1, 1], padding='SAME')
if len(self.input.get_shape()) == 4:
shape = self.input.get_shape().as_list()
fc = tf.reshape(self.input, [shape[0], shape[1] * shape[2] * shape[3]])
if self.activation is not None:
with tf.name_scope('activation'):
non_linearity = tf.nn.tanh(tf.nn.bias_add(conv, self.b))
non_linear_fc = tf.nn.tanh(tf.matmul(fc, self.W) + self.b)
self.output = non_linear_fc
else:
self.output = fc
return self.output
return non_linearity
......@@ -3,7 +3,7 @@
# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
# @date: Wed 11 May 2016 17:38 CEST
import tensoflow as tf
import tensorflow as tf
class Layer(object):
......@@ -12,7 +12,7 @@ class Layer(object):
Layer base class
"""
def __init__(self, input, initialization='xavier', use_gpu=False, seed=10):
def __init__(self, name, activation=None, initialization='xavier', use_gpu=False, seed=10):
"""
Base constructor
......@@ -20,10 +20,23 @@ class Layer(object):
input: Layer input
"""
self.input = input
self.name = name
self.initialization = initialization
self.use_gpu = use_gpu
self.seed = seed
self.input = None
self.activation = None
self.output = None
def create_variables(self, input):
NotImplementedError("Please implement this function in derived classes")
def get_graph(self):
NotImplementedError("Please implement this function in derived classes")
def get_shape(self):
if self.output is None:
NotImplementedError("This class was not implemented properly")
else:
return self.output.get_shape()
......@@ -3,18 +3,25 @@
# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
# @date: Wed 11 May 2016 17:38 CEST
import tensoflow as tf
import tensorflow as tf
from bob.learn.tensorflow.util import *
from .Layer import Layer
class MaxPooling(Layer):
def __init__(self, input, use_gpu=False):
def __init__(self, name, use_gpu=False):
"""
Constructor
"""
super(MaxPooling, self).__init__(input, use_gpu=False)
super(MaxPooling, self).__init__(name, use_gpu=False)
def create_variables(self, input):
self.input = input
return
def get_graph(self):
tf.nn.max_pool(self.input, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
with tf.name_scope(str(self.name)):
self.output = tf.nn.max_pool(self.input, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
return self.output
......@@ -2,6 +2,10 @@
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
from DataShuffler import *
#from DataShuffler import *
from .Layer import Layer
from .Conv2D import Conv2D
from .FullyConnected import FullyConnected
from .MaxPooling import MaxPooling
......@@ -6,7 +6,7 @@
import logging
logger = logging.getLogger("bob.learn.tensorflow")
import tensorflow as tf
from bob.learn.tensorflow.loss.Baseloss as Baseloss
from .Baseloss import Baseloss
class ContrastiveLoss(BaseLoss):
......
......@@ -9,10 +9,11 @@ Class that creates the lenet architecture
from ..util import *
import tensorflow as tf
from bob.learn.tensorflow.network.BaseArchitecture import BaseArchitecture
from .SequenceNetwork import SequenceNetwork
from ..layers import Conv2D, FullyConnected, MaxPooling
class Lenet(BaseArchitecture):
class Lenet(SequenceNetwork):
def __init__(self,
conv1_kernel_size=5,
......@@ -40,80 +41,11 @@ class Lenet(BaseArchitecture):
seed = 10
"""
self.conv1_kernel_size = conv1_kernel_size,
self.conv1_output = conv1_output,
self.conv2_kernel_size = conv2_kernel_size,
self.conv2_output = conv2_output,
self.fc1_output = fc1_output,
self.n_classes = n_classes,
super(Lenet, self).__init__(seed=seed, use_gpu=use_gpu)
def create_variables(self):
# First convolutional
self.W_conv1 = create_weight_variables([self.conv1_kernel_size, self.conv1_kernel_size, 1, self.conv1_output],
seed=self.seed, name="W_conv1", use_gpu=self.use_gpu)
self.b_conv1 = create_bias_variables([self.conv1_output], name="bias_conv1", use_gpu=self.use_gpu)
# Second convolutional
self.W_conv2 = create_weight_variables([self.conv2_kernel_size, self.conv2_kernel_size, self.conv1_output,
self.conv2_output], seed=self.seed, name="W_conv2",
use_gpu=self.use_gpu)
self.b_conv2 = create_bias_variables([self.conv2_output], name="bias_conv2", use_gpu=self.use_gpu)
# First fc
self.W_fc1 = create_weight_variables([(28 // 4) * (28 // 4) * self.conv2_output, self.fc1_output],
seed=self.seed, name="W_fc1", use_gpu=self.use_gpu)
self.b_fc1 = create_bias_variables([self.fc1_output], name="bias_fc1", use_gpu=self.use_gpu)
# Second FC fc
self.W_fc2 = create_weight_variables([self.fc1_output, self.n_classes], seed=self.seed,
name="W_fc2", use_gpu=self.use_gpu)
self.b_fc2 = create_bias_variables([self.n_classes], name="bias_fc2", use_gpu=self.use_gpu)
self.seed = self.seed
def create_graph(self, data):
# Creating the architecture
# First convolutional
with tf.name_scope('conv_1') as scope:
conv1 = create_conv2d(data, self.W_conv1)
with tf.name_scope('tanh_1') as scope:
tanh_1 = create_tanh(conv1, self.b_conv1)
# Pooling
with tf.name_scope('pool_1') as scope:
pool1 = create_max_pool(tanh_1)
# Second convolutional
with tf.name_scope('conv_2') as scope:
conv2 = create_conv2d(pool1, self.W_conv2)
with tf.name_scope('tanh_2') as scope:
tanh_2 = create_tanh(conv2, self.b_conv2)
# Pooling
with tf.name_scope('pool_2') as scope:
pool2 = create_max_pool(tanh_2)
#if train:
#pool2 = tf.nn.dropout(pool2, 0.5, seed=self.seed)
# Reshaping all the convolved images to 2D to feed the FC layers
# FC1
with tf.name_scope('fc_1') as scope:
pool_shape = pool2.get_shape().as_list()
reshape = tf.reshape(pool2, [pool_shape[0], pool_shape[1] * pool_shape[2] * pool_shape[3]])
fc1 = tf.nn.tanh(tf.matmul(reshape, self.W_fc1) + self.b_fc1)
# FC2
with tf.name_scope('fc_2') as scope:
fc2 = tf.matmul(fc1, self.W_fc2) + self.b_fc2
return fc2
super(Lenet, self).__init__()
self.add(Conv2D(name="conv1", kernel_size=conv1_kernel_size, filters=conv1_output))
self.add(MaxPooling(name="pooling1"))
self.add(Conv2D(name="conv2", kernel_size=conv2_kernel_size, filters=conv2_output))
self.add(MaxPooling(name="pooling2"))
self.add(FullyConnected(name="fc1", output_dim=fc1_output, activation=True))
self.add(FullyConnected(name="fc1", output_dim=n_classes, activation=False))
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
# @date: Wed 11 May 2016 09:39:36 CEST
# @date: Thu 11 Aug 2016 09:39:36 CEST
"""
Class that creates the lenet architecture
......@@ -12,35 +12,44 @@ import tensorflow as tf
import abc
import six
from collections import OrderedDict
from bob.learn.tensorflow.layers import *
class BaseArchitecture(six.with_metaclass(abc.ABCMeta, object)):
class SequenceNetwork(six.with_metaclass(abc.ABCMeta, object)):
"""
Base class to create architectures using TensorFlow
"""
def __init__(self, seed=10, use_gpu=False):
def __init__(self):
"""
Base constructor
"""
self.seed = seed
self.use_gpu = use_gpu
self.create_variables()
def create_variables(self):
"""
Create the Tensor flow variables
**Parameters**
input: Place Holder
"""
raise NotImplementedError("Please implement this function in derived classes")
def create_graph(self, data):
"""
Create the Tensor flow variables
self.sequence_net = OrderedDict()
def add(self, layer):
if not isinstance(layer, Layer):
raise ValueError("Input `layer` must be an instance of `bob.learn.tensorflow.layers.Layer`")
self.sequence_net[layer.name] = layer
def compute_graph(self, input_data):
input_offset = input_data
for k in self.sequence_net.keys():
print k
import ipdb;
ipdb.set_trace();
current_layer = self.sequence_net[k]
current_layer.create_variables(input_offset)
input_offset = current_layer.get_graph
return input_offset
**Parameters**
data: Tensorflow Placeholder
**Returns**
Network output
"""
raise NotImplementedError("Please implement this function in derived classes")
......@@ -2,4 +2,6 @@
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
from .SequenceNetwork import SequenceNetwork
from .Lenet import Lenet
......@@ -22,8 +22,9 @@ import tensorflow as tf
from .. import util
SEED = 10
from ..DataShuffler import *
from ..lenet import Lenet
import sys
from bob.learn.tensorflow.network import Lenet
from bob.learn.tensorflow.trainers import Trainer
import numpy
def main():
args = docopt(__doc__, version='Mnist training with TensorFlow')
......@@ -36,59 +37,14 @@ def main():
# Loading data
data, labels = util.load_mnist(data_dir="./src/bob.db.mnist/bob/db/mnist/")
data = numpy.reshape(data, (data.shape[0], 28, 28, 1))
data_shuffler = DataShuffler(data, labels)
# Defining place holders for train and validation
train_data_node = tf.placeholder(tf.float32, shape=(BATCH_SIZE, 28, 28, 1))
train_labels_node = tf.placeholder(tf.int64, shape=BATCH_SIZE)
validation_data_node = tf.placeholder(tf.float32, shape=(data_shuffler.validation_data.shape[0], 28, 28, 1))
# Preparing the architecture
lenet = Lenet()
# Creating the architecture for train and validation
lenet_architecture = Lenet(seed=SEED, use_gpu=USE_GPU)
lenet_train = lenet_architecture.create_lenet(train_data_node)
lenet_validation = lenet_architecture.create_lenet(validation_data_node, train=False)
loss = tf.nn.sparse_softmax_cross_entropy_with_logits
trainer = Trainer(architecture=lenet, loss=loss)
trainer.train(data_shuffler)
# Simple loss
loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(lenet_train, train_labels_node))
#regularizer = (tf.nn.l2_loss(W_fc1) + tf.nn.l2_loss(b_fc1) +
# tf.nn.l2_loss(W_fc2) + tf.nn.l2_loss(b_fc2))
#loss += 5e-4 * regularizer
# Learning rate
batch = tf.Variable(0)
learning_rate = tf.train.exponential_decay(
0.001, # Learning rate
batch * BATCH_SIZE,
data_shuffler.train_data.shape[0],
0.95 # Decay step
)
optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=batch)
train_prediction = tf.nn.softmax(lenet_train)
validation_prediction = tf.nn.softmax(lenet_validation)
print("Initializing !!")
# Training
with tf.Session() as session:
tf.initialize_all_variables().run()
for step in range(ITERATIONS):
train_data, train_labels = data_shuffler.get_batch(BATCH_SIZE)
feed_dict = {train_data_node: train_data,
train_labels_node: train_labels}
_, l, lr, predictions = session.run([optimizer, loss, learning_rate, train_prediction], feed_dict=feed_dict)
if step % VALIDATION_TEST == 0:
validation_data, validation_labels = data_shuffler.get_batch(data_shuffler.validation_data.shape[0],
train_dataset=False)
accuracy = util.evaluate_softmax(validation_data, validation_labels, session, validation_prediction, validation_data_node)
print("Step {0}. Loss = {1}, Lr={2}, Accuracy validation = {3}".format(step, l, lr, accuracy))
sys.stdout.flush()
print("Step {0}. Loss = {1}, Lr={2}, Accuracy validation = {3}".format(step, l, lr, accuracy))
print("End !!")
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
# @date: Tue 09 Aug 2016 15:25:22 CEST
import logging
logger = logging.getLogger("bob.learn.tensorflow")
from ..DataShuffler import DataShuffler
class BaseTrainer(object):
def __init__(self,
data=None,
labels=None,
network=None,
width=28,
height=28,
channels=1,
use_gpu=False,
data_shuffler=DataShuffler(),
optimization=tf.train.MomentumOptimizer(0.001, momentum=0.99, use_locking=False, name='Momentum'),
loss=None,
###### training options ##########
convergence_threshold = 0.01,
percentage_train=0.5,
iterations=5000,
train_batch_size=1,
validation_batch_size=1000,
base_lr=0.00001,
momentum=0.9,
weight_decay=0.0005,
# The learning rate policy
lr_policy="inv",
gamma=0.0001,
power=0.75,
snapshot=5000):
self.data = data
self.labels = labels
self.data_shuffler = DataShuffler(self.data, self.labels)
self.width = width
self.height = height
self.channels = channels
self.network = network
self.use_gpu = use_gpu
#TODO: PREPARE THE CONSTRUCTOR FOR THAT
self.caffe_model_path = None
self.deploy_architecture_path = None
self.net = None
# TODO: Parametrize this variable
self.single_batch = False
self.n_labels = 0
self.train_batch_size = train_batch_size
self.validation_batch_size = validation_batch_size
self.iterations = iterations
self.snapshot = snapshot
self.solver_class = solver
self.solver = None
self.base_lr = base_lr
self.momentum = momentum
self.weight_decay = weight_decay
self.percentage_train = percentage_train
self.convergence_threshold = convergence_threshold
# The learning rate policy
self.lr_policy = lr_policy
self.gamma = gamma
self.power = power
# Shape of the data in the format [c, w, h] --> Channels, Width, Height
self.data_shape = None
def __setup_train(self):
# Loading data
# Defining place holders for train and validation
train_data_node = tf.placeholder(tf.float32, shape=(self.train_batch_size, self.width, self.height, self.channels))
train_labels_node = tf.placeholder(tf.int64, shape=self.train_batch_size)
# Creating the architecture for train and validation
architecture = self.network(use_gpu=self.use_gpu)
train_graph = architecture.create_lenet(train_data_node)
def __call__(self, image):
"""
Forward the CNN uising the image.
**Parameters:**
image:
"""
if self.use_gpu:
caffe.set_device(0)
caffe.set_mode_gpu()
# Dealing with RGB or gray images
if len(image.shape) == 2:
image = numpy.reshape(image, (1, 1, image.shape[0], image.shape[1]))
else:
image = numpy.reshape(image, (1, image.shape[0], image.shape[1], image.shape[2]))