Tests with dropout

parent cebdb323
...@@ -31,7 +31,7 @@ class Constant(Initialization): ...@@ -31,7 +31,7 @@ class Constant(Initialization):
initializer = tf.constant(self.constant_value, shape=shape) initializer = tf.constant(self.constant_value, shape=shape)
if self.use_gpu: if self.use_gpu:
with tf.device("/gpu"): with tf.device("/gpu:0"):
return tf.get_variable(name, initializer=initializer, dtype=tf.float32) return tf.get_variable(name, initializer=initializer, dtype=tf.float32)
else: else:
with tf.device("/cpu"): with tf.device("/cpu"):
......
...@@ -37,7 +37,7 @@ class Gaussian(Initialization): ...@@ -37,7 +37,7 @@ class Gaussian(Initialization):
seed=self.seed) seed=self.seed)
if self.use_gpu: if self.use_gpu:
with tf.device("/gpu"): with tf.device("/gpu:0"):
return tf.get_variable(name, initializer=initializer, dtype=tf.float32) return tf.get_variable(name, initializer=initializer, dtype=tf.float32)
else: else:
with tf.device("/cpu"): with tf.device("/cpu"):
......
...@@ -39,7 +39,7 @@ class SimplerXavier(Initialization): ...@@ -39,7 +39,7 @@ class SimplerXavier(Initialization):
initializer = tf.truncated_normal(shape, stddev=stddev, seed=self.seed) initializer = tf.truncated_normal(shape, stddev=stddev, seed=self.seed)
if self.use_gpu: if self.use_gpu:
with tf.device("/gpu"): with tf.device("/gpu:0"):
return tf.get_variable(name, initializer=initializer, dtype=tf.float32) return tf.get_variable(name, initializer=initializer, dtype=tf.float32)
else: else:
with tf.device("/cpu"): with tf.device("/cpu"):
......
...@@ -35,12 +35,12 @@ class Xavier(Initialization): ...@@ -35,12 +35,12 @@ class Xavier(Initialization):
in_out = shape[0] + shape[1] in_out = shape[0] + shape[1]
import math import math
stddev = math.sqrt(2.0 / in_out) # XAVIER INITIALIZER (GAUSSIAN) stddev = math.sqrt(3.0 / in_out) # XAVIER INITIALIZER (GAUSSIAN)
initializer = tf.truncated_normal(shape, stddev=stddev, seed=self.seed) initializer = tf.truncated_normal(shape, stddev=stddev, seed=self.seed)
if self.use_gpu: if self.use_gpu:
with tf.device("/gpu"): with tf.device("/gpu:0"):
return tf.get_variable(name, initializer=initializer, dtype=tf.float32) return tf.get_variable(name, initializer=initializer, dtype=tf.float32)
else: else:
with tf.device("/cpu"): with tf.device("/cpu"):
......
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
# @date: Wed 11 May 2016 17:38 CEST
import tensorflow as tf
from .Layer import Layer
from operator import mul
class Dropout(Layer):
"""
Dropout
"""
def __init__(self, name,
keep_prob=0.99,
seed=10.
):
"""
Constructor
**Parameters**
input: Layer input
activation: Tensor Flow activation
initialization: Initialization type
use_gpu: Store data in the GPU
seed: Seed for the Random number generation
"""
super(Dropout, self).__init__(name=name)
self.keep_prob = keep_prob
self.seed = seed
def create_variables(self, input_layer):
self.input_layer = input_layer
return
def get_graph(self):
with tf.name_scope(str(self.name)):
output = tf.nn.dropout(self.input_layer, self.keep_prob, name=self.name)
return output
...@@ -10,7 +10,7 @@ from .Layer import Layer ...@@ -10,7 +10,7 @@ from .Layer import Layer
class MaxPooling(Layer): class MaxPooling(Layer):
def __init__(self, name, use_gpu=False): def __init__(self, name):
""" """
Constructor Constructor
""" """
......
...@@ -7,6 +7,7 @@ from .Layer import Layer ...@@ -7,6 +7,7 @@ from .Layer import Layer
from .Conv2D import Conv2D from .Conv2D import Conv2D
from .FullyConnected import FullyConnected from .FullyConnected import FullyConnected
from .MaxPooling import MaxPooling from .MaxPooling import MaxPooling
from .Dropout import Dropout
from .InputLayer import InputLayer from .InputLayer import InputLayer
# gets sphinx autodoc done right - don't remove it # gets sphinx autodoc done right - don't remove it
......
...@@ -29,7 +29,7 @@ class Lenet(SequenceNetwork): ...@@ -29,7 +29,7 @@ class Lenet(SequenceNetwork):
default_feature_layer="fc2", default_feature_layer="fc2",
seed=10, seed=10,
use_gpu = False): use_gpu=False):
""" """
Create all the necessary variables for this CNN Create all the necessary variables for this CNN
...@@ -45,28 +45,29 @@ class Lenet(SequenceNetwork): ...@@ -45,28 +45,29 @@ class Lenet(SequenceNetwork):
seed = 10 seed = 10
""" """
super(Lenet, self).__init__(default_feature_layer=default_feature_layer) super(Lenet, self).__init__(default_feature_layer=default_feature_layer,
use_gpu=use_gpu)
self.add(Conv2D(name="conv1", kernel_size=conv1_kernel_size, self.add(Conv2D(name="conv1", kernel_size=conv1_kernel_size,
filters=conv1_output, filters=conv1_output,
activation=tf.nn.tanh, activation=tf.nn.tanh,
weights_initialization=Xavier(seed=seed), weights_initialization=Xavier(seed=seed, use_gpu=self.use_gpu),
bias_initialization=Constant() bias_initialization=Constant(use_gpu=self.use_gpu)
)) ))
self.add(MaxPooling(name="pooling1")) self.add(MaxPooling(name="pooling1"))
self.add(Conv2D(name="conv2", kernel_size=conv2_kernel_size, self.add(Conv2D(name="conv2", kernel_size=conv2_kernel_size,
filters=conv2_output, filters=conv2_output,
activation=tf.nn.tanh, activation=tf.nn.tanh,
weights_initialization=Xavier(seed=seed), weights_initialization=Xavier(seed=seed, use_gpu=self.use_gpu),
bias_initialization=Constant() bias_initialization=Constant(use_gpu=self.use_gpu)
)) ))
self.add(MaxPooling(name="pooling2")) self.add(MaxPooling(name="pooling2"))
self.add(FullyConnected(name="fc1", output_dim=fc1_output, self.add(FullyConnected(name="fc1", output_dim=fc1_output,
activation=tf.nn.tanh, activation=tf.nn.tanh,
weights_initialization=Xavier(seed=seed), weights_initialization=Xavier(seed=seed, use_gpu=self.use_gpu),
bias_initialization=Constant() bias_initialization=Constant(use_gpu=self.use_gpu)
)) ))
self.add(FullyConnected(name="fc2", output_dim=n_classes, self.add(FullyConnected(name="fc2", output_dim=n_classes,
activation=None, activation=None,
weights_initialization=Xavier(seed=seed), weights_initialization=Xavier(seed=seed, use_gpu=self.use_gpu),
bias_initialization=Constant())) bias_initialization=Constant(use_gpu=self.use_gpu)))
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
# @date: Wed 11 May 2016 09:39:36 CEST
"""
Class that creates the lenet architecture
"""
import tensorflow as tf
from .SequenceNetwork import SequenceNetwork
from ..layers import Conv2D, FullyConnected, MaxPooling, Dropout
import bob.learn.tensorflow
from bob.learn.tensorflow.initialization import Xavier
from bob.learn.tensorflow.initialization import Constant
class LenetDropout(SequenceNetwork):
def __init__(self,
conv1_kernel_size=5,
conv1_output=16,
conv2_kernel_size=5,
conv2_output=32,
fc1_output=400,
n_classes=10,
default_feature_layer="fc2",
seed=10,
use_gpu=False):
"""
Create all the necessary variables for this CNN
**Parameters**
conv1_kernel_size=5,
conv1_output=32,
conv2_kernel_size=5,
conv2_output=64,
fc1_output=400,
n_classes=10
seed = 10
"""
super(LenetDropout, self).__init__(default_feature_layer=default_feature_layer,
use_gpu=use_gpu)
self.add(Conv2D(name="conv1", kernel_size=conv1_kernel_size,
filters=conv1_output,
activation=tf.nn.tanh,
weights_initialization=Xavier(seed=seed, use_gpu=self.use_gpu),
bias_initialization=Constant(use_gpu=self.use_gpu)
))
self.add(MaxPooling(name="pooling1"))
self.add(Conv2D(name="conv2", kernel_size=conv2_kernel_size,
filters=conv2_output,
activation=tf.nn.tanh,
weights_initialization=Xavier(seed=seed, use_gpu=self.use_gpu),
bias_initialization=Constant(use_gpu=self.use_gpu)
))
self.add(MaxPooling(name="pooling2"))
self.add(MaxPooling(name="pooling2"))
self.add(Dropout(name="dropout"))
self.add(FullyConnected(name="fc1", output_dim=fc1_output,
activation=tf.nn.tanh,
weights_initialization=Xavier(seed=seed, use_gpu=self.use_gpu),
bias_initialization=Constant(use_gpu=self.use_gpu)
))
self.add(FullyConnected(name="fc2", output_dim=n_classes,
activation=None,
weights_initialization=Xavier(seed=seed, use_gpu=self.use_gpu),
bias_initialization=Constant(use_gpu=self.use_gpu)))
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
# @date: Wed 11 May 2016 09:39:36 CEST
"""
Class that creates the lenet architecture
"""
import tensorflow as tf
from .SequenceNetwork import SequenceNetwork
from ..layers import Conv2D, FullyConnected, MaxPooling
from bob.learn.tensorflow.initialization import Xavier
from bob.learn.tensorflow.initialization import Constant
class MLP(SequenceNetwork):
def __init__(self,
output_shape,
hidden_layers=[10],
hidden_activation=tf.nn.tanh,
output_activation=None,
weights_initialization=Xavier(),
bias_initialization=Constant(),
use_gpu=False):
"""
Create all the necessary variables for this CNN
**Parameters**
output_shape: Shape of the output
hidden_layers: List that contains the amount of hidden layers, where each element is the number of neurons
hidden_activation: Activation function of the hidden layer. If you set to `None`, the activation will be linear
output_activation: Activation of the output layer. If you set to `None`, the activation will be linear
seed = 10
"""
super(MLP, self).__init__(use_gpu=use_gpu)
if (not (isinstance(hidden_layers, list) or isinstance(hidden_layers, tuple))) or len(hidden_layers) == 0:
raise ValueError("Invalid input for hidden_layers: {0} ".format(hidden_layers))
for i in range(len(hidden_layers)):
l = hidden_layers[i]
self.add(FullyConnected(name="fc{0}".format(i),
output_dim=l,
activation=hidden_activation,
weights_initialization=weights_initialization,
bias_initialization=bias_initialization))
self.add(FullyConnected(name="fc_output",
output_dim=output_shape,
activation=output_activation,
weights_initialization=weights_initialization,
bias_initialization=bias_initialization))
...@@ -13,7 +13,7 @@ import six ...@@ -13,7 +13,7 @@ import six
import os import os
from collections import OrderedDict from collections import OrderedDict
from bob.learn.tensorflow.layers import Layer, MaxPooling from bob.learn.tensorflow.layers import Layer, MaxPooling, Dropout
class SequenceNetwork(six.with_metaclass(abc.ABCMeta, object)): class SequenceNetwork(six.with_metaclass(abc.ABCMeta, object)):
...@@ -22,7 +22,8 @@ class SequenceNetwork(six.with_metaclass(abc.ABCMeta, object)): ...@@ -22,7 +22,8 @@ class SequenceNetwork(six.with_metaclass(abc.ABCMeta, object)):
""" """
def __init__(self, def __init__(self,
default_feature_layer=None): default_feature_layer=None,
use_gpu=False):
""" """
Base constructor Base constructor
...@@ -34,6 +35,7 @@ class SequenceNetwork(six.with_metaclass(abc.ABCMeta, object)): ...@@ -34,6 +35,7 @@ class SequenceNetwork(six.with_metaclass(abc.ABCMeta, object)):
self.default_feature_layer = default_feature_layer self.default_feature_layer = default_feature_layer
self.input_divide = 1. self.input_divide = 1.
self.input_subtract = 0. self.input_subtract = 0.
self.use_gpu=use_gpu
#self.saver = None #self.saver = None
def add(self, layer): def add(self, layer):
...@@ -45,7 +47,7 @@ class SequenceNetwork(six.with_metaclass(abc.ABCMeta, object)): ...@@ -45,7 +47,7 @@ class SequenceNetwork(six.with_metaclass(abc.ABCMeta, object)):
raise ValueError("Input `layer` must be an instance of `bob.learn.tensorflow.layers.Layer`") raise ValueError("Input `layer` must be an instance of `bob.learn.tensorflow.layers.Layer`")
self.sequence_net[layer.name] = layer self.sequence_net[layer.name] = layer
def compute_graph(self, input_data, feature_layer=None): def compute_graph(self, input_data, feature_layer=None, training=True):
""" """
Given the current network, return the Tensorflow graph Given the current network, return the Tensorflow graph
...@@ -57,11 +59,13 @@ class SequenceNetwork(six.with_metaclass(abc.ABCMeta, object)): ...@@ -57,11 +59,13 @@ class SequenceNetwork(six.with_metaclass(abc.ABCMeta, object)):
input_offset = input_data input_offset = input_data
for k in self.sequence_net.keys(): for k in self.sequence_net.keys():
current_layer = self.sequence_net[k] current_layer = self.sequence_net[k]
current_layer.create_variables(input_offset)
input_offset = current_layer.get_graph()
if feature_layer is not None and k == feature_layer: if training or not isinstance(current_layer, Dropout):
return input_offset current_layer.create_variables(input_offset)
input_offset = current_layer.get_graph()
if feature_layer is not None and k == feature_layer:
return input_offset
return input_offset return input_offset
...@@ -80,14 +84,14 @@ class SequenceNetwork(six.with_metaclass(abc.ABCMeta, object)): ...@@ -80,14 +84,14 @@ class SequenceNetwork(six.with_metaclass(abc.ABCMeta, object)):
if feature_layer is None: if feature_layer is None:
feature_layer = self.default_feature_layer feature_layer = self.default_feature_layer
return session.run([self.compute_graph(feature_placeholder, feature_layer)], feed_dict=feed_dict)[0] return session.run([self.compute_graph(feature_placeholder, feature_layer, training=False)], feed_dict=feed_dict)[0]
def dump_variables(self): def dump_variables(self):
variables = {} variables = {}
for k in self.sequence_net: for k in self.sequence_net:
# TODO: IT IS NOT SMART TESTING ALONG THIS PAGE # TODO: IT IS NOT SMART TESTING ALONG THIS PAGE
if not isinstance(self.sequence_net[k], MaxPooling): if not isinstance(self.sequence_net[k], MaxPooling) and not isinstance(self.sequence_net[k], Dropout):
variables[self.sequence_net[k].W.name] = self.sequence_net[k].W variables[self.sequence_net[k].W.name] = self.sequence_net[k].W
variables[self.sequence_net[k].b.name] = self.sequence_net[k].b variables[self.sequence_net[k].b.name] = self.sequence_net[k].b
......
...@@ -4,6 +4,7 @@ __path__ = extend_path(__path__, __name__) ...@@ -4,6 +4,7 @@ __path__ = extend_path(__path__, __name__)
from .SequenceNetwork import SequenceNetwork from .SequenceNetwork import SequenceNetwork
from .Lenet import Lenet from .Lenet import Lenet
from .LenetDropout import LenetDropout
from .MLP import MLP from .MLP import MLP
# gets sphinx autodoc done right - don't remove it # gets sphinx autodoc done right - don't remove it
......
...@@ -64,7 +64,7 @@ def main(): ...@@ -64,7 +64,7 @@ def main():
train_objects = db.objects(protocol="male", groups="world") train_objects = db.objects(protocol="male", groups="world")
train_labels = [o.client_id for o in train_objects] train_labels = [o.client_id for o in train_objects]
train_file_names = [o.make_path( train_file_names = [o.make_path(
directory="/remote/lustre/2/temp/tpereira/FACEREC_EXPERIMENTS/mobio_male/lda/preprocessed", directory="/idiap/user/tpereira/face/baselines/eigenface/preprocessed",
extension=".hdf5") extension=".hdf5")
for o in train_objects] for o in train_objects]
...@@ -77,7 +77,7 @@ def main(): ...@@ -77,7 +77,7 @@ def main():
validation_objects = db.objects(protocol="male", groups="dev") validation_objects = db.objects(protocol="male", groups="dev")
validation_labels = [o.client_id for o in validation_objects] validation_labels = [o.client_id for o in validation_objects]
validation_file_names = [o.make_path( validation_file_names = [o.make_path(
directory="/remote/lustre/2/temp/tpereira/FACEREC_EXPERIMENTS/mobio_male/lda/preprocessed", directory="/idiap/user/tpereira/face/baselines/eigenface/preprocessed",
extension=".hdf5") extension=".hdf5")
for o in validation_objects] for o in validation_objects]
......
...@@ -23,7 +23,7 @@ import tensorflow as tf ...@@ -23,7 +23,7 @@ import tensorflow as tf
from .. import util from .. import util
SEED = 10 SEED = 10
from bob.learn.tensorflow.data import MemoryDataShuffler, TextDataShuffler from bob.learn.tensorflow.data import MemoryDataShuffler, TextDataShuffler
from bob.learn.tensorflow.network import Lenet, MLP from bob.learn.tensorflow.network import Lenet, MLP, LenetDropout
from bob.learn.tensorflow.trainers import SiameseTrainer from bob.learn.tensorflow.trainers import SiameseTrainer
from bob.learn.tensorflow.loss import ContrastiveLoss from bob.learn.tensorflow.loss import ContrastiveLoss
import numpy import numpy
...@@ -39,7 +39,7 @@ def main(): ...@@ -39,7 +39,7 @@ def main():
perc_train = 0.9 perc_train = 0.9
# Loading data # Loading data
mnist = True mnist = False
if mnist: if mnist:
train_data, train_labels, validation_data, validation_labels = \ train_data, train_labels, validation_data, validation_labels = \
...@@ -66,7 +66,7 @@ def main(): ...@@ -66,7 +66,7 @@ def main():
train_objects = db.objects(protocol="male", groups="world") train_objects = db.objects(protocol="male", groups="world")
train_labels = [o.client_id for o in train_objects] train_labels = [o.client_id for o in train_objects]
train_file_names = [o.make_path( train_file_names = [o.make_path(
directory="/remote/lustre/2/temp/tpereira/FACEREC_EXPERIMENTS/mobio_male/lda/preprocessed", directory="/idiap/user/tpereira/face/baselines/eigenface/preprocessed",
extension=".hdf5") extension=".hdf5")
for o in train_objects] for o in train_objects]
...@@ -78,7 +78,7 @@ def main(): ...@@ -78,7 +78,7 @@ def main():
validation_objects = db.objects(protocol="male", groups="dev") validation_objects = db.objects(protocol="male", groups="dev")
validation_labels = [o.client_id for o in validation_objects] validation_labels = [o.client_id for o in validation_objects]
validation_file_names = [o.make_path( validation_file_names = [o.make_path(
directory="/remote/lustre/2/temp/tpereira/FACEREC_EXPERIMENTS/mobio_male/lda/preprocessed", directory="/idiap/user/tpereira/face/baselines/eigenface/preprocessed",
extension=".hdf5") extension=".hdf5")
for o in validation_objects] for o in validation_objects]
...@@ -92,7 +92,8 @@ def main(): ...@@ -92,7 +92,8 @@ def main():
cnn = True cnn = True
if cnn: if cnn:
lenet = Lenet(default_feature_layer="fc2", n_classes=n_classes) lenet = Lenet(default_feature_layer="fc2", n_classes=n_classes, conv1_output=4, conv2_output=8,use_gpu=USE_GPU)
#lenet = LenetDropout(default_feature_layer="fc2", n_classes=n_classes, conv1_output=4, conv2_output=8, use_gpu=USE_GPU)
loss = ContrastiveLoss() loss = ContrastiveLoss()
trainer = SiameseTrainer(architecture=lenet, trainer = SiameseTrainer(architecture=lenet,
......
...@@ -45,7 +45,6 @@ class SiameseTrainer(Trainer): ...@@ -45,7 +45,6 @@ class SiameseTrainer(Trainer):
snapshot=snapshot snapshot=snapshot
) )
def train(self, train_data_shuffler, validation_data_shuffler=None): def train(self, train_data_shuffler, validation_data_shuffler=None):
""" """
Do the loop forward --> backward --| Do the loop forward --> backward --|
...@@ -116,8 +115,11 @@ class SiameseTrainer(Trainer): ...@@ -116,8 +115,11 @@ class SiameseTrainer(Trainer):
train_right_graph) train_right_graph)
# Preparing the optimizer # Preparing the optimizer
step = tf.Variable(0)
self.optimizer._learning_rate = learning_rate self.optimizer._learning_rate = learning_rate
optimizer = self.optimizer.minimize(loss_train, global_step=tf.Variable(0)) optimizer = self.optimizer.minimize(loss_train, global_step=step)
#optimizer = tf.train.MomentumOptimizer(learning_rate, momentum=0.99, use_locking=False,
# name='Momentum').minimize(loss_train, global_step=step)
print("Initializing !!") print("Initializing !!")
# Training # Training
...@@ -146,6 +148,7 @@ class SiameseTrainer(Trainer): ...@@ -146,6 +148,7 @@ class SiameseTrainer(Trainer):
for step in range(self.iterations): for step in range(self.iterations):
_, l, lr, summary = session.run([optimizer, loss_train, learning_rate, merged]) _, l, lr, summary = session.run([optimizer, loss_train, learning_rate, merged])
#_, l, lr= session.run([optimizer, loss_train, learning_rate])
train_writer.add_summary(summary, step) train_writer.add_summary(summary, step)
if validation_data_shuffler is not None and step % self.snapshot == 0: if validation_data_shuffler is not None and step % self.snapshot == 0:
...@@ -160,4 +163,3 @@ class SiameseTrainer(Trainer): ...@@ -160,4 +163,3 @@ class SiameseTrainer(Trainer):
thread_pool.request_stop() thread_pool.request_stop()
thread_pool.join(threads) thread_pool.join(threads)
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment