Added a layer for initialization

parent 7fb3c1de
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
# @date: Mon 05 Sep 2016 16:35 CEST
import logging
logger = logging.getLogger("bob.learn.tensorflow")
from .Initialization import Initialization
import tensorflow as tf
class Constant(Initialization):
"""
Implements the constant initialization.
This is usually used to initialize biases.
This tip were extracted from here
http://www.deeplearningbook.org/contents/optimization.html
page: 302
"""
def __init__(self, constant_value=0.1, use_gpu=False):
self.constant_value = constant_value
super(Constant, self).__init__(seed=None, use_gpu=use_gpu)
def __call__(self, shape, name):
initializer = tf.constant(self.constant_value, shape=shape)
if self.use_gpu:
with tf.device("/gpu"):
return tf.get_variable(name, initializer=initializer, dtype=tf.float32)
else:
with tf.device("/cpu"):
return tf.get_variable(name, initializer=initializer, dtype=tf.float32)
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
# @date: Mon 05 Sep 2016 16:35 CEST
import logging
logger = logging.getLogger("bob.learn.tensorflow")
from .Initialization import Initialization
import tensorflow as tf
class Gaussian(Initialization):
"""
Implements Gaussian normalization
"""
def __init__(self, mean=0.,
std=1.,
seed=10.,
use_gpu=False):
self.mean = mean
self.std = std
super(Gaussian, self).__init__(seed, use_gpu=use_gpu)
def __call__(self, shape, name):
if len(shape) == 4:
in_out = shape[0] * shape[1] * shape[2] + shape[3]
else:
in_out = shape[0] + shape[1]
initializer = tf.truncated_normal(shape,
mean=self.mean,
stddev=self.std,
seed=self.seed)
if self.use_gpu:
with tf.device("/gpu"):
return tf.get_variable(name, initializer=initializer, dtype=tf.float32)
else:
with tf.device("/cpu"):
return tf.get_variable(name, initializer=initializer, dtype=tf.float32)
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
# @date: Mon 05 Sep 2016 16:35 CEST
import logging
logger = logging.getLogger("bob.learn.tensorflow")
class Initialization(object):
"""
Base function for initialization.
"""
def __init__(self, seed=10., use_gpu=False):
"""
Default constructor
**Parameters**
shape: Shape of the input vector
seed: Seed for the pseudo random number generator
use_gpu: Variable stored in the GPU
"""
self.seed = seed
self.use_gpu = use_gpu
def __call__(self, shape, name):
NotImplementedError("Please implement this function in derived classes")
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
# @date: Mon 05 Sep 2016 16:35 CEST
import logging
logger = logging.getLogger("bob.learn.tensorflow")
from .Initialization import Initialization
import tensorflow as tf
class SimplerXavier(Initialization):
"""
Implements the A SIMPLER VERSION of the classic and well used Xavier initialization as in
Glorot, Xavier, and Yoshua Bengio. "Understanding the difficulty of training deep feedforward neural networks." Aistats. Vol. 9. 2010.
Basically the initialization is Gaussian distribution with mean 0 and variance:
Var(W) = 1/sqrt(n_{in});
where n is the total number of parameters for input and output.
"""
def __init__(self, seed=10., use_gpu=False):
super(SimplerXavier, self).__init__(seed, use_gpu=use_gpu)
def __call__(self, shape, name):
if len(shape) == 4:
in_out = shape[0] * shape[1] * shape[2]
else:
in_out = shape[0]
import math
stddev = math.sqrt(1.0 / in_out) # XAVIER INITIALIZER (GAUSSIAN)
initializer = tf.truncated_normal(shape, stddev=stddev, seed=self.seed)
if self.use_gpu:
with tf.device("/gpu"):
return tf.get_variable(name, initializer=initializer, dtype=tf.float32)
else:
with tf.device("/cpu"):
return tf.get_variable(name, initializer=initializer, dtype=tf.float32)
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
# @date: Mon 05 Sep 2016 16:35 CEST
import logging
logger = logging.getLogger("bob.learn.tensorflow")
from .Initialization import Initialization
import tensorflow as tf
class Xavier(Initialization):
"""
Implements the classic and well used Xavier initialization as in
Glorot, Xavier, and Yoshua Bengio. "Understanding the difficulty of training deep feedforward neural networks." Aistats. Vol. 9. 2010.
Basically the initialization is Gaussian distribution with mean 0 and variance:
Var(W) = 1/sqrt(n_{in} + n_{out});
where n is the total number of parameters for input and output.
"""
def __init__(self, seed=10., use_gpu=False):
super(Xavier, self).__init__(seed, use_gpu=use_gpu)
def __call__(self, shape, name):
if len(shape) == 4:
in_out = shape[0] * shape[1] * shape[2] + shape[3]
else:
in_out = shape[0] + shape[1]
import math
stddev = math.sqrt(2.0 / in_out) # XAVIER INITIALIZER (GAUSSIAN)
initializer = tf.truncated_normal(shape, stddev=stddev, seed=self.seed)
if self.use_gpu:
with tf.device("/gpu"):
return tf.get_variable(name, initializer=initializer, dtype=tf.float32)
else:
with tf.device("/cpu"):
return tf.get_variable(name, initializer=initializer, dtype=tf.float32)
# see https://docs.python.org/3/library/pkgutil.html
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
from .Initialization import Initialization
from .Xavier import Xavier
from .SimplerXavier import SimplerXavier
from .Gaussian import Gaussian
from .Constant import Constant
# gets sphinx autodoc done right - don't remove it
__all__ = [_ for _ in dir() if not _.startswith('_')]
......@@ -4,8 +4,9 @@
# @date: Wed 11 May 2016 17:38 CEST
import tensorflow as tf
from bob.learn.tensorflow.util import *
from .Layer import Layer
from bob.learn.tensorflow.initialization import Xavier
from bob.learn.tensorflow.initialization import Constant
class Conv2D(Layer):
......@@ -17,7 +18,8 @@ class Conv2D(Layer):
def __init__(self, name, activation=None,
kernel_size=3,
filters=8,
initialization='xavier',
weights_initialization=Xavier(),
bias_initialization=Constant(),
use_gpu=False,
seed=10
):
......@@ -33,11 +35,14 @@ class Conv2D(Layer):
use_gpu: Store data in the GPU
seed: Seed for the Random number generation
"""
super(Conv2D, self).__init__(name, activation=activation, initialization='xavier',
use_gpu=use_gpu, seed=seed)
super(Conv2D, self).__init__(name=name,
activation=activation,
weights_initialization=weights_initialization,
bias_initialization=bias_initialization,
use_gpu=use_gpu,
seed=seed)
self.kernel_size = kernel_size
self.filters = filters
self.initialization = initialization
self.W = None
self.b = None
......@@ -51,10 +56,12 @@ class Conv2D(Layer):
n_channels = input_layer.get_shape().as_list()[3]
if self.W is None:
self.W = create_weight_variables([self.kernel_size, self.kernel_size, n_channels, self.filters],
seed=self.seed, name="w_" + str(self.name), use_gpu=self.use_gpu)
self.W = self.weights_initialization(shape=[self.kernel_size, self.kernel_size, n_channels, self.filters],
name="w_" + str(self.name))
if self.activation is not None:
self.b = create_bias_variables([self.filters], name="b_" + str(self.name) + "bias", use_gpu=self.use_gpu)
self.b = self.bias_initialization(shape=[self.filters],
name="b_" + str(self.name) + "bias")
def get_graph(self):
......
......@@ -4,9 +4,11 @@
# @date: Wed 11 May 2016 17:38 CEST
import tensorflow as tf
from bob.learn.tensorflow.util import *
from .Layer import Layer
from operator import mul
from bob.learn.tensorflow.initialization import Xavier
from bob.learn.tensorflow.initialization import Constant
class FullyConnected(Layer):
......@@ -15,8 +17,11 @@ class FullyConnected(Layer):
2D Convolution
"""
def __init__(self, name, output_dim, activation=None,
initialization='xavier',
def __init__(self, name,
output_dim,
activation=None,
weights_initialization=Xavier(),
bias_initialization=Constant(),
use_gpu=False,
seed=10
):
......@@ -30,8 +35,14 @@ class FullyConnected(Layer):
use_gpu: Store data in the GPU
seed: Seed for the Random number generation
"""
super(FullyConnected, self).__init__(name, activation=activation,
initialization=initialization, use_gpu=use_gpu, seed=seed)
super(FullyConnected, self).__init__(name=name,
activation=activation,
weights_initialization=weights_initialization,
bias_initialization=bias_initialization,
use_gpu=use_gpu,
seed=seed)
self.output_dim = output_dim
self.W = None
self.b = None
......@@ -41,10 +52,11 @@ class FullyConnected(Layer):
self.input_layer = input_layer
if self.W is None:
input_dim = reduce(mul, self.input_layer.get_shape().as_list()[1:])
self.W = create_weight_variables([input_dim, self.output_dim],
seed=self.seed, name="W_" + str(self.name), use_gpu=self.use_gpu)
#if self.activation is not None:
self.b = create_bias_variables([self.output_dim], name="b_" + str(self.name), use_gpu=self.use_gpu)
self.W = self.weights_initialization(shape=[input_dim, self.output_dim],
name="W_" + str(self.name))
# if self.activation is not None:
self.b = self.bias_initialization(shape=[self.output_dim],
name="b_" + str(self.name))
def get_graph(self):
......
......@@ -4,6 +4,8 @@
# @date: Wed 11 May 2016 17:38 CEST
import tensorflow as tf
from bob.learn.tensorflow.initialization import Xavier
from bob.learn.tensorflow.initialization import Constant
class Layer(object):
......@@ -12,19 +14,26 @@ class Layer(object):
Layer base class
"""
def __init__(self, name, activation=None, initialization='xavier', use_gpu=False, seed=10):
def __init__(self, name,
activation=None,
weights_initialization=Xavier(),
bias_initialization=Constant(),
use_gpu=False,
seed=10):
"""
Base constructor
**Parameters**
name: Name of the layer
activation: Tensorflow activation operation (https://www.tensorflow.org/versions/r0.10/api_docs/python/nn.html)
initialization: STILL TO BE DONE.
weights_initialization: Initialization for the weights
bias_initialization: Initialization for the biases
use_gpu: I think this is not necessary to explain
seed: Initialization seed set in Tensor flow
"""
self.name = name
self.initialization = initialization
self.weights_initialization = weights_initialization
self.bias_initialization = bias_initialization
self.use_gpu = use_gpu
self.seed = seed
......
......@@ -7,11 +7,10 @@
Class that creates the lenet architecture
"""
from ..util import *
import tensorflow as tf
from .SequenceNetwork import SequenceNetwork
from ..layers import Conv2D, FullyConnected, MaxPooling
import bob.learn.tensorflow
class Lenet(SequenceNetwork):
......@@ -44,7 +43,8 @@ class Lenet(SequenceNetwork):
"""
super(Lenet, self).__init__(default_feature_layer=default_feature_layer)
self.add(Conv2D(name="conv1", kernel_size=conv1_kernel_size, filters=conv1_output, activation=tf.nn.tanh))
self.add(Conv2D(name="conv1", kernel_size=conv1_kernel_size,
filters=conv1_output, activation=tf.nn.tanh))
self.add(MaxPooling(name="pooling1"))
self.add(Conv2D(name="conv2", kernel_size=conv2_kernel_size, filters=conv2_output, activation=tf.nn.tanh))
self.add(MaxPooling(name="pooling2"))
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment