Commit 421b1ded authored by Tiago de Freitas Pereira's avatar Tiago de Freitas Pereira
Browse files

Using TF Slim

parent ca6f492d
Pipeline #8064 failed with stages
in 1 minute and 32 seconds
......@@ -9,11 +9,8 @@
import tensorflow as tf
from .SequenceNetwork import SequenceNetwork
from ..layers import Conv2D, FullyConnected, MaxPooling
import bob.learn.tensorflow
from bob.learn.tensorflow.initialization import Xavier
from bob.learn.tensorflow.initialization import Constant
#from bob.learn.tensorflow.initialization import Xavier
#from bob.learn.tensorflow.initialization import Constant
class Chopra(SequenceNetwork):
"""Class that creates the architecture presented in the paper:
......@@ -60,16 +57,16 @@ class Chopra(SequenceNetwork):
seed:
"""
def __init__(self,
conv1_kernel_size=7,
conv1_kernel_size=[7, 7],
conv1_output=15,
pooling1_size=[1, 2, 2, 1],
pooling1_size=[2, 2],
conv2_kernel_size=6,
conv2_kernel_size=[6, 6],
conv2_output=45,
pooling2_size=[1, 4, 3, 1],
pooling2_size=[4, 3],
fc1_output=250,
default_feature_layer="fc1",
......@@ -78,27 +75,18 @@ class Chopra(SequenceNetwork):
use_gpu=False,
batch_norm=False):
slim = tf.contrib.slim
graph = slim.conv2d(data_placeholder, conv1_output, conv1_kernel_size, activation_fn=tf.nn.relu,
stride=2, scope='conv1')
graph = slim.max_pool2d(graph, pooling1_size, scope='pool1')
graph = slim.conv2d(graph, conv2_output, conv2_kernel_size, activation_fn=tf.nn.relu,
stride=2, scope='conv2')
graph = slim.max_pool2d(graph, pooling2_size, scope='pool2')
graph = slim.flatten(graph, scope='flatten1')
graph = slim.fully_connected(graph, fc1_output, scope='fc1')
super(Chopra, self).__init__(default_feature_layer=default_feature_layer,
use_gpu=use_gpu)
self.add(Conv2D(name="conv1", kernel_size=conv1_kernel_size,
filters=conv1_output,
activation=None,
weights_initialization=Xavier(seed=seed, use_gpu=self.use_gpu),
bias_initialization=Constant(use_gpu=self.use_gpu),
batch_norm=batch_norm
))
self.add(MaxPooling(name="pooling1", shape=pooling1_size, activation=tf.nn.tanh, batch_norm=False))
self.add(Conv2D(name="conv2", kernel_size=conv2_kernel_size,
filters=conv2_output,
activation=None,
weights_initialization=Xavier(seed=seed, use_gpu=self.use_gpu),
bias_initialization=Constant(use_gpu=self.use_gpu),
batch_norm=batch_norm))
self.add(MaxPooling(name="pooling2", shape=pooling2_size, activation=tf.nn.tanh, batch_norm=False))
self.add(FullyConnected(name="fc1", output_dim=fc1_output,
activation=None,
weights_initialization=Xavier(seed=seed, use_gpu=self.use_gpu),
bias_initialization=Constant(use_gpu=self.use_gpu), batch_norm=False))
......@@ -10,7 +10,6 @@ import six
import numpy
import pickle
from collections import OrderedDict
from bob.learn.tensorflow.layers import Layer, MaxPooling, Dropout, Conv2D, FullyConnected
from bob.learn.tensorflow.utils.session import Session
......@@ -27,85 +26,17 @@ class SequenceNetwork(six.with_metaclass(abc.ABCMeta, object)):
"""
def __init__(self,
graph=None,
default_feature_layer=None,
use_gpu=False):
self.sequence_net = OrderedDict()
self.base_graph = graph
self.default_feature_layer = default_feature_layer
self.input_divide = 1.
self.input_subtract = 0.
self.use_gpu = use_gpu
self.pickle_architecture = None# The trainer triggers this
self.deployment_shape = None# The trainer triggers this
# Inference graph
self.inference_graph = None
self.inference_placeholder = None
def __del__(self):
tf.reset_default_graph()
def add(self, layer):
"""
Add a :py:class:`bob.learn.tensorflow.layers.Layer` in the sequence network
"""
if not isinstance(layer, Layer):
raise ValueError("Input `layer` must be an instance of `bob.learn.tensorflow.layers.Layer`")
self.sequence_net[layer.name] = layer
def pickle_net(self, shape):
"""
Pickle the class
** Parameters **
shape: Shape of the input data for deployment
"""
self.pickle_architecture = pickle.dumps(self.sequence_net)
self.deployment_shape = shape
def compute_graph(self, input_data, feature_layer=None, training=True):
"""Given the current network, return the Tensorflow graph
**Parameter**
input_data: tensorflow placeholder as input data
feature_layer: Name of the :py:class:`bob.learn.tensorflow.layers.Layer` that you want to "cut".
If `None` will run the graph until the end.
training: If `True` will generating the graph for training
"""
input_offset = input_data
for k in self.sequence_net.keys():
current_layer = self.sequence_net[k]
if training or not isinstance(current_layer, Dropout):
current_layer.create_variables(input_offset)
input_offset = current_layer.get_graph(training_phase=training)
if feature_layer is not None and k == feature_layer:
return input_offset
return input_offset
def compute_inference_graph(self, feature_layer=None):
"""Generate a graph for feature extraction
**Parameters**
placeholder: tensorflow placeholder as input data
"""
if feature_layer is None:
feature_layer = self.default_feature_layer
self.inference_graph = self.compute_graph(self.inference_placeholder, feature_layer, training=False)
def compute_inference_placeholder(self, data_shape):
self.inference_placeholder = tf.placeholder(tf.float32, shape=data_shape, name="feature")
def __call__(self, data, feature_layer=None):
"""Run a graph and compute the embeddings
......@@ -136,28 +67,9 @@ class SequenceNetwork(six.with_metaclass(abc.ABCMeta, object)):
def predict(self, data):
return numpy.argmax(self(data), 1)
def dump_variables(self):
"""
Return all the tensorflow `variables <https://www.tensorflow.org/versions/r0.11/api_docs/python/state_ops.html#Variable>`_ used in the graph
"""
variables = {}
for k in self.sequence_net:
# TODO: IT IS NOT SMART TESTING ALONG THIS PAGE
if not isinstance(self.sequence_net[k], MaxPooling) and not isinstance(self.sequence_net[k], Dropout):
variables[self.sequence_net[k].W.name] = self.sequence_net[k].W
variables[self.sequence_net[k].b.name] = self.sequence_net[k].b
# Dumping batch norm variables
if self.sequence_net[k].batch_norm:
variables[self.sequence_net[k].beta.name] = self.sequence_net[k].beta
variables[self.sequence_net[k].gamma.name] = self.sequence_net[k].gamma
#variables[self.sequence_net[k].mean.name] = self.sequence_net[k].mean
#variables[self.sequence_net[k].var.name] = self.sequence_net[k].var
return variables
"""
def variable_summaries(self, var, name):
"""Attach a lot of summaries to a Tensor."""
#Attach a lot of summaries to a Tensor.
with tf.name_scope('summaries'):
mean = tf.reduce_mean(var)
tf.summary.scalar('mean/' + name, mean)
......@@ -168,13 +80,7 @@ class SequenceNetwork(six.with_metaclass(abc.ABCMeta, object)):
tf.summary.scalar('min/' + name, tf.reduce_min(var))
tf.summary.histogram(name, var)
def generate_summaries(self):
for k in self.sequence_net.keys():
current_layer = self.sequence_net[k]
if not isinstance(self.sequence_net[k], MaxPooling) and not isinstance(self.sequence_net[k], Dropout):
self.variable_summaries(current_layer.W, current_layer.name + '/weights')
self.variable_summaries(current_layer.b, current_layer.name + '/bias')
"""
def compute_magic_number(self, hypothetic_image_dimensions=(28, 28, 1)):
"""
......@@ -213,105 +119,6 @@ class SequenceNetwork(six.with_metaclass(abc.ABCMeta, object)):
return samples_per_sample
def save_hdf5(self, hdf5):
"""
Save the state of the network in HDF5 format
**Parameters**
hdf5: An opened :py:class:`bob.io.base.HDF5File`
step: The current training step. If not `None`, will create a HDF5 group with the current step.
"""
def create_groups(path):
split_path = path.split("/")
for i in range(0, len(split_path)-1):
p = split_path[i]
if not hdf5.has_group(p):
hdf5.create_group(p)
# Saving the architecture
if self.pickle_architecture is not None:
hdf5.set('architecture', self.pickle_architecture)
hdf5.set('deployment_shape', self.deployment_shape)
# Directory that stores the tensorflow variables
hdf5.create_group('/tensor_flow')
hdf5.cd('/tensor_flow')
# Iterating the variables of the model
for v in self.dump_variables().keys():
create_groups(v)
hdf5.set(v, self.dump_variables()[v].eval())
hdf5.cd('..')
hdf5.set('input_divide', self.input_divide)
hdf5.set('input_subtract', self.input_subtract)
def turn_gpu_onoff(self, state=True):
for k in self.sequence_net:
self.sequence_net[k].weights_initialization.use_gpu = state
self.sequence_net[k].bias_initialization.use_gpu = state
def load_variables_only(self, hdf5):
"""
Load the variables of the model
"""
session = Session.instance().session
hdf5.cd('/tensor_flow')
for k in self.sequence_net:
# TODO: IT IS NOT SMART TESTING ALONG THIS PAGE
if not isinstance(self.sequence_net[k], MaxPooling):
self.sequence_net[k].W.assign(hdf5.read(self.sequence_net[k].W.name)).eval(session=session)
session.run(self.sequence_net[k].W)
self.sequence_net[k].b.assign(hdf5.read(self.sequence_net[k].b.name)).eval(session=session)
session.run(self.sequence_net[k].b)
if self.sequence_net[k].batch_norm:
self.sequence_net[k].beta.assign(hdf5.read(self.sequence_net[k].beta.name)).eval(session=session)
self.sequence_net[k].gamma.assign(hdf5.read(self.sequence_net[k].gamma.name)).eval(session=session)
hdf5.cd("..")
def load_hdf5(self, hdf5, shape=None, batch=1, use_gpu=False):
"""
Load the network from scratch.
This will build the graphs
**Parameters**
hdf5: The saved network in the :py:class:`bob.io.base.HDF5File` format
shape: Input shape of the network. If `None`, the internal shape will be assumed
session: An opened tensorflow `session <https://www.tensorflow.org/versions/r0.11/api_docs/python/client.html#Session>`_. If `None`, a new one will be opened
batch: The size of the batch
use_gpu: Load all the variables in the GPU?
"""
session = Session.instance().session
# Loading the normalization parameters
self.input_divide = hdf5.read('input_divide')
self.input_subtract = hdf5.read('input_subtract')
# Loading architecture
self.sequence_net = pickle.loads(hdf5.read('architecture'))
self.deployment_shape = hdf5.read('deployment_shape')
self.turn_gpu_onoff(use_gpu)
if shape is None:
shape = self.deployment_shape
shape[0] = batch
# Loading variables
place_holder = tf.placeholder(tf.float32, shape=shape, name="load")
self.compute_graph(place_holder)
tf.global_variables_initializer().run(session=session)
self.load_variables_only(hdf5, session)
def save(self, saver, path):
......
......@@ -69,7 +69,7 @@ class Trainer(object):
"""
def __init__(self,
architecture,
graph,
optimizer=tf.train.AdamOptimizer(),
use_gpu=False,
loss=None,
......@@ -93,10 +93,10 @@ class Trainer(object):
verbosity_level=2):
if not isinstance(architecture, SequenceNetwork):
raise ValueError("`architecture` should be instance of `SequenceNetwork`")
#if not isinstance(graph, SequenceNetwork):
# raise ValueError("`architecture` should be instance of `SequenceNetwork`")
self.architecture = architecture
self.graph = graph
self.optimizer_class = optimizer
self.use_gpu = use_gpu
self.loss = loss
......@@ -140,8 +140,8 @@ class Trainer(object):
def __del__(self):
tf.reset_default_graph()
"""
def compute_graph(self, data_shuffler, prefetch=False, name="", training=True):
"""
Computes the graph for the trainer.
** Parameters **
......@@ -150,7 +150,6 @@ class Trainer(object):
prefetch: Uses prefetch
name: Name of the graph
training: Is it a training graph?
"""
# Defining place holders
if prefetch:
......@@ -177,6 +176,7 @@ class Trainer(object):
graph = self.loss(network_graph, label_batch)
return graph
"""
def get_feed_dict(self, data_shuffler):
"""
......@@ -214,43 +214,24 @@ class Trainer(object):
logger.info("Loss training set step={0} = {1}".format(step, l))
self.train_summary_writter.add_summary(summary, step)
def compute_validation(self, data_shuffler, step):
"""
Computes the loss in the validation set
** Parameters **
session: Tensorflow session
data_shuffler: The data shuffler to be used
step: Iteration number
"""
# Opening a new session for validation
feed_dict = self.get_feed_dict(data_shuffler)
l = self.session.run(self.validation_graph, feed_dict=feed_dict)
if self.validation_summary_writter is None:
self.validation_summary_writter = tf.summary.FileWriter(os.path.join(self.temp_dir, 'validation'), self.session.graph)
summaries = [summary_pb2.Summary.Value(tag="loss", simple_value=float(l))]
self.validation_summary_writter.add_summary(summary_pb2.Summary(value=summaries), step)
logger.info("Loss VALIDATION set step={0} = {1}".format(step, l))
"""
def create_general_summary(self):
"""
Creates a simple tensorboard summary with the value of the loss and learning rate
"""
# Train summary
tf.summary.scalar('loss', self.training_graph)
tf.summary.scalar('lr', self.learning_rate)
return tf.summary.merge_all()
def start_thread(self):
"""
Start pool of threads for pre-fetching
**Parameters**
session: Tensorflow session
"""
threads = []
for n in range(3):
......@@ -261,12 +242,12 @@ class Trainer(object):
return threads
def load_and_enqueue(self):
"""
Injecting data in the place holder queue
**Parameters**
session: Tensorflow session
"""
while not self.thread_pool.should_stop():
[train_data, train_labels] = self.train_data_shuffler.get_batch()
......@@ -277,51 +258,7 @@ class Trainer(object):
self.session.run(self.enqueue_op, feed_dict=feed_dict)
def bootstrap_graphs(self, train_data_shuffler, validation_data_shuffler):
"""
Create all the necessary graphs for training, validation and inference graphs
"""
# Creating train graph
self.training_graph = self.compute_graph(train_data_shuffler, prefetch=self.prefetch, name="train")
tf.add_to_collection("training_graph", self.training_graph)
# Creating inference graph
self.architecture.compute_inference_placeholder(train_data_shuffler.deployment_shape)
self.architecture.compute_inference_graph()
tf.add_to_collection("inference_placeholder", self.architecture.inference_placeholder)
tf.add_to_collection("inference_graph", self.architecture.inference_graph)
# Creating validation graph
if validation_data_shuffler is not None:
self.validation_graph = self.compute_graph(validation_data_shuffler, name="validation", training=False)
tf.add_to_collection("validation_graph", self.validation_graph)
self.bootstrap_placeholders(train_data_shuffler, validation_data_shuffler)
def bootstrap_placeholders(self, train_data_shuffler, validation_data_shuffler):
"""
Persist the placeholders
** Parameters **
train_data_shuffler: Data shuffler for training
validation_data_shuffler: Data shuffler for validation
"""
# Persisting the placeholders
if self.prefetch:
batch, label = train_data_shuffler.get_placeholders_forprefetch()
else:
batch, label = train_data_shuffler.get_placeholders()
tf.add_to_collection("train_placeholder_data", batch)
tf.add_to_collection("train_placeholder_label", label)
# Creating validation graph
if validation_data_shuffler is not None:
batch, label = validation_data_shuffler.get_placeholders()
tf.add_to_collection("validation_placeholder_data", batch)
tf.add_to_collection("validation_placeholder_label", label)
"""
def bootstrap_graphs_fromfile(self, train_data_shuffler, validation_data_shuffler):
"""
......@@ -352,24 +289,6 @@ class Trainer(object):
return saver
def bootstrap_placeholders_fromfile(self, train_data_shuffler, validation_data_shuffler):
"""
Load placeholders from file
** Parameters **
train_data_shuffler: Data shuffler for training
validation_data_shuffler: Data shuffler for validation
"""
train_data_shuffler.set_placeholders(tf.get_collection("train_placeholder_data")[0],
tf.get_collection("train_placeholder_label")[0])
if validation_data_shuffler is not None:
train_data_shuffler.set_placeholders(tf.get_collection("validation_placeholder_data")[0],
tf.get_collection("validation_placeholder_label")[0])
def train(self, train_data_shuffler, validation_data_shuffler=None):
"""
Train the network:
......@@ -387,7 +306,10 @@ class Trainer(object):
logger.info("Initializing !!")
# Pickle the architecture to save
self.architecture.pickle_net(train_data_shuffler.deployment_shape)
#self.architecture.pickle_net(train_data_shuffler.deployment_shape)
if not isinstance(tf.Tensor, self.graph):
raise NotImplemented("Not tensor still not implemented")
self.session = Session.instance(new=True).session
......@@ -400,8 +322,6 @@ class Trainer(object):
else:
start_step = 0
# Bootstraping all the graphs
self.bootstrap_graphs(train_data_shuffler, validation_data_shuffler)
# TODO: find an elegant way to provide this as a parameter of the trainer
self.global_step = tf.Variable(0, trainable=False, name="global_step")
......@@ -414,24 +334,19 @@ class Trainer(object):
tf.add_to_collection("learning_rate", self.learning_rate)
# Train summary
self.summaries_train = self.create_general_summary()
tf.add_to_collection("summaries_train", self.summaries_train)
tf.add_to_collection("summaries_train", self.summaries_train)
tf.global_variables_initializer().run(session=self.session)
# Original tensorflow saver object
saver = tf.train.Saver(var_list=tf.global_variables())
if isinstance(train_data_shuffler, OnlineSampling):
train_data_shuffler.set_feature_extractor(self.architecture, session=self.session)
#if isinstance(train_data_shuffler, OnlineSampling):
# train_data_shuffler.set_feature_extractor(self.architecture, session=self.session)
# Start a thread to enqueue data asynchronously, and hide I/O latency.
if self.prefetch:
self.thread_pool = tf.train.Coordinator()
tf.train.start_queue_runners(coord=self.thread_pool, sess=self.session)
threads = self.start_thread()
#if self.prefetch:
# self.thread_pool = tf.train.Coordinator()
# tf.train.start_queue_runners(coord=self.thread_pool, sess=self.session)
# threads = self.start_thread()
# TENSOR BOARD SUMMARY
self.train_summary_writter = tf.summary.FileWriter(os.path.join(self.temp_dir, 'train'), self.session.graph)
......@@ -443,12 +358,12 @@ class Trainer(object):
self.train_summary_writter.add_summary(summary_pb2.Summary(value=[summary]), step)
# Running validation
if validation_data_shuffler is not None and step % self.validation_snapshot == 0:
self.compute_validation(validation_data_shuffler, step)
#if validation_data_shuffler is not None and step % self.validation_snapshot == 0:
# self.compute_validation(validation_data_shuffler, step)
if self.analizer is not None:
self.validation_summary_writter.add_summary(self.analizer(
validation_data_shuffler, self.architecture, self.session), step)
# if self.analizer is not None:
# self.validation_summary_writter.add_summary(self.analizer(
# validation_data_shuffler, self.architecture, self.session), step)
# Taking snapshot
if step % self.snapshot == 0:
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment