[sphinx] Fixing issues

bob/learn/tensorflow/datashuffler/OnlineSampling.py

@@ -5,8 +5,6 @@
 
 import tensorflow as tf
 from .Base import Base
-from bob.learn.tensorflow.network import SequenceNetwork
-
 
 class OnlineSampling(object):
     """
@@ -32,8 +30,8 @@ class OnlineSampling(object):
         """
         Set the current feature extraction used in the sampling
         """
-        if not isinstance(feature_extractor, SequenceNetwork):
-            raise ValueError("Feature extractor must be a `bob.learn.tensoflow.network.SequenceNetwork` object")
+        #if not isinstance(feature_extractor, SequenceNetwork):
+        #    raise ValueError("Feature extractor must be a `bob.learn.tensoflow.network.SequenceNetwork` object")
 
         self.feature_extractor = feature_extractor
         self.session = session

bob/learn/tensorflow/network/SequenceNetwork.py

-#!/usr/bin/env python
-# vim: set fileencoding=utf-8 :
-# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
-# @date: Thu 11 Aug 2016 09:39:36 CEST
-
-
-import tensorflow as tf
-import abc
-import six
-import numpy
-import pickle
-
-from bob.learn.tensorflow.layers import Layer, MaxPooling, Dropout, Conv2D, FullyConnected
-from bob.learn.tensorflow.utils.session import Session
-
-
-class SequenceNetwork(six.with_metaclass(abc.ABCMeta, object)):
-    """
-    Sequential model is a linear stack of :py:mod:`bob.learn.tensorflow.layers`.
-
-    **Parameters**
-
-        default_feature_layer: Default layer name (:py:obj:`str`) used as a feature layer.
-
-        use_gpu: If ``True`` uses the GPU in the computation.
-    """
-
-    def __init__(self,
-                 graph=None,
-                 default_feature_layer=None,
-                 use_gpu=False):
-
-        self.base_graph = graph
-        self.default_feature_layer = default_feature_layer
-        self.use_gpu = use_gpu
-
-    def __del__(self):
-        tf.reset_default_graph()
-
-    def __call__(self, data, feature_layer=None):
-        """Run a graph and compute the embeddings
-
-        **Parameters**
-
-        data: tensorflow placeholder as input data
-
-        session: tensorflow `session <https://www.tensorflow.org/versions/r0.11/api_docs/python/client.html#Session>`_
-
-        feature_layer: Name of the :py:class:`bob.learn.tensorflow.layer.Layer` that you want to "cut".
-                       If `None` will run the graph until the end.
-        """
-
-        session = Session.instance().session
-
-        # Feeding the placeholder
-        if self.inference_placeholder is None:
-            self.compute_inference_placeholder(data.shape[1:])
-        feed_dict = {self.inference_placeholder: data}
-
-        if self.inference_graph is None:
-            self.compute_inference_graph(self.inference_placeholder, feature_layer)
-
-        embedding = session.run([self.inference_graph], feed_dict=feed_dict)[0]
-
-        return embedding
-
-    def predict(self, data):
-        return numpy.argmax(self(data), 1)
-
-    """
-    def variable_summaries(self, var, name):
-        #Attach a lot of summaries to a Tensor.
-        with tf.name_scope('summaries'):
-            mean = tf.reduce_mean(var)
-            tf.summary.scalar('mean/' + name, mean)
-            with tf.name_scope('stddev'):
-                stddev = tf.sqrt(tf.reduce_sum(tf.square(var - mean)))
-            tf.summary.scalar('sttdev/' + name, stddev)
-            tf.summary.scalar('max/' + name, tf.reduce_max(var))
-            tf.summary.scalar('min/' + name, tf.reduce_min(var))
-            tf.summary.histogram(name, var)
-
-    """
-
-    def compute_magic_number(self, hypothetic_image_dimensions=(28, 28, 1)):
-        """
-
-        Here it is done an estimative of the capacity of DNN.
-
-        **Parameters**
-
-            hypothetic_image_dimensions: Possible image dimentions `w, h, c` (width x height x channels)
-        """
-
-        stride = 1# ALWAYS EQUALS TO ONE
-        current_image_dimensions = list(hypothetic_image_dimensions)
-
-        samples_per_sample = 0
-        flatten_dimension = numpy.prod(current_image_dimensions)
-        for k in self.sequence_net.keys():
-            current_layer = self.sequence_net[k]
-
-            if isinstance(current_layer, Conv2D):
-                #samples_per_sample += current_layer.filters * current_layer.kernel_size * current_image_dimensions[0] + current_layer.filters
-                #samples_per_sample += current_layer.filters * current_layer.kernel_size * current_image_dimensions[1] + current_layer.filters
-                samples_per_sample += current_layer.filters * current_image_dimensions[0] * current_image_dimensions[1] + current_layer.filters
-
-                current_image_dimensions[2] = current_layer.filters
-                flatten_dimension = numpy.prod(current_image_dimensions)
-
-            if isinstance(current_layer, MaxPooling):
-                current_image_dimensions[0] /= 2
-                current_image_dimensions[1] /= 2
-                flatten_dimension = current_image_dimensions[0] * current_image_dimensions[1] * current_image_dimensions[2]
-
-            if isinstance(current_layer, FullyConnected):
-                samples_per_sample += flatten_dimension * current_layer.output_dim + current_layer.output_dim
-                flatten_dimension = current_layer.output_dim
-
-        return samples_per_sample
-
-
-    def save(self, saver, path):
-
-        session = Session.instance().session
-
-        open(path+"_sequence_net.pickle", 'wb').write(self.pickle_architecture)
-        return saver.save(session, path)
-
-    def load(self, path, clear_devices=False, session_from_scratch=False):
-
-        session = Session.instance(new=session_from_scratch).session
-
-        self.sequence_net = pickle.loads(open(path+"_sequence_net.pickle", 'rb').read())
-        if clear_devices:
-            saver = tf.train.import_meta_graph(path + ".meta", clear_devices=clear_devices)
-        else:
-            saver = tf.train.import_meta_graph(path + ".meta")
-
-        saver.restore(session, path)
-        self.inference_graph = tf.get_collection("inference_graph")[0]
-        self.inference_placeholder = tf.get_collection("inference_placeholder")[0]
-
-        return saver

bob/learn/tensorflow/network/__init__.py

-from .SequenceNetwork import SequenceNetwork
 from .Chopra import Chopra
 from .Dummy import Dummy
 from .MLP import MLP
 from .Embedding import Embedding
-#from .Input import Input
 
 # gets sphinx autodoc done right - don't remove it
 def __appropriate__(*args):
@@ -20,7 +18,6 @@ def __appropriate__(*args):
   for obj in args: obj.__module__ = __name__
 
 __appropriate__(
-    SequenceNetwork,
     Chopra,
     Dummy,
     MLP,

bob/learn/tensorflow/test/test_cnn_load.py

-#!/usr/bin/env python
-# vim: set fileencoding=utf-8 :
-# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
-# @date: Thu 13 Oct 2016 13:35 CEST
-
-
-"""
-Some unit tests that create networks on the fly
-"""
-
-import numpy
-import pkg_resources
-from bob.learn.tensorflow.utils import load_mnist
-from bob.learn.tensorflow.network import SequenceNetwork
-from bob.learn.tensorflow.datashuffler import Memory
-import tensorflow as tf
-
-
-def validate_network(validation_data, validation_labels, network):
-    # Testing
-    validation_data_shuffler = Memory(validation_data, validation_labels,
-                                      input_shape=[28, 28, 1],
-                                      batch_size=400)
-
-    [data, labels] = validation_data_shuffler.get_batch()
-    predictions = network.predict(data)
-    accuracy = 100. * numpy.sum(predictions == labels) / predictions.shape[0]
-
-    return accuracy
-
-
-def test_load_test_cnn():
-    tf.reset_default_graph()
-
-    _, _, validation_data, validation_labels = load_mnist()
-
-    # Creating datashufflers
-    validation_data = numpy.reshape(validation_data, (validation_data.shape[0], 28, 28, 1))
-    network = SequenceNetwork()
-    network.load(pkg_resources.resource_filename(__name__, 'data/cnn_mnist/model.ckp'), session_from_scratch=True)
-
-    accuracy = validate_network(validation_data, validation_labels, network)
-    assert accuracy > 80
-    del network

bob/learn/tensorflow/test/test_cnn_scratch.py

@@ -25,21 +25,20 @@ directory = "./temp/cnn_scratch"
 slim = tf.contrib.slim
 
 
-def scratch_network():
-    # Creating a random network
-    inputs = dict()
-    inputs['data'] = tf.placeholder(tf.float32, shape=[None, 28, 28, 1], name="train_data")
-    inputs['label'] = tf.placeholder(tf.int64, shape=[None], name="train_label")
+def scratch_network(train_data_shuffler):
+
+    inputs = train_data_shuffler("data", from_queue=False)
 
+    # Creating a random network
     initializer = tf.contrib.layers.xavier_initializer(seed=seed)
-    graph = slim.conv2d(inputs['data'], 10, [3, 3], activation_fn=tf.nn.relu, stride=1, scope='conv1',
+    graph = slim.conv2d(inputs, 10, [3, 3], activation_fn=tf.nn.relu, stride=1, scope='conv1',
                         weights_initializer=initializer)
     graph = slim.max_pool2d(graph, [4, 4], scope='pool1')
     graph = slim.flatten(graph, scope='flatten1')
     graph = slim.fully_connected(graph, 10, activation_fn=None, scope='fc1',
                                  weights_initializer=initializer)
 
-    return inputs, graph
+    return graph
 
 
 def validate_network(embedding, validation_data, validation_labels):
@@ -69,20 +68,12 @@ def test_cnn_trainer_scratch():
                                  data_augmentation=data_augmentation,
                                  normalizer=ScaleFactor())
 
-    #validation_data_shuffler = Memory(train_data, train_labels,
-    #                                  input_shape=[28, 28, 1],
-    #                                  batch_size=batch_size,
-    #                                  data_augmentation=data_augmentation,
-    #                                  normalizer=ScaleFactor())
-
     validation_data = numpy.reshape(validation_data, (validation_data.shape[0], 28, 28, 1))
     # Create scratch network
-    inputs, scratch = scratch_network()
+    graph = scratch_network(train_data_shuffler)
 
     # Setting the placeholders
-    train_data_shuffler.data_ph = inputs['data']
-    train_data_shuffler.label_ph = inputs['label']
-    embedding = Embedding(inputs['data'], scratch)
+    embedding = Embedding(train_data_shuffler("data", from_queue=False), graph)
 
     # Loss for the softmax
     loss = BaseLoss(tf.nn.sparse_softmax_cross_entropy_with_logits, tf.reduce_mean)
@@ -93,7 +84,7 @@ def test_cnn_trainer_scratch():
                       analizer=None,
                       temp_dir=directory)
 
-    trainer.create_network_from_scratch(graph=scratch,
+    trainer.create_network_from_scratch(graph=graph,
                                         loss=loss,
                                         learning_rate=constant(0.01, name="regular_lr"),
                                         optimizer=tf.train.GradientDescentOptimizer(0.01),

bob/learn/tensorflow/trainers/SiameseTrainer.py

@@ -21,43 +21,25 @@ class SiameseTrainer(Trainer):
 
     **Parameters**
 
-    architecture:
-      The architecture that you want to run. Should be a :py:class`bob.learn.tensorflow.network.SequenceNetwork`
+    train_data_shuffler:
+      The data shuffler used for batching data for training
 
-    optimizer:
-      One of the tensorflow optimizers https://www.tensorflow.org/versions/r0.10/api_docs/python/train.html
-
-    use_gpu: bool
-      Use GPUs in the training
-
-    loss: :py:class:`bob.learn.tensorflow.loss.BaseLoss`
-      Loss function
-
-    temp_dir: str
-      The output directory
-
-    learning_rate: `bob.learn.tensorflow.trainers.learning_rate`
-      Initial learning rate
-
-    convergence_threshold:
-
-    iterations: int
+    iterations:
       Maximum number of iterations
-
-    snapshot: int
+      
+    snapshot:
       Will take a snapshot of the network at every `n` iterations
-
-    prefetch: bool
-      Use extra Threads to deal with the I/O
-
-    model_from_file: str
-      If you want to use a pretrained model
+      
+    validation_snapshot:
+      Test with validation each `n` iterations
 
     analizer:
       Neural network analizer :py:mod:`bob.learn.tensorflow.analyzers`
 
-    verbosity_level:
+    temp_dir: str
+      The output directory
 
+    verbosity_level:
 
     """
 

bob/learn/tensorflow/trainers/Trainer.py

@@ -4,7 +4,6 @@
 # @date: Tue 09 Aug 2016 15:25:22 CEST
 
 import tensorflow as tf
-from ..network import SequenceNetwork
 import threading
 import os
 import bob.io.base
@@ -29,41 +28,24 @@ class Trainer(object):
 
     **Parameters**
 
-    architecture:
-      The architecture that you want to run. Should be a :py:class`bob.learn.tensorflow.network.SequenceNetwork`
+    train_data_shuffler:
+      The data shuffler used for batching data for training
 
-    optimizer:
-      One of the tensorflow optimizers https://www.tensorflow.org/versions/r0.10/api_docs/python/train.html
-
-    use_gpu: bool
-      Use GPUs in the training
-
-    loss: :py:class:`bob.learn.tensorflow.loss.BaseLoss`
-      Loss function
-
-    temp_dir: str
-      The output directory
-
-    learning_rate: `bob.learn.tensorflow.trainers.learning_rate`
-      Initial learning rate
-
-    convergence_threshold:
-
-    iterations: int
+    iterations:
       Maximum number of iterations
-
-    snapshot: int
+      
+    snapshot:
       Will take a snapshot of the network at every `n` iterations
-
-    prefetch: bool
-      Use extra Threads to deal with the I/O
-
-    model_from_file: str
-      If you want to use a pretrained model
+      
+    validation_snapshot:
+      Test with validation each `n` iterations
 
     analizer:
       Neural network analizer :py:mod:`bob.learn.tensorflow.analyzers`
 
+    temp_dir: str
+      The output directory
+
     verbosity_level:
 
     """

bob/learn/tensorflow/trainers/TripletTrainer.py

@@ -7,7 +7,6 @@ import tensorflow as tf
 from tensorflow.core.framework import summary_pb2
 import threading
 from ..analyzers import ExperimentAnalizer
-from ..network import SequenceNetwork
 from .Trainer import Trainer
 from ..analyzers import SoftmaxAnalizer
 import os
@@ -23,45 +22,29 @@ class TripletTrainer(Trainer):
 
     **Parameters**
 
-    architecture:
-      The architecture that you want to run. Should be a :py:class`bob.learn.tensorflow.network.SequenceNetwork`
+    train_data_shuffler:
+      The data shuffler used for batching data for training
 
-    optimizer:
-      One of the tensorflow optimizers https://www.tensorflow.org/versions/r0.10/api_docs/python/train.html
-
-    use_gpu: bool
-      Use GPUs in the training
-
-    loss: :py:class:`bob.learn.tensorflow.loss.BaseLoss`
-      Loss function
-
-    temp_dir: str
-      The output directory
-
-    learning_rate: `bob.learn.tensorflow.trainers.learning_rate`
-      Initial learning rate
-
-    convergence_threshold:
-
-    iterations: int
+    iterations:
       Maximum number of iterations
-
-    snapshot: int
+      
+    snapshot:
       Will take a snapshot of the network at every `n` iterations
-
-    prefetch: bool
-      Use extra Threads to deal with the I/O
-
-    model_from_file: str
-      If you want to use a pretrained model
+      
+    validation_snapshot:
+      Test with validation each `n` iterations
 
     analizer:
       Neural network analizer :py:mod:`bob.learn.tensorflow.analyzers`
 
+    temp_dir: str
+      The output directory
+
     verbosity_level:
 
     """
 
+
     def __init__(self,
                  train_data_shuffler,
 

doc/nitpick-exceptions.txt

-py:class bob.learn.tensorflow.datashuffler.OnlineSampling
-py:class bob.learn.tensorflow.datashuffler.OnlineSampling.OnlineSampling
\ No newline at end of file
+py:class list
\ No newline at end of file

doc/py_api.rst

@@ -14,14 +14,9 @@ Architectures
 
 .. autosummary::
 
-    bob.learn.tensorflow.network.SequenceNetwork
     bob.learn.tensorflow.network.Chopra
     bob.learn.tensorflow.network.Dummy
-    bob.learn.tensorflow.network.Lenet
-    bob.learn.tensorflow.network.LenetDropout
     bob.learn.tensorflow.network.MLP
-    bob.learn.tensorflow.network.VGG16
-    bob.learn.tensorflow.network.VGG16_mod
 
 
 Trainers

doc/user_guide.rst

@@ -18,27 +18,48 @@ The example consists in training a very simple **CNN** with `MNIST` dataset in 4
 .. code-block:: python
 
     >>> import tensorflow as tf
-    >>> import bob.learn.tensorflow
+    >>> from bob.learn.tensorflow.loss import BaseLoss
+    >>> from bob.learn.tensorflow.trainers import Trainer, constant
+    >>> from bob.learn.tensorflow.utils import load_mnist
+    >>> from bob.learn.tensorflow.datashuffler import Memory
+    >>> from bob.learn.tensorflow.network import Embedding
     >>> import numpy
 
+    >>> # Loading raw data
     >>> train_data, train_labels, _, _ = bob.learn.tensorflow.util.load_mnist()
     >>> train_data = numpy.reshape(train_data, (train_data.shape[0], 28, 28, 1))
-    >>> train_data_shuffler = bob.learn.tensorflow.datashuffler.Memory(train_data, train_labels, input_shape=[28, 28, 1], batch_size=16)
+
+    >>> # Preparing the datashuffler (our batching engine)
+    >>> train_data_shuffler = Memory(train_data, train_labels, input_shape=[None, 28, 28, 1], batch_size=16)
+    >>>
 
 2. Create an architecture
 
 .. code-block:: python
 
-    >>> architecture = bob.learn.tensorflow.network.SequenceNetwork()
-    >>> architecture.add(bob.learn.tensorflow.layers.Conv2D(name="conv1", kernel_size=3, filters=10, activation=tf.nn.tanh))
-    >>> architecture.add(bob.learn.tensorflow.layers.FullyConnected(name="fc1", output_dim=10, activation=None))
+    >>> # Create a function or a class with the graph definition in the `__call__` method
+    >>> def create_dummy_architecture(placeholder):
+    >>>     initializer = tf.contrib.layers.xavier_initializer(seed=10)  # Weights initializer
+    >>>     slim = tf.contrib.slim
+    >>>     graph = slim.conv2d(placeholder, 10, [3, 3], activation_fn=tf.nn.relu, stride=1, scope='conv1', weights_initializer=initializer)
+    >>>     graph = slim.flatten(graph, scope='flatten1')
+    >>>     graph = slim.fully_connected(graph, 10, activation_fn=None, scope='fc1', weights_initializer=initializer)
+    >>>     return graph
+    >>>
+
 
 3. Defining a loss and training algorithm
 
 .. code-block:: python
 
-    >>> loss = bob.learn.tensorflow.loss.BaseLoss(tf.nn.sparse_softmax_cross_entropy_with_logits, tf.reduce_mean)
-    >>> from bob.learn.tensorflow.trainers import Trainer
+    >>> architecture = create_dummy_architecture
+    >>>
+    >>> loss = BaseLoss(tf.nn.sparse_softmax_cross_entropy_with_logits, tf.reduce_mean)
+    >>>
+    >>> optimizer = tf.train.GradientDescentOptimizer(0.001)
+    >>>
+    >>> learning_rate = constant(base_learning_rate=0.001)
+    >>>
     >>> trainer = Trainer
 
 Now that you have defined your data, architecture, loss and training algorithm you can save this in a python file,
@@ -46,7 +67,7 @@ let's say `softmax.py`, and run:
 
 .. code-block:: shell
 
-    >>> ./bin/train.py softmax.py
+    >>> ./bin/train.py softmax.py --iterations 100 --output-dir ./my_first_net
 
 
 4. Predicting and computing the accuracy
@@ -55,15 +76,17 @@ Run the following code to evalutate the network that was just trained.
 
 .. code-block:: python
 
-    >>> # Loading the model
-    >>> architecture = bob.learn.tensorflow.network.SequenceNetwork()
-    >>> architecture.load("./cnn/model.ckp")
-    >>> # Predicting
-    >>> predictions = scratch.predict(validation_data, session=session)
-    >>> # Computing an awesome accuracy for a simple network and 100 iterations
-    >>> accuracy = 100. * numpy.sum(predictions == validation_labels) / predictions.shape[0]
-    >>> print accuracy
-    90.4714285714
+    >>> # Loading the trained model
+    >>> trainer = Trainer(train_data_shuffler)
+    >>> directory = "./my_first_net/"
+    >>> trainer.create_network_from_file(os.path.join(directory, "model.ckp"))
+
+    >>> # Prediction
+    >>> embedding = Embedding(trainer.data_ph, trainer.graph)
+    >>> [data, labels] = train_data_shuffler.get_batch()
+    >>> predictions = embedding(data)
+    >>> accuracy = 100. * numpy.sum(numpy.argmax(predictions, axis=1) == labels) / predictions.shape[0]
+    87.5
 
 
 Understanding what you have done
@@ -73,8 +96,7 @@ Understanding what you have done
 Preparing your input data
 .........................
 
-In this library datasets are wrapped in **data shufflers**. Data shufflers are elements designed to shuffle
-the input data for stochastic training.
+In this library datasets are wrapped in **data shufflers**. Data shufflers are elements designed to do batching.
 It has one basic functionality which is :py:meth:`bob.learn.tensorflow.datashuffler.Base.get_batch` functionality.
 
 It is possible to either use Memory (:py:class:`bob.learn.tensorflow.datashuffler.Memory`) or
@@ -89,9 +111,10 @@ number of channels.
 Creating the architecture
 .........................
 
-Architectures are assembled in the :py:class:`bob.learn.tensorflow.network.SequenceNetwork` object.
-Once the objects are created it is necessary to fill it up with `Layers <py_api.html#layers>`_.
-The library has already some crafted networks implemented in `Architectures <py_api.html#architectures>`_.
+Architectures are assembled using the Tensorflow graphs.
+There are plenty of ways to doing it; you can either use the `tensorflow <https://www.tensorflow.org/api_docs/python/tf/Graph>`_ API directly
+or use one of the several available contribs such as `tf-slim <https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/slim>`_,
+`TFLearn <http://tflearn.org/>`_, etc...
 
 
 Defining a loss and training
@@ -169,28 +192,6 @@ here :py:class:`bob.learn.tensorflow.datashuffler.TripletWithSelectionDisk` and
 This triplet selection relies in the current state of the network and are extensions of `bob.learn.tensorflow.datashuffler.OnlineSampling`.
 
 
-Architecture
-............
-
-As described above, architectures are assembled in the :py:class:`bob.learn.tensorflow.network.SequenceNetwork` object.
-Once the objects are created it is necessary to fill it up with `Layers <py_api.html#layers>`_.
-The library has already some crafted networks implemented in `Architectures <py_api.html#architectures>`_.
-
-It is also possible to craft simple MLPs with this library using the class :py:class:`bob.learn.tensorflow.network.MLP`.
-The example bellow shows how to create a simple MLP with 10 putputs and 2 hidden layers.
-
-.. code-block:: python
-
-    >>> architecture = bob.learn.tensorflow.network.MLP(10, hidden_layers=[20, 40])
-
-
-Layers
-......
-
-There is nothing much to say about layers.
-This library wrapped all the necessary tasks (variable creation, operations, reuse of variables, etc ...) from tensorflow.
-Check `Layers <py_api.html#layers>`_ for more information
-
 Activations
 ...........