Commit 7d115f4d authored by Tiago de Freitas Pereira's avatar Tiago de Freitas Pereira
Browse files

Writing user guide and fixed warnings

Fixed doc tests

Updated the input-fn description
parent 22f0062f
Pipeline #22334 passed with stage
in 20 minutes and 52 seconds
......@@ -12,7 +12,7 @@ import numpy
# -- checkpoint-dir
# YOU CAN DOWNLOAD THE CHECKPOINTS FROM HERE
# https://github.com/tensorflow/models/tree/master/research/slim#pre-trained-models
checkpoint_dir = "[VGG-19-CHECKPOINT]"
checkpoint_dir = "/idiap/user/tpereira/gitlab/bob/bob.learn.tensorflow/vgg_19.ckpt"
# --style-end-points and -- content-end-points
content_end_points = ['vgg_19/conv4/conv4_2', 'vgg_19/conv5/conv5_2']
......@@ -26,8 +26,9 @@ style_end_points = ['vgg_19/conv1/conv1_2',
scopes = {"vgg_19/":"vgg_19/"}
style_image_paths = ["vincent_van_gogh.jpg",
"vincent_van_gogh2.jpg"]
style_image_paths = ["/idiap/user/tpereira/gitlab/bob/bob.learn.tensorflow/bob/learn/tensorflow/configs/style_transfer/vincent_van_gogh.jpg"]
# "/idiap/user/tpereira/gitlab/bob/bob.learn.tensorflow/bob/learn/tensorflow/configs/style_transfer/vincent_van_gogh2.jpg"]
# --preprocess-fn and --un-preprocess-fn
......
......@@ -274,7 +274,7 @@ def blocks_tensorflow(images, block_size):
Parameters
----------
images : :any:`tf.Tensor`
images : `tf.Tensor`
The input color images. It is assumed that the image has a shape of
[?, H, W, C].
block_size : (int, int)
......@@ -282,7 +282,7 @@ def blocks_tensorflow(images, block_size):
Returns
-------
blocks : :any:`tf.Tensor`
blocks : `tf.Tensor`
All the blocks in the batch dimension. The output will be of
size [?, block_size[0], block_size[1], C].
n_blocks : int
......
......@@ -17,22 +17,28 @@ logger = logging.getLogger("bob.learn")
class Logits(estimator.Estimator):
"""
NN Trainer whose with logits as last layer
NN estimator with `Cross entropy loss <https://www.tensorflow.org/api_docs/python/tf/nn/softmax_cross_entropy_with_logits>`_ in the hot-encoded layer :py:class:`bob.learn.tensorflow.estimators.Logits`.
The **architecture** function should follow the following pattern::
The architecture function should follow the following pattern::
```
def my_beautiful_architecture(placeholder, **kwargs):
def my_beautiful_architecture(placeholder, **kwargs):
end_points = dict()
graph = convXX(placeholder)
end_points['conv'] = graph
return graph, end_points
```
end_points = dict()
graph = convXX(placeholder)
end_points['conv'] = graph
....
return graph, end_points
The **loss** function should follow the following pattern::
def my_beautiful_loss(logits, labels, **kwargs):
return loss_set_of_ops(logits, labels)
```
def my_beautiful_loss(logits, labels, **kwargs):
return loss_set_of_ops(logits, labels)
```
Attributes
----------
......@@ -42,10 +48,6 @@ class Logits(estimator.Estimator):
optimizer:
One of the tensorflow solvers
(https://www.tensorflow.org/api_guides/python/train)
- tf.train.GradientDescentOptimizer
- tf.train.AdagradOptimizer
- ....
config:
......@@ -72,18 +74,18 @@ class Logits(estimator.Estimator):
extra_checkpoint: dict
In case you want to use other model to initialize some variables.
This argument should be in the following format
extra_checkpoint = {
"checkpoint_path": <YOUR_CHECKPOINT>,
"scopes": dict({"<SOURCE_SCOPE>/": "<TARGET_SCOPE>/"}),
"trainable_variables": [<LIST OF VARIABLES OR SCOPES THAT YOU WANT TO RETRAIN>]
}
This argument should be in the following format::
```
extra_checkpoint = {"checkpoint_path": <YOUR_CHECKPOINT>,\
"scopes": dict({"<SOURCE_SCOPE>/": "<TARGET_SCOPE>/"}),\
"trainable_variables": [<LIST OF VARIABLES OR SCOPES THAT YOU WANT TO RETRAIN>]\
}
```
apply_moving_averages: bool
Apply exponential moving average in the training variables and in the loss.
https://www.tensorflow.org/api_docs/python/tf/train/ExponentialMovingAverage
By default the decay for the variable averages is 0.9999 and for the loss is 0.9
"""
def __init__(self,
......@@ -237,17 +239,20 @@ class Logits(estimator.Estimator):
class LogitsCenterLoss(estimator.Estimator):
"""
NN Trainer whose with logits as last layer
NN estimator with `Cross entropy loss <https://www.tensorflow.org/api_docs/python/tf/nn/softmax_cross_entropy_with_logits>`_ in the hot-encoded layer :py:class:`bob.learn.tensorflow.estimators.Logits` plus the center loss implemented in: "Wen, Yandong, et al. "A discriminative feature learning approach for deep face recognition." European Conference on Computer Vision. Springer, Cham, 2016."
The **architecture** function should follow the following pattern:
The **architecture** function should follow the following pattern::
```
def my_beautiful_function(placeholder):
end_points = dict()
graph = convXX(placeholder)
end_points['conv'] = graph
....
return graph, end_points
end_points = dict()
graph = convXX(placeholder)
end_points['conv'] = graph
return graph, end_points
```
**Parameters**
architecture:
......@@ -285,20 +290,19 @@ class LogitsCenterLoss(estimator.Estimator):
extra_checkpoint: dict
In case you want to use other model to initialize some variables.
This argument should be in the following format
extra_checkpoint = {
"checkpoint_path": <YOUR_CHECKPOINT>,
"scopes": dict({"<SOURCE_SCOPE>/": "<TARGET_SCOPE>/"}),
"trainable_variables": [<LIST OF VARIABLES OR SCOPES THAT YOU WANT TO TRAIN>]
}
This argument should be in the following format::
```
extra_checkpoint = {"checkpoint_path": <YOUR_CHECKPOINT>,\
"scopes": dict({"<SOURCE_SCOPE>/": "<TARGET_SCOPE>/"}),\
"trainable_variables": [<LIST OF VARIABLES OR SCOPES THAT YOU WANT TO RETRAIN>]\
}
```
apply_moving_averages: bool
Apply exponential moving average in the training variables and in the loss.
https://www.tensorflow.org/api_docs/python/tf/train/ExponentialMovingAverage
By default the decay for the variable averages is 0.9999 and for the loss is 0.9
"""
def __init__(self,
......
......@@ -14,31 +14,33 @@ logger = logging.getLogger("bob.learn")
class Siamese(estimator.Estimator):
"""
NN estimator for Siamese networks
NN estimator for Siamese Networks proposed in: "Chopra, Sumit, Raia Hadsell, and Yann LeCun. "Learning a similarity metric discriminatively, with application to face verification." Computer Vision and Pattern Recognition, 2005. CVPR 2005. IEEE Computer Society Conference on. Vol. 1. IEEE, 2005."
The **architecture** function should follow the following pattern:
The **architecture** function should follow the following pattern::
```
def my_beautiful_function(placeholder):
end_points = dict()
graph = convXX(placeholder)
end_points['conv'] = graph
....
return graph, end_points
```
The **loss** function should follow the following pattern:
The **loss** function should follow the following pattern::
def my_beautiful_loss(logits, labels):
return loss_set_of_ops(logits, labels)
```
def my_beautiful_loss(logits, labels):
return loss_set_of_ops(logits, labels)
extra_checkpoint = {"checkpoint_path":model_dir,
"scopes": dict({"Dummy/": "Dummy/"}),
"trainable_variables": [<LIST OF VARIABLES OR SCOPES THAT YOU WANT TO TRAIN>]
}
```
**Parameters**
architecture:
......@@ -65,20 +67,20 @@ class Siamese(estimator.Estimator):
Size of the batch for validation. This value is used when the
validation with embeddings is used. This is a hack.
params:
Extra params for the model function
(please see https://www.tensorflow.org/extend/estimators for more info)
extra_checkpoint: dict
In case you want to use other model to initialize some variables.
This argument should be in the following format
extra_checkpoint = {
"checkpoint_path": <YOUR_CHECKPOINT>,
"scopes": dict({"<SOURCE_SCOPE>/": "<TARGET_SCOPE>/"}),
"trainable_variables": [<LIST OF VARIABLES OR SCOPES THAT YOU WANT TO TRAIN>]
}
This argument should be in the following format::
```
extra_checkpoint = {"checkpoint_path": <YOUR_CHECKPOINT>,\
"scopes": dict({"<SOURCE_SCOPE>/": "<TARGET_SCOPE>/"}),\
"trainable_variables": [<LIST OF VARIABLES OR SCOPES THAT YOU WANT TO RETRAIN>]\
}
```
"""
def __init__(self,
......
......@@ -66,12 +66,15 @@ class Triplet(estimator.Estimator):
extra_checkpoint: dict
In case you want to use other model to initialize some variables.
This argument should be in the following format
extra_checkpoint = {
"checkpoint_path": <YOUR_CHECKPOINT>,
"scopes": dict({"<SOURCE_SCOPE>/": "<TARGET_SCOPE>/"}),
"trainable_variables": [<LIST OF VARIABLES OR SCOPES THAT YOU WANT TO TRAIN>]
This argument should be in the following format::
```
extra_checkpoint = {\
"checkpoint_path": <YOUR_CHECKPOINT>,\
"scopes": dict({"<SOURCE_SCOPE>/": "<TARGET_SCOPE>/"}),\
"trainable_variables": [<LIST OF VARIABLES OR SCOPES THAT YOU WANT TO TRAIN>]\
}
```
"""
def __init__(self,
......
......@@ -17,7 +17,7 @@ def get_trainable_variables(extra_checkpoint,
mode=tf.estimator.ModeKeys.TRAIN):
"""
Given the extra_checkpoint dictionary provided to the estimator,
extract the content of "trainable_variables" e.
extract the content of "trainable_variables".
If trainable_variables is not provided, all end points are trainable by
default.
......@@ -25,19 +25,21 @@ def get_trainable_variables(extra_checkpoint,
If trainable_variables contains some end_points, ONLY these endpoints will
be trainable.
Parameters
Attributes
----------
extra_checkpoint : dict
The `extra_checkpoint dictionary provided to the estimator
mode
extra_checkpoint: dict
The extra_checkpoint dictionary provided to the estimator
mode:
The estimator mode. TRAIN, EVAL, and PREDICT. If not TRAIN, None is
returned.
Returns
-------
Returns `None` if `trainable_variables` is not in extra_checkpoint;
otherwise returns the content of `extra_checkpoint
Returns `None` if **trainable_variables** is not in extra_checkpoint;
otherwise returns the content of extra_checkpoint .
"""
if mode != tf.estimator.ModeKeys.TRAIN:
return None
......
......@@ -29,6 +29,9 @@ __appropriate__(
light_cnn9,
dummy,
mlp,
inception_resnet_v2, inception_resnet_v2_batch_norm,
inception_resnet_v1, inception_resnet_v1_batch_norm,
SimpleCNN, vgg_19, vgg_16
)
__all__ = [_ for _ in dir() if not _.startswith('_')]
......@@ -207,8 +207,15 @@ def predict_using_tensors(embedding, labels, num=None):
def compute_embedding_accuracy_tensors(embedding, labels, num=None):
"""
Compute the accuracy through exhaustive comparisons between the embeddings
using tensors
Compute the accuracy in a closed-set
**Parameters**
embeddings: `tf.Tensor`
Set of embeddings
labels: `tf.Tensor`
Correspondent labels
"""
# Fitting the main diagonal with infs (removing comparisons with the same
......@@ -224,7 +231,15 @@ def compute_embedding_accuracy_tensors(embedding, labels, num=None):
def compute_embedding_accuracy(embedding, labels):
"""
Compute the accuracy through exhaustive comparisons between the embeddings
Compute the accuracy in a closed-set
**Parameters**
embeddings: :any:`numpy.array`
Set of embeddings
labels: :any:`numpy.array`
Correspondent labels
"""
from scipy.spatial.distance import cdist
......@@ -273,13 +288,13 @@ def to_channels_last(image):
Parameters
----------
image : :any:`tf.Tensor`
image : `tf.Tensor`
At least a 3 dimensional image. If the dimension is more than 3, the
last 3 dimensions are assumed to be [C, H, W].
Returns
-------
image : :any:`tf.Tensor`
image : `tf.Tensor`
The image in [..., H, W, C] format.
Raises
......@@ -303,13 +318,13 @@ def to_channels_first(image):
Parameters
----------
image : :any:`tf.Tensor`
image : `tf.Tensor`
At least a 3 dimensional image. If the dimension is more than 3, the
last 3 dimensions are assumed to be [H, W, C].
Returns
-------
image : :any:`tf.Tensor`
image : `tf.Tensor`
The image in [..., C, H, W] format.
Raises
......
......@@ -12,9 +12,27 @@ Estimators
.. autosummary::
bob.learn.tensorflow.estimators.Logits
bob.learn.tensorflow.estimators.LogitsCenterLoss
bob.learn.tensorflow.estimators.Triplet
bob.learn.tensorflow.estimators.Siamese
Architectures
-------------
=============
.. autosummary::
bob.learn.tensorflow.network.chopra
bob.learn.tensorflow.network.light_cnn9
bob.learn.tensorflow.network.dummy
bob.learn.tensorflow.network.mlp
bob.learn.tensorflow.network.inception_resnet_v2
bob.learn.tensorflow.network.inception_resnet_v1
bob.learn.tensorflow.network.inception_resnet_v2_batch_norm
bob.learn.tensorflow.network.inception_resnet_v1_batch_norm
bob.learn.tensorflow.network.SimpleCNN
bob.learn.tensorflow.network.vgg_19
bob.learn.tensorflow.network.vgg_16
Data
......@@ -22,18 +40,26 @@ Data
.. autosummary::
bob.learn.tensorflow.dataset.bio.BioGenerator
bob.learn.tensorflow.dataset.image.shuffle_data_and_labels_image_augmentation
bob.learn.tensorflow.dataset.siamese_image.shuffle_data_and_labels_image_augmentation
bob.learn.tensorflow.dataset.triplet_image.shuffle_data_and_labels_image_augmentation
bob.learn.tensorflow.dataset.tfrecords.shuffle_data_and_labels_image_augmentation
bob.learn.tensorflow.dataset.tfrecords.shuffle_data_and_labels
bob.learn.tensorflow.utils.util.to_channels_last
bob.learn.tensorflow.utils.util.to_channels_first
Detailed Information
====================
.. automodule:: bob.learn.tensorflow
.. automodule:: bob.learn.tensorflow.estimators
:special-members: __init__
.. automodule:: bob.learn.tensorflow.dataset
.. automodule:: bob.learn.tensorflow.dataset.bio
.. automodule:: bob.learn.tensorflow.dataset.image
.. automodule:: bob.learn.tensorflow.dataset.siamese_image
.. automodule:: bob.learn.tensorflow.dataset.triplet_image
.. automodule:: bob.learn.tensorflow.dataset.tfrecords
.. automodule:: bob.learn.tensorflow.network
.. automodule:: bob.learn.tensorflow.utils
.. automodule:: bob.learn.tensorflow.utils.util
......@@ -20,8 +20,8 @@ data API is here to help you build complex and efficient input pipelines to
your model.
Face recognition example
========================
Face recognition example using bob.db databases
===============================================
Here is a quick code example to build a simple convolutional neural network
(CNN) for recognizing faces from the ATNT database.
......@@ -38,6 +38,8 @@ Here is a quick code example to build a simple convolutional neural network
2. Define the inputs:
.. _input_fn:
.. doctest::
>>> def input_fn(mode):
......@@ -104,7 +106,7 @@ Here is a quick code example to build a simple convolutional neural network
... input_fn=train_input_fn, max_steps=50, hooks=hooks)
>>> eval_spec = tf.estimator.EvalSpec(input_fn=eval_input_fn)
3. Define the estimator:
3. Define the architecture:
.. doctest::
......@@ -139,8 +141,18 @@ Here is a quick code example to build a simple convolutional neural network
... endpoints[name] = net
...
... return net, endpoints
...
...
.. warning ::
Practical advice: use `tf.contrib.slim` to craft your CNNs.
Although Tensorflow's documentation recomend the usage of `tf.layers` and `tf.keras`, in our experience `slim` has better defaults, probably because the guys from Google use them more often.
4. Define the estimator:
.. doctest::
>>> estimator = Logits(
... architecture,
... optimizer=tf.train.GradientDescentOptimizer(1e-4),
......@@ -150,7 +162,7 @@ Here is a quick code example to build a simple convolutional neural network
... validation_batch_size=8,
... ) # doctest: +SKIP
4. Train and evaluate the model:
5. Train and evaluate the model:
.. doctest::
......@@ -164,6 +176,13 @@ and estimators. Let's summarize them.
Data pipeline
-------------
There are several ways to provide data to Tensorflow graphs.
In this section we provide some examples on how to make the bridge between `bob.db` databases
and tensorflow `input_fn`.
The BioGenerator input pipeline
*******************************
The :any:`bob.learn.tensorflow.dataset.bio.BioGenerator` class can be used to
convert any database of bob (not just bob.bio.base's databases) to a
``tf.data.Dataset`` instance.
......@@ -181,12 +200,147 @@ crop, mean normalization, random flip, ...) in the ``transform`` function.
Once these transformations are applied on your data, you can easily cache them
to disk for faster reading of data in your training.
Input pipeline with TFRecords
*****************************
An optimized way to provide data to Tensorflow graphs is using tfrecords.
In this `link <http://warmspringwinds.github.io/tensorflow/tf-slim/2016/12/21/tfrecords-guide/>`_ you have a very nice guide on how TFRecord works.
In `bob.learn.tensorflow` we provide the tool `bob tf db_to_tfrecords` that converts `bob.db` databases in TFRecords. Type the snippet bellow for help::
$ ./bin/bob tf db_to_tfrecords --help
To generate a tfrecord for our `Face recognition example using bob.db databases`_ example use the following snippet.
.. doctest::
>>> from bob.bio.base.utils import read_original_data
>>> database = bob.db.atnt.Database()
>>> groups = 'world'
>>> CLIENT_IDS = (str(f.client_id) for f in database.objects(groups=groups))
>>> CLIENT_IDS = list(set(CLIENT_IDS))
>>> CLIENT_IDS = dict(zip(CLIENT_IDS, range(len(CLIENT_IDS))))
>>> def file_to_label(f):
... return CLIENT_IDS[str(f.client_id)]
>>> def reader(biofile):
... data = read_original_data(biofile, database.original_directory,database.original_extension)
... label = file_to_label(biofile)
... key = biofile.path
... return (data, label, key)
After saving this snippet in a python file (let's say `tfrec.py`) run the following command ::
$ bob tf db_to_tfrecords tfrec.py -o atnt.tfrecord
Once this is done you can replace the `input_fn`_ defined above by the snippet bellow.
.. doctest::
>>>
>>> from bob.learn.tensorflow.dataset.tfrecords import shuffle_data_and_labels_image_augmentation
>>>
>>> tfrecords_filename = ['atnt.tfrecord']
>>> data_shape = (112, 92 , 3)
>>> data_type = tf.uint8
>>> batch_size = 16
>>> epochs = 1
>>>
>>> def train_input_fn():
... return shuffle_data_and_labels_image_augmentation(
... tfrecords_filename,
... data_shape,
... data_type,
... batch_size,
... epochs=epochs)
The Estimator
-------------
The estimators can also be customized using different architectures, loss
functions, and optimizers.
In this package we have crafted 4 types of estimators.
- Logits: `Cross entropy loss <https://www.tensorflow.org/api_docs/python/tf/nn/softmax_cross_entropy_with_logits>`_ in the hot-encoded layer :py:class:`bob.learn.tensorflow.estimators.Logits`
- LogitsCenterLoss: `Cross entropy loss <https://www.tensorflow.org/api_docs/python/tf/nn/softmax_cross_entropy_with_logits>`_ PLUS the `center loss <https://ydwen.github.io/papers/WenECCV16.pdf>`_ in the hot-encoded layer :py:class:`bob.learn.tensorflow.estimators.LogitsCenterLoss`
- Siamese: Siamese network estimator :py:class:`bob.learn.tensorflow.estimators.Siamese`
- Triplet: Triplet network estimator :py:class:`bob.learn.tensorflow.estimators.Triplet`
.. _tensorflow: https://www.tensorflow.org/
Style Transfer
==============
We have implemented the style transfer strategy from::
Gatys, Leon A., Alexander S. Ecker, and Matthias Bethge. "A neural algorithm of artistic style." arXiv preprint arXiv:1508.06576 (2015).
Check as the usage possibilities with the command::
$ bob tf style_transfer --help
Here we have an example on how to do a style transfer using VGG 19 trained with the image net
.. doctest::
>>> from bob.learn.tensorflow.network import vgg_19
>>> # --architecture
>>> architecture = vgg_19
>>> import numpy
>>> # YOU CAN DOWNLOAD THE CHECKPOINTS FROM HERE
>>> # https://github.com/tensorflow/models/tree/master/research/slim#pre-trained-models
>>> checkpoint_dir = "[DOWNLOAD_YOUR_MODEL]"
>>> # --style-end-points and -- content-end-points
>>> content_end_points = ['vgg_19/conv4/conv4_2', 'vgg_19/conv5/conv5_2']
>>> style_end_points = ['vgg_19/conv1/conv1_2',
... 'vgg_19/conv2/conv2_1',
... 'vgg_19/conv3/conv3_1',
... 'vgg_19/conv4/conv4_1',
... 'vgg_19/conv5/conv5_1'
... ]
>>> # Transfering variables
>>> scopes = {"vgg_19/":"vgg_19/"}
>>> # Set if images using
>>> style_image_paths = ["vincent_van_gogh.jpg"]
>>> # Functions used to preprocess the input signal and
>>> # --preprocess-fn and --un-preprocess-fn
>>> # Taken from VGG19
>>> def mean_norm(tensor):
... return tensor - numpy.array([ 123.68 , 116.779, 103.939])
>>> def un_mean_norm(tensor):