Commit aaf932d2 authored by Amir MOHAMMADI's avatar Amir MOHAMMADI

Merge branch 'cleanup' into 'master'

Add necessary files to train DeepPixBis

See merge request !92
parents 4623a148 93845dbc
Pipeline #46486 passed with stages
in 4 minutes and 43 seconds
import json
import logging
import os
import tensorflow as tf
logger = logging.getLogger(__name__)
class CustomBackupAndRestore(tf.keras.callbacks.experimental.BackupAndRestore):
"""This callback is experimental and might be removed in future.
......@@ -44,9 +47,9 @@ class CustomBackupAndRestore(tf.keras.callbacks.experimental.BackupAndRestore):
def on_train_begin(self, logs=None):
super().on_train_begin(logs=logs)
if self.restore():
print(f"Restored callbacks from {self.callbacks_backup_path}")
logger.info(f"Restored callbacks from {self.callbacks_backup_path}")
else:
print("Did not restore callbacks")
logger.info("Did not restore callbacks")
def on_epoch_end(self, epoch, logs=None):
super().on_epoch_end(epoch, logs=logs)
......
import numbers
import tensorflow as tf
def _check_input(
value, name, center=1, bound=(0, float("inf")), clip_first_on_zero=True
):
if isinstance(value, numbers.Number):
if value < 0:
raise ValueError(
"If {} is a single number, it must be non negative.".format(name)
)
value = [center - float(value), center + float(value)]
if clip_first_on_zero:
value[0] = max(value[0], 0.0)
elif isinstance(value, (tuple, list)) and len(value) == 2:
if not bound[0] <= value[0] <= value[1] <= bound[1]:
raise ValueError("{} values should be between {}".format(name, bound))
else:
raise TypeError(
"{} should be a single number or a list/tuple with lenght 2.".format(name)
)
# # if value is 0 or (1., 1.) for brightness/contrast/saturation
# # or (0., 0.) for hue, do nothing
# if value[0] == value[1] == center:
# value = None
return value
class ColorJitter(tf.keras.layers.Layer):
"""Adjust the brightness, contrast, saturation, and hue of an image or images by a random factor.
Equivalent to adjust_brightness() using a delta randomly picked in the interval [-max_delta, max_delta)
For each channel, this layer computes the mean of the image pixels in the
channel and then adjusts each component `x` of each pixel to
`(x - mean) * brightness_factor + mean`.
Input shape:
4D tensor with shape:
`(samples, height, width, channels)`, data_format='channels_last'.
Output shape:
4D tensor with shape:
`(samples, height, width, channels)`, data_format='channels_last'.
Attributes:
brightness (float or tuple of float (min, max)): How much to jitter brightness.
brightness_factor is chosen uniformly from [max(0, 1 - brightness), 1 + brightness]
or the given [min, max]. Should be non negative numbers.
contrast (float or tuple of float (min, max)): How much to jitter contrast.
contrast_factor is chosen uniformly from [max(0, 1 - contrast), 1 + contrast]
or the given [min, max]. Should be non negative numbers.
saturation (float or tuple of float (min, max)): How much to jitter saturation.
saturation_factor is chosen uniformly from [max(0, 1 - saturation), 1 + saturation]
or the given [min, max]. Should be non negative numbers.
hue (float or tuple of float (min, max)): How much to jitter hue.
hue_factor is chosen uniformly from [-hue, hue] or the given [min, max].
Should have 0<= hue <= 0.5 or -0.5 <= min <= max <= 0.5.
seed (int or None): Used to create a random seed.
name (str or None): The name of the layer.
Raise:
ValueError: if lower bound is not between [0, 1], or upper bound is negative.
"""
def __init__(
self,
brightness=0.0,
contrast=0.0,
saturation=0.0,
hue=0.0,
seed=None,
name=None,
**kwargs,
):
super().__init__(name=name, **kwargs)
self.brightness = _check_input(brightness, "brightness")
self.contrast = _check_input(contrast, "contrast")
self.saturation = _check_input(saturation, "saturation")
self.hue = _check_input(
hue, "hue", center=0, bound=(-1.0, 1.0), clip_first_on_zero=False
)
self.seed = seed
self.input_spec = tf.keras.layers.InputSpec(ndim=4)
@tf.function
def call(self, inputs, training=None):
if training is None:
training = tf.keras.backend.learning_phase()
output = inputs
if training:
fn_idx = tf.convert_to_tensor(list(range(4)))
fn_idx = tf.random.shuffle(fn_idx, seed=self.seed)
if inputs.dtype not in (tf.dtypes.float16, tf.dtypes.float32):
output = tf.image.convert_image_dtype(output, dtype=tf.dtypes.float32)
for fn_id in fn_idx:
if fn_id == 0:
brightness_factor = tf.random.uniform(
[],
minval=self.brightness[0],
maxval=self.brightness[1],
seed=self.seed,
)
output = tf.image.adjust_brightness(output, brightness_factor)
if fn_id == 1:
contrast_factor = tf.random.uniform(
[],
minval=self.contrast[0],
maxval=self.contrast[1],
seed=self.seed,
)
output = tf.image.adjust_contrast(output, contrast_factor)
if fn_id == 2:
saturation_factor = tf.random.uniform(
[],
minval=self.saturation[0],
maxval=self.saturation[1],
seed=self.seed,
)
output = tf.image.adjust_saturation(output, saturation_factor)
if fn_id == 3:
hue_factor = tf.random.uniform(
[], minval=self.hue[0], maxval=self.hue[1], seed=self.seed
)
output = tf.image.adjust_hue(output, hue_factor)
output = tf.image.convert_image_dtype(
output, dtype=inputs.dtype, saturate=True
)
output.set_shape(inputs.shape)
return output
def compute_output_shape(self, input_shape):
return input_shape
def get_config(self):
config = {
"brightness": self.brightness,
"contrast": self.contrast,
"saturation": self.saturation,
"hue": self.hue,
"seed": self.seed,
}
base_config = super().get_config()
return dict(list(base_config.items()) + list(config.items()))
def Normalize(mean, std=1.0, **kwargs):
scale = 1.0 / std
offset = mean / std
return tf.keras.layers.experimental.preprocessing.Rescaling(
scale=scale, offset=offset, **kwargs
)
# fmt: off
from .balanced_cross_entropy import \
balanced_sigmoid_cross_entropy_loss_weights # noqa: F401
from .balanced_cross_entropy import \
balanced_softmax_cross_entropy_loss_weights # noqa: F401
# fmt: on
from .center_loss import CenterLoss
from .center_loss import CenterLossLayer
from .pixel_wise import PixelwiseBinaryCrossentropy
# gets sphinx autodoc done right - don't remove it
......@@ -18,5 +25,5 @@ def __appropriate__(*args):
obj.__module__ = __name__
__appropriate__(CenterLoss, CenterLossLayer)
__appropriate__(CenterLoss, CenterLossLayer, PixelwiseBinaryCrossentropy)
__all__ = [_ for _ in dir() if not _.startswith("_")]
import tensorflow as tf
def balanced_softmax_cross_entropy_loss_weights(labels, dtype="float32"):
"""Computes weights that normalizes your loss per class.
Labels must be a batch of one-hot encoded labels. The function takes labels and
computes the weights per batch. Weights will be smaller for classes that have more
samples in this batch. This is useful if you unbalanced classes in your dataset or
batch.
Parameters
----------
labels : ``tf.Tensor``
Labels of your current input. The shape must be [batch_size, n_classes]. If your
labels are not one-hot encoded, you can use ``tf.one_hot`` to convert them first
before giving them to this function.
dtype : ``tf.dtype``
The dtype that weights will have. It should be float. Best is to provide
logits.dtype as input.
Returns
-------
``tf.Tensor``
Computed weights that will cancel your dataset imbalance per batch.
Examples
--------
>>> import numpy
>>> import tensorflow as tf
>>> from bob.learn.tensorflow.losses import balanced_softmax_cross_entropy_loss_weights
>>> labels = numpy.array([[1, 0, 0],
... [1, 0, 0],
... [0, 0, 1],
... [0, 1, 0],
... [0, 0, 1],
... [1, 0, 0],
... [1, 0, 0],
... [0, 0, 1],
... [1, 0, 0],
... [1, 0, 0],
... [1, 0, 0],
... [1, 0, 0],
... [1, 0, 0],
... [1, 0, 0],
... [0, 1, 0],
... [1, 0, 0],
... [0, 1, 0],
... [1, 0, 0],
... [0, 0, 1],
... [0, 0, 1],
... [1, 0, 0],
... [0, 0, 1],
... [1, 0, 0],
... [1, 0, 0],
... [0, 1, 0],
... [1, 0, 0],
... [1, 0, 0],
... [1, 0, 0],
... [0, 1, 0],
... [1, 0, 0],
... [0, 0, 1],
... [1, 0, 0]], dtype="int32")
>>> tf.reduce_sum(labels, axis=0).numpy()
array([20, 5, 7], dtype=int32)
>>> balanced_softmax_cross_entropy_loss_weights(labels, dtype='float32').numpy()
array([0.53333336, 0.53333336, 1.5238096 , 2.1333334 , 1.5238096 ,
0.53333336, 0.53333336, 1.5238096 , 0.53333336, 0.53333336,
0.53333336, 0.53333336, 0.53333336, 0.53333336, 2.1333334 ,
0.53333336, 2.1333334 , 0.53333336, 1.5238096 , 1.5238096 ,
0.53333336, 1.5238096 , 0.53333336, 0.53333336, 2.1333334 ,
0.53333336, 0.53333336, 0.53333336, 2.1333334 , 0.53333336,
1.5238096 , 0.53333336], dtype=float32)
You would use it like this:
>>> #weights = balanced_softmax_cross_entropy_loss_weights(labels, dtype=logits.dtype)
>>> #loss = tf.keras.losses.categorical_crossentropy(y_true=labels, y_pred=logits) * weights
"""
shape = tf.cast(tf.shape(input=labels), dtype=dtype)
batch_size, n_classes = shape[0], shape[1]
weights = tf.cast(tf.reduce_sum(input_tensor=labels, axis=0), dtype=dtype)
weights = batch_size / weights / n_classes
weights = tf.gather(weights, tf.argmax(input=labels, axis=1))
return weights
def balanced_sigmoid_cross_entropy_loss_weights(labels, dtype="float32"):
"""Computes weights that normalizes your loss per class.
Labels must be a batch of binary labels. The function takes labels and
computes the weights per batch. Weights will be smaller for the class that have more
samples in this batch. This is useful if you unbalanced classes in your dataset or
batch.
Parameters
----------
labels : ``tf.Tensor``
Labels of your current input. The shape must be [batch_size] and values must be
either 0 or 1.
dtype : ``tf.dtype``
The dtype that weights will have. It should be float. Best is to provide
logits.dtype as input.
Returns
-------
``tf.Tensor``
Computed weights that will cancel your dataset imbalance per batch.
Examples
--------
>>> import numpy
>>> import tensorflow as tf
>>> from bob.learn.tensorflow.losses import balanced_sigmoid_cross_entropy_loss_weights
>>> labels = numpy.array([1, 1, 0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 0, 0, 1, 0,
... 1, 1, 0, 1, 1, 1, 0, 1, 0, 1], dtype="int32")
>>> sum(labels), len(labels)
(20, 32)
>>> balanced_sigmoid_cross_entropy_loss_weights(labels, dtype='float32').numpy()
array([0.8 , 0.8 , 1.3333334, 1.3333334, 1.3333334, 0.8 ,
0.8 , 1.3333334, 0.8 , 0.8 , 0.8 , 0.8 ,
0.8 , 0.8 , 1.3333334, 0.8 , 1.3333334, 0.8 ,
1.3333334, 1.3333334, 0.8 , 1.3333334, 0.8 , 0.8 ,
1.3333334, 0.8 , 0.8 , 0.8 , 1.3333334, 0.8 ,
1.3333334, 0.8 ], dtype=float32)
You would use it like this:
>>> #weights = balanced_sigmoid_cross_entropy_loss_weights(labels, dtype=logits.dtype)
>>> #loss = tf.losses.sigmoid_cross_entropy(logits=logits, labels=labels, weights=weights)
"""
labels = tf.cast(labels, dtype="int32")
batch_size = tf.cast(tf.shape(input=labels)[0], dtype=dtype)
weights = tf.cast(tf.reduce_sum(input_tensor=labels), dtype=dtype)
weights = tf.convert_to_tensor(value=[batch_size - weights, weights])
weights = batch_size / weights / 2
weights = tf.gather(weights, labels)
return weights
import tensorflow as tf
from ..utils import tf_repeat
from .balanced_cross_entropy import balanced_sigmoid_cross_entropy_loss_weights
def get_pixel_wise_labels(labels, n_pixels):
labels = tf.reshape(labels, (-1, 1))
labels = tf_repeat(labels, [n_pixels, 1])
labels = tf.reshape(labels, (-1, n_pixels))
return labels
class PixelwiseBinaryCrossentropy(tf.keras.losses.Loss):
"""A pixel wise loss which is just a cross entropy loss but applied to all pixels.
Appeared in::
@inproceedings{GeorgeICB2019,
author = {Anjith George, Sebastien Marcel},
title = {Deep Pixel-wise Binary Supervision for Face Presentation Attack Detection},
year = {2019},
booktitle = {ICB 2019},
}
"""
def __init__(
self,
balance_weights=True,
label_smoothing=0.5,
name="pixel_wise_binary_cross_entropy",
**kwargs
):
"""
Parameters
----------
balance_weights : bool, optional
Whether the loss should be balanced per samples of different classes in each batch, by default True
label_smoothing : float, optional
Label smoothing, by default 0.5
"""
super().__init__(name=name, **kwargs)
self.balance_weights = balance_weights
self.label_smoothing = label_smoothing
def call(self, labels, logits):
n_pixels = logits.shape[-1]
pixel_wise_labels = get_pixel_wise_labels(labels, n_pixels)
# per batch weighting, different from Keras's sample weights.
weights = 1.0
if self.balance_weights:
# use labels to figure out the required loss weights
weights = balanced_sigmoid_cross_entropy_loss_weights(
labels, dtype=logits.dtype
)
loss_pixel_wise = tf.keras.losses.binary_crossentropy(
y_true=pixel_wise_labels,
y_pred=logits,
label_smoothing=self.label_smoothing,
from_logits=True,
)
loss_pixel_wise = loss_pixel_wise * weights
return loss_pixel_wise
from .embedding_accuracy import EmbeddingAccuracy
from .embedding_accuracy import predict_using_tensors # noqa: F401
from .pixel_wise import PixelwiseBinaryAccuracy
from .pixel_wise import pixel_wise_binary_accuracy # noqa: F401
# gets sphinx autodoc done right - don't remove it
......@@ -18,5 +20,5 @@ def __appropriate__(*args):
obj.__module__ = __name__
__appropriate__(EmbeddingAccuracy)
__appropriate__(EmbeddingAccuracy, PixelwiseBinaryAccuracy)
__all__ = [_ for _ in dir() if not _.startswith("_")]
......@@ -35,5 +35,5 @@ class EmbeddingAccuracy(MeanMetricWrapper):
available from each class(identity).
"""
def __init__(self, name="embedding_accuracy", dtype=None):
super().__init__(accuracy_from_embeddings, name, dtype=dtype)
def __init__(self, name="embedding_accuracy", dtype=None, **kwargs):
super().__init__(accuracy_from_embeddings, name, dtype=dtype, **kwargs)
import tensorflow as tf
from tensorflow.python.keras.metrics import MeanMetricWrapper
from ..losses.pixel_wise import get_pixel_wise_labels
def pixel_wise_binary_accuracy(labels, logits, threshold=0.5):
n_pixels = logits.shape[-1]
labels = get_pixel_wise_labels(labels, n_pixels)
return tf.keras.metrics.binary_accuracy(labels, logits, threshold=threshold)
class PixelwiseBinaryAccuracy(MeanMetricWrapper):
"""Calculates accuracy from labels and pixel-wise logits.
The labels should not be pixel-wise themselves.
"""
def __init__(self, threshold=0.5, name="pixel_wise_accuracy", dtype=None, **kwargs):
super().__init__(
pixel_wise_binary_accuracy, name, dtype=dtype, threshold=threshold, **kwargs
)
from .alexnet import AlexNet_simplified
from .densenet import DeepPixBiS
from .densenet import DenseNet
from .densenet import densenet161 # noqa: F401
from .mine import MineModel
......@@ -19,5 +21,5 @@ def __appropriate__(*args):
obj.__module__ = __name__
__appropriate__(AlexNet_simplified, DenseNet, MineModel)
__appropriate__(AlexNet_simplified, DenseNet, DeepPixBiS, MineModel)
__all__ = [_ for _ in dir() if not _.startswith("_")]
......@@ -180,25 +180,35 @@ class TransitionBlock(tf.keras.Model):
class DenseNet(tf.keras.Model):
"""Creating the Densenet Architecture.
Arguments:
depth_of_model: number of layers in the model.
growth_rate: number of filters to add per conv block.
num_of_blocks: number of dense blocks.
output_classes: number of output classes.
num_layers_in_each_block: number of layers in each block.
If -1, then we calculate this by (depth-3)/4.
If positive integer, then the it is used as the
number of layers per block.
If list or tuple, then this list is used directly.
data_format: "channels_first" or "channels_last"
bottleneck: boolean, to decide which part of conv block to call.
compression: reducing the number of inputs(filters) to the transition block.
weight_decay: weight decay
rate: dropout rate.
pool_initial: If True add a 7x7 conv with stride 2 followed by 3x3 maxpool
else, do a 3x3 conv with stride 1.
include_top: If true, GlobalAveragePooling Layer and Dense layer are
included.
Parameters
----------
depth_of_model
number of layers in the model.
growth_rate
number of filters to add per conv block.
num_of_blocks
number of dense blocks.
output_classes
number of output classes.
num_layers_in_each_block
number of layers in each block. If -1, then we calculate this by
(depth-3)/4. If positive integer, then the it is used as the number of
layers per block. If list or tuple, then this list is used directly.
data_format
"channels_first" or "channels_last"
bottleneck
boolean, to decide which part of conv block to call.
compression
reducing the number of inputs(filters) to the transition block.
weight_decay
weight decay
rate
dropout rate.
pool_initial
If True add a 7x7 conv with stride 2 followed by 3x3 maxpool else, do a
3x3 conv with stride 1.
include_top
If true, GlobalAveragePooling Layer and Dense layer are included.
"""
def __init__(
......@@ -401,7 +411,9 @@ def densenet161(
class DeepPixBiS(tf.keras.Model):
"""DeepPixBiS"""
def __init__(self, weight_decay=1e-5, data_format="channels_last", **kwargs):
def __init__(
self, weight_decay=1e-5, data_format="channels_last", weights=None, **kwargs
):
super().__init__(**kwargs)
model = densenet161(
......@@ -410,6 +422,10 @@ class DeepPixBiS(tf.keras.Model):
weight_decay=weight_decay,
data_format=data_format,
)
if weights == "imagenet":
status = model.load_weights(rc["bob.learn.tensorflow.densenet161"])
if status is not None:
status.expect_partial()
# create a new model with needed layers
self.sequential_layers = [
......
import numpy as np
import tensorflow as tf
......@@ -21,7 +22,7 @@ def to_channels_last(image):
ValueError
If dim of image is less than 3.
"""
ndim = image.ndim
ndim = len(image.shape)
if ndim < 3:
raise ValueError(
"The image needs to be at least 3 dimensional but it " "was {}".format(ndim)
......@@ -52,7 +53,7 @@ def to_channels_first(image):
ValueError
If dim of image is less than 3.
"""
ndim = image.ndim
ndim = len(image.shape)
if ndim < 3:
raise ValueError(
"The image needs to be at least 3 dimensional but it " "was {}".format(ndim)
......@@ -61,3 +62,104 @@ def to_channels_first(image):
shift = ndim - 3
axis_order = list(range(ndim - 3)) + [n + shift for n in axis_order]
return tf.transpose(a=image, perm=axis_order)
def blocks_tensorflow(images, block_size):
"""Return all non-overlapping blocks of an image using tensorflow
operations.
Parameters
----------
images : `tf.Tensor`
The input color images. It is assumed that the image has a shape of
[?, H, W, C].
block_size : (int, int)
A tuple of two integers indicating the block size.
Returns
-------
blocks : `tf.Tensor`
All the blocks in the batch dimension. The output will be of
size [?, block_size[0], block_size[1], C].
n_blocks : int
The number of blocks that was obtained per image.
"""
# normalize block_size
block_size = [1] + list(block_size) + [1]
output_size = list(block_size)
output_size[0] = -1
output_size[-1] = images.shape[-1]
blocks = tf.image.extract_patches(
images, block_size, block_size, [1, 1, 1, 1], "VALID"
)
n_blocks = int(np.prod(blocks.shape[1:3]))
output = tf.reshape(blocks, output_size)
return output, n_blocks
def tf_repeat(tensor, repeats):
"""
Parameters
----------
tensor
A Tensor. 1-D or higher.
repeats
A list. Number of repeat for each dimension, length must be the same as
the number of dimensions in input
Returns
-------
A Tensor. Has the same type as input. Has the shape of tensor.shape *
repeats
"""
with tf.name_scope("repeat"):
expanded_tensor = tf.expand_dims(tensor, -1)
multiples = [1] + repeats
tiled_tensor = tf.tile(expanded_tensor, multiples=multiples)
repeated_tesnor = tf.reshape(tiled_tensor, tf.shape(input=tensor) * repeats)
return repeated_tesnor
def all_patches(image, label, key, size):
"""Extracts all patches of an image
Parameters
----------
image:
The image should be channels_last format and already batched.
label:
The label for the image
key:
The key for the image
size: (int, int)
The height and width of the blocks.
Returns
-------
blocks:
The non-overlapping blocks of size from image and labels and keys are
repeated.
label:
key:
"""
blocks, n_blocks = blocks_tensorflow(image, size)
# duplicate label and key as n_blocks
def repeats(shape):
r = list(shape)
for i in range(len(r)):
if i == 0:
r[i] = n_blocks
else:
r[i] = 1
return r
label = tf_repeat(label, repeats(label.shape))
key = tf_repeat(key, repeats(key.shape))
return blocks, label, key
......@@ -2,13 +2,13 @@ import tensorflow as tf
def gram_matrix(input_tensor):
"""Computes the gram matrix
"""Computes the gram matrix.
Parameters
----------
input_tensor : object
The input tensor. Usually it's the activation of a conv layer. The input shape
must be ``BHWC``.
input_tensor
The input tensor. Usually it's the activation of a conv layer. The input
shape must be ``BHWC``.
Returns
-------
......@@ -17,15 +17,9 @@ def gram_matrix(input_tensor):
Example
-------
>>>> gram_matrix(tf.zeros((32, 4, 6, 12)))
<tf.Tensor: id=53, shape=(32, 12, 12), dtype=float32, numpy=
array([[[0., 0., 0., ..., 0., 0., 0.],
[0., 0., 0., ..., 0., 0., 0.],
[0., 0., 0., ..., 0., 0., 0.],
...,
[0., 0., 0., ..., 0., 0., 0.],
[0., 0., 0., ..., 0., 0., 0.],
[0., 0., 0., ..., 0., 0., 0.]],
>>> from bob.learn.tensorflow.utils import gram_matrix