Commit 365e6379 authored by Amir MOHAMMADI's avatar Amir MOHAMMADI
Browse files

Merge branch 'new-changes' into 'master'

Add keras-based models, add pixel-wise loss, other improvements

See merge request !79
parents 1fd8ec79 25d35e2a
Pipeline #36819 passed with stages
in 9 minutes and 45 seconds
......@@ -12,3 +12,5 @@ develop-eggs
sphinx
dist
temp/
build/
record.txt
import logging
logging.getLogger("tensorflow").setLevel(logging.WARNING)
def get_config():
"""
Returns a string containing the configuration information.
......
......@@ -86,10 +86,10 @@ def append_image_augmentation(
tf.set_random_seed(0)
if output_shape is not None:
assert len(output_shape) == 2
if random_crop:
image = tf.random_crop(image, size=list(output_shape) + [3])
else:
assert len(output_shape) == 2
image = tf.image.resize_image_with_crop_or_pad(
image, output_shape[0], output_shape[1]
)
......@@ -115,6 +115,19 @@ def append_image_augmentation(
)
image = tf.clip_by_value(image, 0, 1)
if random_rotate:
# from https://stackoverflow.com/a/53855704/1286165
degree = 0.08726646259971647 # math.pi * 5 /180
random_angles = tf.random.uniform(shape=(1,), minval=-degree, maxval=degree)
image = tf.contrib.image.transform(
image,
tf.contrib.image.angles_to_projective_transforms(
random_angles,
tf.cast(tf.shape(image)[-3], tf.float32),
tf.cast(tf.shape(image)[-2], tf.float32),
),
)
if gray_scale:
image = tf.image.rgb_to_grayscale(image, name="rgb_to_gray")
......@@ -329,18 +342,12 @@ def blocks_tensorflow(images, block_size):
block_size = [1] + list(block_size) + [1]
output_size = list(block_size)
output_size[0] = -1
# extract image patches for each color space:
output = []
for i in range(3):
blocks = tf.extract_image_patches(
images[:, :, :, i : i + 1], block_size, block_size, [1, 1, 1, 1], "VALID"
)
if i == 0:
n_blocks = int(numpy.prod(blocks.shape[1:3]))
blocks = tf.reshape(blocks, output_size)
output.append(blocks)
# concatenate the colors back
output = tf.concat(output, axis=3)
output_size[-1] = images.shape[-1]
blocks = tf.extract_image_patches(
images, block_size, block_size, [1, 1, 1, 1], "VALID"
)
n_blocks = int(numpy.prod(blocks.shape[1:3]))
output = tf.reshape(blocks, output_size)
return output, n_blocks
......
import six
import tensorflow as tf
from bob.bio.base import read_original_data
from .generator import Generator
import logging
logger = logging.getLogger(__name__)
class BioGenerator(object):
class BioGenerator(Generator):
"""A generator class which wraps bob.bio.base databases so that they can
be used with tf.data.Dataset.from_generator
......@@ -15,44 +14,37 @@ class BioGenerator(object):
biofile_to_label : :obj:`object`, optional
A callable with the signature of ``label = biofile_to_label(biofile)``.
By default -1 is returned as label.
biofiles : [:any:`bob.bio.base.database.BioFile`]
The list of the bio files .
database : :any:`bob.bio.base.database.BioDatabase`
The database that you want to use.
epoch : int
The number of epochs that have been passed so far.
keys : [str]
The keys of samples obtained by calling ``biofile.make_path("", "")``
labels : [int]
The labels obtained by calling ``label = biofile_to_label(biofile)``
load_data : :obj:`object`, optional
A callable with the signature of
``data = load_data(database, biofile)``.
:any:`bob.bio.base.read_original_data` is wrapped to be used by
default.
multiple_samples : :obj:`bool`, optional
If true, it assumes that the bio database's samples actually contain
multiple samples. This is useful for when you want to for example treat
video databases as image databases.
output_types : (object, object, object)
The types of the returned samples.
output_shapes : ``(tf.TensorShape, tf.TensorShape, tf.TensorShape)``
The shapes of the returned samples.
biofiles : [:any:`bob.bio.base.database.BioFile`]
The list of the bio files .
keys : [str]
The keys of samples obtained by calling ``biofile.make_path("", "")``
labels : [int]
The labels obtained by calling ``label = biofile_to_label(biofile)``
"""
def __init__(self,
database,
biofiles,
load_data=None,
biofile_to_label=None,
multiple_samples=False,
**kwargs):
super(BioGenerator, self).__init__(**kwargs)
def __init__(
self,
database,
biofiles,
load_data=None,
biofile_to_label=None,
multiple_samples=False,
**kwargs
):
if load_data is None:
def load_data(database, biofile):
data = read_original_data(biofile, database.original_directory,
database.original_extension)
data = read_original_data(
biofile, database.original_directory, database.original_extension
)
return data
if biofile_to_label is None:
......@@ -61,29 +53,27 @@ class BioGenerator(object):
return -1
self.database = database
self.biofiles = list(biofiles)
self.load_data = load_data
self.biofile_to_label = biofile_to_label
self.multiple_samples = multiple_samples
self.epoch = 0
# load one data to get its type and shape
data = load_data(database, biofiles[0])
def _reader(f):
label = int(self.biofile_to_label(f))
data = self.load_data(self.database, f)
key = str(f.make_path("", "")).encode("utf-8")
return data, label, key
if multiple_samples:
try:
data = data[0]
except TypeError:
# if the data is a generator
data = six.next(data)
data = tf.convert_to_tensor(data)
self._output_types = (data.dtype, tf.int64, tf.string)
self._output_shapes = (data.shape, tf.TensorShape([]),
tf.TensorShape([]))
logger.info(
"Initializing a dataset with %d files and %s types "
"and %s shapes", len(self.biofiles), self.output_types,
self.output_shapes)
def reader(f):
data, label, key = _reader(f)
for d in data:
yield (d, label, key)
else:
def reader(f):
return _reader(f)
super(BioGenerator, self).__init__(
biofiles, reader, multiple_samples=multiple_samples, **kwargs
)
@property
def labels(self):
......@@ -93,34 +83,11 @@ class BioGenerator(object):
@property
def keys(self):
for f in self.biofiles:
yield str(f.make_path("", "")).encode('utf-8')
@property
def output_types(self):
return self._output_types
yield str(f.make_path("", "")).encode("utf-8")
@property
def output_shapes(self):
return self._output_shapes
def biofiles(self):
return self.samples
def __len__(self):
return len(self.biofiles)
def __call__(self):
"""A generator function that when called will return the samples.
Yields
------
(data, label, key) : tuple
A tuple containing the data, label, and the key.
"""
for f, label, key in six.moves.zip(self.biofiles, self.labels,
self.keys):
data = self.load_data(self.database, f)
if self.multiple_samples:
for d in data:
yield (d, label, key)
else:
yield (data, label, key)
self.epoch += 1
logger.info("Elapsed %d epoch(s)", self.epoch)
import six
import tensorflow as tf
import random
import logging
logger = logging.getLogger(__name__)
......@@ -22,27 +22,37 @@ class Generator:
which takes a sample and loads it.
samples : [:obj:`object`]
A list of samples to be given to ``reader`` to load the data.
shuffle_on_epoch_end : :obj:`bool`, optional
If True, it shuffle the samples at the end of each epoch.
output_types : (object, object, object)
The types of the returned samples.
output_shapes : ``(tf.TensorShape, tf.TensorShape, tf.TensorShape)``
The shapes of the returned samples.
"""
def __init__(self, samples, reader, multiple_samples=False, **kwargs):
def __init__(self, samples, reader, multiple_samples=False, shuffle_on_epoch_end=False, **kwargs):
super().__init__(**kwargs)
self.reader = reader
self.samples = list(samples)
self.multiple_samples = multiple_samples
self.epoch = 0
self.shuffle_on_epoch_end = shuffle_on_epoch_end
# load one data to get its type and shape
dlk = self.reader(self.samples[0])
if self.multiple_samples:
# load samples until one of them is not empty
# this data is used to get the type and shape
for sample in self.samples:
try:
dlk = dlk[0]
except TypeError:
# if the data is a generator
dlk = six.next(dlk)
dlk = self.reader(sample)
if self.multiple_samples:
try:
dlk = dlk[0]
except TypeError:
# if the data is a generator
dlk = next(dlk)
except StopIteration:
continue
else:
break
# Creating a "fake" dataset just to get the types and shapes
dataset = tf.data.Dataset.from_tensors(dlk)
self._output_types = dataset.output_types
......@@ -81,31 +91,34 @@ class Generator:
yield dlk
self.epoch += 1
logger.info("Elapsed %d epoch(s)", self.epoch)
if self.shuffle_on_epoch_end:
logger.info("Shuffling samples")
random.shuffle(self.samples)
def dataset_using_generator(*args, **kwargs):
def dataset_using_generator(samples, reader, **kwargs):
"""
A generator class which wraps samples so that they can
be used with tf.data.Dataset.from_generator
Attributes
Parameters
----------
samples : [:obj:`object`]
A list of samples to be given to ``reader`` to load the data.
samples : [:obj:`object`]
A list of samples to be given to ``reader`` to load the data.
reader : :obj:`object`, optional
A callable with the signature of ``data, label, key = reader(sample)``
which takes a sample and loads it.
multiple_samples : :obj:`bool`, optional
If true, it assumes that the bio database's samples actually contain
multiple samples. This is useful for when you want to for example treat
video databases as image databases.
reader : :obj:`object`, optional
A callable with the signature of ``data, label, key = reader(sample)``
which takes a sample and loads it.
**kwargs
Extra keyword arguments are passed to Generator
Returns
-------
object
A tf.data.Dataset
"""
generator = Generator(*args, **kwargs)
generator = Generator(samples, reader, **kwargs)
dataset = tf.data.Dataset.from_generator(
generator, generator.output_types, generator.output_shapes
)
......
......@@ -197,3 +197,25 @@ def image_augmentation_parser(filename,
features['key'] = filename
return features, label
def load_pngs(img_path, img_shape):
"""Read png files using tensorflow API
You must know the shape of the image beforehand to use this function.
Parameters
----------
img_path : str
Path to the image
img_shape : list
A list or tuple that contains image's shape in channels_last format
Returns
-------
object
The loaded png file
"""
img_raw = tf.read_file(img_path)
img_tensor = tf.image.decode_png(img_raw, channels=img_shape[-1])
img_final = tf.reshape(img_tensor, img_shape)
return img_final
......@@ -87,7 +87,7 @@ def dataset_to_tfrecord(dataset, output):
return writer.write(dataset)
def dataset_from_tfrecord(tfrecord):
def dataset_from_tfrecord(tfrecord, num_parallel_reads=None):
"""Reads TFRecords and returns a dataset.
The TFRecord file must have been created using the :any:`dataset_to_tfrecord`
function.
......@@ -97,6 +97,9 @@ def dataset_from_tfrecord(tfrecord):
tfrecord : str or list
Path to the TFRecord file. Pass a list if you are sure several tfrecords need
the same map function.
num_parallel_reads: int
A `tf.int64` scalar representing the number of files to read in parallel.
Defaults to reading files sequentially.
Returns
-------
......@@ -111,7 +114,9 @@ def dataset_from_tfrecord(tfrecord):
tfrecord = [tfrecord_name_and_json_name(path) for path in tfrecord]
json_output = tfrecord[0][1]
tfrecord = [path[0] for path in tfrecord]
raw_dataset = tf.data.TFRecordDataset(tfrecord)
raw_dataset = tf.data.TFRecordDataset(
tfrecord, num_parallel_reads=num_parallel_reads
)
with open(json_output) as f:
meta = json.load(f)
......
# vim: set fileencoding=utf-8 :
# @author: Amir Mohammadi <amir.mohammadi@idiap.ch>
from . import check_features, get_trainable_variables
from .Logits import moving_average_scaffold
from ..network.utils import append_logits
from ..utils import predict_using_tensors
from ..loss.epsc import epsc_metric, siamese_loss
from tensorflow.python.estimator import estimator
import tensorflow as tf
import logging
logger = logging.getLogger(__name__)
class EPSCBase:
"""A base class for EPSC based estimators"""
def _get_loss(self, bio_logits, pad_logits, bio_labels, pad_labels, mode):
main_loss = self.loss_op(
bio_logits=bio_logits,
pad_logits=pad_logits,
bio_labels=bio_labels,
pad_labels=pad_labels,
)
total_loss = main_loss
if self.add_regularization_losses:
regularization_losses = tf.get_collection(
tf.GraphKeys.REGULARIZATION_LOSSES
)
regularization_losses = [
tf.cast(l, main_loss.dtype) for l in regularization_losses
]
regularization_losses = tf.add_n(
regularization_losses, name="regularization_losses"
)
tf.summary.scalar("regularization_losses", regularization_losses)
total_loss = tf.add_n([main_loss, regularization_losses], name="total_loss")
if self.vat_loss is not None:
vat_loss = self.vat_loss(
self.end_points["features"],
self.end_points["Logits/PAD"],
self.pad_architecture,
mode,
)
total_loss = tf.add_n([main_loss, vat_loss], name="total_loss")
return total_loss
class EPSCLogits(EPSCBase, estimator.Estimator):
"""An logits estimator for epsc problems"""
def __init__(
self,
architecture,
optimizer,
loss_op,
n_classes,
config=None,
embedding_validation=False,
model_dir="",
validation_batch_size=None,
extra_checkpoint=None,
apply_moving_averages=True,
add_histograms="train",
add_regularization_losses=True,
vat_loss=None,
optimize_loss=tf.contrib.layers.optimize_loss,
optimize_loss_learning_rate=None,
):
self.architecture = architecture
self.n_classes = n_classes
self.loss_op = loss_op
self.loss = None
self.embedding_validation = embedding_validation
self.extra_checkpoint = extra_checkpoint
self.add_regularization_losses = add_regularization_losses
self.apply_moving_averages = apply_moving_averages
self.vat_loss = vat_loss
self.optimize_loss = optimize_loss
self.optimize_loss_learning_rate = optimize_loss_learning_rate
if apply_moving_averages and isinstance(optimizer, tf.train.Optimizer):
logger.info(
"Encapsulating the optimizer with " "the MovingAverageOptimizer"
)
optimizer = tf.contrib.opt.MovingAverageOptimizer(optimizer)
self.optimizer = optimizer
def _model_fn(features, labels, mode):
check_features(features)
data = features["data"]
key = features["key"]
# Checking if we have some variables/scope that we may want to shut
# down
trainable_variables = get_trainable_variables(
self.extra_checkpoint, mode=mode
)
prelogits, end_points = self.architecture(
data, mode=mode, trainable_variables=trainable_variables
)
name = "Logits/Bio"
bio_logits = append_logits(
prelogits, n_classes, trainable_variables=trainable_variables, name=name
)
end_points[name] = bio_logits
name = "Logits/PAD"
pad_logits = append_logits(
prelogits, 2, trainable_variables=trainable_variables, name=name
)
end_points[name] = pad_logits
self.end_points = end_points
# for vat_loss
self.end_points["features"] = data
def pad_architecture(features, mode, reuse):
prelogits, end_points = self.architecture(
features,
mode=mode,
trainable_variables=trainable_variables,
reuse=reuse,
)
pad_logits = append_logits(
prelogits,
2,
reuse=reuse,
trainable_variables=trainable_variables,
name="Logits/PAD",
)
return pad_logits, end_points
self.pad_architecture = pad_architecture
if self.embedding_validation and mode != tf.estimator.ModeKeys.TRAIN:
# Compute the embeddings
embeddings = tf.nn.l2_normalize(prelogits, 1)
predictions = {"embeddings": embeddings}
else:
predictions = {
# Generate predictions (for PREDICT and EVAL mode)
"bio_classes": tf.argmax(input=bio_logits, axis=1),
# Add `softmax_tensor` to the graph. It is used for PREDICT
# and by the `logging_hook`.
"bio_probabilities": tf.nn.softmax(
bio_logits, name="bio_softmax_tensor"
),
}
predictions.update(
{
"pad_classes": tf.argmax(input=pad_logits, axis=1),
"pad_probabilities": tf.nn.softmax(
pad_logits, name="pad_softmax_tensor"
),
"key": key,
}
)
# add predictions to end_points
self.end_points.update(predictions)
if mode == tf.estimator.ModeKeys.PREDICT:
return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions)
bio_labels = labels["bio"]
pad_labels = labels["pad"]
if self.embedding_validation and mode != tf.estimator.ModeKeys.TRAIN:
bio_predictions_op = predict_using_tensors(
predictions["embeddings"], bio_labels, num=validation_batch_size
)
else:
bio_predictions_op = predictions["bio_classes"]
pad_predictions_op = predictions["pad_classes"]
metrics = {
"bio_accuracy": tf.metrics.accuracy(
labels=bio_labels, predictions=bio_predictions_op
),
"pad_accuracy": tf.metrics.accuracy(
labels=pad_labels, predictions=pad_predictions_op
),
}
if mode == tf.estimator.ModeKeys.EVAL:
self.loss = self._get_loss(
bio_logits, pad_logits, bio_labels, pad_labels, mode=mode
)
return tf.estimator.EstimatorSpec(
mode=mode,
predictions=predictions,
loss=self.loss,
train_op=None,