Commit 365e6379 authored by Amir MOHAMMADI's avatar Amir MOHAMMADI

Merge branch 'new-changes' into 'master'

Add keras-based models, add pixel-wise loss, other improvements

See merge request !79
parents 1fd8ec79 25d35e2a
Pipeline #36819 passed with stages
in 9 minutes and 45 seconds
......@@ -12,3 +12,5 @@ develop-eggs
sphinx
dist
temp/
build/
record.txt
import logging
logging.getLogger("tensorflow").setLevel(logging.WARNING)
def get_config():
"""
Returns a string containing the configuration information.
......
......@@ -86,10 +86,10 @@ def append_image_augmentation(
tf.set_random_seed(0)
if output_shape is not None:
assert len(output_shape) == 2
if random_crop:
image = tf.random_crop(image, size=list(output_shape) + [3])
else:
assert len(output_shape) == 2
image = tf.image.resize_image_with_crop_or_pad(
image, output_shape[0], output_shape[1]
)
......@@ -115,6 +115,19 @@ def append_image_augmentation(
)
image = tf.clip_by_value(image, 0, 1)
if random_rotate:
# from https://stackoverflow.com/a/53855704/1286165
degree = 0.08726646259971647 # math.pi * 5 /180
random_angles = tf.random.uniform(shape=(1,), minval=-degree, maxval=degree)
image = tf.contrib.image.transform(
image,
tf.contrib.image.angles_to_projective_transforms(
random_angles,
tf.cast(tf.shape(image)[-3], tf.float32),
tf.cast(tf.shape(image)[-2], tf.float32),
),
)
if gray_scale:
image = tf.image.rgb_to_grayscale(image, name="rgb_to_gray")
......@@ -329,18 +342,12 @@ def blocks_tensorflow(images, block_size):
block_size = [1] + list(block_size) + [1]
output_size = list(block_size)
output_size[0] = -1
# extract image patches for each color space:
output = []
for i in range(3):
blocks = tf.extract_image_patches(
images[:, :, :, i : i + 1], block_size, block_size, [1, 1, 1, 1], "VALID"
)
if i == 0:
n_blocks = int(numpy.prod(blocks.shape[1:3]))
blocks = tf.reshape(blocks, output_size)
output.append(blocks)
# concatenate the colors back
output = tf.concat(output, axis=3)
output_size[-1] = images.shape[-1]
blocks = tf.extract_image_patches(
images, block_size, block_size, [1, 1, 1, 1], "VALID"
)
n_blocks = int(numpy.prod(blocks.shape[1:3]))
output = tf.reshape(blocks, output_size)
return output, n_blocks
......
import six
import tensorflow as tf
from bob.bio.base import read_original_data
from .generator import Generator
import logging
logger = logging.getLogger(__name__)
class BioGenerator(object):
class BioGenerator(Generator):
"""A generator class which wraps bob.bio.base databases so that they can
be used with tf.data.Dataset.from_generator
......@@ -15,44 +14,37 @@ class BioGenerator(object):
biofile_to_label : :obj:`object`, optional
A callable with the signature of ``label = biofile_to_label(biofile)``.
By default -1 is returned as label.
biofiles : [:any:`bob.bio.base.database.BioFile`]
The list of the bio files .
database : :any:`bob.bio.base.database.BioDatabase`
The database that you want to use.
epoch : int
The number of epochs that have been passed so far.
keys : [str]
The keys of samples obtained by calling ``biofile.make_path("", "")``
labels : [int]
The labels obtained by calling ``label = biofile_to_label(biofile)``
load_data : :obj:`object`, optional
A callable with the signature of
``data = load_data(database, biofile)``.
:any:`bob.bio.base.read_original_data` is wrapped to be used by
default.
multiple_samples : :obj:`bool`, optional
If true, it assumes that the bio database's samples actually contain
multiple samples. This is useful for when you want to for example treat
video databases as image databases.
output_types : (object, object, object)
The types of the returned samples.
output_shapes : ``(tf.TensorShape, tf.TensorShape, tf.TensorShape)``
The shapes of the returned samples.
biofiles : [:any:`bob.bio.base.database.BioFile`]
The list of the bio files .
keys : [str]
The keys of samples obtained by calling ``biofile.make_path("", "")``
labels : [int]
The labels obtained by calling ``label = biofile_to_label(biofile)``
"""
def __init__(self,
database,
biofiles,
load_data=None,
biofile_to_label=None,
multiple_samples=False,
**kwargs):
super(BioGenerator, self).__init__(**kwargs)
def __init__(
self,
database,
biofiles,
load_data=None,
biofile_to_label=None,
multiple_samples=False,
**kwargs
):
if load_data is None:
def load_data(database, biofile):
data = read_original_data(biofile, database.original_directory,
database.original_extension)
data = read_original_data(
biofile, database.original_directory, database.original_extension
)
return data
if biofile_to_label is None:
......@@ -61,29 +53,27 @@ class BioGenerator(object):
return -1
self.database = database
self.biofiles = list(biofiles)
self.load_data = load_data
self.biofile_to_label = biofile_to_label
self.multiple_samples = multiple_samples
self.epoch = 0
# load one data to get its type and shape
data = load_data(database, biofiles[0])
def _reader(f):
label = int(self.biofile_to_label(f))
data = self.load_data(self.database, f)
key = str(f.make_path("", "")).encode("utf-8")
return data, label, key
if multiple_samples:
try:
data = data[0]
except TypeError:
# if the data is a generator
data = six.next(data)
data = tf.convert_to_tensor(data)
self._output_types = (data.dtype, tf.int64, tf.string)
self._output_shapes = (data.shape, tf.TensorShape([]),
tf.TensorShape([]))
logger.info(
"Initializing a dataset with %d files and %s types "
"and %s shapes", len(self.biofiles), self.output_types,
self.output_shapes)
def reader(f):
data, label, key = _reader(f)
for d in data:
yield (d, label, key)
else:
def reader(f):
return _reader(f)
super(BioGenerator, self).__init__(
biofiles, reader, multiple_samples=multiple_samples, **kwargs
)
@property
def labels(self):
......@@ -93,34 +83,11 @@ class BioGenerator(object):
@property
def keys(self):
for f in self.biofiles:
yield str(f.make_path("", "")).encode('utf-8')
@property
def output_types(self):
return self._output_types
yield str(f.make_path("", "")).encode("utf-8")
@property
def output_shapes(self):
return self._output_shapes
def biofiles(self):
return self.samples
def __len__(self):
return len(self.biofiles)
def __call__(self):
"""A generator function that when called will return the samples.
Yields
------
(data, label, key) : tuple
A tuple containing the data, label, and the key.
"""
for f, label, key in six.moves.zip(self.biofiles, self.labels,
self.keys):
data = self.load_data(self.database, f)
if self.multiple_samples:
for d in data:
yield (d, label, key)
else:
yield (data, label, key)
self.epoch += 1
logger.info("Elapsed %d epoch(s)", self.epoch)
import six
import tensorflow as tf
import random
import logging
logger = logging.getLogger(__name__)
......@@ -22,27 +22,37 @@ class Generator:
which takes a sample and loads it.
samples : [:obj:`object`]
A list of samples to be given to ``reader`` to load the data.
shuffle_on_epoch_end : :obj:`bool`, optional
If True, it shuffle the samples at the end of each epoch.
output_types : (object, object, object)
The types of the returned samples.
output_shapes : ``(tf.TensorShape, tf.TensorShape, tf.TensorShape)``
The shapes of the returned samples.
"""
def __init__(self, samples, reader, multiple_samples=False, **kwargs):
def __init__(self, samples, reader, multiple_samples=False, shuffle_on_epoch_end=False, **kwargs):
super().__init__(**kwargs)
self.reader = reader
self.samples = list(samples)
self.multiple_samples = multiple_samples
self.epoch = 0
self.shuffle_on_epoch_end = shuffle_on_epoch_end
# load one data to get its type and shape
dlk = self.reader(self.samples[0])
if self.multiple_samples:
# load samples until one of them is not empty
# this data is used to get the type and shape
for sample in self.samples:
try:
dlk = dlk[0]
except TypeError:
# if the data is a generator
dlk = six.next(dlk)
dlk = self.reader(sample)
if self.multiple_samples:
try:
dlk = dlk[0]
except TypeError:
# if the data is a generator
dlk = next(dlk)
except StopIteration:
continue
else:
break
# Creating a "fake" dataset just to get the types and shapes
dataset = tf.data.Dataset.from_tensors(dlk)
self._output_types = dataset.output_types
......@@ -81,31 +91,34 @@ class Generator:
yield dlk
self.epoch += 1
logger.info("Elapsed %d epoch(s)", self.epoch)
if self.shuffle_on_epoch_end:
logger.info("Shuffling samples")
random.shuffle(self.samples)
def dataset_using_generator(*args, **kwargs):
def dataset_using_generator(samples, reader, **kwargs):
"""
A generator class which wraps samples so that they can
be used with tf.data.Dataset.from_generator
Attributes
Parameters
----------
samples : [:obj:`object`]
A list of samples to be given to ``reader`` to load the data.
samples : [:obj:`object`]
A list of samples to be given to ``reader`` to load the data.
reader : :obj:`object`, optional
A callable with the signature of ``data, label, key = reader(sample)``
which takes a sample and loads it.
multiple_samples : :obj:`bool`, optional
If true, it assumes that the bio database's samples actually contain
multiple samples. This is useful for when you want to for example treat
video databases as image databases.
reader : :obj:`object`, optional
A callable with the signature of ``data, label, key = reader(sample)``
which takes a sample and loads it.
**kwargs
Extra keyword arguments are passed to Generator
Returns
-------
object
A tf.data.Dataset
"""
generator = Generator(*args, **kwargs)
generator = Generator(samples, reader, **kwargs)
dataset = tf.data.Dataset.from_generator(
generator, generator.output_types, generator.output_shapes
)
......
......@@ -197,3 +197,25 @@ def image_augmentation_parser(filename,
features['key'] = filename
return features, label
def load_pngs(img_path, img_shape):
"""Read png files using tensorflow API
You must know the shape of the image beforehand to use this function.
Parameters
----------
img_path : str
Path to the image
img_shape : list
A list or tuple that contains image's shape in channels_last format
Returns
-------
object
The loaded png file
"""
img_raw = tf.read_file(img_path)
img_tensor = tf.image.decode_png(img_raw, channels=img_shape[-1])
img_final = tf.reshape(img_tensor, img_shape)
return img_final
......@@ -87,7 +87,7 @@ def dataset_to_tfrecord(dataset, output):
return writer.write(dataset)
def dataset_from_tfrecord(tfrecord):
def dataset_from_tfrecord(tfrecord, num_parallel_reads=None):
"""Reads TFRecords and returns a dataset.
The TFRecord file must have been created using the :any:`dataset_to_tfrecord`
function.
......@@ -97,6 +97,9 @@ def dataset_from_tfrecord(tfrecord):
tfrecord : str or list
Path to the TFRecord file. Pass a list if you are sure several tfrecords need
the same map function.
num_parallel_reads: int
A `tf.int64` scalar representing the number of files to read in parallel.
Defaults to reading files sequentially.
Returns
-------
......@@ -111,7 +114,9 @@ def dataset_from_tfrecord(tfrecord):
tfrecord = [tfrecord_name_and_json_name(path) for path in tfrecord]
json_output = tfrecord[0][1]
tfrecord = [path[0] for path in tfrecord]
raw_dataset = tf.data.TFRecordDataset(tfrecord)
raw_dataset = tf.data.TFRecordDataset(
tfrecord, num_parallel_reads=num_parallel_reads
)
with open(json_output) as f:
meta = json.load(f)
......
This diff is collapsed.
......@@ -25,6 +25,7 @@ class Regressor(estimator.Estimator):
add_histograms=None,
optimize_loss=tf.contrib.layers.optimize_loss,
optimize_loss_learning_rate=None,
architecture_has_logits=False,
):
self.architecture = architecture
self.label_dimension = label_dimension
......@@ -51,12 +52,15 @@ class Regressor(estimator.Estimator):
# Checking if we have some variables/scope that we may want to shut
# down
trainable_variables = get_trainable_variables(extra_checkpoint, mode=mode)
prelogits = self.architecture(
prelogits, end_points = self.architecture(
data, mode=mode, trainable_variables=trainable_variables
)[0]
logits = append_logits(
prelogits, label_dimension, trainable_variables=trainable_variables
)
if architecture_has_logits:
logits, prelogits = prelogits, end_points["prelogits"]
else:
logits = append_logits(
prelogits, label_dimension, trainable_variables=trainable_variables
)
predictions = {"predictions": logits, "key": key}
......
......@@ -4,52 +4,7 @@
import tensorflow as tf
def check_features(features):
if "data" not in features or "key" not in features:
raise ValueError(
"The input function needs to contain a dictionary with the keys `data` and `key` "
)
return True
def get_trainable_variables(extra_checkpoint, mode=tf.estimator.ModeKeys.TRAIN):
"""
Given the extra_checkpoint dictionary provided to the estimator,
extract the content of "trainable_variables".
If trainable_variables is not provided, all end points are trainable by
default.
If trainable_variables==[], all end points are NOT trainable.
If trainable_variables contains some end_points, ONLY these endpoints will
be trainable.
Attributes
----------
extra_checkpoint: dict
The extra_checkpoint dictionary provided to the estimator
mode:
The estimator mode. TRAIN, EVAL, and PREDICT. If not TRAIN, None is
returned.
Returns
-------
Returns `None` if **trainable_variables** is not in extra_checkpoint;
otherwise returns the content of extra_checkpoint .
"""
if mode != tf.estimator.ModeKeys.TRAIN:
return None
# If you don't set anything, everything is trainable
if extra_checkpoint is None or "trainable_variables" not in extra_checkpoint:
return None
return extra_checkpoint["trainable_variables"]
from ..utils import get_trainable_variables, check_features
from .utils import MovingAverageOptimizer, learning_rate_decay_fn
from .Logits import Logits, LogitsCenterLoss
from .Siamese import Siamese
......
......@@ -20,7 +20,7 @@ def normalize_checkpoint_path(path):
class Base:
def __init__(self, output_name, input_shape, checkpoint, scopes,
input_transform=None, output_transform=None,
input_dtype='float32', **kwargs):
input_dtype='float32', extra_feed=None, **kwargs):
self.output_name = output_name
self.input_shape = input_shape
......@@ -29,6 +29,7 @@ class Base:
self.input_transform = input_transform
self.output_transform = output_transform
self.input_dtype = input_dtype
self.extra_feed = extra_feed
self.session = None
super().__init__(**kwargs)
......@@ -60,8 +61,11 @@ class Base:
self.load()
data = np.ascontiguousarray(data, dtype=self.input_dtype)
feed_dict = {self.input: data}
if self.extra_feed is not None:
feed_dict.update(self.extra_feed)
return self.session.run(self.output, feed_dict={self.input: data})
return self.session.run(self.output, feed_dict=feed_dict)
def get_output(self, data, mode):
raise NotImplementedError()
from . import spectral_normalization
from . import losses
import tensorflow as tf
def relativistic_discriminator_loss(
discriminator_real_outputs,
discriminator_gen_outputs,
label_smoothing=0.25,
real_weights=1.0,
generated_weights=1.0,
scope=None,
loss_collection=tf.GraphKeys.LOSSES,
reduction=tf.losses.Reduction.SUM_BY_NONZERO_WEIGHTS,
add_summaries=False,
):
"""Relativistic (average) loss
Args:
discriminator_real_outputs: Discriminator output on real data.
discriminator_gen_outputs: Discriminator output on generated data. Expected
to be in the range of (-inf, inf).
label_smoothing: The amount of smoothing for positive labels. This technique
is taken from `Improved Techniques for Training GANs`
(https://arxiv.org/abs/1606.03498). `0.0` means no smoothing.
real_weights: Optional `Tensor` whose rank is either 0, or the same rank as
`real_data`, and must be broadcastable to `real_data` (i.e., all
dimensions must be either `1`, or the same as the corresponding
dimension).
generated_weights: Same as `real_weights`, but for `generated_data`.
scope: The scope for the operations performed in computing the loss.
loss_collection: collection to which this loss will be added.
reduction: A `tf.compat.v1.losses.Reduction` to apply to loss.
add_summaries: Whether or not to add summaries for the loss.
Returns:
A loss Tensor. The shape depends on `reduction`.
"""
with tf.name_scope(
scope,
"discriminator_relativistic_loss",
(
discriminator_real_outputs,
discriminator_gen_outputs,
real_weights,
generated_weights,
label_smoothing,
),
) as scope:
real_logit = discriminator_real_outputs - tf.reduce_mean(
discriminator_gen_outputs
)
fake_logit = discriminator_gen_outputs - tf.reduce_mean(
discriminator_real_outputs
)
loss_on_real = tf.losses.sigmoid_cross_entropy(
tf.ones_like(real_logit),
real_logit,
real_weights,
label_smoothing,
scope,
loss_collection=None,
reduction=reduction,
)
loss_on_generated = tf.losses.sigmoid_cross_entropy(
tf.zeros_like(fake_logit),
fake_logit,
generated_weights,
scope=scope,
loss_collection=None,
reduction=reduction,
)
loss = loss_on_real + loss_on_generated
tf.losses.add_loss(loss, loss_collection)
if add_summaries:
tf.summary.scalar("discriminator_gen_relativistic_loss", loss_on_generated)
tf.summary.scalar("discriminator_real_relativistic_loss", loss_on_real)
tf.summary.scalar("discriminator_relativistic_loss", loss)
return loss
def relativistic_generator_loss(
discriminator_real_outputs,
discriminator_gen_outputs,
label_smoothing=0.0,
real_weights=1.0,
generated_weights=1.0,
scope=None,
loss_collection=tf.GraphKeys.LOSSES,
reduction=tf.losses.Reduction.SUM_BY_NONZERO_WEIGHTS,
add_summaries=False,
confusion_labels=False,
):
"""Relativistic (average) loss
Args:
discriminator_real_outputs: Discriminator output on real data.
discriminator_gen_outputs: Discriminator output on generated data. Expected
to be in the range of (-inf, inf).
label_smoothing: The amount of smoothing for positive labels. This technique
is taken from `Improved Techniques for Training GANs`
(https://arxiv.org/abs/1606.03498). `0.0` means no smoothing.
real_weights: Optional `Tensor` whose rank is either 0, or the same rank as
`real_data`, and must be broadcastable to `real_data` (i.e., all
dimensions must be either `1`, or the same as the corresponding
dimension).
generated_weights: Same as `real_weights`, but for `generated_data`.
scope: The scope for the operations performed in computing the loss.
loss_collection: collection to which this loss will be added.
reduction: A `tf.compat.v1.losses.Reduction` to apply to loss.
add_summaries: Whether or not to add summaries for the loss.
Returns:
A loss Tensor. The shape depends on `reduction`.
"""
with tf.name_scope(
scope,
"generator_relativistic_loss",
(
discriminator_real_outputs,
discriminator_gen_outputs,
real_weights,
generated_weights,
label_smoothing,
),
) as scope:
real_logit = discriminator_real_outputs - tf.reduce_mean(
discriminator_gen_outputs
)
fake_logit = discriminator_gen_outputs - tf.reduce_mean(
discriminator_real_outputs
)
if confusion_labels:
real_labels = tf.ones_like(real_logit) / 2
fake_labels = tf.ones_like(fake_logit) / 2
else:
real_labels = tf.zeros_like(real_logit)
fake_labels = tf.ones_like(fake_logit)
loss_on_real = tf.losses.sigmoid_cross_entropy(
real_labels,
real_logit,
real_weights,
label_smoothing,
scope,
loss_collection=None,
reduction=reduction,
)
loss_on_generated = tf.losses.sigmoid_cross_entropy(
fake_labels,
fake_logit,
generated_weights,
scope=scope,
loss_collection=None,
reduction=reduction,
)
loss = loss_on_real + loss_on_generated
tf.losses.add_loss(loss, loss_collection)
if add_summaries:
tf.summary.scalar("generator_gen_relativistic_loss", loss_on_generated)
tf.summary.scalar("generator_real_relativistic_loss", loss_on_real)
tf.summary.scalar("generator_relativistic_loss", loss)
return loss