Implemented data augmentation

bob/learn/tensorflow/datashuffler/Base.py

@@ -15,7 +15,8 @@ class Base(object):
                  input_dtype="float64",
                  scale=True,
                  batch_size=1,
-                 seed=10):
+                 seed=10,
+                 data_augmentation=None):
         """
          The class provide base functionalities to shuffle the data before to train a neural network
 
@@ -56,6 +57,8 @@ class Base(object):
         self.data_placeholder = None
         self.label_placeholder = None
 
+        self.data_augmentation = data_augmentation
+
     def get_placeholders(self, name=""):
         """
         Returns a place holder with the size of your batch
@@ -83,14 +86,34 @@ class Base(object):
         Convert bob color image to the skcit image
         """
 
-        skimage = numpy.zeros(shape=(bob_image.shape[1], bob_image.shape[2], 3))
+        skimage = numpy.zeros(shape=(bob_image.shape[1], bob_image.shape[2], bob_image.shape[0]))
 
-        skimage[:, :, 0] = bob_image[0, :, :]  # Copying red
-        skimage[:, :, 1] = bob_image[1, :, :]  # Copying green
-        skimage[:, :, 2] = bob_image[2, :, :]  # Copying blue
+        for i in range(bob_image.shape[0]):
+            skimage[:, :, i] = bob_image[i, :, :]  # Copying red
+        #skimage[:, :, 1] = bob_image[1, :, :]  # Copying green
+        #skimage[:, :, 2] = bob_image[2, :, :]  # Copying blue
 
         return skimage
 
+    def skimage2bob(self, sk_image):
+        """
+        Convert bob color image to the skcit image
+        """
+
+        bob_image = numpy.zeros(shape=(sk_image.shape[2], sk_image.shape[0], sk_image.shape[1]))
+
+        for i in range(bob_image.shape[0]):
+            bob_image[i, :, :] = sk_image[:, :, i]  # Copying red
+
+        #bob_image[0, :, :] = sk_image[:, :, 0] # Copying red
+        #if bob_image.shape[0] > 1:
+        #    bob_image[1, :, :] = sk_image[:, :, 1]# Copying green
+        #    bob_image[2, :, :] = sk_image[:, :, 2]  # Copying blue
+
+        return bob_image
+
+
+
     def rescale(self, data):
         """
         Reescale a single sample with input_shape

bob/learn/tensorflow/datashuffler/DataAugmentation.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
+# @date: Sun 16 Oct 2016 14:32:36 CEST
+
+import numpy
+
+
+class DataAugmentation(object):
+    """
+    Base class for applying common real-time data augmentation.
+
+    This class is meant to be used as an argument of `input_data`. When training
+    a model, the defined augmentation methods will be applied at training
+    time only.
+    """
+
+    def __init__(self, seed=10):
+        self.filter_bank = []
+        numpy.random.seed(seed)
+
+    def __call__(self, image):
+        """
+        Apply a random filter to and image
+        """
+
+        if len(self.filter_bank) <= 0:
+            raise ValueError("There is not filters in the filter bank")
+
+        filter = self.filter_bank[numpy.random.randint(len(self.filter_bank))]
+        return filter(image)
+
+

bob/learn/tensorflow/datashuffler/Disk.py

@@ -19,7 +19,8 @@ class Disk(Base):
                  input_dtype="float64",
                  scale=True,
                  batch_size=1,
-                 seed=10):
+                 seed=10,
+                 data_augmentation=None):
         """
          This datashuffler deal with databases that are stored in the disk.
          The data is loaded on the fly,.
@@ -46,7 +47,8 @@ class Disk(Base):
             input_dtype=input_dtype,
             scale=scale,
             batch_size=batch_size,
-            seed=seed
+            seed=seed,
+            data_augmentation=data_augmentation
         )
         # Seting the seed
         numpy.random.seed(seed)
@@ -56,6 +58,11 @@ class Disk(Base):
 
     def load_from_file(self, file_name):
         d = bob.io.base.load(file_name)
+
+        # Applying the data augmentation
+        if self.data_augmentation is not None:
+            d = self.data_augmentation(d)
+
         if d.shape[0] != 3 and self.input_shape[2] != 3: # GRAY SCALE IMAGE
             data = numpy.zeros(shape=(d.shape[0], d.shape[1], 1))
             data[:, :, 0] = d
@@ -70,7 +77,7 @@ class Disk(Base):
 
         return data
 
-    def get_batch(self):
+    def get_batch(self, noise=False):
 
         # Shuffling samples
         indexes = numpy.array(range(self.data.shape[0]))
@@ -83,6 +90,8 @@ class Disk(Base):
             data = self.load_from_file(file_name)
 
             selected_data[i, ...] = data
+
+            # Scaling
             if self.scale:
                 selected_data[i, ...] *= self.scale_value
 

bob/learn/tensorflow/datashuffler/ImageAugmentation.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
+# @date: Wed 11 May 2016 09:39:36 CEST 
+
+import bob.ip.base
+import numpy
+from .DataAugmentation import DataAugmentation
+
+
+class ImageAugmentation(DataAugmentation):
+    """
+    Class for applying common real-time random data augmentation for images.
+    """
+
+    def __init__(self, seed=10):
+
+        super(ImageAugmentation, self).__init__(seed=seed)
+
+        self.filter_bank = [self.__add_none,
+                            self.__add_none,
+                            self.__add_gaussian_blur,
+                            self.__add_left_right_flip,
+                            self.__add_none,
+                            self.__add_salt_and_pepper]
+    #self.__add_rotation,
+
+    def __add_none(self, image):
+        return image
+
+    def __add_gaussian_blur(self, image):
+        possible_sigmas = numpy.arange(0.1, 3., 0.1)
+        possible_radii = [1, 2, 3]
+
+        sigma = possible_sigmas[numpy.random.randint(len(possible_sigmas))]
+        radius = possible_radii[numpy.random.randint(len(possible_radii))]
+
+        gaussian_filter = bob.ip.base.Gaussian(sigma=(sigma, sigma),
+                                               radius=(radius, radius))
+
+        return gaussian_filter(image)
+
+    def __add_left_right_flip(self, image):
+        return bob.ip.base.flop(image)
+
+    def __add_rotation(self, image):
+        possible_angles = numpy.arange(-15, 15, 0.5)
+        angle = possible_angles[numpy.random.randint(len(possible_angles))]
+
+        return bob.ip.base.rotate(image, angle)
+
+    def __add_salt_and_pepper(self, image):
+        possible_levels = numpy.arange(0.01, 0.1, 0.01)
+        level = possible_levels[numpy.random.randint(len(possible_levels))]
+
+        return self.compute_salt_and_peper(image, level)
+
+    def compute_salt_and_peper(self, image, level):
+        """
+        Compute a salt and pepper noise
+        """
+        r = numpy.random.rand(*image.shape)
+
+        # 0 noise
+        indexes_0 = r <= (level/0.5)
+        image[indexes_0] = 0.0
+
+        # 255 noise
+        indexes_255 = (1 - level / 2) <= r;
+        image[indexes_255] = 255.0
+
+        return image

bob/learn/tensorflow/datashuffler/Memory.py

@@ -15,7 +15,8 @@ class Memory(Base):
                  input_dtype="float64",
                  scale=True,
                  batch_size=1,
-                 seed=10):
+                 seed=10,
+                 data_augmentation=None):
         """
          This datashuffler deal with databases that are stored in a :py:class`numpy.array`
 
@@ -35,14 +36,12 @@ class Memory(Base):
             input_dtype=input_dtype,
             scale=scale,
             batch_size=batch_size,
-            seed=seed
+            seed=seed,
+            data_augmentation=data_augmentation
         )
         # Seting the seed
         numpy.random.seed(seed)
-
         self.data = self.data.astype(input_dtype)
-        if self.scale:
-            self.data *= self.scale_value
 
     def get_batch(self):
 
@@ -53,4 +52,14 @@ class Memory(Base):
         selected_data = self.data[indexes[0:self.batch_size], :, :, :]
         selected_labels = self.labels[indexes[0:self.batch_size]]
 
+        # Applying the data augmentation
+        if self.data_augmentation is not None:
+            for i in range(selected_data.shape[0]):
+                img = self.skimage2bob(selected_data[i, ...])
+                img = self.data_augmentation(img)
+                selected_data[i, ...] = self.bob2skimage(img)
+
+        if self.scale:
+            selected_data *= self.scale_value
+
         return [selected_data.astype("float32"), selected_labels.astype("int64")]

bob/learn/tensorflow/datashuffler/SiameseDisk.py

@@ -17,7 +17,8 @@ class SiameseDisk(Siamese, Disk):
                  input_dtype="float64",
                  scale=True,
                  batch_size=1,
-                 seed=10):
+                 seed=10,
+                 data_augmentation=None):
         """
          Shuffler that deal with file list
 
@@ -43,7 +44,8 @@ class SiameseDisk(Siamese, Disk):
             input_dtype=input_dtype,
             scale=scale,
             batch_size=batch_size,
-            seed=seed
+            seed=seed,
+            data_augmentation=data_augmentation
         )
         # Seting the seed
         numpy.random.seed(seed)
@@ -74,6 +76,7 @@ class SiameseDisk(Siamese, Disk):
             labels_siamese[i] = not genuine
             genuine = not genuine
 
+
         if self.scale:
             data *= self.scale_value
             data_p *= self.scale_value

bob/learn/tensorflow/datashuffler/SiameseMemory.py

@@ -14,10 +14,11 @@ class SiameseMemory(Siamese, Memory):
 
     def __init__(self, data, labels,
                  input_shape,
-                 input_dtype="float64",
+                 input_dtype="float",
                  scale=True,
                  batch_size=1,
-                 seed=10):
+                 seed=10,
+                 data_augmentation=None):
         """
          Shuffler that deal with memory datasets
 
@@ -37,14 +38,13 @@ class SiameseMemory(Siamese, Memory):
             input_dtype=input_dtype,
             scale=scale,
             batch_size=batch_size,
-            seed=seed
+            seed=seed,
+            data_augmentation=data_augmentation
         )
         # Seting the seed
         numpy.random.seed(seed)
 
         self.data = self.data.astype(input_dtype)
-        if self.scale:
-            self.data *= self.scale_value
 
     def get_batch(self, zero_one_labels=True):
         """
@@ -55,9 +55,9 @@ class SiameseMemory(Siamese, Memory):
 
         **Return**
         """
-        data = numpy.zeros(shape=self.shape, dtype='float32')
-        data_p = numpy.zeros(shape=self.shape, dtype='float32')
-        labels_siamese = numpy.zeros(shape=self.shape[0], dtype='float32')
+        data = numpy.zeros(shape=self.shape, dtype='float')
+        data_p = numpy.zeros(shape=self.shape, dtype='float')
+        labels_siamese = numpy.zeros(shape=self.shape[0], dtype='float')
 
         genuine = True
         for i in range(self.shape[0]):
@@ -68,4 +68,18 @@ class SiameseMemory(Siamese, Memory):
                 labels_siamese[i] = -1 if genuine else +1
             genuine = not genuine
 
-        return [data, data_p, labels_siamese]
+        # Applying the data augmentation
+        if self.data_augmentation is not None:
+            for i in range(data.shape[0]):
+                d = self.bob2skimage(self.data_augmentation(self.skimage2bob(data[i, ...])))
+                data[i, ...] = d
+
+                d = self.bob2skimage(self.data_augmentation(self.skimage2bob(data_p[i, ...])))
+                data_p[i, ...] = d
+
+        # Scaling
+        if self.scale:
+            data *= self.scale_value
+            data_p *= self.scale_value
+
+        return [data.astype("float32"), data_p.astype("float32"), labels_siamese]

bob/learn/tensorflow/datashuffler/TripletDisk.py

@@ -22,7 +22,8 @@ class TripletDisk(Triplet, Disk):
                  input_dtype="float64",
                  scale=True,
                  batch_size=1,
-                 seed=10):
+                 seed=10,
+                 data_augmentation=None):
         """
          Shuffler that deal with file list
 
@@ -47,7 +48,8 @@ class TripletDisk(Triplet, Disk):
             input_shape=input_shape,
             input_dtype=input_dtype,
             scale=scale,
-            batch_size=batch_size
+            batch_size=batch_size,
+            data_augmentation=data_augmentation
         )
         # Seting the seed
         numpy.random.seed(seed)

bob/learn/tensorflow/datashuffler/TripletMemory.py

@@ -17,7 +17,8 @@ class TripletMemory(Triplet, Memory):
                  input_dtype="float64",
                  scale=True,
                  batch_size=1,
-                 seed=10):
+                 seed=10,
+                 data_augmentation=None):
         """
          Shuffler that deal with memory datasets
 
@@ -37,14 +38,13 @@ class TripletMemory(Triplet, Memory):
             input_dtype=input_dtype,
             scale=scale,
             batch_size=batch_size,
-            seed=seed
+            seed=seed,
+            data_augmentation=data_augmentation
         )
         # Seting the seed
         numpy.random.seed(seed)
 
         self.data = self.data.astype(input_dtype)
-        if self.scale:
-            self.data *= self.scale_value
 
     def get_batch(self):
         """
@@ -63,4 +63,22 @@ class TripletMemory(Triplet, Memory):
         for i in range(self.shape[0]):
             data_a[i, ...], data_p[i, ...], data_n[i, ...] = self.get_one_triplet(self.data, self.labels)
 
-        return data_a, data_p, data_n
+        # Applying the data augmentation
+        if self.data_augmentation is not None:
+            for i in range(data_a.shape[0]):
+                d = self.bob2skimage(self.data_augmentation(self.skimage2bob(data_a[i, ...])))
+                data_a[i, ...] = d
+
+                d = self.bob2skimage(self.data_augmentation(self.skimage2bob(data_p[i, ...])))
+                data_p[i, ...] = d
+
+                d = self.bob2skimage(self.data_augmentation(self.skimage2bob(data_n[i, ...])))
+                data_n[i, ...] = d
+
+        # Scaling
+        if self.scale:
+            data_a *= self.scale_value
+            data_p *= self.scale_value
+            data_n *= self.scale_value
+
+        return [data_a.astype("float32"), data_p.astype("float32"), data_n.astype("float32")]

bob/learn/tensorflow/datashuffler/__init__.py

@@ -13,5 +13,10 @@ from .TripletMemory import TripletMemory
 from .SiameseDisk import SiameseDisk
 from .TripletDisk import TripletDisk
 
+# Data Augmentation
+from .DataAugmentation import DataAugmentation
+from .ImageAugmentation import ImageAugmentation
+
+
 # gets sphinx autodoc done right - don't remove it
 __all__ = [_ for _ in dir() if not _.startswith('_')]

bob/learn/tensorflow/test/test_cnn.py

@@ -4,8 +4,8 @@
 # @date: Thu 13 Oct 2016 13:35 CEST
 
 import numpy
-from bob.learn.tensorflow.datashuffler import Memory, SiameseMemory, TripletMemory, Disk, SiameseDisk, TripletDisk
-from bob.learn.tensorflow.network import Chopra, MLP
+from bob.learn.tensorflow.datashuffler import Memory, SiameseMemory, TripletMemory, Disk, SiameseDisk, TripletDisk, ImageAugmentation
+from bob.learn.tensorflow.network import Chopra, Lenet
 from bob.learn.tensorflow.loss import BaseLoss, ContrastiveLoss, TripletLoss
 from bob.learn.tensorflow.trainers import Trainer, SiameseTrainer, TripletTrainer
 # from ..analyzers import ExperimentAnalizer, SoftmaxAnalizer
@@ -79,9 +79,11 @@ def test_cnn_trainer():
     validation_data = numpy.reshape(validation_data, (validation_data.shape[0], 28, 28, 1))
 
     # Creating datashufflers
+    data_augmentation = ImageAugmentation()
     train_data_shuffler = Memory(train_data, train_labels,
                                  input_shape=[28, 28, 1],
-                                 batch_size=batch_size)
+                                 batch_size=batch_size,
+                                 data_augmentation=data_augmentation)
 
     with tf.Session() as session:
         directory = "./temp/cnn"
@@ -117,6 +119,7 @@ def test_cnn_trainer():
         # At least 80% of accuracy
         assert accuracy > 80.
         shutil.rmtree(directory)
+        session.close()
 
 
 def test_siamesecnn_trainer():
@@ -162,6 +165,7 @@ def test_siamesecnn_trainer():
         # At least 80% of accuracy
         assert eer < 0.25
         shutil.rmtree(directory)
+        session.close()
 
 
 def test_tripletcnn_trainer():
@@ -207,3 +211,4 @@ def test_tripletcnn_trainer():
         # At least 80% of accuracy
         assert eer < 0.25
         shutil.rmtree(directory)
+        session.close()

bob/learn/tensorflow/test/test_datashuffler_augmentation.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
+# @date: Thu 13 Oct 2016 13:35 CEST
+
+import numpy
+from bob.learn.tensorflow.datashuffler import Memory, SiameseMemory, TripletMemory, Disk, SiameseDisk, TripletDisk, ImageAugmentation
+import pkg_resources
+from ..util import load_mnist
+import os
+
+"""
+Some unit tests for the datashuffler
+"""
+
+
+def get_dummy_files():
+
+    base_path = pkg_resources.resource_filename(__name__, 'data/dummy_database')
+    files = []
+    clients = []
+    for f in os.listdir(base_path):
+        if f.endswith(".hdf5"):
+            files.append(os.path.join(base_path, f))
+            clients.append(int(f[1:4]))
+
+    return files, clients
+
+
+def test_memory_shuffler():
+
+    train_data, train_labels, validation_data, validation_labels = load_mnist()
+    train_data = numpy.reshape(train_data, (train_data.shape[0], 28, 28, 1))
+
+    batch_shape = [16, 28, 28, 1]
+
+    data_augmentation = ImageAugmentation()
+    data_shuffler = Memory(train_data, train_labels,
+                           input_shape=batch_shape[1:],
+                           scale=True,
+                           batch_size=batch_shape[0],
+                           data_augmentation=data_augmentation)
+
+    batch = data_shuffler.get_batch()
+    assert len(batch) == 2
+    assert batch[0].shape == tuple(batch_shape)
+    assert batch[1].shape[0] == batch_shape[0]
+
+
+def test_siamesememory_shuffler():
+
+    train_data, train_labels, validation_data, validation_labels = load_mnist()
+    train_data = numpy.reshape(train_data, (train_data.shape[0], 28, 28, 1))
+
+    batch_shape = [16, 28, 28, 1]
+    data_augmentation = ImageAugmentation()
+    data_shuffler = SiameseMemory(train_data, train_labels,
+                                  input_shape=batch_shape[1:],
+                                  scale=True,
+                                  batch_size=batch_shape[0],
+                                  data_augmentation=data_augmentation)
+
+    batch = data_shuffler.get_batch()
+
+    assert len(batch) == 3
+    assert batch[0].shape == tuple(batch_shape)
+    assert batch[1].shape == tuple(batch_shape)
+    assert batch[2].shape[0] == batch_shape[0]
+
+    placeholders = data_shuffler.get_placeholders(name="train")
+    assert placeholders[0].get_shape().as_list() == batch_shape
+    assert placeholders[1].get_shape().as_list() == batch_shape
+    assert placeholders[2].get_shape().as_list()[0] == batch_shape[0]
+
+
+def test_tripletmemory_shuffler():
+
+    train_data, train_labels, validation_data, validation_labels = load_mnist()
+    train_data = numpy.reshape(train_data, (train_data.shape[0], 28, 28, 1))
+
+    batch_shape = [16, 28, 28, 1]
+    data_augmentation = ImageAugmentation()
+    data_shuffler = TripletMemory(train_data, train_labels,
+                                  input_shape=batch_shape[1:],
+                                  scale=True,
+                                  batch_size=batch_shape[0],
+                                  data_augmentation=data_augmentation)
+
+    batch = data_shuffler.get_batch()
+
+    assert len(batch) == 3
+    assert batch[0].shape == tuple(batch_shape)
+    assert batch[1].shape == tuple(batch_shape)
+    assert batch[2].shape == tuple(batch_shape)
+
+    placeholders = data_shuffler.get_placeholders(name="train")
+    assert placeholders[0].get_shape().as_list() == batch_shape
+    assert placeholders[1].get_shape().as_list() == batch_shape
+    assert placeholders[2].get_shape().as_list() == batch_shape
+
+
+def test_disk_shuffler():
+
+    train_data, train_labels = get_dummy_files()
+
+    batch_shape = [2, 125, 125, 3]
+    data_augmentation = ImageAugmentation()
+    data_shuffler = Disk(train_data, train_labels,
+                         input_shape=batch_shape[1:],
+                         scale=True,
+                         batch_size=batch_shape[0],
+                         data_augmentation=data_augmentation)
+
+    batch = data_shuffler.get_batch()
+
+    assert len(batch) == 2
+    assert batch[0].shape == tuple(batch_shape)
+    assert batch[1].shape[0] == batch_shape[0]
+
+    placeholders = data_shuffler.get_placeholders(name="train")
+    assert placeholders[0].get_shape().as_list() == batch_shape
+    assert placeholders[1].get_shape().as_list()[0] == batch_shape[0]
+
+
+def test_siamesedisk_shuffler():
+
+    train_data, train_labels = get_dummy_files()
+
+    batch_shape = [2, 125, 125, 3]
+    data_augmentation = ImageAugmentation()
+    data_shuffler = SiameseDisk(train_data, train_labels,
+                                input_shape=batch_shape[1:],
+                                scale=True,
+                                batch_size=batch_shape[0],
+                                data_augmentation=data_augmentation)
+
+    batch = data_shuffler.get_batch()
+
+    assert len(batch) == 3
+    assert batch[0].shape == tuple(batch_shape)
+    assert batch[1].shape == tuple(batch_shape)
+    assert batch[2].shape[0] == batch_shape[0]
+
+    placeholders = data_shuffler.get_placeholders(name="train")
+    assert placeholders[0].get_shape().as_list() == batch_shape
+    assert placeholders[1].get_shape().as_list() == batch_shape
+    assert placeholders[2].get_shape().as_list()[0] == batch_shape[0]
+
+
+def test_tripletdisk_shuffler():
+
+    train_data, train_labels = get_dummy_files()
+
+    batch_shape = [1, 125, 125, 3]
+    data_augmentation = ImageAugmentation()
+    data_shuffler = TripletDisk(train_data, train_labels,
+                                input_shape=batch_shape[1:],
+                                scale=True,
+                                batch_size=batch_shape[0],
+                                data_augmentation=data_augmentation)
+
+    batch = data_shuffler.get_batch()
+
+    assert len(batch) == 3
+    assert batch[0].shape == tuple(batch_shape)
+    assert batch[1].shape == tuple(batch_shape)
+    assert batch[2].shape == tuple(batch_shape)
+
+    placeholders = data_shuffler.get_placeholders(name="train")
+    assert placeholders[0].get_shape().as_list() == batch_shape
+    assert placeholders[1].get_shape().as_list() == batch_shape
+    assert placeholders[2].get_shape().as_list() == batch_shape

bob/learn/tensorflow/test/test_dnn.py

@@ -71,3 +71,4 @@ def test_dnn_trainer():
         # At least 50% of accuracy for the DNN
         assert accuracy > 50.
         shutil.rmtree(directory)
+        session.close()