Updated triplet trainer

parent bcc04c96
......@@ -149,5 +149,16 @@ class Base(object):
shape = tuple([1] + list(data.shape))
return numpy.reshape(data, shape)
def normalize_sample(self, x):
"""
Normalize the sample.
For the time being I'm only scaling from 0-1
"""
if self.scale:
return x * self.scale_value
return x
......@@ -59,7 +59,6 @@ class Memory(Base):
img = self.data_augmentation(img)
selected_data[i, ...] = self.bob2skimage(img)
if self.scale:
selected_data *= self.scale_value
selected_data = self.normalize_sample(selected_data)
return [selected_data.astype("float32"), selected_labels.astype("int64")]
......@@ -44,9 +44,11 @@ class OnLineSampling(object):
if self.feature_placeholder is None:
shape = tuple([None] + list(data.shape[1:]))
self.feature_placeholder = tf.placeholder(tf.float32, shape=shape, name="feature")
self.graph = self.feature_extractor.compute_graph(self.feature_placeholder, self.feature_extractor.default_feature_layer,
embbeding = self.feature_extractor.compute_graph(self.feature_placeholder, self.feature_extractor.default_feature_layer,
training=False)
self.graph = tf.nn.l2_normalize(embbeding, 1, 1e-10, name='embeddings')
#if self.feature_placeholder.get_shape().as_list() != list(data.shape):
#tf.reshape(self.feature_placeholder, tf.pack([data.shape]))
#self.feature_placeholder.set_shape(data.shape)
......
......@@ -63,8 +63,8 @@ class Siamese(Base):
numpy.random.shuffle(indexes)
# Picking a pair
data = input_data[indexes[0], ...]
data_p = input_data[indexes[1], ...]
sample_l = input_data[indexes[0], ...]
sample_r = input_data[indexes[1], ...]
else:
# Picking a pair of labels from different clients
......@@ -73,13 +73,13 @@ class Siamese(Base):
index[1] = self.possible_labels[int(index[1])]
# Getting the indexes of the two clients
indexes = numpy.where(input_labels == index[0])[0]
indexes_p = numpy.where(input_labels == index[1])[0]
numpy.random.shuffle(indexes)
numpy.random.shuffle(indexes_p)
indexes_l = numpy.where(input_labels == index[0])[0]
indexes_r = numpy.where(input_labels == index[1])[0]
numpy.random.shuffle(indexes_l)
numpy.random.shuffle(indexes_r)
# Picking a pair
data = input_data[indexes[0], ...]
data_p = input_data[indexes_p[0], ...]
sample_l = input_data[indexes_l[0], ...]
sample_r = input_data[indexes_r[0], ...]
return data, data_p
return sample_l, sample_r
......@@ -63,22 +63,20 @@ class SiameseDisk(Siamese, Disk):
**Return**
"""
data = numpy.zeros(shape=self.shape, dtype='float32')
data_p = numpy.zeros(shape=self.shape, dtype='float32')
sample_l = numpy.zeros(shape=self.shape, dtype='float32')
sample_r = numpy.zeros(shape=self.shape, dtype='float32')
labels_siamese = numpy.zeros(shape=self.shape[0], dtype='float32')
genuine = True
for i in range(self.shape[0]):
file_name, file_name_p = self.get_genuine_or_not(self.data, self.labels, genuine=genuine)
data[i, ...] = self.load_from_file(str(file_name))
data_p[i, ...] = self.load_from_file(str(file_name_p))
sample_l[i, ...] = self.load_from_file(str(file_name))
sample_r[i, ...] = self.load_from_file(str(file_name_p))
labels_siamese[i] = not genuine
genuine = not genuine
sample_l = self.normalize_sample(sample_l)
sample_r = self.normalize_sample(sample_r)
if self.scale:
data *= self.scale_value
data_p *= self.scale_value
return data, data_p, labels_siamese
return sample_l, sample_r, labels_siamese
......@@ -55,13 +55,13 @@ class SiameseMemory(Siamese, Memory):
**Return**
"""
data = numpy.zeros(shape=self.shape, dtype='float')
data_p = numpy.zeros(shape=self.shape, dtype='float')
sample_l = numpy.zeros(shape=self.shape, dtype='float')
sample_r = numpy.zeros(shape=self.shape, dtype='float')
labels_siamese = numpy.zeros(shape=self.shape[0], dtype='float')
genuine = True
for i in range(self.shape[0]):
data[i, ...], data_p[i, ...] = self.get_genuine_or_not(self.data, self.labels, genuine=genuine)
sample_l[i, ...], sample_r[i, ...] = self.get_genuine_or_not(self.data, self.labels, genuine=genuine)
if zero_one_labels:
labels_siamese[i] = not genuine
else:
......@@ -70,16 +70,14 @@ class SiameseMemory(Siamese, Memory):
# Applying the data augmentation
if self.data_augmentation is not None:
for i in range(data.shape[0]):
d = self.bob2skimage(self.data_augmentation(self.skimage2bob(data[i, ...])))
data[i, ...] = d
for i in range(sample_l.shape[0]):
d = self.bob2skimage(self.data_augmentation(self.skimage2bob(sample_l[i, ...])))
sample_l[i, ...] = d
d = self.bob2skimage(self.data_augmentation(self.skimage2bob(data_p[i, ...])))
data_p[i, ...] = d
d = self.bob2skimage(self.data_augmentation(self.skimage2bob(sample_r[i, ...])))
sample_r[i, ...] = d
# Scaling
if self.scale:
data *= self.scale_value
data_p *= self.scale_value
sample_l = self.normalize_sample(sample_l)
sample_r = self.normalize_sample(sample_r)
return [data.astype("float32"), data_p.astype("float32"), labels_siamese]
return [sample_l.astype("float32"), sample_r.astype("float32"), labels_siamese]
......@@ -67,19 +67,18 @@ class TripletDisk(Triplet, Disk):
**Return**
"""
data_a = numpy.zeros(shape=self.shape, dtype='float32')
data_p = numpy.zeros(shape=self.shape, dtype='float32')
data_n = numpy.zeros(shape=self.shape, dtype='float32')
sample_a = numpy.zeros(shape=self.shape, dtype='float32')
sample_p = numpy.zeros(shape=self.shape, dtype='float32')
sample_n = numpy.zeros(shape=self.shape, dtype='float32')
for i in range(self.shape[0]):
file_name_a, file_name_p, file_name_n = self.get_one_triplet(self.data, self.labels)
data_a[i, ...] = self.load_from_file(str(file_name_a))
data_p[i, ...] = self.load_from_file(str(file_name_p))
data_n[i, ...] = self.load_from_file(str(file_name_n))
sample_a[i, ...] = self.load_from_file(str(file_name_a))
sample_p[i, ...] = self.load_from_file(str(file_name_p))
sample_n[i, ...] = self.load_from_file(str(file_name_n))
if self.scale:
data_a *= self.scale_value
data_p *= self.scale_value
data_n *= self.scale_value
sample_a = self.normalize_sample(sample_a)
sample_p = self.normalize_sample(sample_p)
sample_n = self.normalize_sample(sample_n)
return [data_a, data_p, data_n]
return [sample_a, sample_p, sample_n]
......@@ -56,29 +56,28 @@ class TripletMemory(Triplet, Memory):
**Return**
"""
data_a = numpy.zeros(shape=self.shape, dtype='float32')
data_p = numpy.zeros(shape=self.shape, dtype='float32')
data_n = numpy.zeros(shape=self.shape, dtype='float32')
sample_a = numpy.zeros(shape=self.shape, dtype='float32')
sample_p = numpy.zeros(shape=self.shape, dtype='float32')
sample_n = numpy.zeros(shape=self.shape, dtype='float32')
for i in range(self.shape[0]):
data_a[i, ...], data_p[i, ...], data_n[i, ...] = self.get_one_triplet(self.data, self.labels)
sample_a[i, ...], sample_p[i, ...], sample_n[i, ...] = self.get_one_triplet(self.data, self.labels)
# Applying the data augmentation
if self.data_augmentation is not None:
for i in range(data_a.shape[0]):
d = self.bob2skimage(self.data_augmentation(self.skimage2bob(data_a[i, ...])))
data_a[i, ...] = d
for i in range(sample_a.shape[0]):
d = self.bob2skimage(self.data_augmentation(self.skimage2bob(sample_a[i, ...])))
sample_a[i, ...] = d
d = self.bob2skimage(self.data_augmentation(self.skimage2bob(data_p[i, ...])))
data_p[i, ...] = d
d = self.bob2skimage(self.data_augmentation(self.skimage2bob(sample_p[i, ...])))
sample_p[i, ...] = d
d = self.bob2skimage(self.data_augmentation(self.skimage2bob(data_n[i, ...])))
data_n[i, ...] = d
d = self.bob2skimage(self.data_augmentation(self.skimage2bob(sample_n[i, ...])))
sample_n[i, ...] = d
# Scaling
if self.scale:
data_a *= self.scale_value
data_p *= self.scale_value
data_n *= self.scale_value
sample_a = self.normalize_sample(sample_a)
sample_p = self.normalize_sample(sample_p)
sample_n = self.normalize_sample(sample_n)
return [data_a.astype("float32"), data_p.astype("float32"), data_n.astype("float32")]
return [sample_a.astype("float32"), sample_p.astype("float32"), sample_n.astype("float32")]
......@@ -92,10 +92,9 @@ class TripletWithFastSelectionDisk(Triplet, Disk, OnLineSampling):
sample_p[i, ...] = self.load_from_file(str(file_name_p))
sample_n[i, ...] = self.load_from_file(str(file_name_n))
if self.scale:
sample_a *= self.scale_value
sample_p *= self.scale_value
sample_n *= self.scale_value
sample_a = self.normalize_sample(sample_a)
sample_p = self.normalize_sample(sample_p)
sample_n = self.normalize_sample(sample_n)
return [sample_a, sample_p, sample_n]
......@@ -133,13 +132,6 @@ class TripletWithFastSelectionDisk(Triplet, Disk, OnLineSampling):
samples_p, embedding_p, d_anchor_positive = self.get_positives(anchor_labels, embedding_a)
samples_n = self.get_negative(anchor_labels, embedding_a, d_anchor_positive)
#import bob.io.base
#import bob.io.image
#for i in range(self.shape[0]):
#bob.io.base.save((self.skimage2bob(samples_a[i, ...]) / 0.00390625).astype("uint8"), "{0}_a.jpg".format(i))
#bob.io.base.save((self.skimage2bob(samples_p[i, ...]) / 0.00390625).astype("uint8"), "{0}_p.jpg".format(i))
#bob.io.base.save((self.skimage2bob(samples_n[i, ...]) / 0.00390625).astype("uint8"), "{0}_n.jpg".format(i))
return samples_a, samples_p, samples_n
def get_anchor(self, label):
......@@ -154,8 +146,7 @@ class TripletWithFastSelectionDisk(Triplet, Disk, OnLineSampling):
file_name = self.data[indexes[0], ...]
anchor = self.load_from_file(str(file_name))
if self.scale:
anchor *= self.scale_value
anchor = self.normalize_sample(anchor)
return anchor
......@@ -171,9 +162,7 @@ class TripletWithFastSelectionDisk(Triplet, Disk, OnLineSampling):
file_name = self.data[indexes[0], ...]
samples_p[i, ...] = self.load_from_file(str(file_name))
if self.scale:
samples_p *= self.scale_value
samples_p = self.normalize_sample(samples_p)
embedding_p = self.project(samples_p)
# Computing the distances
......@@ -204,9 +193,7 @@ class TripletWithFastSelectionDisk(Triplet, Disk, OnLineSampling):
for i in range(shape[0]):
file_name = self.data[indexes[i], ...]
temp_samples_n[i, ...] = self.load_from_file(str(file_name))
if self.scale:
temp_samples_n *= self.scale_value
temp_samples_n = self.normalize_sample(temp_samples_n)
# Computing all the embeddings
embedding_temp_n = self.project(temp_samples_n)
......
......@@ -80,22 +80,21 @@ class TripletWithSelectionDisk(Triplet, Disk, OnLineSampling):
**Return**
"""
data_a = numpy.zeros(shape=self.shape, dtype='float32')
data_p = numpy.zeros(shape=self.shape, dtype='float32')
data_n = numpy.zeros(shape=self.shape, dtype='float32')
sample_a = numpy.zeros(shape=self.shape, dtype='float32')
sample_p = numpy.zeros(shape=self.shape, dtype='float32')
sample_n = numpy.zeros(shape=self.shape, dtype='float32')
for i in range(self.shape[0]):
file_name_a, file_name_p, file_name_n = self.get_one_triplet(self.data, self.labels)
data_a[i, ...] = self.load_from_file(str(file_name_a))
data_p[i, ...] = self.load_from_file(str(file_name_p))
data_n[i, ...] = self.load_from_file(str(file_name_n))
sample_a[i, ...] = self.load_from_file(str(file_name_a))
sample_p[i, ...] = self.load_from_file(str(file_name_p))
sample_n[i, ...] = self.load_from_file(str(file_name_n))
if self.scale:
data_a *= self.scale_value
data_p *= self.scale_value
data_n *= self.scale_value
sample_a = self.normalize_sample(sample_a)
sample_p = self.normalize_sample(sample_p)
sample_n = self.normalize_sample(sample_n)
return [data_a, data_p, data_n]
return [sample_a, sample_p, sample_n]
def get_batch(self):
"""
......@@ -120,9 +119,6 @@ class TripletWithSelectionDisk(Triplet, Disk, OnLineSampling):
anchor_labels = numpy.hstack((anchor_labels,numpy.ones(samples_per_identity) * self.possible_labels[indexes[i]]))
anchor_labels = anchor_labels[0:self.batch_size]
data_a = numpy.zeros(shape=self.shape, dtype='float32')
data_p = numpy.zeros(shape=self.shape, dtype='float32')
data_n = numpy.zeros(shape=self.shape, dtype='float32')
......@@ -167,14 +163,12 @@ class TripletWithSelectionDisk(Triplet, Disk, OnLineSampling):
numpy.random.shuffle(indexes)
file_name = self.data[indexes[0], ...]
anchor = self.load_from_file(str(file_name))
if self.scale:
anchor *= self.scale_value
sample_a = self.load_from_file(str(file_name))
return anchor
sample_a = self.normalize_sample(sample_a)
return sample_a
def get_positive(self, label, anchor_feature):
def get_positive(self, label, embedding_a):
"""
Get the best positive sample given the anchor.
The best positive sample for the anchor is the farthest from the anchor
......@@ -188,31 +182,30 @@ class TripletWithSelectionDisk(Triplet, Disk, OnLineSampling):
0:self.batch_size] # Limiting to the batch size, otherwise the number of comparisons will explode
distances = []
shape = tuple([len(indexes)] + list(self.shape[1:]))
data_p = numpy.zeros(shape=shape, dtype='float32')
sample_p = numpy.zeros(shape=shape, dtype='float32')
#logger.info("****************** search")
for i in range(shape[0]):
#logger.info("****************** fetch")
file_name = self.data[indexes[i], ...]
#logger.info("****************** load")
data_p[i, ...] = self.load_from_file(str(file_name))
sample_p[i, ...] = self.load_from_file(str(file_name))
if self.scale:
data_p *= self.scale_value
sample_p = self.normalize_sample(sample_p)
#logger.info("****************** project")
positive_features = self.project(data_p)
embedding_p = self.project(sample_p)
#logger.info("****************** distances")
# Projecting the positive instances
for p in positive_features:
distances.append(euclidean(anchor_feature, p))
for p in embedding_p:
distances.append(euclidean(embedding_a, p))
# Geting the max
index = numpy.argmax(distances)
#logger.info("****************** return")
return data_p[index, ...], distances[index]
return sample_p[index, ...], distances[index]
def get_negative(self, label, anchor_feature, distance_anchor_positive):
def get_negative(self, label, embedding_a, distance_anchor_positive):
"""
Get the best negative sample for a pair anchor-positive
"""
......@@ -227,24 +220,23 @@ class TripletWithSelectionDisk(Triplet, Disk, OnLineSampling):
0:self.batch_size*3] # Limiting to the batch size, otherwise the number of comparisons will explode
shape = tuple([len(indexes)] + list(self.shape[1:]))
data_n = numpy.zeros(shape=shape, dtype='float32')
sample_n = numpy.zeros(shape=shape, dtype='float32')
#logger.info("****************** search")
for i in range(shape[0]):
#logger.info("****************** fetch")
file_name = self.data[indexes[i], ...]
#logger.info("****************** load")
data_n[i, ...] = self.load_from_file(str(file_name))
sample_n[i, ...] = self.load_from_file(str(file_name))
if self.scale:
data_n *= self.scale_value
sample_n = self.normalize_sample(sample_n)
#logger.info("****************** project")
negative_features = self.project(data_n)
embedding_n = self.project(sample_n)
distances = []
#logger.info("****************** distances")
for n in negative_features:
d = euclidean(anchor_feature, n)
for n in embedding_n:
d = euclidean(embedding_a, n)
# Semi-hard samples criteria
if d > distance_anchor_positive:
......@@ -260,4 +252,4 @@ class TripletWithSelectionDisk(Triplet, Disk, OnLineSampling):
logger.info("SEMI-HARD negative not found, took the first one")
index = 0
#logger.info("****************** return")
return data_n[index, ...]
return sample_n[index, ...]
......@@ -33,12 +33,18 @@ class TripletLoss(BaseLoss):
def __init__(self, margin=5.0):
self.margin = margin
def __call__(self, anchor_feature, positive_feature, negative_feature):
def __call__(self, anchor_embedding, positive_embedding, negative_embedding):
with tf.name_scope("triplet_loss"):
# Normalize
anchor_embedding = tf.nn.l2_normalize(anchor_embedding, 1, 1e-10)
positive_embedding = tf.nn.l2_normalize(positive_embedding, 1, 1e-10)
negative_embedding = tf.nn.l2_normalize(negative_embedding, 1, 1e-10)
d_positive = tf.square(compute_euclidean_distance(anchor_feature, positive_feature))
d_negative = tf.square(compute_euclidean_distance(anchor_feature, negative_feature))
d_positive = tf.reduce_sum(tf.square(tf.sub(anchor_embedding, positive_embedding)), 1)
d_negative = tf.reduce_sum(tf.square(tf.sub(anchor_embedding, negative_embedding)), 1)
loss = tf.maximum(0., d_positive - d_negative + self.margin)
return tf.reduce_mean(loss), tf.reduce_mean(d_negative), tf.reduce_mean(d_positive)
basic_loss = tf.add(tf.sub(d_positive, d_negative), self.margin)
loss = tf.reduce_mean(tf.maximum(basic_loss, 0.0), 0)
return loss, tf.reduce_mean(d_negative), tf.reduce_mean(d_positive)
......@@ -45,7 +45,7 @@ def main():
# Preparing train set
#train_objects = db_mobio.objects(protocol="male", groups="world")
train_objects = db_mobio.objects(protocol="male", groups="world")
train_objects = sorted(db_mobio.objects(protocol="male", groups="world"), key=lambda x: x.id)
train_labels = [int(o.client_id) for o in train_objects]
n_classes = len(set(train_labels))
......@@ -54,16 +54,17 @@ def main():
extension=".hdf5")
for o in train_objects]
#train_data_shuffler = TripletWithSelectionDisk(train_file_names, train_labels,
# input_shape=[125, 125, 3],
# input_shape=[56, 56, 3],
# total_identities=8,
# batch_size=BATCH_SIZE)
train_data_shuffler = TripletWithFastSelectionDisk(train_file_names, train_labels,
input_shape=[224, 224, 3],
input_shape=[112, 112, 3],
batch_size=BATCH_SIZE,
total_identities=8)
# Preparing train set
validation_objects = db_mobio.objects(protocol="male", groups="dev")
validation_objects = sorted(db_mobio.objects(protocol="male", groups="dev"), key=lambda x: x.id)
#validation_objects = db_mobio.objects(protocol="male", groups="world")
validation_labels = [o.client_id for o in validation_objects]
......@@ -72,13 +73,13 @@ def main():
extension=".hdf5")
for o in validation_objects]
validation_data_shuffler = TripletDisk(validation_file_names, validation_labels,
input_shape=[224, 224, 3],
input_shape=[112, 112, 3],
batch_size=VALIDATION_BATCH_SIZE)
# Preparing the architecture
architecture = Chopra(seed=SEED, fc1_output=n_classes)
#architecture = Chopra(seed=SEED, fc1_output=n_classes)
#architecture = Chopra(seed=SEED, fc1_output=n_classes)
architecture = FaceNet(seed=SEED, use_gpu=USE_GPU)
optimizer = tf.train.GradientDescentOptimizer(0.00000001)
#architecture = FaceNet(seed=SEED, use_gpu=USE_GPU)
#optimizer = tf.train.GradientDescentOptimizer(0.0005)
#loss = BaseLoss(tf.nn.sparse_softmax_cross_entropy_with_logits, tf.reduce_mean)
......@@ -95,13 +96,14 @@ def main():
# optimizer=optimizer,
# temp_dir="./LOGS_MOBIO/siamese-cnn-prefetch")
loss = TripletLoss(margin=1.)
loss = TripletLoss(margin=0.5)
#optimizer = tf.train.GradientDescentOptimizer(0.000000000001)
#optimizer = optimizer,
trainer = TripletTrainer(architecture=architecture, loss=loss,
iterations=ITERATIONS,
base_learning_rate=0.0001,
prefetch=False,
temp_dir="./LOGS_MOBIO/triplet-cnn")
trainer.train(train_data_shuffler, validation_data_shuffler)
#trainer.train(train_data_shuffler)
#trainer.train(train_data_shuffler, validation_data_shuffler)
trainer.train(train_data_shuffler)
......@@ -46,7 +46,7 @@ def main():
db_casia = bob.db.casia_webface.Database()
# Preparing train set
train_objects = db_casia.objects(groups="world")
train_objects = sorted(db_casia.objects(groups="world"), key=lambda x: x.id)
#train_objects = db.objects(groups="world")
train_labels = [int(o.client_id) for o in train_objects]
directory = "/idiap/temp/tpereira/DEEP_FACE/CASIA_WEBFACE/casia_webface/preprocessed"
......@@ -67,7 +67,7 @@ def main():
# Preparing train set
directory = "/idiap/temp/tpereira/DEEP_FACE/CASIA_WEBFACE/mobio/preprocessed"
validation_objects = db_mobio.objects(protocol="male", groups="dev")
validation_objects = sorted(db_mobio.objects(protocol="male", groups="dev"), key=lambda x: x.id)
validation_labels = [o.client_id for o in validation_objects]
validation_file_names = [o.make_path(
......
......@@ -14,7 +14,8 @@ from tensorflow.core.framework import summary_pb2
import time
from bob.learn.tensorflow.datashuffler.OnlineSampling import OnLineSampling
os.environ["CUDA_VISIBLE_DEVICES"] = "1,2,3,0"
#os.environ["CUDA_VISIBLE_DEVICES"] = "1,2,3,0"
os.environ["CUDA_VISIBLE_DEVICES"] = ""
logger = bob.core.log.setup("bob.learn.tensorflow")
......
......@@ -6,6 +6,7 @@
parts = scripts
eggs = bob.learn.tensorflow
bob.db.casia_webface
bob.db.mobio
gridtk
extensions = bob.buildout
......@@ -30,7 +31,7 @@ bob.db.base = git git@gitlab.idiap.ch:bob/bob.db.base.git
bob.db.mobio = git git@gitlab.idiap.ch:bob/bob.db.mobio.git
bob.db.lfw = git git@gitlab.idiap.ch:bob/bob.db.lfw.git
bob.db.casia_webface = git git@gitlab.idiap.ch:bob/bob.db.casia_webface.git
gridtk = git git@github.com:bioidiap/gridtk
gridtk = git git@gitlab.idiap.ch:bob/gridtk
[scripts]
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment