util.py 13.4 KB
Newer Older
Tiago de Freitas Pereira's avatar
Scratch  
Tiago de Freitas Pereira committed
1 2 3
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
4
# @date: Wed 11 May 2016 09:39:36 CEST
Tiago de Freitas Pereira's avatar
Scratch  
Tiago de Freitas Pereira committed
5 6 7

import numpy
import tensorflow as tf
8
from tensorflow.python.client import device_lib
9
from tensorflow.python.framework import function
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
10
import logging
11

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
12
logger = logging.getLogger(__name__)
13 14 15 16


@function.Defun(tf.float32, tf.float32)
def norm_grad(x, dy):
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
17 18 19
    return tf.expand_dims(dy, -1) * (
        x / (tf.expand_dims(tf.norm(x, ord=2, axis=-1), -1) + 1.0e-19)
    )
20 21 22 23 24


@function.Defun(tf.float32, grad_func=norm_grad)
def norm(x):
    return tf.norm(x, ord=2, axis=-1)
Tiago de Freitas Pereira's avatar
Scratch  
Tiago de Freitas Pereira committed
25

26

Tiago de Freitas Pereira's avatar
Scratch  
Tiago de Freitas Pereira committed
27 28 29 30 31
def compute_euclidean_distance(x, y):
    """
    Computes the euclidean distance between two tensorflow variables
    """

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
32
    with tf.name_scope("euclidean_distance"):
33 34
        # d = tf.sqrt(tf.reduce_sum(tf.square(tf.subtract(x, y)), 1))
        d = norm(tf.subtract(x, y))
Tiago de Freitas Pereira's avatar
Scratch  
Tiago de Freitas Pereira committed
35 36
        return d

37

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66
def pdist_safe(A, metric="sqeuclidean"):
    if metric != "sqeuclidean":
        raise NotImplementedError()
    r = tf.reduce_sum(A * A, 1)
    r = tf.reshape(r, [-1, 1])
    D = r - 2 * tf.matmul(A, A, transpose_b=True) + tf.transpose(r)
    return D


def cdist(A, B, metric="sqeuclidean"):
    if metric != "sqeuclidean":
        raise NotImplementedError()
    M1, M2 = tf.shape(A)[0], tf.shape(B)[0]
    # code from https://stackoverflow.com/a/43839605/1286165
    p1 = tf.matmul(
        tf.expand_dims(tf.reduce_sum(tf.square(A), 1), 1), tf.ones(shape=(1, M2))
    )
    p2 = tf.transpose(
        tf.matmul(
            tf.reshape(tf.reduce_sum(tf.square(B), 1), shape=[-1, 1]),
            tf.ones(shape=(M1, 1)),
            transpose_b=True,
        )
    )

    D = tf.add(p1, p2) - 2 * tf.matmul(A, B, transpose_b=True)
    return D


67
def load_mnist(perc_train=0.9):
68
    numpy.random.seed(0)
Tiago de Freitas Pereira's avatar
Scratch  
Tiago de Freitas Pereira committed
69
    import bob.db.mnist
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
70

71
    db = bob.db.mnist.Database()
Tiago de Freitas Pereira's avatar
Scratch  
Tiago de Freitas Pereira committed
72 73 74 75 76 77
    raw_data = db.data()

    # data  = raw_data[0].astype(numpy.float64)
    data = raw_data[0]
    labels = raw_data[1]

78 79 80 81 82 83
    # Shuffling
    total_samples = data.shape[0]
    indexes = numpy.array(range(total_samples))
    numpy.random.shuffle(indexes)

    # Spliting train and validation
84
    n_train = int(perc_train * indexes.shape[0])
85 86
    n_validation = total_samples - n_train

87
    train_data = data[0:n_train, :].astype("float32") * 0.00390625
88 89
    train_labels = labels[0:n_train]

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
90 91 92 93
    validation_data = (
        data[n_train : n_train + n_validation, :].astype("float32") * 0.00390625
    )
    validation_labels = labels[n_train : n_train + n_validation]
94 95

    return train_data, train_labels, validation_data, validation_labels
Tiago de Freitas Pereira's avatar
Scratch  
Tiago de Freitas Pereira committed
96 97


98 99 100
def create_mnist_tfrecord(tfrecords_filename, data, labels, n_samples=6000):
    def _bytes_feature(value):
        return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
Tiago de Freitas Pereira's avatar
Scratch  
Tiago de Freitas Pereira committed
101

102 103
    def _int64_feature(value):
        return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
Tiago de Freitas Pereira's avatar
Scratch  
Tiago de Freitas Pereira committed
104

105
    writer = tf.python_io.TFRecordWriter(tfrecords_filename)
Tiago de Freitas Pereira's avatar
Scratch  
Tiago de Freitas Pereira committed
106

107 108 109
    for i in range(n_samples):
        img = data[i]
        img_raw = img.tostring()
110
        feature = {
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
111 112 113
            "data": _bytes_feature(img_raw),
            "label": _int64_feature(labels[i]),
            "key": _bytes_feature(b"-"),
114
        }
115

116 117 118
        example = tf.train.Example(features=tf.train.Features(feature=feature))
        writer.write(example.SerializeToString())
    writer.close()
Tiago de Freitas Pereira's avatar
Scratch  
Tiago de Freitas Pereira committed
119 120


Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
121 122 123
def compute_eer(
    data_train, labels_train, data_validation, labels_validation, n_classes
):
Tiago de Freitas Pereira's avatar
Scratch  
Tiago de Freitas Pereira committed
124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140
    import bob.measure
    from scipy.spatial.distance import cosine

    # Creating client models
    models = []
    for i in range(n_classes):
        indexes = labels_train == i
        models.append(numpy.mean(data_train[indexes, :], axis=0))

    # Probing
    positive_scores = numpy.zeros(shape=0)
    negative_scores = numpy.zeros(shape=0)

    for i in range(n_classes):
        # Positive scoring
        indexes = labels_validation == i
        positive_data = data_validation[indexes, :]
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
141
        p = [cosine(models[i], positive_data[j]) for j in range(positive_data.shape[0])]
Tiago de Freitas Pereira's avatar
Scratch  
Tiago de Freitas Pereira committed
142 143 144 145 146
        positive_scores = numpy.hstack((positive_scores, p))

        # negative scoring
        indexes = labels_validation != i
        negative_data = data_validation[indexes, :]
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
147
        n = [cosine(models[i], negative_data[j]) for j in range(negative_data.shape[0])]
Tiago de Freitas Pereira's avatar
Scratch  
Tiago de Freitas Pereira committed
148 149 150 151 152 153 154 155
        negative_scores = numpy.hstack((negative_scores, n))

    # Computing performance based on EER
    negative_scores = (-1) * negative_scores
    positive_scores = (-1) * positive_scores

    threshold = bob.measure.eer_threshold(negative_scores, positive_scores)
    far, frr = bob.measure.farfrr(negative_scores, positive_scores, threshold)
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
156
    eer = (far + frr) / 2.0
Tiago de Freitas Pereira's avatar
Scratch  
Tiago de Freitas Pereira committed
157 158 159 160

    return eer


Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
161 162 163
def compute_accuracy(
    data_train, labels_train, data_validation, labels_validation, n_classes
):
Tiago de Freitas Pereira's avatar
Scratch  
Tiago de Freitas Pereira committed
164 165 166 167 168 169 170 171 172 173 174 175
    from scipy.spatial.distance import cosine

    # Creating client models
    models = []
    for i in range(n_classes):
        indexes = labels_train == i
        models.append(numpy.mean(data_train[indexes, :], axis=0))

    # Probing
    tp = 0
    for i in range(data_validation.shape[0]):

176
        d = data_validation[i, :]
Tiago de Freitas Pereira's avatar
Scratch  
Tiago de Freitas Pereira committed
177 178 179 180 181 182 183 184 185
        l = labels_validation[i]

        scores = [cosine(m, d) for m in models]
        predict = numpy.argmax(scores)

        if predict == l:
            tp += 1

    return (float(tp) / data_validation.shape[0]) * 100
186 187


Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
188 189 190 191 192
def debug_embbeding(image, architecture, embbeding_dim=2, feature_layer="fc3"):
    """
    """
    import tensorflow as tf
    from bob.learn.tensorflow.utils.session import Session
193 194 195

    session = Session.instance(new=False).session
    inference_graph = architecture.compute_graph(
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
196 197
        architecture.inference_placeholder, feature_layer=feature_layer, training=False
    )
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
198 199 200

    embeddings = numpy.zeros(shape=(image.shape[0], embbeding_dim))
    for i in range(image.shape[0]):
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
201
        feed_dict = {architecture.inference_placeholder: image[i : i + 1, :, :, :]}
202
        embedding = session.run(
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
203 204
            [tf.nn.l2_normalize(inference_graph, 1, 1e-10)], feed_dict=feed_dict
        )[0]
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
205 206 207 208
        embedding = numpy.reshape(embedding, numpy.prod(embedding.shape[1:]))
        embeddings[i] = embedding

    return embeddings
209 210


211
def pdist(A):
212 213
    """
    Compute a pairwise euclidean distance in the same fashion
214
    as in scipy.spation.distance.pdist
215
    """
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
216
    with tf.name_scope("Pairwisedistance"):
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
217 218 219 220
        ones_1 = tf.reshape(tf.cast(tf.ones_like(A), tf.float32)[:, 0], [1, -1])
        p1 = tf.matmul(tf.expand_dims(tf.reduce_sum(tf.square(A), 1), 1), ones_1)

        ones_2 = tf.reshape(tf.cast(tf.ones_like(A), tf.float32)[:, 0], [-1, 1])
221 222 223 224
        p2 = tf.transpose(
            tf.matmul(
                tf.reshape(tf.reduce_sum(tf.square(A), 1), shape=[-1, 1]),
                ones_2,
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
225 226 227
                transpose_b=True,
            )
        )
228 229 230 231

        return tf.sqrt(tf.add(p1, p2) - 2 * tf.matmul(A, A, transpose_b=True))


232
def predict_using_tensors(embedding, labels, num=None):
233
    """
234 235
    Compute the predictions through exhaustive comparisons between
    embeddings using tensors
236 237
    """

238 239
    # Fitting the main diagonal with infs (removing comparisons with the same
    # sample)
240
    inf = tf.cast(tf.ones_like(labels), tf.float32) * numpy.inf
241

242
    distances = pdist(embedding)
243 244
    distances = tf.matrix_set_diag(distances, inf)
    indexes = tf.argmin(distances, axis=1)
245 246 247 248 249
    return [labels[i] for i in tf.unstack(indexes, num=num)]


def compute_embedding_accuracy_tensors(embedding, labels, num=None):
    """
250
    Compute the accuracy in a closed-set
251

252 253 254 255 256 257 258
    **Parameters**

    embeddings: `tf.Tensor`
      Set of embeddings

    labels: `tf.Tensor`
      Correspondent labels
259 260
    """

261 262
    # Fitting the main diagonal with infs (removing comparisons with the same
    # sample)
263
    predictions = predict_using_tensors(embedding, labels, num=num)
264
    matching = [
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
265 266
        tf.equal(p, l)
        for p, l in zip(tf.unstack(predictions, num=num), tf.unstack(labels, num=num))
267
    ]
268

269
    return tf.reduce_sum(tf.cast(matching, tf.uint8)) / len(predictions)
270 271


272 273
def compute_embedding_accuracy(embedding, labels):
    """
274
    Compute the accuracy in a closed-set
275

276 277 278 279 280 281 282
    **Parameters**

    embeddings: :any:`numpy.array`
      Set of embeddings

    labels: :any:`numpy.array`
      Correspondent labels
283 284
    """

285
    from scipy.spatial.distance import pdist, squareform
286

287
    distances = squareform(pdist(embedding))
288

289 290
    n_samples = embedding.shape[0]

291 292
    # Fitting the main diagonal with infs (removing comparisons with the same
    # sample)
293
    numpy.fill_diagonal(distances, numpy.inf)
294

295
    indexes = distances.argmin(axis=1)
296

297 298
    # Computing the argmin excluding comparisons with the same samples
    # Basically, we are excluding the main diagonal
299

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
300
    # valid_indexes = distances[distances>0].reshape(n_samples, n_samples-1).argmin(axis=1)
301 302

    # Getting the original positions of the indexes in the 1-axis
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
303
    # corrected_indexes = [ i if i<j else i+1 for i, j in zip(valid_indexes, range(n_samples))]
304

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
305
    matching = [labels[i] == labels[j] for i, j in zip(range(n_samples), indexes)]
306 307
    accuracy = sum(matching) / float(n_samples)

308
    return accuracy
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
309

310 311 312 313 314 315 316 317 318 319

def get_available_gpus():
    """Returns the number of GPU devices that are available.

    Returns
    -------
    [str]
        The names of available GPU devices.
    """
    local_device_protos = device_lib.list_local_devices()
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
320
    return [x.name for x in local_device_protos if x.device_type == "GPU"]
321 322 323 324 325 326 327 328


def to_channels_last(image):
    """Converts the image to channel_last format. This is the same format as in
    matplotlib, skimage, and etc.

    Parameters
    ----------
329
    image : `tf.Tensor`
330 331 332 333 334
        At least a 3 dimensional image. If the dimension is more than 3, the
        last 3 dimensions are assumed to be [C, H, W].

    Returns
    -------
335
    image : `tf.Tensor`
336 337 338 339 340 341 342 343 344
        The image in [..., H, W, C] format.

    Raises
    ------
    ValueError
        If dim of image is less than 3.
    """
    ndim = len(image.shape)
    if ndim < 3:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
345 346 347
        raise ValueError(
            "The image needs to be at least 3 dimensional but it " "was {}".format(ndim)
        )
348 349 350 351 352 353 354 355 356 357 358 359
    axis_order = [1, 2, 0]
    shift = ndim - 3
    axis_order = list(range(ndim - 3)) + [n + shift for n in axis_order]
    return tf.transpose(image, axis_order)


def to_channels_first(image):
    """Converts the image to channel_first format. This is the same format as
    in bob.io.image and bob.io.video.

    Parameters
    ----------
360
    image : `tf.Tensor`
361 362 363 364 365
        At least a 3 dimensional image. If the dimension is more than 3, the
        last 3 dimensions are assumed to be [H, W, C].

    Returns
    -------
366
    image : `tf.Tensor`
367 368 369 370 371 372 373 374 375
        The image in [..., C, H, W] format.

    Raises
    ------
    ValueError
        If dim of image is less than 3.
    """
    ndim = len(image.shape)
    if ndim < 3:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
376 377 378
        raise ValueError(
            "The image needs to be at least 3 dimensional but it " "was {}".format(ndim)
        )
379 380 381 382 383 384 385 386
    axis_order = [2, 0, 1]
    shift = ndim - 3
    axis_order = list(range(ndim - 3)) + [n + shift for n in axis_order]
    return tf.transpose(image, axis_order)


to_skimage = to_matplotlib = to_channels_last
to_bob = to_channels_first
387 388


Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
389
def bytes2human(n, format="%(value).1f %(symbol)s", symbols="customary"):
390 391 392 393 394 395 396 397 398
    """Convert n bytes into a human readable string based on format.
    From: https://code.activestate.com/recipes/578019-bytes-to-human-human-to-
    bytes-converter/
    Author: Giampaolo Rodola' <g.rodola [AT] gmail [DOT] com>
    License: MIT
    symbols can be either "customary", "customary_ext", "iec" or "iec_ext",
    see: http://goo.gl/kTQMs
    """
    SYMBOLS = {
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422
        "customary": ("B", "K", "M", "G", "T", "P", "E", "Z", "Y"),
        "customary_ext": (
            "byte",
            "kilo",
            "mega",
            "giga",
            "tera",
            "peta",
            "exa",
            "zetta",
            "iotta",
        ),
        "iec": ("Bi", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi"),
        "iec_ext": (
            "byte",
            "kibi",
            "mebi",
            "gibi",
            "tebi",
            "pebi",
            "exbi",
            "zebi",
            "yobi",
        ),
423 424 425 426 427 428 429 430 431 432 433 434 435
    }
    n = int(n)
    if n < 0:
        raise ValueError("n < 0")
    symbols = SYMBOLS[symbols]
    prefix = {}
    for i, s in enumerate(symbols[1:]):
        prefix[s] = 1 << (i + 1) * 10
    for symbol in reversed(symbols[1:]):
        if n >= prefix[symbol]:
            value = float(n) / prefix[symbol]
            return format % locals()
    return format % dict(symbol=symbols[0], value=n)
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
436 437


Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
438
def random_choice_no_replacement(one_dim_input, num_indices_to_drop=3, sort=False):
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467
    """Similar to np.random.choice with no replacement.
    Code from https://stackoverflow.com/a/54755281/1286165
    """
    input_length = tf.shape(one_dim_input)[0]

    # create uniform distribution over the sequence
    uniform_distribution = tf.random.uniform(
        shape=[input_length],
        minval=0,
        maxval=None,
        dtype=tf.float32,
        seed=None,
        name=None,
    )

    # grab the indices of the greatest num_words_to_drop values from the distibution
    _, indices_to_keep = tf.nn.top_k(
        uniform_distribution, input_length - num_indices_to_drop
    )

    # sort the indices
    if sort:
        sorted_indices_to_keep = tf.sort(indices_to_keep)
    else:
        sorted_indices_to_keep = indices_to_keep

    # gather indices from the input array using the filtered actual array
    result = tf.gather(one_dim_input, sorted_indices_to_keep)
    return result