TensorflowEval.py 9.38 KB
Newer Older
1 2 3 4 5 6 7 8
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Pavel Korshunov <pavel.korshunov@idiap.ch>
# @date: Wed 19 Oct 23:43:22 2016

from bob.pad.base.algorithm import Algorithm
import numpy

9
# import tensorflow as tf
Pavel KORSHUNOV's avatar
Pavel KORSHUNOV committed
10
import os
11 12

import logging
13

14 15 16
logger = logging.getLogger("bob.pad.voice")


17
class TensorflowEval(Algorithm):
Pavel KORSHUNOV's avatar
Pavel KORSHUNOV committed
18
    """This class is for evaluating data stored in tensorflow tfrecord format using a pre-trained LSTM model."""
19 20

    def __init__(self,
21
                 architecture_name="mlp",
22
                 input_shape=[200, 81],  # [temporal_length, feature_size]
23
                 network_size=60,  # the output size of LSTM cell
Pavel KORSHUNOV's avatar
Pavel KORSHUNOV committed
24
                 normalization_file=None,  # file with normalization parameters from train set
25 26 27 28 29 30 31 32
                 **kwargs):
        """Generates a test value that is read and written"""

        # call base class constructor registering that this tool performs everything.
        Algorithm.__init__(
            self,
            performs_projection=True,
            requires_projector_training=False,
33
            **kwargs
34
        )
35

36
        self.architecture_name = architecture_name
37 38
        self.input_shape = input_shape
        self.num_time_steps = input_shape[0]
39
        self.network_size = network_size
Pavel KORSHUNOV's avatar
Pavel KORSHUNOV committed
40
        self.data_std = None
41 42
#        import ipdb
#        ipdb.set_trace()
Pavel KORSHUNOV's avatar
Pavel KORSHUNOV committed
43
        features_length = input_shape[1]
Pavel KORSHUNOV's avatar
Pavel KORSHUNOV committed
44
        if normalization_file and os.path.exists(normalization_file):
45
            logger.info("Loading normalization file '%s' " % normalization_file)
Pavel KORSHUNOV's avatar
Pavel KORSHUNOV committed
46 47
            npzfile = numpy.load(normalization_file)
            self.data_mean = npzfile['data_mean']
48
            self.data_std = numpy.array(npzfile['data_std'])
49
            if not self.data_std.shape:  # if std was saved as scalar
Pavel KORSHUNOV's avatar
Pavel KORSHUNOV committed
50 51 52 53 54 55 56
                self.data_std = numpy.ones(features_length)
            # if self.data_mean.shape[0] > input_shape[0]:
            #     self.data_mean = self.data_mean[:input_shape[0]]
            # self.data_mean = numpy.reshape(self.data_mean, input_shape)
            # if self.data_std.shape[0] > input_shape[0]:
            #     self.data_std = self.data_std[:input_shape[0]]
            # self.data_std = numpy.reshape(self.data_std, input_shape)
Pavel KORSHUNOV's avatar
Pavel KORSHUNOV committed
57
        else:
58
            logger.warn("Normalization file '%s' does not exist!" % normalization_file)
Pavel KORSHUNOV's avatar
Pavel KORSHUNOV committed
59 60
            self.data_mean = 0
            self.data_std = 1
61

62
        self.data_reader = None
63
        self.session = None
64
        self.dnn_model = None
65
        self.data_placeholder = None
66

67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89
    # def simple_lstm_network(self, train_data_shuffler, batch_size=10, lstm_cell_size=64,
    #                         num_time_steps=28, num_classes=10, seed=10, reuse=False):
    #     import tensorflow as tf
    #     from bob.learn.tensorflow.layers import lstm
    #     slim = tf.contrib.slim
    #
    #     if isinstance(train_data_shuffler, tf.Tensor):
    #         inputs = train_data_shuffler
    #     else:
    #         inputs = train_data_shuffler("data", from_queue=False)
    #
    #     initializer = tf.contrib.layers.xavier_initializer(seed=seed)
    #
    #     # Creating an LSTM network
    #     graph = lstm(inputs, lstm_cell_size, num_time_steps=num_time_steps, batch_size=batch_size,
    #                  output_activation_size=num_classes, scope='lstm', name='sync_cell',
    #                  weights_initializer=initializer, activation=tf.nn.sigmoid, reuse=reuse)
    #
    #     # fully connect the LSTM output to the classes
    #     graph = slim.fully_connected(graph, num_classes, activation_fn=None, scope='fc1',
    #                                  weights_initializer=initializer, reuse=reuse)
    #
    #     return graph
90

Pavel KORSHUNOV's avatar
Pavel KORSHUNOV committed
91 92 93 94 95
    def normalize_data(self, features):
        mean = numpy.mean(features, axis=0)
        std = numpy.std(features, axis=0)
        return numpy.divide(features - mean, std)

96 97
    def _check_feature(self, feature):
        """Checks that the features are appropriate."""
98
        if not isinstance(feature, numpy.ndarray) or feature.ndim != 2 or feature.dtype != numpy.float32:
99 100
            raise ValueError("The given feature is not appropriate", feature)
        return True
101

102 103
    def restore_trained_model(self, projector_file):
        import tensorflow as tf
104

105 106
        if self.session is None:
            self.session = tf.Session()
107
        data_pl = tf.placeholder(tf.float32, shape=(None,) + tuple(self.input_shape), name="data")
108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131

        # create an empty graph of the correct architecture but with needed batch_size==1
        if self.architecture_name == 'lstm':
            from bob.learn.tensorflow.network import simple_lstm_network
            graph = simple_lstm_network(data_pl, batch_size=1,
                                        lstm_cell_size=self.network_size, num_time_steps=self.num_time_steps,
                                        num_classes=2, reuse=False)

        elif self.architecture_name == 'mlp':
            from bob.learn.tensorflow.network import mlp_network
            graph = mlp_network(data_pl,
                                hidden_layer_size=self.network_size,
                                num_time_steps=self.num_time_steps,
                                num_classes=2, reuse=False)
        elif self.architecture_name == 'simplecnn':
            from bob.learn.tensorflow.network import simple2Dcnn_network
            graph = simple2Dcnn_network(data_pl,
                                        num_classes=2, reuse=False)
        elif self.architecture_name == 'lightcnn':
            from bob.learn.tensorflow.network import LightCNN9
            network = LightCNN9(n_classes=2, device="/cpu:0")
            graph = network(data_pl, reuse=False)
        else:
            return None
132 133

        self.session.run(tf.global_variables_initializer())
134
        saver = tf.train.Saver()
Pavel KORSHUNOV's avatar
Pavel KORSHUNOV committed
135
        #        saver = tf.train.import_meta_graph(projector_file + ".meta", clear_devices=True)
136
        saver.restore(self.session, projector_file)
137
        return tf.nn.softmax(graph, name="softmax"), data_pl
138
#        return graph, data_pl
139

140 141
    def load_projector(self, projector_file):
        logger.info("Loading pretrained model from {0}".format(projector_file))
142 143

        self.dnn_model, self.data_placeholder = self.restore_trained_model(projector_file)
144

145
    def project_feature(self, feature):
146

147
        logger.info(" .... Projecting %d features vector" % feature.shape[0])
148 149
        from bob.learn.tensorflow.datashuffler import DiskAudio
        if not self.data_reader:
150
            self.data_reader = DiskAudio([0], [0], [1] + self.input_shape)
Pavel KORSHUNOV's avatar
Pavel KORSHUNOV committed
151

Pavel KORSHUNOV's avatar
Pavel KORSHUNOV committed
152 153 154 155
        # normalize the feature using pre-loaded normalization parameters
        if self.data_std is not None and self.data_std.all() > 0:
            feature = numpy.divide(feature - self.data_mean, self.data_std)

Pavel KORSHUNOV's avatar
Pavel KORSHUNOV committed
156 157 158
        # split the feature in the sliding window frames
        frames, _ = self.data_reader.split_features_in_windows(features=feature, label=1,
                                                               win_size=self.num_time_steps,
159
                                                               sliding_step=1)
Pavel KORSHUNOV's avatar
Pavel KORSHUNOV committed
160
#        logger.info(" .... And frames of shape {0} are extracted to pass into DNN model".format(frames.shape))
Pavel KORSHUNOV's avatar
Pavel KORSHUNOV committed
161 162 163
        if frames is None:
            return None

Pavel KORSHUNOV's avatar
Pavel KORSHUNOV committed
164 165
        logger.info(" .... And frames of shape {0} are extracted to pass into DNN model".format(frames.shape))

166
        projections = numpy.zeros((len(frames), 2), dtype=numpy.float32)
Pavel KORSHUNOV's avatar
Pavel KORSHUNOV committed
167 168
        for i in range(frames.shape[0]):
            frame = frames[i]
169 170
            frame = numpy.reshape(frame, [1] + self.input_shape)
            #logger.info(" .... projecting frame of shape {0} onto DNN model".format(frame.shape))
171 172 173

            if self.session is not None:
                forward_output = self.session.run(self.dnn_model, feed_dict={self.data_placeholder: frame})
Pavel KORSHUNOV's avatar
Pavel KORSHUNOV committed
174
                projections[i] = forward_output[0]
175 176
            else:
                raise ValueError("Tensorflow session was not initialized, so cannot project on DNN model!")
Pavel KORSHUNOV's avatar
Pavel KORSHUNOV committed
177

178
        logger.info("Projected scores {0}".format(projections))
179
        return numpy.asarray(projections, dtype=numpy.float32)
180

181 182
    def project(self, feature):
        """project(feature) -> projected
183

184 185
        This function will project the given feature.
        It is assured that the :py:meth:`load_projector` was called once before the ``project`` function is executed.
186

187
        **Parameters:**
188

189 190
        feature : object
          The feature to be projected.
191

192
        **Returns:**
193

194 195 196
        projected : object
          The projected features.
          Must be writable with the :py:meth:`write_feature` function and readable with the :py:meth:`read_feature` function.
197

198 199 200 201 202 203 204
        """
        if len(feature) > 0:
            feature = numpy.cast['float32'](feature)
            self._check_feature(feature)
            return self.project_feature(feature)
        else:
            return numpy.zeros(1, dtype=numpy.float64)
205

206 207
    def score_for_multiple_projections(self, toscore):
        """scorescore_for_multiple_projections(toscore) -> score
208

209
        **Returns:**
210

211 212 213 214
        score : float
          A score value for the object ``toscore``.
        """
        scores = numpy.asarray(toscore, dtype=numpy.float32)
215
        real_scores = scores[:, 1]
216 217
        logger.debug("Mean score %f", numpy.mean(real_scores))
        return [numpy.mean(real_scores)]
218

219 220 221 222 223
    def score(self, toscore):
        """Returns the evarage value of the probe"""
        logger.debug("score() score %f", toscore)
        # return only real score
        return [toscore[0]]
224 225


226
algorithm = TensorflowEval()