From 2956f8ea9a217f9d69b33454f4806e90f3239c48 Mon Sep 17 00:00:00 2001 From: Amir MOHAMMADI <amir.mohammadi@idiap.ch> Date: Tue, 10 Oct 2017 12:22:50 +0200 Subject: [PATCH] autopep8 --- .../tensorflow/script/db_to_tfrecords.py | 19 +- bob/learn/tensorflow/script/eval_generic.py | 106 +++++----- bob/learn/tensorflow/script/train_generic.py | 106 +++++----- bob/learn/tensorflow/utils/eval.py | 190 +++++++++--------- bob/learn/tensorflow/utils/hooks.py | 58 +++--- bob/learn/tensorflow/utils/tfrecords.py | 34 ++-- 6 files changed, 256 insertions(+), 257 deletions(-) diff --git a/bob/learn/tensorflow/script/db_to_tfrecords.py b/bob/learn/tensorflow/script/db_to_tfrecords.py index 9f9c2b46..f3b84044 100644 --- a/bob/learn/tensorflow/script/db_to_tfrecords.py +++ b/bob/learn/tensorflow/script/db_to_tfrecords.py @@ -92,20 +92,20 @@ logger = setup(__name__) def bytes_feature(value): - return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value])) + return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value])) def int64_feature(value): - return tf.train.Feature(int64_list=tf.train.Int64List(value=[value])) + return tf.train.Feature(int64_list=tf.train.Int64List(value=[value])) def write_a_sample(writer, data, label, feature=None): - if feature is None: - feature = {'train/data': bytes_feature(data.tostring()), - 'train/label': int64_feature(label)} + if feature is None: + feature = {'train/data': bytes_feature(data.tostring()), + 'train/label': int64_feature(label)} - example = tf.train.Example(features=tf.train.Features(feature=feature)) - writer.write(example.SerializeToString()) + example = tf.train.Example(features=tf.train.Features(feature=feature)) + writer.write(example.SerializeToString()) def main(argv=None): @@ -149,8 +149,7 @@ def main(argv=None): logger.info('Processing file %d out of %d', i + 1, n_files) path = f.make_path(data_dir, data_extension) - data = reader(path) - + data = reader(path) if data is None: if allow_missing_files: logger.debug("... Processing original data file '{0}' was not successful".format(path)) @@ -168,4 +167,4 @@ def main(argv=None): if __name__ == '__main__': - main() + main() diff --git a/bob/learn/tensorflow/script/eval_generic.py b/bob/learn/tensorflow/script/eval_generic.py index 23127563..79ffce82 100644 --- a/bob/learn/tensorflow/script/eval_generic.py +++ b/bob/learn/tensorflow/script/eval_generic.py @@ -32,65 +32,65 @@ from bob.dap.base.util.eval import get_global_step, eval_once def main(argv=None): - arguments = docopt(__doc__, argv=argv) - print(arguments) - input_shape = eval(arguments['--input-shape']) - tfrecord_filenames = arguments['<eval_tfrecords>'] - eval_dir = arguments['--eval-dir'] - batch_size = eval(arguments['--batch-size']) - run_once = arguments['--run-once'] - eval_interval_secs = eval(arguments['--eval-interval-secs']) - checkpoint_dir = arguments['<checkpoint_dir>'] - num_examples = eval(arguments['--num-examples']) + arguments = docopt(__doc__, argv=argv) + print(arguments) + input_shape = eval(arguments['--input-shape']) + tfrecord_filenames = arguments['<eval_tfrecords>'] + eval_dir = arguments['--eval-dir'] + batch_size = eval(arguments['--batch-size']) + run_once = arguments['--run-once'] + eval_interval_secs = eval(arguments['--eval-interval-secs']) + checkpoint_dir = arguments['<checkpoint_dir>'] + num_examples = eval(arguments['--num-examples']) - create_directories_safe(eval_dir) - with tf.Graph().as_default() as g: + create_directories_safe(eval_dir) + with tf.Graph().as_default() as g: - # Get images and labels - with tf.name_scope('input'): - dataset = tf.contrib.data.TFRecordDataset(tfrecord_filenames) - feature = {'train/data': tf.FixedLenFeature([], tf.string), - 'train/label': tf.FixedLenFeature([], tf.int64)} - my_example_parser = partial( - example_parser, feature=feature, data_shape=input_shape) - dataset = dataset.map( - my_example_parser, num_threads=1, output_buffer_size=batch_size) - dataset = dataset.batch(batch_size) - images, labels = dataset.make_one_shot_iterator().get_next() + # Get images and labels + with tf.name_scope('input'): + dataset = tf.contrib.data.TFRecordDataset(tfrecord_filenames) + feature = {'train/data': tf.FixedLenFeature([], tf.string), + 'train/label': tf.FixedLenFeature([], tf.int64)} + my_example_parser = partial( + example_parser, feature=feature, data_shape=input_shape) + dataset = dataset.map( + my_example_parser, num_threads=1, output_buffer_size=batch_size) + dataset = dataset.batch(batch_size) + images, labels = dataset.make_one_shot_iterator().get_next() - # Build a Graph that computes the logits predictions from the - # inference model. - logits = architecture(images, mode=tf.estimator.ModeKeys.EVAL) + # Build a Graph that computes the logits predictions from the + # inference model. + logits = architecture(images, mode=tf.estimator.ModeKeys.EVAL) - # Calculate predictions. - top_k_op = tf.nn.in_top_k(logits, labels, 1) + # Calculate predictions. + top_k_op = tf.nn.in_top_k(logits, labels, 1) - saver = tf.train.Saver() - # Build the summary operation based on the TF collection of Summaries. - summary_op = tf.summary.merge_all() - summary_writer = tf.summary.FileWriter(eval_dir, g) - evaluated_file = os.path.join(eval_dir, 'evaluated') + saver = tf.train.Saver() + # Build the summary operation based on the TF collection of Summaries. + summary_op = tf.summary.merge_all() + summary_writer = tf.summary.FileWriter(eval_dir, g) + evaluated_file = os.path.join(eval_dir, 'evaluated') - while True: - evaluated_steps = [] - if os.path.exists(evaluated_file): - with open(evaluated_file) as f: - evaluated_steps = f.read().split() - ckpt = tf.train.get_checkpoint_state(checkpoint_dir) - if ckpt and ckpt.model_checkpoint_path: - for path in ckpt.all_model_checkpoint_paths: - global_step = get_global_step(path) - if global_step not in evaluated_steps: - ret_val = eval_once(saver, summary_writer, top_k_op, summary_op, - path, global_step, num_examples, - batch_size) - if ret_val == 0: - with open(evaluated_file, 'a') as f: - f.write(global_step + '\n') - if run_once: - break - time.sleep(eval_interval_secs) + while True: + evaluated_steps = [] + if os.path.exists(evaluated_file): + with open(evaluated_file) as f: + evaluated_steps = f.read().split() + ckpt = tf.train.get_checkpoint_state(checkpoint_dir) + if ckpt and ckpt.model_checkpoint_path: + for path in ckpt.all_model_checkpoint_paths: + global_step = get_global_step(path) + if global_step not in evaluated_steps: + ret_val = eval_once(saver, summary_writer, top_k_op, summary_op, + path, global_step, num_examples, + batch_size) + if ret_val == 0: + with open(evaluated_file, 'a') as f: + f.write(global_step + '\n') + if run_once: + break + time.sleep(eval_interval_secs) if __name__ == '__main__': - main() + main() diff --git a/bob/learn/tensorflow/script/train_generic.py b/bob/learn/tensorflow/script/train_generic.py index 246b1c12..c757f45c 100644 --- a/bob/learn/tensorflow/script/train_generic.py +++ b/bob/learn/tensorflow/script/train_generic.py @@ -33,66 +33,66 @@ from bob.dap.base.util.hooks import LoggerHook def main(argv=None): - arguments = docopt(__doc__, argv=argv) - print(arguments) - input_shape = eval(arguments['--input-shape']) - tfrecord_filenames = arguments['<train_tfrecords>'] - save_dir = arguments['--save-dir'] - epochs = eval(arguments['--epochs']) - batch_size = eval(arguments['--batch-size']) - capacity_samples = eval(arguments['--capacity-samples']) - learning_rate = eval(arguments['--learning-rate']) - log_frequency = eval(arguments['--log-frequency']) + arguments = docopt(__doc__, argv=argv) + print(arguments) + input_shape = eval(arguments['--input-shape']) + tfrecord_filenames = arguments['<train_tfrecords>'] + save_dir = arguments['--save-dir'] + epochs = eval(arguments['--epochs']) + batch_size = eval(arguments['--batch-size']) + capacity_samples = eval(arguments['--capacity-samples']) + learning_rate = eval(arguments['--learning-rate']) + log_frequency = eval(arguments['--log-frequency']) - create_directories_safe(save_dir) - with tf.Graph().as_default(): - global_step = tf.contrib.framework.get_or_create_global_step() + create_directories_safe(save_dir) + with tf.Graph().as_default(): + global_step = tf.contrib.framework.get_or_create_global_step() - # Get images and labels - with tf.name_scope('input'): - filename_queue = tf.train.string_input_producer( - tfrecord_filenames, num_epochs=epochs, name="tfrecord_filenames") + # Get images and labels + with tf.name_scope('input'): + filename_queue = tf.train.string_input_producer( + tfrecord_filenames, num_epochs=epochs, name="tfrecord_filenames") - image, label = read_and_decode(filename_queue, input_shape) - images, labels = tf.train.shuffle_batch( - [image, label], batch_size=batch_size, - capacity=capacity_samples // batch_size, - min_after_dequeue=int(capacity_samples // batch_size // 2), - num_threads=1, name="shuffle_batch") + image, label = read_and_decode(filename_queue, input_shape) + images, labels = tf.train.shuffle_batch( + [image, label], batch_size=batch_size, + capacity=capacity_samples // batch_size, + min_after_dequeue=int(capacity_samples // batch_size // 2), + num_threads=1, name="shuffle_batch") - # Build a Graph that computes the logits predictions from the - # inference model. - logits = architecture(images) - tf.add_to_collection('logits', logits) + # Build a Graph that computes the logits predictions from the + # inference model. + logits = architecture(images) + tf.add_to_collection('logits', logits) - # Calculate loss. - predictor = tf.nn.sparse_softmax_cross_entropy_with_logits( - logits=logits, labels=labels) - loss = tf.reduce_mean(predictor) - tf.summary.scalar('loss', loss) + # Calculate loss. + predictor = tf.nn.sparse_softmax_cross_entropy_with_logits( + logits=logits, labels=labels) + loss = tf.reduce_mean(predictor) + tf.summary.scalar('loss', loss) - # Build a Graph that trains the model with one batch of examples and - # updates the model parameters. - optimizer = tf.train.GradientDescentOptimizer(learning_rate) - train_op = optimizer.minimize(loss, global_step=global_step) + # Build a Graph that trains the model with one batch of examples and + # updates the model parameters. + optimizer = tf.train.GradientDescentOptimizer(learning_rate) + train_op = optimizer.minimize(loss, global_step=global_step) - saver = tf.train.Saver(max_to_keep=10**5) - scaffold = tf.train.Scaffold(saver=saver) - with tf.train.MonitoredTrainingSession( - checkpoint_dir=save_dir, - scaffold=scaffold, - hooks=[ - tf.train.CheckpointSaverHook( - save_dir, save_secs=60 * 29, scaffold=scaffold), - tf.train.NanTensorHook(loss), - LoggerHook(loss, batch_size, log_frequency)], - config=session_conf, - save_checkpoint_secs=None, - save_summaries_steps=100, - ) as mon_sess: - while not mon_sess.should_stop(): - mon_sess.run(train_op) + saver = tf.train.Saver(max_to_keep=10**5) + scaffold = tf.train.Scaffold(saver=saver) + with tf.train.MonitoredTrainingSession( + checkpoint_dir=save_dir, + scaffold=scaffold, + hooks=[ + tf.train.CheckpointSaverHook( + save_dir, save_secs=60 * 29, scaffold=scaffold), + tf.train.NanTensorHook(loss), + LoggerHook(loss, batch_size, log_frequency)], + config=session_conf, + save_checkpoint_secs=None, + save_summaries_steps=100, + ) as mon_sess: + while not mon_sess.should_stop(): + mon_sess.run(train_op) if __name__ == '__main__': - main() + main() diff --git a/bob/learn/tensorflow/utils/eval.py b/bob/learn/tensorflow/utils/eval.py index a431f8d2..ce20ed13 100644 --- a/bob/learn/tensorflow/utils/eval.py +++ b/bob/learn/tensorflow/utils/eval.py @@ -9,102 +9,102 @@ from datetime import datetime def get_global_step(path): - """Returns the global number associated with the model checkpoint path. The - checkpoint must have been saved with the - :any:`tf.train.MonitoredTrainingSession`. - - Parameters - ---------- - path : str - The path to model checkpoint, usually ckpt.model_checkpoint_path - - Returns - ------- - global_step : str - The global step number as a string. - """ - # Assuming model_checkpoint_path looks something like: - # /my-favorite-path/train/model.ckpt-0, - # extract global_step from it. - global_step = path.split('/')[-1].split('-')[-1] - return global_step + """Returns the global number associated with the model checkpoint path. The + checkpoint must have been saved with the + :any:`tf.train.MonitoredTrainingSession`. + + Parameters + ---------- + path : str + The path to model checkpoint, usually ckpt.model_checkpoint_path + + Returns + ------- + global_step : str + The global step number as a string. + """ + # Assuming model_checkpoint_path looks something like: + # /my-favorite-path/train/model.ckpt-0, + # extract global_step from it. + global_step = path.split('/')[-1].split('-')[-1] + return global_step def eval_once(saver, summary_writer, prediction_op, summary_op, model_checkpoint_path, global_step, num_examples, batch_size): - """Run Eval once. - - Parameters - ---------- - saver - Saver. - summary_writer - Summary writer. - prediction_op - Prediction operator. - summary_op - Summary operator. - model_checkpoint_path : str - Path to the model checkpoint. - global_step : str - The global step. - num_examples : int or None - The number of examples to try. - batch_size : int - The size of evaluation batch. - - This function requires the ``from __future__ import division`` import. - - Returns - ------- - int - 0 for success, anything else for fail. - - """ - with tf.Session() as sess: - sess.run(tf.local_variables_initializer()) - sess.run(tf.global_variables_initializer()) - - if model_checkpoint_path: - # Restores from checkpoint - saver.restore(sess, model_checkpoint_path) - else: - print('No checkpoint file found') - return -1 - - # Start the queue runners. - coord = tf.train.Coordinator() - try: - threads = [] - for qr in tf.get_collection(tf.GraphKeys.QUEUE_RUNNERS): - threads.extend(qr.create_threads(sess, coord=coord, daemon=True, - start=True)) - - if num_examples is None: - num_iter = float("inf") - else: - num_iter = int(math.ceil(num_examples / batch_size)) - true_count = 0 # Counts the number of correct predictions. - total_sample_count = 0 - step = 0 - while step < num_iter and not coord.should_stop(): - predictions = sess.run([prediction_op]) - true_count += np.sum(predictions) - total_sample_count += np.asarray(predictions).size - step += 1 - - # Compute precision @ 1. - precision = true_count / total_sample_count - print('%s: precision @ 1 = %.3f' % (datetime.now(), precision)) - - summary = tf.Summary() - summary.ParseFromString(sess.run(summary_op)) - summary.value.add(tag='Precision @ 1', simple_value=precision) - summary_writer.add_summary(summary, global_step) - return 0 - except Exception as e: # pylint: disable=broad-except - coord.request_stop(e) - return -1 - finally: - coord.request_stop() - coord.join(threads, stop_grace_period_secs=10) + """Run Eval once. + + Parameters + ---------- + saver + Saver. + summary_writer + Summary writer. + prediction_op + Prediction operator. + summary_op + Summary operator. + model_checkpoint_path : str + Path to the model checkpoint. + global_step : str + The global step. + num_examples : int or None + The number of examples to try. + batch_size : int + The size of evaluation batch. + + This function requires the ``from __future__ import division`` import. + + Returns + ------- + int + 0 for success, anything else for fail. + + """ + with tf.Session() as sess: + sess.run(tf.local_variables_initializer()) + sess.run(tf.global_variables_initializer()) + + if model_checkpoint_path: + # Restores from checkpoint + saver.restore(sess, model_checkpoint_path) + else: + print('No checkpoint file found') + return -1 + + # Start the queue runners. + coord = tf.train.Coordinator() + try: + threads = [] + for qr in tf.get_collection(tf.GraphKeys.QUEUE_RUNNERS): + threads.extend(qr.create_threads(sess, coord=coord, daemon=True, + start=True)) + + if num_examples is None: + num_iter = float("inf") + else: + num_iter = int(math.ceil(num_examples / batch_size)) + true_count = 0 # Counts the number of correct predictions. + total_sample_count = 0 + step = 0 + while step < num_iter and not coord.should_stop(): + predictions = sess.run([prediction_op]) + true_count += np.sum(predictions) + total_sample_count += np.asarray(predictions).size + step += 1 + + # Compute precision @ 1. + precision = true_count / total_sample_count + print('%s: precision @ 1 = %.3f' % (datetime.now(), precision)) + + summary = tf.Summary() + summary.ParseFromString(sess.run(summary_op)) + summary.value.add(tag='Precision @ 1', simple_value=precision) + summary_writer.add_summary(summary, global_step) + return 0 + except Exception as e: # pylint: disable=broad-except + coord.request_stop(e) + return -1 + finally: + coord.request_stop() + coord.join(threads, stop_grace_period_secs=10) diff --git a/bob/learn/tensorflow/utils/hooks.py b/bob/learn/tensorflow/utils/hooks.py index fe15d519..1875e519 100644 --- a/bob/learn/tensorflow/utils/hooks.py +++ b/bob/learn/tensorflow/utils/hooks.py @@ -4,32 +4,32 @@ from datetime import datetime class LoggerHook(tf.train.SessionRunHook): - """Logs loss and runtime.""" - - def __init__(self, loss, batch_size, log_frequency): - self.loss = loss - self.batch_size = batch_size - self.log_frequency = log_frequency - - def begin(self): - self._step = -1 - self._start_time = time.time() - - def before_run(self, run_context): - self._step += 1 - return tf.train.SessionRunArgs(self.loss) # Asks for loss value. - - def after_run(self, run_context, run_values): - if self._step % self.log_frequency == 0: - current_time = time.time() - duration = current_time - self._start_time - self._start_time = current_time - - loss_value = run_values.results - examples_per_sec = self.log_frequency * self.batch_size / duration - sec_per_batch = float(duration / self.log_frequency) - - format_str = ('%s: step %d, loss = %.2f (%.1f examples/sec; %.3f ' - 'sec/batch)') - print(format_str % (datetime.now(), self._step, loss_value, - examples_per_sec, sec_per_batch)) + """Logs loss and runtime.""" + + def __init__(self, loss, batch_size, log_frequency): + self.loss = loss + self.batch_size = batch_size + self.log_frequency = log_frequency + + def begin(self): + self._step = -1 + self._start_time = time.time() + + def before_run(self, run_context): + self._step += 1 + return tf.train.SessionRunArgs(self.loss) # Asks for loss value. + + def after_run(self, run_context, run_values): + if self._step % self.log_frequency == 0: + current_time = time.time() + duration = current_time - self._start_time + self._start_time = current_time + + loss_value = run_values.results + examples_per_sec = self.log_frequency * self.batch_size / duration + sec_per_batch = float(duration / self.log_frequency) + + format_str = ('%s: step %d, loss = %.2f (%.1f examples/sec; %.3f ' + 'sec/batch)') + print(format_str % (datetime.now(), self._step, loss_value, + examples_per_sec, sec_per_batch)) diff --git a/bob/learn/tensorflow/utils/tfrecords.py b/bob/learn/tensorflow/utils/tfrecords.py index 309575a2..dcab80c8 100644 --- a/bob/learn/tensorflow/utils/tfrecords.py +++ b/bob/learn/tensorflow/utils/tfrecords.py @@ -2,24 +2,24 @@ import tensorflow as tf def example_parser(serialized_example, feature, data_shape): - """Parses a single tf.Example into image and label tensors.""" - # Decode the record read by the reader - features = tf.parse_single_example(serialized_example, features=feature) - # Convert the image data from string back to the numbers - image = tf.decode_raw(features['train/data'], tf.float32) - # Cast label data into int64 - label = tf.cast(features['train/label'], tf.int64) - # Reshape image data into the original shape - image = tf.reshape(image, data_shape) - return image, label + """Parses a single tf.Example into image and label tensors.""" + # Decode the record read by the reader + features = tf.parse_single_example(serialized_example, features=feature) + # Convert the image data from string back to the numbers + image = tf.decode_raw(features['train/data'], tf.float32) + # Cast label data into int64 + label = tf.cast(features['train/label'], tf.int64) + # Reshape image data into the original shape + image = tf.reshape(image, data_shape) + return image, label def read_and_decode(filename_queue, data_shape, feature=None): - if feature is None: - feature = {'train/data': tf.FixedLenFeature([], tf.string), - 'train/label': tf.FixedLenFeature([], tf.int64)} - # Define a reader and read the next record - reader = tf.TFRecordReader() - _, serialized_example = reader.read(filename_queue) - return example_parser(serialized_example, feature, data_shape) + if feature is None: + feature = {'train/data': tf.FixedLenFeature([], tf.string), + 'train/label': tf.FixedLenFeature([], tf.int64)} + # Define a reader and read the next record + reader = tf.TFRecordReader() + _, serialized_example = reader.read(filename_queue) + return example_parser(serialized_example, feature, data_shape) -- GitLab