From 2956f8ea9a217f9d69b33454f4806e90f3239c48 Mon Sep 17 00:00:00 2001
From: Amir MOHAMMADI <amir.mohammadi@idiap.ch>
Date: Tue, 10 Oct 2017 12:22:50 +0200
Subject: [PATCH] autopep8
---
.../tensorflow/script/db_to_tfrecords.py | 19 +-
bob/learn/tensorflow/script/eval_generic.py | 106 +++++-----
bob/learn/tensorflow/script/train_generic.py | 106 +++++-----
bob/learn/tensorflow/utils/eval.py | 190 +++++++++---------
bob/learn/tensorflow/utils/hooks.py | 58 +++---
bob/learn/tensorflow/utils/tfrecords.py | 34 ++--
6 files changed, 256 insertions(+), 257 deletions(-)
diff --git a/bob/learn/tensorflow/script/db_to_tfrecords.py b/bob/learn/tensorflow/script/db_to_tfrecords.py
index 9f9c2b46..f3b84044 100644
--- a/bob/learn/tensorflow/script/db_to_tfrecords.py
+++ b/bob/learn/tensorflow/script/db_to_tfrecords.py
@@ -92,20 +92,20 @@ logger = setup(__name__)
def bytes_feature(value):
- return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
+ return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
def int64_feature(value):
- return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
+ return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
def write_a_sample(writer, data, label, feature=None):
- if feature is None:
- feature = {'train/data': bytes_feature(data.tostring()),
- 'train/label': int64_feature(label)}
+ if feature is None:
+ feature = {'train/data': bytes_feature(data.tostring()),
+ 'train/label': int64_feature(label)}
- example = tf.train.Example(features=tf.train.Features(feature=feature))
- writer.write(example.SerializeToString())
+ example = tf.train.Example(features=tf.train.Features(feature=feature))
+ writer.write(example.SerializeToString())
def main(argv=None):
@@ -149,8 +149,7 @@ def main(argv=None):
logger.info('Processing file %d out of %d', i + 1, n_files)
path = f.make_path(data_dir, data_extension)
- data = reader(path)
-
+ data = reader(path)
if data is None:
if allow_missing_files:
logger.debug("... Processing original data file '{0}' was not successful".format(path))
@@ -168,4 +167,4 @@ def main(argv=None):
if __name__ == '__main__':
- main()
+ main()
diff --git a/bob/learn/tensorflow/script/eval_generic.py b/bob/learn/tensorflow/script/eval_generic.py
index 23127563..79ffce82 100644
--- a/bob/learn/tensorflow/script/eval_generic.py
+++ b/bob/learn/tensorflow/script/eval_generic.py
@@ -32,65 +32,65 @@ from bob.dap.base.util.eval import get_global_step, eval_once
def main(argv=None):
- arguments = docopt(__doc__, argv=argv)
- print(arguments)
- input_shape = eval(arguments['--input-shape'])
- tfrecord_filenames = arguments['<eval_tfrecords>']
- eval_dir = arguments['--eval-dir']
- batch_size = eval(arguments['--batch-size'])
- run_once = arguments['--run-once']
- eval_interval_secs = eval(arguments['--eval-interval-secs'])
- checkpoint_dir = arguments['<checkpoint_dir>']
- num_examples = eval(arguments['--num-examples'])
+ arguments = docopt(__doc__, argv=argv)
+ print(arguments)
+ input_shape = eval(arguments['--input-shape'])
+ tfrecord_filenames = arguments['<eval_tfrecords>']
+ eval_dir = arguments['--eval-dir']
+ batch_size = eval(arguments['--batch-size'])
+ run_once = arguments['--run-once']
+ eval_interval_secs = eval(arguments['--eval-interval-secs'])
+ checkpoint_dir = arguments['<checkpoint_dir>']
+ num_examples = eval(arguments['--num-examples'])
- create_directories_safe(eval_dir)
- with tf.Graph().as_default() as g:
+ create_directories_safe(eval_dir)
+ with tf.Graph().as_default() as g:
- # Get images and labels
- with tf.name_scope('input'):
- dataset = tf.contrib.data.TFRecordDataset(tfrecord_filenames)
- feature = {'train/data': tf.FixedLenFeature([], tf.string),
- 'train/label': tf.FixedLenFeature([], tf.int64)}
- my_example_parser = partial(
- example_parser, feature=feature, data_shape=input_shape)
- dataset = dataset.map(
- my_example_parser, num_threads=1, output_buffer_size=batch_size)
- dataset = dataset.batch(batch_size)
- images, labels = dataset.make_one_shot_iterator().get_next()
+ # Get images and labels
+ with tf.name_scope('input'):
+ dataset = tf.contrib.data.TFRecordDataset(tfrecord_filenames)
+ feature = {'train/data': tf.FixedLenFeature([], tf.string),
+ 'train/label': tf.FixedLenFeature([], tf.int64)}
+ my_example_parser = partial(
+ example_parser, feature=feature, data_shape=input_shape)
+ dataset = dataset.map(
+ my_example_parser, num_threads=1, output_buffer_size=batch_size)
+ dataset = dataset.batch(batch_size)
+ images, labels = dataset.make_one_shot_iterator().get_next()
- # Build a Graph that computes the logits predictions from the
- # inference model.
- logits = architecture(images, mode=tf.estimator.ModeKeys.EVAL)
+ # Build a Graph that computes the logits predictions from the
+ # inference model.
+ logits = architecture(images, mode=tf.estimator.ModeKeys.EVAL)
- # Calculate predictions.
- top_k_op = tf.nn.in_top_k(logits, labels, 1)
+ # Calculate predictions.
+ top_k_op = tf.nn.in_top_k(logits, labels, 1)
- saver = tf.train.Saver()
- # Build the summary operation based on the TF collection of Summaries.
- summary_op = tf.summary.merge_all()
- summary_writer = tf.summary.FileWriter(eval_dir, g)
- evaluated_file = os.path.join(eval_dir, 'evaluated')
+ saver = tf.train.Saver()
+ # Build the summary operation based on the TF collection of Summaries.
+ summary_op = tf.summary.merge_all()
+ summary_writer = tf.summary.FileWriter(eval_dir, g)
+ evaluated_file = os.path.join(eval_dir, 'evaluated')
- while True:
- evaluated_steps = []
- if os.path.exists(evaluated_file):
- with open(evaluated_file) as f:
- evaluated_steps = f.read().split()
- ckpt = tf.train.get_checkpoint_state(checkpoint_dir)
- if ckpt and ckpt.model_checkpoint_path:
- for path in ckpt.all_model_checkpoint_paths:
- global_step = get_global_step(path)
- if global_step not in evaluated_steps:
- ret_val = eval_once(saver, summary_writer, top_k_op, summary_op,
- path, global_step, num_examples,
- batch_size)
- if ret_val == 0:
- with open(evaluated_file, 'a') as f:
- f.write(global_step + '\n')
- if run_once:
- break
- time.sleep(eval_interval_secs)
+ while True:
+ evaluated_steps = []
+ if os.path.exists(evaluated_file):
+ with open(evaluated_file) as f:
+ evaluated_steps = f.read().split()
+ ckpt = tf.train.get_checkpoint_state(checkpoint_dir)
+ if ckpt and ckpt.model_checkpoint_path:
+ for path in ckpt.all_model_checkpoint_paths:
+ global_step = get_global_step(path)
+ if global_step not in evaluated_steps:
+ ret_val = eval_once(saver, summary_writer, top_k_op, summary_op,
+ path, global_step, num_examples,
+ batch_size)
+ if ret_val == 0:
+ with open(evaluated_file, 'a') as f:
+ f.write(global_step + '\n')
+ if run_once:
+ break
+ time.sleep(eval_interval_secs)
if __name__ == '__main__':
- main()
+ main()
diff --git a/bob/learn/tensorflow/script/train_generic.py b/bob/learn/tensorflow/script/train_generic.py
index 246b1c12..c757f45c 100644
--- a/bob/learn/tensorflow/script/train_generic.py
+++ b/bob/learn/tensorflow/script/train_generic.py
@@ -33,66 +33,66 @@ from bob.dap.base.util.hooks import LoggerHook
def main(argv=None):
- arguments = docopt(__doc__, argv=argv)
- print(arguments)
- input_shape = eval(arguments['--input-shape'])
- tfrecord_filenames = arguments['<train_tfrecords>']
- save_dir = arguments['--save-dir']
- epochs = eval(arguments['--epochs'])
- batch_size = eval(arguments['--batch-size'])
- capacity_samples = eval(arguments['--capacity-samples'])
- learning_rate = eval(arguments['--learning-rate'])
- log_frequency = eval(arguments['--log-frequency'])
+ arguments = docopt(__doc__, argv=argv)
+ print(arguments)
+ input_shape = eval(arguments['--input-shape'])
+ tfrecord_filenames = arguments['<train_tfrecords>']
+ save_dir = arguments['--save-dir']
+ epochs = eval(arguments['--epochs'])
+ batch_size = eval(arguments['--batch-size'])
+ capacity_samples = eval(arguments['--capacity-samples'])
+ learning_rate = eval(arguments['--learning-rate'])
+ log_frequency = eval(arguments['--log-frequency'])
- create_directories_safe(save_dir)
- with tf.Graph().as_default():
- global_step = tf.contrib.framework.get_or_create_global_step()
+ create_directories_safe(save_dir)
+ with tf.Graph().as_default():
+ global_step = tf.contrib.framework.get_or_create_global_step()
- # Get images and labels
- with tf.name_scope('input'):
- filename_queue = tf.train.string_input_producer(
- tfrecord_filenames, num_epochs=epochs, name="tfrecord_filenames")
+ # Get images and labels
+ with tf.name_scope('input'):
+ filename_queue = tf.train.string_input_producer(
+ tfrecord_filenames, num_epochs=epochs, name="tfrecord_filenames")
- image, label = read_and_decode(filename_queue, input_shape)
- images, labels = tf.train.shuffle_batch(
- [image, label], batch_size=batch_size,
- capacity=capacity_samples // batch_size,
- min_after_dequeue=int(capacity_samples // batch_size // 2),
- num_threads=1, name="shuffle_batch")
+ image, label = read_and_decode(filename_queue, input_shape)
+ images, labels = tf.train.shuffle_batch(
+ [image, label], batch_size=batch_size,
+ capacity=capacity_samples // batch_size,
+ min_after_dequeue=int(capacity_samples // batch_size // 2),
+ num_threads=1, name="shuffle_batch")
- # Build a Graph that computes the logits predictions from the
- # inference model.
- logits = architecture(images)
- tf.add_to_collection('logits', logits)
+ # Build a Graph that computes the logits predictions from the
+ # inference model.
+ logits = architecture(images)
+ tf.add_to_collection('logits', logits)
- # Calculate loss.
- predictor = tf.nn.sparse_softmax_cross_entropy_with_logits(
- logits=logits, labels=labels)
- loss = tf.reduce_mean(predictor)
- tf.summary.scalar('loss', loss)
+ # Calculate loss.
+ predictor = tf.nn.sparse_softmax_cross_entropy_with_logits(
+ logits=logits, labels=labels)
+ loss = tf.reduce_mean(predictor)
+ tf.summary.scalar('loss', loss)
- # Build a Graph that trains the model with one batch of examples and
- # updates the model parameters.
- optimizer = tf.train.GradientDescentOptimizer(learning_rate)
- train_op = optimizer.minimize(loss, global_step=global_step)
+ # Build a Graph that trains the model with one batch of examples and
+ # updates the model parameters.
+ optimizer = tf.train.GradientDescentOptimizer(learning_rate)
+ train_op = optimizer.minimize(loss, global_step=global_step)
- saver = tf.train.Saver(max_to_keep=10**5)
- scaffold = tf.train.Scaffold(saver=saver)
- with tf.train.MonitoredTrainingSession(
- checkpoint_dir=save_dir,
- scaffold=scaffold,
- hooks=[
- tf.train.CheckpointSaverHook(
- save_dir, save_secs=60 * 29, scaffold=scaffold),
- tf.train.NanTensorHook(loss),
- LoggerHook(loss, batch_size, log_frequency)],
- config=session_conf,
- save_checkpoint_secs=None,
- save_summaries_steps=100,
- ) as mon_sess:
- while not mon_sess.should_stop():
- mon_sess.run(train_op)
+ saver = tf.train.Saver(max_to_keep=10**5)
+ scaffold = tf.train.Scaffold(saver=saver)
+ with tf.train.MonitoredTrainingSession(
+ checkpoint_dir=save_dir,
+ scaffold=scaffold,
+ hooks=[
+ tf.train.CheckpointSaverHook(
+ save_dir, save_secs=60 * 29, scaffold=scaffold),
+ tf.train.NanTensorHook(loss),
+ LoggerHook(loss, batch_size, log_frequency)],
+ config=session_conf,
+ save_checkpoint_secs=None,
+ save_summaries_steps=100,
+ ) as mon_sess:
+ while not mon_sess.should_stop():
+ mon_sess.run(train_op)
if __name__ == '__main__':
- main()
+ main()
diff --git a/bob/learn/tensorflow/utils/eval.py b/bob/learn/tensorflow/utils/eval.py
index a431f8d2..ce20ed13 100644
--- a/bob/learn/tensorflow/utils/eval.py
+++ b/bob/learn/tensorflow/utils/eval.py
@@ -9,102 +9,102 @@ from datetime import datetime
def get_global_step(path):
- """Returns the global number associated with the model checkpoint path. The
- checkpoint must have been saved with the
- :any:`tf.train.MonitoredTrainingSession`.
-
- Parameters
- ----------
- path : str
- The path to model checkpoint, usually ckpt.model_checkpoint_path
-
- Returns
- -------
- global_step : str
- The global step number as a string.
- """
- # Assuming model_checkpoint_path looks something like:
- # /my-favorite-path/train/model.ckpt-0,
- # extract global_step from it.
- global_step = path.split('/')[-1].split('-')[-1]
- return global_step
+ """Returns the global number associated with the model checkpoint path. The
+ checkpoint must have been saved with the
+ :any:`tf.train.MonitoredTrainingSession`.
+
+ Parameters
+ ----------
+ path : str
+ The path to model checkpoint, usually ckpt.model_checkpoint_path
+
+ Returns
+ -------
+ global_step : str
+ The global step number as a string.
+ """
+ # Assuming model_checkpoint_path looks something like:
+ # /my-favorite-path/train/model.ckpt-0,
+ # extract global_step from it.
+ global_step = path.split('/')[-1].split('-')[-1]
+ return global_step
def eval_once(saver, summary_writer, prediction_op, summary_op,
model_checkpoint_path, global_step, num_examples, batch_size):
- """Run Eval once.
-
- Parameters
- ----------
- saver
- Saver.
- summary_writer
- Summary writer.
- prediction_op
- Prediction operator.
- summary_op
- Summary operator.
- model_checkpoint_path : str
- Path to the model checkpoint.
- global_step : str
- The global step.
- num_examples : int or None
- The number of examples to try.
- batch_size : int
- The size of evaluation batch.
-
- This function requires the ``from __future__ import division`` import.
-
- Returns
- -------
- int
- 0 for success, anything else for fail.
-
- """
- with tf.Session() as sess:
- sess.run(tf.local_variables_initializer())
- sess.run(tf.global_variables_initializer())
-
- if model_checkpoint_path:
- # Restores from checkpoint
- saver.restore(sess, model_checkpoint_path)
- else:
- print('No checkpoint file found')
- return -1
-
- # Start the queue runners.
- coord = tf.train.Coordinator()
- try:
- threads = []
- for qr in tf.get_collection(tf.GraphKeys.QUEUE_RUNNERS):
- threads.extend(qr.create_threads(sess, coord=coord, daemon=True,
- start=True))
-
- if num_examples is None:
- num_iter = float("inf")
- else:
- num_iter = int(math.ceil(num_examples / batch_size))
- true_count = 0 # Counts the number of correct predictions.
- total_sample_count = 0
- step = 0
- while step < num_iter and not coord.should_stop():
- predictions = sess.run([prediction_op])
- true_count += np.sum(predictions)
- total_sample_count += np.asarray(predictions).size
- step += 1
-
- # Compute precision @ 1.
- precision = true_count / total_sample_count
- print('%s: precision @ 1 = %.3f' % (datetime.now(), precision))
-
- summary = tf.Summary()
- summary.ParseFromString(sess.run(summary_op))
- summary.value.add(tag='Precision @ 1', simple_value=precision)
- summary_writer.add_summary(summary, global_step)
- return 0
- except Exception as e: # pylint: disable=broad-except
- coord.request_stop(e)
- return -1
- finally:
- coord.request_stop()
- coord.join(threads, stop_grace_period_secs=10)
+ """Run Eval once.
+
+ Parameters
+ ----------
+ saver
+ Saver.
+ summary_writer
+ Summary writer.
+ prediction_op
+ Prediction operator.
+ summary_op
+ Summary operator.
+ model_checkpoint_path : str
+ Path to the model checkpoint.
+ global_step : str
+ The global step.
+ num_examples : int or None
+ The number of examples to try.
+ batch_size : int
+ The size of evaluation batch.
+
+ This function requires the ``from __future__ import division`` import.
+
+ Returns
+ -------
+ int
+ 0 for success, anything else for fail.
+
+ """
+ with tf.Session() as sess:
+ sess.run(tf.local_variables_initializer())
+ sess.run(tf.global_variables_initializer())
+
+ if model_checkpoint_path:
+ # Restores from checkpoint
+ saver.restore(sess, model_checkpoint_path)
+ else:
+ print('No checkpoint file found')
+ return -1
+
+ # Start the queue runners.
+ coord = tf.train.Coordinator()
+ try:
+ threads = []
+ for qr in tf.get_collection(tf.GraphKeys.QUEUE_RUNNERS):
+ threads.extend(qr.create_threads(sess, coord=coord, daemon=True,
+ start=True))
+
+ if num_examples is None:
+ num_iter = float("inf")
+ else:
+ num_iter = int(math.ceil(num_examples / batch_size))
+ true_count = 0 # Counts the number of correct predictions.
+ total_sample_count = 0
+ step = 0
+ while step < num_iter and not coord.should_stop():
+ predictions = sess.run([prediction_op])
+ true_count += np.sum(predictions)
+ total_sample_count += np.asarray(predictions).size
+ step += 1
+
+ # Compute precision @ 1.
+ precision = true_count / total_sample_count
+ print('%s: precision @ 1 = %.3f' % (datetime.now(), precision))
+
+ summary = tf.Summary()
+ summary.ParseFromString(sess.run(summary_op))
+ summary.value.add(tag='Precision @ 1', simple_value=precision)
+ summary_writer.add_summary(summary, global_step)
+ return 0
+ except Exception as e: # pylint: disable=broad-except
+ coord.request_stop(e)
+ return -1
+ finally:
+ coord.request_stop()
+ coord.join(threads, stop_grace_period_secs=10)
diff --git a/bob/learn/tensorflow/utils/hooks.py b/bob/learn/tensorflow/utils/hooks.py
index fe15d519..1875e519 100644
--- a/bob/learn/tensorflow/utils/hooks.py
+++ b/bob/learn/tensorflow/utils/hooks.py
@@ -4,32 +4,32 @@ from datetime import datetime
class LoggerHook(tf.train.SessionRunHook):
- """Logs loss and runtime."""
-
- def __init__(self, loss, batch_size, log_frequency):
- self.loss = loss
- self.batch_size = batch_size
- self.log_frequency = log_frequency
-
- def begin(self):
- self._step = -1
- self._start_time = time.time()
-
- def before_run(self, run_context):
- self._step += 1
- return tf.train.SessionRunArgs(self.loss) # Asks for loss value.
-
- def after_run(self, run_context, run_values):
- if self._step % self.log_frequency == 0:
- current_time = time.time()
- duration = current_time - self._start_time
- self._start_time = current_time
-
- loss_value = run_values.results
- examples_per_sec = self.log_frequency * self.batch_size / duration
- sec_per_batch = float(duration / self.log_frequency)
-
- format_str = ('%s: step %d, loss = %.2f (%.1f examples/sec; %.3f '
- 'sec/batch)')
- print(format_str % (datetime.now(), self._step, loss_value,
- examples_per_sec, sec_per_batch))
+ """Logs loss and runtime."""
+
+ def __init__(self, loss, batch_size, log_frequency):
+ self.loss = loss
+ self.batch_size = batch_size
+ self.log_frequency = log_frequency
+
+ def begin(self):
+ self._step = -1
+ self._start_time = time.time()
+
+ def before_run(self, run_context):
+ self._step += 1
+ return tf.train.SessionRunArgs(self.loss) # Asks for loss value.
+
+ def after_run(self, run_context, run_values):
+ if self._step % self.log_frequency == 0:
+ current_time = time.time()
+ duration = current_time - self._start_time
+ self._start_time = current_time
+
+ loss_value = run_values.results
+ examples_per_sec = self.log_frequency * self.batch_size / duration
+ sec_per_batch = float(duration / self.log_frequency)
+
+ format_str = ('%s: step %d, loss = %.2f (%.1f examples/sec; %.3f '
+ 'sec/batch)')
+ print(format_str % (datetime.now(), self._step, loss_value,
+ examples_per_sec, sec_per_batch))
diff --git a/bob/learn/tensorflow/utils/tfrecords.py b/bob/learn/tensorflow/utils/tfrecords.py
index 309575a2..dcab80c8 100644
--- a/bob/learn/tensorflow/utils/tfrecords.py
+++ b/bob/learn/tensorflow/utils/tfrecords.py
@@ -2,24 +2,24 @@ import tensorflow as tf
def example_parser(serialized_example, feature, data_shape):
- """Parses a single tf.Example into image and label tensors."""
- # Decode the record read by the reader
- features = tf.parse_single_example(serialized_example, features=feature)
- # Convert the image data from string back to the numbers
- image = tf.decode_raw(features['train/data'], tf.float32)
- # Cast label data into int64
- label = tf.cast(features['train/label'], tf.int64)
- # Reshape image data into the original shape
- image = tf.reshape(image, data_shape)
- return image, label
+ """Parses a single tf.Example into image and label tensors."""
+ # Decode the record read by the reader
+ features = tf.parse_single_example(serialized_example, features=feature)
+ # Convert the image data from string back to the numbers
+ image = tf.decode_raw(features['train/data'], tf.float32)
+ # Cast label data into int64
+ label = tf.cast(features['train/label'], tf.int64)
+ # Reshape image data into the original shape
+ image = tf.reshape(image, data_shape)
+ return image, label
def read_and_decode(filename_queue, data_shape, feature=None):
- if feature is None:
- feature = {'train/data': tf.FixedLenFeature([], tf.string),
- 'train/label': tf.FixedLenFeature([], tf.int64)}
- # Define a reader and read the next record
- reader = tf.TFRecordReader()
- _, serialized_example = reader.read(filename_queue)
- return example_parser(serialized_example, feature, data_shape)
+ if feature is None:
+ feature = {'train/data': tf.FixedLenFeature([], tf.string),
+ 'train/label': tf.FixedLenFeature([], tf.int64)}
+ # Define a reader and read the next record
+ reader = tf.TFRecordReader()
+ _, serialized_example = reader.read(filename_queue)
+ return example_parser(serialized_example, feature, data_shape)
--
GitLab