Skip to content
Snippets Groups Projects
Commit 0e818979 authored by Tiago de Freitas Pereira's avatar Tiago de Freitas Pereira
Browse files

Merge branch 'predict' into 'master'

biopredict: Handle number of parallel jobs correctly

See merge request !34
parents a102df67 caed652a
Branches
Tags
1 merge request!34biopredict: Handle number of parallel jobs correctly
Pipeline #
import logging
logging.getLogger("tensorflow").setLevel(logging.WARNING)
def get_config(): def get_config():
""" """
Returns a string containing the configuration information. Returns a string containing the configuration information.
""" """
import bob.extension import bob.extension
return bob.extension.get_config(__name__) return bob.extension.get_config(__name__)
\ No newline at end of file
...@@ -196,7 +196,9 @@ def main(argv=None): ...@@ -196,7 +196,9 @@ def main(argv=None):
assert len(biofiles), "biofiles are empty!" assert len(biofiles), "biofiles are empty!"
logger.info("number_of_parallel_jobs: %d", number_of_parallel_jobs) if number_of_parallel_jobs is not None:
number_of_parallel_jobs = int(number_of_parallel_jobs)
logger.info("number_of_parallel_jobs: %d", number_of_parallel_jobs)
if number_of_parallel_jobs > 1: if number_of_parallel_jobs > 1:
start, end = indices(biofiles, number_of_parallel_jobs) start, end = indices(biofiles, number_of_parallel_jobs)
biofiles = biofiles[start:end] biofiles = biofiles[start:end]
......
...@@ -119,7 +119,7 @@ def _eval(tmpdir, model_dir, dummy_tfrecord): ...@@ -119,7 +119,7 @@ def _eval(tmpdir, model_dir, dummy_tfrecord):
eval_generic([config_path]) eval_generic([config_path])
def test_eval_once(): def test_eval():
tmpdir = mkdtemp(prefix='bob_') tmpdir = mkdtemp(prefix='bob_')
try: try:
model_dir = os.path.join(tmpdir, 'model_dir') model_dir = os.path.join(tmpdir, 'model_dir')
...@@ -137,7 +137,7 @@ def test_eval_once(): ...@@ -137,7 +137,7 @@ def test_eval_once():
evaluated_path = os.path.join(eval_dir, 'evaluated') evaluated_path = os.path.join(eval_dir, 'evaluated')
assert os.path.exists(evaluated_path), evaluated_path assert os.path.exists(evaluated_path), evaluated_path
with open(evaluated_path) as f: with open(evaluated_path) as f:
doc = f.read() doc = f.read()
assert '1' in doc, doc assert '1' in doc, doc
assert '200' in doc, doc assert '200' in doc, doc
......
'''Helps training reproducible networks.
'''
import os import os
import random as rn
import numpy as np import numpy as np
import tensorflow as tf import tensorflow as tf
import random as rn
# from tensorflow.contrib import keras
def set_seed(seed=0, python_hash_seed=0, log_device_placement=False):
# reproducible networks """Sets the seeds in python, numpy, and tensorflow in order to help
# The below is necessary in Python 3.2.3 onwards to training reproducible networks.
# have reproducible behavior for certain hash-based operations.
# See these references for further details: Parameters
# https://docs.python.org/3.4/using/cmdline.html#envvar-PYTHONHASHSEED ----------
# https://github.com/fchollet/keras/issues/2280#issuecomment-306959926 seed : :obj:`int`, optional
os.environ['PYTHONHASHSEED'] = '0' The seed to set.
python_hash_seed : :obj:`int`, optional
# The below is necessary for starting Numpy generated random numbers https://docs.python.org/3.4/using/cmdline.html#envvar-PYTHONHASHSEED
# in a well-defined initial state. log_device_placement : :obj:`bool`, optional
np.random.seed(42) Optionally, log device placement of tensorflow variables.
# The below is necessary for starting core Python generated random numbers Returns
# in a well-defined state. -------
rn.seed(12345) :any:`tf.ConfigProto`
Session config.
# Force TensorFlow to use single thread. :any:`tf.estimator.RunConfig`
# Multiple threads are a potential source of A run config to help training estimators.
# non-reproducible results.
# For further details, see: Notes
# https://stackoverflow.com/questions/42022950/which-seeds-have-to-be-set-where-to-realize-100-reproducibility-of-training-res -----
session_config = tf.ConfigProto(intra_op_parallelism_threads=1, This functions return a list and its length might change. Please use
inter_op_parallelism_threads=1, indices to select one of returned values. For example
log_device_placement=True) ``sess_config, run_config = set_seed()[:2]``.
"""
# The below tf.set_random_seed() will make random number generation # reproducible networks
# in the TensorFlow backend have a well-defined initial state. # The below is necessary in Python 3.2.3 onwards to
# For further details, see: # have reproducible behavior for certain hash-based operations.
# https://www.tensorflow.org/api_docs/python/tf/set_random_seed # See these references for further details:
tf_random_seed = 1234 # https://docs.python.org/3.4/using/cmdline.html#envvar-PYTHONHASHSEED
tf.set_random_seed(tf_random_seed) # https://github.com/fchollet/keras/issues/2280#issuecomment-306959926
# sess = tf.Session(graph=tf.get_default_graph(), config=session_config) os.environ['PYTHONHASHSEED'] = '{}'.format(python_hash_seed)
# keras.backend.set_session(sess)
# The below is necessary for starting Numpy generated random numbers
run_config = tf.estimator.RunConfig() # in a well-defined initial state.
run_config = run_config.replace(session_config=session_config) np.random.seed(seed)
run_config = run_config.replace(tf_random_seed=tf_random_seed)
# The below is necessary for starting core Python generated random numbers
# in a well-defined state.
rn.seed(seed)
# Force TensorFlow to use single thread.
# Multiple threads are a potential source of
# non-reproducible results.
# For further details, see:
# https://stackoverflow.com/questions/42022950/which-seeds-have-to-be-set-where-to-realize-100-reproducibility-of-training-res
session_config = tf.ConfigProto(intra_op_parallelism_threads=1,
inter_op_parallelism_threads=1,
log_device_placement=log_device_placement)
# The below tf.set_random_seed() will make random number generation
# in the TensorFlow backend have a well-defined initial state.
# For further details, see:
# https://www.tensorflow.org/api_docs/python/tf/set_random_seed
tf.set_random_seed(seed)
# sess = tf.Session(graph=tf.get_default_graph(), config=session_config)
# keras.backend.set_session(sess)
run_config = tf.estimator.RunConfig()
run_config = run_config.replace(session_config=session_config)
run_config = run_config.replace(tf_random_seed=seed)
return [session_config, run_config, None, None, None]
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment