Skip to content
Snippets Groups Projects

Changes to the biogenerator

Merged Amir MOHAMMADI requested to merge predict into master
All threads resolved!
1 file
+ 16
6
Compare changes
  • Side-by-side
  • Inline
@@ -140,6 +140,14 @@ def make_output_path(output_dir, key):
@@ -140,6 +140,14 @@ def make_output_path(output_dir, key):
return os.path.join(output_dir, key + '.hdf5')
return os.path.join(output_dir, key + '.hdf5')
 
def non_existing_files(paths, force=False):
 
if force:
 
return range(len(paths))
 
for i, path in enumerate(paths):
 
if not os.path.isfile(path):
 
yield i
 
 
def save_predictions(pool, output_dir, key, pred_buffer):
def save_predictions(pool, output_dir, key, pred_buffer):
outpath = make_output_path(output_dir, key)
outpath = make_output_path(output_dir, key)
create_directories_safe(os.path.dirname(outpath))
create_directories_safe(os.path.dirname(outpath))
@@ -172,9 +180,6 @@ def main(argv=None):
@@ -172,9 +180,6 @@ def main(argv=None):
hooks = getattr(config, 'hooks', None)
hooks = getattr(config, 'hooks', None)
load_data = getattr(config, 'load_data', None)
load_data = getattr(config, 'load_data', None)
# TODO(amir): implement force and pre-filtering
raise ValueError("This script is not fully implemented yet!")
# Sets-up logging
# Sets-up logging
set_verbosity_level(logger, verbosity)
set_verbosity_level(logger, verbosity)
@@ -190,6 +195,12 @@ def main(argv=None):
@@ -190,6 +195,12 @@ def main(argv=None):
start, end = indices(biofiles, number_of_parallel_jobs)
start, end = indices(biofiles, number_of_parallel_jobs)
biofiles = biofiles[start:end]
biofiles = biofiles[start:end]
 
# filter the existing files
 
paths = (make_output_path(output_dir, f.make_path("", ""))
 
for f in biofiles)
 
indexes = non_existing_files(paths, force)
 
biofiles = (biofiles[i] for i in indexes)
 
generator = BioGenerator(
generator = BioGenerator(
database, biofiles, load_data=load_data,
database, biofiles, load_data=load_data,
multiple_samples=multiple_samples)
multiple_samples=multiple_samples)
@@ -218,9 +229,8 @@ def main(argv=None):
@@ -218,9 +229,8 @@ def main(argv=None):
else:
else:
save_predictions(pool, output_dir, last_key, pred_buffer)
save_predictions(pool, output_dir, last_key, pred_buffer)
last_key = key
last_key = key
# else below is for the for loop
# save the final returned key as well:
else:
save_predictions(pool, output_dir, key, pred_buffer)
save_predictions(pool, output_dir, key, pred_buffer)
finally:
finally:
pool.close()
pool.close()
pool.join()
pool.join()
Loading