From 2f1d1a974426840257f9f6582435969a0d95d5af Mon Sep 17 00:00:00 2001
From: Manuel Gunther <siebenkopf@googlemail.com>
Date: Fri, 18 Dec 2015 12:27:46 -0700
Subject: [PATCH] Implemented automatic deletion of jobs when run using the
--parallel option
---
bob/bio/base/test/test_scripts.py | 6 +++++-
bob/bio/base/tools/command_line.py | 2 ++
bob/bio/base/tools/grid.py | 6 ++++++
3 files changed, 13 insertions(+), 1 deletion(-)
diff --git a/bob/bio/base/test/test_scripts.py b/bob/bio/base/test/test_scripts.py
index 3c8ede47..127f2e6e 100644
--- a/bob/bio/base/test/test_scripts.py
+++ b/bob/bio/base/test/test_scripts.py
@@ -57,6 +57,8 @@ def _verify(parameters, test_dir, sub_dir, ref_modifier="", score_modifier=('sco
# assert that the values are OK
assert numpy.allclose(d[0][:,3].astype(float), d[1][:,3].astype(float), 1e-5)
+ assert not os.path.exists(os.path.join(test_dir, 'submitted.sql3'))
+
finally:
shutil.rmtree(test_dir)
@@ -133,7 +135,9 @@ def test_verify_parallel():
'-vs', 'test_parallel',
'--temp-directory', test_dir,
'--result-directory', test_dir,
- '-g', 'bob.bio.base.grid.Grid(grid_type = "local", number_of_parallel_processes = 2, scheduler_sleep_time = 0.1)', '-G', test_database, '--run-local-scheduler', '--stop-on-failure',
+ '-g', 'bob.bio.base.grid.Grid(grid_type = "local", number_of_parallel_processes = 2, scheduler_sleep_time = 0.1)',
+ '-G', test_database, '--run-local-scheduler', '--stop-on-failure',
+ '-D', 'success',
'--import', 'bob.io.image'
]
diff --git a/bob/bio/base/tools/command_line.py b/bob/bio/base/tools/command_line.py
index 1f817ef2..6a991baa 100644
--- a/bob/bio/base/tools/command_line.py
+++ b/bob/bio/base/tools/command_line.py
@@ -127,6 +127,8 @@ def command_line_parser(description=__doc__, exclude_resources_from=[]):
help = 'Starts the local scheduler after submitting the jobs to the local queue (by default, local jobs must be started by hand, e.g., using ./bin/jman --local -vv run-scheduler -x)')
flag_group.add_argument('-N', '--nice', type=int, default=10,
help = 'Runs the local scheduler with the given nice value')
+ flag_group.add_argument('-D', '--delete-jobs-finished-with-status', choices = ('all', 'failure', 'success'),
+ help = 'If selected, local scheduler jobs that finished with the given status are deleted from the --gridtk-database-file; otherwise the jobs remain in the database')
flag_group.add_argument('-c', '--calibrate-scores', action='store_true',
help = 'Performs score calibration after the scores are computed.')
flag_group.add_argument('-z', '--zt-norm', action='store_true',
diff --git a/bob/bio/base/tools/grid.py b/bob/bio/base/tools/grid.py
index f9f1620c..8a536d12 100644
--- a/bob/bio/base/tools/grid.py
+++ b/bob/bio/base/tools/grid.py
@@ -127,3 +127,9 @@ class GridSubmission:
if failures:
logger.error("The jobs with the following IDS did not finish successfully: '%s'.", ', '.join([str(f) for f in failures]))
self.job_manager.report(job_ids = failures[:1], output=False)
+
+ # delete the jobs that we have added
+ if self.args.delete_jobs_finished_with_status is not None:
+ logger.info("Deleting jman jobs that we have added")
+ status = ('success', 'failure') if self.args.delete_jobs_finished_with_status == 'all' else (self.args.delete_jobs_finished_with_status,)
+ self.job_manager.delete(job_ids=self.submitted_job_ids, status=status)
--
GitLab