Implemented automatic deletion of jobs when run using the --parallel option

bob/bio/base/test/test_scripts.py

@@ -57,6 +57,8 @@ def _verify(parameters, test_dir, sub_dir, ref_modifier="", score_modifier=('sco
       # assert that the values are OK
       assert numpy.allclose(d[0][:,3].astype(float), d[1][:,3].astype(float), 1e-5)
 
+      assert not os.path.exists(os.path.join(test_dir, 'submitted.sql3'))
+
   finally:
     shutil.rmtree(test_dir)
 
@@ -133,7 +135,9 @@ def test_verify_parallel():
       '-vs', 'test_parallel',
       '--temp-directory', test_dir,
       '--result-directory', test_dir,
-      '-g', 'bob.bio.base.grid.Grid(grid_type = "local", number_of_parallel_processes = 2, scheduler_sleep_time = 0.1)', '-G', test_database, '--run-local-scheduler', '--stop-on-failure',
+      '-g', 'bob.bio.base.grid.Grid(grid_type = "local", number_of_parallel_processes = 2, scheduler_sleep_time = 0.1)',
+      '-G', test_database, '--run-local-scheduler', '--stop-on-failure',
+      '-D', 'success',
       '--import', 'bob.io.image'
   ]
 

bob/bio/base/tools/command_line.py

@@ -127,6 +127,8 @@ def command_line_parser(description=__doc__, exclude_resources_from=[]):
       help = 'Starts the local scheduler after submitting the jobs to the local queue (by default, local jobs must be started by hand, e.g., using ./bin/jman --local -vv run-scheduler -x)')
   flag_group.add_argument('-N', '--nice', type=int, default=10,
       help = 'Runs the local scheduler with the given nice value')
+  flag_group.add_argument('-D', '--delete-jobs-finished-with-status', choices = ('all', 'failure', 'success'),
+      help = 'If selected, local scheduler jobs that finished with the given status are deleted from the --gridtk-database-file; otherwise the jobs remain in the database')
   flag_group.add_argument('-c', '--calibrate-scores', action='store_true',
       help = 'Performs score calibration after the scores are computed.')
   flag_group.add_argument('-z', '--zt-norm', action='store_true',

bob/bio/base/tools/grid.py

@@ -127,3 +127,9 @@ class GridSubmission:
     if failures:
       logger.error("The jobs with the following IDS did not finish successfully: '%s'.", ', '.join([str(f) for f in failures]))
       self.job_manager.report(job_ids = failures[:1], output=False)
+
+    # delete the jobs that we have added
+    if self.args.delete_jobs_finished_with_status is not None:
+      logger.info("Deleting jman jobs that we have added")
+      status = ('success', 'failure') if self.args.delete_jobs_finished_with_status == 'all' else (self.args.delete_jobs_finished_with_status,)
+      self.job_manager.delete(job_ids=self.submitted_job_ids, status=status)