Added a job manager that executes jobs in parallel on the local machine; use...

Added a job manager that executes jobs in parallel on the local machine; use jman -l to enable; This feature is still under development and might be buggy.

gridtk/__init__.py

-from . import setshell
-from . import tools
-from . import manager
-from . import easy
+import setshell
+import tools
+import manager
+import local
+import easy

gridtk/manager.py

@@ -10,43 +10,12 @@ import os
 import time
 import gdbm, anydbm
 from cPickle import dumps, loads
-from .tools import qsub, qstat, qdel, logger
+from .tools import qsub, qstat, qdel, logger, try_get_contents, try_remove_files
 from .setshell import environ
 
 import re
 JOB_ARRAY_SPLIT = re.compile(r'^(?P<m>\d+)-(?P<n>\d+):(?P<s>\d+)$')
 
-def try_get_contents(filename):
-  """Loads contents out of a certain filename"""
-
-  try:
-    return open(filename, 'rt').read()
-  except OSError, e:
-    logger.warn("Could not find file '%s'" % filename)
-
-  return ''
-
-def try_remove_files(filename, recurse, verbose):
-  """Safely removes files from the filesystem"""
-
-  if isinstance(filename, (tuple, list)):
-    for k in filename:
-      if os.path.exists(k):
-        os.unlink(k)
-        if verbose: print verbose + ("removed `%s'" % k)
-      d = os.path.dirname(k)
-      if recurse and os.path.exists(d) and not os.listdir(d):
-        os.removedirs(d)
-        if verbose: print verbose + ("recursively removed `%s'" % d)
-
-  else:
-    if os.path.exists(filename):
-      os.unlink(filename)
-      if verbose: print verbose + ("removed `%s'" % filename)
-    d = os.path.dirname(filename)
-    if recurse and os.path.exists(d) and not os.listdir(d):
-      os.removedirs(d)
-      if verbose: print verbose + ("recursively removed `%s'" % d)
 
 class Job:
   """The job class describes a job"""
@@ -68,6 +37,9 @@ class Job:
 
     return int(self.data['job_number'])
 
+  def command_line(self):
+    return self.args + self.kwargs
+
   def name(self, instance=None):
     """Returns my own numerical id"""
 
@@ -80,7 +52,7 @@ class Job:
       return self.data['job_number']
 
   def given_name(self):
-    """Returns the given name of the job, i.e., whatever was passed as name= to the contructor.
+    """Returns the given name of the job, i.e., whatever was passed as name= to the constructor.
     If no such name was given, self.name() is returned instead."""
     if 'name' in self.kwargs:
       return self.kwargs['name']
@@ -455,10 +427,10 @@ class JobManager:
         if status:
           success.append(self.job[k])
           del self.job[k]
-          logger.debug("Job %d completed successfuly" % k)
+          logger.debug("Job %d completed successfully" % k)
         else:
           error.append(self.job[k])
           del self.job[k]
-          logger.debug("Job %d probably did not complete successfuly" % k)
+          logger.debug("Job %d probably did not complete successfully" % k)
 
     return success, error

gridtk/script/jman.py

@@ -20,7 +20,7 @@ from cPickle import dumps
 
 import argparse
 
-from ..manager import JobManager
+from .. import manager, local
 from ..tools import make_shell, random_logdir, logger
 
 def setup(args):
@@ -28,7 +28,10 @@ def setup(args):
 
   kwargs = {}
   if args.db: kwargs['statefile'] = args.db
-  jm = JobManager(**kwargs)
+  if args.local:
+    jm = local.JobManager(**kwargs)
+  else:
+    jm = manager.JobManager(**kwargs)
 
   # set-up logging
   if args.debug:
@@ -50,7 +53,7 @@ def save_jobs(j, name):
 
   db = anydbm.open(name, 'c')
   for k in j:
-    ki = int(k['job_number'])
+    ki = int(k.id())
     db[dumps(ki)] = dumps(k)
 
 def refresh(args):
@@ -150,7 +153,7 @@ def submit(args):
   if args.python or os.path.splitext(args.job[0])[1] in ('.py',):
     args.job = make_shell(sys.executable, args.job)
 
-  args.stdout, args.stderr = get_logdirs(args.stdout, args.stderr, args.logbase)
+  args.stdout, args.stderr = get_logdirs(args.stdout, args.stderr, args.logbase) if not args.local else (None, None)
 
   jm = setup(args)
   kwargs = {
@@ -202,7 +205,7 @@ def explain(args):
     first_time = False
     J = jm[k[0]]
     print "Job", J
-    print "Command line:", J.args, J.kwargs
+    print "Command line:", J.command_line()
     if args.verbose:
       print "%s stdout (%s)" % (J.name(k[1]), J.stdout_filename(k[1]))
       print J.stdout(k[1])
@@ -244,6 +247,13 @@ def resubmit(args):
       del fromjm[k]
       print '  deleted job %s from database' % O.name()
 
+def execute(args):
+  """Executes the collected jobs on the local machine."""
+  if not args.local:
+    raise ValueError("The execute command can only be used with the '--local' command line option")
+  jm = setup(args)
+  jm.run(parallel_jobs=args.jobs)
+
 class AliasedSubParsersAction(argparse._SubParsersAction):
   """Hack taken from https://gist.github.com/471779 to allow aliases in
   argparse for python 2.x (this has been implemented on python 3.2)
@@ -294,6 +304,9 @@ def main():
       action='store_true', help='prints out lots of debugging information')
   parser.add_argument('-V', '--version', action='version',
       version='GridTk version %s' % __version__)
+
+  parser.add_argument('-l', '--local', action='store_true',
+        help = 'Uses the local job manager instead of the SGE one.')
   cmdparser = parser.add_subparsers(title='commands', help='commands accepted by %(prog)s')
 
   # subcommand 'list'
@@ -357,6 +370,12 @@ def main():
   subparser.add_argument('job', metavar='command', nargs=argparse.REMAINDER)
   subparser.set_defaults(func=submit)
 
+  execute_parser = cmdparser.add_parser('execute', aliases=['exe', 'x'],
+      help='Executes the registered jobs on the local machine; only valid in combination with the \'--local\' option.')
+  execute_parser.add_argument('db', metavar='DATABASE', help='replace the default database to be refreshed by one provided by you; this option is only required if you are running outside the directory where you originally submitted the jobs from or if you have altered manually the location of the JobManager database', nargs='?')
+  execute_parser.add_argument('-j', '--jobs', '--parallel-jobs', type=int, default=1, metavar='jobs', help='Select the number of parallel jobs that you want to execute locally')
+  execute_parser.set_defaults(func=execute)
+
   # subcommand 'resubmit'
   resubparser = cmdparser.add_parser('resubmit', aliases=['resub', 're'],
       help='resubmits all jobs in a given database, exactly like they were submitted the first time')

gridtk/tools.py

@@ -48,6 +48,38 @@ def makedirs_safe(fulldir):
     if exc.errno == errno.EEXIST: pass
     else: raise
 
+def try_get_contents(filename):
+  """Loads contents out of a certain filename"""
+
+  try:
+    return open(filename, 'rt').read()
+  except OSError, e:
+    logger.warn("Could not find file '%s'" % filename)
+
+  return ''
+
+def try_remove_files(filename, recurse, verbose):
+  """Safely removes files from the filesystem"""
+
+  if isinstance(filename, (tuple, list)):
+    for k in filename:
+      if os.path.exists(k):
+        os.unlink(k)
+        if verbose: print verbose + ("removed `%s'" % k)
+      d = os.path.dirname(k)
+      if recurse and os.path.exists(d) and not os.listdir(d):
+        os.removedirs(d)
+        if verbose: print verbose + ("recursively removed `%s'" % d)
+
+  else:
+    if os.path.exists(filename):
+      os.unlink(filename)
+      if verbose: print verbose + ("removed `%s'" % filename)
+    d = os.path.dirname(filename)
+    if recurse and os.path.exists(d) and not os.listdir(d):
+      os.removedirs(d)
+      if verbose: print verbose + ("recursively removed `%s'" % d)
+
 def qsub(command, queue=None, cwd=True, name=None, deps=[], stdout='',
     stderr='', env=[], array=None, context='grid', hostname=None,
     mem=None, memfree=None, hvmem=None, pe_opt=None, io_big=False):