Added possibility to query dependencies between jobs; Now, the job_manager can...

Added possibility to query dependencies between jobs; Now, the job_manager can also be used inside the grid; fixed some small issues.

gridtk/manager.py

@@ -54,9 +54,10 @@ class Job:
 
   def __init__(self, data, args, kwargs):
 
-    self.data = data
-    self.args = args
-    self.kwargs = kwargs
+    import copy
+    self.data = copy.deepcopy(data)
+    self.args = copy.deepcopy(args)
+    self.kwargs = copy.deepcopy(kwargs)
     if self.data.has_key('job-array tasks'):
       b = JOB_ARRAY_SPLIT.match(self.data['job-array tasks']).groupdict()
       self.array =  (int(b['m']), int(b['n']), int(b['s']))
@@ -79,6 +80,15 @@ class Job:
     else:
       return self.data['job_number']
 
+  def given_name(self):
+    """Returns the given name of the job, i.e., whatever was passed as name= to the contructor.
+    If no such name was given, self.name() is returned instead."""
+    if 'name' in self.kwargs:
+      return self.kwargs['name']
+    else:
+      return self.name()
+
+
   def is_array(self):
     """Determines if this job is an array or not."""
 
@@ -90,6 +100,10 @@ class Job:
 
     return self.array
 
+  def is_dependent_on(self, job_id):
+    """Checks if this job is dependent on the given job id."""
+    return 'deps' in self.kwargs and job_id in self.kwargs['deps']
+
   def age(self, short=True):
     """Returns a string representation indicating, approximately, how much time
     has ellapsed since the job was submitted. The input argument must be a
@@ -261,7 +275,7 @@ class Job:
     """Returns a string containing a short job description"""
 
     return "%s @%s (%s ago) %s  %s" % (self.name(),
-        self.queue(), self.age(short=False), self.kwargs['name'], ' '.join(self.args[0]))
+        self.queue(), self.age(short=False), self.given_name(), ' '.join(self.args[0]))
 
   def row(self, fmt, maxcmd=0):
     """Returns a string containing the job description suitable for a table."""
@@ -294,7 +308,7 @@ class JobManager:
   """The JobManager will submit and control the status of submitted jobs"""
 
   def __init__(self, statefile='submitted.db', context='grid'):
-    """Intializes this object with a state file and a method for qsub'bing.
+    """Initializes this object with a state file and a method for qsub'bing.
 
     Keyword parameters:
 

gridtk/script/jman.py

@@ -89,8 +89,8 @@ def delete(args):
           J.rm_stdout(verbose='  ', recurse = not args.keep_log_dir)
           J.rm_stderr(verbose='  ', recurse = not args.keep_log_dir)
         else:
-          J.rm_stdout()
-          J.rm_stderr()
+          J.rm_stdout(recurse = not args.keep_log_dir)
+          J.rm_stderr(recurse = not args.keep_log_dir)
       del jm[k]
       if args.verbose: print "Deleted job %s" % J
       else: print "Deleted job", J.name()

gridtk/setshell.py

 #!/usr/bin/env python
 # vim: set fileencoding=utf-8 :
 # Andre Anjos <andre.anjos@idiap.ch>
-# Wed 24 Aug 2011 09:20:40 CEST 
+# Wed 24 Aug 2011 09:20:40 CEST
 
 """Wrappers for Idiap's SETSHELL functionality
 """
@@ -14,7 +14,26 @@ import logging
 
 def environ(context):
   """Retrieves the environment for a particular SETSHELL context"""
-
+  if 'BASEDIRSETSHELL' not in os.environ:
+    # It seems that we are in a hostile environment
+    # try to source the Idiap-wide shell
+    idiap_source = "/idiap/resource/software/initfiles/shrc"
+    logging.debug("Sourcing: '%s'"%idiap_source)
+    try:
+      command = ['bash', '-c', 'source %s && env' % idiap_source]
+      pi = subprocess.Popen(command, stdout = subprocess.PIPE)
+    except OSError as e:
+      # occurs when the file is not executable or not found
+      raise OSError, "Error executing '%s': %s (%d)" % \
+          (' '.join(command), e.strerror, e.errno)
+
+    # overwrite the default environment
+    for line in pi.stdout:
+      (key, _, value) = line.partition("=")
+      os.environ[key.strip()] = value.strip()
+
+
+  assert 'BASEDIRSETSHELL' in os.environ
   BASEDIRSETSHELL = os.environ['BASEDIRSETSHELL']
   dosetshell = '%s/setshell/bin/dosetshell' % BASEDIRSETSHELL
 

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='gridtk',
-    version='0.3.1',
+    version='0.3.2',
     description='SGE Grid Submission and Monitoring Tools for Idiap',
 
     url='https://github.com/idiap/gridtk',