Commit 33013f70 authored by Manuel Günther's avatar Manuel Günther

Added a job manager that executes jobs in parallel on the local machine; use...

Added a job manager that executes jobs in parallel on the local machine; use jman -l to enable; This feature is still under development and might be buggy.
parent aa20dcdb
from . import setshell
from . import tools
from . import manager
from . import easy
import setshell
import tools
import manager
import local
import easy
This diff is collapsed.
......@@ -10,43 +10,12 @@ import os
import time
import gdbm, anydbm
from cPickle import dumps, loads
from .tools import qsub, qstat, qdel, logger
from .tools import qsub, qstat, qdel, logger, try_get_contents, try_remove_files
from .setshell import environ
import re
JOB_ARRAY_SPLIT = re.compile(r'^(?P<m>\d+)-(?P<n>\d+):(?P<s>\d+)$')
def try_get_contents(filename):
"""Loads contents out of a certain filename"""
try:
return open(filename, 'rt').read()
except OSError, e:
logger.warn("Could not find file '%s'" % filename)
return ''
def try_remove_files(filename, recurse, verbose):
"""Safely removes files from the filesystem"""
if isinstance(filename, (tuple, list)):
for k in filename:
if os.path.exists(k):
os.unlink(k)
if verbose: print verbose + ("removed `%s'" % k)
d = os.path.dirname(k)
if recurse and os.path.exists(d) and not os.listdir(d):
os.removedirs(d)
if verbose: print verbose + ("recursively removed `%s'" % d)
else:
if os.path.exists(filename):
os.unlink(filename)
if verbose: print verbose + ("removed `%s'" % filename)
d = os.path.dirname(filename)
if recurse and os.path.exists(d) and not os.listdir(d):
os.removedirs(d)
if verbose: print verbose + ("recursively removed `%s'" % d)
class Job:
"""The job class describes a job"""
......@@ -68,6 +37,9 @@ class Job:
return int(self.data['job_number'])
def command_line(self):
return self.args + self.kwargs
def name(self, instance=None):
"""Returns my own numerical id"""
......@@ -80,7 +52,7 @@ class Job:
return self.data['job_number']
def given_name(self):
"""Returns the given name of the job, i.e., whatever was passed as name= to the contructor.
"""Returns the given name of the job, i.e., whatever was passed as name= to the constructor.
If no such name was given, self.name() is returned instead."""
if 'name' in self.kwargs:
return self.kwargs['name']
......@@ -455,10 +427,10 @@ class JobManager:
if status:
success.append(self.job[k])
del self.job[k]
logger.debug("Job %d completed successfuly" % k)
logger.debug("Job %d completed successfully" % k)
else:
error.append(self.job[k])
del self.job[k]
logger.debug("Job %d probably did not complete successfuly" % k)
logger.debug("Job %d probably did not complete successfully" % k)
return success, error
......@@ -20,7 +20,7 @@ from cPickle import dumps
import argparse
from ..manager import JobManager
from .. import manager, local
from ..tools import make_shell, random_logdir, logger
def setup(args):
......@@ -28,7 +28,10 @@ def setup(args):
kwargs = {}
if args.db: kwargs['statefile'] = args.db
jm = JobManager(**kwargs)
if args.local:
jm = local.JobManager(**kwargs)
else:
jm = manager.JobManager(**kwargs)
# set-up logging
if args.debug:
......@@ -50,7 +53,7 @@ def save_jobs(j, name):
db = anydbm.open(name, 'c')
for k in j:
ki = int(k['job_number'])
ki = int(k.id())
db[dumps(ki)] = dumps(k)
def refresh(args):
......@@ -150,7 +153,7 @@ def submit(args):
if args.python or os.path.splitext(args.job[0])[1] in ('.py',):
args.job = make_shell(sys.executable, args.job)
args.stdout, args.stderr = get_logdirs(args.stdout, args.stderr, args.logbase)
args.stdout, args.stderr = get_logdirs(args.stdout, args.stderr, args.logbase) if not args.local else (None, None)
jm = setup(args)
kwargs = {
......@@ -202,7 +205,7 @@ def explain(args):
first_time = False
J = jm[k[0]]
print "Job", J
print "Command line:", J.args, J.kwargs
print "Command line:", J.command_line()
if args.verbose:
print "%s stdout (%s)" % (J.name(k[1]), J.stdout_filename(k[1]))
print J.stdout(k[1])
......@@ -244,6 +247,13 @@ def resubmit(args):
del fromjm[k]
print ' deleted job %s from database' % O.name()
def execute(args):
"""Executes the collected jobs on the local machine."""
if not args.local:
raise ValueError("The execute command can only be used with the '--local' command line option")
jm = setup(args)
jm.run(parallel_jobs=args.jobs)
class AliasedSubParsersAction(argparse._SubParsersAction):
"""Hack taken from https://gist.github.com/471779 to allow aliases in
argparse for python 2.x (this has been implemented on python 3.2)
......@@ -294,6 +304,9 @@ def main():
action='store_true', help='prints out lots of debugging information')
parser.add_argument('-V', '--version', action='version',
version='GridTk version %s' % __version__)
parser.add_argument('-l', '--local', action='store_true',
help = 'Uses the local job manager instead of the SGE one.')
cmdparser = parser.add_subparsers(title='commands', help='commands accepted by %(prog)s')
# subcommand 'list'
......@@ -357,6 +370,12 @@ def main():
subparser.add_argument('job', metavar='command', nargs=argparse.REMAINDER)
subparser.set_defaults(func=submit)
execute_parser = cmdparser.add_parser('execute', aliases=['exe', 'x'],
help='Executes the registered jobs on the local machine; only valid in combination with the \'--local\' option.')
execute_parser.add_argument('db', metavar='DATABASE', help='replace the default database to be refreshed by one provided by you; this option is only required if you are running outside the directory where you originally submitted the jobs from or if you have altered manually the location of the JobManager database', nargs='?')
execute_parser.add_argument('-j', '--jobs', '--parallel-jobs', type=int, default=1, metavar='jobs', help='Select the number of parallel jobs that you want to execute locally')
execute_parser.set_defaults(func=execute)
# subcommand 'resubmit'
resubparser = cmdparser.add_parser('resubmit', aliases=['resub', 're'],
help='resubmits all jobs in a given database, exactly like they were submitted the first time')
......
......@@ -48,6 +48,38 @@ def makedirs_safe(fulldir):
if exc.errno == errno.EEXIST: pass
else: raise
def try_get_contents(filename):
"""Loads contents out of a certain filename"""
try:
return open(filename, 'rt').read()
except OSError, e:
logger.warn("Could not find file '%s'" % filename)
return ''
def try_remove_files(filename, recurse, verbose):
"""Safely removes files from the filesystem"""
if isinstance(filename, (tuple, list)):
for k in filename:
if os.path.exists(k):
os.unlink(k)
if verbose: print verbose + ("removed `%s'" % k)
d = os.path.dirname(k)
if recurse and os.path.exists(d) and not os.listdir(d):
os.removedirs(d)
if verbose: print verbose + ("recursively removed `%s'" % d)
else:
if os.path.exists(filename):
os.unlink(filename)
if verbose: print verbose + ("removed `%s'" % filename)
d = os.path.dirname(filename)
if recurse and os.path.exists(d) and not os.listdir(d):
os.removedirs(d)
if verbose: print verbose + ("recursively removed `%s'" % d)
def qsub(command, queue=None, cwd=True, name=None, deps=[], stdout='',
stderr='', env=[], array=None, context='grid', hostname=None,
mem=None, memfree=None, hvmem=None, pe_opt=None, io_big=False):
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment