Commit 2004a114 authored by André Anjos's avatar André Anjos 💬

Smarter job management

parent 3646024f
......@@ -99,7 +99,11 @@ def delete(args):
print "Ignored job %d (not found on manager)" % k
def submit(args):
"""Normal submission"""
"""Submission command"""
# automatically set interpreter if required
if args.python or os.path.splitext(args.job)[1] in ('.py',):
args.job = make_shell(sys.executable, args.job)
if args.stdout is None: args.stdout = os.path.join('logs', random_logdir())
if args.stderr is None: args.stderr = args.stdout
......@@ -115,16 +119,22 @@ def submit(args):
'env': args.env,
'array': args.array,
}
if args.dry_run:
print 'Would submit:', args.job, 'to', args.qname, 'with',
print 'array:', args.array,
print 'cwd:', args.cwd,
print 'deps:', args.deps,
print 'stdout:', args.stdout,
print 'stderr:', args.stderr,
print 'env:', args.env
return
# if you get here, submit the job
job = jm.submit(args.job, **kwargs)
if args.verbose: print 'Submitted', job
else: print 'Job', job.name(), 'submitted'
def pysubmit(args):
"""Wrapped submission"""
args.job = make_shell(sys.executable, args.job)
submit(args)
def explain(args):
"""Explain action"""
......@@ -186,25 +196,6 @@ def resubmit(args):
del fromjm[k]
print ' deleted job %s from database' % O.name()
def add_submission_options(parser):
"""Adds standard submission options to a given parser"""
parser.add_argument('-q', '--queue', metavar='QNAME',
dest='qname', default='all.q', help='the name of the SGE queue to submit the job to (defaults to "%(default)s")')
#this is ON by default as it helps job management
#parser.add_argument('-c', '--cwd', default=False, action='store_true',
# dest='cwd', help='Makes SGE switch to the current working directory before executing the job')
parser.add_argument('-n', '--name', dest='name', help='Sets the jobname')
parser.add_argument('-x', '--dependencies', '--deps', dest='deps', type=int,
default=[], metavar='ID', nargs='*', help='set job dependencies by giving this option an a list of job identifiers separated by spaces')
parser.add_argument('-o', '--stdout', '--out', metavar='DIR', dest='stdout', help='Set the standard output of the job to be placed in the given directory - relative paths are interpreted according to the currently working directory (defaults to a randomly generated hashed directory structure)')
parser.add_argument('-e', '--stderr', '--err', metavar='DIR', dest='stderr', help='Set the standard error of the job to be placed in the given directory - relative paths are interpreted according to the currently working directory (defaults to what stdout will be set to)')
parser.add_argument('-s', '--environment', '--env', metavar='KEY=VALUE',
dest='env', nargs='*', default=[],
help='Passes specific environment variables to the job')
parser.add_argument('-t', '--array', '--parametric', metavar='n[-m[:s]]',
dest='array', help='Creates a parametric (array) job. You must specify the starting range "n" (>=1), the stopping (inclusive) range "m" and the step "s". Read the qsub command man page for details')
class AliasedSubParsersAction(argparse._SubParsersAction):
"""Hack taken from https://gist.github.com/471779 to allow aliases in
argparse for python 2.x (this has been implemented on python 3.2)
......@@ -284,17 +275,28 @@ def main():
# subcommand 'submit'
subparser = cmdparser.add_parser('submit', aliases=['sub'],
help='submits self-contained jobs to the SGE queue and logs them in a private database')
add_submission_options(subparser)
parser.add_argument('-q', '--queue', metavar='QNAME',
dest='qname', default='all.q', help='the name of the SGE queue to submit the job to (defaults to "%(default)s")')
#this is ON by default as it helps job management
#parser.add_argument('-c', '--cwd', default=False, action='store_true',
# dest='cwd', help='Makes SGE switch to the current working directory before executing the job')
parser.add_argument('-n', '--name', dest='name', help='Sets the jobname')
parser.add_argument('-x', '--dependencies', '--deps', dest='deps', type=int,
default=[], metavar='ID', nargs='*', help='set job dependencies by giving this option an a list of job identifiers separated by spaces')
parser.add_argument('-o', '--stdout', '--out', metavar='DIR', dest='stdout', help='Set the standard output of the job to be placed in the given directory - relative paths are interpreted according to the currently working directory (defaults to a randomly generated hashed directory structure)')
parser.add_argument('-e', '--stderr', '--err', metavar='DIR', dest='stderr', help='Set the standard error of the job to be placed in the given directory - relative paths are interpreted according to the currently working directory (defaults to what stdout will be set to)')
parser.add_argument('-s', '--environment', '--env', metavar='KEY=VALUE',
dest='env', nargs='*', default=[],
help='Passes specific environment variables to the job')
parser.add_argument('-t', '--array', '--parametric', metavar='n[-m[:s]]',
dest='array', help='Creates a parametric (array) job. You must specify the starting range "n" (>=1), the stopping (inclusive) range "m" and the step "s". Read the qsub command man page for details')
parser.add_argument('-p', '--py', '--python', dest='python', default=False,
action='store_true', help='Wrap execution of your command using the current python interpreter')
parser.add_argument('-z', '--dry-run', dest='dry_run', default=False,
action='store_true', help='Do not really submit anything, just print out what would submit in this case')
subparser.set_defaults(func=submit)
subparser.add_argument('job', metavar='command', nargs=argparse.REMAINDER)
# subcommand 'pysubmit'
pysubparser = cmdparser.add_parser('pysubmit', aliases=['pysub', 'python'],
help='submits a job that will be executed inside the context of a python interprter (the same as the current one)')
add_submission_options(pysubparser)
pysubparser.set_defaults(func=pysubmit)
pysubparser.add_argument('job', metavar='command', nargs=argparse.REMAINDER)
# subcommand 'resubmit'
resubparser = cmdparser.add_parser('resubmit', aliases=['resub', 're'],
help='resubmits all jobs in a given database, exactly like they were submitted the first time')
......
......@@ -117,7 +117,7 @@ def qsub(command, queue=None, cwd=True, name=None, deps=[], stdout='',
pe_opt
If set, add a -pe option when launching a job (for instance pe_exclusive* 1-)
Returns a list of job ids assigned to this job (integers)
Returns the job id assigned to this job (integer)
"""
scmd = ['qsub']
......@@ -192,10 +192,11 @@ def qsub(command, queue=None, cwd=True, name=None, deps=[], stdout='',
scmd += command
logging.debug("Qsub command '%s'", ' '.join(scmd))
from .setshell import sexec
jobid = sexec(context, scmd)
return int(jobid.split('.',1)[0])
def make_shell(shell, command):
"""Returns a single command given a shell and a command to be qsub'ed
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment