Commit ce2fa1ea authored by Amir MOHAMMADI's avatar Amir MOHAMMADI

Merge branch 'sge-direct-commands' into 'master'

Arbitrary extra arguments for SGE

See merge request !33
parents b3b312cc cf697463
Pipeline #41051 passed with stages
in 4 minutes and 16 seconds
......@@ -125,18 +125,43 @@ This directory can be changed by specifying:
has finished.
In case, SGE backend is used, `--sge-extra-command` or shortly `-e` allows you to send commands to `qsub -l` command
If the SGE backend is used, ``--sge-extra-args`` or shortly ``-e`` allows you to send
extra arguments to ``qsub``.
.. code-block:: sh
$ jman -vv submit -l [log_dir] -e [SGE_command]
$ jman -vv submit -e="<sge_extra_args>"
For example, `jman submit .. -e pytorch=true ...` will be translated to `qsub ... -l pytorch=true ...`.
For example, ``jman submit .. -e="-P project_name -l pytorch" -- ...`` will be
translated to ``qsub ... -P project_name -l pytorch -- ...``.
.. note::
Note that you can pass multiple SGE commands with `--sge-extra-command` or `e`, e.g., `jman submit ... -e <SGE_command_1> <SGE_command_2> <SGE_command_3> ...`
Note that extra options for qsub must be wrapped in single or double quotes **and**
should attach to the ``-e`` option with an ``=`` sign, e.g. ``jman submit -e='-P
project_name -l pytorch'``. Examples like ``jman submit -e '-P project_name -l
pytorch'`` and ``jman submit -e -P project_name -l pytorch`` will not work.
To avoid adding the same ``-e`` option each time you run ``jman submit``, you may also
change its default value using :ref:`bob.extension.rc`. For example, if you run:
.. code-block:: sh
$ bob config set -- gridtk.sge.extra.args.default "-P myproject"
Then, if you do ``jman submit ...``, this will translate to ``qsub -P myproject ...``.
This configuration only changes the default value, you still can provide a new value by
providing the ``-e`` option.
Another (**recommended**) option is to always a prepend a string to this option. For
example, if you run:
.. code-block:: sh
$ bob config set -- gridtk.sge.extra.args.prepend "-P myproject"
Then, if you do ``jman submit -e="-l pytorch"``, this will translate to
``qsub -P myproject -l pytorch``.
Running Jobs Locally
......
......@@ -265,6 +265,8 @@ class Job(Base):
retval['env'] = args['env']
if 'io_big' in args and args['io_big']:
retval['io_big'] = True
if 'sge_extra_args' in args:
retval['sge_extra_args'] = args['sge_extra_args']
# also add the queue
if self.queue_name is not None:
......@@ -332,9 +334,9 @@ class Job(Base):
deps = str(sorted(list(set([dep.unique for dep in self.get_jobs_we_wait_for()]))))
if dependencies < len(deps):
deps = deps[:dependencies-3] + '...'
return format.format(self.unique, job_id, queue[:12], status, self.name, deps, command_line)
return format.format(self.unique, job_id, queue[:12], status, str(self.name), deps, command_line)
else:
return format.format(self.unique, job_id, queue[:12], status, self.name, command_line)
return format.format(self.unique, job_id, queue[:12], status, str(self.name), command_line)
......
......@@ -140,7 +140,7 @@ def submit(args):
'env': args.env,
'memfree': args.memory,
'io_big': args.io_big,
'sge_extra_flags': args.sge_extra_command
'sge_extra_args': args.sge_extra_args
}
if args.array is not None: kwargs['array'] = get_array(args.array)
......@@ -290,6 +290,7 @@ class AliasedSubParsersAction(argparse._SubParsersAction):
def main(command_line_options = None):
from ..config import __version__
from bob.extension import rc
formatter = argparse.ArgumentDefaultsHelpFormatter
parser = argparse.ArgumentParser(description=__doc__, epilog=__epilog__,
......@@ -312,7 +313,7 @@ def main(command_line_options = None):
# subcommand 'submit'
submit_parser = cmdparser.add_parser('submit', aliases=['sub'], formatter_class=formatter, help='Submits jobs to the SGE queue or to the local job scheduler and logs them in a database.')
submit_parser.add_argument('-q', '--queue', metavar='QNAME', dest='qname', default='all.q', choices=QUEUES, help='the name of the SGE queue to submit the job to')
submit_parser.add_argument('-e', '--sge-extra-command', default=[], type=str, nargs="*", help='In case SGE backend is used, this option allows you to send commands to `qsub -l`, e.g., `jman submit -e pytorch=true` will be translated to `qsub -l pytorch=true`')
submit_parser.add_argument('-e', '--sge-extra-args', default=rc.get('gridtk.sge.extra.args.default', ''), type=str, help='Passes extra arguments to qsub. See the documentation of the package for usage and ways of overriding default behavior.')
submit_parser.add_argument('-m', '--memory', help='Sets both the h_vmem and the mem_free parameters when submitting '
'the job to a non-GPU queue, e.g., 8G to set the memory '
'requirements to 8 gigabytes. Sets gpumem parameter when '
......@@ -414,6 +415,9 @@ def main(command_line_options = None):
args = parser.parse_args()
args.wrapper_script = sys.argv[0]
if not hasattr(args, "func"):
return parser.print_help(sys.stderr)
args.func(args)
return 0
......@@ -12,8 +12,7 @@ import re
import hashlib
import random
import math
import itertools
import shlex
# sqlalchemy migration; copied from Bob
try:
......@@ -98,7 +97,8 @@ def str_(name):
def qsub(command, queue=None, cwd=True, name=None, deps=[], stdout='',
stderr='', env=[], array=None, context='grid', hostname=None,
memfree=None, hvmem=None, gpumem=None, pe_opt=None, io_big=False, sge_extra_flags=[]):
memfree=None, hvmem=None, gpumem=None, pe_opt=None, io_big=False,
sge_extra_args=""):
"""Submits a shell job to a given grid queue
Keyword parameters:
......@@ -183,18 +183,23 @@ def qsub(command, queue=None, cwd=True, name=None, deps=[], stdout='',
If set to true, the io_big flag will be set.
Use this flag if your process will need a lot of Input/Output operations.
sge_extra_flags
This is used to add new flags which are developed by SGE admin.
Note that you can pass multiple SGE commands with `--sge-extra-command` or `e`, e.g., `jman submit ... -e <SGE_command_1> <SGE_command_2> <SGE_command_3> ...` which will be translated to `qsub ... -l <SGE_command_1> -l <SGE_command_2> -l <SGE_command_3> ...`
sge_extra_args
This is used to send extra argument to SGE. Note that all its arguments are directly
used in `qsub` command. For example, `jman submit -e "-P project_name -l pytorch=true" -- ...` will
be translated to `qsub -P project_name -l pytorch=true -- ...`
Returns the job id assigned to this job (integer)
"""
import six
from bob.extension import rc
scmd = ['qsub']
scmd += list(itertools.chain(*[['-l', f'{e}'] for e in sge_extra_flags]))
import six
prepend = rc.get('gridtk.sge.extra.args.prepend') or ""
sge_extra_args = f"{prepend} {sge_extra_args or ''}"
scmd += shlex.split(sge_extra_args)
if isinstance(queue, six.string_types) and queue not in ('all.q', 'default'):
scmd += ['-l', queue]
......@@ -268,7 +273,7 @@ def qsub(command, queue=None, cwd=True, name=None, deps=[], stdout='',
from .setshell import sexec
jobid = str_(sexec(context, scmd))
return int(jobid.split('.',1)[0])
return int(jobid.split("\n")[-1].split('.',1)[0])
def make_shell(shell, command):
"""Returns a single command given a shell and a command to be qsub'ed
......
1.7.1b0
\ No newline at end of file
1.8.0b0
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment