Commit ce2fa1ea authored by Amir MOHAMMADI's avatar Amir MOHAMMADI

Merge branch 'sge-direct-commands' into 'master'

Arbitrary extra arguments for SGE

See merge request !33
parents b3b312cc cf697463
Pipeline #41051 passed with stages
in 4 minutes and 16 seconds
...@@ -125,18 +125,43 @@ This directory can be changed by specifying: ...@@ -125,18 +125,43 @@ This directory can be changed by specifying:
has finished. has finished.
In case, SGE backend is used, `--sge-extra-command` or shortly `-e` allows you to send commands to `qsub -l` command If the SGE backend is used, ``--sge-extra-args`` or shortly ``-e`` allows you to send
extra arguments to ``qsub``.
.. code-block:: sh .. code-block:: sh
$ jman -vv submit -l [log_dir] -e [SGE_command] $ jman -vv submit -e="<sge_extra_args>"
For example, `jman submit .. -e pytorch=true ...` will be translated to `qsub ... -l pytorch=true ...`. For example, ``jman submit .. -e="-P project_name -l pytorch" -- ...`` will be
translated to ``qsub ... -P project_name -l pytorch -- ...``.
.. note:: .. note::
Note that you can pass multiple SGE commands with `--sge-extra-command` or `e`, e.g., `jman submit ... -e <SGE_command_1> <SGE_command_2> <SGE_command_3> ...` Note that extra options for qsub must be wrapped in single or double quotes **and**
should attach to the ``-e`` option with an ``=`` sign, e.g. ``jman submit -e='-P
project_name -l pytorch'``. Examples like ``jman submit -e '-P project_name -l
pytorch'`` and ``jman submit -e -P project_name -l pytorch`` will not work.
To avoid adding the same ``-e`` option each time you run ``jman submit``, you may also
change its default value using :ref:`bob.extension.rc`. For example, if you run:
.. code-block:: sh
$ bob config set -- gridtk.sge.extra.args.default "-P myproject"
Then, if you do ``jman submit ...``, this will translate to ``qsub -P myproject ...``.
This configuration only changes the default value, you still can provide a new value by
providing the ``-e`` option.
Another (**recommended**) option is to always a prepend a string to this option. For
example, if you run:
.. code-block:: sh
$ bob config set -- gridtk.sge.extra.args.prepend "-P myproject"
Then, if you do ``jman submit -e="-l pytorch"``, this will translate to
``qsub -P myproject -l pytorch``.
Running Jobs Locally Running Jobs Locally
......
...@@ -265,6 +265,8 @@ class Job(Base): ...@@ -265,6 +265,8 @@ class Job(Base):
retval['env'] = args['env'] retval['env'] = args['env']
if 'io_big' in args and args['io_big']: if 'io_big' in args and args['io_big']:
retval['io_big'] = True retval['io_big'] = True
if 'sge_extra_args' in args:
retval['sge_extra_args'] = args['sge_extra_args']
# also add the queue # also add the queue
if self.queue_name is not None: if self.queue_name is not None:
...@@ -332,9 +334,9 @@ class Job(Base): ...@@ -332,9 +334,9 @@ class Job(Base):
deps = str(sorted(list(set([dep.unique for dep in self.get_jobs_we_wait_for()])))) deps = str(sorted(list(set([dep.unique for dep in self.get_jobs_we_wait_for()]))))
if dependencies < len(deps): if dependencies < len(deps):
deps = deps[:dependencies-3] + '...' deps = deps[:dependencies-3] + '...'
return format.format(self.unique, job_id, queue[:12], status, self.name, deps, command_line) return format.format(self.unique, job_id, queue[:12], status, str(self.name), deps, command_line)
else: else:
return format.format(self.unique, job_id, queue[:12], status, self.name, command_line) return format.format(self.unique, job_id, queue[:12], status, str(self.name), command_line)
......
...@@ -140,7 +140,7 @@ def submit(args): ...@@ -140,7 +140,7 @@ def submit(args):
'env': args.env, 'env': args.env,
'memfree': args.memory, 'memfree': args.memory,
'io_big': args.io_big, 'io_big': args.io_big,
'sge_extra_flags': args.sge_extra_command 'sge_extra_args': args.sge_extra_args
} }
if args.array is not None: kwargs['array'] = get_array(args.array) if args.array is not None: kwargs['array'] = get_array(args.array)
...@@ -290,6 +290,7 @@ class AliasedSubParsersAction(argparse._SubParsersAction): ...@@ -290,6 +290,7 @@ class AliasedSubParsersAction(argparse._SubParsersAction):
def main(command_line_options = None): def main(command_line_options = None):
from ..config import __version__ from ..config import __version__
from bob.extension import rc
formatter = argparse.ArgumentDefaultsHelpFormatter formatter = argparse.ArgumentDefaultsHelpFormatter
parser = argparse.ArgumentParser(description=__doc__, epilog=__epilog__, parser = argparse.ArgumentParser(description=__doc__, epilog=__epilog__,
...@@ -312,7 +313,7 @@ def main(command_line_options = None): ...@@ -312,7 +313,7 @@ def main(command_line_options = None):
# subcommand 'submit' # subcommand 'submit'
submit_parser = cmdparser.add_parser('submit', aliases=['sub'], formatter_class=formatter, help='Submits jobs to the SGE queue or to the local job scheduler and logs them in a database.') submit_parser = cmdparser.add_parser('submit', aliases=['sub'], formatter_class=formatter, help='Submits jobs to the SGE queue or to the local job scheduler and logs them in a database.')
submit_parser.add_argument('-q', '--queue', metavar='QNAME', dest='qname', default='all.q', choices=QUEUES, help='the name of the SGE queue to submit the job to') submit_parser.add_argument('-q', '--queue', metavar='QNAME', dest='qname', default='all.q', choices=QUEUES, help='the name of the SGE queue to submit the job to')
submit_parser.add_argument('-e', '--sge-extra-command', default=[], type=str, nargs="*", help='In case SGE backend is used, this option allows you to send commands to `qsub -l`, e.g., `jman submit -e pytorch=true` will be translated to `qsub -l pytorch=true`') submit_parser.add_argument('-e', '--sge-extra-args', default=rc.get('gridtk.sge.extra.args.default', ''), type=str, help='Passes extra arguments to qsub. See the documentation of the package for usage and ways of overriding default behavior.')
submit_parser.add_argument('-m', '--memory', help='Sets both the h_vmem and the mem_free parameters when submitting ' submit_parser.add_argument('-m', '--memory', help='Sets both the h_vmem and the mem_free parameters when submitting '
'the job to a non-GPU queue, e.g., 8G to set the memory ' 'the job to a non-GPU queue, e.g., 8G to set the memory '
'requirements to 8 gigabytes. Sets gpumem parameter when ' 'requirements to 8 gigabytes. Sets gpumem parameter when '
...@@ -414,6 +415,9 @@ def main(command_line_options = None): ...@@ -414,6 +415,9 @@ def main(command_line_options = None):
args = parser.parse_args() args = parser.parse_args()
args.wrapper_script = sys.argv[0] args.wrapper_script = sys.argv[0]
if not hasattr(args, "func"):
return parser.print_help(sys.stderr)
args.func(args) args.func(args)
return 0 return 0
...@@ -12,8 +12,7 @@ import re ...@@ -12,8 +12,7 @@ import re
import hashlib import hashlib
import random import random
import math import math
import itertools import shlex
# sqlalchemy migration; copied from Bob # sqlalchemy migration; copied from Bob
try: try:
...@@ -98,7 +97,8 @@ def str_(name): ...@@ -98,7 +97,8 @@ def str_(name):
def qsub(command, queue=None, cwd=True, name=None, deps=[], stdout='', def qsub(command, queue=None, cwd=True, name=None, deps=[], stdout='',
stderr='', env=[], array=None, context='grid', hostname=None, stderr='', env=[], array=None, context='grid', hostname=None,
memfree=None, hvmem=None, gpumem=None, pe_opt=None, io_big=False, sge_extra_flags=[]): memfree=None, hvmem=None, gpumem=None, pe_opt=None, io_big=False,
sge_extra_args=""):
"""Submits a shell job to a given grid queue """Submits a shell job to a given grid queue
Keyword parameters: Keyword parameters:
...@@ -183,18 +183,23 @@ def qsub(command, queue=None, cwd=True, name=None, deps=[], stdout='', ...@@ -183,18 +183,23 @@ def qsub(command, queue=None, cwd=True, name=None, deps=[], stdout='',
If set to true, the io_big flag will be set. If set to true, the io_big flag will be set.
Use this flag if your process will need a lot of Input/Output operations. Use this flag if your process will need a lot of Input/Output operations.
sge_extra_flags sge_extra_args
This is used to add new flags which are developed by SGE admin. This is used to send extra argument to SGE. Note that all its arguments are directly
Note that you can pass multiple SGE commands with `--sge-extra-command` or `e`, e.g., `jman submit ... -e <SGE_command_1> <SGE_command_2> <SGE_command_3> ...` which will be translated to `qsub ... -l <SGE_command_1> -l <SGE_command_2> -l <SGE_command_3> ...` used in `qsub` command. For example, `jman submit -e "-P project_name -l pytorch=true" -- ...` will
be translated to `qsub -P project_name -l pytorch=true -- ...`
Returns the job id assigned to this job (integer) Returns the job id assigned to this job (integer)
""" """
import six
from bob.extension import rc
scmd = ['qsub'] scmd = ['qsub']
scmd += list(itertools.chain(*[['-l', f'{e}'] for e in sge_extra_flags]))
import six prepend = rc.get('gridtk.sge.extra.args.prepend') or ""
sge_extra_args = f"{prepend} {sge_extra_args or ''}"
scmd += shlex.split(sge_extra_args)
if isinstance(queue, six.string_types) and queue not in ('all.q', 'default'): if isinstance(queue, six.string_types) and queue not in ('all.q', 'default'):
scmd += ['-l', queue] scmd += ['-l', queue]
...@@ -268,7 +273,7 @@ def qsub(command, queue=None, cwd=True, name=None, deps=[], stdout='', ...@@ -268,7 +273,7 @@ def qsub(command, queue=None, cwd=True, name=None, deps=[], stdout='',
from .setshell import sexec from .setshell import sexec
jobid = str_(sexec(context, scmd)) jobid = str_(sexec(context, scmd))
return int(jobid.split('.',1)[0]) return int(jobid.split("\n")[-1].split('.',1)[0])
def make_shell(shell, command): def make_shell(shell, command):
"""Returns a single command given a shell and a command to be qsub'ed """Returns a single command given a shell and a command to be qsub'ed
......
1.7.1b0 1.8.0b0
\ No newline at end of file
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment