Commit 880cbdc3 authored by Tiago de Freitas Pereira's avatar Tiago de Freitas Pereira

Merge branch 'sge-extra-commands' into 'master'

SGE extra commands

See merge request !32
parents f11199ca 6dc58c2f
Pipeline #40881 passed with stages
in 4 minutes and 31 seconds
......@@ -125,6 +125,20 @@ This directory can be changed by specifying:
has finished.
In case, SGE backend is used, `--sge-extra-command` or shortly `-e` allows you to send commands to `qsub -l` command
.. code-block:: sh
$ jman -vv submit -l [log_dir] -e [SGE_command]
For example, `jman submit .. -e pytorch=true ...` will be translated to `qsub ... -l pytorch=true ...`.
.. note::
Note that you can pass multiple SGE commands with `--sge-extra-command` or `e`, e.g., `jman submit ... -e <SGE_command_1> <SGE_command_2> <SGE_command_3> ...`
Running Jobs Locally
......@@ -140,6 +140,7 @@ def submit(args):
'env': args.env,
'memfree': args.memory,
'io_big': args.io_big,
'sge_extra_flags': args.sge_extra_command
if args.array is not None: kwargs['array'] = get_array(args.array)
......@@ -311,6 +312,7 @@ def main(command_line_options = None):
# subcommand 'submit'
submit_parser = cmdparser.add_parser('submit', aliases=['sub'], formatter_class=formatter, help='Submits jobs to the SGE queue or to the local job scheduler and logs them in a database.')
submit_parser.add_argument('-q', '--queue', metavar='QNAME', dest='qname', default='all.q', choices=QUEUES, help='the name of the SGE queue to submit the job to')
submit_parser.add_argument('-e', '--sge-extra-command', default=[], type=str, nargs="*", help='In case SGE backend is used, this option allows you to send commands to `qsub -l`, e.g., `jman submit -e pytorch=true` will be translated to `qsub -l pytorch=true`')
submit_parser.add_argument('-m', '--memory', help='Sets both the h_vmem and the mem_free parameters when submitting '
'the job to a non-GPU queue, e.g., 8G to set the memory '
'requirements to 8 gigabytes. Sets gpumem parameter when '
......@@ -12,6 +12,7 @@ import re
import hashlib
import random
import math
import itertools
# sqlalchemy migration; copied from Bob
......@@ -97,7 +98,7 @@ def str_(name):
def qsub(command, queue=None, cwd=True, name=None, deps=[], stdout='',
stderr='', env=[], array=None, context='grid', hostname=None,
memfree=None, hvmem=None, gpumem=None, pe_opt=None, io_big=False):
memfree=None, hvmem=None, gpumem=None, pe_opt=None, io_big=False, sge_extra_flags=[]):
"""Submits a shell job to a given grid queue
Keyword parameters:
......@@ -182,10 +183,16 @@ def qsub(command, queue=None, cwd=True, name=None, deps=[], stdout='',
If set to true, the io_big flag will be set.
Use this flag if your process will need a lot of Input/Output operations.
This is used to add new flags which are developed by SGE admin.
Note that you can pass multiple SGE commands with `--sge-extra-command` or `e`, e.g., `jman submit ... -e <SGE_command_1> <SGE_command_2> <SGE_command_3> ...` which will be translated to `qsub ... -l <SGE_command_1> -l <SGE_command_2> -l <SGE_command_3> ...`
Returns the job id assigned to this job (integer)
scmd = ['qsub']
scmd += list(itertools.chain(*[['-l', f'{e}'] for e in sge_extra_flags]))
import six
if isinstance(queue, six.string_types) and queue not in ('all.q', 'default'):
Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment