Commit ba27a8c7 authored by Hatef OTROSHI's avatar Hatef OTROSHI Committed by Amir MOHAMMADI

update doc

add `--sge-direct-commands` option

update doc

update tools.py
add `sge-direct-commands` opyion for `qsub`

update jman.py
add `sge-direct-commands` opyion for `qsub`

update doc:
add example for `sge-direct-commands` option

update doc

update doc

update doc

update doc

add 'gridtk.sge.direct.commands.prepend'

update `--sge-extra-flags`

doc: update `--sge-extra-flags`
parent b3b312cc
......@@ -125,19 +125,26 @@ This directory can be changed by specifying:
has finished.
In case, SGE backend is used, `--sge-extra-command` or shortly `-e` allows you to send commands to `qsub -l` command
In case SGE backend is used, `--sge-extra-flags` or shortly `-e` allows you to send direct commands in `qsub -l`
.. code-block:: sh
$ jman -vv submit -l [log_dir] -e [SGE_command]
$ jman -vv submit -l [log_dir] -e [sge_extra_flags]
For example, `jman submit .. -e pytorch=true ...` will be translated to `qsub ... -l pytorch=true ...`.
For example, `jman submit .. -e "-P project_name -l pytorch=true" -- ...` will be translated to `qsub ... -P project_name -l pytorch=true -- ...`.
.. note::
Please note that you can define `default` and `prepend` values for this command in [`bob.extension.rc`](https://www.idiap.ch/software/bob/docs/bob/docs/master/bob/bob.extension/doc/rc.html).
.. code-block:: sh
Note that you can pass multiple SGE commands with `--sge-extra-command` or `e`, e.g., `jman submit ... -e <SGE_command_1> <SGE_command_2> <SGE_command_3> ...`
$ bob config set gridtk.sge.extra.flags.default "the default expression"
$ bob config set gridtk.sge.extra.args.prepend "the prepend expression"
For example, if you define `"-P biometrics"` as default value, a command like `jman submit -- python train.py` would translate `jman submit --sge-extra-args "-P project_name" -- python train.py`.
Similarly, if you define `"-P biometrics"` as prepend value it will be automatically added as a prepend to `--sge-extra-flags` argument.
.. note::
Note that you can pass multiple SGE commands with `--sge-extra-flags` or `-e`, e.g., `jman submit ... -e "<SGE_command_1> <SGE_command_2> <SGE_command_3> " ...`
Running Jobs Locally
--------------------
......
......@@ -140,7 +140,7 @@ def submit(args):
'env': args.env,
'memfree': args.memory,
'io_big': args.io_big,
'sge_extra_flags': args.sge_extra_command
'sge_extra_flags': args.sge_extra_flags
}
if args.array is not None: kwargs['array'] = get_array(args.array)
......@@ -290,6 +290,7 @@ class AliasedSubParsersAction(argparse._SubParsersAction):
def main(command_line_options = None):
from ..config import __version__
from bob.extension import rc
formatter = argparse.ArgumentDefaultsHelpFormatter
parser = argparse.ArgumentParser(description=__doc__, epilog=__epilog__,
......@@ -312,7 +313,7 @@ def main(command_line_options = None):
# subcommand 'submit'
submit_parser = cmdparser.add_parser('submit', aliases=['sub'], formatter_class=formatter, help='Submits jobs to the SGE queue or to the local job scheduler and logs them in a database.')
submit_parser.add_argument('-q', '--queue', metavar='QNAME', dest='qname', default='all.q', choices=QUEUES, help='the name of the SGE queue to submit the job to')
submit_parser.add_argument('-e', '--sge-extra-command', default=[], type=str, nargs="*", help='In case SGE backend is used, this option allows you to send commands to `qsub -l`, e.g., `jman submit -e pytorch=true` will be translated to `qsub -l pytorch=true`')
submit_parser.add_argument('-e', '--sge-extra-flags', default=rc['gridtk.sge.extra.flags.default'], type=str, nargs="*", help='In case SGE backend is used, this option allows you to send direct commands to `qsub`, e.g., `jman submit -e "-P project_name -l pytorch=true" -- ...` will be translated to `qsub -P project_name -l pytorch=true -- ...`')
submit_parser.add_argument('-m', '--memory', help='Sets both the h_vmem and the mem_free parameters when submitting '
'the job to a non-GPU queue, e.g., 8G to set the memory '
'requirements to 8 gigabytes. Sets gpumem parameter when '
......
......@@ -12,8 +12,7 @@ import re
import hashlib
import random
import math
import itertools
import shlex
# sqlalchemy migration; copied from Bob
try:
......@@ -98,7 +97,8 @@ def str_(name):
def qsub(command, queue=None, cwd=True, name=None, deps=[], stdout='',
stderr='', env=[], array=None, context='grid', hostname=None,
memfree=None, hvmem=None, gpumem=None, pe_opt=None, io_big=False, sge_extra_flags=[]):
memfree=None, hvmem=None, gpumem=None, pe_opt=None, io_big=False,
sge_extra_flags=[]):
"""Submits a shell job to a given grid queue
Keyword parameters:
......@@ -184,15 +184,21 @@ def qsub(command, queue=None, cwd=True, name=None, deps=[], stdout='',
Use this flag if your process will need a lot of Input/Output operations.
sge_extra_flags
This is used to add new flags which are developed by SGE admin.
Note that you can pass multiple SGE commands with `--sge-extra-command` or `e`, e.g., `jman submit ... -e <SGE_command_1> <SGE_command_2> <SGE_command_3> ...` which will be translated to `qsub ... -l <SGE_command_1> -l <SGE_command_2> -l <SGE_command_3> ...`
This is used to send direct commands to SGE. Note that all its arguments are directly
used in `qsub` command. For example, `jman submit -e "-P project_name -l pytorch=true" -- ...` will
be translated to `qsub -P project_name -l pytorch=true -- ...`
Returns the job id assigned to this job (integer)
"""
scmd = ['qsub']
scmd += list(itertools.chain(*[['-l', f'{e}'] for e in sge_extra_flags]))
from bob.extension import rc
scmd += rc['gridtk.sge.extra.flags.prepend']
scmd += list(e for e in shlex.split(sge_extra_flags)]))
import six
if isinstance(queue, six.string_types) and queue not in ('all.q', 'default'):
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment