Merge branch 'sge-extra-commands' into 'master'

SGE extra commands

See merge request bob/gridtk!32

doc/manual.rst

@@ -125,6 +125,20 @@ This directory can be changed by specifying:
    has finished.
 
 
+In case, SGE backend is used, `--sge-extra-command` or shortly `-e` allows you to send commands to `qsub -l` command
+
+.. code-block:: sh
+
+  $ jman -vv submit -l [log_dir] -e [SGE_command]
+
+For example, `jman submit .. -e pytorch=true ...` will be translated to `qsub ... -l pytorch=true ...`.
+
+.. note::
+
+   Note that you can pass multiple SGE commands with `--sge-extra-command` or `e`, e.g., `jman submit ... -e <SGE_command_1> <SGE_command_2> <SGE_command_3> ...`
+
+
+
 Running Jobs Locally
 --------------------
 

gridtk/script/jman.py

@@ -140,6 +140,7 @@ def submit(args):
       'env': args.env,
       'memfree': args.memory,
       'io_big': args.io_big,
+	  'sge_extra_flags': args.sge_extra_command
   }
 
   if args.array is not None:         kwargs['array'] = get_array(args.array)
@@ -311,6 +312,7 @@ def main(command_line_options = None):
   # subcommand 'submit'
   submit_parser = cmdparser.add_parser('submit', aliases=['sub'], formatter_class=formatter, help='Submits jobs to the SGE queue or to the local job scheduler and logs them in a database.')
   submit_parser.add_argument('-q', '--queue', metavar='QNAME', dest='qname', default='all.q', choices=QUEUES, help='the name of the SGE queue to submit the job to')
+  submit_parser.add_argument('-e', '--sge-extra-command', default=[], type=str, nargs="*", help='In case SGE backend is used, this option allows you to send commands to `qsub -l`, e.g., `jman submit -e pytorch=true` will be translated to `qsub -l pytorch=true`')
   submit_parser.add_argument('-m', '--memory', help='Sets both the h_vmem and the mem_free parameters when submitting '
                                                     'the job to a non-GPU queue, e.g., 8G to set the memory '
                                                     'requirements to 8 gigabytes. Sets gpumem parameter when '

gridtk/tools.py

@@ -12,6 +12,7 @@ import re
 import hashlib
 import random
 import math
+import itertools
 
 
 # sqlalchemy migration; copied from Bob
@@ -97,7 +98,7 @@ def str_(name):
 
 def qsub(command, queue=None, cwd=True, name=None, deps=[], stdout='',
     stderr='', env=[], array=None, context='grid', hostname=None,
-    memfree=None, hvmem=None, gpumem=None, pe_opt=None, io_big=False):
+    memfree=None, hvmem=None, gpumem=None, pe_opt=None, io_big=False, sge_extra_flags=[]):
   """Submits a shell job to a given grid queue
 
   Keyword parameters:
@@ -182,10 +183,16 @@ def qsub(command, queue=None, cwd=True, name=None, deps=[], stdout='',
     If set to true, the io_big flag will be set.
     Use this flag if your process will need a lot of Input/Output operations.
 
+  sge_extra_flags
+	This is used to add new flags which are developed by SGE admin.
+	Note that you can pass multiple SGE commands with `--sge-extra-command` or `e`, e.g., `jman submit ... -e <SGE_command_1> <SGE_command_2> <SGE_command_3> ...` which will be translated to `qsub ... -l <SGE_command_1> -l <SGE_command_2> -l <SGE_command_3> ...`
+
+
   Returns the job id assigned to this job (integer)
   """
 
   scmd = ['qsub']
+  scmd += list(itertools.chain(*[['-l', f'{e}'] for e in sge_extra_flags]))
 
   import six
   if isinstance(queue, six.string_types) and queue not in ('all.q', 'default'):