Merge branch 'sge-direct-commands' into 'master'

Arbitrary extra arguments for SGE

See merge request !33

doc/manual.rst

@@ -125,18 +125,43 @@ This directory can be changed by specifying:
    has finished.
 
 
-In case, SGE backend is used, `--sge-extra-command` or shortly `-e` allows you to send commands to `qsub -l` command
+If the SGE backend is used, ``--sge-extra-args`` or shortly ``-e`` allows you to send
+extra arguments to ``qsub``.
 
 .. code-block:: sh
 
-  $ jman -vv submit -l [log_dir] -e [SGE_command]
+  $ jman -vv submit -e="<sge_extra_args>"
 
-For example, `jman submit .. -e pytorch=true ...` will be translated to `qsub ... -l pytorch=true ...`.
+For example, ``jman submit .. -e="-P project_name -l pytorch" -- ...`` will be
+translated to ``qsub ... -P project_name -l pytorch -- ...``.
 
 .. note::
 
-   Note that you can pass multiple SGE commands with `--sge-extra-command` or `e`, e.g., `jman submit ... -e <SGE_command_1> <SGE_command_2> <SGE_command_3> ...`
+   Note that extra options for qsub must be wrapped in single or double quotes **and**
+   should attach to the ``-e`` option with an ``=`` sign, e.g. ``jman submit -e='-P
+   project_name -l pytorch'``. Examples like ``jman submit -e '-P project_name -l
+   pytorch'`` and ``jman submit -e -P project_name -l pytorch`` will not work.
 
+To avoid adding the same ``-e`` option each time you run ``jman submit``, you may also
+change its default value using :ref:`bob.extension.rc`. For example, if you run:
+
+.. code-block:: sh
+
+  $ bob config set -- gridtk.sge.extra.args.default "-P myproject"
+
+Then, if you do ``jman submit ...``, this will translate to ``qsub -P myproject ...``.
+This configuration only changes the default value, you still can provide a new value by
+providing the ``-e`` option.
+
+Another (**recommended**) option is to always a prepend a string to this option. For
+example, if you run:
+
+.. code-block:: sh
+
+  $ bob config set -- gridtk.sge.extra.args.prepend "-P myproject"
+
+Then, if you do ``jman submit -e="-l pytorch"``, this will translate to
+``qsub -P myproject -l pytorch``.
 
 
 Running Jobs Locally

gridtk/models.py

@@ -265,6 +265,8 @@ class Job(Base):
       retval['env'] = args['env']
     if 'io_big' in args and args['io_big']:
       retval['io_big'] = True
+    if 'sge_extra_args' in args:
+      retval['sge_extra_args'] = args['sge_extra_args']
 
     # also add the queue
     if self.queue_name is not None:
@@ -332,9 +334,9 @@ class Job(Base):
       deps = str(sorted(list(set([dep.unique for dep in self.get_jobs_we_wait_for()]))))
       if dependencies < len(deps):
         deps = deps[:dependencies-3] + '...'
-      return format.format(self.unique, job_id, queue[:12], status, self.name, deps, command_line)
+      return format.format(self.unique, job_id, queue[:12], status, str(self.name), deps, command_line)
     else:
-      return format.format(self.unique, job_id, queue[:12], status, self.name, command_line)
+      return format.format(self.unique, job_id, queue[:12], status, str(self.name), command_line)
 
 
 

gridtk/script/jman.py

@@ -140,7 +140,7 @@ def submit(args):
       'env': args.env,
       'memfree': args.memory,
       'io_big': args.io_big,
-	  'sge_extra_flags': args.sge_extra_command
+      'sge_extra_args': args.sge_extra_args
   }
 
   if args.array is not None:         kwargs['array'] = get_array(args.array)
@@ -290,6 +290,7 @@ class AliasedSubParsersAction(argparse._SubParsersAction):
 def main(command_line_options = None):
 
   from ..config import __version__
+  from bob.extension import rc
 
   formatter = argparse.ArgumentDefaultsHelpFormatter
   parser = argparse.ArgumentParser(description=__doc__, epilog=__epilog__,
@@ -312,7 +313,7 @@ def main(command_line_options = None):
   # subcommand 'submit'
   submit_parser = cmdparser.add_parser('submit', aliases=['sub'], formatter_class=formatter, help='Submits jobs to the SGE queue or to the local job scheduler and logs them in a database.')
   submit_parser.add_argument('-q', '--queue', metavar='QNAME', dest='qname', default='all.q', choices=QUEUES, help='the name of the SGE queue to submit the job to')
-  submit_parser.add_argument('-e', '--sge-extra-command', default=[], type=str, nargs="*", help='In case SGE backend is used, this option allows you to send commands to `qsub -l`, e.g., `jman submit -e pytorch=true` will be translated to `qsub -l pytorch=true`')
+  submit_parser.add_argument('-e', '--sge-extra-args', default=rc.get('gridtk.sge.extra.args.default', ''), type=str, help='Passes extra arguments to qsub. See the documentation of the package for usage and ways of overriding default behavior.')
   submit_parser.add_argument('-m', '--memory', help='Sets both the h_vmem and the mem_free parameters when submitting '
                                                     'the job to a non-GPU queue, e.g., 8G to set the memory '
                                                     'requirements to 8 gigabytes. Sets gpumem parameter when '
@@ -414,6 +415,9 @@ def main(command_line_options = None):
     args = parser.parse_args()
     args.wrapper_script = sys.argv[0]
 
+  if not hasattr(args, "func"):
+    return parser.print_help(sys.stderr)
+
   args.func(args)
 
   return 0

gridtk/tools.py

@@ -12,8 +12,7 @@ import re
 import hashlib
 import random
 import math
-import itertools
-
+import shlex
 
 # sqlalchemy migration; copied from Bob
 try:
@@ -98,7 +97,8 @@ def str_(name):
 
 def qsub(command, queue=None, cwd=True, name=None, deps=[], stdout='',
     stderr='', env=[], array=None, context='grid', hostname=None,
-    memfree=None, hvmem=None, gpumem=None, pe_opt=None, io_big=False, sge_extra_flags=[]):
+    memfree=None, hvmem=None, gpumem=None, pe_opt=None, io_big=False,
+    sge_extra_args=""):
   """Submits a shell job to a given grid queue
 
   Keyword parameters:
@@ -183,18 +183,23 @@ def qsub(command, queue=None, cwd=True, name=None, deps=[], stdout='',
     If set to true, the io_big flag will be set.
     Use this flag if your process will need a lot of Input/Output operations.
 
-  sge_extra_flags
-	This is used to add new flags which are developed by SGE admin.
-	Note that you can pass multiple SGE commands with `--sge-extra-command` or `e`, e.g., `jman submit ... -e <SGE_command_1> <SGE_command_2> <SGE_command_3> ...` which will be translated to `qsub ... -l <SGE_command_1> -l <SGE_command_2> -l <SGE_command_3> ...`
+  sge_extra_args
+    This is used to send extra argument to SGE. Note that all its arguments are directly
+    used in `qsub` command. For example, `jman submit -e "-P project_name -l pytorch=true" -- ...` will
+    be translated to `qsub -P project_name -l pytorch=true -- ...`
 
 
   Returns the job id assigned to this job (integer)
   """
+  import six
+  from bob.extension import rc
 
   scmd = ['qsub']
-  scmd += list(itertools.chain(*[['-l', f'{e}'] for e in sge_extra_flags]))
 
-  import six
+  prepend = rc.get('gridtk.sge.extra.args.prepend') or ""
+  sge_extra_args = f"{prepend} {sge_extra_args or ''}"
+  scmd += shlex.split(sge_extra_args)
+
   if isinstance(queue, six.string_types) and queue not in ('all.q', 'default'):
     scmd += ['-l', queue]
 
@@ -268,7 +273,7 @@ def qsub(command, queue=None, cwd=True, name=None, deps=[], stdout='',
 
   from .setshell import sexec
   jobid = str_(sexec(context, scmd))
-  return int(jobid.split('.',1)[0])
+  return int(jobid.split("\n")[-1].split('.',1)[0])
 
 def make_shell(shell, command):
   """Returns a single command given a shell and a command to be qsub'ed

version.txt

-1.7.1b0
\ No newline at end of file
+1.8.0b0