Commit ca3f45c5 authored by Manuel Günther's avatar Manuel Günther

Another improvement of python3 compatibility.

parent 6cc08646
......@@ -40,7 +40,7 @@ Make sure to have your shell environment setup to reach it w/o requiring to type
The first task you may need to pursue is to submit jobs.
Here is how::
$ jman submit --help
$ jman -vv submit --help
... Added job '<Job: 1> : submitted -- /usr/bin/python --help' to the database
... Submitted job '<Job: 6151645> : queued -- /usr/bin/python --help' to the SGE grid.
......@@ -17,7 +17,7 @@ if sys.version_info[0] >= 3:
from cPickle import dumps, loads
from .tools import makedirs_safe, logger
from .tools import makedirs_safe, logger, str_
from .manager import JobManager
......@@ -143,7 +143,7 @@ class JobManagerLocal(JobManager):
"""Finalizes the execution of the job by writing the stdout and stderr results into the according log files."""
def write(file, std, process):
f = std if file is None else open(str(file), 'w')
# get the files to write to
import sqlalchemy
from sqlalchemy import Table, Column, Integer, String, Boolean, ForeignKey, Enum
from sqlalchemy.orm import backref, relationship
from sqlalchemy import Table, Column, Integer, String, Boolean, ForeignKey
from sqlalchemy.orm import backref
from sqlalchemy.ext.declarative import declarative_base
from .tools import Enum, relationship
import os
import sys
......@@ -8,10 +8,9 @@
import os
import sys
import six
import signal
import subprocess
from .tools import logger
from .tools import logger, str_
def environ(context):
"""Retrieves the environment for a particular SETSHELL context"""
......@@ -26,8 +25,7 @@ def environ(context):
pi = subprocess.Popen(command, stdout = subprocess.PIPE)
# overwrite the default environment
for line in pi.stdout:
if isinstance(line, bytes) and not isinstance(line, str):
line = line.decode('utf8')
line = str_(line)
(key, _, value) = line.partition("=")
os.environ[key.strip()] = value.strip()
except OSError as e:
......@@ -54,8 +52,7 @@ def environ(context):
raise OSError("Error executing '%s': %s (%d)" % (' '.join(command), e.strerror, e.errno))
source = p.communicate()[0]
source = source.strip()
source = str_(p.communicate()[0]).strip()
except KeyboardInterrupt: # the user CTRL-C'ed
os.kill(, signal.SIGTERM)
......@@ -72,6 +69,7 @@ def environ(context):
new_environ = dict(os.environ)
for line in p2.stdout:
line = str_(line)
(key, _, value) = line.partition("=")
new_environ[key.strip()] = value.strip()
......@@ -92,6 +90,7 @@ def environ(context):
def sexec(context, command, error_on_nonzero=True):
"""Executes a command within a particular Idiap SETSHELL context"""
import six
if isinstance(context, six.string_types): E = environ(context)
else: E = context
......@@ -102,10 +101,10 @@ def sexec(context, command, error_on_nonzero=True):
(stdout, stderr) = p.communicate() #note: stderr will be 'None'
if p.returncode != 0:
if error_on_nonzero:
raise RuntimeError("Execution of '%s' exited with status != 0 (%d): %s" % (' '.join(command), p.returncode, stdout))
raise RuntimeError("Execution of '%s' exited with status != 0 (%d): %s" % (' '.join(command), p.returncode, str_(stdout)))
logger.debug("Execution of '%s' exited with status != 0 (%d): %s" % \
(' '.join(command), p.returncode, stdout))
(' '.join(command), p.returncode, str_(stdout)))
return stdout.strip()
This diff is collapsed.
......@@ -9,10 +9,49 @@ probing.
import os
import re
import six
import hashlib
import random
# sqlalchemy migration; copied from Bob
from sqlalchemy import Enum
except ImportError:
from sqlalchemy import types
class Enum(types.TypeDecorator):
impl = types.Unicode
def __init__(self, *values):
"""Emulates an Enum type.
A list of valid values for this column
if values is None or len(values) is 0:
raise AssertionError('Enum requires a list of values')
self.values = values[:]
# The length of the string/unicode column should be the longest string
# in values
size = max([len(v) for v in values if v is not None])
super(Enum, self).__init__(size)
def process_bind_param(self, value, dialect):
if value not in self.values:
raise AssertionError('"%s" not in Enum.values' % value)
return value
def process_result_value(self, value, dialect):
return value
from sqlalchemy.orm import relationship
except ImportError:
from sqlalchemy.orm import relation as relationship
# initialize the logging system
import logging
logger = logging.getLogger("gridtk")
......@@ -45,6 +84,16 @@ def makedirs_safe(fulldir):
else: raise
def str_(name):
"""Return the string representation of the given 'name'.
If it is a bytes object, it will be converted into str.
If it is a str object, it will simply be resurned."""
if isinstance(name, bytes) and not isinstance(name, str):
return name.decode('utf8')
return name
def qsub(command, queue=None, cwd=True, name=None, deps=[], stdout='',
stderr='', env=[], array=None, context='grid', hostname=None,
mem=None, memfree=None, hvmem=None, pe_opt=None, io_big=False):
......@@ -138,6 +187,7 @@ def qsub(command, queue=None, cwd=True, name=None, deps=[], stdout='',
scmd = ['qsub']
import six
if isinstance(queue, six.string_types) and queue not in ('all.q', 'default'):
scmd += ['-l', queue]
......@@ -212,7 +262,7 @@ def qsub(command, queue=None, cwd=True, name=None, deps=[], stdout='',
logger.debug("Qsub command '%s'", ' '.join(scmd))
from .setshell import sexec
jobid = sexec(context, scmd)
jobid = str_(sexec(context, scmd))
return int(jobid.split('.',1)[0])
def make_shell(shell, command):
......@@ -253,7 +303,7 @@ def qstat(jobid, context='grid'):
logger.debug("Qstat command '%s'", ' '.join(scmd))
from .setshell import sexec
data = sexec(context, scmd, error_on_nonzero=False)
data = str_(sexec(context, scmd, error_on_nonzero=False))
# some parsing:
retval = {}
......@@ -9,14 +9,14 @@ if sys.version_info[:2] < (2, 7) or ((3,0) <= sys.version_info[:2] < (3,2)):
description='SGE Grid Submission and Monitoring Tools for Idiap',
description='SGE Grid and Local Submission and Monitoring Tools for Idiap',
author='Andre Anjos',
author='Manuel Guenther',
Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment