Commit 4ad67963 authored by Manuel Günther's avatar Manuel Günther

New buildout; few fixes and improvements of jman

parent c1f687fe
include README.rst bootstrap.py buildout.cfg
include README.rst bootstrap-buildout.py buildout.cfg
recursive-include doc *.rst
recursive-include gridtk *.sh
......@@ -27,7 +27,7 @@ Instead of sending jobs to the SGE grid, it executes them in parallel processes
This package uses the Buildout system to install it.
Please call::
$ python bootstrap.py
$ python bootstrap-buildout.py
$ bin/buildout
$ bin/sphinx-build docs sphinx
$ firefox sphinx/index.html
......
......@@ -56,6 +56,11 @@ parser.add_option("-c", "--config-file",
"file to be used."))
parser.add_option("-f", "--find-links",
help=("Specify a URL to search for buildout releases"))
parser.add_option("--allow-site-packages",
action="store_true", default=False,
help=("Let bootstrap.py use existing site packages"))
parser.add_option("--setuptools-version",
help="use a specific setuptools version")
options, args = parser.parse_args()
......@@ -63,45 +68,42 @@ options, args = parser.parse_args()
######################################################################
# load/install setuptools
to_reload = False
try:
import pkg_resources
import setuptools
if options.allow_site_packages:
import setuptools
import pkg_resources
from urllib.request import urlopen
except ImportError:
ez = {}
try:
from urllib.request import urlopen
except ImportError:
from urllib2 import urlopen
# XXX use a more permanent ez_setup.py URL when available.
exec(urlopen('https://bitbucket.org/pypa/setuptools/raw/0.7.2/ez_setup.py'
).read(), ez)
setup_args = dict(to_dir=tmpeggs, download_delay=0)
ez['use_setuptools'](**setup_args)
if to_reload:
reload(pkg_resources)
import pkg_resources
# This does not (always?) update the default working set. We will
# do it.
for path in sys.path:
if path not in pkg_resources.working_set.entries:
pkg_resources.working_set.add_entry(path)
from urllib2 import urlopen
######################################################################
# Try to best guess the version of buildout given setuptools
if options.version is None:
try:
from distutils.version import LooseVersion
package = pkg_resources.require('setuptools')[0]
v = LooseVersion(package.version)
if v < LooseVersion('0.7'):
options.version = '2.1.1'
except:
pass
ez = {}
exec(urlopen('https://bootstrap.pypa.io/ez_setup.py').read(), ez)
if not options.allow_site_packages:
# ez_setup imports site, which adds site packages
# this will remove them from the path to ensure that incompatible versions
# of setuptools are not in the path
import site
# inside a virtualenv, there is no 'getsitepackages'.
# We can't remove these reliably
if hasattr(site, 'getsitepackages'):
for sitepackage_path in site.getsitepackages():
sys.path[:] = [x for x in sys.path if sitepackage_path not in x]
setup_args = dict(to_dir=tmpeggs, download_delay=0)
if options.setuptools_version is not None:
setup_args['version'] = options.setuptools_version
ez['use_setuptools'](**setup_args)
import setuptools
import pkg_resources
# This does not (always?) update the default working set. We will
# do it.
for path in sys.path:
if path not in pkg_resources.working_set.entries:
pkg_resources.working_set.add_entry(path)
######################################################################
# Install buildout
......@@ -132,10 +134,15 @@ if version is None and not options.accept_buildout_test_releases:
_final_parts = '*final-', '*final'
def _final_version(parsed_version):
for part in parsed_version:
if (part[:1] == '*') and (part not in _final_parts):
return False
return True
try:
return not parsed_version.is_prerelease
except AttributeError:
# Older setuptools
for part in parsed_version:
if (part[:1] == '*') and (part not in _final_parts):
return False
return True
index = setuptools.package_index.PackageIndex(
search_path=[setuptools_path])
if find_links:
......@@ -162,8 +169,7 @@ cmd.append(requirement)
import subprocess
if subprocess.call(cmd, env=dict(os.environ, PYTHONPATH=setuptools_path)) != 0:
raise Exception(
"Failed to execute command:\n%s",
repr(cmd)[1:-1])
"Failed to execute command:\n%s" % repr(cmd)[1:-1])
######################################################################
# Import and run buildout
......@@ -181,4 +187,3 @@ if options.config_file is not None:
zc.buildout.buildout.main(args)
shutil.rmtree(tmpeggs)
......@@ -232,7 +232,7 @@ class JobManager:
self.unlock()
def report(self, job_ids=None, array_ids=None, output=True, error=True):
def report(self, job_ids=None, array_ids=None, output=True, error=True, status=Status, name=None):
"""Iterates through the output and error files and write the results to command line."""
def _write_contents(job):
# Writes the contents of the output and error files to command line
......@@ -264,6 +264,10 @@ class JobManager:
# iterate over all jobs
jobs = self.get_jobs(job_ids)
for job in jobs:
if name is not None and job.name != name:
continue
if job.status not in status:
continue
if job.array:
print(job)
_write_array_jobs(job.array)
......@@ -316,7 +320,7 @@ class JobManager:
if job.status in status:
if delete_jobs:
logger.info("Deleting job '%d' from the database." % job.unique)
_delete(job, True)
_delete(job, delete_jobs)
else:
# iterate over all jobs
......@@ -333,7 +337,7 @@ class JobManager:
if job.status in status:
if delete_jobs:
logger.info("Deleting job '%d' from the database." % job.unique)
_delete(job, True)
_delete(job, delete_jobs)
self.session.commit()
self.unlock()
......@@ -3,6 +3,8 @@
# Andre Anjos <andre.anjos@idiap.ch>
# Wed 24 Aug 2011 16:13:31 CEST
from __future__ import print_function
"""A logging Idiap/SGE job manager
"""
......@@ -130,12 +132,15 @@ def submit(args):
# submit the job
job_id = jm.submit(args.job, **kwargs)
if args.print_id:
print (job_id, end='')
def resubmit(args):
"""Re-submits the jobs with the given ids."""
jm = setup(args)
if not args.keep_logs:
jm.delete(job_ids=args.job_ids, delete_jobs=False)
jm.delete(job_ids=get_ids(args.job_ids), delete_jobs=False)
kwargs = {
'cwd': True
......@@ -183,7 +188,7 @@ def communicate(args):
def report(args):
"""Reports the results of the finished (and unfinished) jobs."""
jm = setup(args)
jm.report(job_ids=get_ids(args.job_ids), array_ids=get_ids(args.array_ids), output=not args.errors_only, error=not args.output_only)
jm.report(job_ids=get_ids(args.job_ids), array_ids=get_ids(args.array_ids), output=not args.errors_only, error=not args.output_only, status=args.status, name=args.name)
def stop(args):
......@@ -282,6 +287,7 @@ def main(command_line_options = None):
submit_parser.add_argument('-t', '--array', '--parametric', metavar='(first-)last(:step)', help="Creates a parametric (array) job. You must specify the 'last' value, but 'first' (default=1) and 'step' (default=1) can be specified as well (when specifying 'step', 'first' has to be given, too).")
submit_parser.add_argument('-z', '--dry-run', action='store_true', help='Do not really submit anything, just print out what would submit in this case')
submit_parser.add_argument('-i', '--io-big', action='store_true', help='Sets "io_big" on the submitted jobs so it limits the machines in which the job is submitted to those that can do high-throughput.')
submit_parser.add_argument('-o', '--print-id', action='store_true', help='Prints the new job id (so that they can be parsed by automatic scripts).')
submit_parser.add_argument('job', metavar='command', nargs=argparse.REMAINDER, help = "The job that should be executed. Sometimes a -- is required to separate the job from other command line options.")
submit_parser.set_defaults(func=submit)
......@@ -327,6 +333,8 @@ def main(command_line_options = None):
report_parser.add_argument('-o', '--output-only', action='store_true', help='Only report the output logs (by default, both logs are reported).')
report_parser.add_argument('-j', '--job-ids', metavar='ID', nargs='+', help='Report only the jobs with the given ids (by default, all finished jobs are reported)')
report_parser.add_argument('-a', '--array-ids', metavar='ID', nargs='+', help='Report only the jobs with the given array ids. If specified, a single job-id must be given as well.')
report_parser.add_argument('-n', '--name', help="Report only the jobs with the given name; by default all jobs are reported.")
report_parser.add_argument('-s', '--status', nargs='+', choices = Status, default = Status, help='Report only jobs that have the given statuses; by default all jobs are reported.')
report_parser.set_defaults(func=report)
# subcommand 'delete'
......
......@@ -33,10 +33,6 @@ setup(
'man.py = gridtk.script.grid:main',
],
'bob.test' : [
'gridtk = gridtk.tests:GridTKTest',
],
},
long_description=open('README.rst').read(),
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment