New buildout; few fixes and improvements of jman

MANIFEST.in

-include README.rst bootstrap.py buildout.cfg
+include README.rst bootstrap-buildout.py buildout.cfg
 recursive-include doc *.rst
 recursive-include gridtk *.sh

README.rst

@@ -27,7 +27,7 @@ Instead of sending jobs to the SGE grid, it executes them in parallel processes
 This package uses the Buildout system to install it.
 Please call::
 
-  $ python bootstrap.py
+  $ python bootstrap-buildout.py
   $ bin/buildout
   $ bin/sphinx-build docs sphinx
   $ firefox sphinx/index.html

bootstrap.py → bootstrap-buildout.py

@@ -56,6 +56,11 @@ parser.add_option("-c", "--config-file",
                         "file to be used."))
 parser.add_option("-f", "--find-links",
                   help=("Specify a URL to search for buildout releases"))
+parser.add_option("--allow-site-packages",
+                  action="store_true", default=False,
+                  help=("Let bootstrap.py use existing site packages"))
+parser.add_option("--setuptools-version",
+                  help="use a specific setuptools version")
 
 
 options, args = parser.parse_args()
@@ -63,45 +68,42 @@ options, args = parser.parse_args()
 ######################################################################
 # load/install setuptools
 
-to_reload = False
 try:
-    import pkg_resources
-    import setuptools
+    if options.allow_site_packages:
+        import setuptools
+        import pkg_resources
+    from urllib.request import urlopen
 except ImportError:
-    ez = {}
-
-    try:
-        from urllib.request import urlopen
-    except ImportError:
-        from urllib2 import urlopen
-
-    # XXX use a more permanent ez_setup.py URL when available.
-    exec(urlopen('https://bitbucket.org/pypa/setuptools/raw/0.7.2/ez_setup.py'
-                ).read(), ez)
-    setup_args = dict(to_dir=tmpeggs, download_delay=0)
-    ez['use_setuptools'](**setup_args)
-
-    if to_reload:
-        reload(pkg_resources)
-    import pkg_resources
-    # This does not (always?) update the default working set.  We will
-    # do it.
-    for path in sys.path:
-        if path not in pkg_resources.working_set.entries:
-            pkg_resources.working_set.add_entry(path)
+    from urllib2 import urlopen
 
-######################################################################
-# Try to best guess the version of buildout given setuptools
-if options.version is None:
-
-  try:
-    from distutils.version import LooseVersion
-    package = pkg_resources.require('setuptools')[0]
-    v = LooseVersion(package.version)
-    if v < LooseVersion('0.7'):
-      options.version = '2.1.1'
-  except:
-    pass
+ez = {}
+exec(urlopen('https://bootstrap.pypa.io/ez_setup.py').read(), ez)
+
+if not options.allow_site_packages:
+    # ez_setup imports site, which adds site packages
+    # this will remove them from the path to ensure that incompatible versions
+    # of setuptools are not in the path
+    import site
+    # inside a virtualenv, there is no 'getsitepackages'.
+    # We can't remove these reliably
+    if hasattr(site, 'getsitepackages'):
+        for sitepackage_path in site.getsitepackages():
+            sys.path[:] = [x for x in sys.path if sitepackage_path not in x]
+
+setup_args = dict(to_dir=tmpeggs, download_delay=0)
+
+if options.setuptools_version is not None:
+    setup_args['version'] = options.setuptools_version
+
+ez['use_setuptools'](**setup_args)
+import setuptools
+import pkg_resources
+
+# This does not (always?) update the default working set.  We will
+# do it.
+for path in sys.path:
+    if path not in pkg_resources.working_set.entries:
+        pkg_resources.working_set.add_entry(path)
 
 ######################################################################
 # Install buildout
@@ -132,10 +134,15 @@ if version is None and not options.accept_buildout_test_releases:
     _final_parts = '*final-', '*final'
 
     def _final_version(parsed_version):
-        for part in parsed_version:
-            if (part[:1] == '*') and (part not in _final_parts):
-                return False
-        return True
+        try:
+            return not parsed_version.is_prerelease
+        except AttributeError:
+            # Older setuptools
+            for part in parsed_version:
+                if (part[:1] == '*') and (part not in _final_parts):
+                    return False
+            return True
+
     index = setuptools.package_index.PackageIndex(
         search_path=[setuptools_path])
     if find_links:
@@ -162,8 +169,7 @@ cmd.append(requirement)
 import subprocess
 if subprocess.call(cmd, env=dict(os.environ, PYTHONPATH=setuptools_path)) != 0:
     raise Exception(
-        "Failed to execute command:\n%s",
-        repr(cmd)[1:-1])
+        "Failed to execute command:\n%s" % repr(cmd)[1:-1])
 
 ######################################################################
 # Import and run buildout
@@ -181,4 +187,3 @@ if options.config_file is not None:
 
 zc.buildout.buildout.main(args)
 shutil.rmtree(tmpeggs)
-

gridtk/manager.py

@@ -232,7 +232,7 @@ class JobManager:
     self.unlock()
 
 
-  def report(self, job_ids=None, array_ids=None, output=True, error=True):
+  def report(self, job_ids=None, array_ids=None, output=True, error=True, status=Status, name=None):
     """Iterates through the output and error files and write the results to command line."""
     def _write_contents(job):
       # Writes the contents of the output and error files to command line
@@ -264,6 +264,10 @@ class JobManager:
       # iterate over all jobs
       jobs = self.get_jobs(job_ids)
       for job in jobs:
+        if name is not None and job.name != name:
+          continue
+        if job.status not in status:
+          continue
         if job.array:
           print(job)
           _write_array_jobs(job.array)
@@ -316,7 +320,7 @@ class JobManager:
           if job.status in status:
             if delete_jobs:
               logger.info("Deleting job '%d' from the database." % job.unique)
-            _delete(job, True)
+            _delete(job, delete_jobs)
 
     else:
       # iterate over all jobs
@@ -333,7 +337,7 @@ class JobManager:
         if job.status in status:
           if delete_jobs:
             logger.info("Deleting job '%d' from the database." % job.unique)
-          _delete(job, True)
+          _delete(job, delete_jobs)
 
     self.session.commit()
     self.unlock()

gridtk/script/jman.py

@@ -3,6 +3,8 @@
 # Andre Anjos <andre.anjos@idiap.ch>
 # Wed 24 Aug 2011 16:13:31 CEST
 
+from __future__ import print_function
+
 """A logging Idiap/SGE job manager
 """
 
@@ -130,12 +132,15 @@ def submit(args):
   # submit the job
   job_id = jm.submit(args.job, **kwargs)
 
+  if args.print_id:
+    print (job_id, end='')
+
 
 def resubmit(args):
   """Re-submits the jobs with the given ids."""
   jm = setup(args)
   if not args.keep_logs:
-    jm.delete(job_ids=args.job_ids, delete_jobs=False)
+    jm.delete(job_ids=get_ids(args.job_ids), delete_jobs=False)
 
   kwargs = {
       'cwd': True
@@ -183,7 +188,7 @@ def communicate(args):
 def report(args):
   """Reports the results of the finished (and unfinished) jobs."""
   jm = setup(args)
-  jm.report(job_ids=get_ids(args.job_ids), array_ids=get_ids(args.array_ids), output=not args.errors_only, error=not args.output_only)
+  jm.report(job_ids=get_ids(args.job_ids), array_ids=get_ids(args.array_ids), output=not args.errors_only, error=not args.output_only, status=args.status, name=args.name)
 
 
 def stop(args):
@@ -282,6 +287,7 @@ def main(command_line_options = None):
   submit_parser.add_argument('-t', '--array', '--parametric', metavar='(first-)last(:step)', help="Creates a parametric (array) job. You must specify the 'last' value, but 'first' (default=1) and 'step' (default=1) can be specified as well (when specifying 'step', 'first' has to be given, too).")
   submit_parser.add_argument('-z', '--dry-run', action='store_true', help='Do not really submit anything, just print out what would submit in this case')
   submit_parser.add_argument('-i', '--io-big', action='store_true', help='Sets "io_big" on the submitted jobs so it limits the machines in which the job is submitted to those that can do high-throughput.')
+  submit_parser.add_argument('-o', '--print-id', action='store_true', help='Prints the new job id (so that they can be parsed by automatic scripts).')
   submit_parser.add_argument('job', metavar='command', nargs=argparse.REMAINDER, help = "The job that should be executed. Sometimes a -- is required to separate the job from other command line options.")
   submit_parser.set_defaults(func=submit)
 
@@ -327,6 +333,8 @@ def main(command_line_options = None):
   report_parser.add_argument('-o', '--output-only', action='store_true', help='Only report the output logs  (by default, both logs are reported).')
   report_parser.add_argument('-j', '--job-ids', metavar='ID', nargs='+', help='Report only the jobs with the given ids (by default, all finished jobs are reported)')
   report_parser.add_argument('-a', '--array-ids', metavar='ID', nargs='+', help='Report only the jobs with the given array ids. If specified, a single job-id must be given as well.')
+  report_parser.add_argument('-n', '--name', help="Report only the jobs with the given name; by default all jobs are reported.")
+  report_parser.add_argument('-s', '--status', nargs='+', choices = Status, default = Status, help='Report only jobs that have the given statuses; by default all jobs are reported.')
   report_parser.set_defaults(func=report)
 
   # subcommand 'delete'

setup.py

@@ -33,10 +33,6 @@ setup(
         'man.py = gridtk.script.grid:main',
       ],
 
-      'bob.test' : [
-        'gridtk = gridtk.tests:GridTKTest',
-      ],
-
     },
 
     long_description=open('README.rst').read(),