Commit cee2bdfe authored by Tiago de Freitas Pereira's avatar Tiago de Freitas Pereira
Browse files

Merge branch 'master' of github.com:bioidiap/gridtk

parents 0730f113 469a7adb
...@@ -6,7 +6,7 @@ ...@@ -6,7 +6,7 @@
.. image:: http://travis-ci.org/bioidiap/gridtk.svg?branch=master .. image:: http://travis-ci.org/bioidiap/gridtk.svg?branch=master
:target: https://travis-ci.org/bioidiap/gridtk?branch=master :target: https://travis-ci.org/bioidiap/gridtk?branch=master
.. image:: https://coveralls.io/repos/github/bioidiap/gridtk/badge.svg?branch=master .. image:: https://coveralls.io/repos/github/bioidiap/gridtk/badge.svg?branch=master
:target: https://coveralls.io/github/bioidiap/gridtk?branch=master :target: https://coveralls.io/github/bioidiap/gridtk?branch=master
.. image:: https://img.shields.io/badge/github-master-0000c0.png .. image:: https://img.shields.io/badge/github-master-0000c0.png
:target: https://github.com/bioidiap/gridtk/tree/master :target: https://github.com/bioidiap/gridtk/tree/master
.. image:: http://img.shields.io/pypi/v/gridtk.png .. image:: http://img.shields.io/pypi/v/gridtk.png
......
...@@ -159,6 +159,9 @@ You can use it to select a range of job ids, e.g., ``-j 1-4 6-8``. ...@@ -159,6 +159,9 @@ You can use it to select a range of job ids, e.g., ``-j 1-4 6-8``.
In this case, please assert that there are no spaces between job ids and the ``-`` separator. In this case, please assert that there are no spaces between job ids and the ``-`` separator.
If any job id is specified, which is not available in the database, it will simply be ignored, including job ids that in the ranges. If any job id is specified, which is not available in the database, it will simply be ignored, including job ids that in the ranges.
Since version 1.3.0, GridTK also saves timing information about jobs, i.e., time stamps when jobs were submitted, started and finished.
You can use the ``-t`` option of ``jman ls`` to add the time stamps to the listing, which are both written for jobs and parametric jobs (i.e., when using the ``-a`` option).
Inspecting log files Inspecting log files
-------------------- --------------------
......
...@@ -4,7 +4,7 @@ from __future__ import print_function ...@@ -4,7 +4,7 @@ from __future__ import print_function
import os, sys import os, sys
import subprocess import subprocess
import socket # to get the host name import socket # to get the host name
from .models import Base, Job, ArrayJob, Status from .models import Base, Job, ArrayJob, Status, times
from .tools import logger from .tools import logger
...@@ -200,7 +200,7 @@ class JobManager: ...@@ -200,7 +200,7 @@ class JobManager:
self.unlock() self.unlock()
def list(self, job_ids, print_array_jobs = False, print_dependencies = False, long = False, status=Status, names=None, ids_only=False): def list(self, job_ids, print_array_jobs = False, print_dependencies = False, long = False, print_times = False, status=Status, names=None, ids_only=False):
"""Lists the jobs currently added to the database.""" """Lists the jobs currently added to the database."""
# configuration for jobs # configuration for jobs
if print_dependencies: if print_dependencies:
...@@ -231,7 +231,6 @@ class JobManager: ...@@ -231,7 +231,6 @@ class JobManager:
print(' '.join(header)) print(' '.join(header))
print(delimiter) print(delimiter)
self.lock() self.lock()
for job in self.get_jobs(job_ids): for job in self.get_jobs(job_ids):
job.refresh() job.refresh()
...@@ -240,11 +239,16 @@ class JobManager: ...@@ -240,11 +239,16 @@ class JobManager:
print(job.unique, end=" ") print(job.unique, end=" ")
else: else:
print(job.format(format, dependency_length, None if long else 43)) print(job.format(format, dependency_length, None if long else 43))
if print_times:
print(times(job))
if (not ids_only) and print_array_jobs and job.array: if (not ids_only) and print_array_jobs and job.array:
print(array_delimiter) print(array_delimiter)
for array_job in job.array: for array_job in job.array:
if array_job.status in status: if array_job.status in status:
print(array_job.format(array_format)) print(array_job.format(array_format))
if print_times:
print(times(array_job))
print(array_delimiter) print(array_delimiter)
self.unlock() self.unlock()
......
import sqlalchemy import sqlalchemy
from sqlalchemy import Table, Column, Integer, String, Boolean, ForeignKey from sqlalchemy import Table, Column, Integer, DateTime, String, Boolean, ForeignKey
from sqlalchemy.orm import backref from sqlalchemy.orm import backref
from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.ext.declarative import declarative_base
from .tools import Enum, relationship from .tools import Enum, relationship
import os import os
import sys import sys
from datetime import datetime
if sys.version_info[0] >= 3: if sys.version_info[0] >= 3:
from pickle import dumps, loads from pickle import dumps, loads
...@@ -29,6 +30,10 @@ class ArrayJob(Base): ...@@ -29,6 +30,10 @@ class ArrayJob(Base):
result = Column(Integer) result = Column(Integer)
machine_name = Column(String(10)) machine_name = Column(String(10))
submit_time = Column(DateTime)
start_time = Column(DateTime)
finish_time = Column(DateTime)
job = relationship("Job", backref='array', order_by=id) job = relationship("Job", backref='array', order_by=id)
def __init__(self, id, job_id): def __init__(self, id, job_id):
...@@ -38,6 +43,11 @@ class ArrayJob(Base): ...@@ -38,6 +43,11 @@ class ArrayJob(Base):
self.result = None self.result = None
self.machine_name = None # will be set later, by the Job class self.machine_name = None # will be set later, by the Job class
self.submit_time = datetime.now()
self.start_time = None
self.finish_time = None
def std_out_file(self): def std_out_file(self):
return self.job.std_out_file() + "." + str(self.id) if self.job.log_dir else None return self.job.std_out_file() + "." + str(self.id) if self.job.log_dir else None
...@@ -76,6 +86,11 @@ class Job(Base): ...@@ -76,6 +86,11 @@ class Job(Base):
array_string = Column(String(255)) # The array string (only needed for re-submission) array_string = Column(String(255)) # The array string (only needed for re-submission)
stop_on_failure = Column(Boolean) # An indicator whether to stop depending jobs when this job finishes with an error stop_on_failure = Column(Boolean) # An indicator whether to stop depending jobs when this job finishes with an error
submit_time = Column(DateTime)
start_time = Column(DateTime)
finish_time = Column(DateTime)
status = Column(Enum(*Status)) status = Column(Enum(*Status))
result = Column(Integer) result = Column(Integer)
...@@ -105,6 +120,9 @@ class Job(Base): ...@@ -105,6 +120,9 @@ class Job(Base):
array_job.result = None array_job.result = None
array_job.machine_name = None array_job.machine_name = None
self.id = self.unique self.id = self.unique
self.submit_time = datetime.now()
self.start_time = None
self.finish_time = None
def queue(self, new_job_id = None, new_job_name = None, queue_name = None): def queue(self, new_job_id = None, new_job_name = None, queue_name = None):
...@@ -148,8 +166,11 @@ class Job(Base): ...@@ -148,8 +166,11 @@ class Job(Base):
array_job.status = 'executing' array_job.status = 'executing'
if machine_name is not None: if machine_name is not None:
array_job.machine_name = machine_name array_job.machine_name = machine_name
array_job.start_time = datetime.now()
elif machine_name is not None: elif machine_name is not None:
self.machine_name = machine_name self.machine_name = machine_name
if self.start_time is None:
self.start_time = datetime.now()
# sometimes, the 'finish' command did not work for array jobs, # sometimes, the 'finish' command did not work for array jobs,
# so check if any old job still has the 'executing' flag set # so check if any old job still has the 'executing' flag set
...@@ -169,6 +190,7 @@ class Job(Base): ...@@ -169,6 +190,7 @@ class Job(Base):
if array_job.id == array_id: if array_job.id == array_id:
array_job.status = new_status array_job.status = new_status
array_job.result = result array_job.result = result
array_job.finish_time = datetime.now()
if array_job.status not in ('success', 'failure'): if array_job.status not in ('success', 'failure'):
finished = False finished = False
elif new_result == 0: elif new_result == 0:
...@@ -178,6 +200,7 @@ class Job(Base): ...@@ -178,6 +200,7 @@ class Job(Base):
# There was no array job, or all array jobs finished # There was no array job, or all array jobs finished
self.status = 'success' if new_result == 0 else 'failure' self.status = 'success' if new_result == 0 else 'failure'
self.result = new_result self.result = new_result
self.finish_time = datetime.now()
# update all waiting jobs # update all waiting jobs
for job in self.get_jobs_waiting_for_us(): for job in self.get_jobs_waiting_for_us():
...@@ -362,3 +385,12 @@ def add_job(session, command_line, name = 'job', dependencies = [], array = None ...@@ -362,3 +385,12 @@ def add_job(session, command_line, name = 'job', dependencies = [], array = None
session.commit() session.commit()
return job return job
def times(job):
"""Returns a string containing timing information for teh given job, which might be a :py:class:`Job` or an :py:class:`ArrayJob`."""
timing = "Submitted: %s" % job.submit_time.ctime()
if job.start_time is not None:
timing += "\nStarted : %s \t Job waited : %s" % (job.start_time.ctime(), job.start_time - job.submit_time)
if job.finish_time is not None:
timing += "\nFinished : %s \t Job executed: %s" % (job.finish_time.ctime(), job.finish_time - job.start_time)
return timing
...@@ -181,7 +181,7 @@ def run_scheduler(args): ...@@ -181,7 +181,7 @@ def run_scheduler(args):
def list(args): def list(args):
"""Lists the jobs in the given database.""" """Lists the jobs in the given database."""
jm = setup(args) jm = setup(args)
jm.list(job_ids=get_ids(args.job_ids), print_array_jobs=args.print_array_jobs, print_dependencies=args.print_dependencies, status=args.status, long=args.long, ids_only=args.ids_only, names=args.names) jm.list(job_ids=get_ids(args.job_ids), print_array_jobs=args.print_array_jobs, print_dependencies=args.print_dependencies, status=args.status, long=args.long, print_times=args.print_times, ids_only=args.ids_only, names=args.names)
def communicate(args): def communicate(args):
...@@ -324,6 +324,7 @@ def main(command_line_options = None): ...@@ -324,6 +324,7 @@ def main(command_line_options = None):
list_parser.add_argument('-n', '--names', metavar='NAME', nargs='+', help='List only the jobs with the given names (by default, all jobs are listed)') list_parser.add_argument('-n', '--names', metavar='NAME', nargs='+', help='List only the jobs with the given names (by default, all jobs are listed)')
list_parser.add_argument('-a', '--print-array-jobs', action='store_true', help='Also list the array ids.') list_parser.add_argument('-a', '--print-array-jobs', action='store_true', help='Also list the array ids.')
list_parser.add_argument('-l', '--long', action='store_true', help='Prints additional information about the submitted job.') list_parser.add_argument('-l', '--long', action='store_true', help='Prints additional information about the submitted job.')
list_parser.add_argument('-t', '--print-times', action='store_true', help='Prints timing information on when jobs were submited, executed and finished')
list_parser.add_argument('-x', '--print-dependencies', action='store_true', help='Print the dependencies of the jobs as well.') list_parser.add_argument('-x', '--print-dependencies', action='store_true', help='Print the dependencies of the jobs as well.')
list_parser.add_argument('-o', '--ids-only', action='store_true', help='Prints ONLY the job ids (so that they can be parsed by automatic scripts).') list_parser.add_argument('-o', '--ids-only', action='store_true', help='Prints ONLY the job ids (so that they can be parsed by automatic scripts).')
list_parser.add_argument('-s', '--status', nargs='+', choices = Status, default = Status, help='Delete only jobs that have the given statuses; by default all jobs are deleted.') list_parser.add_argument('-s', '--status', nargs='+', choices = Status, default = Status, help='Delete only jobs that have the given statuses; by default all jobs are deleted.')
......
...@@ -60,7 +60,7 @@ class GridTKTest(unittest.TestCase): ...@@ -60,7 +60,7 @@ class GridTKTest(unittest.TestCase):
print() print()
# test that the list command works (should also work with the "default" grid manager # test that the list command works (should also work with the "default" grid manager
jman.main(['./bin/jman', '--database', self.database, 'list', '--job-ids', '1']) jman.main(['./bin/jman', '--database', self.database, 'list', '--job-ids', '1'])
jman.main(['./bin/jman', '--database', self.database, 'list', '--job-ids', '2', '--print-array-jobs', '--print-dependencies']) jman.main(['./bin/jman', '--database', self.database, 'list', '--job-ids', '2', '--print-array-jobs', '--print-dependencies', '--print-times'])
# get insight into the database # get insight into the database
job_manager = gridtk.local.JobManagerLocal(database=self.database) job_manager = gridtk.local.JobManagerLocal(database=self.database)
...@@ -74,6 +74,12 @@ class GridTKTest(unittest.TestCase): ...@@ -74,6 +74,12 @@ class GridTKTest(unittest.TestCase):
self.assertEqual(jobs[0].status, 'submitted') self.assertEqual(jobs[0].status, 'submitted')
self.assertEqual(jobs[1].status, 'submitted') self.assertEqual(jobs[1].status, 'submitted')
self.assertEqual(jobs[2].status, 'submitted') self.assertEqual(jobs[2].status, 'submitted')
self.assertTrue(all(j.submit_time is not None for j in jobs))
self.assertTrue(all(j.start_time is None for j in jobs))
self.assertTrue(all(j.finish_time is None for j in jobs))
self.assertTrue(all(j.submit_time is not None for j in jobs[1].array))
self.assertTrue(all(j.start_time is None for j in jobs[1].array))
self.assertTrue(all(j.finish_time is None for j in jobs[1].array))
# check that the job dependencies are correct # check that the job dependencies are correct
waiting = jobs[0].get_jobs_waiting_for_us() waiting = jobs[0].get_jobs_waiting_for_us()
...@@ -107,6 +113,13 @@ class GridTKTest(unittest.TestCase): ...@@ -107,6 +113,13 @@ class GridTKTest(unittest.TestCase):
self.assertEqual(jobs[0].status, 'failure') self.assertEqual(jobs[0].status, 'failure')
self.assertEqual(jobs[1].status, 'queued') self.assertEqual(jobs[1].status, 'queued')
self.assertEqual(jobs[2].status, 'waiting') self.assertEqual(jobs[2].status, 'waiting')
self.assertTrue(jobs[0].start_time is not None)
self.assertTrue(jobs[0].finish_time is not None)
self.assertTrue(jobs[1].start_time is None)
self.assertTrue(jobs[1].finish_time is None)
self.assertTrue(jobs[2].start_time is None)
self.assertTrue(jobs[2].finish_time is None)
# the result files should already be there # the result files should already be there
self.assertTrue(os.path.exists(jobs[0].std_out_file())) self.assertTrue(os.path.exists(jobs[0].std_out_file()))
self.assertTrue(os.path.exists(jobs[0].std_err_file())) self.assertTrue(os.path.exists(jobs[0].std_err_file()))
...@@ -193,6 +206,14 @@ class GridTKTest(unittest.TestCase): ...@@ -193,6 +206,14 @@ class GridTKTest(unittest.TestCase):
self.assertEqual(jobs[1].array[i].result, 0) self.assertEqual(jobs[1].array[i].result, 0)
self.assertEqual(jobs[2].status, 'success') self.assertEqual(jobs[2].status, 'success')
self.assertEqual(jobs[2].result, 0) self.assertEqual(jobs[2].result, 0)
self.assertTrue(all(j.submit_time is not None for j in jobs))
self.assertTrue(all(j.start_time is not None for j in jobs))
self.assertTrue(all(j.finish_time is not None for j in jobs))
self.assertTrue(all(j.submit_time is not None for j in jobs[1].array))
self.assertTrue(all(j.start_time is not None for j in jobs[1].array))
self.assertTrue(all(j.finish_time is not None for j in jobs[1].array))
job_manager.unlock() job_manager.unlock()
print() print()
......
1.3.0b0 1.3.1b0
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment