From 5886b5e5115bd64c3a1492887ef90c0734de0b25 Mon Sep 17 00:00:00 2001
From: Andre Anjos <andre.anjos@idiap.ch>
Date: Tue, 16 Aug 2016 16:17:39 +0200
Subject: [PATCH] Standardise
---
.gitlab-ci.yml | 254 ++++++++++++++++++++++++++++++++++++
.travis.yml | 33 -----
LICENSE.txt => COPYING | 0
MANIFEST.in | 2 +-
README.rst | 290 +++++------------------------------------
bootstrap-buildout.py | 51 +++++---
buildout.cfg | 14 +-
doc/conf.py | 159 +++++++++++-----------
doc/img/favicon.ico | Bin 0 -> 4286 bytes
doc/img/logo.png | Bin 0 -> 6266 bytes
doc/index.rst | 18 +--
doc/manual.rst | 277 +++++++++++++++++++++++++--------------
doc/program.rst | 24 ++--
gridtk/script/grid.py | 0
requirements.txt | 3 +-
setup.cfg | 8 --
setup.py | 22 ++--
17 files changed, 617 insertions(+), 538 deletions(-)
create mode 100644 .gitlab-ci.yml
delete mode 100644 .travis.yml
rename LICENSE.txt => COPYING (100%)
create mode 100644 doc/img/favicon.ico
create mode 100644 doc/img/logo.png
mode change 100755 => 100644 gridtk/script/grid.py
delete mode 100644 setup.cfg
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
new file mode 100644
index 0000000..f26a01e
--- /dev/null
+++ b/.gitlab-ci.yml
@@ -0,0 +1,254 @@
+# This build file is defined in two parts: 1) a generic set of instructions you
+# probably **don't** need to change and 2) a part you may have to tune to your
+# project. It heavily uses template features from YAML to help you in only
+# changing a minimal part of it and avoid code duplication to a maximum while
+# still providing a nice pipeline display on your package.
+
+
+# 1) Generic instructions (only change if you know what you're doing)
+# -------------------------------------------------------------------
+
+# Definition of our build pipeline
+stages:
+ - build
+ - test
+ - docs
+ - wheels
+
+
+# Global variables
+variables:
+ CONDA_PREFIX: env
+
+
+# Template for the build stage
+# Needs to run on all supported architectures, platforms and python versions
+.build_template: &build_job
+ stage: build
+ before_script:
+ - git clean -ffdx
+ - curl --silent https://gitlab.idiap.ch/bob/bob/snippets/7/raw | tr -d '\r' > bootstrap-conda.sh
+ - chmod 755 ./bootstrap-conda.sh
+ - ./bootstrap-conda.sh ${CONDA_FOLDER} ${PYTHON_VER} ${CONDA_PREFIX}
+ variables: &build_variables
+ BOB_DOCUMENTATION_SERVER: "http://www.idiap.ch/software/bob/docs/latest/bob/%s/master/"
+ script:
+ - ./bin/buildout
+ - if [ -x ./bin/bob_dbmanage.py ]; then ./bin/bob_dbmanage.py all download --force; fi
+ - ./bin/sphinx-build doc sphinx
+ - ./bin/python setup.py bdist_wheel --python-tag ${WHEEL_TAG}
+ after_script:
+ - rm -rf ${CONDA_PREFIX}
+ artifacts:
+ expire_in: 1 day
+ paths:
+ - bootstrap-conda.sh
+ - dist/
+ - sphinx/
+
+
+# Template for building on a Linux machine
+.build_linux_template: &linux_build_job
+ <<: *build_job
+ variables: &linux_build_variables
+ <<: *build_variables
+ CONDA_FOLDER: "/local/conda"
+ CFLAGS: "-D_GLIBCXX_USE_CXX11_ABI=0 -coverage"
+ CXXFLAGS: "-D_GLIBCXX_USE_CXX11_ABI=0 -coverage"
+
+
+# Template for building on a Mac OSX machine
+.build_mac_template: &macosx_build_job
+ <<: *build_job
+ variables: &macosx_build_variables
+ <<: *build_variables
+ CONDA_FOLDER: "/opt/conda"
+ MACOSX_DEPLOYMENT_TARGET: "10.9"
+ CFLAGS: "-pthread -coverage"
+ CXXFLAGS: "-pthread -coverage"
+ LDFLAGS: "-lpthread"
+
+
+# Template for the test stage - re-install from uploaded wheels
+# Needs to run on all supported architectures, platforms and python versions
+.test_template: &test_job
+ stage: test
+ before_script:
+ - ./bootstrap-conda.sh ${CONDA_FOLDER} ${PYTHON_VER} ${CONDA_PREFIX}
+ - source ${CONDA_FOLDER}/bin/activate ${CONDA_PREFIX}
+ - pip install --use-wheel --no-index --pre dist/*.whl
+ script:
+ - cd ${CONDA_PREFIX}
+ - python -c "from ${CI_PROJECT_NAME} import get_config; print(get_config())"
+ - coverage run --source=${CI_PROJECT_NAME} ./bin/nosetests -sv ${CI_PROJECT_NAME}
+ - coverage report
+ - sphinx-build -b doctest ../doc ../sphinx
+ after_script:
+ - rm -rf ${CONDA_PREFIX}
+
+
+# Template for the wheel uploading stage
+# Needs to run against one combination of python 2.x and 3.x if it is a python
+# only package, otherwise, needs to run in both pythons to all supported
+# architectures (Linux and Mac OSX 64-bit)
+.wheels_template: &wheels_job
+ stage: wheels
+ only:
+ - master
+ - tags
+ before_script:
+ - curl --silent https://gitlab.idiap.ch/bob/bob/snippets/8/raw | tr -d '\r' > upload-wheel.sh
+ - chmod 755 upload-wheel.sh
+ script:
+ - ./upload-wheel.sh
+
+
+# Template for (latest) documentation upload stage
+# Only one real job needs to do this
+.docs_template: &docs_job
+ stage: docs
+ only:
+ - master
+ before_script:
+ - curl --silent https://gitlab.idiap.ch/bob/bob/snippets/9/raw | tr -d '\r' > upload-sphinx.sh
+ - chmod 755 upload-sphinx.sh
+ script:
+ - ./upload-sphinx.sh
+
+
+# 2) Package specific instructions (you may tune this if needed)
+# --------------------------------------------------------------
+
+# Linux + Python 2.7: Builds, tests, uploads wheel
+build_linux_27:
+ <<: *linux_build_job
+ variables: &linux_27_build_variables
+ <<: *linux_build_variables
+ PYTHON_VER: "2.7"
+ WHEEL_TAG: "py27"
+ tags:
+ - conda-linux
+
+test_linux_27:
+ <<: *test_job
+ variables: *linux_27_build_variables
+ dependencies:
+ - build_linux_27
+ tags:
+ - conda-linux
+
+wheels_linux_27:
+ <<: *wheels_job
+ dependencies:
+ - build_linux_27
+ tags:
+ - conda-linux
+
+
+# Linux + Python 3.4: Builds and tests
+build_linux_34:
+ <<: *linux_build_job
+ variables: &linux_34_build_variables
+ <<: *linux_build_variables
+ PYTHON_VER: "3.4"
+ WHEEL_TAG: "py3"
+ tags:
+ - conda-linux
+
+test_linux_34:
+ <<: *test_job
+ variables: *linux_34_build_variables
+ dependencies:
+ - build_linux_34
+ tags:
+ - conda-linux
+
+
+# Linux + Python 3.5: Builds, tests, uploads wheel
+build_linux_35:
+ <<: *linux_build_job
+ variables: &linux_35_build_variables
+ <<: *linux_build_variables
+ PYTHON_VER: "3.5"
+ WHEEL_TAG: "py3"
+ tags:
+ - conda-linux
+
+test_linux_35:
+ <<: *test_job
+ variables: *linux_35_build_variables
+ dependencies:
+ - build_linux_35
+ tags:
+ - conda-linux
+
+wheels_linux_35:
+ <<: *wheels_job
+ dependencies:
+ - build_linux_35
+ tags:
+ - conda-linux
+
+docs_linux_35:
+ <<: *docs_job
+ dependencies:
+ - build_linux_35
+ tags:
+ - conda-linux
+
+
+# Mac OSX + Python 2.7: Builds and tests
+build_macosx_27:
+ <<: *macosx_build_job
+ variables: &macosx_27_build_variables
+ <<: *macosx_build_variables
+ PYTHON_VER: "2.7"
+ WHEEL_TAG: "py27"
+ tags:
+ - conda-macosx
+
+test_macosx_27:
+ <<: *test_job
+ variables: *macosx_27_build_variables
+ dependencies:
+ - build_macosx_27
+ tags:
+ - conda-macosx
+
+
+# Mac OSX + Python 3.4: Builds and tests
+build_macosx_34:
+ <<: *macosx_build_job
+ variables: &macosx_34_build_variables
+ <<: *macosx_build_variables
+ PYTHON_VER: "3.4"
+ WHEEL_TAG: "py3"
+ tags:
+ - conda-macosx
+
+test_macosx_34:
+ <<: *test_job
+ variables: *macosx_34_build_variables
+ dependencies:
+ - build_macosx_34
+ tags:
+ - conda-macosx
+
+
+# Mac OSX + Python 3.5: Builds and tests
+build_macosx_35:
+ <<: *macosx_build_job
+ variables: &macosx_35_build_variables
+ <<: *macosx_build_variables
+ PYTHON_VER: "3.5"
+ WHEEL_TAG: "py3"
+ tags:
+ - conda-macosx
+
+test_macosx_35:
+ <<: *test_job
+ variables: *macosx_35_build_variables
+ dependencies:
+ - build_macosx_35
+ tags:
+ - conda-macosx
\ No newline at end of file
diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index c6a2392..0000000
--- a/.travis.yml
+++ /dev/null
@@ -1,33 +0,0 @@
-language: python
-matrix:
- include:
- - python: 2.7
- env:
- - secure: CchM+63gNGQp44wkG/vzx52uHg6R2XrFaV9P5Anid88h2wiqQi8N6UYD7oF7cNMc+e/wSaa4sv802blsk+8HUv6e6RUIWHqXsDQM9iXWemAAwrT8zvL+Nc/CtTKWGBhVn+rtwr7eEZRcAUii+obDSxc5R7lJGMJzqxO5x6tl77LDEHNcSguF58RJQOifcabThYT7VnZkV4DqGzfaG5ngZwcIGF34zlkOTvtVwnmfM7EdPjzQitKlurM1w2ViyiClby9OO/nAk7T7HgqT6ZkzW0G9exdgeztTzqjtS9DL4FIDdm17cCDWEdLeCgXBO3DKJ9J/XNS0ND7WQooMSFSf8dLQTYMfSDP1AZxPeyb260hIVwMSyJxeS6l8HnAphuBuOo/f8gNteWNxShxcAC8uiVm/EcCLmrwLWSmfPjGyje0QRaJzQ45kQXEf25lm1uPlH3cyYCIn52Y2yP7vwDk1JIbk9G6E6oIZjRq4myZUJCfOzyATZUlZ/EzzZS4fnSxGc9YGOERxAc1E8VcEAq+WbGuNmYJidOGRm5NKzL7rvXWAkfvbSDdNZPZPETAPSKFAqLyYZbCAKzefAQRTyA6CQE8d9HBkEr5iNevlxbuVBF70cUcnsTyLaFpA3t+XvbiR3FhOGjLrfJ/yiZjJihl7bgYimIlntI8vAasgjUxBZ3Q=
- - secure: j4yg2mFK9jAzgB+aGJj9ieSGYq0iJBTopUSznd9HwT0xAwXGFkOhV2D8t+a0Ww1tuD8eYHKVfLBCMXV9ahelFbIBOM4ajMRsSJf8YcxdqaGFU/4BoKJd6G6chpz3HtScB/VGGO5uRLIqDQWgii6+pEAk3uj5csLp0amL0LCWdhCYHrh94NbeVsJvxPN4LTOTChKZjbo3yhUCPWuKTvbhPteE0NGuf1Ko6NqxaCzT9Iq11YbdNSmJmZnNbOqLkvFFQLdtzV3NXHFstep1YCRcDHLz/wNxkvh9iROUmrdT1b0D8IxexmwFMfVt8GsnrTRAe9jS7mQq7ccE90cJU2nHwh+pzAKyg/wmDdRBkDP08flEZprCUvYWGwoVv5E7GUGxG0voVdLJd2CD2NUHsvyvLZqY1R0EcRvc61aeVY9p/S/r2JzlzZjJTrdh3TUw0OcQ4cymM0VPXKU7N+x89KOhJMbGNwxibd4BtILY5A52nuTy+u1uiiUDFh96rFd3bl3ZwwmnY/CaOmclNILTpa9USsZGngn3Q+juoA8HPKZ7murNl+/mZpopb6tBqQNys5y9AHpX/wdWb36cupmysccmO54cmvfVIuVnOgxFMxlZpBhI4B/TGEeioED6gwJs092TaXBKs2eR5fVuwevZISQOpoH07cmp2bN7fwXQpgdVU6I=
- - BOB_DOCUMENTATION_SERVER=https://www.idiap.ch/software/bob/docs/latest/bioidiap/%s/master
- - BOB_UPLOAD_WHEEL="--universal"
- - python: 3.3
- - python: 3.4
- - python: 3.5
-before_install:
-# - sudo add-apt-repository -y ppa:biometrics/bob
-- sudo apt-get update -qq
-- sudo apt-get install -qq --force-yes dvipng texlive-latex-base texlive-latex-extra texlive-math-extra texlive-latex-recommended texlive-fonts-recommended
-- pip install --upgrade pip
-- pip install --find-links https://www.idiap.ch/software/bob/wheels/travis/ --use-wheel sphinx nose numpy scipy matplotlib coverage
-- pip install --find-links https://www.idiap.ch/software/bob/wheels/travis/ --use-wheel --pre -r requirements.txt coveralls
-install:
-- python bootstrap-buildout.py
-- ./bin/buildout buildout:develop=. buildout:extensions=bob.buildout buildout:auto-checkout=
-script:
-# - ./bin/python -c 'import pkg_resources; from bob.bio.base import get_config; print(get_config())'
-- ./bin/coverage run --source=gridtk ./bin/nosetests -sv
-- ./bin/sphinx-build -b doctest doc sphinx
-- ./bin/sphinx-build -b html doc sphinx
-after_success:
-- coveralls
-- wget https://raw.githubusercontent.com/bioidiap/bob.extension/master/scripts/upload-{sphinx,wheel}.sh
-- chmod a+x upload-sphinx.sh upload-wheel.sh
-- ./upload-sphinx.sh
-- ./upload-wheel.sh
diff --git a/LICENSE.txt b/COPYING
similarity index 100%
rename from LICENSE.txt
rename to COPYING
diff --git a/MANIFEST.in b/MANIFEST.in
index 79d2ceb..7455025 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,3 +1,3 @@
-include README.rst bootstrap-buildout.py buildout.cfg version.txt
+include README.rst bootstrap-buildout.py buildout.cfg develop.cfg version.txt
recursive-include doc *.rst
recursive-include gridtk *.sh
diff --git a/README.rst b/README.rst
index cd62d5a..6532218 100644
--- a/README.rst
+++ b/README.rst
@@ -1,286 +1,54 @@
+.. vim: set fileencoding=utf-8 :
+.. Tue 16 Aug 16:07:37 CEST 2016
.. image:: http://img.shields.io/badge/docs-stable-yellow.png
:target: http://pythonhosted.org/gridtk/index.html
.. image:: http://img.shields.io/badge/docs-latest-orange.png
:target: https://www.idiap.ch/software/bob/docs/latest/bioidiap/gridtk/master/index.html
-.. image:: http://travis-ci.org/bioidiap/gridtk.svg?branch=master
- :target: https://travis-ci.org/bioidiap/gridtk?branch=master
-.. image:: https://coveralls.io/repos/github/bioidiap/gridtk/badge.svg?branch=master
- :target: https://coveralls.io/github/bioidiap/gridtk?branch=master
-.. image:: https://img.shields.io/badge/github-master-0000c0.png
- :target: https://github.com/bioidiap/gridtk/tree/master
+.. image:: https://gitlab.idiap.ch/bob/gridtk/badges/master/build.svg
+ :target: https://gitlab.idiap.ch/bob/gridtk/commits/master
+.. image:: https://img.shields.io/badge/gitlab-project-0000c0.svg
+ :target: https://gitlab.idiap.ch/bob/gridtk
.. image:: http://img.shields.io/pypi/v/gridtk.png
:target: https://pypi.python.org/pypi/gridtk
.. image:: http://img.shields.io/pypi/dm/gridtk.png
:target: https://pypi.python.org/pypi/gridtk
+
======================
Parallel Job Manager
======================
-The Job Manager is python wrapper around SGE utilities like ``qsub``, ``qstat`` and ``qdel``.
-It interacts with these tools to submit and manage grid jobs making up a complete workflow ecosystem.
-Currently, it is set up to work with the SGE grid at Idiap, but it is also possible to modify it to be used in other SGE grids.
-
-Since version 1.0 there is also a local submission system introduced.
-Instead of sending jobs to the SGE grid, it executes them in parallel processes on the local machine, using a simple scheduling system.
+This package is part of the signal-processing and machine learning toolbox
+Bob_. It provides a set of python wrappers around SGE utilities like ``qsub``,
+``qstat`` and ``qdel``. It interacts with these tools to submit and manage
+grid jobs making up a complete workflow ecosystem. Currently, it is set up to
+work with the SGE grid at Idiap, but it is also possible to modify it to be
+used in other SGE grids.
-.. warning::
- The new version of gridtk was completely rewritten and is no longer compatible with older versions of gridtk.
- In particular, the database type has changed.
- If you still have old ``submitted.db``, ``success.db`` or ``failure.db`` databases, please use an older version of gridtk to handle them.
+Since version 1.0.x there is also a local submission system introduced. Instead
+of sending jobs to the SGE grid, it executes them in parallel processes on the
+local machine, using a simple scheduling system.
-.. warning::
- Though tested thoroughly, this version might still be unstable and the reported statuses of the grid jobs might be incorrect.
- If you are in doubt that the status is correct, please double-check with other grid utilities (like ``bin/grid qmon``).
- In case you found any problem, please report it using the `bug reporting system <http://github.com/bioidiap/gridtk/issues>`.
-.. note::
- In the current version, gridtk is compatible with python3.
- Anyways, due to limitations of the working environment, the grid functionality is not tested with python 3.
- However, with python 2.7 everything should work out fine.
+Installation
+------------
-This package uses the Buildout system to install it.
-Please call::
+Follow our `installation`_ instructions. Then, using the Python interpreter
+provided by the distribution, bootstrap and buildout this package::
$ python bootstrap-buildout.py
- $ bin/buildout
- $ bin/sphinx-build docs sphinx
- $ firefox sphinx/index.html
-
-to create and open the documentation including even more information than given in this README below.
-
-Submitting jobs to the SGE grid
-+++++++++++++++++++++++++++++++
-
-Every time you interact with the Job Manager, a local database file (normally named ``submitted.sql3``) is read or written so it preserves its state during decoupled calls.
-The database contains all information about jobs that is required for the Job Manager to:
-
-* submit jobs of any kind
-* probe for submitted jobs
-* query SGE for submitted jobs
-* identify problems with submitted jobs
-* cleanup logs from submitted jobs
-* easily re-submit jobs if problems occur
-* support for parametric (array) jobs
-* submit jobs with dependencies, which automatically get killed on failures
-
-Many of these features are also achievable using the stock SGE utilities, the Job Manager only makes it dead simple.
-
-If you really want to use the stock SGE utilities, the gridtk defines some wrapper scripts that allows to use ``qsub``, ``qstat`` and ``qdel`` without the need of the SETSHELL command.
-For example, you can easily use ``qstat.py`` to query the list of your jobs running in the SGE grid.
-
-
-Submitting a simple job
------------------------
-
-To interact with the Job Manager we use the ``jman`` utility.
-Make sure to have your shell environment setup to reach it w/o requiring to type-in the full path.
-The first task you may need to pursue is to submit jobs.
-Here is how::
-
- $ jman -vv submit myscript.py --help
- ... Added job '<Job: 1> : submitted -- /usr/bin/python myscript.py --help' to the database
- ... Submitted job '<Job: 6151645> : queued -- /usr/bin/python myscript.py --help' to the SGE grid.
-
-.. note::
-
- The command ``submit`` of the Job Manager will submit a job that will run in a python environment.
- It is not the only way to submit a job using the Job Manager.
- You can also use ``submit`` a job that considers the command as a self sufficient application.
- Read the full help message of ``jman`` for details and instructions.
-
-
-Submitting a parametric job
----------------------------
-
-Parametric or array jobs are jobs that execute the same way, except for the environment variable ``SGE_TASK_ID``, which changes for every job.
-This way, your program controls, which bit of the full job has to be executed in each (parallel) instance.
-It is great for forking thousands of jobs into the grid.
-
-The next example sends 10 copies of the ``myscript.py`` job to the grid with the same parameters.
-Only the variable ``SGE_TASK_ID`` changes between them::
-
- $ jman -vv submit -t 10 myscript.py --help
- ... Added job '<Job: 2> : submitted -- /usr/bin/python myscript.py --help' to the database
- ... Submitted job '<Job: 6151646> : queued -- /usr/bin/python myscript.py --help' to the SGE grid.
-
-The ``-t`` option in ``jman`` accepts different kinds of job array descriptions.
-Have a look at the help documentation for details with ``jman --help``.
-
-
-Probing for jobs
-----------------
-
-Once the job has been submitted you will noticed a database file (by default called ``submitted.sql3``) has been created in the current working directory.
-It contains the information for the job you just submitted::
-
- $ jman list
-
- job-id queue status job-name dependencies submitted command line
- ==================== ========= ============== ==================== ============================== ===========================================
- 6151645 all.q queued None [] /usr/bin/python myscript.py --help
- 6151646 [1-10:1] all.q queued None [] /usr/bin/python myscript.py --help
-
-From this dump you can see the SGE job identifier including the number of array jobs, the queue the job has been submitted to, the current status of the job in the SGE grid, the dependencies of the job and the command that was executed in the SGE grid.
-The ``list`` command from ``jman`` will show the current status of the job, which is updated automatically as soon as the grid job finishes.
-Several calls to ``list`` might end up in
-
-.. note::
-
- This feature is new since version 1.0.0. There is no need to refresh the
- database any more.
-
-
-Submitting dependent jobs
--------------------------
-
-Sometimes, the execution of one job might depend on the execution of another job.
-The JobManager can take care of this, simply by adding the id of the job that we have to wait for::
-
- $ jman -vv submit --dependencies 6151645 -- /usr/bin/python myscript.py --help
- ... Added job '<Job: 3> : submitted -- /usr/bin/python myscript.py --help' to the database
- ... Submitted job '<Job: 6151647> : queued -- /usr/bin/python myscript.py --help' to the SGE grid.
-
-Now, the new job will only be run after the first one finished.
-
-.. note::
-
- Please note the ``--`` between the list of dependencies and the command.
-
-
-Inspecting log files
---------------------
-
-If jobs finish, the result of the executed job will be shown in the ``list``.
-In case it is non-zero, might want to inspect the log files as follows::
-
- $ jman report --errors-only
- ...
- <Job: 6151646 - 'jman'> : failure (2) -- /usr/bin/python myscript.py --help
- /usr/bin/python: can't open file 'myscript.py': [Errno 2] No such file or directory
-
-Hopefully, that helps in debugging the problem!
-
-
-Re-submitting the job
----------------------
-
-If you are convinced the job did not work because of external conditions (e.g. temporary network outage), you may re-submit it, *exactly* like it was submitted the first time::
-
- $ jman -vv resubmit --job-id 6151645
- ... Deleting job '6151645'
- ... Submitted job '<Job: 6151673> : queued -- /usr/bin/python myscript.py --help' to the SGE grid.
-
-By default, the log files of the old job are deleted during re-submission.
-If for any reason you want to keep the old log files, use the ``--keep-logs`` option.
-Notice the new job identifier has changed as expected.
-
-
-Stopping a grid job
--------------------
-In case you found an error in the code of a grid job that is currently executing, you might want to kill the job in the grid.
-For this purpose, you can use the command::
-
- $ jman stop
-
-The job is removed from the grid, but all log files are still available.
-A common use case is to stop the grid job, fix the bugs, and re-submit it.
-
-
-Cleaning-up
------------
-
-If the job in question will not work no matter how many times we re-submit it, you may just want to clean it up and do something else.
-The Job Manager is here for you again::
-
- $ jman -vvv delete
- ... Deleting job '8258327' from the database.
-
-In case, jobs are still running or queued in the grid, they will be stopped before they are removed from the database.
-By default, all logs will be deleted with the job.
-Inspection on the current directory will now show you everything concerning the jobs is gone.
-
-
-New from version 1.0
-++++++++++++++++++++
-
-If you know the gridtk in versions below 1.0, you might experience some differences.
-The main advantages of the new version are:
-
-* When run in the grid, the jobs now register themselves in the database.
- There is no need to refresh the database by hand any more.
- This includes that the result (an integral value) of the job execution is available once the job is finished.
- Hence, there is no need to rely on the output of the error log any more.
-
- .. note::
- In case the job died in the grid, e.g., because of a timeout, this mechanism unfortunately still doesn't work.
- Please try to use ``jman -vv communicate`` to see if these kinds of errors happened.
-
-* Jobs are now stored in a proper .sql3 database.
- Additionally to the jobs, each array job now has its own SQL model, which allows to store status and results of each array job.
- To ``list`` the array jobs as well, please use the ``--print-array-jobs`` option.
-
-* In case you have submitted a long list of commands with inter-dependencies, the Job Manager can now kill waiting jobs in case a dependent job failed.
- Simply use the ``--stop-on-failure`` option during the submission of the jobs.
-
-* Now, the verbosity of the gridtk can be selected more detailed.
- Simply use the ``-v`` option several times to get 0: ERROR, 1: WARNING, 2: INFO, 3: DEBUG outputs.
- A good choose is probably the ``-vv`` option to enable INFO output.
- Please note that this is not propagated to the jobs that are run in the grid.
-
- .. note::
-
- The ``-v`` options must directly follow the ``jman`` command, and it has to be before the action (like ``submit`` or ``list``) is chosen.
- The ``--database`` is now also a default option, which has to be at the same position.
-
-* One important improvement is that you now have the possibility to execute the jobs **in parallel** on the **local machine**.
- Please see next section for details.
-
-Running jobs on the local machine
----------------------------------
-
-The JobManager is designed such that it supports mainly the same infrastructure when submitting jobs locally or in the SGE grid.
-To submit jobs locally, just add the ``--local`` option to the jman command::
-
- $ jman --local -vv submit /usr/bin/python myscript.py --help
-
-
-One important difference to the grid submission is that the jobs that are submitted to the local machine **do not run immediately**, but are only collected in the ``submitted.sql3`` database.
-To run the collected jobs using 4 parallel processes, simply use::
-
- $ jman --local -vv run-scheduler --parallel 4
-
-and all jobs that have not run yet are executed, keeping an eye on the dependencies.
-
-.. note::
-
- The scheduler will run until it is stopped using Ctrl-C.
- Hence, as soon as you submit new (local) jobs to the database, it will continue running these jobs.
- If you want the scheduler to stop after all scheduled jobs ran, please use the ``--die-when-finished`` option.
-
-Another difference is that by default, the jobs write their results into the command line and not into log files.
-If you want the log file behavior back, specify the log directory during the submission::
-
- $ jman --local -vv submit --log-dir logs myscript.py --help
-
-Of course, you can choose a different log directory (also for the SGE submission).
-
-Furthermore, the job identifiers during local submission usually start from 1 and increase.
-Also, during local re-submission, the job ID does not change.
-
-
-Using the local machine for debugging
--------------------------------------
-
-One possible use case for the local job submission is the re-submission of jobs to the local machine.
-In this case, you might re-submit the grid job locally::
+ $ ./bin/buildout
- $ jman --local -vv resubmit --job-id 6151646 --keep-logs
-(as mentioned above, no new ID is assigned) and run the local scheduler::
+Contact
+-------
- $ jman --local -vv run-scheduler --no-log-files --job-ids 6151646
+For questions or reporting issues to this software package, contact our
+development `mailing list`_.
-to print the output and the error to console instead of to log files.
+.. Place your references here:
+.. _bob: https://www.idiap.ch/software/bob
+.. _installation: https://gitlab.idiap.ch/bob/bob/wikis/Installation
+.. _mailing list: https://groups.google.com/forum/?fromgroups#!forum/bob-devel
diff --git a/bootstrap-buildout.py b/bootstrap-buildout.py
index a629566..a459921 100644
--- a/bootstrap-buildout.py
+++ b/bootstrap-buildout.py
@@ -25,7 +25,10 @@ import tempfile
from optparse import OptionParser
-tmpeggs = tempfile.mkdtemp()
+__version__ = '2015-07-01'
+# See zc.buildout's changelog if this version is up to date.
+
+tmpeggs = tempfile.mkdtemp(prefix='bootstrap-')
usage = '''\
[DESIRED PYTHON FOR BUILDOUT] bootstrap.py [options]
@@ -40,8 +43,9 @@ this script from going over the network.
'''
parser = OptionParser(usage=usage)
-parser.add_option("-v", "--version", help="use a specific zc.buildout version")
-
+parser.add_option("--version",
+ action="store_true", default=False,
+ help=("Return bootstrap.py version."))
parser.add_option("-t", "--accept-buildout-test-releases",
dest='accept_buildout_test_releases',
action="store_true", default=False,
@@ -59,25 +63,33 @@ parser.add_option("-f", "--find-links",
parser.add_option("--allow-site-packages",
action="store_true", default=False,
help=("Let bootstrap.py use existing site packages"))
+parser.add_option("--buildout-version",
+ help="Use a specific zc.buildout version")
parser.add_option("--setuptools-version",
- help="use a specific setuptools version")
-
+ help="Use a specific setuptools version")
+parser.add_option("--setuptools-to-dir",
+ help=("Allow for re-use of existing directory of "
+ "setuptools versions"))
options, args = parser.parse_args()
+if options.version:
+ print("bootstrap.py version %s" % __version__)
+ sys.exit(0)
+
######################################################################
# load/install setuptools
try:
- if options.allow_site_packages:
- import setuptools
- import pkg_resources
from urllib.request import urlopen
except ImportError:
from urllib2 import urlopen
ez = {}
-exec(urlopen('https://bootstrap.pypa.io/ez_setup.py').read(), ez)
+if os.path.exists('ez_setup.py'):
+ exec(open('ez_setup.py').read(), ez)
+else:
+ exec(urlopen('https://bootstrap.pypa.io/ez_setup.py').read(), ez)
if not options.allow_site_packages:
# ez_setup imports site, which adds site packages
@@ -88,12 +100,19 @@ if not options.allow_site_packages:
# We can't remove these reliably
if hasattr(site, 'getsitepackages'):
for sitepackage_path in site.getsitepackages():
- sys.path[:] = [x for x in sys.path if sitepackage_path not in x]
+ # Strip all site-packages directories from sys.path that
+ # are not sys.prefix; this is because on Windows
+ # sys.prefix is a site-package directory.
+ if sitepackage_path != sys.prefix:
+ sys.path[:] = [x for x in sys.path
+ if sitepackage_path not in x]
setup_args = dict(to_dir=tmpeggs, download_delay=0)
if options.setuptools_version is not None:
setup_args['version'] = options.setuptools_version
+if options.setuptools_to_dir is not None:
+ setup_args['to_dir'] = options.setuptools_to_dir
ez['use_setuptools'](**setup_args)
import setuptools
@@ -110,7 +129,12 @@ for path in sys.path:
ws = pkg_resources.working_set
+setuptools_path = ws.find(
+ pkg_resources.Requirement.parse('setuptools')).location
+
+# Fix sys.path here as easy_install.pth added before PYTHONPATH
cmd = [sys.executable, '-c',
+ 'import sys; sys.path[0:0] = [%r]; ' % setuptools_path +
'from setuptools.command.easy_install import main; main()',
'-mZqNxd', tmpeggs]
@@ -123,11 +147,8 @@ find_links = os.environ.get(
if find_links:
cmd.extend(['-f', find_links])
-setuptools_path = ws.find(
- pkg_resources.Requirement.parse('setuptools')).location
-
requirement = 'zc.buildout'
-version = options.version
+version = options.buildout_version
if version is None and not options.accept_buildout_test_releases:
# Figure out the most recent final version of zc.buildout.
import setuptools.package_index
@@ -167,7 +188,7 @@ if version:
cmd.append(requirement)
import subprocess
-if subprocess.call(cmd, env=dict(os.environ, PYTHONPATH=setuptools_path)) != 0:
+if subprocess.call(cmd) != 0:
raise Exception(
"Failed to execute command:\n%s" % repr(cmd)[1:-1])
diff --git a/buildout.cfg b/buildout.cfg
index 675be9c..87c3aa9 100644
--- a/buildout.cfg
+++ b/buildout.cfg
@@ -1,12 +1,14 @@
+; vim: set fileencoding=utf-8 :
+; Tue 16 Aug 16:07:37 CEST 2016
+
[buildout]
parts = scripts
develop = .
-
-; required packages: our package
-; optional package: ipdb (for debugging)
-eggs = gridtk
- ipdb
+eggs = gridtk
+extensions = bob.buildout
+newest = false
+verbose = true
[scripts]
recipe = bob.buildout:scripts
-
+dependent-scripts = true
\ No newline at end of file
diff --git a/doc/conf.py b/doc/conf.py
index 9a413ac..4a34a90 100644
--- a/doc/conf.py
+++ b/doc/conf.py
@@ -1,9 +1,5 @@
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
-# Manuel Guenther <manuel.guenther@idiap.ch>
-# Tue Nov 4 18:34:42 CET 2014
-#
-# Copyright (C) 2011-2014 Idiap Research Institute, Martigny, Switzerland
import os
import sys
@@ -11,33 +7,40 @@ import glob
import pkg_resources
-# If extensions (or modules to document with autodoc) are in another directory,
-# add these directories to sys.path here. If the directory is relative to the
-# documentation root, use os.path.abspath to make it absolute, like shown here.
-#sys.path.insert(0, os.path.abspath('.'))
-
# -- General configuration -----------------------------------------------------
# If your documentation needs a minimal Sphinx version, state it here.
-#needs_sphinx = '1.0'
+needs_sphinx = '1.3'
# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = [
- 'sphinx.ext.todo',
- 'sphinx.ext.coverage',
- 'sphinx.ext.pngmath',
- 'sphinx.ext.ifconfig',
- 'sphinx.ext.autodoc',
- 'sphinx.ext.autosummary',
- 'sphinx.ext.doctest',
- 'sphinx.ext.intersphinx',
- ]
+ 'sphinx.ext.todo',
+ 'sphinx.ext.coverage',
+ 'sphinx.ext.ifconfig',
+ 'sphinx.ext.autodoc',
+ 'sphinx.ext.autosummary',
+ 'sphinx.ext.doctest',
+ 'sphinx.ext.graphviz',
+ 'sphinx.ext.intersphinx',
+ 'sphinx.ext.napoleon',
+ 'sphinx.ext.viewcode',
+ ]
-# The viewcode extension appeared only on Sphinx >= 1.0.0
import sphinx
-if sphinx.__version__ >= "1.0":
- extensions.append('sphinx.ext.viewcode')
+if sphinx.__version__ >= "1.4.1":
+ extensions.append('sphinx.ext.imgmath')
+else:
+ extensions.append('sphinx.ext.pngmath')
+
+# Always includes todos
+todo_include_todos = True
+
+# Generates auto-summary automatically
+autosummary_generate = True
+
+# Create numbers on figures with captions
+numfig = True
# If we are on OSX, the 'dvipng' path maybe different
dvipng_osx = '/opt/local/libexec/texlive/binaries/dvipng'
@@ -56,12 +59,12 @@ source_suffix = '.rst'
master_doc = 'index'
# General information about the project.
-project = u'GridTk'
+project = u'gridtk'
import time
copyright = u'%s, Idiap Research Institute' % time.strftime('%Y')
# Grab the setup entry
-distribution = pkg_resources.require('gridtk')[0]
+distribution = pkg_resources.require(project)[0]
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
@@ -84,7 +87,7 @@ release = distribution.version
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
-exclude_patterns = []
+exclude_patterns = ['links.rst']
# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None
@@ -106,12 +109,18 @@ pygments_style = 'sphinx'
# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []
+# Some variables which are useful for generated material
+project_variable = project.replace('.', '_')
+short_description = u'Parallel Job Manager'
+owner = [u'Idiap Research Institute']
+
# -- Options for HTML output ---------------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
-html_theme = 'default'
+import sphinx_rtd_theme
+html_theme = 'sphinx_rtd_theme'
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
@@ -119,28 +128,28 @@ html_theme = 'default'
#html_theme_options = {}
# Add any paths that contain custom themes here, relative to this directory.
-#html_theme_path = []
+html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
# The name for this set of Sphinx documents. If None, it defaults to
# "<project> v<release> documentation".
#html_title = None
# A shorter title for the navigation bar. Default is the same as html_title.
-#html_short_title = None
+#html_short_title = project_variable
# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
-#html_logo = None
+html_logo = 'img/logo.png'
# The name of an image file (within the static path) to use as favicon of the
# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
-#html_favicon = None
+html_favicon = 'img/favicon.ico'
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
-# html_static_path = ['_static']
+#html_static_path = ['_static']
# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
@@ -184,68 +193,48 @@ html_theme = 'default'
#html_file_suffix = None
# Output file base name for HTML help builder.
-htmlhelp_basename = 'gridtk_doc'
-
-
-# -- Options for LaTeX output --------------------------------------------------
-
-# The paper size ('letter' or 'a4').
-#latex_paper_size = 'letter'
-
-# The font size ('10pt', '11pt' or '12pt').
-#latex_font_size = '10pt'
+htmlhelp_basename = project_variable + u'_doc'
-# Grouping the document tree into LaTeX files. List of tuples
-# (source start file, target name, title, author, documentclass [howto/manual]).
-latex_documents = [
- ('index', 'gridtk.tex', u'Grid TookKit Documentation',
- u'Idiap Research Institute', 'manual'),
-]
-# The name of an image file (relative to this directory) to place at the top of
-# the title page.
-#latex_logo = None
-
-# For "manual" documents, if this is true, then toplevel headings are parts,
-# not chapters.
-#latex_use_parts = False
-
-# If true, show page references after internal links.
-#latex_show_pagerefs = False
-
-# If true, show URL addresses after external links.
-#latex_show_urls = False
-
-# Additional stuff for the LaTeX preamble.
-#latex_preamble = ''
-
-# Documents to append as an appendix to all manuals.
-#latex_appendices = []
-
-# If false, no module index is generated.
-#latex_domain_indices = True
+# -- Post configuration --------------------------------------------------------
# Included after all input documents
rst_epilog = """
-.. Some variables
-.. |project| replace:: GridTk
-.. |current-year| date:: %Y
-"""
-
-# -- Options for manual page output --------------------------------------------
-
-# One entry per manual page. List of tuples
-# (source start file, name, description, authors, manual section).
-man_pages = [
- ('index', 'idiap', u'GridTk Documentation',
- [u'Idiap Research Institute'], 1)
-]
+.. |project| replace:: Bob
+.. |version| replace:: %s
+.. |current-year| date:: %%Y
+""" % (version,)
# Default processing flags for sphinx
-autoclass_content = 'both'
+autoclass_content = 'class'
autodoc_member_order = 'bysource'
-autodoc_default_flags = ['members', 'undoc-members', 'inherited-members', 'show-inheritance']
+autodoc_default_flags = [
+ 'members',
+ 'undoc-members',
+ 'inherited-members',
+ 'show-inheritance',
+ ]
+# For inter-documentation mapping:
+from bob.extension.utils import link_documentation
+intersphinx_mapping = link_documentation()
+
+# We want to remove all private (i.e. _. or __.__) members
+# that are not in the list of accepted functions
+accepted_private_functions = ['__array__']
+
+def member_function_test(app, what, name, obj, skip, options):
+ # test if we have a private function
+ if len(name) > 1 and name[0] == '_':
+ # test if this private function should be allowed
+ if name not in accepted_private_functions:
+ # omit privat functions that are not in the list of accepted private functions
+ return skip
+ else:
+ # test if the method is documented
+ if not hasattr(obj, '__doc__') or not obj.__doc__:
+ return skip
+ return False
def setup(app):
- pass
+ app.connect('autodoc-skip-member', member_function_test)
diff --git a/doc/img/favicon.ico b/doc/img/favicon.ico
new file mode 100644
index 0000000000000000000000000000000000000000..4cc3264302627d40868261add69eb755856611b6
GIT binary patch
literal 4286
zcmZQzU}RuqP*4ET3Jfa*7#PGD7#K7d7#I{77#JKFAmR)lAi%)D#|Xh7E(rhs|DS<W
z%)r3FDZt5@C&<G!Rfvycs~8X0Wl?_4{Sy2<QzZnsvZMrgZ5S9Bm`OF481?*|oWX+J
z?3aXiIR6Xta{d?L<@ztm$Ms)~pZmW!KlgtL0iOSoAS}rHTuO+y1mt~U%q2#RAU9`)
z5D({HVIGjZoc~1;c7K%M=Q<-H!1DuSx0E2ye`z7!|1v_n|7C^wb}NhX=n!K*J~bj-
zTt>n?93Mq^k?iH;{x8nQvw@$VUl62MmY-i)il4VsT8QV7j4&U_?*DQkeE;Qz`QIvv
z@t0B(79zad`@rGM#{~{&P*{ueb3YdsXW$0ejf`bPc<kkb`QFKk@c&m3<^QiJD)3)P
zOyIwYsNfW2b;Pm-1Ox=c`MJMIfa8}N8oxaMr3865;W9%(ls{Zil>fV;7{qR6al!v8
z;)4HFB?YT->A@u~#n101DG0KU=fAWdFFbCgMfhrQ=>dr=i3+wu?ftJRA@pBOQs}?B
zq|i?d84(ANJh50-m?s;WmLPE|%m;~AQQl;HW+;k@iK|KozJl8OUqedxzowM%e{E^e
zV+;%oO!)L7%PENPMS|iIghBBM!q7NvL{^85tu84%0c5Y1wD5mz8Ik|GvLgTWWkvrR
z$cd+7Q;$tdO@>oiNlf4;C=Dr#3H*nJsfxJZ3T*0;#kAyvv~^`f{^^136#Z`~FZ$nD
zLF~VYqUaiAb%fZel7bgNaR|bo@C0FXNuhVTa*(`-Prs48$P=)g3S$4w6vh6VD~bQN
zR1$w{EG5N-Pam?JhLq4eP&k6HrnK;XEol*OcxuZEXCte_#<oxvKVSi}Q(64KwTk$E
z8x@KFcB&Hp?KPw=v8l%<rYj?^t0yb+85DQ=a-jHwgr$+Z=zU`;DG_XXU}82Z602=h
zCH~u~N&I(Ell<?fF8SX{L-N0ihD;Dl4KcK_f_S^Bg4lmkMMOA){AHmmF&&?NM|H{5
zAp4v(r2e~VO8t*El>T3BEAzk8Nh%SaK6E*2brD``Rk0h^DiYu@1p5i%ub&PY5?J%J
zx2~L0imBA6278(Roz61<J6&b|cezXd@Ai=X-|H!5h;9cVzN4yC6gd3UCI35XfUwkm
z7fq@Eu3FNkLPIg@+%9+NHDLQZrT_PMOaJfnk^bN7C-uM2U+N(tyKu=@+sSU~aDw{B
z1?nd^>Hpns(%`bSp@D&^*H_x2*GFncuP@lH|NQ|{|NDca{!a*&{68^N^8chz$qgX$
zyIf^bI$UH<p<q{;Qys1{r@GvwQE<11^r>!7=~G>v(l<bU0b#IzKz2a<^r=5U>imR2
zsW%gXrT$L{f!Y-=`F~P`<p0T$lK&@1OZ=Y_EpdNxq?888{sspImR=vJ;~;mUV32tr
z><@8&AISZIQvdrw80-&-`7k>shDrXP1hXef0%A{$#Q&+W691>hN&KIdAn|`%qWJ%5
zNn%f@rAQis?1tfqp;9&zLnXg~!Ubdw41>*snhAG5$m}SvAO25?k@!CaYCf_Z)04#i
zPfr&AKO;r_|BO`e|1;CX|IbVp`#CF9Z2i1!adVix$n?~BiJqx(5}&|0UgFc#c!^Kb
z5|D7B_@`-!;-97`iGP}&48b!}#6QhQ75@t{U;O{f46*;SGR6MS$`boOJ4@{KoE-7f
zb8^Mj&dC$&oSQ4Ak8B?Xn}LCGeu4rAsd#p-c>kO{(f@PvMgPw&5dA-|Q1t)2BGLZ~
zN<>#;*g-~Hmyj7ta)tO7lnY;9ST6E^VTH*5g_R=z7gdS;UsNsfZ*i?~93lPGkYCy$
z9JHiC`0vsN;r~k;h5s*W68gWaS?K?=7NG}gTSX+PVFw{IR<sK)S=lM{e`S~8|CQZ>
z|5t%<kKq3`eFBRJ>7|DJ+DSq(t0xFNSvyJK|Jum{|JO|s_`hx{82?>6LnwqAcHlB&
z?M!~74KoDdHq7K-v2hmv|BbWx|8Jbb|9{h5{{Nfi@%`UCkMH*81p;wkJfBY$mpSB$
z@88eBvSlHE>z0LlU$-vi`wzxTc>ix(%KLxYGT#5&m-GJLzJmAvwpF|*7A?YTuaj#p
zy1uO|dF*zq;W@i=4bT6bYaw{oI-dW#*7N+|y@BWTuJzmo=z6HlU$u&XbMIE}346D4
z|Jb{g`#%)_+q;!3h1z!EGjHE^Zuh<0xUYch-?No_4nDQClH0b8fp6b-?za7_7<h2n
z@q_XI{|^lH{~s{e|39F>|NlS*|Nn+R?Ef47G5>G)$N0bDKLj7BZ}<<!_6`3ZfUrEn
z{}23(|9@~Wg8G%9{(b`k1N#962KfUFj8N+u>i_>g0K)eF|2MEh@B{w;|3Aq8|NqDS
I|NnpX0RF|L5C8xG
literal 0
HcmV?d00001
diff --git a/doc/img/logo.png b/doc/img/logo.png
new file mode 100644
index 0000000000000000000000000000000000000000..b60858a7068bf45c1ed8e3da12fe244ccdcfe85d
GIT binary patch
literal 6266
zcmeAS@N?(olHy`uVBq!ia0y~yVA##Tz>vbh!oa{#^08Bzfq{W3$=lt9;eUJonf(k5
z44efXk;M!QOfx{3(d@GOeFg>w_7YEDSM~=S+}t7z@=~s03=9(7o-U3d6}R5bt(+4R
zI`w${^Q&>OSF@`;ms@j3HVClrw5Xm)n<6L?8N#Hr=}bE#W1wT3V9KJJV~nLwn46Ro
z+WKo;%vdM-vM@C{MND$=P*T+1yJYgalI#2SS3m1t^{VRYt9#+=cUv{i^Iv7>8M^oB
zyzl$I-zh$C`~2SZBR9U!Iw5{_p(yj5O?F<l*|VG$Zm)egt2poI#H)S9@0?8&PyC(x
zJGR#S`9;gx>>0l*F5A7SX0CcIDK1~;nr!~y(8=X_f7JiZDvy@gdw!cgLs9<kp7r-`
z?!56<JMZSMr`z5?F5KFBzkP4ni(o(N_aEMyoHfW3{&i|9f9(17Z;Xwq*>)~p`FfT0
z?!M1IeCKsNzTmt)&g1W&hjwp-`S(73vROR#whgxcgU0@~8_qx9zUjm7o--8_cQ;=?
z_q*~t=R5|V6PaIT<VNT17p$KW`J?v6arV7Gms}RVd%@=RhNCSq3LJmePVM*5{S#+q
z6q0UP;$4&{q)`6LN`K$l>`wg^(;cTbl<QcOymmF>-XFg4a*RyE#^?lAyN5SA4c?wO
zFZcQHvzsnV*LLv<*!-L9cXvnYyXXCOX3CpO{(V_qn!j_;CH-Y5ZvFOu_(om2diL+`
z^6m8(FZ*~s3AxBrw&wRw(?8SuJ#TmJ+iPPmnRmr$ZG~OGuS(m$ywK>acWL*g_0L%t
z*1xux?!NSG%Ln&$<)#cRRp%r7`qz5z{qjrG+)~E%PTI;{|IS;_+cC@j<6rB*k5cPv
zR{aby{v`Cx<;m3b`#yd-E;sq-p)Eh}tF&u8c*d1`f9tzW`41`utGcdN9NZ~8Uo`8q
zUe)}lpL;?rt2uw4xxdT$i_znx6dC!-ztS@EY~DQ<{ObPK<MXS4$~T8+%kMdLE8(5!
z8}XP+Rckbwb6xbm%xgE>V>olqH~+h@=P$kXHKgQgY1qZ%dt3hAc;>v1e|y}+y(0GI
z^&g&m)luXBvUz&RoJHOff)Db%{c&2_KU6zeKVklN>GLItO>w{MC99ru@%hTV*lgqd
zu7u5{wLPw;b-P_%Wods-e{y`4+w12oC#OwcuzBC(kEQjqeb~}IzTH>3>&Ss8`%@pi
z&hy~E^`Yb2n%@NpcMVs+ezxg||M~x1wI3c_v@xn&Fn`yE&#eB7PZg%TtN+BGUc9Wz
zvf85c@5}ff&$y=C`}HE)K|!$cfAnNdtIV_CEv37|*N5K;&G`L!`pw3D?Y(DSc1NEN
z+1bk@W<9^^ee21TgnfVR-(z*M)H{2`{H}=PhRDY1b0@a!@{0@I?r^PF%xb=GzwF7w
zli7cL+`N4MyUfs^uMV6`|5qZm{O#1O>|6i-m|yYybr3%rgV99Ya38f74!@b#g>7AT
zeB0m0_un{~CoUJOP=BSwKfmn2Pbcm7PTem!&sfLBZk@KwSMJ5~Ik)$0crh{5?fT}E
z{O_cE0-1|EHrlBv8}EPD@cS#9S>%IL4@AQoru@yjp>MF8W9OFjJM3lO{%@CF6Ix}F
zrg?RHnDOM4>eW{#>bC#5m9o#O(Xr!g%>NJTdUvx=JYv8$(Sc90<*I1Hfqz@?o6SGZ
zwnFFNxl8fKjJa;DS)n6$ReV}sMBv4bQ7uc4y=uR4%l*<@)w^k#YMl%(R^}K@KjCIP
z@yv&HU!}u7r{y>Y*@?&;t$X(9TKb)@F>UK*#JtbAevDVmcr3W&MMz{<|M7->w+u~|
zy;68-nNld?xw`Ry(njlBo1Y#%`#wRn&2rHJzjL3%ugd0Ji;5JTa*J{1s>2m+Tlbwl
z%o6q@`})_cs{yi>n#@coKaMZB_`b^Q&i0o(d-W4kf1O_J$NDDyfBxlikL1qEi3hi>
zVyT>R@s8V?51Sqp?-6w{`#$+m#UzFBM*a0HY>ww!s;}9Yd|=Np@VleHwjt&Jy8XMO
zpFDGoWBnOlbmiImH7`n@+O|mjbm=!Z!L)bH>n)34f70@3Gq?G9uxL-Pk=YT3bG3Wl
zY0Rut^gJGw8I!Ur|6SVK3fJV<5tU6oJ*DYct#>Mx?G!o_I>9&R|8MK^hn0Q(zuu<D
zTrb%X@M@dPHmR+#wcB4sJU#HeZQJR66Lmi?x>m~c;KO;FgMu~2T}}o^Dx-q`h8A$C
zzI#-jczNQQ9L}pc|88uV`<<!4!IXD#k+$cTc`3SwuU5Wz)1ASc&&ZH|`tB}u<0aQh
zcYfEq?`5}^&#QX9x?sCIci++F7lY@tS$;n9M7{gsmU%8-+!Idg)iF)*4&YkERJ8Y*
z+R~dBm8V>7dZM=U-t_;M_7wZQi+BBAAfjL)*kUKxTNZYCj(KcuQSu>yU+d!cls}&H
zYg_BN#VcnoyI8Y!>n?xc_x0tGw;xBg7uFxX^Wbt{Tg`7_(OicVw`oV;Wxx6$w^(-p
zU#^y{OYhGg+rk<a^eSCkAX?{Yw*DVqoL+E+-ItzY)u#QYmZcw_eV%je9CPi0EoBoH
zGY89ZZ)rHQXG-{i3+?OwTJPtvux#h-s+j*{!De5NTr1{|`XBd<`V^Yt{%|t%&3wPh
z*IMpf(Eot0Q=eb_ugbo=v6jF1ibnc5;gf7%Lzi6pa`#{%Pg3r`?`JCCrIv7+9y&B5
z<ALA*MMpLtlblf;yHEPYVxN13PuAP&7w{=ZOub)lN9jB7Eupjilf1M-9csfrR8+NA
z9%9;md`{T+70*ITm*0GGZK>_~S?4Vu2^E^kPVkp7C|UW{FYn(ABQuw$>2dG&`Q|!&
zdDy7B@#?d+#s6FLmRDXgv)i`q{p&r;zEpqHuf4BevB0sJ`DcFLMdP5`_ieQT1*7d>
zD!pxwG7l};mvZ6fzp1OIefq|EO0U!+#$Iu~;ccZJNr}EaSF}Ad_LSy7>Ud$zxn!bi
zUdZE}Z{EK7P;*S+aDKrb_LyUTQ+I0H{`08uT{?$<{$b8X^Lak+zOqYVo?d%mWb$>J
z+)Td6B@-NY1X~Py?%wE4Y!P%evHWnkMP=c~OGW>Z*7?it&ENax)u;W;mkWQisD4fR
zd!y80ser7cis0dd_f@WDwaz^8LeY_3Ht}6y*9&xdSEb%vzeVzCP)~Y_8Gnra{OE0k
z?D1>f3r)(>^(r!9<-HO6J+Sy^U7Legri#GP3#)SXPb{t|<c*)(J#(V;{_5yi;(h|(
zSnsamF3}R3bA6(L8`mvCU;ckOm)Bc;Z+`njbu&-p$KBUj4xL(XPWfes<hDQ2{jUFl
z=gz$Q#p++g%ExY<?_M5#HMeeJ^2A79ult*Kx9L4z;&rae{m1%ac3$S840qPD`v`n{
z@b}gfnV=5QgmwW(rk70*J=WbTU4QlFBeUAMk;i)-R`9)LaCq`ZmG$Z}rL9VTwxqs!
z#<p?uC!6LuPG7hEkTpM)+ZOvruc)&<>3#G`&i~;mlN^q*>{;@8p4lZehTl#4%f<T>
zSQL*;aM%@adNFT(#tS3Ah`IYU+s)UW;rV%imtpSp%=Q^;D_{2HyuToNE%LYS;XnI)
zPIuqdXy9Y`aNl$Dlec~j3ZEYyFv^z8zSp1ImmqPNL$OVzNKL+%X~lZW>9^iw{*=GF
zxJ!!5_GiGOy{j&MvRl1Tqw*q)Pv*zk$+BAey_X1YPp@8e>SmnQY%RAVPn@;Q%4ObI
zcDo0cJ7_wyF;uCP<bJix`6DRLmNL;*s)5<h;$h3`2b>e-8)t1azxQf}_7uh2XWE(n
zht@J}jYt)Jmc(Av@Z0>#N<EF;RZsM!PfnQUCCT=OFNNXx-YWr8vjQzE{Pv4f-ea56
z-xEH|Sx<ht=7MymU2bnKKl^zuF}1};eCw+xJRD&~Hv4Dzu!W^;*j(G0&8eL{A*uYw
zr=*<oLFcmBj!xYo@x^!VrL6%TU*|ABscK)hW%Hw!?PBMRyG|>%n>3U#O_bVL{qFLt
zNXdI%>pASRmrr?Xy~{mdIagb!37hQchv95TzFb+n(zNVJ$9BP`Rgq5x876)(j6Hqs
z)L+(?l}B=K+F2L<2<6zic~WO_?k&5=HP@B4u1jB4R>Y=oVaiIri8*_l_)=2yG`D9y
zVVkY(SoA80Ln*9&w%J_YXG^E#onIrl^Y*>FA?KzWX6=<*S6DTvEcf2UNvxY))avrL
zU5~Sj_4^TN>DnxOt~erisilS7{*&)a)brBkYq=bFz*FluH9+yj;f47(4O18n6fao0
zHgel`wKv7fF3Oau*zQ&?>AyQ^$JP$#WgnxMm#vy=d3|!lrEPQac0@Fa7L)`epL{Z-
z|8$<u`c1m&+m71VUJVxhc4xb(yta^d&l@Yb4Yf0kA10=9HNRR|B6D!&bj!5pb4;Nc
zJ&cU<-abx+d=8Hie`_Cc-ENhucWGv+Y2tI?$eBw7KAy1s<}H*yX<3ni>zS;!84{hn
zuICJAT?!Oc7BMNA|LnxGb4QYsClqf`+xm)Og6z3<YZ!m8ahrAglu7J1^UtRwv<em;
z-Pdw$k(uEyvp9P;OZLr1$5|G&bu~tvEWOiUw>xN=@cGAGe3c@<)xKP9n7hAk!JRz}
z4$t0hT;u+za*2c%qk=_<obBtC>2J?(%FT8#_x9moZg^l}?(O6M&sehlz`s?Ig$B27
zv{aYG#i)I&ekSFVXd|`VA+m=#;l;-jo4<Ky{k18TIN!d|#a7Sk^fvJ%m4#2Zeiwc5
zzOX>co4@n!rng6*cs^Tecw*<GDS`&mr^$BNGhbQocG27gUUtP^Z)BVU@1KotJ#+j*
zdDfbFvT8vuB%2q_IW6kEAn^Nw^Dl!XB^-`yJp66P+yKLQdYN^3!S^D1qi?&q?`)GU
zb@jiwy(QOC=3J_=XisC<M>X>)*>RF53?F2~Iehkg+;}Pa`h2GiF~?88)9YH1`~O|e
ztiYRkpPzR4Ji1<~D^>qs`_UU6r&v2%YGr=$uk@U^jlre<PxaA0zi(oKH_GMeIr5jb
zBpqLK|1{%IQwAPEsl!E2Pd{8S$1;k;Z+*Az#QS#t^roKao8c(GxZ{Sl>VwZUt_x2-
zh<Vm{XIq-@&6`c`Z^}vz>OJp_dAMI>o%PIpN2e#W?}|R3yWKnWl;OEnovBU&6JKmS
zYANve`|ZO;Pk0>sZ(QZOzV^z}S+6zBj^#6+X;yq}_-n!4?yK({g}v99F+2%Vm;bVK
z;uEW>(<g7vHQE{-nEuATN4fD#*O3*$wsmimU%mYQL^XeFx)rm-hP!`LjZX!wQLPnj
zXZShUoXg$x=9RC?TOWrwD;m!EWv1S+%wpLBB@+jR1(~{LrL`*`8zlQ|&at>AziXAr
zIobWW{GsRS<Xwd}upcSpUpAw<?c$C{(ks;&8r&9a*vYVU0`FEu^}B&R&%&o1{&o3g
zIYWbT8vCX%v9mK@=hgghEZjA9CU5%X)0^9#xAxx8zpv?jj^RZ2x>u@K;#cT49;z0&
zeR=vbcBehg?=~0jtBPKIdS$d*`${dp`xADk<qND7pYYqL;hK|Kn1R{7w>RS5m>G)l
zTkB%ddXBAq+pn}*$SLm3%CcSc+rBMMdUdE~b5i=V?3rg<gW{)#^E5YF$%KTSOWM^m
z*XVzX;<lydgj)YKhM%6u%yT8$?xH^b-<r2;7R^pvwrD+n{n8qzhd(y%SKW4cg0aMv
zH+Iu%rt#W8sET;q^2M!Er8xA{){D;ag1bZIOt)sZPkbtC5xw<k#-#qnqYl6JoMB&F
z{-N4$<DnB4w=x5#o|^l3N9(I6Y2i8D^YZ4Yo8-N*o^fxgB0ED-wf}*GVi%;FzGt>i
zy&tuaMezK+DMl{84+%!`uiSpt;lS7RH_kixb^i`LGnJj?NWr&HU+j5`?mhRN5W>e0
zF!?#pCWG{Ie(#f`)V;-}I~$K&N;#_hC3~j&W|_<oh4b}R%P)0aS=Mk-W=FiW@T`&*
zQ%~KU`p0$W@00ass-GLZ_A&kt8B}sVYWMd-tFXLz7PF$K)g3B1^ts42!C>dJWKPzn
zMR$yw&p%23Fhi5!*SjZ6H=3*RWX=3|`=5<ndzVIv;+Nf;McngJCa3$>W%1p4Z*g$O
z4ud-hPBs&E+_IUex!_^0%rXfh&cA=ZoUJ$~^L~lK)++gh{-=$yS@^o=FU>PN&&a{H
zVb-%5izl?^?>sa6<opM&pEa7KVvJwO6~2kD`?BJ?{gl0Wdt%I%tmgflR5z_@Rja{F
z-;1+9$^>~7GAY={e5gM6A!eOy$g8TJ7nK2jG(KuaDr|q3bceaU<^;>a^=I$&gimmr
z6wL7Rq1%Dd#>Y;xl>65f&)=c-cZp88$REaOE5q~)&3?OmT3#WXsbcBNVt;4F_xP7_
zt4dFcPwmv5$-Q{}$>SGI&*kUu<`)-Dd*|WsY+LF)zx`)-Z(xn{DV2X==|4NZW`@gA
zp;^8+xr6J&ts2iiOu2eW?M2e$e@p)9v2EmaEV1A(pW|EXCAUR$*$%VsQ{S+61+=z@
zJ$T{uq}6QF)$6JM=EU~LzHk(_s_BkRxBAxXylack_3&foJ+&V%-kNm%hosfbq@ybz
ztdI~>T+7HSV#23(%C7Y9*DpIQyDQo1?Ekg3zH&_!ka>2uBT!jNgJHT-^OT4CQ}o1~
z#S9mf#VwT8xcu!<x?f9WN6-}yUJ;e{KNfx6?M~sFU1L@~yDF1gXU6!@u&ztMe#(x%
z)hy?0Vk0KX^R7s5_sri=a!2o9=(pq7E+;e|SXjYz`O0aPtJ&Y&&IabcTXrIU?dCY)
z4IRt2cFG^~biHkD_Co2L_Se;yU%yFty7%|P?-45QXQv%tY^iA75q3XGJxS7n>HEV6
zSqGjzsGKn0GiLs~^->#V%-HI!YkoO3W6hS2(_Z@rC-wf@d*$`F%?sny-#uJ<)Rk@9
zdYd-$BjsO9zp2Nt{P`j4*Ib<QKu~D?lKr!9<t)!q&p8=!I=#SFOj+Y|mP&WC`-Z^(
zk)K{>`R@Dl^rz6eC&~{ux6U(c`MSra{&3yZzOqNtFS-6GDJ+Z<{m@kYare#ls@HC<
zy~SDU68(7j7U=?!QnBUvUJtnC9`ODN`c^)(apPi%-b?cS*FQLwIQ;#xROZ)4x$_;z
zD>DB0x30JD);p|tg)1nlf^F}fY5%9xXqlf$j<{?6(Er$H>)7tCmp9rkD`c>%c=$-<
zQHRnQWeG9;W%I7Kt3T@8$0t9xCD=gxEyscLauyR@m;_r4niBs+?&<m9Vl(T`<p+Ok
zbQfCQabx`PK#`a8h~QFzHp3P6QuW=+$@-=Jwi70B<XOblxhn_@IY^zf{ajOa=|S7%
zR}UTw3K%7uN1Hvhc+hlNR>9wj#p#rvRQmd4b#<u=Eov1HnzZeE1zsMq?mKu(@0P;x
zRyC7_jB-03oUylCprG2U!S>eg%Q~&&&1xnK<??oLEpEzj^pZ;C>HVtz;(}a7%n1=6
z_vESVVy=cAihb9v%(`{4_m@^|mV~p!CTqv2g-?Sow(im@ax>^q{C&z~;SJ6g4(5Nh
z`FkX@S4GORf69E(bo?8q=a;18H+p3(FZBBD$eAzT%wP8L-cdijGh1henC;p0;^2XZ
z2@}MBD;uO)YSy$I+;~U#>cfUYn?94iw^y%c`%08?ISJKm>Rlqf`)w?zX|lb=F-`SG
zx!u7@D`hVqpWEB9;CR)wzbuK%8J^v$da+1!lI~S&hpa89)NZ<LT3f~L@pi>m#~p`%
zc)Zo{=O`;m?mOBeEz5ai0>eRW{-#rA3zMrwWj`iwbA9?uz+HyZ;z29>*HhZ35_bQZ
z$;jZqJ0meu>~V)Or}~}`3mk8%1>IN{w1o43pVYU_2Kt6yBy`OMr7o^=*%bWOJmy4d
zW}MrG=F{s}N$Pezmi+t5-ohpB%%&B0Vr^1hl`UW3*%;@({!aHSjyJj)8x3S@r`}Z+
zT&7`tU!&}lw0dd8j&4Do4QGTi!^`$5O9Z9V?OHhRtHBxPOMzbPm+GY_tyNwlX>EC7
zZBAO|m(?{QSuUF|alBbkqIELY@Tq3s+%<BpSG4bXb+WtK`GlV0TzyMP@Ya-lwQDY|
zTq9d8+$6?plzNkQ^}+Wh%nLQe9yivl_M0T7dmwzrlq(BOX7t!~`w3+PHucYnJtZZ+
zU-&`bzuwC-I_|kL${GzB8H*Cl+~Y!uE?t|)+1?@Q5R>6>BSp_-sm$rhGd#C=!+!j<
zJ2lOkd#hdIVNS-YM~|%9wnSnM*WDteh;2=WQa`fitk&1<IIwU>F#px5rB!*{mZ45O
zJX{MS6AS)MG2Zs)*2d>LZU0tOElga>{7h;7!Lyqu1z)|LzRcj2=)KPNqP7$Joi*E6
zAAcJe;E{RZ&ywk0D@1pvH6C^gnL9JdYxdhkcV4tkV&#**EBsm{_?NoerO#<ypOyuh
zuG;YNk<8Jl8<q55x*CUSmjz1mH55+?iff%*)EabU*_pIcEt{VNRK@6>vafE|d2oyO
z#?pnFd+Wk4GF|D44Ug-q71zuzT*D@-pb+gnTZ;A3mI=yS*Mk<zur0mXY;}oi^ULf)
zg_EEB8E^D!2}Ryo=&5#m*OVGV6V87^pZ*`7zK#Fj@sGdk>#m&_J^Ek0jDdlH!PC{x
JWt~$(69D)+N{avh
literal 0
HcmV?d00001
diff --git a/doc/index.rst b/doc/index.rst
index 1ff97b2..2a40458 100644
--- a/doc/index.rst
+++ b/doc/index.rst
@@ -4,15 +4,18 @@
.. _gridtk:
-Parallel Execution of Jobs in the SGE Grid or on a Local Machine
-================================================================
+======================
+ Parallel Job Manager
+======================
-The GridTK serves as a tool to submit jobs and keep track of their dependencies and their current statuses.
-These jobs can either be submitted to an SGE grid, or to be run in parallel on the local machine.
+The GridTK serves as a tool to submit jobs and keep track of their dependencies
+and their current statuses. These jobs can either be submitted to an SGE grid,
+or to be run in parallel on the local machine.
-There are two main ways to interact with the GridTK.
-The easiest way is surely to use the command line interface, for details please read the :ref:`command_line` section.
-It is also possible to use the GridTK in a program, the developer interface is described in the :ref:`developer` section.
+There are two main ways to interact with the GridTK. The easiest way is surely
+to use the command line interface, for details please read the
+:ref:`command_line` section. It is also possible to use the GridTK in a
+program, the developer interface is described in the :ref:`developer` section.
Contents:
@@ -29,4 +32,3 @@ Indices and tables
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`
-
diff --git a/doc/manual.rst b/doc/manual.rst
index 532eae3..a0af273 100644
--- a/doc/manual.rst
+++ b/doc/manual.rst
@@ -4,9 +4,9 @@
.. _command_line:
-==========================
-The Command Line Interface
-==========================
+============================
+ The Command Line Interface
+============================
The command line interface requires the package to be installed properly.
Fortunately, this is easy to be done, using the Buildout tools in the main directory of GridTK:
@@ -16,18 +16,26 @@ Fortunately, this is easy to be done, using the Buildout tools in the main direc
$ python boostrap.py
$ ./bin/buildout
-These two commands will download all required dependencies and create a ``bin`` directory containing all the command line utilities that we will need in this section.
-To verify the installation, you can call out nose tests:
+These two commands will download all required dependencies and create a ``bin``
+directory containing all the command line utilities that we will need in this
+section. To verify the installation, you can call out nose tests:
.. code-block:: sh
- $ ./bin/nosetests -v
+ $ ./bin/nosetests -sv
+
+To build the package documentation, do:
+
+.. code-block:: sh
+
+ $ ./bin/sphinx-build docs sphinx
The Job Manager
===============
-The most important utility is the Job Manager ``bin/jman``.
-This Job Manager can be used to:
+
+The most important utility is the Job Manager ``bin/jman``. This Job Manager
+can be used to:
* submit jobs
* probe for submitted jobs
@@ -36,71 +44,82 @@ This Job Manager can be used to:
* easily re-submit jobs if problems occur
* support for parametric (array) jobs
-The Job Manager has a common set of parameters, which will be explained in the next section.
-Additionally, several commands can be issued, each of which has its own set of options.
-These commands will be explained afterwards.
+The Job Manager has a common set of parameters, which will be explained in the
+next section. Additionally, several commands can be issued, each of which has
+its own set of options. These commands will be explained afterwards.
+
Basic Job Manager Parameters
----------------------------
-There are two versions of Job Managers: One that submits jobs to the SGE grid, and one that submits jobs so that they are run in parallel on the local machine.
-By default, the SGE manager is engaged.
-If you don't have access to the SGE grid, or you want to submit locally, please issue the ``bin/jman --local`` (or shortly ``bin/jman -l``) command.
-To keep track of the submitted jobs, an SQL3 database is written.
-This database is by default called ``submitted.sql3`` and put in the current directory, but this can be changed using the ``bin/jman --database`` (``bin/jman -d``) flag.
+There are two versions of Job Managers: One that submits jobs to the SGE grid,
+and one that submits jobs so that they are run in parallel on the local
+machine. By default, the SGE manager is engaged. If you don't have access to
+the SGE grid, or you want to submit locally, please issue the ``bin/jman
+--local`` (or shortly ``bin/jman -l``) command.
+
+To keep track of the submitted jobs, an SQL3 database is written. This
+database is by default called ``submitted.sql3`` and put in the current
+directory, but this can be changed using the ``bin/jman --database``
+(``bin/jman -d``) flag.
Normally, the Job Manager acts silently, and only error messages are reported.
-To make the Job Manager more verbose, you can use the ``--verbose`` (``-v``) option several times, to increase the verbosity level to 1) WARNING, 2) INFO, 3) DEBUG.
+To make the Job Manager more verbose, you can use the ``--verbose`` (``-v``)
+option several times, to increase the verbosity level to 1) WARNING, 2) INFO,
+3) DEBUG.
Submitting Jobs
---------------
+
To submit a job, the ``bin/jman submit`` command is used.
The simplest way to submit a job to be run in the SGE grid is:
.. code-block:: sh
- $ bin/jman -vv submit myscript.py
+ $ bin/jman -vv submit myscript.py
This command will create an SQL3 database, submit the job to the grid and register it in the database.
To be more easily separable from other jobs in the database, you can give your job a name:
.. code-block:: sh
- $ bin/jman -vv submit -n [name] myscript.py
+ $ bin/jman -vv submit -n [name] myscript.py
If the job requires certain machine specifications, you can add these (please see the SGE manual for possible specifications of [key] and [value] pairs).
Please note the ``--`` option that separates specifications from the command:
.. code-block:: sh
- $ bin/jman -vv submit -q [queue-name] -m [memory] --io-big -s [key1]=[value1] [key2]=[value2] -- myscript.py
+ $ bin/jman -vv submit -q [queue-name] -m [memory] --io-big -s [key1]=[value1] [key2]=[value2] -- myscript.py
-To have jobs run in parallel, you can submit a parametric job.
-Simply call:
+To have jobs run in parallel, you can submit a parametric job. Simply call:
.. code-block:: sh
- $ bin/jman -vv submit -t 10 myscript.py
+ $ bin/jman -vv submit -t 10 myscript.py
-to run ``myscript.py`` 10 times in parallel.
-Each of the parallel jobs will have a different environment variable called ``SGE_TASK_ID``, which will range from 1 to 10 in this case.
-If your script can handle this environment variable, it can actually execute 10 different tasks.
+to run ``myscript.py`` 10 times in parallel. Each of the parallel jobs will
+have a different environment variable called ``SGE_TASK_ID``, which will range
+from 1 to 10 in this case. If your script can handle this environment
+variable, it can actually execute 10 different tasks.
-Also, jobs with dependencies can be submitted.
-When submitted to the grid, each job has its own job id.
-These job ids can be used to create dependencies between the jobs (i.e., one job needs to finish before the next one can be started):
+Also, jobs with dependencies can be submitted. When submitted to the grid,
+each job has its own job id. These job ids can be used to create dependencies
+between the jobs (i.e., one job needs to finish before the next one can be
+started):
.. code-block:: sh
$ bin/jman -vv submit -x [job_id_1] [job_id_2] -- myscript.py
-In case the first job fails, it can automatically stop the depending jobs from being executed.
-Just submit jobs with the ``--stop-on-failure`` option.
+In case the first job fails, it can automatically stop the depending jobs from
+being executed. Just submit jobs with the ``--stop-on-failure`` option.
.. note::
- The ``--stop-on-failure`` option is under development and might not work properly.
- Use this option with care.
+
+ The ``--stop-on-failure`` option is under development and might not work
+ properly. Use this option with care.
While the jobs run, the output and error stream are captured in log files, which are written into a ``logs`` directory.
@@ -111,126 +130,194 @@ This directory can be changed by specifying:
$ bin/jman -vv submit -l [log_dir]
.. note::
- When submitting jobs locally, by default the output and error streams are written to console and no log directory is created.
- To get back the SGE grid logging behavior, please specify the log directory.
- In this case, output and error streams are written into the log files **after** the job has finished.
+
+ When submitting jobs locally, by default the output and error streams are
+ written to console and no log directory is created. To get back the SGE
+ grid logging behavior, please specify the log directory. In this case,
+ output and error streams are written into the log files **after** the job
+ has finished.
Running Jobs Locally
--------------------
-When jobs are submitted to the SGE grid, they are run immediately.
-However, when jobs are submitted locally, (using the ``--local`` option, see above), a local scheduler needs to be run.
-This is achieved by issuing the command:
+
+When jobs are submitted to the SGE grid, they are run immediately. However,
+when jobs are submitted locally, (using the ``--local`` option, see above), a
+local scheduler needs to be run. This is achieved by issuing the command:
.. code-block:: sh
- $ bin/jman -vv run-scheduler -p [parallel_jobs] -s [sleep_time]
+ $ bin/jman -vv run-scheduler -p [parallel_jobs] -s [sleep_time]
-This will start the scheduler in the daemon mode.
-This will constantly monitor the SQL3 database and execute jobs after submission, starting every ``[sleep_time]`` second.
-Use ``Ctrl-C`` to stop the scheduler (if jobs are still running locally, they will automatically be stopped).
+This will start the scheduler in the daemon mode. This will constantly monitor
+the SQL3 database and execute jobs after submission, starting every
+``[sleep_time]`` second. Use ``Ctrl-C`` to stop the scheduler (if jobs are
+still running locally, they will automatically be stopped).
-If you want to submit a list of jobs and have the scheduler to run the jobs and stop afterward, simply use the ``--die-when-finished`` option.
-Also, it is possible to run only specific jobs (and array jobs), which can be specified with the ``--j`` and ``--a`` option, respectively.
+If you want to submit a list of jobs and have the scheduler to run the jobs and
+stop afterward, simply use the ``--die-when-finished`` option. Also, it is
+possible to run only specific jobs (and array jobs), which can be specified
+with the ``--j`` and ``--a`` option, respectively.
Probing for Jobs
----------------
-To list the contents of the job database, you can use the ``jman list`` command.
-This will show you the job-id, the queue, the current status, the name and the command line of each job.
-Since the database is automatically updated when jobs finish, you can use the ``jman list`` again after some time.
-Normally, long command lines are cut so that each job is listed in a single line.
-To get the full command line, please use the ``-vv`` option:
+To list the contents of the job database, you can use the ``jman list``
+command. This will show you the job-id, the queue, the current status, the
+name and the command line of each job. Since the database is automatically
+updated when jobs finish, you can use the ``jman list`` again after some time.
+
+Normally, long command lines are cut so that each job is listed in a single
+line. To get the full command line, please use the ``-vv`` option:
+
+.. code-block:: sh
+
+ $ bin/jman -vv list
+
+By default, array jobs are not listed, but the ``-a`` option changes this
+behavior. Usually, it is a good idea to combine the ``-a`` option with ``-j``,
+which will list only the jobs of the given job id(s):
.. code-block:: sh
- $ bin/jman -vv list
+ $ bin/jman -vv list -a -j [job_id_1] [job_id_2]
+
+Note that the ``-j`` option is in general relatively smart. You can use it to
+select a range of job ids, e.g., ``-j 1-4 6-8``. In this case, please assert
+that there are no spaces between job ids and the ``-`` separator. If any job
+id is specified, which is not available in the database, it will simply be
+ignored, including job ids that in the ranges.
-By default, array jobs are not listed, but the ``-a`` option changes this behavior.
-Usually, it is a good idea to combine the ``-a`` option with ``-j``, which will list only the jobs of the given job id(s):
+Since version 1.3.0, GridTK also saves timing information about jobs, i.e.,
+time stamps when jobs were submitted, started and finished. You can use the
+``-t`` option of ``jman ls`` to add the time stamps to the listing, which are
+both written for jobs and parametric jobs (i.e., when using the ``-a`` option).
+
+
+Submitting dependent jobs
+-------------------------
+
+Sometimes, the execution of one job might depend on the execution of another
+job. The JobManager can take care of this, simply by adding the id of the job
+that we have to wait for:
.. code-block:: sh
- $ bin/jman -vv list -a -j [job_id_1] [job_id_2]
+ $ jman -vv submit --dependencies 6151645 -- /usr/bin/python myscript.py --help
+ ... Added job '<Job: 3> : submitted -- /usr/bin/python myscript.py --help' to the database
+ ... Submitted job '<Job: 6151647> : queued -- /usr/bin/python myscript.py --help' to the SGE grid.
-Note that the ``-j`` option is in general relatively smart.
-You can use it to select a range of job ids, e.g., ``-j 1-4 6-8``.
-In this case, please assert that there are no spaces between job ids and the ``-`` separator.
-If any job id is specified, which is not available in the database, it will simply be ignored, including job ids that in the ranges.
+Now, the new job will only be run after the first one finished.
+
+.. note::
-Since version 1.3.0, GridTK also saves timing information about jobs, i.e., time stamps when jobs were submitted, started and finished.
-You can use the ``-t`` option of ``jman ls`` to add the time stamps to the listing, which are both written for jobs and parametric jobs (i.e., when using the ``-a`` option).
+ Note the ``--`` between the list of dependencies and the command.
Inspecting log files
--------------------
-When a job fails, the status will be ``failure``.
-In this case, you might want to know, what happened.
-As a first indicator, the exit code of the program is reported as well.
-Also, the output and error streams of the job has been recorded and can be seen using the utilities.
-E.g.:
+
+When a job fails, the status will be ``failure``. In this case, you might want
+to know, what happened. As a first indicator, the exit code of the program is
+reported as well. Also, the output and error streams of the job has been
+recorded and can be seen using the utilities. E.g.:
.. code-block:: sh
- $ bin/jman -vv report -j [job_id] -a [array_id]
+ $ bin/jman -vv report -j [job_id] -a [array_id]
-will print the contents of the output and error log file from the job with the desired ID (and only the array job with the given ID).
+will print the contents of the output and error log file from the job with the
+desired ID (and only the array job with the given ID).
-To report only the output or only the error logs, you can use the ``-o`` or ``-e`` option, respectively.
-Hopefully, that helps in debugging the problem!
+To report only the output or only the error logs, you can use the ``-o`` or
+``-e`` option, respectively. Hopefully, that helps in debugging the problem!
Re-submitting the job
---------------------
-After correcting your code you might want to submit the same command line again.
-For this purpose, the ``bin/jman resubmit`` command exists.
-Simply specify the job id(s) that you want to resubmit:
+
+After correcting your code you might want to submit the same command line
+again. For this purpose, the ``bin/jman resubmit`` command exists. Simply
+specify the job id(s) that you want to resubmit:
.. code-block:: sh
- $ bin/jman -vv resubmit -j [job_id_1] [job_id_2]
+ $ bin/jman -vv resubmit -j [job_id_1] [job_id_2]
+
+This will clean up the old log files (if you didn't specify the ``--keep-logs``
+option) and re-submit the job. If the submission is done in the grid the job
+id(s) will change during this process.
+
-This will clean up the old log files (if you didn't specify the ``--keep-logs`` option) and re-submit the job.
-If the submission is done in the grid the job id(s) will change during this process.
+Stopping a grid job
+-------------------
+
+In case you found an error in the code of a grid job that is currently
+executing, you might want to kill the job in the grid. For this purpose, you
+can use the command:
+
+.. code-block:: sh
+
+ $ jman stop
+
+The job is removed from the grid, but all log files are still available. A
+common use case is to stop the grid job, fix the bugs, and re-submit it.
Note about verbosity and time stamps
------------------------------------
-For some jobs, it might be interesting to get the time stamps when the job has started and when it has finished.
-These time stamps are added to the log files (usually the error log file) automatically, when you use the ``-vv`` option, one when starting the process and one when it is finished.
-However, there is a difference between the ``SGE`` operation and the ``--local`` operation.
-For the ``SGE`` operation, you need to use the ``-vv`` option during the submission or re-submission of a job.
-In ``--local`` mode, the ``-vv`` flag during execution (using ``--run-local-scheduler``) is used instead.
+
+For some jobs, it might be interesting to get the time stamps when the job has
+started and when it has finished. These time stamps are added to the log files
+(usually the error log file) automatically, when you use the ``-vv`` option,
+one when starting the process and one when it is finished. However, there is a
+difference between the ``SGE`` operation and the ``--local`` operation. For
+the ``SGE`` operation, you need to use the ``-vv`` option during the submission
+or re-submission of a job. In ``--local`` mode, the ``-vv`` flag during
+execution (using ``--run-local-scheduler``) is used instead.
.. note::
- Why writing info logs the error log file, and not to the default output log file?
- This is the default behavior of python's logging module.
- All logs, independent of whether they are error, warning, info or debug logs are written to ``sys.stderr``, which in turn will be written into the error log files.
+
+ Why writing info logs the error log file, and not to the default output log
+ file? This is the default behavior of python's logging module. All logs,
+ independent of whether they are error, warning, info or debug logs are
+ written to ``sys.stderr``, which in turn will be written into the error log
+ files.
Cleaning up
-----------
-After the job was successfully (or not) executed, you should clean up the database using the ``bin/jman delete`` command.
-If not specified otherwise (i.e., using the ``--keep-logs`` option), this command will delete all jobs from the database and delete the log files (including the log directory in case it is empty), and remove the database as well.
-Again, job ids and array ids can be specified to limit the deleted jobs with the ``-j`` and ``-a`` option, respectively.
-It is also possible to clean up only those jobs (and array jobs) with a certain status.
-E.g. use:
+After the job was successfully (or not) executed, you should clean up the
+database using the ``bin/jman delete`` command. If not specified otherwise
+(i.e., using the ``--keep-logs`` option), this command will delete all jobs
+from the database and delete the log files (including the log directory in case
+it is empty), and remove the database as well.
+
+Again, job ids and array ids can be specified to limit the deleted jobs with
+the ``-j`` and ``-a`` option, respectively. It is also possible to clean up
+only those jobs (and array jobs) with a certain status. E.g. use:
.. code-block:: sh
$ bin/jman -vv delete -s success -j 10-20
-to delete all jobs and the logs of all successfully finished jobs with job ids from 10 to 20 from the database.
+to delete all jobs and the logs of all successfully finished jobs with job ids
+from 10 to 20 from the database.
Other command line tools
========================
-For convenience, we also provide additional command line tools, which are mainly useful at Idiap.
-These tools are:
-- ``bin/qstat.py``: writes the statuses of the jobs that are currently running in the SGE grid
-- ``bin/qsub.py``: submit job to the SGE grid without logging them into the database
-- ``bin/qdel.py``: delete job from the SGE grid without logging them into the database
-- ``bin/grid``: executes the command in an grid environment (i.e., as if a ``SETSHELL grid`` command would have been issued before)
+For convenience, we also provide additional command line tools, which are
+mainly useful at Idiap. These tools are:
+
+- ``bin/qstat.py``: writes the statuses of the jobs that are currently running
+ in the SGE grid
+- ``bin/qsub.py``: submit job to the SGE grid without logging them into the
+ database
+- ``bin/qdel.py``: delete job from the SGE grid without logging them into the
+ database
+- ``bin/grid``: executes the command in an grid environment (i.e., as if a
+ ``SETSHELL grid`` command would have been issued before)
diff --git a/doc/program.rst b/doc/program.rst
index 18d178a..38f3168 100644
--- a/doc/program.rst
+++ b/doc/program.rst
@@ -4,22 +4,24 @@
.. _developer:
-=====================
-The GridTk User Guide
-=====================
+=======================
+ The GridTk User Guide
+=======================
-The ``gridtk`` framework is a python library to help submitting, tracking and querying SGE.
-Here is quick example on how to use the ``gridtk`` framework to submit a python script:
+The ``gridtk`` framework is a python library to help submitting, tracking and
+querying SGE. Here is quick example on how to use the ``gridtk`` framework to
+submit a python script:
.. code-block:: python
- import sys
- from gridtk.sge import JobManager
- from gridtk.tools import make_shell
+ import sys
+ from gridtk.sge import JobManager
+ from gridtk.tools import make_shell
+
+ manager = JobManager()
+ command = make_shell(sys.executable, ['myscript.py', '--help'])
+ job = manager.submit(command)
- manager = JobManager()
- command = make_shell(sys.executable, ['myscript.py', '--help'])
- job = manager.submit(command)
You can do, programatically, everything you can do with the job manager - just
browse the help messages and the ``jman`` script for more information.
diff --git a/gridtk/script/grid.py b/gridtk/script/grid.py
old mode 100755
new mode 100644
diff --git a/requirements.txt b/requirements.txt
index b10607c..30d555d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1 +1,2 @@
-sqlalchemy
\ No newline at end of file
+sqlalchemy
+bob.extension
diff --git a/setup.cfg b/setup.cfg
deleted file mode 100644
index b12d119..0000000
--- a/setup.cfg
+++ /dev/null
@@ -1,8 +0,0 @@
-[build_sphinx]
-source-dir = docs
-build-dir = sphinx
-all_files = 1
-
-[upload_sphinx]
-upload-dir = sphinx/html
-
diff --git a/setup.py b/setup.py
index 1c50a4a..a28d007 100644
--- a/setup.py
+++ b/setup.py
@@ -2,27 +2,25 @@ from setuptools import setup, find_packages
import sys
-# If Python < 2.7 or 3.0 <= Python < 3.2, require some more stuff
-DEPS = ['six']
-if sys.version_info[:2] < (2, 7) or ((3,0) <= sys.version_info[:2] < (3,2)):
- DEPS.append('argparse')
-
version = open("version.txt").read().rstrip()
+requirements = [k.strip() for k in open("requirements.txt".read().split()]
setup(
name='gridtk',
version=version,
- description='SGE Grid and Local Submission and Monitoring Tools for Idiap',
-
- url='http://github.com/bioidiap/gridtk',
+ description='Parallel Job Manager',
+ long_description=open('README.rst').read(),
+ url='https://gitlab.idiap.ch/bob/gridtk',
license='GPLv3',
- author='Manuel Guenther',
- author_email='manuel.guenther@idiap.ch',
+ author='Manuel Guenther,Andre Anjos',
+ author_email='manuel.guenther@idiap.ch,andre.anjos@idiap.ch',
packages=find_packages(),
include_package_data=True,
+ install_requires=requirements,
+
entry_points={
'console_scripts': [
'jman = gridtk.script.jman:main',
@@ -37,10 +35,6 @@ setup(
},
- long_description=open('README.rst').read(),
-
- install_requires=DEPS,
-
classifiers = [
'Development Status :: 4 - Beta',
'Intended Audience :: Developers',
--
GitLab