diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..12d31c52ddd90977f97100f7e591bda8ca7e52a2 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,130 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = PYTHONPATH=.. /idiap/group/torch5spro/nightlies/externals/v2/linux-x86_64/pyenv/bin/sphinx-build +PAPER = +BUILDDIR = _build + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest + +help: + @echo "Please use \`make <target>' where <target> is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + -rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/GridTk.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/GridTk.qhc" + +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/GridTk" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/GridTk" + @echo "# devhelp" + +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + make -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000000000000000000000000000000000000..4e50c66a9608efc964d0edc111461e2c362e7de3 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,260 @@ +# -*- coding: utf-8 -*- +# +# Torch5spro documentation build configuration file, created by +# sphinx-quickstart on Mon Mar 21 18:07:34 2011. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys, os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +#sys.path.insert(0, os.path.abspath('.')) + +# -- General configuration ----------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = [ + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'sphinx.ext.pngmath', + 'sphinx.ext.ifconfig', + 'sphinx.ext.autodoc', + 'sphinx.ext.autosummary', + 'sphinx.ext.doctest', + ] + +# The viewcode extension appeared only on Sphinx >= 1.0.0 +import sphinx +if sphinx.__version__ >= "1.0": + extensions.append('sphinx.ext.viewcode') + +# If we are on OSX, the 'dvipng' path maybe different +dvipng_osx = '/opt/local/libexec/texlive/binaries/dvipng' +if os.path.exists(dvipng_osx): pngmath_dvipng = dvipng_osx + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'GridTk' +import time +copyright = u'%s, Idiap Research Institute' % time.strftime('%Y') + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '0.1' +# The full version, including alpha/beta/rc tags. +release = 'beta' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = [] + +# The reST default role (used for this markup: `text`) to use for all documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'default' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# "<project> v<release> documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a <link> tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = 'gridtkdoc' + + +# -- Options for LaTeX output -------------------------------------------------- + +# The paper size ('letter' or 'a4'). +#latex_paper_size = 'letter' + +# The font size ('10pt', '11pt' or '12pt'). +#latex_font_size = '10pt' + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ('index', 'gridtk.tex', u'Grid TookKit Documentation', + u'Idiap Research Institute', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# If true, show page references after internal links. +#latex_show_pagerefs = False + +# If true, show URL addresses after external links. +#latex_show_urls = False + +# Additional stuff for the LaTeX preamble. +#latex_preamble = '' + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_domain_indices = True + +# Included after all input documents +rst_epilog = """ +.. Some variables +.. |project| replace:: GridTk +.. |torchweb| replace:: http://www.idiap.ch/software/torch5spro +.. |current-year| date:: %Y +""" + +# -- Options for manual page output -------------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ('index', 'torch5spro', u'GridTk Documentation', + [u'Idiap Research Institute'], 1) +] + +# Skips selected members in auto-generated documentation. Unfortunately, old +# versions of Boost.Python will note generate a __self__ member for static +# methods and that screws-up Sphinx processing +def skip_member(app, what, name, obj, skip, options): + # We don't use this technique if the version of Sphinx is relatively new + if sphinx.__version__ >= "1.0": return False + + # Else, we have to remove objects that do not have a __self__ attribute set + import types + if isinstance(obj, types.BuiltinFunctionType) and \ + not hasattr(obj, '__self__') and what == 'class': + app.warn("Skipping %s %s (no __self__)" % (what, name)) + return True + + return False + +def setup(app): + app.connect('autodoc-skip-member', skip_member) diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..513e778209012388674502e70347a7547c19b930 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,23 @@ +.. GridTk documentation master file, created by + sphinx-quickstart on Thu Aug 25 15:51:27 2011. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to GridTk's documentation! +================================== + +Contents: + +.. toctree:: + :maxdepth: 2 + + manual + program + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/readme.rst b/docs/manual.rst similarity index 83% rename from readme.rst rename to docs/manual.rst index 22387beb82ccab1172c8ec197f5a7410a9641a27..632f9b3dad924e28f38f63677d7a0b5ef279ef2b 100644 --- a/readme.rst +++ b/docs/manual.rst @@ -32,7 +32,7 @@ To interact with the Job Manager we use the `jman` utility. Make sure to have your shell environment setup to reach it w/o requiring to type-in the full path. The first task you may need to pursue is to submit jobs. Here is how: -.. code-block:: shell +.. code-block:: sh $ jman torch -- dbmanage.py --help Submitted (torch'd) 6151645 @all.q (0 seconds ago) -S /usr/bin/python /idiap/group/torch5spro/nightlies/last/bin/shell.py -- dbmanage.py --help @@ -55,7 +55,7 @@ Once the job has been submitted you will noticed a database file (by default called `.jobmanager.db`) has been created in the current working directory. It contains the information for the job you just submitted: -.. code-block:: shell +.. code-block:: sh $ jman list job-id queue age arguments @@ -74,7 +74,7 @@ it is monitoring. Finished jobs will be reported to the screen and removed from the job manager database and placed on a second database (actually two) containing jobs that failed and jobs that succeeded. -.. code-block:: shell +.. code-block:: sh $ jman refresh These jobs require attention: @@ -93,7 +93,7 @@ requires attention. If jobs fail, they are copied to a database named `failure.db` in the current directory. Otherwise, they are copied to `success.db`. You can inspect the job log files like this: -.. code-block:: shell +.. code-block:: sh $ jman explain failure.db Job 6151645 @all.q (34 minutes ago) -S /usr/bin/python /idiap/group/torch5spro/nightlies/last/bin/shell.py -- dbmanage.py --help @@ -117,7 +117,7 @@ If you are convinced the job did not work because of external conditions (e.g. temporary network outage), you may re-submit it, *exactly* like it was submitted the first time: -.. code-block:: shell +.. code-block:: sh $ jman resubmit --clean failure.db Re-submitted job 6151663 @all.q (1 second ago) -S /usr/bin/python /idiap/group/torch5spro/nightlies/last/bin/shell.py -- dbmanage.py --help @@ -136,7 +136,7 @@ The job in question will not work no matter how many times we re-submit it. It is not a temporary error. In these circumstances, I may just want to clean the job and do something else. The job manager is here for you again: -.. code-block:: shell +.. code-block:: sh $ jman cleanup --remove-job failure.db Cleaning-up logs for job 6151663 @all.q (5 minutes ago) -S /usr/bin/python /idiap/group/torch5spro/nightlies/last/bin/shell.py -- dbmanage.py --help @@ -147,34 +147,3 @@ job and do something else. The job manager is here for you again: Inspection on the current directory will now show you everything concerning the said job is gone. -Programatically submit and query for jobs ------------------------------------------ - -The Job Manager is part of the `gridtk` framework, which is a python library to -help submitting, tracking and querying SGE. Here is quick example on how to use -the `gridtk` framework: - -.. code-block:: python - - # This variable points to the torch5spro root directory you want to use - TORCH = '/idiap/group/torch5spro/nightlies/last' - - from gridtk.manager import JobManager - - # This helps constructing the command line with bracket'ed by Torch - from gridtk.tools import make_torch_wrapper - - man = JobManager() - command = ['dbmange.py', '--help'] - command, kwargs = make_torch_wrapper(TORCH, False, command, kwargs) - - # For more options look do help(gridtk.qsub) - job = man.submit(command, cwd=True, stdout='logs', name='testjob') - -You can do, programatically, everything you can do with the job manager just -browse the help messages and the `jman` script for more information. - -.. note:: - - To be able to import the `gridtk` library, you must have it on your - PYTHONPATH. diff --git a/docs/program.rst b/docs/program.rst new file mode 100644 index 0000000000000000000000000000000000000000..912f541d5d514b6a564846bc717ece2ae49e43aa --- /dev/null +++ b/docs/program.rst @@ -0,0 +1,56 @@ +.. vim: set fileencoding=utf-8 : +.. Andre Anjos <andre.anjos@idiap.ch> +.. Thu 25 Aug 2011 15:58:21 CEST + +======================= + The GridTk User Guide +======================= + +The `gridtk` framework is a python library to help submitting, tracking and +querying SGE. Here is quick example on how to use the `gridtk` framework: + +.. code-block:: python + + # This variable points to the torch5spro root directory you want to use + TORCH = '/idiap/group/torch5spro/nightlies/last' + + from gridtk.manager import JobManager + + # This helps constructing the command line with bracket'ed by Torch + from gridtk.tools import make_torch_wrapper + + man = JobManager() + command = ['dbmange.py', '--help'] + command, kwargs = make_torch_wrapper(TORCH, False, command, kwargs) + + # For more options look do help(gridtk.qsub) + job = man.submit(command, cwd=True, stdout='logs', name='testjob') + +You can do, programatically, everything you can do with the job manager just +browse the help messages and the `jman` script for more information. + +.. note:: + + To be able to import the `gridtk` library, you must have it on your + PYTHONPATH. + +Reference Manual +---------------- + +API to the Job Manager +====================== + +.. automodule:: gridtk.manager + :members: + +Middleware +========== + +.. automodule:: gridtk.tools + :members: + +Low-level Utilities +=================== + +.. automodule:: gridtk.setshell + :members: