diff --git a/MANIFEST.in b/MANIFEST.in index b872f66f580f15239ae85f7f6c3103daf353808e..86435a434c234fdf295768d6e538e0dbe2cc84f5 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1,2 @@ include LICENSE README.rst bootstrap.py buildout.cfg +recursive-include xbob/extension *.h \ No newline at end of file diff --git a/README.rst b/README.rst index 9f18971c36e889bf2135f5f93dbedcfd6e63d590..295b6d5c903037f5162c21f7307dfac6bdf2f528 100644 --- a/README.rst +++ b/README.rst @@ -43,6 +43,7 @@ so that you include the following:: setup_requires=[ 'xbob.extension', + 'numpydoc' # see below ], ... @@ -123,6 +124,106 @@ After inclusion, you can just instantiate an object of type ``pkgconfig``:: >>> zlib > '1.2.10' False +Documenting your Python extension +--------------------------------- +One part of this package are some functions that makes it easy to generate a proper python documentation for your bound C++ functions. +This documentation can be used after:: + + #include <xbob.extension/documentation.h> + +The generated documentation relies on the ``numpydoc`` sphinx extension http://pypi.python.org/pypi/numpydoc, which is documented `here <http://github.com/numpy/numpy/blob/master/doc/HOWTO_DOCUMENT.rst.txt>`_. +To use this package, please add the following lines in the ``conf.py`` file of your documentation (which is usually located in ``doc/conf.py``):: + + extensions = [ + ... + 'numpydoc', + ] + # Removes some warnings, see: https://github.com/phn/pytpm/issues/3 + numpydoc_show_class_members = False + + +Function documentation +====================== +To generate a properly aligned function documentation, you can use:: + + static xbob::extension::FunctionDoc description( + "function_name", + "Short function description", + "Optional long function description" + ); + +.. note:: + Please assure that you define this variable as ``static``. + +Using this object, you can add several parts of the function that need documentation: + +1. ``description.add_prototype("variable1, variable2", "return1, return2");`` can be used to add function definitions (i.e., ways how to use your function). + This function needs to be called at least once. + If the function does not define a return value, it can be left out (in which case the default ``"None"`` is used). + +2. ``description.add_parameter("variable1, variable2", "datatype", "Variable description");`` should be defined for each variable that you have used in the prototypes. + +3. ``description.add_return("return1", "datatype", "Return value description");`` should be defined for each return value that you have used in the prototypes. + +Finally, when binding you function, you can use: + +a) ``description.name()`` to get the name of the function + +b) ``description.doc()`` to get the aligned documentation of the function, properly indented and broken at 80 characters (by default). + By default, this call will check that all parameters and return values are documented, and add a ``.. todo`` directive if not. + You can call ``description.doc(false)`` to disable the checks. + +Sphinx directives like ``.. note::``, ``.. warning::`` or ``.. math::`` will be automatically detected and aligned, when they are used as one-line directive, e.g.: + + "(more text)\n.. note:: This is a note\n(more text)" + +.. note:: + The ``.. todo::`` directive seems not to like being broken at 80 characters. + If you want to use ``.. todo::``, please call ``description.doc(true, 10000)`` to avoid line breaking. + + +Class documentation +=================== +To document a bound C++ class, you can use the ``xbob::extension::ClassDoc("class_name", "Short class description", "Optional long class description")`` function to align and wrap your documentation. +Again, during binding you can use the functions``description.name()`` and ``description.doc()`` as above. + +Additionally, the class documentation has a function to add constructor definitions, which takes an ``xbob::extension::FunctionDoc`` object. +The shortest way to get a proper class documentation is:: + + static auto my_class_doc = + xbob::extension::ClassDoc("class_name", "Short description", "Long Description") + .add_constructor( + xbob::extension::FunctionDoc("class_name", "Constructor Description") + .add_prototype("param1", "") + .add_parameter("param1", "type1", "Description of param1") + ) + ; + +.. note:: The second ``""`` in ``add_prototype`` prevents the output type (which otherwise defaults to ``"None"``) to be written. + +Possible speed issues +===================== + +In order to speed up the loading time of the modules, you might want to reduce the amount of documentation that is generated (though I haven't experienced any speed differences). +For this purpose, just compile your bindings using the "-DXBOB_SHORT_DOCSTRINGS" compiler option, e.g. by adding it to the setup.py as follows (see also above):: + + ... + ext_modules=[ + Extension("xbob.myext._myext", + [ + ... + ], + ... + define_macros = [('XBOB_SHORT_DOCSTRINGS',1)], + ), + ], + ... + +or simply define an environment variable ``XBOB_SHORT_DOCSTRINGS=1`` before invoking buildout. + +In any of these cases, only the short descriptions will be returned as the doc string. + + Using the ``boost`` class ========================= diff --git a/setup.py b/setup.py index 465676f7c13c4810d8ecb3f983a7cc04d1dedd74..3c8d25cf605f7cfb7b365b9b020f3069dc2a7c37 100644 --- a/setup.py +++ b/setup.py @@ -28,6 +28,7 @@ setup( install_requires=[ 'setuptools', + 'numpydoc' ], classifiers = [ diff --git a/xbob/extension/__init__.py b/xbob/extension/__init__.py index 530b38d3431fa4ad281cf23d069d60549dd369ee..3f4c64dfaac2bf1adab1a325a14c0f543ad01bf6 100644 --- a/xbob/extension/__init__.py +++ b/xbob/extension/__init__.py @@ -11,6 +11,7 @@ import os import platform import pkg_resources from distutils.extension import Extension as DistutilsExtension +from pkg_resources import resource_filename from .pkgconfig import pkgconfig from .boost import boost @@ -236,8 +237,12 @@ class Extension(DistutilsExtension): kwargs[key] = uniq(kwargs[key]) + # add our include dir by default + self_include_dir = resource_filename(__name__, 'include') + kwargs.setdefault('include_dirs', []).append(self_include_dir) + # Uniq'fy parameters that are not on our parameter list - kwargs['include_dirs'] = uniq(kwargs.get('include_dirs', [])) + kwargs['include_dirs'] = uniq(kwargs['include_dirs']) # Make sure the language is correctly set to C++ kwargs['language'] = 'c++' diff --git a/xbob/extension/include/xbob.extension/documentation.h b/xbob/extension/include/xbob.extension/documentation.h new file mode 100644 index 0000000000000000000000000000000000000000..6e25d41fd5f0d7287b77410c4af863e72b491893 --- /dev/null +++ b/xbob/extension/include/xbob.extension/documentation.h @@ -0,0 +1,614 @@ +/** + * @file xbob/extension/include/xbob.extension/documentation.h + * @date Fri Feb 21 18:29:37 CET 2014 + * @author Manuel Guenther <manuel.guenther@idiap.ch> + * + * @brief Implements a few functions to generate doc strings + * + * Copyright (C) 2011-2014 Idiap Research Institute, Martigny, Switzerland + */ + +#ifndef XBOB_EXTENSION_DOCUMENTATION_H_INCLUDED +#define XBOB_EXTENSION_DOCUMENTATION_H_INCLUDED + +#include <string> +#include <set> +#include <stdexcept> +#include <initializer_list> +#include <bob/core/logging.h> + +namespace xbob{ + namespace extension{ + + /** + * Use a static object of this class to document a variable. + * This class can be used to document both global variables as well as class member variables. + */ + class VariableDoc { + friend class ClassDoc; + public: + /** + * Generates a VariableDoc object. Please assure that use use this as a static member variable. + * @param variable_name The name of the variable + * @param variable_type The type of the variable, e.g., "float" or "array_like (float, 2D)" + * @param short_description A short description of the variable + * @param long_description An optional long description of the variable + */ + VariableDoc( + const char* const variable_name, + const char* const variable_type, + const char* const short_description, + const char* const long_description = 0 + ); + + /** + * Returns the name of the variable that is documented, i.e., the "variable_name" parameter of the constructor. + */ + char* name() const {return const_cast<char*>(variable_name.c_str());} + + /** + * Generates and returns the documentation string. + * @param checks Currently ignored. + * @param alignment The default alignment is 80 characters. + * Since the documentation is automatically indented by 4 spaces in the python documentation, we need to subtract these values here... + * @return The documentation string, properly aligned, possibly including "ToDo's" for detected problems. + */ + char* doc(bool checks = true, const unsigned alignment = 76) const; + + private: + // variable name and type + const std::string variable_name; + const std::string variable_type; + // variable description + std::string variable_description; + + // an internal string that is generated and returned. + mutable std::string description; + + }; + + /** + * Use a static object of this class to generate a properly aligned and numpydoc compatible function documentation. + * Documentation generated by this class can be used for non-member functions as well as for member functions and constructors. + */ + class FunctionDoc { + friend class ClassDoc; + public: + /** + * Generates a FunctionDoc object. Please assure that use use this as a static member variable. + * @param function_name The name of the function you want to document + * @param short_description A short description of what the function does + * @param long_description An optional long description of the function + */ + FunctionDoc( + const char* const function_name, + const char* const short_description, + const char* const long_description = 0 + ); + + /** + * Add a prototypical call for this function by defining the parameters and the return values. + * This function has to be called at least ones. + * @param variables A string containing a comma-separated list of parameters, e.g., "param1, param2" + * @param return_value A string containing a comma-separated list of return values, e.g., "retval1, retval2". + * If the function does not return anything, this value can be left at its default "None". + * To document a constructor, please use "" as return value. + */ + FunctionDoc& add_prototype( + const char* const variables, + const char* const return_value = "None" + ); + + /** + * Add the documentation for a parameter added with the add_prototype function + * @param parameter_name The name of the parameter, e.g. "param1" + * @param parameter_type The type of the parameter, e.g. "float" or "array_like (float, 2D)"; indicate if the parameter is optional here + * @param parameter_description The description of the parameter + */ + FunctionDoc& add_parameter( + const char* const parameter_name, + const char* const parameter_type, + const char* const parameter_description + ); + + /** + * Add the documentation of a return value added with the add_prototype function + * @param return_name The name assigned to the return value + * @param return_type The tape of the returned value + * @param return_description The description of the return value + */ + FunctionDoc& add_return( + const char* const return_name, + const char* const return_type, + const char* const return_description + ); + + + /** + * Returns the name of the function that is documented (i.e., the function_name parameter of the constructor) + */ + const char* const name() const {return function_name.c_str();} + + /** + * Generates and returns the documentation string. + * @param checks If enabled, perform sanity checks, i.e., that at least one prototype is given, or that at all parameters are documented. + * In this case, a ..todo:: directive is added for each detected mistake, and a warning is emitted. + * @param alignment The default alignment is 80 characters. + * Since the documentation is automatically indented by 4 spaces in the python documentation, we need to subtract these values here... + * @return The documentation string, properly aligned, possibly including "ToDo's" for detected problems. + */ + const char* const doc(bool checks = true, const unsigned alignment = 76) const; + + + private: + // the function name + const std::string function_name; + // the description + std::string function_description; + // prototypes + std::vector<std::string> prototype_variables; + std::vector<std::string> prototype_returns; + // parameter documentation + std::vector<std::string> parameter_names; + std::vector<std::string> parameter_types; + std::vector<std::string> parameter_descriptions; + // return value documentation + std::vector<std::string> return_names; + std::vector<std::string> return_types; + std::vector<std::string> return_descriptions; + + // an internal string that is generated and returned. + mutable std::string description; + }; + + + /** + * Use a static object of this class to document a class. + * Documenting a class includes the documentation of the constructor, + * but not the documentation of the other member functions. + * For those, please use the FunctionDoc class. + */ + class ClassDoc{ + public: + /** + * Generates a ClassDoc object. Please assure that use use this as a static member variable. + * @param class_name The name of the class to be documented + * @param short_description A short description of the class + * @param long_description An optional long description of the class + */ + ClassDoc( + const char* const class_name, + const char* const short_description, + const char* const long_description = 0 + ); + + /** + * Add the documentation of the constructor. + * This function can be called only once. + * @param constructor_documentation An instance of the FunctionDoc class that contains the documentation of the constructor. + * Please read the documentation of that class on how to generate constructor documentations. + */ + ClassDoc& add_constructor( + const FunctionDoc& constructor_documentation + ); + + /** + * Adds the given function to the highlighted section. + * @param function_documentation An instance of the FunctionDoc class that should be highlighted. + */ + ClassDoc& highlight( + const FunctionDoc& function_documentation + ); + + /** + * Adds the given variable to the highlighted section. + * @param function_documentation An instance of the FunctionDoc class that should be highlighted. + */ + ClassDoc& highlight( + const VariableDoc& variable_documentation + ); + + /** + * Returns the name of the class that is documented, i.e., the "class_name" parameter of the constructor. + */ + char* name() const {return const_cast<char*>(class_name.c_str());} + + /** + * Generates and returns the documentation string. + * @param checks If enabled, performs checks in the constructor documentation; see FunctionDoc.doc() for details. + * @param alignment The default alignment is 80 characters. + * Since the documentation is automatically indented by 4 spaces in the python documentation, we need to subtract these values here... + * @return The documentation string, properly aligned, possibly including "ToDo's" for detected problems. + */ + char* doc(bool checks = true, const unsigned alignment = 76) const; + + + private: + // class name + const std::string class_name; + // class description + std::string class_description; + // constructor + std::shared_ptr<FunctionDoc> constructor; + + // highlighting + std::vector<FunctionDoc> highlighted_functions; + std::vector<VariableDoc> highlighted_variables; + + // an internal string that is generated and returned. + mutable std::string description; + }; + + } +} + + +///////////////////////////////////////////////////////////// +//////////////////// Implementations //////////////////////// +///////////////////////////////////////////////////////////// + + +///////////////////////////////////////////////////////////// +/// helper functions +// TODO: remove +#include <iostream> + +#ifndef XBOB_SHORT_DOCSTRINGS +// removes leading and trailing spaces +static std::string _strip(const std::string& str, char sep = ' '){ + unsigned first = 0, last = str.size(); + while (first < str.size() && str[first] == sep) ++first; + while (last > 0 && str[last-1] == sep) --last; + return str.substr(first, last-first); +} + +// splits the given string by the given separator +static std::vector<std::string> _split(const std::string& str, char limit = ' '){ + std::vector<std::string> splits; + int i = str.find(limit); + unsigned j = 0; + while (i != (int)std::string::npos){ + splits.push_back(str.substr(j, i-j)); + j = i+1; + i = str.find(limit, j); + } + if (j < str.size()) splits.push_back(str.substr(j)); + return splits; +} + +// aligns the given string using the given indent to the given alignment length; +// line breaks are handled carefully. +//static std::string _align(std::string str, unsigned first_line_indent=0, unsigned other_line_indent=4, unsigned alignment=76){ +static std::string _align(std::string str, unsigned indent=0, unsigned alignment=76){ + // first, split the newlines + auto lines = _split(str, '\n'); + + std::string aligned; + unsigned current_indent = indent; + bool first_line = true; + // now, split each line + for (auto line : lines){ + auto words = _split(line); + // fill in one line + unsigned len = 0; + for (auto word : words){ + if (aligned.empty() || len + word.size() >= alignment || !first_line){ + // line reached alignment + if (!aligned.empty()){ + aligned += "\n"; + } + // add indent and start new line + for (unsigned j = current_indent; j--;) aligned += " "; + len = current_indent; + first_line = true; + } + // increase indent? + const std::string& w = words[0]; + if ((w.size() == 2 && w[0] == '.' && w[1] == '.') || + (w.size() >= 1 && '0' <= w[0] && '9' >= w[0]) || + (w.size() == 1 && '*' == w[0]) ){ + current_indent = indent + 3; + } + // add word + aligned += word + " "; + len += word.size() + 1; + } + current_indent = indent; + aligned += "\n"; + first_line = false; + } + + return aligned; +} + +// Aligns the parameter description +static void _align_parameter(std::string& str, const std::string& name, const std::string& type, const std::string& description, unsigned alignment=76){ + str += _align(name + " : " + type + "", 0, alignment); + str += _align(description, 4, alignment); +} + +static std::string _prototype(const std::string& name, const std::string& variables, const std::string& retval){ + if (retval.empty()) + return name + "(" + variables + ")"; + else + return name + "(" + variables + ") -> " + retval; +} + +static void _check(std::string& doc, const std::vector<std::string>& vars, const std::vector<std::string>& docs, const std::string& type){ + // check that all parameters are documented. If not, add a TODO + std::set<std::string> undoc; + std::set<std::string> unused; + // gather parameters + for (auto p : vars){ + for (auto s : _split(p, ',')){ + undoc.insert(_strip(s)); + } + } + for (auto p : docs){ + for (auto s : _split(p, ',')){ + std::string x = _strip(s); + if (undoc.find(x) == undoc.end()){ + unused.insert(x); + } else { + undoc.erase(x); + } + } + } + if (undoc.size()){ + std::string all; + for (auto p : undoc){ + if (p != "None"){ + if (!all.empty()) all += ", "; + all += p; + } + } + if (!all.empty()){ + doc += _align(".. todo:: The " + type + "(s) '" + all + "' are used, but not documented.", 0, (unsigned)-1); + bob::core::warn << "The " << type << "(s) '" << all << "' are used, but not documented." << std::endl; + } + } + if (unused.size()){ + std::string all; + for (auto p : unused){ + if (!all.empty()) all += ", "; + all += p; + } + doc += _align(".. todo:: The " + type + "(s) '" + all + "' are documented, but nowhere used.", 0, (unsigned)-1); + bob::core::warn << "The " << type << "(s) '" << all << "' are documented, but nowhere used." << std::endl; + } +} + +#endif // ! XBOB_SHORT_DOCSTRINGS + + +///////////////////////////////////////////////////////////// +/// FunctionDoc + +inline xbob::extension::FunctionDoc::FunctionDoc( + const char* const function_name, + const char* const short_description, + const char* const long_description +) : function_name(function_name), function_description(short_description) +{ +#ifndef XBOB_SHORT_DOCSTRINGS + if (long_description){ + function_description += "\n"; + function_description += long_description; + } +#endif +} + +inline xbob::extension::FunctionDoc& xbob::extension::FunctionDoc::add_prototype( + const char* const variables, + const char* const return_values +){ +#ifndef XBOB_SHORT_DOCSTRINGS + prototype_variables.push_back(variables); + prototype_returns.push_back(return_values); +#endif // XBOB_SHORT_DOCSTRINGS + return *this; +} + +inline xbob::extension::FunctionDoc& xbob::extension::FunctionDoc::add_parameter( + const char* const parameter_name, + const char* const parameter_type, + const char* const parameter_description +) +{ +#ifndef XBOB_SHORT_DOCSTRINGS + parameter_names.push_back(parameter_name); + parameter_types.push_back(parameter_type); + parameter_descriptions.push_back(parameter_description); +#endif // XBOB_SHORT_DOCSTRINGS + return *this; +} + +inline xbob::extension::FunctionDoc& xbob::extension::FunctionDoc::add_return( + const char* const parameter_name, + const char* const parameter_type, + const char* const parameter_description +) +{ +#ifndef XBOB_SHORT_DOCSTRINGS + return_names.push_back(parameter_name); + return_types.push_back(parameter_type); + return_descriptions.push_back(parameter_description); +#endif // XBOB_SHORT_DOCSTRINGS + return *this; +} + +inline const char* const xbob::extension::FunctionDoc::doc( + bool checks, + const unsigned alignment +) const +{ +#ifdef XBOB_SHORT_DOCSTRINGS + return function_description.c_str(); +#else + description = ""; + switch(prototype_variables.size()){ + case 0: + if (checks){ + description = _align(".. todo:: Please use ``FunctionDoc.add_prototype`` to add at least one prototypical way to call this function", 0, (unsigned)-1); + bob::core::warn << "The bound function '" << function_name << "' has no prototype." << std::endl; + } + break; + case 1: + // only one way to call; use the default way + description = _align(_prototype(function_name, prototype_variables[0], prototype_returns[0]) + "\n", 0, alignment); + break; + default: + // several ways to call; list them + for (unsigned n = 0; n < prototype_variables.size(); ++n) + description += _align("* " + _prototype(function_name, prototype_variables[n], prototype_returns[n]) + "\n", 0, alignment); + } + // add function description + description += "\n" + _align(function_description, 0, alignment) + "\n"; + + if (checks){ + + // check that all parameters are documented + _check(description, prototype_variables, parameter_names, "parameter"); + + // check that all return values are documented + _check(description, prototype_returns, return_names, "return value"); + } + + if (!parameter_names.empty()){ + // add parameter description + description += "\n" + _align("Parameters") + _align("----------"); + for (unsigned i = 0; i < parameter_names.size(); ++i){ + _align_parameter(description, parameter_names[i], parameter_types[i], parameter_descriptions[i], alignment); + } + } + + if (!return_names.empty()){ + // add return value description + description += "\n" + _align("Returns") + _align("--------"); + for (unsigned i = 0; i < return_names.size(); ++i){ + _align_parameter(description, return_names[i], return_types[i], return_descriptions[i], alignment); + } + } + +// std::cout << description << std::endl; + + // return the description + return description.c_str(); +#endif // XBOB_SHORT_DOCSTRINGS +} + + +///////////////////////////////////////////////////////////// +/// ClassDoc + +inline xbob::extension::ClassDoc::ClassDoc( + const char* const class_name, + const char* const short_description, + const char* const long_description +) : class_name(class_name), class_description(short_description) +{ +#ifndef XBOB_SHORT_DOCSTRINGS + if (long_description){ + class_description += "\n"; + class_description += long_description; + } +#endif // ! XBOB_SHORT_DOCSTRINGS +} + +inline xbob::extension::ClassDoc& xbob::extension::ClassDoc::add_constructor( + const xbob::extension::FunctionDoc& constructor_documentation +) +{ +#ifndef XBOB_SHORT_DOCSTRINGS + if (constructor){ + bob::core::warn << "The class documentation can have only a single constructor documentation" << std::endl; + } + constructor.reset(new xbob::extension::FunctionDoc(constructor_documentation)); +#endif // XBOB_SHORT_DOCSTRINGS + return *this; +} + +inline xbob::extension::ClassDoc& xbob::extension::ClassDoc::highlight( + const xbob::extension::FunctionDoc& function_documentation +) +{ +#ifndef XBOB_SHORT_DOCSTRINGS + highlighted_functions.push_back(function_documentation); +#endif // XBOB_SHORT_DOCSTRINGS + return *this; +} + +inline xbob::extension::ClassDoc& xbob::extension::ClassDoc::highlight( + const xbob::extension::VariableDoc& variable_documentation +) +{ +#ifndef XBOB_SHORT_DOCSTRINGS + highlighted_variables.push_back(variable_documentation); +#endif // XBOB_SHORT_DOCSTRINGS + return *this; +} + + +inline char* xbob::extension::ClassDoc::doc( + bool checks, + const unsigned alignment +) const +{ +#ifdef XBOB_SHORT_DOCSTRINGS + return const_cast<char*>(class_description.c_str()); +#else + description = _align(class_description, 0, alignment); + if (constructor){ + description += _align("\n**Constructor Documentation** :\n\n"); + description += constructor->doc(checks, alignment); + } + if (!highlighted_variables.empty()){ + description += "\n" + _align("Attributes") + _align("----------"); + for (auto hightlight : highlighted_variables){ + description += _align(hightlight.variable_name, 0, alignment) + _align(_split(hightlight.variable_description, '\n')[0], 4, alignment); + } + } + if (!highlighted_functions.empty()){ + description += "\n" + _align("Methods") + _align("-------"); + for (auto hightlight : highlighted_functions){ + description += _align(hightlight.function_name, 0, alignment) + _align(_split(hightlight.function_description, '\n')[0], 4, alignment); + } + } + return const_cast<char*>(description.c_str()); +#endif // XBOB_SHORT_DOCSTRINGS +} + +///////////////////////////////////////////////////////////// +/// VariableDoc + +inline xbob::extension::VariableDoc::VariableDoc( + const char* const variable_name, + const char* const variable_type, + const char* const short_description, + const char* const long_description +) : variable_name(variable_name), variable_type(variable_type), variable_description(short_description) +{ +#ifndef XBOB_SHORT_DOCSTRINGS + if (long_description){ + variable_description += "\n"; + variable_description += long_description; + } +#endif // ! XBOB_SHORT_DOCSTRINGS +} + +inline char* xbob::extension::VariableDoc::doc( + bool checks, + const unsigned alignment +) const +{ +#ifdef XBOB_SHORT_DOCSTRINGS + return const_cast<char*>(variable_description.c_str()); +#else + // The numpydoc standard does not use variable types, so I came up with my own way of it + description = _align("**" + variable_type + "** <-- " + variable_description, 0, alignment); + return const_cast<char*>(description.c_str()); +#endif // XBOB_SHORT_DOCSTRINGS +} + +#endif // XBOB_EXTENSION_DOCUMENTATION_H_INCLUDED +