Commit 413a698c authored by Manuel Günther's avatar Manuel Günther
Browse files

Improved documentation; updated python bindings; fixed issue in multithread...

Improved documentation; updated python bindings; fixed issue in multithread logging (wrong logger was used)
parent 57dd5a01
......@@ -50,116 +50,67 @@ PyObject* inner_convert (PyBlitzArrayObject* src,
//use all defaults
auto bz_dst = convert<Tdst,Tsrc>(*bz_src);
return PyBlitzArrayCxx_NewFromArray(bz_dst);
return PyBlitzArrayCxx_AsNumpy(bz_dst);
}
template <typename Tdst, typename Tsrc>
PyObject* convert_dim (PyBlitzArrayObject* src,
PyObject* dst_min, PyObject* dst_max,
PyObject* src_min, PyObject* src_max) {
PyObject* retval = 0;
switch (src->ndim) {
case 1:
retval = inner_convert<Tdst, Tsrc, 1>(src, dst_min, dst_max, src_min, src_max);
break;
case 2:
retval = inner_convert<Tdst, Tsrc, 2>(src, dst_min, dst_max, src_min, src_max);
break;
case 3:
retval = inner_convert<Tdst, Tsrc, 3>(src, dst_min, dst_max, src_min, src_max);
break;
case 4:
retval = inner_convert<Tdst, Tsrc, 4>(src, dst_min, dst_max, src_min, src_max);
break;
case 1: return inner_convert<Tdst, Tsrc, 1>(src, dst_min, dst_max, src_min, src_max);
case 2: return inner_convert<Tdst, Tsrc, 2>(src, dst_min, dst_max, src_min, src_max);
case 3: return inner_convert<Tdst, Tsrc, 3>(src, dst_min, dst_max, src_min, src_max);
case 4: return inner_convert<Tdst, Tsrc, 4>(src, dst_min, dst_max, src_min, src_max);
default:
PyErr_Format(PyExc_TypeError, "conversion does not support %" PY_FORMAT_SIZE_T "d dimensions", src->ndim);
}
return retval;
return 0;
}
template <typename T> PyObject* convert_to(PyBlitzArrayObject* src,
PyObject* dst_min, PyObject* dst_max,
PyObject* src_min, PyObject* src_max) {
PyObject* retval = 0;
switch (src->type_num) {
case NPY_BOOL:
retval = convert_dim<T, bool>(src, dst_min, dst_max, src_min, src_max);
break;
case NPY_INT8:
retval = convert_dim<T, int8_t>(src, dst_min, dst_max, src_min, src_max);
break;
case NPY_INT16:
retval = convert_dim<T, int16_t>(src, dst_min, dst_max, src_min, src_max);
break;
case NPY_INT32:
retval = convert_dim<T, int32_t>(src, dst_min, dst_max, src_min, src_max);
break;
case NPY_INT64:
retval = convert_dim<T, int64_t>(src, dst_min, dst_max, src_min, src_max);
break;
case NPY_UINT8:
retval = convert_dim<T, uint8_t>(src, dst_min, dst_max, src_min, src_max);
break;
case NPY_UINT16:
retval = convert_dim<T, uint16_t>(src, dst_min, dst_max, src_min, src_max);
break;
case NPY_UINT32:
retval = convert_dim<T, uint32_t>(src, dst_min, dst_max, src_min, src_max);
break;
case NPY_UINT64:
retval = convert_dim<T, uint64_t>(src, dst_min, dst_max, src_min, src_max);
break;
case NPY_FLOAT32:
retval = convert_dim<T, float>(src, dst_min, dst_max, src_min, src_max);
break;
case NPY_FLOAT64:
retval = convert_dim<T, double>(src, dst_min, dst_max, src_min, src_max);
break;
case NPY_BOOL: return convert_dim<T, bool>(src, dst_min, dst_max, src_min, src_max);
case NPY_INT8: return convert_dim<T, int8_t>(src, dst_min, dst_max, src_min, src_max);
case NPY_INT16: return convert_dim<T, int16_t>(src, dst_min, dst_max, src_min, src_max);
case NPY_INT32: return convert_dim<T, int32_t>(src, dst_min, dst_max, src_min, src_max);
case NPY_INT64: return convert_dim<T, int64_t>(src, dst_min, dst_max, src_min, src_max);
case NPY_UINT8: return convert_dim<T, uint8_t>(src, dst_min, dst_max, src_min, src_max);
case NPY_UINT16: return convert_dim<T, uint16_t>(src, dst_min, dst_max, src_min, src_max);
case NPY_UINT32: return convert_dim<T, uint32_t>(src, dst_min, dst_max, src_min, src_max);
case NPY_UINT64: return convert_dim<T, uint64_t>(src, dst_min, dst_max, src_min, src_max);
case NPY_FLOAT32: return convert_dim<T, float>(src, dst_min, dst_max, src_min, src_max);
case NPY_FLOAT64: return convert_dim<T, double>(src, dst_min, dst_max, src_min, src_max);
default:
PyErr_Format(PyExc_TypeError, "conversion from `%s' (%d) is not supported", PyBlitzArray_TypenumAsString(src->type_num), src->type_num);
}
return retval;
return 0;
}
static auto convert_doc = bob::extension::FunctionDoc(
"convert",
"Converts array data type, with optional range squash/expansion",
"This function allows to convert/rescale a array of a given type into another array of a possibly different type. "
"Typically, this can be used to rescale a 16 bit precision grayscale image (2D array) into an 8 bit precision grayscale image."
)
.add_prototype("src, dtype, [dest_range], [source_range]", "converted")
.add_parameter("src", "array_like", "Input array")
.add_parameter("dtype", ":py:class:`numpy.dtype` or anything convertible", "The element data type for the returned ``converted`` array")
.add_parameter("dest_range", "(dtype, dtype)", "[Default: full range of ``dtype``] The range ``[min, max]`` to be deployed at the ``converted`` array")
.add_parameter("source_range", "(X, X)", "[Default: full range of ``src`` data type] Determines the input range ``[min,max]`` that will be used for scaling")
.add_return("converted", "array_like", "A new array with the same shape as ``src``, but re-scaled and with its element type as given by the ``dtype``")
;
static PyObject* py_convert(PyObject*, PyObject* args, PyObject* kwds) {
BOB_TRY
char** kwlist = convert_doc.kwlist();
/* Parses input arguments in a single shot */
static const char* const_kwlist[] = {
"src",
"dtype",
"dest_range",
"source_range",
0 /* Sentinel */
};
static char** kwlist = const_cast<char**>(const_kwlist);
PyBlitzArrayObject* src = 0;
int type_num = NPY_NOTYPE;
PyBlitzArrayObject* src;
int type_num;
PyObject* dst_min = 0;
PyObject* dst_max = 0;
PyObject* src_min = 0;
......@@ -174,62 +125,23 @@ static PyObject* py_convert(PyObject*, PyObject* args, PyObject* kwds) {
)) return 0;
auto src_ = make_safe(src);
PyObject* retval = 0;
switch (type_num) {
case NPY_UINT8:
retval = convert_to<uint8_t>(src, dst_min, dst_max, src_min, src_max);
case NPY_UINT8: return convert_to<uint8_t>(src, dst_min, dst_max, src_min, src_max);
break;
case NPY_UINT16:
retval = convert_to<uint16_t>(src, dst_min, dst_max, src_min, src_max);
case NPY_UINT16: return convert_to<uint16_t>(src, dst_min, dst_max, src_min, src_max);
break;
case NPY_FLOAT64:
retval = convert_to<double>(src, dst_min, dst_max, src_min, src_max);
case NPY_FLOAT64: return convert_to<double>(src, dst_min, dst_max, src_min, src_max);
break;
default:
PyErr_Format(PyExc_TypeError, "conversion to `%s' (%d) is not supported", PyBlitzArray_TypenumAsString(type_num), type_num);
}
if (!retval) return 0;
return PyBlitzArray_NUMPY_WRAP(retval);
return 0;
BOB_CATCH_FUNCTION("convert", 0)
}
PyDoc_STRVAR(s_convert_str, "convert");
PyDoc_STRVAR(s_convert__doc__,
"convert(array, dtype, [dst_range, [src_range]]) -> array\n\
\n\
Converts array data type, with optional range squash/expansion.\n\
\n\
Function which allows to convert/rescale a array of a given type into\n\
another array of a possibly different type with re-scaling. Typically,\n\
this can be used to rescale a 16 bit precision grayscale image (2D\n\
array) into an 8 bit precision grayscale image.\n\
\n\
Keyword Parameters:\n\
\n\
array\n\
(array) Input array\n\
\n\
dtype\n\
(object) Any object that can be convertible to a\n\
:py:class:`numpy.dtype`. Controls the output element type for the\n\
returned array.\n\
\n\
dest_range\n\
(tuple) Determines the range to be deployed at the returned array.\n\
\n\
source_range\n\
(tuple) Determines the input range that will be used for scaling.\n\
\n\
Returns a new array with the same shape as this one, but re-scaled and\n\
with its element type as indicated by the user.\n\
"
);
static auto sort_doc = bob::extension::FunctionDoc(
......@@ -267,10 +179,10 @@ BOB_CATCH_FUNCTION("sort", 0)
static PyMethodDef module_methods[] = {
{
s_convert_str,
convert_doc.name(),
(PyCFunction)py_convert,
METH_VARARGS|METH_KEYWORDS,
s_convert__doc__
convert_doc.doc()
},
{
sort_doc.name(),
......
......@@ -10,7 +10,7 @@ import sys
import logging
from ._logging import reset
# get the default logger of Bob
# get the default root logger of Bob
_logger = logging.getLogger('bob')
# by default, warning and error messages should be written to sys.stderr
......@@ -38,18 +38,22 @@ del atexit
# helper functions to instantiate and set-up logging
def setup(logger_name, format="%(name)s@%(asctime)s -- %(levelname)s: %(message)s"):
"""This function returns a logger object that is set up to perform logging using Bob loggers.
"""setup(logger_name, [format]) -> logger
**Keyword parameters:**
This function returns a logger object that is set up to perform logging using Bob loggers.
logger_name : str
**Parameters:**
``logger_name`` : str
The name of the module to generate logs for
format : str
``format`` : str
The format of the logs, see :py:class:`logging.LogRecord` for more details.
By default, the log contains the logger name, the log time, the log level and the massage.
Returns : :py:class:`logging.Logger`
**Returns:**
``logger`` : :py:class:`logging.Logger`
The logger configured for logging.
The same logger can be retrieved using the :py:func:`logging.getLogger` function.
"""
......@@ -76,33 +80,35 @@ def setup(logger_name, format="%(name)s@%(asctime)s -- %(levelname)s: %(message)
def add_command_line_option(parser, short_option = '-v'):
"""Adds the verbosity command line option to the given parser.
The verbosity can by set to 0 (error), 1 (warning), 2 (info) or 3 (debug) by including the according number of --verbose command line arguments (e.g., ``-vv`` for info level).
**Keyword parameters:**
**Parameters:**
parser : :py:class:`argparse.ArgumentParser` or one of its derivatives
A command line parser that you want to add a verbose option to.
``parser`` : :py:class:`argparse.ArgumentParser` or one of its derivatives
A command line parser that you want to add a verbose option to
short_option : str
``short_option`` : str
The short command line option that should be used for increasing the verbosity.
By default, ``'-v'`` is considered as the shortcut
"""
parser.add_argument(short_option, '--verbose', action = 'count', default = 0,
help = "Increase the verbosity level from 0 (only error messages) to 1 (warnings), 2 (log messages), 3 (debug information) by adding the --verbose option as often as desired (e.g. '-vvv' for debug).")
def set_verbosity_level(logger, level):
"""Sets the log level.
"""Sets the log level for the given logger.
**Keyword parameters:**
**Parameters:**
logger : :py:class:`logging.Logger` or str
``logger`` : :py:class:`logging.Logger` or str
The logger to generate logs for, or the name of the module to generate logs for.
level : int
``level`` : int
Possible log levels are: 0: Error; 1: Warning; 2: Info; 3: Debug.
"""
if level not in range(0,4):
raise ValueError("The verbosity level %d does not exist. Please reduce the number of '--verbose' parameters in your call" % level)
raise ValueError("The verbosity level %d does not exist. Please reduce the number of '--verbose' parameters in your command line" % level)
# set up the verbosity level of the logging system
log_level = {
0: logging.ERROR,
......
......@@ -13,6 +13,7 @@
#endif
#include <bob.blitz/capi.h>
#include <bob.blitz/cleanup.h>
#include <bob.extension/documentation.h>
#include <boost/shared_array.hpp>
#include <boost/make_shared.hpp>
......@@ -77,10 +78,10 @@ struct PythonLoggingOutputDevice: public bob::core::OutputDevice {
#if PYTHON_LOGGING_DEBUG != 0
pthread_t thread_id = pthread_self();
static_log << "(" << std::hex << thread_id << std::dec
<< ") Constructing new PythonLoggingOutputDevice from logger `logging.logger('"
<< PyString_AsString(PyObject_GetAttrString(m_logger, "name")) << "')."
<< name << "' (@" << std::hex << m_logger << std::dec
<< ")" << std::endl;
<< ") Constructing new PythonLoggingOutputDevice from logger `logging.logger('"
<< PyString_AsString(PyObject_GetAttrString(m_logger, "name")) << "')."
<< name << "' (@" << std::hex << m_logger << std::dec
<< ")" << std::endl;
#endif
}
......@@ -96,8 +97,8 @@ struct PythonLoggingOutputDevice: public bob::core::OutputDevice {
_name = PyString_AsString(PyObject_GetAttrString(m_logger, "name"));
}
static_log << "(" << std::hex << thread_id << std::dec
<< ") Destroying PythonLoggingOutputDevice with logger `" << _name
<< "' (" << std::hex << m_logger << std::dec << ")" << std::endl;
<< ") Destroying PythonLoggingOutputDevice with logger `" << _name
<< "' (" << std::hex << m_logger << std::dec << ")" << std::endl;
#endif
if (m_logger) close();
}
......@@ -113,8 +114,8 @@ struct PythonLoggingOutputDevice: public bob::core::OutputDevice {
_name = PyString_AsString(PyObject_GetAttrString(m_logger, "name"));
}
static_log << "(" << std::hex << thread_id << std::dec
<< ") Closing PythonLoggingOutputDevice with logger `" << _name
<< "' (" << std::hex << m_logger << std::dec << ")" << std::endl;
<< ") Closing PythonLoggingOutputDevice with logger `" << _name
<< "' (" << std::hex << m_logger << std::dec << ")" << std::endl;
#endif
Py_XDECREF(m_logger);
m_logger = 0;
......@@ -141,21 +142,21 @@ struct PythonLoggingOutputDevice: public bob::core::OutputDevice {
pthread_t thread_id = pthread_self();
std::string message(s, n);
static_log << "(" << std::hex << thread_id << std::dec
<< ") Processing message `" << boost::algorithm::trim_right_copy(message)
<< "' (size = " << n << ") with method `logging.logger('"
<< PyString_AsString(PyObject_GetAttrString(m_logger, "name")) << "')."
<< PyString_AsString(m_name) << "'" << std::endl;
<< ") Processing message `" << boost::algorithm::trim_right_copy(message)
<< "' (size = " << n << ") with method `logging.logger('"
<< PyString_AsString(PyObject_GetAttrString(m_logger, "name")) << "')."
<< PyString_AsString(m_name) << "'" << std::endl;
#endif
int len = n;
while (std::isspace(s[len-1])) len -= 1;
PyObject* value = Py_BuildValue("s#", s, len);
auto value_ = make_safe(value);
PyObject* result = PyObject_CallMethodObjArgs(m_logger, m_name, value, 0);
Py_DECREF(value);
auto result_ = make_xsafe(result);
if (!result) len = 0;
else Py_DECREF(result);
PyGILState_Release(gil);
......@@ -164,15 +165,28 @@ struct PythonLoggingOutputDevice: public bob::core::OutputDevice {
};
static int set_stream(boost::iostreams::stream<bob::core::AutoOutputDevice>& s, PyObject* o, const char* n) {
static auto reset_doc = bob::extension::FunctionDoc(
"reset",
"Resets the standard C++ logging devices, or sets it to the given callable",
"This function allows you to manipulate the sinks for messages emitted in C++, using Python callables. "
"The first variant (without parameters) will reset all logging output to :py:data:`sys.stderr`. "
"The second variant will reset the given logger to the given callable. "
"If ``stream`` is not specified, it resets all loggers.\n\n"
"This function raises a :py:exc:`ValueError` in case of problems setting or resetting any of the streams."
)
.add_prototype("")
.add_prototype("callable, [stream]")
.add_parameter("callable", "callable", "A python callable that receives an ``str`` and dumps messages to the desired output channel")
.add_parameter("stream", "one of ('debug', 'info', warn', 'error')", "[optional] If specified, only the given logger is send to the given callable. Otherwise all loggers are reset to that callable.")
;
static int set_stream(boost::iostreams::stream<bob::core::AutoOutputDevice>& s, PyObject* o, const char* n) {
// if no argument or None, write everything else to stderr
if (!o || o == Py_None) {
#if PYTHON_LOGGING_DEBUG != 0
pthread_t thread_id = pthread_self();
static_log << "(" << std::hex << thread_id << std::dec
<< ") Resetting stream `" << n << "' to stderr" << std::endl;
<< ") Resetting stream `" << n << "' to stderr" << std::endl;
#endif
s.close();
s.open("stderr");
......@@ -181,18 +195,17 @@ static int set_stream(boost::iostreams::stream<bob::core::AutoOutputDevice>& s,
if (PyObject_HasAttrString(o, n)) {
PyObject* callable = PyObject_GetAttrString(o, n);
auto callable_ = make_safe(callable);
if (callable && PyCallable_Check(callable)) {
#if PYTHON_LOGGING_DEBUG != 0
pthread_t thread_id = pthread_self();
static_log << "(" << std::hex << thread_id << std::dec
<< ") Setting stream `" << n << "' to logger at " << std::hex
<< o << std::dec << std::endl;
pthread_t thread_id = pthread_self();
static_log << "(" << std::hex << thread_id << std::dec
<< ") Setting stream `" << n << "' to logger at " << std::hex
<< o << std::dec << std::endl;
#endif
s.close();
s.open(boost::make_shared<PythonLoggingOutputDevice>(o, n));
Py_DECREF(callable);
return 1;
}
}
......@@ -200,63 +213,35 @@ static int set_stream(boost::iostreams::stream<bob::core::AutoOutputDevice>& s,
// if you get to this point, set an error
PyErr_Format(PyExc_ValueError, "argument to set stream `%s' needs to be either None or an object with a callable named `%s'", n, n);
return 0;
}
static PyObject* reset(PyObject*, PyObject* args, PyObject* kwds) {
BOB_TRY
char** kwlist = reset_doc.kwlist(1);
/* Parses input arguments in a single shot */
static const char* const_kwlist[] = {
"logger",
0 /* Sentinel */
};
static char** kwlist = const_cast<char**>(const_kwlist);
PyObject* logger = 0;
if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O",
kwlist, &logger)) return 0;
PyObject* callable = 0;
const char* stream = 0;
if (!set_stream(bob::core::debug, logger, "debug")) return 0;
if (!set_stream(bob::core::info, logger, "info")) return 0;
if (!set_stream(bob::core::warn, logger, "warn")) return 0;
if (!set_stream(bob::core::error, logger, "error")) return 0;
if (!PyArg_ParseTupleAndKeywords(args, kwds, "|Os", kwlist, &callable, &stream)) return 0;
if (!stream){
// reset all streams to either stderr or the given callable
if (!set_stream(bob::core::debug, callable, "debug")) return 0;
if (!set_stream(bob::core::info, callable, "info")) return 0;
if (!set_stream(bob::core::warn, callable, "warn")) return 0;
if (!set_stream(bob::core::error, callable, "error")) return 0;
} else {
if (strcmp(stream, "debug") && strcmp(stream, "info") && strcmp(stream, "warn") && strcmp(stream, "error")){
PyErr_Format(PyExc_ValueError, "If given, the parameter 'stream' needs to be one of ('debug', 'info', warn', 'error), not %s", stream);
return 0;
}
if (!set_stream(bob::core::error, callable, stream)) return 0;
}
Py_RETURN_NONE;
BOB_CATCH_FUNCTION("reset", 0)
}
PyDoc_STRVAR(s_reset_str, "reset");
PyDoc_STRVAR(s_reset__doc__,
"reset([debug, [info, [warn, [error]]]]) -> None\n\
\n\
Resets the standard C++ logging devices.\n\
\n\
This function allows you to manipulate the sinks for messages emitted\n\
in C++, using Python callables.\n\
\n\
Keyword Parameters:\n\
\n\
debug\n\
[optional] (callable) A callable that receives a string and dumps\n\
messages to the desired output channel.\n\
\n\
info\n\
[optional] (callable) A callable that receives a string and dumps\n\
messages to the desired output channel.\n\
\n\
warn\n\
[optional] (callable) A callable that receives a string and dumps\n\
messages to the desired output channel.\n\
\n\
error\n\
[optional] (callable) A callable that receives a string and dumps\n\
messages to the desired output channel.\n\
\n\
Raises a :py:class:`ValueError` in case of problems setting or resetting\n\
any of the streams.\n\
"
);
/**************************
* Testing Infrastructure *
**************************/
......@@ -290,9 +275,9 @@ static void* log_message_inner(void* cookie) {
*(mi->s) << mi->message
# if PYTHON_LOGGING_DEBUG != 0
<< " (thread " << mi->thread_id << "; iteration " << i << ")"
<< " (thread " << mi->thread_id << "; iteration " << i << ")"
# endif
<< std::endl;
<< std::endl;
mi->s->flush();
}
......@@ -311,23 +296,23 @@ static void* log_message_inner(void* cookie) {
return 0;
}
/**
* A test function for your python bindings
*/
static auto _logmsg_doc = bob::extension::FunctionDoc(
"_log_message",
"Logs a message into Bob's logging system from C++",
"This function is bound for testing purposes only and is not part of the Python API for bob.core"
)
.add_prototype("ntimes, stream, message")
.add_parameter("ntimes", "int", "The number of times to print the given message")
.add_parameter("stream", "str", "The stream to use for logging the message. Choose from ``('debug', 'info', 'warn', 'error')")
.add_parameter("message", "str", "The message to be logged")
;
static PyObject* log_message(PyObject*, PyObject* args, PyObject* kwds) {
BOB_TRY
char** kwlist = _logmsg_doc.kwlist();
/* Parses input arguments in a single shot */
static const char* const_kwlist[] = {
"ntimes",
"stream",
"message",
0 /* Sentinel */
};
static char** kwlist = const_cast<char**>(const_kwlist);
unsigned int ntimes = 0;
const char* stream = 0;
const char* message = 0;
unsigned int ntimes;
const char* stream;
const char* message;
if (!PyArg_ParseTupleAndKeywords(args, kwds, "Iss",
kwlist, &ntimes, &stream, &message)) return 0;
......@@ -340,7 +325,7 @@ static PyObject* log_message(PyObject*, PyObject* args, PyObject* kwds) {
else if (strncmp(stream, "error", 5) == 0) s = &bob::core::error;
else if (strncmp(stream, "fatal", 5) == 0) s = &bob::core::error;
else {
PyErr_SetString(PyExc_ValueError, "parameter `stream' must be one of 'debug', 'info', 'warn', 'error' or 'fatal' (synomym for 'error')");
PyErr_Format(PyExc_ValueError, "parameter `stream' must be one of 'debug', 'info', 'warn', 'error' or 'fatal' (synomym for 'error'), not '%s'", stream);
return 0;
}
......@@ -356,62 +341,41 @@ static PyObject* log_message(PyObject*, PyObject* args, PyObject* kwds) {
PyEval_RestoreThread(no_gil);
Py_RETURN_NONE;
BOB_CATCH_FUNCTION("_log_message", 0)
}
PyDoc_STRVAR(s_logmsg_str, "__log_message__");
PyDoc_STRVAR(s_logmsg__doc__,
"__log_message__(ntimes, stream, message) -> None\n\
\n\
Logs a message into Bob's logging system from C++.\n\
\n\
This method is included for testing purposes only and should not be\n\
considered part of the Python API for Bob.\n\
\n\
Keyword parameters:\n\
\n\
ntimes\n\
(integer) The number of times to print the given message\n\
\n\
stream\n\
(string) The stream to use for logging the message. Choose from:\n\
``'debug'``, ``'info'``, ``'warn'`` or ``'error'``\n\
\n\
message\n\
(string) The message to be logged.\n\
\n"
);
/**