From c5e0e8138ca58cfe2ed1f362d2e4da74ce1c4c72 Mon Sep 17 00:00:00 2001
From: Andre Anjos <andre.dos.anjos@gmail.com>
Date: Thu, 14 Nov 2013 18:21:04 +0100
Subject: [PATCH] Append/Set/Replace implemented; More tests passing
---
xbob/io/bobskin.h | 6 +-
xbob/io/hdf5.cpp | 655 +++++++++++++++++++++++++++++++++++---
xbob/io/test/test_hdf5.py | 12 +-
3 files changed, 611 insertions(+), 62 deletions(-)
diff --git a/xbob/io/bobskin.h b/xbob/io/bobskin.h
index bede271..8a9fe7b 100644
--- a/xbob/io/bobskin.h
+++ b/xbob/io/bobskin.h
@@ -22,17 +22,17 @@ class bobskin: public bob::core::array::interface {
public: //api
/**
- * @brief Builds a new array an array like object
+ * @brief Builds a new skin from an array like object
*/
bobskin(PyObject* array, bob::core::array::ElementType eltype);
/**
- * @brief Builds a new array an array like object
+ * @brief Builds a new skin from a numpy array object
*/
bobskin(PyArrayObject* array, bob::core::array::ElementType eltype);
/**
- * @brief Builds a new array an array like object
+ * @brief Builds a new skin around a blitz array object
*/
bobskin(PyBlitzArrayObject* array);
diff --git a/xbob/io/hdf5.cpp b/xbob/io/hdf5.cpp
index 9c57816..e578074 100644
--- a/xbob/io/hdf5.cpp
+++ b/xbob/io/hdf5.cpp
@@ -288,51 +288,60 @@ If the given path is relative, it is take w.r.t. to the current\n\
working directory.\n\
");
+static bob::io::hdf5type PyBobIo_H5FromTypenum (int type_num) {
+
+ switch(type_num) {
+ case NPY_STRING: return bob::io::s;
+ case NPY_BOOL: return bob::io::b;
+ case NPY_INT8: return bob::io::i8;
+ case NPY_INT16: return bob::io::i16;
+ case NPY_INT32: return bob::io::i32;
+ case NPY_INT64: return bob::io::i64;
+ case NPY_UINT8: return bob::io::u8;
+ case NPY_UINT16: return bob::io::u16;
+ case NPY_UINT32: return bob::io::u32;
+ case NPY_UINT64: return bob::io::u64;
+ case NPY_FLOAT32: return bob::io::f32;
+ case NPY_FLOAT64: return bob::io::f64;
+#ifdef NPY_FLOAT128
+ case NPY_FLOAT128: return bob::io::f128;
+#endif
+ case NPY_COMPLEX64: return bob::io::c64;
+ case NPY_COMPLEX128: return bob::io::c128;
+#ifdef NPY_COMPLEX256
+ case NPY_COMPLEX256: return bob::io::c256;
+#endif
+ default: return bob::io::unsupported;
+ }
+
+}
+
static int PyBobIo_H5AsTypenum (bob::io::hdf5type type) {
switch(type) {
- case bob::io::s:
- return NPY_STRING;
- case bob::io::b:
- return NPY_BOOL;
- case bob::io::i8:
- return NPY_INT8;
- case bob::io::i16:
- return NPY_INT16;
- case bob::io::i32:
- return NPY_INT32;
- case bob::io::i64:
- return NPY_INT64;
- case bob::io::u8:
- return NPY_UINT8;
- case bob::io::u16:
- return NPY_UINT16;
- case bob::io::u32:
- return NPY_UINT32;
- case bob::io::u64:
- return NPY_UINT64;
- case bob::io::f32:
- return NPY_FLOAT32;
- case bob::io::f64:
- return NPY_FLOAT64;
+ case bob::io::s: return NPY_STRING;
+ case bob::io::b: return NPY_BOOL;
+ case bob::io::i8: return NPY_INT8;
+ case bob::io::i16: return NPY_INT16;
+ case bob::io::i32: return NPY_INT32;
+ case bob::io::i64: return NPY_INT64;
+ case bob::io::u8: return NPY_UINT8;
+ case bob::io::u16: return NPY_UINT16;
+ case bob::io::u32: return NPY_UINT32;
+ case bob::io::u64: return NPY_UINT64;
+ case bob::io::f32: return NPY_FLOAT32;
+ case bob::io::f64: return NPY_FLOAT64;
#ifdef NPY_FLOAT128
- case bob::io::f128:
- return NPY_FLOAT128;
+ case bob::io::f128: return NPY_FLOAT128;
#endif
- case bob::io::c64:
- return NPY_COMPLEX64;
- case bob::io::c128:
- return NPY_COMPLEX128;
+ case bob::io::c64: return NPY_COMPLEX64;
+ case bob::io::c128: return NPY_COMPLEX128;
#ifdef NPY_COMPLEX256
- case bob::io::c256:
- return NPY_COMPLEX256;
+ case bob::io::c256: return NPY_COMPLEX256;
#endif
- default:
- PyErr_Format(PyExc_TypeError, "unsupported Bob/C++ element type (%s)", bob::io::stringize(type));
+ default: return NPY_NOTYPE;
}
- return NPY_NOTYPE;
-
}
static PyObject* PyBobIo_HDF5TypeAsTuple (const bob::io::HDF5Type& t) {
@@ -341,7 +350,11 @@ static PyObject* PyBobIo_HDF5TypeAsTuple (const bob::io::HDF5Type& t) {
const hsize_t* shptr = sh.get();
int type_num = PyBobIo_H5AsTypenum(t.type());
- if (type_num == NPY_NOTYPE) return 0;
+ if (type_num == NPY_NOTYPE) {
+ PyErr_Format(PyExc_TypeError, "unsupported HDF5 element type (%d) found during conversion to numpy type number", (int)t.type());
+ return 0;
+ }
+
PyObject* dtype = reinterpret_cast<PyObject*>(PyArray_DescrFromType(type_num));
if (!dtype) return 0;
@@ -689,20 +702,17 @@ static PyObject* PyBobIoHDF5File_Xread(PyBobIoHDF5FileObject* self,
}
//read as an numpy array
- bob::core::array::typeinfo atype;
- type.copy_to(atype);
-
- int type_num = PyBobIo_AsTypenum(atype.dtype);
+ int type_num = PyBobIo_H5AsTypenum(type.type());
if (type_num == NPY_NOTYPE) return 0; ///< failure
npy_intp pyshape[NPY_MAXDIMS];
- for (int k=0; k<atype.nd; ++k) pyshape[k] = atype.shape[k];
+ for (int k=0; k<shape.n(); ++k) pyshape[k] = shape.get()[k];
- PyObject* retval = PyArray_SimpleNew(atype.nd, pyshape, type_num);
+ PyObject* retval = PyArray_SimpleNew(shape.n(), pyshape, type_num);
if (!retval) return 0;
try {
- self->f->read_buffer(p, pos, atype, PyArray_DATA((PyArrayObject*)retval));
+ self->f->read_buffer(p, pos, type, PyArray_DATA((PyArrayObject*)retval));
}
catch (std::exception& e) {
PyErr_SetString(PyExc_RuntimeError, e.what());
@@ -812,18 +822,313 @@ with N-1 dimensions. It returns a single\n\
value >= 0, or a list of arrays otherwise.\n\
");
-static PyObject* PyBobIoHDF5File_Replace(PyBobIoHDF5FileObject* self, PyObject *args, PyObject* kwds) {
+/**
+ * Sets at 't', the type of the object 'o' according to our support types.
+ * Raise in case of problems. Furthermore, returns 'true' if the object is as
+ * simple scalar.
+ */
+
+static char* PyBobIo_GetString(PyObject* o) {
+#if PY_VERSION_HEX < 0x03000000
+ return PyString_AsString(o);
+#else
+ return PyBytes_AsString(o);
+#endif
+}
+
+static int PyBobIoHDF5File_SetStringType(bob::io::HDF5Type& t, PyObject* o) {
+ char* s = PyBobIo_GetString(o);
+ if (!s) return -1;
+ t = bob::io::HDF5Type(s);
+ return 0;
+}
+
+template <typename T> int PyBobIoHDF5File_SetType(bob::io::HDF5Type& t) {
+ T v;
+ t = bob::io::HDF5Type(v);
+ return 0;
+}
+
+/**
+ * A function to check for python scalars that works with numpy-1.6.x
+ */
+static bool PyBobIoHDF5File_IsPythonScalar(PyObject* obj) {
+ return (
+ PyBool_Check(obj) ||
+#if PY_VERSION_HEX < 0x03000000
+ PyString_Check(obj) ||
+#else
+ PyBytes_Check(obj) ||
+#endif
+ PyUnicode_Check(obj) ||
+#if PY_VERSION_HEX < 0x03000000
+ PyInt_Check(obj) ||
+#endif
+ PyLong_Check(obj) ||
+ PyFloat_Check(obj) ||
+ PyComplex_Check(obj)
+ );
+}
+
+/**
+ * Returns the type of object `op' is - a scalar (return value = 0), a
+ * blitz.array (return value = 1), a numpy.ndarray (return value = 2), an
+ * object which is convertible to a numpy.ndarray (return value = 3) or returns
+ * -1 if the object cannot be converted. No error is set on the python stack.
+ *
+ * If the object is convertible into a numpy.ndarray, then it is converted into
+ * a numpy ndarray and the resulting object is placed in `converted'. If
+ * `*converted' is set to 0 (NULL), then we don't try a conversion, returning
+ * -1.
+ */
+static int PyBobIoHDF5File_GetObjectType(PyObject* o, bob::io::HDF5Type& t,
+ PyObject** converted=0) {
+
+ if (PyArray_IsScalar(o, Generic) || PyBobIoHDF5File_IsPythonScalar(o)) {
+
+ if (PyArray_IsScalar(o, String))
+ return PyBobIoHDF5File_SetStringType(t, o);
+
+ else if (PyBool_Check(o))
+ return PyBobIoHDF5File_SetType<bool>(t);
+
+#if PY_VERSION_HEX < 0x03000000
+ else if (PyString_Check(o))
+ return PyBobIoHDF5File_SetStringType(t, o);
+
+#else
+ else if (PyBytes_Check(o))
+ return PyBobIoHDF5File_SetStringType(t, o);
+
+#endif
+ else if (PyUnicode_Check(o))
+ return PyBobIoHDF5File_SetStringType(t, o);
+
+#if PY_VERSION_HEX < 0x03000000
+ else if (PyInt_Check(o))
+ return PyBobIoHDF5File_SetType<int32_t>(t);
+
+#endif
+ else if (PyLong_Check(o))
+ return PyBobIoHDF5File_SetType<int64_t>(t);
+
+ else if (PyFloat_Check(o))
+ return PyBobIoHDF5File_SetType<double>(t);
+
+ else if (PyComplex_Check(o))
+ return PyBobIoHDF5File_SetType<std::complex<double> >(t);
+
+ else if (PyArray_IsScalar(o, Bool))
+ return PyBobIoHDF5File_SetType<bool>(t);
+
+ else if (PyArray_IsScalar(o, Int8))
+ return PyBobIoHDF5File_SetType<int8_t>(t);
+
+ else if (PyArray_IsScalar(o, UInt8))
+ return PyBobIoHDF5File_SetType<uint8_t>(t);
+
+ else if (PyArray_IsScalar(o, Int16))
+ return PyBobIoHDF5File_SetType<int16_t>(t);
+
+ else if (PyArray_IsScalar(o, UInt16))
+ return PyBobIoHDF5File_SetType<uint16_t>(t);
+
+ else if (PyArray_IsScalar(o, Int32))
+ return PyBobIoHDF5File_SetType<int32_t>(t);
+
+ else if (PyArray_IsScalar(o, UInt32))
+ return PyBobIoHDF5File_SetType<uint32_t>(t);
+
+ else if (PyArray_IsScalar(o, Int64))
+ return PyBobIoHDF5File_SetType<int64_t>(t);
+
+ else if (PyArray_IsScalar(o, UInt64))
+ return PyBobIoHDF5File_SetType<uint64_t>(t);
+
+ else if (PyArray_IsScalar(o, Float))
+ return PyBobIoHDF5File_SetType<float>(t);
+
+ else if (PyArray_IsScalar(o, Double))
+ return PyBobIoHDF5File_SetType<double>(t);
+
+ else if (PyArray_IsScalar(o, LongDouble))
+ return PyBobIoHDF5File_SetType<long double>(t);
+
+ else if (PyArray_IsScalar(o, CFloat))
+ return PyBobIoHDF5File_SetType<std::complex<float> >(t);
+
+ else if (PyArray_IsScalar(o, CDouble))
+ return PyBobIoHDF5File_SetType<std::complex<double> >(t);
+
+ else if (PyArray_IsScalar(o, CLongDouble))
+ return PyBobIoHDF5File_SetType<std::complex<long double> >(t);
+
+ //if you get to this, point, it is an unsupported scalar
+ return -1;
+
+ }
+
+ else if (PyBlitzArray_Check(o)) {
+
+ PyBlitzArrayObject* bz = reinterpret_cast<PyBlitzArrayObject*>(o);
+ bob::io::hdf5type h5type = PyBobIo_H5FromTypenum(bz->type_num);
+ if (h5type == bob::io::unsupported) return -1;
+ bob::io::HDF5Shape h5shape(bz->ndim, bz->shape);
+ t = bob::io::HDF5Type(h5type, h5shape);
+ return 1;
+
+ }
+
+ else if (PyArray_CheckExact(o) && PyArray_ISCARRAY_RO((PyArrayObject*)o)) {
+
+ PyArrayObject* np = reinterpret_cast<PyArrayObject*>(o);
+ bob::io::hdf5type h5type = PyBobIo_H5FromTypenum(PyArray_DESCR(np)->type_num);
+ if (h5type == bob::io::unsupported) return -1;
+ bob::io::HDF5Shape h5shape(PyArray_NDIM(np), PyArray_DIMS(np));
+ t = bob::io::HDF5Type(h5type, h5shape);
+ return 2;
+
+ }
+
+ else if (converted) {
+
+ *converted = PyArray_FromAny(o, 0, 1, 0,
+#if NPY_FEATURE_VERSION >= NUMPY17_API /* NumPy C-API version >= 1.7 */
+ NPY_ARRAY_CARRAY_RO,
+# else
+ NPY_CARRAY_RO,
+# endif
+ 0);
+ if (!*converted) return -1; ///< error condition
+
+ PyArrayObject* np = reinterpret_cast<PyArrayObject*>(*converted);
+ bob::io::hdf5type h5type = PyBobIo_H5FromTypenum(PyArray_DESCR(np)->type_num);
+ if (h5type == bob::io::unsupported) return -1;
+ bob::io::HDF5Shape h5shape(PyArray_NDIM(np), PyArray_DIMS(np));
+ t = bob::io::HDF5Type(h5type, h5shape);
+ return 3;
+
+ }
+
+ //if you get to this, point, it is an unsupported type
+ return -1;
+
+}
+
+template <typename T>
+static PyObject* PyBobIoHDF5File_ReplaceScalar(PyBobIoHDF5FileObject* self,
+ const char* path, Py_ssize_t pos, PyObject* o) {
+
+ T value = PyBlitzArrayCxx_AsCScalar<T>(o);
+ if (PyErr_Occurred()) return 0;
+ self->f->replace(path, pos, value);
+
+ Py_RETURN_NONE;
+
+}
+
+static PyObject* PyBobIoHDF5File_Replace(PyBobIoHDF5FileObject* self, PyObject* args, PyObject* kwds) {
/* Parses input arguments in a single shot */
static const char* const_kwlist[] = {"path", "pos", "data", 0};
static char** kwlist = const_cast<char**>(const_kwlist);
- PyObject* key = 0;
+ char* path = 0;
Py_ssize_t pos = -1;
PyObject* data = 0;
- if (!PyArg_ParseTupleAndKeywords(args, kwds, "OnO", kwlist, &key, &pos, &data)) return 0;
+ if (!PyArg_ParseTupleAndKeywords(args, kwds, "snO", kwlist, &path, &pos, &data)) return 0;
+
+ bob::io::HDF5Type type;
+ PyObject* converted = 0;
+ int is_array = PyBobIoHDF5File_GetObjectType(data, type, &converted);
+ if (is_array < 0) { ///< error condition, signal
+ PyErr_Format(PyExc_TypeError, "error replacing position %" PY_FORMAT_SIZE_T "d of dataset `%s' at HDF5 file `%s': no support for storing objects of type `%s' on HDF5 files", pos, path, self->f->filename().c_str(), data->ob_type->tp_name);
+ return 0;
+ }
+
+ try {
+
+ if (!is_array) { //write as a scalar
+
+ switch(type.type()) {
+ case bob::io::s:
+ {
+ char* value = PyBobIo_GetString(data);
+ if (!value) return 0;
+ self->f->replace(path, pos, value);
+ Py_RETURN_NONE;
+ }
+ case bob::io::b:
+ return PyBobIoHDF5File_ReplaceScalar<bool>(self, path, pos, data);
+ case bob::io::i8:
+ return PyBobIoHDF5File_ReplaceScalar<int8_t>(self, path, pos, data);
+ case bob::io::i16:
+ return PyBobIoHDF5File_ReplaceScalar<int16_t>(self, path, pos, data);
+ case bob::io::i32:
+ return PyBobIoHDF5File_ReplaceScalar<int32_t>(self, path, pos, data);
+ case bob::io::i64:
+ return PyBobIoHDF5File_ReplaceScalar<int64_t>(self, path, pos, data);
+ case bob::io::u8:
+ return PyBobIoHDF5File_ReplaceScalar<uint8_t>(self, path, pos, data);
+ case bob::io::u16:
+ return PyBobIoHDF5File_ReplaceScalar<uint16_t>(self, path, pos, data);
+ case bob::io::u32:
+ return PyBobIoHDF5File_ReplaceScalar<uint32_t>(self, path, pos, data);
+ case bob::io::u64:
+ return PyBobIoHDF5File_ReplaceScalar<uint64_t>(self, path, pos, data);
+ case bob::io::f32:
+ return PyBobIoHDF5File_ReplaceScalar<float>(self, path, pos, data);
+ case bob::io::f64:
+ return PyBobIoHDF5File_ReplaceScalar<double>(self, path, pos, data);
+ case bob::io::f128:
+ return PyBobIoHDF5File_ReplaceScalar<long double>(self, path, pos, data);
+ case bob::io::c64:
+ return PyBobIoHDF5File_ReplaceScalar<std::complex<float> >(self, path, pos, data);
+ case bob::io::c128:
+ return PyBobIoHDF5File_ReplaceScalar<std::complex<double> >(self, path, pos, data);
+ case bob::io::c256:
+ return PyBobIoHDF5File_ReplaceScalar<std::complex<long double> >(self, path, pos, data);
+ default:
+ break;
+ }
+
+ }
+
+ else { //write as array
+
+ switch (is_array) {
+ case 1: //blitz.array
+ self->f->write_buffer(path, pos, type, ((PyBlitzArrayObject*)data)->data);
+ break;
+
+ case 2: //numpy.ndarray
+ self->f->write_buffer(path, pos, type, PyArray_DATA((PyArrayObject*)data));
+ break;
+
+ case 3: //converted numpy.ndarray
+ self->f->write_buffer(path, pos, type, PyArray_DATA((PyArrayObject*)converted));
+ Py_DECREF(converted);
+ break;
+
+ default:
+ PyErr_Format(PyExc_NotImplementedError, "error replacing position %" PY_FORMAT_SIZE_T "d of dataset `%s' at HDF5 file `%s': HDF5 replace function is uncovered for array type %d (DEBUG ME)", pos, path, self->f->filename().c_str(), is_array);
+ return 0;
+ }
+
+ }
+
+ }
+ catch (std::exception& e) {
+ PyErr_SetString(PyExc_RuntimeError, e.what());
+ return 0;
+ }
+ catch (...) {
+ PyErr_Format(PyExc_RuntimeError, "cannot replace object in position %" PY_FORMAT_SIZE_T "d at HDF5 file `%s': unknown exception caught", pos, self->f->filename().c_str());
+ return 0;
+ }
Py_RETURN_NONE;
+
}
PyDoc_STRVAR(s_replace_str, "replace");
@@ -851,18 +1156,152 @@ data\n\
\n\
");
-static PyObject* PyBobIoHDF5File_Append(PyBobIoHDF5FileObject* self, PyObject *args, PyObject* kwds) {
+template <typename T>
+static int PyBobIoHDF5File_AppendScalar(PyBobIoHDF5FileObject* self,
+ const char* path, PyObject* o) {
+
+ T value = PyBlitzArrayCxx_AsCScalar<T>(o);
+ if (PyErr_Occurred()) return 0;
+ self->f->append(path, value);
+
+ return 1;
+
+}
+
+static int PyBobIoHDF5File_InnerAppend(PyBobIoHDF5FileObject* self, const char* path, PyObject* data, Py_ssize_t compression) {
+
+ bob::io::HDF5Type type;
+ PyObject* converted = 0;
+ int is_array = PyBobIoHDF5File_GetObjectType(data, type, &converted);
+ if (is_array < 0) { ///< error condition, signal
+ PyErr_Format(PyExc_TypeError, "error appending to object `%s' of HDF5 file `%s': no support for storing objects of type `%s' on HDF5 files", path, self->f->filename().c_str(), data->ob_type->tp_name);
+ return 0;
+ }
+ try {
+
+ if (!is_array) { //write as a scalar
+
+ switch(type.type()) {
+ case bob::io::s:
+ {
+ char* value = PyBobIo_GetString(data);
+ if (!value) return 0;
+ self->f->append(path, value);
+ return 1;
+ }
+ case bob::io::b:
+ return PyBobIoHDF5File_AppendScalar<bool>(self, path, data);
+ case bob::io::i8:
+ return PyBobIoHDF5File_AppendScalar<int8_t>(self, path, data);
+ case bob::io::i16:
+ return PyBobIoHDF5File_AppendScalar<int16_t>(self, path, data);
+ case bob::io::i32:
+ return PyBobIoHDF5File_AppendScalar<int32_t>(self, path, data);
+ case bob::io::i64:
+ return PyBobIoHDF5File_AppendScalar<int64_t>(self, path, data);
+ case bob::io::u8:
+ return PyBobIoHDF5File_AppendScalar<uint8_t>(self, path, data);
+ case bob::io::u16:
+ return PyBobIoHDF5File_AppendScalar<uint16_t>(self, path, data);
+ case bob::io::u32:
+ return PyBobIoHDF5File_AppendScalar<uint32_t>(self, path, data);
+ case bob::io::u64:
+ return PyBobIoHDF5File_AppendScalar<uint64_t>(self, path, data);
+ case bob::io::f32:
+ return PyBobIoHDF5File_AppendScalar<float>(self, path, data);
+ case bob::io::f64:
+ return PyBobIoHDF5File_AppendScalar<double>(self, path, data);
+ case bob::io::f128:
+ return PyBobIoHDF5File_AppendScalar<long double>(self, path, data);
+ case bob::io::c64:
+ return PyBobIoHDF5File_AppendScalar<std::complex<float> >(self, path, data);
+ case bob::io::c128:
+ return PyBobIoHDF5File_AppendScalar<std::complex<double> >(self, path, data);
+ case bob::io::c256:
+ return PyBobIoHDF5File_AppendScalar<std::complex<long double> >(self, path, data);
+ default:
+ break;
+ }
+
+ }
+
+ else { //write as array
+
+ switch (is_array) {
+ case 1: //blitz.array
+ if (!self->f->contains(path)) self->f->create(path, type, false, compression);
+ self->f->extend_buffer(path, type, ((PyBlitzArrayObject*)data)->data);
+ break;
+
+ case 2: //numpy.ndarray
+ if (!self->f->contains(path)) self->f->create(path, type, false, compression);
+ self->f->extend_buffer(path, type, PyArray_DATA((PyArrayObject*)data));
+ break;
+
+ case 3: //converted numpy.ndarray
+ if (!self->f->contains(path)) self->f->create(path, type, false, compression);
+ self->f->extend_buffer(path, type, PyArray_DATA((PyArrayObject*)converted));
+ Py_DECREF(converted);
+ break;
+
+ default:
+ PyErr_Format(PyExc_NotImplementedError, "error appending to object `%s' at HDF5 file `%s': HDF5 replace function is uncovered for array type %d (DEBUG ME)", path, self->f->filename().c_str(), is_array);
+ return 0;
+ }
+
+ }
+
+ }
+ catch (std::exception& e) {
+ PyErr_SetString(PyExc_RuntimeError, e.what());
+ return 0;
+ }
+ catch (...) {
+ PyErr_Format(PyExc_RuntimeError, "cannot append to object `%s' at HDF5 file `%s': unknown exception caught", path, self->f->filename().c_str());
+ return 0;
+ }
+
+ return 1;
+
+}
+
+static PyObject* PyBobIoHDF5File_Append(PyBobIoHDF5FileObject* self, PyObject *args, PyObject* kwds) {
+
/* Parses input arguments in a single shot */
static const char* const_kwlist[] = {"path", "data", "compression", 0};
static char** kwlist = const_cast<char**>(const_kwlist);
- PyObject* key = 0;
+ char* path = 0;
PyObject* data = 0;
Py_ssize_t compression = 0;
- if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO|n", kwlist, &key, &data, &compression)) return 0;
+ if (!PyArg_ParseTupleAndKeywords(args, kwds, "sO|n", kwlist, &path, &data, &compression)) return 0;
+ if (compression < 0 || compression > 9) {
+ PyErr_SetString(PyExc_ValueError, "compression should be set to an integer value between and including 0 and 9");
+ return 0;
+ }
+
+ // special case: user passes a tuple or list of arrays or scalars to append
+ if (PyTuple_Check(data) || PyList_Check(data)) {
+ PyObject* iter = PyObject_GetIter(data);
+ if (!iter) return 0;
+ while (PyObject* item = PyIter_Next(iter)) {
+ int ok = PyBobIoHDF5File_InnerAppend(self, path, item, compression);
+ Py_DECREF(item);
+ if (!ok) {
+ Py_DECREF(iter);
+ return 0;
+ }
+ }
+ Py_DECREF(iter);
+ Py_RETURN_NONE;
+ }
+
+ int ok = PyBobIoHDF5File_InnerAppend(self, path, data, compression);
+ if (!ok) return 0;
Py_RETURN_NONE;
+
}
PyDoc_STRVAR(s_append_str, "append");
@@ -895,18 +1334,128 @@ compression\n\
\n\
");
-static PyObject* PyBobIoHDF5File_Set(PyBobIoHDF5FileObject* self, PyObject *args, PyObject* kwds) {
+template <typename T>
+static PyObject* PyBobIoHDF5File_SetScalar(PyBobIoHDF5FileObject* self,
+ const char* path, PyObject* o) {
+
+ T value = PyBlitzArrayCxx_AsCScalar<T>(o);
+ if (PyErr_Occurred()) return 0;
+ self->f->set(path, value);
+
+ Py_RETURN_NONE;
+
+}
+
+static PyObject* PyBobIoHDF5File_Set(PyBobIoHDF5FileObject* self, PyObject* args, PyObject* kwds) {
/* Parses input arguments in a single shot */
static const char* const_kwlist[] = {"path", "data", "compression", 0};
static char** kwlist = const_cast<char**>(const_kwlist);
- PyObject* key = 0;
+ char* path = 0;
PyObject* data = 0;
Py_ssize_t compression = 0;
- if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO|n", kwlist, &key, &data, &compression)) return 0;
+ if (!PyArg_ParseTupleAndKeywords(args, kwds, "sO|n", kwlist, &path, &data, &compression)) return 0;
+
+ if (compression < 0 || compression > 9) {
+ PyErr_SetString(PyExc_ValueError, "compression should be set to an integer value between and including 0 and 9");
+ return 0;
+ }
+
+ bob::io::HDF5Type type;
+ PyObject* converted = 0;
+ int is_array = PyBobIoHDF5File_GetObjectType(data, type, &converted);
+ if (is_array < 0) { ///< error condition, signal
+ PyErr_Format(PyExc_TypeError, "error setting object `%s' of HDF5 file `%s': no support for storing objects of type `%s' on HDF5 files", path, self->f->filename().c_str(), data->ob_type->tp_name);
+ return 0;
+ }
+
+ try {
+
+ if (!is_array) { //write as a scalar
+
+ switch(type.type()) {
+ case bob::io::s:
+ {
+ char* value = PyBobIo_GetString(data);
+ if (!value) return 0;
+ self->f->set(path, value);
+ Py_RETURN_NONE;
+ }
+ case bob::io::b:
+ return PyBobIoHDF5File_SetScalar<bool>(self, path, data);
+ case bob::io::i8:
+ return PyBobIoHDF5File_SetScalar<int8_t>(self, path, data);
+ case bob::io::i16:
+ return PyBobIoHDF5File_SetScalar<int16_t>(self, path, data);
+ case bob::io::i32:
+ return PyBobIoHDF5File_SetScalar<int32_t>(self, path, data);
+ case bob::io::i64:
+ return PyBobIoHDF5File_SetScalar<int64_t>(self, path, data);
+ case bob::io::u8:
+ return PyBobIoHDF5File_SetScalar<uint8_t>(self, path, data);
+ case bob::io::u16:
+ return PyBobIoHDF5File_SetScalar<uint16_t>(self, path, data);
+ case bob::io::u32:
+ return PyBobIoHDF5File_SetScalar<uint32_t>(self, path, data);
+ case bob::io::u64:
+ return PyBobIoHDF5File_SetScalar<uint64_t>(self, path, data);
+ case bob::io::f32:
+ return PyBobIoHDF5File_SetScalar<float>(self, path, data);
+ case bob::io::f64:
+ return PyBobIoHDF5File_SetScalar<double>(self, path, data);
+ case bob::io::f128:
+ return PyBobIoHDF5File_SetScalar<long double>(self, path, data);
+ case bob::io::c64:
+ return PyBobIoHDF5File_SetScalar<std::complex<float> >(self, path, data);
+ case bob::io::c128:
+ return PyBobIoHDF5File_SetScalar<std::complex<double> >(self, path, data);
+ case bob::io::c256:
+ return PyBobIoHDF5File_SetScalar<std::complex<long double> >(self, path, data);
+ default:
+ break;
+ }
+
+ }
+
+ else { //write as array
+
+ switch (is_array) {
+ case 1: //blitz.array
+ if (!self->f->contains(path)) self->f->create(path, type, false, compression);
+ self->f->write_buffer(path, 0, type, ((PyBlitzArrayObject*)data)->data);
+ break;
+
+ case 2: //numpy.ndarray
+ if (!self->f->contains(path)) self->f->create(path, type, false, compression);
+ self->f->write_buffer(path, 0, type, PyArray_DATA((PyArrayObject*)data));
+ break;
+
+ case 3: //converted numpy.ndarray
+ if (!self->f->contains(path)) self->f->create(path, type, false, compression);
+ self->f->write_buffer(path, 0, type, PyArray_DATA((PyArrayObject*)converted));
+ Py_DECREF(converted);
+ break;
+
+ default:
+ PyErr_Format(PyExc_NotImplementedError, "error setting object `%s' at HDF5 file `%s': HDF5 replace function is uncovered for array type %d (DEBUG ME)", path, self->f->filename().c_str(), is_array);
+ return 0;
+ }
+
+ }
+
+ }
+ catch (std::exception& e) {
+ PyErr_SetString(PyExc_RuntimeError, e.what());
+ return 0;
+ }
+ catch (...) {
+ PyErr_Format(PyExc_RuntimeError, "cannot set object `%s' at HDF5 file `%s': unknown exception caught", path, self->f->filename().c_str());
+ return 0;
+ }
Py_RETURN_NONE;
+
}
PyDoc_STRVAR(s_set_str, "set");
diff --git a/xbob/io/test/test_hdf5.py b/xbob/io/test/test_hdf5.py
index 42f56b8..30dce84 100644
--- a/xbob/io/test/test_hdf5.py
+++ b/xbob/io/test/test_hdf5.py
@@ -278,13 +278,13 @@ def test_can_load_hdf5_from_matlab():
# interestingly enough, if you load those files as arrays, you will read
# the whole data at once:
- t = peek_all(testutils.datafile('matlab_1d.hdf5', __name__))
- assert t.shape == (512,)
- assert t.dtype == numpy.dtype('float64')
+ dtype, shape, stride = peek_all(testutils.datafile('matlab_1d.hdf5', __name__))
+ assert shape == (512,)
+ assert dtype == numpy.dtype('float64')
- t = peek_all(testutils.datafile('matlab_2d.hdf5', __name__))
- assert t.shape == (512, 2)
- assert t.dtype == numpy.dtype('float64')
+ dtype, shape, stride = peek_all(testutils.datafile('matlab_2d.hdf5', __name__))
+ assert shape == (512, 2)
+ assert dtype == numpy.dtype('float64')
def test_matlab_import():
--
GitLab