Commit 9cdb0e76 authored by Manuel Günther's avatar Manuel Günther
Browse files

Reverted some modifications for which we now found a better solution.

parent ba995e7d
......@@ -5,46 +5,129 @@
* @brief Implementation of our bobskin class
*/
#define BOB_IO_BASE_MODULE
#include "bobskin.h"
#include <stdexcept>
bobskin::bobskin(PyObject* array, int dtype) {
bobskin::bobskin(PyObject* array, bob::io::base::array::ElementType eltype) {
if (!PyArray_CheckExact(array)) {
PyErr_SetString(PyExc_TypeError, "input object to bobskin constructor is not (exactly) a numpy.ndarray");
throw std::runtime_error("error is already set");
}
if (!BobIoTypeinfo_SignedSetWithStrides(&m_type, dtype,
PyArray_NDIM((PyArrayObject*)array),
PyArray_DIMS((PyArrayObject*)array),
PyArray_STRIDES((PyArrayObject*)array))) {
throw std::runtime_error("error is already set");
}
m_type.set<npy_intp>(eltype, PyArray_NDIM((PyArrayObject*)array),
PyArray_DIMS((PyArrayObject*)array),
PyArray_STRIDES((PyArrayObject*)array));
m_ptr = PyArray_DATA((PyArrayObject*)array);
}
bobskin::bobskin(PyArrayObject* array, int dtype) {
bobskin::bobskin(PyArrayObject* array, bob::io::base::array::ElementType eltype) {
if (!BobIoTypeinfo_SignedSetWithStrides(&m_type, dtype,
PyArray_NDIM((PyArrayObject*)array),
PyArray_DIMS((PyArrayObject*)array),
PyArray_STRIDES((PyArrayObject*)array))) {
throw std::runtime_error("error is already set");
}
m_type.set<npy_intp>(eltype, PyArray_NDIM((PyArrayObject*)array),
PyArray_DIMS((PyArrayObject*)array),
PyArray_STRIDES((PyArrayObject*)array));
m_ptr = PyArray_DATA((PyArrayObject*)array);
}
static bob::io::base::array::ElementType signed_integer_type(int bits) {
switch(bits) {
case 8:
return bob::io::base::array::t_int8;
case 16:
return bob::io::base::array::t_int16;
case 32:
return bob::io::base::array::t_int32;
case 64:
return bob::io::base::array::t_int64;
default:
PyErr_Format(PyExc_TypeError, "unsupported signed integer element type with %d bits", bits);
}
return bob::io::base::array::t_unknown;
}
static bob::io::base::array::ElementType unsigned_integer_type(int bits) {
switch(bits) {
case 8:
return bob::io::base::array::t_uint8;
case 16:
return bob::io::base::array::t_uint16;
case 32:
return bob::io::base::array::t_uint32;
case 64:
return bob::io::base::array::t_uint64;
default:
PyErr_Format(PyExc_TypeError, "unsupported unsigned signed integer element type with %d bits", bits);
}
return bob::io::base::array::t_unknown;
}
static bob::io::base::array::ElementType num_to_type (int num) {
switch(num) {
case NPY_BOOL:
return bob::io::base::array::t_bool;
//signed integers
case NPY_BYTE:
return signed_integer_type(NPY_BITSOF_CHAR);
case NPY_SHORT:
return signed_integer_type(NPY_BITSOF_SHORT);
case NPY_INT:
return signed_integer_type(NPY_BITSOF_INT);
case NPY_LONG:
return signed_integer_type(NPY_BITSOF_LONG);
case NPY_LONGLONG:
return signed_integer_type(NPY_BITSOF_LONGLONG);
//unsigned integers
case NPY_UBYTE:
return unsigned_integer_type(NPY_BITSOF_CHAR);
case NPY_USHORT:
return unsigned_integer_type(NPY_BITSOF_SHORT);
case NPY_UINT:
return unsigned_integer_type(NPY_BITSOF_INT);
case NPY_ULONG:
return unsigned_integer_type(NPY_BITSOF_LONG);
case NPY_ULONGLONG:
return unsigned_integer_type(NPY_BITSOF_LONGLONG);
//floats
case NPY_FLOAT32:
return bob::io::base::array::t_float32;
case NPY_FLOAT64:
return bob::io::base::array::t_float64;
#ifdef NPY_FLOAT128
case NPY_FLOAT128:
return bob::io::base::array::t_float128;
#endif
//complex
case NPY_COMPLEX64:
return bob::io::base::array::t_complex64;
case NPY_COMPLEX128:
return bob::io::base::array::t_complex128;
#ifdef NPY_COMPLEX256
case NPY_COMPLEX256:
return bob::io::base::array::t_complex256;
#endif
default:
PyErr_Format(PyExc_TypeError, "unsupported NumPy element type (%d)", num);
}
return bob::io::base::array::t_unknown;
}
bobskin::bobskin(PyBlitzArrayObject* array) {
if (!BobIoTypeinfo_SignedSetWithStrides(&m_type, array->type_num,
array->ndim, array->shape, array->stride)) {
bob::io::base::array::ElementType eltype = num_to_type(array->type_num);
if (eltype == bob::io::base::array::t_unknown) {
throw std::runtime_error("error is already set");
}
m_type.set<Py_ssize_t>(num_to_type(array->type_num), array->ndim,
array->shape, array->stride);
m_ptr = array->data;
}
......@@ -60,8 +143,8 @@ void bobskin::set(boost::shared_ptr<interface>) {
throw std::runtime_error("error is already set");
}
void bobskin::set (const BobIoTypeinfo&) {
PyErr_SetString(PyExc_NotImplementedError, "setting C++ bobskin with (const type-information&) implemented - DEBUG ME!");
void bobskin::set (const bob::io::base::array::typeinfo&) {
PyErr_SetString(PyExc_NotImplementedError, "setting C++ bobskin with (const typeinfo&) implemented - DEBUG ME!");
throw std::runtime_error("error is already set");
}
......
......@@ -10,8 +10,13 @@
#define PYTHON_BOB_IO_BOBSKIN_H
#include <Python.h>
#include <bob.io.base/array.h>
#include <bob.blitz/cppapi.h>
extern "C" {
#include <bob.blitz/capi.h>
}
/**
* Wraps a PyArrayObject such that we can access it from bob::io
......@@ -23,12 +28,12 @@ class bobskin: public bob::io::base::array::interface {
/**
* @brief Builds a new skin from an array like object
*/
bobskin(PyObject* array, int dtype);
bobskin(PyObject* array, bob::io::base::array::ElementType eltype);
/**
* @brief Builds a new skin from a numpy array object
*/
bobskin(PyArrayObject* array, int dtype);
bobskin(PyArrayObject* array, bob::io::base::array::ElementType eltype);
/**
* @brief Builds a new skin around a blitz array object
......@@ -55,12 +60,12 @@ class bobskin: public bob::io::base::array::interface {
* @brief Re-allocates this interface taking into consideration new
* requirements. The internal memory should be considered uninitialized.
*/
virtual void set (const BobIoTypeinfo& req);
virtual void set (const bob::io::base::array::typeinfo& req);
/**
* @brief Type information for this interface.
*/
virtual const BobIoTypeinfo& type() const { return m_type; }
virtual const bob::io::base::array::typeinfo& type() const { return m_type; }
/**
* @brief Borrows a reference from the underlying memory. This means
......@@ -80,7 +85,7 @@ class bobskin: public bob::io::base::array::interface {
private: //representation
BobIoTypeinfo m_type; ///< type information
bob::io::base::array::typeinfo m_type; ///< type information
void* m_ptr; ///< pointer to the data
};
......
......@@ -7,9 +7,8 @@
#define BOB_IO_BASE_MODULE
#include <bob.io.base/api.h>
#include "cpp/CodecRegistry.h"
int PyBobIoCodec_Register (const char* extension, const char* description, BobIoFileFactory factory) {
int PyBobIoCodec_Register (const char* extension, const char* description, bob::io::base::file_factory_t factory) {
boost::shared_ptr<bob::io::base::CodecRegistry> instance =
bob::io::base::CodecRegistry::instance();
......
......@@ -8,10 +8,6 @@
* Copyright (C) Idiap Research Institute, Martigny, Switzerland
*/
#define BOB_IO_BASE_MODULE
#include <bob.io.base/api.h>
#include <bob.io.base/File.h>
#include <sstream>
#include <fstream>
#include <string>
......@@ -24,6 +20,8 @@
#include <boost/shared_array.hpp>
#include <boost/algorithm/string.hpp>
#include <bob.io.base/CodecRegistry.h>
typedef boost::tokenizer<boost::escaped_list_separator<char> > Tokenizer;
class CSVFile: public bob::io::base::File {
......@@ -64,16 +62,16 @@ class CSVFile: public bob::io::base::File {
return;
}
m_arrayset_type.dtype = NPY_FLOAT64;
m_arrayset_type.dtype = bob::io::base::array::t_float64;
m_arrayset_type.nd = 1;
m_arrayset_type.shape[0] = entries;
BobIoTypeinfo_UpdateStrides(&m_arrayset_type);
m_arrayset_type.update_strides();
m_array_type = m_arrayset_type;
m_array_type.nd = 2;
m_array_type.shape[0] = m_pos.size();
m_array_type.shape[1] = entries;
BobIoTypeinfo_UpdateStrides(&m_array_type);
m_array_type.update_strides();
}
CSVFile(const char* path, char mode):
......@@ -118,11 +116,11 @@ class CSVFile: public bob::io::base::File {
return m_filename.c_str();
}
virtual const BobIoTypeinfo& type() const {
virtual const bob::io::base::array::typeinfo& type() const {
return m_arrayset_type;
}
virtual const BobIoTypeinfo& type_all() const {
virtual const bob::io::base::array::typeinfo& type_all() const {
return m_array_type;
}
......@@ -138,7 +136,7 @@ class CSVFile: public bob::io::base::File {
if (m_newfile)
throw std::runtime_error("uninitialized CSV file cannot be read");
if (!BobIoTypeinfo_IsCompatible(&buffer.type(), &m_array_type)) buffer.set(m_array_type);
if (!buffer.type().is_compatible(m_array_type)) buffer.set(m_array_type);
//read contents
std::string line;
......@@ -158,7 +156,7 @@ class CSVFile: public bob::io::base::File {
if (m_newfile)
throw std::runtime_error("uninitialized CSV file cannot be read");
if (!BobIoTypeinfo_IsCompatible(&buffer.type(), &m_arrayset_type))
if (!buffer.type().is_compatible(m_arrayset_type))
buffer.set(m_arrayset_type);
if (index >= m_pos.size()) {
......@@ -186,12 +184,12 @@ class CSVFile: public bob::io::base::File {
virtual size_t append (const bob::io::base::array::interface& buffer) {
const BobIoTypeinfo& type = buffer.type();
const bob::io::base::array::typeinfo& type = buffer.type();
if (m_newfile) {
if (type.nd != 1 || type.dtype != NPY_FLOAT64) {
if (type.nd != 1 || type.dtype != bob::io::base::array::t_float64) {
boost::format m("cannot append %s to file '%s' - CSV files only accept 1D double precision float arrays");
m % BobIoTypeinfo_Str(&type) % m_filename;
m % type.str() % m_filename;
throw std::runtime_error(m.str());
}
m_pos.clear();
......@@ -203,9 +201,9 @@ class CSVFile: public bob::io::base::File {
else {
//check compatibility
if (!BobIoTypeinfo_IsCompatible(&m_arrayset_type, &buffer.type())) {
if (!m_arrayset_type.is_compatible(buffer.type())) {
boost::format m("CSV file '%s' only accepts arrays of type %s");
m % m_filename % BobIoTypeinfo_Str(&m_arrayset_type);
m % m_filename % m_arrayset_type.str();
throw std::runtime_error(m.str());
}
......@@ -217,19 +215,19 @@ class CSVFile: public bob::io::base::File {
for (size_t k=1; k<type.shape[0]; ++k) m_file << *(p++) << ",";
m_file << *(p++);
m_array_type.shape[0] = m_pos.size();
BobIoTypeinfo_UpdateStrides(&m_array_type);
m_array_type.update_strides();
return (m_pos.size()-1);
}
virtual void write (const bob::io::base::array::interface& buffer) {
const BobIoTypeinfo& type = buffer.type();
const bob::io::base::array::typeinfo& type = buffer.type();
if (m_newfile) {
if (type.nd != 2 || type.dtype != NPY_FLOAT64) {
if (type.nd != 2 || type.dtype != bob::io::base::array::t_float64) {
boost::format m("cannot write %s to file '%s' - CSV files only accept a single 2D double precision float array as input");
m % BobIoTypeinfo_Str(&type) % m_filename;
m % type.str() % m_filename;
throw std::runtime_error(m.str());
}
const double* p = static_cast<const double*>(buffer.ptr());
......@@ -243,7 +241,7 @@ class CSVFile: public bob::io::base::File {
m_arrayset_type = type;
m_arrayset_type.nd = 1;
m_arrayset_type.shape[0] = type.shape[1];
BobIoTypeinfo_UpdateStrides(&m_arrayset_type);
m_arrayset_type.update_strides();
m_array_type = type;
m_newfile = false;
return;
......@@ -258,8 +256,8 @@ class CSVFile: public bob::io::base::File {
std::fstream m_file;
std::string m_filename;
bool m_newfile;
BobIoTypeinfo m_array_type;
BobIoTypeinfo m_arrayset_type;
bob::io::base::array::typeinfo m_array_type;
bob::io::base::array::typeinfo m_arrayset_type;
std::vector<std::streampos> m_pos; ///< dictionary of line starts
static std::string s_codecname;
......@@ -269,10 +267,48 @@ class CSVFile: public bob::io::base::File {
std::string CSVFile::s_codecname = "bob.csv";
/**
* Registration method: use an unique name. Copy the definition to "plugin.h"
* and then call it on "main.cpp" to register the codec.
* From this point onwards we have the registration procedure. If you are
* looking at this file for a coding example, just follow the procedure bellow,
* minus local modifications you may need to apply.
*/
/**
* This defines the factory method F that can create codecs of this type.
*
* Here are the meanings of the mode flag that should be respected by your
* factory implementation:
*
* 'r': opens for reading only - no modifications can occur; it is an
* error to open a file that does not exist for read-only operations.
* 'w': opens for reading and writing, but truncates the file if it
* exists; it is not an error to open files that do not exist with
* this flag.
* 'a': opens for reading and writing - any type of modification can
* occur. If the file does not exist, this flag is effectively like
* 'w'.
*
* Returns a newly allocated File object that can read and write data to the
* file using a specific backend.
*
* @note: This method can be static.
*/
boost::shared_ptr<bob::io::base::File>
make_csv_file (const char* path, char mode) {
static boost::shared_ptr<bob::io::base::File> make_file (const char* path, char mode) {
return boost::make_shared<CSVFile>(path, mode);
}
/**
* Takes care of codec registration per se.
*/
static bool register_codec() {
boost::shared_ptr<bob::io::base::CodecRegistry> instance =
bob::io::base::CodecRegistry::instance();
instance->registerExtension(".csv", "Comma-Separated Values", &make_file);
instance->registerExtension(".txt", "Comma-Separated Values", &make_file);
return true;
}
static bool codec_registered = register_codec();
......@@ -7,16 +7,15 @@
* Copyright (C) Idiap Research Institute, Martigny, Switzerland
*/
#define BOB_IO_BASE_MODULE
#include <bob.core/logging.h>
#include "CodecRegistry.h"
#include <vector>
#include <boost/filesystem.hpp>
#include <boost/format.hpp>
#include <bob.io.base/CodecRegistry.h>
#include <bob/core/logging.h>
boost::shared_ptr<bob::io::base::CodecRegistry> bob::io::base::CodecRegistry::instance() {
static boost::shared_ptr<bob::io::base::CodecRegistry> s_instance(new CodecRegistry());
return s_instance;
......@@ -33,7 +32,7 @@ const char* bob::io::base::CodecRegistry::getDescription(const char* ext) {
return it->second.c_str();
}
void bob::io::base::CodecRegistry::deregisterFactory(BobIoFileFactory factory) {
void bob::io::base::CodecRegistry::deregisterFactory(bob::io::base::file_factory_t factory) {
std::vector<std::string> to_remove;
for (auto it = s_extension2codec.begin(); it != s_extension2codec.end(); ++it) {
......@@ -48,7 +47,7 @@ void bob::io::base::CodecRegistry::deregisterFactory(BobIoFileFactory factory) {
}
void bob::io::base::CodecRegistry::registerExtension(const char* extension,
const char* description, BobIoFileFactory codec) {
const char* description, bob::io::base::file_factory_t codec) {
auto it = s_extension2codec.find(extension);
......@@ -59,8 +58,7 @@ void bob::io::base::CodecRegistry::registerExtension(const char* extension,
else if (!s_ignore) {
boost::format m("extension already registered: %s - ignoring second registration with description `%s'");
m % extension % description;
auto& error_stream = PyBobCoreLogging_Error();
error_stream << m.str() << std::endl;
bob::core::error << m.str() << std::endl;
throw std::runtime_error(m.str());
}
......@@ -73,13 +71,13 @@ bool bob::io::base::CodecRegistry::isRegistered(const char* ext) {
return (s_extension2codec.find(lower_extension) != s_extension2codec.end());
}
BobIoFileFactory bob::io::base::CodecRegistry::findByExtension (const char* ext) {
bob::io::base::file_factory_t bob::io::base::CodecRegistry::findByExtension (const char* ext) {
std::string extension(ext);
std::string lower_extension = extension;
std::transform(extension.begin(), extension.end(), lower_extension.begin(), ::tolower);
std::map<std::string, BobIoFileFactory>::iterator it =
std::map<std::string, bob::io::base::file_factory_t >::iterator it =
s_extension2codec.find(lower_extension);
if (it == s_extension2codec.end()) {
......@@ -92,7 +90,7 @@ BobIoFileFactory bob::io::base::CodecRegistry::findByExtension (const char* ext)
}
BobIoFileFactory bob::io::base::CodecRegistry::findByFilenameExtension
bob::io::base::file_factory_t bob::io::base::CodecRegistry::findByFilenameExtension
(const char* filename) {
return findByExtension(boost::filesystem::path(filename).extension().c_str());
......
/**
* @date Tue Oct 25 23:25:46 2011 +0200
* @author Andre Anjos <andre.anjos@idiap.ch>
*
* Copyright (C) Idiap Research Institute, Martigny, Switzerland
*/
#include <bob.io.base/File.h>
bob::io::base::File::~File() { }
......@@ -7,15 +7,13 @@
* Copyright (C) Idiap Research Institute, Martigny, Switzerland
*/
#define BOB_IO_BASE_MODULE
#include <bob.io.base/api.h>
#include <bob.io.base/File.h>
#include <bob.io.base/HDF5File.h>
#include <boost/make_shared.hpp>
#include <boost/filesystem.hpp>
#include <boost/format.hpp>
#include <bob.io.base/CodecRegistry.h>
#include <bob.io.base/HDF5File.h>
/**
* Read and write arrays in HDF5 format
*/
......@@ -67,11 +65,11 @@ class HDF5ArrayFile: public bob::io::base::File {
return m_filename.c_str();
}
virtual const BobIoTypeinfo& type_all () const {
virtual const bob::io::base::array::typeinfo& type_all () const {
return m_type_array;
}
virtual const BobIoTypeinfo& type () const {
virtual const bob::io::base::array::typeinfo& type () const {
return m_type_arrayset;
}
......@@ -91,7 +89,7 @@ class HDF5ArrayFile: public bob::io::base::File {
throw std::runtime_error(f.str());
}
if(!BobIoTypeinfo_IsCompatible(&buffer.type(), &m_type_array)) buffer.set(m_type_array);
if(!buffer.type().is_compatible(m_type_array)) buffer.set(m_type_array);
m_file.read_buffer(m_path, 0, buffer.type(), buffer.ptr());
}
......@@ -104,7 +102,7 @@ class HDF5ArrayFile: public bob::io::base::File {
throw std::runtime_error(f.str());
}
if(!BobIoTypeinfo_IsCompatible(&buffer.type(), &m_type_arrayset)) buffer.set(m_type_arrayset);
if(!buffer.type().is_compatible(m_type_arrayset)) buffer.set(m_type_arrayset);
m_file.read_buffer(m_path, index, buffer.type(), buffer.ptr());
}
......@@ -154,8 +152,8 @@ class HDF5ArrayFile: public bob::io::base::File {
bob::io::base::HDF5File m_file;
std::string m_filename;
BobIoTypeinfo m_type_array; ///< type for reading all data at once
BobIoTypeinfo m_type_arrayset; ///< type for reading data by sub-arrays
bob::io::base::array::typeinfo m_type_array; ///< type for reading all data at once
bob::io::base::array::typeinfo m_type_arrayset; ///< type for reading data by sub-arrays
size_t m_size_arrayset; ///< number of arrays in arrayset mode
std::string m_path; ///< default path to use
bool m_newfile; ///< path check optimization
......@@ -167,11 +165,32 @@ class HDF5ArrayFile: public bob::io::base::File {
std::string HDF5ArrayFile::s_codecname = "bob.hdf5";
/**
* Registration method: use an unique name. Copy the definition to "plugin.h"
* and then call it on "main.cpp" to register the codec.
* From this point onwards we have the registration procedure. If you are
* looking at this file for a coding example, just follow the procedure bellow,
* minus local modifications you may need to apply.
*/
/**
* This defines the factory method F that can create codecs of this type.
*
* Here are the meanings of the mode flag that should be respected by your
* factory implementation:
*
* 'r': opens for reading only - no modifications can occur; it is an
* error to open a file that does not exist for read-only operations.
* 'w': opens for reading and writing, but truncates the file if it
* exists; it is not an error to open files that do not exist with
* this flag.
* 'a': opens for reading and writing - any type of modification can
* occur. If the file does not exist, this flag is effectively like
* 'w'.
*
* Returns a newly allocated File object that can read and write data to the
* file using a specific backend.
*
* @note: This method can be static.
*/
boost::shared_ptr<bob::io::base::File>
make_hdf5_file (const char* path, char mode) {
static boost::shared_ptr<bob::io::base::File> make_file (const char* path, char mode) {
bob::io::base::HDF5File::mode_t h5mode;
if (mode == 'r') h5mode = bob::io::base::HDF5File::in;
......@@ -182,3 +201,22 @@ boost::shared_ptr<bob::io::base::File>
return boost::make_shared<HDF5ArrayFile>(path, h5mode);
}
/**
* Takes care of codec registration per se.
*/
static bool register_codec() {
static const char* description = "Hierarchical Data Format v5 (default)";