Commit f943a35d authored by Tiago de Freitas Pereira's avatar Tiago de Freitas Pereira
Browse files

Merge branch...

Merge branch '18-load-images-based-on-magic-numbers-instead-of-filename-extension-in-c' into 'master'

Resolve "Load images based on magic numbers instead of filename extension in C++"

Closes #18 

I have implemented the C++ interface of loading images based on their magic numbers. I have tested this functionality and it works well, even in parallel mode. I have also changed the python interface.

Could someone have a look on the modifications?

See merge request !17
parents 812f9614 077a7ca5
Pipeline #3684 passed with stages
in 48 minutes and 51 seconds
......@@ -9,6 +9,8 @@ from . import _library
from . import version
from .version import module as __version__
from ._library import *
import os
def get_config():
......@@ -17,13 +19,6 @@ def get_config():
import bob.extension
return bob.extension.get_config(__name__, version.externals)
# fix imghdr's jpeg detection to use the first two bytes (according to https://en.wikipedia.org/wiki/List_of_file_signatures)
import imghdr
def _test_jpeg(h, f):
if h.startswith('\xff\xd8'):
return "jpeg"
imghdr.tests.append(_test_jpeg)
def load(filename, extension=None):
"""load(filename) -> image
......@@ -54,10 +49,7 @@ def load(filename, extension=None):
f = bob.io.base.File(filename, 'r')
else:
if extension == 'auto':
extension = imghdr.what(filename)
if extension is None:
raise IOError("Could not detect the image type of file %s" % filename)
extension = "." + extension
extension = get_correct_image_extension(filename)
f = bob.io.base.File(filename, 'r', extension)
return f.read()
......
/**
* @date Thu Sep 8 12:05:08 MDT 2016
* @author Manuel Gunther <siebenkopf@googlemail.com>
*
* @brief Implements an generic image functionalities.
*
* Copyright (c) 2016, Regents of the University of Colorado on behalf of the University of Colorado Colorado Springs.
*/
#include <bob.io.image/image.h>
namespace bob { namespace io { namespace image {
static std::map<std::string, std::vector<std::vector<std::uint8_t>>> _initialize_magic_numbers(){
// these numbers are based on: http://stackoverflow.com/questions/26350342/determining-the-extension-type-of-an-image-file-using-binary/26350431#26350431
std::map<std::string, std::vector<std::vector<std::uint8_t>>> magic_numbers;
// BMP
magic_numbers[".bmp"].push_back({ 0x42, 0x4D });
// NetPBM, see https://en.wikipedia.org/wiki/Netpbm_format
magic_numbers[".pbm"].push_back({ 0x50, 0x31 });// P1
magic_numbers[".pbm"].push_back({ 0x50, 0x34 });// P4
magic_numbers[".pgm"].push_back({ 0x50, 0x32 });// P2
magic_numbers[".pgm"].push_back({ 0x50, 0x35 });// P5
magic_numbers[".ppm"].push_back({ 0x50, 0x33 });// P3
magic_numbers[".ppm"].push_back({ 0x50, 0x36 });// P6
// TODO: what about P7?
#ifdef HAVE_GIFLIB
// GIF
magic_numbers[".gif"].push_back({ 0x47, 0x49, 0x46, 0x38, 0x37, 0x61 });
magic_numbers[".gif"].push_back({ 0x47, 0x49, 0x46, 0x38, 0x39, 0x61 });
#endif // HAVE_GIFLIB
#ifdef HAVE_LIBPNG
// PNG
magic_numbers[".png"].push_back({ 0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A });
#endif // HAVE_LIB_PNG
#ifdef HAVE_LIBJPEG
// JPEG
magic_numbers[".jpg"].push_back({ 0xFF, 0xD8, 0xFF });
magic_numbers[".jpg"].push_back({ 0x00, 0x00, 0x00, 0x0C, 0x6A, 0x50, 0x20, 0x20 });
#endif // HAVE_LIBJPEG
#ifdef HAVE_LIBTIFF
// TIFF
magic_numbers[".tiff"].push_back({ 0x0C, 0xED });
magic_numbers[".tiff"].push_back({ 0x49, 0x20, 0x49 });
magic_numbers[".tiff"].push_back({ 0x49, 0x49, 0x2A, 0x00 });
magic_numbers[".tiff"].push_back({ 0x4D, 0x4D, 0x00, 0x2A });
magic_numbers[".tiff"].push_back({ 0x4D, 0x4D, 0x00, 0x2B });
#endif // HAVE_LIBTIFF
return magic_numbers;
}
// global static thread-safe map of known magic numbers
static std::map<std::string, std::vector<std::vector<std::uint8_t>>> known_magic_numbers = _initialize_magic_numbers();
const std::string& get_correct_image_extension(const std::string& image_name){
// read first 8 bytes from file
uint8_t image_bytes[8];
std::ifstream f(image_name);
if (!f) throw std::runtime_error("The given image '" + image_name + "' could not be opened for reading");
f.read(reinterpret_cast<char*>(image_bytes), 8);
// iterate over all extensions
for (auto eit = known_magic_numbers.begin(); eit != known_magic_numbers.end(); ++eit){
// iterate over all magic bytes
for (auto mit = eit->second.begin(); mit != eit->second.end(); ++mit){
// check magic number
if (std::equal(mit->begin(), mit->end(), image_bytes))
return eit->first;
}
}
throw std::runtime_error("The given image '" + image_name + "' does not contain an image of a known type");
}
bool is_color_image(const std::string& filename, std::string extension){
if (extension.empty())
extension = boost::filesystem::path(filename).extension().string();
boost::algorithm::to_lower(extension);
if (extension == ".bmp") return true;
#ifdef HAVE_GIFLIB
if (extension == ".gif") return true;
#endif
#ifdef HAVE_LIBPNG
if (extension == ".png") return is_color_png(filename);
#endif
#ifdef HAVE_LIBJPEG
if (extension == ".jpg" || extension == ".jpeg") return is_color_jpeg(filename);
#endif
#ifdef HAVE_LIBTIFF
if (extension == ".tif" || extension == ".tiff") return is_color_tiff(filename);
#endif
if (extension == ".pgm" || extension == ".pbm") return false;
if (extension == ".ppm") return true;
throw std::runtime_error("The filename extension '" + extension + "' is not known");
}
} } } // namespaces
......@@ -9,7 +9,7 @@
#define BOB_IO_IMAGE_CONFIG_H
/* Macros that define versions and important names */
#define BOB_IO_IMAGE_API_VERSION 0x0200
#define BOB_IO_IMAGE_API_VERSION 0x0201
#ifdef BOB_IMPORT_VERSION
......
......@@ -29,34 +29,18 @@
#include <bob.io.image/tiff.h>
#include <boost/filesystem/path.hpp>
#include <boost/algorithm/string.hpp>
#include <fstream>
#include <algorithm>
namespace bob { namespace io { namespace image {
inline bool is_color_image(const std::string& filename){
std::string extension = boost::filesystem::path(filename).extension().string();
boost::algorithm::to_lower(extension);
if (extension == ".bmp") return true;
#ifdef HAVE_GIFLIB
if (extension == ".gif") return true;
#endif
#ifdef HAVE_LIBPNG
if (extension == ".png") return is_color_png(filename);
#endif
#ifdef HAVE_LIBJPEG
if (extension == ".jpg" || extension == ".jpeg") return is_color_jpeg(filename);
#endif
#ifdef HAVE_LIBTIFF
if (extension == ".tif" || extension == ".tiff") return is_color_tiff(filename);
#endif
if (extension == ".pgm" || extension == ".pbm") return false;
if (extension == ".ppm") return true;
const std::string& get_correct_image_extension(const std::string& image_name);
throw std::runtime_error("The filename extension '" + extension + "' is not known");
}
bool is_color_image(const std::string& filename, std::string extension="");
inline blitz::Array<uint8_t,3> read_color_image(const std::string& filename){
std::string extension = boost::filesystem::path(filename).extension().string();
inline blitz::Array<uint8_t,3> read_color_image(const std::string& filename, std::string extension=""){
if (extension.empty())
extension = boost::filesystem::path(filename).extension().string();
boost::algorithm::to_lower(extension);
if (extension == ".bmp") return read_bmp(filename);
#ifdef HAVE_GIFLIB
......@@ -76,8 +60,9 @@ inline blitz::Array<uint8_t,3> read_color_image(const std::string& filename){
throw std::runtime_error("The filename extension '" + extension + "' is not known or not supported for color images");
}
blitz::Array<uint8_t,2> read_gray_image(const std::string& filename){
std::string extension = boost::filesystem::path(filename).extension().string();
blitz::Array<uint8_t,2> read_gray_image(const std::string& filename, std::string extension=""){
if (extension.empty())
extension = boost::filesystem::path(filename).extension().string();
boost::algorithm::to_lower(extension);
#ifdef HAVE_LIBPNG
if (extension == ".png") return read_png<uint8_t,2>(filename);
......@@ -95,8 +80,9 @@ blitz::Array<uint8_t,2> read_gray_image(const std::string& filename){
}
void write_color_image(const blitz::Array<uint8_t,3>& image, const std::string& filename){
std::string extension = boost::filesystem::path(filename).extension().string();
void write_color_image(const blitz::Array<uint8_t,3>& image, const std::string& filename, std::string extension=""){
if (extension.empty())
extension = boost::filesystem::path(filename).extension().string();
boost::algorithm::to_lower(extension);
if (extension == ".bmp") return write_bmp(image, filename); // this will only work for T=uint8_t
#ifdef HAVE_GIFLIB
......@@ -116,8 +102,9 @@ void write_color_image(const blitz::Array<uint8_t,3>& image, const std::string&
throw std::runtime_error("The filename extension '" + extension + "' is not known or not supported for color images");
}
void write_gray_image(const blitz::Array<uint8_t,2>& image, const std::string& filename){
std::string extension = boost::filesystem::path(filename).extension().string();
void write_gray_image(const blitz::Array<uint8_t,2>& image, const std::string& filename, std::string extension=""){
if (extension.empty())
extension = boost::filesystem::path(filename).extension().string();
boost::algorithm::to_lower(extension);
#ifdef HAVE_LIBPNG
if (extension == ".png") return write_png(image, filename);
......
......@@ -88,7 +88,7 @@ namespace bob { namespace io { namespace image {
static std::string s_codecname;
};
bool is_color_jpeg(const std::string& filename){
inline bool is_color_jpeg(const std::string& filename){
JPEGFile jpeg(filename.c_str(), 'r');
return jpeg.type().nd == 3;
}
......
......@@ -123,7 +123,7 @@ namespace bob { namespace io { namespace image {
}
bool is_color_p_m(const std::string& filename){
inline bool is_color_p_m(const std::string& filename){
NetPBMFile p_m(filename.c_str(), 'r');
return p_m.type().nd == 3;
}
......
......@@ -88,7 +88,7 @@ namespace bob { namespace io { namespace image {
static std::string s_codecname;
};
bool is_color_png(const std::string& filename){
inline bool is_color_png(const std::string& filename){
PNGFile png(filename.c_str(), 'r');
return png.type().nd == 3;
}
......
......@@ -88,7 +88,7 @@ namespace bob { namespace io { namespace image {
static std::string s_codecname;
};
bool is_color_tiff(const std::string& filename){
inline bool is_color_tiff(const std::string& filename){
TIFFFile tiff(filename.c_str(), 'r');
return tiff.type().nd == 3;
}
......
......@@ -59,8 +59,12 @@ BOB_TRY
// BMP; only color images are supported
boost::filesystem::path bmp(tempdir); bmp /= std::string("color.bmp");
bob::io::image::write_color_image(color_image, bmp.string());
if (bob::io::image::get_correct_image_extension(bmp.string()) != ".bmp")
throw std::runtime_error("BMP image type check did not succeed, check " + bmp.string());
if (!bob::io::image::is_color_image(bmp.string()))
throw std::runtime_error("BMP image " + bmp.string() + " is not color as expected");
blitz::Array<uint8_t, 3> color_bmp = bob::io::image::read_color_image(bmp.string());
blitz::Array<uint8_t, 3> color_bmp = bob::io::image::read_color_image(bmp.string(), ".bmp");
if (blitz::any(blitz::abs(color_image - color_bmp) > 0))
throw std::runtime_error("BMP image IO did not succeed, check " + bmp.string());
......@@ -69,8 +73,12 @@ BOB_TRY
// GIF; only color images are supported
boost::filesystem::path gif(tempdir); gif /= std::string("color.gif");
bob::io::image::write_color_image(color_image, gif.string());
if (bob::io::image::get_correct_image_extension(gif.string()) != ".gif")
throw std::runtime_error("GIF image type check did not succeed, check " + gif.string());
if (!bob::io::image::is_color_image(gif.string(), ".gif"))
throw std::runtime_error("GIF image " + gif.string() + " is not color as expected");
blitz::Array<uint8_t, 3> color_gif = bob::io::image::read_color_image(gif.string());
blitz::Array<uint8_t, 3> color_gif = bob::io::image::read_color_image(gif.string(), ".gif");
if (blitz::any(blitz::abs(color_image - color_gif) > 8)) // TODO: why is GIF not lossless?
throw std::runtime_error("GIF image IO did not succeed, check " + gif.string());
#endif
......@@ -78,19 +86,23 @@ BOB_TRY
// NetPBM
boost::filesystem::path pgm(tempdir); pgm /= std::string("gray.pgm");
bob::io::image::write_gray_image(gray_image, pgm.string());
if (bob::io::image::is_color_p_m(pgm.string()))
if (bob::io::image::get_correct_image_extension(pgm.string()) != ".pgm")
throw std::runtime_error("PGM image type check did not succeed, check " + pgm.string());
if (bob::io::image::is_color_p_m(pgm.string()) || bob::io::image::is_color_image(pgm.string(), ".pgm"))
throw std::runtime_error("PGM image " + pgm.string() + " is not gray as expected");
blitz::Array<uint8_t, 2> gray_pgm = bob::io::image::read_gray_image(pgm.string());
blitz::Array<uint8_t, 2> gray_pgm = bob::io::image::read_gray_image(pgm.string(), ".pgm");
if (blitz::any(blitz::abs(gray_image - gray_pgm) > 0))
throw std::runtime_error("PGM image IO did not succeed, check " + pgm.string());
boost::filesystem::path ppm(tempdir); ppm /= std::string("color.ppm");
bob::io::image::write_color_image(color_image, ppm.string());
if (!bob::io::image::is_color_p_m(ppm.string()))
if (bob::io::image::get_correct_image_extension(ppm.string()) != ".ppm")
throw std::runtime_error("PPM image type check did not succeed, check " + ppm.string());
if (!bob::io::image::is_color_p_m(ppm.string()) || !bob::io::image::is_color_image(ppm.string()))
throw std::runtime_error("PPM image " + ppm.string() + " is not color as expected");
blitz::Array<uint8_t, 3> color_ppm = bob::io::image::read_color_image(ppm.string());
blitz::Array<uint8_t, 3> color_ppm = bob::io::image::read_color_image(ppm.string(), ".ppm");
if (blitz::any(blitz::abs(color_image - color_ppm) > 0))
throw std::runtime_error("PPM image IO did not succeed, check " + ppm.string());
......@@ -99,7 +111,9 @@ BOB_TRY
// JPEG
boost::filesystem::path jpeg_gray(tempdir); jpeg_gray /= std::string("gray.jpg");
bob::io::image::write_gray_image(gray_image, jpeg_gray.string());
if (bob::io::image::is_color_image(jpeg_gray.string()))
if (bob::io::image::get_correct_image_extension(jpeg_gray.string()) != ".jpg")
throw std::runtime_error("JPEG image type check did not succeed, check " + jpeg_gray.string());
if (bob::io::image::is_color_image(jpeg_gray.string()) || bob::io::image::is_color_image(jpeg_gray.string()))
throw std::runtime_error("JPEG image " + jpeg_gray.string() + " is not gray as expected");
blitz::Array<uint8_t, 2> gray_jpeg = bob::io::image::read_gray_image(jpeg_gray.string());
......@@ -108,7 +122,9 @@ BOB_TRY
boost::filesystem::path jpeg_color(tempdir); jpeg_color /= std::string("color.jpg");
bob::io::image::write_color_image(color_image, jpeg_color.string());
if (!bob::io::image::is_color_image(jpeg_color.string()))
if (bob::io::image::get_correct_image_extension(jpeg_color.string()) != ".jpg")
throw std::runtime_error("JPEG image type check did not succeed, check " + jpeg_color.string());
if (!bob::io::image::is_color_image(jpeg_color.string()) || !bob::io::image::is_color_image(jpeg_color.string(), ".jpeg"))
throw std::runtime_error("JPEG image " + jpeg_color.string() + " is not color as expected");
blitz::Array<uint8_t, 3> color_jpeg = bob::io::image::read_color_image(jpeg_color.string());
......@@ -120,7 +136,9 @@ BOB_TRY
// PNG
boost::filesystem::path png_gray(tempdir); png_gray /= std::string("gray.png");
bob::io::image::write_gray_image(gray_image, png_gray.string());
if (bob::io::image::is_color_image(png_gray.string()))
if (bob::io::image::get_correct_image_extension(png_gray.string()) != ".png")
throw std::runtime_error("PNG image type check did not succeed, check " + png_gray.string());
if (bob::io::image::is_color_image(png_gray.string()) || bob::io::image::is_color_image(png_gray.string(), ".png"))
throw std::runtime_error("PNG image " + png_gray.string() + " is not gray as expected");
blitz::Array<uint8_t, 2> gray_png = bob::io::image::read_gray_image(png_gray.string());
......@@ -129,7 +147,9 @@ BOB_TRY
boost::filesystem::path png_color(tempdir); png_color /= std::string("color.png");
bob::io::image::write_color_image(color_image, png_color.string());
if (!bob::io::image::is_color_png(png_color.string()))
if (bob::io::image::get_correct_image_extension(png_color.string()) != ".png")
throw std::runtime_error("PNG image type check did not succeed, check " + png_color.string());
if (!bob::io::image::is_color_png(png_color.string()) || !bob::io::image::is_color_image(png_color.string()))
throw std::runtime_error("PNG image " + png_color.string() + " is not color as expected");
blitz::Array<uint8_t, 3> color_png = bob::io::image::read_color_image(png_color.string());
......@@ -141,7 +161,9 @@ BOB_TRY
// TIFF
boost::filesystem::path tiff_gray(tempdir); tiff_gray /= std::string("gray.tiff");
bob::io::image::write_gray_image(gray_image, tiff_gray.string());
if (bob::io::image::is_color_image(tiff_gray.string()))
if (bob::io::image::get_correct_image_extension(tiff_gray.string()) != ".tiff")
throw std::runtime_error("TIFF image type check did not succeed, check " + tiff_gray.string());
if (bob::io::image::is_color_image(tiff_gray.string()) || bob::io::image::is_color_image(tiff_gray.string(), ".tif"))
throw std::runtime_error("TIFF image " + tiff_gray.string() + " is not gray as expected");
blitz::Array<uint8_t, 2> gray_tiff = bob::io::image::read_gray_image(tiff_gray.string());
......@@ -150,7 +172,9 @@ BOB_TRY
boost::filesystem::path tiff_color(tempdir); tiff_color /= std::string("color.tiff");
bob::io::image::write_color_image(color_image, tiff_color.string());
if (!bob::io::image::is_color_image(tiff_color.string()))
if (bob::io::image::get_correct_image_extension(tiff_color.string()) != ".tiff")
throw std::runtime_error("TIFF image type check did not succeed, check " + tiff_color.string());
if (!bob::io::image::is_color_image(tiff_color.string()) || !bob::io::image::is_color_image(tiff_color.string(), ".tiff"))
throw std::runtime_error("TIFF image " + tiff_color.string() + " is not color as expected");
blitz::Array<uint8_t, 3> color_tiff = bob::io::image::read_color_image(tiff_color.string());
......@@ -163,6 +187,29 @@ BOB_CATCH_FUNCTION("_test_io", 0)
}
static auto s_image_extension = bob::extension::FunctionDoc(
"get_correct_image_extension",
"Estimates the image type and return a corresponding extension based on file content",
"This function loads the first bytes of the given image, and matches it with known magic numbers of image files. "
"If a match is found, it returns the corresponding image extension (including the leading ``'.'`` that can be used, e.g., in :py:func:`bob.io.image.load`."
)
.add_prototype("image_name", "extension")
.add_parameter("image_name", "str", "The name (including path) of the image to check")
.add_return("extension", "str", "The extension of the image based on the file content")
;
static PyObject* image_extension(PyObject*, PyObject *args, PyObject* kwds) {
BOB_TRY
static char** kwlist = s_image_extension.kwlist();
const char* image_name;
if (!PyArg_ParseTupleAndKeywords(args, kwds, "s", kwlist, &image_name)) return 0;
return Py_BuildValue("s", bob::io::image::get_correct_image_extension(image_name).c_str());
BOB_CATCH_FUNCTION("get_correct_image_extension", 0)
}
static PyMethodDef module_methods[] = {
{
s_test_io.name(),
......@@ -170,6 +217,12 @@ static PyMethodDef module_methods[] = {
METH_VARARGS|METH_KEYWORDS,
s_test_io.doc(),
},
{
s_image_extension.name(),
(PyCFunction)image_extension,
METH_VARARGS|METH_KEYWORDS,
s_image_extension.doc(),
},
{0} /* Sentinel */
};
......
......@@ -397,6 +397,7 @@ setup(
"bob/io/image/cpp/bmp.cpp",
"bob/io/image/cpp/pnmio.cpp",
"bob/io/image/cpp/netpbm.cpp",
"bob/io/image/cpp/image.cpp",
],
packages = packages,
boost_modules = boost_modules,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment