From 51ade7675fc6d7bd8df0fbf92ea1753042a46132 Mon Sep 17 00:00:00 2001
From: Andre Anjos <andre.dos.anjos@gmail.com>
Date: Sun, 24 Nov 2013 05:22:25 +0100
Subject: [PATCH] Loading correctly for the first time
---
setup.py | 40 ++---
src/xbob.blitz | 2 +-
xbob/measure/__init__.py | 2 +-
xbob/measure/calibration.py | 8 +-
xbob/measure/main.cpp | 349 +++++++++++++++++++++++++-----------
5 files changed, 265 insertions(+), 136 deletions(-)
diff --git a/setup.py b/setup.py
index ba5d5b8..19d4b3a 100644
--- a/setup.py
+++ b/setup.py
@@ -7,21 +7,15 @@ from setuptools import setup, find_packages, dist
dist.Distribution(dict(setup_requires=['xbob.blitz']))
from xbob.blitz.extension import Extension
-import os
-package_dir = os.path.dirname(os.path.realpath(__file__))
-package_dir = os.path.join(package_dir, 'xbob', 'io', 'include')
-include_dirs = [package_dir]
-
-packages = ['bob-io >= 1.3']
+packages = ['bob-measure >= 1.3']
version = '2.0.0a0'
-define_macros = [("XBOB_IO_VERSION", '"%s"' % version)]
setup(
- name='xbob.io',
+ name='xbob.measure',
version=version,
- description='Bindings for bob.io',
- url='http://github.com/anjos/xbob.io',
+ description='Bindings for bob.measure',
+ url='http://github.com/anjos/xbob.measure',
license='BSD',
author='Andre Anjos',
author_email='andre.anjos@idiap.ch',
@@ -34,6 +28,7 @@ setup(
install_requires=[
'setuptools',
'xbob.blitz',
+ #'xbob.math',
],
namespace_packages=[
@@ -41,32 +36,21 @@ setup(
],
ext_modules = [
- Extension("xbob.io._externals",
- [
- "xbob/io/externals.cpp",
- ],
- packages = packages,
- define_macros = define_macros,
- include_dirs = include_dirs,
- ),
- Extension("xbob.io._library",
+ Extension("xbob.measure._library",
[
- "xbob/io/bobskin.cpp",
- "xbob/io/file.cpp",
- "xbob/io/videoreader.cpp",
- "xbob/io/videowriter.cpp",
- "xbob/io/hdf5.cpp",
- "xbob/io/main.cpp",
+ "xbob/measure/main.cpp",
],
packages = packages,
- define_macros = define_macros,
- include_dirs = include_dirs,
+ version = version,
),
],
entry_points={
'console_scripts': [
- 'xbob_video_test.py = xbob.io.script.video_test:main',
+ 'xbob_compute_perf.py = xbob.measure.script.compute_perf:main',
+ 'xbob_eval_threshold.py = xbob.measure.script.eval_threshold:main',
+ 'xbob_apply_threshold.py = xbob.measure.script.apply_threshold:main',
+ 'xbob_plot_cmc.py = xbob.measure.script.plot_cmc:main',
],
},
diff --git a/src/xbob.blitz b/src/xbob.blitz
index 0c025f7..6df5905 160000
--- a/src/xbob.blitz
+++ b/src/xbob.blitz
@@ -1 +1 @@
-Subproject commit 0c025f75ef8fc80c34ef0460febae775568450a1
+Subproject commit 6df5905976b1ea66ed2cc1550fd944a2dc939f19
diff --git a/xbob/measure/__init__.py b/xbob/measure/__init__.py
index b760997..ce1757b 100644
--- a/xbob/measure/__init__.py
+++ b/xbob/measure/__init__.py
@@ -1,4 +1,4 @@
-from ._library import __version__, __api_version__
+from ._library import __version__
from . import plot
from . import load
diff --git a/xbob/measure/calibration.py b/xbob/measure/calibration.py
index 4215cf5..c7a9aa7 100644
--- a/xbob/measure/calibration.py
+++ b/xbob/measure/calibration.py
@@ -21,7 +21,6 @@
import math
import numpy
-from ..math import pavx
def cllr(negatives, positives):
"""Computes the 'cost of log likelihood ratio' measure as given in the bosaris toolkit"""
@@ -35,6 +34,9 @@ def cllr(negatives, positives):
def min_cllr(negatives, positives):
"""Computes the 'minimum cost of log likelihood ratio' measure as given in the bosaris toolkit"""
+
+ from ..math import pavx
+
# first, sort both scores
neg = sorted(negatives)
pos = sorted(positives)
@@ -71,8 +73,8 @@ def min_cllr(negatives, positives):
llrs = posterior_log_odds - log_prior_odds;
# some weired addition
-# for i in range(I):
-# llrs[i] += float(i)*1e-6/float(I)
+ # for i in range(I):
+ # llrs[i] += float(i)*1e-6/float(I)
# unmix positive and negative scores
new_neg = numpy.zeros(N)
diff --git a/xbob/measure/main.cpp b/xbob/measure/main.cpp
index 31064f6..11fd1d6 100644
--- a/xbob/measure/main.cpp
+++ b/xbob/measure/main.cpp
@@ -5,122 +5,265 @@
* @brief Bindings to bob::io
*/
-#define XBOB_IO_MODULE
-#include <xbob.io/api.h>
-
#ifdef NO_IMPORT_ARRAY
#undef NO_IMPORT_ARRAY
#endif
-#include <xbob.blitz/capi.h>
-
-static PyMethodDef module_methods[] = {
- {0} /* Sentinel */
-};
-
-PyDoc_STRVAR(module_docstr, "bob::io classes and methods");
-
-int PyXbobIo_APIVersion = XBOB_IO_API_VERSION;
-
-#define ENTRY_FUNCTION_INNER(a) init ## a
-#define ENTRY_FUNCTION(a) ENTRY_FUNCTION_INNER(a)
-
-PyMODINIT_FUNC ENTRY_FUNCTION(XBOB_IO_MODULE_NAME) (void) {
-
- PyBobIoFile_Type.tp_new = PyType_GenericNew;
- if (PyType_Ready(&PyBobIoFile_Type) < 0) return;
-
- PyBobIoFileIterator_Type.tp_new = PyType_GenericNew;
- if (PyType_Ready(&PyBobIoFileIterator_Type) < 0) return;
-
-#if WITH_FFMPEG
- PyBobIoVideoReader_Type.tp_new = PyType_GenericNew;
- if (PyType_Ready(&PyBobIoVideoReader_Type) < 0) return;
-
- PyBobIoVideoReaderIterator_Type.tp_new = PyType_GenericNew;
- if (PyType_Ready(&PyBobIoVideoReaderIterator_Type) < 0) return;
-
- PyBobIoVideoWriter_Type.tp_new = PyType_GenericNew;
- if (PyType_Ready(&PyBobIoVideoWriter_Type) < 0) return;
-#endif /* WITH_FFMPEG */
-
- PyBobIoHDF5File_Type.tp_new = PyType_GenericNew;
- if (PyType_Ready(&PyBobIoHDF5File_Type) < 0) return;
-
- PyObject* m = Py_InitModule3(BOOST_PP_STRINGIZE(XBOB_IO_MODULE_NAME),
- module_methods, module_docstr);
-
- /* register some constants */
- PyModule_AddIntConstant(m, "__api_version__", XBOB_IO_API_VERSION);
- PyModule_AddStringConstant(m, "__version__", XBOB_IO_VERSION);
-
- /* register the types to python */
- Py_INCREF(&PyBobIoFile_Type);
- PyModule_AddObject(m, "File", (PyObject *)&PyBobIoFile_Type);
-
- Py_INCREF(&PyBobIoFileIterator_Type);
- PyModule_AddObject(m, "File.iter", (PyObject *)&PyBobIoFileIterator_Type);
-
-#if WITH_FFMPEG
- Py_INCREF(&PyBobIoVideoReader_Type);
- PyModule_AddObject(m, "VideoReader", (PyObject *)&PyBobIoVideoReader_Type);
-
- Py_INCREF(&PyBobIoVideoReaderIterator_Type);
- PyModule_AddObject(m, "VideoReader.iter", (PyObject *)&PyBobIoVideoReaderIterator_Type);
+#include <xbob.blitz/cppapi.h>
+#include <bob/measure/error.h>
- Py_INCREF(&PyBobIoVideoWriter_Type);
- PyModule_AddObject(m, "VideoWriter", (PyObject *)&PyBobIoVideoWriter_Type);
-#endif /* WITH_FFMPEG */
-
- Py_INCREF(&PyBobIoHDF5File_Type);
- PyModule_AddObject(m, "HDF5File", (PyObject *)&PyBobIoHDF5File_Type);
-
- static void* PyXbobIo_API[PyXbobIo_API_pointers];
-
- /* exhaustive list of C APIs */
-
- /**************
- * Versioning *
- **************/
-
- PyXbobIo_API[PyXbobIo_APIVersion_NUM] = (void *)&PyXbobIo_APIVersion;
-
- /*****************************
- * Bindings for xbob.io.file *
- *****************************/
-
- PyXbobIo_API[PyBobIoFile_Type_NUM] = (void *)&PyBobIoFile_Type;
+/**
+static tuple farfrr(
+ bob::python::const_ndarray negatives,
+ bob::python::const_ndarray positives,
+ double threshold
+){
+ std::pair<double, double> retval = bob::measure::farfrr(negatives.cast<double,1>(), positives.cast<double,1>(), threshold);
+ return make_tuple(retval.first, retval.second);
+}
- PyXbobIo_API[PyBobIoFileIterator_Type_NUM] = (void *)&PyBobIoFileIterator_Type;
+static tuple precision_recall(
+ bob::python::const_ndarray negatives,
+ bob::python::const_ndarray positives,
+ double threshold
+){
+ std::pair<double, double> retval = bob::measure::precision_recall(negatives.cast<double,1>(), positives.cast<double,1>(), threshold);
+ return make_tuple(retval.first, retval.second);
+}
+**/
- /************************
- * I/O generic bindings *
- ************************/
+/**
+void bind_measure_error() {
+ def(
+ "farfrr",
+ &farfrr,
+ (arg("negatives"), arg("positives"), arg("threshold")),
+ "Calculates the FA ratio and the FR ratio given positive and negative scores and a threshold. 'positives' holds the score information for samples that are labelled to belong to a certain class (a.k.a., 'signal' or 'client'). 'negatives' holds the score information for samples that are labelled *not* to belong to the class (a.k.a., 'noise' or 'impostor').\n\nIt is expected that 'positive' scores are, at least by design, greater than 'negative' scores. So, every positive value that falls bellow the threshold is considered a false-rejection (FR). 'negative' samples that fall above the threshold are considered a false-accept (FA).\n\nPositives that fall on the threshold (exactly) are considered correctly classified. Negatives that fall on the threshold (exactly) are considered *incorrectly* classified. This equivalent to setting the comparision like this pseudo-code:\n\nforeach (positive as K) if K < threshold: falseRejectionCount += 1\nforeach (negative as K) if K >= threshold: falseAcceptCount += 1\n\nThe 'threshold' value does not necessarily have to fall in the range covered by the input scores (negatives and positives altogether), but if it does not, the output will be either (1.0, 0.0) or (0.0, 1.0) depending on the side the threshold falls.\n\nThe output is in form of a std::pair of two double-precision real numbers. The numbers range from 0 to 1. The first element of the pair is the false-accept ratio. The second element of the pair is the false-rejection ratio.\n\nIt is possible that scores are inverted in the negative/positive sense. In some setups the designer may have setup the system so 'positive' samples have a smaller score than the 'negative' ones. In this case, make sure you normalize the scores so positive samples have greater scores before feeding them into this method."
+ );
- PyXbobIo_API[PyBobIo_AsTypenum_NUM] = (void *)PyBobIo_AsTypenum;
-
- PyXbobIo_API[PyBobIo_TypeInfoAsTuple_NUM] = (void *)PyBobIo_TypeInfoAsTuple;
-
-#if WITH_FFMPEG
- /******************
- * Video bindings *
- ******************/
+ def(
+ "precision_recall",
+ &precision_recall,
+ (arg("negatives"), arg("positives"), arg("threshold")),
+ "Calculates the precision and recall (sensitiveness) values given positive and negative scores and a threshold. 'positives' holds the score information for samples that are labeled to belong to a certain class (a.k.a., 'signal' or 'client'). 'negatives' holds the score information for samples that are labeled *not* to belong to the class (a.k.a., 'noise' or 'impostor'). For more precise details about how the method considers error rates, please refer to the documentation of the method bob.measure.farfrr."
+ );
+
+ def(
+ "f_score",
+ &f_score,
+ (arg("negatives"), arg("positives"), arg("threshold"), arg("weight")=1.0),
+ "This method computes F score of the accuracy of the classification. It is a weighted mean of precision and recall measurements. The weight parameter needs to be non-negative real value. In case the weight parameter is 1, the F-score is called F1 score and is a harmonic mean between precision and recall values."
+ );
+
+ def(
+ "correctly_classified_positives",
+ &bob_correctly_classified_positives,
+ (arg("positives"), arg("threshold")),
+ "This method returns a blitz::Array composed of booleans that pin-point which positives where correctly classified in a 'positive' score sample, given a threshold. It runs the formula: foreach (element k in positive) if positive[k] >= threshold: returnValue[k] = true else: returnValue[k] = false"
+ );
+
+ def(
+ "correctly_classified_negatives",
+ &bob_correctly_classified_negatives,
+ (arg("negatives"), arg("threshold")),
+ "This method returns a blitz::Array composed of booleans that pin-point which negatives where correctly classified in a 'negative' score sample, given a threshold. It runs the formula: foreach (element k in negative) if negative[k] < threshold: returnValue[k] = true else: returnValue[k] = false"
+ );
+
+ def(
+ "eer_threshold",
+ &bob_eer_threshold,
+ (arg("negatives"), arg("positives")),
+ "Calculates the threshold that is as close as possible to the equal-error-rate (EER) given the input data. The EER should be the point where the FAR equals the FRR. Graphically, this would be equivalent to the intersection between the ROC (or DET) curves and the identity."
+ );
+
+ def(
+ "eer_rocch",
+ &bob_eer_rocch,
+ (arg("negatives"), arg("positives")),
+ "Calculates the equal-error-rate (EER) given the input data, on the ROC Convex Hull as done in the Bosaris toolkit (https://sites.google.com/site/bosaristoolkit/)."
+ );
+
+ def(
+ "min_weighted_error_rate_threshold",
+ &bob_min_weighted_error_rate_threshold,
+ (arg("negatives"), arg("positives"), arg("cost")),
+ "Calculates the threshold that minimizes the error rate, given the input data. An optional parameter 'cost' determines the relative importance between false-accepts and false-rejections. This number should be between 0 and 1 and will be clipped to those extremes. The value to minimize becomes: ER_cost = [cost * FAR] + [(1-cost) * FRR]. The higher the cost, the higher the importance given to *not* making mistakes classifying negatives/noise/impostors."
+ );
+
+ def(
+ "min_hter_threshold",
+ &bob_min_hter_threshold,
+ (arg("negatives"), arg("positives")),
+ "Calculates the min_weighted_error_rate_threshold() when the cost is 0.5."
+ );
+
+ def(
+ "far_threshold",
+ &bob_far_threshold,
+ bob_far_threshold_overloads(
+ (arg("negatives"), arg("positives"), arg("far_value")=0.001),
+ "Computes the threshold such that the real FAR is *at least* the requested ``far_value``.\n\nKeyword parameters:\n\nnegatives\n The impostor scores to be used for computing the FAR\n\npositives\n The client scores; ignored by this function\n\nfar_value\n The FAR value where the threshold should be computed\n\nReturns the computed threshold (float)"
+ )
+ );
+
+ def(
+ "frr_threshold",
+ &bob_frr_threshold,
+ bob_frr_threshold_overloads(
+ (arg("negatives"), arg("positives"), arg("frr_value")=0.001),
+ "Computes the threshold such that the real FRR is *at least* the requested ``frr_value``.\n\nKeyword parameters:\n\nnegatives\n The impostor scores; ignored by this function\n\npositives\n The client scores to be used for computing the FRR\n\nfrr_value\n\n The FRR value where the threshold should be computed\n\nReturns the computed threshold (float)"
+ )
+ );
+
+ def(
+ "roc",
+ &bob_roc,
+ (arg("negatives"), arg("positives"), arg("n_points")),
+ "Calculates the ROC curve given a set of positive and negative scores and a desired number of points. Returns a two-dimensional blitz::Array of doubles that express the X (FRR) and Y (FAR) coordinates in this order. The points in which the ROC curve are calculated are distributed uniformily in the range [min(negatives, positives), max(negatives, positives)]."
+ );
+
+ def(
+ "precision_recall_curve",
+ &bob_precision_recall_curve,
+ (arg("negatives"), arg("positives"), arg("n_points")),
+ "Calculates the precision-recall curve given a set of positive and negative scores and a number of desired points. Returns a two-dimensional blitz::Array of doubles that express the X (precision) and Y (recall) coordinates in this order. The points in which the curve is calculated are distributed uniformly in the range [min(negatives, positives), max(negatives, positives)]."
+ );
+
+ def(
+ "rocch",
+ &bob_rocch,
+ (arg("negatives"), arg("positives")),
+ "Calculates the ROC Convex Hull curve given a set of positive and negative scores. Returns a two-dimensional blitz::Array of doubles that express the X (FRR) and Y (FAR) coordinates in this order."
+ );
+
+ def(
+ "rocch2eer",
+ &bob_rocch2eer,
+ (arg("pmiss_pfa")),
+ "Calculates the threshold that is as close as possible to the equal-error-rate (EER) given the input data."
+ );
+
+ def(
+ "roc_for_far",
+ &bob_roc_for_far,
+ (arg("negatives"), arg("positives"), arg("far_list")),
+ "Calculates the ROC curve given a set of positive and negative scores and the FAR values for which the CAR should be computed. The resulting ROC curve holds a copy of the given FAR values (row 0), and the corresponding FRR values (row 1)."
+ );
+
+ def(
+ "ppndf",
+ &bob::measure::ppndf,
+ (arg("value")),
+ "Returns the Deviate Scale equivalent of a false rejection/acceptance ratio.\n\nThe algorithm that calculates the deviate scale is based on function ppndf() from the NIST package DETware version 2.1, freely available on the internet. Please consult it for more details."
+ );
+
+ def(
+ "det",
+ &bob_det,
+ (arg("negatives"), arg("positives"), arg("n_points")),
+ "Calculates the DET curve given a set of positive and negative scores and a desired number of points. Returns a two-dimensional blitz::Array of doubles that express on its rows:\n\n0. X axis values in the normal deviate scale for the false-rejections\n1. Y axis values in the normal deviate scale for the false-accepts\n\nYou can plot the results using your preferred tool to first create a plot using rows 0 and 1 from the returned value and then replace the X/Y axis annotation using a pre-determined set of tickmarks as recommended by NIST.\n\nThe algorithm that calculates the deviate scale is based on function ppndf() from the NIST package DETware version 2.1, freely available on the internet. Please consult it for more details.\n\nBy 20.04.2011, you could find such package here: http://www.itl.nist.gov/iad/mig/tools/"
+ );
- PyXbobIo_API[PyBobIoVideoReader_Type_NUM] = (void *)&PyBobIoVideoReader_Type;
+}
+**/
+
+static int double1d_converter(PyObject* o, PyBlitzArrayObject** a) {
+ if (PyBlitzArray_Converter(o, a) != 0) return 1;
+ // in this case, *a is set to a new reference
+ if ((*a)->type_num != NPY_FLOAT64 || (*a)->ndim != 1) {
+ PyErr_Format(PyExc_TypeError, "cannot convert blitz::Array<%s,%" PY_FORMAT_SIZE_T "d> to a blitz::Array<double,1>", PyBlitzArray_TypenumAsString((*a)->type_num), (*a)->ndim);
+ return 1;
+ }
+ return 0;
+}
- PyXbobIo_API[PyBobIoVideoReaderIterator_Type_NUM] = (void *)&PyBobIoVideoReaderIterator_Type;
+static PyObject* epc(PyObject*, PyObject* args, PyObject* kwds) {
+
+ /* Parses input arguments in a single shot */
+ static const char* const_kwlist[] = {
+ "dev_negatives",
+ "dev_positives",
+ "test_positives",
+ "test_negatives",
+ "n_points",
+ 0 /* Sentinel */
+ };
+ static char** kwlist = const_cast<char**>(const_kwlist);
+
+ PyBlitzArrayObject* dev_neg = 0;
+ PyBlitzArrayObject* dev_pos = 0;
+ PyBlitzArrayObject* test_neg = 0;
+ PyBlitzArrayObject* test_pos = 0;
+ Py_ssize_t n_points = 0;
+
+ if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&O&O&O&n",
+ kwlist,
+ &double1d_converter, &dev_neg,
+ &double1d_converter, &dev_pos,
+ &double1d_converter, &test_neg,
+ &double1d_converter, &test_pos,
+ &n_points
+ )) return 0;
+
+ blitz::Array<double,2> retval = bob::measure::epc(
+ *PyBlitzArrayCxx_AsBlitz<double,1>(dev_neg),
+ *PyBlitzArrayCxx_AsBlitz<double,1>(dev_pos),
+ *PyBlitzArrayCxx_AsBlitz<double,1>(test_neg),
+ *PyBlitzArrayCxx_AsBlitz<double,1>(test_pos),
+ n_points);
+
+ PyObject* pyret = reinterpret_cast<PyObject*>(PyBlitzArrayCxx_NewFromArray(retval));
+
+ Py_DECREF(dev_neg);
+ Py_DECREF(dev_pos);
+ Py_DECREF(test_neg);
+ Py_DECREF(test_pos);
+
+ return pyret;
- PyXbobIo_API[PyBobIoVideoWriter_Type_NUM] = (void *)&PyBobIoVideoWriter_Type;
-#endif /* WITH_FFMPEG */
+}
- /*****************
- * HDF5 bindings *
- *****************/
+PyDoc_STRVAR(s_epc_str, "epc");
+PyDoc_STRVAR(s_epc_doc,
+"epc(dev_negatives, dev_positives, test_negatives, test_positives, n_points) -> numpy.ndarray\n\
+\n\
+Calculates points of an Expected Performance Curve (EPC).\n\
+\n\
+Calculates the EPC curve given a set of positive and negative scores\n\
+and a desired number of points. Returns a two-dimensional\n\
+blitz::Array of doubles that express the X (cost) and Y (HTER on\n\
+the test set given the min. HTER threshold on the development set)\n\
+coordinates in this order. Please note that, in order to calculate\n\
+the EPC curve, one needs two sets of data comprising a development\n\
+set and a test set. The minimum weighted error is calculated on the\n\
+development set and then applied to the test set to evaluate the\n\
+half-total error rate at that position.\n\
+\n\
+The EPC curve plots the HTER on the test set for various values of\n\
+'cost'. For each value of 'cost', a threshold is found that provides\n\
+the minimum weighted error (see\n\
+:py:func:`xbob.measure.min_weighted_error_rate_threshold()`)\n\
+on the development set. Each threshold is consecutively applied to\n\
+the test set and the resulting HTER values are plotted in the EPC.\n\
+\n\
+The cost points in which the EPC curve are calculated are\n\
+distributed uniformily in the range :math:`[0.0, 1.0]`.\n\
+");
+
+static PyMethodDef library_methods[] = {
+ {
+ s_epc_str,
+ (PyCFunction)epc,
+ METH_VARARGS|METH_KEYWORDS,
+ s_epc_doc
+ },
+ {0} /* Sentinel */
+};
- PyXbobIo_API[PyBobIoHDF5File_Type_NUM] = (void *)&PyBobIoHDF5File_Type;
-
- PyXbobIo_API[PyBobIoHDF5File_Check_NUM] = (void *)&PyBobIoHDF5File_Check;
+PyMODINIT_FUNC XBOB_EXT_ENTRY_NAME (void) {
- PyXbobIo_API[PyBobIoHDF5File_Converter_NUM] = (void *)&PyBobIoHDF5File_Converter;
+ PyObject* m = Py_InitModule3(XBOB_EXT_MODULE_NAME,
+ library_methods, "bob::measure bindings");
+ PyModule_AddStringConstant(m, "__version__", XBOB_EXT_MODULE_VERSION);
/* imports the NumPy C-API */
import_array();
--
GitLab