diff --git a/doc/c_cpp_api.rst b/doc/c_cpp_api.rst
deleted file mode 100644
index 88a6870aa92f65e77f30e7c3ab624f3b1b091845..0000000000000000000000000000000000000000
--- a/doc/c_cpp_api.rst
+++ /dev/null
@@ -1,206 +0,0 @@
-.. vim: set fileencoding=utf-8 :
-.. Andre Anjos <andre.dos.anjos@gmail.com>
-.. Tue 15 Oct 14:59:05 2013
-
-=========
- C++ API
-=========
-
-The C++ API of ``xbob.io`` allows users to leverage from automatic converters
-for classes in :py:class:`xbob.io`. To use the C API, clients should first,
-include the header file ``<xbob.io/api.h>`` on their compilation units and
-then, make sure to call once ``import_xbob_io()`` at their module
-instantiation, as explained at the `Python manual
-<http://docs.python.org/2/extending/extending.html#using-capsules>`_.
-
-Here is a dummy C example showing how to include the header and where to call
-the import function:
-
-.. code-block:: c++
-
- #include <xbob.io/api.h>
-
- PyMODINIT_FUNC initclient(void) {
-
- PyObject* m Py_InitModule("client", ClientMethods);
-
- if (!m) return;
-
- // imports the NumPy C-API
- import_array();
-
- // imports blitz.array C-API
- import_blitz_array();
-
- // imports xbob.core.random C-API
- import_xbob_io();
-
- }
-
-.. note::
-
- The include directory can be discovered using
- :py:func:`xbob.io.get_include`.
-
-Generic Functions
------------------
-
-.. cpp:function:: int PyBobIo_AsTypenum(bob::core::array::ElementType et)
-
- Converts the input Bob element type into a ``NPY_<TYPE>`` enumeration value.
- Returns ``NPY_NOTYPE`` in case of problems, and sets a
- :py:class:`RuntimeError`.
-
-.. cpp:function:: PyObject* PyBobIo_TypeInfoAsTuple (const bob::core::array::typeinfo& ti)
-
- Converts the ``bob::core::array::typeinfo&`` object into a **new reference**
- to a :py:class:`tuple` with 3 elements:
-
- [0]
- The data type as a :py:class:`numpy.dtype` object
-
- [1]
- The shape of the object, as a tuple of integers
-
- [2]
- The strides of the object, as a tuple of integers
-
- Returns ``0`` in case of failure, or a **new reference** to the tuple
- described above in case of success.
-
-
-Bob File Support
-----------------
-
-.. cpp:type:: PyBobIoFileObject
-
- The pythonic object representation for a ``bob::io::File`` object.
-
- .. code-block:: cpp
-
- typedef struct {
- PyObject_HEAD
- boost::shared_ptr<bob::io::File> f;
- } PyBobIoFileObject;
-
- .. cpp:member:: boost::shared_ptr<bob::io::File> f
-
- A pointer to a file being read or written.
-
-.. cpp:type:: PyBobIoFileIteratorObject
-
- The pythonic object representation for an iterator over a ``bob::io::File``
- object.
-
- .. code-block:: cpp
-
- typedef struct {
- PyObject_HEAD
- PyBobIoFileObject* pyfile;
- Py_ssize_t curpos;
- } PyBobIoFileIteratorObject;
-
- .. cpp:member:: PyBobIoFileObject* pyfile
-
- A pointer to the pythonic representation of a file.
-
- .. cpp:member:: Py_ssize_t curpos
-
- The current position at the file being pointed to.
-
-
-Bob HDF5 Support
-----------------
-
-.. cpp:type:: PyBobIoHDF5FileObject
-
- The pythonic object representation for a ``bob::io::HDF5File`` object.
-
- .. code-block:: cpp
-
- typedef struct {
- PyObject_HEAD
- boost::shared_ptr<bob::io::HDF5File> f;
- } PyBobIoHDF5FileObject;
-
- .. cpp:member:: boost::shared_ptr<bob::io::HDF5File> f
-
- A pointer to a Bob object being used to read/write data into an HDF5
- file.
-
-
-.. cpp:function:: int PyBobIoHDF5File_Check(PyObject* o)
-
- Checks if the input object ``o`` is a ``PyBobIoHDF5FileObject``. Returns
- ``1`` if it is, and ``0`` otherwise.
-
-
-.. cpp:function:: int PyBobIoHDF5File_Converter(PyObject* o, PyBobIoHDF5FileObject** a)
-
- This function is meant to be used with :c:func:`PyArg_ParseTupleAndKeywords`
- family of functions in the Python C-API. It checks the input object to be of
- type ``PyBobIoHDF5FileObject`` and sets a **new reference** to it (in
- ``*a``) if it is the case. Returns ``0`` in case of failure, ``1`` in case
- of success.
-
-Bob VideoReader Support
------------------------
-
-.. note::
-
- The video C-API (and Python) is only available if the package was compiled
- with FFMPEG or LibAV support.
-
-.. cpp:type:: PyBobIoVideoReaderObject
-
- The pythonic object representation for a ``bob::io::VideoReader`` object.
-
- .. code-block:: cpp
-
- typedef struct {
- PyObject_HEAD
- boost::shared_ptr<bob::io::VideoReader> v;
- } PyBobIoVideoReaderObject;
-
- .. cpp:member:: boost::shared_ptr<bob::io::VideoReader> v
-
- A pointer to a Bob object being used to read the video contents
-
-.. cpp:type:: PyBobIoVideoReaderIteratorObject
-
- The pythonic object representation for an iterator over a
- ``bob::io::VideoReader`` object.
-
- .. code-block:: cpp
-
- typedef struct {
- PyObject_HEAD
- PyBobIoVideoReaderObject* pyreader;
- boost::shared_ptr<bob::io::VideoReader::const_iterator> iter;
- } PyBobIoFileIteratorObject;
-
- .. cpp:member:: PyBobIoVideoReaderObject* pyreader
-
- A pointer to the pythonic representation of the video reader.
-
- .. cpp:member:: boost::shared_ptr<bob::io::VideoReader::const_iterator> iter
-
- The current position at the file being pointed to, represented by a
- formal iterator over the VideoReader.
-
-.. cpp:type:: PyBobIoVideoReaderObject
-
- The pythonic object representation for a ``bob::io::VideoWriter`` object.
-
- .. code-block:: cpp
-
- typedef struct {
- PyObject_HEAD
- boost::shared_ptr<bob::io::VideoWriter> v;
- } PyBobIoVideoWriterObject;
-
- .. cpp:member:: boost::shared_ptr<bob::io::VideoWriter> v
-
- A pointer to a Bob object being used to write contents to the video.
-
-.. include:: links.rst
diff --git a/doc/guide.rst b/doc/guide.rst
index 3dc1a2fe3dbd6a876ae961563463bd0ac87ba0b8..f62a5469b9b804770ffe9641f9da5df679632a30 100644
--- a/doc/guide.rst
+++ b/doc/guide.rst
@@ -2,479 +2,348 @@
.. Andre Anjos <andre.dos.anjos@gmail.com>
.. Tue 15 Oct 17:41:52 2013
-.. testsetup:: iotest
+.. testsetup:: measuretest
import numpy
- import xbob.io
+ import xbob.measure
============
User Guide
============
-This section gives an overview of the operations for storing and retrieving the
-basic data structures in |project|, such as `NumPy`_ arrays. |project| uses
-`HDF5`_ format for storing binary coded data. Using the |project| support for
-`HDF5`_, it is very simple to import and export data.
-
-`HDF5`_ uses a neat descriptive language for representing the data in the HDF5
-files, called Data Description Language (`DDL`_).
-
-To perform the functionalities given in this section, you should have `NumPy`_
-and |project| loaded into the `Python`_ environment.
-
-.. testsetup:: *
-
- import numpy
- import xbob.io
- import tempfile
- import os
-
- current_directory = os.path.realpath(os.curdir)
- temp_dir = tempfile.mkdtemp(prefix='bob_doctest_')
- os.chdir(temp_dir)
-
-HDF5 standard utilities
------------------------
-
-Before explaining the basics of reading and writing to `HDF5`_ files, it is
-important to list some `HDF5`_ standard utilities for checking the content of
-an `HDF5`_ file. These are supplied by the `HDF5`_ project.
-
-``h5dump``
- Dumps the content of the file using the DDL.
-
-``h5ls``
- Lists the content of the file using DDL, but does not show the data.
-
-``h5diff``
- Finds the differences between HDF5 files.
-
-I/O operations using the class `xbob.io.HDF5File`
--------------------------------------------------
-
-Writing operations
-------------------
-
-Let's take a look at how to write simple scalar data such as integers or
-floats.
-
-.. doctest::
-
- >>> an_integer = 5
- >>> a_float = 3.1416
- >>> f = xbob.io.HDF5File('testfile1.hdf5', 'w')
- >>> f.set('my_integer', an_integer)
- >>> f.set('my_float', a_float)
- >>> del f
-
-If after this you use the **h5dump** utility on the file ``testfile1.hdf5``,
-you will verify that the file now contains:
-
-.. code-block:: none
-
- HDF5 "testfile1.hdf5" {
- GROUP "/" {
- DATASET "my_float" {
- DATATYPE H5T_IEEE_F64LE
- DATASPACE SIMPLE { ( 1 ) / ( 1 ) }
- DATA {
- (0): 3.1416
- }
- }
- DATASET "my_integer" {
- DATATYPE H5T_STD_I32LE
- DATASPACE SIMPLE { ( 1 ) / ( 1 ) }
- DATA {
- (0): 5
- }
- }
- }
- }
+Methods in the :py:mod:`xbob.measure` module can help you to quickly and easily
+evaluate error for multi-class or binary classification problems. If you are
+not yet familiarized with aspects of performance evaluation, we recommend the
+following papers for an overview of some of the methods implemented.
+
+* Bengio, S., Keller, M., Mariéthoz, J. (2004). `The Expected Performance
+ Curve`_. International Conference on Machine Learning ICML Workshop on ROC
+ Analysis in Machine Learning, 136(1), 1963–1966.
+* Martin, A., Doddington, G., Kamm, T., Ordowski, M., & Przybocki, M. (1997).
+ `The DET curve in assessment of detection task performance`_. Fifth European
+ Conference on Speech Communication and Technology (pp. 1895-1898).
+
+Overview
+--------
+
+A classifier is subject to two types of errors, either the real access/signal
+is rejected (false rejection) or an impostor attack/a false access is accepted
+(false acceptance). A possible way to measure the detection performance is to
+use the Half Total Error Rate (HTER), which combines the False Rejection Rate
+(FRR) and the False Acceptance Rate (FAR) and is defined in the following
+formula:
+
+.. math::
+
+ HTER(\tau, \mathcal{D}) = \frac{FAR(\tau, \mathcal{D}) + FRR(\tau, \mathcal{D})}{2} \quad \textrm{[\%]}
+
+where :math:`\mathcal{D}` denotes the dataset used. Since both the FAR and the
+FRR depends on the threshold :math:`\tau`, they are strongly related to each
+other: increasing the FAR will reduce the FRR and vice-versa. For this reason,
+results are often presented using either a Receiver Operating Characteristic
+(ROC) or a Detection-Error Tradeoff (DET) plot, these two plots basically
+present the FAR versus the FRR for different values of the threshold. Another
+widely used measure to summarise the performance of a system is the Equal Error
+Rate (EER), defined as the point along the ROC or DET curve where the FAR
+equals the FRR.
+
+However, it was noted in by Bengio et al. (2004) that ROC and DET curves may be
+misleading when comparing systems. Hence, the so-called Expected Performance
+Curve (EPC) was proposed and consists of an unbiased estimate of the reachable
+performance of a system at various operating points. Indeed, in real-world
+scenarios, the threshold :math:`\tau` has to be set a priori: this is typically
+done using a development set (also called cross-validation set). Nevertheless,
+the optimal threshold can be different depending on the relative importance
+given to the FAR and the FRR. Hence, in the EPC framework, the cost
+:math:`\beta \in [0;1]` is defined as the tradeoff between the FAR and FRR. The
+optimal threshold :math:`\tau^*` is then computed using different values of
+:math:`\beta`, corresponding to different operating points:
+
+.. math::
+ \tau^{*} = \arg\!\min_{\tau} \quad \beta \cdot \textrm{FAR}(\tau, \mathcal{D}_{d}) + (1-\beta) \cdot \textrm{FRR}(\tau, \mathcal{D}_{d})
+
+where :math:`\mathcal{D}_{d}` denotes the development set and should be
+completely separate to the evaluation set `\mathcal{D}`.
+
+Performance for different values of :math:`\beta` is then computed on the test
+set :math:`\mathcal{D}_{t}` using the previously derived threshold. Note that
+setting :math:`\beta` to 0.5 yields to the Half Total Error Rate (HTER) as
+defined in the first equation.
.. note::
- In |project|, when you open a HDF5 file, you can choose one of the following
- options:
+ Most of the methods availabe in this module require as input a set of 2
+ :py:class:`numpy.ndarray` objects that contain the scores obtained by the
+ classification system to be evaluated, without specific order. Most of the
+ classes that are defined to deal with two-class problems. Therefore, in this
+ setting, and throughout this manual, we have defined that the **negatives**
+ represents the impostor attacks or false class accesses (that is when a
+ sample of class A is given to the classifier of another class, such as class
+ B) for of the classifier. The second set, refered as the **positives**
+ represents the true class accesses or signal response of the classifier. The
+ vectors are called this way because the procedures implemented in this module
+ expects that the scores of **negatives** to be statistically distributed to
+ the left of the signal scores (the **positives**). If that is not the case,
+ one should either invert the input to the methods or multiply all scores
+ available by -1, in order to have them inverted.
+
+ The input to create these two vectors is generated by experiments conducted
+ by the user and normally sits in files that may need some parsing before
+ these vectors can be extracted.
+
+ While it is not possible to provide a parser for every individual file that
+ may be generated in different experimental frameworks, we do provide a few
+ parsers for formats we use the most. Please refer to the documentation of
+ :py:mod:`bob.measure.load` for a list of formats and details.
+
+ In the remainder of this section we assume you have successfuly parsed and
+ loaded your scores in two 1D float64 vectors and are ready to evaluate the
+ performance of the classifier.
- **'r'** Open the file in reading mode; writing operations will fail (this is the default).
-
- **'a'** Open the file in reading and writing mode with appending.
+.. testsetup:: *
- **'w'** Open the file in reading and writing mode, but truncate it.
+ import numpy
+ positives = numpy.random.normal(1,1,100)
+ negatives = numpy.random.normal(-1,1,100)
+ import bob
+ import matplotlib
+ if not hasattr(matplotlib, 'backends'):
+ matplotlib.use('pdf') #non-interactive avoids exception on display
- **'x'** Read/write/append with exclusive access.
+Evaluation
+----------
-The dump shows that there are two datasets inside a group named ``/`` in the
-file. HDF5 groups are like file system directories. They create namespaces for
-the data. In the root group (or directory), you will find the two variables,
-named as you set them to be. The variable names are the complete path to the
-location where they live. You could write a new variable in the same file but
-in a different directory like this:
+To count the number of correctly classified positives and negatives you can use
+the following techniques:
.. doctest::
- >>> f = xbob.io.HDF5File('testfile1.hdf5', 'a')
- >>> f.create_group('/test')
- >>> f.set('/test/my_float', numpy.float32(6.28))
- >>> del f
-
-Line 1 opens the file for reading and writing, but without truncating it. This
-will allow you to access the file contents. Next, the directory ``/test`` is
-created and a new variable is written inside the subdirectory. As you can
-verify, **for simple scalars**, you can also force the storage type. Where
-normally one would have a 64-bit real value, you can impose that this variable
-is saved as a 32-bit real value. You can verify the dump correctness with
-``h5dump``:
-
-.. code-block:: none
-
- GROUP "/" {
- ...
- GROUP "test" {
- DATASET "my_float" {
- DATATYPE H5T_IEEE_F32LE
- DATASPACE SIMPLE { ( 1 ) / ( 1 ) }
- DATA {
- (0): 6.28
- }
- }
- }
- }
-
-Notice the subdirectory ``test`` has been created and inside it a floating
-point number has been stored. Such a float point number has a 32-bit precision
-as it was defined.
-
-.. note::
-
- If you need to place lots of variables in a subfolder, it may be better to
- setup the prefix folder before starting the writing operations on the
- :py:class:`xbob.io.HDF5File` object. You can do this using the method
- :py:meth:`HDF5File.cd`. Look up its help for more information and usage
- instructions.
+ >>> # negatives, positives = parse_my_scores(...) # write parser if not provided!
+ >>> T = 0.0 #Threshold: later we explain how one can calculate these
+ >>> correct_negatives = bob.measure.correctly_classified_negatives(negatives, T)
+ >>> FAR = 1 - (float(correct_negatives.sum())/negatives.size)
+ >>> correct_positives = bob.measure.correctly_classified_positives(positives, T)
+ >>> FRR = 1 - (float(correct_positives.sum())/positives.size)
-Writing arrays is a little simpler as the :py:class:`numpy.ndarray` objects
-encode all the type information we need to write and read them correctly. Here
-is an example:
+We do provide a method to calculate the FAR and FRR in a single shot:
.. doctest::
- >>> A = numpy.array(range(4), 'int8').reshape(2,2)
- >>> f = xbob.io.HDF5File('testfile1.hdf5', 'a')
- >>> f.set('my_array', A)
- >>> del f
+ >>> FAR, FRR = bob.measure.farfrr(negatives, positives, T)
-The result of running ``h5dump`` on the file ``testfile3.hdf5`` should be:
+The threshold ``T`` is normally calculated by looking at the distribution of
+negatives and positives in a development (or validation) set, selecting a
+threshold that matches a certain criterion and applying this derived threshold
+to the test (or evaluation) set. This technique gives a better overview of the
+generalization of a method. We implement different techniques for the
+calculation of the threshold:
-.. code-block:: none
+* Threshold for the EER
- ...
- DATASET "my_array" {
- DATATYPE H5T_STD_I8LE
- DATASPACE SIMPLE { ( 2, 2 ) / ( 2, 2 ) }
- DATA {
- (0,0): 0, 1,
- (1,0): 2, 3
- }
- }
- ...
+ .. doctest::
-You don't need to limit yourself to single variables, you can also save lists
-of scalars and arrays using the function :py:meth:`xbob.io.HDF5.append` instead
-of :py:meth:`xbob.io.HDF5.set`.
+ >>> T = bob.measure.eer_threshold(negatives, positives)
-Reading operations
-------------------
+* Threshold for the minimum HTER
-Reading data from a file that you just wrote to is just as easy. For this task
-you should use :py:meth:`xbob.io.HDF5File.read`. The read method will read all
-the contents of the variable pointed to by the given path. This is the normal
-way to read a variable you have written with :py:meth:`xbob.io.HDF5File.set`. If
-you decided to create a list of scalar or arrays, the way to read that up would
-be using :py:meth:`xbob.io.HDF5File.lread` instead. Here is an example:
+ .. doctest::
-.. doctest::
+ >>> T = bob.measure.min_hter_threshold(negatives, positives)
- >>> f = xbob.io.HDF5File('testfile1.hdf5') #read only
- >>> f.read('my_integer') #reads integer
- 5
- >>> print(f.read('my_array')) # reads the array
- [[0 1]
- [2 3]]
- >>> del f
+* Threshold for the minimum weighted error rate (MWER) given a certain cost
+ :math:`\beta`.
-Now let's look at an example where we have used
-:py:meth:`xbob.io.HDF5File.append` instead of :py:meth:`xbob.io.HDF5File.set`
-to write data to a file. That is normally the case when you write lists of
-variables to a dataset.
+ .. code-block:: python
-.. doctest::
+ >>> cost = 0.3 #or "beta"
+ >>> T = bob.measure.min_weighted_error_rate_threshold(negatives, positives, cost)
- >>> f = xbob.io.HDF5File('testfile2.hdf5', 'w')
- >>> f.append('arrayset', numpy.array(range(10), 'float64'))
- >>> f.append('arrayset', 2*numpy.array(range(10), 'float64'))
- >>> f.append('arrayset', 3*numpy.array(range(10), 'float64'))
- >>> print(f.lread('arrayset', 0))
- [ 0. 1. 2. 3. 4. 5. 6. 7. 8. 9.]
- >>> print(f.lread('arrayset', 2))
- [ 0. 3. 6. 9. 12. 15. 18. 21. 24. 27.]
- >>> del f
-
-This is what the ``h5dump`` of the file would look like:
-
-.. code-block:: none
-
- HDF5 "testfile4.hdf5" {
- GROUP "/" {
- DATASET "arrayset" {
- DATATYPE H5T_IEEE_F64LE
- DATASPACE SIMPLE { ( 3, 10 ) / ( H5S_UNLIMITED, 10 ) }
- DATA {
- (0,0): 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
- (1,0): 0, 2, 4, 6, 8, 10, 12, 14, 16, 18,
- (2,0): 0, 3, 6, 9, 12, 15, 18, 21, 24, 27
- }
- }
- }
- }
-
-Notice that the expansion limits for the first dimension have been correctly
-set by |project| so you can insert an *unlimited* number of 1D float vectors.
-Of course, you can also read the whole contents of the arrayset in a single
-shot:
+ .. note::
-.. doctest::
+ By setting cost to 0.5 is equivalent to use
+ :py:meth:`bob.measure.min_hter_threshold`.
- >>> f = xbob.io.HDF5File('testfile2.hdf5')
- >>> print(f.read('arrayset'))
- [[ 0. 1. 2. 3. 4. 5. 6. 7. 8. 9.]
- [ 0. 2. 4. 6. 8. 10. 12. 14. 16. 18.]
- [ 0. 3. 6. 9. 12. 15. 18. 21. 24. 27.]]
-
-As you can see, the only difference between :py:meth:`xbob.io.HDF5File.read`
-and :py:meth:`xbob.io.HDF5File.lread` is on how |project| considers the
-available data (as a single array with N dimensions or as a set of arrays with
-N-1 dimensions). In the first example, you would have also been able to read
-the variable `my_array` as an arrayset using :py:meth:`xbob.io.HDF5File.lread`
-instead of :py:meth:`xbob.io.HDF5File.read`. In this case, each position
-readout would return a 1D uint8 array instead of a 2D array.
-
-Array interfaces
-----------------
-
-What we have shown so far is the generic API to read and write data using HDF5.
-You will use it when you want to import or export data from |project| into
-other software frameworks, debug your data or just implement your own classes
-that can serialize and de-serialize from HDF5 file containers. In |project|,
-most of the time you will be working with :py:class:`numpy.ndarrays`\s. In
-special situations though, you may be asked to handle
-:py:class:`xbob.io.File`\s. :py:class:`xbob.io.File` objects create a
-transparent connection between C++ (`Blitz++`_) / Python (`NumPy`_) arrays and
-file access. You specify the filename from which you want to input data and
-the :py:class:`xbob.io.File` object decides what is the best codec to be used
-(from the extension) and how to read the data back into your array.
-
-To create an :py:class:`xbob.io.File` from a file path, just do the following:
+Plotting
+--------
-.. doctest::
+An image is worth 1000 words, they say. You can combine the capabilities of
+`Matplotlib`_ with |project| to plot a number of curves. However, you must have that
+package installed though. In this section we describe a few recipes.
- >>> a = xbob.io.File('testfile2.hdf5', 'r')
- >>> a.filename
- 'testfile2.hdf5'
+ROC
+===
-:py:class:`xbob.io.File`\s simulate containers for :py:class:`numpy.ndarray`\s,
-transparently accessing the file data when requested. Note, however, that when
-you instantiate an :py:class:`xbob.io.File` it does **not** load the file
-contents into memory. It waits until you emit another explicit instruction to
-do so. We do this with the :py:meth:`xbob.io.File.read` method:
+The Receiver Operating Characteristic (ROC) curve is one of the oldest plots in
+town. To plot an ROC curve, in possession of your **negatives** and
+**positives**, just do something along the lines of:
.. doctest::
- >>> array = a.read()
- >>> array
- array([[ 0., 1., 2., 3., 4., 5., 6., 7., 8., 9.],
- [ 0., 2., 4., 6., 8., 10., 12., 14., 16., 18.],
- [ 0., 3., 6., 9., 12., 15., 18., 21., 24., 27.]])
+ >>> from matplotlib import pyplot
+ >>> # we assume you have your negatives and positives already split
+ >>> npoints = 100
+ >>> bob.measure.plot.roc(negatives, positives, npoints, color=(0,0,0), linestyle='-', label='test') # doctest: +SKIP
+ >>> pyplot.xlabel('FRR (%)') # doctest: +SKIP
+ >>> pyplot.ylabel('FAR (%)') # doctest: +SKIP
+ >>> pyplot.grid(True)
+ >>> pyplot.show() # doctest: +SKIP
-Every time you say :py:meth:`xbob.io.File.read`, the file contents will be read
-from the file and into a new array.
+You should see an image like the following one:
-Saving arrays to the :py:class:`xbob.io.File` is as easy, just call the
-:py:meth:`xbob.io.File.write` method:
+.. plot:: plot/perf_roc.py
+ :include-source: False
-.. doctest::
+As can be observed, plotting methods live in the namespace
+:py:mod:`bob.measure.plot`. They work like `Matplotlib`_'s `plot()`_ method
+itself, except that instead of receiving the x and y point coordinates as
+parameters, they receive the two :py:class:`numpy.ndarray` arrays with
+negatives and positives, as well as an indication of the number of points the
+curve must contain.
- >>> f = xbob.io.File('copy1.hdf5', 'w')
- >>> f.write(a)
+As in `Matplotlib`_'s `plot()`_ command, you can pass optional parameters for
+the line as shown in the example to setup its color, shape and even the label.
+For an overview of the keywords accepted, please refer to the `Matplotlib`_'s
+Documentation. Other plot properties such as the plot title, axis labels,
+grids, legends should be controlled directly using the relevant `Matplotlib`_'s
+controls.
-Numpy ndarray shortcuts
------------------------
+DET
+===
-To just load an :py:class:`numpy.ndarray` in memory, you can use a short cut
-that lives at :py:func:`xbob.io.load`. With it, you don't have to go through
-the :py:class:`xbob.io.File` container:
+A DET curve can be drawn using similar commands such as the ones for the ROC curve:
.. doctest::
- >>> t = xbob.io.load('testfile2.hdf5')
- >>> t
- array([[ 0., 1., 2., 3., 4., 5., 6., 7., 8., 9.],
- [ 0., 2., 4., 6., 8., 10., 12., 14., 16., 18.],
- [ 0., 3., 6., 9., 12., 15., 18., 21., 24., 27.]])
-
-You can also directly save :py:class:`numpy.ndarray`\s without going
-through the :py:class:`xbob.io.Array` container:
+ >>> from matplotlib import pyplot
+ >>> # we assume you have your negatives and positives already split
+ >>> npoints = 100
+ >>> bob.measure.plot.det(negatives, positives, npoints, color=(0,0,0), linestyle='-', label='test') # doctest: +SKIP
+ >>> bob.measure.plot.det_axis([0.01, 40, 0.01, 40]) # doctest: +SKIP
+ >>> pyplot.xlabel('FAR (%)') # doctest: +SKIP
+ >>> pyplot.ylabel('FRR (%)') # doctest: +SKIP
+ >>> pyplot.grid(True)
+ >>> pyplot.show() # doctest: +SKIP
-.. doctest::
+This will produce an image like the following one:
- >>> xbob.io.save(t, 'copy2.hdf5')
+.. plot:: plot/perf_det.py
+ :include-source: False
.. note::
- Under the hood, we still use the :py:class:`xbob.io.File` API to execute
- the read and write operations. Have a look at the manual section for
- :py:mod:`xbob.io` for more details and other shortcuts available.
+ If you wish to reset axis zooming, you must use the Gaussian scale rather
+ than the visual marks showed at the plot, which are just there for
+ displaying purposes. The real axis scale is based on the
+ ``bob.measure.ppndf()`` method. For example, if you wish to set the x and y
+ axis to display data between 1% and 40% here is the recipe:
-Reading and writing images
---------------------------
+ .. doctest::
-|project| provides support to load and save data from many different file types
-including Matlab ``.mat`` files, various image file types and video data. File
-types and specific serialization and de-serialization is switched automatically
-using filename extensions. Knowing this, saving an array in a different format
-is just a matter of choosing the right extension. This is illustrated in the
-following example, where an image generated randomly using the method `NumPy`
-:py:meth:`numpy.random.random_integers`, is saved in JPEG format. The image
-must be of type uint8 or uint16.
-
-.. doctest::
+ >>> #AFTER you plot the DET curve, just set the axis in this way:
+ >>> pyplot.axis([bob.measure.ppndf(k/100.0) for k in (1, 40, 1, 40)]) # doctest: +SKIP
- >>> my_image = numpy.random.random_integers(0,255,(3,256,256))
- >>> xbob.io.save(my_image.astype('uint8'), 'testimage.jpg') # saving the image in jpeg format
- >>> my_image_copy = xbob.io.load('testimage.jpg')
+ We provide a convenient way for you to do the above in this module. So,
+ optionally, you may use the ``bob.measure.plot.det_axis`` method like this:
-.. tip::
+ .. doctest::
- To find out about which formats and extensions are supported in a given
- installation of |project|, just call ``bob_config.py`` on your prompt. It
- will print a list of compiled-in software and supported extensions.
+ >>> bob.measure.plot.det_axis([1, 40, 1, 40]) # doctest: +SKIP
-The loaded image files can be 3D arrays (for RGB format) or 2D arrays (for
-greyscale) of type ``uint8`` or ``uint16``.
+EPC
+===
-Dealing with videos
--------------------
-
-|project| has support for dealing with videos in an equivalent way to dealing
-with images:
+Drawing an EPC requires that both the development set negatives and positives are provided alognside
+the test (or evaluation) set ones. Because of this the API is slightly modified:
.. doctest::
- >>> my_video = numpy.random.random_integers(0,255,(30,3,256,256))
- >>> xbob.io.save(my_video.astype('uint8'), 'testvideo.avi') # saving the video avi format with a default codec
- >>> my_video_copy = xbob.io.load('testvideo.avi')
-
-Video reading and writing is performed using an `FFmpeg`_ (or `libav`_ if
-`FFmpeg`_ is not available) bridge. |project|'s :py:meth:`xbob.io.save` method
-will allow you to choose the output format with the same extension mechanism as
-mentioned earlier. `FFmpeg`_ will then choose a default codec for the format
-and perform encoding. The output file can be as easily loaded using
-:py:meth:`xbob.io.load`.
-
-For finer control over the loading, saving, format and codecs used for a
-specific encoding or decoding operation, you must directly use either
-:py:class:`xbob.io.VideoReader` or :py:class:`xbob.io.VideoWriter` classes. For
-example, it is possible to use :py:class:`xbob.io.VideoReader` to read videos
-frame by frame and avoid overloading your machine's memory. In the following
-example you can see how to create a video, save it using the class
-:py:class:`xbob.io.VideoWriter` and load it again using the class
-:py:class:`xbob.io.VideoReader`. The created video will have 30 frames
-generated randomly.
+ >>> bob.measure.plot.epc(dev_neg, dev_pos, test_neg, test_pos, npoints, color=(0,0,0), linestyle='-') # doctest: +SKIP
+ >>> pyplot.show() # doctest: +SKIP
-.. note::
+This will produce an image like the following one:
- Due to `FFmpeg`_ constrains, the width and height of the video need to be
- multiples of two.
+.. plot:: plot/perf_epc.py
+ :include-source: False
-.. doctest::
+Fine-tunning
+============
- >>> width = 50; height = 50;
- >>> framerate = 24
- >>> outv = xbob.io.VideoWriter('testvideo.avi', height, width, framerate, codec='mpeg1video') # output video
- >>> for i in range(0, 30):
- ... newframe = (numpy.random.random_integers(0,255,(3,height,width)))
- ... outv.append(newframe.astype('uint8'))
- >>> outv.close()
- >>> input = xbob.io.VideoReader('testvideo.avi')
- >>> input.number_of_frames
- 30
- >>> inv = input.load()
- >>> inv.shape
- (30, 3, 50, 50)
- >>> type(inv)
- <... 'numpy.ndarray'>
-
-Videos in |project| are represented as sequences of colored images, i.e. 4D
-arrays of type ``uint8``. All the extensions and formats for videos supported
-in version of |project| installed on your machine can be listed using the
-|project|'s utility ``bob_config.py``.
-
-.. testcleanup:: *
-
- import shutil
- os.chdir(current_directory)
- shutil.rmtree(temp_dir)
-
-.. warning::
-
- Please read :doc:`video` for details on choosing codecs and formats that are
- adequate to your application, as well as drawbacks and pitfalls with video
- encoding and decoding.
-
-Loading and saving Matlab data
-------------------------------
-
-An alternative for saving data in ``.mat`` files using :py:meth:`xbob.io.save`,
-would be to save them as a `HDF5`_ file which then can be easily read in
-Matlab. Similarly, instead of having to read ``.mat`` files using
-:py:meth:`xbob.io.load`, you can save your Matlab data in `HDF5`_ format, which
-then can be easily read from |project|. Detailed instructions about how to save
-and load data from Matlab to and from `HDF5`_ files can be found `here`__.
-
-.. _audiosignal:
-
-Loading and saving audio files
-------------------------------
-
-|project| does not yet support audio files (no wav codec). However, it is
-possible to use the `SciPy`_ module :py:mod:`scipy.io.wavfile` to do the job.
-For instance, to read a wave file, just use the
-:py:func:`scipy.io.wavfile.read` function.
-
-.. code-block:: python
-
- >>> import scipy.io.wavfile
- >>> filename = '/home/user/sample.wav'
- >>> samplerate, data = scipy.io.wavfile.read(filename)
- >>> print(type(data))
- <... 'numpy.ndarray'>
- >>> print(data.shape)
- (132474, 2)
-
-In the above example, the stereo audio signal is represented as a 2D `NumPy`
-:py:class:`numpy.ndarray`. The first dimension corresponds to the time index
-(132474 frames) and the second dimesnion correpsonds to one of the audio
-channel (2 channels, stereo). The values in the array correpsond to the wave
-magnitudes.
-
-To save a `NumPy` :py:class:`numpy.ndarray` into a wave file, the
-:py:func:`scipy.io.wavfile.write` could be used, which also requires the
-framerate to be specified.
+The methods inside :py:mod:`bob.measure.plot` are only provided as a
+`Matplotlib`_ wrapper to equivalent methods in :py:mod:`bob.measure` that can
+only calculate the points without doing any plotting. You may prefer to tweak
+the plotting or even use a different plotting system such as gnuplot. Have a
+look at the implementations at :py:mod:`bob.measure.plot` to understand how
+to use the |project| methods to compute the curves and interlace that in the
+way that best suits you.
+
+Full applications
+-----------------
+
+We do provide a few scripts that can be used to quickly evaluate a set of
+scores. We present these scripts in this section. The scripts take as input
+either a 4-column or 5-column data format as specified in the documentation of
+:py:mod:`bob.measure.load.four_column` or
+:py:mod:`bob.measure.load.five_column`.
+
+To calculate the threshold using a certain criterion (EER, min.HTER or weighted
+Error Rate) on a set, after setting up |project|, just do:
+
+.. code-block:: sh
+
+ $ bob_eval_threshold.py --scores=development-scores-4col.txt
+ Threshold: -0.004787956164
+ FAR : 6.731% (35/520)
+ FRR : 6.667% (26/390)
+ HTER: 6.699%
+
+The output will present the threshold together with the FAR, FRR and HTER on
+the given set, calculated using such a threshold. The relative counts of FAs
+and FRs are also displayed between parenthesis.
+
+To evaluate the performance of a new score file with a given threshold, use the
+application ``bob_apply_threshold.py``:
+
+.. code-block:: sh
+
+ $ bob_apply_threshold.py --scores=test-scores-4col.txt --threshold=-0.0047879
+ FAR : 2.115% (11/520)
+ FRR : 7.179% (28/390)
+ HTER: 4.647%
+
+In this case, only the error figures are presented. You can conduct the
+evaluation and plotting of development and test set data using our combined
+``bob_compute_perf.py`` script. You pass both sets and it does the rest:
+
+.. code-block:: sh
+
+ $ bob_compute_perf.py --devel=development-scores-4col.txt --test=test-scores-4col.txt
+ [Min. criterium: EER] Threshold on Development set: -4.787956e-03
+ | Development | Test
+ -------+-----------------+------------------
+ FAR | 6.731% (35/520) | 2.500% (13/520)
+ FRR | 6.667% (26/390) | 6.154% (24/390)
+ HTER | 6.699% | 4.327%
+ [Min. criterium: Min. HTER] Threshold on Development set: 3.411070e-03
+ | Development | Test
+ -------+-----------------+------------------
+ FAR | 4.231% (22/520) | 1.923% (10/520)
+ FRR | 7.949% (31/390) | 7.692% (30/390)
+ HTER | 6.090% | 4.808%
+ [Plots] Performance curves => 'curves.pdf'
+
+Inside that script we evaluate 2 different thresholds based on the EER and the
+minimum HTER on the development set and apply the output to the test set. As
+can be seen from the toy-example above, the system generalizes reasonably well.
+A single PDF file is generated containing an EPC as well as ROC and DET plots of such a
+system.
+
+Use the ``--help`` option on the above-cited scripts to find-out about more
+options.
.. include:: links.rst
-.. Place here your external references
+.. Place youre references here:
-.. _ddl: http://www.hdfgroup.org/HDF5/doc/ddl.html
-.. _matlab-hdf5: http://www.mathworks.ch/help/techdoc/ref/hdf5write.html
-__ matlab-hdf5_
+.. _`The Expected Performance Curve`: http://publications.idiap.ch/downloads/reports/2005/bengio_2005_icml.pdf
+.. _`The DET curve in assessment of detection task performance`: http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.117.4489&rep=rep1&type=pdf
+.. _`plot()`: http://matplotlib.sourceforge.net/api/pyplot_api.html#matplotlib.pyplot.plot
diff --git a/doc/index.rst b/doc/index.rst
index a3169f534e00017e293b1bf1fa5b8c68dbe4e646..8bff957df9c6a0ea486550311c48d8f24f8c9d33 100644
--- a/doc/index.rst
+++ b/doc/index.rst
@@ -1,7 +1,7 @@
.. vim: set fileencoding=utf-8 :
.. Andre Anjos <andre.anjos@idiap.ch>
.. Mon 4 Nov 20:58:04 2013 CET
-..
+..
.. Copyright (C) 2011-2013 Idiap Research Institute, Martigny, Switzerland
====================
@@ -19,9 +19,7 @@ Reference
:maxdepth: 2
guide
- video
py_api
- c_cpp_api
Indices and tables
------------------
diff --git a/doc/plot/perf_det.py b/doc/plot/perf_det.py
new file mode 100644
index 0000000000000000000000000000000000000000..b842c51dad886fc7a1b40eb83b8aafe0eb4e8971
--- /dev/null
+++ b/doc/plot/perf_det.py
@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+"""Tutorial for plotting a DET curve
+"""
+
+import bob
+import numpy
+from matplotlib import pyplot
+
+positives = numpy.random.normal(1,1,100)
+negatives = numpy.random.normal(-1,1,100)
+
+npoints = 100
+bob.measure.plot.det(negatives, positives, npoints, color=(0,0,0), linestyle='-', label='test')
+bob.measure.plot.det_axis([0.1, 80, 0.1, 80])
+pyplot.grid(True)
+pyplot.xlabel('FAR (%)')
+pyplot.ylabel('FRR (%)')
+pyplot.title('DET')
diff --git a/doc/plot/perf_epc.py b/doc/plot/perf_epc.py
new file mode 100644
index 0000000000000000000000000000000000000000..df3729dc4733ccfbd343bb29a49ed3bfa119fe72
--- /dev/null
+++ b/doc/plot/perf_epc.py
@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+"""Tutorial for plotting an EPC curve
+"""
+
+import bob
+import numpy
+from matplotlib import pyplot
+
+dev_pos = numpy.random.normal(1,1,100)
+dev_neg = numpy.random.normal(-1,1,100)
+test_pos = numpy.random.normal(0.9,1,100)
+test_neg = numpy.random.normal(-1.1,1,100)
+npoints = 100
+bob.measure.plot.epc(dev_neg, dev_pos, test_neg, test_pos, npoints, color=(0,0,0), linestyle='-')
+pyplot.grid(True)
+pyplot.title('EPC')
diff --git a/doc/plot/perf_roc.py b/doc/plot/perf_roc.py
new file mode 100644
index 0000000000000000000000000000000000000000..b708d290e1434227e53205c158b4c4aaf0c9ea7a
--- /dev/null
+++ b/doc/plot/perf_roc.py
@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+"""Tutorial for plotting a ROC curve
+"""
+
+import bob
+import numpy
+from matplotlib import pyplot
+
+positives = numpy.random.normal(1,1,100)
+negatives = numpy.random.normal(-1,1,100)
+npoints = 100
+bob.measure.plot.roc(negatives, positives, npoints, color=(0,0,0), linestyle='-', label='test')
+pyplot.grid(True)
+pyplot.xlabel('FAR (%)')
+pyplot.ylabel('FRR (%)')
+pyplot.title('ROC')
diff --git a/doc/py_api.rst b/doc/py_api.rst
index 701f2e15cf7ba681ca32ac88b7a92ece0f19fc37..c8039163e0539f5ea0ceff34ca5cd43beb7f73d0 100644
--- a/doc/py_api.rst
+++ b/doc/py_api.rst
@@ -1,6 +1,6 @@
.. vim: set fileencoding=utf-8 :
.. Andre Anjos <andre.dos.anjos@gmail.com>
-.. Sat 16 Nov 20:52:58 2013
+.. Sat 16 Nov 20:52:58 2013
============
Python API
@@ -9,37 +9,5 @@
This section includes information for using the pure Python API of ``xbob.io``.
-Classes
--------
+.. automodule:: xbob.io
-.. autoclass:: xbob.io.File
-
-.. autoclass:: xbob.io.HDF5File
-
-.. autoclass:: xbob.io.VideoReader
-
-.. autoclass:: xbob.io.VideoWriter
-
-
-Functions
----------
-
-.. autofunction:: xbob.io.load
-
-.. autofunction:: xbob.io.merge
-
-.. autofunction:: xbob.io.save
-
-.. autofunction:: xbob.io.append
-
-.. autofunction:: xbob.io.peek
-
-.. autofunction:: xbob.io.peek_all
-
-.. autofunction:: xbob.io.create_directories_save
-
-
-C++ API Helpers
----------------
-
-.. autofunction:: xbob.io.get_include
diff --git a/doc/video.rst b/doc/video.rst
deleted file mode 100644
index ab72d859bed48b9d04d390edb9cf26ea6edade57..0000000000000000000000000000000000000000
--- a/doc/video.rst
+++ /dev/null
@@ -1,568 +0,0 @@
-.. vim: set fileencoding=utf-8 :
-.. Andre Anjos <andre.anjos@idiap.ch>
-.. Wed 20 Mar 2013 11:30:02 CET
-..
-.. Copyright (C) 2011-2013 Idiap Research Institute, Martigny, Switzerland
-..
-.. This program is free software: you can redistribute it and/or modify
-.. it under the terms of the GNU General Public License as published by
-.. the Free Software Foundation, version 3 of the License.
-..
-.. This program is distributed in the hope that it will be useful,
-.. but WITHOUT ANY WARRANTY; without even the implied warranty of
-.. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-.. GNU General Public License for more details.
-..
-.. You should have received a copy of the GNU General Public License
-.. along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-=============================
- Using Videos with |project|
-=============================
-
-Video read and write support in |project| uses FFmpeg_ as implementation
-backend. In Ubuntu-based distributions, FFmpeg_ was replaced by libav_, which
-is a fork based on FFmpeg_ version 0.8. |project| can detect and use libav_
-when FFmpeg_ is not available on the machine. We currently support a variety of
-FFmpeg_ (and libav_) releases, ranging from FFmpeg_ 0.5 until the most recent
-branches.
-
-FFmpeg_ (and libav_) provide a (reasonably) uniform API for reading and writing
-data into a variety of video container formats, using different video and audio
-codecs. |project| leverages on this API to propose a sub-range of formats and
-codecs that work well together, with low distortion patterns and accross
-platforms.
-
-.. note::
-
- As much as we strive to make video formats and codecs available to all
- platforms in which |project| is available, codecs, in particular may be
- disabled by compilation options on FFmpeg_ or libav_, in which case |project|
- builds will not be able to use them.
-
-.. note::
-
- Currently, |project| does not support reading or writing of audio streams on
- video data - only images.
-
-This section provides guidance in choosing a set of formats and codecs for your
-project, so you will be able to leverage the maximum from |project|.
-
-Codec and Format Availability
------------------------------
-
-To get a list of all FFmpeg_ (or libav_) supported formats for a given build of
-|project|, use the ``bob_video_test.py`` application:
-
-.. code-block:: sh
-
- $ bob_video_test.py --list-all-codecs # lists all codecs available
-
- $ bob_video_test.py --list-all-formats # lists all formats available
-
-These listings represent all that is compiled with your current installation of
-FFmpeg_ or libav_. To list supported formats and codecs by |project| use
-another set of command-line options:
-
-.. code-block:: sh
-
- $ bob_video_test.py --list-codecs # lists all codecs currently supported
-
- $ bob_video_test.py --list-formats # lists all formats currently supported
-
-The program ``bob_video_test.py`` can be used to run a sequence of tests using
-all combinations of *supported* formats and tests:
-
-.. code-block:: sh
-
- $ bob_video_test.py # runs all tests
-
-This will run through all combinations of supported codecs and formats and will
-report average distortion figures for each of 4 different tests, which exercise
-different aspects of each combination of format and codec. Here is a an example
-output:
-
-.. code-block:: text
-
- Video Encoding/Decoding Test Tool v1.2.0a0 (bob_video_test)
- Settings:
- Width : 128 pixels
- Height : 128 pixels
- Length : 30 frames
- Framerate: 30.000000 Hz
- Legend:
- C: Color test
- N: Noise test
- U: User test
- S: Frameskip test
- Running 4 test(s)...CSNU
-
- =========== ===== ======= =======================================
- test fmt codec figure (lower means better quality)
- =========== ===== ======= =======================================
- color mov h264 4.603 min=0.890@22 max=8.387@9
- frameskip mov h264 0.108 min=0.009@11 max=0.344@0
- noise mov h264 44.900 min=43.916@4 max=46.103@29
- user mov h264 1.983 min=1.525@0 max=2.286@7
- =========== ===== ======= =======================================
-
-Each line in the output table represents the average distortion patterns for
-the particular test using the format and codec described. The lower the
-distortion, the better the combination of format and codecs is. Different tests
-have different levels of baseline performance. The figures above were obtained
-in a healthy (no know bugs) system, running libav_ 0.8.13 (Ubuntu 12.10). Each
-line indicates, besides the average distortion per frame, the minimum and the
-maximum obtained and in which frame (counting from 0 - zero), that figure was
-obtained.
-
-The video tests are made on temporary files that are discarded after the test
-is completed. You can use the option ``--output=<directory>`` to specify a
-directory in which the generated files will be saved. You can then go to these
-directories and explore potential problems you may find.
-
-Each test creates a video from an artificially generated test signal, encodes
-it using the defined format and codec and reads it back, comparing the output
-result with the original sequence. The sole exception is the ``user`` test. In
-this test, a user test sequence is (partially) loaded and tested. If you don't
-specify any sequence, a default sequence from |project| is used. If you want to
-test a specific sequence of your own, use ``--user-video`` to specify the path
-of the video sequence you want to test with. By default, only the first 10
-frames of the sequence are used for the test, to speed-up execution. You can
-change this behavior with the option ``--user-frames``. Here is an example:
-
-.. code-block:: sh
-
- $ bob_video_test.py --user-video=test_sample.avi
-
-All tests are executed by default, on all combination of formats and codecs.
-That can be long. You can limit the test execution by properly choosing the
-format (``--format``), the codec (``--codec``) and the tests to execute. For
-example:
-
-.. code-block:: sh
-
- # execute only the user video test with a user provided video and
- # using the H.264 built-in codec and a MOV output file format.
- $ xbob_video_test.py --format mov --codec h264 --user-video=test_sample.avi -- user
-
-.. note::
-
- Not all codecs can be used by all formats available. For example, the ``mp4``
- file format cannot use the ``vp8`` codec, even if both are supported by
- |project|. To know which formats support each codec, you can execute the
- following python code:
-
- .. code-block:: python
-
- import xbob.io
- xbob.io.supported_videowriter_formats()['mp4']['supported_codecs'].keys()
- ['h264', 'libx264', 'mjpeg', 'mpeg1video']
-
- You can see from the output command that only 4 codecs are supported by the
- file format ``mp4``.
-
-You can test new combinations of formats and codecs which are not currently
-supported by |project|, as long as they are supported by the underlying FFmpeg_
-or libav_ installations. In this case, just specify the format and/or codec
-names using ``--format`` and ``--codec`` options in the application
-``xbob_video_test.py``. The advantage of using *supported* formats and codecs is
-that we make sure a minimal distortion figure is respected in all platform
-nightly builds, with our unit and integration tests. We cannot, currently,
-test all possible combinations of codecs and formats.
-
-Know Your Platforms
--------------------
-
-One important aspect when working with videos is to know there will be some
-lossy compression applied to the output. This means you will **loose**
-information when re-encoding. When working with videos, you will want to choose
-the combination of format and codec that will work well accross different
-platforms. We recommend to run ``xbob_video_test.py`` with a few of your video
-inputs to make sure they can be decoded with low distortion where you plan to
-work.
-
-.. note::
-
- The only codec that supports lossless compression in |project| is ``zlib``.
- Of course, the output files are considerably bigger, but they continue to be
- readable using any FFmpeg_-based movie player or even QuickTime (on OSX), if
- Perian is installed.
-
-Example Output in Different Platforms
--------------------------------------
-
-In what follows, you will find some tabbed output for different combinations of
-operating systems and FFmpeg_/libav_ versions. To run these tests we only
-executed:
-
-.. code-block:: sh
-
- $ xbob_video_test.py
-
-Idiap Linux (Xubuntu), version 12.10 + libav 0.8.3
-==================================================
-
-=========== ===== ============ ================================================
- test fmt codec figure (lower means better quality)
-=========== ===== ============ ================================================
- color avi ffv1 4.569 min=0.888\@22 max=8.377\@9
- color avi h264 4.603 min=0.890\@22 max=8.388\@9
- color avi libvpx 4.657 min=0.955\@26 max=8.528\@9
- color avi libx264 4.603 min=0.890\@22 max=8.388\@9
- color avi mjpeg 4.676 min=0.965\@22 max=8.469\@9
- color avi mpeg1video 4.781 min=1.103\@28 max=8.483\@9
- color avi mpeg2video 4.741 min=1.004\@16 max=8.466\@9
- color avi mpeg4 4.892 min=1.087\@24 max=8.658\@9
- color avi msmpeg4 4.921 min=1.073\@24 max=8.717\@9
- color avi msmpeg4v2 4.921 min=1.073\@24 max=9.181\@17
- color avi vp8 4.657 min=0.955\@26 max=8.528\@9
- color avi wmv1 4.871 min=1.087\@24 max=8.729\@9
- color avi wmv2 4.884 min=1.093\@24 max=8.823\@9
- color avi zlib 0.000 min=0.000\@0 max=0.000\@0
- color mov ffv1 4.569 min=0.888\@22 max=8.377\@9
- color mov h264 4.603 min=0.890\@22 max=8.387\@9
- color mov libvpx 4.657 min=0.955\@26 max=8.528\@9
- color mov libx264 4.603 min=0.890\@22 max=8.387\@9
- color mov mjpeg 4.676 min=0.965\@22 max=8.469\@9
- color mov mpeg1video 4.781 min=1.103\@28 max=8.483\@9
- color mov mpeg2video 4.741 min=1.004\@16 max=8.466\@9
- color mov mpeg4 4.892 min=1.087\@24 max=8.658\@9
- color mov msmpeg4 4.921 min=1.073\@24 max=8.717\@9
- color mov msmpeg4v2 4.921 min=1.073\@24 max=9.181\@17
- color mov vp8 4.657 min=0.955\@26 max=8.528\@9
- color mov wmv1 4.871 min=1.087\@24 max=8.729\@9
- color mov wmv2 4.884 min=1.093\@24 max=8.823\@9
- color mov zlib 0.000 min=0.000\@0 max=0.000\@0
- color mp4 ffv1 format+codec unsupported
- color mp4 h264 4.603 min=0.890\@22 max=8.387\@9
- color mp4 libvpx format+codec unsupported
- color mp4 libx264 4.603 min=0.890\@22 max=8.387\@9
- color mp4 mjpeg 4.676 min=0.965\@22 max=8.469\@9
- color mp4 mpeg1video 4.781 min=1.103\@28 max=8.483\@9
- color mp4 mpeg2video 4.741 min=1.004\@16 max=8.466\@9
- color mp4 mpeg4 4.892 min=1.087\@24 max=8.658\@9
- color mp4 msmpeg4 format+codec unsupported
- color mp4 msmpeg4v2 format+codec unsupported
- color mp4 vp8 format+codec unsupported
- color mp4 wmv1 format+codec unsupported
- color mp4 wmv2 format+codec unsupported
- color mp4 zlib format+codec unsupported
- frameskip avi ffv1 0.018 min=0.002\@11 max=0.029\@8
- frameskip avi h264 0.108 min=0.009\@11 max=0.344\@0
- frameskip avi libvpx 0.129 min=0.042\@11 max=0.198\@8
- frameskip avi libx264 0.108 min=0.009\@11 max=0.344\@0
- frameskip avi mjpeg 0.380 min=0.141\@11 max=1.108\@0
- frameskip avi mpeg1video 0.426 min=0.237\@17 max=1.338\@0
- frameskip avi mpeg2video 0.411 min=0.223\@15 max=1.284\@0
- frameskip avi mpeg4 0.454 min=0.263\@17 max=0.858\@0
- frameskip avi msmpeg4 1.684 min=0.257\@12 max=3.766\@15
- frameskip avi msmpeg4v2 1.683 min=0.257\@12 max=3.765\@15
- frameskip avi vp8 0.129 min=0.042\@11 max=0.198\@8
- frameskip avi wmv1 0.627 min=0.191\@11 max=1.568\@8
- frameskip avi wmv2 0.626 min=0.191\@11 max=1.566\@8
- frameskip avi zlib 0.000 min=0.000\@0 max=0.000\@0
- frameskip mov ffv1 0.018 min=0.002\@11 max=0.029\@8
- frameskip mov h264 0.108 min=0.009\@11 max=0.344\@0
- frameskip mov libvpx 0.129 min=0.042\@11 max=0.198\@8
- frameskip mov libx264 0.108 min=0.009\@11 max=0.344\@0
- frameskip mov mjpeg 0.380 min=0.141\@11 max=1.108\@0
- frameskip mov mpeg1video 0.426 min=0.237\@17 max=1.338\@0
- frameskip mov mpeg2video 0.411 min=0.223\@15 max=1.284\@0
- frameskip mov mpeg4 0.454 min=0.263\@17 max=0.858\@0
- frameskip mov msmpeg4 1.684 min=0.257\@12 max=3.766\@15
- frameskip mov msmpeg4v2 1.683 min=0.257\@12 max=3.765\@15
- frameskip mov vp8 0.129 min=0.042\@11 max=0.198\@8
- frameskip mov wmv1 0.627 min=0.191\@11 max=1.568\@8
- frameskip mov wmv2 0.626 min=0.191\@11 max=1.566\@8
- frameskip mov zlib 0.000 min=0.000\@0 max=0.000\@0
- frameskip mp4 ffv1 format+codec unsupported
- frameskip mp4 h264 0.108 min=0.009\@11 max=0.344\@0
- frameskip mp4 libvpx format+codec unsupported
- frameskip mp4 libx264 0.108 min=0.009\@11 max=0.344\@0
- frameskip mp4 mjpeg 0.380 min=0.141\@11 max=1.108\@0
- frameskip mp4 mpeg1video 0.426 min=0.237\@17 max=1.338\@0
- frameskip mp4 mpeg2video 0.411 min=0.223\@15 max=1.284\@0
- frameskip mp4 mpeg4 0.454 min=0.263\@17 max=0.858\@0
- frameskip mp4 msmpeg4 format+codec unsupported
- frameskip mp4 msmpeg4v2 format+codec unsupported
- frameskip mp4 vp8 format+codec unsupported
- frameskip mp4 wmv1 format+codec unsupported
- frameskip mp4 wmv2 format+codec unsupported
- frameskip mp4 zlib format+codec unsupported
- noise avi ffv1 44.192 min=43.887\@0 max=44.568\@8
- noise avi h264 44.882 min=43.738\@2 max=45.848\@27
- noise avi libvpx 48.629 min=44.156\@12 max=54.365\@27
- noise avi libx264 44.883 min=44.089\@2 max=45.857\@29
- noise avi mjpeg 45.723 min=43.942\@3 max=48.283\@28
- noise avi mpeg1video 46.270 min=44.412\@2 max=48.632\@29
- noise avi mpeg2video 45.227 min=44.008\@5 max=48.528\@29
- noise avi mpeg4 45.769 min=43.720\@4 max=48.472\@27
- noise avi msmpeg4 45.757 min=44.034\@7 max=48.055\@24
- noise avi msmpeg4v2 45.789 min=43.908\@6 max=48.423\@27
- noise avi vp8 48.796 min=43.765\@0 max=50.864\@15
- noise avi wmv1 45.729 min=43.878\@6 max=47.921\@29
- noise avi wmv2 46.105 min=44.205\@3 max=48.261\@28
- noise avi zlib 0.000 min=0.000\@0 max=0.000\@0
- noise mov ffv1 44.200 min=43.869\@20 max=44.719\@22
- noise mov h264 44.882 min=43.991\@6 max=46.183\@27
- noise mov libvpx 48.692 min=43.934\@0 max=50.906\@15
- noise mov libx264 44.909 min=43.773\@3 max=46.079\@29
- noise mov mjpeg 45.754 min=43.823\@8 max=48.278\@28
- noise mov mpeg1video 46.353 min=44.326\@1 max=48.712\@29
- noise mov mpeg2video 45.970 min=43.952\@4 max=50.645\@29
- noise mov mpeg4 45.772 min=43.961\@4 max=48.414\@28
- noise mov msmpeg4 45.764 min=43.867\@5 max=48.156\@29
- noise mov msmpeg4v2 45.844 min=44.009\@6 max=48.317\@27
- noise mov vp8 48.323 min=43.985\@12 max=50.512\@19
- noise mov wmv1 45.803 min=44.109\@3 max=48.334\@29
- noise mov wmv2 46.081 min=43.950\@4 max=48.293\@26
- noise mov zlib 0.000 min=0.000\@0 max=0.000\@0
- noise mp4 ffv1 format+codec unsupported
- noise mp4 h264 44.856 min=43.749\@1 max=46.045\@27
- noise mp4 libvpx format+codec unsupported
- noise mp4 libx264 44.785 min=43.820\@0 max=46.093\@28
- noise mp4 mjpeg 45.725 min=43.979\@7 max=48.208\@28
- noise mp4 mpeg1video 46.227 min=44.144\@2 max=48.241\@27
- noise mp4 mpeg2video 46.060 min=43.991\@5 max=51.358\@29
- noise mp4 mpeg4 45.690 min=44.072\@6 max=47.974\@28
- noise mp4 msmpeg4 format+codec unsupported
- noise mp4 msmpeg4v2 format+codec unsupported
- noise mp4 vp8 format+codec unsupported
- noise mp4 wmv1 format+codec unsupported
- noise mp4 wmv2 format+codec unsupported
- noise mp4 zlib format+codec unsupported
- user avi ffv1 1.174 min=1.166\@2 max=1.187\@7
- user avi h264 1.988 min=1.525\@0 max=2.290\@7
- user avi libvpx 1.614 min=1.464\@0 max=1.711\@8
- user avi libx264 1.988 min=1.525\@0 max=2.290\@7
- user avi mjpeg 1.067 min=1.014\@2 max=1.444\@0
- user avi mpeg1video 1.586 min=1.447\@1 max=1.895\@0
- user avi mpeg2video 1.743 min=1.515\@3 max=1.891\@8
- user avi mpeg4 1.794 min=1.606\@1 max=1.906\@9
- user avi msmpeg4 1.802 min=1.599\@1 max=1.925\@8
- user avi msmpeg4v2 1.775 min=1.599\@1 max=1.868\@9
- user avi vp8 1.614 min=1.464\@0 max=1.711\@8
- user avi wmv1 1.802 min=1.599\@1 max=1.925\@8
- user avi wmv2 1.799 min=1.596\@1 max=1.921\@8
- user avi zlib 0.000 min=0.000\@0 max=0.000\@0
- user mov ffv1 1.174 min=1.166\@2 max=1.187\@7
- user mov h264 1.983 min=1.525\@0 max=2.286\@7
- user mov libvpx 1.614 min=1.464\@0 max=1.711\@8
- user mov libx264 1.983 min=1.525\@0 max=2.286\@7
- user mov mjpeg 1.067 min=1.014\@2 max=1.444\@0
- user mov mpeg1video 1.586 min=1.447\@1 max=1.895\@0
- user mov mpeg2video 1.743 min=1.515\@3 max=1.891\@8
- user mov mpeg4 1.794 min=1.606\@1 max=1.906\@9
- user mov msmpeg4 1.802 min=1.599\@1 max=1.925\@8
- user mov msmpeg4v2 1.775 min=1.599\@1 max=1.868\@9
- user mov vp8 1.614 min=1.464\@0 max=1.711\@8
- user mov wmv1 1.802 min=1.599\@1 max=1.925\@8
- user mov wmv2 1.799 min=1.596\@1 max=1.921\@8
- user mov zlib 0.000 min=0.000\@0 max=0.000\@0
- user mp4 ffv1 format+codec unsupported
- user mp4 h264 1.983 min=1.525\@0 max=2.286\@7
- user mp4 libvpx format+codec unsupported
- user mp4 libx264 1.983 min=1.525\@0 max=2.286\@7
- user mp4 mjpeg 1.067 min=1.014\@2 max=1.444\@0
- user mp4 mpeg1video 1.586 min=1.447\@1 max=1.895\@0
- user mp4 mpeg2video 1.743 min=1.515\@3 max=1.891\@8
- user mp4 mpeg4 1.794 min=1.606\@1 max=1.906\@9
- user mp4 msmpeg4 format+codec unsupported
- user mp4 msmpeg4v2 format+codec unsupported
- user mp4 vp8 format+codec unsupported
- user mp4 wmv1 format+codec unsupported
- user mp4 wmv2 format+codec unsupported
- user mp4 zlib format+codec unsupported
-=========== ===== ============ ================================================
-
-MacOSX 10.8.3 + FFmpeg 1.1.2
-============================
-
-=========== ===== ================== ========================================
- test fmt codec figure (lower is better quality)
-=========== ===== ================== ========================================
- color avi ffv1 4.643 min=0.999\@24 max=8.420\@9
- color avi h264 4.685 min=1.001\@24 max=8.473\@9
- color avi libvpx 4.736 min=1.079\@26 max=8.503\@9
- color avi libx264 4.685 min=1.001\@24 max=8.473\@9
- color avi mjpeg 4.617 min=0.934\@24 max=8.440\@9
- color avi mpeg1video 4.820 min=1.125\@16 max=8.548\@9
- color avi mpeg2video 4.787 min=1.130\@16 max=8.465\@9
- color avi mpeg4 4.956 min=1.129\@24 max=8.725\@9
- color avi mpegvideo 4.787 min=1.130\@16 max=8.465\@9
- color avi msmpeg4 4.987 min=1.114\@24 max=8.731\@9
- color avi msmpeg4v2 4.949 min=1.114\@24 max=8.667\@9
- color avi vp8 4.736 min=1.079\@26 max=8.503\@9
- color avi wmv1 4.925 min=1.129\@24 max=8.728\@9
- color avi wmv2 4.936 min=1.138\@24 max=8.796\@9
- color avi zlib 0.000 min=0.000\@0 max=0.000\@0
- color mov ffv1 4.643 min=0.999\@24 max=8.420\@9
- color mov h264 4.645 min=1.001\@24 max=8.424\@9
- color mov libvpx 4.736 min=1.079\@26 max=8.503\@9
- color mov libx264 4.645 min=1.001\@24 max=8.424\@9
- color mov mjpeg 4.617 min=0.934\@24 max=8.440\@9
- color mov mpeg1video 4.820 min=1.125\@16 max=8.548\@9
- color mov mpeg2video 4.787 min=1.130\@16 max=8.465\@9
- color mov mpeg4 4.956 min=1.129\@24 max=8.725\@9
- color mov mpegvideo 4.787 min=1.130\@16 max=8.465\@9
- color mov msmpeg4 4.987 min=1.114\@24 max=8.731\@9
- color mov msmpeg4v2 4.949 min=1.114\@24 max=8.667\@9
- color mov vp8 4.736 min=1.079\@26 max=8.503\@9
- color mov wmv1 4.925 min=1.129\@24 max=8.728\@9
- color mov wmv2 4.936 min=1.138\@24 max=8.796\@9
- color mov zlib 0.000 min=0.000\@0 max=0.000\@0
- color mp4 ffv1 format+codec unsupported
- color mp4 h264 4.645 min=1.001\@24 max=8.424\@9
- color mp4 libvpx format+codec unsupported
- color mp4 libx264 4.645 min=1.001\@24 max=8.424\@9
- color mp4 mjpeg 4.617 min=0.934\@24 max=8.440\@9
- color mp4 mpeg1video 4.820 min=1.125\@16 max=8.548\@9
- color mp4 mpeg2video 4.787 min=1.130\@16 max=8.465\@9
- color mp4 mpeg4 4.956 min=1.129\@24 max=8.725\@9
- color mp4 mpegvideo 4.787 min=1.130\@16 max=8.465\@9
- color mp4 msmpeg4 format+codec unsupported
- color mp4 msmpeg4v2 format+codec unsupported
- color mp4 vp8 format+codec unsupported
- color mp4 wmv1 format+codec unsupported
- color mp4 wmv2 format+codec unsupported
- color mp4 zlib format+codec unsupported
- frameskip avi ffv1 0.018 min=0.002\@11 max=0.029\@8
- frameskip avi h264 0.120 min=0.017\@21 max=0.300\@0
- frameskip avi libvpx 0.122 min=0.051\@11 max=0.181\@0
- frameskip avi libx264 0.120 min=0.017\@21 max=0.300\@0
- frameskip avi mjpeg 0.386 min=0.147\@11 max=1.085\@0
- frameskip avi mpeg1video 0.427 min=0.243\@11 max=1.310\@0
- frameskip avi mpeg2video 0.408 min=0.229\@17 max=1.258\@0
- frameskip avi mpeg4 0.456 min=0.253\@12 max=0.849\@0
- frameskip avi mpegvideo 0.408 min=0.229\@17 max=1.258\@0
- frameskip avi msmpeg4 1.608 min=0.434\@24 max=3.409\@25
- frameskip avi msmpeg4v2 1.600 min=0.434\@24 max=3.708\@5
- frameskip avi vp8 0.122 min=0.051\@11 max=0.181\@0
- frameskip avi wmv1 0.617 min=0.191\@11 max=1.981\@8
- frameskip avi wmv2 0.614 min=0.191\@11 max=1.978\@8
- frameskip avi zlib 0.000 min=0.000\@0 max=0.000\@0
- frameskip mov ffv1 0.018 min=0.002\@11 max=0.029\@8
- frameskip mov h264 0.042 min=0.011\@11 max=0.085\@9
- frameskip mov libvpx 0.122 min=0.051\@11 max=0.181\@0
- frameskip mov libx264 0.042 min=0.011\@11 max=0.085\@9
- frameskip mov mjpeg 0.386 min=0.147\@11 max=1.085\@0
- frameskip mov mpeg1video 0.427 min=0.243\@11 max=1.310\@0
- frameskip mov mpeg2video 0.408 min=0.229\@17 max=1.258\@0
- frameskip mov mpeg4 0.456 min=0.253\@12 max=0.849\@0
- frameskip mov mpegvideo 0.408 min=0.229\@17 max=1.258\@0
- frameskip mov msmpeg4 1.608 min=0.434\@24 max=3.409\@25
- frameskip mov msmpeg4v2 1.600 min=0.434\@24 max=3.708\@5
- frameskip mov vp8 0.122 min=0.051\@11 max=0.181\@0
- frameskip mov wmv1 0.617 min=0.191\@11 max=1.981\@8
- frameskip mov wmv2 0.614 min=0.191\@11 max=1.978\@8
- frameskip mov zlib 0.000 min=0.000\@0 max=0.000\@0
- frameskip mp4 ffv1 format+codec unsupported
- frameskip mp4 h264 0.042 min=0.011\@11 max=0.085\@9
- frameskip mp4 libvpx format+codec unsupported
- frameskip mp4 libx264 0.042 min=0.011\@11 max=0.085\@9
- frameskip mp4 mjpeg 0.386 min=0.147\@11 max=1.085\@0
- frameskip mp4 mpeg1video 0.427 min=0.243\@11 max=1.310\@0
- frameskip mp4 mpeg2video 0.408 min=0.229\@17 max=1.258\@0
- frameskip mp4 mpeg4 0.456 min=0.253\@12 max=0.849\@0
- frameskip mp4 mpegvideo 0.408 min=0.229\@17 max=1.258\@0
- frameskip mp4 msmpeg4 format+codec unsupported
- frameskip mp4 msmpeg4v2 format+codec unsupported
- frameskip mp4 vp8 format+codec unsupported
- frameskip mp4 wmv1 format+codec unsupported
- frameskip mp4 wmv2 format+codec unsupported
- frameskip mp4 zlib format+codec unsupported
- noise avi ffv1 44.108 min=43.717\@16 max=44.565\@22
- noise avi h264 44.509 min=43.859\@4 max=45.146\@27
- noise avi libvpx 46.882 min=43.812\@1 max=49.422\@18
- noise avi libx264 44.572 min=43.917\@5 max=45.236\@29
- noise avi mjpeg 45.739 min=43.819\@4 max=48.211\@29
- noise avi mpeg1video 46.320 min=44.273\@3 max=48.996\@29
- noise avi mpeg2video 46.054 min=43.987\@7 max=51.580\@29
- noise avi mpeg4 45.755 min=44.071\@2 max=48.502\@28
- noise avi mpegvideo 44.951 min=43.775\@1 max=46.796\@24
- noise avi msmpeg4 45.749 min=43.934\@5 max=48.267\@29
- noise avi msmpeg4v2 45.846 min=43.987\@0 max=48.264\@27
- noise avi vp8 46.457 min=43.931\@12 max=48.857\@27
- noise avi wmv1 45.804 min=44.219\@10 max=48.252\@28
- noise avi wmv2 46.091 min=44.113\@3 max=48.380\@25
- noise avi zlib 0.000 min=0.000\@0 max=0.000\@0
- noise mov ffv1 44.128 min=43.657\@15 max=44.513\@21
- noise mov h264 44.168 min=43.794\@24 max=44.577\@7
- noise mov libvpx 47.009 min=44.127\@4 max=49.547\@17
- noise mov libx264 44.143 min=43.813\@23 max=44.529\@16
- noise mov mjpeg 44.378 min=44.020\@18 max=44.670\@0
- noise mov mpeg1video 44.564 min=43.903\@9 max=45.314\@0
- noise mov mpeg2video 44.340 min=44.021\@26 max=44.733\@0
- noise mov mpeg4 44.338 min=43.923\@3 max=44.677\@11
- noise mov mpegvideo 44.343 min=43.978\@8 max=44.904\@29
- noise mov msmpeg4 44.293 min=43.870\@9 max=44.669\@24
- noise mov msmpeg4v2 44.256 min=43.859\@5 max=44.596\@21
- noise mov vp8 47.558 min=43.955\@0 max=52.720\@25
- noise mov wmv1 44.283 min=43.848\@24 max=44.643\@14
- noise mov wmv2 44.323 min=43.957\@10 max=44.727\@0
- noise mov zlib 0.000 min=0.000\@0 max=0.000\@0
- noise mp4 ffv1 format+codec unsupported
- noise mp4 h264 44.118 min=43.717\@18 max=44.439\@1
- noise mp4 libvpx format+codec unsupported
- noise mp4 libx264 44.218 min=43.870\@8 max=44.730\@19
- noise mp4 mjpeg 44.374 min=44.061\@2 max=44.902\@0
- noise mp4 mpeg1video 44.537 min=44.157\@18 max=45.222\@0
- noise mp4 mpeg2video 44.397 min=43.834\@5 max=44.825\@0
- noise mp4 mpeg4 44.276 min=43.875\@9 max=44.912\@17
- noise mp4 mpegvideo 44.339 min=43.812\@2 max=45.328\@0
- noise mp4 msmpeg4 format+codec unsupported
- noise mp4 msmpeg4v2 format+codec unsupported
- noise mp4 vp8 format+codec unsupported
- noise mp4 wmv1 format+codec unsupported
- noise mp4 wmv2 format+codec unsupported
- noise mp4 zlib format+codec unsupported
- user avi ffv1 1.463 min=1.457\@5 max=1.472\@7
- user avi h264 2.028 min=1.666\@0 max=2.201\@9
- user avi libvpx 1.999 min=1.646\@0 max=2.420\@2
- user avi libx264 2.028 min=1.666\@0 max=2.201\@9
- user avi mjpeg 1.197 min=1.149\@6 max=1.532\@0
- user avi mpeg1video 1.760 min=1.641\@1 max=2.061\@0
- user avi mpeg2video 1.882 min=1.694\@3 max=2.026\@0
- user avi mpeg4 1.960 min=1.782\@1 max=2.076\@9
- user avi mpegvideo 1.882 min=1.694\@3 max=2.026\@0
- user avi msmpeg4 1.964 min=1.773\@1 max=2.088\@8
- user avi msmpeg4v2 1.921 min=1.773\@1 max=2.008\@9
- user avi vp8 1.999 min=1.646\@0 max=2.420\@2
- user avi wmv1 1.964 min=1.773\@1 max=2.088\@8
- user avi wmv2 1.958 min=1.768\@1 max=2.082\@8
- user avi zlib 0.000 min=0.000\@0 max=0.000\@0
- user mov ffv1 1.463 min=1.457\@5 max=1.472\@7
- user mov h264 1.533 min=1.477\@0 max=1.566\@7
- user mov libvpx 2.103 min=1.646\@0 max=2.547\@2
- user mov libx264 1.533 min=1.477\@0 max=1.566\@7
- user mov mjpeg 1.197 min=1.149\@6 max=1.532\@0
- user mov mpeg1video 1.760 min=1.641\@1 max=2.061\@0
- user mov mpeg2video 1.882 min=1.694\@3 max=2.026\@0
- user mov mpeg4 1.960 min=1.782\@1 max=2.076\@9
- user mov mpegvideo 1.882 min=1.694\@3 max=2.026\@0
- user mov msmpeg4 1.964 min=1.773\@1 max=2.088\@8
- user mov msmpeg4v2 1.921 min=1.773\@1 max=2.008\@9
- user mov vp8 2.103 min=1.646\@0 max=2.547\@2
- user mov wmv1 1.964 min=1.773\@1 max=2.088\@8
- user mov wmv2 1.958 min=1.768\@1 max=2.082\@8
- user mov zlib 0.000 min=0.000\@0 max=0.000\@0
- user mp4 ffv1 format+codec unsupported
- user mp4 h264 1.533 min=1.477\@0 max=1.566\@7
- user mp4 libvpx format+codec unsupported
- user mp4 libx264 1.533 min=1.477\@0 max=1.566\@7
- user mp4 mjpeg 1.197 min=1.149\@6 max=1.532\@0
- user mp4 mpeg1video 1.760 min=1.641\@1 max=2.061\@0
- user mp4 mpeg2video 1.882 min=1.694\@3 max=2.026\@0
- user mp4 mpeg4 1.960 min=1.782\@1 max=2.076\@9
- user mp4 mpegvideo 1.882 min=1.694\@3 max=2.026\@0
- user mp4 msmpeg4 format+codec unsupported
- user mp4 msmpeg4v2 format+codec unsupported
- user mp4 vp8 format+codec unsupported
- user mp4 wmv1 format+codec unsupported
- user mp4 wmv2 format+codec unsupported
- user mp4 zlib format+codec unsupported
-=========== ===== ================== ========================================
-
-.. include:: links.rst
-
-.. Place here your external references
diff --git a/xbob/measure/main.cpp b/xbob/measure/main.cpp
index 5608e83d65b0afbfc200e3f1cb3f6d6d05245e7a..e1cf79d968f77fc686897e058284574cb38f0f15 100644
--- a/xbob/measure/main.cpp
+++ b/xbob/measure/main.cpp
@@ -53,9 +53,9 @@ static PyObject* epc(PyObject*, PyObject* args, PyObject* kwds) {
/* Parses input arguments in a single shot */
static const char* const_kwlist[] = {
- "dev_negatives",
- "dev_positives",
- "test_positives",
+ "dev_negatives",
+ "dev_positives",
+ "test_positives",
"test_negatives",
"n_points",
0 /* Sentinel */
@@ -124,8 +124,8 @@ static PyObject* det(PyObject*, PyObject* args, PyObject* kwds) {
/* Parses input arguments in a single shot */
static const char* const_kwlist[] = {
- "negatives",
- "positives",
+ "negatives",
+ "positives",
"n_points",
0 /* Sentinel */
};
@@ -202,8 +202,8 @@ static PyObject* roc(PyObject*, PyObject* args, PyObject* kwds) {
/* Parses input arguments in a single shot */
static const char* const_kwlist[] = {
- "negatives",
- "positives",
+ "negatives",
+ "positives",
"n_points",
0 /* Sentinel */
};
@@ -278,8 +278,8 @@ static PyObject* farfrr(PyObject*, PyObject* args, PyObject* kwds) {
/* Parses input arguments in a single shot */
static const char* const_kwlist[] = {
- "negatives",
- "positives",
+ "negatives",
+ "positives",
"threshold",
0 /* Sentinel */
};
@@ -327,8 +327,8 @@ static PyObject* eer_threshold(PyObject*, PyObject* args, PyObject* kwds) {
/* Parses input arguments in a single shot */
static const char* const_kwlist[] = {
- "negatives",
- "positives",
+ "negatives",
+ "positives",
0 /* Sentinel */
};
static char** kwlist = const_cast<char**>(const_kwlist);
@@ -372,8 +372,8 @@ static PyObject* min_weighted_error_rate_threshold(PyObject*, PyObject* args, Py
/* Parses input arguments in a single shot */
static const char* const_kwlist[] = {
- "negatives",
- "positives",
+ "negatives",
+ "positives",
"cost",
0 /* Sentinel */
};
@@ -415,8 +415,8 @@ static PyObject* min_hter_threshold(PyObject*, PyObject* args, PyObject* kwds) {
/* Parses input arguments in a single shot */
static const char* const_kwlist[] = {
- "negatives",
- "positives",
+ "negatives",
+ "positives",
0 /* Sentinel */
};
static char** kwlist = const_cast<char**>(const_kwlist);
@@ -460,8 +460,8 @@ static PyObject* precision_recall(PyObject*, PyObject* args, PyObject* kwds) {
/* Parses input arguments in a single shot */
static const char* const_kwlist[] = {
- "negatives",
- "positives",
+ "negatives",
+ "positives",
"threshold",
0 /* Sentinel */
};
@@ -509,8 +509,8 @@ static PyObject* f_score(PyObject*, PyObject* args, PyObject* kwds) {
/* Parses input arguments in a single shot */
static const char* const_kwlist[] = {
- "negatives",
- "positives",
+ "negatives",
+ "positives",
"threshold",
"weight",
0 /* Sentinel */
@@ -557,7 +557,7 @@ static PyObject* correctly_classified_negatives(PyObject*, PyObject* args, PyObj
/* Parses input arguments in a single shot */
static const char* const_kwlist[] = {
- "negatives",
+ "negatives",
"threshold",
0 /* Sentinel */
};
@@ -598,7 +598,7 @@ static PyObject* correctly_classified_positives(PyObject*, PyObject* args, PyObj
/* Parses input arguments in a single shot */
static const char* const_kwlist[] = {
- "positives",
+ "positives",
"threshold",
0 /* Sentinel */
};
@@ -640,8 +640,8 @@ static PyObject* precision_recall_curve(PyObject*, PyObject* args, PyObject* kwd
/* Parses input arguments in a single shot */
static const char* const_kwlist[] = {
- "negatives",
- "positives",
+ "negatives",
+ "positives",
"n_points",
0 /* Sentinel */
};
@@ -696,8 +696,8 @@ static PyObject* far_threshold(PyObject*, PyObject* args, PyObject* kwds) {
/* Parses input arguments in a single shot */
static const char* const_kwlist[] = {
- "negatives",
- "positives",
+ "negatives",
+ "positives",
"threshold",
0 /* Sentinel */
};
@@ -752,8 +752,8 @@ static PyObject* frr_threshold(PyObject*, PyObject* args, PyObject* kwds) {
/* Parses input arguments in a single shot */
static const char* const_kwlist[] = {
- "negatives",
- "positives",
+ "negatives",
+ "positives",
"frr_value",
0 /* Sentinel */
};
@@ -796,8 +796,8 @@ static PyObject* eer_rocch(PyObject*, PyObject* args, PyObject* kwds) {
/* Parses input arguments in a single shot */
static const char* const_kwlist[] = {
- "negatives",
- "positives",
+ "negatives",
+ "positives",
0 /* Sentinel */
};
static char** kwlist = const_cast<char**>(const_kwlist);
@@ -837,8 +837,8 @@ static PyObject* rocch(PyObject*, PyObject* args, PyObject* kwds) {
/* Parses input arguments in a single shot */
static const char* const_kwlist[] = {
- "negatives",
- "positives",
+ "negatives",
+ "positives",
0 /* Sentinel */
};
static char** kwlist = const_cast<char**>(const_kwlist);
@@ -887,7 +887,7 @@ static PyObject* rocch2eer(PyObject*, PyObject* args, PyObject* kwds) {
/* Parses input arguments in a single shot */
static const char* const_kwlist[] = {
- "pmiss_pfa",
+ "pmiss_pfa",
0 /* Sentinel */
};
static char** kwlist = const_cast<char**>(const_kwlist);
@@ -921,8 +921,8 @@ static PyObject* roc_for_far(PyObject*, PyObject* args, PyObject* kwds) {
/* Parses input arguments in a single shot */
static const char* const_kwlist[] = {
- "negatives",
- "positives",
+ "negatives",
+ "positives",
"far_list",
0 /* Sentinel */
};
@@ -1074,7 +1074,7 @@ static PyMethodDef library_methods[] = {
PyMODINIT_FUNC XBOB_EXT_ENTRY_NAME (void) {
- PyObject* m = Py_InitModule3(XBOB_EXT_MODULE_NAME,
+ PyObject* m = Py_InitModule3(XBOB_EXT_MODULE_NAME,
library_methods, "bob::measure bindings");
PyModule_AddStringConstant(m, "__version__", XBOB_EXT_MODULE_VERSION);
diff --git a/xbob/measure/plot.py b/xbob/measure/plot.py
index 990a16384f1c63888fa938671d2b867719892525..121a92019fa85a70e419ae8235e67741c176327f 100644
--- a/xbob/measure/plot.py
+++ b/xbob/measure/plot.py
@@ -59,7 +59,7 @@ def roc(negatives, positives, npoints=100, CAR=False, **kwargs):
if not CAR:
return mpl.plot(100.0*out[0,:], 100.0*out[1,:], **kwargs)
else:
- return mpl.semilogx(100.0*out[1,:], 100.0*(1-out[0,:]), **kwargs)
+ return mpl.semilogx(100.0*out[0,:], 100.0*(1-out[1,:]), **kwargs)
def precision_recall_curve(negatives, positives, npoints=100, **kwargs):
"""Plots Precision-Recall curve.
diff --git a/xbob/measure/script/compute_perf.py b/xbob/measure/script/compute_perf.py
index 9521c5ea7f6b57e8bf9256b4a968e0e1bee19222..35b52059c94bbf0bdfe8469affc6bf2cfa931b97 100644
--- a/xbob/measure/script/compute_perf.py
+++ b/xbob/measure/script/compute_perf.py
@@ -102,8 +102,8 @@ def plots(dev_neg, dev_pos, test_neg, test_pos, npoints, filename):
linestyle='-', label='test')
mpl.axis([0,40,0,40])
mpl.title("ROC Curve")
- mpl.xlabel('FRR (%)')
- mpl.ylabel('FAR (%)')
+ mpl.xlabel('FAR (%)')
+ mpl.ylabel('FRR (%)')
mpl.grid(True, color=(0.3,0.3,0.3))
mpl.legend()
pp.savefig(fig)
@@ -116,8 +116,8 @@ def plots(dev_neg, dev_pos, test_neg, test_pos, npoints, filename):
linestyle='-', label='test')
plot.det_axis([0.01, 40, 0.01, 40])
mpl.title("DET Curve")
- mpl.xlabel('FRR (%)')
- mpl.ylabel('FAR (%)')
+ mpl.xlabel('FAR (%)')
+ mpl.ylabel('FRR (%)')
mpl.grid(True, color=(0.3,0.3,0.3))
mpl.legend()
pp.savefig(fig)
diff --git a/xbob/measure/test/data/nonsep-det.hdf5 b/xbob/measure/test/data/nonsep-det.hdf5
index 98b50b44eeb7784b931c86ac334aa043d9075d8b..4664fe84b9b0787e190dcdcb6b18208aa9c6fb61 100644
Binary files a/xbob/measure/test/data/nonsep-det.hdf5 and b/xbob/measure/test/data/nonsep-det.hdf5 differ
diff --git a/xbob/measure/test/data/nonsep-roc.hdf5 b/xbob/measure/test/data/nonsep-roc.hdf5
index adcf17bbffb5d805b025632bb73035537efb2343..46ea38d6ff16287725e9e5883ab55ae1ebedb711 100644
Binary files a/xbob/measure/test/data/nonsep-roc.hdf5 and b/xbob/measure/test/data/nonsep-roc.hdf5 differ
diff --git a/xbob/measure/test/test_error.py b/xbob/measure/test/test_error.py
index 11def628b546d509cf63f8a453041aa6ca626321..2f632d56bb3f455455a8a3537f7d05d7e3e5d1b0 100644
--- a/xbob/measure/test/test_error.py
+++ b/xbob/measure/test/test_error.py
@@ -45,7 +45,7 @@ def test_basic_ratios():
prec, recall = precision_recall(negatives, positives, minimum-0.1)
nose.tools.eq_(prec, 0.5)
nose.tools.eq_(recall, 1.0)
-
+
# Similarly, if we take a threshold on the maximum, the FRR should be 1.0
# while the FAR should be 0.0. Both precision and recall should be 0.0.
far, frr = farfrr(negatives, positives, maximum+0.1)
@@ -63,23 +63,23 @@ def test_basic_ratios():
prec, recall = precision_recall(negatives, positives, 3.0)
nose.tools.eq_(prec, 1.0)
nose.tools.eq_(recall, 1.0)
-
+
# Testing the values of F-score depending on different choices of the threshold
f_score_ = f_score(negatives, positives, minimum-0.1)
nose.tools.assert_almost_equal(f_score_, 0.66666667)
f_score_ = f_score(negatives, positives, minimum-0.1, 2)
nose.tools.assert_almost_equal(f_score_, 0.83333333)
-
+
f_score_ = f_score(negatives, positives, maximum+0.1)
nose.tools.eq_(f_score_, 0.0)
f_score_ = f_score(negatives, positives, maximum+0.1, 2)
nose.tools.eq_(f_score_, 0.0)
-
+
f_score_ = f_score(negatives, positives, 3.0)
nose.tools.eq_(f_score_, 1.0)
f_score_ = f_score(negatives, positives, 3.0, 2)
nose.tools.eq_(f_score_, 1.0)
-
+
def test_indexing():
@@ -185,14 +185,14 @@ def test_plots():
# save('nonsep-roc.hdf5', xy)
xyref = xbob.io.load(F('nonsep-roc.hdf5'))
assert numpy.array_equal(xy, xyref)
-
+
# This example will test the Precision-Recall plot calculation functionality.
xy = precision_recall_curve(negatives, positives, 100)
# uncomment the next line to save a reference value
# save('nonsep-roc.hdf5', xy)
xyref = xbob.io.load(F('nonsep-precisionrecall.hdf5'))
assert numpy.array_equal(xy, xyref)
-
+
# This example will test the DET plot calculation functionality.
det_xyzw = det(negatives, positives, 100)
# uncomment the next line to save a reference value
@@ -227,7 +227,7 @@ def test_rocch():
positives = xbob.io.load(F('linsep-positives.hdf5'))
negatives = xbob.io.load(F('linsep-negatives.hdf5'))
# References obtained using Bosaris 1.06
- pmiss_pfa_ref = numpy.array([[0., 0, 1], [1, 0, 0]])
+ pmiss_pfa_ref = numpy.array([[1., 0., 0.], [0., 0., 1.]])
eer_ref = 0.
# Computes
pmiss_pfa = rocch(negatives, positives)
@@ -241,7 +241,7 @@ def test_rocch():
positives = xbob.io.load(F('nonsep-positives.hdf5'))
negatives = xbob.io.load(F('nonsep-negatives.hdf5'))
# References obtained using Bosaris 1.06
- pmiss_pfa_ref = numpy.array([[0, 0, 0.08, 0.12, 0.22, 0.48, 1.], [1., 0.68, 0.28, 0.1, 0.06, 0., 0.]])
+ pmiss_pfa_ref = numpy.array([[1., 0.68, 0.28, 0.1, 0.06, 0., 0.], [0, 0, 0.08, 0.12, 0.22, 0.48, 1.]])
eer_ref = 0.116363636363636
# Computes
pmiss_pfa = rocch(negatives, positives)
@@ -250,7 +250,7 @@ def test_rocch():
assert abs(eer-eer_ref) < 1e-4
eer = eer_rocch(negatives, positives)
assert abs(eer-eer_ref) < 1e-4
-
+
def test_cmc():