Commit 034dfc55 authored by André Anjos's avatar André Anjos 💬

Merge branch 'utils' into 'master'

Add utilities to compute ROI performance metrics

See merge request !19
parents cc356cc3 500296b7
Pipeline #5266 passed with stages
in 13 minutes and 8 seconds
......@@ -17,7 +17,7 @@ You can download the raw data of the `UTFVP`_ database by following the link.
.. include:: links.rst
"""
from bob.bio.vein.database.utfvp import Database
from ..database.utfvp import Database
utfvp_directory = "[YOUR_UTFVP_DIRECTORY]"
"""Value of ``~/.bob_bio_databases.txt`` for this database"""
......
......@@ -15,7 +15,7 @@ the link.
"""
from bob.bio.vein.database.verafinger import Database
from ..database.verafinger import Database
verafinger_directory = "[YOUR_VERAFINGER_DIRECTORY]"
"""Value of ``~/.bob_bio_databases.txt`` for this database"""
......
......@@ -165,7 +165,12 @@ def mask_to_image(mask, dtype=numpy.uint8):
def show_image(image):
"""Shows a single image
"""Shows a single image using :py:meth:`PIL.Image.Image.show`
.. warning::
This function opens a new window. You must be operating interactively in a
windowing system for it to work properly.
Parameters:
......@@ -179,7 +184,7 @@ def show_image(image):
img.show()
def show_mask_over_image(image, mask, color='red'):
def draw_mask_over_image(image, mask, color='red'):
"""Plots the mask over the image of a finger, for debugging purposes
Parameters:
......@@ -190,6 +195,11 @@ def show_mask_over_image(image, mask, color='red'):
mask (numpy.ndarray): A 2D numpy.ndarray compose of boolean values
containing the calculated mask
Returns:
PIL.Image: An image in PIL format
"""
from PIL import Image
......@@ -198,7 +208,29 @@ def show_mask_over_image(image, mask, color='red'):
msk = Image.fromarray((~mask).astype('uint8')*80)
red = Image.new('RGBA', img.size, color=color)
img.paste(red, mask=msk)
img.show()
return img
def show_mask_over_image(image, mask, color='red'):
"""Plots the mask over the image of a finger using :py:meth:`PIL.Image.Image.show`
.. warning::
This function opens a new window. You must be operating interactively in a
windowing system for it to work properly.
Parameters:
image (numpy.ndarray): A 2D numpy.ndarray compose of 8-bit unsigned
integers containing the original image
mask (numpy.ndarray): A 2D numpy.ndarray compose of boolean values
containing the calculated mask
"""
draw_mask_over_image(image, mask, color).show()
def jaccard_index(a, b):
......@@ -232,12 +264,12 @@ def jaccard_index(a, b):
def intersect_ratio(a, b):
"""Calculates the intersection ratio between a probe and ground-truth
"""Calculates the intersection ratio between the ground-truth and a probe
This function calculates the intersection ratio between a probe mask
(:math:`B`) and a ground-truth mask (:math:`A`; probably generated from an
annotation), and returns the ratio of overlap when the probe is compared to
the ground-truth data:
This function calculates the intersection ratio between a ground-truth mask
(:math:`A`; probably generated from an annotation) and a probe mask
(:math:`B`), returning the ratio of overlap when the probe is compared to the
ground-truth data:
.. math::
......@@ -271,13 +303,12 @@ def intersect_ratio(a, b):
def intersect_ratio_of_complement(a, b):
"""Calculates the intersection ratio between a probe and the ground-truth
complement
"""Calculates the intersection ratio between the complement of ground-truth and a probe
This function calculates the intersection ratio between a probe mask
(:math:`B`) and *the complement* of a ground-truth mask (:math:`A`; probably
generated from an annotation), and returns the ratio of overlap when the
probe is compared to the ground-truth data:
This function calculates the intersection ratio between *the complement* of a
ground-truth mask (:math:`A`; probably generated from an annotation) and a
probe mask (:math:`B`), returning the ratio of overlap when the probe is
compared to the ground-truth data:
.. math::
......
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
"""Compares two set of masks and prints some error metrics
This program requires that the masks for both databases (one representing the
ground-truth and a second the database with an automated method) are
represented as HDF5 files containing a ``mask`` object, which should be boolean
in nature.
Usage: %(prog)s [-v...] [-n X] <ground-truth> <database>
%(prog)s --help
%(prog)s --version
Arguments:
<ground-truth> Path to a set of files that contain ground truth annotations
for the ROIs you wish to compare.
<database> Path to a similar set of files as in `<ground-truth>`, but
with ROIs calculated automatically. Every HDF5 in this
directory will be matched to an equivalent file in the
`<ground-truth>` database and their masks will be compared
Options:
-h, --help Shows this help message and exits
-V, --version Prints the version and exits
-v, --verbose Increases the output verbosity level
-n N, --annotate=N Print out the N worst cases available in the database,
taking into consideration the various metrics analyzed
Example:
1. Just run for basic statistics:
$ %(prog)s -vvv gt/ automatic/
2. Identify worst 5 samples in the database according to a certain criterion:
$ %(prog)s -vv -n 5 gt/ automatic/
"""
import os
import sys
import fnmatch
import operator
import numpy
import bob.core
logger = bob.core.log.setup("bob.measure")
import bob.io.base
def make_catalog(d):
"""Returns a catalog dictionary containing the file stems available in ``d``
Parameters:
d (str): A path representing a directory that will be scanned for .hdf5
files
Returns
list: A list of stems, from the directory ``d``, that represent files of
type HDF5 in that directory. Each file should contain two objects:
``image`` and ``mask``.
"""
logger.info("Scanning directory `%s'..." % d)
retval = []
for path, dirs, files in os.walk(d):
basedir = os.path.relpath(path, d)
logger.debug("Scanning sub-directory `%s'..." % basedir)
candidates = fnmatch.filter(files, '*.hdf5')
if not candidates: continue
logger.debug("Found %d files" % len(candidates))
retval += [os.path.join(basedir, k) for k in candidates]
logger.info("Found a total of %d files at `%s'" % (len(retval), d))
return sorted(retval)
def sort_table(table, cols):
"""Sorts a table by multiple columns
Parameters:
table (:py:class:`list` of :py:class:`list`): Or tuple of tuples, where
each inner list represents a row
cols (list, tuple): Specifies the column numbers to sort by e.g. (1,0)
would sort by column 1, then by column 0
Returns:
list: of lists, with the table re-ordered as you see fit.
"""
for col in reversed(cols):
table = sorted(table, key=operator.itemgetter(col))
return table
def mean_std_for_column(table, column):
"""Calculates the mean and standard deviation for the column in question
Parameters:
table (:py:class:`list` of :py:class:`list`): Or tuple of tuples, where
each inner list represents a row
col (int): The number of the column from where to extract the data for
calculating the mean and the standard-deviation.
Returns:
float: mean
float: (unbiased) standard deviation
"""
z = numpy.array([k[column] for k in table])
return z.mean(), z.std(ddof=1)
def main(user_input=None):
if user_input is not None:
argv = user_input
else:
argv = sys.argv[1:]
import docopt
import pkg_resources
completions = dict(
prog=os.path.basename(sys.argv[0]),
version=pkg_resources.require('bob.bio.vein')[0].version
)
args = docopt.docopt(
__doc__ % completions,
argv=argv,
version=completions['version'],
)
# Sets-up logging
verbosity = int(args['--verbose'])
bob.core.log.set_verbosity_level(logger, verbosity)
# Catalogs
gt = make_catalog(args['<ground-truth>'])
db = make_catalog(args['<database>'])
if gt != db:
raise RuntimeError("Ground-truth and database have different files!")
# Calculate all metrics required
from ..preprocessor import utils
metrics = []
for k in gt:
logger.info("Evaluating metrics for `%s'..." % k)
gt_file = os.path.join(args['<ground-truth>'], k)
db_file = os.path.join(args['<database>'], k)
gt_roi = bob.io.base.HDF5File(gt_file).read('mask')
db_roi = bob.io.base.HDF5File(db_file).read('mask')
metrics.append((
k,
utils.jaccard_index(gt_roi, db_roi),
utils.intersect_ratio(gt_roi, db_roi),
utils.intersect_ratio_of_complement(gt_roi, db_roi),
))
# Print statistics
names = (
(1, 'Jaccard index'),
(2, 'Intersection ratio (m1)'),
(3, 'Intersection ratio of complement (m2)'),
)
print("Statistics:")
for k, name in names:
mean, std = mean_std_for_column(metrics, k)
print(name + ': ' + '%.2e +- %.2e' % (mean, std))
# Print worst cases, if the user asked so
if args['--annotate'] is not None:
N = int(args['--annotate'])
if N <= 0:
raise docopt.DocoptExit("Argument to --annotate should be >0")
print("Worst cases by metric:")
for k, name in names:
print(name + ':')
if k in (1,2):
worst = sort_table(metrics, (k,))[:N]
else:
worst = reversed(sort_table(metrics, (k,))[-N:])
for n, l in enumerate(worst):
fname = os.path.join(args['<database>'], l[0])
print(' %d. [%.2e] %s' % (n, l[k], fname))
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Mon 07 Nov 2016 15:20:26 CET
"""Visualizes masks applied to vein imagery
Usage: %(prog)s [-v...] [options] <file> [<file>...]
%(prog)s --help
%(prog)s --version
Arguments:
<file> The HDF5 file to load image and mask from
Options:
-h, --help Shows this help message and exits
-V, --version Prints the version and exits
-v, --verbose Increases the output verbosity level
-s path, --save=path If set, saves image into a file instead of displaying
it
Examples:
Visualize the mask on a single image:
$ %(prog)s data.hdf5
Visualize multiple masks (like in a proof-sheet):
$ %(prog)s *.hdf5
"""
import os
import sys
import bob.core
logger = bob.core.log.setup("bob.bio.vein")
from ..preprocessor import utils
def main(user_input=None):
if user_input is not None:
argv = user_input
else:
argv = sys.argv[1:]
import docopt
import pkg_resources
completions = dict(
prog=os.path.basename(sys.argv[0]),
version=pkg_resources.require('bob.bio.vein')[0].version
)
args = docopt.docopt(
__doc__ % completions,
argv=argv,
version=completions['version'],
)
# Sets-up logging
verbosity = int(args['--verbose'])
bob.core.log.set_verbosity_level(logger, verbosity)
# Loads the image, the mask and save it to a PNG file
from ..preprocessor import utils
for filename in args['<file>']:
f = bob.io.base.HDF5File(filename)
image = f.read('image')
mask = f.read('mask')
img = utils.draw_mask_over_image(image, mask)
if args['--save']:
img.save(args['--save'])
else:
img.show()
......@@ -245,4 +245,124 @@ Wide Line Detector Histogram Eq. 00:04 00:01 00:01 02:04 00:06
======================== ================= ====== ====== ====== ====== ======
Modifying Baseline Experiments
------------------------------
It is fairly easy to modify baseline experiments available in this package. To
do so, you must copy the configuration files for the given baseline you want to
modify, edit them to make the desired changes and run the experiment again.
For example, suppose you'd like to change the protocol on the Vera Fingervein
database and use the protocol ``full`` instead of the default protocol ``nom``.
First, you identify where the configuration file sits:
.. code-block:: sh
$ ./bin/resources.py -tc -p bob.bio.vein
- bob.bio.vein X.Y.Z @ /path/to/bob.bio.vein:
+ mc --> bob.bio.vein.configurations.maximum_curvature
+ parallel --> bob.bio.vein.configurations.parallel
+ rlt --> bob.bio.vein.configurations.repeated_line_tracking
+ utfvp --> bob.bio.vein.configurations.utfvp
+ verafinger --> bob.bio.vein.configurations.verafinger
+ wld --> bob.bio.vein.configurations.wide_line_detector
The listing above tells the ``verafinger`` configuration file sits on the
file ``/path/to/bob.bio.vein/bob/bio/vein/configurations/verafinger.py``. In
order to modify it, make a local copy. For example:
.. code-block:: sh
$ cp /path/to/bob.bio.vein/bob/bio/vein/configurations/verafinger.py verafinger_full.py
$ # edit verafinger_full.py, change the value of "protocol" to "full"
Also, don't forget to change all relative module imports (such as ``from
..database.verafinger import Database``) to absolute imports (e.g. ``from
bob.bio.vein.database.verafinger import Database``). This will make the
configuration file work irrespectively of its location w.r.t. ``bob.bio.vein``.
The final version of the modified file could look like this:
.. code-block:: python
from bob.bio.vein.database.verafinger import Database
database = Database(original_directory='/where/you/have/the/raw/files',
original_extension='.png', #don't change this
)
protocol = 'full'
Now, re-run the experiment using your modified database descriptor:
.. code-block:: sh
$ ./bin/verify.py ./verafinger_full.py wld -vv
Notice we replace the use of the registered configuration file named
``verafinger`` by the local file ``verafinger_full.py``. This makes the program
``verify.py`` take that into consideration instead of the original file.
Other Resources
---------------
This package contains other resources that can be used to evaluate different
bits of the vein processing toolchain.
Region of Interest Goodness of Fit
==================================
Automatic region of interest (RoI) finding and cropping can be evaluated using
a couple of scripts available in this package. The program ``compare_rois.py``
compares two sets of ``preprocessed`` images and masks, generated by
*different* preprocessors (see
:py:class:`bob.bio.base.preprocessor.Preprocessor`) and calculates a few
metrics to help you determine how both techniques compare. Normally, the
program is used to compare the result of automatic RoI to manually annoted
regions on the same images. To use it, just point it to the outputs of two
experiments representing the manually annotated regions and automatically
extracted ones. E.g.:
.. code-block:: sh
$ ./bin/compare_rois.py ~/verafinger/mc_annot/preprocessed ~/verafinger/mc/preprocessed
Jaccard index: 9.60e-01 +- 5.98e-02
Intersection ratio (m1): 9.79e-01 +- 5.81e-02
Intersection ratio of complement (m2): 1.96e-02 +- 1.53e-02
Values printed by the script correspond to the `Jaccard index`_
(:py:func:`bob.bio.vein.preprocessor.utils.jaccard_index`), as well as the
intersection ratio between the manual and automatically generated masks
(:py:func:`bob.bio.vein.preprocessor.utils.intersect_ratio`) and the ratio to
the complement of the intersection with respect to the automatically generated
mask
(:py:func:`bob.bio.vein.preprocessor.utils.intersect_ratio_of_complement`). You
can use the option ``-n 5`` to print the 5 worst cases according to each of the
metrics.
You can use the program ``view_mask.py`` to display the images after the
preprocessing step using:
.. code-block:: sh
$ ./bin/view_mask.py /path/to/verafinger/mc/preprocessed/098-F/098_R_1.hdf5 --save=example.png
$ # open example.png
And you should be able to view an image like this (example taken from the Vera
fingervein database, using the automatic annotator):
.. figure:: img/vein-mask.*
:scale: 50%
Example RoI overlayed on finger vein image of the Vera fingervein database,
as produced by the script ``view_mask.py``.
.. include:: links.rst
......@@ -249,6 +249,12 @@ skimage_version = '.'.join(skimage_version.split('.')[:2])
intersphinx_mapping['http://scikit-image.org/docs/%s.x' % skimage_version] = \
None
# Add PIL link
pil_version = pkg_resources.require('pillow')[0].version
pil_version = '.'.join(pil_version.split('.')[:2])
intersphinx_mapping['http://pillow.readthedocs.io/en/%s.x' % pil_version] = \
None
# We want to remove all private (i.e. _. or __.__) members
# that are not in the list of accepted functions
accepted_private_functions = ['__array__']
......
......@@ -24,3 +24,4 @@
.. _dependencies: https://gitlab.idiap.ch/bob/bob/wikis/Dependencies
.. _mailing list: https://groups.google.com/forum/?fromgroups#!forum/bob-devel
.. _bob.bio.base: https://pypi.python.org/pypi/bob.bio.base
.. _jaccard index: https://en.wikipedia.org/wiki/Jaccard_index
......@@ -45,6 +45,10 @@ setup(
'parallel = bob.bio.vein.configurations.parallel',
],
'console_scripts': [
'compare_rois.py = bob.bio.vein.script.compare_rois:main',
'view_mask.py = bob.bio.vein.script.view_mask:main',
]
},
classifiers = [
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment