Commit ffd2b1cc authored by André Anjos's avatar André Anjos

Merge branch 'fprfnr' into 'master'

Update documentation and commands: FAR->FPR, FRR->FNR

Closes #54

See merge request !87
parents 180c790f fdcceedb
Pipeline #21940 passed with stages
in 10 minutes and 47 seconds
......@@ -440,9 +440,9 @@ def false_alarm_rate(cmc_scores, threshold):
def eer(negatives, positives, is_sorted=False, also_farfrr=False):
"""Calculates the Equal Error Rate (EER).
Please note that it is possible that eer != far != frr.
This function returns (far + frr) / 2 as eer.
If you also need the far and frr values, set ``also_farfrr`` to ``True``.
Please note that it is possible that eer != fpr != fnr.
This function returns (fpr + fnr) / 2 as eer.
If you also need the fpr and fnr values, set ``also_farfrr`` to ``True``.
Parameters
----------
......@@ -459,11 +459,11 @@ def eer(negatives, positives, is_sorted=False, also_farfrr=False):
-------
eer : float
The Equal Error Rate (EER).
far : float
The False Accept Rate (FAR). Returned only when ``also_farfrr`` is
fpr : float
The False Positive Rate (FPR). Returned only when ``also_farfrr`` is
``True``.
frr : float
The False Reject Rate (FAR). Returned only when ``also_farfrr`` is
fnr : float
The False Negative Rate (FNR). Returned only when ``also_farfrr`` is
``True``.
"""
threshold = eer_threshold(negatives, positives, is_sorted)
......
......@@ -39,13 +39,13 @@ bob::measure::farfrr(const blitz::Array<double, 1> &negatives,
const blitz::Array<double, 1> &positives,
double threshold) {
if (std::isnan(threshold)){
bob::core::error << "Cannot compute FAR or FRR with threshold NaN.\n";
bob::core::error << "Cannot compute FPR (FAR) or FNR (FRR) with threshold NaN.\n";
return std::make_pair(1.,1.);
}
if (!negatives.size())
throw std::runtime_error("Cannot compute FAR when no negatives are given");
throw std::runtime_error("Cannot compute FPR (FAR) when no negatives are given");
if (!positives.size())
throw std::runtime_error("Cannot compute FRR when no positives are given");
throw std::runtime_error("Cannot compute FNR (FRR) when no positives are given");
blitz::sizeType total_negatives = negatives.extent(blitz::firstDim);
blitz::sizeType total_positives = positives.extent(blitz::firstDim);
blitz::sizeType false_accepts = blitz::count(negatives >= threshold);
......@@ -395,13 +395,13 @@ double bob::measure::rocch2eer(const blitz::Array<double, 2> &pfa_pmiss) {
/**
* This function computes the ROC coordinates for the given positive and
* negative values at the given FAR positions.
* negative values at the given FPR (FAR) positions.
*
* @param negatives Impostor scores
* @param positives Client scores
* @param far_list The list of FAR values where the FRR should be calculated
* @param far_list The list of FPR (FAR) values where the FNR (FRR) should be calculated
*
* @return The ROC curve with the FAR in the first row and the FRR in the
* @return The ROC curve with the FPR in the first row and the FNR in the
* second.
*/
blitz::Array<double, 2>
......
......@@ -229,7 +229,7 @@ double minimizingThreshold(const blitz::Array<double, 1> &negatives,
/**
* Calculates the threshold that is, as close as possible, to the
* equal-error-rate (EER) given the input data. The EER should be the point
* where the FAR equals the FRR. Graphically, this would be equivalent to the
* where the FPR equals the FNR. Graphically, this would be equivalent to the
* intersection between the R.O.C. (or D.E.T.) curves and the identity.
*/
double eerThreshold(const blitz::Array<double, 1> &negatives,
......@@ -252,7 +252,7 @@ double eerRocch(const blitz::Array<double, 1> &negatives,
*
* The value to minimize becomes:
*
* ER_cost = [cost * FAR] + [(1-cost) * FRR]
* ER_cost = [cost * FPR] + [(1-cost) * FNR]
*
* The higher the cost, the higher the importance given to *not* making
* mistakes classifying negatives/noise/impostors.
......@@ -271,12 +271,12 @@ inline double minHterThreshold(const blitz::Array<double, 1> &negatives,
}
/**
* Computes the threshold such that the real FAR is as close as possible
* Computes the threshold such that the real FPR is as close as possible
* to the requested far_value.
*
* @param negatives The impostor scores to be used for computing the FAR
* @param negatives The impostor scores to be used for computing the FPR
* @param positives The client scores; ignored by this function
* @param far_value The FAR value where the threshold should be computed
* @param far_value The FPR value where the threshold should be computed
*
* @return The computed threshold
*/
......@@ -301,7 +301,7 @@ double frrThreshold(const blitz::Array<double, 1> &negatives,
/**
* Calculates the ROC curve given a set of positive and negative scores and a
* number of desired points. Returns a two-dimensional blitz::Array of
* doubles that express the X (FRR) and Y (FAR) coordinates in this order.
* doubles that express the X (FNR) and Y (FPR) coordinates in this order.
* The points in which the ROC curve are calculated are distributed
* uniformly in the range [min(negatives, positives), max(negatives,
* positives)].
......@@ -349,8 +349,8 @@ double rocch2eer(const blitz::Array<double, 2> &pmiss_pfa);
/**
* Calculates the ROC curve given a set of positive and negative scores at
* the given FAR coordinates. Returns a two-dimensional blitz::Array of
* doubles that express the X (FAR) and Y (CAR) coordinates in this order.
* the given FPR coordinates. Returns a two-dimensional blitz::Array of
* doubles that express the X (FPR) and Y (CPR) coordinates in this order.
*/
blitz::Array<double, 2> roc_for_far(const blitz::Array<double, 1> &negatives,
const blitz::Array<double, 1> &positives,
......
This diff is collapsed.
......@@ -80,9 +80,9 @@ def roc(negatives, positives, npoints=100, CAR=False, **kwargs):
npoints (:py:class:`int`, optional): The number of points for the plot. See
(:py:func:`bob.measure.roc`)
CAR (:py:class:`bool`, optional): If set to ``True``, it will plot the CAR
over FAR in using :py:func:`matplotlib.pyplot.semilogx`, otherwise the
FAR over FRR linearly using :py:func:`matplotlib.pyplot.plot`.
CAR (:py:class:`bool`, optional): If set to ``True``, it will plot the CPR
(CAR) over FPR in using :py:func:`matplotlib.pyplot.semilogx`, otherwise the
FPR over FNR linearly using :py:func:`matplotlib.pyplot.plot`.
kwargs (:py:class:`dict`, optional): Extra plotting parameters, which are
passed directly to :py:func:`matplotlib.pyplot.plot`.
......@@ -107,7 +107,7 @@ def roc(negatives, positives, npoints=100, CAR=False, **kwargs):
def roc_for_far(negatives, positives, far_values=log_values(), CAR=True,
**kwargs):
"""Plots the ROC curve for the given list of False Acceptance Rates (FAR).
"""Plots the ROC curve for the given list of False Positive Rates (FAR).
This method will call ``matplotlib`` to plot the ROC curve for a system which
contains a particular set of negatives (impostors) and positives (clients)
......@@ -115,8 +115,8 @@ def roc_for_far(negatives, positives, far_values=log_values(), CAR=True,
All parameters passed with exception of the three first parameters of this
method will be directly passed to the plot command.
The plot will represent the False Acceptance Rate (FAR) on the horizontal
axis and the Correct Acceptance Rate (CAR) on the vertical axis. The values
The plot will represent the False Positive Rate (FPR) on the horizontal
axis and the Correct Positive Rate (CPR) on the vertical axis. The values
for the axis will be computed using :py:func:`bob.measure.roc_for_far`.
.. note::
......@@ -136,12 +136,12 @@ def roc_for_far(negatives, positives, far_values=log_values(), CAR=True,
"positive" (signal, class) samples of your classifier. See
(:py:func:`bob.measure.roc`)
far_values (:py:class:`list`, optional): The values for the FAR, where the
CAR should be plotted; each value should be in range [0,1].
far_values (:py:class:`list`, optional): The values for the FPR, where the
CPR (CAR) should be plotted; each value should be in range [0,1].
CAR (:py:class:`bool`, optional): If set to ``True``, it will plot the CAR
over FAR in using :py:func:`matplotlib.pyplot.semilogx`, otherwise the
FAR over FRR linearly using :py:func:`matplotlib.pyplot.plot`.
CAR (:py:class:`bool`, optional): If set to ``True``, it will plot the CPR
(CAR) over FPR in using :py:func:`matplotlib.pyplot.semilogx`, otherwise the
FPR over FNR linearly using :py:func:`matplotlib.pyplot.plot`.
kwargs (:py:class:`dict`, optional): Extra plotting parameters, which are
passed directly to :py:func:`matplotlib.pyplot.plot`.
......@@ -510,14 +510,14 @@ def cmc(cmc_scores, logx=True, **kwargs):
def detection_identification_curve(cmc_scores, far_values=log_values(), rank=1, logx=True, **kwargs):
"""Plots the Detection & Identification curve over the FAR
"""Plots the Detection & Identification curve over the FPR
This curve is designed to be used in an open set identification protocol, and
defined in Chapter 14.1 of [LiJain2005]_. It requires to have at least one
open set probe item, i.e., with no corresponding gallery, such that the
positives for that pair are ``None``.
The detection and identification curve first computes FAR thresholds based on
The detection and identification curve first computes FPR thresholds based on
the out-of-set probe scores (negative scores). For each probe item, the
**maximum** negative score is used. Then, it plots the detection and
identification rates for those thresholds, which are based on the in-set
......@@ -534,8 +534,8 @@ def detection_identification_curve(cmc_scores, far_values=log_values(), rank=1,
rank (:py:class:`int`, optional): The rank for which the curve should be
plotted
far_values (:py:class:`list`, optional): The values for the FAR, where the
CAR should be plotted; each value should be in range [0,1].
far_values (:py:class:`list`, optional): The values for the FPR (FAR), where the
CPR (CAR) should be plotted; each value should be in range [0,1].
logx (:py:class:`bool`, optional): If set (the default), plots the rank
axis in logarithmic scale using :py:func:`matplotlib.pyplot.semilogx` or
......
......@@ -17,7 +17,8 @@ CRITERIA = ('eer', 'min-hter', 'far')
criteria=CRITERIA, score_format=SCORE_FORMAT,
hter_note=' ',
command='bob measure metrics'),
criteria=CRITERIA)
criteria=CRITERIA,
far_name="FPR")
def metrics(ctx, scores, evaluation, **kwargs):
process = figure.Metrics(ctx, scores, evaluation, load.split)
process.run()
......@@ -25,7 +26,7 @@ def metrics(ctx, scores, evaluation, **kwargs):
@common_options.roc_command(
common_options.ROC_HELP.format(
score_format=SCORE_FORMAT, command='bob measure roc'))
score_format=SCORE_FORMAT, command='bob measure roc'), far_name="FPR")
def roc(ctx, scores, evaluation, **kwargs):
process = figure.Roc(ctx, scores, evaluation, load.split)
process.run()
......@@ -33,7 +34,7 @@ def roc(ctx, scores, evaluation, **kwargs):
@common_options.det_command(
common_options.DET_HELP.format(
score_format=SCORE_FORMAT, command='bob measure det'))
score_format=SCORE_FORMAT, command='bob measure det'), far_name="FPR")
def det(ctx, scores, evaluation, **kwargs):
process = figure.Det(ctx, scores, evaluation, load.split)
process.run()
......@@ -49,7 +50,7 @@ def epc(ctx, scores, **kwargs):
@common_options.hist_command(
common_options.HIST_HELP.format(
score_format=SCORE_FORMAT, command='bob measure hist'))
score_format=SCORE_FORMAT, command='bob measure hist'), far_name="FPR")
def hist(ctx, scores, evaluation, **kwargs):
process = figure.Hist(ctx, scores, evaluation, load.split)
process.run()
......@@ -58,7 +59,7 @@ def hist(ctx, scores, evaluation, **kwargs):
@common_options.evaluate_command(
common_options.EVALUATE_HELP.format(
score_format=SCORE_FORMAT, command='bob measure evaluate'),
criteria=CRITERIA)
criteria=CRITERIA, far_name="FPR")
def evaluate(ctx, scores, evaluation, **kwargs):
common_options.evaluate_flow(
ctx, scores, evaluation, metrics, roc, det, epc, hist, **kwargs)
......@@ -69,7 +70,7 @@ def evaluate(ctx, scores, evaluation, **kwargs):
names='FtA, FAR, FRR, FMR, FMNR, HTER',
criteria=CRITERIA, score_format=SCORE_FORMAT,
command='bob measure multi-metrics'),
criteria=CRITERIA)
criteria=CRITERIA, far_name="FPR")
def multi_metrics(ctx, scores, evaluation, protocols_number, **kwargs):
ctx.meta['min_arg'] = protocols_number * (2 if evaluation else 1)
process = figure.MultiMetrics(ctx, scores, evaluation, load.split)
......
......@@ -220,16 +220,16 @@ def subplot_option(dflt=111, **kwargs):
def cost_option(**kwargs):
'''Get option to get cost for FAR'''
'''Get option to get cost for FPR'''
def custom_cost_option(func):
def callback(ctx, param, value):
if value < 0 or value > 1:
raise click.BadParameter("Cost for FAR must be betwen 0 and 1")
raise click.BadParameter("Cost for FPR must be betwen 0 and 1")
ctx.meta['cost'] = value
return value
return click.option(
'-C', '--cost', type=float, default=0.99, show_default=True,
help='Cost for FAR in minDCF',
help='Cost for FPR in minDCF',
callback=callback, **kwargs)(func)
return custom_cost_option
......@@ -388,34 +388,40 @@ def decimal_option(dflt=1, **kwargs):
return custom_decimal_option
def far_option(**kwargs):
def far_option(far_name="FAR", **kwargs):
'''Get option to get far value'''
def custom_far_option(func):
def callback(ctx, param, value):
if value is not None and (value > 1 or value < 0):
raise click.BadParameter("FAR value should be between 0 and 1")
raise click.BadParameter(
"{} value should be between 0 and 1".format(far_name)
)
ctx.meta['far_value'] = value
return value
return click.option(
'-f', '--far-value', type=click.FLOAT, default=None,
help='The FAR value for which to compute threshold. This option '
'must be used alongside `--criterion far`.',
'-f', '--{}-value'.format(far_name.lower()), 'far_value',
type=click.FLOAT, default=None,
help='The {} value for which to compute threshold. This option '
'must be used alongside `--criterion far`.'.format(far_name),
callback=callback, show_default=True, **kwargs)(func)
return custom_far_option
def min_far_option(dflt=1e-4, **kwargs):
def min_far_option(far_name="FAR",dflt=1e-4, **kwargs):
'''Get option to get min far value'''
def custom_min_far_option(func):
def callback(ctx, param, value):
if value is not None and (value > 1 or value < 0):
raise click.BadParameter("FAR value should be between 0 and 1")
raise click.BadParameter(
"{} value should be between 0 and 1".format(far_name)
)
ctx.meta['min_far_value'] = value
return value
return click.option(
'-M', '--min-far-value', type=click.FLOAT, default=dflt,
help='Select the minimum FAR value used in ROC and DET plots; '
'should be a power of 10.',
'-M', '--min-{}-value'.format(far_name.lower()), 'min_far_value',
type=click.FLOAT, default=dflt,
help='Select the minimum {} value used in ROC and DET plots; '
'should be a power of 10.'.format(far_name),
callback=callback, show_default=True, **kwargs)(func)
return custom_min_far_option
......@@ -585,7 +591,8 @@ def style_option(**kwargs):
return custom_style_option
def metrics_command(docstring, criteria=('eer', 'min-hter', 'far')):
def metrics_command(docstring, criteria=('eer', 'min-hter', 'far'),
far_name="FAR"):
def custom_metrics_command(func):
func.__doc__ = docstring
......@@ -596,7 +603,7 @@ def metrics_command(docstring, criteria=('eer', 'min-hter', 'far')):
@output_log_metric_option()
@criterion_option(criteria)
@thresholds_option()
@far_option()
@far_option(far_name=far_name)
@legends_option()
@open_file_mode_option()
@verbosity_option()
......@@ -632,7 +639,7 @@ METRICS_HELP = """Prints a table that contains {names} for a given
"""
def roc_command(docstring):
def roc_command(docstring, far_name="FAR"):
def custom_roc_command(func):
func.__doc__ = docstring
......@@ -648,7 +655,7 @@ def roc_command(docstring):
@semilogx_option(True)
@lines_at_option()
@axes_val_option()
@min_far_option()
@min_far_option(far_name=far_name)
@x_rotation_option()
@x_label_option()
@y_label_option()
......@@ -687,7 +694,7 @@ ROC_HELP = """Plot ROC (receiver operating characteristic) curve.
"""
def det_command(docstring):
def det_command(docstring, far_name="FAR"):
def custom_det_command(func):
func.__doc__ = docstring
......@@ -701,7 +708,7 @@ def det_command(docstring):
@sep_dev_eval_option()
@eval_option()
@axes_val_option(dflt='0.01,95,0.01,95')
@min_far_option()
@min_far_option(far_name=far_name)
@x_rotation_option(dflt=45)
@x_label_option()
@y_label_option()
......@@ -785,7 +792,7 @@ EPC_HELP = """Plot EPC (expected performance curve).
"""
def hist_command(docstring):
def hist_command(docstring, far_name="FAR"):
def custom_hist_command(func):
func.__doc__ = docstring
......@@ -799,7 +806,7 @@ def hist_command(docstring):
@no_legend_option()
@legend_ncols_option()
@criterion_option()
@far_option()
@far_option(far_name=far_name)
@no_line_option()
@thresholds_option()
@subplot_option()
......@@ -841,7 +848,8 @@ HIST_HELP = """ Plots histograms of positive and negatives along with threshold
"""
def evaluate_command(docstring, criteria=('eer', 'min-hter', 'far')):
def evaluate_command(docstring, criteria=('eer', 'min-hter', 'far'),
far_name="FAR"):
def custom_evaluate_command(func):
func.__doc__ = docstring
......@@ -852,7 +860,7 @@ def evaluate_command(docstring, criteria=('eer', 'min-hter', 'far')):
@table_option()
@eval_option()
@criterion_option(criteria)
@far_option()
@far_option(far_name=far_name)
@output_log_metric_option()
@output_plot_file_option(default_out='eval_plots.pdf')
@lines_at_option()
......@@ -955,7 +963,8 @@ def n_protocols_option(required=True, **kwargs):
return custom_n_protocols_option
def multi_metrics_command(docstring, criteria=('eer', 'min-hter', 'far')):
def multi_metrics_command(docstring, criteria=('eer', 'min-hter', 'far'),
far_name="FAR"):
def custom_metrics_command(func):
func.__doc__ = docstring
......@@ -967,7 +976,7 @@ def multi_metrics_command(docstring, criteria=('eer', 'min-hter', 'far')):
@output_log_metric_option()
@criterion_option(criteria)
@thresholds_option()
@far_option()
@far_option(far_name=far_name)
@legends_option()
@open_file_mode_option()
@verbosity_option()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment