diff --git a/bob/bio/base/script/evaluate.py b/bob/bio/base/script/evaluate.py
index 4e1da2ae6db0fcd7cc5cf0ba5388358577bb031a..b4c4da659b9ceb0e2eea71624ac2154f027075d1 100644
--- a/bob/bio/base/script/evaluate.py
+++ b/bob/bio/base/script/evaluate.py
@@ -21,9 +21,8 @@ from matplotlib import pyplot
from matplotlib.backends.backend_pdf import PdfPages
if not os.environ.get('BOB_NO_STYLE_CHANGES'):
- # increase the default line width and font size
- matplotlib.rc('lines', linewidth=4)
- matplotlib.rc('font', size=18)
+ # make the fig size smaller so that everything becomes bigger
+ matplotlib.rc('figure', figsize=(4, 3))
import bob.core
@@ -43,20 +42,22 @@ def command_line_arguments(command_line_parameters):
parser.add_argument('-s', '--directory', default = '.', help = "A directory, where to find the --dev-files and the --eval-files")
parser.add_argument('-c', '--criterion', choices = ('EER', 'HTER', 'FAR'), help = "If given, the threshold of the development set will be computed with this criterion.")
- parser.add_argument('-f', '--far-value', type=float, default=0.1, help = "The FAR value in %% for which to evaluate (only for --criterion FAR)")
+ parser.add_argument('-f', '--far-value', type=float, default=0.001, help = "The FAR value for which to evaluate (only for --criterion FAR)")
parser.add_argument('-x', '--cllr', action = 'store_true', help = "If given, Cllr and minCllr will be computed.")
parser.add_argument('-m', '--mindcf', action = 'store_true', help = "If given, minDCF will be computed.")
parser.add_argument('--cost', default=0.99, help='Cost for FAR in minDCF')
parser.add_argument('-r', '--rr', action = 'store_true', help = "If given, the Recognition Rate will be computed.")
parser.add_argument('-t', '--thresholds', type=float, nargs='+', help = "If given, the Recognition Rate will incorporate an Open Set handling, rejecting all scores that are below the given threshold; when multiple thresholds are given, they are applied in the same order as the --dev-files.")
parser.add_argument('-l', '--legends', nargs='+', help = "A list of legend strings used for ROC, CMC and DET plots; if given, must be the same number than --dev-files.")
- parser.add_argument('-F', '--legend-font-size', type=int, default=18, help = "Set the font size of the legends.")
+ parser.add_argument('-F', '--legend-font-size', type=int, default=10, help = "Set the font size of the legends.")
parser.add_argument('-P', '--legend-position', type=int, help = "Set the font size of the legends.")
parser.add_argument('-T', '--title', nargs = '+', help = "Overwrite the default title of the plot for development (and evaluation) set")
parser.add_argument('-R', '--roc', help = "If given, ROC curves will be plotted into the given pdf file.")
parser.add_argument('-D', '--det', help = "If given, DET curves will be plotted into the given pdf file.")
parser.add_argument('-C', '--cmc', help = "If given, CMC curves will be plotted into the given pdf file.")
parser.add_argument('-E', '--epc', help = "If given, EPC curves will be plotted into the given pdf file. For this plot --eval-files is mandatory.")
+ parser.add_argument('-M', '--min-far-value', type=float, default=1e-4, help = "Select the minimum FAR value used in ROC plots; should be a power of 10.")
+ parser.add_argument('-L', '--far-line-at', type=float, help = "If given, draw a veritcal line at this FAR value in the ROC plots.")
parser.add_argument('--parser', default = '4column', choices = ('4column', '5column'), help="The style of the resulting score files. The default fits to the usual output of score files.")
# add verbose option
@@ -99,102 +100,110 @@ def command_line_arguments(command_line_parameters):
return args
-def _plot_roc(frrs, colors, labels, title, fontsize=18, position=None, farfrrs=None):
+def _plot_roc(frrs, colors, labels, title, fontsize=10, position=None, farfrrs=None):
if position is None: position = 'lower right'
figure = pyplot.figure()
+
# plot FAR and CAR for each algorithm
for i in range(len(frrs)):
- pyplot.semilogx([100.0*f for f in frrs[i][0]], [100. - 100.0*f for f in frrs[i][1]], color=colors[i], lw=2, ms=10, mew=1.5, label=labels[i])
- if farfrrs is not None:
- pyplot.plot(farfrrs[i][0]*100, (1-farfrrs[i][1])*100, 'o', color=colors[i], markeredgecolor='black')
+ pyplot.semilogx([f for f in frrs[i][0]], [1. - f for f in frrs[i][1]], color=colors[i], label=labels[i])
+ if isinstance(farfrrs, list):
+ pyplot.plot(farfrrs[i][0], (1.-farfrrs[i][1]), 'o', color=colors[i], markeredgecolor=colors[i])
+ # plot vertical bar, if desired
if farfrrs is not None:
- pyplot.plot([x[0]*100 for x in farfrrs], [(1-x[1])*100 for x in farfrrs], '--', color='black', linewidth=1.5)
-
- # finalize plot
- if farfrrs is None:
- pyplot.plot([0.1,0.1],[0,100], "--", color='black')
- pyplot.axis([frrs[0][0][0]*100,100,0,100])
- pyplot.xticks((0.01, 0.1, 1, 10, 100), ('0.01', '0.1', '1', '10', '100'))
- pyplot.xlabel('FMR (%)')
- pyplot.ylabel('1 - FNMR (%)')
+ if isinstance(farfrrs, float):
+ pyplot.plot([farfrrs,farfrrs],[0.,1.], "--", color='black')
+ else:
+ pyplot.plot([x[0] for x in farfrrs], [(1.-x[1]) for x in farfrrs], '--', color='black')
+
+ # compute and apply tick marks
+ min_far = frrs[0][0][0]
+ ticks = [min_far]
+ while ticks[-1] < 1.: ticks.append(ticks[-1] * 10.)
+ pyplot.axis([min_far, 1., -0.01, 1.01])
+ pyplot.xticks(ticks)
+
+ # set label, legend and title
+ pyplot.xlabel('FMR')
+ pyplot.ylabel('1 - FNMR')
pyplot.grid(True, color=(0.6,0.6,0.6))
pyplot.legend(loc=position, prop = {'size':fontsize})
pyplot.title(title)
- figure.set_tight_layout(True)
return figure
-def _plot_det(dets, colors, labels, title, fontsize=18, position=None):
- if position is None: position = 1
+def _plot_det(dets, colors, labels, title, fontsize=10, position=None):
+ if position is None: position = 'upper right'
# open new page for current plot
- figure = pyplot.figure(figsize=(8.2,8))
+ figure = pyplot.figure(figsize=(matplotlib.rcParams['figure.figsize'][0],
+ matplotlib.rcParams['figure.figsize'][0] * 0.975))
+ pyplot.grid(True)
# plot the DET curves
for i in range(len(dets)):
- pyplot.plot(dets[i][0], dets[i][1], color=colors[i], lw=2, ms=10, mew=1.5, label=labels[i])
+ pyplot.plot(dets[i][0], dets[i][1], color=colors[i], label=labels[i])
# change axes accordingly
det_list = [0.0002, 0.001, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 0.7, 0.9, 0.95]
ticks = [bob.measure.ppndf(d) for d in det_list]
- labels = [("%.5f" % (d*100)).rstrip('0').rstrip('.') for d in det_list]
- pyplot.xticks(ticks, labels)
+ labels = [("%.5f" % d).rstrip('0').rstrip('.') for d in det_list]
+ pyplot.xticks(ticks, [l if i % 2 else "" for i,l in enumerate(labels)])
pyplot.yticks(ticks, labels)
pyplot.axis((ticks[0], ticks[-1], ticks[0], ticks[-1]))
- pyplot.xlabel('FMR (%)')
- pyplot.ylabel('FNMR (%)')
+ pyplot.xlabel('FMR')
+ pyplot.ylabel('FNMR')
pyplot.legend(loc=position, prop = {'size':fontsize})
pyplot.title(title)
- figure.set_tight_layout(True)
return figure
-def _plot_cmc(cmcs, colors, labels, title, fontsize=18, position=None):
- if position is None: position = 4
+
+def _plot_cmc(cmcs, colors, labels, title, fontsize=10, position=None):
+ if position is None: position = 'lower right'
# open new page for current plot
figure = pyplot.figure()
- max_x = 0
- # plot the DET curves
+ max_R = 0
+ # plot the CMC curves
for i in range(len(cmcs)):
- x = bob.measure.plot.cmc(cmcs[i], figure=figure, color=colors[i], lw=2, ms=10, mew=1.5, label=labels[i])
- max_x = max(x, max_x)
+ probs = bob.measure.cmc(cmcs[i])
+ R = len(probs)
+ pyplot.semilogx(range(1, R+1), probs, figure=figure, color=colors[i], label=labels[i])
+ max_R = max(R, max_R)
# change axes accordingly
ticks = [int(t) for t in pyplot.xticks()[0]]
pyplot.xlabel('Rank')
- pyplot.ylabel('Probability (%)')
+ pyplot.ylabel('Probability')
pyplot.xticks(ticks, [str(t) for t in ticks])
- pyplot.axis([0, max_x, 0, 100])
+ pyplot.axis([0, max_R, -0.01, 1.01])
pyplot.legend(loc=position, prop = {'size':fontsize})
pyplot.title(title)
- figure.set_tight_layout(True)
return figure
-
-def _plot_epc(scores_dev, scores_eval, colors, labels, title, fontsize=18, position=None):
+def _plot_epc(scores_dev, scores_eval, colors, labels, title, fontsize=10, position=None):
if position is None: position = 'upper center'
# open new page for current plot
figure = pyplot.figure()
# plot the DET curves
for i in range(len(scores_dev)):
- bob.measure.plot.epc(scores_dev[i][0], scores_dev[i][1], scores_eval[i][0], scores_eval[i][1], 100, color=colors[i], label=labels[i], lw=2)
+ x,y = bob.measure.epc(scores_dev[i][0], scores_dev[i][1], scores_eval[i][0], scores_eval[i][1], 100)
+ pyplot.plot(x, y, color=colors[i], label=labels[i])
# change axes accordingly
pyplot.xlabel('alpha')
- pyplot.ylabel('HTER (%)')
+ pyplot.ylabel('HTER')
pyplot.title(title)
+ pyplot.axis([-0.01, 1.01, -0.01, 0.51])
pyplot.grid(True)
pyplot.legend(loc=position, prop = {'size':fontsize})
pyplot.title(title)
- pyplot.xlim([-0.01, 1.01])
- pyplot.ylim([0, 51])
- figure.set_tight_layout(True)
return figure
@@ -261,7 +270,7 @@ def main(command_line_parameters=None):
# apply threshold to development set
far, frr = bob.measure.farfrr(scores_dev[i][0], scores_dev[i][1], threshold)
if args.criterion == 'FAR':
- print("The FRR at FAR=%2.3f%% of the development set of '%s' is %2.3f%% (CAR: %2.3f%%)" % (args.far_value, args.legends[i], frr * 100., 100.*(1-frr)))
+ print("The FRR at FAR=%.1E of the development set of '%s' is %2.3f%% (CAR: %2.3f%%)" % (args.far_value, args.legends[i], frr * 100., 100.*(1-frr)))
else:
print("The %s of the development set of '%s' is %2.3f%%" % (args.criterion, args.legends[i], (far + frr) * 50.)) # / 2 * 100%
if args.eval_files:
@@ -303,7 +312,8 @@ def main(command_line_parameters=None):
if args.roc:
logger.info("Computing CAR curves on the development " + ("and on the evaluation set" if args.eval_files else "set"))
- fars = [math.pow(10., i * 0.25) for i in range(-17,0)] + [1.]
+ min_far = int(math.floor(math.log(args.min_far_value, 10)))
+ fars = [math.pow(10., i * 0.25) for i in range(min_far * 4, 0)] + [1.]
frrs_dev = [bob.measure.roc_for_far(scores[0], scores[1], fars) for scores in scores_dev]
if args.eval_files:
frrs_eval = [bob.measure.roc_for_far(scores[0], scores[1], fars) for scores in scores_eval]
@@ -313,18 +323,21 @@ def main(command_line_parameters=None):
# create a multi-page PDF for the ROC curve
pdf = PdfPages(args.roc)
# create a separate figure for dev and eval
- pdf.savefig(_plot_roc(frrs_dev, colors, args.legends, args.title[0] if args.title is not None else "ROC for development set", args.legend_font_size, args.legend_position), bbox_inches='tight')
+ pdf.savefig(_plot_roc(frrs_dev, colors, args.legends, args.title[0] if args.title is not None else "ROC for development set", args.legend_font_size, args.legend_position, args.far_line_at), bbox_inches='tight')
del frrs_dev
if args.eval_files:
- farfrrs = []
- for i, scores in enumerate(scores_eval):
- threshold = bob.measure.far_threshold(scores_dev[i][0], scores_dev[i][1], float(args.far_value) / 100)
- farfrrs.append(bob.measure.farfrr(scores_eval[i][0], scores_eval[i][1], threshold))
+ if args.far_line_at is not None:
+ farfrrs = []
+ for i in range(len(scores_dev)):
+ threshold = bob.measure.far_threshold(scores_dev[i][0], scores_dev[i][1], args.far_line_at)
+ farfrrs.append(bob.measure.farfrr(scores_eval[i][0], scores_eval[i][1], threshold))
+ else:
+ farfrrs = None
pdf.savefig(_plot_roc(frrs_eval, colors, args.legends, args.title[1] if args.title is not None else "ROC for evaluation set", args.legend_font_size, args.legend_position, farfrrs), bbox_inches='tight')
del frrs_eval
pdf.close()
except RuntimeError as e:
- raise RuntimeError("During plotting of ROC curves, the following exception occured:\n%s\nUsually this happens when the label contains characters that LaTeX cannot parse." % e)
+ raise RuntimeError("During plotting of ROC curves, the following exception occured:\n%s" % e)
if args.det:
logger.info("Computing DET curves on the development " + ("and on the evaluation set" if args.eval_files else "set"))
@@ -334,7 +347,7 @@ def main(command_line_parameters=None):
logger.info("Plotting DET curves to file '%s'", args.det)
try:
- # create a multi-page PDF for the ROC curve
+ # create a multi-page PDF for the DET curve
pdf = PdfPages(args.det)
# create a separate figure for dev and eval
pdf.savefig(_plot_det(dets_dev, colors, args.legends, args.title[0] if args.title is not None else "DET for development set", args.legend_font_size, args.legend_position), bbox_inches='tight')
@@ -344,7 +357,7 @@ def main(command_line_parameters=None):
del dets_eval
pdf.close()
except RuntimeError as e:
- raise RuntimeError("During plotting of ROC curves, the following exception occured:\n%s\nUsually this happens when the label contains characters that LaTeX cannot parse." % e)
+ raise RuntimeError("During plotting of DET curves, the following exception occured:\n%s" % e)
if args.epc:
@@ -354,12 +367,12 @@ def main(command_line_parameters=None):
raise ValueError("To plot the EPC curve the evaluation scores are necessary. Please, set it with the --eval-files option.")
try:
- # create a multi-page PDF for the ROC curve
+ # create a multi-page PDF for the EPC curve
pdf = PdfPages(args.epc)
pdf.savefig(_plot_epc(scores_dev, scores_eval, colors, args.legends, args.title if args.title is not None else "" , args.legend_font_size, args.legend_position), bbox_inches='tight')
pdf.close()
except RuntimeError as e:
- raise RuntimeError("During plotting of EPC curves, the following exception occured:\n%s\nUsually this happens when the label contains characters that LaTeX cannot parse." % e)
+ raise RuntimeError("During plotting of EPC curves, the following exception occured:\n%s" % e)
@@ -377,9 +390,9 @@ def main(command_line_parameters=None):
# create a multi-page PDF for the ROC curve
pdf = PdfPages(args.cmc)
# create a separate figure for dev and eval
- pdf.savefig(_plot_cmc(cmcs_dev, colors, args.legends, args.title[0] if args.title is not None else "CMC curve for development set", args.legend_font_size, args.legend_position))
+ pdf.savefig(_plot_cmc(cmcs_dev, colors, args.legends, args.title[0] if args.title is not None else "CMC curve for development set", args.legend_font_size, args.legend_position), bbox_inches='tight')
if args.eval_files:
- pdf.savefig(_plot_cmc(cmcs_eval, colors, args.legends, args.title[1] if args.title is not None else "CMC curve for evaluation set", args.legend_font_size, args.legend_position))
+ pdf.savefig(_plot_cmc(cmcs_eval, colors, args.legends, args.title[1] if args.title is not None else "CMC curve for evaluation set", args.legend_font_size, args.legend_position), bbox_inches='tight')
pdf.close()
except RuntimeError as e:
raise RuntimeError("During plotting of ROC curves, the following exception occured:\n%s\nUsually this happens when the label contains characters that LaTeX cannot parse." % e)
diff --git a/bob/bio/base/test/test_scripts.py b/bob/bio/base/test/test_scripts.py
index 7b647065b95b4f29f7982e502649992cc2f0a809..214ebc8997648b9dd2995db9f5f7ba90959d86ee 100644
--- a/bob/bio/base/test/test_scripts.py
+++ b/bob/bio/base/test/test_scripts.py
@@ -466,7 +466,7 @@ def test_evaluate():
# tests our 'evaluate' script using the reference files
test_dir = tempfile.mkdtemp(prefix='bobtest_')
reference_files = ('scores-nonorm-dev', 'scores-ztnorm-dev')
- plots = [os.path.join(test_dir, '%s.pdf')%f for f in ['roc', 'cmc', 'det']]
+ plots = [os.path.join(test_dir, '%s.pdf')%f for f in ['roc', 'cmc', 'det', 'epc']]
parameters = [
'--dev-files', reference_files[0], reference_files[1],
'--eval-files', reference_files[0], reference_files[1],
@@ -474,20 +474,27 @@ def test_evaluate():
'--legends', 'no norm', 'ZT norm',
'--criterion', 'HTER',
'--roc', plots[0],
- '--det', plots[1],
- '--cmc', plots[2],
+ '--cmc', plots[1],
+ '--det', plots[2],
+ '--epc', plots[3],
'--rr',
'--thresholds', '5000', '0',
+ '--min-far-value', '1e-6',
+ '--far-line-at', '1e-5',
'-v',
]
# execute the script
from bob.bio.base.script.evaluate import main
- main(parameters)
- for i in range(3):
- assert os.path.exists(plots[i])
- os.remove(plots[i])
- os.rmdir(test_dir)
+ try:
+ main(parameters)
+ for i in range(4):
+ assert os.path.exists(plots[i])
+ os.remove(plots[i])
+ finally:
+ if os.path.exists(test_dir):
+ shutil.rmtree(test_dir)
+
def test_resources():