Commit 797e257a authored by Amir MOHAMMADI's avatar Amir MOHAMMADI
Browse files

improve the evaluate script plots

parent 72c66d7b
Pipeline #11505 passed with stages
in 12 minutes and 4 seconds
......@@ -99,23 +99,29 @@ def command_line_arguments(command_line_parameters):
return args
def _plot_roc(frrs, colors, labels, title, fontsize=18, position=None):
if position is None: position = 4
def _plot_roc(frrs, colors, labels, title, fontsize=18, position=None, farfrrs=None):
if position is None: position = 'best'
figure = pyplot.figure()
# plot FAR and CAR for each algorithm
for i in range(len(frrs)):
pyplot.semilogx([100.0*f for f in frrs[i][0]], [100. - 100.0*f for f in frrs[i][1]], color=colors[i], lw=2, ms=10, mew=1.5, label=labels[i])
if farfrrs is not None:
pyplot.plot(farfrrs[i][0]*100, (1-farfrrs[i][1])*100, 'o', color=colors[i], markeredgecolor='black')
if farfrrs is not None:
pyplot.plot([x[0]*100 for x in farfrrs], [(1-x[1])*100 for x in farfrrs], '--', color='black', linewidth=1.5)
# finalize plot
pyplot.plot([0.1,0.1],[0,100], "--", color=(0.3,0.3,0.3))
if farfrrs is None:
pyplot.plot([0.1,0.1],[0,100], "--", color='black')
pyplot.axis([frrs[0][0][0]*100,100,0,100])
pyplot.xticks((0.01, 0.1, 1, 10, 100), ('0.01', '0.1', '1', '10', '100'))
pyplot.xlabel('FAR (%)')
pyplot.ylabel('CAR (%)')
pyplot.xlabel('FMR (%)')
pyplot.ylabel('1 - FNMR (%)')
pyplot.grid(True, color=(0.6,0.6,0.6))
pyplot.legend(loc=position, prop = {'size':fontsize})
pyplot.title(title)
figure.set_tight_layout(True)
pyplot.tight_layout()
return figure
......@@ -137,11 +143,11 @@ def _plot_det(dets, colors, labels, title, fontsize=18, position=None):
pyplot.yticks(ticks, labels)
pyplot.axis((ticks[0], ticks[-1], ticks[0], ticks[-1]))
pyplot.xlabel('FAR (%)')
pyplot.ylabel('FRR (%)')
pyplot.xlabel('FMR (%)')
pyplot.ylabel('FNMR (%)')
pyplot.legend(loc=position, prop = {'size':fontsize})
pyplot.title(title)
figure.set_tight_layout(True)
pyplot.tight_layout()
return figure
......@@ -171,7 +177,7 @@ def _plot_cmc(cmcs, colors, labels, title, fontsize=18, position=None):
def _plot_epc(scores_dev, scores_eval, colors, labels, title, fontsize=18, position=None):
if position is None: position = 4
if position is None: position = 'upper center'
# open new page for current plot
figure = pyplot.figure()
......@@ -186,11 +192,32 @@ def _plot_epc(scores_dev, scores_eval, colors, labels, title, fontsize=18, posit
pyplot.grid(True)
pyplot.legend(loc=position, prop = {'size':fontsize})
pyplot.title(title)
figure.set_tight_layout(True)
pyplot.xlim([-0.01, 1.01])
pyplot.ylim([0, 51])
pyplot.tight_layout()
return figure
def remove_nan(scores):
"""removes the NaNs from the scores"""
nans = numpy.isnan(scores)
sum_nans = sum(nans)
total = len(scores)
return scores[numpy.where(~nans)], sum_nans, total
def get_fta(scores):
"""calculates the Failure To Acquire (FtA) rate"""
fta_sum, fta_total = 0, 0
neg, sum_nans, total = remove_nan(scores[0])
fta_sum += sum_nans
fta_total += total
pos, sum_nans, total = remove_nan(scores[1])
fta_sum += sum_nans
fta_total += total
return (neg, pos, fta_sum * 100 / float(fta_total))
def main(command_line_parameters=None):
"""Reads score files, computes error measures and plots curves."""
......@@ -198,8 +225,12 @@ def main(command_line_parameters=None):
args = command_line_arguments(command_line_parameters)
# get some colors for plotting
cmap = pyplot.cm.get_cmap(name='hsv')
colors = [cmap(i) for i in numpy.linspace(0, 1.0, len(args.dev_files)+1)]
# cmap = pyplot.cm.get_cmap(name='magma')
# colors = [cmap(i) for i in numpy.linspace(0, 1.0, len(args.dev_files)+1)]
# matplotlib 2.0 default color cycler list: Vega category10 palette
colors = ['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728',
'#9467bd', '#8c564b', '#e377c2', '#7f7f7f',
'#bcbd22', '#17becf']
if args.criterion or args.roc or args.det or args.epc or args.cllr or args.mindcf:
score_parser = {'4column' : bob.measure.load.split_four_column, '5column' : bob.measure.load.split_five_column}[args.parser]
......@@ -207,10 +238,14 @@ def main(command_line_parameters=None):
# First, read the score files
logger.info("Loading %d score files of the development set", len(args.dev_files))
scores_dev = [score_parser(os.path.join(args.directory, f)) for f in args.dev_files]
# remove nans
scores_dev = [get_fta(s) for s in scores_dev]
if args.eval_files:
logger.info("Loading %d score files of the evaluation set", len(args.eval_files))
scores_eval = [score_parser(os.path.join(args.directory, f)) for f in args.eval_files]
# remove nans
scores_eval = [get_fta(s) for s in scores_eval]
if args.criterion:
......@@ -266,7 +301,7 @@ def main(command_line_parameters=None):
if args.roc:
logger.info("Computing CAR curves on the development " + ("and on the evaluation set" if args.eval_files else "set"))
fars = [math.pow(10., i * 0.25) for i in range(-16,0)] + [1.]
fars = [math.pow(10., i * 0.25) for i in range(-17,0)] + [1.]
frrs_dev = [bob.measure.roc_for_far(scores[0], scores[1], fars) for scores in scores_dev]
if args.eval_files:
frrs_eval = [bob.measure.roc_for_far(scores[0], scores[1], fars) for scores in scores_eval]
......@@ -276,10 +311,14 @@ def main(command_line_parameters=None):
# create a multi-page PDF for the ROC curve
pdf = PdfPages(args.roc)
# create a separate figure for dev and eval
pdf.savefig(_plot_roc(frrs_dev, colors, args.legends, args.title[0] if args.title is not None else "ROC curve for development set", args.legend_font_size, args.legend_position))
pdf.savefig(_plot_roc(frrs_dev, colors, args.legends, args.title[0] if args.title is not None else "ROC for development set", args.legend_font_size, args.legend_position), bbox_inches='tight')
del frrs_dev
if args.eval_files:
pdf.savefig(_plot_roc(frrs_eval, colors, args.legends, args.title[1] if args.title is not None else "ROC curve for evaluation set", args.legend_font_size, args.legend_position))
farfrrs = []
for i, scores in enumerate(scores_eval):
threshold = bob.measure.far_threshold(scores_dev[i][0], scores_dev[i][1], float(args.far_value) / 100)
farfrrs.append(bob.measure.farfrr(scores_eval[i][0], scores_eval[i][1], threshold))
pdf.savefig(_plot_roc(frrs_eval, colors, args.legends, args.title[1] if args.title is not None else "ROC for evaluation set", args.legend_font_size, args.legend_position, farfrrs), bbox_inches='tight')
del frrs_eval
pdf.close()
except RuntimeError as e:
......@@ -296,10 +335,10 @@ def main(command_line_parameters=None):
# create a multi-page PDF for the ROC curve
pdf = PdfPages(args.det)
# create a separate figure for dev and eval
pdf.savefig(_plot_det(dets_dev, colors, args.legends, args.title[0] if args.title is not None else "DET plot for development set", args.legend_font_size, args.legend_position))
pdf.savefig(_plot_det(dets_dev, colors, args.legends, args.title[0] if args.title is not None else "DET for development set", args.legend_font_size, args.legend_position), bbox_inches='tight')
del dets_dev
if args.eval_files:
pdf.savefig(_plot_det(dets_eval, colors, args.legends, args.title[1] if args.title is not None else "DET plot for evaluation set", args.legend_font_size, args.legend_position))
pdf.savefig(_plot_det(dets_eval, colors, args.legends, args.title[1] if args.title is not None else "DET for evaluation set", args.legend_font_size, args.legend_position), bbox_inches='tight')
del dets_eval
pdf.close()
except RuntimeError as e:
......@@ -315,7 +354,7 @@ def main(command_line_parameters=None):
try:
# create a multi-page PDF for the ROC curve
pdf = PdfPages(args.epc)
pdf.savefig(_plot_epc(scores_dev, scores_eval, colors, args.legends, args.title if args.title is not None else "EPC Curves" , args.legend_font_size, args.legend_position))
pdf.savefig(_plot_epc(scores_dev, scores_eval, colors, args.legends, args.title if args.title is not None else "" , args.legend_font_size, args.legend_position), bbox_inches='tight')
pdf.close()
except RuntimeError as e:
raise RuntimeError("During plotting of EPC curves, the following exception occured:\n%s\nUsually this happens when the label contains characters that LaTeX cannot parse." % e)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment