Commit 25a71f54 authored by Amir MOHAMMADI's avatar Amir MOHAMMADI

Merge branch 'various' into 'master'

Various improvements

Closes #41 and #42

See merge request !72
parents 36b268e9 7667556d
Pipeline #21200 passed with stages
in 15 minutes and 45 seconds
...@@ -171,6 +171,7 @@ def epc(ctx, scores, **kwargs): ...@@ -171,6 +171,7 @@ def epc(ctx, scores, **kwargs):
@common_options.no_legend_option() @common_options.no_legend_option()
@common_options.criterion_option() @common_options.criterion_option()
@common_options.no_line_option() @common_options.no_line_option()
@common_options.hide_dev_option()
@common_options.far_option() @common_options.far_option()
@common_options.thresholds_option() @common_options.thresholds_option()
@common_options.const_layout_option() @common_options.const_layout_option()
...@@ -213,6 +214,7 @@ def hist(ctx, scores, evaluation, **kwargs): ...@@ -213,6 +214,7 @@ def hist(ctx, scores, evaluation, **kwargs):
@common_options.sep_dev_eval_option() @common_options.sep_dev_eval_option()
@common_options.table_option() @common_options.table_option()
@common_options.eval_option() @common_options.eval_option()
@common_options.criterion_option()
@common_options.output_log_metric_option() @common_options.output_log_metric_option()
@common_options.output_plot_file_option(default_out='eval_plots.pdf') @common_options.output_plot_file_option(default_out='eval_plots.pdf')
@common_options.points_curve_option() @common_options.points_curve_option()
...@@ -254,14 +256,11 @@ def evaluate(ctx, scores, evaluation, **kwargs): ...@@ -254,14 +256,11 @@ def evaluate(ctx, scores, evaluation, **kwargs):
''' '''
# first time erase if existing file # first time erase if existing file
ctx.meta['open_mode'] = 'w' ctx.meta['open_mode'] = 'w'
click.echo("Computing metrics with EER...") criterion = ctx.meta.get('criterion')
ctx.meta['criterion'] = 'eer' # no criterion passed to evaluate if criterion is not None:
ctx.invoke(metrics, scores=scores, evaluation=evaluation) click.echo("Computing metrics with %s..." % criterion)
# second time, appends the content ctx.invoke(metrics, scores=scores, evaluation=evaluation)
ctx.meta['open_mode'] = 'a'
click.echo("Computing metrics with min-HTER...")
ctx.meta['criterion'] = 'min-hter' # no criterion passed in evaluate
ctx.invoke(metrics, scores=scores, evaluation=evaluation)
if 'log' in ctx.meta: if 'log' in ctx.meta:
click.echo("[metrics] => %s" % ctx.meta['log']) click.echo("[metrics] => %s" % ctx.meta['log'])
......
...@@ -79,6 +79,18 @@ def eval_option(**kwargs): ...@@ -79,6 +79,18 @@ def eval_option(**kwargs):
return custom_eval_option return custom_eval_option
def hide_dev_option(dflt=False, **kwargs):
'''Get option flag to say if dev plot should be hidden'''
def custom_hide_dev_option(func):
def callback(ctx, param, value):
ctx.meta['hide_dev'] = value
return value
return click.option(
'--hide-dev', is_flag=True, default=dflt,
show_default=True,
help='If set, hide dev related plots',
callback=callback, **kwargs)(func)
return custom_hide_dev_option
def sep_dev_eval_option(dflt=True, **kwargs): def sep_dev_eval_option(dflt=True, **kwargs):
......
...@@ -570,6 +570,9 @@ class Hist(PlotBase): ...@@ -570,6 +570,9 @@ class Hist(PlotBase):
self._no_line = ctx.meta.get('no_line', False) self._no_line = ctx.meta.get('no_line', False)
self._nrows = ctx.meta.get('n_row', 1) self._nrows = ctx.meta.get('n_row', 1)
self._ncols = ctx.meta.get('n_col', 1) self._ncols = ctx.meta.get('n_col', 1)
self._hide_dev = ctx.meta.get('hide_dev', False)
# dev hist are displayed next to eval hist
self._ncols *= 1 if self._hide_dev else 2
self._nlegends = ctx.meta.get('legends_ncol', 10) self._nlegends = ctx.meta.get('legends_ncol', 10)
self._legend_loc = self._legend_loc or 'upper center' self._legend_loc = self._legend_loc or 'upper center'
self._step_print = int(self._nrows * self._ncols) self._step_print = int(self._nrows * self._ncols)
...@@ -582,14 +585,23 @@ class Hist(PlotBase): ...@@ -582,14 +585,23 @@ class Hist(PlotBase):
''' Draw histograms of negative and positive scores.''' ''' Draw histograms of negative and positive scores.'''
dev_neg, dev_pos, eval_neg, eval_pos, threshold = \ dev_neg, dev_pos, eval_neg, eval_pos, threshold = \
self._get_neg_pos_thres(idx, input_scores, input_names) self._get_neg_pos_thres(idx, input_scores, input_names)
dev_file = input_names[0] idx *= 1 if self._hide_dev or not self._eval else 2
eval_file = None if len(input_names) != 2 else input_names[1]
if not self._hide_dev or not self._eval:
self._print_subplot(idx, dev_neg, dev_pos, threshold, False,
dflt_title="Dev scores")
idx += 1 if self._eval and not self._hide_dev else 0
if self._eval:
self._print_subplot(idx, eval_neg, eval_pos, threshold,
not self._no_line, dflt_title="Eval scores")
def _print_subplot(self, idx, neg, pos, threshold, draw_line, dflt_title):
n = idx % self._step_print n = idx % self._step_print
col = n % self._ncols col = n % self._ncols
sub_plot_idx = n + 1 sub_plot_idx = n + 1
axis = mpl.subplot(self._nrows, self._ncols, sub_plot_idx) axis = mpl.subplot(self._nrows, self._ncols, sub_plot_idx)
neg = eval_neg if eval_neg is not None else dev_neg
pos = eval_pos if eval_pos is not None else dev_pos
self._setup_hist(neg, pos) self._setup_hist(neg, pos)
if col == 0: if col == 0:
axis.set_ylabel(self._y_label) axis.set_ylabel(self._y_label)
...@@ -598,23 +610,25 @@ class Hist(PlotBase): ...@@ -598,23 +610,25 @@ class Hist(PlotBase):
int(idx / self._step_print) * self._step_print int(idx / self._step_print) * self._step_print
if n + self._ncols >= min(self._step_print, rest_print): if n + self._ncols >= min(self._step_print, rest_print):
axis.set_xlabel(self._x_label) axis.set_xlabel(self._x_label)
axis.set_title(self._get_title(idx, dev_file, eval_file)) axis.set_title(self._get_title(idx, dflt_title))
label = "%s threshold%s" % ( label = "%s threshold%s" % (
'' if self._criterion is None else '' if self._criterion is None else
self._criterion.upper(), ' (dev)' if self._eval else '' self._criterion.upper(), ' (dev)' if self._eval else ''
) )
if self._eval and not self._no_line: if draw_line:
self._lines(threshold, label, neg, pos, idx) self._lines(threshold, label, neg, pos, idx)
if sub_plot_idx == 1: if sub_plot_idx == 1:
self._plot_legends() self._plot_legends()
if self._step_print == sub_plot_idx or idx == self.n_systems - 1: mult = 2 if self._eval and not self._hide_dev else 1
if self._step_print == sub_plot_idx or idx == self.n_systems * mult - 1:
mpl.tight_layout() mpl.tight_layout()
self._pdf_page.savefig(mpl.gcf(), bbox_inches='tight') self._pdf_page.savefig(mpl.gcf(), bbox_inches='tight')
mpl.clf() mpl.clf()
mpl.figure() mpl.figure()
def _get_title(self, idx, dev_file, eval_file): def _get_title(self, idx, dflt=None):
title = self._legends[idx] if self._legends is not None else None title = self._legends[idx] if self._legends is not None \
and idx < len(self._legends) else dflt
title = title or self._title_base title = title or self._title_base
title = '' if title is not None and not title.replace( title = '' if title is not None and not title.replace(
' ', '') else title ' ', '') else title
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment