From ab9cd9e7122f1af3c3836fa6ce82ad85e46f333c Mon Sep 17 00:00:00 2001
From: Theophile GENTILHOMME <tgentilhomme@jurasix08.idiap.ch>
Date: Mon, 16 Apr 2018 14:38:45 +0200
Subject: [PATCH] move click options to bob.extension, change default axis for
ROC and DET to be ISO conform, add command to generate fakes score files, add
options to change title and axis labels
---
bob/measure/script/commands.py | 20 +++-
bob/measure/script/common_options.py | 143 +++++++++------------------
bob/measure/script/figure.py | 69 ++++++++-----
bob/measure/script/gen.py | 97 ++++++++++++++++++
bob/measure/test_script.py | 35 ++++---
setup.py | 1 +
6 files changed, 226 insertions(+), 139 deletions(-)
create mode 100644 bob/measure/script/gen.py
diff --git a/bob/measure/script/commands.py b/bob/measure/script/commands.py
index f081228..ae83cde 100644
--- a/bob/measure/script/commands.py
+++ b/bob/measure/script/commands.py
@@ -5,19 +5,19 @@ import click
from .. import load
from . import figure
from . import common_options
-from bob.extension.scripts.click_helper import verbosity_option
-
+from bob.extension.scripts.click_helper import (verbosity_option,
+ open_file_mode_option)
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.eval_option()
@common_options.table_option()
-@common_options.open_file_mode_option()
@common_options.output_plot_metric_option()
@common_options.criterion_option()
@common_options.thresholds_option()
@common_options.far_option()
@common_options.titles_option()
+@open_file_mode_option()
@verbosity_option()
@click.pass_context
def metrics(ctx, scores, evaluation, **kwargs):
@@ -45,6 +45,7 @@ def metrics(ctx, scores, evaluation, **kwargs):
@click.command()
@common_options.scores_argument(nargs=-1)
+@common_options.title_option()
@common_options.titles_option()
@common_options.sep_dev_eval_option()
@common_options.output_plot_file_option(default_out='roc.pdf')
@@ -54,7 +55,9 @@ def metrics(ctx, scores, evaluation, **kwargs):
@common_options.axes_val_option(dflt=[1e-4, 1, 1e-4, 1])
@common_options.axis_fontsize_option()
@common_options.x_rotation_option()
-@common_options.fmr_line_at_option()
+@common_options.x_label_option()
+@common_options.y_label_option()
+@common_options.lines_at_option()
@common_options.const_layout_option()
@verbosity_option()
@click.pass_context
@@ -83,12 +86,15 @@ def roc(ctx, scores, evaluation, **kwargs):
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.output_plot_file_option(default_out='det.pdf')
+@common_options.title_option()
@common_options.titles_option()
@common_options.sep_dev_eval_option()
@common_options.eval_option()
@common_options.axes_val_option(dflt=[0.01, 95, 0.01, 95])
@common_options.axis_fontsize_option(dflt=6)
@common_options.x_rotation_option(dflt=45)
+@common_options.x_label_option()
+@common_options.y_label_option()
@common_options.points_curve_option()
@common_options.const_layout_option()
@verbosity_option()
@@ -117,10 +123,13 @@ def det(ctx, scores, evaluation, **kwargs):
@click.command()
@common_options.scores_argument(eval_mandatory=True, nargs=-1)
@common_options.output_plot_file_option(default_out='epc.pdf')
+@common_options.title_option()
@common_options.titles_option()
@common_options.points_curve_option()
@common_options.axis_fontsize_option()
@common_options.const_layout_option()
+@common_options.x_label_option()
+@common_options.y_label_option()
@verbosity_option()
@click.pass_context
def epc(ctx, scores, **kwargs):
@@ -152,6 +161,7 @@ def epc(ctx, scores, **kwargs):
@common_options.const_layout_option()
@common_options.show_dev_option()
@common_options.print_filenames_option()
+@common_options.title_option()
@common_options.titles_option()
@verbosity_option()
@click.pass_context
@@ -191,7 +201,7 @@ def hist(ctx, scores, evaluation, **kwargs):
@common_options.points_curve_option()
@common_options.semilogx_option(dflt=True)
@common_options.n_bins_option()
-@common_options.fmr_line_at_option()
+@common_options.lines_at_option()
@common_options.const_layout_option()
@verbosity_option()
@click.pass_context
diff --git a/bob/measure/script/common_options.py b/bob/measure/script/common_options.py
index 9ac30bf..86aca83 100644
--- a/bob/measure/script/common_options.py
+++ b/bob/measure/script/common_options.py
@@ -7,7 +7,8 @@ import click
from click.types import INT, FLOAT, Choice, File
import matplotlib.pyplot as plt
from matplotlib.backends.backend_pdf import PdfPages
-from bob.extension.scripts.click_helper import verbosity_option
+from bob.extension.scripts.click_helper import (verbosity_option, bool_option,
+ list_float_option)
logger = logging.getLogger(__name__)
@@ -75,30 +76,6 @@ def scores_argument(eval_mandatory=False, min_len=1, **kwargs):
)(func)
return custom_scores_argument
-def bool_option(name, short_name, desc, dflt=False, **kwargs):
- '''Generic provider for boolean options
- Parameters
- ----------
-
- name: str
- name of the option
- short_name: str
- short name for the option
- desc: str
- short description for the option
- dflt: bool or None
- Default value
- '''
- def custom_bool_option(func):
- def callback(ctx, param, value):
- ctx.meta[name.replace('-', '_')] = value
- return value
- return click.option(
- '-' + short_name, '--%s/--no-%s' % (name, name), default=dflt,
- help=desc,
- show_default=True, callback=callback, is_eager=True, **kwargs)(func)
- return custom_bool_option
-
def eval_option(**kwargs):
'''Get option flag to say if eval-scores are provided'''
return bool_option(
@@ -134,47 +111,6 @@ def const_layout_option(dflt=True, **kwargs):
'''Option to set matplotlib constrained_layout'''
return bool_option('clayout', 'Y', '(De)Activate constrained layout', dflt)
-def list_float_option(name, short_name, desc, nitems=None, dflt=None, **kwargs):
- '''Get option to get a list of float f
- Parameters
- ----------
-
- name: str
- name of the option
- short_name: str
- short name for the option
- desc: str
- short description for the option
- nitems: :obj:`int`
- if given, the parsed list must contains this number of items
- dflt: :any:`list`
- List of default values for axes.
- '''
- def custom_list_float_option(func):
- def callback(ctx, param, value):
- if value is not None:
- tmp = value.split(',')
- if nitems is not None and len(tmp) != nitems:
- raise click.BadParameter(
- '%s Must provide %d axis limits' % (name, nitems)
- )
- try:
- value = [float(i) for i in tmp]
- except:
- raise click.BadParameter('Inputs of %s be floats' % name)
- if None in value:
- value = None
- elif dflt is not None and None not in dflt and len(dflt) == nitems:
- value = dflt if not all(
- isinstance(x, float) for x in dflt
- ) else None
- ctx.meta[name.replace('-', '_')] = value
- return value
- return click.option(
- '-'+short_name, '--'+name, default=None, show_default=True,
- help=desc, callback=callback, **kwargs)(func)
- return custom_list_float_option
-
def axes_val_option(dflt=None, **kwargs):
return list_float_option(
name='axlim', short_name='L',
@@ -189,6 +125,14 @@ def thresholds_option(**kwargs):
nitems=None, dflt=None, **kwargs
)
+def lines_at_option(**kwargs):
+ '''Get option to draw const far line'''
+ return list_float_option(
+ name='lines-at', short_name='la',
+ desc='If given, draw a veritcal lines on ROC plots',
+ nitems=None, dflt=None, **kwargs
+ )
+
def axis_fontsize_option(dflt=8, **kwargs):
'''Get option for axis font size'''
def custom_axis_fontsize_option(func):
@@ -215,19 +159,6 @@ def x_rotation_option(dflt=0, **kwargs):
callback=callback, **kwargs)(func)
return custom_x_rotation_option
-def fmr_line_at_option(**kwargs):
- '''Get option to draw const fmr line'''
- def custom_fmr_line_at_option(func):
- def callback(ctx, param, value):
- if value is not None:
- value = min(max(0.0, value), 1.0)
- ctx.meta['fmr_at'] = value
- return value
- return click.option(
- '-L', '--fmr-at', type=float, default=None, show_default=True,
- help='If given, draw a veritcal line for constant FMR on ROC plots',
- callback=callback, **kwargs)(func)
- return custom_fmr_line_at_option
def cost_option(**kwargs):
'''Get option to get cost for FAR'''
@@ -343,20 +274,6 @@ def output_plot_metric_option(**kwargs):
callback=callback, **kwargs)(func)
return custom_output_plot_file_option
-def open_file_mode_option(**kwargs):
- '''Get open mode file option'''
- def custom_open_file_mode_option(func):
- def callback(ctx, param, value):
- if value not in ['w', 'a', 'w+', 'a+']:
- raise click.BadParameter('Incorrect open file mode')
- ctx.meta['open_mode'] = value
- return value
- return click.option(
- '-om', '--open-mode', default='w',
- help='File open mode',
- callback=callback, **kwargs)(func)
- return custom_open_file_mode_option
-
def criterion_option(lcriteria=['eer', 'hter', 'far'], **kwargs):
"""Get option flag to tell which criteriom is used (default:eer)
@@ -494,7 +411,7 @@ def marker_style_option(**kwargs):
return custom_marker_style_option
def titles_option(**kwargs):
- '''Get the titles otpion for the different systems'''
+ '''Get the titles option for the different systems'''
def custom_titles_option(func):
def callback(ctx, param, value):
if value is not None:
@@ -502,12 +419,48 @@ def titles_option(**kwargs):
ctx.meta['titles'] = value
return value
return click.option(
- '-t', '--titles', type=click.STRING, default=None,
+ '-ts', '--titles', type=click.STRING, default=None,
help='The title for each system comma separated. '
'Example: --titles ISV,CNN',
callback=callback, **kwargs)(func)
return custom_titles_option
+def title_option(**kwargs):
+ '''Get the title option for the different systems'''
+ def custom_title_option(func):
+ def callback(ctx, param, value):
+ ctx.meta['title'] = value
+ return value
+ return click.option(
+ '-t', '--title', type=click.STRING, default=None,
+ help='The title of the plots',
+ callback=callback, **kwargs)(func)
+ return custom_title_option
+
+def x_label_option(dflt=None, **kwargs):
+ '''Get the label option for X axis '''
+ def custom_x_label_option(func):
+ def callback(ctx, param, value):
+ ctx.meta['x_label'] = value
+ return value
+ return click.option(
+ '-xl', '--x-lable', type=click.STRING, default=dflt,
+ help='Label for x-axis',
+ callback=callback, **kwargs)(func)
+ return custom_x_label_option
+
+def y_label_option(dflt=None, **kwargs):
+ '''Get the label option for Y axis '''
+ def custom_y_label_option(func):
+ def callback(ctx, param, value):
+ ctx.meta['y_label'] = value
+ return value
+ return click.option(
+ '-yl', '--y-lable', type=click.STRING, default=dflt,
+ help='Label for y-axis',
+ callback=callback, **kwargs)(func)
+ return custom_y_label_option
+
def style_option(**kwargs):
'''Get option for matplotlib style'''
def custom_style_option(func):
diff --git a/bob/measure/script/figure.py b/bob/measure/script/figure.py
index 7540ddb..2ce0afc 100644
--- a/bob/measure/script/figure.py
+++ b/bob/measure/script/figure.py
@@ -403,9 +403,11 @@ class PlotBase(MeasureBase):
self._multi_plots = len(self.dev_scores) > 1
self._colors = utils.get_colors(len(self.dev_scores))
self._states = ['Development', 'Evaluation']
- self._title = ''
- self._x_label = 'FMR (%)'
- self._y_label = 'FNMR (%)'
+ self._title = None if 'title' not in ctx.meta else ctx.meta['title']
+ self._x_label = None if 'x_label' not in ctx.meta else\
+ ctx.meta['x_label']
+ self._y_label = None if 'y_label' not in ctx.meta else\
+ ctx.meta['y_label']
self._grid_color = 'silver'
self._pdf_page = None
self._end_setup_plot = True
@@ -445,7 +447,7 @@ class PlotBase(MeasureBase):
mpl.xlabel(self._x_label)
mpl.ylabel(self._y_label)
mpl.grid(True, color=self._grid_color)
- mpl.legend()
+ mpl.legend(loc='best')
self._set_axis()
#gives warning when applied with mpl
fig.set_tight_layout(True)
@@ -490,14 +492,18 @@ class Roc(PlotBase):
super(Roc, self).__init__(ctx, scores, evaluation, func_load)
self._semilogx = True if 'semilogx' not in ctx.meta else\
ctx.meta['semilogx']
- self._fmr_at = None if 'fmr_at' not in ctx.meta else\
- ctx.meta['fmr_at']
- self._title = 'ROC'
- self._x_label = 'FMR'
- self._y_label = ("1 - FNMR" if self._semilogx else "FNMR")
+ self._far_at = None if 'lines_at' not in ctx.meta else\
+ ctx.meta['lines_at']
+ self._title = self._title or 'ROC'
+ self._x_label = self._x_label or 'False Positive Rate'
+ self._y_label = self._y_label or (
+ "1 - False Negative Rate" if self._semilogx else "False Negative Rate"
+ )
#custom defaults
if self._axlim is None:
self._axlim = [1e-4, 1.0, 1e-4, 1.0]
+ if self._far_at is not None:
+ self._eval_points = {line: [] for line in self._far_at}
def compute(self, idx, dev_score, dev_file=None,
eval_score=None, eval_file=None):
@@ -523,12 +529,14 @@ class Roc(PlotBase):
color=self._colors[idx], linestyle=linestyle,
label=self._label('eval', eval_file, idx, **self._kwargs)
)
- if self._fmr_at is not None:
+ if self._far_at is not None:
from .. import farfrr
- eval_fmr, eval_fnmr = farfrr(eval_neg, eval_pos, self._fmr_at)
- if self._semilogx:
- eval_fnmr = 1 - eval_fnmr
- mpl.scatter(eval_fmr, eval_fnmr, c=self._colors[idx], s=30)
+ for line in self._far_at:
+ eval_fmr, eval_fnmr = farfrr(eval_neg, eval_pos, line)
+ if self._semilogx:
+ eval_fnmr = 1 - eval_fnmr
+ mpl.scatter(eval_fmr, eval_fnmr, c=self._colors[idx], s=30)
+ self._eval_points[line].append((eval_fmr, eval_fnmr))
else:
plot.roc(
dev_neg, dev_pos, self._points, self._semilogx,
@@ -540,16 +548,31 @@ class Roc(PlotBase):
''' Draw vertical line on the dev plot at the given fmr and print the
corresponding points on the eval plot for all the systems '''
#draw vertical lines
- if self._fmr_at is not None:
- mpl.figure(1)
- mpl.plot([self._fmr_at, self._fmr_at], [0., 1.], "--", color='black')
+ if self._far_at is not None:
+ for line in self._far_at:
+ mpl.figure(1)
+ mpl.plot([line, line], [0., 1.], "--", color='black')
+ if self._eval and self._split:
+ mpl.figure(2)
+ x_values = [i for i, _ in self._eval_points[line]]
+ y_values = [j for _, j in self._eval_points[line]]
+ sort_indice = sorted(
+ range(len(x_values)), key=x_values.__getitem__
+ )
+ x_values = [x_values[i] for i in sort_indice]
+ y_values = [y_values[i] for i in sort_indice]
+ mpl.plot(x_values,
+ y_values, '--',
+ color='black')
super(Roc, self).end_process()
class Det(PlotBase):
''' Handles the plotting of DET '''
def __init__(self, ctx, scores, evaluation, func_load):
super(Det, self).__init__(ctx, scores, evaluation, func_load)
- self._title = 'DET'
+ self._title = self._title or 'DET'
+ self._x_label = self._x_label or 'False Positive Rate'
+ self._y_label = self._y_label or 'False Negative Rate'
#custom defaults here
if self._x_rotation is None:
self._x_rotation = 50
@@ -595,9 +618,9 @@ class Epc(PlotBase):
super(Epc, self).__init__(ctx, scores, evaluation, func_load)
if 'eval_scores_0' not in self._ctx.meta:
raise click.UsageError("EPC requires dev and eval score files")
- self._title = 'EPC'
- self._x_label = 'Cost'
- self._y_label = 'Min. HTER (%)'
+ self._title = self._title or 'EPC'
+ self._x_label = self._x_label or 'Cost'
+ self._y_label = self._y_label or 'Min. HTER (%)'
self._eval = True #always eval data with EPC
self._split = False
self._nb_figs = 1
@@ -645,9 +668,9 @@ class Hist(PlotBase):
)
self._criter = None if 'criter' not in ctx.meta else ctx.meta['criter']
self._y_label = 'Dev. probability density' if self._eval else \
- 'density'
+ 'density' or self._y_label
self._x_label = 'Scores' if not self._eval else ''
- self._title_base = 'Scores'
+ self._title_base = self._title or 'Scores'
self._end_setup_plot = False
def compute(self, idx, dev_score, dev_file=None,
diff --git a/bob/measure/script/gen.py b/bob/measure/script/gen.py
new file mode 100644
index 0000000..bab9195
--- /dev/null
+++ b/bob/measure/script/gen.py
@@ -0,0 +1,97 @@
+"""Generate random scores.
+"""
+import pkg_resources # to make sure bob gets imported properly
+import os
+import logging
+import numpy
+import click
+from click.types import FLOAT
+from bob.extension.scripts.click_helper import verbosity_option
+from bob.core import random
+from bob.io.base import create_directories_safe
+
+logger = logging.getLogger(__name__)
+
+NUM_NEG = 5000
+NUM_POS = 5000
+
+def gen_score_distr(mean_neg, mean_pos, sigma_neg=1, sigma_pos=1):
+ """Generate scores from normal distributions
+
+ Parameters
+ ----------
+
+ mean_neg : float
+ Mean for negative scores
+ mean_pos : float
+ Mean for positive scores
+ sigma_neg : float
+ STDev for negative scores
+ sigma_pos : float
+ STDev for positive scores
+
+ Returns
+ -------
+
+ neg_scores : array_like
+ Negatives scores
+ pos_scores : array_like
+ Positive scores
+ """
+ mt = random.mt19937() # initialise the random number generator
+
+ neg_generator = random.normal(numpy.float32, mean_neg, sigma_neg)
+ pos_generator = random.normal(numpy.float32, mean_pos, sigma_pos)
+
+ neg_scores = [neg_generator(mt) for _ in range(NUM_NEG)]
+ pos_scores = [pos_generator(mt) for _ in range(NUM_NEG)]
+
+ return neg_scores, pos_scores
+
+
+def write_scores_to_file(pos, neg, filename):
+ """Writes score distributions into 2-column score files. For the format of
+ the 2-column score files, please refer to Bob's documentation. See
+ :py:func:`bob.measure.load.split`.
+
+ Parameters
+ ----------
+ pos : array_like
+ Scores for positive samples.
+ neg : array_like
+ Scores for negative samples.
+ filename : str
+ The path to write the score to.
+ """
+ create_directories_safe(os.path.dirname(filename))
+ mt = random.mt19937()
+ nan_dist = random.uniform(numpy.float32, 0, 1)
+ with open(filename, 'wt') as f:
+ for i in pos:
+ text = '1 %f\n' % i if nan_dist(mt) > 0.01 else '1 nan\n'
+ f.write(text)
+ for i in neg:
+ text = '-1 %f\n' % i if nan_dist(mt) > 0.01 else '1 nan\n'
+ f.write(text)
+
+@click.command()
+@click.argument('outdir')
+@click.option('--mean-neg', default=-1, type=FLOAT, show_default=True)
+@click.option('--mean-pos', default=1, type=FLOAT, show_default=True)
+@verbosity_option()
+def gen(outdir, mean_neg, mean_pos):
+ """Generate random scores.
+ Generates random scores for negative and positive scores, whatever they
+ could be. The
+ scores are generated using Gaussian distribution whose mean is an input
+ parameter. The generated scores can be used as hypothetical datasets.
+ """
+ # Generate the data
+ neg_dev, pos_dev = gen_score_distr(mean_neg, mean_pos)
+ neg_eval, pos_eval = gen_score_distr(mean_neg, mean_pos)
+
+ # Write the data into files
+ write_scores_to_file(neg_dev, pos_dev,
+ os.path.join(outdir, 'scores-dev'))
+ write_scores_to_file(neg_eval, pos_eval,
+ os.path.join(outdir, 'scores-eval'))
diff --git a/bob/measure/test_script.py b/bob/measure/test_script.py
index 4eb790e..dd29f46 100644
--- a/bob/measure/test_script.py
+++ b/bob/measure/test_script.py
@@ -15,7 +15,7 @@ def test_metrics():
with open('tmp', 'w') as f:
f.write(result.output)
test_ref = bob.io.base.test_utils.datafile('test_m1.txt', 'bob.measure')
- assert result.exit_code == 0
+ assert result.exit_code == 0, (result.exit_code, result.output)
dev2 = bob.io.base.test_utils.datafile('dev-2.txt', 'bob.measure')
test1 = bob.io.base.test_utils.datafile('test-1.txt', 'bob.measure')
@@ -30,15 +30,16 @@ def test_metrics():
assert result.exit_code == 0
with runner.isolated_filesystem():
result = runner.invoke(
- commands.metrics, ['-l', 'tmp', dev1, test1, dev2, test2, '-t',
+ commands.metrics, ['-l', 'tmp', dev1, test1, dev2, test2, '-ts',
'A,B']
)
- assert result.exit_code == 0
+ assert result.exit_code == 0, (result.exit_code, result.output)
+
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['-l', 'tmp', '--no-evaluation', dev1, dev2]
)
- assert result.exit_code == 0
+ assert result.exit_code == 0, (result.exit_code, result.output)
def test_roc():
dev1 = bob.io.base.test_utils.datafile('dev-1.txt', 'bob.measure')
@@ -58,7 +59,7 @@ def test_roc():
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
- assert result.exit_code == 0
+ assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(commands.roc, ['--output',
@@ -66,7 +67,7 @@ def test_roc():
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
- assert result.exit_code == 0
+ assert result.exit_code == 0, (result.exit_code, result.output)
def test_det():
@@ -86,14 +87,15 @@ def test_det():
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
- assert result.exit_code == 0
+ assert result.exit_code == 0, (result.exit_code, result.output)
+
with runner.isolated_filesystem():
result = runner.invoke(commands.det, ['--output',
'test.pdf',
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
- assert result.exit_code == 0
+ assert result.exit_code == 0, (result.exit_code, result.output)
def test_epc():
dev1 = bob.io.base.test_utils.datafile('dev-1.txt', 'bob.measure')
@@ -103,7 +105,8 @@ def test_epc():
result = runner.invoke(commands.epc, [dev1, test1])
if result.output:
click.echo(result.output)
- assert result.exit_code == 0
+ assert result.exit_code == 0, (result.exit_code, result.output)
+
dev2 = bob.io.base.test_utils.datafile('dev-2.txt', 'bob.measure')
test2 = bob.io.base.test_utils.datafile('test-2.txt', 'bob.measure')
with runner.isolated_filesystem():
@@ -112,7 +115,7 @@ def test_epc():
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
- assert result.exit_code == 0
+ assert result.exit_code == 0, (result.exit_code, result.output)
def test_hist():
dev1 = bob.io.base.test_utils.datafile('dev-1.txt', 'bob.measure')
@@ -124,7 +127,7 @@ def test_hist():
result = runner.invoke(commands.hist, ['--no-evaluation', dev1])
if result.output:
click.echo(result.output)
- assert result.exit_code == 0
+ assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(commands.hist, ['--no-evaluation', '--criter', 'hter',
@@ -132,7 +135,7 @@ def test_hist():
30, dev1, dev2])
if result.output:
click.echo(result.output)
- assert result.exit_code == 0
+ assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(commands.hist, ['--criter', 'eer','--output',
@@ -140,7 +143,7 @@ def test_hist():
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
- assert result.exit_code == 0
+ assert result.exit_code == 0, (result.exit_code, result.output)
def test_evaluate():
@@ -151,15 +154,15 @@ def test_evaluate():
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(commands.evaluate, ['--no-evaluation', dev1])
- assert result.exit_code == 0
+ assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(
commands.evaluate, ['--no-evaluation', '--output', 'my_plots.pdf', '-b',
30, '-n', 300, dev1, dev2])
- assert result.exit_code == 0
+ assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(
commands.evaluate, [dev1, test1, dev2, test2])
- assert result.exit_code == 0
+ assert result.exit_code == 0, (result.exit_code, result.output)
diff --git a/setup.py b/setup.py
index 6b2d558..6602182 100644
--- a/setup.py
+++ b/setup.py
@@ -78,6 +78,7 @@ setup(
'det = bob.measure.script.commands:det',
'epc = bob.measure.script.commands:epc',
'hist = bob.measure.script.commands:hist',
+ 'gen = bob.measure.script.gen:gen',
],
},
--
GitLab