From 3e8418977d2abaea27ccd075dc93b17261a024ed Mon Sep 17 00:00:00 2001
From: Theophile GENTILHOMME <tgentilhomme@jurasix08.idiap.ch>
Date: Mon, 16 Apr 2018 14:47:27 +0200
Subject: [PATCH] Add options for axis labels, modifications accoding to the
move of click options to bob.extension, print assert outputs for tests
---
bob/bio/base/script/commands.py | 17 +++--
bob/bio/base/script/figure.py | 28 ++++++--
bob/bio/base/script/gen.py | 102 +++++++++++++++++++++++++++++
bob/bio/base/test/test_commands.py | 62 +++++++++---------
setup.py | 1 +
5 files changed, 167 insertions(+), 43 deletions(-)
create mode 100644 bob/bio/base/script/gen.py
diff --git a/bob/bio/base/script/commands.py b/bob/bio/base/script/commands.py
index 7dfc1665..3f35444c 100644
--- a/bob/bio/base/script/commands.py
+++ b/bob/bio/base/script/commands.py
@@ -6,7 +6,8 @@ import bob.bio.base.script.figure as bio_figure
import bob.measure.script.figure as measure_figure
from ..score import load
from bob.measure.script import common_options
-from bob.extension.scripts.click_helper import verbosity_option
+from bob.extension.scripts.click_helper import (verbosity_option,
+ open_file_mode_option)
FUNC_SPLIT = lambda x: load.load_files(x, load.split)
FUNC_CMC = lambda x: load.load_files(x, load.cmc)
@@ -15,13 +16,13 @@ FUNC_CMC = lambda x: load.load_files(x, load.cmc)
@common_options.scores_argument(nargs=-1)
@common_options.table_option()
@common_options.eval_option()
-@common_options.open_file_mode_option()
@common_options.output_plot_metric_option()
@common_options.criterion_option(['eer', 'hter', 'far', 'mindcf', 'cllr', 'rr'])
@common_options.cost_option()
@common_options.thresholds_option()
@common_options.far_option()
@common_options.titles_option()
+@open_file_mode_option()
@verbosity_option()
@click.pass_context
def metrics(ctx, scores, evaluation, **kargs):
@@ -65,7 +66,9 @@ def metrics(ctx, scores, evaluation, **kargs):
@common_options.axes_val_option(dflt=[1e-4, 1, 1e-4, 1])
@common_options.axis_fontsize_option()
@common_options.x_rotation_option()
-@common_options.fmr_line_at_option()
+@common_options.lines_at_option()
+@common_options.x_label_option()
+@common_options.y_label_option()
@common_options.const_layout_option()
@verbosity_option()
@click.pass_context
@@ -90,13 +93,15 @@ def roc(ctx, scores, evaluation, **kargs):
$ bob bio roc -o my_roc.pdf dev-scores1 eval-scores1
"""
- process = measure_figure.Roc(ctx, scores, evaluation, FUNC_SPLIT)
+ process = bio_figure.Roc(ctx, scores, evaluation, FUNC_SPLIT)
process.run()
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.output_plot_file_option(default_out='det.pdf')
@common_options.titles_option()
+@common_options.x_label_option()
+@common_options.y_label_option()
@common_options.sep_dev_eval_option()
@common_options.eval_option()
@common_options.axis_fontsize_option(dflt=6)
@@ -126,7 +131,7 @@ def det(ctx, scores, evaluation, **kargs):
$ bob bio det -o my_det.pdf dev-scores1 eval-scores1
"""
- process = measure_figure.Det(ctx, scores, evaluation, FUNC_SPLIT)
+ process = bio_figure.Det(ctx, scores, evaluation, FUNC_SPLIT)
process.run()
@click.command()
@@ -294,7 +299,7 @@ def hist(ctx, scores, evaluation, **kwargs):
@common_options.output_plot_metric_option()
@common_options.output_plot_file_option(default_out='eval_plots.pdf')
@common_options.points_curve_option()
-@common_options.fmr_line_at_option()
+@common_options.lines_at_option()
@common_options.cost_option()
@common_options.rank_option()
@common_options.far_option()
diff --git a/bob/bio/base/script/figure.py b/bob/bio/base/script/figure.py
index 950586d3..e6c82756 100644
--- a/bob/bio/base/script/figure.py
+++ b/bob/bio/base/script/figure.py
@@ -7,6 +7,22 @@ import bob.measure
from bob.measure import plot
from tabulate import tabulate
+class Roc(measure_figure.Roc):
+ def __init__(self, ctx, scores, evaluation, func_load):
+ super(Roc, self).__init__(ctx, scores, evaluation, func_load)
+ self._x_label = 'False Match Rate' if 'x_label' not in ctx.meta or \
+ ctx.meta['x_label'] is None else ctx.meta['x_label']
+ self._y_label = '1 - False Non Match Rate' if 'y_label' not in \
+ ctx.meta or ctx.meta['y_label'] is None else ctx.meta['y_label']
+
+class Det(measure_figure.Det):
+ def __init__(self, ctx, scores, evaluation, func_load):
+ super(Det, self).__init__(ctx, scores, evaluation, func_load)
+ self._x_label = 'False Match Rate' if 'x_label' not in ctx.meta or \
+ ctx.meta['x_label'] is None else ctx.meta['x_label']
+ self._y_label = 'False Non Match Rate' if 'y_label' not in ctx.meta or\
+ ctx.meta['y_label'] is None else ctx.meta['y_label']
+
class Cmc(measure_figure.PlotBase):
''' Handles the plotting of Cmc
@@ -20,9 +36,9 @@ class Cmc(measure_figure.PlotBase):
super(Cmc, self).__init__(ctx, scores, evaluation, func_load)
self._semilogx = True if 'semilogx' not in ctx.meta else\
ctx.meta['semilogx']
- self._title = 'CMC'
- self._x_label = 'Rank'
- self._y_label = 'Probability'
+ self._title = self._title or 'CMC'
+ self._x_label = self._x_label or 'Rank'
+ self._y_label = self._y_label or 'Identification rate'
self._max_R = 0
def compute(self, idx, dev_score, dev_file=None,
@@ -79,9 +95,9 @@ class Dic(measure_figure.PlotBase):
self._semilogx = True if 'semilogx' not in ctx.meta else\
ctx.meta['semilogx']
self._rank = 1 if 'rank' not in ctx.meta else ctx.meta['rank']
- self._title = 'DIC'
- self._x_label = 'FAR'
- self._y_label = 'DIR'
+ self._title = self._title or 'DIC'
+ self._x_label = self._title or 'FAR'
+ self._y_label = self._title or 'DIR'
def compute(self, idx, dev_score, dev_file=None,
eval_score=None, eval_file=None):
diff --git a/bob/bio/base/script/gen.py b/bob/bio/base/script/gen.py
new file mode 100644
index 00000000..479ced91
--- /dev/null
+++ b/bob/bio/base/script/gen.py
@@ -0,0 +1,102 @@
+"""Generate random scores.
+"""
+import pkg_resources # to make sure bob gets imported properly
+import os
+import logging
+import numpy
+import random as rd
+import click
+from click.types import FLOAT
+from bob.extension.scripts.click_helper import verbosity_option
+from bob.core import random
+from bob.io.base import create_directories_safe
+
+logger = logging.getLogger(__name__)
+
+NUM_NEG = 5000
+NUM_POS = 5000
+
+def gen_score_distr(mean_neg, mean_pos, sigma_neg=10, sigma_pos=10):
+ """Generate scores from normal distributions
+
+ Parameters
+ ----------
+
+ mean_neg : float
+ Mean for negative scores
+ mean_pos : float
+ Mean for positive scores
+ sigma_neg : float
+ STDev for negative scores
+ sigma_pos : float
+ STDev for positive scores
+
+ Returns
+ -------
+
+ neg_scores : array_like
+ Negatives scores
+ pos_scores : array_like
+ Positive scores
+ """
+ mt = random.mt19937() # initialise the random number generator
+
+ neg_generator = random.normal(numpy.float32, mean_neg, sigma_neg)
+ pos_generator = random.normal(numpy.float32, mean_pos, sigma_pos)
+
+ neg_scores = [neg_generator(mt) for _ in range(NUM_NEG)]
+ pos_scores = [pos_generator(mt) for _ in range(NUM_NEG)]
+
+ return neg_scores, pos_scores
+
+def write_scores_to_file(pos, neg, filename, n_sys=1, five_col=False):
+ """ Writes score distributions
+
+ Parameters
+ ----------
+
+ pos : array_like
+ Scores for positive samples.
+ neg : array_like
+ Scores for negative samples.
+ filename : str
+ The path to write the score to.
+ n_sys : int
+ Number of different systems
+ five_col : bool
+ If 5-colum format, else 4-column
+ """
+ create_directories_safe(os.path.dirname(filename))
+ s_names = ['s%d' % i for i in range(n_sys)]
+ with open(filename, 'wt') as f:
+ for i in pos:
+ s_name = rd.choice(s_names)
+ s_five = ' ' if not five_col else ' d' + rd.choice(s_names) + ' '
+ f.write('x%sx %s %f\n' % (s_five, s_name, i))
+ for i in neg:
+ s_name = rd.choice(s_names)
+ s_five = ' ' if not five_col else ' d' + rd.choice(s_names) + ' '
+ f.write('x%sy %s %f\n' % (s_five, s_name, i))
+
+@click.command()
+@click.argument('outdir')
+@click.option('-mm', '--mean-match', default=10, type=FLOAT, show_default=True)
+@click.option('-mnm', '--mean-non-match', default=-10, type=FLOAT, show_default=True)
+@click.option('-n', '--n-sys', default=1, type=click.INT, show_default=True)
+@click.option('--five-col/--four-col', default=False, show_default=True)
+@verbosity_option()
+def gen(outdir, mean_match, mean_non_match, n_sys, five_col):
+ """Generate random scores.
+ Generates random scores in 4col or 5col format. The scores are generated
+ using Gaussian distribution whose mean is an input
+ parameter. The generated scores can be used as hypothetical datasets.
+ """
+ # Generate the data
+ neg_dev, pos_dev = gen_score_distr(mean_non_match, mean_match)
+ neg_eval, pos_eval = gen_score_distr(mean_non_match, mean_match)
+
+ # Write the data into files
+ write_scores_to_file(neg_dev, pos_dev,
+ os.path.join(outdir, 'scores-dev'), n_sys, five_col)
+ write_scores_to_file(neg_eval, pos_eval,
+ os.path.join(outdir, 'scores-eval'), n_sys, five_col)
diff --git a/bob/bio/base/test/test_commands.py b/bob/bio/base/test/test_commands.py
index 7c018f0a..49a21709 100644
--- a/bob/bio/base/test/test_commands.py
+++ b/bob/bio/base/test/test_commands.py
@@ -15,7 +15,7 @@ def test_metrics():
with runner.isolated_filesystem():
with open('tmp', 'w') as f:
f.write(result.output)
- assert result.exit_code == 0
+ assert result.exit_code == 0, (result.exit_code, result.output)
dev2 = pkg_resources.resource_filename('bob.bio.base.test',
'data/dev-5col.txt')
test1 = pkg_resources.resource_filename('bob.bio.base.test',
@@ -28,19 +28,19 @@ def test_metrics():
)
with open('tmp', 'w') as f:
f.write(result.output)
- assert result.exit_code == 0
+ assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(
- commands.metrics, ['-l', 'tmp', '-t', 'A,B',
+ commands.metrics, ['-l', 'tmp', '-ts', 'A,B',
dev1, test1, dev2, test2]
)
- assert result.exit_code == 0
+ assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['-l', 'tmp', dev1, test2]
)
- assert result.exit_code == 0
+ assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(
@@ -48,7 +48,7 @@ def test_metrics():
'--criter', 'mindcf', '--cost', 0.9,
dev1, test2]
)
- assert result.exit_code == 0
+ assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(
@@ -56,34 +56,34 @@ def test_metrics():
'--criter', 'mindcf', '--cost', 0.9,
dev1]
)
- assert result.exit_code == 0
+ assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['--criter', 'cllr', dev1, test2]
)
- assert result.exit_code == 0
+ assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['--no-evaluation', '-l', 'tmp', '--criter', 'cllr',
'--cost', 0.9, dev1]
)
- assert result.exit_code == 0
+ assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['--criter', 'rr', '-T',
'0.1', dev1, test2]
)
- assert result.exit_code == 0
+ assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['--no-evaluation', '-l', 'tmp', '--criter', 'rr',
dev1, dev2]
)
- assert result.exit_code == 0
+ assert result.exit_code == 0, (result.exit_code, result.output)
def test_roc():
@@ -95,7 +95,7 @@ def test_roc():
'test.pdf',dev1])
if result.output:
click.echo(result.output)
- assert result.exit_code == 0
+ assert result.exit_code == 0, (result.exit_code, result.output)
dev2 = pkg_resources.resource_filename('bob.bio.base.test',
'data/dev-5col.txt')
test1 = pkg_resources.resource_filename('bob.bio.base.test',
@@ -108,7 +108,7 @@ def test_roc():
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
- assert result.exit_code == 0
+ assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(commands.roc, ['--output',
@@ -116,7 +116,7 @@ def test_roc():
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
- assert result.exit_code == 0
+ assert result.exit_code == 0, (result.exit_code, result.output)
def test_det():
@@ -127,7 +127,7 @@ def test_det():
result = runner.invoke(commands.det, ['--no-evaluation', dev1])
if result.output:
click.echo(result.output)
- assert result.exit_code == 0
+ assert result.exit_code == 0, (result.exit_code, result.output)
dev2 = pkg_resources.resource_filename('bob.bio.base.test',
'data/dev-5col.txt')
test1 = pkg_resources.resource_filename('bob.bio.base.test',
@@ -140,14 +140,14 @@ def test_det():
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
- assert result.exit_code == 0
+ assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(commands.det, ['--output',
'test.pdf',
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
- assert result.exit_code == 0
+ assert result.exit_code == 0, (result.exit_code, result.output)
def test_epc():
dev1 = pkg_resources.resource_filename('bob.bio.base.test',
@@ -159,7 +159,7 @@ def test_epc():
result = runner.invoke(commands.epc, [dev1, test1])
if result.output:
click.echo(result.output)
- assert result.exit_code == 0
+ assert result.exit_code == 0, (result.exit_code, result.output)
dev2 = pkg_resources.resource_filename('bob.bio.base.test',
'data/dev-4col.tar.gz')
test2 = pkg_resources.resource_filename('bob.bio.base.test',
@@ -170,7 +170,7 @@ def test_epc():
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
- assert result.exit_code == 0
+ assert result.exit_code == 0, (result.exit_code, result.output)
def test_hist():
dev1 = pkg_resources.resource_filename('bob.bio.base.test',
@@ -186,7 +186,7 @@ def test_hist():
result = runner.invoke(commands.hist, ['--no-evaluation', dev1])
if result.output:
click.echo(result.output)
- assert result.exit_code == 0
+ assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(commands.hist, ['--criter', 'hter', '--output',
@@ -194,16 +194,16 @@ def test_hist():
30,'--no-evaluation', dev1, dev2])
if result.output:
click.echo(result.output)
- assert result.exit_code == 0
+ assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(commands.hist, ['--criter', 'eer', '--output',
'HISTO.pdf', '-b', 30, '-F', 3,
- '-t', 'A,B', dev1, test1, dev2,
+ '-ts', 'A,B', dev1, test1, dev2,
test2])
if result.output:
click.echo(result.output)
- assert result.exit_code == 0
+ assert result.exit_code == 0, (result.exit_code, result.output)
def test_cmc():
dev1 = pkg_resources.resource_filename('bob.bio.base.test',
@@ -213,7 +213,7 @@ def test_cmc():
result = runner.invoke(commands.cmc, ['--no-evaluation', dev1])
if result.output:
click.echo(result.output)
- assert result.exit_code == 0
+ assert result.exit_code == 0, (result.exit_code, result.output)
test1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/scores-cmc-4col.txt')
with runner.isolated_filesystem():
@@ -222,7 +222,7 @@ def test_cmc():
dev1, test1, dev1, test1])
if result.output:
click.echo(result.output)
- assert result.exit_code == 0
+ assert result.exit_code == 0, (result.exit_code, result.output)
def test_dic():
dev1 = pkg_resources.resource_filename('bob.bio.base.test',
@@ -232,7 +232,7 @@ def test_dic():
result = runner.invoke(commands.dic, ['--no-evaluation', dev1, '--rank', 2])
if result.output:
click.echo(result.output)
- assert result.exit_code == 0
+ assert result.exit_code == 0, (result.exit_code, result.output)
test1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/scores-nonorm-openset-dev')
with runner.isolated_filesystem():
@@ -241,7 +241,7 @@ def test_dic():
dev1, test1, dev1, test1])
if result.output:
click.echo(result.output)
- assert result.exit_code == 0
+ assert result.exit_code == 0, (result.exit_code, result.output)
def test_evaluate():
dev1 = pkg_resources.resource_filename('bob.bio.base.test',
@@ -257,15 +257,15 @@ def test_evaluate():
with runner.isolated_filesystem():
result = runner.invoke(commands.evaluate, ['-l', 'tmp', '-f', 0.03,
'--no-evaluation', dev1, dev2])
- assert result.exit_code == 0
+ assert result.exit_code == 0, (result.exit_code, result.output)
result = runner.invoke(commands.evaluate, ['--no-evaluation', '-f', 0.02,
dev1, dev2])
- assert result.exit_code == 0
+ assert result.exit_code == 0, (result.exit_code, result.output)
result = runner.invoke(commands.evaluate, ['-l', 'tmp', '-f', 0.04,
dev1, test1, dev2, test2])
- assert result.exit_code == 0
+ assert result.exit_code == 0, (result.exit_code, result.output)
result = runner.invoke(commands.evaluate, ['-f', 0.01,
dev1, test1, dev2, test2])
- assert result.exit_code == 0
+ assert result.exit_code == 0, (result.exit_code, result.output)
diff --git a/setup.py b/setup.py
index c27e5ab2..042c6336 100644
--- a/setup.py
+++ b/setup.py
@@ -145,6 +145,7 @@ setup(
'hist = bob.bio.base.script.commands:hist',
'cmc = bob.bio.base.script.commands:cmc',
'dic = bob.bio.base.script.commands:dic',
+ 'gen = bob.bio.base.script.gen:gen',
'evaluate = bob.bio.base.script.commands:evaluate',
],
--
GitLab