Commit 3e841897 authored by Theophile GENTILHOMME's avatar Theophile GENTILHOMME
Browse files

Add options for axis labels, modifications accoding to the move of click...

Add options for axis labels, modifications accoding to the move of click options to bob.extension, print assert outputs for tests
parent fce02f14
Pipeline #19064 failed with stage
in 34 minutes and 51 seconds
......@@ -6,7 +6,8 @@ import bob.bio.base.script.figure as bio_figure
import bob.measure.script.figure as measure_figure
from ..score import load
from bob.measure.script import common_options
from bob.extension.scripts.click_helper import verbosity_option
from bob.extension.scripts.click_helper import (verbosity_option,
open_file_mode_option)
FUNC_SPLIT = lambda x: load.load_files(x, load.split)
FUNC_CMC = lambda x: load.load_files(x, load.cmc)
......@@ -15,13 +16,13 @@ FUNC_CMC = lambda x: load.load_files(x, load.cmc)
@common_options.scores_argument(nargs=-1)
@common_options.table_option()
@common_options.eval_option()
@common_options.open_file_mode_option()
@common_options.output_plot_metric_option()
@common_options.criterion_option(['eer', 'hter', 'far', 'mindcf', 'cllr', 'rr'])
@common_options.cost_option()
@common_options.thresholds_option()
@common_options.far_option()
@common_options.titles_option()
@open_file_mode_option()
@verbosity_option()
@click.pass_context
def metrics(ctx, scores, evaluation, **kargs):
......@@ -65,7 +66,9 @@ def metrics(ctx, scores, evaluation, **kargs):
@common_options.axes_val_option(dflt=[1e-4, 1, 1e-4, 1])
@common_options.axis_fontsize_option()
@common_options.x_rotation_option()
@common_options.fmr_line_at_option()
@common_options.lines_at_option()
@common_options.x_label_option()
@common_options.y_label_option()
@common_options.const_layout_option()
@verbosity_option()
@click.pass_context
......@@ -90,13 +93,15 @@ def roc(ctx, scores, evaluation, **kargs):
$ bob bio roc -o my_roc.pdf dev-scores1 eval-scores1
"""
process = measure_figure.Roc(ctx, scores, evaluation, FUNC_SPLIT)
process = bio_figure.Roc(ctx, scores, evaluation, FUNC_SPLIT)
process.run()
@click.command()
@common_options.scores_argument(nargs=-1)
@common_options.output_plot_file_option(default_out='det.pdf')
@common_options.titles_option()
@common_options.x_label_option()
@common_options.y_label_option()
@common_options.sep_dev_eval_option()
@common_options.eval_option()
@common_options.axis_fontsize_option(dflt=6)
......@@ -126,7 +131,7 @@ def det(ctx, scores, evaluation, **kargs):
$ bob bio det -o my_det.pdf dev-scores1 eval-scores1
"""
process = measure_figure.Det(ctx, scores, evaluation, FUNC_SPLIT)
process = bio_figure.Det(ctx, scores, evaluation, FUNC_SPLIT)
process.run()
@click.command()
......@@ -294,7 +299,7 @@ def hist(ctx, scores, evaluation, **kwargs):
@common_options.output_plot_metric_option()
@common_options.output_plot_file_option(default_out='eval_plots.pdf')
@common_options.points_curve_option()
@common_options.fmr_line_at_option()
@common_options.lines_at_option()
@common_options.cost_option()
@common_options.rank_option()
@common_options.far_option()
......
......@@ -7,6 +7,22 @@ import bob.measure
from bob.measure import plot
from tabulate import tabulate
class Roc(measure_figure.Roc):
def __init__(self, ctx, scores, evaluation, func_load):
super(Roc, self).__init__(ctx, scores, evaluation, func_load)
self._x_label = 'False Match Rate' if 'x_label' not in ctx.meta or \
ctx.meta['x_label'] is None else ctx.meta['x_label']
self._y_label = '1 - False Non Match Rate' if 'y_label' not in \
ctx.meta or ctx.meta['y_label'] is None else ctx.meta['y_label']
class Det(measure_figure.Det):
def __init__(self, ctx, scores, evaluation, func_load):
super(Det, self).__init__(ctx, scores, evaluation, func_load)
self._x_label = 'False Match Rate' if 'x_label' not in ctx.meta or \
ctx.meta['x_label'] is None else ctx.meta['x_label']
self._y_label = 'False Non Match Rate' if 'y_label' not in ctx.meta or\
ctx.meta['y_label'] is None else ctx.meta['y_label']
class Cmc(measure_figure.PlotBase):
''' Handles the plotting of Cmc
......@@ -20,9 +36,9 @@ class Cmc(measure_figure.PlotBase):
super(Cmc, self).__init__(ctx, scores, evaluation, func_load)
self._semilogx = True if 'semilogx' not in ctx.meta else\
ctx.meta['semilogx']
self._title = 'CMC'
self._x_label = 'Rank'
self._y_label = 'Probability'
self._title = self._title or 'CMC'
self._x_label = self._x_label or 'Rank'
self._y_label = self._y_label or 'Identification rate'
self._max_R = 0
def compute(self, idx, dev_score, dev_file=None,
......@@ -79,9 +95,9 @@ class Dic(measure_figure.PlotBase):
self._semilogx = True if 'semilogx' not in ctx.meta else\
ctx.meta['semilogx']
self._rank = 1 if 'rank' not in ctx.meta else ctx.meta['rank']
self._title = 'DIC'
self._x_label = 'FAR'
self._y_label = 'DIR'
self._title = self._title or 'DIC'
self._x_label = self._title or 'FAR'
self._y_label = self._title or 'DIR'
def compute(self, idx, dev_score, dev_file=None,
eval_score=None, eval_file=None):
......
"""Generate random scores.
"""
import pkg_resources # to make sure bob gets imported properly
import os
import logging
import numpy
import random as rd
import click
from click.types import FLOAT
from bob.extension.scripts.click_helper import verbosity_option
from bob.core import random
from bob.io.base import create_directories_safe
logger = logging.getLogger(__name__)
NUM_NEG = 5000
NUM_POS = 5000
def gen_score_distr(mean_neg, mean_pos, sigma_neg=10, sigma_pos=10):
"""Generate scores from normal distributions
Parameters
----------
mean_neg : float
Mean for negative scores
mean_pos : float
Mean for positive scores
sigma_neg : float
STDev for negative scores
sigma_pos : float
STDev for positive scores
Returns
-------
neg_scores : array_like
Negatives scores
pos_scores : array_like
Positive scores
"""
mt = random.mt19937() # initialise the random number generator
neg_generator = random.normal(numpy.float32, mean_neg, sigma_neg)
pos_generator = random.normal(numpy.float32, mean_pos, sigma_pos)
neg_scores = [neg_generator(mt) for _ in range(NUM_NEG)]
pos_scores = [pos_generator(mt) for _ in range(NUM_NEG)]
return neg_scores, pos_scores
def write_scores_to_file(pos, neg, filename, n_sys=1, five_col=False):
""" Writes score distributions
Parameters
----------
pos : array_like
Scores for positive samples.
neg : array_like
Scores for negative samples.
filename : str
The path to write the score to.
n_sys : int
Number of different systems
five_col : bool
If 5-colum format, else 4-column
"""
create_directories_safe(os.path.dirname(filename))
s_names = ['s%d' % i for i in range(n_sys)]
with open(filename, 'wt') as f:
for i in pos:
s_name = rd.choice(s_names)
s_five = ' ' if not five_col else ' d' + rd.choice(s_names) + ' '
f.write('x%sx %s %f\n' % (s_five, s_name, i))
for i in neg:
s_name = rd.choice(s_names)
s_five = ' ' if not five_col else ' d' + rd.choice(s_names) + ' '
f.write('x%sy %s %f\n' % (s_five, s_name, i))
@click.command()
@click.argument('outdir')
@click.option('-mm', '--mean-match', default=10, type=FLOAT, show_default=True)
@click.option('-mnm', '--mean-non-match', default=-10, type=FLOAT, show_default=True)
@click.option('-n', '--n-sys', default=1, type=click.INT, show_default=True)
@click.option('--five-col/--four-col', default=False, show_default=True)
@verbosity_option()
def gen(outdir, mean_match, mean_non_match, n_sys, five_col):
"""Generate random scores.
Generates random scores in 4col or 5col format. The scores are generated
using Gaussian distribution whose mean is an input
parameter. The generated scores can be used as hypothetical datasets.
"""
# Generate the data
neg_dev, pos_dev = gen_score_distr(mean_non_match, mean_match)
neg_eval, pos_eval = gen_score_distr(mean_non_match, mean_match)
# Write the data into files
write_scores_to_file(neg_dev, pos_dev,
os.path.join(outdir, 'scores-dev'), n_sys, five_col)
write_scores_to_file(neg_eval, pos_eval,
os.path.join(outdir, 'scores-eval'), n_sys, five_col)
......@@ -15,7 +15,7 @@ def test_metrics():
with runner.isolated_filesystem():
with open('tmp', 'w') as f:
f.write(result.output)
assert result.exit_code == 0
assert result.exit_code == 0, (result.exit_code, result.output)
dev2 = pkg_resources.resource_filename('bob.bio.base.test',
'data/dev-5col.txt')
test1 = pkg_resources.resource_filename('bob.bio.base.test',
......@@ -28,19 +28,19 @@ def test_metrics():
)
with open('tmp', 'w') as f:
f.write(result.output)
assert result.exit_code == 0
assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['-l', 'tmp', '-t', 'A,B',
commands.metrics, ['-l', 'tmp', '-ts', 'A,B',
dev1, test1, dev2, test2]
)
assert result.exit_code == 0
assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['-l', 'tmp', dev1, test2]
)
assert result.exit_code == 0
assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(
......@@ -48,7 +48,7 @@ def test_metrics():
'--criter', 'mindcf', '--cost', 0.9,
dev1, test2]
)
assert result.exit_code == 0
assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(
......@@ -56,34 +56,34 @@ def test_metrics():
'--criter', 'mindcf', '--cost', 0.9,
dev1]
)
assert result.exit_code == 0
assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['--criter', 'cllr', dev1, test2]
)
assert result.exit_code == 0
assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['--no-evaluation', '-l', 'tmp', '--criter', 'cllr',
'--cost', 0.9, dev1]
)
assert result.exit_code == 0
assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['--criter', 'rr', '-T',
'0.1', dev1, test2]
)
assert result.exit_code == 0
assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['--no-evaluation', '-l', 'tmp', '--criter', 'rr',
dev1, dev2]
)
assert result.exit_code == 0
assert result.exit_code == 0, (result.exit_code, result.output)
def test_roc():
......@@ -95,7 +95,7 @@ def test_roc():
'test.pdf',dev1])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
assert result.exit_code == 0, (result.exit_code, result.output)
dev2 = pkg_resources.resource_filename('bob.bio.base.test',
'data/dev-5col.txt')
test1 = pkg_resources.resource_filename('bob.bio.base.test',
......@@ -108,7 +108,7 @@ def test_roc():
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(commands.roc, ['--output',
......@@ -116,7 +116,7 @@ def test_roc():
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
assert result.exit_code == 0, (result.exit_code, result.output)
def test_det():
......@@ -127,7 +127,7 @@ def test_det():
result = runner.invoke(commands.det, ['--no-evaluation', dev1])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
assert result.exit_code == 0, (result.exit_code, result.output)
dev2 = pkg_resources.resource_filename('bob.bio.base.test',
'data/dev-5col.txt')
test1 = pkg_resources.resource_filename('bob.bio.base.test',
......@@ -140,14 +140,14 @@ def test_det():
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(commands.det, ['--output',
'test.pdf',
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
assert result.exit_code == 0, (result.exit_code, result.output)
def test_epc():
dev1 = pkg_resources.resource_filename('bob.bio.base.test',
......@@ -159,7 +159,7 @@ def test_epc():
result = runner.invoke(commands.epc, [dev1, test1])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
assert result.exit_code == 0, (result.exit_code, result.output)
dev2 = pkg_resources.resource_filename('bob.bio.base.test',
'data/dev-4col.tar.gz')
test2 = pkg_resources.resource_filename('bob.bio.base.test',
......@@ -170,7 +170,7 @@ def test_epc():
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
assert result.exit_code == 0, (result.exit_code, result.output)
def test_hist():
dev1 = pkg_resources.resource_filename('bob.bio.base.test',
......@@ -186,7 +186,7 @@ def test_hist():
result = runner.invoke(commands.hist, ['--no-evaluation', dev1])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(commands.hist, ['--criter', 'hter', '--output',
......@@ -194,16 +194,16 @@ def test_hist():
30,'--no-evaluation', dev1, dev2])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
assert result.exit_code == 0, (result.exit_code, result.output)
with runner.isolated_filesystem():
result = runner.invoke(commands.hist, ['--criter', 'eer', '--output',
'HISTO.pdf', '-b', 30, '-F', 3,
'-t', 'A,B', dev1, test1, dev2,
'-ts', 'A,B', dev1, test1, dev2,
test2])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
assert result.exit_code == 0, (result.exit_code, result.output)
def test_cmc():
dev1 = pkg_resources.resource_filename('bob.bio.base.test',
......@@ -213,7 +213,7 @@ def test_cmc():
result = runner.invoke(commands.cmc, ['--no-evaluation', dev1])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
assert result.exit_code == 0, (result.exit_code, result.output)
test1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/scores-cmc-4col.txt')
with runner.isolated_filesystem():
......@@ -222,7 +222,7 @@ def test_cmc():
dev1, test1, dev1, test1])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
assert result.exit_code == 0, (result.exit_code, result.output)
def test_dic():
dev1 = pkg_resources.resource_filename('bob.bio.base.test',
......@@ -232,7 +232,7 @@ def test_dic():
result = runner.invoke(commands.dic, ['--no-evaluation', dev1, '--rank', 2])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
assert result.exit_code == 0, (result.exit_code, result.output)
test1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/scores-nonorm-openset-dev')
with runner.isolated_filesystem():
......@@ -241,7 +241,7 @@ def test_dic():
dev1, test1, dev1, test1])
if result.output:
click.echo(result.output)
assert result.exit_code == 0
assert result.exit_code == 0, (result.exit_code, result.output)
def test_evaluate():
dev1 = pkg_resources.resource_filename('bob.bio.base.test',
......@@ -257,15 +257,15 @@ def test_evaluate():
with runner.isolated_filesystem():
result = runner.invoke(commands.evaluate, ['-l', 'tmp', '-f', 0.03,
'--no-evaluation', dev1, dev2])
assert result.exit_code == 0
assert result.exit_code == 0, (result.exit_code, result.output)
result = runner.invoke(commands.evaluate, ['--no-evaluation', '-f', 0.02,
dev1, dev2])
assert result.exit_code == 0
assert result.exit_code == 0, (result.exit_code, result.output)
result = runner.invoke(commands.evaluate, ['-l', 'tmp', '-f', 0.04,
dev1, test1, dev2, test2])
assert result.exit_code == 0
assert result.exit_code == 0, (result.exit_code, result.output)
result = runner.invoke(commands.evaluate, ['-f', 0.01,
dev1, test1, dev2, test2])
assert result.exit_code == 0
assert result.exit_code == 0, (result.exit_code, result.output)
......@@ -145,6 +145,7 @@ setup(
'hist = bob.bio.base.script.commands:hist',
'cmc = bob.bio.base.script.commands:cmc',
'dic = bob.bio.base.script.commands:dic',
'gen = bob.bio.base.script.gen:gen',
'evaluate = bob.bio.base.script.commands:evaluate',
],
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment