diff --git a/bob/bio/base/script/commands.py b/bob/bio/base/script/commands.py index 95b9b33dcc703c96f0195a5c73ab31b48a868b25..7fe761e87c3bf8a8c269eb8ca96ceadd97c60fe7 100644 --- a/bob/bio/base/script/commands.py +++ b/bob/bio/base/script/commands.py @@ -213,3 +213,50 @@ def cmc(ctx, scores, test, **kargs): """ process = bio_figure.Cmc(ctx, scores, test, FUNC_CMC) process.run() + +@click.command() +@common_options.scores_argument(nargs=-1) +@common_options.titles_option() +@common_options.sep_dev_test_option() +@common_options.output_plot_file_option(default_out='cmc.pdf') +@common_options.test_option() +@common_options.semilogx_option(True) +@common_options.axes_val_option(dflt=None) +@common_options.axis_fontsize_option() +@common_options.x_rotation_option() +@common_options.rank_option() +@verbosity_option() +@click.pass_context +def dic(ctx, scores, test, **kargs): + """Plots the Detection & Identification curve over the FAR + + This curve is designed to be used in an open set identification protocol, and + defined in Chapter 14.1 of [LiJain2005]_. It requires to have at least one + open set probe item, i.e., with no corresponding gallery, such that the + positives for that pair are ``None``. + + The detection and identification curve first computes FAR thresholds based on + the out-of-set probe scores (negative scores). For each probe item, the + **maximum** negative score is used. Then, it plots the detection and + identification rates for those thresholds, which are based on the in-set + probe scores only. See [LiJain2005]_ for more details. + + .. [LiJain2005] **Stan Li and Anil K. Jain**, *Handbook of Face Recognition*, Springer, 2005 + + You need provide one or more development score file(s) for each experiment. + You can also provide test files along with dev files but the flag `--test` + is required in that case.Files must be 4- or 5- columns format, see + :py:func:`bob.bio.base.score.load.four_column` and + :py:func:`bob.bio.base.score.load.five_column` for details. + + + Examples: + $ bob bio dic dev-scores + + $ bob bio dic --test dev-scores1 test-scores1 dev-scores2 + test-scores2 + + $ bob bio dic --test -o my_roc.pdf dev-scores1 test-scores1 + """ + process = bio_figure.Dic(ctx, scores, test, FUNC_CMC) + process.run() diff --git a/bob/bio/base/script/evaluate.py b/bob/bio/base/script/evaluate.py index 8d285f70953599bb36cb1eed02e8cee5a046d6c7..5ec3b2467b111886272565a0b53ead079b3937cc 100644 --- a/bob/bio/base/script/evaluate.py +++ b/bob/bio/base/script/evaluate.py @@ -25,6 +25,7 @@ import bob.measure from .. import score + if not os.environ.get('BOB_NO_STYLE_CHANGES'): # make the fig size smaller so that everything becomes bigger matplotlib.rc('figure', figsize=(4, 3)) @@ -369,7 +370,11 @@ def main(command_line_parameters=None): # create a multi-page PDF for the ROC curve pdf = PdfPages(args.roc) # create a separate figure for dev and eval - pdf.savefig(_plot_roc(frrs_dev, colors, args.legends, args.title[0] if args.title is not None else "ROC for development set", args.legend_font_size, args.legend_position, args.far_line_at, min_far=args.min_far_value), bbox_inches='tight') + pdf.savefig(_plot_roc( + frrs_dev, colors, args.legends, + args.title[0] if args.title is not None else "ROC for development set", + args.legend_font_size, args.legend_position, args.far_line_at, + min_far=args.min_far_value), bbox_inches='tight') del frrs_dev if args.eval_files: if args.far_line_at is not None: diff --git a/bob/bio/base/script/figure.py b/bob/bio/base/script/figure.py index 17675284d6197dde8b65dc9a17053bef9d4e6b80..2ab16e3f4fa7211075a8a02679cf6dd922bca23e 100644 --- a/bob/bio/base/script/figure.py +++ b/bob/bio/base/script/figure.py @@ -59,3 +59,52 @@ class Cmc(measure_figure.PlotBase): if self._axlim is None: self._axlim = [0, self._max_R, -0.01, 1.01] super(Cmc, self).end_process() + +class Dic(measure_figure.PlotBase): + ''' Handles the plotting of DIC + + Attributes + ---------- + + _semilogx: :obj:`bool` + If true (default), X-axis will be semilog10 + _rank: :obj:`int` + Rank to be used to plot DIC (default: 1) + ''' + def __init__(self, ctx, scores, test, func_load): + super(Dic, self).__init__(ctx, scores, test, func_load) + self._semilogx = True if 'semilogx' not in ctx.meta else\ + ctx.meta['semilogx'] + self._rank = 1 if 'rank' not in ctx.meta else ctx.meta['rank'] + self._title = 'DIC' + self._x_label = 'FAR' + self._y_label = 'DIR' + + def compute(self, idx, dev_score, dev_file=None, + test_score=None, test_file=None): + ''' Plot DIC for dev and eval data using + :py:func:`bob.measure.plot.detection_identification_curve`''' + mpl.figure(1) + if self._test: + linestyle = '-' if not self._split else measure_figure.LINESTYLES[idx % 14] + plot.detection_identification_curve( + dev_score, rank=self._rank, logx=self._semilogx, + color=self._colors[idx], linestyle=linestyle, + label=self._label('development', dev_file, idx) + ) + linestyle = '--' + if self._split: + mpl.figure(2) + linestyle = measure_figure.LINESTYLES[idx % 14] + + plot.detection_identification_curve( + test_score, rank=self._rank, logx=self._semilogx, + color=self._colors[idx], linestyle=linestyle, + label=self._label('test', test_file, idx) + ) + else: + rank = plot.detection_identification_curve( + dev_score, rank=self._rank, logx=self._semilogx, + color=self._colors[idx], linestyle=measure_figure.LINESTYLES[idx % 14], + label=self._label('development', dev_file, idx) + ) diff --git a/bob/bio/base/test/test_commands.py b/bob/bio/base/test/test_commands.py index 48b62300b7fd4b8ebf23cd1e8d1856f5e0432cbb..159927f44eeab6f22458b4503dd51149993ed78a 100644 --- a/bob/bio/base/test/test_commands.py +++ b/bob/bio/base/test/test_commands.py @@ -176,3 +176,22 @@ def test_cmc(): if result.output: click.echo(result.output) assert result.exit_code == 0 + +def test_dic(): + dev1 = pkg_resources.resource_filename('bob.bio.base.test', + 'data/scores-nonorm-openset-dev') + runner = CliRunner() + with runner.isolated_filesystem(): + result = runner.invoke(commands.dic, [dev1, '--rank', 2]) + if result.output: + click.echo(result.output) + assert result.exit_code == 0 + test1 = pkg_resources.resource_filename('bob.bio.base.test', + 'data/scores-nonorm-openset-dev') + with runner.isolated_filesystem(): + result = runner.invoke(commands.dic, ['--output', 'test.pdf', '-t', + '--titles', 'A,B', '-F', 3, + dev1, test1, dev1, test1]) + if result.output: + click.echo(result.output) + assert result.exit_code == 0 diff --git a/setup.py b/setup.py index 7382e7052395909e48382334bca82b1cba8127a8..3ce128aff101ff981724a240ee38347690665a65 100644 --- a/setup.py +++ b/setup.py @@ -145,6 +145,7 @@ setup( 'epc = bob.bio.base.script.commands:epc', 'hist = bob.bio.base.script.commands:hist', 'cmc = bob.bio.base.script.commands:cmc', + 'dic = bob.bio.base.script.commands:dic', ], # annotators