diff --git a/bob/bio/base/script/evaluate.py b/bob/bio/base/script/evaluate.py index f6bae080c5fc664a0dbe599ba5b2b17e81ac362f..f2f7ac42f82e2d0b801b46348a3d2ef5a90966d0 100644 --- a/bob/bio/base/script/evaluate.py +++ b/bob/bio/base/script/evaluate.py @@ -61,6 +61,7 @@ def command_line_arguments(command_line_parameters): parser.add_argument('-m', '--mindcf', action = 'store_true', help = "If given, minDCF will be computed.") parser.add_argument('--cost', default=0.99, help='Cost for FAR in minDCF') parser.add_argument('-r', '--rr', action = 'store_true', help = "If given, the Recognition Rate will be computed.") + parser.add_argument('-t', '--thresholds', type=float, nargs='+', help = "If given, the Recognition Rate will incorporate an Open Set handling, rejecting all scores that are below the given threshold; when multiple thresholds are given, they are applied in the same order as the --dev-files.") parser.add_argument('-l', '--legends', nargs='+', help = "A list of legend strings used for ROC, CMC and DET plots; if given, must be the same number than --dev-files.") parser.add_argument('-F', '--legend-font-size', type=int, default=18, help = "Set the font size of the legends.") parser.add_argument('-P', '--legend-position', type=int, help = "Set the font size of the legends.") @@ -93,6 +94,14 @@ def command_line_arguments(command_line_parameters): if len(args.dev_files) != len(args.legends): logger.error("The number of --dev-files (%d) and --legends (%d) are not identical", len(args.dev_files), len(args.legends)) + if args.thresholds is not None: + if len(args.thresholds) == 1: + args.thresholds = args.thresholds * len(args.dev_files) + elif len(args.thresholds) != len(args.dev_files): + logger.error("If given, the number of --thresholds imust be either 1, or the same as --dev-files (%d), but it is %d", len(args.dev_files), len(args.thresholds)) + else: + args.thresholds = [None] * len(args.dev_files) + return args @@ -314,24 +323,24 @@ def main(command_line_parameters=None): if args.eval_files: cmcs_eval = [cmc_parser(os.path.join(args.directory, f)) for f in args.eval_files] - if args.cmc: - logger.info("Plotting CMC curves to file '%s'", args.cmc) - try: - # create a multi-page PDF for the ROC curve - pdf = PdfPages(args.cmc) - # create a separate figure for dev and eval - pdf.savefig(_plot_cmc(cmcs_dev, colors, args.legends, "CMC curve for development set", args.legend_font_size, args.legend_position)) - if args.eval_files: - pdf.savefig(_plot_cmc(cmcs_eval, colors, args.legends, "CMC curve for evaluation set", args.legend_font_size, args.legend_position)) - pdf.close() - except RuntimeError as e: - raise RuntimeError("During plotting of ROC curves, the following exception occured:\n%s\nUsually this happens when the label contains characters that LaTeX cannot parse." % e) - - if args.rr: - logger.info("Computing recognition rate on the development " + ("and on the evaluation set" if args.eval_files else "set")) - for i in range(len(cmcs_dev)): - rr = bob.measure.recognition_rate(cmcs_dev[i]) - print("The Recognition Rate of the development set of '%s' is %2.3f%%" % (args.legends[i], rr * 100.)) - if args.eval_files: - rr = bob.measure.recognition_rate(cmcs_eval[i]) + if args.cmc: + logger.info("Plotting CMC curves to file '%s'", args.cmc) + try: + # create a multi-page PDF for the ROC curve + pdf = PdfPages(args.cmc) + # create a separate figure for dev and eval + pdf.savefig(_plot_cmc(cmcs_dev, colors, args.legends, "CMC curve for development set", args.legend_font_size, args.legend_position)) + if args.eval_files: + pdf.savefig(_plot_cmc(cmcs_eval, colors, args.legends, "CMC curve for evaluation set", args.legend_font_size, args.legend_position)) + pdf.close() + except RuntimeError as e: + raise RuntimeError("During plotting of ROC curves, the following exception occured:\n%s\nUsually this happens when the label contains characters that LaTeX cannot parse." % e) + + if args.rr: + logger.info("Computing recognition rate on the development " + ("and on the evaluation set" if args.eval_files else "set")) + for i in range(len(cmcs_dev)): + rr = bob.measure.recognition_rate(cmcs_dev[i], args.thresholds[i]) print("The Recognition Rate of the development set of '%s' is %2.3f%%" % (args.legends[i], rr * 100.)) + if args.eval_files: + rr = bob.measure.recognition_rate(cmcs_eval[i], args.thresholds[i]) + print("The Recognition Rate of the development set of '%s' is %2.3f%%" % (args.legends[i], rr * 100.)) diff --git a/bob/bio/base/test/test_scripts.py b/bob/bio/base/test/test_scripts.py index 888ca886ae7d2e18e3f0ea14d264b537bfc19c5b..2f755da0777f57e779732c30fc28c07016d4b44e 100644 --- a/bob/bio/base/test/test_scripts.py +++ b/bob/bio/base/test/test_scripts.py @@ -306,6 +306,8 @@ def test_evaluate(): '--roc', plots[0], '--det', plots[1], '--cmc', plots[2], + '--rr', + '--thresholds', '5000', '0', '-v', ]