diff --git a/bob/bio/base/score/load.py b/bob/bio/base/score/load.py index b041ee8ecaf0bf502a05cfcbcaaa50511926b109..ecdfd519f635ac54831168863dc2fb3327aeb5d8 100644 --- a/bob/bio/base/score/load.py +++ b/bob/bio/base/score/load.py @@ -168,7 +168,7 @@ def get_split_dataframe(filename): return impostors, genuines -def split_csv_writer(filename): +def split_csv_scores(filename): """Loads a score set that was written with :any:`bob.bio.base.pipelines.vanilla_biometrics.CSVScoreWriter` Parameters @@ -335,9 +335,7 @@ def cmc_five_column(filename): def scores(filename, ncolumns=None): - """scores(filename, ncolumns=None) -> tuple - - Loads the scores from the given score file and yield its lines. + """Loads the scores from the given score file and yield its lines. Depending on the score file format, four or five elements are yielded, see :py:func:`bob.bio.base.score.load.four_column` and :py:func:`bob.bio.base.score.load.five_column` for details. diff --git a/bob/bio/base/script/commands.py b/bob/bio/base/script/commands.py index 9dbf07888364ef80dd543281c4c18f5dd2e63e73..d1f9958a0a98c6f470c23c779cc195eb84e4f097 100644 --- a/bob/bio/base/script/commands.py +++ b/bob/bio/base/script/commands.py @@ -56,7 +56,7 @@ def metrics(ctx, scores, evaluation, **kwargs): if "criterion" in ctx.meta and ctx.meta["criterion"] == "rr": process = bio_figure.Metrics(ctx, scores, evaluation, load.cmc) else: - func = load.split_csv_writer if iscsv(scores[0]) else load.split + func = load.split_csv_scores if iscsv(scores[0]) else load.split process = bio_figure.Metrics(ctx, scores, evaluation, func) process.run() @@ -65,7 +65,7 @@ def metrics(ctx, scores, evaluation, **kwargs): common_options.ROC_HELP.format(score_format=SCORE_FORMAT, command="bob bio roc") ) def roc(ctx, scores, evaluation, **kargs): - func = load.split_csv_writer if iscsv(scores[0]) else load.split + func = load.split_csv_scores if iscsv(scores[0]) else load.split process = bio_figure.Roc(ctx, scores, evaluation, func) process.run() @@ -74,7 +74,7 @@ def roc(ctx, scores, evaluation, **kargs): common_options.DET_HELP.format(score_format=SCORE_FORMAT, command="bob bio det") ) def det(ctx, scores, evaluation, **kargs): - func = load.split_csv_writer if iscsv(scores[0]) else load.split + func = load.split_csv_scores if iscsv(scores[0]) else load.split process = bio_figure.Det(ctx, scores, evaluation, func) process.run() @@ -83,7 +83,7 @@ def det(ctx, scores, evaluation, **kargs): common_options.EPC_HELP.format(score_format=SCORE_FORMAT, command="bob bio epc") ) def epc(ctx, scores, **kargs): - func = load.split_csv_writer if iscsv(scores[0]) else load.split + func = load.split_csv_scores if iscsv(scores[0]) else load.split process = measure_figure.Epc(ctx, scores, True, func) process.run() @@ -92,7 +92,7 @@ def epc(ctx, scores, **kargs): common_options.HIST_HELP.format(score_format=SCORE_FORMAT, command="bob bio hist") ) def hist(ctx, scores, evaluation, **kwargs): - func = load.split_csv_writer if iscsv(scores[0]) else load.split + func = load.split_csv_scores if iscsv(scores[0]) else load.split process = bio_figure.Hist(ctx, scores, evaluation, func) process.run() @@ -120,7 +120,7 @@ def evaluate(ctx, scores, evaluation, **kwargs): criteria=CRITERIA, ) def multi_metrics(ctx, scores, evaluation, protocols_number, **kwargs): - func = load.split_csv_writer if iscsv(scores[0]) else load.split + func = load.split_csv_scores if iscsv(scores[0]) else load.split ctx.meta["min_arg"] = protocols_number * (2 if evaluation else 1) process = bio_figure.MultiMetrics(ctx, scores, evaluation, func) process.run() diff --git a/bob/bio/base/script/vanilla_biometrics.py b/bob/bio/base/script/vanilla_biometrics.py index 7bdc89083a7b3eeffa521b6a444112dd2ae0e0d2..b84df23efb65861427616cff7142b37210076cc9 100644 --- a/bob/bio/base/script/vanilla_biometrics.py +++ b/bob/bio/base/script/vanilla_biometrics.py @@ -100,10 +100,10 @@ It is possible to do it via configuration file cls=ResourceOption, ) @click.option( - "--write-metadata-scores", - "-m", - is_flag=True, - help="If set, all the scores will be written with all its metadata using the `CSVScoreWriter`", + "--write-metadata-scores/--write-column-scores", + "-m/-nm", + default=True, + help="If set, all the scores will be written with all their metadata using the `CSVScoreWriter`", cls=ResourceOption, ) @click.option( diff --git a/bob/bio/base/script/vanilla_biometrics_ztnorm.py b/bob/bio/base/script/vanilla_biometrics_ztnorm.py index e7ccf7878f6fdc06b30c0c04ced4c7480c7c085a..c370a840c6a70d7085577021252707e5c3891532 100644 --- a/bob/bio/base/script/vanilla_biometrics_ztnorm.py +++ b/bob/bio/base/script/vanilla_biometrics_ztnorm.py @@ -106,10 +106,10 @@ It is possible to do it via configuration file cls=ResourceOption, ) @click.option( - "--write-metadata-scores", - "-m", - is_flag=True, - help="If set, all the scores will be written with all its metadata", + "--write-metadata-scores/--write-column-scores", + "-m/-nm", + default=True, + help="If set, all the scores will be written with all their metadata using the `CSVScoreWriter`", cls=ResourceOption, ) @click.option( diff --git a/doc/vanilla_biometrics_features.rst b/doc/vanilla_biometrics_features.rst index ed2f8a7aab227905d2ea9a67f298808f01e8f4e4..bc93d9de471ea04e47ea227fc3fbdcf3caf1831f 100644 --- a/doc/vanilla_biometrics_features.rst +++ b/doc/vanilla_biometrics_features.rst @@ -445,6 +445,8 @@ The page available contains some useful tools like a graphical visualization of The input are on the left side, and the resulting scores are generated by the jobs on the right. +.. _bob.bio.base.score_formats: + Writing scores in a customized manner ===================================== @@ -460,16 +462,15 @@ Common :py:class:`bob.bio.base.pipelines.vanilla_biometrics.ScoreWriter` are ava - A four columns format ScoreWriter -Using a Score Writer --------------------- +Using a specific Score Writer +----------------------------- -By default (omitting the ``--write-metadata-scores`` option), vanilla-biometrics will use the four-column format ScoreWriter. +By default, vanilla-biometrics will use the CSV format ScoreWriter. -To indicate to a vanilla-biometrics pipeline to use the CSV ScoreWriter instead -of the default four-column ScoreWriter, you can pass the -``--write-metadata-scores`` option like so:: +To indicate to a vanilla-biometrics pipeline to use the four-columns ScoreWriter instead +of the default CSV ScoreWriter, you can pass the ``--write-column-scores`` option like so:: - $ bob bio pipelines vanilla-biometrics --write-metadata-scores --output + $ bob bio pipelines vanilla-biometrics --write-column-scores --output CSV Score Writer @@ -495,7 +496,7 @@ Here is a short example of such file format with metadata on the ``age`` and ``g Four Columns Score Writer ------------------------- -:py:class:`bob.bio.base.pipelines.vanilla_biometrics.FourColumnsScoreWriter` is the default score file format used by bob. +:py:class:`bob.bio.base.pipelines.vanilla_biometrics.FourColumnsScoreWriter` is the old score file format used by bob. It consists of a text file with four columns separated by spaces and no header. Each row represents a comparison between a *probe* and a *model*, and the similarity score resulting from that comparison. The four columns are, in order: @@ -521,8 +522,10 @@ Evaluation Once scores are generated for each probe evaluated against the corresponding models, conclusions on the performance of the system can be drawn by using metrics like the FMR and FNMR or tools like the ROC curve. -In this section, commands that help to quickly evaluate a set of scores by generating metrics or plots are presented. -The scripts take as input a four-column file data format as specified in the documentation of :py:func:`bob.bio.base.score.load.four_column`. +In this section, commands that help to quickly evaluate a set of scores by +generating metrics or plots are presented. The scripts take as input either a +csv file or a legacy four/five column format. See +:ref:`bob.bio.base.score_formats` for the explanation of their formats. .. note:: @@ -536,7 +539,7 @@ min.HTER) on a development set and apply it on an evaluation set, just do: .. code-block:: sh - $ bob bio metrics -e output/scores-{dev,eval} --legends ExpA --criterion min-hter + $ bob bio metrics -e output/scores-{dev,eval}.csv --legends ExpA --criterion min-hter [Min. criterion: MIN-HTER ] Threshold on Development set `ExpA`: -1.399200e-01 ===================== ============== ============== @@ -557,8 +560,8 @@ min.HTER) on a development set and apply it on an evaluation set, just do: When evaluation scores are provided, the ``--eval`` option must be passed. See ``metrics --help`` for further options. - The ``scores-{dev,eval}`` brace expansion expands the path to every element - in the list. here: ``scores-dev scores-eval``. + The ``scores-{dev,eval}.csv`` brace expansion (a bash feature) expands the + path to every element in the list. here: ``scores-dev.csv scores-eval.csv``. .. _bob.bio.base.bob_bio_plots: @@ -587,7 +590,7 @@ For example, to generate a CMC curve from development and evaluation datasets: .. code-block:: sh - $ bob bio cmc -v --eval --output 'my_cmc.pdf' dev-1.txt eval-1.txt dev-2.txt eval-2.txt + $ bob bio cmc -v --eval --output 'my_cmc.pdf' dev-1.csv eval-1.csv dev-2.csv eval-2.csv where ``my_cmc.pdf`` will contain CMC curves for the two experiments represented by their respective *dev* and *eval* scores-file. @@ -612,7 +615,7 @@ plots for a list of experiments. It generates two ``metrics`` outputs with EER, HTER, minDCF criteria, along with ``roc``, ``det``, ``epc``, and ``hist`` plots for each experiment. For example:: - $ bob bio evaluate -v --eval --log 'my_metrics.txt' --output 'my_plots.pdf' {sys1,sys2}/{dev,eval} + $ bob bio evaluate -v --eval --log 'my_metrics.txt' --output 'my_plots.pdf' {sys1,sys2}/scores-{dev,eval}.csv will output metrics and plots for the two experiments (dev and eval pairs) in ``my_metrics.txt`` and ``my_plots.pdf``, respectively. diff --git a/doc/vulnerability_analysis.rst b/doc/vulnerability_analysis.rst index 5172c361dd224ec355a4c4321f59d46d49d56676..c6f680c3f244b25c8ec9208558162abfd0108809 100644 --- a/doc/vulnerability_analysis.rst +++ b/doc/vulnerability_analysis.rst @@ -195,7 +195,7 @@ parameter to the vanilla-biometrics command: .. code-block:: sh - $ bob bio pipelines vanilla-biometrics --write-metadata-scores custom_database_config.py facenet-sanderberg + $ bob bio pipelines vanilla-biometrics custom_database_config.py facenet-sanderberg Computing the scores @@ -220,14 +220,9 @@ following command: .. code-block:: sh - $ bob bio pipelines vanilla-biometrics --write-metadata-scores -v --output ./results \ + $ bob bio pipelines vanilla-biometrics -v --output ./results \ --group dev --group eval replaymobile-img facenet-sanderberg -The ``--write-metadata-scores`` option is necessary, as the scores -samples will need to retain the information of being an attack or not (``attack_type`` -field). This information is passed as metadata through the samples, into the score -files. - ``replaymobile-img`` points to a python configuration file defining a ``database`` object.