Commit 335937fd authored by Tiago de Freitas Pereira's avatar Tiago de Freitas Pereira
Browse files

Merge branch 'csv-default' into 'master'

Use the CSV ScoreWriter by default in bob pipelines vanilla-biometrics

See merge request !251
parents d647dc76 27798250
Pipeline #52430 passed with stages
in 17 minutes and 2 seconds
......@@ -168,7 +168,7 @@ def get_split_dataframe(filename):
return impostors, genuines
def split_csv_writer(filename):
def split_csv_scores(filename):
"""Loads a score set that was written with :any:`bob.bio.base.pipelines.vanilla_biometrics.CSVScoreWriter`
Parameters
......@@ -335,9 +335,7 @@ def cmc_five_column(filename):
def scores(filename, ncolumns=None):
"""scores(filename, ncolumns=None) -> tuple
Loads the scores from the given score file and yield its lines.
"""Loads the scores from the given score file and yield its lines.
Depending on the score file format, four or five elements are yielded, see
:py:func:`bob.bio.base.score.load.four_column` and
:py:func:`bob.bio.base.score.load.five_column` for details.
......
......@@ -56,7 +56,7 @@ def metrics(ctx, scores, evaluation, **kwargs):
if "criterion" in ctx.meta and ctx.meta["criterion"] == "rr":
process = bio_figure.Metrics(ctx, scores, evaluation, load.cmc)
else:
func = load.split_csv_writer if iscsv(scores[0]) else load.split
func = load.split_csv_scores if iscsv(scores[0]) else load.split
process = bio_figure.Metrics(ctx, scores, evaluation, func)
process.run()
......@@ -65,7 +65,7 @@ def metrics(ctx, scores, evaluation, **kwargs):
common_options.ROC_HELP.format(score_format=SCORE_FORMAT, command="bob bio roc")
)
def roc(ctx, scores, evaluation, **kargs):
func = load.split_csv_writer if iscsv(scores[0]) else load.split
func = load.split_csv_scores if iscsv(scores[0]) else load.split
process = bio_figure.Roc(ctx, scores, evaluation, func)
process.run()
......@@ -74,7 +74,7 @@ def roc(ctx, scores, evaluation, **kargs):
common_options.DET_HELP.format(score_format=SCORE_FORMAT, command="bob bio det")
)
def det(ctx, scores, evaluation, **kargs):
func = load.split_csv_writer if iscsv(scores[0]) else load.split
func = load.split_csv_scores if iscsv(scores[0]) else load.split
process = bio_figure.Det(ctx, scores, evaluation, func)
process.run()
......@@ -83,7 +83,7 @@ def det(ctx, scores, evaluation, **kargs):
common_options.EPC_HELP.format(score_format=SCORE_FORMAT, command="bob bio epc")
)
def epc(ctx, scores, **kargs):
func = load.split_csv_writer if iscsv(scores[0]) else load.split
func = load.split_csv_scores if iscsv(scores[0]) else load.split
process = measure_figure.Epc(ctx, scores, True, func)
process.run()
......@@ -92,7 +92,7 @@ def epc(ctx, scores, **kargs):
common_options.HIST_HELP.format(score_format=SCORE_FORMAT, command="bob bio hist")
)
def hist(ctx, scores, evaluation, **kwargs):
func = load.split_csv_writer if iscsv(scores[0]) else load.split
func = load.split_csv_scores if iscsv(scores[0]) else load.split
process = bio_figure.Hist(ctx, scores, evaluation, func)
process.run()
......@@ -120,7 +120,7 @@ def evaluate(ctx, scores, evaluation, **kwargs):
criteria=CRITERIA,
)
def multi_metrics(ctx, scores, evaluation, protocols_number, **kwargs):
func = load.split_csv_writer if iscsv(scores[0]) else load.split
func = load.split_csv_scores if iscsv(scores[0]) else load.split
ctx.meta["min_arg"] = protocols_number * (2 if evaluation else 1)
process = bio_figure.MultiMetrics(ctx, scores, evaluation, func)
process.run()
......
......@@ -100,10 +100,10 @@ It is possible to do it via configuration file
cls=ResourceOption,
)
@click.option(
"--write-metadata-scores",
"-m",
is_flag=True,
help="If set, all the scores will be written with all its metadata using the `CSVScoreWriter`",
"--write-metadata-scores/--write-column-scores",
"-m/-nm",
default=True,
help="If set, all the scores will be written with all their metadata using the `CSVScoreWriter`",
cls=ResourceOption,
)
@click.option(
......
......@@ -106,10 +106,10 @@ It is possible to do it via configuration file
cls=ResourceOption,
)
@click.option(
"--write-metadata-scores",
"-m",
is_flag=True,
help="If set, all the scores will be written with all its metadata",
"--write-metadata-scores/--write-column-scores",
"-m/-nm",
default=True,
help="If set, all the scores will be written with all their metadata using the `CSVScoreWriter`",
cls=ResourceOption,
)
@click.option(
......
......@@ -445,6 +445,8 @@ The page available contains some useful tools like a graphical visualization of
The input are on the left side, and the resulting scores are generated by the jobs on the right.
.. _bob.bio.base.score_formats:
Writing scores in a customized manner
=====================================
......@@ -460,16 +462,15 @@ Common :py:class:`bob.bio.base.pipelines.vanilla_biometrics.ScoreWriter` are ava
- A four columns format ScoreWriter
Using a Score Writer
--------------------
Using a specific Score Writer
-----------------------------
By default (omitting the ``--write-metadata-scores`` option), vanilla-biometrics will use the four-column format ScoreWriter.
By default, vanilla-biometrics will use the CSV format ScoreWriter.
To indicate to a vanilla-biometrics pipeline to use the CSV ScoreWriter instead
of the default four-column ScoreWriter, you can pass the
``--write-metadata-scores`` option like so::
To indicate to a vanilla-biometrics pipeline to use the four-columns ScoreWriter instead
of the default CSV ScoreWriter, you can pass the ``--write-column-scores`` option like so::
$ bob bio pipelines vanilla-biometrics --write-metadata-scores <database> <pipeline> --output <output_dir>
$ bob bio pipelines vanilla-biometrics --write-column-scores <database> <pipeline> --output <output_dir>
CSV Score Writer
......@@ -495,7 +496,7 @@ Here is a short example of such file format with metadata on the ``age`` and ``g
Four Columns Score Writer
-------------------------
:py:class:`bob.bio.base.pipelines.vanilla_biometrics.FourColumnsScoreWriter` is the default score file format used by bob.
:py:class:`bob.bio.base.pipelines.vanilla_biometrics.FourColumnsScoreWriter` is the old score file format used by bob.
It consists of a text file with four columns separated by spaces and no header.
Each row represents a comparison between a *probe* and a *model*, and the similarity score resulting from that comparison.
The four columns are, in order:
......@@ -521,8 +522,10 @@ Evaluation
Once scores are generated for each probe evaluated against the corresponding models, conclusions on the performance of the system can be drawn by using metrics like the FMR and FNMR or tools like the ROC curve.
In this section, commands that help to quickly evaluate a set of scores by generating metrics or plots are presented.
The scripts take as input a four-column file data format as specified in the documentation of :py:func:`bob.bio.base.score.load.four_column`.
In this section, commands that help to quickly evaluate a set of scores by
generating metrics or plots are presented. The scripts take as input either a
csv file or a legacy four/five column format. See
:ref:`bob.bio.base.score_formats` for the explanation of their formats.
.. note::
......@@ -536,7 +539,7 @@ min.HTER) on a development set and apply it on an evaluation set, just do:
.. code-block:: sh
$ bob bio metrics -e output/scores-{dev,eval} --legends ExpA --criterion min-hter
$ bob bio metrics -e output/scores-{dev,eval}.csv --legends ExpA --criterion min-hter
[Min. criterion: MIN-HTER ] Threshold on Development set `ExpA`: -1.399200e-01
===================== ============== ==============
......@@ -557,8 +560,8 @@ min.HTER) on a development set and apply it on an evaluation set, just do:
When evaluation scores are provided, the ``--eval`` option must be passed.
See ``metrics --help`` for further options.
The ``scores-{dev,eval}`` brace expansion expands the path to every element
in the list. here: ``scores-dev scores-eval``.
The ``scores-{dev,eval}.csv`` brace expansion (a bash feature) expands the
path to every element in the list. here: ``scores-dev.csv scores-eval.csv``.
.. _bob.bio.base.bob_bio_plots:
......@@ -587,7 +590,7 @@ For example, to generate a CMC curve from development and evaluation datasets:
.. code-block:: sh
$ bob bio cmc -v --eval --output 'my_cmc.pdf' dev-1.txt eval-1.txt dev-2.txt eval-2.txt
$ bob bio cmc -v --eval --output 'my_cmc.pdf' dev-1.csv eval-1.csv dev-2.csv eval-2.csv
where ``my_cmc.pdf`` will contain CMC curves for the two experiments represented
by their respective *dev* and *eval* scores-file.
......@@ -612,7 +615,7 @@ plots for a list of experiments. It generates two ``metrics`` outputs with EER,
HTER, minDCF criteria, along with ``roc``, ``det``, ``epc``, and ``hist`` plots
for each experiment. For example::
$ bob bio evaluate -v --eval --log 'my_metrics.txt' --output 'my_plots.pdf' {sys1,sys2}/{dev,eval}
$ bob bio evaluate -v --eval --log 'my_metrics.txt' --output 'my_plots.pdf' {sys1,sys2}/scores-{dev,eval}.csv
will output metrics and plots for the two experiments (dev and eval pairs) in
``my_metrics.txt`` and ``my_plots.pdf``, respectively.
......
......@@ -195,7 +195,7 @@ parameter to the vanilla-biometrics command:
.. code-block:: sh
$ bob bio pipelines vanilla-biometrics --write-metadata-scores custom_database_config.py facenet-sanderberg
$ bob bio pipelines vanilla-biometrics custom_database_config.py facenet-sanderberg
Computing the scores
......@@ -220,14 +220,9 @@ following command:
.. code-block:: sh
$ bob bio pipelines vanilla-biometrics --write-metadata-scores -v --output ./results \
$ bob bio pipelines vanilla-biometrics -v --output ./results \
--group dev --group eval replaymobile-img facenet-sanderberg
The ``--write-metadata-scores`` option is necessary, as the scores
samples will need to retain the information of being an attack or not (``attack_type``
field). This information is passed as metadata through the samples, into the score
files.
``replaymobile-img`` points to a python configuration file defining a ``database``
object.
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment