diff --git a/bob/measure/__init__.py b/bob/measure/__init__.py index 8d8c74b8ef0190283bc063977564efb63e0484a9..a514d59a2a23942ba342a44accef72b6b01dd1fd 100644 --- a/bob/measure/__init__.py +++ b/bob/measure/__init__.py @@ -13,10 +13,9 @@ from . import openbr import numpy def mse (estimation, target): - """mse(estimation, target) -> error + """Mean square error between a set of outputs and target values - Calculates the mean square error between a set of outputs and target - values using the following formula: + Uses the formula: .. math:: @@ -26,14 +25,30 @@ def mse (estimation, target): have 2 dimensions. Different examples are organized as rows while different features in the estimated values or targets are organized as different columns. + + + Parameters: + + estimation (array): an N-dimensional array that corresponds to the value + estimated by your procedure + + target (array): an N-dimensional array that corresponds to the expected + value + + + Returns: + + float: The average of the squared error between the estimated value and the + target + """ return numpy.mean((estimation - target)**2, 0) + def rmse (estimation, target): - """rmse(estimation, target) -> error + """Calculates the root mean square error between a set of outputs and target - Calculates the root mean square error between a set of outputs and target - values using the following formula: + Uses the formula: .. math:: @@ -43,14 +58,30 @@ def rmse (estimation, target): have 2 dimensions. Different examples are organized as rows while different features in the estimated values or targets are organized as different columns. + + + Parameters: + + estimation (array): an N-dimensional array that corresponds to the value + estimated by your procedure + + target (array): an N-dimensional array that corresponds to the expected + value + + + Returns: + + float: The square-root of the average of the squared error between the + estimated value and the target + """ return numpy.sqrt(mse(estimation, target)) + def relevance (input, machine): - """relevance (input, machine) -> relevances + """Calculates the relevance of every input feature to the estimation process - Calculates the relevance of every input feature to the estimation process - using the following definition from: + Uses the formula: Neural Triggering System Operating on High Resolution Calorimetry Information, Anjos et al, April 2006, Nuclear Instruments and Methods in @@ -65,6 +96,22 @@ def relevance (input, machine): input vectors. For this to work, the `input` parameter has to be a 2D array with features arranged column-wise while different examples are arranged row-wise. + + + Parameters: + + input (array): an N-dimensional array that corresponds to the value + estimated by your model + + machine (object): A machine that can be called to "process" your input + + + Returns: + + array: An 1D float array as large as the number of columns (second + dimension) of your input array, estimating the "relevance" of each input + column (or feature) to the score provided by the machine. + """ o = machine(input) @@ -80,51 +127,74 @@ def relevance (input, machine): def recognition_rate(cmc_scores, threshold = None, rank = 1): - """recognition_rate(cmc_scores, rank, threshold) -> RR + """Calculates the recognition rate from the given input - Calculates the recognition rate from the given input, which is identical - to the CMC value for the given ``rank``. + It is identical to the CMC value for the given ``rank``. - The input has a specific format, which is a list of two-element tuples. - Each of the tuples contains the negative :math:`\\{S_p^-\\}` and the positive :math:`\\{S_p^+\\}` scores for one probe item :math:`p`, or ``None`` in case of open set recognition. - To read the lists from score files in 4 or 5 column format, please use the :py:func:`bob.measure.load.cmc_four_column` or :py:func:`bob.measure.load.cmc_five_column` function. + The input has a specific format, which is a list of two-element tuples. Each + of the tuples contains the negative :math:`\\{S_p^-\\}` and the positive + :math:`\\{S_p^+\\}` scores for one probe item :math:`p`, or ``None`` in case + of open set recognition. To read the lists from score files in 4 or 5 column + format, please use the :py:func:`bob.measure.load.cmc_four_column` or + :py:func:`bob.measure.load.cmc_five_column` function. - If **threshold** is set to ``None``, the rank 1 recognition rate is defined as the number of test items, for which the highest positive :math:`\\max\\{S_p^+\\}` score is greater than or equal to all negative scores, divided by the number of all probe items :math:`P`: + If ``threshold`` is set to ``None``, the rank 1 recognition rate is defined + as the number of test items, for which the highest positive + :math:`\\max\\{S_p^+\\}` score is greater than or equal to all negative + scores, divided by the number of all probe items :math:`P`: .. math:: \\mathrm{RR} = \\frac{1}{P} \\sum_{p=1}^{P} \\begin{cases} 1 & \\mathrm{if } \\max\\{S_p^+\\} >= \\max\\{S_p^-\\}\\\\ 0 & \\mathrm{otherwise} \\end{cases} - For a given rank :math:`r>1`, up to :math:`r` negative scores that are higher than the highest positive score are allowed to still count as correctly classified in the top :math:`r` rank. + For a given rank :math:`r>1`, up to :math:`r` negative scores that are higher + than the highest positive score are allowed to still count as correctly + classified in the top :math:`r` rank. - If ``threshold`` :math:`\\theta` is given, **all** scores below threshold will be filtered out. - Hence, if all positive scores are below threshold :math:`\\max\\{S_p^+\\} < \\theta`, the probe will be misclassified **at any rank**. + If ``threshold`` :math:`\\theta` is given, **all** scores below threshold + will be filtered out. Hence, if all positive scores are below threshold + :math:`\\max\\{S_p^+\\} < \\theta`, the probe will be misclassified **at any + rank**. - For open set recognition, i.e., when there exist a tuple including negative scores without corresponding positive scores (``None``), and **all** negative scores are below ``threshold`` :math:`\\max\\{S_p^+\\} < \\theta`, the probe item is correctly rejected, **and it does not count into the denominator** :math:`P`. - When no ``threshold`` is provided, the open set probes will **always** count as misclassified, regardless of the ``rank``. + For open set recognition, i.e., when there exist a tuple including negative + scores without corresponding positive scores (``None``), and **all** negative + scores are below ``threshold`` :math:`\\max\\{S_p^+\\} < \\theta`, the probe + item is correctly rejected, **and it does not count into the denominator** + :math:`P`. When no ``threshold`` is provided, the open set probes will + **always** count as misclassified, regardless of the ``rank``. .. warn: For open set tests, this rate does not correspond to a standard rate. - Please use :py:func:`detection_identification_rate` and :py:func:`false_alarm_rate` instead. + Please use :py:func:`detection_identification_rate` and + :py:func:`false_alarm_rate` instead. + + + Parameters: - **Parameters:** + cmc_scores (list): A list in the format ``[(negatives, positives), ...]`` + containing the CMC scores loaded with one of the functions + (:py:func:`bob.measure.load.cmc_four_column` or + :py:func:`bob.measure.load.cmc_five_column`). - ``cmc_scores`` : [(array_like(1D, float), array_like(1D, float))] - CMC scores loaded with one of the functions (:py:func:`bob.measure.load.cmc_four_column` or :py:func:`bob.measure.load.cmc_five_column`). - Each pair contains the ``negative`` and the ``positive`` scores for **one probe item**. - Each pair can contain up to one empty array (or ``None``), i.e., in case of open set recognition. + Each pair contains the ``negative`` and the ``positive`` scores for **one + probe item**. Each pair can contain up to one empty array (or ``None``), + i.e., in case of open set recognition. - ``threshold`` : float or ``None`` - Decision threshold. If not ``None``, **all** scores will be filtered by the threshold. - In an open set recognition problem, all open set scores (negatives with no corresponding positive) for which all scores are below threshold, will be counted as correctly rejected and **removed** from the probe list (i.e., the denominator). + threshold (:obj:`float`, optional): Decision threshold. If not ``None``, **all** + scores will be filtered by the threshold. In an open set recognition + problem, all open set scores (negatives with no corresponding positive) + for which all scores are below threshold, will be counted as correctly + rejected and **removed** from the probe list (i.e., the denominator). - ``rank`` : int or ``None`` - The rank for which the recognition rate should be computed, 1 by default. + rank (:obj:`int`, optional): + The rank for which the recognition rate should be computed, 1 by default. - **Returns:** - ``RR`` : float - The (open set) recognition rate for the given rank, a value between 0 and 1. + Returns: + + float: The (open set) recognition rate for the given rank, a value between + 0 and 1. + """ # If no scores are given, the recognition rate is exactly 0. if not cmc_scores: @@ -184,9 +254,7 @@ def recognition_rate(cmc_scores, threshold = None, rank = 1): def cmc(cmc_scores): - """cmc(cmc_scores) -> curve - - Calculates the cumulative match characteristic (CMC) from the given input. + """Calculates the cumulative match characteristic (CMC) from the given input. The input has a specific format, which is a list of two-element tuples. Each of the tuples contains the negative and the positive scores for one probe @@ -194,25 +262,38 @@ def cmc(cmc_scores): the :py:func:`bob.measure.load.cmc_four_column` or :py:func:`bob.measure.load.cmc_five_column` function. - For each probe item the probability that the rank :math:`r` of the positive score is - calculated. The rank is computed as the number of negative scores that are - higher than the positive score. If several positive scores for one test item - exist, the **highest** positive score is taken. The CMC finally computes how - many test items have rank r or higher, divided by the total number of test values. + For each probe item the probability that the rank :math:`r` of the positive + score is calculated. The rank is computed as the number of negative scores + that are higher than the positive score. If several positive scores for one + test item exist, the **highest** positive score is taken. The CMC finally + computes how many test items have rank r or higher, divided by the total + number of test values. .. note:: - The CMC is not available for open set classification. - Please use the :py:func:`detection_identification_rate` and :py:func:`false_alarm_rate` instead. - **Parameters:** + The CMC is not available for open set classification. Please use the + :py:func:`detection_identification_rate` and :py:func:`false_alarm_rate` + instead. + + + Parameters: + + cmc_scores (list): A list in the format ``[(negatives, positives), ...]`` + containing the CMC scores loaded with one of the functions + (:py:func:`bob.measure.load.cmc_four_column` or + :py:func:`bob.measure.load.cmc_five_column`). - ``cmc_scores`` : [(array_like(1D, float), array_like(1D, float))] - A list of tuples, where each tuple contains the ``negative`` and ``positive`` scores for one probe of the database + Each pair contains the ``negative`` and the ``positive`` scores for **one + probe item**. Each pair can contain up to one empty array (or ``None``), + i.e., in case of open set recognition. - **Returns:** - ``curve`` : array_like(2D, float) - The CMC curve, with the Rank in the first column and the number of correctly classified clients (in this rank) in the second column. + Returns: + + array: A 2D float array representing the CMC curve, with the Rank in the + first column and the number of correctly classified clients (in this + rank) in the second column. + """ # If no scores are given, we cannot plot anything @@ -243,33 +324,41 @@ def cmc(cmc_scores): def detection_identification_rate(cmc_scores, threshold, rank = 1): - """detection_identification_rate(cmc_scores, threshold, rank) -> dir + """Computes the `detection and identification rate` for the given threshold. + + This value is designed to be used in an open set identification protocol, and + defined in Chapter 14.1 of [LiJain2005]_. + + Although the detection and identification rate is designed to be computed on + an open set protocol, it uses only the probe elements, for which a + corresponding gallery element exists. For closed set identification + protocols, this function is identical to :py:func:`recognition_rate`. The + only difference is that for this function, a ``threshold`` for the scores + need to be defined, while for :py:func:`recognition_rate` it is optional. + + + Parameters: - Computes the `detection and identification rate` for the given threshold. - This value is designed to be used in an open set identification protocol, and defined in Chapter 14.1 of [LiJain2005]_. + cmc_scores (list): A list in the format ``[(negatives, positives), ...]`` + containing the CMC scores loaded with one of the functions + (:py:func:`bob.measure.load.cmc_four_column` or + :py:func:`bob.measure.load.cmc_five_column`). - Although the detection and identification rate is designed to be computed on an open set protocol, it uses only the probe elements, for which a corresponding gallery element exists. - For closed set identification protocols, this function is identical to :py:func:`recognition_rate`. - The only difference is that for this function, a ``threshold`` for the scores need to be defined, while for :py:func:`recognition_rate` it is optional. + Each pair contains the ``negative`` and the ``positive`` scores for **one + probe item**. Each pair can contain up to one empty array (or ``None``), + i.e., in case of open set recognition. - **Parameters:** + threshold (float): The decision threshold :math:`\\tau``. - ``cmc_scores`` : [(array_like(1D, float), array_like(1D, float))] - CMC scores loaded with one of the functions (:py:func:`bob.measure.load.cmc_four_column` or :py:func:`bob.measure.load.cmc_five_column`). - Each pair contains the ``negative`` and the ``positive`` scores for **one probe item**. - There need to be at least one probe item, for which positive and negative scores exist. + rank (:obj:`int`, optional): The rank for which the curve should be plotted - ``threshold`` : float - The decision threshold :math:`\\tau``. - ``rank`` : int - The rank for which the curve should be plotted, by default 1. + Returns: - **Returns:** + float: The detection and identification rate for the given threshold. - ``dir`` : float - The detection and identification rate for the given threshold. """ + # count the correctly classifier probes correct = 0 counter = 0 @@ -299,27 +388,34 @@ def detection_identification_rate(cmc_scores, threshold, rank = 1): def false_alarm_rate(cmc_scores, threshold): - """false_alarm_rate(cmc_scores, threshold) -> far + """Computes the `false alarm rate` for the given threshold,. + + This value is designed to be used in an open set identification protocol, and + defined in Chapter 14.1 of [LiJain2005]_. + + The false alarm rate is designed to be computed on an open set protocol, it + uses only the probe elements, for which **no** corresponding gallery element + exists. + + + Parameters: - Computes the `false alarm rate` for the given threshold,. - This value is designed to be used in an open set identification protocol, and defined in Chapter 14.1 of [LiJain2005]_. + cmc_scores (list): A list in the format ``[(negatives, positives), ...]`` + containing the CMC scores loaded with one of the functions + (:py:func:`bob.measure.load.cmc_four_column` or + :py:func:`bob.measure.load.cmc_five_column`). - The false alarm rate is designed to be computed on an open set protocol, it uses only the probe elements, for which **no** corresponding gallery element exists. + Each pair contains the ``negative`` and the ``positive`` scores for **one + probe item**. Each pair can contain up to one empty array (or ``None``), + i.e., in case of open set recognition. - **Parameters:** + threshold (float): The decision threshold :math:`\\tau``. - ``cmc_scores`` : [(array_like(1D, float), array_like(1D, float))] - CMC scores loaded with one of the functions (:py:func:`bob.measure.load.cmc_four_column` or :py:func:`bob.measure.load.cmc_five_column`). - Each pair contains the ``negative`` and the ``positive`` scores for **one probe item**. - There need to be at least one probe item, for which only negative scores exist. - ``threshold`` : float - The decision threshold :math:`\\tau``. + Returns: - **Returns:** + float: The false alarm rate. - ``far`` : float - The false alarm rate. """ incorrect = 0 counter = 0 diff --git a/bob/measure/calibration.py b/bob/measure/calibration.py index 35a555648078b02e00f63384038b3cd66c46dead..e025e0c8cbc9d7c0167235fa6b3ba20fda3dd70c 100644 --- a/bob/measure/calibration.py +++ b/bob/measure/calibration.py @@ -1,29 +1,33 @@ #!/usr/bin/env python # vim: set fileencoding=utf-8 : -# Manuel Guenther # Thu May 16 11:41:49 CEST 2013 -# -# Copyright (C) 2011-2013 Idiap Research Institute, Martigny, Switzerland """Measures for calibration""" import math import numpy + def cllr(negatives, positives): - """cllr(negatives, positives) -> cllr + """Cost of log likelihood ratio as defined by the Bosaris toolkit + + Computes the 'cost of log likelihood ratio' (:math:`C_{llr}`) measure as + given in the Bosaris toolkit + + + Parameters: + + negatives (array): 1D float array that contains the scores of the + "negative" (noise, non-class) samples of your classifier. - Computes the 'cost of log likelihood ratio' (:math:`C_{llr}`) measure as given in the Bosaris toolkit + positives (array): 1D float array that contains the scores of the + "positive" (signal, class) samples of your classifier. - **Parameters:** - ``negatives, positives`` : array_like(1D, float) - The scores computed by comparing elements from different classes and the same class, respectively. + Returns: - **Returns** + float: The computed :math:`C_{llr}` value. - ``cllr`` : float - The computed :math:`C_{llr}` value. """ sum_pos, sum_neg = 0., 0. for pos in positives: @@ -34,19 +38,25 @@ def cllr(negatives, positives): def min_cllr(negatives, positives): - """min_cllr(negatives, positives) -> min_cllr + """Minimum cost of log likelihood ratio as defined by the Bosaris toolkit + + Computes the 'minimum cost of log likelihood ratio' (:math:`C_{llr}^{min}`) + measure as given in the bosaris toolkit + + + Parameters: + + negatives (array): 1D float array that contains the scores of the + "negative" (noise, non-class) samples of your classifier. - Computes the 'minimum cost of log likelihood ratio' (:math:`C_{llr}^{min}`) measure as given in the bosaris toolkit + positives (array): 1D float array that contains the scores of the + "positive" (signal, class) samples of your classifier. - **Parameters:** - ``negatives, positives`` : array_like(1D, float) - The scores computed by comparing elements from different classes and the same class, respectively. + Returns: - **Returns** + float: The computed :math:`C_{llr}^{min}` value. - ``min_cllr`` : float - The computed :math:`C_{llr}^{min}` value. """ from bob.math import pavx diff --git a/bob/measure/load.py b/bob/measure/load.py index 4a170cd1420e8c4dff2cbc2d76d1ade529afddb5..3cbc4300e55db2b2a85d8522f39f6a2540fdf8de 100644 --- a/bob/measure/load.py +++ b/bob/measure/load.py @@ -1,6 +1,5 @@ #!/usr/bin/env python # vim: set fileencoding=utf-8 : -# Andre Anjos # Mon 23 May 2011 16:23:05 CEST """A set of utilities to load score files with different formats. @@ -13,23 +12,30 @@ import os import logging logger = logging.getLogger('bob.measure') + def open_file(filename, mode='rt'): - """open_file(filename) -> file_like + """Opens the given score file for reading. + + Score files might be raw text files, or a tar-file including a single score + file inside. + + + Parameters: + + filename (:py:class:`str`, ``file-like``): The name of the score file to + open, or a file-like object open for reading. If a file name is given, + the according file might be a raw text file or a (compressed) tar file + containing a raw text file. - Opens the given score file for reading. - Score files might be raw text files, or a tar-file including a single score file inside. - **Parameters:** + Returns: - ``filename`` : str or file-like - The name of the score file to open, or a file-like object open for reading. - If a file name is given, the according file might be a raw text file or a (compressed) tar file containing a raw text file. - **Returns:** + ``file-like``: A read-only file-like object as it would be returned by + :py:func:`open`. - ``file_like`` : file-like - A read-only file-like object as it would be returned by open(). """ + if not isinstance(filename, str) and hasattr(filename, 'read'): # It seems that this is an open file return filename @@ -54,32 +60,36 @@ def open_file(filename, mode='rt'): def four_column(filename): - """four_column(filename) -> claimed_id, real_id, test_label, score + """Loads a score set from a single file and yield its lines + + Loads a score set from a single file and yield its lines (to avoid loading + the score file at once into memory). This function verifies that all fields + are correctly placed and contain valid fields. The score file must contain + the following information in each line: + + .. code-block:: text + + claimed_id real_id test_label score - Loads a score set from a single file and yield its lines (to avoid loading the score file at once into memory). - This function verifies that all fields are correctly placed and contain valid fields. - The score file must contain the following information in each line: - claimed_id real_id test_label score + Parameters: - **Parametes:** + filename (:py:class:`str`, ``file-like``): The file object that will be + opened with :py:func:`open_file` containing the scores. - ``filename`` : str or file-like - The file object that will be opened with :py:func:`open_file` containing the scores. - **Yields:** + Returns: - ``claimed_id`` : str - The claimed identity -- the client name of the model that was used in the comparison + str: The claimed identity -- the client name of the model that was used in + the comparison - ``real_id`` : str - The real identity -- the client name of the probe that was used in the comparison + str: The real identity -- the client name of the probe that was used in the + comparison - ``test_label`` : str - A label of the probe -- usually the probe file name, or the probe id + str: A label of the probe -- usually the probe file name, or the probe id + + float: The result of the comparison of the model and the probe - ``score`` : float - The result of the comparison of the model and the probe """ for i, l in enumerate(open_file(filename)): @@ -97,57 +107,67 @@ def four_column(filename): def split_four_column(filename): - """split_four_column(filename) -> negatives, positives + """Loads a score set from a single file and splits the scores - Loads a score set from a single file and splits the scores - between negatives and positives. The score file has to respect the 4 column - format as defined in the method :py:func:`four_column`. + Loads a score set from a single file and splits the scores between negatives + and positives. The score file has to respect the 4 column format as defined + in the method :py:func:`four_column`. This method avoids loading and allocating memory for the strings present in the file. We only keep the scores. - **Parameters:** - ``filename`` : str or file-like - The file that will be opened with :py:func:`open_file` containing the scores. + Parameters: + + filename (:py:class:`str`, ``file-like``): The file object that will be + opened with :py:func:`open_file` containing the scores. + - **Returns:** + Returns: - ``negatives`` : array_like(1D, float) - The list of ``score``'s, for which the ``claimed_id`` and the ``real_id`` differed (see :py:func:`four_column`). + negatives (array): 1D float array containing the list of scores, for which + the ``claimed_id`` and the ``real_id`` differed (see + :py:func:`four_column`) + + positivies (array): 1D float array containing the list of scores, for which + the ``claimed_id`` and the ``real_id`` are identical (see + :py:func:`four_column`) - ``positives`` : array_like(1D, float) - The list of ``score``'s, for which the ``claimed_id`` and the ``real_id`` are identical (see :py:func:`four_column`). """ + score_lines = load_score(filename, 4) return get_negatives_positives(score_lines) + def cmc_four_column(filename): - """cmc_four_column(filename) -> cmc_scores + """Loads scores to compute CMC curves from a file in four column format. - Loads scores to compute CMC curves from a file in four column format. - The four column file needs to be in the same format as described in :py:func:`four_column`, - and the ``test_label`` (column 3) has to contain the test/probe file name or a probe id. + The four column file needs to be in the same format as described in + :py:func:`four_column`, and the ``test_label`` (column 3) has to contain the + test/probe file name or a probe id. - This function returns a list of tuples. - For each probe file, the tuple consists of a list of negative scores and a list of positive scores. - Usually, the list of positive scores should contain only one element, but more are allowed. - The result of this function can directly be passed to, e.g., the :py:func:`bob.measure.cmc` function. + This function returns a list of tuples. For each probe file, the tuple + consists of a list of negative scores and a list of positive scores. + Usually, the list of positive scores should contain only one element, but + more are allowed. The result of this function can directly be passed to, + e.g., the :py:func:`bob.measure.cmc` function. - **Parameters:** + Parameters: - ``filename`` : str or file-like - The file that will be opened with :py:func:`open_file` containing the scores. + filename (:py:class:`str`, ``file-like``): The file object that will be + opened with :py:func:`open_file` containing the scores. - **Returns:** + Returns: - ``cmc_scores`` : [(negatives, positives)] - A list of tuples, where each tuple contains the ``negative`` and ``positive`` scores for one probe of the database. - Both ``negatives`` and ``positives`` can be either an 1D :py:class:`numpy.ndarray` of type ``float``, or ``None``. + list: A list of tuples, where each tuple contains the ``negative`` and + ``positive`` scores for one probe of the database. Both ``negatives`` and + ``positives`` can be either an 1D :py:class:`numpy.ndarray` of type + ``float``, or ``None``. """ + # extract positives and negatives pos_dict = {} neg_dict = {} @@ -166,37 +186,39 @@ def cmc_four_column(filename): return _convert_cmc_scores(neg_dict, pos_dict) - def five_column(filename): - """five_column(filename) -> claimed_id, model_label, real_id, test_label, score + """Loads a score set from a single file and yield its lines + + Loads a score set from a single file and yield its lines (to avoid loading + the score file at once into memory). This function verifies that all fields + are correctly placed and contain valid fields. The score file must contain + the following information in each line: - Loads a score set from a single file and yield its lines (to avoid loading the score file at once into memory). - This function verifies that all fields are correctly placed and contain valid fields. - The score file must contain the following information in each line: + .. code-block:: text - claimed_id model_label real_id test_label score + claimed_id model_label real_id test_label score - **Parametes:** - ``filename`` : str or file-like - The file object that will be opened with :py:func:`open_file` containing the scores. + Parameters: - **Yields:** + filename (:py:class:`str`, ``file-like``): The file object that will be + opened with :py:func:`open_file` containing the scores. - ``claimed_id`` : str - The claimed identity -- the client name of the model that was used in the comparison - ``model_label`` : str - A label for the model -- usually the model file name, or the model id + Returns: - ``real_id`` : str - The real identity -- the client name of the probe that was used in the comparison + str: The claimed identity -- the client name of the model that was used in + the comparison - ``test_label`` : str - A label of the probe -- usually the probe file name, or the probe id + str: A label for the model -- usually the model file name, or the model id + + str: The real identity -- the client name of the probe that was used in the + comparison + + str: A label of the probe -- usually the probe file name, or the probe id + + float: The result of the comparison of the model and the probe - ``score`` : float - The result of the comparison of the model and the probe. """ for i, l in enumerate(open_file(filename)): @@ -214,53 +236,62 @@ def five_column(filename): def split_five_column(filename): - """split_five_column(filename) -> negatives, positives + """Loads a score set from a single file and splits the scores - Loads a score set from a single file in five column format and splits the scores - between negatives and positives. The score file has to respect the 4 column - format as defined in the method :py:func:`five_column`. + Loads a score set from a single file in five column format and splits the + scores between negatives and positives. The score file has to respect the 5 + column format as defined in the method :py:func:`five_column`. This method avoids loading and allocating memory for the strings present in the file. We only keep the scores. - **Parameters:** - ``filename`` : str or file-like - The file that will be opened with :py:func:`open_file` containing the scores. + Parameters: + + filename (:py:class:`str`, ``file-like``): The file object that will be + opened with :py:func:`open_file` containing the scores. + - **Returns:** + Returns: - ``negatives`` : array_like(1D, float) - The list of ``score``'s, for which the ``claimed_id`` and the ``real_id`` differed (see :py:func:`five_column`). + negatives (array): 1D float array containing the list of scores, for which + the ``claimed_id`` and the ``real_id`` differed (see + :py:func:`four_column`) + + positivies (array): 1D float array containing the list of scores, for which + the ``claimed_id`` and the ``real_id`` are identical (see + :py:func:`four_column`) - ``positives`` : array_like(1D, float) - The list of ``score``'s, for which the ``claimed_id`` and the ``real_id`` are identical (see :py:func:`five_column`). """ + score_lines = load_score(filename, 5) return get_negatives_positives(score_lines) def cmc_five_column(filename): - """cmc_four_column(filename) -> cmc_scores + """Loads scores to compute CMC curves from a file in five column format. + + The five column file needs to be in the same format as described in + :py:func:`five_column`, and the ``test_label`` (column 4) has to contain the + test/probe file name or a probe id. + + This function returns a list of tuples. For each probe file, the tuple + consists of a list of negative scores and a list of positive scores. + Usually, the list of positive scores should contain only one element, but + more are allowed. The result of this function can directly be passed to, + e.g., the :py:func:`bob.measure.cmc` function. - Loads scores to compute CMC curves from a file in five column format. - The four column file needs to be in the same format as described in :py:func:`five_column`, - and the ``test_label`` (column 4) has to contain the test/probe file name or a probe id. - This function returns a list of tuples. - For each probe file, the tuple consists of a list of negative scores and a list of positive scores. - Usually, the list of positive scores should contain only one element, but more are allowed. - The result of this function can directly be passed to, e.g., the :py:func:`bob.measure.cmc` function. + Parameters: - **Parameters:** + filename (:py:class:`str`, ``file-like``): The file object that will be + opened with :py:func:`open_file` containing the scores. - ``filename`` : str or file-like - The file that will be opened with :py:func:`open_file` containing the scores. - **Returns:** + Returns: - ``cmc_scores`` : [(array_like(1D, float), array_like(1D, float))] - A list of tuples, where each tuple contains the ``negative`` and ``positive`` scores for one probe of the database + list: A list of tuples, where each tuple contains the ``negative`` and + ``positive`` scores for one probe of the database. """ # extract positives and negatives @@ -284,21 +315,20 @@ def cmc_five_column(filename): def load_score(filename, ncolumns=None): """Load scores using numpy.loadtxt and return the data as a numpy array. - **Parameters:** + Parameters: - ``filename`` : str or file-like - A path or file-like object that will be read with :py:func:`numpy.loadtxt` - containing the scores. + filename (:py:class:`str`, ``file-like``): The file object that will be + opened with :py:func:`open_file` containing the scores. - ``ncolumns`` : 4 or 5 or None [default: None] - Specify the number of columns in the score file. If None is provided, - the number of columns will be guessed. + ncolumns (:py:class:`int`, optional): 4, 5 or None (the default), + specifying the number of columns in the score file. If None is provided, + the number of columns will be guessed. - **Returns:** - ``score_lines`` : numpy.array - An array which contains not only the actual scores but also the - 'claimed_id', 'real_id', 'test_label', and ['model_label'] + Returns: + + array: An array which contains not only the actual scores but also the + ``claimed_id``, ``real_id``, ``test_label`` and ``['model_label']`` """ @@ -348,10 +378,11 @@ def load_score(filename, ncolumns=None): def get_negatives_positives(score_lines): - """Take the output of load_score and return negatives and positives. - This function aims to replace split_four_column and split_five_column - but takes a different input. It's up to you to use which one. + """Take the output of load_score and return negatives and positives. This + function aims to replace split_four_column and split_five_column but takes a + different input. It's up to you to use which one. """ + pos_mask = score_lines['claimed_id'] == score_lines['real_id'] positives = score_lines['score'][pos_mask] negatives = score_lines['score'][numpy.logical_not(pos_mask)] @@ -360,7 +391,9 @@ def get_negatives_positives(score_lines): def get_negatives_positives_all(score_lines_list): """Take a list of outputs of load_score and return stacked negatives and - positives.""" + positives. + """ + negatives, positives = [], [] for score_lines in score_lines_list: neg_pos = get_negatives_positives(score_lines) @@ -373,6 +406,7 @@ def get_negatives_positives_all(score_lines_list): def get_all_scores(score_lines_list): """Take a list of outputs of load_score and return stacked scores""" + return numpy.vstack([score_lines['score'] for score_lines in score_lines_list]).T @@ -381,6 +415,7 @@ def dump_score(filename, score_lines): """Dump scores that were loaded using :py:func:`load_score` The number of columns is automatically detected. """ + if len(score_lines.dtype) == 5: fmt = '%s %s %s %s %.9f' elif len(score_lines.dtype) == 4: @@ -389,8 +424,13 @@ def dump_score(filename, score_lines): raise ValueError("Only scores with 4 and 5 columns are supported.") numpy.savetxt(filename, score_lines, fmt=fmt) + def _convert_cmc_scores(neg_dict, pos_dict): - """Converts the negative and positive scores read with :py:func:`cmc_four_column` or :py:func:`cmc_four_column` into a format that is handled by the :py:func:`bob.measure.cmc` and similar functions.""" + """Converts the negative and positive scores read with + :py:func:`cmc_four_column` or :py:func:`cmc_four_column` into a format that + is handled by the :py:func:`bob.measure.cmc` and similar functions. + """ + # convert to lists of tuples of ndarrays (or None) probe_names = sorted(set(neg_dict.keys()).union(set(pos_dict.keys()))) # get all scores in the desired format diff --git a/bob/measure/main.cpp b/bob/measure/main.cpp index 558502cb430fc01e2d90453126d87e4dc129dccf..9baa2f14b1efbf055520d4c438d52d3b94091683 100644 --- a/bob/measure/main.cpp +++ b/bob/measure/main.cpp @@ -102,7 +102,7 @@ static auto det_doc = bob::extension::FunctionDoc( "[0] X axis values in the normal deviate scale for the false-accepts\n\n" "[1] Y axis values in the normal deviate scale for the false-rejections\n\n" "You can plot the results using your preferred tool to first create a plot using rows 0 and 1 from the returned value and then replace the X/Y axis annotation using a pre-determined set of tickmarks as recommended by NIST. " - "The derivative scales are computed with the :py:func:`ppndf` function." + "The derivative scales are computed with the :py:func:`bob.measure.ppndf` function." ) .add_prototype("negatives, positives, n_points", "curve") .add_parameter("negatives, positives", "array_like(1D, float)", "The list of negative and positive scores to compute the DET for") @@ -264,7 +264,7 @@ static auto eer_threshold_doc = bob::extension::FunctionDoc( .add_prototype("negatives, positives, [is_sorted]", "threshold") .add_parameter("negatives, positives", "array_like(1D, float)", "The set of negative and positive scores to compute the threshold") .add_parameter("is_sorted", "bool", "[Default: ``False``] Are both sets of scores already in ascendantly sorted order?") -.add_return("threshold", "float", "The threshold (i.e., as used in :py:func:`farfrr`) where FAR and FRR are as close as possible") +.add_return("threshold", "float", "The threshold (i.e., as used in :py:func:`bob.measure.farfrr`) where FAR and FRR are as close as possible") ; static PyObject* eer_threshold(PyObject*, PyObject* args, PyObject* kwds) { BOB_TRY @@ -344,7 +344,7 @@ BOB_CATCH_FUNCTION("min_weighted_error_rate_threshold", 0) static auto min_hter_threshold_doc = bob::extension::FunctionDoc( "min_hter_threshold", - "Calculates the :py:func:`min_weighted_error_rate_threshold` with ``cost=0.5``" + "Calculates the :py:func:`bob.measure.min_weighted_error_rate_threshold` with ``cost=0.5``" ) .add_prototype("negatives, positives, [is_sorted]", "threshold") .add_parameter("negatives, positives", "array_like(1D, float)", "The set of negative and positive scores to compute the threshold") @@ -390,7 +390,7 @@ static auto precision_recall_doc = bob::extension::FunctionDoc( "where :math:`tp` are the true positives, :math:`fp` are the false positives and :math:`fn` are the false negatives.\n\n" "``positives`` holds the score information for samples that are labeled to belong to a certain class (a.k.a., 'signal' or 'client'). " "``negatives`` holds the score information for samples that are labeled **not** to belong to the class (a.k.a., 'noise' or 'impostor'). " - "For more precise details about how the method considers error rates, see :py:func:`farfrr`." + "For more precise details about how the method considers error rates, see :py:func:`bob.measure.farfrr`." ) .add_prototype("negatives, positives, threshold", "precision, recall") .add_parameter("negatives, positives", "array_like(1D, float)", "The set of negative and positive scores to compute the measurements") @@ -429,7 +429,7 @@ BOB_CATCH_FUNCTION("precision_recall", 0) static auto f_score_doc = bob::extension::FunctionDoc( "f_score", "This method computes the F-score of the accuracy of the classification", - "The F-score is a weighted mean of precision and recall measurements, see :py:func:`precision_recall`. " + "The F-score is a weighted mean of precision and recall measurements, see :py:func:`bob.measure.precision_recall`. " "It is computed as:\n\n" ".. math::\n\n" " \\mathrm{f-score} = (1 + w^2)\\frac{\\mathrm{precision}\\cdot{}\\mathrm{recall}}{w^2\\cdot{}\\mathrm{precision} + \\mathrm{recall}}\n\n" diff --git a/bob/measure/openbr.py b/bob/measure/openbr.py index 27ed5d7b4a330c4887ff0380cb678ad002621b51..22fe26e925626d739974b894896ea723e3648d55 100644 --- a/bob/measure/openbr.py +++ b/bob/measure/openbr.py @@ -1,4 +1,10 @@ -"""This file includes functionality to convert between Bob's four column or five column score files and the Matrix files used in OpenBR.""" +#!/usr/bin/env python +# vim: set fileencoding=utf-8 : + + +"""This file includes functionality to convert between Bob's four column or + five column score files and the Matrix files used in OpenBR.""" + import numpy import sys @@ -7,6 +13,7 @@ logger = logging.getLogger("bob.measure") from .load import open_file, four_column, five_column + def write_matrix( score_file, matrix_file, @@ -16,58 +23,67 @@ def write_matrix( score_file_format = '4column', gallery_file_name = 'unknown-gallery.lst', probe_file_name = 'unknown-probe.lst', - search = None -): - """Writes the OpenBR matrix and mask files (version 2), given the score file. - If gallery and probe names are provided, the matrices in both files will be sorted by gallery and probe names. - Otherwise, the order will be the same as given in the score file. + search = None): + """Writes the OpenBR matrix and mask files (version 2), given a score file. + + If gallery and probe names are provided, the matrices in both files will be + sorted by gallery and probe names. Otherwise, the order will be the same as + given in the score file. - If ``search`` is given (as an integer), the resulting matrix files will be in the *search* format, keeping the given number of gallery scores with the highest values for each probe. + If ``search`` is given (as an integer), the resulting matrix files will be in + the *search* format, keeping the given number of gallery scores with the + highest values for each probe. .. warning:: - When provided with a 4-column score file, this function will work only, if there is only a single model id for each client. - **Parameters:** + When provided with a 4-column score file, this function will work only, if + there is only a single model id for each client. + + Parameters: - ``score_file`` : str - The 4 or 5 column style score file written by bob. + score_file (str): The 4 or 5 column style score file written by bob. - ``matrix_file`` : str - The OpenBR matrix file that should be written. - Usually, the file name extension is ``.mtx`` + matrix_file (str): The OpenBR matrix file that should be written. + Usually, the file name extension is ``.mtx`` - ``mask_file`` : str - The OpenBR mask file that should be written. - The mask file defines, which values are positives, negatives or to be ignored. - Usually, the file name extension is ``.mask`` + mask_file (str): The OpenBR mask file that should be written. + The mask file defines, which values are positives, negatives or to be + ignored. Usually, the file name extension is ``.mask`` - ``model_names`` : [str] or ``None`` - If given, the matrix will be written in the same order as the given model names. - The model names must be identical with the second column in the 5-column ``score_file``. + model_names (:py:class:`str`, optional): If given, the matrix will be + written in the same order as the given model names. The model names must + be identical with the second column in the 5-column ``score_file``. - .. note:: - If the score file is in four column format, the model_names must be the client ids stored in the first column. - In this case, there might be only a single model per client + .. note:: - Only the scores of the given models will be considered. + If the score file is in four column format, the model_names must be + the client ids stored in the first column. In this case, there might + be only a single model per client - ``probe_names`` : [str] or ``None`` - If given, the matrix will be written in the same order as the given probe names (the ``path`` of the probe). - The probe names are identical to the third column of the 4-column (or the fourth column of the 5-column) ``score_file``. - Only the scores of the given probe names will be considered in this case. + Only the scores of the given models will be considered. - ``score_file_format`` : one of ``('4column', '5column')`` - The format, in which the ``score_file`` is; defaults to ``'4column'`` + probe_names (:py:class:`list`, optional): A list of strings. If given, + the matrix will be written in the same order as the given probe names + (the ``path`` of the probe). The probe names are identical to the third + column of the 4-column (or the fourth column of the 5-column) + ``score_file``. Only the scores of the given probe names will be + considered in this case. - ``gallery_file_name`` : str - The name of the gallery file that will be written in the header of the OpenBR files. + score_file_format (:py:class:`str`, optional): One of ``('4column', + '5column')``. The format, in which the ``score_file`` is; defaults to + ``'4column'`` - ``probe_file_name`` : str - The name of the probe file that will be written in the header of the OpenBR files. + gallery_file_name (:py:class:`str`, optional): The name of the gallery file + that will be written in the header of the OpenBR files. + + probe_file_name (:py:class:`str`, optional): The name of the probe file that + will be written in the header of the OpenBR files. + + search (:py:class:`int`, optional): If given, the scores will be sorted per + probe, keeping the specified number of highest scores. If the given + number is higher than the models, ``NaN`` values will be added, and the + mask will contain ``0x00`` values. - ``search`` : int or ``None`` - If given, the scores will be sorted per probe, keeping the specified number of highest scores. - If the given number is higher than the models, ``NaN`` values will be added, and the mask will contain ``0x00`` values. """ def _write_matrix(filename, matrix): @@ -178,62 +194,77 @@ def write_score_file( score_file_format = '4column', replace_nan = None ): - """Writes the Bob score file in the desired ``score_file_format`` (four or five column), given the OpenBR matrix and mask files. + """Writes the Bob score file in the desired format from OpenBR files. + + Writes a Bob score file in the desired format (four or five column), given + the OpenBR matrix and mask files. + + In principle, the score file can be written based on the matrix and mask + files, and the format suffice the requirements to compute CMC curves. + However, the contents of the score files can be adapted. If given, the + ``models_ids`` and ``probes_ids`` define the **client ids** of model and + probe, and they have to be in the same order as used to compute the OpenBR + matrix. The ``model_names`` and ``probe_names`` define the **paths** of + model and probe, and they should be in the same order as the ids. - In principle, the score file can be written based on the matrix and mask files, and the format suffice the requirements to compute CMC curves. - However, the contents of the score files can be adapted. - If given, the ``models_ids`` and ``probes_ids`` define the **client ids** of model and probe, and they have to be in the same order as used to compute the OpenBR matrix. - The ``model_names`` and ``probe_names`` define the **paths** of model and probe, and they should be in the same order as the ids. + In rare cases, the OpenBR matrix contains NaN values, which Bob's score files + cannot handle. You can use the ``replace_nan`` parameter to decide, what to + do with these values. By default (``None``), these values are ignored, i.e., + not written into the score file. This is, what OpenBR is doing as well. + However, you can also set ``replace_nan`` to any value, which will be written + instead of the NaN values. - In rare cases, the OpenBR matrix contains NaN values, which Bob's score files cannot handle. - You can use the ``replace_nan`` parameter to decide, what to do with these values. - By default (``None``), these values are ignored, i.e., not written into the score file. - This is, what OpenBR is doing as well. - However, you can also set ``replace_nan`` to any value, which will be written instead of the NaN values. - **Parameters:** + Parameters: - ``matrix_file`` : str - The OpenBR matrix file that should be read. - Usually, the file name extension is ``.mtx`` + matrix_file (str): The OpenBR matrix file that should be read. Usually, the + file name extension is ``.mtx`` - ``mask_file`` : str - The OpenBR mask file that should be read. - Usually, the file name extension is ``.mask`` + mask_file (str): The OpenBR mask file that should be read. Usually, the + file name extension is ``.mask`` - ``score_file`` : str - The 4 or 5 column style score file that should be written. + score_file (str): Path to the 4 or 5 column style score file that should be + written. - ``models_ids`` : [str] or ``None`` - The client ids of the models that will be written in the first column of the score file. - If given, the size must be identical to the number of models (gallery templates) in the OpenBR files. - If not given, client ids of the model will be identical to the **gallery index** in the matrix file. + models_ids (:py:class:`list`, optional): A list of strings with the client + ids of the models that will be written in the first column of the score + file. If given, the size must be identical to the number of models + (gallery templates) in the OpenBR files. If not given, client ids of the + model will be identical to the **gallery index** in the matrix file. - ``probes_ids`` : [str] or ``None``: - The client ids of the probes that will be written in the second/third column of the four/five column score file. - If given, the size must be identical to the number of probe templates in the OpenBR files. - It will be checked that the OpenBR mask fits to the model/probe client ids. - If not given, the probe ids will be estimated automatically, i.e., to fit the OpenBR matrix. + probes_ids (:py:class:`list`, optional): A list of strings with the client + ids of the probes that will be written in the second/third column of the + four/five column score file. If given, the size must be identical to the + number of probe templates in the OpenBR files. It will be checked that + the OpenBR mask fits to the model/probe client ids. If not given, the + probe ids will be estimated automatically, i.e., to fit the OpenBR + matrix. - ``model_names`` : [str] or ``None`` - A list of model path written in the second column of the five column score file. - If not given, the model index in the OpenBR file will be used. + model_names (:py:class:`list`, optional): A list of strings with the model + path written in the second column of the five column score file. If not + given, the model index in the OpenBR file will be used. - .. note:: - This entry is ignored in the four column score file format. + .. note:: - ``probe_names`` : [str] or ``None`` - A list of probe path to be written in the third/fourth column in the four/five column score file. - If given, the size must be identical to the number of probe templates in the OpenBR files. - If not given, the probe index in the OpenBR file will be used. + This entry is ignored in the four column score file format. - ``score_file_format`` : one of ``('4column', '5column')`` - The format, in which the ``score_file`` should be written; defaults to ``'4column'``. + probe_names (:py:class:`list`, optional): A list of probe path to be + written in the third/fourth column in the four/five column score file. If + given, the size must be identical to the number of probe templates in the + OpenBR files. If not given, the probe index in the OpenBR file will be + used. + + score_file_format (:py:class:`str`, optional): One of ``('4column', + '5column')``. The format, in which the ``score_file`` is; defaults to + ``'4column'`` + + replace_nan (:py:class:`float`, optional): If NaN values are encountered in + the OpenBR matrix (which are not ignored due to the mask being non-NULL), + this value will be written instead. If ``None``, the values will not be + written in the score file at all. - ``replace_nan`` : float or ``None``: - If NaN values are encountered in the OpenBR matrix (which are not ignored due to the mask being non-NULL), this value will be written instead. - If ``None``, the values will not be written in the score file at all. """ + def _read_matrix(filename): py3 = sys.version_info[0] >=3 ## Helper function to read a matrix file as written by OpenBR diff --git a/bob/measure/plot.py b/bob/measure/plot.py index 0a66544c1612bfd85dbf40f42a64615f53ba0569..e103c028a95375e358c5f34e772a58ed8e862e0a 100644 --- a/bob/measure/plot.py +++ b/bob/measure/plot.py @@ -1,45 +1,52 @@ #!/usr/bin/env python # vim: set fileencoding=utf-8 : -# Chakka Murali Mohan, Trainee, IDIAP Research Institute, Switzerland. # Mon 23 May 2011 14:36:14 CEST + def log_values(min_step = -4, counts_per_step = 4): - """log_values(min_step, counts_per_step) -> log_list + """Computes log-scaled values between :math:`10^{M}` and 1 + + This function computes log-scaled values between :math:`10^{M}` and 1 + (including), where :math:`M` is the ``min_ste`` argument, which needs to be a + negative integer. The integral ``counts_per_step`` value defines how many + values between two adjacent powers of 10 will be created. The total number + of values will be ``-min_step * counts_per_step + 1``. + - This function computes log-scaled values between :math:`10^{M}` and 1 (including), where :math:`M` is the ``min_ste`` argument, which needs to be a negative integer. - The integral ``counts_per_step`` value defines how many values between two adjacent powers of 10 will be created. - The total number of values will be ``-min_step * counts_per_step + 1``. + Parameters: - **Parameters:** + min_step (:py:class:`int`, optional): The power of 10 that will be the + minimum value. E.g., the default ``-4`` will result in the first number + to be :math:`10^{-4}` = ``0.00001`` or ``0.01%`` - ``min_step`` : int (negative) - The power of 10 that will be the minimum value. E.g., the default ``-4`` will result in the first number to be :math:`10^{-4}` = ``0.00001`` or ``0.01%`` + counts_per_step (:py:class:`int`, optional): The number of values that will + be put between two adjacent powers of 10. With the default value ``4`` + (and default values of ``min_step``), we will get ``log_list[0] == + 1e-4``, ``log_list[4] == 1e-3``, ..., ``log_list[16] == 1``. - ``counts_per_step`` : int (positive) - The number of values that will be put between two adjacent powers of 10. - With the default value ``4`` (and default values of ``min_step``), we will get ``log_list[0] == 1e-4``, ``log_list[4] == 1e-3``, ..., ``log_list[16] == 1``. - **Returns** + Returns: + + list: A list of logarithmically scaled values between :math:`10^{M}` and 1. - ``log_list`` : [float] - A list of logarithmically scaled values between :math:`10^{M}` and 1. """ + import math return [math.pow(10., i * 1./counts_per_step) for i in range(min_step*counts_per_step,0)] + [1.] -"""Methods to plot error analysis figures such as ROC, precision-recall curve, EPC and DET""" def roc(negatives, positives, npoints=100, CAR=False, **kwargs): """Plots Receiver Operating Characteristic (ROC) curve. This method will call ``matplotlib`` to plot the ROC curve for a system which contains a particular set of negatives (impostors) and positives (clients) - scores. We use the standard :py:func:`matplotlib.pyplot.plot` command. All parameters - passed with exception of the three first parameters of this method will be - directly passed to the plot command. + scores. We use the standard :py:func:`matplotlib.pyplot.plot` command. All + parameters passed with exception of the three first parameters of this method + will be directly passed to the plot command. - The plot will represent the false-alarm on the horizontal axis and the false-rejection on the vertical axis. - The values for the axis will be computed using :py:func:`bob.measure.roc`. + The plot will represent the false-alarm on the horizontal axis and the + false-rejection on the vertical axis. The values for the axis will be + computed using :py:func:`bob.measure.roc`. .. note:: @@ -47,23 +54,34 @@ def roc(negatives, positives, npoints=100, CAR=False, **kwargs): issues the plotting command. You are the responsible for setting up and saving the figure as you see fit. - **Parameters:** - ``negatives, positives`` : array_like(1D, float) - The list of negative and positive scores forwarded to :py:func:`bob.measure.roc` + Parameters: - ``npoints`` : int - The number of points forwarded to :py:func:`bob.measure.roc` + negatives (array): 1D float array that contains the scores of the + "negative" (noise, non-class) samples of your classifier. See + (:py:func:`bob.measure.roc`) - ``CAR`` : bool - If set to ``True``, it will plot the CAR over FAR in using :py:func:`matplotlib.pyplot.semilogx`, otherwise the FAR over FRR linearly using :py:func:`matplotlib.pyplot.plot`. + positives (array): 1D float array that contains the scores of the + "positive" (signal, class) samples of your classifier. See + (:py:func:`bob.measure.roc`) - ``kwargs`` : keyword arguments - Extra plotting parameters, which are passed directly to :py:func:`matplotlib.pyplot.plot`. + npoints (:py:class:`int`, optional): The number of points for the plot. See + (:py:func:`bob.measure.roc`) - **Returns:** + CAR (:py:class:`bool`, optional): If set to ``True``, it will plot the CAR + over FAR in using :py:func:`matplotlib.pyplot.semilogx`, otherwise the + FAR over FRR linearly using :py:func:`matplotlib.pyplot.plot`. + + kwargs (:py:class:`dict`, optional): Extra plotting parameters, which are + passed directly to :py:func:`matplotlib.pyplot.plot`. + + + Returns: + + :py:class:`list` of :py:class:`matplotlib.lines.Line2D`: The lines that + were added as defined by the return value of + :py:func:`matplotlib.pyplot.plot`. - The return value is the matplotlib line that was added as defined by :py:func:`matplotlib.pyplot.plot` or :py:func:`matplotlib.pyplot.semilogx`. """ from matplotlib import pyplot @@ -76,16 +94,17 @@ def roc(negatives, positives, npoints=100, CAR=False, **kwargs): def roc_for_far(negatives, positives, far_values = log_values(), **kwargs): - """Plots Receiver Operating Characteristic (ROC) curve for the given list of False Acceptance Rates (FAR). + """Plots the ROC curve for the given list of False Acceptance Rates (FAR). This method will call ``matplotlib`` to plot the ROC curve for a system which contains a particular set of negatives (impostors) and positives (clients) - scores. We use the standard :py:func:`matplotlib.pyplot.semilogx` command. All parameters - passed with exception of the three first parameters of this method will be - directly passed to the plot command. + scores. We use the standard :py:func:`matplotlib.pyplot.semilogx` command. + All parameters passed with exception of the three first parameters of this + method will be directly passed to the plot command. - The plot will represent the False Acceptance Rate (FAR) on the horizontal axis and the Correct Acceptance Rate (CAR) on the vertical axis. - The values for the axis will be computed using :py:func:`bob.measure.roc_for_far`. + The plot will represent the False Acceptance Rate (FAR) on the horizontal + axis and the Correct Acceptance Rate (CAR) on the vertical axis. The values + for the axis will be computed using :py:func:`bob.measure.roc_for_far`. .. note:: @@ -93,20 +112,30 @@ def roc_for_far(negatives, positives, far_values = log_values(), **kwargs): issues the plotting command. You are the responsible for setting up and saving the figure as you see fit. - **Parameters:** - ``negatives, positives`` : array_like(1D, float) - The list of negative and positive scores forwarded to :py:func:`bob.measure.roc` + Parameters: + + negatives (array): 1D float array that contains the scores of the + "negative" (noise, non-class) samples of your classifier. See + (:py:func:`bob.measure.roc`) - ``far_values`` : [float] - The values for the FAR, where the CAR should be plotted; each value should be in range [0,1]. + positives (array): 1D float array that contains the scores of the + "positive" (signal, class) samples of your classifier. See + (:py:func:`bob.measure.roc`) - ``kwargs`` : keyword arguments - Extra plotting parameters, which are passed directly to :py:func:`matplotlib.pyplot.plot`. + far_values (:py:class:`list`, optional): The values for the FAR, where the + CAR should be plotted; each value should be in range [0,1]. - **Returns:** + kwargs (:py:class:`dict`, optional): Extra plotting parameters, which are + passed directly to :py:func:`matplotlib.pyplot.plot`. + + + Returns: + + :py:class:`list` of :py:class:`matplotlib.lines.Line2D`: The lines that + were added as defined by the return value of + :py:func:`matplotlib.pyplot.semilogx`. - The return value is the matplotlib line that was added as defined by :py:func:`matplotlib.pyplot.semilogx`. """ from matplotlib import pyplot @@ -116,13 +145,14 @@ def roc_for_far(negatives, positives, far_values = log_values(), **kwargs): def precision_recall_curve(negatives, positives, npoints=100, **kwargs): - """Plots Precision-Recall curve. + """Plots a Precision-Recall curve. - This method will call ``matplotlib`` to plot the precision-recall curve for a system which - contains a particular set of ``negatives`` (impostors) and ``positives`` (clients) - scores. We use the standard :py:func:`matplotlib.pyplot.plot` command. All parameters - passed with exception of the three first parameters of this method will be - directly passed to the plot command. + This method will call ``matplotlib`` to plot the precision-recall curve for a + system which contains a particular set of ``negatives`` (impostors) and + ``positives`` (clients) scores. We use the standard + :py:func:`matplotlib.pyplot.plot` command. All parameters passed with + exception of the three first parameters of this method will be directly + passed to the plot command. .. note:: @@ -130,20 +160,30 @@ def precision_recall_curve(negatives, positives, npoints=100, **kwargs): issues the plotting command. You are the responsible for setting up and saving the figure as you see fit. - **Parameters:** - ``negatives, positives`` : array_like(1D, float) - The list of negative and positive scores forwarded to :py:func:`bob.measure.precision_recall_curve` + Parameters: + + negatives (array): 1D float array that contains the scores of the + "negative" (noise, non-class) samples of your classifier. See + (:py:func:`bob.measure.precision_recall_curve`) + + positives (array): 1D float array that contains the scores of the + "positive" (signal, class) samples of your classifier. See + (:py:func:`bob.measure.precision_recall_curve`) - ``npoints`` : int - The number of points forwarded to :py:func:`bob.measure.precision_recall_curve` + npoints (:py:class:`int`, optional): The number of points for the plot. See + (:py:func:`bob.measure.precision_recall_curve`) - ``kwargs`` : keyword arguments - Extra plotting parameters, which are passed directly to :py:func:`matplotlib.pyplot.plot`. + kwargs (:py:class:`dict`, optional): Extra plotting parameters, which are + passed directly to :py:func:`matplotlib.pyplot.plot`. - **Returns:** - The return value is the ``matplotlib`` line that was added as defined by :py:func:`matplotlib.pyplot.plot`. + Returns: + + :py:class:`list` of :py:class:`matplotlib.lines.Line2D`: The lines that + were added as defined by the return value of + :py:func:`matplotlib.pyplot.plot`. + """ from matplotlib import pyplot @@ -164,11 +204,12 @@ def epc(dev_negatives, dev_positives, test_negatives, test_positives, This method will call ``matplotlib`` to plot the EPC curve for a system which contains a particular set of negatives (impostors) and positives (clients) for both the development and test sets. We use the standard - :py:func:`matplotlib.pyplot.plot` command. All parameters passed with exception of - the five first parameters of this method will be directly passed to the plot - command. + :py:func:`matplotlib.pyplot.plot` command. All parameters passed with + exception of the five first parameters of this method will be directly passed + to the plot command. - The plot will represent the minimum HTER on the vertical axis and the cost on the horizontal axis. + The plot will represent the minimum HTER on the vertical axis and the cost on + the horizontal axis. .. note:: @@ -176,26 +217,45 @@ def epc(dev_negatives, dev_positives, test_negatives, test_positives, issues the plotting commands. You are the responsible for setting up and saving the figure as you see fit. - **Parameters:** - ``dev_negatives, dev_positvies, test_negatives, test_positives`` : array_like(1D, float) - See :py:func:bob.measure.epc` for details + Parameters: + + dev_negatives (array): 1D float array that contains the scores of the + "negative" (noise, non-class) samples of your classifier, from the + development set. See (:py:func:`bob.measure.epc`) + + dev_positives (array): 1D float array that contains the scores of the + "positive" (signal, class) samples of your classifier, from the + development set. See (:py:func:`bob.measure.epc`) + + test_negatives (array): 1D float array that contains the scores of the + "negative" (noise, non-class) samples of your classifier, from the test + set. See (:py:func:`bob.measure.epc`) - ``npoints`` : int - See :py:func:bob.measure.epc` for details + test_positives (array): 1D float array that contains the scores of the + "positive" (signal, class) samples of your classifier, from the test set. + See (:py:func:`bob.measure.epc`) - ``kwargs`` : keyword arguments - Extra plotting parameters, which are passed directly to :py:func:`matplotlib.pyplot.plot`. + npoints (:py:class:`int`, optional): The number of points for the plot. See + (:py:func:`bob.measure.epc`) - **Returns:** + kwargs (:py:class:`dict`, optional): Extra plotting parameters, which are + passed directly to :py:func:`matplotlib.pyplot.plot`. + + + Returns: + + :py:class:`list` of :py:class:`matplotlib.lines.Line2D`: The lines that + were added as defined by the return value of + :py:func:`matplotlib.pyplot.plot`. - The return value is the ``matplotlib`` line that was added as defined by :py:func:`matplotlib.pyplot.plot`. """ from matplotlib import pyplot from . import epc as calc - out = calc(dev_negatives, dev_positives, test_negatives, test_positives, npoints) + out = calc(dev_negatives, dev_positives, test_negatives, test_positives, + npoints) return pyplot.plot(out[0,:], 100.0*out[1,:], **kwargs) @@ -207,11 +267,11 @@ def det(negatives, positives, npoints=100, axisfontsize='x-small', **kwargs): Conference on Speech Communication and Technology (pp. 1895-1898). Available: http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.117.4489&rep=rep1&type=pdf - This method will call ``matplotlib`` to plot the DET curve(s) for a system which - contains a particular set of negatives (impostors) and positives (clients) - scores. We use the standard :py:func:`matplotlib.pyplot.plot` command. All parameters - passed with exception of the three first parameters of this method will be - directly passed to the plot command. + This method will call ``matplotlib`` to plot the DET curve(s) for a system + which contains a particular set of negatives (impostors) and positives + (clients) scores. We use the standard :py:func:`matplotlib.pyplot.plot` + command. All parameters passed with exception of the three first parameters + of this method will be directly passed to the plot command. The plot will represent the false-alarm on the horizontal axis and the false-rejection on the vertical axis. @@ -231,8 +291,9 @@ def det(negatives, positives, npoints=100, axisfontsize='x-small', **kwargs): If you wish to reset axis zooming, you must use the Gaussian scale rather than the visual marks showed at the plot, which are just there for - displaying purposes. The real axis scale is based on :py:func:`bob.measure.ppndf`. - For example, if you wish to set the x and y axis to display data between 1% and 40% here is the recipe: + displaying purposes. The real axis scale is based on + :py:func:`bob.measure.ppndf`. For example, if you wish to set the x and y + axis to display data between 1% and 40% here is the recipe: .. code-block:: python @@ -243,7 +304,8 @@ def det(negatives, positives, npoints=100, axisfontsize='x-small', **kwargs): pyplot.axis([bob.measure.ppndf(k/100.0) for k in (1, 40, 1, 40)]) We provide a convenient way for you to do the above in this module. So, - optionally, you may use the :py:func:`bob.measure.plot.det_axis` method like this: + optionally, you may use the :py:func:`bob.measure.plot.det_axis` method + like this: .. code-block:: python @@ -252,23 +314,33 @@ def det(negatives, positives, npoints=100, axisfontsize='x-small', **kwargs): # please note we convert percentage values in det_axis() bob.measure.plot.det_axis([1, 40, 1, 40]) - **Parameters:** - ``negatives, positives`` : array_like(1D, float) - The list of negative and positive scores forwarded to :py:func:`bob.measure.det` + Parameters: + + negatives (array): 1D float array that contains the scores of the + "negative" (noise, non-class) samples of your classifier. See + (:py:func:`bob.measure.det`) + + positives (array): 1D float array that contains the scores of the + "positive" (signal, class) samples of your classifier. See + (:py:func:`bob.measure.det`) - ``npoints`` : int - The number of points forwarded to :py:func:`bob.measure.det` + npoints (:py:class:`int`, optional): The number of points for the plot. See + (:py:func:`bob.measure.det`) - ``axisfontsize`` : str - The size to be used by x/y-tick-labels to set the font size on the axis + axisfontsize (:py:class:`str`, optional): The size to be used by + x/y-tick-labels to set the font size on the axis - ``kwargs`` : keyword arguments - Extra plotting parameters, which are passed directly to :py:func:`matplotlib.pyplot.plot`. + kwargs (:py:class:`dict`, optional): Extra plotting parameters, which are + passed directly to :py:func:`matplotlib.pyplot.plot`. - **Returns:** - The return value is the ``matplotlib`` line that was added as defined by :py:func:`matplotlib.pyplot.plot`. + Returns: + + :py:class:`list` of :py:class:`matplotlib.lines.Line2D`: The lines that + were added as defined by the return value of + :py:func:`matplotlib.pyplot.plot`. + """ # these are some constants required in this method @@ -319,23 +391,26 @@ def det_axis(v, **kwargs): """Sets the axis in a DET plot. This method wraps the :py:func:`matplotlib.pyplot.axis` by calling - :py:func:`bob.measure.ppndf` on the values passed by the user so they are meaningful - in a DET plot as performed by :py:func:`bob.measure.plot.det`. + :py:func:`bob.measure.ppndf` on the values passed by the user so they are + meaningful in a DET plot as performed by :py:func:`bob.measure.plot.det`. + + + Parameters: + + v (``sequence``): A sequence (list, tuple, array or the like) containing + the X and Y limits in the order ``(xmin, xmax, ymin, ymax)``. Expected + values should be in percentage (between 0 and 100%). If ``v`` is not a + list or tuple that contains 4 numbers it is passed without further + inspection to :py:func:`matplotlib.pyplot.axis`. - **Parameters:** + kwargs (:py:class:`dict`, optional): Extra plotting parameters, which are + passed directly to :py:func:`matplotlib.pyplot.axis`. - ``v`` : (int, int, int, int) - The X and Y limits in the order ``(xmin, xmax, ymin, ymax)``. - Expected values should be in percentage (between 0 and 100%). - If ``v`` is not a list or tuple that contains 4 numbers it is passed - without further inspection to :py:func:`matplotlib.pyplot.axis`. - ``kwargs`` : keyword arguments - Extra parameters, which are passed directly to :py:func:`matplotlib.pyplot.axis`. + Returns: - **Returns:** + object: Whatever is returned by :py:func:`matplotlib.pyplot.axis`. - Returns whatever :py:func:`matplotlib.pyplot.axis` returns. """ import logging @@ -372,28 +447,36 @@ def det_axis(v, **kwargs): def cmc(cmc_scores, logx = True, **kwargs): - """Plots the (cumulative) match characteristics curve and returns the maximum rank. + """Plots the (cumulative) match characteristics and returns the maximum rank. - This function plots a CMC curve using the given CMC scores, which can be read from the our score files using the :py:func:`bob.measure.load.cmc_four_column` or :py:func:`bob.measure.load.cmc_five_column` methods. - The structure of the ``cmc_scores`` parameter is relatively complex. - It contains a list of pairs of lists. - For each probe object, a pair of list negative and positive scores is required. + This function plots a CMC curve using the given CMC scores, which can be read + from the our score files using the + :py:func:`bob.measure.load.cmc_four_column` or + :py:func:`bob.measure.load.cmc_five_column` methods. The structure of the + ``cmc_scores`` parameter is relatively complex. It contains a list of pairs + of lists. For each probe object, a pair of list negative and positive scores + is required. - **Parameters:** - ``cmc_scores`` : [(array_like(1D, float), array_like(1D, float))] - See :py:func:`bob.measure.cmc` + Parameters: - ``logx`` : bool - Plot the rank axis in logarithmic scale using :py:func:`matplotlib.pyplot.semilogx` or in linear scale using :py:func:`matplotlib.pyplot.plot`? (Default: ``True``) + cmc_scores (array): 1D float array containing the CMC values (See + :py:func:`bob.measure.cmc`) - ``kwargs`` : keyword arguments - Extra plotting parameters, which are passed directly to :py:func:`matplotlib.pyplot.plot` or :py:func:`matplotlib.pyplot.semilogx`. + logx (:py:class:`bool`, optional): If set (the default), plots the rank + axis in logarithmic scale using :py:func:`matplotlib.pyplot.semilogx` or + in linear scale using :py:func:`matplotlib.pyplot.plot` - **Returns:** + kwargs (:py:class:`dict`, optional): Extra plotting parameters, which are + passed directly to :py:func:`matplotlib.pyplot.plot`. + + + Returns: + + int: The number of classes (clients) in the given scores. - The number of classes (clients) in the given scores. """ + from matplotlib import pyplot from . import cmc as calc @@ -407,38 +490,49 @@ def cmc(cmc_scores, logx = True, **kwargs): return len(out) -def detection_identification_curve(cmc_scores, far_values = log_values(), rank = 1, logx = True, **kwargs): - """Plots the Detection & Identification curve over the FAR for the given FAR values. - This curve is designed to be used in an open set identification protocol, and defined in Chapter 14.1 of [LiJain2005]_. - It requires to have at least one open set probe item, i.e., with no corresponding gallery, such that the positives for that pair are ``None``. +def detection_identification_curve(cmc_scores, far_values = log_values(), rank + = 1, logx = True, **kwargs): + """Plots the Detection & Identification curve over the FAR + + This curve is designed to be used in an open set identification protocol, and + defined in Chapter 14.1 of [LiJain2005]_. It requires to have at least one + open set probe item, i.e., with no corresponding gallery, such that the + positives for that pair are ``None``. - The detection and identification curve first computes FAR thresholds based on the out-of-set probe scores (negative scores). - For each probe item, the **maximum** negative score is used. - Then, it plots the detection and identification rates for those thresholds, which are based on the in-set probe scores only. - See [LiJain2005]_ for more details. + The detection and identification curve first computes FAR thresholds based on + the out-of-set probe scores (negative scores). For each probe item, the + **maximum** negative score is used. Then, it plots the detection and + identification rates for those thresholds, which are based on the in-set + probe scores only. See [LiJain2005]_ for more details. - **Parameters:** + .. [LiJain2005] **Stan Li and Anil K. Jain**, *Handbook of Face Recognition*, Springer, 2005 - ``cmc_scores`` : [(array_like(1D, float), array_like(1D, float))] - See :py:func:`bob.measure.detection_identification_rate` - ``far_values`` : [float] - The values for the FAR, where the CAR should be plotted; each value should be in range [0,1]. + Parameters: - ``rank`` : int or ``None`` - The rank for which the curve should be plotted, 1 by default. + cmc_scores (array): 1D float array containing the CMC values (See + :py:func:`bob.measure.cmc`) - ``logx`` : bool - Plot the FAR axis in logarithmic scale using :py:func:`matplotlib.pyplot.semilogx` or in linear scale using :py:func:`matplotlib.pyplot.plot`? (Default: ``True``) + rank (:py:class:`int`, optional): The rank for which the curve should be + plotted - ``kwargs`` : keyword arguments - Extra plotting parameters, which are passed directly to :py:func:`matplotlib.pyplot.plot` or :py:func:`matplotlib.pyplot.semilogx`. + far_values (:py:class:`list`, optional): The values for the FAR, where the + CAR should be plotted; each value should be in range [0,1]. - **Returns:** + logx (:py:class:`bool`, optional): If set (the default), plots the rank + axis in logarithmic scale using :py:func:`matplotlib.pyplot.semilogx` or + in linear scale using :py:func:`matplotlib.pyplot.plot` - The return value is the ``matplotlib`` line that was added as defined by :py:func:`matplotlib.pyplot.plot`. + kwargs (:py:class:`dict`, optional): Extra plotting parameters, which are + passed directly to :py:func:`matplotlib.pyplot.plot`. + + + Returns: + + :py:class:`list` of :py:class:`matplotlib.lines.Line2D`: The lines that + were added as defined by the return value of + :py:func:`matplotlib.pyplot.plot`. - .. [LiJain2005] **Stan Li and Anil K. Jain**, *Handbook of Face Recognition*, Springer, 2005 """ import numpy diff --git a/bob/measure/script/apply_threshold.py b/bob/measure/script/apply_threshold.py index c88acaf0162db00682f6c0d6f36cb5a3c4485d3d..dbf03a2b3b41dc322d89c5e7527bbcaa85631e69 100644 --- a/bob/measure/script/apply_threshold.py +++ b/bob/measure/script/apply_threshold.py @@ -1,107 +1,81 @@ #!/usr/bin/env python # vim: set fileencoding=utf-8 : -# Andre Anjos -# Wed May 25 13:27:46 2011 +0200 -# -# Copyright (C) 2011-2013 Idiap Research Institute, Martigny, Switzerland +# Wed 28 Sep 2016 17:55:17 CEST + + +"""Applies a threshold to score file and reports error rates + +Usage: %(prog)s [-v...] [options] + %(prog)s --help + %(prog)s --version + + +Arguments: + The threshold value to apply (float) + Path to the file containing the scores where to apply the + threshold and calculate error rates + +Options: + -h, --help Shows this help message and exits + -V, --version Prints the version and exits + -v, --verbose Increases the output verbosity level -"""This script applies a threshold to score file and reports error rates -""" -__epilog__ = """ Examples: - 1. Standard usage + Applies the threshold of 0.5 to the scores file in scores.txt and reports: + + $ %(prog)s 0.5 scores.txt - $ %(prog)s --scores=my-scores.txt --threshold=0.5 """ + import os import sys -from .. import farfrr, load - -def apthres(neg, pos, thres): - """Prints a single output line that contains all info for the threshold""" - - far, frr = farfrr(neg, pos, thres) - hter = (far + frr)/2.0 - - ni = neg.shape[0] #number of impostors - fa = int(round(far*ni)) #number of false accepts - nc = pos.shape[0] #number of clients - fr = int(round(frr*nc)) #number of false rejects - - print("FAR : %.3f%% (%d/%d)" % (100*far, fa, ni)) - print("FRR : %.3f%% (%d/%d)" % (100*frr, fr, nc)) - print("HTER: %.3f%%" % (100*hter,)) - -def get_options(user_input): - """Parse the program options""" - - usage = 'usage: %s [arguments]' % os.path.basename(sys.argv[0]) - - import argparse - parser = argparse.ArgumentParser(usage=usage, - description=(__doc__ % {'prog': os.path.basename(sys.argv[0])}), - epilog=(__epilog__ % {'prog': os.path.basename(sys.argv[0])}), - formatter_class=argparse.RawDescriptionHelpFormatter) - - parser.add_argument('-s', '--scores', dest="ifile", default=None, - help="Name of the file containing the scores (defaults to %(default)s)", - metavar="FILE") - parser.add_argument('-t', '--threshold', dest='thres', default=None, - type=float, help="The threshold value to apply", metavar="FLOAT") - parser.add_argument('-p', '--parser', dest="parser", default="4column", - help="Name of a known parser or of a python-importable function that can parse your input files and return a tuple (negatives, positives) as blitz 1-D arrays of 64-bit floats. Consult the API of bob.measure.load.split_four_column() for details", metavar="NAME.FUNCTION") - - # This option is not normally shown to the user... - parser.add_argument("--self-test", - action="store_true", dest="test", default=False, help=argparse.SUPPRESS) - #help="if set, runs an internal verification test and erases any output") - - args = parser.parse_args(args=user_input) - - if args.test: - # then we go into test mode, all input is preset - args.thres = 0.0 - - if args.ifile is None: - parser.error("you should give an input score set with --scores") - - if args.thres is None: - parser.error("you should give a threshold value with --threshold") - - #parse the score-parser - if args.parser.lower() in ('4column', '4col'): - args.parser = load.split_four_column - elif args.parser.lower() in ('5column', '5col'): - args.parser = load.split_five_column - else: #try an import - if args.parser.find('.') == -1: - parser.error("parser module should be either '4column', '5column' or a valid python function identifier in the format 'module.function': '%s' is invalid" % args.parser) - - mod, fct = args.parser.rsplit('.', 2) - import imp - try: - fp, pathname, description = imp.find_module(mod, ['.'] + sys.path) - except Exception as e: - parser.error("import error for '%s': %s" % (args.parser, e)) - - try: - pmod = imp.load_module(mod, fp, pathname, description) - args.parser = getattr(pmod, fct) - except Exception as e: - parser.error("loading error for '%s': %s" % (args.parser, e)) - finally: - fp.close() - - return args + +import logging +__logging_format__='[%(levelname)s] %(message)s' +logging.basicConfig(format=__logging_format__) +logger = logging.getLogger('bob') + +from .eval_threshold import apthres + def main(user_input=None): - options = get_options(user_input) + if user_input is not None: + argv = user_input + else: + argv = sys.argv[1:] + + import docopt + import pkg_resources + + completions = dict( + prog=os.path.basename(sys.argv[0]), + version=pkg_resources.require('bob.measure')[0].version + ) + + args = docopt.docopt( + __doc__ % completions, + argv=argv, + version=completions['version'], + ) + + # Sets-up logging + if args['--verbose'] == 1: logging.getLogger().setLevel(logging.INFO) + elif args['--verbose'] >= 2: logging.getLogger().setLevel(logging.DEBUG) + + # handles threshold validation + try: + args[''] = float(args['']) + except: + raise docopt.DocoptExit("cannot convert %s into float for threshold" % \ + args['']) + + from ..load import load_score, get_negatives_positives + neg, pos = get_negatives_positives(load_score(args[''])) - neg, pos = options.parser(options.ifile) - apthres(neg, pos, options.thres) + apthres(neg, pos, args['']) return 0 diff --git a/bob/measure/script/compute_perf.py b/bob/measure/script/compute_perf.py index bd79f8fd9fc68a7aa9d93333f3642c86876d6ba0..a3e2b12a8a2931963a0f9166e23648f466e3b8ff 100644 --- a/bob/measure/script/compute_perf.py +++ b/bob/measure/script/compute_perf.py @@ -1,40 +1,64 @@ #!/usr/bin/env python # vim: set fileencoding=utf-8 : -# Andre Anjos -# Wed May 25 13:27:46 2011 +0200 -# -# Copyright (C) 2011-2013 Idiap Research Institute, Martigny, Switzerland +# Wed 28 Sep 2016 15:39:05 CEST + +"""Runs error analysis on score sets -"""This script runs error analysis on the development and test set scores, in a -four column format: 1. Computes the threshold using either EER or min. HTER criteria on develoment set scores; 2. Applies the above threshold on test set scores to compute the HTER 3. Plots ROC, EPC and DET curves to a multi-page PDF file -""" -__epilog__ = """ + +Usage: %(prog)s [-v...] [options] + %(prog)s --help + %(prog)s --version + + +Arguments: + Path to the file containing the development scores + Path to the file containing the test scores + + +Options: + -h, --help Shows this help message and exits + -V, --version Prints the version and exits + -v, --verbose Increases the output verbosity level + -n , --points= Number of points to use in the curves + [default: 100] + -o , --output= Name of the output file that will contain the + plots [default: curves.pdf] + -x, --no-plot If set, then I'll execute no plotting + + Examples: 1. Specify a different output filename - $ %(prog)s --output=mycurves.pdf --devel=dev.scores --test=test.scores + $ %(prog)s -vv --output=mycurves.pdf dev.scores test.scores 2. Specify a different number of points - $ %(prog)s --points=500 --devel=dev.scores --test=test.scores + $ %(prog)s --points=500 dev.scores test.scores 3. Don't plot (only calculate thresholds) - $ %(prog)s --no-plot --devel=dev.scores --test=test.scores + $ %(prog)s --no-plot dev.scores test.scores + """ import os import sys import numpy +import logging +__logging_format__='[%(levelname)s] %(message)s' +logging.basicConfig(format=__logging_format__) +logger = logging.getLogger('bob') + + def print_crit(dev_neg, dev_pos, test_neg, test_pos, crit): - """Prints a single output line that contains all info for a given criterium""" + """Prints a single output line that contains all info for a given criterion""" from .. import eer_threshold, min_hter_threshold, farfrr @@ -49,7 +73,7 @@ def print_crit(dev_neg, dev_pos, test_neg, test_pos, crit): test_far, test_frr = farfrr(test_neg, test_pos, thres) test_hter = (test_far + test_frr)/2.0 - print("[Min. criterium: %s] Threshold on Development set: %e" % (crit, thres)) + print("[Min. criterion: %s] Threshold on Development set: %e" % (crit, thres)) dev_ni = dev_neg.shape[0] #number of impostors dev_fa = int(round(dev_far*dev_ni)) #number of false accepts @@ -82,7 +106,8 @@ def print_crit(dev_neg, dev_pos, test_neg, test_pos, crit): print(" HTER | %s | %s" % (fmt(dev_hter_str, -1*dev_max_len), fmt(test_hter_str, -1*test_max_len))) -def plots(dev_neg, dev_pos, test_neg, test_pos, npoints, filename): + +def plots(dev_neg, dev_pos, test_neg, test_pos, points, filename): """Saves ROC, DET and EPC curves on the file pointed out by filename.""" from .. import plot @@ -96,9 +121,9 @@ def plots(dev_neg, dev_pos, test_neg, test_pos, npoints, filename): # ROC fig = mpl.figure() - plot.roc(dev_neg, dev_pos, npoints, color=(0.3,0.3,0.3), + plot.roc(dev_neg, dev_pos, points, color=(0.3,0.3,0.3), linestyle='--', dashes=(6,2), label='development') - plot.roc(test_neg, test_pos, npoints, color=(0,0,0), + plot.roc(test_neg, test_pos, points, color=(0,0,0), linestyle='-', label='test') mpl.axis([0,40,0,40]) mpl.title("ROC Curve") @@ -110,9 +135,9 @@ def plots(dev_neg, dev_pos, test_neg, test_pos, npoints, filename): # DET fig = mpl.figure() - plot.det(dev_neg, dev_pos, npoints, color=(0.3,0.3,0.3), + plot.det(dev_neg, dev_pos, points, color=(0.3,0.3,0.3), linestyle='--', dashes=(6,2), label='development') - plot.det(test_neg, test_pos, npoints, color=(0,0,0), + plot.det(test_neg, test_pos, points, color=(0,0,0), linestyle='-', label='test') plot.det_axis([0.01, 40, 0.01, 40]) mpl.title("DET Curve") @@ -124,7 +149,7 @@ def plots(dev_neg, dev_pos, test_neg, test_pos, npoints, filename): # EPC fig = mpl.figure() - plot.epc(dev_neg, dev_pos, test_neg, test_pos, npoints, + plot.epc(dev_neg, dev_pos, test_neg, test_pos, points, color=(0,0,0), linestyle='-') mpl.title('EPC Curve') mpl.xlabel('Cost') @@ -134,86 +159,53 @@ def plots(dev_neg, dev_pos, test_neg, test_pos, npoints, filename): pp.close() -def get_options(user_input): - """Parse the program options""" - - usage = 'usage: %s [arguments]' % os.path.basename(sys.argv[0]) - - import argparse - parser = argparse.ArgumentParser(usage=usage, - description=(__doc__ % {'prog': os.path.basename(sys.argv[0])}), - epilog=(__epilog__ % {'prog': os.path.basename(sys.argv[0])}), - formatter_class=argparse.RawDescriptionHelpFormatter) - - parser.add_argument('-d', '--devel', dest="dev", default=None, - help="Name of the file containing the development scores (defaults to %(default)s)", metavar="FILE") - parser.add_argument('-t', '--test', dest="test", default=None, - help="Name of the file containing the test scores (defaults to %(default)s)", metavar="FILE") - parser.add_argument('-n', '--points', dest="npoints", default=100, type=int, - help="Number of points to use in the curves (defaults to %(default)s)", - metavar="INT(>0)") - parser.add_argument('-o', '--output', dest="plotfile", default="curves.pdf", - help="Name of the output file that will contain the plots (defaults to %(default)s)", metavar="FILE") - parser.add_argument('-x', '--no-plot', dest="doplot", default=True, - action='store_false', help="If set, then I'll execute no plotting") - parser.add_argument('-p', '--parser', dest="parser", default="4column", - help="Name of a known parser or of a python-importable function that can parse your input files and return a tuple (negatives, positives) as blitz 1-D arrays of 64-bit floats. Consult the API of bob.measure.load.split_four_column() for details", metavar="NAME.FUNCTION") - - # This option is not normally shown to the user... - parser.add_argument("--self-test", - action="store_true", dest="selftest", default=False, - help=argparse.SUPPRESS) - #help="if set, runs an internal verification test and erases any output") - - args = parser.parse_args(args=user_input) - - if args.selftest: - # then we go into test mode, all input is preset - import tempfile - outputdir = tempfile.mkdtemp(prefix='bobtest_') - args.plotfile = os.path.join(outputdir, "curves.pdf") - - if args.dev is None: - parser.error("you should give a development score set with --devel") - - if args.test is None: - parser.error("you should give a test score set with --test") - - #parse the score-parser - from .. import load - - if args.parser.lower() in ('4column', '4col'): - args.parser = load.split_four_column - elif args.parser.lower() in ('5column', '5col'): - args.parser = load.split_five_column - else: #try an import - if args.parser.find('.') == -1: - parser.error("parser module should be either '4column', '5column' or a valid python function identifier in the format 'module.function': '%s' is invalid" % arg.parser) - - mod, fct = args.parser.rsplit('.', 1) - try: - args.parser = getattr(__import__(mod, fromlist=['*']), fct) - except Exception as e: - parser.error("error importing '%s': %s" % (args.parser, e)) - - return args def main(user_input=None): - options = get_options(user_input) + if user_input is not None: + argv = user_input + else: + argv = sys.argv[1:] + + import docopt + import pkg_resources + + completions = dict( + prog=os.path.basename(sys.argv[0]), + version=pkg_resources.require('bob.measure')[0].version + ) - dev_neg, dev_pos = options.parser(options.dev) - test_neg, test_pos = options.parser(options.test) + args = docopt.docopt( + __doc__ % completions, + argv=argv, + version=completions['version'], + ) + + # Sets-up logging + if args['--verbose'] == 1: logging.getLogger().setLevel(logging.INFO) + elif args['--verbose'] >= 2: logging.getLogger().setLevel(logging.DEBUG) + + # Checks number of points option + try: + args['--points'] = int(args['--points']) + except: + raise docopt.DocoptExit("cannot convert %s into int for points" % \ + args['--points']) + + if args['--points'] <= 0: + raise docopt.DocoptExit('Number of points (--points) should greater ' \ + 'than zero') + + from ..load import load_score, get_negatives_positives + dev_neg, dev_pos = get_negatives_positives(load_score(args[''])) + test_neg, test_pos = get_negatives_positives(load_score(args[''])) print_crit(dev_neg, dev_pos, test_neg, test_pos, 'EER') print_crit(dev_neg, dev_pos, test_neg, test_pos, 'Min. HTER') - if options.doplot: - plots(dev_neg, dev_pos, test_neg, test_pos, options.npoints, - options.plotfile) - print("[Plots] Performance curves => '%s'" % options.plotfile) - - if options.selftest: #remove output file + tmp directory - import shutil - shutil.rmtree(os.path.dirname(options.plotfile)) + + if not args['--no-plot']: + plots(dev_neg, dev_pos, test_neg, test_pos, args['--points'], + args['--output']) + print("[Plots] Performance curves => '%s'" % args['--output']) return 0 diff --git a/bob/measure/script/eval_threshold.py b/bob/measure/script/eval_threshold.py index 1c14f9cace07c2d2dfaf3c0ed87870eb0234b8c6..d2a6a5d548a5008ba1956a11a78153f6e2d20516 100644 --- a/bob/measure/script/eval_threshold.py +++ b/bob/measure/script/eval_threshold.py @@ -1,32 +1,58 @@ #!/usr/bin/env python # vim: set fileencoding=utf-8 : -# Andre Anjos -# Wed May 25 13:27:46 2011 +0200 -# -# Copyright (C) 2011-2013 Idiap Research Institute, Martigny, Switzerland +# Wed 28 Sep 2016 16:56:52 CEST + + +"""Computes the threshold following a minimization criteria on input scores + +Usage: %(prog)s [-v...] [options] + %(prog)s --help + %(prog)s --version + + +Arguments: + Path to the file containing the scores to be used for calculating + the threshold + + +Options: + -h, --help Shows this help message and exits + -V, --version Prints the version and exits + -v, --verbose Increases the output verbosity level + -c , --criterion= The minimization criterion to use (choose + between mhter, mwer or eer) [default: eer] + -w , --cost= The value w of the cost when minimizing using + the minimum weighter error rate (mwer) + criterion. This value is ignored for eer or + mhter criteria. [default: 0.5] -"""This script computes the threshold following a certain minimization criteria -on the given input data.""" -__epilog__ = """ Examples: 1. Specify a different criteria (only mhter, mwer or eer accepted): - $ %(prog)s --scores=dev.scores --criterium=mhter + $ %(prog)s --criterion=mhter scores.txt 2. Calculate the threshold that minimizes the weither HTER for a cost of 0.4: - $ %(prog)s --scores=dev.scores --criterium=mwer --cost=0.4 + $ %(prog)s --criterion=mwer --cost=0.4 scores.txt 3. Parse your input using a 5-column format - $ %(prog)s --scores=dev.scores --parser=5column + $ %(prog)s scores.txt + """ + import os import sys +import logging +__logging_format__='[%(levelname)s] %(message)s' +logging.basicConfig(format=__logging_format__) +logger = logging.getLogger('bob') + + def apthres(neg, pos, thres): """Prints a single output line that contains all info for the threshold""" @@ -44,87 +70,67 @@ def apthres(neg, pos, thres): print("FRR : %.3f%% (%d/%d)" % (100*frr, fr, nc)) print("HTER: %.3f%%" % (100*hter,)) + def calculate(neg, pos, crit, cost): """Returns the threshold given a certain criteria""" - from .. import eer_threshold, min_hter_threshold, min_weighted_error_rate_threshold - if crit == 'eer': + from .. import eer_threshold return eer_threshold(neg, pos) elif crit == 'mhter': + from .. import min_hter_threshold return min_hter_threshold(neg, pos) # defaults to the minimum of the weighter error rate + from .. import min_weighted_error_rate_threshold return min_weighted_error_rate_threshold(neg, pos, cost) -def get_options(user_input): - """Parse the program options""" - - usage = 'usage: %s [arguments]' % os.path.basename(sys.argv[0]) - - import argparse - parser = argparse.ArgumentParser(usage=usage, - description=(__doc__ % {'prog': os.path.basename(sys.argv[0])}), - epilog=(__epilog__ % {'prog': os.path.basename(sys.argv[0])}), - formatter_class=argparse.RawDescriptionHelpFormatter) - - parser.add_argument('-s', '--scores', dest="ifile", default=None, - help="Name of the file containing the scores (defaults to %(default)s)", - metavar="FILE") - parser.add_argument('-c', '--criterium', dest='crit', default='eer', - choices=('eer', 'mhter', 'mwer'), - help="The minimization criterium to use", metavar="CRITERIUM") - parser.add_argument('-w', '--cost', dest='cost', default=0.5, - type=float, help="The value w of the cost when minimizing using the minimum weighter error rate (mwer) criterium. This value is ignored for eer or mhter criteria.", metavar="FLOAT") - parser.add_argument('-p', '--parser', dest="parser", default="4column", - help="Name of a known parser or of a python-importable function that can parse your input files and return a tuple (negatives, positives) as blitz 1-D arrays of 64-bit floats. Consult the API of bob.measure.load.split_four_column() for details", metavar="NAME.FUNCTION") - - # This option is not normally shown to the user... - parser.add_argument("--self-test", - action="store_true", dest="test", default=False, help=argparse.SUPPRESS) - #help="if set, runs an internal verification test and erases any output") - - args = parser.parse_args(args=user_input) - - if args.ifile is None: - parser.error("you should give an input score set with --scores") - - if args.cost < 0.0 or args.cost > 1.0: - parser.error("cost should lie between 0.0 and 1.0") - - #parse the score-parser - from .. import load - if args.parser.lower() in ('4column', '4col'): - args.parser = load.split_four_column - elif args.parser.lower() in ('5column', '5col'): - args.parser = load.split_five_column - else: #try an import - if args.parser.find('.') == -1: - parser.error("parser module should be either '4column', '5column' or a valid python function identifier in the format 'module.function': '%s' is invalid" % args.parser) - - mod, fct = args.parser.rsplit('.', 2) - import imp - try: - fp, pathname, description = imp.find_module(mod, ['.'] + sys.path) - except Exception as e: - parser.error("import error for '%s': %s" % (args.parser, e)) - - try: - pmod = imp.load_module(mod, fp, pathname, description) - args.parser = getattr(pmod, fct) - except Exception as e: - parser.error("loading error for '%s': %s" % (args.parser, e)) - finally: - fp.close() - - return args def main(user_input=None): - options = get_options(user_input) - - neg, pos = options.parser(options.ifile) - t = calculate(neg, pos, options.crit, options.cost) + if user_input is not None: + argv = user_input + else: + argv = sys.argv[1:] + + import docopt + import pkg_resources + + completions = dict( + prog=os.path.basename(sys.argv[0]), + version=pkg_resources.require('bob.measure')[0].version + ) + + args = docopt.docopt( + __doc__ % completions, + argv=argv, + version=completions['version'], + ) + + # Sets-up logging + if args['--verbose'] == 1: logging.getLogger().setLevel(logging.INFO) + elif args['--verbose'] >= 2: logging.getLogger().setLevel(logging.DEBUG) + + # validates criterion + valid_criteria = ('eer', 'mhter', 'mwer') + if args['--criterion'] not in valid_criteria: + raise docopt.DocoptExit("--criterion must be one of %s" % \ + ', '.join(valid_criteria)) + + # handles cost validation + try: + args['--cost'] = float(args['--cost']) + except: + raise docopt.DocoptExit("cannot convert %s into float for cost" % \ + args['--cost']) + + if args['--cost'] < 0.0 or args['--cost'] > 1.0: + docopt.DocoptExit("cost should lie between 0.0 and 1.0") + + from ..load import load_score, get_negatives_positives + neg, pos = get_negatives_positives(load_score(args[''])) + + t = calculate(neg, pos, args['--criterion'], args['--cost']) print("Threshold:", t) apthres(neg, pos, t) diff --git a/bob/measure/script/plot_cmc.py b/bob/measure/script/plot_cmc.py index 71ca794c5aaf5577d06beb6493b379ed4e59f0bd..712fb9285f4b84aa2b704451013f25a203b952f1 100644 --- a/bob/measure/script/plot_cmc.py +++ b/bob/measure/script/plot_cmc.py @@ -1,81 +1,118 @@ #!/usr/bin/env python # vim: set fileencoding=utf-8 : -# Manuel Guenther -# Tue Jan 8 13:36:12 CET 2013 -# -# Copyright (C) 2011-2014 Idiap Research Institute, Martigny, Switzerland +# Wed 28 Sep 2016 21:24:46 CEST -from __future__ import print_function +"""Computes and plots a cumulative rank characteristics (CMC) curve -"""This script computes and plot a cumulative rank characteristics (CMC) curve -from a score file in four or five column format. +Usage: %(prog)s [-v...] [options] + %(prog)s --help + %(prog)s --version -Note: The score file has to contain the exact probe file names as the 3rd -(4column) or 4th (5column) column. -""" +Arguments: -import os -import sys + The score file in 4 or 5 column format to test -def parse_command_line(command_line_options): - """Parse the program options""" - usage = 'usage: %s [arguments]' % os.path.basename(sys.argv[0]) +Options: - import argparse - parser = argparse.ArgumentParser(usage=usage, description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter) + -h, --help Shows this help message and exits + -V, --version Prints the version and exits + -v, --verbose Increases the output verbosity level + -o , --output= Name of the output file that will contain the + plots [default: cmc.pdf] + -x, --no-plot If set, then I'll execute no plotting + -l, --log-x-scale If set, plots logarithmic rank axis + -r , --rank= Plot detection & identification rate curve for + the given rank instead of the CMC curve. - # This option is not normally shown to the user... - parser.add_argument('--self-test', action = 'store_true', help = argparse.SUPPRESS) - parser.add_argument('-s', '--score-file', required = True, help = 'The score file in 4 or 5 column format to test.') - parser.add_argument('-o', '--output-pdf-file', default = 'cmc.pdf', help = 'The PDF file to write.') - parser.add_argument('-l', '--log-x-scale', action='store_true', help = 'Plot logarithmic Rank axis.') - parser.add_argument('-r', '--rank', type=int, help = 'Plot Detection & Identification rate curve for the given rank instead of the CMC curve.') - parser.add_argument('-x', '--no-plot', action = 'store_true', help = 'Do not print a PDF file, but only report the results.') - parser.add_argument('-p', '--parser', default = '4column', choices = ('4column', '5column'), help = 'The type of the score file.') - - args = parser.parse_args(command_line_options) +""" - if args.self_test: - # then we go into test mode, all input is preset - import tempfile - temp_dir = tempfile.mkdtemp(prefix="bobtest_") - args.output_pdf_file = os.path.join(temp_dir, "cmc.pdf") - print("temporary using file", args.output_pdf_file) +from __future__ import print_function - return args +import os +import sys -def main(command_line_options = None): - """Computes and plots the CMC curve.""" - from .. import load, plot, recognition_rate +def main(user_input=None): + + if user_input is not None: + argv = user_input + else: + argv = sys.argv[1:] + + import docopt + import pkg_resources + + completions = dict( + prog=os.path.basename(sys.argv[0]), + version=pkg_resources.require('bob.measure')[0].version + ) + + args = docopt.docopt( + __doc__ % completions, + argv=argv, + version=completions['version'], + ) + + # Sets-up logging + if args['--verbose'] == 1: logging.getLogger().setLevel(logging.INFO) + elif args['--verbose'] >= 2: logging.getLogger().setLevel(logging.DEBUG) + + # Validates rank + if args['--rank'] is not None: + try: + args['--rank'] = int(args['--rank']) + except: + raise docopt.DocoptExit("cannot convert %s into int for rank" % \ + args['--rank']) + + if args['--rank'] <= 0: + raise docopt.DocoptExit('Rank (--rank) should greater than zero') + + from .. import load + + # Loads score file + f = load.open_file(args['']) + try: + line = f.readline() + ncolumns = len(line.split()) + except Exception: + logger.warn('Could not guess the number of columns in file: {}. ' + 'Assuming 4 column format.'.format(args[''])) + ncolumns = 4 + finally: + f.close() + + if ncolumns == 4: + data = load.cmc_four_column(args['']) + else: + data = load.cmc_five_column(args['']) - args = parse_command_line(command_line_options) + # compute recognition rate + from .. import recognition_rate + rr = recognition_rate(data, args['--rank']) + print("Recognition rate for score file %s is %3.2f%%" % (args[''], + rr * 100)) - # read data - if not os.path.isfile(args.score_file): raise IOError("The given score file does not exist") - # pythonic way: create inline dictionary "{...}", index with desired value "[...]", execute function "(...)" - data = {'4column' : load.cmc_four_column, '5column' : load.cmc_five_column}[args.parser](args.score_file) + if not args['--no-plot']: - # compute recognition rate - rr = recognition_rate(data, args.rank) - print("Recognition rate for score file", args.score_file, "is %3.2f%%" % (rr * 100)) + from .. import plot - if not args.no_plot: # compute CMC import matplotlib if not hasattr(matplotlib, 'backends'): matplotlib.use('pdf') import matplotlib.pyplot as mpl from matplotlib.backends.backend_pdf import PdfPages - pp = PdfPages(args.output_pdf_file) + pp = PdfPages(args['--output']) # CMC fig = mpl.figure() - if args.rank is None: - max_rank = plot.cmc(data, color=(0,0,1), linestyle='--', dashes=(6,2), logx = args.log_x_scale) + if args['--rank'] is None: + max_rank = plot.cmc(data, color=(0,0,1), linestyle='--', dashes=(6,2), + logx = args['--log-x-scale']) mpl.title("CMC Curve") - if args.log_x_scale: + if args['--log-x-scale']: mpl.xlabel('Rank (log)') else: mpl.xlabel('Rank') @@ -84,10 +121,13 @@ def main(command_line_options = None): ticks = [int(t) for t in mpl.xticks()[0]] mpl.xticks(ticks, ticks) mpl.xlim([1, max_rank]) + else: - plot.detection_identification_curve(data, rank = args.rank, color=(0,0,1), linestyle='--', dashes=(6,2), logx = args.log_x_scale) + plot.detection_identification_curve(data, rank = args['--rank'], + color=(0,0,1), linestyle='--', dashes=(6,2), + logx = args['--log-x-scale']) mpl.title("Detection \& Identification Curve") - if args.log_x_scale: + if args['--log-x-scale']: mpl.xlabel('False Acceptance Rate (log) in %') else: mpl.xlabel('False Acceptance Rate in %') @@ -104,11 +144,4 @@ def main(command_line_options = None): pp.savefig(fig) pp.close() - if args.self_test: #remove output file + tmp directory - import shutil - shutil.rmtree(os.path.dirname(args.output_pdf_file)) - return 0 - -if __name__ == '__main__': - main(sys.argv[1:]) diff --git a/bob/measure/script/recurse_results.py b/bob/measure/script/recurse_results.py old mode 100755 new mode 100644 index 4ff4065f108ae5dc44139da614ef282c72c1b821..0aa68c500a09aa8e252da528b547ea37aaa3a0c3 --- a/bob/measure/script/recurse_results.py +++ b/bob/measure/script/recurse_results.py @@ -1,12 +1,9 @@ #!/usr/bin/env python # vim: set fileencoding=utf-8 : -# Manuel Guenther # Tue Jul 2 14:52:49 CEST 2013 -# -# Copyright (C) 2011-2013 Idiap Research Institute, Martigny, Switzerland +"""Collects results of verification experiments recursively, reports results -""" This script parses through the given directory, collects all results of verification experiments that are stored in file with the given file name. It supports the split into development and test set of the data, as well as @@ -23,90 +20,62 @@ information are given in columns: The measure type of the development set can be changed to compute "HTER" or "FAR" thresholds instead, using the --criterion option. + +Usage: %(prog)s [-v...] [options] + %(prog)s --help + %(prog)s --version + + +Options: + -h, --help Shows this help message and exit + -V, --version Prints the version and exits + -v, --verbose Increases the output verbosity level + -d , --devel-name= Name of the file containing the development + scores [default: scores-dev] + -e , --eval-name= Name of the file containing the evaluation + scores [default: scores-eval] + -D , --directory= The directory where the results should be + collected from [default: .] + -n , --nonorm-dir= Directory where the unnormalized scores are + found [default: nonorm] + -z , --ztnorm-dir= Directory where the normalized scores are + found [default: ztnorm] + -s, --sort If set, sorts the results + -k , --sort-key= Sorts the results according to the given key. + May be one of "nonorm_dev", "nonorm_eval", + "ztnorm_dev", "ztnorm_eval" or "dir" + [default: dir] + -c , --criterion= Report Equal Rates (EER) rather than Half + Total Error Rate (HTER). Choose between + "HTER", "EER" or "FAR" [default: HTER] + -o , --output= If set, outputs results to a file named after + the option. If not set, writes to the console + """ import os import sys -#from apport.hookutils import default - -def get_args(): - """Parse the program options""" - - import argparse - from .. import load - - # set up command line parser - parser = argparse.ArgumentParser(description=__doc__, - formatter_class=argparse.ArgumentDefaultsHelpFormatter) - - parser.add_argument('-d', '--devel-name', dest="dev", default="scores-dev", metavar="FILE", - help = "Name of the file containing the development scores") - parser.add_argument('-e', '--eval-name', dest="eval", default="scores-eval", metavar="FILE", - help = "Name of the file containing the evaluation scores") - parser.add_argument('-D', '--directory', default=".", - help = "The directory where the results should be collected from.") - parser.add_argument('-n', '--nonorm-dir', dest="nonorm", default="nonorm", metavar = "DIR", - help = "Directory where the unnormalized scores are found") - parser.add_argument('-z', '--ztnorm-dir', dest="ztnorm", default = "ztnorm", metavar = "DIR", - help = "Directory where the normalized scores are found") - parser.add_argument('-s', '--sort', dest="sort", action='store_true', - help = "Sort the results") - parser.add_argument('-k', '--sort-key', dest='key', default = 'dir', choices=['nonorm_dev','nonorm_eval','ztnorm_dev','ztnorm_eval','dir'], - help = "Sort the results accordign to the given key") - parser.add_argument('-c', '--criterion', dest='criterion', default = 'HTER', choices=['HTER', 'EER', 'FAR'], - help = "Report Equal Rates (EER) rather than Half Total Error Rate (HTER)") - - parser.add_argument('-o', '--output', dest="output", - help="Name of the output file that will contain the HTER scores") - - parser.add_argument('--self-test', action='store_true', help=argparse.SUPPRESS) - - parser.add_argument('-p', '--parser', dest="parser", default="4column", metavar="NAME.FUNCTION", - help="Name of a known parser or of a python-importable function that can parse your input files and return a tuple (negatives, positives) as blitz 1-D arrays of 64-bit floats. Consult the API of bob.measure.load.split_four_column() for details") - - # parse arguments - args = parser.parse_args() - - # parse the score-parser - if args.parser.lower() in ('4column', '4col'): - args.parser = load.split_four_column - elif args.parser.lower() in ('5column', '5col'): - args.parser = load.split_five_column - else: #try an import - if args.parser.find('.') == -1: - parser.error("parser module should be either '4column', '5column' or a valid python function identifier in the format 'module.function': '%s' is invalid" % args.parser) - - mod, fct = args.parser.rsplit('.', 2) - import imp - try: - fp, pathname, description = imp.find_module(mod, ['.'] + sys.path) - except Exception as e: - parser.error("import error for '%s': %s" % (args.parser, e)) - - try: - pmod = imp.load_module(mod, fp, pathname, description) - args.parser = getattr(pmod, fct) - except Exception as e: - parser.error("loading error for '%s': %s" % (args.parser, e)) - finally: - fp.close() - - return args + +import logging +__logging_format__='[%(levelname)s] %(message)s' +logging.basicConfig(format=__logging_format__) +logger = logging.getLogger('bob') class Result: def __init__(self, dir, args): self.dir = dir - self.m_args = args + self.args = args self.nonorm_dev = None self.nonorm_eval = None self.ztnorm_dev = None self.ztnorm_eval = None def __calculate__(self, dev_file, eval_file = None): + from ..load import load_score, get_negatives_positives from .. import eer_threshold, min_hter_threshold, far_threshold, farfrr - dev_neg, dev_pos = self.m_args.parser(dev_file) + dev_neg, dev_pos = get_negatives_positives(load_score(dev_file)) # switch which threshold function to use; # THIS f***ing piece of code really is what python authors propose: @@ -114,29 +83,31 @@ class Result: 'EER' : eer_threshold, 'HTER' : min_hter_threshold, 'FAR' : far_threshold - } [self.m_args.criterion](dev_neg, dev_pos) + } [self.args['--criterion']](dev_neg, dev_pos) # compute far and frr for the given threshold dev_far, dev_frr = farfrr(dev_neg, dev_pos, threshold) dev_hter = (dev_far + dev_frr)/2.0 if eval_file: - eval_neg, eval_pos = self.m_args.parser(eval_file) + eval_neg, eval_pos = get_negatives_positives(load_score(eval_file)) eval_far, eval_frr = farfrr(eval_neg, eval_pos, threshold) eval_hter = (eval_far + eval_frr)/2.0 else: eval_hter = None - if self.m_args.criterion == 'FAR': + if self.args['--criterion'] == 'FAR': return (dev_frr, eval_frr) else: return (dev_hter, eval_hter) def nonorm(self, dev_file, eval_file = None): - (self.nonorm_dev, self.nonorm_eval) = self.__calculate__(dev_file, eval_file) + (self.nonorm_dev, self.nonorm_eval) = \ + self.__calculate__(dev_file, eval_file) def ztnorm(self, dev_file, eval_file = None): - (self.ztnorm_dev, self.ztnorm_eval) = self.__calculate__(dev_file, eval_file) + (self.ztnorm_dev, self.ztnorm_eval) = \ + self.__calculate__(dev_file, eval_file) def __str__(self): str = "" @@ -153,12 +124,14 @@ class Result: results = [] + def add_results(args, nonorm, ztnorm = None): + r = Result(os.path.dirname(nonorm).replace(os.getcwd()+"/", ""), args) print("Adding results from directory", r.dir) # check if the results files are there - dev_file = os.path.join(nonorm, args.dev) - eval_file = os.path.join(nonorm, args.eval) + dev_file = os.path.join(nonorm, args['--devel-name']) + eval_file = os.path.join(nonorm, args['--eval-name']) if os.path.isfile(dev_file): if os.path.isfile(eval_file): r.nonorm(dev_file, eval_file) @@ -166,8 +139,8 @@ def add_results(args, nonorm, ztnorm = None): r.nonorm(dev_file) if ztnorm: - dev_file = os.path.join(ztnorm, args.dev) - eval_file = os.path.join(ztnorm, args.eval) + dev_file = os.path.join(ztnorm, args['--devel-name']) + eval_file = os.path.join(ztnorm, args['--eval-name']) if os.path.isfile(dev_file): if os.path.isfile(eval_file): r.ztnorm(dev_file, eval_file) @@ -176,15 +149,17 @@ def add_results(args, nonorm, ztnorm = None): results.append(r) + def recurse(args, path): dir_list = os.listdir(path) # check if the score directories are included in the current path - if args.nonorm in dir_list: - if args.ztnorm in dir_list: - add_results(args, os.path.join(path, args.nonorm), os.path.join(path, args.ztnorm)) + if args['--nonorm-dir'] in dir_list: + if args['--ztnorm-dir'] in dir_list: + add_results(args, os.path.join(path, args['--nonorm-dir']), + os.path.join(path, args['--ztnorm-dir'])) else: - add_results(args, os.path.join(path, args.nonorm)) + add_results(args, os.path.join(path, args['--nonorm-dir'])) for e in dir_list: real_path = os.path.join(path, e) @@ -199,19 +174,52 @@ def table(): A += str(r) + "\n" return A -def main(): - args = get_args() - recurse(args, args.directory) +def main(user_input=None): + + if user_input is not None: + argv = user_input + else: + argv = sys.argv[1:] + + import docopt + import pkg_resources + + completions = dict( + prog=os.path.basename(sys.argv[0]), + version=pkg_resources.require('bob.measure')[0].version + ) + + args = docopt.docopt( + __doc__ % completions, + argv=argv, + version=completions['version'], + ) + + # Sets-up logging + if args['--verbose'] == 1: logging.getLogger().setLevel(logging.INFO) + elif args['--verbose'] >= 2: logging.getLogger().setLevel(logging.DEBUG) + + # checks sort-key + valid_sort_keys = 'nonorm_dev nonorm_eval ztnorm_dev ztnorm_eval dir'.split() + if args['--sort-key'] not in valid_sort_keys: + raise docopt.DocoptExit('--sort-key must be one of %s' % \ + ', '.join(valid_sort_keys)) + + # checks criterion + valid_criterion = 'HTER EER FAR'.split() + if args['--criterion'] not in valid_criterion: + raise docopt.DocoptExit('--criterion must be one of %s' % \ + ', '.join(valid_criterion)) + + recurse(args, args['--directory']) - if args.sort: + if args['--sort']: import operator - results.sort(key=operator.attrgetter(args.key)) + results.sort(key=operator.attrgetter(args['--sort-key'])) - if args.self_test: - _ = table() - elif args.output: - f = open(args.output, "w") + if args['--output']: + f = open(args['--output'], "w") f.writelines(table()) f.close() else: diff --git a/bob/measure/test_scripts.py b/bob/measure/test_scripts.py index 1408316406e33f42a7decce7975aa11d4f00d8e8..2a6d240ce2db39e93c0875df00cd18ccfad1fad5 100644 --- a/bob/measure/test_scripts.py +++ b/bob/measure/test_scripts.py @@ -1,17 +1,17 @@ #!/usr/bin/env python # vim: set fileencoding=utf-8 : -# Andre Anjos # Tue 21 Aug 2012 12:14:43 CEST -# -# Copyright (C) 2011-2013 Idiap Research Institute, Martigny, Switzerland """Script tests for bob.measure """ import os +import tempfile + import nose.tools import pkg_resources + def F(f): """Returns the test file on the "data" subdirectory""" return pkg_resources.resource_filename(__name__, os.path.join('data', f)) @@ -27,33 +27,49 @@ SCORES_5COL_CMC = F('scores-cmc-5col.txt') SCORES_4COL_CMC_OS = F('scores-cmc-4col-open-set.txt') + def test_compute_perf(): # sanity checks assert os.path.exists(DEV_SCORES) assert os.path.exists(TEST_SCORES) + tmp_output = tempfile.NamedTemporaryFile(prefix=__name__, suffix='.pdf') + + cmdline = [ + DEV_SCORES, + TEST_SCORES, + '--output=' + tmp_output.name, + ] + from .script.compute_perf import main - cmdline = '--devel=%s --test=%s --self-test' % (DEV_SCORES, TEST_SCORES) - nose.tools.eq_(main(cmdline.split()), 0) + nose.tools.eq_(main(cmdline), 0) + def test_eval_threshold(): # sanity checks assert os.path.exists(DEV_SCORES) + cmdline = [DEV_SCORES] + from .script.eval_threshold import main - cmdline = '--scores=%s --self-test' % (DEV_SCORES,) - nose.tools.eq_(main(cmdline.split()), 0) + nose.tools.eq_(main(cmdline), 0) + def test_apply_threshold(): # sanity checks assert os.path.exists(TEST_SCORES) + cmdline = [ + '0.5', + TEST_SCORES, + ] + from .script.apply_threshold import main - cmdline = '--scores=%s --self-test' % (TEST_SCORES,) - nose.tools.eq_(main(cmdline.split()), 0) + nose.tools.eq_(main(cmdline), 0) + def test_compute_perf_5col(): @@ -61,9 +77,17 @@ def test_compute_perf_5col(): assert os.path.exists(DEV_SCORES_5COL) assert os.path.exists(TEST_SCORES_5COL) + tmp_output = tempfile.NamedTemporaryFile(prefix=__name__, suffix='.pdf') + + cmdline = [ + DEV_SCORES_5COL, + TEST_SCORES_5COL, + '--output=' + tmp_output.name, + ] + from .script.compute_perf import main - cmdline = '--devel=%s --test=%s --parser=bob.measure.load.split_five_column --self-test' % (DEV_SCORES_5COL, TEST_SCORES_5COL) - nose.tools.eq_(main(cmdline.split()), 0) + nose.tools.eq_(main(cmdline), 0) + def test_compute_cmc(): @@ -73,6 +97,26 @@ def test_compute_cmc(): assert os.path.exists(SCORES_4COL_CMC_OS) from .script.plot_cmc import main - nose.tools.eq_(main(['--self-test', '--score-file', SCORES_4COL_CMC, '--log-x-scale']), 0) - nose.tools.eq_(main(['--self-test', '--score-file', SCORES_5COL_CMC, '--parser', '5column']), 0) - nose.tools.eq_(main(['--self-test', '--score-file', SCORES_4COL_CMC_OS, '--rank', '1']), 0) + + tmp_output = tempfile.NamedTemporaryFile(prefix=__name__, suffix='.pdf') + + nose.tools.eq_(main([ + SCORES_4COL_CMC, + '--log-x-scale', + '--output=%s' % tmp_output.name, + ]), 0) + + tmp_output = tempfile.NamedTemporaryFile(prefix=__name__, suffix='.pdf') + + nose.tools.eq_(main([ + SCORES_5COL_CMC, + '--output=%s' % tmp_output.name, + ]), 0) + + tmp_output = tempfile.NamedTemporaryFile(prefix=__name__, suffix='.pdf') + + nose.tools.eq_(main([ + SCORES_4COL_CMC_OS, + '--rank=1', + '--output=%s' % tmp_output.name, + ]), 0) diff --git a/doc/conf.py b/doc/conf.py index f940ef0ce8002229e330610695300222682e1048..592d54edf05d02ec4da3a771764f3ffe157d238d 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -25,11 +25,13 @@ extensions = [ 'sphinx.ext.intersphinx', 'sphinx.ext.napoleon', 'sphinx.ext.viewcode', + 'matplotlib.sphinxext.plot_directive', ] import sphinx if sphinx.__version__ >= "1.4.1": extensions.append('sphinx.ext.imgmath') + imgmath_image_format = 'svg' else: extensions.append('sphinx.ext.pngmath') diff --git a/doc/guide.rst b/doc/guide.rst index bc88fca2924a41d07f2c596023def48d13e42fac..e99c90abf9357207e1bf3d0de1fe4320627eefab 100644 --- a/doc/guide.rst +++ b/doc/guide.rst @@ -45,6 +45,7 @@ formula: HTER(\tau, \mathcal{D}) = \frac{FAR(\tau, \mathcal{D}) + FRR(\tau, \mathcal{D})}{2} \quad \textrm{[\%]} + where :math:`\mathcal{D}` denotes the dataset used. Since both the FAR and the FRR depends on the threshold :math:`\tau`, they are strongly related to each other: increasing the FAR will reduce the FRR and vice-versa. For this reason, @@ -70,8 +71,9 @@ optimal threshold :math:`\tau^*` is then computed using different values of .. math:: \tau^{*} = \arg\!\min_{\tau} \quad \beta \cdot \textrm{FAR}(\tau, \mathcal{D}_{d}) + (1-\beta) \cdot \textrm{FRR}(\tau, \mathcal{D}_{d}) + where :math:`\mathcal{D}_{d}` denotes the development set and should be -completely separate to the evaluation set `\mathcal{D}`. +completely separate to the evaluation set :math:`\mathcal{D}`. Performance for different values of :math:`\beta` is then computed on the test set :math:`\mathcal{D}_{t}` using the previously derived threshold. Note that @@ -213,8 +215,8 @@ Plotting -------- An image is worth 1000 words, they say. You can combine the capabilities of -`Matplotlib`_ with |project| to plot a number of curves. However, you must have that -package installed though. In this section we describe a few recipes. +`Matplotlib`_ with |project| to plot a number of curves. However, you must have +that package installed though. In this section we describe a few recipes. ROC === @@ -253,18 +255,18 @@ You should see an image like the following one: pyplot.title('ROC') As can be observed, plotting methods live in the namespace -:py:mod:`bob.measure.plot`. They work like the :py:func:`matplotlib.pyplot.plot` -itself, except that instead of receiving the x and y point coordinates as -parameters, they receive the two :py:class:`numpy.ndarray` arrays with -negatives and positives, as well as an indication of the number of points the -curve must contain. - -As in the :py:func:`matplotlib.pyplot.plot` command, you can pass optional parameters for -the line as shown in the example to setup its color, shape and even the label. -For an overview of the keywords accepted, please refer to the `Matplotlib`_'s -Documentation. Other plot properties such as the plot title, axis labels, -grids, legends should be controlled directly using the relevant `Matplotlib`_'s -controls. +:py:mod:`bob.measure.plot`. They work like the +:py:func:`matplotlib.pyplot.plot` itself, except that instead of receiving the +x and y point coordinates as parameters, they receive the two +:py:class:`numpy.ndarray` arrays with negatives and positives, as well as an +indication of the number of points the curve must contain. + +As in the :py:func:`matplotlib.pyplot.plot` command, you can pass optional +parameters for the line as shown in the example to setup its color, shape and +even the label. For an overview of the keywords accepted, please refer to the +`Matplotlib`_'s Documentation. Other plot properties such as the plot title, +axis labels, grids, legends should be controlled directly using the relevant +`Matplotlib`_'s controls. DET === @@ -356,9 +358,11 @@ This will produce an image like the following one: CMC === -The Cumulative Match Characteristics (CMC) curve estimates the probability that the correct model is in the *N* models with the highest similarity to a given probe. -A CMC curve can be plotted using the :py:func:`bob.measure.plot.cmc` function. -The CMC can be calculated from a relatively complex data structure, which defines a pair of positive and negative scores **per probe**: +The Cumulative Match Characteristics (CMC) curve estimates the probability that +the correct model is in the *N* models with the highest similarity to a given +probe. A CMC curve can be plotted using the :py:func:`bob.measure.plot.cmc` +function. The CMC can be calculated from a relatively complex data structure, +which defines a pair of positive and negative scores **per probe**: .. plot:: @@ -373,6 +377,7 @@ The CMC can be calculated from a relatively complex data structure, which define negatives = numpy.random.normal(0, 1, 19) cmc_scores.append((negatives, positives)) bob.measure.plot.cmc(cmc_scores, logx=False) + pyplot.grid(True) pyplot.title('CMC') pyplot.xlabel('Rank') pyplot.xticks([1,5,10,20]) @@ -383,15 +388,23 @@ The CMC can be calculated from a relatively complex data structure, which define Usually, there is only a single positive score per probe, but this is not a fixed restriction. .. note:: - The complex data structure can be read from our default 4 or 5 column score files using the :py:func:`bob.measure.load.cmc_four_column` or :py:func:`bob.measure.load.cmc_five_column` function. + + The complex data structure can be read from our default 4 or 5 column score + files using the :py:func:`bob.measure.load.cmc_four_column` or + :py:func:`bob.measure.load.cmc_five_column` function. Detection & Identification Curve ================================ -The detection & identification curve is designed to evaluate open set identification tasks. -It can be plotted using the :py:func:`bob.measure.plot.detection_identification_curve` function, but it requires at least one open-set probe, i.e., where no corresponding positive score exists, for which the FAR values are computed. -Here, we plot the detection and identification curve for rank 1, so that the recognition rate for FAR=1 will be identical to the rank one :py:func:`bob.measure.recognition_rate` obtained in the CMC plot above. +The detection & identification curve is designed to evaluate open set +identification tasks. It can be plotted using the +:py:func:`bob.measure.plot.detection_identification_curve` function, but it +requires at least one open-set probe, i.e., where no corresponding positive +score exists, for which the FAR values are computed. Here, we plot the +detection and identification curve for rank 1, so that the recognition rate for +FAR=1 will be identical to the rank one :py:func:`bob.measure.recognition_rate` +obtained in the CMC plot above. .. plot:: @@ -423,9 +436,10 @@ The methods inside :py:mod:`bob.measure.plot` are only provided as a `Matplotlib`_ wrapper to equivalent methods in :py:mod:`bob.measure` that can only calculate the points without doing any plotting. You may prefer to tweak the plotting or even use a different plotting system such as gnuplot. Have a -look at the implementations at :py:mod:`bob.measure.plot` to understand how -to use the |project| methods to compute the curves and interlace that in the -way that best suits you. +look at the implementations at :py:mod:`bob.measure.plot` to understand how to +use the |project| methods to compute the curves and interlace that in the way +that best suits you. + Full applications ----------------- @@ -468,13 +482,13 @@ evaluation and plotting of development and test set data using our combined .. code-block:: sh $ bob_compute_perf.py --devel=development-scores-4col.txt --test=test-scores-4col.txt - [Min. criterium: EER] Threshold on Development set: -4.787956e-03 + [Min. criterion: EER] Threshold on Development set: -4.787956e-03 | Development | Test -------+-----------------+------------------ FAR | 6.731% (35/520) | 2.500% (13/520) FRR | 6.667% (26/390) | 6.154% (24/390) HTER | 6.699% | 4.327% - [Min. criterium: Min. HTER] Threshold on Development set: 3.411070e-03 + [Min. criterion: Min. HTER] Threshold on Development set: 3.411070e-03 | Development | Test -------+-----------------+------------------ FAR | 4.231% (22/520) | 1.923% (10/520) diff --git a/doc/py_api.rst b/doc/py_api.rst index 25a369d2bc48599762ac04a7eb149fd5d09867e0..55875a92b94c8a24bf4aec29a617777b37265529 100644 --- a/doc/py_api.rst +++ b/doc/py_api.rst @@ -1,13 +1,12 @@ .. vim: set fileencoding=utf-8 : -.. Andre Anjos .. Sat 16 Nov 20:52:58 2013 ============ Python API ============ -This section includes information for using the Python API of :py:mod:`bob.measure`. - +This section includes information for using the Python API of +:py:mod:`bob.measure`. Measurement diff --git a/requirements.txt b/requirements.txt index e615c4484debdbc4fe13fb42488ff5b8d300030c..ad7def2dff2cf21ed949a115e628f5c225c67943 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ bob.core>2.0.4 bob.math bob.io.base matplotlib +docopt