__init__.py 16 KB
Newer Older
1 2
# import Libraries of other lib packages
import bob.math
3
import bob.io.base
4

André Anjos's avatar
André Anjos committed
5
from ._library import *
André Anjos's avatar
André Anjos committed
6 7
from . import version
from .version import module as __version__
André Anjos's avatar
André Anjos committed
8 9 10

from . import plot
from . import calibration
11
from . import load
André Anjos's avatar
André Anjos committed
12 13
import numpy

14 15 16 17 18 19

def fprfnr(negatives, positives, threshold):
  """Alias for :py:func:`bob.measure.farfrr`"""
  return farfrr(negatives, positives, threshold)


André Anjos's avatar
André Anjos committed
20
def mse (estimation, target):
André Anjos's avatar
André Anjos committed
21
  """Mean square error between a set of outputs and target values
22

André Anjos's avatar
André Anjos committed
23
  Uses the formula:
André Anjos's avatar
André Anjos committed
24 25 26 27 28 29 30 31 32

  .. math::

    MSE(\hat{\Theta}) = E[(\hat{\Theta} - \Theta)^2]

  Estimation (:math:`\hat{\Theta}`) and target (:math:`\Theta`) are supposed to
  have 2 dimensions. Different examples are organized as rows while different
  features in the estimated values or targets are organized as different
  columns.
André Anjos's avatar
André Anjos committed
33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48


  Parameters:

    estimation (array): an N-dimensional array that corresponds to the value
      estimated by your procedure

    target (array): an N-dimensional array that corresponds to the expected
      value


  Returns:

    float: The average of the squared error between the estimated value and the
    target

André Anjos's avatar
André Anjos committed
49 50 51
  """
  return numpy.mean((estimation - target)**2, 0)

André Anjos's avatar
André Anjos committed
52

André Anjos's avatar
André Anjos committed
53
def rmse (estimation, target):
André Anjos's avatar
André Anjos committed
54
  """Calculates the root mean square error between a set of outputs and target
55

André Anjos's avatar
André Anjos committed
56
  Uses the formula:
André Anjos's avatar
André Anjos committed
57 58 59 60 61 62 63 64 65

  .. math::

    RMSE(\hat{\Theta}) = \sqrt(E[(\hat{\Theta} - \Theta)^2])

  Estimation (:math:`\hat{\Theta}`) and target (:math:`\Theta`) are supposed to
  have 2 dimensions. Different examples are organized as rows while different
  features in the estimated values or targets are organized as different
  columns.
André Anjos's avatar
André Anjos committed
66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81


  Parameters:

    estimation (array): an N-dimensional array that corresponds to the value
      estimated by your procedure

    target (array): an N-dimensional array that corresponds to the expected
      value


  Returns:

    float: The square-root of the average of the squared error between the
    estimated value and the target

André Anjos's avatar
André Anjos committed
82 83 84
  """
  return numpy.sqrt(mse(estimation, target))

André Anjos's avatar
André Anjos committed
85

André Anjos's avatar
André Anjos committed
86
def relevance (input, machine):
André Anjos's avatar
André Anjos committed
87
  """Calculates the relevance of every input feature to the estimation process
88

André Anjos's avatar
André Anjos committed
89
  Uses the formula:
André Anjos's avatar
André Anjos committed
90

91 92 93
    Neural Triggering System Operating on High Resolution Calorimetry
    Information, Anjos et al, April 2006, Nuclear Instruments and Methods in
    Physics Research, volume 559, pages 134-138
André Anjos's avatar
André Anjos committed
94 95 96 97 98 99 100 101 102 103

  .. math::

    R(x_{i}) = |E[(o(x) - o(x|x_{i}=E[x_{i}]))^2]|

  In other words, the relevance of a certain input feature **i** is the change
  on the machine output value when such feature is replaced by its mean for all
  input vectors. For this to work, the `input` parameter has to be a 2D array
  with features arranged column-wise while different examples are arranged
  row-wise.
André Anjos's avatar
André Anjos committed
104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119


  Parameters:

    input (array): an N-dimensional array that corresponds to the value
      estimated by your model

    machine (object): A machine that can be called to "process" your input


  Returns:

    array: An 1D float array as large as the number of columns (second
    dimension) of your input array, estimating the "relevance" of each input
    column (or feature) to the score provided by the machine.

André Anjos's avatar
André Anjos committed
120 121 122 123 124 125 126 127 128 129 130 131 132
  """

  o = machine(input)
  i2 = input.copy()
  retval = numpy.ndarray((input.shape[1],), 'float64')
  retval.fill(0)
  for k in range(input.shape[1]):
    i2[:,:] = input #reset
    i2[:,k] = numpy.mean(input[:,k])
    retval[k] = (mse(machine(i2), o).sum())**0.5

  return retval

133

134
def recognition_rate(cmc_scores, threshold = None, rank = 1):
André Anjos's avatar
André Anjos committed
135
  """Calculates the recognition rate from the given input
136

André Anjos's avatar
André Anjos committed
137
  It is identical to the CMC value for the given ``rank``.
138

André Anjos's avatar
André Anjos committed
139 140 141
  The input has a specific format, which is a list of two-element tuples.  Each
  of the tuples contains the negative :math:`\\{S_p^-\\}` and the positive
  :math:`\\{S_p^+\\}` scores for one probe item :math:`p`, or ``None`` in case
142
  of open set recognition.
143

André Anjos's avatar
André Anjos committed
144 145 146 147
  If ``threshold`` is set to ``None``, the rank 1 recognition rate is defined
  as the number of test items, for which the highest positive
  :math:`\\max\\{S_p^+\\}` score is greater than or equal to all negative
  scores, divided by the number of all probe items :math:`P`:
148 149

  .. math::
150

151
    \\mathrm{RR} = \\frac{1}{P} \\sum_{p=1}^{P} \\begin{cases} 1 & \\mathrm{if } \\max\\{S_p^+\\} >= \\max\\{S_p^-\\}\\\\ 0 & \\mathrm{otherwise} \\end{cases}
152

André Anjos's avatar
André Anjos committed
153 154 155
  For a given rank :math:`r>1`, up to :math:`r` negative scores that are higher
  than the highest positive score are allowed to still count as correctly
  classified in the top :math:`r` rank.
156

André Anjos's avatar
André Anjos committed
157 158 159 160
  If ``threshold`` :math:`\\theta` is given, **all** scores below threshold
  will be filtered out.  Hence, if all positive scores are below threshold
  :math:`\\max\\{S_p^+\\} < \\theta`, the probe will be misclassified **at any
  rank**.
161

André Anjos's avatar
André Anjos committed
162 163 164 165 166 167
  For open set recognition, i.e., when there exist a tuple including negative
  scores without corresponding positive scores (``None``), and **all** negative
  scores are below ``threshold`` :math:`\\max\\{S_p^+\\} < \\theta`, the probe
  item is correctly rejected, **and it does not count into the denominator**
  :math:`P`.  When no ``threshold`` is provided, the open set probes will
  **always** count as misclassified, regardless of the ``rank``.
168

169 170
  .. warn:
     For open set tests, this rate does not correspond to a standard rate.
André Anjos's avatar
André Anjos committed
171 172 173 174 175
     Please use :py:func:`detection_identification_rate` and
     :py:func:`false_alarm_rate` instead.


  Parameters:
176

177
    cmc_scores (:py:class:`list`): A list in the format ``[(negatives,
178
      positives), ...]`` containing the CMC scores (i.e. :py:class:`list`:
179 180
      A list of tuples, where each tuple contains the
      ``negative`` and ``positive`` scores for one probe of the database).
181

André Anjos's avatar
André Anjos committed
182 183 184
      Each pair contains the ``negative`` and the ``positive`` scores for **one
      probe item**.  Each pair can contain up to one empty array (or ``None``),
      i.e., in case of open set recognition.
185

186
    threshold (:obj:`float`, optional): Decision threshold. If not ``None``, **all**
André Anjos's avatar
André Anjos committed
187 188 189 190
      scores will be filtered by the threshold. In an open set recognition
      problem, all open set scores (negatives with no corresponding positive)
      for which all scores are below threshold, will be counted as correctly
      rejected and **removed** from the probe list (i.e., the denominator).
191

192
    rank (:obj:`int`, optional):
André Anjos's avatar
André Anjos committed
193
      The rank for which the recognition rate should be computed, 1 by default.
194

195

André Anjos's avatar
André Anjos committed
196 197 198 199 200
  Returns:

    float: The (open set) recognition rate for the given rank, a value between
    0 and 1.

André Anjos's avatar
André Anjos committed
201
  """
202 203
  # If no scores are given, the recognition rate is exactly 0.
  if not cmc_scores:
204
    return 0.
205

206
  correct = 0
207
  counter = 0
André Anjos's avatar
André Anjos committed
208
  for neg, pos in cmc_scores:
209 210 211 212 213 214
    # set all values that are empty before to None
    if pos is not None and not numpy.array(pos).size:
      pos = None
    if neg is not None and not numpy.array(neg).size:
      neg = None

215 216 217
    if pos is None and neg is None:
      raise ValueError("One pair of the CMC scores has neither positive nor negative values")

218 219 220 221 222 223 224
    # filter out any negative or positive scores below threshold; scores with exactly the threshold are also filtered out
    # now, None and an empty array have different meanings.
    if threshold is not None:
      if neg is not None:
        neg = numpy.array(neg)[neg > threshold]
      if pos is not None:
        pos = numpy.array(pos)[pos > threshold]
225 226 227 228

    if pos is None:
      # no positives, so we definitely do not have a match;
      # check if we have negatives above threshold
229
      if not neg.size:
230 231 232 233 234 235 236 237 238
        # we have no negative scores over the threshold, so we have correctly rejected the probe
        # don't increase any of the two counters...
        continue
      # we have negatives over threshold, so we have incorrect classifications; independent on the actual rank
      counter += 1
    else:
      # we have a positive, so we need to count the probe
      counter += 1

239
      if not numpy.array(pos).size:
240 241 242
        # all positive scores have been filtered out by the threshold, we definitely have a mis-match
        continue

243 244 245 246
      # get the maximum positive score for the current probe item
      # (usually, there is only one positive score, but just in case...)
      max_pos = numpy.max(pos)

247
      if neg is None or not numpy.array(neg).size:
248 249 250 251 252
        # if we had no negatives, or all negatives were below threshold, we have a match at rank 1
        correct += 1
      else:
        # count the number of negative scores that are higher than the best positive score
        index = numpy.sum(neg >= max_pos)
253
        if index < rank:
254 255 256
          correct += 1

  return float(correct) / float(counter)
257 258


259
def cmc(cmc_scores):
260
  """Calculates the cumulative match characteristic (CMC) from the given input.
261

262
  The input has a specific format, which is a list of two-element tuples. Each
263
  of the tuples contains the negative and the positive scores for one probe
264
  item.
265

André Anjos's avatar
André Anjos committed
266 267 268 269 270 271
  For each probe item the probability that the rank :math:`r` of the positive
  score is calculated.  The rank is computed as the number of negative scores
  that are higher than the positive score.  If several positive scores for one
  test item exist, the **highest** positive score is taken. The CMC finally
  computes how many test items have rank r or higher, divided by the total
  number of test values.
272

273 274
  .. note::

275
     The CMC is not available for open set classification. Please use the
André Anjos's avatar
André Anjos committed
276 277 278 279
     :py:func:`detection_identification_rate` and :py:func:`false_alarm_rate`
     instead.


280 281
  Parameters
  ----------
André Anjos's avatar
André Anjos committed
282

283 284
  cmc_scores : :py:class:`list`
    A list in the format ``[(negatives, positives), ...]`` containing the CMC
285
    scores.
286

287 288 289
    Each pair contains the ``negative`` and the ``positive`` scores for **one
    probe item**.  Each pair can contain up to one empty array (or ``None``),
    i.e., in case of open set recognition.
290 291


292 293
  Returns
  -------
André Anjos's avatar
André Anjos committed
294

295 296 297 298 299
  1D :py:class:`numpy.ndarray` of `float`
    A 1D float array representing the CMC curve.
    The rank 1 recognition rate can be found in ``array[0]``, rank 2 rate in
    ``array[1]``, and so on. The number of ranks (``array.shape[0]``) is the
    number of gallery items. Values are in range ``[0,1]``.
André Anjos's avatar
André Anjos committed
300
  """
301

302 303 304 305 306
  # If no scores are given, we cannot plot anything
  probe_count = float(len(cmc_scores))
  if not probe_count:
    raise ValueError("The given set of scores is empty")

André Anjos's avatar
André Anjos committed
307
  # compute MC
308
  match_characteristic = numpy.zeros((max([len(neg) for neg, _ in cmc_scores if neg is not None])+1,), numpy.int)
309

André Anjos's avatar
André Anjos committed
310
  for neg, pos in cmc_scores:
311
    if pos is None or not numpy.array(pos).size:
312
      raise ValueError("For the CMC computation at least one positive score per pair is necessary.")
313 314
    if neg is None:
      neg = []
315

André Anjos's avatar
André Anjos committed
316
    # get the maximum positive score for the current probe item
317
    # (usually, there is only one positive score, but just in case...)
André Anjos's avatar
André Anjos committed
318 319
    max_pos = numpy.max(pos)

320
    # count the number of negative scores that are higher than the best positive score
321
    index = numpy.sum(neg >= max_pos)
322
    match_characteristic[index] += 1
323

André Anjos's avatar
André Anjos committed
324
  # cumulate
325 326 327 328 329
  cumulative_match_characteristic = numpy.cumsum(match_characteristic, dtype=numpy.float64)
  return cumulative_match_characteristic / probe_count


def detection_identification_rate(cmc_scores, threshold, rank = 1):
André Anjos's avatar
André Anjos committed
330 331 332 333 334 335 336 337 338 339 340 341 342 343
  """Computes the `detection and identification rate` for the given threshold.

  This value is designed to be used in an open set identification protocol, and
  defined in Chapter 14.1 of [LiJain2005]_.

  Although the detection and identification rate is designed to be computed on
  an open set protocol, it uses only the probe elements, for which a
  corresponding gallery element exists.  For closed set identification
  protocols, this function is identical to :py:func:`recognition_rate`.  The
  only difference is that for this function, a ``threshold`` for the scores
  need to be defined, while for :py:func:`recognition_rate` it is optional.


  Parameters:
344

345
    cmc_scores (:py:class:`list`): A list in the format ``[(negatives,
346
      positives), ...]`` containing the CMC.
347

André Anjos's avatar
André Anjos committed
348 349 350
      Each pair contains the ``negative`` and the ``positive`` scores for **one
      probe item**.  Each pair can contain up to one empty array (or ``None``),
      i.e., in case of open set recognition.
351

André Anjos's avatar
André Anjos committed
352
    threshold (float): The decision threshold :math:`\\tau``.
353

354
    rank (:obj:`int`, optional): The rank for which the curve should be plotted
355 356


André Anjos's avatar
André Anjos committed
357
  Returns:
358

André Anjos's avatar
André Anjos committed
359
    float: The detection and identification rate for the given threshold.
360 361

  """
André Anjos's avatar
André Anjos committed
362

363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391
  # count the correctly classifier probes
  correct = 0
  counter = 0
  for neg, pos in cmc_scores:
    if pos is None or not numpy.array(pos).size:
      # we only consider probes with corresponding gallery items
      continue
    # we have an in-gallery probe
    counter += 1
    # check, if it is correctly classified
    if neg is None:
      neg = []

    # get the maximum positive score for the current probe item
    # (usually, there is only one positive score, but just in case...)
    max_pos = numpy.max(pos)

    index = numpy.sum(neg >= max_pos) # compute the rank (in fact, rank - 1)
    if max_pos >= threshold and index < rank:
      correct += 1

  if not counter:
    logger.warn("No in-gallery probe was found")
    return 0.

  return float(correct) / float(counter)


def false_alarm_rate(cmc_scores, threshold):
André Anjos's avatar
André Anjos committed
392 393 394 395 396 397 398 399 400 401 402
  """Computes the `false alarm rate` for the given threshold,.

  This value is designed to be used in an open set identification protocol, and
  defined in Chapter 14.1 of [LiJain2005]_.

  The false alarm rate is designed to be computed on an open set protocol, it
  uses only the probe elements, for which **no** corresponding gallery element
  exists.


  Parameters:
403

404
    cmc_scores (:py:class:`list`): A list in the format ``[(negatives,
405
      positives), ...]`` containing the CMC scores (i.e. :py:class:`list`:
406 407
      A list of tuples, where each tuple contains the
      ``negative`` and ``positive`` scores for one probe of the database).
408

André Anjos's avatar
André Anjos committed
409 410 411
      Each pair contains the ``negative`` and the ``positive`` scores for **one
      probe item**.  Each pair can contain up to one empty array (or ``None``),
      i.e., in case of open set recognition.
412

André Anjos's avatar
André Anjos committed
413
    threshold (float): The decision threshold :math:`\\tau``.
414 415


André Anjos's avatar
André Anjos committed
416
  Returns:
417

André Anjos's avatar
André Anjos committed
418
    float: The false alarm rate.
419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436

  """
  incorrect = 0
  counter = 0
  for neg, pos in cmc_scores:
    # we only consider the out-of-gallery probes, i.e., with no positive scores
    if pos is None or not numpy.array(pos).size:
      counter += 1

      # check if the probe is above threshold
      if neg is None or not numpy.array(neg).size:
        raise ValueError("One pair of the CMC scores has neither positive nor negative values")
      if numpy.max(neg) >= threshold:
        incorrect += 1

  if not counter:
    logger.warn("No out-of-gallery probe was found")
    return 0.
André Anjos's avatar
André Anjos committed
437

438
  return float(incorrect) / float(counter)
439

440

441 442 443
def eer(negatives, positives, is_sorted=False, also_farfrr=False):
  """Calculates the Equal Error Rate (EER).

444 445 446
  Please note that it is possible that eer != fpr != fnr.
  This function returns (fpr + fnr) / 2 as eer.
  If you also need the fpr and fnr values, set ``also_farfrr`` to ``True``.
447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462

  Parameters
  ----------
  negatives : ``array_like (1D, float)``
      The scores for comparisons of objects of different classes.
  positives : ``array_like (1D, float)``
      The scores for comparisons of objects of the same class.
  is_sorted : bool
      Are both sets of scores already in ascendantly sorted order?
  also_farfrr : bool
      If True, it will also return far and frr.

  Returns
  -------
  eer : float
      The Equal Error Rate (EER).
463 464
  fpr : float
      The False Positive Rate (FPR). Returned only when ``also_farfrr`` is
465
      ``True``.
466 467
  fnr : float
      The False Negative Rate (FNR). Returned only when ``also_farfrr`` is
468 469 470 471 472 473 474 475 476
      ``True``.
  """
  threshold = eer_threshold(negatives, positives, is_sorted)
  far, frr = farfrr(negatives, positives, threshold)
  if also_farfrr:
    return (far + frr) / 2.0, far, frr
  return (far + frr) / 2.0


477 478 479
def get_config():
  """Returns a string containing the configuration information.
  """
480
  import bob.extension
481
  return bob.extension.get_config(__name__, version.externals)
482 483


484 485
# gets sphinx autodoc done right - don't remove it
__all__ = [_ for _ in dir() if not _.startswith('_')]