plot.py 17.8 KB
Newer Older
André Anjos's avatar
André Anjos committed
1
2
3
4
5
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Chakka Murali Mohan, Trainee, IDIAP Research Institute, Switzerland.
# Mon 23 May 2011 14:36:14 CEST

6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
def log_values(min_step = -4, counts_per_step = 4):
  """log_values(min_step, counts_per_step) -> log_list

  This function computes log-scaled values between :math:`10^{M}` and 1 (including), where :math:`M` is the ``min_ste`` argument, which needs to be a negative integer.
  The integral ``counts_per_step`` value defines how many values between two adjacent powers of 10 will be created.
  The total number of values will be ``-min_step * counts_per_step + 1``.

  **Parameters:**

  ``min_step`` : int (negative)
    The power of 10 that will be the minimum value. E.g., the default ``-4`` will result in the first number to be :math:`10^{-4}` = ``0.00001`` or ``0.01%``

  ``counts_per_step`` : int (positive)
    The number of values that will be put between two adjacent powers of 10.
    With the default value ``4`` (and default values of ``min_step``), we will get ``log_list[0] == 1e-4``, ``log_list[4] == 1e-3``, ..., ``log_list[16] == 1``.

  **Returns**

  ``log_list`` : [float]
    A list of logarithmically scaled values between :math:`10^{M}` and 1.
  """
  import math
  return [math.pow(10., i * 1./counts_per_step) for i in range(min_step*counts_per_step,0)] + [1.]

André Anjos's avatar
André Anjos committed
30
31
32
"""Methods to plot error analysis figures such as ROC, precision-recall curve, EPC and DET"""

def roc(negatives, positives, npoints=100, CAR=False, **kwargs):
33
  """Plots Receiver Operating Characteristic (ROC) curve.
André Anjos's avatar
André Anjos committed
34

35
  This method will call ``matplotlib`` to plot the ROC curve for a system which
André Anjos's avatar
André Anjos committed
36
  contains a particular set of negatives (impostors) and positives (clients)
37
38
39
  scores. We use the standard :py:func:`matplotlib.pyplot.plot` command. All parameters
  passed with exception of the three first parameters of this method will be
  directly passed to the plot command.
André Anjos's avatar
André Anjos committed
40

41
42
  The plot will represent the false-alarm on the horizontal axis and the false-rejection on the vertical axis.
  The values for the axis will be computed using :py:func:`bob.measure.roc`.
André Anjos's avatar
André Anjos committed
43

44
  .. note::
André Anjos's avatar
André Anjos committed
45

46
47
48
    This function does not initiate and save the figure instance, it only
    issues the plotting command. You are the responsible for setting up and
    saving the figure as you see fit.
André Anjos's avatar
André Anjos committed
49

50
  **Parameters:**
André Anjos's avatar
André Anjos committed
51

52
53
  ``negatives, positives`` : array_like(1D, float)
    The list of negative and positive scores forwarded to :py:func:`bob.measure.roc`
André Anjos's avatar
André Anjos committed
54

55
56
  ``npoints`` : int
    The number of points forwarded to :py:func:`bob.measure.roc`
André Anjos's avatar
André Anjos committed
57

58
59
  ``CAR`` : bool
    If set to ``True``, it will plot the CAR over FAR in using :py:func:`matplotlib.pyplot.semilogx`, otherwise the FAR over FRR linearly using :py:func:`matplotlib.pyplot.plot`.
André Anjos's avatar
André Anjos committed
60

61
62
  ``kwargs`` : keyword arguments
    Extra plotting parameters, which are passed directly to :py:func:`matplotlib.pyplot.plot`.
André Anjos's avatar
André Anjos committed
63

64
  **Returns:**
André Anjos's avatar
André Anjos committed
65

66
  The return value is the matplotlib line that was added as defined by :py:func:`matplotlib.pyplot.plot` or :py:func:`matplotlib.pyplot.semilogx`.
André Anjos's avatar
André Anjos committed
67
68
  """

69
  from matplotlib import pyplot
André Anjos's avatar
André Anjos committed
70
  from . import roc as calc
71
  out = calc(negatives, positives, npoints)
André Anjos's avatar
André Anjos committed
72
  if not CAR:
73
    return pyplot.plot(100.0*out[0,:], 100.0*out[1,:], **kwargs)
André Anjos's avatar
André Anjos committed
74
  else:
75
76
    return pyplot.semilogx(100.0*out[0,:], 100.0*(1-out[1,:]), **kwargs)

André Anjos's avatar
André Anjos committed
77

78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
def roc_for_far(negatives, positives, far_values = log_values(), **kwargs):
  """Plots Receiver Operating Characteristic (ROC) curve for the given list of False Acceptance Rates (FAR).

  This method will call ``matplotlib`` to plot the ROC curve for a system which
  contains a particular set of negatives (impostors) and positives (clients)
  scores. We use the standard :py:func:`matplotlib.pyplot.semilogx` command. All parameters
  passed with exception of the three first parameters of this method will be
  directly passed to the plot command.

  The plot will represent the False Acceptance Rate (FAR) on the horizontal axis and the Correct Acceptance Rate (CAR) on the vertical axis.
  The values for the axis will be computed using :py:func:`bob.measure.roc_for_far`.

  .. note::

    This function does not initiate and save the figure instance, it only
    issues the plotting command. You are the responsible for setting up and
    saving the figure as you see fit.

  **Parameters:**

  ``negatives, positives`` : array_like(1D, float)
    The list of negative and positive scores forwarded to :py:func:`bob.measure.roc`

  ``far_values`` : [float]
    The values for the FAR, where the CAR should be plotted; each value should be in range [0,1].

  ``kwargs`` : keyword arguments
    Extra plotting parameters, which are passed directly to :py:func:`matplotlib.pyplot.plot`.

  **Returns:**

  The return value is the matplotlib line that was added as defined by :py:func:`matplotlib.pyplot.semilogx`.
  """

  from matplotlib import pyplot
  from . import roc_for_far as calc
  out = calc(negatives, positives, far_values)
  return pyplot.semilogx(100.0*out[0,:], 100.0*(1-out[1,:]), **kwargs)


André Anjos's avatar
André Anjos committed
118
119
120
def precision_recall_curve(negatives, positives, npoints=100, **kwargs):
  """Plots Precision-Recall curve.

121
122
123
124
125
  This method will call ``matplotlib`` to plot the precision-recall curve for a system which
  contains a particular set of ``negatives`` (impostors) and ``positives`` (clients)
  scores. We use the standard :py:func:`matplotlib.pyplot.plot` command. All parameters
  passed with exception of the three first parameters of this method will be
  directly passed to the plot command.
André Anjos's avatar
André Anjos committed
126

127
  .. note::
André Anjos's avatar
André Anjos committed
128

129
130
131
    This function does not initiate and save the figure instance, it only
    issues the plotting command. You are the responsible for setting up and
    saving the figure as you see fit.
André Anjos's avatar
André Anjos committed
132

133
  **Parameters:**
André Anjos's avatar
André Anjos committed
134

135
136
  ``negatives, positives`` : array_like(1D, float)
    The list of negative and positive scores forwarded to :py:func:`bob.measure.precision_recall_curve`
André Anjos's avatar
André Anjos committed
137

138
139
  ``npoints`` : int
    The number of points forwarded to :py:func:`bob.measure.precision_recall_curve`
André Anjos's avatar
André Anjos committed
140

141
142
  ``kwargs`` : keyword arguments
    Extra plotting parameters, which are passed directly to :py:func:`matplotlib.pyplot.plot`.
André Anjos's avatar
André Anjos committed
143

144
  **Returns:**
André Anjos's avatar
André Anjos committed
145

146
  The return value is the ``matplotlib`` line that was added as defined by :py:func:`matplotlib.pyplot.plot`.
André Anjos's avatar
André Anjos committed
147
148
  """

149
  from matplotlib import pyplot
André Anjos's avatar
André Anjos committed
150
151
  from . import precision_recall_curve as calc
  out = calc(negatives, positives, npoints)
152
  return pyplot.plot(100.0*out[0,:], 100.0*out[1,:], **kwargs)
André Anjos's avatar
André Anjos committed
153
154
155
156
157
158
159
160
161
162
163


def epc(dev_negatives, dev_positives, test_negatives, test_positives,
    npoints=100, **kwargs):
  """Plots Expected Performance Curve (EPC) as defined in the paper:

  Bengio, S., Keller, M., Mariéthoz, J. (2004). The Expected Performance Curve.
  International Conference on Machine Learning ICML Workshop on ROC Analysis in
  Machine Learning, 136(1), 1963–1966. IDIAP RR. Available:
  http://eprints.pascal-network.org/archive/00000670/

164
  This method will call ``matplotlib`` to plot the EPC curve for a system which
André Anjos's avatar
André Anjos committed
165
166
  contains a particular set of negatives (impostors) and positives (clients)
  for both the development and test sets. We use the standard
167
  :py:func:`matplotlib.pyplot.plot` command. All parameters passed with exception of
André Anjos's avatar
André Anjos committed
168
  the five first parameters of this method will be directly passed to the plot
169
  command.
André Anjos's avatar
André Anjos committed
170

171
  The plot will represent the minimum HTER on the vertical axis and the cost on the horizontal axis.
André Anjos's avatar
André Anjos committed
172

173
  .. note::
André Anjos's avatar
André Anjos committed
174

175
176
177
    This function does not initiate and save the figure instance, it only
    issues the plotting commands. You are the responsible for setting up and
    saving the figure as you see fit.
André Anjos's avatar
André Anjos committed
178

179
  **Parameters:**
André Anjos's avatar
André Anjos committed
180

181
182
  ``dev_negatives, dev_positvies, test_negatives, test_positives`` : array_like(1D, float)
    See :py:func:bob.measure.epc` for details
André Anjos's avatar
André Anjos committed
183

184
185
  ``npoints`` : int
    See :py:func:bob.measure.epc` for details
André Anjos's avatar
André Anjos committed
186

187
188
  ``kwargs`` : keyword arguments
    Extra plotting parameters, which are passed directly to :py:func:`matplotlib.pyplot.plot`.
André Anjos's avatar
André Anjos committed
189

190
  **Returns:**
André Anjos's avatar
André Anjos committed
191

192
  The return value is the ``matplotlib`` line that was added as defined by :py:func:`matplotlib.pyplot.plot`.
André Anjos's avatar
André Anjos committed
193
194
  """

195
  from matplotlib import pyplot
André Anjos's avatar
André Anjos committed
196
197
  from . import epc as calc

198
  out = calc(dev_negatives, dev_positives, test_negatives, test_positives, npoints)
199
200
  return pyplot.plot(out[0,:], 100.0*out[1,:], **kwargs)

André Anjos's avatar
André Anjos committed
201
202
203
204
205
206
207
208
209

def det(negatives, positives, npoints=100, axisfontsize='x-small', **kwargs):
  """Plots Detection Error Trade-off (DET) curve as defined in the paper:

  Martin, A., Doddington, G., Kamm, T., Ordowski, M., & Przybocki, M. (1997).
  The DET curve in assessment of detection task performance. Fifth European
  Conference on Speech Communication and Technology (pp. 1895-1898). Available:
  http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.117.4489&rep=rep1&type=pdf

210
  This method will call ``matplotlib`` to plot the DET curve(s) for a system which
André Anjos's avatar
André Anjos committed
211
  contains a particular set of negatives (impostors) and positives (clients)
212
  scores. We use the standard :py:func:`matplotlib.pyplot.plot` command. All parameters
André Anjos's avatar
André Anjos committed
213
  passed with exception of the three first parameters of this method will be
214
  directly passed to the plot command.
André Anjos's avatar
André Anjos committed
215

216
217
  The plot will represent the false-alarm on the horizontal axis and the
  false-rejection on the vertical axis.
André Anjos's avatar
André Anjos committed
218
219
220
221
222
223
224
225
226
227
228
229
230
231

  This method is strongly inspired by the NIST implementation for Matlab,
  called DETware, version 2.1 and available for download at the NIST website:

  http://www.itl.nist.gov/iad/mig/tools/

  .. note::

    This function does not initiate and save the figure instance, it only
    issues the plotting commands. You are the responsible for setting up and
    saving the figure as you see fit.

  .. note::

232
    If you wish to reset axis zooming, you must use the Gaussian scale rather
André Anjos's avatar
André Anjos committed
233
    than the visual marks showed at the plot, which are just there for
234
235
    displaying purposes. The real axis scale is based on :py:func:`bob.measure.ppndf`.
    For example, if you wish to set the x and y  axis to display data between 1% and 40% here is the recipe:
André Anjos's avatar
André Anjos committed
236
237
238

    .. code-block:: python

André Anjos's avatar
André Anjos committed
239
      import bob.measure
240
      from matplotlib import pyplot
André Anjos's avatar
André Anjos committed
241
      bob.measure.plot.det(...) #call this as many times as you need
André Anjos's avatar
André Anjos committed
242
      #AFTER you plot the DET curve, just set the axis in this way:
243
      pyplot.axis([bob.measure.ppndf(k/100.0) for k in (1, 40, 1, 40)])
André Anjos's avatar
André Anjos committed
244
245

    We provide a convenient way for you to do the above in this module. So,
246
    optionally, you may use the :py:func:`bob.measure.plot.det_axis` method like this:
André Anjos's avatar
André Anjos committed
247
248
249

    .. code-block:: python

André Anjos's avatar
André Anjos committed
250
251
      import bob.measure
      bob.measure.plot.det(...)
André Anjos's avatar
André Anjos committed
252
      # please note we convert percentage values in det_axis()
André Anjos's avatar
André Anjos committed
253
      bob.measure.plot.det_axis([1, 40, 1, 40])
André Anjos's avatar
André Anjos committed
254

255
  **Parameters:**
André Anjos's avatar
André Anjos committed
256

257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
  ``negatives, positives`` : array_like(1D, float)
    The list of negative and positive scores forwarded to :py:func:`bob.measure.det`

  ``npoints`` : int
    The number of points forwarded to :py:func:`bob.measure.det`

  ``axisfontsize`` : str
    The size to be used by x/y-tick-labels to set the font size on the axis

  ``kwargs`` : keyword arguments
    Extra plotting parameters, which are passed directly to :py:func:`matplotlib.pyplot.plot`.

  **Returns:**

  The return value is the ``matplotlib`` line that was added as defined by :py:func:`matplotlib.pyplot.plot`.
  """
André Anjos's avatar
André Anjos committed
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299

  # these are some constants required in this method
  desiredTicks = [
      "0.00001", "0.00002", "0.00005",
      "0.0001", "0.0002", "0.0005",
      "0.001", "0.002", "0.005",
      "0.01", "0.02", "0.05",
      "0.1", "0.2", "0.4", "0.6", "0.8", "0.9",
      "0.95", "0.98", "0.99",
      "0.995", "0.998", "0.999",
      "0.9995", "0.9998", "0.9999",
      "0.99995", "0.99998", "0.99999"
      ]

  desiredLabels = [
      "0.001", "0.002", "0.005",
      "0.01", "0.02", "0.05",
      "0.1", "0.2", "0.5",
      "1", "2", "5",
      "10", "20", "40", "60", "80", "90",
      "95", "98", "99",
      "99.5", "99.8", "99.9",
      "99.95", "99.98", "99.99",
      "99.995", "99.998", "99.999"
      ]

  # this will actually do the plotting
300
  from matplotlib import pyplot
André Anjos's avatar
André Anjos committed
301
302
303
  from . import det as calc
  from . import ppndf

304
  out = calc(negatives, positives, npoints)
305
  retval = pyplot.plot(out[0,:], out[1,:], **kwargs)
André Anjos's avatar
André Anjos committed
306
307
308

  # now the trick: we must plot the tick marks by hand using the PPNDF method
  pticks = [ppndf(float(v)) for v in desiredTicks]
309
  ax = pyplot.gca() #and finally we set our own tick marks
André Anjos's avatar
André Anjos committed
310
311
312
313
314
315
316
  ax.set_xticks(pticks)
  ax.set_xticklabels(desiredLabels, size=axisfontsize)
  ax.set_yticks(pticks)
  ax.set_yticklabels(desiredLabels, size=axisfontsize)

  return retval

317

André Anjos's avatar
André Anjos committed
318
319
320
def det_axis(v, **kwargs):
  """Sets the axis in a DET plot.

321
322
323
324
325
  This method wraps the :py:func:`matplotlib.pyplot.axis` by calling
  :py:func:`bob.measure.ppndf` on the values passed by the user so they are meaningful
  in a DET plot as performed by :py:func:`bob.measure.plot.det`.

  **Parameters:**
André Anjos's avatar
André Anjos committed
326

327
328
329
330
331
  ``v`` : (int, int, int, int)
    The X and Y limits in the order ``(xmin, xmax, ymin, ymax)``.
    Expected values should be in percentage (between 0 and 100%).
    If ``v`` is not a list or tuple that contains 4 numbers it is passed
    without further inspection to :py:func:`matplotlib.pyplot.axis`.
André Anjos's avatar
André Anjos committed
332

333
334
  ``kwargs`` : keyword arguments
    Extra parameters, which are passed directly to :py:func:`matplotlib.pyplot.axis`.
André Anjos's avatar
André Anjos committed
335

336
  **Returns:**
André Anjos's avatar
André Anjos committed
337

338
  Returns whatever :py:func:`matplotlib.pyplot.axis` returns.
André Anjos's avatar
André Anjos committed
339
340
341
  """

  import logging
342
  logger = logging.getLogger("bob.measure")
André Anjos's avatar
André Anjos committed
343

344
  from matplotlib import pyplot
André Anjos's avatar
André Anjos committed
345
346
347
348
349
350
351
  from . import ppndf

  # treat input
  try:
    tv = list(v) #normal input
    if len(tv) != 4: raise IndexError
    tv = [ppndf(float(k)/100) for k in tv]
352
    cur = pyplot.axis()
André Anjos's avatar
André Anjos committed
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370

    # limits must be within bounds
    if tv[0] < cur[0]:
      logger.warn("Readjusting xmin: the provided value is out of bounds")
      tv[0] = cur[0]
    if tv[1] > cur[1]:
      logger.warn("Readjusting xmax: the provided value is out of bounds")
      tv[1] = cur[1]
    if tv[2] < cur[2]:
      logger.warn("Readjusting ymin: the provided value is out of bounds")
      tv[2] = cur[2]
    if tv[3] > cur[3]:
      logger.warn("Readjusting ymax: the provided value is out of bounds")
      tv[3] = cur[3]

  except:
    tv = v

371
372
  return pyplot.axis(tv, **kwargs)

André Anjos's avatar
André Anjos committed
373
374

def cmc(cmc_scores, logx = True, **kwargs):
375
376
377
378
379
380
381
  """Plots the (cumulative) match characteristics curve and returns the maximum rank.

  This function plots a CMC curve using the given CMC scores, which can be read from the our score files using the :py:func:`bob.measure.load.cmc_four_column` or :py:func:`bob.measure.load.cmc_five_column` methods.
  The structure of the ``cmc_scores`` parameter is relatively complex.
  It contains a list of pairs of lists.
  For each probe object, a pair of list negative and positive scores is required.

382
  **Parameters:**
383

384
385
  ``cmc_scores`` : [(array_like(1D, float), array_like(1D, float))]
    See :py:func:`bob.measure.cmc`
386

387
388
  ``logx`` : bool
    Plot the rank axis in logarithmic scale using :py:func:`matplotlib.pyplot.semilogx` or in linear scale using :py:func:`matplotlib.pyplot.plot`? (Default: ``True``)
389

390
391
392
393
  ``kwargs`` : keyword arguments
    Extra plotting parameters, which are passed directly to :py:func:`matplotlib.pyplot.plot` or :py:func:`matplotlib.pyplot.semilogx`.

  **Returns:**
André Anjos's avatar
André Anjos committed
394

395
396
397
  The number of classes (clients) in the given scores.
  """
  from matplotlib import pyplot
André Anjos's avatar
André Anjos committed
398
  from . import cmc as calc
399

André Anjos's avatar
André Anjos committed
400
401
402
  out = calc(cmc_scores)

  if logx:
403
    pyplot.semilogx(range(1, len(out)+1), out * 100, **kwargs)
André Anjos's avatar
André Anjos committed
404
  else:
405
    pyplot.plot(range(1, len(out)+1), out * 100, **kwargs)
André Anjos's avatar
André Anjos committed
406
407

  return len(out)
408
409


410
411
def detection_identification_curve(cmc_scores, far_values = log_values(), rank = 1, logx = True, **kwargs):
  """Plots the Detection & Identification curve over the FAR for the given FAR values.
412
  This curve is designed to be used in an open set identification protocol, and defined in Chapter 14.1 of [LiJain2005]_.
413
414
415
416
417
418
  It requires to have at least one open set probe item, i.e., with no corresponding gallery, such that the positives for that pair are ``None``.

  The detection and identification curve first computes FAR thresholds based on the out-of-set probe scores (negative scores).
  For each probe item, the **maximum** negative score is used.
  Then, it plots the detection and identification rates for those thresholds, which are based on the in-set probe scores only.
  See [LiJain2005]_ for more details.
419
420
421
422

  **Parameters:**

  ``cmc_scores`` : [(array_like(1D, float), array_like(1D, float))]
423
    See :py:func:`bob.measure.detection_identification_rate`
424
425
426
427
428

  ``far_values`` : [float]
    The values for the FAR, where the CAR should be plotted; each value should be in range [0,1].

  ``rank`` : int or ``None``
429
    The rank for which the curve should be plotted, 1 by default.
430
431
432
433
434
435
436
437
438
439
440
441
442
443

  ``logx`` : bool
    Plot the FAR axis in logarithmic scale using :py:func:`matplotlib.pyplot.semilogx` or in linear scale using :py:func:`matplotlib.pyplot.plot`? (Default: ``True``)

  ``kwargs`` : keyword arguments
    Extra plotting parameters, which are passed directly to :py:func:`matplotlib.pyplot.plot` or :py:func:`matplotlib.pyplot.semilogx`.

  **Returns:**

  The return value is the ``matplotlib`` line that was added as defined by :py:func:`matplotlib.pyplot.plot`.

  .. [LiJain2005] **Stan Li and Anil K. Jain**, *Handbook of Face Recognition*, Springer, 2005
  """

444
  import numpy
445
  from matplotlib import pyplot
446
  from . import far_threshold, detection_identification_rate
447

448
449
450
451
  # for each probe, for which no positives exists, get the highest negative score; and sort them to compute the FAR thresholds
  negatives = sorted(max(neg) for neg,pos in cmc_scores if (pos is None or not numpy.array(pos).size) and neg is not None)
  if not negatives:
    raise ValueError("There need to be at least one pair with only negative scores")
452
453
454
455

  # compute thresholds based on FAR values
  thresholds = [far_threshold(negatives, [], v, True) for v in far_values]

456
457
  # compute detection and identification rate based on the thresholds for the given rank
  rates = [100.*detection_identification_rate(cmc_scores, t, rank) for t in thresholds]
458
459
460
461
462
463

  # plot curve
  if logx:
    return pyplot.semilogx(far_values, rates, **kwargs)
  else:
    return pyplot.plot(far_values, rates, **kwargs)