Merge branch '87-no-way-of-plotting-open-set-dir-curves' into 'master'

Resolve "No way of plotting open-set DIR curves"

Closes #87

See merge request !94

bob/bio/base/script/evaluate.py

@@ -47,6 +47,7 @@ def command_line_arguments(command_line_parameters):
   parser.add_argument('-m', '--mindcf', action = 'store_true', help = "If given, minDCF will be computed.")
   parser.add_argument('--cost', default=0.99,  help='Cost for FAR in minDCF')
   parser.add_argument('-r', '--rr', action = 'store_true', help = "If given, the Recognition Rate will be computed.")
+  parser.add_argument('-o', '--rank', type=int, default=1, help = "The rank for which to plot the DIR curve")
   parser.add_argument('-t', '--thresholds', type=float, nargs='+', help = "If given, the Recognition Rate will incorporate an Open Set handling, rejecting all scores that are below the given threshold; when multiple thresholds are given, they are applied in the same order as the --dev-files.")
   parser.add_argument('-l', '--legends', nargs='+', help = "A list of legend strings used for ROC, CMC and DET plots; if given, must be the same number than --dev-files.")
   parser.add_argument('-F', '--legend-font-size', type=int, default=10, help = "Set the font size of the legends.")
@@ -55,6 +56,7 @@ def command_line_arguments(command_line_parameters):
   parser.add_argument('-R', '--roc', help = "If given, ROC curves will be plotted into the given pdf file.")
   parser.add_argument('-D', '--det', help = "If given, DET curves will be plotted into the given pdf file.")
   parser.add_argument('-C', '--cmc', help = "If given, CMC curves will be plotted into the given pdf file.")
+  parser.add_argument('-O', '--dir', help = "If given, DIR curves will be plotted into the given pdf file; This is an open-set measure, which cannot be applied to closed set score files.")
   parser.add_argument('-E', '--epc', help = "If given, EPC curves will be plotted into the given pdf file. For this plot --eval-files is mandatory.")
   parser.add_argument('-M', '--min-far-value', type=float, default=1e-4, help = "Select the minimum FAR value used in ROC plots; should be a power of 10.")
   parser.add_argument('-L', '--far-line-at', type=float, help = "If given, draw a veritcal line at this FAR value in the ROC plots.")
@@ -68,8 +70,11 @@ def command_line_arguments(command_line_parameters):
   # set verbosity level
   bob.core.log.set_verbosity_level(logger, args.verbose)
 
-
   # some sanity checks:
+  for f in args.dev_files + (args.eval_files or []):
+    if not os.path.exists(f):
+      raise ValueError("The provided score file '%s' does not exist", f)
+
   if args.eval_files is not None and len(args.dev_files) != len(args.eval_files):
     logger.error("The number of --dev-files (%d) and --eval-files (%d) are not identical", len(args.dev_files), len(args.eval_files))
 
@@ -98,6 +103,14 @@ def command_line_arguments(command_line_parameters):
 
   return args
 
+def _add_far_labels(min_far):
+  # compute and apply tick marks
+  ticks = [min_far]
+  while ticks[-1] < 1.: ticks.append(ticks[-1] * 10.)
+  pyplot.xticks(ticks)
+  pyplot.axis([min_far, 1., -0.01, 1.01])
+
+
 
 def _plot_roc(frrs, colors, labels, title, fontsize=10, position=None, farfrrs=None):
   if position is None: position = 'lower right'
@@ -116,12 +129,7 @@ def _plot_roc(frrs, colors, labels, title, fontsize=10, position=None, farfrrs=N
     else:
       pyplot.plot([x[0] for x in farfrrs], [(1.-x[1]) for x in farfrrs], '--', color='black')
 
-  # compute and apply tick marks
-  min_far = frrs[0][0][0]
-  ticks = [min_far]
-  while ticks[-1] < 1.: ticks.append(ticks[-1] * 10.)
-  pyplot.axis([min_far, 1., -0.01, 1.01])
-  pyplot.xticks(ticks)
+  _add_far_labels(frrs[0][0][0])
 
   # set label, legend and title
   pyplot.xlabel('FMR')
@@ -185,6 +193,39 @@ def _plot_cmc(cmcs, colors, labels, title, fontsize=10, position=None):
   return figure
 
 
+def _plot_dir(cmc_scores, far_values, rank, colors, labels, title, fontsize=10, position=None):
+  if position is None: position = 'lower right'
+  # open new page for current plot
+  figure = pyplot.figure()
+
+  # for each probe, for which no positives exists, get the highest negative
+  # score; and sort them to compute the FAR thresholds
+  for i, cmcs in enumerate(cmc_scores):
+    negatives = sorted(max(neg) for neg, pos in cmcs if (pos is None or not numpy.array(pos).size) and neg is not None)
+    if not negatives:
+      raise ValueError("There need to be at least one pair with only negative scores")
+
+    # compute thresholds based on FAR values
+    thresholds = [bob.measure.far_threshold(negatives, [], v, True) for v in far_values]
+
+    # compute detection and identification rate based on the thresholds for
+    # the given rank
+    rates = [bob.measure.detection_identification_rate(cmcs, t, rank) for t in thresholds]
+
+    # plot DIR curve
+    pyplot.semilogx(far_values, rates, figure=figure, color=colors[i], label=labels[i])
+
+  # finalize plot
+  _add_far_labels(far_values[0])
+
+  pyplot.xlabel('FAR')
+  pyplot.ylabel('DIR')
+  pyplot.legend(loc=position, prop = {'size':fontsize})
+  pyplot.title(title)
+
+  return figure
+
+
 def _plot_epc(scores_dev, scores_eval, colors, labels, title, fontsize=10, position=None):
   if position is None: position = 'upper center'
   # open new page for current plot
@@ -367,15 +408,14 @@ def main(command_line_parameters=None):
       try:
         # create a multi-page PDF for the EPC curve
         pdf = PdfPages(args.epc)
-        pdf.savefig(_plot_epc(scores_dev, scores_eval, colors, args.legends, args.title if args.title is not None else "" , args.legend_font_size, args.legend_position), bbox_inches='tight')
+        pdf.savefig(_plot_epc(scores_dev, scores_eval, colors, args.legends, args.title[0] if args.title is not None else "" , args.legend_font_size, args.legend_position), bbox_inches='tight')
         pdf.close()
       except RuntimeError as e:
         raise RuntimeError("During plotting of EPC curves, the following exception occured:\n%s" % e)
 
 
 
-
-  if args.cmc or args.rr:
+  if args.cmc or args.rr or args.dir:
     logger.info("Loading CMC data on the development " + ("and on the evaluation set" if args.eval_files else "set"))
     cmcs_dev = [bob.measure.load.cmc(os.path.join(args.directory, f)) for f in args.dev_files]
     if args.eval_files:
@@ -384,7 +424,7 @@ def main(command_line_parameters=None):
     if args.cmc:
       logger.info("Plotting CMC curves to file '%s'", args.cmc)
       try:
-        # create a multi-page PDF for the ROC curve
+        # create a multi-page PDF for the CMC curve
         pdf = PdfPages(args.cmc)
         # create a separate figure for dev and eval
         pdf.savefig(_plot_cmc(cmcs_dev, colors, args.legends, args.title[0] if args.title is not None else "CMC curve for development set", args.legend_font_size, args.legend_position), bbox_inches='tight')
@@ -392,7 +432,7 @@ def main(command_line_parameters=None):
           pdf.savefig(_plot_cmc(cmcs_eval, colors, args.legends, args.title[1] if args.title is not None else "CMC curve for evaluation set", args.legend_font_size, args.legend_position), bbox_inches='tight')
         pdf.close()
       except RuntimeError as e:
-        raise RuntimeError("During plotting of ROC curves, the following exception occured:\n%s\nUsually this happens when the label contains characters that LaTeX cannot parse." % e)
+        raise RuntimeError("During plotting of CMC curves, the following exception occured:\n%s\nUsually this happens when the label contains characters that LaTeX cannot parse." % e)
 
     if args.rr:
       logger.info("Computing recognition rate on the development " + ("and on the evaluation set" if args.eval_files else "set"))
@@ -402,3 +442,19 @@ def main(command_line_parameters=None):
         if args.eval_files:
           rr = bob.measure.recognition_rate(cmcs_eval[i], args.thresholds[i])
           print("The Recognition Rate of the development set of '%s' is %2.3f%%" % (args.legends[i], rr * 100.))
+
+    if args.dir:
+      # compute false alarm values to evaluate
+      min_far = int(math.floor(math.log(args.min_far_value, 10)))
+      fars = [math.pow(10., i * 0.25) for i in range(min_far * 4, 0)] + [1.]
+      logger.info("Plotting DIR curves to file '%s'", args.dir)
+      try:
+        # create a multi-page PDF for the DIR curve
+        pdf = PdfPages(args.dir)
+        # create a separate figure for dev and eval
+        pdf.savefig(_plot_dir(cmcs_dev, fars, args.rank, colors, args.legends, args.title[0] if args.title is not None else "DIR curve for development set", args.legend_font_size, args.legend_position), bbox_inches='tight')
+        if args.eval_files:
+          pdf.savefig(_plot_dir(cmcs_eval, fars, args.rank, colors, args.legends, args.title[1] if args.title is not None else "DIR curve for evaluation set", args.legend_font_size, args.legend_position), bbox_inches='tight')
+        pdf.close()
+      except RuntimeError as e:
+        raise RuntimeError("During plotting of DIR curves, the following exception occured:\n%s")

bob/bio/base/test/data/scores-nonorm-openset-dev

+3 3 s3/1 2915.37699106
+3 3 s3/3 2198.08257352
+3 3 s3/6 3725.56009212
+3 3 s3/8 3923.53259194
+3 3 s3/10 3281.35520784
+3 4 s4/1 4593.48734623
+3 4 s4/3 4763.32520410
+3 4 s4/6 4935.13393942
+3 4 s4/8 4721.70964800
+3 4 s4/10 3801.80614971
+3 7 s7/1 4735.99915541
+3 7 s7/3 4893.73282883
+3 7 s7/6 4726.88893460
+3 7 s7/8 5008.08047060
+3 7 s7/10 4650.56770728
+3 8 s8/1 5843.10148808
+3 8 s8/3 5674.83982153
+3 8 s8/6 5739.77804449
+3 8 s8/8 5858.49067593
+3 8 s8/10 5410.76565746
+3 9 s9/1 4368.39776119
+3 9 s9/3 4214.54766256
+3 9 s9/6 4734.12357253
+3 9 s9/8 4432.64830547
+3 9 s9/10 3922.38932285
+3 13 s13/1 5116.34410492
+3 13 s13/3 5538.12513401
+3 13 s13/6 4537.41842902
+3 13 s13/8 4981.18831204
+3 13 s13/10 4745.49965757
+3 15 s15/1 4795.29936500
+3 15 s15/3 4823.72698232
+3 15 s15/6 4826.18420701
+3 15 s15/8 5030.10317986
+3 15 s15/10 4379.85502043
+3 18 s18/1 4394.62489867
+3 18 s18/3 4969.12597949
+3 18 s18/6 4945.15581150
+3 18 s18/8 5010.17993689
+3 18 s18/10 4955.47333763
+3 19 s19/1 5574.85524476
+3 19 s19/3 5953.73504617
+3 19 s19/6 5793.52699139
+3 19 s19/8 5821.70645086
+3 19 s19/10 5525.87468189
+3 22 s22/1 5320.28608253
+3 22 s22/3 5112.94553071
+3 22 s22/6 5759.13005583
+3 22 s22/8 5413.43596988
+3 22 s22/10 5729.36584623
+3 23 s23/1 4558.52190957
+3 23 s23/3 4897.26913698
+3 23 s23/6 4778.95856856
+3 23 s23/8 4109.44266294
+3 23 s23/10 4439.30118375
+3 25 s25/1 4085.08188412
+3 25 s25/3 4031.13371150
+3 25 s25/6 4255.67197984
+3 25 s25/8 4992.09154563
+3 25 s25/10 4041.30733798
+3 28 s28/1 5189.92697444
+3 28 s28/3 5179.55413139
+3 28 s28/6 5033.20921481
+3 28 s28/8 4864.09045969
+3 28 s28/10 5663.88788731
+3 30 s30/1 4407.45425387
+3 30 s30/3 4819.81576411
+3 30 s30/6 4456.45172755
+3 30 s30/8 5087.76247480
+3 30 s30/10 4706.95899706
+3 31 s31/1 4605.94626543
+3 31 s31/3 6012.36085078
+3 31 s31/6 4740.99715250
+3 31 s31/8 4587.20285141
+3 31 s31/10 5987.25563176
+3 32 s32/1 5714.79597186
+3 32 s32/3 5864.20830121
+3 32 s32/6 5359.54540983
+3 32 s32/8 5635.99831441
+3 32 s32/10 5212.69776603
+3 35 s35/1 5195.81485428
+3 35 s35/3 5011.40539170
+3 35 s35/6 4653.00397593
+3 35 s35/8 4710.81075825
+3 35 s35/10 4852.58271439
+3 37 s37/1 4903.12604366
+3 37 s37/3 5129.18619276
+3 37 s37/6 5481.63652571
+3 37 s37/8 5138.48537995
+3 37 s37/10 4825.35967571
+3 38 s38/1 4761.21560108
+3 38 s38/3 4717.32074381
+3 38 s38/6 5068.80518466
+3 38 s38/8 4452.82314942
+3 38 s38/10 4629.55645824
+3 40 s40/1 4335.91339858
+3 40 s40/3 4126.24441835
+3 40 s40/6 4808.05802794
+3 40 s40/8 4465.80082404
+3 40 s40/10 4990.20680934
+4 3 s3/1 4048.09646624
+4 3 s3/3 4013.26064441
+4 3 s3/6 4343.19720943
+4 3 s3/8 4416.68880498
+4 3 s3/10 3859.08771603
+4 4 s4/1 3663.81795399
+4 4 s4/3 3939.19852254
+4 4 s4/6 4509.90698352
+4 4 s4/8 3718.33591812
+4 4 s4/10 3739.79063585
+4 7 s7/1 5419.34424077
+4 7 s7/3 5390.47038764
+4 7 s7/6 5253.95251216
+4 7 s7/8 5610.14099645
+4 7 s7/10 5107.09839341
+4 8 s8/1 5748.42613243
+4 8 s8/3 5124.25457994
+4 8 s8/6 5257.00694312
+4 8 s8/8 5125.55977431
+4 8 s8/10 4738.50155640
+4 9 s9/1 4320.94422551
+4 9 s9/3 4211.65098269
+4 9 s9/6 4782.23316036
+4 9 s9/8 4403.20190316
+4 9 s9/10 4257.52604220
+4 13 s13/1 5403.17230893
+4 13 s13/3 5922.15467545
+4 13 s13/6 4995.89291318
+4 13 s13/8 5472.91065156
+4 13 s13/10 5106.55686348
+4 15 s15/1 4807.36123045
+4 15 s15/3 4714.66075132
+4 15 s15/6 4930.23386869
+4 15 s15/8 4708.58683683
+4 15 s15/10 4569.94945267
+4 18 s18/1 4815.76743625
+4 18 s18/3 5143.28912273
+4 18 s18/6 4965.95569855
+4 18 s18/8 5101.29150314
+4 18 s18/10 5146.67873487
+4 19 s19/1 5491.25459253
+4 19 s19/3 6007.58362405
+4 19 s19/6 5856.52653029
+4 19 s19/8 6048.22916233
+4 19 s19/10 5396.44503354
+4 22 s22/1 5245.79889054
+4 22 s22/3 4933.86298959
+4 22 s22/6 5691.65274766
+4 22 s22/8 5327.80151657
+4 22 s22/10 5727.51316018
+4 23 s23/1 4874.57300694
+4 23 s23/3 5294.26397151
+4 23 s23/6 5224.94813371
+4 23 s23/8 4971.11707768
+4 23 s23/10 4852.05657428
+4 25 s25/1 3762.09542675
+4 25 s25/3 3719.78319261
+4 25 s25/6 3781.26486774
+4 25 s25/8 4939.84453197
+4 25 s25/10 3605.58469600
+4 28 s28/1 5844.08281940
+4 28 s28/3 5627.97148180
+4 28 s28/6 5601.64627587
+4 28 s28/8 5522.59359359
+4 28 s28/10 5857.10098940
+4 30 s30/1 4225.57132232
+4 30 s30/3 4680.01730766
+4 30 s30/6 4409.06928954
+4 30 s30/8 4895.47188737
+4 30 s30/10 4535.89340704
+4 31 s31/1 4748.92840544
+4 31 s31/3 5789.32837210
+4 31 s31/6 4513.52567291
+4 31 s31/8 4474.08918105
+4 31 s31/10 5778.18622061
+4 32 s32/1 5594.66540554
+4 32 s32/3 5710.45173344
+4 32 s32/6 5333.17250799
+4 32 s32/8 5545.33849282
+4 32 s32/10 5254.69066644
+4 35 s35/1 5660.85417583
+4 35 s35/3 5234.81269961
+4 35 s35/6 4290.81484103
+4 35 s35/8 4427.54559547
+4 35 s35/10 5426.33439810
+4 37 s37/1 5528.29928640
+4 37 s37/3 5818.64855443
+4 37 s37/6 5866.06674016
+4 37 s37/8 5512.79021912
+4 37 s37/10 5400.01351850
+4 38 s38/1 5091.32909956
+4 38 s38/3 4797.06295560
+4 38 s38/6 5422.96302772
+4 38 s38/8 4731.86876403
+4 38 s38/10 4986.97413268
+4 40 s40/1 4799.25900530
+4 40 s40/3 4619.81828647
+4 40 s40/6 5061.94112965
+4 40 s40/8 4778.30963836
+4 40 s40/10 5421.03790800
+7 3 s3/1 4547.50711929
+7 3 s3/3 4660.79574751
+7 3 s3/6 4755.76912812
+7 3 s3/8 4948.60748090
+7 3 s3/10 4638.28502790
+7 4 s4/1 5136.11662640
+7 4 s4/3 5368.81541869
+7 4 s4/6 5500.63350897
+7 4 s4/8 5367.84016155
+7 4 s4/10 5075.31141902
+7 7 s7/1 3316.07539118
+7 7 s7/3 3414.17940360
+7 7 s7/6 3157.47478216
+7 7 s7/8 4291.28768553
+7 7 s7/10 3342.47902013
+7 8 s8/1 5727.08852734
+7 8 s8/3 5479.08103609
+7 8 s8/6 5569.16259414
+7 8 s8/8 5821.02542513
+7 8 s8/10 5475.16383317
+7 9 s9/1 5111.03971810
+7 9 s9/3 5141.62250656
+7 9 s9/6 5383.78119912
+7 9 s9/8 4989.69167384
+7 9 s9/10 5194.18251508
+7 13 s13/1 6316.17534589
+7 13 s13/3 6391.66112994
+7 13 s13/6 6038.34679362
+7 13 s13/8 6115.59522859
+7 13 s13/10 5880.47557601
+7 15 s15/1 4604.60790948
+7 15 s15/3 4527.70272876
+7 15 s15/6 4738.72366782
+7 15 s15/8 4585.57215623
+7 15 s15/10 4804.06182308
+7 18 s18/1 5853.43745162
+7 18 s18/3 6074.09260713
+7 18 s18/6 5692.91190868
+7 18 s18/8 5984.31758850
+7 18 s18/10 5876.17715866
+7 19 s19/1 5835.07746307
+7 19 s19/3 6287.45393621
+7 19 s19/6 6265.14445165
+7 19 s19/8 5662.23860324
+7 19 s19/10 5866.40733328
+7 22 s22/1 5296.87341740
+7 22 s22/3 5201.58398952
+7 22 s22/6 5329.34996036
+7 22 s22/8 5314.47560913
+7 22 s22/10 5495.95387535
+7 23 s23/1 4595.53914138
+7 23 s23/3 5005.64051846
+7 23 s23/6 5085.30028612
+7 23 s23/8 4573.53637790
+7 23 s23/10 4564.13989707
+7 25 s25/1 5115.93666888
+7 25 s25/3 5193.18813447
+7 25 s25/6 5357.14625524
+7 25 s25/8 5795.12829884
+7 25 s25/10 5287.69259697
+7 28 s28/1 6043.13478254
+7 28 s28/3 6044.56921542
+7 28 s28/6 5902.28709908
+7 28 s28/8 5906.41329404
+7 28 s28/10 6408.48125534
+7 30 s30/1 4732.13049271
+7 30 s30/3 4924.21465820
+7 30 s30/6 4645.33357252
+7 30 s30/8 5226.90376801
+7 30 s30/10 4882.50335381
+7 31 s31/1 5003.19308042
+7 31 s31/3 6028.70442135
+7 31 s31/6 5252.74594855
+7 31 s31/8 5145.27880683
+7 31 s31/10 6122.78596719
+7 32 s32/1 5824.69887634
+7 32 s32/3 6193.41319468
+7 32 s32/6 5234.42652064
+7 32 s32/8 5742.66654090
+7 32 s32/10 5022.67637819
+7 35 s35/1 5575.19093843
+7 35 s35/3 5502.22991159
+7 35 s35/6 5506.26025538
+7 35 s35/8 5257.81684732
+7 35 s35/10 5293.56751161
+7 37 s37/1 5802.33340993
+7 37 s37/3 5975.79526088
+7 37 s37/6 5675.25937733
+7 37 s37/8 5383.19756279
+7 37 s37/10 5736.34517790
+7 38 s38/1 4969.65451515
+7 38 s38/3 4973.19967425
+7 38 s38/6 5186.53583811
+7 38 s38/8 4689.37735739
+7 38 s38/10 4661.73776611
+7 40 s40/1 5457.86286013
+7 40 s40/3 5212.55705005
+7 40 s40/6 5449.14488704
+7 40 s40/8 5415.74436250
+7 40 s40/10 5810.16798380
+8 3 s3/1 5463.49613343
+8 3 s3/3 5881.26534004
+8 3 s3/6 5068.93312246
+8 3 s3/8 5019.59171646
+8 3 s3/10 5918.82800899
+8 4 s4/1 5071.55962205
+8 4 s4/3 5153.68373108
+8 4 s4/6 5405.66240159
+8 4 s4/8 5277.98001133
+8 4 s4/10 6346.43151700
+8 7 s7/1 5859.09813879
+8 7 s7/3 5646.09103717
+8 7 s7/6 5264.26462101
+8 7 s7/8 5381.47962925
+8 7 s7/10 6258.01637901
+8 8 s8/1 1852.38764841
+8 8 s8/3 3994.23409429
+8 8 s8/6 3822.74103230
+8 8 s8/8 4274.80198372
+8 8 s8/10 4440.59005088
+8 9 s9/1 6007.37396872
+8 9 s9/3 6057.28709902
+8 9 s9/6 5248.83768086
+8 9 s9/8 5166.13627385
+8 9 s9/10 6396.69844529
+8 13 s13/1 6159.57644648
+8 13 s13/3 6074.79110752
+8 13 s13/6 6276.45831660
+8 13 s13/8 6090.79223090
+8 13 s13/10 5875.34271341
+8 15 s15/1 5400.88353883
+8 15 s15/3 5170.35153544
+8 15 s15/6 5323.37872032
+8 15 s15/8 5245.38301747
+8 15 s15/10 4583.99552792
+8 18 s18/1 6689.77480936
+8 18 s18/3 5701.92932261
+8 18 s18/6 5613.40725407
+8 18 s18/8 5776.63517976
+8 18 s18/10 5715.05686761
+8 19 s19/1 5826.78487676
+8 19 s19/3 6374.41918923
+8 19 s19/6 6007.45370353
+8 19 s19/8 5348.66067348
+8 19 s19/10 5454.01338466
+8 22 s22/1 5757.47974378
+8 22 s22/3 5770.25987283
+8 22 s22/6 5842.24580106
+8 22 s22/8 5792.68953078
+8 22 s22/10 5981.31707904
+8 23 s23/1 4825.51603458
+8 23 s23/3 5841.88462741
+8 23 s23/6 5424.27506677
+8 23 s23/8 6128.56084248
+8 23 s23/10 4807.79034485
+8 25 s25/1 5491.90231159
+8 25 s25/3 5627.62383249
+8 25 s25/6 5394.70749902
+8 25 s25/8 5459.25406993
+8 25 s25/10 5709.54464034
+8 28 s28/1 5870.09889184
+8 28 s28/3 5860.05238884
+8 28 s28/6 5850.76165982
+8 28 s28/8 6876.91929573
+8 28 s28/10 6164.75263088
+8 30 s30/1 5418.36377516
+8 30 s30/3 5544.38734217
+8 30 s30/6 5324.33047434
+8 30 s30/8 5818.69023063
+8 30 s30/10 5235.46062921
+8 31 s31/1 6190.44667209
+8 31 s31/3 6542.20467427
+8 31 s31/6 6131.17729315
+8 31 s31/8 6198.49731790
+8 31 s31/10 6500.07269190
+8 32 s32/1 5899.56083111
+8 32 s32/3 5802.84016668
+8 32 s32/6 5224.61405273
+8 32 s32/8 5153.73806086
+8 32 s32/10 5097.85651034
+8 35 s35/1 6150.31096775
+8 35 s35/3 5552.10887862
+8 35 s35/6 5540.33979102
+8 35 s35/8 5469.50482219
+8 35 s35/10 5978.10588732
+8 37 s37/1 6404.46063303
+8 37 s37/3 6460.87331558
+8 37 s37/6 6298.55348473
+8 37 s37/8 5952.06325571
+8 37 s37/10 6151.83509207
+8 38 s38/1 4914.14051488
+8 38 s38/3 4956.73138268
+8 38 s38/6 5137.97148688
+8 38 s38/8 5488.61612795
+8 38 s38/10 5227.97723790
+8 40 s40/1 6227.74951327
+8 40 s40/3 6257.40553265
+8 40 s40/6 5641.62928594
+8 40 s40/8 5820.00085911
+8 40 s40/10 5409.03605091
+9 3 s3/1 4006.14490502
+9 3 s3/3 4387.83944556
+9 3 s3/6 4301.73034952
+9 3 s3/8 4493.85246754
+9 3 s3/10 4583.63065702
+9 4 s4/1 4054.62156064
+9 4 s4/3 4377.70727665
+9 4 s4/6 4959.40813001
+9 4 s4/8 4393.64677688
+9 4 s4/10 4487.09393706
+9 7 s7/1 5112.41352005
+9 7 s7/3 5196.95516625
+9 7 s7/6 5077.56802022
+9 7 s7/8 5157.37278079
+9 7 s7/10 5130.30583884
+9 8 s8/1 5640.50113022
+9 8 s8/3 5288.17463781
+9 8 s8/6 5635.84226181
+9 8 s8/8 5445.72061347
+9 8 s8/10 5134.69853059
+9 9 s9/1 2838.15485835
+9 9 s9/3 2413.77049448
+9 9 s9/6 3753.16666297
+9 9 s9/8 3283.64660096
+9 9 s9/10 3463.86229518
+9 13 s13/1 5674.80228730
+9 13 s13/3 6270.98748205
+9 13 s13/6 5550.79111479
+9 13 s13/8 5821.60484746
+9 13 s13/10 5526.70453344
+9 15 s15/1 5036.25773765
+9 15 s15/3 4968.56599030
+9 15 s15/6 5149.03136522
+9 15 s15/8 5130.54597485
+9 15 s15/10 4310.29581351
+9 18 s18/1 5724.79397009
+9 18 s18/3 5489.02459459
+9 18 s18/6 5572.64174337
+9 18 s18/8 5479.59241185
+9 18 s18/10 5290.66819976
+9 19 s19/1 5967.41007473
+9 19 s19/3 6530.60188650
+9 19 s19/6 6248.94671125
+9 19 s19/8 6398.89381065
+9 19 s19/10 5795.00327869
+9 22 s22/1 5271.74752810
+9 22 s22/3 4991.07182878
+9 22 s22/6 5471.72815480
+9 22 s22/8 5284.24952098
+9 22 s22/10 5667.61687131
+9 23 s23/1 4252.15004439
+9 23 s23/3 4053.49466510
+9 23 s23/6 4630.02343407
+9 23 s23/8 4296.81800871
+9 23 s23/10 4367.78055767
+9 25 s25/1 3636.89537930
+9 25 s25/3 3674.33408933
+9 25 s25/6 3974.29364793
+9 25 s25/8 4996.09307359
+9 25 s25/10 3657.56763437
+9 28 s28/1 6113.99787373
+9 28 s28/3 5873.92654023
+9 28 s28/6 5861.30659495
+9 28 s28/8 6017.28526829
+9 28 s28/10 6323.63170971
+9 30 s30/1 3759.00957700
+9 30 s30/3 3848.59896586
+9 30 s30/6 3892.35507116
+9 30 s30/8 4410.52049083
+9 30 s30/10 3917.35229460
+9 31 s31/1 3981.03340855
+9 31 s31/3 6088.26798030
+9 31 s31/6 4571.48116041
+9 31 s31/8 4160.56101986
+9 31 s31/10 6126.35242212
+9 32 s32/1 6294.10366931
+9 32 s32/3 6339.36029896
+9 32 s32/6 5783.45787570
+9 32 s32/8 6037.37434652
+9 32 s32/10 5697.96454885
+9 35 s35/1 4651.39419959
+9 35 s35/3 5589.54846119
+9 35 s35/6 4659.66565324
+9 35 s35/8 4223.84564112
+9 35 s35/10 5544.75544997
+9 37 s37/1 5867.44484422
+9 37 s37/3 6082.60380100
+9 37 s37/6 6184.75795808
+9 37 s37/8 5896.29069840
+9 37 s37/10 5932.32719934
+9 38 s38/1 4274.81110694
+9 38 s38/3 4081.14162950
+9 38 s38/6 4447.71087190
+9 38 s38/8 3844.63652378
+9 38 s38/10 3853.71029010
+9 40 s40/1 4460.74309953
+9 40 s40/3 4656.37165613
+9 40 s40/6 3937.79506831
+9 40 s40/8 3995.57317540
+9 40 s40/10 4883.37260508
+13 3 s3/1 4875.47269503
+13 3 s3/3 5175.63348780
+13 3 s3/6 5189.72744949
+13 3 s3/8 5119.46178812
+13 3 s3/10 5470.63917655
+13 4 s4/1 5216.47898491
+13 4 s4/3 5192.14926596
+13 4 s4/6 5333.25435358
+13 4 s4/8 5210.71597768
+13 4 s4/10 6048.25503761
+13 7 s7/1 6587.06770878
+13 7 s7/3 6534.96212690
+13 7 s7/6 6282.49440907
+13 7 s7/8 6506.60026435
+13 7 s7/10 6253.85920852
+13 8 s8/1 6290.95954525
+13 8 s8/3 6490.43203493
+13 8 s8/6 6417.94741331
+13 8 s8/8 6424.16656073
+13 8 s8/10 6541.94420643
+13 9 s9/1 6091.12649680
+13 9 s9/3 5954.22631414
+13 9 s9/6 5906.95632285
+13 9 s9/8 5620.91682913
+13 9 s9/10 6197.66464081
+13 13 s13/1 2386.22924297
+13 13 s13/3 4036.88431838
+13 13 s13/6 3805.73974412
+13 13 s13/8 3726.97169294
+13 13 s13/10 3292.50056948
+13 15 s15/1 5778.09008237
+13 15 s15/3 5818.97061343
+13 15 s15/6 5604.24562274
+13 15 s15/8 6078.41517174
+13 15 s15/10 5094.60744317
+13 18 s18/1 5244.50026218
+13 18 s18/3 4298.43948428
+13 18 s18/6 4616.24598565
+13 18 s18/8 4163.33688284
+13 18 s18/10 4659.76962950
+13 19 s19/1 5379.57637737
+13 19 s19/3 5285.12667776
+13 19 s19/6 5095.12865392
+13 19 s19/8 5321.04604378
+13 19 s19/10 5252.35394847
+13 22 s22/1 6475.90233095
+13 22 s22/3 6438.99611741
+13 22 s22/6 6807.75557728
+13 22 s22/8 6542.09232585
+13 22 s22/10 6245.14419369
+13 23 s23/1 6101.10539165
+13 23 s23/3 6464.78073874
+13 23 s23/6 6006.06976316
+13 23 s23/8 6742.16330268
+13 23 s23/10 6073.20640189
+13 25 s25/1 4925.68360738
+13 25 s25/3 4809.54363739
+13 25 s25/6 4371.80866461
+13 25 s25/8 4588.55805237
+13 25 s25/10 4904.70855403
+13 28 s28/1 4598.39939544
+13 28 s28/3 4816.85270690
+13 28 s28/6 4735.73859076
+13 28 s28/8 5224.31593608
+13 28 s28/10 4514.05128460
+13 30 s30/1 6513.68958425
+13 30 s30/3 6698.41466319
+13 30 s30/6 6459.47157281
+13 30 s30/8 6623.22866886
+13 30 s30/10 6449.31453722
+13 31 s31/1 6602.70156830
+13 31 s31/3 6171.66606355
+13 31 s31/6 6069.84719742
+13 31 s31/8 6336.40221261
+13 31 s31/10 6271.01339498
+13 32 s32/1 5749.68799153
+13 32 s32/3 5380.34422691
+13 32 s32/6 5656.66615596
+13 32 s32/8 5366.91941434
+13 32 s32/10 5555.10710968
+13 35 s35/1 5223.20581635
+13 35 s35/3 5128.17950154
+13 35 s35/6 4914.26525536
+13 35 s35/8 5140.09036886
+13 35 s35/10 5529.25673124
+13 37 s37/1 4678.40507011
+13 37 s37/3 4580.72723484
+13 37 s37/6 5304.15026182
+13 37 s37/8 5209.55190012
+13 37 s37/10 4587.82508385
+13 38 s38/1 6140.76143487
+13 38 s38/3 5685.85209094
+13 38 s38/6 6347.02331806
+13 38 s38/8 6328.72317296
+13 38 s38/10 5928.37549418
+13 40 s40/1 4675.91552533
+13 40 s40/3 5250.29789631
+13 40 s40/6 4512.21331499
+13 40 s40/8 4465.36583048
+13 40 s40/10 4420.14332347
+15 3 s3/1 4228.92066608
+15 3 s3/3 4530.28718736
+15 3 s3/6 4320.43389025
+15 3 s3/8 4405.13711478
+15 3 s3/10 4711.76601711
+15 4 s4/1 4065.27994116
+15 4 s4/3 4260.79476154
+15 4 s4/6 4645.01474702
+15 4 s4/8 4323.28960399
+15 4 s4/10 4883.38827045
+15 7 s7/1 4928.63682979
+15 7 s7/3 4791.71493309
+15 7 s7/6 4558.92070561
+15 7 s7/8 4682.20962794
+15 7 s7/10 4920.22001541
+15 8 s8/1 4603.31576149
+15 8 s8/3 5154.69184336
+15 8 s8/6 5367.86056078
+15 8 s8/8 5180.61367021
+15 8 s8/10 5243.76086411
+15 9 s9/1 4571.56734611
+15 9 s9/3 4563.98389568
+15 9 s9/6 4231.21483737
+15 9 s9/8 3941.90157158
+15 9 s9/10 4696.34719756
+15 13 s13/1 4933.64996732
+15 13 s13/3 4965.15508318
+15 13 s13/6 5004.52924859
+15 13 s13/8 4907.69783096
+15 13 s13/10 4744.68692329
+15 15 s15/1 4504.51007325
+15 15 s15/3 4390.45179907
+15 15 s15/6 4465.37602000
+15 15 s15/8 4124.67295673
+15 15 s15/10 2118.67198971
+15 18 s18/1 5237.39524955
+15 18 s18/3 4283.67482426
+15 18 s18/6 4387.91784335
+15 18 s18/8 4264.28845178
+15 18 s18/10 4465.66736334
+15 19 s19/1 5428.66852921
+15 19 s19/3 5775.01099566
+15 19 s19/6 5652.10898692
+15 19 s19/8 5617.01673489
+15 19 s19/10 5313.35769547
+15 22 s22/1 4867.23648491
+15 22 s22/3 4738.44710849
+15 22 s22/6 4905.17400303
+15 22 s22/8 4904.40577440
+15 22 s22/10 5055.64951317
+15 23 s23/1 4012.45847331
+15 23 s23/3 4484.76286999
+15 23 s23/6 4090.97347828
+15 23 s23/8 4708.44093092
+15 23 s23/10 4117.70761468
+15 25 s25/1 3957.65650354
+15 25 s25/3 4192.03315827
+15 25 s25/6 4149.18534173
+15 25 s25/8 4419.11744583
+15 25 s25/10 4332.19597895
+15 28 s28/1 5295.93079638
+15 28 s28/3 5275.72592161
+15 28 s28/6 5183.53007129
+15 28 s28/8 5774.26488828
+15 28 s28/10 5675.51891901
+15 30 s30/1 3930.40201506
+15 30 s30/3 4158.61166737
+15 30 s30/6 3792.90851458
+15 30 s30/8 4273.64294250
+15 30 s30/10 3879.40227355
+15 31 s31/1 4393.72598144
+15 31 s31/3 5171.52956097
+15 31 s31/6 4469.53834305
+15 31 s31/8 4449.02832088
+15 31 s31/10 5285.37501035
+15 32 s32/1 5394.18316337
+15 32 s32/3 5229.42884071
+15 32 s32/6 5049.35481027
+15 32 s32/8 5054.60502908
+15 32 s32/10 4940.17054361
+15 35 s35/1 4182.12111255
+15 35 s35/3 4030.92408760
+15 35 s35/6 4407.73309083
+15 35 s35/8 4004.81997098
+15 35 s35/10 4287.09108837
+15 37 s37/1 5238.54617237
+15 37 s37/3 5310.70183686
+15 37 s37/6 5557.32525591
+15 37 s37/8 5297.47496455
+15 37 s37/10 5279.50272280
+15 38 s38/1 4314.32323777
+15 38 s38/3 4234.71156043
+15 38 s38/6 4450.34841333
+15 38 s38/8 4454.78832269
+15 38 s38/10 4318.88689363
+15 40 s40/1 4474.80904621
+15 40 s40/3 4447.91299375
+15 40 s40/6 4273.03697620
+15 40 s40/8 4366.64722642
+15 40 s40/10 4031.56706505
+18 3 s3/1 4594.38211297
+18 3 s3/3 4215.23961359
+18 3 s3/6 4930.28579293
+18 3 s3/8 4990.70295650
+18 3 s3/10 4366.12047475
+18 4 s4/1 5143.72063783
+18 4 s4/3 5253.04702054
+18 4 s4/6 4959.44140000
+18 4 s4/8 5090.89579544
+18 4 s4/10 4542.34124654
+18 7 s7/1 5436.85626074
+18 7 s7/3 5439.10792318
+18 7 s7/6 5543.79139218
+18 7 s7/8 5784.41976347
+18 7 s7/10 5017.41586875
+18 8 s8/1 6295.85903591
+18 8 s8/3 5832.93828186
+18 8 s8/6 5843.73185559
+18 8 s8/8 5806.10824908
+18 8 s8/10 5544.32042725
+18 9 s9/1 5620.60646194
+18 9 s9/3 5479.25925651
+18 9 s9/6 5837.53937888
+18 9 s9/8 5328.36888738
+18 9 s9/10 4845.51070580
+18 13 s13/1 4552.54906618
+18 13 s13/3 4647.58517942
+18 13 s13/6 3629.54432402
+18 13 s13/8 4393.25881323
+18 13 s13/10 4165.65637085
+18 15 s15/1 5366.14871207
+18 15 s15/3 5413.41149369
+18 15 s15/6 5348.41471840
+18 15 s15/8 5522.04237579
+18 15 s15/10 4951.93820640
+18 18 s18/1 2543.34110964
+18 18 s18/3 4006.31264382
+18 18 s18/6 3804.21949945
+18 18 s18/8 3875.04361266
+18 18 s18/10 4056.80736540
+18 19 s19/1 5578.93385872
+18 19 s19/3 5566.08111691
+18 19 s19/6 5810.01815832
+18 19 s19/8 5569.47430194
+18 19 s19/10 5722.38822521
+18 22 s22/1 6720.29701725
+18 22 s22/3 6562.26515770
+18 22 s22/6 7174.90522585
+18 22 s22/8 6803.90439380
+18 22 s22/10 6886.72251510
+18 23 s23/1 5680.87352438
+18 23 s23/3 5756.95692185
+18 23 s23/6 5822.56240842
+18 23 s23/8 5274.72075090
+18 23 s23/10 5706.59960046
+18 25 s25/1 4326.58641425
+18 25 s25/3 4177.47304001
+18 25 s25/6 4218.27405463
+18 25 s25/8 4917.27058438
+18 25 s25/10 4173.23891001
+18 28 s28/1 5424.89096665
+18 28 s28/3 5544.76013909
+18 28 s28/6 5405.19537112
+18 28 s28/8 5077.34497548
+18 28 s28/10 5624.08036927
+18 30 s30/1 5815.62490193
+18 30 s30/3 6205.18267257
+18 30 s30/6 5822.27326738
+18 30 s30/8 6451.99449783
+18 30 s30/10 6070.63530448
+18 31 s31/1 6275.69605701
+18 31 s31/3 6042.07257487
+18 31 s31/6 5890.94949902
+18 31 s31/8 6053.47437427
+18 31 s31/10 6009.36053170
+18 32 s32/1 4913.63236313
+18 32 s32/3 5165.70914783
+18 32 s32/6 5352.50586174
+18 32 s32/8 5252.33957394
+18 32 s32/10 5494.55293905
+18 35 s35/1 5627.57709854
+18 35 s35/3 5230.80758583
+18 35 s35/6 4395.33752970
+18 35 s35/8 4606.89439862
+18 35 s35/10 5066.11261225
+18 37 s37/1 5048.76826563
+18 37 s37/3 5223.17709828
+18 37 s37/6 6118.06627947
+18 37 s37/8 5972.22990180
+18 37 s37/10 4925.60777976
+18 38 s38/1 6129.36962501
+18 38 s38/3 6147.37952302
+18 38 s38/6 6356.92677321
+18 38 s38/8 5719.25711959
+18 38 s38/10 5990.98581203
+18 40 s40/1 4017.45528911
+18 40 s40/3 3535.49586338
+18 40 s40/6 4584.68384951
+18 40 s40/8 4237.69406635
+18 40 s40/10 5070.39544809
+19 3 s3/1 5437.44250544
+19 3 s3/3 5637.11247005
+19 3 s3/6 5426.82024762
+19 3 s3/8 5550.44718919
+19 3 s3/10 5592.01287552
+19 4 s4/1 5603.08379377
+19 4 s4/3 5164.17457102
+19 4 s4/6 4877.38833804
+19 4 s4/8 5135.89894760
+19 4 s4/10 5874.75139900
+19 7 s7/1 6400.83525800
+19 7 s7/3 6287.73830562
+19 7 s7/6 6011.11445574
+19 7 s7/8 6502.39755782
+19 7 s7/10 6050.27751430
+19 8 s8/1 6176.81009907
+19 8 s8/3 5889.87122100
+19 8 s8/6 5770.51141581
+19 8 s8/8 5552.83504167
+19 8 s8/10 5941.51925016
+19 9 s9/1 6235.32845967
+19 9 s9/3 6046.01124710
+19 9 s9/6 6116.19865603
+19 9 s9/8 6184.09613444
+19 9 s9/10 6349.76251524
+19 13 s13/1 5030.71992860
+19 13 s13/3 5174.08040139
+19 13 s13/6 5247.83917437
+19 13 s13/8 4917.32478081
+19 13 s13/10 5122.65722062
+19 15 s15/1 4828.64867225
+19 15 s15/3 4710.05753680
+19 15 s15/6 4585.96511108
+19 15 s15/8 5149.14983274
+19 15 s15/10 5472.39856005
+19 18 s18/1 5644.58590155
+19 18 s18/3 5208.63331403
+19 18 s18/6 5172.94268285
+19 18 s18/8 5087.84129076
+19 18 s18/10 5589.33019243
+19 19 s19/1 1862.67146862
+19 19 s19/3 2492.18839577
+19 19 s19/6 3493.54962753
+19 19 s19/8 4407.45119088
+19 19 s19/10 2716.94884015
+19 22 s22/1 5779.88079462
+19 22 s22/3 5901.72178267
+19 22 s22/6 6407.55827129
+19 22 s22/8 6064.95136007
+19 22 s22/10 5762.56114935
+19 23 s23/1 6524.63056425
+19 23 s23/3 6734.59360318
+19 23 s23/6 6394.09235154
+19 23 s23/8 6813.16527027
+19 23 s23/10 6305.76030309
+19 25 s25/1 5532.19558584
+19 25 s25/3 5242.00314765
+19 25 s25/6 5110.60250851
+19 25 s25/8 5054.46634176
+19 25 s25/10 5270.09990418
+19 28 s28/1 5514.16884036
+19 28 s28/3 5503.89053307
+19 28 s28/6 5656.67614417
+19 28 s28/8 5925.65557555
+19 28 s28/10 5199.28187349
+19 30 s30/1 6222.57028888
+19 30 s30/3 6597.99484692
+19 30 s30/6 6233.52147666
+19 30 s30/8 6350.19913074
+19 30 s30/10 6153.12709116
+19 31 s31/1 6605.90364750
+19 31 s31/3 5066.93664851
+19 31 s31/6 5662.34862932
+19 31 s31/8 6199.01476043
+19 31 s31/10 5419.78763422
+19 32 s32/1 5226.51805699
+19 32 s32/3 4426.54278190
+19 32 s32/6 4975.45867232
+19 32 s32/8 4592.05607544
+19 32 s32/10 4851.39031619
+19 35 s35/1 6307.73588540
+19 35 s35/3 5181.67752760
+19 35 s35/6 4945.13316302
+19 35 s35/8 5326.90228932
+19 35 s35/10 5892.27774294
+19 37 s37/1 5902.10106657
+19 37 s37/3 5971.60707013
+19 37 s37/6 5252.80563128
+19 37 s37/8 5191.39961860
+19 37 s37/10 5447.09170108
+19 38 s38/1 6670.34286975
+19 38 s38/3 6382.99514335
+19 38 s38/6 6887.15122529
+19 38 s38/8 6741.52697836
+19 38 s38/10 6577.49412771
+19 40 s40/1 5899.44565192
+19 40 s40/3 6033.55077877
+19 40 s40/6 5899.71100987
+19 40 s40/8 5826.40515241
+19 40 s40/10 5833.24798033
+22 3 s3/1 5190.25847141
+22 3 s3/3 5495.19162541
+22 3 s3/6 5738.48969678
+22 3 s3/8 5687.55923046
+22 3 s3/10 5833.38958068
+22 4 s4/1 5294.55040584
+22 4 s4/3 5084.08762710
+22 4 s4/6 5821.78726853
+22 4 s4/8 5183.04119220
+22 4 s4/10 6128.90740671
+22 7 s7/1 6004.17221605
+22 7 s7/3 6141.33658091
+22 7 s7/6 5805.96029955
+22 7 s7/8 6031.76872899
+22 7 s7/10 6202.76341641
+22 8 s8/1 6013.51818822
+22 8 s8/3 6598.93938448
+22 8 s8/6 6597.09390565
+22 8 s8/8 6599.14456578
+22 8 s8/10 6582.71349825
+22 9 s9/1 5359.37057872
+22 9 s9/3 5345.70107657
+22 9 s9/6 5483.33292806
+22 9 s9/8 5705.84866606
+22 9 s9/10 5937.90771232
+22 13 s13/1 5987.98388441
+22 13 s13/3 6209.33200916
+22 13 s13/6 6389.19948037
+22 13 s13/8 6450.61214149
+22 13 s13/10 6200.28942873
+22 15 s15/1 4921.48900232
+22 15 s15/3 4755.85512816
+22 15 s15/6 4946.88993207
+22 15 s15/8 4666.23167020
+22 15 s15/10 4737.21183820
+22 18 s18/1 7048.69335409
+22 18 s18/3 6430.18187923
+22 18 s18/6 6584.47811144
+22 18 s18/8 6469.92681566
+22 18 s18/10 6792.34363088
+22 19 s19/1 5481.64072154
+22 19 s19/3 5980.19222099
+22 19 s19/6 5561.52488082
+22 19 s19/8 5979.88662100
+22 19 s19/10 5188.72922786
+22 22 s22/1 2617.90183162
+22 22 s22/3 3123.85467012
+22 22 s22/6 2985.79620872
+22 22 s22/8 3121.59975013
+22 22 s22/10 2624.46013496
+22 23 s23/1 5934.79317247
+22 23 s23/3 5606.03826245
+22 23 s23/6 5358.47879533
+22 23 s23/8 6153.82848315
+22 23 s23/10 5766.26161391
+22 25 s25/1 5809.43233027
+22 25 s25/3 5884.17326393
+22 25 s25/6 5573.15673564
+22 25 s25/8 5634.92076253
+22 25 s25/10 5882.77307058
+22 28 s28/1 5634.35994590
+22 28 s28/3 5463.86227865
+22 28 s28/6 5518.26195464
+22 28 s28/8 5917.94153401
+22 28 s28/10 5803.91367958
+22 30 s30/1 4735.89410777
+22 30 s30/3 4964.82789228
+22 30 s30/6 4784.31290783
+22 30 s30/8 4338.30946337
+22 30 s30/10 4636.49512024
+22 31 s31/1 5058.46943255
+22 31 s31/3 4905.78056990
+22 31 s31/6 4704.30249878
+22 31 s31/8 4880.73150255
+22 31 s31/10 5016.54622225
+22 32 s32/1 6676.87254633
+22 32 s32/3 6270.33611539
+22 32 s32/6 5991.93215916
+22 32 s32/8 6046.71456247
+22 32 s32/10 5695.67010983
+22 35 s35/1 5765.74678598
+22 35 s35/3 4913.02350900
+22 35 s35/6 5770.18491905
+22 35 s35/8 5794.17863032
+22 35 s35/10 6165.04914822
+22 37 s37/1 5805.67300147
+22 37 s37/3 5861.86156438
+22 37 s37/6 5003.12192536
+22 37 s37/8 4587.70792444
+22 37 s37/10 5776.03566471
+22 38 s38/1 5643.43565570
+22 38 s38/3 5201.67078928
+22 38 s38/6 5812.37232118
+22 38 s38/8 5819.60084542
+22 38 s38/10 5310.76312784
+22 40 s40/1 5997.00433550
+22 40 s40/3 6416.15079312
+22 40 s40/6 5991.85196746
+22 40 s40/8 6015.42442393
+22 40 s40/10 5754.37868062

bob/bio/base/test/test_scripts.py

@@ -462,10 +462,10 @@ def test_fusion():
 
 
 
-def test_evaluate():
+def test_evaluate_closedset():
   # tests our 'evaluate' script using the reference files
   test_dir = tempfile.mkdtemp(prefix='bobtest_')
-  reference_files = ('scores-nonorm-dev', 'scores-ztnorm-dev')
+  reference_files = [os.path.join(data_dir, s) for s in ('scores-nonorm-dev', 'scores-ztnorm-dev')]
   plots = [os.path.join(test_dir, '%s.pdf')%f for f in ['roc', 'cmc', 'det', 'epc']]
   parameters = [
     '--dev-files', reference_files[0], reference_files[1],
@@ -495,6 +495,32 @@ def test_evaluate():
     if os.path.exists(test_dir):
       shutil.rmtree(test_dir)
 
+def test_evaluate_openset():
+  # tests our 'evaluate' script using the reference files
+  test_dir = tempfile.mkdtemp(prefix='bobtest_')
+  reference_file = os.path.join(data_dir, 'scores-nonorm-openset-dev')
+  plot = os.path.join(test_dir, 'dir.pdf')
+  parameters = [
+    '--dev-files', reference_file,
+    '--eval-files', reference_file,
+    '--directory', os.path.join(data_dir),
+    '--legends', 'Test',
+    '--dir', plot,
+    '--min-far-value', '1e-6',
+    '-v',
+  ]
+
+  # execute the script
+  from bob.bio.base.script.evaluate import main
+  try:
+    main(parameters)
+    assert os.path.exists(plot)
+    os.remove(plot)
+  finally:
+    if os.path.exists(test_dir):
+      shutil.rmtree(test_dir)
+
+
 
 
 def test_resources():

doc/experiments.rst

@@ -144,7 +144,7 @@ Evaluating Experiments
 ----------------------
 After the experiment has finished successfully, one or more text file containing all the scores are written.
 
-To evaluate the experiment, you can use the generic ``evaluate.py`` script, which has properties for all prevalent evaluation types, such as CMC, ROC and DET plots, as well as computing recognition rates, EER/HTER, Cllr and minDCF.
+To evaluate the experiment, you can use the generic ``evaluate.py`` script, which has properties for all prevalent evaluation types, such as CMC, DIR, ROC and DET plots, as well as computing recognition rates, EER/HTER, Cllr and minDCF.
 Additionally, a combination of different algorithms can be plotted into the same files.
 Just specify all the score files that you want to evaluate using the ``--dev-files`` option, and possible legends for the plots (in the same order) using the ``--legends`` option, and the according plots will be generated.
 For example, to create a ROC curve for the experiment above, use: