Commit 64b1ad66 authored by Pavel KORSHUNOV's avatar Pavel KORSHUNOV

cleaned up scripts and setup

parent 34040878
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Pavel Korshunov <pavel.korshunov@idiap.ch>
# Mon 7 Sep 15:19:22 CEST 2015
#
# Copyright (C) 2012-2015 Idiap Research Institute, Martigny, Switzerland
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the ipyplotied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from __future__ import print_function
"""This script evaluates the given score files and computes EER and Spoofing FAR with regards to 10 types of voice attacks"""
import bob.measure
import argparse
import numpy, math
import os
import os.path
import sys
import matplotlib.pyplot as mpl
import matplotlib.font_manager as fm
import bob.core
def load_attacks_file(filename, support="all", adevice="all", recdevice="all"):
# split in positives and negatives
positives = []
negatives = []
# read four column list line by line
for (client_id, probe_id, filename, score) in bob.measure.load.four_column(filename):
if client_id == probe_id:
if (support in filename or support == "all") and \
(adevice in filename or adevice == "all") and \
(recdevice in filename or recdevice == "all"):
positives.append(score)
else:
negatives.append(score)
return (numpy.array(negatives, numpy.float64), numpy.array(positives, numpy.float64))
def plot_det_curves(scores_real, scores_attack, thresholds, labels, title, outname):
from matplotlib.backends.backend_pdf import PdfPages
pdf_name = outname+'.pdf'
pp = PdfPages(pdf_name)
font_size = 16
fig = mpl.figure()
ax1 = mpl.subplot(111)
linestyles = ['-', '--', ':', '-.']
colors = ('b', 'g', 'r', 'c', 'm', 'y', 'k')
for i in range(len(scores_real)):
if thresholds is None:
eer_threshold = bob.measure.eer_threshold(scores_attack[i], scores_real[i])
else:
eer_threshold = float(thresholds[i])
far, frr = bob.measure.farfrr(scores_attack[i], scores_real[i], eer_threshold)
bob.measure.plot.det(scores_attack[i], scores_real[i], 100,
color=colors[i % len(colors)], linestyle=linestyles[i % len(linestyles)],
label='%s'%(labels[i]), linewidth=2)
# label='%s, HTER=%.2f%%'%(labels[i], (far+frr)*50), linewidth=2)
if far < 0.0001:
far_scaled = bob.measure.ppndf(far+0.001)
else:
far_scaled = bob.measure.ppndf(far)
if frr < 0.0001:
frr_scaled = bob.measure.ppndf(frr+0.001)
else:
frr_scaled = bob.measure.ppndf(frr)
ax1.plot(far_scaled, frr_scaled, color=colors[i % len(colors)], marker='o')
text_coords = (far_scaled+0.1, frr_scaled)
if i % 2:
text_coords = (far_scaled, frr_scaled+0.15)
if thresholds is None:
# str_name = "EER = %.2f%%"
str_name = "%.2f%%"
else:
# str_name = "HTER = %.2f%%"
str_name = "%.2f%%"
mpl.annotate(str_name % ((far+frr)*50), xy=(far_scaled, frr_scaled), xycoords='data', xytext=text_coords, color=colors[i % len(colors)], fontsize=font_size)
bob.measure.plot.det_axis([0.1, 99, 0.1, 99])
mpl.xlabel('FAR (%)', fontsize=font_size)
mpl.ylabel('FRR (%)', fontsize=font_size)
mpl.legend(loc='best')
# ax1.set_title(title, fontsize=12)
mpl.grid()
pp.savefig()
pp.close()
def command_line_arguments(command_line_parameters):
"""Parse the program options"""
basedir = os.path.dirname(os.path.dirname(os.path.realpath(sys.argv[0])))
OUTPUT_DIR = os.path.join(basedir, 'plots')
# set up command line parser
parser = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('-r', '--real-files', required=True, nargs = '+', help = "The first score file (or set) of the set.")
parser.add_argument('-a', '--attack-files', required=True, nargs = '+', help = "The second score file (or set) of the set.")
parser.add_argument('-l', '--labels', required=True, nargs = '+', help = "The name(s) of det curves corresponding to the score file (or set).")
parser.add_argument('-e', '--thresholds', required=False, nargs = '+', help = "The value(s) of thersholds when computing scores for Eval sets.")
parser.add_argument('-t', '--plot-title', required=False, type=str, default="all", help = "Title of the plot.")
parser.add_argument('-o', '--out-directory', dest="directory", default=OUTPUT_DIR, help="This path will be prepended to every file output by this procedure (defaults to '%(default)s')")
parser.add_argument('-s', '--support', required=False, type=str, default="all", help = "Type of attack.")
parser.add_argument('-k', '--attackdevice', required=False, type=str, default="all", help = "Attack device.")
# add verbose option
bob.core.log.add_command_line_option(parser)
# parse arguments
args = parser.parse_args(command_line_parameters)
return args
def main(command_line_parameters=None):
"""Reads score files, computes error measures and plots curves."""
args = command_line_arguments(command_line_parameters)
if not os.path.exists(args.directory):
os.makedirs(args.directory)
support=args.support
adevice=args.attackdevice
thresholds = None
if args.thresholds:
thresholds = args.thresholds
## Read scores ###
print("Loading real score files")
# take only positive values
scores_real = [bob.measure.load.split_four_column(real_file)[1] for real_file in args.real_files]
print("Loading attack score files")
scores_attack = [load_attacks_file(attack_file, support, adevice)[1] for attack_file in args.attack_files]
####################
## Plot DET ###
####################
if thresholds is None:
title = 'Dev set, attack "%s" and attack device "%s"' % (support, adevice)
outname = 'det_dev_attack_%s_adevice_%s' % (support, adevice)
else:
title = 'Test set, attack "%s" and attack device "%s"' % (support, adevice)
outname = 'det_test_attack_%s_adevice_%s' % (support, adevice)
outname = os.path.join(args.directory, outname)
plot_det_curves(scores_real, scores_attack, thresholds, args.labels, title, outname)
if __name__ == '__main__':
main()
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Pavel Korshunov <pavel.korshunov@idiap.ch>
# Mon 7 Sep 15:19:22 CEST 2015
#
# Copyright (C) 2012-2015 Idiap Research Institute, Martigny, Switzerland
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the ipyplotied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from __future__ import print_function
"""This script evaluates the given score files and computes EER and Spoofing FAR with regards to 10 types of voice attacks"""
import bob.measure
import argparse
import numpy, math
import os
import os.path
import sys
import matplotlib.pyplot as mpl
import matplotlib.font_manager as fm
import bob.core
# load and split scores in positives and negatives
def load_attack_scores(scores_filename, support="all", adevice="all", recdevice="all"):
positives = []
negatives = []
# read four column list line by line
for (client_id, probe_id, filename, score) in bob.measure.load.four_column(scores_filename):
if client_id == probe_id:
if (support in filename or support == "all") and \
(adevice in filename or adevice == "all") and \
(recdevice in filename or recdevice == "all"):
positives.append(score)
else:
negatives.append(score)
return numpy.array(negatives, numpy.float64), numpy.array(positives, numpy.float64)
def plot_det_curves(scores_dev, scores_eval, outname):
from matplotlib.backends.backend_pdf import PdfPages
pdf_name = outname+'_det_curves.pdf'
pp = PdfPages(pdf_name)
fig = mpl.figure()
ax1 = mpl.subplot(111)
bob.measure.plot.det(scores_dev[0], scores_dev[1], 100,
color='blue', linestyle='-', label='Dev set', linewidth=2)
bob.measure.plot.det(scores_eval[0], scores_eval[1], 100,
color='red', linestyle='--', label='Eval set', linewidth=2)
bob.measure.plot.det_axis([0.1, 99, 0.1, 99])
mpl.xlabel('FRR (%)')
mpl.ylabel('FAR (%)')
mpl.legend()
mpl.grid()
pp.savefig()
pp.close()
def plot_histograms(settype, scores, threshold, outname, histbins_real=100, histbins_attacks=100, attack_scores=None, impostor_scores=None, attacktype="all", realtype="all"):
from matplotlib.backends.backend_pdf import PdfPages
pdf_name = outname + '_distributions_' + settype + '.pdf'
pp = PdfPages(pdf_name)
negatives = attack_scores
positives = scores
fig = mpl.figure()
ax1 = mpl.subplot(111)
mpl.rcParams.update({'font.size': 18})
if attack_scores is not None:
mpl.hist(attack_scores, bins=histbins_attacks, color='black', alpha=0.4, label="Spoofing Attacks", normed=True)
# mpl.hist(impostor_scores, bins=histbins_real, color='red', alpha=0.8, label="Impostors", normed=True)
mpl.hist(scores, bins=histbins_real, color='blue', alpha=0.6, label="Genuine Accesses", normed=True)
# plot the line
mpl.axvline(x=threshold, ymin=0, ymax=1, linewidth=2, color='black', linestyle='--', label="EER threshold")
mpl.xlabel("Scores")
mpl.ylabel("Normalized Count")
# mpl.ylim([0, 0.10])
# mpl.xlim([-80, 40])
mpl.legend(loc='upper left', prop=fm.FontProperties(size=16))
mpl.title("Score distributions and EER, %s set" % (settype))
mpl.grid()
pp.savefig()
pp.close()
def command_line_arguments(command_line_parameters):
"""Parse the program options"""
basedir = os.path.dirname(os.path.dirname(os.path.realpath(sys.argv[0])))
OUTPUT_DIR = os.path.join(basedir, 'plots')
# set up command line parser
parser = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('-d', '--real-dev-file', required=True, help = "The score file of the development set.")
parser.add_argument('-e', '--real-eval-file', required=True, help = "The score files of the evaluation set.")
parser.add_argument('-f', '--attacks-eval-file', type=str, required=True, help = "The score file with attacks scores.")
parser.add_argument('-t', '--attacks-dev-file', type=str, required=True, help = "The score file with attacks scores.")
parser.add_argument('-n', '--histogram-bins-real', required=False, type=int, default=20, help = "The number of bins in computed histogram of score distribution for real data.")
parser.add_argument('-m', '--histogram-bins-attacks', required=False, type=int, default=20, help = "The number of bins in computed histogram of score distribution for spoof data.")
parser.add_argument('-o', '--out-directory', dest="directory", default=OUTPUT_DIR, help="This path will be prepended to every file output by this procedure (defaults to '%(default)s')")
parser.add_argument('-s', '--support', required=False, type=str, default="all", help = "Type of attack.")
parser.add_argument('-k', '--attackdevice', required=False, type=str, default="all", help = "Attack device.")
parser.add_argument('-r', '--device', required=False, type=str, default="all", help = "Recording device.")
parser.add_argument('-b', '--session', required=False, type=str, default="all", help = "Recording session.")
parser.add_argument('-p', '--pretty-title', required=False, type=str, help = "Title of the results. Default is the auto-generated file name.")
# add verbose option
bob.core.log.add_command_line_option(parser)
# parse arguments
args = parser.parse_args(command_line_parameters)
return args
def main(command_line_parameters=None):
"""Reads score files, computes error measures and plots curves."""
args = command_line_arguments(command_line_parameters)
if not os.path.exists(args.directory):
os.makedirs(args.directory)
histbins_real = int(args.histogram_bins_real)
histbins_attacks = int(args.histogram_bins_attacks)
####################
## Read scores ###
####################
print("Loading %s real score file of the development set" % (args.real_dev_file))
scores_dev_zimp, scores_dev = bob.measure.load.split_four_column(args.real_dev_file)
print("Loading %s real score file of the evaluation set" % (args.real_eval_file))
scores_eval_zimp, scores_eval = bob.measure.load.split_four_column(args.real_eval_file)
support=args.support
adevice=args.attackdevice
recdevice=args.device
attacktype = 'a:' + support+ ', ad:' + adevice # + ', rd:' + recdevice
outname = 'attack_%s_adevice_%s' % (support, adevice)
print("Loading %s score file of the development set with attacks" % (args.attacks_dev_file))
scores_dev_attacks = load_attack_scores(args.attacks_dev_file)[1] # only positive values
print("Loading %s score file of the evaluation set with attacks" % (args.attacks_eval_file))
scores_eval_attacks = load_attack_scores(args.attacks_eval_file, support, adevice, recdevice)[1]
####################
## Compute Stats ###
####################
resfile = open(os.path.join(args.directory, outname+'_results.txt'), "w")
# print the title of the experiment
results_title = outname
if args.pretty_title:
results_title = args.pretty_title
print (results_title)
resfile.write(results_title + "\n")
eer_threshold = bob.measure.eer_threshold(scores_dev_attacks, scores_dev)
far, frr = bob.measure.farfrr(scores_dev_attacks, scores_dev, eer_threshold)
print("Development set: FAR = %2.12f \t FRR = %2.12f \t HTER = %2.12f \t EER threshold = %2.12f" % (far, frr, (far+frr)/2, eer_threshold))
resfile.write("Development set: FAR = %2.12f \t FRR = %2.12f \t EER threshold = %2.12f\n" % (far, frr, eer_threshold))
sfar, sfrr = bob.measure.farfrr(scores_eval_attacks, scores_eval, eer_threshold)
print("Evaluation set with attacks: %s, SFAR = %2.12f, SFRR = %2.12f, HTER = %2.12f \t " % (attacktype, sfar, sfrr, (sfar+sfrr)/2))
resfile.write("Evaluation set with attacks: %s, SFAR = %2.12f, SFRR = %2.12f" % (attacktype, sfar, sfrr))
resfile.close()
####################
## Plot graphs ###
####################
outname = os.path.join(args.directory, outname)
plot_histograms("Dev", scores_dev, eer_threshold, outname, histbins_real, histbins_attacks, scores_dev_attacks, scores_dev_zimp, attacktype=attacktype)
plot_histograms("Eval", scores_eval, eer_threshold, outname, histbins_real, histbins_attacks, scores_eval_attacks, scores_eval_zimp, attacktype=attacktype)
plot_det_curves([scores_dev_attacks, scores_dev], [scores_eval_attacks, scores_eval], outname)
if __name__ == '__main__':
main()
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Pavel Korshunov <pavel.korshunov@idiap.ch>
# Mon 7 Sep 15:19:22 CEST 2015
#
# Copyright (C) 2012-2015 Idiap Research Institute, Martigny, Switzerland
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the ipyplotied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from __future__ import print_function
"""This script evaluates the given score files and computes EER and Spoofing FAR with regards to 10 types of voice attacks"""
import bob.measure
import argparse
import numpy, math
import os
import os.path
import sys
import re
import matplotlib.pyplot as mpl
import matplotlib.font_manager as fm
import bob.core
def command_line_arguments(command_line_parameters):
"""Parse the program options"""
basedir = os.path.dirname(os.path.dirname(os.path.realpath(sys.argv[0])))
OUTPUT_FILE = os.path.join(basedir, 'statistics.txt')
# set up command line parser
parser = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('-t', '--score-types', required=False, nargs = '+', default=[], help = "The type of the scores, can be a set of values.")
parser.add_argument('-p', '--parameters', required=False, nargs = '+', default=[], help = "The parameter for this type of the scores, can be a set of values.")
parser.add_argument('-d', '--directory', type=str, required=True, help = "The the directory prefix with txt files that have FAR and FRR scores.")
parser.add_argument('-o', '--out-file', type=str, default=OUTPUT_FILE, help = "The the directory to ouput the resulted plot (defaults to '%(default)s').")
parser.add_argument('-s', '--style', required=False, type=str, help = "The plot line-style.")
# add verbose option
bob.core.log.add_command_line_option(parser)
# parse arguments
args = parser.parse_args(command_line_parameters)
return args
def main(command_line_parameters=None):
"""Reads score files, computes error measures and plots curves."""
args = command_line_arguments(command_line_parameters)
fid = open(args.out_file, "w")
# fid.write("{\bf Feat.\ type} & & {\bf DEV Threshold} & {\bf DEV EER (\%)} & {\bf EVAL HTER (\%)} \\ \hline \n")
for score_type in args.score_types:
for param in args.parameters:
cur_dir = args.directory + "_" + score_type + "_" + param
if not os.path.exists(cur_dir):
continue
for fn in os.listdir(cur_dir):
if fn.endswith(".txt"):
print (cur_dir)
# print (fn)
resfile = open(os.path.join(cur_dir, fn), "r")
text = resfile.read()
text_lines = text.splitlines()
title = text_lines[0]
if 'Development' in text_lines[0]:
title = score_type
print("Results of " + title + ":")
m=re.search(r"FAR = (\w+\.\w+)", text)
far = 100.0 * float(m.group(1)) # the value in parenthesis
m=re.search(r"FRR = (\w+\.\w+)", text)
frr = 100.0 * float(m.group(1)) # the value in parenthesis
m=re.search(r"threshold = (-*\w+\.\w+)", text)
if not m:
continue
eer_thres = float(m.group(1)) # the value in parenthesis
print ("far=%.5f, frr=%.5f, eer_thres=%.3f" % (far, frr, eer_thres))
m=re.search(r"SFAR = (\w+\.\w+)", text)
sfar = 100.0 * float(m.group(1)) # the value in parenthesis
m=re.search(r"SFRR = (\w+\.\w+)", text)
sfrr = 100.0 * float(m.group(1)) # the value in parenthesis
m = re.search(r"attacks: a:(\w+), ad:(\w+)", text)
type_attack = m.group(1)
type_device = m.group(2)
hter = (sfar + sfrr)*0.5
print ("%s, %s: sfar=%.5f, sfrr=%.5f, hter=%.3f" % (type_attack, type_device, sfar, sfrr, hter))
# if hter < 15.0:
# tableline = "%s & {\\bf %s} & {\\bf %s} & {\\bf %s} & {\\bf %5.2f} & {\\bf %5.3f} & {\\bf %5.2f} & {\\bf %5.2f} & {\\bf %5.2f} \\\\ \\hline \n" % \
# (score_type, param, type_attack, type_device, (far+frr)*0.5, eer_thres, sfar, sfrr, hter)
# tableline = "%s & {\\bf %s} & {\\bf %5.3f} & {\\bf %5.2f} \\\\ \\hline \n" % \
# (score_type, param, eer_thres, (far+frr)*0.5)
# tableline = "%s & {\\bf %s} & {\\bf %5.3f} & {\\bf %5.2f} \\\\ \\hline \n" % \
# (title, param, eer_thres, (far+frr)*0.5)
# else:
# tableline = "%s & %s & %5.3f & %5.2f \\\\ \\hline \n" % \
# (score_type, param, eer_thres, (far+frr)*0.5)
if type_attack == 'all':
# tableline = "%s %.3f & %.5f & %.5f \\\\ \\hline \n" % \
# (title, eer_thres, (far+frr)*0.5, hter)
tableline = "%s & all & all & %.3f & %.3f & %.3f \\\\ \\hline \n" % \
(title, sfrr, sfar, hter)
else:
tableline = "%s & %s & %s & %.3f & %.3f & %.3f \\\\ \\hline \n" % \
(title, type_attack, type_device, sfrr, sfar, hter)
fid.write(tableline)
fid.write(" \hline \n")
if __name__ == '__main__':
main()
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment