Commit 34040878 authored by Pavel KORSHUNOV's avatar Pavel KORSHUNOV

cleaned up scripts and setup

parent 1242a6d0
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Pavel Korshunov <pavel.korshunov@idiap.ch>
# Mon 7 Sep 15:19:22 CEST 2015
#
# Copyright (C) 2012-2015 Idiap Research Institute, Martigny, Switzerland
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the ipyplotied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from __future__ import print_function
"""This script evaluates the given score files and computes EER and Spoofing FAR with regards to 10 types of voice attacks"""
import bob.measure
import argparse
import numpy, math
import os
import os.path
import sys
import matplotlib.pyplot as mpl
import matplotlib.font_manager as fm
import bob.core
logger = bob.core.log.setup("bob.spoof.speech")
#attacks = ['replay_phone1', 'replay_phone2', 'replay_laptop', 'replay_laptop_HQ', 'speech_synthesis_logical_access', 'speech_synthesis_physical_access', 'speech_synthesis_physical_access_HQ', 'voice_conversion_logical_access', 'voice_conversion_physical_access', 'voice_conversion_physical_access_HQ']
def load_attacks_file(filename, support="all", adevice="all", recdevice="all"):
# split in positives and negatives
positives = []
negatives = []
# read four column list line by line
for (client_id, probe_id, filename, score) in bob.measure.load.four_column(filename):
if client_id == probe_id:
if (support in filename or support == "all") and \
(adevice in filename or adevice == "all") and \
(recdevice in filename or recdevice == "all"):
positives.append(score)
else:
negatives.append(score)
return (numpy.array(negatives, numpy.float64), numpy.array(positives, numpy.float64))
def plot_det_curves(scores_real, scores_attack, thresholds, labels, title, outname):
from matplotlib.backends.backend_pdf import PdfPages
pdf_name = outname+'.pdf'
pp = PdfPages(pdf_name)
font_size = 16
fig = mpl.figure()
ax1 = mpl.subplot(111)
linestyles = ['-', '--', ':', '-.']
colors = ('b', 'g', 'r', 'c', 'm', 'y', 'k')
for i in range(len(scores_real)):
if thresholds is None:
eer_threshold = bob.measure.eer_threshold(scores_attack[i], scores_real[i])
else:
eer_threshold = float(thresholds[i])
far, frr = bob.measure.farfrr(scores_attack[i], scores_real[i], eer_threshold)
bob.measure.plot.det(scores_attack[i], scores_real[i], 100,
color=colors[i % len(colors)], linestyle=linestyles[i % len(linestyles)],
label='%s'%(labels[i]), linewidth=2)
# label='%s, HTER=%.2f%%'%(labels[i], (far+frr)*50), linewidth=2)
if far < 0.0001:
far_scaled = bob.measure.ppndf(far+0.001)
else:
far_scaled = bob.measure.ppndf(far)
if frr < 0.0001:
frr_scaled = bob.measure.ppndf(frr+0.001)
else:
frr_scaled = bob.measure.ppndf(frr)
ax1.plot(far_scaled, frr_scaled, color=colors[i % len(colors)], marker='o')
text_coords = (far_scaled+0.1, frr_scaled)
if i % 2:
text_coords = (far_scaled, frr_scaled+0.15)
if thresholds is None:
# str_name = "EER = %.2f%%"
str_name = "%.2f%%"
else:
# str_name = "HTER = %.2f%%"
str_name = "%.2f%%"
mpl.annotate(str_name % ((far+frr)*50), xy=(far_scaled, frr_scaled), xycoords='data', xytext=text_coords, color=colors[i % len(colors)], fontsize=font_size)
bob.measure.plot.det_axis([0.1, 99, 0.1, 99])
mpl.xlabel('FAR (%)', fontsize=font_size)
mpl.ylabel('FRR (%)', fontsize=font_size)
mpl.legend(loc='best')
# ax1.set_title(title, fontsize=12)
mpl.grid()
pp.savefig()
pp.close()
def command_line_arguments(command_line_parameters):
"""Parse the program options"""
basedir = os.path.dirname(os.path.dirname(os.path.realpath(sys.argv[0])))
OUTPUT_DIR = os.path.join(basedir, 'plots')
# set up command line parser
parser = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('-r', '--real-files', required=True, nargs = '+', help = "The first score file (or set) of the set.")
parser.add_argument('-a', '--attack-files', required=True, nargs = '+', help = "The second score file (or set) of the set.")
parser.add_argument('-l', '--labels', required=True, nargs = '+', help = "The name(s) of det curves corresponding to the score file (or set).")
parser.add_argument('-e', '--thresholds', required=False, nargs = '+', help = "The value(s) of thersholds when computing scores for Eval sets.")
parser.add_argument('-t', '--plot-title', required=False, type=str, default="all", help = "Title of the plot.")
parser.add_argument('-o', '--out-directory', dest="directory", default=OUTPUT_DIR, help="This path will be prepended to every file output by this procedure (defaults to '%(default)s')")
parser.add_argument('-s', '--support', required=False, type=str, default="all", help = "Type of attack.")
parser.add_argument('-k', '--attackdevice', required=False, type=str, default="all", help = "Attack device.")
# add verbose option
bob.core.log.add_command_line_option(parser)
# parse arguments
args = parser.parse_args(command_line_parameters)
# set verbosity level
bob.core.log.set_verbosity_level(logger, args.verbose)
return args
def main(command_line_parameters=None):
"""Reads score files, computes error measures and plots curves."""
args = command_line_arguments(command_line_parameters)
if not os.path.exists(args.directory):
os.makedirs(args.directory)
support=args.support
adevice=args.attackdevice
thresholds = None
if args.thresholds:
thresholds = args.thresholds
## Read scores ###
print("Loading real score files")
# take only positive values
scores_real = [bob.measure.load.split_four_column(real_file)[1] for real_file in args.real_files]
print("Loading attack score files")
scores_attack = [load_attacks_file(attack_file, support, adevice)[1] for attack_file in args.attack_files]
####################
## Plot DET ###
####################
if thresholds is None:
title = 'Dev set, attack "%s" and attack device "%s"' % (support, adevice)
outname = 'det_dev_attack_%s_adevice_%s' % (support, adevice)
else:
title = 'Test set, attack "%s" and attack device "%s"' % (support, adevice)
outname = 'det_test_attack_%s_adevice_%s' % (support, adevice)
outname = os.path.join(args.directory, outname)
plot_det_curves(scores_real, scores_attack, thresholds, args.labels, title, outname)
if __name__ == '__main__':
main()
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Pavel Korshunov <pavel.korshunov@idiap.ch>
# Mon 7 Sep 15:19:22 CEST 2015
#
# Copyright (C) 2012-2015 Idiap Research Institute, Martigny, Switzerland
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the ipyplotied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from __future__ import print_function
"""This script evaluates the given score files and computes EER and Spoofing FAR with regards to 10 types of voice attacks"""
import bob.measure
import argparse
import numpy, math
import os
import os.path
import sys
import re
import matplotlib.pyplot as mpl
import matplotlib.font_manager as fm
import bob.core
logger = bob.core.log.setup("bob.spoof.speech")
def command_line_arguments(command_line_parameters):
"""Parse the program options"""
basedir = os.path.dirname(os.path.dirname(os.path.realpath(sys.argv[0])))
OUTPUT_FILE = os.path.join(basedir, 'statistics.txt')
# set up command line parser
parser = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('-t', '--score-types', required=False, nargs = '+', default=[], help = "The type of the scores, can be a set of values.")
parser.add_argument('-p', '--parameters', required=False, nargs = '+', default=[], help = "The parameter for this type of the scores, can be a set of values.")
parser.add_argument('-d', '--directory', type=str, required=True, help = "The the directory prefix with txt files that have FAR and FRR scores.")
parser.add_argument('-o', '--out-file', type=str, default=OUTPUT_FILE, help = "The the directory to ouput the resulted plot (defaults to '%(default)s').")
parser.add_argument('-s', '--style', required=False, type=str, help = "The plot line-style.")
# add verbose option
bob.core.log.add_command_line_option(parser)
# parse arguments
args = parser.parse_args(command_line_parameters)
# set verbosity level
bob.core.log.set_verbosity_level(logger, args.verbose)
return args
def main(command_line_parameters=None):
"""Reads score files, computes error measures and plots curves."""
args = command_line_arguments(command_line_parameters)
# if not os.path.exists(args.out_directory):
# os.makedirs(args.out_directory)
fid = open(args.out_file, "w")
# fid.write("{\bf Feat.\ type} & & {\bf DEV Threshold} & {\bf DEV EER (\%)} & {\bf EVAL HTER (\%)} \\ \hline \n")
for score_type in args.score_types:
for param in args.parameters:
cur_dir = args.directory + "_" + score_type + "_" + param
if not os.path.exists(cur_dir):
continue
for fn in os.listdir(cur_dir):
if fn.endswith(".txt"):
print (cur_dir)
# print (fn)
resfile = open(os.path.join(cur_dir, fn), "r")
text = resfile.read()
text_lines = text.splitlines()
title = text_lines[0]
if 'Development' in text_lines[0]:
title = score_type
print("Results of " + title + ":")
m=re.search(r"FAR = (\w+\.\w+)", text)
far = 100.0 * float(m.group(1)) # the value in parenthesis
m=re.search(r"FRR = (\w+\.\w+)", text)
frr = 100.0 * float(m.group(1)) # the value in parenthesis
m=re.search(r"threshold = (-*\w+\.\w+)", text)
if not m:
continue
eer_thres = float(m.group(1)) # the value in parenthesis
print ("far=%.5f, frr=%.5f, eer_thres=%.3f" % (far, frr, eer_thres))
m=re.search(r"SFAR = (\w+\.\w+)", text)
sfar = 100.0 * float(m.group(1)) # the value in parenthesis
m=re.search(r"SFRR = (\w+\.\w+)", text)
sfrr = 100.0 * float(m.group(1)) # the value in parenthesis
m = re.search(r"attacks: a:(\w+), ad:(\w+)", text)
type_attack = m.group(1)
type_device = m.group(2)
hter = (sfar + sfrr)*0.5
print ("%s, %s: sfar=%.5f, sfrr=%.5f, hter=%.3f" % (type_attack, type_device, sfar, sfrr, hter))
# if hter < 15.0:
# tableline = "%s & {\\bf %s} & {\\bf %s} & {\\bf %s} & {\\bf %5.2f} & {\\bf %5.3f} & {\\bf %5.2f} & {\\bf %5.2f} & {\\bf %5.2f} \\\\ \\hline \n" % \
# (score_type, param, type_attack, type_device, (far+frr)*0.5, eer_thres, sfar, sfrr, hter)
# tableline = "%s & {\\bf %s} & {\\bf %5.3f} & {\\bf %5.2f} \\\\ \\hline \n" % \
# (score_type, param, eer_thres, (far+frr)*0.5)
# tableline = "%s & {\\bf %s} & {\\bf %5.3f} & {\\bf %5.2f} \\\\ \\hline \n" % \
# (title, param, eer_thres, (far+frr)*0.5)
# else:
# tableline = "%s & %s & %5.3f & %5.2f \\\\ \\hline \n" % \
# (score_type, param, eer_thres, (far+frr)*0.5)
if type_attack == 'all':
# tableline = "%s %.3f & %.5f & %.5f \\\\ \\hline \n" % \
# (title, eer_thres, (far+frr)*0.5, hter)
tableline = "%s & all & all & %.3f & %.3f & %.3f \\\\ \\hline \n" % \
(title, sfrr, sfar, hter)
else:
tableline = "%s & %s & %s & %.3f & %.3f & %.3f \\\\ \\hline \n" % \
(title, type_attack, type_device, sfrr, sfar, hter)
fid.write(tableline)
# fid.write(" \hline \n")
fid.write(" \hline \n")
# fid.close()
# plot_det_curves([scores_dev_attacks, scores_dev], [scores_eval_attacks, scores_eval], outname)
if __name__ == '__main__':
main()
This diff is collapsed.
......@@ -4,11 +4,12 @@
[buildout]
parts = scripts
eggs = bob.spoof.speech
bob.db.asvspoof
bob.db.avspoof
eggs = bob.paper.interspeech_2016
bob.pad.base
bob.sp
bob.ap
bob.spoof.speech
bob.bio.db
bob.db.base
bob.pad.db
......@@ -19,8 +20,6 @@ extensions = bob.buildout
auto-checkout = *
develop = src/bob.spoof.speech
src/bob.db.asvspoof
src/bob.db.avspoof
src/bob.pad.base
src/bob.ap
src/bob.bio.db
......@@ -35,8 +34,6 @@ newest = false
[sources]
bob.spoof.speech = git git@gitlab.idiap.ch:pkorshunov/bob.spoof.speech.git
bob.db.avspoof = git git@github.com:bioidiap/bob.db.avspoof.git
bob.db.asvspoof = git git@gitlab.idiap.ch:pkorshunov/bob.db.asvspoof.git
bob.pad.base = git git@gitlab.idiap.ch:biometric/bob.pad.base.git
bob.ap = git git@github.com:pkorshunov/bob.ap.git
bob.bio.db = git git@gitlab.idiap.ch:biometric/bob.bio.db.git
......
......@@ -23,24 +23,24 @@ for p in "${postfixes[@]}"; do
command="-n 10 -m 60 -t ${scorepath_dev}dev-attack -d ${scorepath_dev}dev-real -f ${scorepath}eval-attack -e ${scorepath}eval-real -o plots_${p}_${f}_20"
echo $command --pretty-title "${prettytitles[indxfeat]}${prettypostfixes[indxfixes]}"
bin/plot_pad_results.py $command --pretty-title "${prettytitles[indxfeat]}${prettypostfixes[indxfixes]}"
bin/pad_process_scores.py $command --pretty-title "${prettytitles[indxfeat]}${prettypostfixes[indxfixes]}"
# if need to plot for each attack
# bin/plot_pad_results.py $command -s replay
# bin/plot_pad_results.py $command -s all -k all --pretty-title "${prettytitles[indxfeat]}${prettypostfixes[indxfixes]}"
# bin/plot_pad_results.py $command -s replay -k laptop --pretty-title "${prettytitles[indxfeat]}${prettypostfixes[indxfixes]}"
# bin/plot_pad_results.py $command -s replay -k laptop_HQ_speaker --pretty-title "${prettytitles[indxfeat]}${prettypostfixes[indxfixes]}"
# bin/plot_pad_results.py $command -s replay -k phone1 --pretty-title "${prettytitles[indxfeat]}${prettypostfixes[indxfixes]}"
# bin/plot_pad_results.py $command -s replay -k phone2 --pretty-title "${prettytitles[indxfeat]}${prettypostfixes[indxfixes]}"
# bin/plot_pad_results.py $command -s voice_conversion -k physical_access --pretty-title "${prettytitles[indxfeat]}${prettypostfixes[indxfixes]}"
# bin/plot_pad_results.py $command -s speech_synthesis -k physical_access --pretty-title "${prettytitles[indxfeat]}${prettypostfixes[indxfixes]}"
# bin/plot_pad_results.py $command -s voice_conversion -k physical_access_HQ_speaker --pretty-title "${prettytitles[indxfeat]}${prettypostfixes[indxfixes]}"
# bin/plot_pad_results.py $command -s speech_synthesis -k physical_access_HQ_speaker --pretty-title "${prettytitles[indxfeat]}${prettypostfixes[indxfixes]}"
# bin/plot_pad_results.py $command -s voice_conversion -k logical_access --pretty-title "${prettytitles[indxfeat]}${prettypostfixes[indxfixes]}"
# bin/plot_pad_results.py $command -s speech_synthesis -k logical_access --pretty-title "${prettytitles[indxfeat]}${prettypostfixes[indxfixes]}"
# bin/pad_process_scores.py $command -s replay
# bin/pad_process_scores.py $command -s all -k all --pretty-title "${prettytitles[indxfeat]}${prettypostfixes[indxfixes]}"
# bin/pad_process_scores.py $command -s replay -k laptop --pretty-title "${prettytitles[indxfeat]}${prettypostfixes[indxfixes]}"
# bin/pad_process_scores.py $command -s replay -k laptop_HQ_speaker --pretty-title "${prettytitles[indxfeat]}${prettypostfixes[indxfixes]}"
# bin/pad_process_scores.py $command -s replay -k phone1 --pretty-title "${prettytitles[indxfeat]}${prettypostfixes[indxfixes]}"
# bin/pad_process_scores.py $command -s replay -k phone2 --pretty-title "${prettytitles[indxfeat]}${prettypostfixes[indxfixes]}"
# bin/pad_process_scores.py $command -s voice_conversion -k physical_access --pretty-title "${prettytitles[indxfeat]}${prettypostfixes[indxfixes]}"
# bin/pad_process_scores.py $command -s speech_synthesis -k physical_access --pretty-title "${prettytitles[indxfeat]}${prettypostfixes[indxfixes]}"
# bin/pad_process_scores.py $command -s voice_conversion -k physical_access_HQ_speaker --pretty-title "${prettytitles[indxfeat]}${prettypostfixes[indxfixes]}"
# bin/pad_process_scores.py $command -s speech_synthesis -k physical_access_HQ_speaker --pretty-title "${prettytitles[indxfeat]}${prettypostfixes[indxfixes]}"
# bin/pad_process_scores.py $command -s voice_conversion -k logical_access --pretty-title "${prettytitles[indxfeat]}${prettypostfixes[indxfixes]}"
# bin/pad_process_scores.py $command -s speech_synthesis -k logical_access --pretty-title "${prettytitles[indxfeat]}${prettypostfixes[indxfixes]}"
indxfeat=$(( indxfeat + 1 ))
done
indxfixes=$(( indxfixes + 1 ))
done
echo bin/plot_far_frr_pad.py -t ${plotnames[@]} -p 20 -d plots -o stats.txt
bin/plot_far_frr_pad.py -t ${plotnames[@]} -p 20 -d plots -o stats.txt
echo bin/pad_stats_summary.py -t ${plotnames[@]} -p 20 -d plots -o stats.txt
bin/pad_stats_summary.py -t ${plotnames[@]} -p 20 -d plots -o stats.txt
......@@ -24,9 +24,9 @@ function plot {
titles="$titles"
thr="$thr"
echo "bin/plot_det_from_sets.py -r $realfiles -a $attackfiles -l $titles --thresholds $thr -o plots_compare_pads"
echo "bin/pad_diff_sys_scores.py -r $realfiles -a $attackfiles -l $titles --thresholds $thr -o plots_compare_pads"
bin/plot_det_from_sets.py -r $realfiles -a $attackfiles -l $titles --thresholds $thr -o plots_compare_pads
bin/pad_diff_sys_scores.py -r $realfiles -a $attackfiles -l $titles --thresholds $thr -o plots_compare_pads
}
# uncomment to produce Figure 2(a) of the paper
......
......@@ -100,9 +100,9 @@ setup(
# the version of bob.
entry_points={
'console_scripts': [
'plot_pad_results.py = bob.paper.interspeech_2016.scripts.plot_pad_results:main',
'plot_far_frr_pad.py = bob.paper.interspeech_2016.scripts.plot_far_frr_pad:main',
'plot_det_from_sets.py = bob.paper.interspeech_2016.scripts.plot_det_from_sets:main',
'pad_process_scores.py = bob.paper.interspeech_2016.scripts.pad_process_scores:main',
'pad_stats_summary.py = bob.paper.interspeech_2016.scripts.pad_stats_summary:main',
'pad_diff_sys_scores.py = bob.paper.interspeech_2016.scripts.pad_diff_sys_scores:main',
],
},
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment