Commit fcbec2dc authored by Amir MOHAMMADI's avatar Amir MOHAMMADI
Browse files

Before separating still faces and talking faces

parent a90474ce
import scipy.io.wavfile
import scipy.signal
from bob.db.base import read_annotation_file
from bob.io.base import load
from bob.io.video import reader
......@@ -9,34 +10,27 @@ import subprocess
import tempfile
from os.path import split, splitext
from . import SWAN_FRAME_SHAPE
import logging
SITE_MAPPING = {
'1': 'NTNU',
'2': 'UIO',
'3': 'MPH-FRA',
'4': 'IDIAP',
'6': 'MPH-IND',
}
DEVICE_MAPPING = {
'p': 'iPhone',
't': 'iPad',
}
MODALITY_MAPPING = {
'1': 'face',
'2': 'voice',
'3': 'eye',
'4': 'finger',
}
def read_audio(video_path):
with tempfile.NamedTemporaryFile(suffix='.wav') as f:
cmd = ['ffmpeg', '-v', 'quiet', '-i', video_path, '-y', '-vn', f.name]
logger = logging.getLogger(__name__)
SITE_MAPPING = {"1": "NTNU", "2": "UIO", "3": "MPH-FRA", "4": "IDIAP", "6": "MPH-IND"}
DEVICE_MAPPING = {"p": "iPhone", "t": "iPad"}
MODALITY_MAPPING = {"1": "face", "2": "voice", "3": "eye", "4": "finger"}
def read_audio(video_path, new_rate=None):
with tempfile.NamedTemporaryFile(suffix=".wav") as f:
cmd = ["ffmpeg", "-v", "quiet", "-i", video_path, "-y", "-vn", f.name]
subprocess.call(cmd)
f.seek(0)
rate, signal = scipy.io.wavfile.read(f.name)
if new_rate is not None and rate != new_rate:
logger.debug("Resampling audio from %d to %d", rate, new_rate)
samps = round(len(signal) * new_rate / rate) # Number of samples to resample
signal, rate = scipy.signal.resample(signal, samps), new_rate
return rate, signal
......@@ -51,7 +45,7 @@ class Client(object):
@property
def id(self):
return '{}_{}'.format(self.institute, self.id_in_site)
return "{}_{}".format(self.institute, self.id_in_site)
def swan_file_metadata(path):
......@@ -76,7 +70,7 @@ def swan_file_metadata(path):
_, path = split(path)
# path: 4_00001_m_01_01_t_2.mp4
path, extension = splitext(path)
parts = path.split('_')
parts = path.split("_")
if len(parts) == 8:
parts = parts[1:]
# path: 4_00001_m_01_01_t_2
......@@ -95,8 +89,11 @@ class SwanFile(object):
def __init__(self, **kwargs):
super(SwanFile, self).__init__(**kwargs)
(
self.client, self.session, self.nrecording,
self.device, self.modality
self.client,
self.session,
self.nrecording,
self.device,
self.modality,
) = swan_file_metadata(self.path)
......@@ -107,11 +104,14 @@ class SwanVideoFile(VideoBioFile, SwanFile):
# rotate the video or image since SWAN videos are not upright!
return np.swapaxes(data, -2, -1)
def load(self, directory=None, extension=None,
frame_selector=FrameSelector(selection_style='all')):
def load(
self,
directory=None,
extension=None,
frame_selector=FrameSelector(selection_style="all"),
):
if extension is None:
video_path = self.make_path(directory or self.original_directory,
extension)
video_path = self.make_path(directory or self.original_directory, extension)
for _ in range(100):
try:
video = load(video_path)
......@@ -121,8 +121,7 @@ class SwanVideoFile(VideoBioFile, SwanFile):
video = self.swap(video)
return frame_selector(video)
else:
return super(SwanVideoFile, self).load(
directory, extension, frame_selector)
return super(SwanVideoFile, self).load(directory, extension, frame_selector)
@property
def frames(self):
......@@ -182,19 +181,23 @@ class SwanVideoFile(VideoBioFile, SwanFile):
``{'0': {'reye':(re_y,re_x), 'leye':(le_y,le_x)}, ...}``
"""
return read_annotation_file(
self.make_path(self.annotation_directory,
self.annotation_extension),
self.annotation_type)
self.make_path(self.annotation_directory, self.annotation_extension),
self.annotation_type,
)
class SwanAudioFile(SwanVideoFile):
"""A base class that extracts audio from SWAN video files"""
def __init__(self, new_rate=None, **kwargs):
super().__init__(**kwargs)
self.new_rate = new_rate
def load(self, directory=None, extension=None):
if extension is None:
video_path = self.make_path(directory, extension)
rate, audio = read_audio(video_path)
return rate, np.cast['float'](audio)
rate, audio = read_audio(video_path, new_rate=self.new_rate)
return rate, np.cast["float"](audio)
else:
return super(SwanAudioFile, self).load(directory, extension)
......@@ -202,6 +205,10 @@ class SwanAudioFile(SwanVideoFile):
class SwanVideoDatabase(object):
"""SwanVideoDatabase"""
def __init__(self, new_rate=None, **kwargs):
super().__init__(**kwargs)
self.new_rate = new_rate
def frames(self, padfile):
return padfile.frames
......@@ -218,4 +225,5 @@ class SwanVideoDatabase(object):
f.annotation_directory = self.annotation_directory
f.annotation_extension = self.annotation_extension
f.annotation_type = self.annotation_type
f.new_rate = self.new_rate
return files
......@@ -5,4 +5,5 @@ from bob.db.swan import Database, SwanAudioBioFile
database = Database(
bio_file_class=SwanAudioBioFile,
annotation_directory=None, # no annotations for the voice part
new_rate=16000,
)
......@@ -5,4 +5,5 @@ from bob.db.swan.query_pad import Database, SwanAudioPadFile
database = Database(
pad_file_class=SwanAudioPadFile,
annotation_directory=None, # no annotations for the voice part
new_rate=16000,
)
......@@ -103,9 +103,11 @@ def enrollment_probes(folder, files, group, pattern, ids, cls='enroll'):
f.write('{0} {1} {1}\n'.format(path, client_id))
def licit_protocols(out_folder, files, patterns, attack=False):
def licit_protocols(
out_folder, files, patterns, attack=False, modalities=('eye', 'face', 'voice')
):
for fold, dev_ids, eval_ids in BIO_FOLDS:
for modality in ('eye', 'face', 'voice'):
for modality in modalities:
folder = '{}_{}_f{}'.format(out_folder, modality, fold)
# create empty norm folder
empty_norm(folder)
......@@ -207,8 +209,7 @@ def bio_protocol_2(out_folder, files):
def bio_protocol_3(out_folder, files):
# This will give variation for indoor controlled versus indoor/outdoor
# uncontrolled
# This will give variation for indoor controlled versus indoor/outdoor uncontrolled
# enroll with session 2
# probe with session 3,4,5,6
# Data Partition: 30% development and 70% evaluation.
......@@ -227,6 +228,15 @@ def bio_protocol_3(out_folder, files):
licit_protocols(out_folder, files, patterns)
def bio_protocol_4(out_folder, files):
# This is just like protocol 3 but faces are talking faces
patterns = {
('face', 'enroll'): r'.*session_02/iPhone/.*/.*_02_0[1-2]_p_2\.mp4',
('face', 'probe'): r'.*session_0[3-6]/iPhone/.*/.*_0[3-6]_0[3-4]_p_2.*',
}
licit_protocols(out_folder, files, patterns, modalities=['face'])
def spoof_protocol_3(out_folder, files):
# This will give variation for indoor controlled versus indoor/outdoor
# uncontrolled
......@@ -242,12 +252,21 @@ def spoof_protocol_3(out_folder, files):
('face', 'enroll'): r'.*session_02/iPhone/.*/.*_02_0[1-2]_p_1\.mp4',
('voice', 'enroll'): r'.*session_02/iPhone/.*/.*_02_0[1-2]_p_2\.mp4',
('eye', 'probe'): r'pa-database/Eye/.*',
('face', 'probe'): r'pa-database/Face/.*',
('face', 'probe'): r'pa-database/StillFace/.*',
('voice', 'probe'): r'pa-database/Voice/.*',
}
licit_protocols(out_folder, files, patterns, attack=True)
def spoof_protocol_4(out_folder, files):
# spoof protocol for talking faces that matches bio_protocol_4
patterns = {
('face', 'enroll'): r'.*session_02/iPhone/.*/.*_02_0[1-2]_p_2\.mp4',
('face', 'probe'): r'pa-database/TalkingFace/.*',
}
licit_protocols(out_folder, files, patterns, attack=True)
def all_pad_protocols(out_folder, files):
# protocol 1
# eye
......@@ -283,27 +302,27 @@ def all_pad_protocols(out_folder, files):
('train', 'bf'): r'.*session_02/iPhone/.*/.*_02_0[1-2]_p_1\.mp4',
('dev', 'bf'): r'.*session_02/iPhone/.*/.*_02_0[1-2]_p_1\.mp4',
('eval', 'bf'): r'.*session_0[2-6]/iPhone/.*/.*_0[2-6]_0[1-2]_p_1\.mp4',
('train', 'pa'): r'pa-database/Face/PA\.F\.1/.*',
('dev', 'pa'): r'pa-database/Face/PA\.F\.1/.*',
('eval', 'pa'): r'pa-database/Face/PA\.F\.1/.*',
('train', 'pa'): r'pa-database/TalkingFace/PA\.F\.1/.*',
('dev', 'pa'): r'pa-database/TalkingFace/PA\.F\.1/.*',
('eval', 'pa'): r'pa-database/TalkingFace/PA\.F\.1/.*',
}
pad_protocols(out_folder + 'pad_p1_paf1', files, patterns)
patterns = {
('train', 'bf'): r'.*session_02/iPhone/.*/.*_02_0[1-8]_p_2\.mp4',
('dev', 'bf'): r'.*session_02/iPhone/.*/.*_02_0[1-8]_p_2\.mp4',
('eval', 'bf'): r'.*session_0[2-6]/iPhone/.*/.*_0[2-6]_0[1-8]_p_2\.mp4',
('train', 'pa'): r'pa-database/Face/PA\.F\.5/.*',
('dev', 'pa'): r'pa-database/Face/PA\.F\.5/.*',
('eval', 'pa'): r'pa-database/Face/PA\.F\.5/.*',
('train', 'pa'): r'pa-database/TalkingFace/PA\.F\.5/.*',
('dev', 'pa'): r'pa-database/TalkingFace/PA\.F\.5/.*',
('eval', 'pa'): r'pa-database/TalkingFace/PA\.F\.5/.*',
}
pad_protocols(out_folder + 'pad_p1_paf5', files, patterns)
patterns = {
('train', 'bf'): r'.*session_02/iPhone/.*/.*_02_0[1-8]_p_2\.mp4',
('dev', 'bf'): r'.*session_02/iPhone/.*/.*_02_0[1-8]_p_2\.mp4',
('eval', 'bf'): r'.*session_0[2-6]/iPhone/.*/.*_0[2-6]_0[1-8]_p_2\.mp4',
('train', 'pa'): r'pa-database/Face/PA\.F\.6/.*',
('dev', 'pa'): r'pa-database/Face/PA\.F\.6/.*',
('eval', 'pa'): r'pa-database/Face/PA\.F\.6/.*',
('train', 'pa'): r'pa-database/TalkingFace/PA\.F\.6/.*',
('dev', 'pa'): r'pa-database/TalkingFace/PA\.F\.6/.*',
('eval', 'pa'): r'pa-database/TalkingFace/PA\.F\.6/.*',
}
pad_protocols(out_folder + 'pad_p1_paf6', files, patterns)
# voice
......@@ -373,3 +392,5 @@ def _create(args):
spoof_protocol_3(path, files)
path = pkg_resources.resource_filename(__name__, 'lists/')
all_pad_protocols(path, files)
path = pkg_resources.resource_filename(__name__, 'lists/licit_p4')
bio_protocol_4(path, files)
This diff is collapsed.
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment