Skip to content
GitLab
Commit 7002fdd7 authored by Olegs NIKISINS's avatar Olegs NIKISINS
Browse files

Merge branch 'batl_LOO_protocols' into 'master' 

Batl loo protocols

See merge request !86
parents 3e3b04a9 1ec2a411
Pipeline #26493 passed with stages
in 16 minutes and 15 seconds
Hide whitespace changes
Inline Side-by-side
bob/pad/face/database/batl.py
View file @ 7002fdd7
......@@ -202,6 +202,14 @@ class BatlPadDatabase(PadDatabase):
infrared data only, use 50 frames, join train and dev sets forming
a single large training set.
"grandtest-infrared-50-LOO_<unseen_attack>", for example "grandtest-infrared-50-LOO_fakehead"
- Leave One Out (LOO) protocol with fakehead attacks present only in the `test` set. The original partitioning
in the `grandtest` protocol is queried first and subselects the file list such
that the specified `unknown_attack` is removed from both `train` and `dev` sets.
The `test` set will consist of only the selected `unknown_attack` and `bonafide` files.
This protocol is used to evaluate the robustness against attacks unseen in training.
.
"grandtest-color*infrared-50" - baseline protocol,
load both "color" and "infrared" channels,
use 50 frames.
......@@ -286,6 +294,15 @@ class BatlPadDatabase(PadDatabase):
self.append_color_face_roi_annot = append_color_face_roi_annot
#map file to identify attack grouping
map_file = pkg_resources.resource_filename('bob.pad.face', 'lists/batl/idiap_converter_v2.json')
with open(map_file, 'r') as fp:
map_dict=json.load(fp)
self.map_dict=map_dict
@property
def original_directory(self):
return self.db.original_directory
......@@ -295,6 +312,52 @@ class BatlPadDatabase(PadDatabase):
def original_directory(self, value):
self.db.original_directory = value
def unseen_attack_list_maker(self,files,unknown_attack,train=True):
"""
Selects and returns a list of files for Leave One Out (LOO) protocols.
This utilizes the original partitioning in the `grandtest` protocol and subselects
the file list such that the specified `unknown_attack` is removed from both `train` and `dev` sets.
The `test` set will consist of only the selected `unknown_attack` and `bonafide` files.
**Parameters:**
``files`` : pyclass::list
A list of files, db.objects()
``unknown_attack`` : str
The unknown attack protocol name example:'rigidmask' .
``train`` : bool
Denotes whether files are from training or development partition
**Returns:**
``mod_files`` : pyclass::list
A list of files selected for the protocol
"""
mod_files=[]
for file in files:
attack_category=self.map_dict["_"+"_".join(os.path.split(file.path)[-1].split("_")[-2:])].split("_")[-1]
if train:
if attack_category==unknown_attack:
pass
else:
mod_files.append(file) # everything except the attack specified is there
if not train:
if attack_category==unknown_attack or attack_category=='bonafide':
mod_files.append(file) # only the attack mentioned and bonafides in testing
else:
pass
return mod_files
def parse_protocol(self, protocol):
"""
......@@ -329,7 +392,7 @@ class BatlPadDatabase(PadDatabase):
forming a single training set.
"""
possible_extras = ['join_train_dev']
possible_extras = ['join_train_dev','LOO_fakehead','LOO_flexiblemask','LOO_glasses','LOO_papermask','LOO_prints','LOO_replay','LOO_rigidmask','LOO_makeup']
components = protocol.split("-")
......@@ -490,10 +553,11 @@ class BatlPadDatabase(PadDatabase):
protocol = 'nowig' if protocol == 'grandtest' else protocol
# Convert group names to low-level group names here.
groups = self.convert_names_to_lowlevel(
groups, self.low_level_group_names, self.high_level_group_names)
groups = self.convert_names_to_lowlevel(groups, self.low_level_group_names, self.high_level_group_names)
if not isinstance(groups, list) and groups is not None: # if a single group is given make it a list
if not isinstance(groups, list) and groups is not None and not isinstance(groups,str): # if a single group is given make it a list
groups = list(groups)
if extra is not None and "join_train_dev" in extra:
......@@ -515,6 +579,48 @@ class BatlPadDatabase(PadDatabase):
groups=['test'],
purposes=purposes, **kwargs)
# for the LOO protocols
elif extra is not None and "LOO_" in extra:
batl_files=[]
unknown_attack=extra.split("_")[-1]
if 'train' in groups:
tbatl_files = self.db.objects(protocol=protocol,
groups=['train'],
purposes=purposes, **kwargs)
tbatl_files=self.unseen_attack_list_maker(tbatl_files,unknown_attack,train=True)
batl_files=batl_files+tbatl_files
if 'validation' in groups:
tbatl_files = self.db.objects(protocol=protocol,
groups=['validation'],
purposes=purposes, **kwargs)
tbatl_files=self.unseen_attack_list_maker(tbatl_files,unknown_attack,train=True)
batl_files=batl_files+tbatl_files
if 'test' in groups:
tbatl_files = self.db.objects(protocol=protocol,
groups=['test'],
purposes=purposes, **kwargs)
tbatl_files=self.unseen_attack_list_maker(tbatl_files,unknown_attack,train=False)
batl_files=batl_files+tbatl_files
files=batl_files
else:
# files = self._fix_funny_eyes_in_objects(protocol=protocol,
# groups=groups,
......
bob/pad/face/lists/batl/idiap_converter_v2.json 0 → 100644
View file @ 7002fdd7
{"_0_00": "Unknown_bonafide", "_0_01": "Clientownmedicalglasse_bonafide", "_0_02": "UnisexglassesG01_bonafide", "_1_01": "FunnyeyesglassesG02_glasses", "_1_02": "PlastichalloweenmaskB082200_rigidmask", "_1_03": "unassigne_glasses", "_1_04": "FunnyeyesglassesG03_glasses", "_1_05": "FunnyeyesglassesG04_glasses", "_1_06": "FunnyeyesglassesG05_glasses", "_1_07": "FunnyeyesglassesG06_glasses", "_1_13": "ExoticfemailtransparentmaskB0853200_rigidmask", "_1_14": "ExoticfemailtransparentmaskwithmakeupB082301_rigidmask", "_1_15": "PaperglassesG10_glasses", "_1_16": "PaperglassesG11_glasses", "_2_01": "MannequinMq07_fakehead", "_2_02": "MannequinMq01_fakehead", "_2_03": "MannequinMq02_fakehead", "_2_04": "MannequinMq03_fakehead", "_2_05": "MannequinMq04_fakehead", "_2_06": "MannequinMq05_fakehead", "_2_07": "MannequinMq06_fakehead", "_2_08": "PapercraftmaskB030100_papermask", "_2_09": "PapercraftmaskB030300_papermask", "_2_10": "PapercraftmaskB031100_papermask", "_2_11": "PapercraftmaskB031200_papermask", "_2_12": "CustomwearablemaskB050100_rigidmask", "_2_13": "CustomwearablemaskB050200_rigidmask", "_2_14": "CustomwearablemaskB051200_rigidmask", "_2_15": "CustomwearablemaskB051600_rigidmask", "_2_16": "CustomwearablemaskB051800_rigidmask", "_2_17": "CustomwearablemaskB051900_rigidmask", "_2_18": "FlexiblesiliconmaskB072100_flexiblemask", "_2_75": "FlexiblesiliconmaskB063000_flexiblemask", "_2_76": "FlexiblesiliconmaskB063100_flexiblemask", "_2_77": "FlexiblesiliconmaskB073200_flexiblemask", "_2_78": "FlexiblesiliconmaskB073300_flexiblemask", "_2_79": "FlexiblesiliconmaskB073400_flexiblemask", "_2_80": "FlexiblesiliconmaskB060100_flexiblemask", "_2_81": "FlexiblesiliconmaskB060200_flexiblemask", "_2_82": "FlexiblesiliconmaskB060202_flexiblemask", "_2_83": "FlexiblesiliconmaskB061200_flexiblemask", "_2_84": "FlexiblesiliconmaskB062400_flexiblemask", "_2_85": "FlexiblesiliconmaskB061600_flexiblemask", "_2_86": "FlexiblesiliconmaskB061800_flexiblemask", "_2_87": "FlexiblesiliconmaskB061900_flexiblemask", "_2_88": "FlexiblesiliconmaskB070100_flexiblemask", "_2_89": "FlexiblesiliconmaskB071200_flexiblemask", "_2_90": "FlexiblesiliconmaskB071900_flexiblemask", "_2_91": "FlexiblesiliconmaskB062500_flexiblemask", "_2_92": "FlexiblesiliconmaskB062601_flexiblemask", "_2_93": "FlexiblesiliconmaskB072701_flexiblemask", "_2_94": "FlexiblesiliconmaskB072801_flexiblemask", "_2_95": "FlexiblesiliconmaskB072901_flexiblemask", "_2_96": "FlexiblesiliconmaskB062600_flexiblemask", "_2_97": "FlexiblesiliconmaskB072700_flexiblemask", "_2_98": "FlexiblesiliconmaskB072800_flexiblemask", "_2_99": "FlexiblesiliconmaskB072900_flexiblemask", "_3_01": "PrintedP0000100_prints", "_3_02": "PrintedP0000101_prints", "_3_03": "PrintedP0000102_prints", "_3_04": "PrintedP0000103_prints", "_3_05": "ElectronicP0100100_replay", "_3_06": "PrintedP0001800_prints", "_3_07": "PrintedP0001801_prints", "_3_08": "PrintedP0001802_prints", "_3_09": "PrintedP0001803_prints", "_3_10": "ElectronicP0101800_replay", "_3_11": "PrintedP0009800_prints", "_3_12": "PrintedP0009801_prints", "_3_13": "PrintedP0009802_prints", "_3_14": "PrintedP0009803_prints", "_3_15": "ElectronicP0109800_replay", "_4_01": "VideoplayV0001900_replay", "_4_02": "VideopauseV0101900_replay", "_4_03": "VideoplayV0001200_replay", "_4_04": "VideopauseV0101200_replay", "_4_05": "VideoplayV0009800_replay", "_4_06": "VideopauseV0109800_replay", "_4_07": "VideoplayV0001901_replay", "_4_08": "VideopauseV0101901_replay", "_4_09": "VideoplayV0000501_replay", "_4_10": "VideopauseV0100501_replay", "_4_11": "VideoplayV0009801_replay", "_4_12": "VideopauseV0109801_replay", "_4_13": "VideoplayV0101902_replay", "_4_14": "VideopauseV0101902_replay", "_4_15": "VideoplayV0101202_replay", "_4_16": "VideopauseV0101202_replay", "_4_17": "VideoplayV0109802_replay", "_4_18": "VideopauseV0109802_replay", "_5_01": "Makeup_makeup", "_5_02": "Makeup_makeup", "_5_03": "Makeup_makeup", "_5_04": "Makeup_makeup", "_5_05": "Makeup_makeup", "_5_06": "Makeup_makeup", "_5_07": "Makeup_makeup", "_5_08": "Makeup_makeup", "_5_09": "Makeup_makeup", "_5_10": "Makeup_makeup", "_5_100": "Makeup_makeup", "_5_101": "Makeup_makeup", "_5_102": "Makeup_makeup", "_5_103": "Makeup_makeup", "_5_104": "Makeup_makeup", "_5_105": "Makeup_makeup", "_5_106": "Makeup_makeup", "_5_107": "Makeup_makeup", "_5_108": "Makeup_makeup", "_5_109": "Makeup_makeup", "_5_11": "Makeup_makeup", "_5_110": "Makeup_makeup", "_5_111": "Makeup_makeup", "_5_112": "Makeup_makeup", "_5_113": "Makeup_makeup", "_5_114": "Makeup_makeup", "_5_115": "Makeup_makeup", "_5_116": "Makeup_makeup", "_5_117": "Makeup_makeup", "_5_118": "Makeup_makeup", "_5_119": "Makeup_makeup", "_5_12": "Makeup_makeup", "_5_120": "Makeup_makeup", "_5_121": "Makeup_makeup", "_5_122": "Makeup_makeup", "_5_123": "Makeup_makeup", "_5_124": "Makeup_makeup", "_5_125": "Makeup_makeup", "_5_126": "Makeup_makeup", "_5_127": "Makeup_makeup", "_5_128": "Makeup_makeup", "_5_129": "Makeup_makeup", "_5_13": "Makeup_makeup", "_5_130": "Makeup_makeup", "_5_131": "Makeup_makeup", "_5_132": "Makeup_makeup", "_5_133": "Makeup_makeup", "_5_134": "Makeup_makeup", "_5_135": "Makeup_makeup", "_5_136": "Makeup_makeup", "_5_137": "Makeup_makeup", "_5_138": "Makeup_makeup", "_5_139": "Makeup_makeup", "_5_14": "Makeup_makeup", "_5_15": "Makeup_makeup", "_5_16": "Makeup_makeup", "_5_17": "Makeup_makeup", "_5_18": "Makeup_makeup", "_5_19": "Makeup_makeup", "_5_20": "Makeup_makeup", "_5_21": "Makeup_makeup", "_5_22": "Makeup_makeup", "_5_23": "Makeup_makeup", "_5_24": "Makeup_makeup", "_5_25": "Makeup_makeup", "_5_26": "Makeup_makeup", "_5_27": "Makeup_makeup", "_5_28": "Makeup_makeup", "_5_29": "Makeup_makeup", "_5_30": "Makeup_makeup", "_5_31": "Makeup_makeup", "_5_32": "Makeup_makeup", "_5_33": "Makeup_makeup", "_5_34": "Makeup_makeup", "_5_35": "Makeup_makeup", "_5_36": "Makeup_makeup", "_5_37": "Makeup_makeup", "_5_38": "Makeup_makeup", "_5_39": "Makeup_makeup", "_5_40": "Makeup_makeup", "_5_41": "Makeup_makeup", "_5_42": "Makeup_makeup", "_5_43": "Makeup_makeup", "_5_44": "Makeup_makeup", "_5_45": "Makeup_makeup", "_5_46": "Makeup_makeup", "_5_47": "Makeup_makeup", "_5_48": "Makeup_makeup", "_5_49": "Makeup_makeup", "_5_50": "Makeup_makeup", "_5_51": "Makeup_makeup", "_5_52": "Makeup_makeup", "_5_53": "Makeup_makeup", "_5_54": "Makeup_makeup", "_5_55": "Makeup_makeup", "_5_56": "Makeup_makeup", "_5_57": "Makeup_makeup", "_5_58": "Makeup_makeup", "_5_59": "Makeup_makeup", "_5_60": "Makeup_makeup", "_5_61": "Makeup_makeup", "_5_62": "Makeup_makeup", "_5_63": "Makeup_makeup", "_5_64": "Makeup_makeup", "_5_65": "Makeup_makeup", "_5_66": "Makeup_makeup", "_5_67": "Makeup_makeup", "_5_68": "Makeup_makeup", "_5_69": "Makeup_makeup", "_5_70": "Makeup_makeup", "_5_71": "Makeup_makeup", "_5_72": "Makeup_makeup", "_5_73": "Makeup_makeup", "_5_74": "Makeup_makeup", "_5_75": "Makeup_makeup", "_5_76": "Makeup_makeup", "_5_77": "Makeup_makeup", "_5_78": "Makeup_makeup", "_5_79": "Makeup_makeup", "_5_80": "Makeup_makeup", "_5_81": "Makeup_makeup", "_5_82": "Makeup_makeup", "_5_83": "Makeup_makeup", "_5_84": "Makeup_makeup", "_5_85": "Makeup_makeup", "_5_86": "Makeup_makeup", "_5_87": "Makeup_makeup", "_5_88": "Makeup_makeup", "_5_89": "Makeup_makeup", "_5_90": "Makeup_makeup", "_5_91": "Makeup_makeup", "_5_92": "Makeup_makeup", "_5_93": "Makeup_makeup", "_5_94": "Makeup_makeup", "_5_95": "Makeup_makeup", "_5_96": "Makeup_makeup", "_5_97": "Makeup_makeup", "_5_98": "Makeup_makeup", "_5_99": "Makeup_makeup", "_1_08": "Wig", "_1_09": "Wig", "_1_10": "Wig", "_1_11": "Wig", "_1_12": "Wig"}
\ No newline at end of file
bob/pad/face/test/test_databases.py
View file @ 7002fdd7
......@@ -219,3 +219,136 @@ def test_casiasurf():
raise SkipTest(
"The database could not be queried; probably the db.sql3 file is missing. Here is the error: '%s'"
% e)
# # Test the BATL database
# @db_available('batl-db')
# def test_aggregated_db():
# batl_db = bob.bio.base.load_resource(
# 'batl-db',
# 'database',
# preferred_package='bob.pad.face',
# package_prefix='bob.pad.')
# try:
# assert len(
# batl_db.objects(groups=['train', 'dev', 'eval'])) == 1679
# assert len(batl_db.objects(groups=['train', 'dev'])) == 1122
# assert len(batl_db.objects(groups=['train'])) == 565
# assert len(batl_db.objects(groups='train')) == 565
# assert len(batl_db.objects(groups='dev')) == 557
# assert len(batl_db.objects(groups='eval')) == 557
# assert len(
# batl_db.objects(
# groups=['train', 'dev', 'eval'], protocol='grandtest')) == 1679
# assert len(
# batl_db.objects(
# groups=['train', 'dev', 'eval'],
# protocol='grandtest',
# purposes='real')) == 347
# assert len(
# batl_db.objects(
# groups=['train', 'dev', 'eval'],
# protocol='grandtest',
# purposes='attack')) == 1332
# #tests for join_train_dev protocols
# assert len(
# batl_db.objects(
# groups=['train', 'dev', 'eval'],
# protocol='grandtest-color-50-join_train_dev')) == 1679
# assert len(
# batl_db.objects(
# groups=['train', 'dev'], protocol='grandtest-color-50-join_train_dev')) == 1679
# assert len(
# batl_db.objects(groups='eval',
# protocol='grandtest-color-50-join_train_dev')) == 557
# # test for LOO_fakehead
# assert len(
# batl_db.objects(
# groups=['train', 'dev', 'eval'],
# protocol='grandtest-color-50-LOO_fakehead')) == 1149
# assert len(
# batl_db.objects(
# groups=['train', 'dev'], protocol='grandtest-color-50-LOO_fakehead')) == 1017
# assert len(
# batl_db.objects(groups='eval',
# protocol='grandtest-color-50-LOO_fakehead')) == 132
# # test for LOO_flexiblemask
# assert len(
# batl_db.objects(
# groups=['train', 'dev', 'eval'],
# protocol='grandtest-color-50-LOO_flexiblemask')) == 1132
# assert len(
# batl_db.objects(
# groups=['train', 'dev'], protocol='grandtest-color-50-LOO_flexiblemask')) == 880
# assert len(
# batl_db.objects(groups='eval',
# protocol='grandtest-color-50-LOO_flexiblemask')) == 252
# # test for LOO_glasses
# assert len(
# batl_db.objects(
# groups=['train', 'dev', 'eval'],
# protocol='grandtest-color-50-LOO_glasses')) == 1206
# assert len(
# batl_db.objects(
# groups=['train', 'dev'], protocol='grandtest-color-50-LOO_glasses')) == 1069
# assert len(
# batl_db.objects(groups='eval',
# protocol='grandtest-color-50-LOO_glasses')) == 137
# # test for LOO_papermask
# assert len(
# batl_db.objects(
# groups=['train', 'dev', 'eval'],
# protocol='grandtest-color-50-LOO_papermask')) == 1308
# assert len(
# batl_db.objects(
# groups=['train', 'dev'], protocol='grandtest-color-50-LOO_papermask')) == 1122
# assert len(
# batl_db.objects(groups='eval',
# protocol='grandtest-color-50-LOO_papermask')) == 186
# # test for LOO_prints
# assert len(
# batl_db.objects(
# groups=['train', 'dev', 'eval'],
# protocol='grandtest-color-50-LOO_prints')) == 1169
# assert len(
# batl_db.objects(
# groups=['train', 'dev'], protocol='grandtest-color-50-LOO_prints')) == 988
# assert len(
# batl_db.objects(groups='eval',
# protocol='grandtest-color-50-LOO_prints')) == 181
# # test for LOO_replay
# assert len(
# batl_db.objects(
# groups=['train', 'dev', 'eval'],
# protocol='grandtest-color-50-LOO_replay')) == 1049
# assert len(
# batl_db.objects(
# groups=['train', 'dev'], protocol='grandtest-color-50-LOO_replay')) == 854
# assert len(
# batl_db.objects(groups='eval',
# protocol='grandtest-color-50-LOO_replay')) == 195
# # test for LOO_rigidmask
# assert len(
# batl_db.objects(
# groups=['train', 'dev', 'eval'],
# protocol='grandtest-color-50-LOO_rigidmask')) == 1198
# assert len(
# batl_db.objects(
# groups=['train', 'dev'], protocol='grandtest-color-50-LOO_rigidmask')) == 1034
# assert len(
# batl_db.objects(groups='eval',
# protocol='grandtest-color-50-LOO_rigidmask')) == 164
# except IOError as e:
# raise SkipTest(
# "The database could not be queried; probably the db.sql3 file is missing. Here is the error: '%s'"
# % e)
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment