Merge branch 'batl_LOO_protocols' into 'master'

Batl loo protocols

See merge request !86

bob/pad/face/database/batl.py

@@ -202,6 +202,14 @@ class BatlPadDatabase(PadDatabase):
             infrared data only, use 50 frames, join train and dev sets forming
             a single large training set.
 
+            "grandtest-infrared-50-LOO_<unseen_attack>", for example "grandtest-infrared-50-LOO_fakehead"
+            - Leave One Out (LOO) protocol with fakehead attacks present only in the `test` set. The original partitioning
+            in the `grandtest` protocol is queried first and subselects the file list such
+            that the specified `unknown_attack` is removed from both `train` and `dev` sets. 
+            The `test` set will consist of only the selected `unknown_attack` and `bonafide` files.
+            This protocol is used to evaluate the robustness against attacks unseen in training. 
+            .
+
             "grandtest-color*infrared-50" - baseline protocol,
             load both "color" and "infrared" channels,
             use 50 frames.
@@ -286,6 +294,15 @@ class BatlPadDatabase(PadDatabase):
         self.append_color_face_roi_annot = append_color_face_roi_annot
 
 
+        #map file to identify attack grouping
+        map_file = pkg_resources.resource_filename('bob.pad.face', 'lists/batl/idiap_converter_v2.json')
+
+        with open(map_file, 'r') as fp:
+            map_dict=json.load(fp)
+
+        self.map_dict=map_dict
+
+
     @property
     def original_directory(self):
         return self.db.original_directory
@@ -295,6 +312,52 @@ class BatlPadDatabase(PadDatabase):
     def original_directory(self, value):
         self.db.original_directory = value
 
+    def unseen_attack_list_maker(self,files,unknown_attack,train=True):
+        """
+        Selects and returns a list of files for Leave One Out (LOO) protocols. 
+        This utilizes the original partitioning in the `grandtest` protocol and subselects 
+        the file list such that the specified `unknown_attack` is removed from both `train` and `dev` sets. 
+        The `test` set will consist of only the selected `unknown_attack` and `bonafide` files.
+
+        **Parameters:**
+
+        ``files`` : pyclass::list
+            A list of files, db.objects()
+        ``unknown_attack`` : str
+            The unknown attack protocol name example:'rigidmask' .
+        ``train`` : bool
+            Denotes whether files are from training or development partition
+
+        **Returns:**
+
+        ``mod_files`` : pyclass::list
+            A list of files selected for the protocol
+
+        """
+
+
+        mod_files=[]
+
+
+        for file in files:
+
+            attack_category=self.map_dict["_"+"_".join(os.path.split(file.path)[-1].split("_")[-2:])].split("_")[-1]
+
+            if train:
+                if  attack_category==unknown_attack:
+                    pass
+                else:
+                    mod_files.append(file) # everything except the attack specified is there 
+
+            if not train:
+
+                if  attack_category==unknown_attack or attack_category=='bonafide':
+                    mod_files.append(file) # only the attack mentioned and bonafides in testing
+                else:
+                    pass
+
+
+        return mod_files
 
     def parse_protocol(self, protocol):
         """
@@ -329,7 +392,7 @@ class BatlPadDatabase(PadDatabase):
             forming a single training set.
         """
 
-        possible_extras = ['join_train_dev']
+        possible_extras = ['join_train_dev','LOO_fakehead','LOO_flexiblemask','LOO_glasses','LOO_papermask','LOO_prints','LOO_replay','LOO_rigidmask','LOO_makeup']
 
         components = protocol.split("-")
 
@@ -490,10 +553,11 @@ class BatlPadDatabase(PadDatabase):
         protocol = 'nowig' if protocol == 'grandtest' else protocol
 
         # Convert group names to low-level group names here.
-        groups = self.convert_names_to_lowlevel(
-            groups, self.low_level_group_names, self.high_level_group_names)
+        groups = self.convert_names_to_lowlevel(groups, self.low_level_group_names, self.high_level_group_names)
 
-        if not isinstance(groups, list) and groups is not None:  # if a single group is given make it a list
+            
+
+        if not isinstance(groups, list) and groups is not None and not isinstance(groups,str):  # if a single group is given make it a list
             groups = list(groups)
 
         if extra is not None and "join_train_dev" in extra:
@@ -515,6 +579,48 @@ class BatlPadDatabase(PadDatabase):
                                         groups=['test'],
                                         purposes=purposes, **kwargs)
 
+
+        # for the LOO protocols
+        elif extra is not None and "LOO_" in extra:
+
+            batl_files=[]
+
+            unknown_attack=extra.split("_")[-1]
+
+            if 'train' in groups:
+                tbatl_files = self.db.objects(protocol=protocol,
+                                        groups=['train'],
+                                        purposes=purposes, **kwargs)
+
+                tbatl_files=self.unseen_attack_list_maker(tbatl_files,unknown_attack,train=True)
+
+                batl_files=batl_files+tbatl_files
+
+
+            if 'validation' in groups: 
+                tbatl_files = self.db.objects(protocol=protocol,
+                                        groups=['validation'],
+                                        purposes=purposes, **kwargs)
+
+                tbatl_files=self.unseen_attack_list_maker(tbatl_files,unknown_attack,train=True)
+
+
+                batl_files=batl_files+tbatl_files
+
+            if 'test' in groups:
+                tbatl_files = self.db.objects(protocol=protocol,
+                                        groups=['test'],
+                                        purposes=purposes, **kwargs)
+
+                tbatl_files=self.unseen_attack_list_maker(tbatl_files,unknown_attack,train=False)
+
+                batl_files=batl_files+tbatl_files
+
+
+            files=batl_files
+
+
+
         else:
 #            files = self._fix_funny_eyes_in_objects(protocol=protocol,
 #                                                    groups=groups,

bob/pad/face/lists/batl/idiap_converter_v2.json

+{"_0_00": "Unknown_bonafide", "_0_01": "Clientownmedicalglasse_bonafide", "_0_02": "UnisexglassesG01_bonafide", "_1_01": "FunnyeyesglassesG02_glasses", "_1_02": "PlastichalloweenmaskB082200_rigidmask", "_1_03": "unassigne_glasses", "_1_04": "FunnyeyesglassesG03_glasses", "_1_05": "FunnyeyesglassesG04_glasses", "_1_06": "FunnyeyesglassesG05_glasses", "_1_07": "FunnyeyesglassesG06_glasses", "_1_13": "ExoticfemailtransparentmaskB0853200_rigidmask", "_1_14": "ExoticfemailtransparentmaskwithmakeupB082301_rigidmask", "_1_15": "PaperglassesG10_glasses", "_1_16": "PaperglassesG11_glasses", "_2_01": "MannequinMq07_fakehead", "_2_02": "MannequinMq01_fakehead", "_2_03": "MannequinMq02_fakehead", "_2_04": "MannequinMq03_fakehead", "_2_05": "MannequinMq04_fakehead", "_2_06": "MannequinMq05_fakehead", "_2_07": "MannequinMq06_fakehead", "_2_08": "PapercraftmaskB030100_papermask", "_2_09": "PapercraftmaskB030300_papermask", "_2_10": "PapercraftmaskB031100_papermask", "_2_11": "PapercraftmaskB031200_papermask", "_2_12": "CustomwearablemaskB050100_rigidmask", "_2_13": "CustomwearablemaskB050200_rigidmask", "_2_14": "CustomwearablemaskB051200_rigidmask", "_2_15": "CustomwearablemaskB051600_rigidmask", "_2_16": "CustomwearablemaskB051800_rigidmask", "_2_17": "CustomwearablemaskB051900_rigidmask", "_2_18": "FlexiblesiliconmaskB072100_flexiblemask", "_2_75": "FlexiblesiliconmaskB063000_flexiblemask", "_2_76": "FlexiblesiliconmaskB063100_flexiblemask", "_2_77": "FlexiblesiliconmaskB073200_flexiblemask", "_2_78": "FlexiblesiliconmaskB073300_flexiblemask", "_2_79": "FlexiblesiliconmaskB073400_flexiblemask", "_2_80": "FlexiblesiliconmaskB060100_flexiblemask", "_2_81": "FlexiblesiliconmaskB060200_flexiblemask", "_2_82": "FlexiblesiliconmaskB060202_flexiblemask", "_2_83": "FlexiblesiliconmaskB061200_flexiblemask", "_2_84": "FlexiblesiliconmaskB062400_flexiblemask", "_2_85": "FlexiblesiliconmaskB061600_flexiblemask", "_2_86": "FlexiblesiliconmaskB061800_flexiblemask", "_2_87": "FlexiblesiliconmaskB061900_flexiblemask", "_2_88": "FlexiblesiliconmaskB070100_flexiblemask", "_2_89": "FlexiblesiliconmaskB071200_flexiblemask", "_2_90": "FlexiblesiliconmaskB071900_flexiblemask", "_2_91": "FlexiblesiliconmaskB062500_flexiblemask", "_2_92": "FlexiblesiliconmaskB062601_flexiblemask", "_2_93": "FlexiblesiliconmaskB072701_flexiblemask", "_2_94": "FlexiblesiliconmaskB072801_flexiblemask", "_2_95": "FlexiblesiliconmaskB072901_flexiblemask", "_2_96": "FlexiblesiliconmaskB062600_flexiblemask", "_2_97": "FlexiblesiliconmaskB072700_flexiblemask", "_2_98": "FlexiblesiliconmaskB072800_flexiblemask", "_2_99": "FlexiblesiliconmaskB072900_flexiblemask", "_3_01": "PrintedP0000100_prints", "_3_02": "PrintedP0000101_prints", "_3_03": "PrintedP0000102_prints", "_3_04": "PrintedP0000103_prints", "_3_05": "ElectronicP0100100_replay", "_3_06": "PrintedP0001800_prints", "_3_07": "PrintedP0001801_prints", "_3_08": "PrintedP0001802_prints", "_3_09": "PrintedP0001803_prints", "_3_10": "ElectronicP0101800_replay", "_3_11": "PrintedP0009800_prints", "_3_12": "PrintedP0009801_prints", "_3_13": "PrintedP0009802_prints", "_3_14": "PrintedP0009803_prints", "_3_15": "ElectronicP0109800_replay", "_4_01": "VideoplayV0001900_replay", "_4_02": "VideopauseV0101900_replay", "_4_03": "VideoplayV0001200_replay", "_4_04": "VideopauseV0101200_replay", "_4_05": "VideoplayV0009800_replay", "_4_06": "VideopauseV0109800_replay", "_4_07": "VideoplayV0001901_replay", "_4_08": "VideopauseV0101901_replay", "_4_09": "VideoplayV0000501_replay", "_4_10": "VideopauseV0100501_replay", "_4_11": "VideoplayV0009801_replay", "_4_12": "VideopauseV0109801_replay", "_4_13": "VideoplayV0101902_replay", "_4_14": "VideopauseV0101902_replay", "_4_15": "VideoplayV0101202_replay", "_4_16": "VideopauseV0101202_replay", "_4_17": "VideoplayV0109802_replay", "_4_18": "VideopauseV0109802_replay", "_5_01": "Makeup_makeup", "_5_02": "Makeup_makeup", "_5_03": "Makeup_makeup", "_5_04": "Makeup_makeup", "_5_05": "Makeup_makeup", "_5_06": "Makeup_makeup", "_5_07": "Makeup_makeup", "_5_08": "Makeup_makeup", "_5_09": "Makeup_makeup", "_5_10": "Makeup_makeup", "_5_100": "Makeup_makeup", "_5_101": "Makeup_makeup", "_5_102": "Makeup_makeup", "_5_103": "Makeup_makeup", "_5_104": "Makeup_makeup", "_5_105": "Makeup_makeup", "_5_106": "Makeup_makeup", "_5_107": "Makeup_makeup", "_5_108": "Makeup_makeup", "_5_109": "Makeup_makeup", "_5_11": "Makeup_makeup", "_5_110": "Makeup_makeup", "_5_111": "Makeup_makeup", "_5_112": "Makeup_makeup", "_5_113": "Makeup_makeup", "_5_114": "Makeup_makeup", "_5_115": "Makeup_makeup", "_5_116": "Makeup_makeup", "_5_117": "Makeup_makeup", "_5_118": "Makeup_makeup", "_5_119": "Makeup_makeup", "_5_12": "Makeup_makeup", "_5_120": "Makeup_makeup", "_5_121": "Makeup_makeup", "_5_122": "Makeup_makeup", "_5_123": "Makeup_makeup", "_5_124": "Makeup_makeup", "_5_125": "Makeup_makeup", "_5_126": "Makeup_makeup", "_5_127": "Makeup_makeup", "_5_128": "Makeup_makeup", "_5_129": "Makeup_makeup", "_5_13": "Makeup_makeup", "_5_130": "Makeup_makeup", "_5_131": "Makeup_makeup", "_5_132": "Makeup_makeup", "_5_133": "Makeup_makeup", "_5_134": "Makeup_makeup", "_5_135": "Makeup_makeup", "_5_136": "Makeup_makeup", "_5_137": "Makeup_makeup", "_5_138": "Makeup_makeup", "_5_139": "Makeup_makeup", "_5_14": "Makeup_makeup", "_5_15": "Makeup_makeup", "_5_16": "Makeup_makeup", "_5_17": "Makeup_makeup", "_5_18": "Makeup_makeup", "_5_19": "Makeup_makeup", "_5_20": "Makeup_makeup", "_5_21": "Makeup_makeup", "_5_22": "Makeup_makeup", "_5_23": "Makeup_makeup", "_5_24": "Makeup_makeup", "_5_25": "Makeup_makeup", "_5_26": "Makeup_makeup", "_5_27": "Makeup_makeup", "_5_28": "Makeup_makeup", "_5_29": "Makeup_makeup", "_5_30": "Makeup_makeup", "_5_31": "Makeup_makeup", "_5_32": "Makeup_makeup", "_5_33": "Makeup_makeup", "_5_34": "Makeup_makeup", "_5_35": "Makeup_makeup", "_5_36": "Makeup_makeup", "_5_37": "Makeup_makeup", "_5_38": "Makeup_makeup", "_5_39": "Makeup_makeup", "_5_40": "Makeup_makeup", "_5_41": "Makeup_makeup", "_5_42": "Makeup_makeup", "_5_43": "Makeup_makeup", "_5_44": "Makeup_makeup", "_5_45": "Makeup_makeup", "_5_46": "Makeup_makeup", "_5_47": "Makeup_makeup", "_5_48": "Makeup_makeup", "_5_49": "Makeup_makeup", "_5_50": "Makeup_makeup", "_5_51": "Makeup_makeup", "_5_52": "Makeup_makeup", "_5_53": "Makeup_makeup", "_5_54": "Makeup_makeup", "_5_55": "Makeup_makeup", "_5_56": "Makeup_makeup", "_5_57": "Makeup_makeup", "_5_58": "Makeup_makeup", "_5_59": "Makeup_makeup", "_5_60": "Makeup_makeup", "_5_61": "Makeup_makeup", "_5_62": "Makeup_makeup", "_5_63": "Makeup_makeup", "_5_64": "Makeup_makeup", "_5_65": "Makeup_makeup", "_5_66": "Makeup_makeup", "_5_67": "Makeup_makeup", "_5_68": "Makeup_makeup", "_5_69": "Makeup_makeup", "_5_70": "Makeup_makeup", "_5_71": "Makeup_makeup", "_5_72": "Makeup_makeup", "_5_73": "Makeup_makeup", "_5_74": "Makeup_makeup", "_5_75": "Makeup_makeup", "_5_76": "Makeup_makeup", "_5_77": "Makeup_makeup", "_5_78": "Makeup_makeup", "_5_79": "Makeup_makeup", "_5_80": "Makeup_makeup", "_5_81": "Makeup_makeup", "_5_82": "Makeup_makeup", "_5_83": "Makeup_makeup", "_5_84": "Makeup_makeup", "_5_85": "Makeup_makeup", "_5_86": "Makeup_makeup", "_5_87": "Makeup_makeup", "_5_88": "Makeup_makeup", "_5_89": "Makeup_makeup", "_5_90": "Makeup_makeup", "_5_91": "Makeup_makeup", "_5_92": "Makeup_makeup", "_5_93": "Makeup_makeup", "_5_94": "Makeup_makeup", "_5_95": "Makeup_makeup", "_5_96": "Makeup_makeup", "_5_97": "Makeup_makeup", "_5_98": "Makeup_makeup", "_5_99": "Makeup_makeup", "_1_08": "Wig", "_1_09": "Wig", "_1_10": "Wig", "_1_11": "Wig", "_1_12": "Wig"}
\ No newline at end of file

bob/pad/face/test/test_databases.py

@@ -219,3 +219,136 @@ def test_casiasurf():
         raise SkipTest(
             "The database could not be queried; probably the db.sql3 file is missing. Here is the error: '%s'"
             % e)
+
+# # Test the BATL database
+# @db_available('batl-db')
+# def test_aggregated_db():
+#     batl_db = bob.bio.base.load_resource(
+#         'batl-db',
+#         'database',
+#         preferred_package='bob.pad.face',
+#         package_prefix='bob.pad.')
+#     try:
+
+#         assert len(
+#             batl_db.objects(groups=['train', 'dev', 'eval'])) == 1679
+#         assert len(batl_db.objects(groups=['train', 'dev'])) == 1122
+#         assert len(batl_db.objects(groups=['train'])) == 565
+
+#         assert len(batl_db.objects(groups='train')) == 565
+#         assert len(batl_db.objects(groups='dev')) == 557
+#         assert len(batl_db.objects(groups='eval')) == 557
+
+#         assert len(
+#             batl_db.objects(
+#                 groups=['train', 'dev', 'eval'], protocol='grandtest')) == 1679
+#         assert len(
+#             batl_db.objects(
+#                 groups=['train', 'dev', 'eval'],
+#                 protocol='grandtest',
+#                 purposes='real')) == 347
+#         assert len(
+#             batl_db.objects(
+#                 groups=['train', 'dev', 'eval'],
+#                 protocol='grandtest',
+#                 purposes='attack')) == 1332
+#         #tests for join_train_dev protocols
+#         assert len(
+#             batl_db.objects(
+#                 groups=['train', 'dev', 'eval'],
+#                 protocol='grandtest-color-50-join_train_dev')) == 1679
+#         assert len(
+#             batl_db.objects(
+#                 groups=['train', 'dev'], protocol='grandtest-color-50-join_train_dev')) == 1679
+#         assert len(
+#             batl_db.objects(groups='eval',
+#                                   protocol='grandtest-color-50-join_train_dev')) == 557
+#         # test for LOO_fakehead
+#         assert len(
+#             batl_db.objects(
+#                 groups=['train', 'dev', 'eval'],
+#                 protocol='grandtest-color-50-LOO_fakehead')) == 1149
+#         assert len(
+#             batl_db.objects(
+#                 groups=['train', 'dev'], protocol='grandtest-color-50-LOO_fakehead')) == 1017
+#         assert len(
+#             batl_db.objects(groups='eval',
+#                                   protocol='grandtest-color-50-LOO_fakehead')) == 132
+
+#         # test for LOO_flexiblemask
+#         assert len(
+#             batl_db.objects(
+#                 groups=['train', 'dev', 'eval'],
+#                 protocol='grandtest-color-50-LOO_flexiblemask')) == 1132
+#         assert len(
+#             batl_db.objects(
+#                 groups=['train', 'dev'], protocol='grandtest-color-50-LOO_flexiblemask')) == 880
+#         assert len(
+#             batl_db.objects(groups='eval',
+#                                   protocol='grandtest-color-50-LOO_flexiblemask')) == 252
+
+#         # test for LOO_glasses
+#         assert len(
+#             batl_db.objects(
+#                 groups=['train', 'dev', 'eval'],
+#                 protocol='grandtest-color-50-LOO_glasses')) == 1206
+#         assert len(
+#             batl_db.objects(
+#                 groups=['train', 'dev'], protocol='grandtest-color-50-LOO_glasses')) == 1069
+#         assert len(
+#             batl_db.objects(groups='eval',
+#                                   protocol='grandtest-color-50-LOO_glasses')) == 137
+
+#         # test for LOO_papermask
+#         assert len(
+#             batl_db.objects(
+#                 groups=['train', 'dev', 'eval'],
+#                 protocol='grandtest-color-50-LOO_papermask')) == 1308
+#         assert len(
+#             batl_db.objects(
+#                 groups=['train', 'dev'], protocol='grandtest-color-50-LOO_papermask')) == 1122
+#         assert len(
+#             batl_db.objects(groups='eval',
+#                                   protocol='grandtest-color-50-LOO_papermask')) == 186
+
+#         # test for LOO_prints
+#         assert len(
+#             batl_db.objects(
+#                 groups=['train', 'dev', 'eval'],
+#                 protocol='grandtest-color-50-LOO_prints')) == 1169
+#         assert len(
+#             batl_db.objects(
+#                 groups=['train', 'dev'], protocol='grandtest-color-50-LOO_prints')) == 988
+#         assert len(
+#             batl_db.objects(groups='eval',
+#                                   protocol='grandtest-color-50-LOO_prints')) == 181
+
+#         # test for LOO_replay
+#         assert len(
+#             batl_db.objects(
+#                 groups=['train', 'dev', 'eval'],
+#                 protocol='grandtest-color-50-LOO_replay')) == 1049
+#         assert len(
+#             batl_db.objects(
+#                 groups=['train', 'dev'], protocol='grandtest-color-50-LOO_replay')) == 854
+#         assert len(
+#             batl_db.objects(groups='eval',
+#                                   protocol='grandtest-color-50-LOO_replay')) == 195
+
+#         # test for LOO_rigidmask
+#         assert len(
+#             batl_db.objects(
+#                 groups=['train', 'dev', 'eval'],
+#                 protocol='grandtest-color-50-LOO_rigidmask')) == 1198
+#         assert len(
+#             batl_db.objects(
+#                 groups=['train', 'dev'], protocol='grandtest-color-50-LOO_rigidmask')) == 1034
+#         assert len(
+#             batl_db.objects(groups='eval',
+#                                   protocol='grandtest-color-50-LOO_rigidmask')) == 164
+
+
+#     except IOError as e:
+#         raise SkipTest(
+#             "The database could not be queried; probably the db.sql3 file is missing. Here is the error: '%s'"
+#             % e)
\ No newline at end of file