diff --git a/bob/bio/base/annotator/FailSafe.py b/bob/bio/base/annotator/FailSafe.py
index 6b9cec696a1ffac825b66166617e9f32e8291157..04a443a9327bde19cdb0c93d19ee46328e6a8c5e 100644
--- a/bob/bio/base/annotator/FailSafe.py
+++ b/bob/bio/base/annotator/FailSafe.py
@@ -42,4 +42,7 @@ class FailSafe(Base):
             # check if we have all the required annotations
             if all(key in kwargs['annotations'] for key in self.required_keys):
                 break
+        else:  # this else is for the for loop
+            # we don't want to return half of the annotations
+            kwargs['annotations'] = {}
         return kwargs['annotations']