Merge branch 'data_improve_file_handling' into 'master'

Improve file name handling and removal in CachedDataSink

See merge request !70

beat/backend/python/data.py

@@ -496,12 +496,13 @@ class CachedDataSource(DataSource):
 
             try:
                 self.current_file = open(self.filenames[infos.file_index], "rb")
-                self.current_file_index = infos.file_index
             except Exception as e:
                 raise IOError(
                     "Could not read `%s': %s" % (self.filenames[infos.file_index], e)
                 )
 
+            self.current_file_index = infos.file_index
+
         self.current_file.seek(infos.offset, 0)
 
         t1 = time.time()
@@ -939,8 +940,8 @@ class CachedDataSink(DataSink):
             logger.error("Failed to open data file {}: {}".format(self.filename, e))
             return False
 
-        try:
         index_filename = self.filename.replace(".data", ".index")
+        try:
             self.index_file = open(index_filename, "wt")
         except Exception as e:
             logger.error("Failed to open index file {}: {}".format(index_filename, e))
@@ -960,6 +961,9 @@ class CachedDataSink(DataSink):
             self.data_file.close()
             self.index_file.close()
 
+            data_filename = self.data_file.name
+            index_filename = self.index_file.name
+
             # If file is not complete, delete it
             if (self.last_written_data_index is None) or (
                 self.last_written_data_index < self.end_index
@@ -973,19 +977,21 @@ class CachedDataSink(DataSink):
 
                 logger.warning("Removing cache files: {}".format(message))
 
+                artifacts_removed = True
+
+                for filename in [data_filename, index_filename]:
                     try:
-                    os.remove(self.filename)
-                    os.remove(self.filename.replace(".data", ".index"))
-                    return True
-                except Exception:
-                    return False
+                        os.remove(filename)
+                    except Exception as e:
+                        logger.warning("Failed to remove {}: {}".format(filename, e))
+                        artifacts_removed = False
+                return artifacts_removed
 
             # Creates the checksums for all data and indexes
-            chksum_data = hashFileContents(self.filename)
-            with open(self.filename + ".checksum", "wt") as f:
+            chksum_data = hashFileContents(data_filename)
+            with open(data_filename + ".checksum", "wt") as f:
                 f.write(chksum_data)
 
-            index_filename = self.filename.replace(".data", ".index")
             chksum_index = hashFileContents(index_filename)
             with open(index_filename + ".checksum", "wt") as f:
                 f.write(chksum_index)