Skip to content
Snippets Groups Projects
Commit d5cf537b authored by Flavio TARSETTI's avatar Flavio TARSETTI
Browse files

Merge branch 'data_improve_file_handling' into 'master'

Improve file name handling and removal in CachedDataSink

See merge request !70
parents 928a2797 0500977a
Branches
Tags
1 merge request!70Improve file name handling and removal in CachedDataSink
Pipeline #38049 passed
...@@ -496,12 +496,13 @@ class CachedDataSource(DataSource): ...@@ -496,12 +496,13 @@ class CachedDataSource(DataSource):
try: try:
self.current_file = open(self.filenames[infos.file_index], "rb") self.current_file = open(self.filenames[infos.file_index], "rb")
self.current_file_index = infos.file_index
except Exception as e: except Exception as e:
raise IOError( raise IOError(
"Could not read `%s': %s" % (self.filenames[infos.file_index], e) "Could not read `%s': %s" % (self.filenames[infos.file_index], e)
) )
self.current_file_index = infos.file_index
self.current_file.seek(infos.offset, 0) self.current_file.seek(infos.offset, 0)
t1 = time.time() t1 = time.time()
...@@ -939,8 +940,8 @@ class CachedDataSink(DataSink): ...@@ -939,8 +940,8 @@ class CachedDataSink(DataSink):
logger.error("Failed to open data file {}: {}".format(self.filename, e)) logger.error("Failed to open data file {}: {}".format(self.filename, e))
return False return False
try:
index_filename = self.filename.replace(".data", ".index") index_filename = self.filename.replace(".data", ".index")
try:
self.index_file = open(index_filename, "wt") self.index_file = open(index_filename, "wt")
except Exception as e: except Exception as e:
logger.error("Failed to open index file {}: {}".format(index_filename, e)) logger.error("Failed to open index file {}: {}".format(index_filename, e))
...@@ -960,6 +961,9 @@ class CachedDataSink(DataSink): ...@@ -960,6 +961,9 @@ class CachedDataSink(DataSink):
self.data_file.close() self.data_file.close()
self.index_file.close() self.index_file.close()
data_filename = self.data_file.name
index_filename = self.index_file.name
# If file is not complete, delete it # If file is not complete, delete it
if (self.last_written_data_index is None) or ( if (self.last_written_data_index is None) or (
self.last_written_data_index < self.end_index self.last_written_data_index < self.end_index
...@@ -973,19 +977,21 @@ class CachedDataSink(DataSink): ...@@ -973,19 +977,21 @@ class CachedDataSink(DataSink):
logger.warning("Removing cache files: {}".format(message)) logger.warning("Removing cache files: {}".format(message))
artifacts_removed = True
for filename in [data_filename, index_filename]:
try: try:
os.remove(self.filename) os.remove(filename)
os.remove(self.filename.replace(".data", ".index")) except Exception as e:
return True logger.warning("Failed to remove {}: {}".format(filename, e))
except Exception: artifacts_removed = False
return False return artifacts_removed
# Creates the checksums for all data and indexes # Creates the checksums for all data and indexes
chksum_data = hashFileContents(self.filename) chksum_data = hashFileContents(data_filename)
with open(self.filename + ".checksum", "wt") as f: with open(data_filename + ".checksum", "wt") as f:
f.write(chksum_data) f.write(chksum_data)
index_filename = self.filename.replace(".data", ".index")
chksum_index = hashFileContents(index_filename) chksum_index = hashFileContents(index_filename)
with open(index_filename + ".checksum", "wt") as f: with open(index_filename + ".checksum", "wt") as f:
f.write(chksum_index) f.write(chksum_index)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment