Skip to content
Snippets Groups Projects
Verified Commit 1ffc8bd3 authored by Yannick DAYER's avatar Yannick DAYER
Browse files

fix: download no longer tries to get existing files

parent e2fa582c
Branches
Tags
1 merge request!329Download no longer tries to get existing files
Pipeline #89062 passed
...@@ -561,13 +561,20 @@ def download_file( ...@@ -561,13 +561,20 @@ def download_file(
local_file = destination_directory / destination_filename local_file = destination_directory / destination_filename
needs_download = True needs_download = True
if not force and local_file.is_file(): if force or not local_file.is_file():
logger.info( if not force:
"File %s already exists, skipping download (force=%s).", logger.info(f"File {local_file} is not present. Needs download.")
local_file, needs_download = True
force, elif local_file.is_file():
) file_ok = verify_file(local_file, checksum, hash_fct=checksum_fct)
needs_download = False if not file_ok:
logger.info(
f"File {local_file} does not checksum to '{checksum=}'."
)
needs_download = True
elif not force and checksum is not None and file_ok:
logger.info(f"File {local_file} already exists, skipping download.")
needs_download = False
if needs_download: if needs_download:
for current_download_try in range(checksum_mismatch_download_attempts): for current_download_try in range(checksum_mismatch_download_attempts):
...@@ -613,31 +620,19 @@ def download_file( ...@@ -613,31 +620,19 @@ def download_file(
with local_file.open("wb") as f: with local_file.open("wb") as f:
f.write(response.content) f.write(response.content)
# Check the created file integrity, re-download if needed if checksum is not None:
if checksum is None or verify_file( if not verify_file(local_file, checksum, hash_fct=checksum_fct):
local_file, checksum, hash_fct=checksum_fct if not needs_download:
): raise ValueError(
break # Exit the re-download loop f"The local file hash does not correspond to '{checksum}' "
logger.warning( f"and {force=} prevents overwriting."
"Downloading %s created a file with a wrong checksum. Retry %d", )
url,
current_download_try + 1,
)
if current_download_try >= checksum_mismatch_download_attempts - 1:
raise ValueError( raise ValueError(
"The downloaded file hash " "The downloaded file hash ('"
f"({compute_crc(local_file, hash_fct=checksum_fct)}) for " f"{compute_crc(local_file, hash_fct=checksum_fct)}') does not "
f"'{url}' does not correspond to '{checksum}', even after " f"correspond to '{checksum}'."
f"{checksum_mismatch_download_attempts} retries."
) )
elif checksum is not None:
if not verify_file(local_file, checksum, hash_fct=checksum_fct):
raise ValueError(
f"The local file hash does not correspond to '{checksum}' and "
f"{force=} prevents overwriting."
)
if extract: if extract:
# Extract only if the file was re-downloaded # Extract only if the file was re-downloaded
if needs_download: if needs_download:
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment