Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
bob.bio.base
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Package registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
bob
bob.bio.base
Commits
1ffc8bd3
Verified
Commit
1ffc8bd3
authored
1 year ago
by
Yannick DAYER
Browse files
Options
Downloads
Patches
Plain Diff
fix: download no longer tries to get existing files
parent
e2fa582c
Branches
Branches containing commit
Tags
Tags containing commit
1 merge request
!329
Download no longer tries to get existing files
Pipeline
#89062
passed
11 months ago
Stage: qa
Stage: test
Stage: doc
Stage: dist
Changes
1
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
src/bob/bio/base/database/utils.py
+24
-29
24 additions, 29 deletions
src/bob/bio/base/database/utils.py
with
24 additions
and
29 deletions
src/bob/bio/base/database/utils.py
+
24
−
29
View file @
1ffc8bd3
...
@@ -561,13 +561,20 @@ def download_file(
...
@@ -561,13 +561,20 @@ def download_file(
local_file
=
destination_directory
/
destination_filename
local_file
=
destination_directory
/
destination_filename
needs_download
=
True
needs_download
=
True
if
not
force
and
local_file
.
is_file
():
if
force
or
not
local_file
.
is_file
():
logger
.
info
(
if
not
force
:
"
File %s already exists, skipping download (force=%s).
"
,
logger
.
info
(
f
"
File
{
local_file
}
is not present. Needs download.
"
)
local_file
,
needs_download
=
True
force
,
elif
local_file
.
is_file
():
)
file_ok
=
verify_file
(
local_file
,
checksum
,
hash_fct
=
checksum_fct
)
needs_download
=
False
if
not
file_ok
:
logger
.
info
(
f
"
File
{
local_file
}
does not checksum to
'
{
checksum
=
}
'
.
"
)
needs_download
=
True
elif
not
force
and
checksum
is
not
None
and
file_ok
:
logger
.
info
(
f
"
File
{
local_file
}
already exists, skipping download.
"
)
needs_download
=
False
if
needs_download
:
if
needs_download
:
for
current_download_try
in
range
(
checksum_mismatch_download_attempts
):
for
current_download_try
in
range
(
checksum_mismatch_download_attempts
):
...
@@ -613,31 +620,19 @@ def download_file(
...
@@ -613,31 +620,19 @@ def download_file(
with
local_file
.
open
(
"
wb
"
)
as
f
:
with
local_file
.
open
(
"
wb
"
)
as
f
:
f
.
write
(
response
.
content
)
f
.
write
(
response
.
content
)
# Check the created file integrity, re-download if needed
if
checksum
is
not
None
:
if
checksum
is
None
or
verify_file
(
if
not
verify_file
(
local_file
,
checksum
,
hash_fct
=
checksum_fct
):
local_file
,
checksum
,
hash_fct
=
checksum_fct
if
not
needs_download
:
):
raise
ValueError
(
break
# Exit the re-download loop
f
"
The local file hash does not correspond to
'
{
checksum
}
'
"
logger
.
warning
(
f
"
and
{
force
=
}
prevents overwriting.
"
"
Downloading %s created a file with a wrong checksum. Retry %d
"
,
)
url
,
current_download_try
+
1
,
)
if
current_download_try
>=
checksum_mismatch_download_attempts
-
1
:
raise
ValueError
(
raise
ValueError
(
"
The downloaded file hash
"
"
The downloaded file hash (
'"
f
"
(
{
compute_crc
(
local_file
,
hash_fct
=
checksum_fct
)
}
) for
"
f
"
{
compute_crc
(
local_file
,
hash_fct
=
checksum_fct
)
}
'
) does not
"
f
"'
{
url
}
'
does not correspond to
'
{
checksum
}
'
, even after
"
f
"
correspond to
'
{
checksum
}
'
.
"
f
"
{
checksum_mismatch_download_attempts
}
retries.
"
)
)
elif
checksum
is
not
None
:
if
not
verify_file
(
local_file
,
checksum
,
hash_fct
=
checksum_fct
):
raise
ValueError
(
f
"
The local file hash does not correspond to
'
{
checksum
}
'
and
"
f
"
{
force
=
}
prevents overwriting.
"
)
if
extract
:
if
extract
:
# Extract only if the file was re-downloaded
# Extract only if the file was re-downloaded
if
needs_download
:
if
needs_download
:
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment