Commit 0436c422 authored by André Anjos's avatar André Anjos 💬
Browse files

[mirror] Implement download retries with random pauses

parent 22944814
Pipeline #34636 failed with stages
in 2 minutes and 56 seconds
...@@ -11,6 +11,8 @@ https://github.com/valassis-digital-media/conda-mirror ...@@ -11,6 +11,8 @@ https://github.com/valassis-digital-media/conda-mirror
import os import os
import bz2 import bz2
import json import json
import time
import random
import hashlib import hashlib
import fnmatch import fnmatch
import tempfile import tempfile
...@@ -207,11 +209,14 @@ def download_packages(packages, repodata, channel_url, dest_dir, arch, dry_run): ...@@ -207,11 +209,14 @@ def download_packages(packages, repodata, channel_url, dest_dir, arch, dry_run):
temp_dest = os.path.join(download_dir, p) temp_dest = os.path.join(download_dir, p)
logger.info('[download: %d/%d] %s -> %s', k, total, url, temp_dest) logger.info('[download: %d/%d] %s -> %s', k, total, url, temp_dest)
package_retries = 10
while package_retries:
if not dry_run: if not dry_run:
logger.debug('[checking: %d/%d] %s', k, total, url) logger.debug('[checking: %d/%d] %s', k, total, url)
r = requests.get(url, stream=True, allow_redirects=True) r = requests.get(url, stream=True, allow_redirects=True)
logger.info('[download: %d/%d] %s -> %s (%s bytes)', k, total, logger.info('[download: %d/%d] %s -> %s (%s bytes)', k,
url, temp_dest, r.headers['Content-length']) total, url, temp_dest, r.headers['Content-length'])
open(temp_dest, 'wb').write(r.raw.read()) open(temp_dest, 'wb').write(r.raw.read())
# verify that checksum matches # verify that checksum matches
...@@ -227,8 +232,24 @@ def download_packages(packages, repodata, channel_url, dest_dir, arch, dry_run): ...@@ -227,8 +232,24 @@ def download_packages(packages, repodata, channel_url, dest_dir, arch, dry_run):
actual_hash = _md5sum(temp_dest) actual_hash = _md5sum(temp_dest)
else: #sha256 else: #sha256
actual_hash = _sha256sum(temp_dest) actual_hash = _sha256sum(temp_dest)
assert actual_hash == expected_hash, 'Checksum of locally' \
' downloaded version of %s does not match ' \ if actual_hash != expected_hash:
wait_time = random.randint(10,61)
logger.warning('Checksum of locally downloaded ' \
' version of %s does not match ' \
'(actual:%r != %r:expected) - retrying ' \
'after %d seconds', (url, actual_hash,
expected_hash, wait_time)
os.unlink(temp_dest)
time.sleep(wait_time)
package_retries -= 1
continue
else:
break
# final check, before we continue
assert actual_hash == expected_hash, 'Checksum of locally ' \
'downloaded version of %s does not match ' \
'(actual:%r != %r:expected)' % (url, actual_hash, '(actual:%r != %r:expected)' % (url, actual_hash,
expected_hash) expected_hash)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment