Commit 086cf496 authored by André Anjos's avatar André Anjos 💬

[mirror] Minor fixes to the new checksum feature

parent 11995597
Pipeline #34680 passed with stages
in 3 minutes and 51 seconds
......@@ -325,15 +325,31 @@ def copy_and_clean_json(url, dest_dir, arch, name):
return _save_json(data, dest_dir, arch, name)
def checksum(repodata, basepath, packages):
def copy_and_clean_patch(url, dest_dir, arch, name):
"""Copies and cleans conda patch_instructions JSON file"""
data = get_json(url, arch, name)
packages = get_local_contents(dest_dir, arch)
data = _cleanup_json(data, packages)
# cleanup specific patch_instructions.json fields
for key in ["remove", "revoke"]:
data[key] = [k for k in data[key] if k in packages]
return _save_json(data, dest_dir, arch, name)
def checksum_packages(repodata, dest_dir, arch, packages):
"""Checksums packages on the local mirror and compare to remote repository
Parameters
----------
repodata : dict
Data loaded from `repodata.json` on the remote repository
basepath : str
Path leading to the packages in the package list
dest_dir : str
Path leading to local mirror
arch : str
Current architecture being considered (e.g. noarch, linux-64 or osx-64)
packages : list
List of packages that are available locally, by name
......@@ -343,11 +359,11 @@ def checksum(repodata, basepath, packages):
List of matching errors
"""
issues = []
issues = set()
total = len(packages)
for k, p in enumerate(packages):
path_to_package = os.path.join(basepath, p)
path_to_package = os.path.join(dest_dir, arch, p)
# checksum to verify
if p.endswith('.tar.bz2'):
......@@ -374,6 +390,6 @@ def checksum(repodata, basepath, packages):
logger.warning('Checksum of %s does not match remote ' \
'repository description (actual:%r != %r:expected)',
path_to_package, actual_hash, expected_hash)
issues.append(p)
issues.add(p)
return issues
......@@ -17,8 +17,8 @@ from ..mirror import (
whitelist_filter,
download_packages,
remove_packages,
copy_and_clean_json,
checksum,
copy_and_clean_patch,
checksum_packages,
)
from ..log import verbosity_option, get_logger, echo_info, echo_warning
......@@ -180,8 +180,13 @@ def mirror(
if checksum:
# double-check if, among packages I should keep, everything looks
# already with respect to expected checksums from the remote repo
issues = checksum(remote_repodata, os.path.join(dest_dir, arch),
issues = checksum_packages(remote_repodata, dest_dir, arch,
to_keep)
if issues:
echo_warning("Detected %d packages with checksum issues - " \
"re-downloading after erasing..." % len(issues))
else:
echo_info("All local package checksums match expected values")
remove_packages(issues, dest_dir, arch, dry_run)
to_download |= issues
......@@ -206,7 +211,8 @@ def mirror(
# go crazy. Do this before the indexing, that will use that file
# to do its magic.
patch_file = 'patch_instructions.json'
name = copy_and_clean_json(channel_url, dest_dir, arch, patch_file)
name = copy_and_clean_patch(channel_url, dest_dir, arch,
patch_file)
echo_info("Cleaned copy of %s/%s/%s installed at %s" %
(channel_url, arch, patch_file, name))
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment