diff --git a/bob/devtools/mirror.py b/bob/devtools/mirror.py index 778ea60a8830d87b9e8ee0617b43b47c79205aae..c83a7b4b474dd374ea157304949f265cb612fe97 100644 --- a/bob/devtools/mirror.py +++ b/bob/devtools/mirror.py @@ -325,15 +325,31 @@ def copy_and_clean_json(url, dest_dir, arch, name): return _save_json(data, dest_dir, arch, name) -def checksum(repodata, basepath, packages): +def copy_and_clean_patch(url, dest_dir, arch, name): + """Copies and cleans conda patch_instructions JSON file""" + + data = get_json(url, arch, name) + packages = get_local_contents(dest_dir, arch) + data = _cleanup_json(data, packages) + + # cleanup specific patch_instructions.json fields + for key in ["remove", "revoke"]: + data[key] = [k for k in data[key] if k in packages] + + return _save_json(data, dest_dir, arch, name) + + +def checksum_packages(repodata, dest_dir, arch, packages): """Checksums packages on the local mirror and compare to remote repository Parameters ---------- repodata : dict Data loaded from `repodata.json` on the remote repository - basepath : str - Path leading to the packages in the package list + dest_dir : str + Path leading to local mirror + arch : str + Current architecture being considered (e.g. noarch, linux-64 or osx-64) packages : list List of packages that are available locally, by name @@ -343,11 +359,11 @@ def checksum(repodata, basepath, packages): List of matching errors """ - issues = [] + issues = set() total = len(packages) for k, p in enumerate(packages): - path_to_package = os.path.join(basepath, p) + path_to_package = os.path.join(dest_dir, arch, p) # checksum to verify if p.endswith('.tar.bz2'): @@ -374,6 +390,6 @@ def checksum(repodata, basepath, packages): logger.warning('Checksum of %s does not match remote ' \ 'repository description (actual:%r != %r:expected)', path_to_package, actual_hash, expected_hash) - issues.append(p) + issues.add(p) return issues diff --git a/bob/devtools/scripts/mirror.py b/bob/devtools/scripts/mirror.py index 5dded9fb905b5cea50a3fe1db25c3c51c05401a6..8fa1c00b7dea8333ef169e7219734e5a845a789b 100644 --- a/bob/devtools/scripts/mirror.py +++ b/bob/devtools/scripts/mirror.py @@ -17,8 +17,8 @@ from ..mirror import ( whitelist_filter, download_packages, remove_packages, - copy_and_clean_json, - checksum, + copy_and_clean_patch, + checksum_packages, ) from ..log import verbosity_option, get_logger, echo_info, echo_warning @@ -180,8 +180,13 @@ def mirror( if checksum: # double-check if, among packages I should keep, everything looks # already with respect to expected checksums from the remote repo - issues = checksum(remote_repodata, os.path.join(dest_dir, arch), + issues = checksum_packages(remote_repodata, dest_dir, arch, to_keep) + if issues: + echo_warning("Detected %d packages with checksum issues - " \ + "re-downloading after erasing..." % len(issues)) + else: + echo_info("All local package checksums match expected values") remove_packages(issues, dest_dir, arch, dry_run) to_download |= issues @@ -206,7 +211,8 @@ def mirror( # go crazy. Do this before the indexing, that will use that file # to do its magic. patch_file = 'patch_instructions.json' - name = copy_and_clean_json(channel_url, dest_dir, arch, patch_file) + name = copy_and_clean_patch(channel_url, dest_dir, arch, + patch_file) echo_info("Cleaned copy of %s/%s/%s installed at %s" % (channel_url, arch, patch_file, name))