From b50e19d363d8325804357e600aca90a499748658 Mon Sep 17 00:00:00 2001
From: Amir MOHAMMADI <amir.mohammadi@idiap.ch>
Date: Wed, 14 Mar 2018 11:38:24 +0100
Subject: [PATCH] Trying to see what was wrong in the old script
---
conda/clean-betas.py | 103 +++++++++++++++++++++++++++++++++++--------
1 file changed, 85 insertions(+), 18 deletions(-)
diff --git a/conda/clean-betas.py b/conda/clean-betas.py
index 0c0caf6..03d0ebc 100755
--- a/conda/clean-betas.py
+++ b/conda/clean-betas.py
@@ -4,34 +4,56 @@
import os
import sys
+import re
from distutils.version import StrictVersion
-def main(scandir, dry_run):
+def _remove_all_packages(scandir, dry_run, pyver=True):
+ """This function implements the core idea of the main function.
+ See its documentation.
+
+ Parameters
+ ----------
+ scandir : str
+ Path to a folder of a conda channel. This should be a platform specific
+ path.
+ dry_run : bool
+ See main.
+ pyver : bool, optional
+ See main.
+ """
+ if not os.path.isdir(scandir):
+ return
betas = dict()
+ # python version regular expression:
+ pyver_finder = re.compile('py[1-9][0-9]h.*')
for (path, dirnames, filenames) in os.walk(scandir, followlinks=False):
for f in filenames:
- if f.startswith('.'): continue
- if not f.endswith('.tar.bz2'): continue
+ if f.startswith('.'):
+ continue
+ if not f.endswith('.tar.bz2'):
+ continue
name, version, build_string = f[:-8].rsplit('-', 2)
- hash_, build = build_string.rsplit('_')
- pyver, _ = hash_.rsplit('h', 1)
+ hash_, build = build_string.rsplit('_', 1)
- name = os.path.basename(path) + '/' + name
- if pyver: name += '/' + pyver
+ if pyver:
+ # try to find the python version if it exists
+ result = pyver_finder.match(hash_)
+ if result is not None:
+ name += '/' + result.string[:4]
target = os.path.join(path, f)
betas.setdefault(name, []).append((
- StrictVersion(version),
- int(build), #build number
- os.path.getmtime(target), #cross-platform last modification time
- target,
- ))
+ StrictVersion(version),
+ int(build), # build number
+ os.path.getmtime(target), # cross-platform last modification time
+ target,
+ ))
count = sum([len(k) for k in betas.values()]) - len(betas)
- print('end of scan')
+ print('end of scan for {}'.format(scandir))
print(' - %d variants' % len(betas))
print(' - %d packages found' % count)
@@ -44,16 +66,61 @@ def main(scandir, dry_run):
print('[keep] %s (time=%u)' % (path, mtime))
else:
print('remove %s (time=%u)' % (path, mtime))
- if not dry_run: os.unlink(path)
+ if not dry_run:
+ os.unlink(path)
-if __name__ == '__main__':
+def main(scandir, dry_run, pyver=True):
+ """Removes old conda packages from a conda channel.
+
+ What is an old package depends on how the packages are produced.
+ In bob, we build new beta packages with every commit in the CI and we want to
+ delete the old ones using this script so that we do not run out of space.
+
+ The core idea is to remove packages that are not (the latest version AND the
+ latest build number) for each package name.
+
+ Our CI distributes its build into several jobs. Since each job runs
+ independently of each other (per OS and per Python version), the build
+ numbers are estimated independently and they will end up to be different
+ between jobs.
- #check_regex()
+ So the core idea is needed to be applied on each CI job independently.
+
+ Parameters
+ ----------
+ scandir : str
+ The path to the root of a conda channel.
+ dry_run : bool
+ If True, the code will not delete files but print what would be deleted.
+ pyver : bool, optional
+ If True, the python version of a package will be a part of a package's
+ name. This is need to account for the fact that our CI jobs run per
+ Python version.
+ """
+ for os_sub_dir in (
+ 'linux-64',
+ 'linux-32',
+ 'linux-armv6l',
+ 'linux-armv7l',
+ 'linux-ppc64le',
+ 'osx-64',
+ 'osx-32',
+ 'win-64',
+ 'win-32',
+ 'noarch',
+
+ ):
+ _remove_all_packages(os.path.join(scandir, os_sub_dir), dry_run, pyver)
+
+
+if __name__ == '__main__':
- if len(sys.argv) not in (2,3) or sys.argv[1] in ('-h', '--help'):
+ if len(sys.argv) not in (2, 3) or sys.argv[1] in ('-h', '--help'):
print ("usage: %s <dir> [run]" % sys.argv[0])
- print ("When no command line argument is specified, this script is executed in dry-run mode (i.e., it does not actually delete files). Add option 'run' to actually delete the files.")
+ print ("When no command line argument is specified, this script is "
+ "executed in dry-run mode (i.e., it does not actually delete files)"
+ ". Add option 'run' to actually delete the files.")
sys.exit(1)
main(sys.argv[1], len(sys.argv) != 3 or sys.argv[2] != 'run')
--
GitLab