Merge branch 'issue-77' into 'master'

Trying to see what was wrong in the old script

Closes #77

See merge request !71

conda/clean-betas.py

@@ -4,34 +4,56 @@
 
 import os
 import sys
+import re
 from distutils.version import StrictVersion
 
 
-def main(scandir, dry_run):
+def _remove_all_packages(scandir, dry_run, pyver=True):
+  """This function implements the core idea of the main function.
+  See its documentation.
+
+  Parameters
+  ----------
+  scandir : str
+      Path to a folder of a conda channel. This should be a platform specific
+      path.
+  dry_run : bool
+      See main.
+  pyver : bool, optional
+      See main.
+  """
+  if not os.path.isdir(scandir):
+    return
 
   betas = dict()
+  # python version regular expression:
+  pyver_finder = re.compile('py[1-9][0-9]h.*')
 
   for (path, dirnames, filenames) in os.walk(scandir, followlinks=False):
 
     for f in filenames:
-      if f.startswith('.'): continue
-      if not f.endswith('.tar.bz2'): continue
+      if f.startswith('.'):
+        continue
+      if not f.endswith('.tar.bz2'):
+        continue
       name, version, build_string = f[:-8].rsplit('-', 2)
-      hash_, build = build_string.rsplit('_')
-      pyver, _ = hash_.rsplit('h', 1)
+      hash_, build = build_string.rsplit('_', 1)
 
-      name = os.path.basename(path) + '/' + name
-      if pyver: name += '/' + pyver
+      if pyver:
+        # try to find the python version if it exists
+        result = pyver_finder.match(hash_)
+        if result is not None:
+          name += '/' + result.string[:4]
       target = os.path.join(path, f)
       betas.setdefault(name, []).append((
-        StrictVersion(version),
-        int(build), #build number
-        os.path.getmtime(target), #cross-platform last modification time
-        target,
-        ))
+          StrictVersion(version),
+          int(build),  # build number
+          os.path.getmtime(target),  # cross-platform last modification time
+          target,
+      ))
 
   count = sum([len(k) for k in betas.values()]) - len(betas)
-  print('end of scan')
+  print('end of scan for {}'.format(scandir))
   print(' - %d variants' % len(betas))
   print(' - %d packages found' % count)
 
@@ -44,16 +66,61 @@ def main(scandir, dry_run):
         print('[keep] %s (time=%u)' % (path, mtime))
       else:
         print('remove %s (time=%u)' % (path, mtime))
-        if not dry_run: os.unlink(path)
+        if not dry_run:
+          os.unlink(path)
 
 
-if __name__ == '__main__':
+def main(scandir, dry_run, pyver=True):
+  """Removes old conda packages from a conda channel.
+
+  What is an old package depends on how the packages are produced.
+  In bob, we build new beta packages with every commit in the CI and we want to
+  delete the old ones using this script so that we do not run out of space.
+
+  The core idea is to remove packages that are not (the latest version AND the
+  latest build number) for each package name.
+
+  Our CI distributes its build into several jobs. Since each job runs
+  independently of each other (per OS and per Python version), the build
+  numbers are estimated independently and they will end up to be different
+  between jobs.
 
-  #check_regex()
+  So the core idea is needed to be applied on each CI job independently.
+
+  Parameters
+  ----------
+  scandir : str
+      The path to the root of a conda channel.
+  dry_run : bool
+      If True, the code will not delete files but print what would be deleted.
+  pyver : bool, optional
+      If True, the python version of a package will be a part of a package's
+      name. This is need to account for the fact that our CI jobs run per
+      Python version.
+  """
+  for os_sub_dir in (
+      'linux-64',
+      'linux-32',
+      'linux-armv6l',
+      'linux-armv7l',
+      'linux-ppc64le',
+      'osx-64',
+      'osx-32',
+      'win-64',
+      'win-32',
+      'noarch',
+
+  ):
+    _remove_all_packages(os.path.join(scandir, os_sub_dir), dry_run, pyver)
+
+
+if __name__ == '__main__':
 
-  if len(sys.argv) not in (2,3) or sys.argv[1] in ('-h', '--help'):
+  if len(sys.argv) not in (2, 3) or sys.argv[1] in ('-h', '--help'):
     print ("usage: %s <dir> [run]" % sys.argv[0])
-    print ("When no command line argument is specified, this script is executed in dry-run mode (i.e., it does not actually delete files). Add option 'run' to actually delete the files.")
+    print ("When no command line argument is specified, this script is "
+           "executed in dry-run mode (i.e., it does not actually delete files)"
+           ". Add option 'run' to actually delete the files.")
     sys.exit(1)
 
   main(sys.argv[1], len(sys.argv) != 3 or sys.argv[2] != 'run')