[dav] Implement clean-betas on WebDAV support

bob/devtools/dav.py

@@ -4,7 +4,7 @@
 import os
 import configparser
 
-from .log import get_logger
+from .log import get_logger, echo_warning, echo_info
 from .deploy import _setup_webdav_client
 
 logger = get_logger(__name__)
@@ -20,25 +20,29 @@ def _get_config():
         if os.path.exists(k):
             data = configparser.ConfigParser()
             data.read(k)
-            if 'global' not in data or \
-                'server' not in data['global'] or \
-                'username' not in data['global'] or \
-                'password' not in data['global']:
-                assert KeyError, 'The file %s should contain a single ' \
-                    '"global" section with 3 variables defined inside: ' \
+            if (
+                "global" not in data
+                or "server" not in data["global"]
+                or "username" not in data["global"]
+                or "password" not in data["global"]
+            ):
+                assert KeyError, (
+                    "The file %s should contain a single "
+                    '"global" section with 3 variables defined inside: '
                     '"server", "username", "password".' % (k,)
-            return data['global']
+                )
+            return data["global"]
 
     # ask the user for the information, cache credentials for future use
     retval = dict()
-    retval['server'] = input("The base address of the server: ")
-    retval['username'] = input("Username: ")
-    retval['password'] = input("Password: ")
+    retval["server"] = input("The base address of the server: ")
+    retval["username"] = input("Username: ")
+    retval["password"] = input("Password: ")
 
     # record file for the user
     data = configparser.ConfigParser()
-    data['global'] = retval
-    with open(cfgs[0], 'w') as f:
+    data["global"] = retval
+    with open(cfgs[0], "w") as f:
         logger.warn('Recorded "%s" configuration file for next queries')
         data.write(f)
     os.chmod(cfgs[0], 0o600)
@@ -51,7 +55,103 @@ def setup_webdav_client(private):
     """Returns a ready-to-use WebDAV client"""
 
     config = _get_config()
-    root = '/private-upload' if private else '/public-upload'
-    c = _setup_webdav_client(config['server'], root, config['username'],
-        config['password'])
+    root = "/private-upload" if private else "/public-upload"
+    c = _setup_webdav_client(
+        config["server"], root, config["username"], config["password"]
+    )
     return c
+
+
+def remove_old_beta_packages(client, path, dry_run, pyver=True):
+    """Removes old conda packages from a conda channel.
+
+    What is an old package depends on how the packages are produced.  In
+    BEAT/Bob, we build new beta packages with every commit in the CI and we
+    want to delete the old ones using this script so that we do not run out of
+    space.
+
+    The core idea is to remove packages that are not (the latest version AND
+    the latest build number) for each package name.
+
+    Our CI distributes its build into several jobs.  Since each job runs
+    independently of each other (per OS and per Python version), the build
+    numbers are estimated independently and they will end up to be different
+    between jobs.
+
+    So the core idea is needed to be applied on each CI job independently.
+
+
+    Parameters:
+
+        client (object): The WebDAV client with a preset public/private path
+
+        path (str): A path, within the preset root of the client, where to
+        search for beta packages.  Beta packages are searched in the directory
+        itself.
+
+        dry_run (bool): A flag indicating if we should just list what we will
+        be doing, or really execute the deletions
+
+        pyver (:py:class:`bool`, Optional): If ``True``, the python version of
+        a package will be a part of a package's name. This is need to account
+        for the fact that our CI jobs run per Python version.
+
+    """
+
+    server_path = client.get_url(path)
+
+    if not client.is_dir(path):
+        echo_warning("Path %s is not a directory - ignoring...", server_path)
+        return
+
+    betas = dict()
+    # python version regular expression:
+    pyver_finder = re.compile("py[1-9][0-9]h.*")
+
+    for f in client.list(path):
+
+        if f.startswith("."):
+            continue
+        if not f.endswith(".tar.bz2"):
+            continue
+
+        name, version, build_string = f[:-8].rsplit("-", 2)
+        hash_, build = build_string.rsplit("_", 1)
+
+        if pyver:
+            # try to find the python version if it exists
+            result = pyver_finder.match(hash_)
+            if result is not None:
+                name += "/" + result.string[:4]
+
+        target = '/'.join(path, f)
+        info = client.info(target)
+
+        betas.setdefault(name, []).append(
+            (
+                StrictVersion(version),
+                int(build),  # build number
+                info['modified'],
+                target,
+            )
+        )
+
+    import ipdb; ipdb.set_trace()
+
+    count = sum([len(k) for k in betas.values()]) - len(betas)
+    echo_info(" - %d variants" % len(betas))
+    echo_info(" - %d packages found" % count)
+    echo_info(" --------------------")
+
+    for name in sorted(betas.keys()):
+        echo_info(" - packages for %s (%d) ====" % (name, len(betas[name])))
+        sorted_packages = sorted(betas[name])
+        keep_version, keep_build, _, _ = sorted_packages[-1]
+        for version, build, mtime, target in sorted_packages:
+            if version == keep_version and build == keep_build:
+                echo_info("[keep] %s (time=%u)" % (target, mtime))
+            else:
+                echo_info("rm %s (time=%u)" % (target, mtime))
+                if not dry_run:
+                    #client.clean(target)
+                    echo_info("boooom")

bob/devtools/scripts/dav.py

@@ -10,7 +10,7 @@ from click_plugins import with_plugins
 
 from . import bdt
 
-from ..dav import setup_webdav_client
+from ..dav import setup_webdav_client, remove_old_beta_packages
 from ..log import verbosity_option, get_logger, echo_normal, echo_info, \
     echo_warning
 
@@ -271,3 +271,88 @@ def upload(private, execute, local, remote):
             echo_info('cp %s %s' % (k, remote_path))
             if execute:
                 cl.upload_file(local_path=k, remote_path=actual_remote)
+
+
+@dav.command(
+    epilog="""
+Examples:
+
+  1. Cleans-up the excess of beta packages from our conda channels via WebDAV:
+
+     $ bdt dav -vv clean-betas remote/path/foo/bar
+
+     Notice this does not do anything for security.  It just displays what it
+     would do.  To actually run the rmtree comment pass the --execute flag (or
+     -x)
+
+
+  2. Realy removes (recursively), everything under the 'remote/path/foo/bar'
+     path:
+
+     $ bdt dav -vv rmtree --execute remote/path/foo/bar
+
+
+"""
+)
+@click.option(
+    "-p",
+    "--private/--no-private",
+    default=False,
+    help="If set, use the 'private' area instead of the public one",
+)
+@click.option(
+    "-x",
+    "--execute/--no-execute",
+    default=False,
+    help="If this flag is set, then execute the removal",
+)
+@click.argument(
+    "path",
+    required=True,
+)
+@verbosity_option()
+@bdt.raise_on_error
+def clean_betas(private, execute, path):
+    """Cleans-up the excess of beta packages from a conda channel via WebDAV
+
+    ATTENTION: There is no undo!  Use --execute to execute.
+    """
+
+    if not execute:
+        echo_warning("!!!! DRY RUN MODE !!!!")
+        echo_warning("Nothing is being executed on server.  Use -x to execute.")
+
+    if not path.startswith('/'): path = '/' + path
+    cl = setup_webdav_client(private)
+    remote_path = cl.get_url(path)
+
+    if not cl.is_dir(path):
+        echo_warning('Path %s is not a directory - ignoring...', remote_path)
+        return
+
+    # go through all possible variants:
+    archs = [
+            'linux-64',
+            'linux-32',
+            'linux-armv6l',
+            'linux-armv7l',
+            'linux-ppc64le',
+            'osx-64',
+            'osx-32',
+            'win-64',
+            'win-32',
+            'noarch',
+            ]
+
+    for arch in archs:
+
+        arch_path = '/'.join((path, arch))
+
+        if not cl.is_dir(arch_path):
+            # it is normal if the directory does not exist
+            continue
+
+        server_path = cl.get_url(arch_path)
+        echo_info('Cleaning beta packages from %s' % server_path)
+        remove_old_beta_packages(client=cl, path=arch_path,
+                dry_run=(not execute), pyver=True)

conda/meta.yaml

@@ -93,6 +93,12 @@ test:
     - bdt ci clean --help
     - bdt ci nightlies --help
     - bdt ci docs --help
+    - bdt dav --help
+    - bdt dav list --help
+    - bdt dav makedirs --help
+    - bdt dav rmtree --help
+    - bdt dav clean-betas --help
+    - bdt dav upload --help
     - sphinx-build -aEW ${PREFIX}/share/doc/{{ name }}/doc sphinx
     - if [ -n "${CI_PROJECT_DIR}" ]; then mv sphinx "${CI_PROJECT_DIR}/"; fi
 

setup.py

@@ -92,6 +92,7 @@ setup(
           'makedirs = bob.devtools.scripts.dav:makedirs',
           'rmtree = bob.devtools.scripts.dav:rmtree',
           'upload = bob.devtools.scripts.dav:upload',
+          'clean-betas = bob.devtools.scripts.dav:clean_betas',
           ],
 
     },