Merge branch 'dav-improvements' into 'master'

WebDAV support improvements

See merge request !99

bob/devtools/ci.py

@@ -7,7 +7,7 @@
 import git
 import distutils.version
 
-from .log import get_logger
+from .log import get_logger, echo_info
 from .build import load_order_file
 
 logger = get_logger(__name__)
@@ -213,3 +213,46 @@ def select_user_condarc(paths, branch):
     """
 
     return select_build_file("condarc", paths, branch)
+
+
+def clean_betas(dry_run, username, password):
+    """Cleans-up betas (through the CI).  Executes if ``dry_run==False`` only.
+    """
+
+    from .deploy import _setup_webdav_client
+    from .constants import WEBDAV_PATHS, SERVER
+    from .dav import remove_old_beta_packages
+
+    for public in (True, False):
+
+        server_info = WEBDAV_PATHS[False][public]
+        davclient = _setup_webdav_client(
+            SERVER, server_info["root"], username, password
+        )
+
+        # go through all possible variants:
+        archs = [
+                'linux-64',
+                'linux-32',
+                'linux-armv6l',
+                'linux-armv7l',
+                'linux-ppc64le',
+                'osx-64',
+                'osx-32',
+                'win-64',
+                'win-32',
+                'noarch',
+                ]
+
+        for arch in archs:
+
+            arch_path = '/'.join((path, arch))
+
+            if not (davclient.check(arch_path) and davclient.is_dir(arch_path)):
+                # it is normal if the directory does not exist
+                continue
+
+            server_path = davclient.get_url(arch_path)
+            echo_info('Cleaning beta packages from %s' % server_path)
+            remove_old_beta_packages(client=davclient, path=arch_path,
+                    dry_run=dry_run, pyver=True)

bob/devtools/data/gitlab-ci/nightlies.yaml

@@ -10,9 +10,30 @@ variables:
 
 # Definition of our build pipeline order
 stages:
+  - cleanup
   - build
 
 
+# Periodic cleanup of beta packages
+cleanup:
+  stage: cleanup
+  tags:
+    - docker
+  image: continuumio/conda-concourse-ci
+  script:
+    - curl --silent "${BOOTSTRAP}" --output "bootstrap.py"
+    - python3 bootstrap.py -vv channel base
+    - source ${CONDA_ROOT}/etc/profile.d/conda.sh
+    - conda activate base
+    - bdt ci clean-betas -vv --dry-run
+  cache: &test_caches
+    key: "linux-cache"
+    paths:
+      - miniconda.sh
+      - ${CONDA_ROOT}/pkgs/*.tar.bz2
+      - ${CONDA_ROOT}/pkgs/urls.txt
+
+
 # Build targets
 .build_template:
   stage: build

bob/devtools/dav.py

@@ -2,9 +2,13 @@
 # -*- coding: utf-8 -*-
 
 import os
+import re
 import configparser
+import dateutil.parser
 
-from .log import get_logger
+from distutils.version import StrictVersion
+
+from .log import get_logger, echo_warning, echo_info, echo_normal
 from .deploy import _setup_webdav_client
 
 logger = get_logger(__name__)
@@ -13,32 +17,36 @@ logger = get_logger(__name__)
 def _get_config():
     """Returns a dictionary with server parameters, or ask them to the user"""
 
-    # tries to figure if we can authenticate using a global configuration
-    cfgs = ["~/.bdt-webdav.cfg"]
+    # tries to figure if we can authenticate using a configuration file
+    cfgs = ["~/.bdtrc"]
     cfgs = [os.path.expanduser(k) for k in cfgs]
     for k in cfgs:
         if os.path.exists(k):
             data = configparser.ConfigParser()
             data.read(k)
-            if 'global' not in data or \
-                'server' not in data['global'] or \
-                'username' not in data['global'] or \
-                'password' not in data['global']:
-                assert KeyError, 'The file %s should contain a single ' \
-                    '"global" section with 3 variables defined inside: ' \
+            if (
+                "webdav" not in data
+                or "server" not in data["webdav"]
+                or "username" not in data["webdav"]
+                or "password" not in data["webdav"]
+            ):
+                assert KeyError, (
+                    "The file %s should contain a single "
+                    '"webdav" section with 3 variables defined inside: '
                     '"server", "username", "password".' % (k,)
-            return data['global']
+                )
+            return data["webdav"]
 
     # ask the user for the information, cache credentials for future use
     retval = dict()
-    retval['server'] = input("The base address of the server: ")
-    retval['username'] = input("Username: ")
-    retval['password'] = input("Password: ")
+    retval["server"] = input("The base address of the server: ")
+    retval["username"] = input("Username: ")
+    retval["password"] = input("Password: ")
 
     # record file for the user
     data = configparser.ConfigParser()
-    data['global'] = retval
-    with open(cfgs[0], 'w') as f:
+    data["webdav"] = retval
+    with open(cfgs[0], "w") as f:
         logger.warn('Recorded "%s" configuration file for next queries')
         data.write(f)
     os.chmod(cfgs[0], 0o600)
@@ -51,7 +59,101 @@ def setup_webdav_client(private):
     """Returns a ready-to-use WebDAV client"""
 
     config = _get_config()
-    root = '/private-upload' if private else '/public-upload'
-    c = _setup_webdav_client(config['server'], root, config['username'],
-        config['password'])
+    root = "/private-upload" if private else "/public-upload"
+    c = _setup_webdav_client(
+        config["server"], root, config["username"], config["password"]
+    )
     return c
+
+
+def remove_old_beta_packages(client, path, dry_run, pyver=True):
+    """Removes old conda packages from a conda channel.
+
+    What is an old package depends on how the packages are produced.  In
+    BEAT/Bob, we build new beta packages with every commit in the CI and we
+    want to delete the old ones using this script so that we do not run out of
+    space.
+
+    The core idea is to remove packages that are not (the latest version AND
+    the latest build number) for each package name.
+
+    Our CI distributes its build into several jobs.  Since each job runs
+    independently of each other (per OS and per Python version), the build
+    numbers are estimated independently and they will end up to be different
+    between jobs.
+
+    So the core idea is needed to be applied on each CI job independently.
+
+
+    Parameters:
+
+        client (object): The WebDAV client with a preset public/private path
+
+        path (str): A path, within the preset root of the client, where to
+        search for beta packages.  Beta packages are searched in the directory
+        itself.
+
+        dry_run (bool): A flag indicating if we should just list what we will
+        be doing, or really execute the deletions
+
+        pyver (:py:class:`bool`, Optional): If ``True``, the python version of
+        a package will be a part of a package's name. This is need to account
+        for the fact that our CI jobs run per Python version.
+
+    """
+
+    server_path = client.get_url(path)
+
+    if not client.is_dir(path):
+        echo_warning("Path %s is not a directory - ignoring...", server_path)
+        return
+
+    betas = dict()
+    # python version regular expression:
+    pyver_finder = re.compile("py[1-9][0-9]h.*")
+
+    for f in client.list(path):
+
+        if f.startswith("."):
+            continue
+        if not f.endswith(".tar.bz2"):
+            continue
+
+        name, version, build_string = f[:-8].rsplit("-", 2)
+        hash_, build = build_string.rsplit("_", 1)
+
+        if pyver:
+            # try to find the python version if it exists
+            result = pyver_finder.match(hash_)
+            if result is not None:
+                name += "/" + result.string[:4]
+
+        target = '/'.join((path, f))
+        info = client.info(target)
+
+        betas.setdefault(name, []).append(
+            (
+                StrictVersion(version),
+                int(build),  # build number
+                dateutil.parser.parse(info['modified']).timestamp(),
+                target,
+            )
+        )
+
+    count = sum([len(k) for k in betas.values()]) - len(betas)
+    echo_normal(" - %d variants" % len(betas))
+    echo_normal(" - %d packages found" % count)
+    echo_normal(" ---------------------")
+
+    for name in sorted(betas.keys()):
+        echo_normal(" - packages for %s (%d)" % (name, len(betas[name])))
+        sorted_packages = sorted(betas[name])
+        keep_version, keep_build, _, _ = sorted_packages[-1]
+        for version, build, mtime, target in sorted_packages:
+            if version == keep_version and build == keep_build:
+                echo_normal("[keep] %s (time=%u)" % (target, mtime))
+            else:
+                echo_warning("rm %s (time=%u)" % (target, mtime))
+                if not dry_run:
+                    #client.clean(target)
+                    echo_info("boooom")

bob/devtools/scripts/ci.py

@@ -20,6 +20,7 @@ from ..ci import (
     select_conda_build_config,
     select_conda_recipe_append,
     select_user_condarc,
+    clean_betas,
 )
 
 from ..log import verbosity_option, get_logger, echo_normal
@@ -946,3 +947,53 @@ def docs(ctx, requirement, dry_run):
 
     logger.info("Building documentation...")
     ctx.invoke(build, dry_run=dry_run)
+
+
+@ci.command(
+    epilog="""
+Examples:
+
+  1. Cleans-up the excess of beta packages from all conda channels via WebDAV:
+
+     $ bdt ci -vv clean-betas --dry-run
+
+     Notice this does not do anything.  Remove the --dry-run flag to execute
+
+
+  2. Really removes (recursively), the excess of beta packages
+
+     $ bdt ci -vv clean-betas
+
+"""
+)
+@click.option(
+    "-d",
+    "--dry-run/--no-dry-run",
+    default=False,
+    help="Only goes through the actions, but does not execute them "
+    "(combine with the verbosity flags - e.g. ``-vvv``) to enable "
+    "printing to help you understand what will be done",
+)
+@verbosity_option()
+@bdt.raise_on_error
+def clean_betas(dry_run):
+    """Cleans-up the excess of beta packages from a conda channel via WebDAV
+
+    ATTENTION: There is no undo!  Use --dry-run to test before using.
+    """
+
+    is_master = os.environ["CI_COMMIT_REF_NAME"] == "master"
+    if not is_master and dry_run == False:
+        echo_warning("Forcing dry-run mode - not in master branch")
+        echo_warning("... considering this is **not** a periodic run!")
+        dry_run = True
+
+    if dry_run:
+        echo_warning("!!!! DRY RUN MODE !!!!")
+        echo_warning("Nothing is being executed on server.")
+
+    clean_betas(
+            dry_run=dry_run,
+            username=os.environ["DOCUSER"],
+            password=os.environ["DOCPASS"],
+            )

bob/devtools/scripts/dav.py

@@ -10,7 +10,7 @@ from click_plugins import with_plugins
 
 from . import bdt
 
-from ..dav import setup_webdav_client
+from ..dav import setup_webdav_client, remove_old_beta_packages
 from ..log import verbosity_option, get_logger, echo_normal, echo_info, \
     echo_warning
 
@@ -277,9 +277,18 @@ def upload(private, execute, local, remote):
     epilog="""
 Examples:
 
-  1. Lists the amount of free disk space on the WebDAV server:
+  1. Cleans-up the excess of beta packages from our conda channels via WebDAV:
 
-     $ bdt dav -vv free
+     $ bdt dav -vv clean-betas remote/path/foo/bar
+
+     Notice this does not do anything for security.  It just displays what it
+     would do.  To actually run the rmtree comment pass the --execute flag (or
+     -x)
+
+
+  2. Really removes (recursively), the excess of beta packages
+
+     $ bdt dav -vv clean-betas --execute remote/path/foo/bar
 
 """
 )
@@ -289,13 +298,59 @@ Examples:
     default=False,
     help="If set, use the 'private' area instead of the public one",
 )
+@click.option(
+    "-x",
+    "--execute/--no-execute",
+    default=False,
+    help="If this flag is set, then execute the removal",
+)
+@click.argument(
+    "path",
+    required=True,
+)
 @verbosity_option()
 @bdt.raise_on_error
-def free(private):
-    """Lists the amount of free space on the webserver disk
+def clean_betas(private, execute, path):
+    """Cleans-up the excess of beta packages from a conda channel via WebDAV
+
+    ATTENTION: There is no undo!  Use --execute to execute.
     """
 
+    if not execute:
+        echo_warning("!!!! DRY RUN MODE !!!!")
+        echo_warning("Nothing is being executed on server.  Use -x to execute.")
+
+    if not path.startswith('/'): path = '/' + path
     cl = setup_webdav_client(private)
-    echo_info('free')
-    data = cl.free()
-    echo_normal(data)
+    remote_path = cl.get_url(path)
+
+    if not cl.is_dir(path):
+        echo_warning('Path %s is not a directory - ignoring...', remote_path)
+        return
+
+    # go through all possible variants:
+    archs = [
+            'linux-64',
+            'linux-32',
+            'linux-armv6l',
+            'linux-armv7l',
+            'linux-ppc64le',
+            'osx-64',
+            'osx-32',
+            'win-64',
+            'win-32',
+            'noarch',
+            ]
+
+    for arch in archs:
+
+        arch_path = '/'.join((path, arch))
+
+        if not (cl.check(arch_path) and cl.is_dir(arch_path)):
+            # it is normal if the directory does not exist
+            continue
+
+        server_path = cl.get_url(arch_path)
+        echo_info('Cleaning beta packages from %s' % server_path)
+        remove_old_beta_packages(client=cl, path=arch_path,
+                dry_run=(not execute), pyver=True)

conda/meta.yaml

@@ -93,6 +93,13 @@ test:
     - bdt ci clean --help
     - bdt ci nightlies --help
     - bdt ci docs --help
+    - bdt ci clean-betas --help
+    - bdt dav --help
+    - bdt dav list --help
+    - bdt dav makedirs --help
+    - bdt dav rmtree --help
+    - bdt dav clean-betas --help
+    - bdt dav upload --help
     - sphinx-build -aEW ${PREFIX}/share/doc/{{ name }}/doc sphinx
     - if [ -n "${CI_PROJECT_DIR}" ]; then mv sphinx "${CI_PROJECT_DIR}/"; fi
 

setup.py

@@ -79,6 +79,7 @@ setup(
           'pypi = bob.devtools.scripts.ci:pypi',
           'nightlies = bob.devtools.scripts.ci:nightlies',
           'docs = bob.devtools.scripts.ci:docs',
+          'clean-betas = bob.devtools.scripts.ci:clean_betas',
           ],
 
         'bdt.local.cli': [
@@ -92,7 +93,7 @@ setup(
           'makedirs = bob.devtools.scripts.dav:makedirs',
           'rmtree = bob.devtools.scripts.dav:rmtree',
           'upload = bob.devtools.scripts.dav:upload',
-          #'free = bob.devtools.scripts.dav:free',
+          'clean-betas = bob.devtools.scripts.dav:clean_betas',
           ],
 
     },