[ci] Move clean-betas to CI (pre-nightlies)

bob/devtools/ci.py

@@ -7,7 +7,7 @@
 import git
 import distutils.version
 
-from .log import get_logger
+from .log import get_logger, echo_info
 from .build import load_order_file
 
 logger = get_logger(__name__)
@@ -213,3 +213,46 @@ def select_user_condarc(paths, branch):
     """
 
     return select_build_file("condarc", paths, branch)
+
+
+def clean_betas(dry_run, username, password):
+    """Cleans-up betas (through the CI).  Executes if ``dry_run==False`` only.
+    """
+
+    from .deploy import _setup_webdav_client
+    from .constants import WEBDAV_PATHS, SERVER
+    from .dav import remove_old_beta_packages
+
+    for public in (True, False):
+
+        server_info = WEBDAV_PATHS[False][public]
+        davclient = _setup_webdav_client(
+            SERVER, server_info["root"], username, password
+        )
+
+        # go through all possible variants:
+        archs = [
+                'linux-64',
+                'linux-32',
+                'linux-armv6l',
+                'linux-armv7l',
+                'linux-ppc64le',
+                'osx-64',
+                'osx-32',
+                'win-64',
+                'win-32',
+                'noarch',
+                ]
+
+        for arch in archs:
+
+            arch_path = '/'.join((path, arch))
+
+            if not (davclient.check(arch_path) and davclient.is_dir(arch_path)):
+                # it is normal if the directory does not exist
+                continue
+
+            server_path = davclient.get_url(arch_path)
+            echo_info('Cleaning beta packages from %s' % server_path)
+            remove_old_beta_packages(client=davclient, path=arch_path,
+                    dry_run=dry_run, pyver=True)

bob/devtools/data/gitlab-ci/nightlies.yaml

@@ -10,9 +10,30 @@ variables:
 
 # Definition of our build pipeline order
 stages:
+  - cleanup
   - build
 
 
+# Periodic cleanup of beta packages
+cleanup:
+  stage: cleanup
+  tags:
+    - docker
+  image: continuumio/conda-concourse-ci
+  script:
+    - curl --silent "${BOOTSTRAP}" --output "bootstrap.py"
+    - python3 bootstrap.py -vv channel base
+    - source ${CONDA_ROOT}/etc/profile.d/conda.sh
+    - conda activate base
+    - bdt ci clean-betas -vv --dry-run
+  cache: &test_caches
+    key: "linux-cache"
+    paths:
+      - miniconda.sh
+      - ${CONDA_ROOT}/pkgs/*.tar.bz2
+      - ${CONDA_ROOT}/pkgs/urls.txt
+
+
 # Build targets
 .build_template:
   stage: build

bob/devtools/scripts/ci.py

@@ -20,6 +20,7 @@ from ..ci import (
     select_conda_build_config,
     select_conda_recipe_append,
     select_user_condarc,
+    clean_betas,
 )
 
 from ..log import verbosity_option, get_logger, echo_normal
@@ -946,3 +947,53 @@ def docs(ctx, requirement, dry_run):
 
     logger.info("Building documentation...")
     ctx.invoke(build, dry_run=dry_run)
+
+
+@ci.command(
+    epilog="""
+Examples:
+
+  1. Cleans-up the excess of beta packages from all conda channels via WebDAV:
+
+     $ bdt ci -vv clean-betas --dry-run
+
+     Notice this does not do anything.  Remove the --dry-run flag to execute
+
+
+  2. Really removes (recursively), the excess of beta packages
+
+     $ bdt ci -vv clean-betas
+
+"""
+)
+@click.option(
+    "-d",
+    "--dry-run/--no-dry-run",
+    default=False,
+    help="Only goes through the actions, but does not execute them "
+    "(combine with the verbosity flags - e.g. ``-vvv``) to enable "
+    "printing to help you understand what will be done",
+)
+@verbosity_option()
+@bdt.raise_on_error
+def clean_betas(dry_run):
+    """Cleans-up the excess of beta packages from a conda channel via WebDAV
+
+    ATTENTION: There is no undo!  Use --dry-run to test before using.
+    """
+
+    is_master = os.environ["CI_COMMIT_REF_NAME"] == "master"
+    if not is_master and dry_run == False:
+        echo_warning("Forcing dry-run mode - not in master branch")
+        echo_warning("... considering this is **not** a periodic run!")
+        dry_run = True
+
+    if dry_run:
+        echo_warning("!!!! DRY RUN MODE !!!!")
+        echo_warning("Nothing is being executed on server.")
+
+    clean_betas(
+            dry_run=dry_run,
+            username=os.environ["DOCUSER"],
+            password=os.environ["DOCPASS"],
+            )

bob/devtools/scripts/dav.py

@@ -286,11 +286,9 @@ Examples:
      -x)
 
 
-  2. Realy removes (recursively), everything under the 'remote/path/foo/bar'
-     path:
-
-     $ bdt dav -vv rmtree --execute remote/path/foo/bar
+  2. Really removes (recursively), the excess of beta packages
 
+     $ bdt dav -vv clean-betas --execute remote/path/foo/bar
 
 """
 )

conda/meta.yaml

@@ -93,6 +93,7 @@ test:
     - bdt ci clean --help
     - bdt ci nightlies --help
     - bdt ci docs --help
+    - bdt ci clean-betas --help
     - bdt dav --help
     - bdt dav list --help
     - bdt dav makedirs --help

setup.py

@@ -79,6 +79,7 @@ setup(
           'pypi = bob.devtools.scripts.ci:pypi',
           'nightlies = bob.devtools.scripts.ci:nightlies',
           'docs = bob.devtools.scripts.ci:docs',
+          'clean-betas = bob.devtools.scripts.ci:clean_betas',
           ],
 
         'bdt.local.cli': [