Merge branch 'dav-improvements' into 'master'

More cleanup improvements

See merge request !100

bob/devtools/ci.py

@@ -215,8 +215,23 @@ def select_user_condarc(paths, branch):
     return select_build_file("condarc", paths, branch)
 
 
-def clean_betas(dry_run, username, password):
+def clean_betas(dry_run, username, password, includes):
     """Cleans-up betas (through the CI).  Executes if ``dry_run==False`` only.
+
+    Parameters:
+
+        dry_run (bool): If set, then does not execute any action, just print
+          what it would do instead.
+
+        username (str): The user to use for interacting with the WebDAV service
+
+        password (str): Password the the above user
+
+        includes (re.SRE_Pattern): A regular expression that matches the names
+          of packages that should be considered for clean-up.  For example: for
+          Bob and BATL packages, you may use ``^(bob|batl|gridtk).*`` For BEAT
+          packages you may use ``^beat.*``
+
     """
 
     from .deploy import _setup_webdav_client
@@ -255,4 +270,4 @@ def clean_betas(dry_run, username, password):
             server_path = davclient.get_url(arch_path)
             echo_info('Cleaning beta packages from %s' % server_path)
             remove_old_beta_packages(client=davclient, path=arch_path,
-                    dry_run=dry_run, pyver=True)
+                    dry_run=dry_run, pyver=True, includes=includes)

bob/devtools/data/gitlab-ci/nightlies.yaml

@@ -10,28 +10,8 @@ variables:
 
 # Definition of our build pipeline order
 stages:
-  - cleanup
   - build
-
-
-# Periodic cleanup of beta packages
-cleanup:
-  stage: cleanup
-  tags:
-    - docker
-  image: continuumio/conda-concourse-ci
-  script:
-    - curl --silent "${BOOTSTRAP}" --output "bootstrap.py"
-    - python3 bootstrap.py -vv channel base
-    - source ${CONDA_ROOT}/etc/profile.d/conda.sh
-    - conda activate base
-    - bdt ci clean-betas -vv --dry-run
-  cache: &test_caches
-    key: "linux-cache"
-    paths:
-      - miniconda.sh
-      - ${CONDA_ROOT}/pkgs/*.tar.bz2
-      - ${CONDA_ROOT}/pkgs/urls.txt
+  - cleanup
 
 
 # Build targets
@@ -64,3 +44,34 @@ macosx:
   extends: .build_template
   tags:
     - macosx
+
+
+# Periodic cleanup of beta packages
+.cleanup_template:
+  stage: cleanup
+  tags:
+    - docker
+  image: continuumio/conda-concourse-ci
+  script:
+    - curl --silent "${BOOTSTRAP}" --output "bootstrap.py"
+    - python3 bootstrap.py -vv channel base
+    - source ${CONDA_ROOT}/etc/profile.d/conda.sh
+    - conda activate base
+    - bdt ci clean-betas -vv --dry-run
+  cache: &test_caches
+    key: "linux-cache"
+    paths:
+      - miniconda.sh
+      - ${CONDA_ROOT}/pkgs/*.tar.bz2
+      - ${CONDA_ROOT}/pkgs/urls.txt
+
+
+# The automatic clean-up takes place when the nightly successfuly runs
+cleanup:
+  extends: .cleanup_template
+
+
+manual_cleanup:
+  extends: .cleanup_template
+  when: manual
+  allow_failure: true

bob/devtools/dav.py

@@ -66,7 +66,7 @@ def setup_webdav_client(private):
     return c
 
 
-def remove_old_beta_packages(client, path, dry_run, pyver=True):
+def remove_old_beta_packages(client, path, dry_run, pyver=True, includes=None):
     """Removes old conda packages from a conda channel.
 
     What is an old package depends on how the packages are produced.  In
@@ -100,6 +100,11 @@ def remove_old_beta_packages(client, path, dry_run, pyver=True):
         a package will be a part of a package's name. This is need to account
         for the fact that our CI jobs run per Python version.
 
+        includes (re.SRE_Pattern): A regular expression that matches the names
+          of packages that should be considered for clean-up.  For example: for
+          Bob and BATL packages, you may use ``^(bob|batl|gridtk).*`` For BEAT
+          packages you may use ``^beat.*``
+
     """
 
     server_path = client.get_url(path)
@@ -120,6 +125,11 @@ def remove_old_beta_packages(client, path, dry_run, pyver=True):
             continue
 
         name, version, build_string = f[:-8].rsplit("-", 2)
+
+        # see if this package should be included or not in our clean-up
+        if (includes is not None) and (not includes.match(name)):
+            continue
+
         hash_, build = build_string.rsplit("_", 1)
 
         if pyver:

bob/devtools/scripts/ci.py

@@ -992,8 +992,15 @@ def clean_betas(dry_run):
         echo_warning("!!!! DRY RUN MODE !!!!")
         echo_warning("Nothing is being executed on server.")
 
+    import re
+    if os.environ["CI_PROJECT_NAMESPACE"] == "beat":
+        includes = re.compile(r'^beat.*')
+    else:
+        includes = re.compile(r'^(bob|batl|gridtk).*')
+
     clean_betas(
             dry_run=dry_run,
             username=os.environ["DOCUSER"],
             password=os.environ["DOCPASS"],
+            includes=includes,
             )