Commit 5d6796d2 authored by André Anjos's avatar André Anjos 💬
Browse files

Merge branch 'dav-improvements' into 'master'

More cleanup improvements

See merge request !100
parents 1cd9e861 c7e043f8
Pipeline #32638 passed with stages
in 10 minutes and 15 seconds
......@@ -215,8 +215,23 @@ def select_user_condarc(paths, branch):
return select_build_file("condarc", paths, branch)
def clean_betas(dry_run, username, password):
def clean_betas(dry_run, username, password, includes):
"""Cleans-up betas (through the CI). Executes if ``dry_run==False`` only.
Parameters:
dry_run (bool): If set, then does not execute any action, just print
what it would do instead.
username (str): The user to use for interacting with the WebDAV service
password (str): Password the the above user
includes (re.SRE_Pattern): A regular expression that matches the names
of packages that should be considered for clean-up. For example: for
Bob and BATL packages, you may use ``^(bob|batl|gridtk).*`` For BEAT
packages you may use ``^beat.*``
"""
from .deploy import _setup_webdav_client
......@@ -255,4 +270,4 @@ def clean_betas(dry_run, username, password):
server_path = davclient.get_url(arch_path)
echo_info('Cleaning beta packages from %s' % server_path)
remove_old_beta_packages(client=davclient, path=arch_path,
dry_run=dry_run, pyver=True)
dry_run=dry_run, pyver=True, includes=includes)
......@@ -10,28 +10,8 @@ variables:
# Definition of our build pipeline order
stages:
- cleanup
- build
# Periodic cleanup of beta packages
cleanup:
stage: cleanup
tags:
- docker
image: continuumio/conda-concourse-ci
script:
- curl --silent "${BOOTSTRAP}" --output "bootstrap.py"
- python3 bootstrap.py -vv channel base
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- conda activate base
- bdt ci clean-betas -vv --dry-run
cache: &test_caches
key: "linux-cache"
paths:
- miniconda.sh
- ${CONDA_ROOT}/pkgs/*.tar.bz2
- ${CONDA_ROOT}/pkgs/urls.txt
- cleanup
# Build targets
......@@ -64,3 +44,34 @@ macosx:
extends: .build_template
tags:
- macosx
# Periodic cleanup of beta packages
.cleanup_template:
stage: cleanup
tags:
- docker
image: continuumio/conda-concourse-ci
script:
- curl --silent "${BOOTSTRAP}" --output "bootstrap.py"
- python3 bootstrap.py -vv channel base
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- conda activate base
- bdt ci clean-betas -vv --dry-run
cache: &test_caches
key: "linux-cache"
paths:
- miniconda.sh
- ${CONDA_ROOT}/pkgs/*.tar.bz2
- ${CONDA_ROOT}/pkgs/urls.txt
# The automatic clean-up takes place when the nightly successfuly runs
cleanup:
extends: .cleanup_template
manual_cleanup:
extends: .cleanup_template
when: manual
allow_failure: true
......@@ -66,7 +66,7 @@ def setup_webdav_client(private):
return c
def remove_old_beta_packages(client, path, dry_run, pyver=True):
def remove_old_beta_packages(client, path, dry_run, pyver=True, includes=None):
"""Removes old conda packages from a conda channel.
What is an old package depends on how the packages are produced. In
......@@ -100,6 +100,11 @@ def remove_old_beta_packages(client, path, dry_run, pyver=True):
a package will be a part of a package's name. This is need to account
for the fact that our CI jobs run per Python version.
includes (re.SRE_Pattern): A regular expression that matches the names
of packages that should be considered for clean-up. For example: for
Bob and BATL packages, you may use ``^(bob|batl|gridtk).*`` For BEAT
packages you may use ``^beat.*``
"""
server_path = client.get_url(path)
......@@ -120,6 +125,11 @@ def remove_old_beta_packages(client, path, dry_run, pyver=True):
continue
name, version, build_string = f[:-8].rsplit("-", 2)
# see if this package should be included or not in our clean-up
if (includes is not None) and (not includes.match(name)):
continue
hash_, build = build_string.rsplit("_", 1)
if pyver:
......
......@@ -992,8 +992,15 @@ def clean_betas(dry_run):
echo_warning("!!!! DRY RUN MODE !!!!")
echo_warning("Nothing is being executed on server.")
import re
if os.environ["CI_PROJECT_NAMESPACE"] == "beat":
includes = re.compile(r'^beat.*')
else:
includes = re.compile(r'^(bob|batl|gridtk).*')
clean_betas(
dry_run=dry_run,
username=os.environ["DOCUSER"],
password=os.environ["DOCPASS"],
includes=includes,
)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment