Commit d3f2f407 authored by André Anjos's avatar André Anjos 💬

[dav] Implement clean-betas on WebDAV support

parent cd7cf914
Pipeline #32608 passed with stage
in 7 minutes and 8 seconds
......@@ -4,7 +4,7 @@
import os
import configparser
from .log import get_logger
from .log import get_logger, echo_warning, echo_info
from .deploy import _setup_webdav_client
logger = get_logger(__name__)
......@@ -20,25 +20,29 @@ def _get_config():
if os.path.exists(k):
data = configparser.ConfigParser()
if 'global' not in data or \
'server' not in data['global'] or \
'username' not in data['global'] or \
'password' not in data['global']:
assert KeyError, 'The file %s should contain a single ' \
'"global" section with 3 variables defined inside: ' \
if (
"global" not in data
or "server" not in data["global"]
or "username" not in data["global"]
or "password" not in data["global"]
assert KeyError, (
"The file %s should contain a single "
'"global" section with 3 variables defined inside: '
'"server", "username", "password".' % (k,)
return data['global']
return data["global"]
# ask the user for the information, cache credentials for future use
retval = dict()
retval['server'] = input("The base address of the server: ")
retval['username'] = input("Username: ")
retval['password'] = input("Password: ")
retval["server"] = input("The base address of the server: ")
retval["username"] = input("Username: ")
retval["password"] = input("Password: ")
# record file for the user
data = configparser.ConfigParser()
data['global'] = retval
with open(cfgs[0], 'w') as f:
data["global"] = retval
with open(cfgs[0], "w") as f:
logger.warn('Recorded "%s" configuration file for next queries')
os.chmod(cfgs[0], 0o600)
......@@ -51,7 +55,103 @@ def setup_webdav_client(private):
"""Returns a ready-to-use WebDAV client"""
config = _get_config()
root = '/private-upload' if private else '/public-upload'
c = _setup_webdav_client(config['server'], root, config['username'],
root = "/private-upload" if private else "/public-upload"
c = _setup_webdav_client(
config["server"], root, config["username"], config["password"]
return c
def remove_old_beta_packages(client, path, dry_run, pyver=True):
"""Removes old conda packages from a conda channel.
What is an old package depends on how the packages are produced. In
BEAT/Bob, we build new beta packages with every commit in the CI and we
want to delete the old ones using this script so that we do not run out of
The core idea is to remove packages that are not (the latest version AND
the latest build number) for each package name.
Our CI distributes its build into several jobs. Since each job runs
independently of each other (per OS and per Python version), the build
numbers are estimated independently and they will end up to be different
between jobs.
So the core idea is needed to be applied on each CI job independently.
client (object): The WebDAV client with a preset public/private path
path (str): A path, within the preset root of the client, where to
search for beta packages. Beta packages are searched in the directory
dry_run (bool): A flag indicating if we should just list what we will
be doing, or really execute the deletions
pyver (:py:class:`bool`, Optional): If ``True``, the python version of
a package will be a part of a package's name. This is need to account
for the fact that our CI jobs run per Python version.
server_path = client.get_url(path)
if not client.is_dir(path):
echo_warning("Path %s is not a directory - ignoring...", server_path)
betas = dict()
# python version regular expression:
pyver_finder = re.compile("py[1-9][0-9]h.*")
for f in client.list(path):
if f.startswith("."):
if not f.endswith(".tar.bz2"):
name, version, build_string = f[:-8].rsplit("-", 2)
hash_, build = build_string.rsplit("_", 1)
if pyver:
# try to find the python version if it exists
result = pyver_finder.match(hash_)
if result is not None:
name += "/" + result.string[:4]
target = '/'.join(path, f)
info =
betas.setdefault(name, []).append(
int(build), # build number
import ipdb; ipdb.set_trace()
count = sum([len(k) for k in betas.values()]) - len(betas)
echo_info(" - %d variants" % len(betas))
echo_info(" - %d packages found" % count)
echo_info(" --------------------")
for name in sorted(betas.keys()):
echo_info(" - packages for %s (%d) ====" % (name, len(betas[name])))
sorted_packages = sorted(betas[name])
keep_version, keep_build, _, _ = sorted_packages[-1]
for version, build, mtime, target in sorted_packages:
if version == keep_version and build == keep_build:
echo_info("[keep] %s (time=%u)" % (target, mtime))
echo_info("rm %s (time=%u)" % (target, mtime))
if not dry_run:
......@@ -10,7 +10,7 @@ from click_plugins import with_plugins
from . import bdt
from ..dav import setup_webdav_client
from ..dav import setup_webdav_client, remove_old_beta_packages
from ..log import verbosity_option, get_logger, echo_normal, echo_info, \
......@@ -271,3 +271,88 @@ def upload(private, execute, local, remote):
echo_info('cp %s %s' % (k, remote_path))
if execute:
cl.upload_file(local_path=k, remote_path=actual_remote)
1. Cleans-up the excess of beta packages from our conda channels via WebDAV:
$ bdt dav -vv clean-betas remote/path/foo/bar
Notice this does not do anything for security. It just displays what it
would do. To actually run the rmtree comment pass the --execute flag (or
2. Realy removes (recursively), everything under the 'remote/path/foo/bar'
$ bdt dav -vv rmtree --execute remote/path/foo/bar
help="If set, use the 'private' area instead of the public one",
help="If this flag is set, then execute the removal",
def clean_betas(private, execute, path):
"""Cleans-up the excess of beta packages from a conda channel via WebDAV
ATTENTION: There is no undo! Use --execute to execute.
if not execute:
echo_warning("!!!! DRY RUN MODE !!!!")
echo_warning("Nothing is being executed on server. Use -x to execute.")
if not path.startswith('/'): path = '/' + path
cl = setup_webdav_client(private)
remote_path = cl.get_url(path)
if not cl.is_dir(path):
echo_warning('Path %s is not a directory - ignoring...', remote_path)
# go through all possible variants:
archs = [
for arch in archs:
arch_path = '/'.join((path, arch))
if not cl.is_dir(arch_path):
# it is normal if the directory does not exist
server_path = cl.get_url(arch_path)
echo_info('Cleaning beta packages from %s' % server_path)
remove_old_beta_packages(client=cl, path=arch_path,
dry_run=(not execute), pyver=True)
......@@ -93,6 +93,12 @@ test:
- bdt ci clean --help
- bdt ci nightlies --help
- bdt ci docs --help
- bdt dav --help
- bdt dav list --help
- bdt dav makedirs --help
- bdt dav rmtree --help
- bdt dav clean-betas --help
- bdt dav upload --help
- sphinx-build -aEW ${PREFIX}/share/doc/{{ name }}/doc sphinx
- if [ -n "${CI_PROJECT_DIR}" ]; then mv sphinx "${CI_PROJECT_DIR}/"; fi
......@@ -92,6 +92,7 @@ setup(
'makedirs = bob.devtools.scripts.dav:makedirs',
'rmtree = bob.devtools.scripts.dav:rmtree',
'upload = bob.devtools.scripts.dav:upload',
'clean-betas = bob.devtools.scripts.dav:clean_betas',
Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment