diff --git a/bob/devtools/ci.py b/bob/devtools/ci.py
index 0dea4c9cdb705627346f20c5017f9454cdc201d7..b61ff159035fa9d111cd2ce1822e24fea6191516 100644
--- a/bob/devtools/ci.py
+++ b/bob/devtools/ci.py
@@ -7,7 +7,7 @@
import git
import distutils.version
-from .log import get_logger
+from .log import get_logger, echo_info
from .build import load_order_file
logger = get_logger(__name__)
@@ -213,3 +213,46 @@ def select_user_condarc(paths, branch):
"""
return select_build_file("condarc", paths, branch)
+
+
+def clean_betas(dry_run, username, password):
+ """Cleans-up betas (through the CI). Executes if ``dry_run==False`` only.
+ """
+
+ from .deploy import _setup_webdav_client
+ from .constants import WEBDAV_PATHS, SERVER
+ from .dav import remove_old_beta_packages
+
+ for public in (True, False):
+
+ server_info = WEBDAV_PATHS[False][public]
+ davclient = _setup_webdav_client(
+ SERVER, server_info["root"], username, password
+ )
+
+ # go through all possible variants:
+ archs = [
+ 'linux-64',
+ 'linux-32',
+ 'linux-armv6l',
+ 'linux-armv7l',
+ 'linux-ppc64le',
+ 'osx-64',
+ 'osx-32',
+ 'win-64',
+ 'win-32',
+ 'noarch',
+ ]
+
+ for arch in archs:
+
+ arch_path = '/'.join((path, arch))
+
+ if not (davclient.check(arch_path) and davclient.is_dir(arch_path)):
+ # it is normal if the directory does not exist
+ continue
+
+ server_path = davclient.get_url(arch_path)
+ echo_info('Cleaning beta packages from %s' % server_path)
+ remove_old_beta_packages(client=davclient, path=arch_path,
+ dry_run=dry_run, pyver=True)
diff --git a/bob/devtools/data/gitlab-ci/nightlies.yaml b/bob/devtools/data/gitlab-ci/nightlies.yaml
index aee65c5d5f702a18afa729e38cb964fd15c02213..57693946496ec837ed86144ebbacbb61c35f723e 100644
--- a/bob/devtools/data/gitlab-ci/nightlies.yaml
+++ b/bob/devtools/data/gitlab-ci/nightlies.yaml
@@ -10,9 +10,30 @@ variables:
# Definition of our build pipeline order
stages:
+ - cleanup
- build
+# Periodic cleanup of beta packages
+cleanup:
+ stage: cleanup
+ tags:
+ - docker
+ image: continuumio/conda-concourse-ci
+ script:
+ - curl --silent "${BOOTSTRAP}" --output "bootstrap.py"
+ - python3 bootstrap.py -vv channel base
+ - source ${CONDA_ROOT}/etc/profile.d/conda.sh
+ - conda activate base
+ - bdt ci clean-betas -vv --dry-run
+ cache: &test_caches
+ key: "linux-cache"
+ paths:
+ - miniconda.sh
+ - ${CONDA_ROOT}/pkgs/*.tar.bz2
+ - ${CONDA_ROOT}/pkgs/urls.txt
+
+
# Build targets
.build_template:
stage: build
diff --git a/bob/devtools/dav.py b/bob/devtools/dav.py
index cd5c3d4463ffc83b3ff4db9cad7dec3dc697c7f2..40bec89eee9b40f9cf43923882951e910cc72a36 100644
--- a/bob/devtools/dav.py
+++ b/bob/devtools/dav.py
@@ -2,9 +2,13 @@
# -*- coding: utf-8 -*-
import os
+import re
import configparser
+import dateutil.parser
-from .log import get_logger
+from distutils.version import StrictVersion
+
+from .log import get_logger, echo_warning, echo_info, echo_normal
from .deploy import _setup_webdav_client
logger = get_logger(__name__)
@@ -13,32 +17,36 @@ logger = get_logger(__name__)
def _get_config():
"""Returns a dictionary with server parameters, or ask them to the user"""
- # tries to figure if we can authenticate using a global configuration
- cfgs = ["~/.bdt-webdav.cfg"]
+ # tries to figure if we can authenticate using a configuration file
+ cfgs = ["~/.bdtrc"]
cfgs = [os.path.expanduser(k) for k in cfgs]
for k in cfgs:
if os.path.exists(k):
data = configparser.ConfigParser()
data.read(k)
- if 'global' not in data or \
- 'server' not in data['global'] or \
- 'username' not in data['global'] or \
- 'password' not in data['global']:
- assert KeyError, 'The file %s should contain a single ' \
- '"global" section with 3 variables defined inside: ' \
+ if (
+ "webdav" not in data
+ or "server" not in data["webdav"]
+ or "username" not in data["webdav"]
+ or "password" not in data["webdav"]
+ ):
+ assert KeyError, (
+ "The file %s should contain a single "
+ '"webdav" section with 3 variables defined inside: '
'"server", "username", "password".' % (k,)
- return data['global']
+ )
+ return data["webdav"]
# ask the user for the information, cache credentials for future use
retval = dict()
- retval['server'] = input("The base address of the server: ")
- retval['username'] = input("Username: ")
- retval['password'] = input("Password: ")
+ retval["server"] = input("The base address of the server: ")
+ retval["username"] = input("Username: ")
+ retval["password"] = input("Password: ")
# record file for the user
data = configparser.ConfigParser()
- data['global'] = retval
- with open(cfgs[0], 'w') as f:
+ data["webdav"] = retval
+ with open(cfgs[0], "w") as f:
logger.warn('Recorded "%s" configuration file for next queries')
data.write(f)
os.chmod(cfgs[0], 0o600)
@@ -51,7 +59,101 @@ def setup_webdav_client(private):
"""Returns a ready-to-use WebDAV client"""
config = _get_config()
- root = '/private-upload' if private else '/public-upload'
- c = _setup_webdav_client(config['server'], root, config['username'],
- config['password'])
+ root = "/private-upload" if private else "/public-upload"
+ c = _setup_webdav_client(
+ config["server"], root, config["username"], config["password"]
+ )
return c
+
+
+def remove_old_beta_packages(client, path, dry_run, pyver=True):
+ """Removes old conda packages from a conda channel.
+
+ What is an old package depends on how the packages are produced. In
+ BEAT/Bob, we build new beta packages with every commit in the CI and we
+ want to delete the old ones using this script so that we do not run out of
+ space.
+
+ The core idea is to remove packages that are not (the latest version AND
+ the latest build number) for each package name.
+
+ Our CI distributes its build into several jobs. Since each job runs
+ independently of each other (per OS and per Python version), the build
+ numbers are estimated independently and they will end up to be different
+ between jobs.
+
+ So the core idea is needed to be applied on each CI job independently.
+
+
+ Parameters:
+
+ client (object): The WebDAV client with a preset public/private path
+
+ path (str): A path, within the preset root of the client, where to
+ search for beta packages. Beta packages are searched in the directory
+ itself.
+
+ dry_run (bool): A flag indicating if we should just list what we will
+ be doing, or really execute the deletions
+
+ pyver (:py:class:`bool`, Optional): If ``True``, the python version of
+ a package will be a part of a package's name. This is need to account
+ for the fact that our CI jobs run per Python version.
+
+ """
+
+ server_path = client.get_url(path)
+
+ if not client.is_dir(path):
+ echo_warning("Path %s is not a directory - ignoring...", server_path)
+ return
+
+ betas = dict()
+ # python version regular expression:
+ pyver_finder = re.compile("py[1-9][0-9]h.*")
+
+ for f in client.list(path):
+
+ if f.startswith("."):
+ continue
+ if not f.endswith(".tar.bz2"):
+ continue
+
+ name, version, build_string = f[:-8].rsplit("-", 2)
+ hash_, build = build_string.rsplit("_", 1)
+
+ if pyver:
+ # try to find the python version if it exists
+ result = pyver_finder.match(hash_)
+ if result is not None:
+ name += "/" + result.string[:4]
+
+ target = '/'.join((path, f))
+ info = client.info(target)
+
+ betas.setdefault(name, []).append(
+ (
+ StrictVersion(version),
+ int(build), # build number
+ dateutil.parser.parse(info['modified']).timestamp(),
+ target,
+ )
+ )
+
+ count = sum([len(k) for k in betas.values()]) - len(betas)
+ echo_normal(" - %d variants" % len(betas))
+ echo_normal(" - %d packages found" % count)
+ echo_normal(" ---------------------")
+
+ for name in sorted(betas.keys()):
+ echo_normal(" - packages for %s (%d)" % (name, len(betas[name])))
+ sorted_packages = sorted(betas[name])
+ keep_version, keep_build, _, _ = sorted_packages[-1]
+ for version, build, mtime, target in sorted_packages:
+ if version == keep_version and build == keep_build:
+ echo_normal("[keep] %s (time=%u)" % (target, mtime))
+ else:
+ echo_warning("rm %s (time=%u)" % (target, mtime))
+ if not dry_run:
+ #client.clean(target)
+ echo_info("boooom")
diff --git a/bob/devtools/scripts/ci.py b/bob/devtools/scripts/ci.py
index cffe0c09ec437a8a4317c4ecf05ad10f2b031a64..5534ad41774dbb372f18205356109fcdcff36a81 100644
--- a/bob/devtools/scripts/ci.py
+++ b/bob/devtools/scripts/ci.py
@@ -20,6 +20,7 @@ from ..ci import (
select_conda_build_config,
select_conda_recipe_append,
select_user_condarc,
+ clean_betas,
)
from ..log import verbosity_option, get_logger, echo_normal
@@ -946,3 +947,53 @@ def docs(ctx, requirement, dry_run):
logger.info("Building documentation...")
ctx.invoke(build, dry_run=dry_run)
+
+
+@ci.command(
+ epilog="""
+Examples:
+
+ 1. Cleans-up the excess of beta packages from all conda channels via WebDAV:
+
+ $ bdt ci -vv clean-betas --dry-run
+
+ Notice this does not do anything. Remove the --dry-run flag to execute
+
+
+ 2. Really removes (recursively), the excess of beta packages
+
+ $ bdt ci -vv clean-betas
+
+"""
+)
+@click.option(
+ "-d",
+ "--dry-run/--no-dry-run",
+ default=False,
+ help="Only goes through the actions, but does not execute them "
+ "(combine with the verbosity flags - e.g. ``-vvv``) to enable "
+ "printing to help you understand what will be done",
+)
+@verbosity_option()
+@bdt.raise_on_error
+def clean_betas(dry_run):
+ """Cleans-up the excess of beta packages from a conda channel via WebDAV
+
+ ATTENTION: There is no undo! Use --dry-run to test before using.
+ """
+
+ is_master = os.environ["CI_COMMIT_REF_NAME"] == "master"
+ if not is_master and dry_run == False:
+ echo_warning("Forcing dry-run mode - not in master branch")
+ echo_warning("... considering this is **not** a periodic run!")
+ dry_run = True
+
+ if dry_run:
+ echo_warning("!!!! DRY RUN MODE !!!!")
+ echo_warning("Nothing is being executed on server.")
+
+ clean_betas(
+ dry_run=dry_run,
+ username=os.environ["DOCUSER"],
+ password=os.environ["DOCPASS"],
+ )
diff --git a/bob/devtools/scripts/dav.py b/bob/devtools/scripts/dav.py
index cb66d5b70190c123fd4bf626c3000126d0ab8d68..0642af521c27fe4d44ae521942c61fe31c5dc508 100644
--- a/bob/devtools/scripts/dav.py
+++ b/bob/devtools/scripts/dav.py
@@ -10,7 +10,7 @@ from click_plugins import with_plugins
from . import bdt
-from ..dav import setup_webdav_client
+from ..dav import setup_webdav_client, remove_old_beta_packages
from ..log import verbosity_option, get_logger, echo_normal, echo_info, \
echo_warning
@@ -277,9 +277,18 @@ def upload(private, execute, local, remote):
epilog="""
Examples:
- 1. Lists the amount of free disk space on the WebDAV server:
+ 1. Cleans-up the excess of beta packages from our conda channels via WebDAV:
- $ bdt dav -vv free
+ $ bdt dav -vv clean-betas remote/path/foo/bar
+
+ Notice this does not do anything for security. It just displays what it
+ would do. To actually run the rmtree comment pass the --execute flag (or
+ -x)
+
+
+ 2. Really removes (recursively), the excess of beta packages
+
+ $ bdt dav -vv clean-betas --execute remote/path/foo/bar
"""
)
@@ -289,13 +298,59 @@ Examples:
default=False,
help="If set, use the 'private' area instead of the public one",
)
+@click.option(
+ "-x",
+ "--execute/--no-execute",
+ default=False,
+ help="If this flag is set, then execute the removal",
+)
+@click.argument(
+ "path",
+ required=True,
+)
@verbosity_option()
@bdt.raise_on_error
-def free(private):
- """Lists the amount of free space on the webserver disk
+def clean_betas(private, execute, path):
+ """Cleans-up the excess of beta packages from a conda channel via WebDAV
+
+ ATTENTION: There is no undo! Use --execute to execute.
"""
+ if not execute:
+ echo_warning("!!!! DRY RUN MODE !!!!")
+ echo_warning("Nothing is being executed on server. Use -x to execute.")
+
+ if not path.startswith('/'): path = '/' + path
cl = setup_webdav_client(private)
- echo_info('free')
- data = cl.free()
- echo_normal(data)
+ remote_path = cl.get_url(path)
+
+ if not cl.is_dir(path):
+ echo_warning('Path %s is not a directory - ignoring...', remote_path)
+ return
+
+ # go through all possible variants:
+ archs = [
+ 'linux-64',
+ 'linux-32',
+ 'linux-armv6l',
+ 'linux-armv7l',
+ 'linux-ppc64le',
+ 'osx-64',
+ 'osx-32',
+ 'win-64',
+ 'win-32',
+ 'noarch',
+ ]
+
+ for arch in archs:
+
+ arch_path = '/'.join((path, arch))
+
+ if not (cl.check(arch_path) and cl.is_dir(arch_path)):
+ # it is normal if the directory does not exist
+ continue
+
+ server_path = cl.get_url(arch_path)
+ echo_info('Cleaning beta packages from %s' % server_path)
+ remove_old_beta_packages(client=cl, path=arch_path,
+ dry_run=(not execute), pyver=True)
diff --git a/conda/meta.yaml b/conda/meta.yaml
index ab99512be00d7ed3a0960eb5dc494d168638e6af..67b2b5afc904d35b55930af82631e84264304701 100644
--- a/conda/meta.yaml
+++ b/conda/meta.yaml
@@ -93,6 +93,13 @@ test:
- bdt ci clean --help
- bdt ci nightlies --help
- bdt ci docs --help
+ - bdt ci clean-betas --help
+ - bdt dav --help
+ - bdt dav list --help
+ - bdt dav makedirs --help
+ - bdt dav rmtree --help
+ - bdt dav clean-betas --help
+ - bdt dav upload --help
- sphinx-build -aEW ${PREFIX}/share/doc/{{ name }}/doc sphinx
- if [ -n "${CI_PROJECT_DIR}" ]; then mv sphinx "${CI_PROJECT_DIR}/"; fi
diff --git a/setup.py b/setup.py
index 4d825c5833c1b7eb5f07133f702f15ddf7a22b65..a3d490ec52158faf0f15f483e47af5b5336b4804 100644
--- a/setup.py
+++ b/setup.py
@@ -79,6 +79,7 @@ setup(
'pypi = bob.devtools.scripts.ci:pypi',
'nightlies = bob.devtools.scripts.ci:nightlies',
'docs = bob.devtools.scripts.ci:docs',
+ 'clean-betas = bob.devtools.scripts.ci:clean_betas',
],
'bdt.local.cli': [
@@ -92,7 +93,7 @@ setup(
'makedirs = bob.devtools.scripts.dav:makedirs',
'rmtree = bob.devtools.scripts.dav:rmtree',
'upload = bob.devtools.scripts.dav:upload',
- #'free = bob.devtools.scripts.dav:free',
+ 'clean-betas = bob.devtools.scripts.dav:clean_betas',
],
},