diff --git a/bob/devtools/dav.py b/bob/devtools/dav.py
index cd5c3d4463ffc83b3ff4db9cad7dec3dc697c7f2..7a3578f94474c2a12a946a1c36b93d685b75e9c4 100644
--- a/bob/devtools/dav.py
+++ b/bob/devtools/dav.py
@@ -4,7 +4,7 @@
import os
import configparser
-from .log import get_logger
+from .log import get_logger, echo_warning, echo_info
from .deploy import _setup_webdav_client
logger = get_logger(__name__)
@@ -20,25 +20,29 @@ def _get_config():
if os.path.exists(k):
data = configparser.ConfigParser()
data.read(k)
- if 'global' not in data or \
- 'server' not in data['global'] or \
- 'username' not in data['global'] or \
- 'password' not in data['global']:
- assert KeyError, 'The file %s should contain a single ' \
- '"global" section with 3 variables defined inside: ' \
+ if (
+ "global" not in data
+ or "server" not in data["global"]
+ or "username" not in data["global"]
+ or "password" not in data["global"]
+ ):
+ assert KeyError, (
+ "The file %s should contain a single "
+ '"global" section with 3 variables defined inside: '
'"server", "username", "password".' % (k,)
- return data['global']
+ )
+ return data["global"]
# ask the user for the information, cache credentials for future use
retval = dict()
- retval['server'] = input("The base address of the server: ")
- retval['username'] = input("Username: ")
- retval['password'] = input("Password: ")
+ retval["server"] = input("The base address of the server: ")
+ retval["username"] = input("Username: ")
+ retval["password"] = input("Password: ")
# record file for the user
data = configparser.ConfigParser()
- data['global'] = retval
- with open(cfgs[0], 'w') as f:
+ data["global"] = retval
+ with open(cfgs[0], "w") as f:
logger.warn('Recorded "%s" configuration file for next queries')
data.write(f)
os.chmod(cfgs[0], 0o600)
@@ -51,7 +55,103 @@ def setup_webdav_client(private):
"""Returns a ready-to-use WebDAV client"""
config = _get_config()
- root = '/private-upload' if private else '/public-upload'
- c = _setup_webdav_client(config['server'], root, config['username'],
- config['password'])
+ root = "/private-upload" if private else "/public-upload"
+ c = _setup_webdav_client(
+ config["server"], root, config["username"], config["password"]
+ )
return c
+
+
+def remove_old_beta_packages(client, path, dry_run, pyver=True):
+ """Removes old conda packages from a conda channel.
+
+ What is an old package depends on how the packages are produced. In
+ BEAT/Bob, we build new beta packages with every commit in the CI and we
+ want to delete the old ones using this script so that we do not run out of
+ space.
+
+ The core idea is to remove packages that are not (the latest version AND
+ the latest build number) for each package name.
+
+ Our CI distributes its build into several jobs. Since each job runs
+ independently of each other (per OS and per Python version), the build
+ numbers are estimated independently and they will end up to be different
+ between jobs.
+
+ So the core idea is needed to be applied on each CI job independently.
+
+
+ Parameters:
+
+ client (object): The WebDAV client with a preset public/private path
+
+ path (str): A path, within the preset root of the client, where to
+ search for beta packages. Beta packages are searched in the directory
+ itself.
+
+ dry_run (bool): A flag indicating if we should just list what we will
+ be doing, or really execute the deletions
+
+ pyver (:py:class:`bool`, Optional): If ``True``, the python version of
+ a package will be a part of a package's name. This is need to account
+ for the fact that our CI jobs run per Python version.
+
+ """
+
+ server_path = client.get_url(path)
+
+ if not client.is_dir(path):
+ echo_warning("Path %s is not a directory - ignoring...", server_path)
+ return
+
+ betas = dict()
+ # python version regular expression:
+ pyver_finder = re.compile("py[1-9][0-9]h.*")
+
+ for f in client.list(path):
+
+ if f.startswith("."):
+ continue
+ if not f.endswith(".tar.bz2"):
+ continue
+
+ name, version, build_string = f[:-8].rsplit("-", 2)
+ hash_, build = build_string.rsplit("_", 1)
+
+ if pyver:
+ # try to find the python version if it exists
+ result = pyver_finder.match(hash_)
+ if result is not None:
+ name += "/" + result.string[:4]
+
+ target = '/'.join(path, f)
+ info = client.info(target)
+
+ betas.setdefault(name, []).append(
+ (
+ StrictVersion(version),
+ int(build), # build number
+ info['modified'],
+ target,
+ )
+ )
+
+ import ipdb; ipdb.set_trace()
+
+ count = sum([len(k) for k in betas.values()]) - len(betas)
+ echo_info(" - %d variants" % len(betas))
+ echo_info(" - %d packages found" % count)
+ echo_info(" --------------------")
+
+ for name in sorted(betas.keys()):
+ echo_info(" - packages for %s (%d) ====" % (name, len(betas[name])))
+ sorted_packages = sorted(betas[name])
+ keep_version, keep_build, _, _ = sorted_packages[-1]
+ for version, build, mtime, target in sorted_packages:
+ if version == keep_version and build == keep_build:
+ echo_info("[keep] %s (time=%u)" % (target, mtime))
+ else:
+ echo_info("rm %s (time=%u)" % (target, mtime))
+ if not dry_run:
+ #client.clean(target)
+ echo_info("boooom")
diff --git a/bob/devtools/scripts/dav.py b/bob/devtools/scripts/dav.py
index 41027a7d2170ff0ea0f5dd0dab8f2e8600ca32f7..a9687bb03fb70d7d5f4239ebdd0fa037e72b78d0 100644
--- a/bob/devtools/scripts/dav.py
+++ b/bob/devtools/scripts/dav.py
@@ -10,7 +10,7 @@ from click_plugins import with_plugins
from . import bdt
-from ..dav import setup_webdav_client
+from ..dav import setup_webdav_client, remove_old_beta_packages
from ..log import verbosity_option, get_logger, echo_normal, echo_info, \
echo_warning
@@ -271,3 +271,88 @@ def upload(private, execute, local, remote):
echo_info('cp %s %s' % (k, remote_path))
if execute:
cl.upload_file(local_path=k, remote_path=actual_remote)
+
+
+@dav.command(
+ epilog="""
+Examples:
+
+ 1. Cleans-up the excess of beta packages from our conda channels via WebDAV:
+
+ $ bdt dav -vv clean-betas remote/path/foo/bar
+
+ Notice this does not do anything for security. It just displays what it
+ would do. To actually run the rmtree comment pass the --execute flag (or
+ -x)
+
+
+ 2. Realy removes (recursively), everything under the 'remote/path/foo/bar'
+ path:
+
+ $ bdt dav -vv rmtree --execute remote/path/foo/bar
+
+
+"""
+)
+@click.option(
+ "-p",
+ "--private/--no-private",
+ default=False,
+ help="If set, use the 'private' area instead of the public one",
+)
+@click.option(
+ "-x",
+ "--execute/--no-execute",
+ default=False,
+ help="If this flag is set, then execute the removal",
+)
+@click.argument(
+ "path",
+ required=True,
+)
+@verbosity_option()
+@bdt.raise_on_error
+def clean_betas(private, execute, path):
+ """Cleans-up the excess of beta packages from a conda channel via WebDAV
+
+ ATTENTION: There is no undo! Use --execute to execute.
+ """
+
+ if not execute:
+ echo_warning("!!!! DRY RUN MODE !!!!")
+ echo_warning("Nothing is being executed on server. Use -x to execute.")
+
+ if not path.startswith('/'): path = '/' + path
+ cl = setup_webdav_client(private)
+ remote_path = cl.get_url(path)
+
+ if not cl.is_dir(path):
+ echo_warning('Path %s is not a directory - ignoring...', remote_path)
+ return
+
+ # go through all possible variants:
+ archs = [
+ 'linux-64',
+ 'linux-32',
+ 'linux-armv6l',
+ 'linux-armv7l',
+ 'linux-ppc64le',
+ 'osx-64',
+ 'osx-32',
+ 'win-64',
+ 'win-32',
+ 'noarch',
+ ]
+
+ for arch in archs:
+
+ arch_path = '/'.join((path, arch))
+
+ if not cl.is_dir(arch_path):
+ # it is normal if the directory does not exist
+ continue
+
+ server_path = cl.get_url(arch_path)
+ echo_info('Cleaning beta packages from %s' % server_path)
+ remove_old_beta_packages(client=cl, path=arch_path,
+ dry_run=(not execute), pyver=True)
diff --git a/conda/meta.yaml b/conda/meta.yaml
index ab99512be00d7ed3a0960eb5dc494d168638e6af..6b06ea7a23e2e90c2f768392b91ae99ffca63255 100644
--- a/conda/meta.yaml
+++ b/conda/meta.yaml
@@ -93,6 +93,12 @@ test:
- bdt ci clean --help
- bdt ci nightlies --help
- bdt ci docs --help
+ - bdt dav --help
+ - bdt dav list --help
+ - bdt dav makedirs --help
+ - bdt dav rmtree --help
+ - bdt dav clean-betas --help
+ - bdt dav upload --help
- sphinx-build -aEW ${PREFIX}/share/doc/{{ name }}/doc sphinx
- if [ -n "${CI_PROJECT_DIR}" ]; then mv sphinx "${CI_PROJECT_DIR}/"; fi
diff --git a/setup.py b/setup.py
index 9d29c85fe9228575eef1e2f80cf5ce71edb56399..badd6d24d4beb7093deeee670d09b629dacbdb6c 100644
--- a/setup.py
+++ b/setup.py
@@ -92,6 +92,7 @@ setup(
'makedirs = bob.devtools.scripts.dav:makedirs',
'rmtree = bob.devtools.scripts.dav:rmtree',
'upload = bob.devtools.scripts.dav:upload',
+ 'clean-betas = bob.devtools.scripts.dav:clean_betas',
],
},