From 41ccce249516c34669a7ff44f51d2667e9dea63d Mon Sep 17 00:00:00 2001
From: Andre Anjos <andre.dos.anjos@gmail.com>
Date: Wed, 21 Aug 2019 10:10:44 +0200
Subject: [PATCH] [ci] Move clean-betas to CI (pre-nightlies)
---
bob/devtools/ci.py | 45 ++++++++++++++++++-
bob/devtools/data/gitlab-ci/nightlies.yaml | 21 +++++++++
bob/devtools/scripts/ci.py | 51 ++++++++++++++++++++++
bob/devtools/scripts/dav.py | 6 +--
conda/meta.yaml | 1 +
setup.py | 1 +
6 files changed, 120 insertions(+), 5 deletions(-)
diff --git a/bob/devtools/ci.py b/bob/devtools/ci.py
index 0dea4c9c..b61ff159 100644
--- a/bob/devtools/ci.py
+++ b/bob/devtools/ci.py
@@ -7,7 +7,7 @@
import git
import distutils.version
-from .log import get_logger
+from .log import get_logger, echo_info
from .build import load_order_file
logger = get_logger(__name__)
@@ -213,3 +213,46 @@ def select_user_condarc(paths, branch):
"""
return select_build_file("condarc", paths, branch)
+
+
+def clean_betas(dry_run, username, password):
+ """Cleans-up betas (through the CI). Executes if ``dry_run==False`` only.
+ """
+
+ from .deploy import _setup_webdav_client
+ from .constants import WEBDAV_PATHS, SERVER
+ from .dav import remove_old_beta_packages
+
+ for public in (True, False):
+
+ server_info = WEBDAV_PATHS[False][public]
+ davclient = _setup_webdav_client(
+ SERVER, server_info["root"], username, password
+ )
+
+ # go through all possible variants:
+ archs = [
+ 'linux-64',
+ 'linux-32',
+ 'linux-armv6l',
+ 'linux-armv7l',
+ 'linux-ppc64le',
+ 'osx-64',
+ 'osx-32',
+ 'win-64',
+ 'win-32',
+ 'noarch',
+ ]
+
+ for arch in archs:
+
+ arch_path = '/'.join((path, arch))
+
+ if not (davclient.check(arch_path) and davclient.is_dir(arch_path)):
+ # it is normal if the directory does not exist
+ continue
+
+ server_path = davclient.get_url(arch_path)
+ echo_info('Cleaning beta packages from %s' % server_path)
+ remove_old_beta_packages(client=davclient, path=arch_path,
+ dry_run=dry_run, pyver=True)
diff --git a/bob/devtools/data/gitlab-ci/nightlies.yaml b/bob/devtools/data/gitlab-ci/nightlies.yaml
index aee65c5d..57693946 100644
--- a/bob/devtools/data/gitlab-ci/nightlies.yaml
+++ b/bob/devtools/data/gitlab-ci/nightlies.yaml
@@ -10,9 +10,30 @@ variables:
# Definition of our build pipeline order
stages:
+ - cleanup
- build
+# Periodic cleanup of beta packages
+cleanup:
+ stage: cleanup
+ tags:
+ - docker
+ image: continuumio/conda-concourse-ci
+ script:
+ - curl --silent "${BOOTSTRAP}" --output "bootstrap.py"
+ - python3 bootstrap.py -vv channel base
+ - source ${CONDA_ROOT}/etc/profile.d/conda.sh
+ - conda activate base
+ - bdt ci clean-betas -vv --dry-run
+ cache: &test_caches
+ key: "linux-cache"
+ paths:
+ - miniconda.sh
+ - ${CONDA_ROOT}/pkgs/*.tar.bz2
+ - ${CONDA_ROOT}/pkgs/urls.txt
+
+
# Build targets
.build_template:
stage: build
diff --git a/bob/devtools/scripts/ci.py b/bob/devtools/scripts/ci.py
index cffe0c09..5534ad41 100644
--- a/bob/devtools/scripts/ci.py
+++ b/bob/devtools/scripts/ci.py
@@ -20,6 +20,7 @@ from ..ci import (
select_conda_build_config,
select_conda_recipe_append,
select_user_condarc,
+ clean_betas,
)
from ..log import verbosity_option, get_logger, echo_normal
@@ -946,3 +947,53 @@ def docs(ctx, requirement, dry_run):
logger.info("Building documentation...")
ctx.invoke(build, dry_run=dry_run)
+
+
+@ci.command(
+ epilog="""
+Examples:
+
+ 1. Cleans-up the excess of beta packages from all conda channels via WebDAV:
+
+ $ bdt ci -vv clean-betas --dry-run
+
+ Notice this does not do anything. Remove the --dry-run flag to execute
+
+
+ 2. Really removes (recursively), the excess of beta packages
+
+ $ bdt ci -vv clean-betas
+
+"""
+)
+@click.option(
+ "-d",
+ "--dry-run/--no-dry-run",
+ default=False,
+ help="Only goes through the actions, but does not execute them "
+ "(combine with the verbosity flags - e.g. ``-vvv``) to enable "
+ "printing to help you understand what will be done",
+)
+@verbosity_option()
+@bdt.raise_on_error
+def clean_betas(dry_run):
+ """Cleans-up the excess of beta packages from a conda channel via WebDAV
+
+ ATTENTION: There is no undo! Use --dry-run to test before using.
+ """
+
+ is_master = os.environ["CI_COMMIT_REF_NAME"] == "master"
+ if not is_master and dry_run == False:
+ echo_warning("Forcing dry-run mode - not in master branch")
+ echo_warning("... considering this is **not** a periodic run!")
+ dry_run = True
+
+ if dry_run:
+ echo_warning("!!!! DRY RUN MODE !!!!")
+ echo_warning("Nothing is being executed on server.")
+
+ clean_betas(
+ dry_run=dry_run,
+ username=os.environ["DOCUSER"],
+ password=os.environ["DOCPASS"],
+ )
diff --git a/bob/devtools/scripts/dav.py b/bob/devtools/scripts/dav.py
index cecf4017..0642af52 100644
--- a/bob/devtools/scripts/dav.py
+++ b/bob/devtools/scripts/dav.py
@@ -286,11 +286,9 @@ Examples:
-x)
- 2. Realy removes (recursively), everything under the 'remote/path/foo/bar'
- path:
-
- $ bdt dav -vv rmtree --execute remote/path/foo/bar
+ 2. Really removes (recursively), the excess of beta packages
+ $ bdt dav -vv clean-betas --execute remote/path/foo/bar
"""
)
diff --git a/conda/meta.yaml b/conda/meta.yaml
index 6b06ea7a..67b2b5af 100644
--- a/conda/meta.yaml
+++ b/conda/meta.yaml
@@ -93,6 +93,7 @@ test:
- bdt ci clean --help
- bdt ci nightlies --help
- bdt ci docs --help
+ - bdt ci clean-betas --help
- bdt dav --help
- bdt dav list --help
- bdt dav makedirs --help
diff --git a/setup.py b/setup.py
index badd6d24..a3d490ec 100644
--- a/setup.py
+++ b/setup.py
@@ -79,6 +79,7 @@ setup(
'pypi = bob.devtools.scripts.ci:pypi',
'nightlies = bob.devtools.scripts.ci:nightlies',
'docs = bob.devtools.scripts.ci:docs',
+ 'clean-betas = bob.devtools.scripts.ci:clean_betas',
],
'bdt.local.cli': [
--
GitLab