diff --git a/bob/devtools/ci.py b/bob/devtools/ci.py index b61ff159035fa9d111cd2ce1822e24fea6191516..f82fba466b1c763a41a629252dc843aaf901e8e2 100644 --- a/bob/devtools/ci.py +++ b/bob/devtools/ci.py @@ -215,8 +215,23 @@ def select_user_condarc(paths, branch): return select_build_file("condarc", paths, branch) -def clean_betas(dry_run, username, password): +def clean_betas(dry_run, username, password, includes): """Cleans-up betas (through the CI). Executes if ``dry_run==False`` only. + + Parameters: + + dry_run (bool): If set, then does not execute any action, just print + what it would do instead. + + username (str): The user to use for interacting with the WebDAV service + + password (str): Password the the above user + + includes (re.SRE_Pattern): A regular expression that matches the names + of packages that should be considered for clean-up. For example: for + Bob and BATL packages, you may use ``^(bob|batl|gridtk).*`` For BEAT + packages you may use ``^beat.*`` + """ from .deploy import _setup_webdav_client @@ -255,4 +270,4 @@ def clean_betas(dry_run, username, password): server_path = davclient.get_url(arch_path) echo_info('Cleaning beta packages from %s' % server_path) remove_old_beta_packages(client=davclient, path=arch_path, - dry_run=dry_run, pyver=True) + dry_run=dry_run, pyver=True, includes=includes) diff --git a/bob/devtools/data/gitlab-ci/nightlies.yaml b/bob/devtools/data/gitlab-ci/nightlies.yaml index 57693946496ec837ed86144ebbacbb61c35f723e..a469b4b4ce0d6abff33ebe505f16ecb864dc431c 100644 --- a/bob/devtools/data/gitlab-ci/nightlies.yaml +++ b/bob/devtools/data/gitlab-ci/nightlies.yaml @@ -10,28 +10,8 @@ variables: # Definition of our build pipeline order stages: - - cleanup - build - - -# Periodic cleanup of beta packages -cleanup: - stage: cleanup - tags: - - docker - image: continuumio/conda-concourse-ci - script: - - curl --silent "${BOOTSTRAP}" --output "bootstrap.py" - - python3 bootstrap.py -vv channel base - - source ${CONDA_ROOT}/etc/profile.d/conda.sh - - conda activate base - - bdt ci clean-betas -vv --dry-run - cache: &test_caches - key: "linux-cache" - paths: - - miniconda.sh - - ${CONDA_ROOT}/pkgs/*.tar.bz2 - - ${CONDA_ROOT}/pkgs/urls.txt + - cleanup # Build targets @@ -64,3 +44,34 @@ macosx: extends: .build_template tags: - macosx + + +# Periodic cleanup of beta packages +.cleanup_template: + stage: cleanup + tags: + - docker + image: continuumio/conda-concourse-ci + script: + - curl --silent "${BOOTSTRAP}" --output "bootstrap.py" + - python3 bootstrap.py -vv channel base + - source ${CONDA_ROOT}/etc/profile.d/conda.sh + - conda activate base + - bdt ci clean-betas -vv --dry-run + cache: &test_caches + key: "linux-cache" + paths: + - miniconda.sh + - ${CONDA_ROOT}/pkgs/*.tar.bz2 + - ${CONDA_ROOT}/pkgs/urls.txt + + +# The automatic clean-up takes place when the nightly successfuly runs +cleanup: + extends: .cleanup_template + + +manual_cleanup: + extends: .cleanup_template + when: manual + allow_failure: true diff --git a/bob/devtools/dav.py b/bob/devtools/dav.py index 40bec89eee9b40f9cf43923882951e910cc72a36..b659bd3c7e666d4bde9f643543a74f0976303e29 100644 --- a/bob/devtools/dav.py +++ b/bob/devtools/dav.py @@ -66,7 +66,7 @@ def setup_webdav_client(private): return c -def remove_old_beta_packages(client, path, dry_run, pyver=True): +def remove_old_beta_packages(client, path, dry_run, pyver=True, includes=None): """Removes old conda packages from a conda channel. What is an old package depends on how the packages are produced. In @@ -100,6 +100,11 @@ def remove_old_beta_packages(client, path, dry_run, pyver=True): a package will be a part of a package's name. This is need to account for the fact that our CI jobs run per Python version. + includes (re.SRE_Pattern): A regular expression that matches the names + of packages that should be considered for clean-up. For example: for + Bob and BATL packages, you may use ``^(bob|batl|gridtk).*`` For BEAT + packages you may use ``^beat.*`` + """ server_path = client.get_url(path) @@ -120,6 +125,11 @@ def remove_old_beta_packages(client, path, dry_run, pyver=True): continue name, version, build_string = f[:-8].rsplit("-", 2) + + # see if this package should be included or not in our clean-up + if (includes is not None) and (not includes.match(name)): + continue + hash_, build = build_string.rsplit("_", 1) if pyver: diff --git a/bob/devtools/scripts/ci.py b/bob/devtools/scripts/ci.py index 5534ad41774dbb372f18205356109fcdcff36a81..5edda16525fec58088b427a55f97c391a81a8a68 100644 --- a/bob/devtools/scripts/ci.py +++ b/bob/devtools/scripts/ci.py @@ -992,8 +992,15 @@ def clean_betas(dry_run): echo_warning("!!!! DRY RUN MODE !!!!") echo_warning("Nothing is being executed on server.") + import re + if os.environ["CI_PROJECT_NAMESPACE"] == "beat": + includes = re.compile(r'^beat.*') + else: + includes = re.compile(r'^(bob|batl|gridtk).*') + clean_betas( dry_run=dry_run, username=os.environ["DOCUSER"], password=os.environ["DOCPASS"], + includes=includes, )