Commit d3f2f407 authored by André Anjos's avatar André Anjos 💬

[dav] Implement clean-betas on WebDAV support

parent cd7cf914
Pipeline #32608 passed with stage
in 7 minutes and 8 seconds
...@@ -4,7 +4,7 @@ ...@@ -4,7 +4,7 @@
import os import os
import configparser import configparser
from .log import get_logger from .log import get_logger, echo_warning, echo_info
from .deploy import _setup_webdav_client from .deploy import _setup_webdav_client
logger = get_logger(__name__) logger = get_logger(__name__)
...@@ -20,25 +20,29 @@ def _get_config(): ...@@ -20,25 +20,29 @@ def _get_config():
if os.path.exists(k): if os.path.exists(k):
data = configparser.ConfigParser() data = configparser.ConfigParser()
data.read(k) data.read(k)
if 'global' not in data or \ if (
'server' not in data['global'] or \ "global" not in data
'username' not in data['global'] or \ or "server" not in data["global"]
'password' not in data['global']: or "username" not in data["global"]
assert KeyError, 'The file %s should contain a single ' \ or "password" not in data["global"]
'"global" section with 3 variables defined inside: ' \ ):
assert KeyError, (
"The file %s should contain a single "
'"global" section with 3 variables defined inside: '
'"server", "username", "password".' % (k,) '"server", "username", "password".' % (k,)
return data['global'] )
return data["global"]
# ask the user for the information, cache credentials for future use # ask the user for the information, cache credentials for future use
retval = dict() retval = dict()
retval['server'] = input("The base address of the server: ") retval["server"] = input("The base address of the server: ")
retval['username'] = input("Username: ") retval["username"] = input("Username: ")
retval['password'] = input("Password: ") retval["password"] = input("Password: ")
# record file for the user # record file for the user
data = configparser.ConfigParser() data = configparser.ConfigParser()
data['global'] = retval data["global"] = retval
with open(cfgs[0], 'w') as f: with open(cfgs[0], "w") as f:
logger.warn('Recorded "%s" configuration file for next queries') logger.warn('Recorded "%s" configuration file for next queries')
data.write(f) data.write(f)
os.chmod(cfgs[0], 0o600) os.chmod(cfgs[0], 0o600)
...@@ -51,7 +55,103 @@ def setup_webdav_client(private): ...@@ -51,7 +55,103 @@ def setup_webdav_client(private):
"""Returns a ready-to-use WebDAV client""" """Returns a ready-to-use WebDAV client"""
config = _get_config() config = _get_config()
root = '/private-upload' if private else '/public-upload' root = "/private-upload" if private else "/public-upload"
c = _setup_webdav_client(config['server'], root, config['username'], c = _setup_webdav_client(
config['password']) config["server"], root, config["username"], config["password"]
)
return c return c
def remove_old_beta_packages(client, path, dry_run, pyver=True):
"""Removes old conda packages from a conda channel.
What is an old package depends on how the packages are produced. In
BEAT/Bob, we build new beta packages with every commit in the CI and we
want to delete the old ones using this script so that we do not run out of
space.
The core idea is to remove packages that are not (the latest version AND
the latest build number) for each package name.
Our CI distributes its build into several jobs. Since each job runs
independently of each other (per OS and per Python version), the build
numbers are estimated independently and they will end up to be different
between jobs.
So the core idea is needed to be applied on each CI job independently.
Parameters:
client (object): The WebDAV client with a preset public/private path
path (str): A path, within the preset root of the client, where to
search for beta packages. Beta packages are searched in the directory
itself.
dry_run (bool): A flag indicating if we should just list what we will
be doing, or really execute the deletions
pyver (:py:class:`bool`, Optional): If ``True``, the python version of
a package will be a part of a package's name. This is need to account
for the fact that our CI jobs run per Python version.
"""
server_path = client.get_url(path)
if not client.is_dir(path):
echo_warning("Path %s is not a directory - ignoring...", server_path)
return
betas = dict()
# python version regular expression:
pyver_finder = re.compile("py[1-9][0-9]h.*")
for f in client.list(path):
if f.startswith("."):
continue
if not f.endswith(".tar.bz2"):
continue
name, version, build_string = f[:-8].rsplit("-", 2)
hash_, build = build_string.rsplit("_", 1)
if pyver:
# try to find the python version if it exists
result = pyver_finder.match(hash_)
if result is not None:
name += "/" + result.string[:4]
target = '/'.join(path, f)
info = client.info(target)
betas.setdefault(name, []).append(
(
StrictVersion(version),
int(build), # build number
info['modified'],
target,
)
)
import ipdb; ipdb.set_trace()
count = sum([len(k) for k in betas.values()]) - len(betas)
echo_info(" - %d variants" % len(betas))
echo_info(" - %d packages found" % count)
echo_info(" --------------------")
for name in sorted(betas.keys()):
echo_info(" - packages for %s (%d) ====" % (name, len(betas[name])))
sorted_packages = sorted(betas[name])
keep_version, keep_build, _, _ = sorted_packages[-1]
for version, build, mtime, target in sorted_packages:
if version == keep_version and build == keep_build:
echo_info("[keep] %s (time=%u)" % (target, mtime))
else:
echo_info("rm %s (time=%u)" % (target, mtime))
if not dry_run:
#client.clean(target)
echo_info("boooom")
...@@ -10,7 +10,7 @@ from click_plugins import with_plugins ...@@ -10,7 +10,7 @@ from click_plugins import with_plugins
from . import bdt from . import bdt
from ..dav import setup_webdav_client from ..dav import setup_webdav_client, remove_old_beta_packages
from ..log import verbosity_option, get_logger, echo_normal, echo_info, \ from ..log import verbosity_option, get_logger, echo_normal, echo_info, \
echo_warning echo_warning
...@@ -271,3 +271,88 @@ def upload(private, execute, local, remote): ...@@ -271,3 +271,88 @@ def upload(private, execute, local, remote):
echo_info('cp %s %s' % (k, remote_path)) echo_info('cp %s %s' % (k, remote_path))
if execute: if execute:
cl.upload_file(local_path=k, remote_path=actual_remote) cl.upload_file(local_path=k, remote_path=actual_remote)
@dav.command(
epilog="""
Examples:
1. Cleans-up the excess of beta packages from our conda channels via WebDAV:
$ bdt dav -vv clean-betas remote/path/foo/bar
Notice this does not do anything for security. It just displays what it
would do. To actually run the rmtree comment pass the --execute flag (or
-x)
2. Realy removes (recursively), everything under the 'remote/path/foo/bar'
path:
$ bdt dav -vv rmtree --execute remote/path/foo/bar
"""
)
@click.option(
"-p",
"--private/--no-private",
default=False,
help="If set, use the 'private' area instead of the public one",
)
@click.option(
"-x",
"--execute/--no-execute",
default=False,
help="If this flag is set, then execute the removal",
)
@click.argument(
"path",
required=True,
)
@verbosity_option()
@bdt.raise_on_error
def clean_betas(private, execute, path):
"""Cleans-up the excess of beta packages from a conda channel via WebDAV
ATTENTION: There is no undo! Use --execute to execute.
"""
if not execute:
echo_warning("!!!! DRY RUN MODE !!!!")
echo_warning("Nothing is being executed on server. Use -x to execute.")
if not path.startswith('/'): path = '/' + path
cl = setup_webdav_client(private)
remote_path = cl.get_url(path)
if not cl.is_dir(path):
echo_warning('Path %s is not a directory - ignoring...', remote_path)
return
# go through all possible variants:
archs = [
'linux-64',
'linux-32',
'linux-armv6l',
'linux-armv7l',
'linux-ppc64le',
'osx-64',
'osx-32',
'win-64',
'win-32',
'noarch',
]
for arch in archs:
arch_path = '/'.join((path, arch))
if not cl.is_dir(arch_path):
# it is normal if the directory does not exist
continue
server_path = cl.get_url(arch_path)
echo_info('Cleaning beta packages from %s' % server_path)
remove_old_beta_packages(client=cl, path=arch_path,
dry_run=(not execute), pyver=True)
...@@ -93,6 +93,12 @@ test: ...@@ -93,6 +93,12 @@ test:
- bdt ci clean --help - bdt ci clean --help
- bdt ci nightlies --help - bdt ci nightlies --help
- bdt ci docs --help - bdt ci docs --help
- bdt dav --help
- bdt dav list --help
- bdt dav makedirs --help
- bdt dav rmtree --help
- bdt dav clean-betas --help
- bdt dav upload --help
- sphinx-build -aEW ${PREFIX}/share/doc/{{ name }}/doc sphinx - sphinx-build -aEW ${PREFIX}/share/doc/{{ name }}/doc sphinx
- if [ -n "${CI_PROJECT_DIR}" ]; then mv sphinx "${CI_PROJECT_DIR}/"; fi - if [ -n "${CI_PROJECT_DIR}" ]; then mv sphinx "${CI_PROJECT_DIR}/"; fi
......
...@@ -92,6 +92,7 @@ setup( ...@@ -92,6 +92,7 @@ setup(
'makedirs = bob.devtools.scripts.dav:makedirs', 'makedirs = bob.devtools.scripts.dav:makedirs',
'rmtree = bob.devtools.scripts.dav:rmtree', 'rmtree = bob.devtools.scripts.dav:rmtree',
'upload = bob.devtools.scripts.dav:upload', 'upload = bob.devtools.scripts.dav:upload',
'clean-betas = bob.devtools.scripts.dav:clean_betas',
], ],
}, },
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment