Commit 1cd9e861 authored by André Anjos's avatar André Anjos 💬
Browse files

Merge branch 'dav-improvements' into 'master'

WebDAV support improvements

See merge request !99
parents 3d6f8eed 41ccce24
Pipeline #32622 passed with stages
in 12 minutes and 47 seconds
......@@ -7,7 +7,7 @@
import git
import distutils.version
from .log import get_logger
from .log import get_logger, echo_info
from .build import load_order_file
logger = get_logger(__name__)
......@@ -213,3 +213,46 @@ def select_user_condarc(paths, branch):
"""
return select_build_file("condarc", paths, branch)
def clean_betas(dry_run, username, password):
"""Cleans-up betas (through the CI). Executes if ``dry_run==False`` only.
"""
from .deploy import _setup_webdav_client
from .constants import WEBDAV_PATHS, SERVER
from .dav import remove_old_beta_packages
for public in (True, False):
server_info = WEBDAV_PATHS[False][public]
davclient = _setup_webdav_client(
SERVER, server_info["root"], username, password
)
# go through all possible variants:
archs = [
'linux-64',
'linux-32',
'linux-armv6l',
'linux-armv7l',
'linux-ppc64le',
'osx-64',
'osx-32',
'win-64',
'win-32',
'noarch',
]
for arch in archs:
arch_path = '/'.join((path, arch))
if not (davclient.check(arch_path) and davclient.is_dir(arch_path)):
# it is normal if the directory does not exist
continue
server_path = davclient.get_url(arch_path)
echo_info('Cleaning beta packages from %s' % server_path)
remove_old_beta_packages(client=davclient, path=arch_path,
dry_run=dry_run, pyver=True)
......@@ -10,9 +10,30 @@ variables:
# Definition of our build pipeline order
stages:
- cleanup
- build
# Periodic cleanup of beta packages
cleanup:
stage: cleanup
tags:
- docker
image: continuumio/conda-concourse-ci
script:
- curl --silent "${BOOTSTRAP}" --output "bootstrap.py"
- python3 bootstrap.py -vv channel base
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- conda activate base
- bdt ci clean-betas -vv --dry-run
cache: &test_caches
key: "linux-cache"
paths:
- miniconda.sh
- ${CONDA_ROOT}/pkgs/*.tar.bz2
- ${CONDA_ROOT}/pkgs/urls.txt
# Build targets
.build_template:
stage: build
......
......@@ -2,9 +2,13 @@
# -*- coding: utf-8 -*-
import os
import re
import configparser
import dateutil.parser
from .log import get_logger
from distutils.version import StrictVersion
from .log import get_logger, echo_warning, echo_info, echo_normal
from .deploy import _setup_webdav_client
logger = get_logger(__name__)
......@@ -13,32 +17,36 @@ logger = get_logger(__name__)
def _get_config():
"""Returns a dictionary with server parameters, or ask them to the user"""
# tries to figure if we can authenticate using a global configuration
cfgs = ["~/.bdt-webdav.cfg"]
# tries to figure if we can authenticate using a configuration file
cfgs = ["~/.bdtrc"]
cfgs = [os.path.expanduser(k) for k in cfgs]
for k in cfgs:
if os.path.exists(k):
data = configparser.ConfigParser()
data.read(k)
if 'global' not in data or \
'server' not in data['global'] or \
'username' not in data['global'] or \
'password' not in data['global']:
assert KeyError, 'The file %s should contain a single ' \
'"global" section with 3 variables defined inside: ' \
if (
"webdav" not in data
or "server" not in data["webdav"]
or "username" not in data["webdav"]
or "password" not in data["webdav"]
):
assert KeyError, (
"The file %s should contain a single "
'"webdav" section with 3 variables defined inside: '
'"server", "username", "password".' % (k,)
return data['global']
)
return data["webdav"]
# ask the user for the information, cache credentials for future use
retval = dict()
retval['server'] = input("The base address of the server: ")
retval['username'] = input("Username: ")
retval['password'] = input("Password: ")
retval["server"] = input("The base address of the server: ")
retval["username"] = input("Username: ")
retval["password"] = input("Password: ")
# record file for the user
data = configparser.ConfigParser()
data['global'] = retval
with open(cfgs[0], 'w') as f:
data["webdav"] = retval
with open(cfgs[0], "w") as f:
logger.warn('Recorded "%s" configuration file for next queries')
data.write(f)
os.chmod(cfgs[0], 0o600)
......@@ -51,7 +59,101 @@ def setup_webdav_client(private):
"""Returns a ready-to-use WebDAV client"""
config = _get_config()
root = '/private-upload' if private else '/public-upload'
c = _setup_webdav_client(config['server'], root, config['username'],
config['password'])
root = "/private-upload" if private else "/public-upload"
c = _setup_webdav_client(
config["server"], root, config["username"], config["password"]
)
return c
def remove_old_beta_packages(client, path, dry_run, pyver=True):
"""Removes old conda packages from a conda channel.
What is an old package depends on how the packages are produced. In
BEAT/Bob, we build new beta packages with every commit in the CI and we
want to delete the old ones using this script so that we do not run out of
space.
The core idea is to remove packages that are not (the latest version AND
the latest build number) for each package name.
Our CI distributes its build into several jobs. Since each job runs
independently of each other (per OS and per Python version), the build
numbers are estimated independently and they will end up to be different
between jobs.
So the core idea is needed to be applied on each CI job independently.
Parameters:
client (object): The WebDAV client with a preset public/private path
path (str): A path, within the preset root of the client, where to
search for beta packages. Beta packages are searched in the directory
itself.
dry_run (bool): A flag indicating if we should just list what we will
be doing, or really execute the deletions
pyver (:py:class:`bool`, Optional): If ``True``, the python version of
a package will be a part of a package's name. This is need to account
for the fact that our CI jobs run per Python version.
"""
server_path = client.get_url(path)
if not client.is_dir(path):
echo_warning("Path %s is not a directory - ignoring...", server_path)
return
betas = dict()
# python version regular expression:
pyver_finder = re.compile("py[1-9][0-9]h.*")
for f in client.list(path):
if f.startswith("."):
continue
if not f.endswith(".tar.bz2"):
continue
name, version, build_string = f[:-8].rsplit("-", 2)
hash_, build = build_string.rsplit("_", 1)
if pyver:
# try to find the python version if it exists
result = pyver_finder.match(hash_)
if result is not None:
name += "/" + result.string[:4]
target = '/'.join((path, f))
info = client.info(target)
betas.setdefault(name, []).append(
(
StrictVersion(version),
int(build), # build number
dateutil.parser.parse(info['modified']).timestamp(),
target,
)
)
count = sum([len(k) for k in betas.values()]) - len(betas)
echo_normal(" - %d variants" % len(betas))
echo_normal(" - %d packages found" % count)
echo_normal(" ---------------------")
for name in sorted(betas.keys()):
echo_normal(" - packages for %s (%d)" % (name, len(betas[name])))
sorted_packages = sorted(betas[name])
keep_version, keep_build, _, _ = sorted_packages[-1]
for version, build, mtime, target in sorted_packages:
if version == keep_version and build == keep_build:
echo_normal("[keep] %s (time=%u)" % (target, mtime))
else:
echo_warning("rm %s (time=%u)" % (target, mtime))
if not dry_run:
#client.clean(target)
echo_info("boooom")
......@@ -20,6 +20,7 @@ from ..ci import (
select_conda_build_config,
select_conda_recipe_append,
select_user_condarc,
clean_betas,
)
from ..log import verbosity_option, get_logger, echo_normal
......@@ -946,3 +947,53 @@ def docs(ctx, requirement, dry_run):
logger.info("Building documentation...")
ctx.invoke(build, dry_run=dry_run)
@ci.command(
epilog="""
Examples:
1. Cleans-up the excess of beta packages from all conda channels via WebDAV:
$ bdt ci -vv clean-betas --dry-run
Notice this does not do anything. Remove the --dry-run flag to execute
2. Really removes (recursively), the excess of beta packages
$ bdt ci -vv clean-betas
"""
)
@click.option(
"-d",
"--dry-run/--no-dry-run",
default=False,
help="Only goes through the actions, but does not execute them "
"(combine with the verbosity flags - e.g. ``-vvv``) to enable "
"printing to help you understand what will be done",
)
@verbosity_option()
@bdt.raise_on_error
def clean_betas(dry_run):
"""Cleans-up the excess of beta packages from a conda channel via WebDAV
ATTENTION: There is no undo! Use --dry-run to test before using.
"""
is_master = os.environ["CI_COMMIT_REF_NAME"] == "master"
if not is_master and dry_run == False:
echo_warning("Forcing dry-run mode - not in master branch")
echo_warning("... considering this is **not** a periodic run!")
dry_run = True
if dry_run:
echo_warning("!!!! DRY RUN MODE !!!!")
echo_warning("Nothing is being executed on server.")
clean_betas(
dry_run=dry_run,
username=os.environ["DOCUSER"],
password=os.environ["DOCPASS"],
)
......@@ -10,7 +10,7 @@ from click_plugins import with_plugins
from . import bdt
from ..dav import setup_webdav_client
from ..dav import setup_webdav_client, remove_old_beta_packages
from ..log import verbosity_option, get_logger, echo_normal, echo_info, \
echo_warning
......@@ -277,9 +277,18 @@ def upload(private, execute, local, remote):
epilog="""
Examples:
1. Lists the amount of free disk space on the WebDAV server:
1. Cleans-up the excess of beta packages from our conda channels via WebDAV:
$ bdt dav -vv free
$ bdt dav -vv clean-betas remote/path/foo/bar
Notice this does not do anything for security. It just displays what it
would do. To actually run the rmtree comment pass the --execute flag (or
-x)
2. Really removes (recursively), the excess of beta packages
$ bdt dav -vv clean-betas --execute remote/path/foo/bar
"""
)
......@@ -289,13 +298,59 @@ Examples:
default=False,
help="If set, use the 'private' area instead of the public one",
)
@click.option(
"-x",
"--execute/--no-execute",
default=False,
help="If this flag is set, then execute the removal",
)
@click.argument(
"path",
required=True,
)
@verbosity_option()
@bdt.raise_on_error
def free(private):
"""Lists the amount of free space on the webserver disk
def clean_betas(private, execute, path):
"""Cleans-up the excess of beta packages from a conda channel via WebDAV
ATTENTION: There is no undo! Use --execute to execute.
"""
if not execute:
echo_warning("!!!! DRY RUN MODE !!!!")
echo_warning("Nothing is being executed on server. Use -x to execute.")
if not path.startswith('/'): path = '/' + path
cl = setup_webdav_client(private)
echo_info('free')
data = cl.free()
echo_normal(data)
remote_path = cl.get_url(path)
if not cl.is_dir(path):
echo_warning('Path %s is not a directory - ignoring...', remote_path)
return
# go through all possible variants:
archs = [
'linux-64',
'linux-32',
'linux-armv6l',
'linux-armv7l',
'linux-ppc64le',
'osx-64',
'osx-32',
'win-64',
'win-32',
'noarch',
]
for arch in archs:
arch_path = '/'.join((path, arch))
if not (cl.check(arch_path) and cl.is_dir(arch_path)):
# it is normal if the directory does not exist
continue
server_path = cl.get_url(arch_path)
echo_info('Cleaning beta packages from %s' % server_path)
remove_old_beta_packages(client=cl, path=arch_path,
dry_run=(not execute), pyver=True)
......@@ -93,6 +93,13 @@ test:
- bdt ci clean --help
- bdt ci nightlies --help
- bdt ci docs --help
- bdt ci clean-betas --help
- bdt dav --help
- bdt dav list --help
- bdt dav makedirs --help
- bdt dav rmtree --help
- bdt dav clean-betas --help
- bdt dav upload --help
- sphinx-build -aEW ${PREFIX}/share/doc/{{ name }}/doc sphinx
- if [ -n "${CI_PROJECT_DIR}" ]; then mv sphinx "${CI_PROJECT_DIR}/"; fi
......
......@@ -79,6 +79,7 @@ setup(
'pypi = bob.devtools.scripts.ci:pypi',
'nightlies = bob.devtools.scripts.ci:nightlies',
'docs = bob.devtools.scripts.ci:docs',
'clean-betas = bob.devtools.scripts.ci:clean_betas',
],
'bdt.local.cli': [
......@@ -92,7 +93,7 @@ setup(
'makedirs = bob.devtools.scripts.dav:makedirs',
'rmtree = bob.devtools.scripts.dav:rmtree',
'upload = bob.devtools.scripts.dav:upload',
#'free = bob.devtools.scripts.dav:free',
'clean-betas = bob.devtools.scripts.dav:clean_betas',
],
},
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment