From 283fb134dbd3bc74c01bed1bb4c01e343ab8b823 Mon Sep 17 00:00:00 2001
From: Andre Anjos <andre.dos.anjos@gmail.com>
Date: Fri, 1 Mar 2019 07:27:09 +0100
Subject: [PATCH] [scripts][ci] Handle rebuilt artefacts properly
---
bob/devtools/build.py | 2 +-
bob/devtools/scripts/ci.py | 50 ++++++++++++++++++++++++++++++---
bob/devtools/scripts/rebuild.py | 16 +++++++++--
3 files changed, 60 insertions(+), 8 deletions(-)
diff --git a/bob/devtools/build.py b/bob/devtools/build.py
index cb21a664..3fd99723 100644
--- a/bob/devtools/build.py
+++ b/bob/devtools/build.py
@@ -139,7 +139,7 @@ def make_conda_config(config, python, append_file, condarc_options):
def get_output_path(metadata, config):
- '''Renders the recipe and returns the interpreted YAML file'''
+ '''Renders the recipe and returns the name of the output file'''
from conda_build.api import get_output_file_paths
return get_output_file_paths(metadata, config=config)[0]
diff --git a/bob/devtools/scripts/ci.py b/bob/devtools/scripts/ci.py
index a1248633..9f753fa5 100644
--- a/bob/devtools/scripts/ci.py
+++ b/bob/devtools/scripts/ci.py
@@ -3,6 +3,7 @@
import os
import re
import glob
+import shutil
import yaml
import click
@@ -11,7 +12,8 @@ import conda_build.api
from click_plugins import with_plugins
from . import bdt
-from ..constants import SERVER, CONDA_BUILD_CONFIG, CONDA_RECIPE_APPEND
+from ..constants import SERVER, CONDA_BUILD_CONFIG, CONDA_RECIPE_APPEND, \
+ WEBDAV_PATHS
from ..log import verbosity_option, get_logger, echo_normal
logger = get_logger(__name__)
@@ -57,7 +59,6 @@ def base_deploy(dry_run):
package = os.environ['CI_PROJECT_PATH']
- from ..constants import WEBDAV_PATHS
server_info = WEBDAV_PATHS[True][True] #stable=True, visible=True
logger.info('Deploying dependence packages to %s%s%s...', SERVER,
@@ -138,7 +139,6 @@ def deploy(dry_run):
# determine if building branch or tag
stable = ('CI_COMMIT_TAG' in os.environ)
- from ..constants import WEBDAV_PATHS
server_info = WEBDAV_PATHS[stable][visible]
logger.info('Deploying conda packages to %s%s%s...', SERVER,
@@ -564,6 +564,7 @@ def nightlies(ctx, order, dry_run):
# determine package visibility
private = urlopen('https://gitlab.idiap.ch/%s' % package).getcode() != 200
+ stable = 'STABLE' in os.environ
ctx.invoke(rebuild,
recipe_dir=[os.path.join(clone_to, 'conda')],
@@ -574,7 +575,48 @@ def nightlies(ctx, order, dry_run):
server=SERVER,
group=group,
private=private,
- stable='STABLE' in os.environ,
+ stable=stable,
dry_run=dry_run,
ci=True,
)
+
+ sphinx_output = os.path.join(os.environ['CI_PROJECT_DIR'], 'sphinx')
+ if os.path.exists(sphinx_output):
+ logger.debug('Sphinx output was generated during test/rebuild of %s - ' \
+ 'Erasing...', package)
+ shutil.rmtree(sphinx_output)
+
+ # re-deploys a new conda package if it was rebuilt
+ # n.b.: can only arrive here if dry_run was ``False`` (no need to check
+ # again)
+ if 'BDT_REBUILD' in os.environ:
+
+ tarball = os.environ['BDT_REBUILD']
+ del os.environ['BDT_REBUILD']
+
+ server_info = WEBDAV_PATHS[stable][not private]
+
+ logger.info('Deploying conda package to %s%s%s...', SERVER,
+ server_info['root'], server_info['conda'])
+
+ # setup webdav connection
+ webdav_options = {
+ 'webdav_hostname': SERVER,
+ 'webdav_root': server_info['root'],
+ 'webdav_login': os.environ['DOCUSER'],
+ 'webdav_password': os.environ['DOCPASS'],
+ }
+ from ..webdav3 import client as webdav
+ davclient = webdav.Client(webdav_options)
+ assert davclient.valid()
+
+ remote_path = '%s/%s/%s' % (server_info['conda'], arch,
+ os.path.basename(tarball))
+ if davclient.check(remote_path):
+ raise RuntimeError('The file %s/%s already exists on the server ' \
+ '- this can be due to more than one rebuild with deployment ' \
+ 'running at the same time. Re-running the broken builds ' \
+ 'normally fixes it' % (SERVER, remote_path))
+ logger.info('[dav] %s -> %s%s%s', k, SERVER, server_info['root'],
+ remote_path)
+ davclient.upload(local_path=tarball, remote_path=remote_path)
diff --git a/bob/devtools/scripts/rebuild.py b/bob/devtools/scripts/rebuild.py
index 2039dd0c..cd11dbf8 100644
--- a/bob/devtools/scripts/rebuild.py
+++ b/bob/devtools/scripts/rebuild.py
@@ -175,7 +175,7 @@ def rebuild(recipe_dir, python, condarc, config, append_file,
if should_build: #something wrong happened, run a full build
- logger.info('Building %s-%s-py%s (build: %d) for %s',
+ logger.info('Re-building %s-%s-py%s (build: %d) for %s',
rendered_recipe['package']['name'],
rendered_recipe['package']['version'], python.replace('.',''),
build_number, arch)
@@ -184,12 +184,22 @@ def rebuild(recipe_dir, python, condarc, config, append_file,
# already resolved the "wrong" build number. We'll have to reparse after
# setting the environment variable BOB_BUILD_NUMBER.
set_environment('BOB_BUILD_NUMBER', str(build_number))
+ metadata = get_rendered_metadata(d, conda_config)
+ candidate = get_output_path(metadata, conda_config)
+
+ if os.path.exists(candidate):
+ logger.debug('Removing existing conda package at %s for rebuild...',
+ candidate)
+ os.unlink(candidate)
if not dry_run:
- conda_build.api.build(d, config=conda_config, notest=no_test)
+ conda_build.api.build(metadata, config=conda_config, notest=no_test)
+ # if you get to this point, the package was successfully rebuilt
+ # set environment to signal caller we can upload it
+ os.environ['BDT_REBUILD'] = candidate
else: #skip build, test worked
- logger.info('Skipping build of %s-%s-py%s (build: %d) for %s',
+ logger.info('Skipping rebuild of %s-%s-py%s (build: %d) for %s',
rendered_recipe['package']['name'],
rendered_recipe['package']['version'], python.replace('.',''),
build_number, arch)
--
GitLab