Commit 15e53189 authored by André Anjos's avatar André Anjos 💬
Browse files

[ci/buildout] Use centralized script to pull required images for test (from beat.core)

parent 49a95a46
Pipeline #25373 failed with stages
in 4 minutes and 20 seconds
......@@ -32,3 +32,5 @@ toolchains/
plotters/
plotterparameters/
.noseids
scripts/_core_docker_pull.sh
_ci/
......@@ -4,12 +4,12 @@
# Definition of global variables (all stages)
variables:
CONDA_ROOT: "${CI_PROJECT_DIR}/miniconda"
DOCKER_REGISTRY: docker.idiap.ch
# Definition of our build pipeline order
stages:
- build
- docker
- deploy
- pypi
......@@ -23,6 +23,8 @@ stages:
- chmod 755 _ci/install.sh
- ./_ci/install.sh _ci master #installs ci support scripts
- ./_ci/before_build.sh
- docker info
- ./scripts/docker_pull.sh master #pulls required test images
script:
- ./_ci/build.sh
after_script:
......@@ -37,7 +39,7 @@ stages:
.build_linux_template: &linux_build_job
<<: *build_job
tags:
- docker
- docker-build
image: continuumio/conda-concourse-ci
artifacts:
expire_in: 1 week
......@@ -63,23 +65,6 @@ stages:
key: "macosx-cache"
# Docker host based testing (must be run inside dind or docker-enabled host)
.docker_test_linux_template: &linux_docker_job
stage: docker
before_script:
# safe keep artifacts as before_build.sh will erase those...
- mv ${CONDA_ROOT}/conda-bld .
- ./_ci/install.sh _ci master #updates ci support scripts
- ./_ci/before_build.sh
- mv conda-bld ${CONDA_ROOT}
- ./scripts/before_test.sh
script:
- export BEAT_DOCKER_TESTS=true
- BOB_TEST_ONLY=true ./_ci/build.sh
after_script:
- ./_ci/after_build.sh
build_linux_36:
<<: *linux_build_job
variables:
......@@ -100,17 +85,6 @@ build_macosx_36:
PYTHON_VERSION: "3.6"
# Docker host based testing
docker_linux_36:
<<: *linux_docker_job
variables:
PYTHON_VERSION: "3.6"
dependencies:
- build_linux_36
tags:
- docker-build
# Deploy targets
.deploy_template: &deploy_job
stage: deploy
......
[buildout]
parts = scripts
parts = scripts docker
eggs = beat.cmdline
develop = .
newest = false
[scripts]
recipe = bob.buildout:scripts
[docker]
recipe = collective.recipe.cmd
cmds = BUILDOUT=true ./scripts/docker_pull.sh 1.7.x
on_install = true
on_update = true
#!/usr/bin/env bash
basedir=`pwd`
source ${basedir}/_ci/functions.sh
REGISTRY="docker.idiap.ch"
check_env REGISTRY
# Select here images that are required for minimal operation (or tests)
IMAGES=(
"${REGISTRY}/beat/beat.env.system.python:1.3.0r0"
"${REGISTRY}/beat/beat.env.db.examples:1.4.0r0"
)
check_array_env IMAGES
missing=()
log_info "Checking for minimal set of docker images..."
for k in ${IMAGES[@]}; do
hash_=$(docker images -q ${k})
if [ -z "${hash_}" ]; then
missing+=(${k})
log_warn "${k} NOT available - downloading...";
else
log_info "${k} already available - not re-downloading";
fi
done
# If there are some images missing, ask the user if he wants to download them
if [[ "${#missing[@]}" > 0 ]]; then
# Log in the registry if needed
if [ -z "$CI_SERVER" ]; then
if ! grep -q "${REGISTRY}" ~/.docker/config.json ; then
run_cmd docker login "${REGISTRY}"
fi
else
check_pass CI_BUILD_TOKEN
log_info docker login -u gitlab-ci-token -p "*************" "${REGISTRY}"
docker login -u gitlab-ci-token -p "${CI_BUILD_TOKEN}" "${REGISTRY}"
fi
# Pull the images
for k in "${missing[@]}"; do
run_cmd docker pull ${k}
done
fi
# Makes sure we activate the base environment if available
run_cmd source ${CONDA_ROOT}/etc/profile.d/conda.sh
run_cmd conda activate base
export_env PATH
for subchannel in linux-64 osx-64 noarch; do
if [ ! -d ${CONDA_ROOT}/conda-bld/${subchannel} ]; then
run_cmd mkdir -p ${CONDA_ROOT}/conda-bld/${subchannel}
fi
run_cmd conda index ${CONDA_ROOT}/conda-bld/${subchannel}
done
#!/usr/bin/env bash
if [ $# == 0 ]; then
echo "usage: $0 <beat-core-branch>"
exit 1
fi
basedir=`pwd`
if [ ! -e ${basedir}/_ci/functions.sh ]; then
# this will happen when executing buildout for the first time
mkdir ${basedir}/_ci
curl --silent "https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/install.sh" > ${basedir}/_ci/install.sh
chmod 755 ${basedir}/_ci/install.sh
${basedir}/_ci/install.sh ${basedir}/_ci master #installs ci support scripts
else
${basedir}/_ci/install.sh ${basedir}/_ci master #updates ci support scripts
fi
if [ "${BUILDOUT}" == "true" ]; then
# when executing in the context of buildout, define dummies
export CI_PROJECT_URL=https://gitlab.idiap.ch/beat/beat.core
export CI_PROJECT_DIR=beat
export CI_PROJECT_PATH=beat/beat.core
export CI_PROJECT_NAME=beat.core
export CI_COMMIT_REF_NAME=master
export PYPIUSER=pypiuser
export PYPIPASS=pypipass
export DOCUSER=docuser
export DOCPASS=docpass
export CONDA_ROOT=${basedir}/miniconda
export DOCKER_REGISTRY=docker.idiap.ch
export DOCSERVER=https://www.idiap.ch/software/bob
fi
source ${basedir}/_ci/functions.sh
branch=$1
check_env branch
destdir="$(cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd)"
check_env destdir
run_cmd curl -o ${destdir}/_core_docker_pull.sh --silent "${CI_PROJECT_URL}/raw/${branch}/scripts/docker_pull.sh"
run_cmd chmod 755 ${destdir}/_core_docker_pull.sh
run_cmd ${destdir}/_core_docker_pull.sh
#!/usr/bin/env bash
# Runs the CI build instructions on your local machine, for tests
# The aim of this script is to serve as basis for testing the CI system without
# actually incurring in builds. You may help yourself by modifying it for
# different purposes
export PYTHON_VERSION=3.6
export CI_PROJECT_URL=https://gitlab.idiap.ch/beat/beat.cmdline
export CI_PROJECT_DIR=beat
export CI_PROJECT_PATH=beat/beat.cmdline
export CI_PROJECT_NAME=beat.cmdline
export CI_COMMIT_REF_NAME=master
export PYPIUSER=pypiuser
export PYPIPASS=pypipass
export DOCUSER=docuser
export DOCPASS=docpass
export CONDA_ROOT=${PWD}/miniconda
export DOCSERVER=https://www.idiap.ch/software/bob
# Build stage
mkdir _ci
curl --silent "https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/install.sh" > _ci/install.sh
chmod 755 _ci/install.sh
./_ci/install.sh _ci master #installs ci support scripts
sed -i '~' -e 's;public/;;g' _ci/functions.sh
./_ci/before_build.sh
./_ci/build.sh
./_ci/after_build.sh
# Docker-based Test stage
mv ${CONDA_ROOT}/conda-bld .
./_ci/before_build.sh
mv conda-bld ${CONDA_ROOT}
./scripts/before_test.sh
BOB_TEST_ONLY=true
export BEAT_DOCKER_TESTS=true
./_ci/build.sh
./_ci/after_build.sh
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment