Commit e60b7933 authored by André Anjos's avatar André Anjos 💬

[test] Add automatic test image pulling if not already available (closes #63)

parent a2a253bd
Pipeline #27129 failed
# This build file uses template features from YAML so it is generic enough for
# any Bob project. Don't modify it unless you know what you're doing.
# Definition of global variables (all stages)
variables:
CONDA_ROOT: "${CI_PROJECT_DIR}/miniconda"
DOCKER_REGISTRY: docker.idiap.ch
# Definition of our build pipeline order
stages:
- build
- deploy
- pypi
# Build targets
.build_template: &build_job
stage: build
before_script:
- mkdir _ci
- curl --silent "https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/install.sh" > _ci/install.sh
- chmod 755 _ci/install.sh
- ./_ci/install.sh _ci master #installs ci support scripts
- ./_ci/before_build.sh
- docker info
- ./scripts/docker_pull.sh #pulls required test images
script:
- ./_ci/build.sh
after_script:
- ./_ci/after_build.sh
cache: &build_caches
paths:
- miniconda.sh
- ${CONDA_ROOT}/pkgs/*.tar.bz2
- ${CONDA_ROOT}/pkgs/urls.txt
.build_linux_template: &linux_build_job
<<: *build_job
tags:
- docker-build
artifacts:
expire_in: 1 week
paths:
- _ci/
- ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
cache:
<<: *build_caches
key: "linux-cache"
.build_macosx_template: &macosx_build_job
<<: *build_job
tags:
- macosx
artifacts:
expire_in: 1 week
paths:
- _ci/
- ${CONDA_ROOT}/conda-bld/osx-64/*.tar.bz2
cache:
<<: *build_caches
key: "macosx-cache"
build_linux_36:
<<: *linux_build_job
variables:
PYTHON_VERSION: "3.6"
BUILD_EGG: "true"
artifacts:
expire_in: 1 week
paths:
- _ci/
- dist/*.zip
- sphinx
- ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
build_macosx_36:
<<: *macosx_build_job
variables:
PYTHON_VERSION: "3.6"
# Deploy targets
.deploy_template: &deploy_job
stage: deploy
before_script:
- ./_ci/install.sh _ci master #updates ci support scripts
script:
- ./_ci/deploy.sh
dependencies:
- build_linux_36
- build_macosx_36
tags:
- deployer
deploy_beta:
<<: *deploy_job
environment: beta
only:
- master
deploy_stable:
<<: *deploy_job
environment: stable
only:
- /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags)
except:
- branches
pypi:
stage: pypi
environment: pypi
only:
- /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags)
except:
- branches
before_script:
- ./_ci/install.sh _ci master #updates ci support scripts
script:
- ./_ci/pypi.sh
dependencies:
- build_linux_36
tags:
- deployer
include: 'https://gitlab.idiap.ch/bob/bob.devtools/raw/master/bob/devtools/data/gitlab-ci/beat-docker-package.yaml'
......@@ -73,6 +73,16 @@ if VERBOSE_TEST_LOGGING:
logger.addHandler(handler)
TEST_IMAGES = {
"docker.idiap.ch/beat/beat.env.system.python": "1.3.0r4",
"docker.idiap.ch/beat/beat.env.db.examples": "1.4.0r4",
"docker.idiap.ch/beat/beat.env.cxx": "2.0.0r1",
"docker.idiap.ch/beat/beat.env.client": "2.0.0r1",
}
"""Images used for docker-enabled tests
"""
def setup_package():
prefixes = [
pkg_resources.resource_filename('beat.backend.python.test', 'prefix'),
......@@ -82,9 +92,21 @@ def setup_package():
for path in prefixes:
subprocess.check_call(['rsync', '-arz', path, prefix_folder])
# Docker setup - pull images that are not locally available yet
# This technique prevents errors if docker.idiap.ch is not available
# e.g. when running outside the Idiap network
import docker
client = docker.from_env()
for image, tag in TEST_IMAGES.items():
has_image = False
for installed_image in client.images.list():
for installed_tag in installed_image.tags:
if installed_tag == ('%s:%s' % (image, tag)):
has_image = True
if not has_image:
client.images.pull(image, tag)
if DOCKER_NETWORK_TEST_ENABLED:
import docker
client = docker.from_env()
try:
network = client.networks.get(network_name)
except docker.errors.NotFound:
......
[buildout]
parts = scripts docker
parts = scripts
eggs = beat.core
develop = .
newest = false
[scripts]
recipe = bob.buildout:scripts
[docker]
recipe = collective.recipe.cmd
cmds = BUILDOUT=true ./scripts/docker_pull.sh
on_install = true
on_update = true
#!/usr/bin/env bash
basedir=`pwd`
if [ ! -e ${basedir}/_ci/functions.sh ]; then
# this will happen when executing buildout for the first time
mkdir ${basedir}/_ci
curl --silent "https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/install.sh" > ${basedir}/_ci/install.sh
chmod 755 ${basedir}/_ci/install.sh
${basedir}/_ci/install.sh ${basedir}/_ci master #installs ci support scripts
fi
if [ "${BUILDOUT}" == "true" ]; then
# when executing in the context of buildout, define dummies
export CI_PROJECT_URL=https://gitlab.idiap.ch/beat/beat.core
export CI_PROJECT_DIR=beat
export CI_PROJECT_PATH=beat/beat.core
export CI_PROJECT_NAME=beat.core
export CI_COMMIT_REF_NAME=master
export PYPIUSER=pypiuser
export PYPIPASS=pypipass
export DOCUSER=docuser
export DOCPASS=docpass
export CONDA_ROOT=${basedir}/miniconda
export DOCKER_REGISTRY=docker.idiap.ch
export DOCSERVER=https://www.idiap.ch/software/bob
fi
source ${basedir}/_ci/functions.sh
check_env DOCKER_REGISTRY
# Select here images that are required for minimal operation (or tests)
IMAGES=(
"${DOCKER_REGISTRY}/beat/beat.env.system.python:1.3.0r4"
"${DOCKER_REGISTRY}/beat/beat.env.db.examples:1.4.0r4"
"${DOCKER_REGISTRY}/beat/beat.env.cxx:2.0.0r1"
"${DOCKER_REGISTRY}/beat/beat.env.client:2.0.0r1"
)
check_array_env IMAGES
missing=()
log_info "Checking for minimal set of docker images..."
for k in ${IMAGES[@]}; do
hash_=$(docker images -q ${k})
if [ -z "${hash_}" ]; then
missing+=(${k})
log_warn "${k} NOT available - downloading...";
else
log_info "${k} already available - not re-downloading";
fi
done
# If there are some images missing, download them
if [[ "${#missing[@]}" > 0 ]]; then
# Log in the registry if needed
if [ -z "$CI_SERVER" ]; then
if ! grep -q "${DOCKER_REGISTRY}" ~/.docker/config.json ; then
run_cmd docker login "${DOCKER_REGISTRY}"
fi
else
check_pass CI_BUILD_TOKEN
log_info docker login -u gitlab-ci-token -p "*************" "${DOCKER_REGISTRY}"
docker login -u gitlab-ci-token -p "${CI_BUILD_TOKEN}" "${DOCKER_REGISTRY}"
fi
# Pull the images
for k in "${missing[@]}"; do
run_cmd docker pull ${k}
done
fi
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment