Commit b3d51ab1 authored by Samuel GAIST's avatar Samuel GAIST

Merge branch 'issue-63' into 'master'

Add automatic test image pulling if not already available (closes #63)

Closes #63

See merge request !51
parents a2a253bd 90161f61
Pipeline #27231 passed with stages
in 20 minutes and 7 seconds
# This build file uses template features from YAML so it is generic enough for
# any Bob project. Don't modify it unless you know what you're doing.
# Definition of global variables (all stages)
variables:
CONDA_ROOT: "${CI_PROJECT_DIR}/miniconda"
DOCKER_REGISTRY: docker.idiap.ch
# Definition of our build pipeline order
stages:
- build
- deploy
- pypi
# Build targets
.build_template: &build_job
stage: build
before_script:
- mkdir _ci
- curl --silent "https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/install.sh" > _ci/install.sh
- chmod 755 _ci/install.sh
- ./_ci/install.sh _ci master #installs ci support scripts
- ./_ci/before_build.sh
- docker info
- ./scripts/docker_pull.sh #pulls required test images
script:
- ./_ci/build.sh
after_script:
- ./_ci/after_build.sh
cache: &build_caches
paths:
- miniconda.sh
- ${CONDA_ROOT}/pkgs/*.tar.bz2
- ${CONDA_ROOT}/pkgs/urls.txt
.build_linux_template: &linux_build_job
<<: *build_job
tags:
- docker-build
artifacts:
expire_in: 1 week
paths:
- _ci/
- ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
cache:
<<: *build_caches
key: "linux-cache"
.build_macosx_template: &macosx_build_job
<<: *build_job
tags:
- macosx
artifacts:
expire_in: 1 week
paths:
- _ci/
- ${CONDA_ROOT}/conda-bld/osx-64/*.tar.bz2
cache:
<<: *build_caches
key: "macosx-cache"
build_linux_36:
<<: *linux_build_job
variables:
PYTHON_VERSION: "3.6"
BUILD_EGG: "true"
artifacts:
expire_in: 1 week
paths:
- _ci/
- dist/*.zip
- sphinx
- ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
build_macosx_36:
<<: *macosx_build_job
variables:
PYTHON_VERSION: "3.6"
# Deploy targets
.deploy_template: &deploy_job
stage: deploy
before_script:
- ./_ci/install.sh _ci master #updates ci support scripts
script:
- ./_ci/deploy.sh
dependencies:
- build_linux_36
- build_macosx_36
tags:
- deployer
deploy_beta:
<<: *deploy_job
environment: beta
only:
- master
deploy_stable:
<<: *deploy_job
environment: stable
only:
- /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags)
except:
- branches
pypi:
stage: pypi
environment: pypi
only:
- /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags)
except:
- branches
before_script:
- ./_ci/install.sh _ci master #updates ci support scripts
script:
- ./_ci/pypi.sh
dependencies:
- build_linux_36
tags:
- deployer
include:
- 'https://gitlab.idiap.ch/bob/bob.devtools/raw/master/bob/devtools/data/gitlab-ci/single-package.yaml'
- 'https://gitlab.idiap.ch/bob/bob.devtools/raw/master/bob/devtools/data/gitlab-ci/beat-docker-package.yaml'
......@@ -34,6 +34,49 @@ import shutil
import nose
import docker
#----------------------------------------------------------
DOCKER_TEST_IMAGES = {
"docker.idiap.ch/beat/beat.env.system.python": "1.3.0r4",
"docker.idiap.ch/beat/beat.env.db.examples": "1.4.0r4",
"docker.idiap.ch/beat/beat.env.cxx": "2.0.0r1",
"docker.idiap.ch/beat/beat.env.client": "2.0.0r1",
}
"""Images used for docker-enabled tests within this and other BEAT packages
"""
def pull_docker_test_images():
"""To be called when you need to set up tests using ``DOCKER_TEST_IMAGES``
This function will pull images that are not locally available yet
This technique prevents errors if docker.idiap.ch is not available,
e.g. when running outside the Idiap network
"""
import docker
client = docker.from_env()
for image, tag in DOCKER_TEST_IMAGES.items():
has_image = False
for installed_image in client.images.list():
for installed_tag in installed_image.tags:
if installed_tag == ('%s:%s' % (image, tag)):
has_image = True
if not has_image: #must pull (network connection required)
token = os.environ.get('CI_BUILD_TOKEN')
params = (image, tag)
if token is not None: #running on CI, setup
auth_config = dict(username='gitlab-ci-token', password=token)
params += (auth_config,)
client.images.pull(*params)
#----------------------------------------------------------
......
......@@ -9,6 +9,6 @@ recipe = bob.buildout:scripts
[docker]
recipe = collective.recipe.cmd
cmds = BUILDOUT=true ./scripts/docker_pull.sh
cmds = ./bin/python -c 'from beat.core.test.utils import pull_docker_test_images as f; f()'
on_install = true
on_update = true
......@@ -52,6 +52,8 @@ test:
- {{ name }}
commands:
# pulls required images once before running the tests
- python -c "from beat.core.test.utils import pull_docker_test_images as f; f()"
- worker --help
- nosetests --with-coverage --cover-package={{ name }} -sv {{ name }}
- sphinx-build -aEW {{ project_dir }}/doc {{ project_dir }}/sphinx
......
......@@ -16,6 +16,6 @@ recipe = bob.buildout:scripts
[docker]
recipe = collective.recipe.cmd
cmds = BUILDOUT=true ./scripts/docker_pull.sh
cmds = ./bin/python -c 'from beat.core.test.utils import pull_docker_test_images as f; f()'
on_install = true
on_update = true
#!/usr/bin/env bash
basedir=`pwd`
if [ ! -e ${basedir}/_ci/functions.sh ]; then
# this will happen when executing buildout for the first time
mkdir ${basedir}/_ci
curl --silent "https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/install.sh" > ${basedir}/_ci/install.sh
chmod 755 ${basedir}/_ci/install.sh
${basedir}/_ci/install.sh ${basedir}/_ci master #installs ci support scripts
fi
if [ "${BUILDOUT}" == "true" ]; then
# when executing in the context of buildout, define dummies
export CI_PROJECT_URL=https://gitlab.idiap.ch/beat/beat.core
export CI_PROJECT_DIR=beat
export CI_PROJECT_PATH=beat/beat.core
export CI_PROJECT_NAME=beat.core
export CI_COMMIT_REF_NAME=master
export PYPIUSER=pypiuser
export PYPIPASS=pypipass
export DOCUSER=docuser
export DOCPASS=docpass
export CONDA_ROOT=${basedir}/miniconda
export DOCKER_REGISTRY=docker.idiap.ch
export DOCSERVER=https://www.idiap.ch/software/bob
fi
source ${basedir}/_ci/functions.sh
check_env DOCKER_REGISTRY
# Select here images that are required for minimal operation (or tests)
IMAGES=(
"${DOCKER_REGISTRY}/beat/beat.env.system.python:1.3.0r4"
"${DOCKER_REGISTRY}/beat/beat.env.db.examples:1.4.0r4"
"${DOCKER_REGISTRY}/beat/beat.env.cxx:2.0.0r1"
"${DOCKER_REGISTRY}/beat/beat.env.client:2.0.0r1"
)
check_array_env IMAGES
missing=()
log_info "Checking for minimal set of docker images..."
for k in ${IMAGES[@]}; do
hash_=$(docker images -q ${k})
if [ -z "${hash_}" ]; then
missing+=(${k})
log_warn "${k} NOT available - downloading...";
else
log_info "${k} already available - not re-downloading";
fi
done
# If there are some images missing, download them
if [[ "${#missing[@]}" > 0 ]]; then
# Log in the registry if needed
if [ -z "$CI_SERVER" ]; then
if ! grep -q "${DOCKER_REGISTRY}" ~/.docker/config.json ; then
run_cmd docker login "${DOCKER_REGISTRY}"
fi
else
check_pass CI_BUILD_TOKEN
log_info docker login -u gitlab-ci-token -p "*************" "${DOCKER_REGISTRY}"
docker login -u gitlab-ci-token -p "${CI_BUILD_TOKEN}" "${DOCKER_REGISTRY}"
fi
# Pull the images
for k in "${missing[@]}"; do
run_cmd docker pull ${k}
done
fi
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment