Commit 3fbc1172 authored by André Anjos's avatar André Anjos 💬
Browse files

Merge branch 'docker-test' into 'soft_loop'

Docker test

See merge request !44
parents 62bb0f6e 6a42bfb4
Pipeline #25361 passed with stage
in 27 minutes and 49 seconds
......@@ -4,12 +4,12 @@
# Definition of global variables (all stages)
variables:
CONDA_ROOT: "${CI_PROJECT_DIR}/miniconda"
DOCKER_REGISTRY: docker.idiap.ch
# Definition of our build pipeline order
stages:
- build
- docker
- deploy
- pypi
......@@ -23,6 +23,9 @@ stages:
- chmod 755 _ci/install.sh
- ./_ci/install.sh _ci master #installs ci support scripts
- ./_ci/before_build.sh
- docker info
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN docker.idiap.ch
- ./scripts/before_test.sh build #pulls required docker images and compiles
script:
- ./_ci/build.sh
after_script:
......@@ -37,8 +40,7 @@ stages:
.build_linux_template: &linux_build_job
<<: *build_job
tags:
- docker
image: continuumio/conda-concourse-ci
- docker-build
artifacts:
expire_in: 1 week
paths:
......@@ -63,23 +65,6 @@ stages:
key: "macosx-cache"
# Docker host based testing (must be run inside dind or docker-enabled host)
.docker_test_linux_template: &linux_docker_job
stage: docker
before_script:
# safe keep artifacts as before_build.sh will erase those...
- mv ${CONDA_ROOT}/conda-bld .
- ./_ci/install.sh _ci master #updates ci support scripts
- ./_ci/before_build.sh
- mv conda-bld ${CONDA_ROOT}
- ./scripts/before_test.sh
script:
- export BEAT_DOCKER_TESTS=true
- BOB_TEST_ONLY=true ./_ci/build.sh
after_script:
- ./_ci/after_build.sh
build_linux_36:
<<: *linux_build_job
variables:
......@@ -100,17 +85,6 @@ build_macosx_36:
PYTHON_VERSION: "3.6"
# Docker host based testing
docker_linux_36:
<<: *linux_docker_job
variables:
PYTHON_VERSION: "3.6"
dependencies:
- build_linux_36
tags:
- docker-build
# Deploy targets
.deploy_template: &deploy_job
stage: deploy
......@@ -129,7 +103,7 @@ deploy_beta:
<<: *deploy_job
environment: beta
only:
- 1.7.x
- master
deploy_stable:
......
{
"root_folder": "/path/not/set",
"root_folder": "/tmp/path/not/set",
"protocols": [
{
"name": "double",
......
{
"root_folder": "/path/not/set",
"root_folder": "/tmp/path/not/set",
"protocols": [
]
}
{
"root_folder": "/path/not/set",
"root_folder": "/tmp/path/not/set",
"protocols": [
{
"name": "double",
......
{
"root_folder": "/path/not/set",
"root_folder": "/tmp/path/not/set",
"protocols": [
{
"name": "large",
......
{
"root_folder": "/path/not/set",
"root_folder": "/tmp/path/not/set",
"protocols": [
{
"sets": [
......
{
"root_folder": "/path/not/set"
"root_folder": "/tmp/path/not/set"
}
{
"root_folder": "/path/not/set",
"root_folder": "/tmp/path/not/set",
"protocols": [
{
"name": "double",
......
{
"root_folder": "/path/not/set",
"root_folder": "/tmp/path/not/set",
"protocols": [
{
"name": "double",
......
{
"root_folder": "/path/not/set",
"root_folder": "/tmp/path/not/set",
"protocols": [
{
"name": "double",
......
{
"root_folder": "/path/not/set",
"root_folder": "/tmp/path/not/set",
"protocols": [
{
"name": "protocol",
......
{
"root_folder": "/path/not/set",
"root_folder": "/tmp/path/not/set",
"protocols": [
{
"name": "double",
......
{
"root_folder": "/path/not/set",
"root_folder": "/tmp/path/not/set",
"protocols": [
{
"name": "double",
......
{
"root_folder": "/path/not/set",
"root_folder": "/tmp/path/not/set",
"protocols": [
{
"name": "double",
......
{
"root_folder": "/foo/bar",
"root_folder": "/tmp/foo/bar",
"protocols": [
{
"name": "protocol",
......
[buildout]
parts = scripts
parts = scripts docker
eggs = beat.core
develop = .
newest = false
[scripts]
recipe = bob.buildout:scripts
[docker]
recipe = collective.recipe.cmd
cmds = BUILDOUT=true ./scripts/before_test.sh build
uninstall_cmds = BUILDOUT=true ./scripts/before_test.sh cleanup
on_install = true
on_update = true
#! /bin/bash
ALGORITHMS_ROOT="beat/core/test/prefix/algorithms/user"
BEAT_CORE_PATH=$(dirname "$0")
cd $BEAT_CORE_PATH
# Compilation of the algorithm
if [ "$1" == "build" ]; then
cd $ALGORITHMS_ROOT
# # V1 build
# echo "Building v1 algorithm"
# DOCKER_IMAGE="docker.idiap.ch/beat/beat.env.client:1.3.0r1"
# tar -cf cxx_integers_echo.tar cxx_integers_echo/
# docker run -dti --name build $DOCKER_IMAGE > /dev/null
# docker cp cxx_integers_echo.tar build:/tmp/cxx_integers_echo.tar
# docker exec build bash -c 'cd /tmp ; tar -xf /tmp/cxx_integers_echo.tar'
# docker exec build bash -c 'cd /tmp/cxx_integers_echo ; mkdir build ; cd build ; cmake .. ; make'
# docker cp build:/tmp/cxx_integers_echo/1.so cxx_integers_echo/.
# docker stop build > /dev/null
# docker rm build > /dev/null
# rm cxx_integers_echo.tar
# V2 builds
echo "Building v2 algorithm"
DOCKER_IMAGE="docker.idiap.ch/beat/beat.env.client:2.0.0"
types=('legacy' 'autonomous' 'sequential' 'analyzer')
# types=('autonomous')
for type in "${types[@]}"; do
algorithm_name="cxx_integers_echo_$type"
archive_name="$algorithm_name.tar"
echo "$archive_name" "$algorithm_name/"
tar -cf "$archive_name" "$algorithm_name/"
docker run -dti --name build $DOCKER_IMAGE > /dev/null
docker cp "$archive_name" "build:/tmp/$archive_name"
docker exec build bash -c "cd /tmp ; tar -xf /tmp/$archive_name"
docker exec build bash -c "cd /tmp/$algorithm_name ; mkdir build ; cd build ; cmake .. ; make"
docker cp "build:/tmp/$algorithm_name/1.so" "$algorithm_name/."
docker stop build > /dev/null
docker rm build > /dev/null
rm "$archive_name"
done
fi
# Cleanup of the compiled algorithms
if [ "$1" == "clean" ]; then
rm -f "$ALGORITHMS_ROOT/cxx_integers_echo/1.so"
for type in ['legacy', 'autonomous', 'sequential', 'analyzer']; do
rm -f "$ALGORITHMS_ROOT/cxx_integers_echo_$type/1.so"
done
fi
#! /bin/bash
IMAGES=(
docker.idiap.ch/beat/beat.env.system.python:1.3.0r1
docker.idiap.ch/beat/beat.env.db.examples:1.4.0r1
docker.idiap.ch/beat/beat.env.cxx:2.0.0r1
docker.idiap.ch/beat/beat.env.client:2.0.0r1
)
# Find all the missing images
declare -a MISSING_IMAGES
for IMAGE in ${IMAGES[@]}; do
IMAGE_HASH=$(docker images -q $IMAGE)
if [ -z "$IMAGE_HASH" ]; then
MISSING_IMAGES+=($IMAGE)
fi
done
# If there are some images missing, ask the user if he wants to download them
if [ ${#MISSING_IMAGES[@]} -gt 0 ]; then
echo "--------------------------------------------------------------------------------------"
echo "The following docker images are required (at least for the tests), but were not found:"
echo
for IMAGE in ${MISSING_IMAGES[@]}; do
echo " - $IMAGE"
done
echo
MUST_PULL=0
if [ -z "$CI_SERVER" ]; then
read -p "Do you want to install them now? (yes/no) " -n 1 -r
echo
if [[ $REPLY =~ ^[Yy]$ ]]; then
MUST_PULL=1
fi
else
MUST_PULL=1
fi
if [ $MUST_PULL -eq 1 ]; then
# Log in the registry if needed
if [ -z "$CI_SERVER" ]; then
if ! grep -q "docker.idiap.ch" ~/.docker/config.json ; then
docker login docker.idiap.ch
fi
else
docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN docker.idiap.ch
fi
# Pull the images
for IMAGE in ${MISSING_IMAGES[@]}; do
echo
CMD="docker pull $IMAGE"
echo "> $CMD"
$CMD
done
fi
fi
......@@ -53,10 +53,9 @@ test:
commands:
- worker --help
- if [ -n "${BEAT_DOCKER_TESTS}" ]; then nosetests --with-coverage --cover-package={{ name }} -sv {{ name }}; fi
- if [ -z "${BEAT_DOCKER_TESTS}" ]; then nosetests --with-coverage --cover-package={{ name }} -sv {{ name }} --exclude=".*test_docker.*"; fi
- if [ -z "${BEAT_DOCKER_TESTS}" ]; then sphinx-build -aEW {{ project_dir }}/doc {{ project_dir }}/sphinx; fi
- if [ -z "${BEAT_DOCKER_TESTS}" ]; then sphinx-build -aEb doctest {{ project_dir }}/doc sphinx; fi
- nosetests --with-coverage --cover-package={{ name }} -sv {{ name }}
- sphinx-build -aEW {{ project_dir }}/doc {{ project_dir }}/sphinx
- sphinx-build -aEb doctest {{ project_dir }}/doc sphinx
- conda inspect linkages -p $PREFIX {{ name }} # [not win]
- conda inspect objects -p $PREFIX {{ name }} # [osx]
......
#!/usr/bin/env bash
basedir=`pwd`
if [ ! -e ${basedir}/_ci/functions.sh ]; then
# this will happen when executing buildout for the first time
mkdir ${basedir}/_ci
curl --silent "https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/install.sh" > ${basedir}/_ci/install.sh
chmod 755 ${basedir}/_ci/install.sh
${basedir}/_ci/install.sh ${basedir}/_ci master #installs ci support scripts
else
${basedir}/_ci/install.sh ${basedir}/_ci master #updates ci support scripts
fi
if [ "${BUILDOUT}" == "true" ]; then
# when executing in the context of buildout, define dummies
export CI_PROJECT_URL=https://gitlab.idiap.ch/beat/beat.core
export CI_PROJECT_DIR=beat
export CI_PROJECT_PATH=beat/beat.core
export CI_PROJECT_NAME=beat.core
export CI_COMMIT_REF_NAME=master
export PYPIUSER=pypiuser
export PYPIPASS=pypipass
export DOCUSER=docuser
export DOCPASS=docpass
export CONDA_ROOT=${basedir}/miniconda
export DOCSERVER=https://www.idiap.ch/software/bob
fi
source ${basedir}/_ci/functions.sh
if [ -e ${basedir}/bin ]; then
log_info "Found ${basedir}/bin directory, prepending to PATH..."
PATH=${basedir}/bin:${PATH}
export_env PATH
fi
REGISTRY="docker.idiap.ch"
check_env REGISTRY
......@@ -10,11 +42,15 @@ check_env REGISTRY
IMAGES=(
"${REGISTRY}/beat/beat.env.system.python:1.3.0r3"
"${REGISTRY}/beat/beat.env.db.examples:1.4.0r3"
"${REGISTRY}/beat/beat.env.cxx:2.0.0r0"
"${REGISTRY}/beat/beat.env.client:2.0.0r0"
"${REGISTRY}/beat/beat.env.cxx:2.0.0r1"
"${REGISTRY}/beat/beat.env.client:2.0.0r1"
)
check_array_env IMAGES
# Docker image to be used for compiling the algorithm
BEAT_CXX_DEV_ENVIRONMENT="${REGISTRY}/beat/beat.env.client:2.0.0r1"
check_env BEAT_CXX_DEV_ENVIRONMENT
missing=()
log_info "Checking for minimal set of docker images..."
for k in ${IMAGES[@]}; do
......@@ -47,10 +83,6 @@ if [[ "${#missing[@]}" > 0 ]]; then
done
fi
# Docker image to be used for compiling the algorithm
BEAT_CXX_DEV_ENVIRONMENT="${REGISTRY}/beat/beat.env.client:2.0.0r0"
check_env BEAT_CXX_DEV_ENVIRONMENT
# These are the algorithms that need to be compiled and which we'll test
ALGORITHMS=(
"prefix/algorithms/user/cxx_integers_echo_legacy/1"
......@@ -61,34 +93,26 @@ ALGORITHMS=(
check_array_env ALGORITHMS
# Check where the package root is
BEAT_CORE_TEST_DIR=$(python -c 'import beat.core.test as f; import os; print os.path.abspath(os.path.dirname(f.__file__))')
if [ "${BUILDOUT}" == "true" ]; then
BEAT_CORE_TEST_DIR="${basedir}/beat/core/test"
else
BEAT_CORE_TEST_DIR=$(python -c 'import beat.core.test as f; import os; print(os.path.abspath(os.path.dirname(f.__file__)))')
fi
check_env BEAT_CORE_TEST_DIR
# Compilation of the algorithms in the list above
if [ "$1" == "build" ]; then
for k in "${ALGORITHMS[@]}"; do
algodir=$(dirname "${k}")
run_cmd docker run --rm --user=$(id -u gitlab-runner):$(id -g gitlab-runner) --volume="${BEAT_CORE_TEST_DIR}/scripts/build.sh:/build.sh" --volume="${BEAT_CORE_TEST_DIR}/${algodir}:/sources" --workdir="/sources" --entrypoint="bash" "${BEAT_CXX_DEV_ENVIRONMENT}" /build.sh
done
for k in "${ALGORITHMS[@]}"; do
algodir=$(dirname "${k}")
run_cmd docker run --rm --user=$(id -u):$(id -g) --volume="${BEAT_CORE_TEST_DIR}/scripts/build.sh:/build.sh" --volume="${BEAT_CORE_TEST_DIR}/${algodir}:/sources" --workdir="/sources" --entrypoint="bash" "${BEAT_CXX_DEV_ENVIRONMENT}" /build.sh
done
fi
# Cleanup of the compiled algorithms
if [ "$1" == "clean" ]; then
for k in "${ALGORITHMS[@]}"; do
algodir=$(dirname "${k}")
run_cmd rm -f "${BEAT_CORE_TEST_DIR}/${algodir}/*.so"
run_cmd rm -rf "${BEAT_CORE_TEST_DIR}/${algodir}/build/"
done
for k in "${ALGORITHMS[@]}"; do
algodir=$(dirname "${k}")
run_cmd rm -f "${BEAT_CORE_TEST_DIR}/${algodir}/*.so"
run_cmd rm -rf "${BEAT_CORE_TEST_DIR}/${algodir}/build/"
done
fi
# Makes sure we activate the base environment if available
run_cmd source ${CONDA_ROOT}/etc/profile.d/conda.sh
run_cmd conda activate base
export_env PATH
for subchannel in linux-64 osx-64 noarch; do
if [ ! -d ${CONDA_ROOT}/conda-bld/${subchannel} ]; then
run_cmd mkdir -p ${CONDA_ROOT}/conda-bld/${subchannel}
fi
run_cmd conda index ${CONDA_ROOT}/conda-bld/${subchannel}
done
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment