Commit 0da0f76d authored by André Anjos's avatar André Anjos 💬 Committed by Samuel GAIST
Browse files

[ci/buildout] Buildout and CI now share the same setup scripts

Image pulling and cxx algorithm compilation are done in only
one place now.
parent 00585f9e
......@@ -22,7 +22,8 @@ stages:
- chmod 755 _ci/install.sh
- ./_ci/install.sh _ci master #installs ci support scripts
- ./_ci/before_build.sh
- ./scripts/before_test.sh #pulls required docker images
- docker info
- ./scripts/before_test.sh build #pulls required docker images and compiles
script:
- ./_ci/build.sh
after_script:
......
[buildout]
parts = scripts
parts = scripts docker
eggs = beat.core
develop = .
newest = false
[scripts]
recipe = bob.buildout:scripts
[docker]
recipe = collective.recipe.cmd
cmds = BUILDOUT=true ./scripts/before_test.sh build
uninstall_cmds =
on_install = true
on_update = true
#! /bin/bash
ALGORITHMS_ROOT="beat/core/test/prefix/algorithms/user"
BEAT_CORE_PATH=$(dirname "$0")
cd $BEAT_CORE_PATH
# Compilation of the algorithm
if [ "$1" == "build" ]; then
cd $ALGORITHMS_ROOT
# # V1 build
# echo "Building v1 algorithm"
# DOCKER_IMAGE="docker.idiap.ch/beat/beat.env.client:1.3.0r1"
# tar -cf cxx_integers_echo.tar cxx_integers_echo/
# docker run -dti --name build $DOCKER_IMAGE > /dev/null
# docker cp cxx_integers_echo.tar build:/tmp/cxx_integers_echo.tar
# docker exec build bash -c 'cd /tmp ; tar -xf /tmp/cxx_integers_echo.tar'
# docker exec build bash -c 'cd /tmp/cxx_integers_echo ; mkdir build ; cd build ; cmake .. ; make'
# docker cp build:/tmp/cxx_integers_echo/1.so cxx_integers_echo/.
# docker stop build > /dev/null
# docker rm build > /dev/null
# rm cxx_integers_echo.tar
# V2 builds
echo "Building v2 algorithm"
DOCKER_IMAGE="docker.idiap.ch/beat/beat.env.client:2.0.0r1"
types=('legacy' 'autonomous' 'sequential' 'analyzer')
# types=('autonomous')
for type in "${types[@]}"; do
algorithm_name="cxx_integers_echo_$type"
archive_name="$algorithm_name.tar"
echo "$archive_name" "$algorithm_name/"
tar -cf "$archive_name" "$algorithm_name/"
docker run -dti --name build $DOCKER_IMAGE > /dev/null
docker cp "$archive_name" "build:/tmp/$archive_name"
docker exec build bash -c "cd /tmp ; tar -xf /tmp/$archive_name"
docker exec build bash -c "cd /tmp/$algorithm_name ; mkdir build ; cd build ; cmake .. ; make"
docker cp "build:/tmp/$algorithm_name/1.so" "$algorithm_name/."
docker stop build > /dev/null
docker rm build > /dev/null
rm "$archive_name"
done
fi
# Cleanup of the compiled algorithms
if [ "$1" == "clean" ]; then
rm -f "$ALGORITHMS_ROOT/cxx_integers_echo/1.so"
for type in ['legacy', 'autonomous', 'sequential', 'analyzer']; do
rm -f "$ALGORITHMS_ROOT/cxx_integers_echo_$type/1.so"
done
fi
#! /bin/bash
IMAGES=(
docker.idiap.ch/beat/beat.env.system.python:1.3.0r3
docker.idiap.ch/beat/beat.env.db.examples:1.4.0r3
docker.idiap.ch/beat/beat.env.cxx:2.0.0r1
docker.idiap.ch/beat/beat.env.client:2.0.0r1
)
# Find all the missing images
declare -a MISSING_IMAGES
for IMAGE in ${IMAGES[@]}; do
IMAGE_HASH=$(docker images -q $IMAGE)
if [ -z "$IMAGE_HASH" ]; then
MISSING_IMAGES+=($IMAGE)
fi
done
# If there are some images missing, ask the user if he wants to download them
if [ ${#MISSING_IMAGES[@]} -gt 0 ]; then
echo "--------------------------------------------------------------------------------------"
echo "The following docker images are required (at least for the tests), but were not found:"
echo
for IMAGE in ${MISSING_IMAGES[@]}; do
echo " - $IMAGE"
done
echo
MUST_PULL=0
if [ -z "$CI_SERVER" ]; then
read -p "Do you want to install them now? (y/n) " -n 1 -r
echo
if [[ $REPLY =~ ^[Yy]$ ]]; then
MUST_PULL=1
fi
else
MUST_PULL=1
fi
if [ $MUST_PULL -eq 1 ]; then
# Log in the registry if needed
if [ -z "$CI_SERVER" ]; then
if ! grep -q "docker.idiap.ch" ~/.docker/config.json ; then
docker login docker.idiap.ch
fi
else
docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN docker.idiap.ch
fi
# Pull the images
for IMAGE in ${MISSING_IMAGES[@]}; do
echo
CMD="docker pull $IMAGE"
echo "> $CMD"
$CMD
done
fi
fi
#!/usr/bin/env bash
basedir=`pwd`
if [ ! -e ${basedir}/_ci/functions.sh ]; then
# this will happen when executing buildout for the first time
mkdir ${basedir}/_ci
curl --silent "https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/install.sh" > ${basedir}/_ci/install.sh
chmod 755 ${basedir}/_ci/install.sh
${basedir}/_ci/install.sh ${basedir}/_ci master #installs ci support scripts
else
${basedir}/_ci/install.sh ${basedir}/_ci master #updates ci support scripts
fi
if [ "${BUILDOUT}" == "true" ]; then
export PATH=${basedir}/bin:${PATH} #use python from here
# when executing in the context of buildout, define dummies
export CI_PROJECT_URL=https://gitlab.idiap.ch/beat/beat.core
export CI_PROJECT_DIR=beat
export CI_PROJECT_PATH=beat/beat.core
export CI_PROJECT_NAME=beat.core
export CI_COMMIT_REF_NAME=master
export PYPIUSER=pypiuser
export PYPIPASS=pypipass
export DOCUSER=docuser
export DOCPASS=docpass
export CONDA_ROOT=${basedir}/miniconda
export DOCSERVER=https://www.idiap.ch/software/bob
fi
source ${basedir}/_ci/functions.sh
REGISTRY="docker.idiap.ch"
......@@ -15,6 +42,10 @@ IMAGES=(
)
check_array_env IMAGES
# Docker image to be used for compiling the algorithm
BEAT_CXX_DEV_ENVIRONMENT="${REGISTRY}/beat/beat.env.client:2.0.0r1"
check_env BEAT_CXX_DEV_ENVIRONMENT
missing=()
log_info "Checking for minimal set of docker images..."
for k in ${IMAGES[@]}; do
......@@ -47,10 +78,6 @@ if [[ "${#missing[@]}" > 0 ]]; then
done
fi
# Docker image to be used for compiling the algorithm
BEAT_CXX_DEV_ENVIRONMENT="${REGISTRY}/beat/beat.env.client:2.0.0r0"
check_env BEAT_CXX_DEV_ENVIRONMENT
# These are the algorithms that need to be compiled and which we'll test
ALGORITHMS=(
"prefix/algorithms/user/cxx_integers_echo_legacy/1"
......@@ -61,34 +88,40 @@ ALGORITHMS=(
check_array_env ALGORITHMS
# Check where the package root is
BEAT_CORE_TEST_DIR=$(python -c 'import beat.core.test as f; import os; print os.path.abspath(os.path.dirname(f.__file__))')
if [ "${BUILDOUT}" == "true" ]; then
BEAT_CORE_TEST_DIR="${basedir}/beat/core/test"
else
BEAT_CORE_TEST_DIR=$(python -c 'import beat.core.test as f; import os; print os.path.abspath(os.path.dirname(f.__file__))')
fi
check_env BEAT_CORE_TEST_DIR
# Compilation of the algorithms in the list above
if [ "$1" == "build" ]; then
for k in "${ALGORITHMS[@]}"; do
algodir=$(dirname "${k}")
run_cmd docker run --rm --user=$(id -u gitlab-runner):$(id -g gitlab-runner) --volume="${BEAT_CORE_TEST_DIR}/scripts/build.sh:/build.sh" --volume="${BEAT_CORE_TEST_DIR}/${algodir}:/sources" --workdir="/sources" --entrypoint="bash" "${BEAT_CXX_DEV_ENVIRONMENT}" /build.sh
done
for k in "${ALGORITHMS[@]}"; do
algodir=$(dirname "${k}")
run_cmd docker run --rm --user=$(id -u):$(id -g) --volume="${BEAT_CORE_TEST_DIR}/scripts/build.sh:/build.sh" --volume="${BEAT_CORE_TEST_DIR}/${algodir}:/sources" --workdir="/sources" --entrypoint="bash" "${BEAT_CXX_DEV_ENVIRONMENT}" /build.sh
done
fi
# Cleanup of the compiled algorithms
if [ "$1" == "clean" ]; then
for k in "${ALGORITHMS[@]}"; do
algodir=$(dirname "${k}")
run_cmd rm -f "${BEAT_CORE_TEST_DIR}/${algodir}/*.so"
run_cmd rm -rf "${BEAT_CORE_TEST_DIR}/${algodir}/build/"
done
for k in "${ALGORITHMS[@]}"; do
algodir=$(dirname "${k}")
run_cmd rm -f "${BEAT_CORE_TEST_DIR}/${algodir}/*.so"
run_cmd rm -rf "${BEAT_CORE_TEST_DIR}/${algodir}/build/"
done
fi
# Makes sure we activate the base environment if available
run_cmd source ${CONDA_ROOT}/etc/profile.d/conda.sh
run_cmd conda activate base
export_env PATH
if [ "${BUILDOUT}" != "true" ]; then
# Makes sure we activate the base environment if available
run_cmd source ${CONDA_ROOT}/etc/profile.d/conda.sh
run_cmd conda activate base
export_env PATH
for subchannel in linux-64 osx-64 noarch; do
if [ ! -d ${CONDA_ROOT}/conda-bld/${subchannel} ]; then
run_cmd mkdir -p ${CONDA_ROOT}/conda-bld/${subchannel}
fi
run_cmd conda index ${CONDA_ROOT}/conda-bld/${subchannel}
done
for subchannel in linux-64 osx-64 noarch; do
if [ ! -d ${CONDA_ROOT}/conda-bld/${subchannel} ]; then
run_cmd mkdir -p ${CONDA_ROOT}/conda-bld/${subchannel}
fi
run_cmd conda index ${CONDA_ROOT}/conda-bld/${subchannel}
done
fi
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment