diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 00ab79ce6221a89665c25e9a6c601109549c060c..4eaf3cfce34f45fe5a1bb7fd9cfa4eed2b497a13 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,12 +4,12 @@ # Definition of global variables (all stages) variables: CONDA_ROOT: "${CI_PROJECT_DIR}/miniconda" + DOCKER_REGISTRY: docker.idiap.ch # Definition of our build pipeline order stages: - build - - docker - deploy - pypi @@ -23,6 +23,9 @@ stages: - chmod 755 _ci/install.sh - ./_ci/install.sh _ci master #installs ci support scripts - ./_ci/before_build.sh + - docker info + - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN docker.idiap.ch + - ./scripts/before_test.sh build #pulls required docker images and compiles script: - ./_ci/build.sh after_script: @@ -37,8 +40,7 @@ stages: .build_linux_template: &linux_build_job <<: *build_job tags: - - docker - image: continuumio/conda-concourse-ci + - docker-build artifacts: expire_in: 1 week paths: @@ -63,23 +65,6 @@ stages: key: "macosx-cache" -# Docker host based testing (must be run inside dind or docker-enabled host) -.docker_test_linux_template: &linux_docker_job - stage: docker - before_script: - # safe keep artifacts as before_build.sh will erase those... - - mv ${CONDA_ROOT}/conda-bld . - - ./_ci/install.sh _ci master #updates ci support scripts - - ./_ci/before_build.sh - - mv conda-bld ${CONDA_ROOT} - - ./scripts/before_test.sh - script: - - export BEAT_DOCKER_TESTS=true - - BOB_TEST_ONLY=true ./_ci/build.sh - after_script: - - ./_ci/after_build.sh - - build_linux_36: <<: *linux_build_job variables: @@ -100,17 +85,6 @@ build_macosx_36: PYTHON_VERSION: "3.6" -# Docker host based testing -docker_linux_36: - <<: *linux_docker_job - variables: - PYTHON_VERSION: "3.6" - dependencies: - - build_linux_36 - tags: - - docker-build - - # Deploy targets .deploy_template: &deploy_job stage: deploy @@ -129,7 +103,7 @@ deploy_beta: <<: *deploy_job environment: beta only: - - 1.7.x + - master deploy_stable: diff --git a/beat/core/test/prefix/databases/empty_protocol_sets/1.json b/beat/core/test/prefix/databases/empty_protocol_sets/1.json index fef2d3f284852ac1150bbedfe267e0710b47a564..ef4c2af004079e6d84fcd15a48709f2070d5091f 100644 --- a/beat/core/test/prefix/databases/empty_protocol_sets/1.json +++ b/beat/core/test/prefix/databases/empty_protocol_sets/1.json @@ -1,5 +1,5 @@ { - "root_folder": "/path/not/set", + "root_folder": "/tmp/path/not/set", "protocols": [ { "name": "double", diff --git a/beat/core/test/prefix/databases/empty_protocols/1.json b/beat/core/test/prefix/databases/empty_protocols/1.json index 6884a1bb52279c126cf2463c3482bf86b5e32e39..aa02e6a9e9064df0a0ed7a620f64497f644d2e78 100644 --- a/beat/core/test/prefix/databases/empty_protocols/1.json +++ b/beat/core/test/prefix/databases/empty_protocols/1.json @@ -1,5 +1,5 @@ { - "root_folder": "/path/not/set", + "root_folder": "/tmp/path/not/set", "protocols": [ ] } diff --git a/beat/core/test/prefix/databases/empty_set_outputs/1.json b/beat/core/test/prefix/databases/empty_set_outputs/1.json index 6fde45ea3dfcd0a35f0bc99afbf657be4c5ebee8..c5ea0efea32f82fed5ee738bee7f410cdd73e9fa 100644 --- a/beat/core/test/prefix/databases/empty_set_outputs/1.json +++ b/beat/core/test/prefix/databases/empty_set_outputs/1.json @@ -1,5 +1,5 @@ { - "root_folder": "/path/not/set", + "root_folder": "/tmp/path/not/set", "protocols": [ { "name": "double", diff --git a/beat/core/test/prefix/databases/large/1.json b/beat/core/test/prefix/databases/large/1.json index f12c594f5917fee21ee5193dc70354a27715397f..2c8bc3a821dd9d4a4cb9a03aa5b7d5c06f830ca4 100644 --- a/beat/core/test/prefix/databases/large/1.json +++ b/beat/core/test/prefix/databases/large/1.json @@ -1,5 +1,5 @@ { - "root_folder": "/path/not/set", + "root_folder": "/tmp/path/not/set", "protocols": [ { "name": "large", diff --git a/beat/core/test/prefix/databases/missing_protocol_name/1.json b/beat/core/test/prefix/databases/missing_protocol_name/1.json index 91559e22a773546c095458b97ae1e22ba3146bbe..49a46140be6f6c4cac2991da93968b98b617678f 100644 --- a/beat/core/test/prefix/databases/missing_protocol_name/1.json +++ b/beat/core/test/prefix/databases/missing_protocol_name/1.json @@ -1,5 +1,5 @@ { - "root_folder": "/path/not/set", + "root_folder": "/tmp/path/not/set", "protocols": [ { "sets": [ diff --git a/beat/core/test/prefix/databases/missing_protocols/1.json b/beat/core/test/prefix/databases/missing_protocols/1.json index ebadcb19b4357bae80dfb6fef9aa94ee8924a84f..7c90729be9831c20fb3e86be8ce0423a2889dfb6 100644 --- a/beat/core/test/prefix/databases/missing_protocols/1.json +++ b/beat/core/test/prefix/databases/missing_protocols/1.json @@ -1,3 +1,3 @@ { - "root_folder": "/path/not/set" + "root_folder": "/tmp/path/not/set" } diff --git a/beat/core/test/prefix/databases/missing_set_name/1.json b/beat/core/test/prefix/databases/missing_set_name/1.json index c083acdb91d9e5d6cefed89e44e561519b6d2c9e..b81a112bd790bda7c26143d61f584958930e24c5 100644 --- a/beat/core/test/prefix/databases/missing_set_name/1.json +++ b/beat/core/test/prefix/databases/missing_set_name/1.json @@ -1,5 +1,5 @@ { - "root_folder": "/path/not/set", + "root_folder": "/tmp/path/not/set", "protocols": [ { "name": "double", diff --git a/beat/core/test/prefix/databases/missing_set_outputs/1.json b/beat/core/test/prefix/databases/missing_set_outputs/1.json index c2132825934cfbe8a330b9264fa2bde9f4d35de8..3b2cfa28d412382025b319a4da47da0f49fe9284 100644 --- a/beat/core/test/prefix/databases/missing_set_outputs/1.json +++ b/beat/core/test/prefix/databases/missing_set_outputs/1.json @@ -1,5 +1,5 @@ { - "root_folder": "/path/not/set", + "root_folder": "/tmp/path/not/set", "protocols": [ { "name": "double", diff --git a/beat/core/test/prefix/databases/missing_set_view/1.json b/beat/core/test/prefix/databases/missing_set_view/1.json index a910e587e0c73cb82ff10476b321dc5f8fc7c474..22051b0fa0428b99d76b22d100ffb604fdd77532 100644 --- a/beat/core/test/prefix/databases/missing_set_view/1.json +++ b/beat/core/test/prefix/databases/missing_set_view/1.json @@ -1,5 +1,5 @@ { - "root_folder": "/path/not/set", + "root_folder": "/tmp/path/not/set", "protocols": [ { "name": "double", diff --git a/beat/core/test/prefix/databases/mixed_protocol_names/1.json b/beat/core/test/prefix/databases/mixed_protocol_names/1.json index ec816823512b2510cc40ce8fd35535d60ec2f948..5fdd6c7840b8bc82c68b279b9667aa41a33afa39 100644 --- a/beat/core/test/prefix/databases/mixed_protocol_names/1.json +++ b/beat/core/test/prefix/databases/mixed_protocol_names/1.json @@ -1,5 +1,5 @@ { - "root_folder": "/path/not/set", + "root_folder": "/tmp/path/not/set", "protocols": [ { "name": "protocol", diff --git a/beat/core/test/prefix/databases/mixed_set_names/1.json b/beat/core/test/prefix/databases/mixed_set_names/1.json index b97af0da0801b24c07b4429f60d129476600024b..2bc02b0cf32148682b03666dbb73903aec35b330 100644 --- a/beat/core/test/prefix/databases/mixed_set_names/1.json +++ b/beat/core/test/prefix/databases/mixed_set_names/1.json @@ -1,5 +1,5 @@ { - "root_folder": "/path/not/set", + "root_folder": "/tmp/path/not/set", "protocols": [ { "name": "double", diff --git a/beat/core/test/prefix/databases/same_protocol_names/1.json b/beat/core/test/prefix/databases/same_protocol_names/1.json index 024f95ae570a6d9df4c454042b9bf3b47b3394d5..ea8f93f372d1578504bd8774575d7fa9639e58bf 100644 --- a/beat/core/test/prefix/databases/same_protocol_names/1.json +++ b/beat/core/test/prefix/databases/same_protocol_names/1.json @@ -1,5 +1,5 @@ { - "root_folder": "/path/not/set", + "root_folder": "/tmp/path/not/set", "protocols": [ { "name": "double", diff --git a/beat/core/test/prefix/databases/same_set_names/1.json b/beat/core/test/prefix/databases/same_set_names/1.json index bbbe91e89e55c8e8079698fcaf4286d8aa4db254..deabf9aa1b2c3c6a337a594e24485e5a1aefa63f 100644 --- a/beat/core/test/prefix/databases/same_set_names/1.json +++ b/beat/core/test/prefix/databases/same_set_names/1.json @@ -1,5 +1,5 @@ { - "root_folder": "/path/not/set", + "root_folder": "/tmp/path/not/set", "protocols": [ { "name": "double", diff --git a/beat/core/test/prefix/databases/simple/1.json b/beat/core/test/prefix/databases/simple/1.json index dcc64d5710e2d1cf62fba3339f96de34389475aa..db336902b9731606954d62abb2971aaffd752069 100644 --- a/beat/core/test/prefix/databases/simple/1.json +++ b/beat/core/test/prefix/databases/simple/1.json @@ -1,5 +1,5 @@ { - "root_folder": "/foo/bar", + "root_folder": "/tmp/foo/bar", "protocols": [ { "name": "protocol", diff --git a/buildout.cfg b/buildout.cfg index f4dd83034743049f04ae98f007b84f941d12386b..9deabb2658a1abdce81aa98ece6eae34d6d51da8 100644 --- a/buildout.cfg +++ b/buildout.cfg @@ -1,8 +1,15 @@ [buildout] -parts = scripts +parts = scripts docker eggs = beat.core develop = . newest = false [scripts] recipe = bob.buildout:scripts + +[docker] +recipe = collective.recipe.cmd +cmds = BUILDOUT=true ./scripts/before_test.sh build +uninstall_cmds = BUILDOUT=true ./scripts/before_test.sh cleanup +on_install = true +on_update = true diff --git a/buildout_compile_cxx_algorithm.sh b/buildout_compile_cxx_algorithm.sh deleted file mode 100755 index 84cd57fd64cb8572b4d5009c89445c1700df3502..0000000000000000000000000000000000000000 --- a/buildout_compile_cxx_algorithm.sh +++ /dev/null @@ -1,55 +0,0 @@ -#! /bin/bash - -ALGORITHMS_ROOT="beat/core/test/prefix/algorithms/user" - - -BEAT_CORE_PATH=$(dirname "$0") -cd $BEAT_CORE_PATH - - -# Compilation of the algorithm -if [ "$1" == "build" ]; then - cd $ALGORITHMS_ROOT - -# # V1 build -# echo "Building v1 algorithm" -# DOCKER_IMAGE="docker.idiap.ch/beat/beat.env.client:1.3.0r1" -# tar -cf cxx_integers_echo.tar cxx_integers_echo/ -# docker run -dti --name build $DOCKER_IMAGE > /dev/null -# docker cp cxx_integers_echo.tar build:/tmp/cxx_integers_echo.tar -# docker exec build bash -c 'cd /tmp ; tar -xf /tmp/cxx_integers_echo.tar' -# docker exec build bash -c 'cd /tmp/cxx_integers_echo ; mkdir build ; cd build ; cmake .. ; make' -# docker cp build:/tmp/cxx_integers_echo/1.so cxx_integers_echo/. -# docker stop build > /dev/null -# docker rm build > /dev/null -# rm cxx_integers_echo.tar - -# V2 builds - echo "Building v2 algorithm" - DOCKER_IMAGE="docker.idiap.ch/beat/beat.env.client:2.0.0" - types=('legacy' 'autonomous' 'sequential' 'analyzer') - # types=('autonomous') - for type in "${types[@]}"; do - algorithm_name="cxx_integers_echo_$type" - archive_name="$algorithm_name.tar" - echo "$archive_name" "$algorithm_name/" - tar -cf "$archive_name" "$algorithm_name/" - docker run -dti --name build $DOCKER_IMAGE > /dev/null - docker cp "$archive_name" "build:/tmp/$archive_name" - docker exec build bash -c "cd /tmp ; tar -xf /tmp/$archive_name" - docker exec build bash -c "cd /tmp/$algorithm_name ; mkdir build ; cd build ; cmake .. ; make" - docker cp "build:/tmp/$algorithm_name/1.so" "$algorithm_name/." - docker stop build > /dev/null - docker rm build > /dev/null - rm "$archive_name" - done -fi - - -# Cleanup of the compiled algorithms -if [ "$1" == "clean" ]; then - rm -f "$ALGORITHMS_ROOT/cxx_integers_echo/1.so" - for type in ['legacy', 'autonomous', 'sequential', 'analyzer']; do - rm -f "$ALGORITHMS_ROOT/cxx_integers_echo_$type/1.so" - done -fi diff --git a/buildout_pull_images.sh b/buildout_pull_images.sh deleted file mode 100755 index f407a4816ca4976d2e45d10ef5d312615d60d643..0000000000000000000000000000000000000000 --- a/buildout_pull_images.sh +++ /dev/null @@ -1,64 +0,0 @@ -#! /bin/bash - -IMAGES=( - docker.idiap.ch/beat/beat.env.system.python:1.3.0r1 - docker.idiap.ch/beat/beat.env.db.examples:1.4.0r1 - docker.idiap.ch/beat/beat.env.cxx:2.0.0r1 - docker.idiap.ch/beat/beat.env.client:2.0.0r1 -) - - -# Find all the missing images -declare -a MISSING_IMAGES - -for IMAGE in ${IMAGES[@]}; do - IMAGE_HASH=$(docker images -q $IMAGE) - - if [ -z "$IMAGE_HASH" ]; then - MISSING_IMAGES+=($IMAGE) - fi -done - - -# If there are some images missing, ask the user if he wants to download them -if [ ${#MISSING_IMAGES[@]} -gt 0 ]; then - echo "--------------------------------------------------------------------------------------" - echo "The following docker images are required (at least for the tests), but were not found:" - echo - for IMAGE in ${MISSING_IMAGES[@]}; do - echo " - $IMAGE" - done - echo - - MUST_PULL=0 - - if [ -z "$CI_SERVER" ]; then - read -p "Do you want to install them now? (yes/no) " -n 1 -r - echo - - if [[ $REPLY =~ ^[Yy]$ ]]; then - MUST_PULL=1 - fi - else - MUST_PULL=1 - fi - - if [ $MUST_PULL -eq 1 ]; then - # Log in the registry if needed - if [ -z "$CI_SERVER" ]; then - if ! grep -q "docker.idiap.ch" ~/.docker/config.json ; then - docker login docker.idiap.ch - fi - else - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN docker.idiap.ch - fi - - # Pull the images - for IMAGE in ${MISSING_IMAGES[@]}; do - echo - CMD="docker pull $IMAGE" - echo "> $CMD" - $CMD - done - fi -fi diff --git a/conda/meta.yaml b/conda/meta.yaml index ca26dc15de766e1d68350bf41ba0088e7e3565d3..41dad405f83681d76cbc4ce1475a0df4de8774b9 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -53,10 +53,9 @@ test: commands: - worker --help - - if [ -n "${BEAT_DOCKER_TESTS}" ]; then nosetests --with-coverage --cover-package={{ name }} -sv {{ name }}; fi - - if [ -z "${BEAT_DOCKER_TESTS}" ]; then nosetests --with-coverage --cover-package={{ name }} -sv {{ name }} --exclude=".*test_docker.*"; fi - - if [ -z "${BEAT_DOCKER_TESTS}" ]; then sphinx-build -aEW {{ project_dir }}/doc {{ project_dir }}/sphinx; fi - - if [ -z "${BEAT_DOCKER_TESTS}" ]; then sphinx-build -aEb doctest {{ project_dir }}/doc sphinx; fi + - nosetests --with-coverage --cover-package={{ name }} -sv {{ name }} + - sphinx-build -aEW {{ project_dir }}/doc {{ project_dir }}/sphinx + - sphinx-build -aEb doctest {{ project_dir }}/doc sphinx - conda inspect linkages -p $PREFIX {{ name }} # [not win] - conda inspect objects -p $PREFIX {{ name }} # [osx] diff --git a/scripts/before_test.sh b/scripts/before_test.sh index a4900189c9609fa8417769f7162de9d12411d430..f7b19f9c03d76709b30c594c4d92c79e0599846f 100755 --- a/scripts/before_test.sh +++ b/scripts/before_test.sh @@ -1,8 +1,40 @@ #!/usr/bin/env bash basedir=`pwd` + +if [ ! -e ${basedir}/_ci/functions.sh ]; then + # this will happen when executing buildout for the first time + mkdir ${basedir}/_ci + curl --silent "https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/install.sh" > ${basedir}/_ci/install.sh + chmod 755 ${basedir}/_ci/install.sh + ${basedir}/_ci/install.sh ${basedir}/_ci master #installs ci support scripts +else + ${basedir}/_ci/install.sh ${basedir}/_ci master #updates ci support scripts +fi + +if [ "${BUILDOUT}" == "true" ]; then + # when executing in the context of buildout, define dummies + export CI_PROJECT_URL=https://gitlab.idiap.ch/beat/beat.core + export CI_PROJECT_DIR=beat + export CI_PROJECT_PATH=beat/beat.core + export CI_PROJECT_NAME=beat.core + export CI_COMMIT_REF_NAME=master + export PYPIUSER=pypiuser + export PYPIPASS=pypipass + export DOCUSER=docuser + export DOCPASS=docpass + export CONDA_ROOT=${basedir}/miniconda + export DOCSERVER=https://www.idiap.ch/software/bob +fi + source ${basedir}/_ci/functions.sh +if [ -e ${basedir}/bin ]; then + log_info "Found ${basedir}/bin directory, prepending to PATH..." + PATH=${basedir}/bin:${PATH} + export_env PATH +fi + REGISTRY="docker.idiap.ch" check_env REGISTRY @@ -10,11 +42,15 @@ check_env REGISTRY IMAGES=( "${REGISTRY}/beat/beat.env.system.python:1.3.0r3" "${REGISTRY}/beat/beat.env.db.examples:1.4.0r3" - "${REGISTRY}/beat/beat.env.cxx:2.0.0r0" - "${REGISTRY}/beat/beat.env.client:2.0.0r0" + "${REGISTRY}/beat/beat.env.cxx:2.0.0r1" + "${REGISTRY}/beat/beat.env.client:2.0.0r1" ) check_array_env IMAGES +# Docker image to be used for compiling the algorithm +BEAT_CXX_DEV_ENVIRONMENT="${REGISTRY}/beat/beat.env.client:2.0.0r1" +check_env BEAT_CXX_DEV_ENVIRONMENT + missing=() log_info "Checking for minimal set of docker images..." for k in ${IMAGES[@]}; do @@ -47,10 +83,6 @@ if [[ "${#missing[@]}" > 0 ]]; then done fi -# Docker image to be used for compiling the algorithm -BEAT_CXX_DEV_ENVIRONMENT="${REGISTRY}/beat/beat.env.client:2.0.0r0" -check_env BEAT_CXX_DEV_ENVIRONMENT - # These are the algorithms that need to be compiled and which we'll test ALGORITHMS=( "prefix/algorithms/user/cxx_integers_echo_legacy/1" @@ -61,34 +93,26 @@ ALGORITHMS=( check_array_env ALGORITHMS # Check where the package root is -BEAT_CORE_TEST_DIR=$(python -c 'import beat.core.test as f; import os; print os.path.abspath(os.path.dirname(f.__file__))') +if [ "${BUILDOUT}" == "true" ]; then + BEAT_CORE_TEST_DIR="${basedir}/beat/core/test" +else + BEAT_CORE_TEST_DIR=$(python -c 'import beat.core.test as f; import os; print(os.path.abspath(os.path.dirname(f.__file__)))') +fi check_env BEAT_CORE_TEST_DIR # Compilation of the algorithms in the list above if [ "$1" == "build" ]; then - for k in "${ALGORITHMS[@]}"; do - algodir=$(dirname "${k}") - run_cmd docker run --rm --user=$(id -u gitlab-runner):$(id -g gitlab-runner) --volume="${BEAT_CORE_TEST_DIR}/scripts/build.sh:/build.sh" --volume="${BEAT_CORE_TEST_DIR}/${algodir}:/sources" --workdir="/sources" --entrypoint="bash" "${BEAT_CXX_DEV_ENVIRONMENT}" /build.sh - done + for k in "${ALGORITHMS[@]}"; do + algodir=$(dirname "${k}") + run_cmd docker run --rm --user=$(id -u):$(id -g) --volume="${BEAT_CORE_TEST_DIR}/scripts/build.sh:/build.sh" --volume="${BEAT_CORE_TEST_DIR}/${algodir}:/sources" --workdir="/sources" --entrypoint="bash" "${BEAT_CXX_DEV_ENVIRONMENT}" /build.sh + done fi # Cleanup of the compiled algorithms if [ "$1" == "clean" ]; then - for k in "${ALGORITHMS[@]}"; do - algodir=$(dirname "${k}") - run_cmd rm -f "${BEAT_CORE_TEST_DIR}/${algodir}/*.so" - run_cmd rm -rf "${BEAT_CORE_TEST_DIR}/${algodir}/build/" - done + for k in "${ALGORITHMS[@]}"; do + algodir=$(dirname "${k}") + run_cmd rm -f "${BEAT_CORE_TEST_DIR}/${algodir}/*.so" + run_cmd rm -rf "${BEAT_CORE_TEST_DIR}/${algodir}/build/" + done fi - -# Makes sure we activate the base environment if available -run_cmd source ${CONDA_ROOT}/etc/profile.d/conda.sh -run_cmd conda activate base -export_env PATH - -for subchannel in linux-64 osx-64 noarch; do - if [ ! -d ${CONDA_ROOT}/conda-bld/${subchannel} ]; then - run_cmd mkdir -p ${CONDA_ROOT}/conda-bld/${subchannel} - fi - run_cmd conda index ${CONDA_ROOT}/conda-bld/${subchannel} -done