diff --git a/gitlab/after_build.sh b/gitlab/after_build.sh index c9f0e880517fe5fd62f816e2b6569bf5e277984a..45a69db147b4f8ff809df9f59b09337a50057c6e 100755 --- a/gitlab/after_build.sh +++ b/gitlab/after_build.sh @@ -1,6 +1,2 @@ #!/usr/bin/env bash # Sat 24 Sep 08:12:24 2016 CEST - -source $(dirname ${0})/functions.sh - -log_info "*** Not yet implemented ***" diff --git a/gitlab/after_deploy.sh b/gitlab/after_deploy.sh index dc97ec3a5e72f02e23d5efe1fc691bdec613ca2a..8a57965e9d0af0397c64d571849c3511fda5d615 100755 --- a/gitlab/after_deploy.sh +++ b/gitlab/after_deploy.sh @@ -1,6 +1,2 @@ #!/usr/bin/env bash # Thu 22 Sep 2016 15:11:53 CEST - -source $(dirname ${0})/functions.sh - -log_info "*** Not yet implemented ***" diff --git a/gitlab/after_docs.sh b/gitlab/after_docs.sh index cd563fb91dbdc844066e4a4a8d506f28d420cade..73a037624bb88e0aed6b9d09f5c6aa0291b38784 100755 --- a/gitlab/after_docs.sh +++ b/gitlab/after_docs.sh @@ -1,6 +1,2 @@ #!/usr/bin/env bash # Thu 22 Sep 2016 18:23:57 CEST - -source $(dirname ${0})/functions.sh - -log_info "*** Not yet implemented ***" diff --git a/gitlab/after_test.sh b/gitlab/after_test.sh index 67a04cb79ada656a7be2d1cf2296f653acf399bb..9f03f60bf4cfab46998e4079c79e9dd29ede2e23 100755 --- a/gitlab/after_test.sh +++ b/gitlab/after_test.sh @@ -1,6 +1,2 @@ #!/usr/bin/env bash # Thu 22 Sep 2016 13:59:03 CEST - -source $(dirname ${0})/functions.sh - -log_info "*** Not yet implemented ***" diff --git a/gitlab/after_wheels.sh b/gitlab/after_wheels.sh index cd563fb91dbdc844066e4a4a8d506f28d420cade..73a037624bb88e0aed6b9d09f5c6aa0291b38784 100755 --- a/gitlab/after_wheels.sh +++ b/gitlab/after_wheels.sh @@ -1,6 +1,2 @@ #!/usr/bin/env bash # Thu 22 Sep 2016 18:23:57 CEST - -source $(dirname ${0})/functions.sh - -log_info "*** Not yet implemented ***" diff --git a/gitlab/before_build.sh b/gitlab/before_build.sh index 86a45993d6e67a6ab36e16fc3f512f71827c7e39..47538e6c1f528bb46ea18c72d21857978fce26d0 100755 --- a/gitlab/before_build.sh +++ b/gitlab/before_build.sh @@ -10,27 +10,15 @@ if [ -z "${WHEELS_REPOSITORY}" ]; then fi check_env WHEELS_REPOSITORY -# Readies the virtualenv to use for installation +# Readies a conda environment to use for installation if [ ! -d ${PREFIX} ]; then - log_info "Creating virtualenv installation at ${PREFIX}..." - run_cmd ${BOB_PREFIX_PATH}/bin/virtualenv --system-site-packages ${PREFIX} + log_info "Creating conda installation at ${PREFIX}..." + run_cmd ${CONDA_FOLDER}/bin/conda create --clone ${CONDA_ENV} --prefix ${PREFIX} --yes else log_info "Prefix directory ${PREFIX} exists, not re-installing..." fi -# Configures CCACHE -use_ccache=`which ccache` -if [ -z "${use_ccache}" ]; then - log_warn "Cannot find ccache, compiling from scratch..." -else - log_info "ccache installed at ${use_ccache}, caching compilations..." - run_cmd ln -sf ${use_ccache} ${PREFIX}/bin/gcc - run_cmd ln -sf ${use_ccache} ${PREFIX}/bin/g++ -fi - -# Source the newly created virtualenv -log_info "$ source ${PREFIX}/bin/activate" -source ${PREFIX}/bin/activate +prepare_build_env ${CONDA_FOLDER} ${PREFIX} # Verify where pip is installed use_pip=`which pip` @@ -41,9 +29,6 @@ else log_info "Using pip: ${use_pip}" fi -# Upgrade wheel generation library to avoid bugs -run_cmd ${use_pip} install --upgrade setuptools pip wheel - use_python=`which python` if [ -z "${use_python}" ]; then log_error "Cannot find python, aborting..." @@ -55,9 +40,15 @@ fi # Install this package's build dependencies PIPOPTS="--find-links ${WHEELS_REPOSITORY}" if [ ! -z "${WHEELS_SERVER}" ]; then - PIPOPTS="${PIPOPTS} --trusted-host ${WHEELS_SERVER}" + PIPOPTS="${PIPOPTS} --trusted-host ${WHEELS_SERVER}" fi -PIPOPTS="${PIPOPTS} --pre --use-wheel --no-index" + +# When building a tag, do not use beta wheels +PIPOPTS="${PIPOPTS} --use-wheel --no-index" +if [ -z "${CI_BUILD_TAG}" ]; then + PIPOPTS="${PIPOPTS} --pre" +fi + if [ -e requirements.txt ]; then run_cmd ${use_pip} install ${PIPOPTS} --requirement requirements.txt else diff --git a/gitlab/before_docs.sh b/gitlab/before_docs.sh index cd563fb91dbdc844066e4a4a8d506f28d420cade..73a037624bb88e0aed6b9d09f5c6aa0291b38784 100755 --- a/gitlab/before_docs.sh +++ b/gitlab/before_docs.sh @@ -1,6 +1,2 @@ #!/usr/bin/env bash # Thu 22 Sep 2016 18:23:57 CEST - -source $(dirname ${0})/functions.sh - -log_info "*** Not yet implemented ***" diff --git a/gitlab/before_test.sh b/gitlab/before_test.sh index 26cd4f692dbd9a09671075409d541100cbca2779..e31bc9dd7bbef54a1a720012f4bebf1fb050b269 100755 --- a/gitlab/before_test.sh +++ b/gitlab/before_test.sh @@ -5,9 +5,7 @@ source $(dirname ${0})/functions.sh run_cmd $(dirname ${0})/before_build.sh -# Source the newly created virtualenv -log_info "$ source ${PREFIX}/bin/activate" -source ${PREFIX}/bin/activate +prepare_build_env ${CONDA_FOLDER} ${PREFIX} # Verify where pip is installed use_pip=`which pip` diff --git a/gitlab/before_wheels.sh b/gitlab/before_wheels.sh index cd563fb91dbdc844066e4a4a8d506f28d420cade..73a037624bb88e0aed6b9d09f5c6aa0291b38784 100755 --- a/gitlab/before_wheels.sh +++ b/gitlab/before_wheels.sh @@ -1,6 +1,2 @@ #!/usr/bin/env bash # Thu 22 Sep 2016 18:23:57 CEST - -source $(dirname ${0})/functions.sh - -log_info "*** Not yet implemented ***" diff --git a/gitlab/build.sh b/gitlab/build.sh index 6d45957122a54ece216884168058bb796aed9c73..fb300c9bd9340c6e40d536223a3f5eb0a86624d2 100755 --- a/gitlab/build.sh +++ b/gitlab/build.sh @@ -3,6 +3,8 @@ source $(dirname ${0})/functions.sh +prepare_build_env ${CONDA_FOLDER} ${PREFIX} + run_cmd ./bin/buildout if [ -x ./bin/bob_dbmanage.py ]; then diff --git a/gitlab/functions.sh b/gitlab/functions.sh index 00fa382e8f2ae6426c75e376ed9f14453b87c680..6eaa8d82a95f392248a59eb42b541344f2129cbc 100644 --- a/gitlab/functions.sh +++ b/gitlab/functions.sh @@ -176,6 +176,36 @@ doc_upload() { } +# Activates environment and sets up compilation +# $1: root of the conda installation +# $2: your current build prefix +prepare_build_env() { + # Activates conda environment for the build + log_info "$ source $1/bin/activate $2" + source $1/bin/activate $2 + + # Configures CCACHE + # use_ccache=`which ccache` + # if [ -z "${use_ccache}" ]; then + # log_warn "Cannot find ccache, compiling from scratch..." + # else + # local ccache_bin=$2/lib/ccache + # if [ ! -d ${ccache_bin} ]; then + # run_cmd mkdir -pv ${ccache_bin} + # ln -sf ${use_ccache} ${ccache_bin}/gcc + # ln -sf ${use_ccache} ${ccache_bin}/g++ + # ln -sf ${use_ccache} ${ccache_bin}/cc + # ln -sf ${use_ccache} ${ccache_bin}/c++ + # fi + # use_gcc=`which gcc` + # PATH=${ccache_bin}:${PATH} + # export_env PATH + # log_info "ccache installed at ${use_ccache}, caching compilations..." + # log_info "gcc installed at ${use_gcc}..." + # fi +} + + # Checks if an array contains a value # taken from here: https://stackoverflow.com/questions/3685970/check-if-an-array-contains-a-value # Parameters: <value-to-check> <array-variable> @@ -205,49 +235,28 @@ if [ -z "${CONDA_FOLDER}" ]; then CONDA_FOLDER=/opt/conda fi -if [ -z "${VIRTUALENV_PATH}" ]; then - VIRTUALENV_PATH=env +PYVER=py$(echo ${PYTHON_VERSION} | tr -d '.') +if [ -z "${CONDA_ENV}" ]; then + if [ -d "${CONDA_FOLDER}/envs/bob-devel-${PYVER}-${CI_BUILD_REF_NAME}" ]; then + CONDA_ENV=bob-devel-${PYVER}-${CI_BUILD_REF_NAME} + else + CONDA_ENV=bob-devel-${PYVER} + fi fi +BOB_PREFIX_PATH=${CONDA_FOLDER}/envs/${CONDA_ENV} if [ -z "${DOCSERVER}" ]; then DOCSERVER=http://www.idiap.ch export_env DOCSERVER fi -PREFIX=${CI_PROJECT_DIR}/${VIRTUALENV_PATH} - -if [ "${OSNAME}" == "linux" ]; then - # Temporary hack to get building done right with gcc-5 compilers - if [ -z "${CFLAGS}" ]; then - CFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" - else - if [[ "${CFLAGS}" != *-D_GLIBCXX_USE_CXX11_ABI=0* ]]; then - CFLAGS="${CFLAGS} -D_GLIBCXX_USE_CXX11_ABI=0" - fi - fi -else #macosx - if [ "${CFLAGS}" == *-DBZ_DEBUG* ]; then - # -DBZ_DEBUG does not work well with clang - CFLAGS=`echo ${CFLAGS} | sed -e s/\s*-DBZ_DEBUG//g` - fi -fi -CXXFLAGS=${CFLAGS} - -PYVER=py$(echo ${PYTHON_VERSION} | tr -d '.') -if [ -z "${BOB_PREFIX_PATH}" ]; then - # Default on stock installations for our CIs - BOB_PREFIX_PATH=${CONDA_FOLDER}/envs/bob-devel-${PYVER} - - # Unless, a special environment (to be tested) exists matching the branch - if [ -d ${BOB_PREFIX_PATH}-${CI_BUILD_REF_NAME} ]; then - BOB_PREFIX_PATH=${BOB_PREFIX_PATH}-${CI_BUILD_REF_NAME} +if [ -z "{PREFIX}" ]; then + PREFIX=${CI_PROJECT_DIR}/build-prefix + # In case we're setting-up the prefix, erase it if it exists + if [ -d "${PREFIX}" ]; then + run_cmd rm -rf ${PREFIX} fi fi -LD_LIBRARY_PATH=${BOB_PREFIX_PATH}/lib -if [ "${OSNAME}" == "macosx" ]; then - DYLD_FALLBACK_LIBRARY_PATH=${BOB_PREFIX_PATH}/lib - MACOSX_DEPLOYMENT_TARGET=10.9 -fi if [ -z "${PYPISERVER}" ]; then PYPISERVER="https://pypi.python.org/pypi" @@ -258,18 +267,12 @@ TESTSERVER=https://testpypi.python.org/pypi check_env OSNAME check_env PYVER check_env PREFIX -export_env CFLAGS -export_env CXXFLAGS +export_env PREFIX check_env DOCSERVER check_env PYPISERVER check_env TESTSERVER check_env CONDA_FOLDER -check_env VIRTUALENV_PATH -export_env LD_LIBRARY_PATH -if [ "${OSNAME}" == "macosx" ]; then - export_env DYLD_FALLBACK_LIBRARY_PATH - export_env MACOSX_DEPLOYMENT_TARGET -fi +check_env CONDA_ENV export_env BOB_PREFIX_PATH if [ -z "${CI_BUILD_TAG}" ]; then @@ -284,6 +287,3 @@ if [ -z "${CI_BUILD_TAG}" ]; then else log_info "Building tag, not setting BOB_DOCUMENTATION_SERVER" fi - -# Activates conda environment -run_cmd source ${BOB_PREFIX_PATH}/bin/activate `basename ${BOB_PREFIX_PATH}` diff --git a/gitlab/install.sh b/gitlab/install.sh index 8adb34b7f3cebf8ad4ab118feaefe90726231282..6b5e27ce8e334475552a00d47d11e42af9ab27a9 100755 --- a/gitlab/install.sh +++ b/gitlab/install.sh @@ -3,12 +3,19 @@ # Installation script for our build tools -if [ "${#}" -ne 1 ]; then - echo "usage: ${0} <ci-support-directory>" +if [ "${#}" -eq 0 ]; then + echo "usage: ${0} <ci-support-directory> [<branch>]" echo "example: ${0} _ci" + echo "example: ${0} _ci staging" exit 1 fi +if [ -n "$2" ]; then + BRANCH=$2; +else + BRANCH=master; +fi + # Functions for coloring echo commands log_info() { @@ -34,7 +41,7 @@ run_cmd() { get_script() { - local url="https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/${2}" + local url="https://gitlab.idiap.ch/bob/bob.admin/raw/${BRANCH}/gitlab/${2}" local curlopt="--location --silent --show-error --output ${1}/${2}" if [ -e ${1}/${2} ]; then rm -f ${1}/${2} diff --git a/gitlab/test.sh b/gitlab/test.sh index 85ac251a22e2d266cfd12bc0d394d4b91c7051d3..4a76151d483f3d59f19945ac802d55b65ed2a990 100755 --- a/gitlab/test.sh +++ b/gitlab/test.sh @@ -3,11 +3,28 @@ source $(dirname ${0})/functions.sh +prepare_build_env ${CONDA_FOLDER} ${PREFIX} + run_cmd cd ${PREFIX} +# Checks some programs + +use_python=`which python` +check_env use_python + +use_coverage=`which coverage` +check_env use_coverage + +use_nosetests=`which nosetests` +check_env use_nosetests + +use_sphinx=`which sphinx-build` +check_env use_sphinx + # The tests: -run_cmd ${PREFIX}/bin/python ${BOB_PREFIX_PATH}/bin/coverage run --source=${CI_PROJECT_NAME} ${BOB_PREFIX_PATH}/bin/nosetests -sv ${CI_PROJECT_NAME} -run_cmd ${PREFIX}/bin/python ${BOB_PREFIX_PATH}/bin/coverage report -run_cmd ${PREFIX}/bin/python ${BOB_PREFIX_PATH}/bin/sphinx-build -b doctest ${CI_PROJECT_DIR}/doc ${CI_PROJECT_NAME}/sphinx + +run_cmd ${use_coverage} run --source=${CI_PROJECT_NAME} ${use_nosetests} -sv ${CI_PROJECT_NAME} +run_cmd ${use_coverage} report +run_cmd ${use_sphinx} -b doctest ${CI_PROJECT_DIR}/doc ${CI_PROJECT_NAME}/sphinx run_cmd cd ${CI_PROJECT_DIR}