diff --git a/gitlab/after_build.sh b/gitlab/after_build.sh new file mode 100755 index 0000000000000000000000000000000000000000..fd14ce964a3cab326e87b929df26090395eb5692 --- /dev/null +++ b/gitlab/after_build.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# Thu 22 Sep 2016 13:59:03 CEST + +source $(dirname ${0})/functions.sh + +check_env CONDA_PREFIX +run_cmd rm -rf ${CONDA_PREFIX} +run_cmd rm -rf install.sh diff --git a/gitlab/after_deploy.sh b/gitlab/after_deploy.sh new file mode 100755 index 0000000000000000000000000000000000000000..d161c1fe59ca3bac2bf12377fd40ae064f2c4ac4 --- /dev/null +++ b/gitlab/after_deploy.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# Thu 22 Sep 2016 15:11:53 CEST + +source $(dirname ${0})/functions.sh + +check_env CONDA_PREFIX +run_cmd rm -rf ${CONDA_PREFIX} +run_cmd rm -rf ~/.pypirc diff --git a/gitlab/after_docs.sh b/gitlab/after_docs.sh new file mode 100755 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/gitlab/after_test.sh b/gitlab/after_test.sh new file mode 100755 index 0000000000000000000000000000000000000000..743024b0955cbfe0bf1a5104b37fcf9fb3c46add --- /dev/null +++ b/gitlab/after_test.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# Thu 22 Sep 2016 13:59:03 CEST + +source $(dirname ${0})/functions.sh + +check_env CONDA_PREFIX +run_cmd rm -rf ${CONDA_PREFIX} diff --git a/gitlab/after_wheels.sh b/gitlab/after_wheels.sh new file mode 100755 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/gitlab/before_build.sh b/gitlab/before_build.sh new file mode 100755 index 0000000000000000000000000000000000000000..9ec066005f7dbc4ed9b7b1631fd688ed65e4db68 --- /dev/null +++ b/gitlab/before_build.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# Mon 8 Aug 17:40:24 2016 CEST + +source $(dirname ${0})/functions.sh + +check DOCSERVER +check CONDA_FOLDER +check PYTHON_VER +check CONDA_PREFIX +check ARCH +check PREFIX + +WHEELS_REPOSITORY="${DOCSERVER}/software/bob/wheels/gitlab/" +WHEELS_SERVER=`echo ${DOCSERVER} | sed 's;https\?://;;'` + +# Clones the conda dev environment to use +if [ ! -d ${PREFIX} ]; then + log_info "[++] Downloading environment list into file \`env.txt'..." + pyver=$(echo ${PYTHON_VER} | tr -d '.') + run_cmd curl --silent --output env.txt https://gitlab.idiap.ch/bob/bob.admin/raw/master/install/${ARCH}/devel-py${pyver}.txt + log_info "[++] Bootstrapping conda installation at ${PREFIX}..." + run_cmd ${CONDA_FOLDER}/bin/conda clean --lock + run_cmd ${CONDA_FOLDER}/bin/conda create --prefix ${PREFIX} --file env.txt --yes + + # Dirty fix to libjpeg.8 compilation issues: + # https://github.com/ContinuumIO/anaconda-issues/issues/1042 + if [ "${ARCH}" == "macosx" ] && [ -e "${PREFIX}/lib/libjpeg.8.dylib" ]; then + run_cmd install_name_tool -id @rpath/libjpeg.8.dylib ${PREFIX}/lib/libjpeg.8.dylib + fi + run_cmd ${CONDA_FOLDER}/bin/conda clean --lock +else + log_warn "Prefix directory ${PREFIX} exists, not re-installing..." +fi + +# Source the newly created conda environment +log_info "Running \"source ${CONDA_FOLDER}/bin/activate ${PREFIX}\"..." +source ${CONDA_FOLDER}/bin/activate ${PREFIX} +log_info "Environment ${PREFIX} activated" + +# Verify where pip is installed +use_pip=`which pip` +if [ -z "${use_pip}" ]; then + log_error "Cannot find pip, aborting..." + exit 1 +else + log_info "Using pip: ${use_pip}" +fi + +use_python=`which python` +if [ -z "${use_python}" ]; then + log_error "Cannot find python, aborting..." + exit 1 +else + log_info "Using python: ${use_python}" +fi + +# Install this package's build dependencies +if [ -e requirements.txt ]; then + run_cmd ${use_pip} install --find-links ${WHEELS_REPOSITORY} --use-wheel --no-index --trusted-host ${WHEELS_SERVER} --pre --requirement requirements.txt +else + log_info "No requirements.txt file found, skipping 'pip install <build-deps>'..." +fi + +# Install this package's test dependencies +if [ -e test-requirements.txt ]; then + run_cmd ${use_pip} install --find-links ${WHEELS_REPOSITORY} --use-wheel --no-index --trusted-host ${WHEELS_SERVER} --pre --requirement test-requirements.txt +else + log_info "No test-requirements.txt file found, skipping 'pip install <test-deps>'..." +fi + +# Finally, bootstrap the installation from the new environment +if [ -e bootstrap-buildout.py ]; then + run_cmd ${use_python} bootstrap-buildout.py +else + log_error "No bootstrap-buildout.py file found, stopping..." + exit 1 +fi diff --git a/gitlab/before_deploy.sh b/gitlab/before_deploy.sh new file mode 100755 index 0000000000000000000000000000000000000000..f2b32a186075957d777ef3b60698e86d780e6776 --- /dev/null +++ b/gitlab/before_deploy.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Wed 21 Sep 2016 13:08:05 CEST + +source $(dirname ${0})/functions.sh + +check_env DOCSERVER +check_env CONDA_PREFIX + +BOB_DOCUMENTATION_SERVER="${DOCSERVER}/software/bob/docs/latest/bob/%s/master/" +PREFIX=`pwd`/${CONDA_PREFIX} + +run_cmd $(dirname ${0})/before_build.sh +run_cmd ${PREFIX}/bin/pip install --use-wheel --no-index --pre dist/*.whl diff --git a/gitlab/before_docs.sh b/gitlab/before_docs.sh new file mode 100755 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/gitlab/before_test.sh b/gitlab/before_test.sh new file mode 100755 index 0000000000000000000000000000000000000000..f2b32a186075957d777ef3b60698e86d780e6776 --- /dev/null +++ b/gitlab/before_test.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Wed 21 Sep 2016 13:08:05 CEST + +source $(dirname ${0})/functions.sh + +check_env DOCSERVER +check_env CONDA_PREFIX + +BOB_DOCUMENTATION_SERVER="${DOCSERVER}/software/bob/docs/latest/bob/%s/master/" +PREFIX=`pwd`/${CONDA_PREFIX} + +run_cmd $(dirname ${0})/before_build.sh +run_cmd ${PREFIX}/bin/pip install --use-wheel --no-index --pre dist/*.whl diff --git a/gitlab/before_wheels.sh b/gitlab/before_wheels.sh new file mode 100755 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/gitlab/build.sh b/gitlab/build.sh new file mode 100755 index 0000000000000000000000000000000000000000..68c6251b2b3809c393444505470b6a92ab23d78e --- /dev/null +++ b/gitlab/build.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Wed 21 Sep 2016 13:08:05 CEST + +source $(dirname ${0})/functions.sh + +check_env DOCSERVER +check_env CONDA_FOLDER +check_env PYTHON_VER + +BOB_DOCUMENTATION_SERVER="${DOCSERVER}/software/bob/docs/latest/bob/%s/master/" +pyver=$(echo ${PYTHON_VER} | tr -d '.') +BOB_PREFIX_PATH=${CONDA_FOLDER}/envs/bob-devel-py${pyver} + +run_cmd ./bin/buildout + +if [ -x ./bin/bob_dbmanage.py ]; then + run_cmd ./bin/bob_dbmanage.py all download --force; +fi + +if [ -d ./doc ]; then + run_cmd ./bin/sphinx-build doc sphinx +fi + +if [ -z "${WHEEL_TAG}" ]; then + # C/C++ extensions + run_cmd ./bin/python setup.py bdist_wheel +else + # Python-only packages + run_cmd ./bin/python setup.py bdist_wheel --python-tag ${WHEEL_TAG} +fi diff --git a/gitlab/deploy.sh b/gitlab/deploy.sh new file mode 100755 index 0000000000000000000000000000000000000000..ef0bed70cd0126f0226cff22671418d9f2c567fc --- /dev/null +++ b/gitlab/deploy.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Thu 22 Sep 2016 13:59:03 CEST + +source $(dirname ${0})/functions.sh + +check_env PYPIUSER +check_env CI_BUILD_TAG + +if [ -z "${PYPISERVER}" ]; then + PYPISERVER="https://pypi.python.org" +fi + +TESTSERVER=https://testpypi.python.org + +dot_pypirc + +setup check sdist --formats zip upload --repository staging + +# if that worked, uploads documentation to pythonhosted if any exists +if [ -d sphinx ]; then + log_info "Uploading documentation to pythonhosted on behalf of ${PYPIUSER}..." + setup upload_docs --upload-dir sphinx +fi + +# if that worked, uploads source package to the production index +log_info "Uploading package to ${PYPISERVER} on behalf of ${PYPIUSER}..." +setup check sdist --formats zip upload --repository production + +# cleanup +run_cmd rm -f ~/.pypirc diff --git a/gitlab/docs.sh b/gitlab/docs.sh new file mode 100755 index 0000000000000000000000000000000000000000..c70203cac98fc4c1de04e2b85e37c17adc715a0a --- /dev/null +++ b/gitlab/docs.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Wed 21 Sep 2016 13:08:05 CEST + +source $(dirname ${0})/functions.sh + +info=sphinx/.gitlab-ci.info + +echo "repo=${CI_PROJECT_PATH}" > ${info} +echo "branch=${CI_BUILD_REF_NAME}" >> ${info} +echo "tag=${CI_BUILD_TAG}" >> ${info} +echo "build=${CI_BUILD_ID}" >> ${info} +echo "commit=${CI_BUILD_REF}" >> ${info} +echo "runner=${CI_RUNNER_DESCRIPTION}" >> ${info} + +check_env CI_PROJECT_NAME +check_env CI_BUILD_REF + +file=${CI_PROJECT_NAME}-${CI_BUILD_REF}.tar.bz2 +run_cmd tar cfj ${file} sphinx + +doc_upload ${file} software/bob/docs-upload/ diff --git a/gitlab/functions.sh b/gitlab/functions.sh new file mode 100644 index 0000000000000000000000000000000000000000..8fad3fa8a6bb3e03d48682c360e5e5d63380d79a --- /dev/null +++ b/gitlab/functions.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash +# Thu 22 Sep 2016 13:05:54 CEST + +# Build utilities + +# Determines the architecture we're using +arch() { + [[ "$(uname -s)" == "Darwin" ]] && echo "macosx" || echo "linux" +} + + +# Functions for coloring echo commands +log_info() { + echo -e "\033[1;32m[(`date +%c`)]${@}\033[0m" +} + + +log_warn() { + echo -e "\033[1;33m[(`date +%c`)]${@}\033[0m" +} + + +log_error() { + echo -e "\033[1;31m[(`date +%c`)]${@}\033[0m" +} + + +# Checks a given environment variable is set (non-zero size) +check_env() { + if [ -z "${!${1}}" ]; then + log_error "Variable ${1} is undefined - aborting..."; + exit 1 + else + log_info "Variable ${1}: OK"; + fi +} + + +# Function for running command and echoing results +run_cmd() { + log_info "Running \"${@}\"..." + ${@} + local status=$? + if [ ${status} != 0 ]; then + log_error "Command Failed \"${@}\"" + exit ${status} + else + log_info "Finished comand \"${@}\"" + fi +} + + +# Runs setup.py +setup() { + check_env CONDA_PREFIX + local python=`pwd`/${CONDA_PREFIX}/bin/python + log_info "Running \"${@}\"..." + ${python} setup.py ${@} + local status=$? + if [ ${status} != 0 ]; then + log_error "Command Failed \"${python} setup.py ${@}\"" + log_info rm -f ~/.pypirc + exit ${status} + else + log_info "Finished comand \"${python} setup.py ${@}\"" + fi +} + + +# Prepares ~/.pypirc +dot_pypirc() { + check_env PYPISERVER + check_env PYPIUSER + check_env PYPIPASS + log_info "Creating ~/.pypirc..." + cat <<EOT >> ~/.pypirc +[distutils] +index-servers = + production + staging + +[production] +repository: ${PYPISERVER} +username: ${PYPIUSER} +password: ${PYPIPASS} + +[staging] +repository: https://testpypi.python.org/ +username: ${PYPIUSER} +password: ${PYPIPASS} +EOT + run_cmd chmod 600 ~/.pypirc +} + + +# Uploads a file to our intranet location via curl +doc_upload() { + check_env DOCSERVER + check_env DOCUSER + check_env DOCPASS + log_info "${1} -> ${DOCSERVER}/${2}, via curl..." + curl --silent --user "${DOCUSER}:${DOCPASS}" --upload-file ${1} ${DOCSERVER}/${2} + local status=$? + if [ ${status} != 0 ]; then + log_error "Curl command finished with an error condition (status=${status})" + exit ${status} + fi + log_info "Successfully uploaded ${1} with curl" +} + +# Sets up variables +ARCH=`arch` +if [ -z "${CONDA_FOLDER}" ]; then + if [ "${ARCH}" == "linux" ]; then + CONDA_FOLDER=/local/conda + else + CONDA_FOLDER=/opt/conda + fi +fi + +if [ -z "${CONDA_PREFIX}" ]; then + CONDA_PREFIX=env +fi + +PREFIX=`pwd`/${CONDA_PREFIX} + +# Add "-coverage" support +if [ "${ARCH}" == "linux" ]; then + # Temporary hack to get building done right with gcc-5 compilers + if [ -z "${CFLAGS}" ]; then + CFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -coverage" + else + CFLAGS="${CFLAGS} -D_GLIBCXX_USE_CXX11_ABI=0 -coverage" + fi + CXXFLAGS=${CFLAGS} +else + if [ -z "${CFLAGS}" ]; then + CFLAGS="-coverage" + else + CFLAGS="${CFLAGS} -coverage" + fi + CXXFLAGS=${CFLAGS} +fi diff --git a/gitlab/install.sh b/gitlab/install.sh new file mode 100755 index 0000000000000000000000000000000000000000..f90205089068f267447e44190f6f4d5eb66c31f6 --- /dev/null +++ b/gitlab/install.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# Thu 22 Sep 2016 13:05:54 CEST + +# Installation script for our build tools + +if [ "${#}" -ne 1 ]; then + echo "usage: ${0} <ci-support-directory>" + echo "example: ${0} _ci" + exit 1 +fi + +# Functions for coloring echo commands +log_info() { + echo -e "\033[1;32m[(`date +%c`)]${@}\033[0m" +} + +log_error() { + echo -e "\033[1;31m[(`date +%c`)]${@}\033[0m" +} + +# Function for running command and echoing results +run_cmd() { + log_info "Running \"${@}\"..." + ${@} + local status=$? + if [ ${status} != 0 ]; then + log_error "Command Failed \"${@}\"" + exit ${status} + else + log_info "Finished comand \"${@}\"" + fi +} + +get_script() { + run_cmd curl --silent --output ${1}/${2} "https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/${2}" +} + +get_exec() { + get_script ${1} ${2} + chmod 755 ${1}/${2} +} + +get_script ${1} functions.sh +get_exec ${1} before_build.sh +get_exec ${1} build.sh +get_exec ${1} after_build.sh +get_exec ${1} before_test.sh +get_exec ${1} test.sh +get_exec ${1} after_test.sh +get_exec ${1} before_wheels.sh +get_exec ${1} wheels.sh +get_exec ${1} after_wheels.sh +get_exec ${1} before_docs.sh +get_exec ${1} docs.sh +get_exec ${1} after_docs.sh +get_exec ${1} before_deploy.sh +get_exec ${1} deploy.sh +get_exec ${1} after_deploy.sh diff --git a/gitlab/test.sh b/gitlab/test.sh new file mode 100755 index 0000000000000000000000000000000000000000..bde7b93194282f4e684402d2fce693cd9392b518 --- /dev/null +++ b/gitlab/test.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Wed 21 Sep 2016 13:08:05 CEST + +source $(dirname ${0})/functions.sh + +check_env DOCSERVER +check_env CONDA_PREFIX +check_env CI_PROJECT_NAME + +BOB_DOCUMENTATION_SERVER="${DOCSERVER}/software/bob/docs/latest/bob/%s/master/" + +bindir=`pwd`/${CONDA_PREFIX}/bin + +run_cmd ${bindir}/coverage run --source=${CI_PROJECT_NAME} ./bin/nosetests -sv ${CI_PROJECT_NAME} +run_cmd ${bindir}/coverage report +run_cmd ${bindir}/sphinx-build -b doctest ../doc ../sphinx diff --git a/gitlab/wheels.sh b/gitlab/wheels.sh new file mode 100755 index 0000000000000000000000000000000000000000..34fb958f109f24c281b06cc3dcc437f6ed6dfe4c --- /dev/null +++ b/gitlab/wheels.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# Thu 22 Sep 2016 13:58:56 CEST + +source $(dirname ${0})/functions.sh + +for file in dist/*.whl; do + doc_upload ${file} software/bob/wheels-upload/gitlab/ +done diff --git a/templates/ci-for-cxx-extensions.yml b/templates/ci-for-cxx-extensions.yml index e03e34569b4c60ad78540fec464eddc1f423674e..dfeb26f035ee69843c1d1ae3cdf1bdc7bf82c670 100644 --- a/templates/ci-for-cxx-extensions.yml +++ b/templates/ci-for-cxx-extensions.yml @@ -17,70 +17,38 @@ stages: - deploy -# Global variables -variables: - CONDA_PREFIX: env - - # Template for the build stage # Needs to run on all supported architectures, platforms and python versions .build_template: &build_job stage: build before_script: - git clean -ffdx - - curl --silent "https://gitlab.idiap.ch/bob/bob.admin/raw/master/ci/bootstrap.sh" > bootstrap.sh - - chmod 755 ./bootstrap.sh - - ./bootstrap.sh ${CONDA_FOLDER} ${PYTHON_VER} ${CONDA_PREFIX} - variables: &build_variables - BOB_DOCUMENTATION_SERVER: "http://www.idiap.ch/software/bob/docs/latest/bob/%s/master/" + - curl --silent "https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/install.sh" > install.sh + - chmod 755 ./install.sh + - ./install.sh _ci + - ./_ci/before_build.sh script: - - BOB_PREFIX_PATH=${CONDA_FOLDER}/envs/`cat ${CONDA_FOLDER}/envs/latest-devel-${PYTHON_VER}.txt` ./bin/buildout - - ./bin/sphinx-build doc sphinx - - ./bin/python setup.py bdist_wheel + - ./_ci/build.sh after_script: - - rm -rf ${CONDA_PREFIX} + - ./_ci/after_build.sh artifacts: expire_in: 1 day paths: - - bootstrap.sh + - _ci/ - dist/ - sphinx/ -# Template for building on a Linux machine -.build_linux_template: &linux_build_job - <<: *build_job - variables: &linux_build_variables - <<: *build_variables - CONDA_FOLDER: "/local/conda" - CFLAGS: "-D_GLIBCXX_USE_CXX11_ABI=0 -coverage" - CXXFLAGS: "-D_GLIBCXX_USE_CXX11_ABI=0 -coverage" - - -# Template for building on a Mac OSX machine -.build_mac_template: &macosx_build_job - <<: *build_job - variables: &macosx_build_variables - <<: *build_variables - CONDA_FOLDER: "/opt/conda" - - # Template for the test stage - re-install from uploaded wheels # Needs to run on all supported architectures, platforms and python versions .test_template: &test_job stage: test before_script: - - ./bootstrap.sh ${CONDA_FOLDER} ${PYTHON_VER} ${CONDA_PREFIX} - - source ${CONDA_FOLDER}/bin/activate ${CONDA_PREFIX} - - pip install --use-wheel --no-index --pre dist/*.whl + - ./_ci/before_test.sh script: - - cd ${CONDA_PREFIX} - - python -c "from ${CI_PROJECT_NAME} import get_config; print(get_config())" - - coverage run --source=${CI_PROJECT_NAME} ./bin/nosetests -sv ${CI_PROJECT_NAME} - - coverage report - - sphinx-build -b doctest ../doc ../sphinx + - ./_ci/test.sh after_script: - - rm -rf ${CONDA_PREFIX} + - ./_ci/after_test.sh # Template for the wheel uploading stage @@ -94,10 +62,11 @@ variables: - master - /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags) before_script: - - curl --silent https://gitlab.idiap.ch/bob/bob.admin/raw/master/ci/wheels.sh > wheels.sh - - chmod 755 wheels.sh + - ./_ci/before_wheels.sh script: - - ./wheels.sh + - ./_ci/wheels.sh + after_script: + - ./_ci/after_wheels.sh # Template for (latest) documentation upload stage @@ -108,43 +77,37 @@ variables: only: - master before_script: - - curl --silent https://gitlab.idiap.ch/bob/bob.admin/raw/master/ci/docs.sh > docs.sh - - chmod 755 docs.sh + - ./_ci/before_docs.sh script: - - ./docs.sh + - ./_ci/docs.sh + after_script: + - ./_ci/after_docs.sh # Template for the pypi stage - re-install from uploaded wheels # Needs to run on a single architecture .deploy_template: &deploy_job stage: deploy - environment: pypi + environment: internet only: - /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags) except: - branches before_script: - - ./bootstrap.sh ${CONDA_FOLDER} ${PYTHON_VER} ${CONDA_PREFIX} - - source ${CONDA_FOLDER}/bin/activate ${CONDA_PREFIX} - - pip install --use-wheel --no-index --pre dist/*.whl - - curl --silent https://gitlab.idiap.ch/bob/bob.admin/raw/master/ci/pypi.sh > pypi.sh - - chmod 755 pypi.sh + - ./_ci/before_deploy.sh script: - - source ${CONDA_FOLDER}/bin/activate ${CONDA_PREFIX} - - ./pypi.sh + - ./_ci/deploy.sh after_script: - - rm -rf ~/.pypirc - - rm -rf ${CONDA_PREFIX} + - ./_ci/after_deply.sh # 2) Package specific instructions (you may tune this if needed) # -------------------------------------------------------------- -# Linux + Python 2.7: Builds and tests +# Linux + Python 2.7: Builds, tests, uploads wheel build_linux_27: - <<: *linux_build_job + <<: *build_job variables: &linux_27_build_variables - <<: *linux_build_variables PYTHON_VER: "2.7" tags: - conda-linux @@ -167,9 +130,8 @@ wheels_linux_27: # Linux + Python 3.4: Builds and tests build_linux_34: - <<: *linux_build_job + <<: *build_job variables: &linux_34_build_variables - <<: *linux_build_variables PYTHON_VER: "3.4" tags: - conda-linux @@ -182,19 +144,11 @@ test_linux_34: tags: - conda-linux -wheels_linux_34: - <<: *wheels_job - dependencies: - - build_linux_34 - tags: - - conda-linux - -# Linux + Python 3.5: Builds, tests and deploys +# Linux + Python 3.5: Builds, tests, uploads wheel and deploys build_linux_35: - <<: *linux_build_job + <<: *build_job variables: &linux_35_build_variables - <<: *linux_build_variables PYTHON_VER: "3.5" tags: - conda-linux @@ -223,17 +177,17 @@ docs_linux_35: deploy_linux_35: <<: *deploy_job + variables: *linux_35_build_variables dependencies: - build_linux_35 tags: - conda-linux -# Mac OSX + Python 2.7: Builds, tests and uploads the wheel +# Mac OSX + Python 2.7: Builds and tests build_macosx_27: - <<: *macosx_build_job + <<: *build_job variables: &macosx_27_build_variables - <<: *macosx_build_variables PYTHON_VER: "2.7" tags: - conda-macosx @@ -246,19 +200,11 @@ test_macosx_27: tags: - conda-macosx -wheels_macosx_27: - <<: *wheels_job - dependencies: - - build_macosx_27 - tags: - - conda-macosx - # Mac OSX + Python 3.4: Builds and tests build_macosx_34: - <<: *macosx_build_job + <<: *build_job variables: &macosx_34_build_variables - <<: *macosx_build_variables PYTHON_VER: "3.4" tags: - conda-macosx @@ -271,19 +217,11 @@ test_macosx_34: tags: - conda-macosx -wheels_macosx_34: - <<: *wheels_job - dependencies: - - build_macosx_34 - tags: - - conda-macosx - -# Mac OSX + Python 3.5: Builds, tests, uploads the wheel and the latest docs +# Mac OSX + Python 3.5: Builds and tests build_macosx_35: - <<: *macosx_build_job + <<: *build_job variables: &macosx_35_build_variables - <<: *macosx_build_variables PYTHON_VER: "3.5" tags: - conda-macosx @@ -295,10 +233,3 @@ test_macosx_35: - build_macosx_35 tags: - conda-macosx - -wheels_macosx_35: - <<: *wheels_job - dependencies: - - build_macosx_35 - tags: - - conda-macosx diff --git a/templates/ci-for-python-only.yml b/templates/ci-for-python-only.yml index f9935e1a943524c55593213deb9669f528b1f7d2..84dcfa0d045640a66922584136fe99bf7b610850 100644 --- a/templates/ci-for-python-only.yml +++ b/templates/ci-for-python-only.yml @@ -17,71 +17,38 @@ stages: - deploy -# Global variables -variables: - CONDA_PREFIX: env - - # Template for the build stage # Needs to run on all supported architectures, platforms and python versions .build_template: &build_job stage: build before_script: - git clean -ffdx - - curl --silent "https://gitlab.idiap.ch/bob/bob.admin/raw/master/ci/bootstrap.sh" > bootstrap.sh - - chmod 755 ./bootstrap.sh - - ./bootstrap.sh ${CONDA_FOLDER} ${PYTHON_VER} ${CONDA_PREFIX} - variables: &build_variables - BOB_DOCUMENTATION_SERVER: "http://www.idiap.ch/software/bob/docs/latest/bob/%s/master/" + - curl --silent "https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/install.sh" > install.sh + - chmod 755 ./install.sh + - ./install.sh _ci + - ./_ci/before_build.sh script: - - ./bin/buildout - - if [ -x ./bin/bob_dbmanage.py ]; then ./bin/bob_dbmanage.py all download --force; fi - - ./bin/sphinx-build doc sphinx - - ./bin/python setup.py bdist_wheel --python-tag ${WHEEL_TAG} + - ./_ci/build.sh after_script: - - rm -rf ${CONDA_PREFIX} + - ./_ci/after_build.sh artifacts: expire_in: 1 day paths: - - bootstrap.sh + - _ci/ - dist/ - sphinx/ -# Template for building on a Linux machine -.build_linux_template: &linux_build_job - <<: *build_job - variables: &linux_build_variables - <<: *build_variables - CONDA_FOLDER: "/local/conda" - CFLAGS: "-D_GLIBCXX_USE_CXX11_ABI=0 -coverage" - CXXFLAGS: "-D_GLIBCXX_USE_CXX11_ABI=0 -coverage" - - -# Template for building on a Mac OSX machine -.build_mac_template: &macosx_build_job - <<: *build_job - variables: &macosx_build_variables - <<: *build_variables - CONDA_FOLDER: "/opt/conda" - - # Template for the test stage - re-install from uploaded wheels # Needs to run on all supported architectures, platforms and python versions .test_template: &test_job stage: test before_script: - - ./bootstrap.sh ${CONDA_FOLDER} ${PYTHON_VER} ${CONDA_PREFIX} - - source ${CONDA_FOLDER}/bin/activate ${CONDA_PREFIX} - - pip install --use-wheel --no-index --pre dist/*.whl + - ./_ci/before_test.sh script: - - cd ${CONDA_PREFIX} - - python -c "from ${CI_PROJECT_NAME} import get_config; print(get_config())" - - coverage run --source=${CI_PROJECT_NAME} ./bin/nosetests -sv ${CI_PROJECT_NAME} - - coverage report - - sphinx-build -b doctest ../doc ../sphinx + - ./_ci/test.sh after_script: - - rm -rf ${CONDA_PREFIX} + - ./_ci/after_test.sh # Template for the wheel uploading stage @@ -95,10 +62,11 @@ variables: - master - /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags) before_script: - - curl --silent https://gitlab.idiap.ch/bob/bob.admin/raw/master/ci/wheels.sh > wheels.sh - - chmod 755 wheels.sh + - ./_ci/before_wheels.sh script: - - ./wheels.sh + - ./_ci/wheels.sh + after_script: + - ./_ci/after_wheels.sh # Template for (latest) documentation upload stage @@ -109,33 +77,28 @@ variables: only: - master before_script: - - curl --silent https://gitlab.idiap.ch/bob/bob.admin/raw/master/ci/docs.sh > docs.sh - - chmod 755 docs.sh + - ./_ci/before_docs.sh script: - - ./docs.sh + - ./_ci/docs.sh + after_script: + - ./_ci/after_docs.sh # Template for the pypi stage - re-install from uploaded wheels # Needs to run on a single architecture .deploy_template: &deploy_job stage: deploy - environment: pypi + environment: internet only: - /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags) except: - branches before_script: - - ./bootstrap.sh ${CONDA_FOLDER} ${PYTHON_VER} ${CONDA_PREFIX} - - source ${CONDA_FOLDER}/bin/activate ${CONDA_PREFIX} - - pip install --use-wheel --no-index --pre dist/*.whl - - curl --silent https://gitlab.idiap.ch/bob/bob.admin/raw/master/ci/pypi.sh > pypi.sh - - chmod 755 pypi.sh + - ./_ci/before_deploy.sh script: - - source ${CONDA_FOLDER}/bin/activate ${CONDA_PREFIX} - - ./pypi.sh + - ./_ci/deploy.sh after_script: - - rm -rf ~/.pypirc - - rm -rf ${CONDA_PREFIX} + - ./_ci/after_deply.sh # 2) Package specific instructions (you may tune this if needed) @@ -143,9 +106,8 @@ variables: # Linux + Python 2.7: Builds, tests, uploads wheel build_linux_27: - <<: *linux_build_job + <<: *build_job variables: &linux_27_build_variables - <<: *linux_build_variables PYTHON_VER: "2.7" WHEEL_TAG: "py27" tags: @@ -169,9 +131,8 @@ wheels_linux_27: # Linux + Python 3.4: Builds and tests build_linux_34: - <<: *linux_build_job + <<: *build_job variables: &linux_34_build_variables - <<: *linux_build_variables PYTHON_VER: "3.4" WHEEL_TAG: "py3" tags: @@ -188,9 +149,8 @@ test_linux_34: # Linux + Python 3.5: Builds, tests, uploads wheel and deploys build_linux_35: - <<: *linux_build_job + <<: *build_job variables: &linux_35_build_variables - <<: *linux_build_variables PYTHON_VER: "3.5" WHEEL_TAG: "py3" tags: @@ -220,6 +180,7 @@ docs_linux_35: deploy_linux_35: <<: *deploy_job + variables: *linux_35_build_variables dependencies: - build_linux_35 tags: @@ -228,9 +189,8 @@ deploy_linux_35: # Mac OSX + Python 2.7: Builds and tests build_macosx_27: - <<: *macosx_build_job + <<: *build_job variables: &macosx_27_build_variables - <<: *macosx_build_variables PYTHON_VER: "2.7" WHEEL_TAG: "py27" tags: @@ -247,9 +207,8 @@ test_macosx_27: # Mac OSX + Python 3.4: Builds and tests build_macosx_34: - <<: *macosx_build_job + <<: *build_job variables: &macosx_34_build_variables - <<: *macosx_build_variables PYTHON_VER: "3.4" WHEEL_TAG: "py3" tags: @@ -266,9 +225,8 @@ test_macosx_34: # Mac OSX + Python 3.5: Builds and tests build_macosx_35: - <<: *macosx_build_job + <<: *build_job variables: &macosx_35_build_variables - <<: *macosx_build_variables PYTHON_VER: "3.5" WHEEL_TAG: "py3" tags: