diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e52b3cb1969f120f2a244bd6e98d382d2c1aa38a..b0825bfc741f4d67b38ee054bc34033c41553842 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,9 +10,8 @@ # Definition of our build pipeline stages: - - build - - test - - wheels + - build-bob + # Global variables variables: @@ -21,26 +20,20 @@ variables: # Template for the build stage # Needs to run on all supported architectures, platforms and python versions -.build_template: &build_job - stage: build +.build_bob: &build_job + stage: build-bob before_script: - - git clean -ffdx - - curl --silent https://gitlab.idiap.ch/bob/bob/snippets/7/raw | tr -d '\r' > bootstrap-conda.sh - - chmod 755 ./bootstrap-conda.sh - - ./bootstrap-conda.sh ${CONDA_FOLDER} ${PYTHON_VER} ${CONDA_PREFIX} - variables: &build_variables - BOB_DOCUMENTATION_SERVER: "http://www.idiap.ch/software/bob/docs/latest/bob/%s/master/" + - git clean -ffdx + - chmod 755 ci/install.sh + - ./ci/install.sh ci #updates + - ./ci/before_build.sh 0 script: - - BOB_PREFIX_PATH=${CONDA_FOLDER}/envs/`cat ${CONDA_FOLDER}/envs/latest-devel-${PYTHON_VER}.txt` ./bin/buildout - - ./bin/bob_dbmanage.py all download - - ./bin/bob_dbmanage.py ijba download - - ./wheels_dependencies.sh ${WHEEL_TAG} + - ./ci/build.sh after_script: - - rm -rf ${CONDA_PREFIX} + - ./ci/after_build.sh artifacts: expire_in: 1 day paths: - - bootstrap-conda.sh - dist/ @@ -66,35 +59,6 @@ variables: LDFLAGS: "-lpthread" -# Template for the test stage - re-install from uploaded wheels -# Needs to run on all supported architectures, platforms and python versions -.test_template: &test_job - stage: test - before_script: - - ./bootstrap-conda.sh ${CONDA_FOLDER} ${PYTHON_VER} ${CONDA_PREFIX} - - source ${CONDA_FOLDER}/bin/activate ${CONDA_PREFIX} - - pip install --use-wheel --no-index --pre dist/*.whl - script: - - cd ${CONDA_PREFIX} - - ./bin/nosetests -sv bob - after_script: - - rm -rf ${CONDA_PREFIX} - - -# Template for the wheel uploading stage -# Needs to run against one combination of python 2.x and 3.x if it is a python -# only package, otherwise, needs to run in both pythons to all supported -# architectures (Linux and Mac OSX 64-bit) -.wheels_template: &wheels_job - stage: wheels - only: - - master - - tags - before_script: - - curl --silent https://gitlab.idiap.ch/bob/bob/snippets/8/raw | tr -d '\r' > upload-wheel.sh - - chmod 755 upload-wheel.sh - script: - - ./upload-wheel.sh # 2) Package specific instructions (you may tune this if needed) @@ -110,22 +74,6 @@ build_linux_27: tags: - conda-linux -test_linux_27: - <<: *test_job - variables: *linux_27_build_variables - dependencies: - - build_linux_27 - tags: - - conda-linux - -wheels_linux_27: - <<: *wheels_job - dependencies: - - build_linux_27 - tags: - - conda-linux - - # Linux + Python 3.4: Builds and tests build_linux_34: <<: *linux_build_job @@ -136,22 +84,6 @@ build_linux_34: tags: - conda-linux -test_linux_34: - <<: *test_job - variables: *linux_34_build_variables - dependencies: - - build_linux_34 - tags: - - conda-linux - -wheels_linux_34: - <<: *wheels_job - dependencies: - - build_linux_34 - tags: - - conda-linux - - # Linux + Python 3.5: Builds and tests build_linux_35: <<: *linux_build_job @@ -162,21 +94,6 @@ build_linux_35: tags: - conda-linux -test_linux_35: - <<: *test_job - variables: *linux_35_build_variables - dependencies: - - build_linux_35 - tags: - - conda-linux - -wheels_linux_35: - <<: *wheels_job - dependencies: - - build_linux_35 - tags: - - conda-linux - # Mac OSX + Python 2.7: Builds, tests and uploads the wheel build_macosx_27: <<: *macosx_build_job @@ -187,22 +104,6 @@ build_macosx_27: tags: - conda-macosx -test_macosx_27: - <<: *test_job - variables: *macosx_27_build_variables - dependencies: - - build_macosx_27 - tags: - - conda-macosx - -wheels_macosx_27: - <<: *wheels_job - dependencies: - - build_macosx_27 - tags: - - conda-macosx - - # Mac OSX + Python 3.4: Builds and tests build_macosx_34: <<: *macosx_build_job @@ -213,22 +114,6 @@ build_macosx_34: tags: - conda-macosx -test_macosx_34: - <<: *test_job - variables: *macosx_34_build_variables - dependencies: - - build_macosx_34 - tags: - - conda-macosx - -wheels_macosx_34: - <<: *wheels_job - dependencies: - - build_macosx_34 - tags: - - conda-macosx - - # Mac OSX + Python 3.5: Builds, tests, uploads the wheel and the latest docs build_macosx_35: <<: *macosx_build_job @@ -238,18 +123,3 @@ build_macosx_35: WHEEL_TAG: "py3" tags: - conda-macosx - -test_macosx_35: - <<: *test_job - variables: *macosx_35_build_variables - dependencies: - - build_macosx_35 - tags: - - conda-macosx - -wheels_macosx_35: - <<: *wheels_job - dependencies: - - build_macosx_35 - tags: - - conda-macosx diff --git a/buildout-bob.cfg b/buildout-bob.cfg new file mode 100644 index 0000000000000000000000000000000000000000..7150927a1680b14d3e65364d1b041bf937108315 --- /dev/null +++ b/buildout-bob.cfg @@ -0,0 +1,128 @@ +; vim: set fileencoding=utf-8 : +; Andre Anjos <andre.bioidiap@idiap.ch> +; Mon 16 Apr 08:29:18 2012 CEST + +[buildout] +parts = scripts +develop = src/bob.buildout + src/bob.extension + src/bob.blitz + src/bob.core + src/bob.io.base + src/bob.sp + src/bob.ap + src/bob.math + src/bob.measure + src/bob.db.base + src/bob.io.image + src/bob.io.video + src/bob.io.matlab + src/bob.io.audio + src/bob.ip.base + src/bob.ip.color + src/bob.ip.draw + src/bob.ip.gabor + src/bob.learn.activation + src/bob.learn.libsvm + src/bob.learn.linear + src/bob.learn.mlp + src/bob.learn.em + src/bob.learn.boosting + src/bob.ip.facedetect + src/bob.ip.flandmark + src/bob.ip.skincolorfilter + src/bob.ip.optflow.hornschunck + src/bob.ip.optflow.liu + src/bob.db.iris + src/bob.db.mnist + src/bob.db.wine + src/bob.db.atnt + src/bob.db.arface + + +eggs = bob.extension + bob.blitz + bob.core + bob.io.base + bob.sp + bob.ap + bob.math + bob.measure + bob.db.base + bob.io.image + bob.io.video + bob.io.matlab + bob.io.audio + bob.ip.base + bob.ip.color + bob.ip.draw + bob.ip.gabor + bob.learn.activation + bob.learn.libsvm + bob.learn.linear + bob.learn.mlp + bob.learn.em + bob.learn.boosting + bob.ip.facedetect + bob.ip.flandmark + bob.ip.optflow.hornschunck + bob.ip.optflow.liu + bob.db.iris + bob.db.mnist + bob.db.wine + bob.db.atnt + bob.db.arface + + + +extensions = bob.buildout + mr.developer +auto-checkout = * + + +; options for bob.buildout extension +newest = false +debug = false +verbose = true + +[scripts] +recipe = bob.buildout:scripts + + +[sources] +bob.buildout = git https://gitlab.idiap.ch/bob/bob.buildout +bob.extension = git https://gitlab.idiap.ch/bob/bob.extension +bob.blitz = git https://gitlab.idiap.ch/bob/bob.blitz +bob.core = git https://gitlab.idiap.ch/bob/bob.core +bob.io.base = git https://gitlab.idiap.ch/bob/bob.io.base +bob.sp = git https://gitlab.idiap.ch/bob/bob.sp +bob.ap = git https://gitlab.idiap.ch/bob/bob.ap +bob.math = git https://gitlab.idiap.ch/bob/bob.math +bob.measure = git https://gitlab.idiap.ch/bob/bob.measure +bob.db.base = git https://gitlab.idiap.ch/bob/bob.db.base +bob.io.image = git https://gitlab.idiap.ch/bob/bob.io.image +bob.io.video = git https://gitlab.idiap.ch/bob/bob.io.video +bob.io.matlab = git https://gitlab.idiap.ch/bob/bob.io.matlab +bob.io.audio = git https://gitlab.idiap.ch/bob/bob.io.audio +bob.ip.base = git https://gitlab.idiap.ch/bob/bob.ip.base +bob.ip.color = git https://gitlab.idiap.ch/bob/bob.ip.color +bob.ip.draw = git https://gitlab.idiap.ch/bob/bob.ip.draw +bob.ip.gabor = git https://gitlab.idiap.ch/bob/bob.ip.gabor +bob.learn.activation = git https://gitlab.idiap.ch/bob/bob.learn.activation +bob.learn.libsvm = git https://gitlab.idiap.ch/bob/bob.learn.libsvm +bob.learn.linear = git https://gitlab.idiap.ch/bob/bob.learn.linear +bob.learn.mlp = git https://gitlab.idiap.ch/bob/bob.learn.mlp +bob.learn.em = git https://gitlab.idiap.ch/bob/bob.learn.em +bob.learn.boosting = git https://gitlab.idiap.ch/bob/bob.learn.boosting +bob.ip.facedetect = git https://gitlab.idiap.ch/bob/bob.ip.facedetect +bob.ip.flandmark = git https://gitlab.idiap.ch/bob/bob.ip.flandmark +bob.ip.facedetect = git https://gitlab.idiap.ch/bob/bob.ip.facedetect +bob.ip.optflow.hornschunck = git https://gitlab.idiap.ch/bob/bob.ip.optflow.hornschunck +bob.ip.optflow.liu = git https://gitlab.idiap.ch/bob/bob.ip.optflow.liu +bob.db.iris = git https://gitlab.idiap.ch/bob/bob.db.iris +bob.db.mnist = git https://gitlab.idiap.ch/bob/bob.db.mnist +bob.db.wine = git https://gitlab.idiap.ch/bob/bob.db.wine +bob.db.atnt = git https://gitlab.idiap.ch/bob/bob.db.atnt +bob.db.arface = git https://gitlab.idiap.ch/bob/bob.db.arface + + diff --git a/buildout-satellites.cfg b/buildout-satellites.cfg new file mode 100644 index 0000000000000000000000000000000000000000..20c26fe2376124079bbb7f70081629658751c714 --- /dev/null +++ b/buildout-satellites.cfg @@ -0,0 +1,130 @@ +; vim: set fileencoding=utf-8 : +; Andre Anjos <andre.bioidiap@idiap.ch> +; Mon 16 Apr 08:29:18 2012 CEST + +[buildout] +parts = scripts +develop = src/bob.db.bio_filelist + src/bob.db.banca + src/bob.db.biosecure + src/bob.db.caspeal + src/bob.db.frgc + src/bob.db.gbu + src/bob.db.lfw + src/bob.db.mobio + src/bob.db.multipie + src/bob.db.scface + src/bob.db.xm2vts + src/bob.db.youtube + src/bob.db.voxforge + src/bob.db.utfvp + src/bob.db.livdet2013 + src/bob.db.atvskeystroke + src/bob.db.biosecurid.face + src/bob.db.casme2 + src/bob.db.replay + src/bob.db.biowave_test + src/bob.db.casia_fasd + src/bob.db.verafinger + src/bob.bio.db + src/bob.bio.base + src/bob.bio.gmm + src/bob.bio.spear + src/bob.bio.face + src/bob.bio.video + src/bob.db.msu_mfsd_mod + src/bob.db.avspoof + src/bob.db.asvspoof + src/bob.db.kboc16 + src/bob.db.putvein + src/bob.db.ijba + src/bob.db.cuhk_cufs + +eggs = bob.db.bio_filelist + bob.db.banca + bob.db.biosecure + bob.db.caspeal + bob.db.frgc + bob.db.gbu + bob.db.lfw + bob.db.mobio + bob.db.multipie + bob.db.scface + bob.db.xm2vts + bob.db.youtube + bob.db.voxforge + bob.db.utfvp + bob.db.livdet2013 + bob.db.atvskeystroke + bob.db.biosecurid.face + bob.db.casme2 + bob.db.replay + bob.db.casia_fasd + bob.db.verafinger + bob.db.biowave_test + bob.bio.base + bob.bio.gmm + bob.bio.spear + bob.bio.face + bob.bio.video + bob.db.msu_mfsd_mod + bob.db.avspoof + bob.db.asvspoof + bob.db.kboc16 + bob.db.putvein + bob.db.ijba + bob.db.cuhk_cufs + + +extensions = bob.buildout + mr.developer + +auto-checkout = * + + +; options for bob.buildout extension +newest = false +debug = false +verbose = true + +[scripts] +recipe = bob.buildout:scripts + +[sources] +bob.db.bio_filelist = git https://gitlab.idiap.ch/bob/bob.db.bio_filelist +bob.db.banca = git https://gitlab.idiap.ch/bob/bob.db.banca +bob.db.biosecure = git https://gitlab.idiap.ch/bob/bob.db.biosecure +bob.db.caspeal = git https://gitlab.idiap.ch/bob/bob.db.caspeal +bob.db.frgc = git https://gitlab.idiap.ch/bob/bob.db.frgc +bob.db.gbu = git https://gitlab.idiap.ch/bob/bob.db.gbu +bob.db.lfw = git https://gitlab.idiap.ch/bob/bob.db.lfw +bob.db.mobio = git https://gitlab.idiap.ch/bob/bob.db.mobio +bob.db.multipie = git https://gitlab.idiap.ch/bob/bob.db.multipie +bob.db.scface = git https://gitlab.idiap.ch/bob/bob.db.scface +bob.db.xm2vts = git https://gitlab.idiap.ch/bob/bob.db.xm2vts +bob.db.youtube = git https://gitlab.idiap.ch/bob/bob.db.youtube +;bob.db.nist_sre12 = git https://gitlab.idiap.ch/bob/bob.db.nist_sre12 +bob.db.voxforge = git https://gitlab.idiap.ch/bob/bob.db.voxforge +bob.db.utfvp = git https://gitlab.idiap.ch/bob/bob.db.utfvp +bob.db.livdet2013 = git https://gitlab.idiap.ch/bob/bob.db.livdet2013 +bob.db.atvskeystroke = git https://gitlab.idiap.ch/bob/bob.db.atvskeystroke +bob.db.biosecurid.face = git https://gitlab.idiap.ch/bob/bob.db.biosecurid.face +bob.db.casme2 = git https://gitlab.idiap.ch/bob/bob.db.casme2 +bob.db.replay = git https://gitlab.idiap.ch/bob/bob.db.replay +bob.db.biowave_test = git https://gitlab.idiap.ch/bob/bob.db.biowave_test +;bob.db.casia_fasd = git https://gitlab.idiap.ch/bob/bob.db.casia_fasd +bob.db.verafinger = git git@gitlab.idiap.ch:bob/bob.db.verafinger +bob.bio.base = git https://gitlab.idiap.ch/bob/bob.bio.base +bob.bio.gmm = git https://gitlab.idiap.ch/bob/bob.bio.gmm +bob.bio.spear = git https://gitlab.idiap.ch/bob/bob.bio.spear +bob.bio.face = git https://gitlab.idiap.ch/bob/bob.bio.face +bob.bio.video = git https://gitlab.idiap.ch/bob/bob.bio.video +bob.db.msu_mfsd_mod = git https://gitlab.idiap.ch/bob/bob.db.msu_mfsd_mod +bob.db.avspoof = git https://gitlab.idiap.ch/bob/bob.db.avspoof +bob.db.asvspoof = git https://gitlab.idiap.ch/bob/bob.db.asvspoof +bob.db.kboc16 = git https://gitlab.idiap.ch/bob/bob.db.kboc16 +bob.db.putvein = git https://gitlab.idiap.ch/bob/bob.db.putvein +bob.db.ijba = git https://gitlab.idiap.ch/bob/bob.db.ijba +bob.db.cuhk_cufs = git https://gitlab.idiap.ch/bob/bob.db.cuhk_cufs +;bob.db.cbsr_nir_vis_2 = git https://gitlab.idiap.ch/bob/bob.db.cbsr_nir_vis_2 +;bob.example.cmake = git https://gitlab.idiap.ch/bob/bob.example.cmake diff --git a/buildout.cfg b/buildout.cfg deleted file mode 100644 index d81a5afc29b2c1553c112e99437814d829455500..0000000000000000000000000000000000000000 --- a/buildout.cfg +++ /dev/null @@ -1,20 +0,0 @@ -; vim: set fileencoding=utf-8 : -; Andre Anjos <andre.bioidiap@idiap.ch> -; Mon 16 Apr 08:29:18 2012 CEST - -[buildout] -parts = scripts -extends = layer2.cfg - -extensions = bob.buildout - mr.developer -auto-checkout = * - - -; options for bob.buildout extension -newest = false -debug = false -verbose = true - -[scripts] -recipe = bob.buildout:scripts diff --git a/ci/README.md b/ci/README.md new file mode 100644 index 0000000000000000000000000000000000000000..59ea50aaabf1f09bcc282716b5fa65ae332af00e --- /dev/null +++ b/ci/README.md @@ -0,0 +1,23 @@ +# Scripts for Continous Integration with Gitlab + +This directory contains scripts that are used by our continuous integration +(CI) builds. There are various sets of scripts named as follows: + +* `before_<stage>.sh` +* `<stage>.sh` +* `after_<stage>.sh` + +Each of these sets correpond to one of the build stages inside the CI builds. + +The script `install.sh` contains a basic set of routines to install and update +the scripts on the target host. If you add more scripts, it is sensible to +add those in there. + +The script `functions.sh` contains a set of functions that are shared between +all scripts. All scripts *source* `functions.sh` before anything is done. +`functions.sh` also set important local and global (export) variables that are +required by the build system. + +Before changing any script, try to understand the role of each of the stages by +looking at our template CI yaml files (in the directory `../templates`) and +then how to use the functions declared in `functions.sh`. diff --git a/ci/after_build.sh b/ci/after_build.sh new file mode 100755 index 0000000000000000000000000000000000000000..c9f0e880517fe5fd62f816e2b6569bf5e277984a --- /dev/null +++ b/ci/after_build.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# Sat 24 Sep 08:12:24 2016 CEST + +source $(dirname ${0})/functions.sh + +log_info "*** Not yet implemented ***" diff --git a/ci/after_deploy.sh b/ci/after_deploy.sh new file mode 100755 index 0000000000000000000000000000000000000000..dc97ec3a5e72f02e23d5efe1fc691bdec613ca2a --- /dev/null +++ b/ci/after_deploy.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# Thu 22 Sep 2016 15:11:53 CEST + +source $(dirname ${0})/functions.sh + +log_info "*** Not yet implemented ***" diff --git a/ci/after_docs.sh b/ci/after_docs.sh new file mode 100755 index 0000000000000000000000000000000000000000..cd563fb91dbdc844066e4a4a8d506f28d420cade --- /dev/null +++ b/ci/after_docs.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# Thu 22 Sep 2016 18:23:57 CEST + +source $(dirname ${0})/functions.sh + +log_info "*** Not yet implemented ***" diff --git a/ci/after_test.sh b/ci/after_test.sh new file mode 100755 index 0000000000000000000000000000000000000000..67a04cb79ada656a7be2d1cf2296f653acf399bb --- /dev/null +++ b/ci/after_test.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# Thu 22 Sep 2016 13:59:03 CEST + +source $(dirname ${0})/functions.sh + +log_info "*** Not yet implemented ***" diff --git a/ci/after_wheels.sh b/ci/after_wheels.sh new file mode 100755 index 0000000000000000000000000000000000000000..cd563fb91dbdc844066e4a4a8d506f28d420cade --- /dev/null +++ b/ci/after_wheels.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# Thu 22 Sep 2016 18:23:57 CEST + +source $(dirname ${0})/functions.sh + +log_info "*** Not yet implemented ***" diff --git a/ci/before_build.sh b/ci/before_build.sh new file mode 100755 index 0000000000000000000000000000000000000000..0e05f3d62afc844a700ee858b27b5e662d1cfe2e --- /dev/null +++ b/ci/before_build.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# Mon 8 Aug 17:40:24 2016 CEST + +source $(dirname ${0})/functions.sh + +PIPINSTALL = $1 + +WHEELS_REPOSITORY="${DOCSERVER}/software/bob/wheels/gitlab/" +WHEELS_SERVER=`echo ${DOCSERVER} | sed 's;https\?://;;'` +check_env WHEELS_REPOSITORY +check_env WHEELS_SERVER + +# Readies the virtualenv to use for installation +if [ ! -d ${PREFIX} ]; then + log_info "Creating virtualenv installation at ${PREFIX}..." + run_cmd ${BOB_PREFIX_PATH}/bin/virtualenv --system-site-packages ${PREFIX} +else + log_warn "Prefix directory ${PREFIX} exists, not re-installing..." +fi + +# Source the newly created virtualenv +log_info "$ source ${PREFIX}/bin/activate" +source ${PREFIX}/bin/activate + +# Verify where pip is installed +use_pip=`which pip` +if [ -z "${use_pip}" ]; then + log_error "Cannot find pip, aborting..." + exit 1 +else + log_info "Using pip: ${use_pip}" +fi + +# Upgrade wheel generation library to avoid bugs +run_cmd ${use_pip} install --upgrade setuptools pip wheel + +use_python=`which python` +if [ -z "${use_python}" ]; then + log_error "Cannot find python, aborting..." + exit 1 +else + log_info "Using python: ${use_python}" +fi + +if PIPINSTALL ; then + # Install this package's build dependencies + PIPOPTS="--find-links ${WHEELS_REPOSITORY} --trusted-host ${WHEELS_SERVER}" + PIPOPTS="${PIPOPTS} --pre --use-wheel --no-index" + if [ -e requirements.txt ]; then + run_cmd ${use_pip} install ${PIPOPTS} --requirement requirements-bob.txt + else + log_info "No requirements.txt file found, skipping 'pip install <build-deps>'..." + fi + + # Finally, bootstrap the installation from the new environment + if [ -e bootstrap-buildout.py ]; then + run_cmd ${use_python} bootstrap-buildout.py + else + log_error "No bootstrap-buildout.py file found, stopping..." + exit 1 + fi +fi diff --git a/ci/before_deploy.sh b/ci/before_deploy.sh new file mode 100755 index 0000000000000000000000000000000000000000..cac81087be5533b36b8ab026d458748b8f607f25 --- /dev/null +++ b/ci/before_deploy.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Wed 21 Sep 2016 13:08:05 CEST + +source $(dirname ${0})/functions.sh + +run_cmd $(dirname ${0})/before_test.sh + +# setup database locally and run `bob_dbmanage.py all download` +# if this is a database package - need auxiliary file for package +if [[ ${CI_PROJECT_NAME} == bob.db.* ]]; then + run_cmd ./bin/buildout + if [ -x ./bin/bob_dbmanage.py ]; then + run_cmd ./bin/bob_dbmanage.py all download --force; + fi +fi diff --git a/ci/before_docs.sh b/ci/before_docs.sh new file mode 100755 index 0000000000000000000000000000000000000000..cd563fb91dbdc844066e4a4a8d506f28d420cade --- /dev/null +++ b/ci/before_docs.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# Thu 22 Sep 2016 18:23:57 CEST + +source $(dirname ${0})/functions.sh + +log_info "*** Not yet implemented ***" diff --git a/ci/before_test.sh b/ci/before_test.sh new file mode 100755 index 0000000000000000000000000000000000000000..3709c10f2ba1b13c92d1b24b23a642033dac1a8c --- /dev/null +++ b/ci/before_test.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Wed 21 Sep 2016 13:08:05 CEST + +source $(dirname ${0})/functions.sh + +run_cmd $(dirname ${0})/before_build.sh + +# Source the newly created virtualenv +log_info "$ source ${PREFIX}/bin/activate" +source ${PREFIX}/bin/activate + +# Verify where pip is installed +use_pip=`which pip` +if [ -z "${use_pip}" ]; then + log_error "Cannot find pip, aborting..." + exit 1 +else + log_info "Using pip: ${use_pip}" +fi + +# zc.recipe.egg needs some special installation instructions +if [ "${CI_PROJECT_NAME}" == "bob.buildout" ]; then + run_cmd ${use_pip} install --upgrade --no-binary ":all:" zc.recipe.egg +fi + +run_cmd ${use_pip} install --upgrade --use-wheel --no-index --pre dist/*.whl diff --git a/ci/before_wheels.sh b/ci/before_wheels.sh new file mode 100755 index 0000000000000000000000000000000000000000..cd563fb91dbdc844066e4a4a8d506f28d420cade --- /dev/null +++ b/ci/before_wheels.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# Thu 22 Sep 2016 18:23:57 CEST + +source $(dirname ${0})/functions.sh + +log_info "*** Not yet implemented ***" diff --git a/ci/build.sh b/ci/build.sh new file mode 100644 index 0000000000000000000000000000000000000000..32ac88cac0748f64b70dce4d6a6a987c7967d81b --- /dev/null +++ b/ci/build.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Wed 21 Sep 2016 13:08:05 CEST + +source $(dirname ${0})/functions.sh + +run_cmd ./bin/buildout + +if [ -x ./bin/bob_dbmanage.py ]; then + run_cmd ./bin/bob_dbmanage.py all download --force; +fi + diff --git a/ci/deploy.sh b/ci/deploy.sh new file mode 100755 index 0000000000000000000000000000000000000000..34b1ca2372e35333c200e5188de5c57fd133f73c --- /dev/null +++ b/ci/deploy.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# Thu 22 Sep 2016 13:59:03 CEST + +source $(dirname ${0})/functions.sh + +lock_pypirc + +setup_deploy register --repository staging +setup_deploy check sdist --formats zip upload --repository staging + +# if that worked, uploads source package to the production index +log_info "Uploading package to ${PYPISERVER} on behalf of ${PYPIUSER}..." +setup_deploy register --repository production +setup_deploy check sdist --formats zip upload --repository production + +# if that worked, uploads documentation to pythonhosted if any exists +if [ -d sphinx ]; then + log_info "Uploading documentation to ${PYPISERVER} on behalf of ${PYPIUSER}..." + setup_deploy upload_docs --upload-dir sphinx --repository production +fi + +unlock_pypirc + +condaforge_packages=("bob" \ +"bob.extension" \ +"bob.blitz" \ +"bob.core" \ +"bob.ip.draw" \ +"bob.io.base" \ +"bob.sp" \ +"bob.math" \ +"bob.ap" \ +"bob.measure" \ +"bob.db.base" \ +"bob.io.image" \ +"bob.io.video" \ +"bob.io.matlab" \ +"bob.ip.base" \ +"bob.ip.color" \ +"bob.ip.gabor" \ +"bob.learn.activation" \ +"bob.learn.libsvm" \ +"bob.learn.boosting" \ +"bob.io.audio" \ +"bob.learn.linear" \ +"bob.learn.mlp" \ +"bob.db.wine" \ +"bob.db.mnist" \ +"bob.db.atnt" \ +"bob.ip.flandmark" \ +"bob.ip.facedetect" \ +"bob.ip.optflow.hornschunck" \ +"bob.ip.optflow.liu" \ +"bob.learn.em" \ +"bob.db.iris") + +if contains_element ${CI_PROJECT_NAME} "${condaforge_packages[@]}"; then + run_cmd ${CONDA_FOLDER}/bin/python _ci/update_feedstock.py ${CI_PROJECT_NAME} +fi diff --git a/ci/docs.sh b/ci/docs.sh new file mode 100755 index 0000000000000000000000000000000000000000..d2a62c3f7cd34ba6ca10eddded1daf66e21acf3f --- /dev/null +++ b/ci/docs.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Wed 21 Sep 2016 13:08:05 CEST + +source $(dirname ${0})/functions.sh + +info=sphinx/.gitlab-ci.info + +echo "repo=${CI_PROJECT_PATH}" > ${info} +echo "branch=${CI_BUILD_REF_NAME}" >> ${info} +echo "tag=${CI_BUILD_TAG}" >> ${info} +echo "build=${CI_BUILD_ID}" >> ${info} +echo "commit=${CI_BUILD_REF}" >> ${info} +echo "runner=${CI_RUNNER_DESCRIPTION}" >> ${info} + +file=${CI_PROJECT_NAME}-${CI_BUILD_REF}.tar.bz2 +run_cmd tar cfj ${file} sphinx + +doc_upload ${file} software/bob/docs-upload/ diff --git a/ci/functions.sh b/ci/functions.sh new file mode 100644 index 0000000000000000000000000000000000000000..e82893c3b1e9213ea7a833b1d4e9be1cf14325d4 --- /dev/null +++ b/ci/functions.sh @@ -0,0 +1,255 @@ +#!/usr/bin/env bash +# Thu 22 Sep 2016 13:05:54 CEST + +# Build utilities + +# Determines the architecture we're using +arch() { + [[ "$(uname -s)" == "Darwin" ]] && echo "macosx" || echo "linux" +} + + +# Functions for coloring echo commands +log_info() { + echo -e "(`date +%T`) \033[1;34m${@}\033[0m" +} + + +log_warn() { + echo -e "(`date +%T`) \033[1;35mWarning: ${@}\033[0m" >&2 +} + + +log_error() { + echo -e "(`date +%T`) \033[1;31mError: ${@}\033[0m" >&2 +} + + +# Checks a given environment variable is set (non-zero size) +check_env() { + if [ -z "${!1}" ]; then + log_error "Variable ${1} is undefined - aborting..."; + exit 1 + else + log_info "${1}=${!1}"; + fi +} + + +# Exports a given environment variable, verbosely +export_env() { + if [ -z "${!1}" ]; then + log_error "Variable ${1} is undefined - aborting..."; + exit 1 + else + export ${1} + log_info "export ${1}=${!1}"; + fi +} + + +# Checks a given environment variable is set (non-zero size) +check_pass() { + if [ -z "${!1}" ]; then + log_error "Variable ${1} is undefined - aborting..."; + exit 1 + else + log_info "${1}=********"; + fi +} + + +# Function for running command and echoing results +run_cmd() { + log_info "$ ${@}" + ${@} + local status=$? + if [ ${status} != 0 ]; then + log_error "Command Failed \"${@}\"" + exit ${status} + fi +} + + +# Prepares ~/.pypirc +lock_pypirc() { + local lockfile=/var/tmp/pypirc_lock + local rc=${HOME}/.pypirc + local maxtries=10 + local try=0 + local sleeptime=30 #seconds + + while true; do + if [[ ${try} -lt ${maxtries} ]]; then + ((try++)) + if ( set -o noclobber; echo "$$" > "${lockfile}") 2> /dev/null; then + log_info "Successfully acquired ${lockfile}" + echo $$ > ${lockfile} + log_info "trapping on ${lockfile}..." + trap 'rm -f "${lockfile}"; exit $?' INT TERM EXIT + + # start: protected code + log_info "Creating ${rc}..." + if [ -e ${rc} ]; then + run_cmd rm -f ${rc} + fi + cat <<EOT >> ${rc} +[distutils] +index-servers = + production + staging + +[production] +repository: ${PYPISERVER} +username: ${PYPIUSER} +password: ${PYPIPASS} + +[staging] +repository: ${TESTSERVER} +username: ${PYPIUSER} +password: ${PYPIPASS} +EOT + run_cmd chmod 600 ${rc} + # end: protected code + break + else + log_warn "${lockfile} exists, owned by process $(cat $lockfile)" + log_info "Will retry after a ${sleeptime} seconds sleep (${try}/${maxtries})" + run_cmd sleep ${sleeptime} + fi + else + log_error "I already retried deploying ${try} times. Aborting..." + log_error "You can retry this step when less packages are building." + exit 1 + fi + done +} + + +# Cleans ~/.pypirc, if the lock file belongs to us +unlock_pypirc() { + local lockfile=/var/tmp/pypirc_lock + local rc=${HOME}/.pypirc + + # untrap if lock belongs to the running process + if [[ $(cat ${lockfile}) == $$ ]]; then + run_cmd rm -r ${lockfile} + run_cmd rm -rf ${rc} + log_info "$ trap - INT TERM EXIT" + trap - INT TERM EXIT + fi +} + + +# Runs setup.py in a deployment context. If fails, tries to unlock +# the ${HOME}/.pypirc file lock +setup_deploy() { + log_info "$ ${@}" + ${PREFIX}/bin/python setup.py ${@} + local status=$? + if [ ${status} != 0 ]; then + log_error "Command Failed \"${@}\"" + unlock_pypirc #just tries + exit ${status} + fi +} + + +# Uploads a file to our intranet location via curl +doc_upload() { + log_info "curl: ${1} -> ${DOCSERVER}/${2}..." + curl --location --silent --show-error --user "${DOCUSER}:${DOCPASS}" --upload-file ${1} ${DOCSERVER}/${2} + local status=$? + if [ ${status} != 0 ]; then + log_error "Curl command finished with an error condition (status=${status})" + exit ${status} + fi + log_info "Successfully uploaded ${1} with curl" +} + +check_env PYTHON_VERSION +check_env CI_PROJECT_DIR +check_env CI_PROJECT_PATH +check_env CI_PROJECT_NAME +check_env CI_BUILD_REF +check_env CI_BUILD_REF_NAME +check_pass PYPIUSER +check_pass PYPIPASS +check_pass DOCUSER +check_pass DOCPASS + +# Sets up variables +ARCH=`arch` + +if [ -z "${CONDA_FOLDER}" ]; then + if [ "${ARCH}" == "linux" ]; then + CONDA_FOLDER=/local/conda + else + CONDA_FOLDER=/opt/conda + fi +fi + +if [ -z "${CONDA_PREFIX}" ]; then + CONDA_PREFIX=env +fi + +if [ -z "${DOCSERVER}" ]; then + DOCSERVER=https://www.idiap.ch +fi + +PREFIX=${CI_PROJECT_DIR}/${CONDA_PREFIX} + +# Add "-coverage" support +if [ "${ARCH}" == "linux" ]; then + # Temporary hack to get building done right with gcc-5 compilers + if [ -z "${CFLAGS}" ]; then + CFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -coverage" + else + CFLAGS="${CFLAGS} -D_GLIBCXX_USE_CXX11_ABI=0 -coverage" + fi + CXXFLAGS=${CFLAGS} +else + if [ -z "${CFLAGS}" ]; then + CFLAGS="-coverage" + else + CFLAGS="${CFLAGS} -coverage" + fi + CXXFLAGS=${CFLAGS} +fi + +PYVER=py$(echo ${PYTHON_VERSION} | tr -d '.') +BOB_PREFIX_PATH=${CONDA_FOLDER}/envs/bob-devel-${PYVER} +LD_LIBRARY_PATH=${BOB_PREFIX_PATH}/lib + +if [ -z "${PYPISERVER}" ]; then + PYPISERVER="https://pypi.python.org/pypi" +fi + +TESTSERVER=https://testpypi.python.org/pypi + +check_env ARCH +check_env PYVER +check_env PREFIX +export_env CFLAGS +export_env CXXFLAGS +check_env DOCSERVER +check_env PYPISERVER +check_env TESTSERVER +check_env CONDA_FOLDER +check_env CONDA_PREFIX +export_env LD_LIBRARY_PATH +export_env BOB_PREFIX_PATH + +if [ -z "${CI_BUILD_TAG}" ]; then + BOB_DOCUMENTATION_SERVER="${DOCSERVER}/software/bob/docs/latest/bob/%s/master/" + export_env BOB_DOCUMENTATION_SERVER +else + log_info "Building tag, not setting BOB_DOCUMENTATION_SERVER" +fi + +# taken from here: https://stackoverflow.com/questions/3685970/check-if-an-array-contains-a-value +contains_element () { + local e + for e in "${@:2}"; do [[ "$e" == "$1" ]] && return 0; done + return 1 +} diff --git a/ci/install.sh b/ci/install.sh new file mode 100755 index 0000000000000000000000000000000000000000..8adb34b7f3cebf8ad4ab118feaefe90726231282 --- /dev/null +++ b/ci/install.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# Thu 22 Sep 2016 13:05:54 CEST + +# Installation script for our build tools + +if [ "${#}" -ne 1 ]; then + echo "usage: ${0} <ci-support-directory>" + echo "example: ${0} _ci" + exit 1 +fi + + +# Functions for coloring echo commands +log_info() { + echo -e "(`date +%T`) \033[1;34m${@}\033[0m" +} + + +log_error() { + echo -e "(`date +%T`) \033[1;31mError: ${@}\033[0m" +} + + +# Function for running command and echoing results +run_cmd() { + log_info "$ ${@}" + ${@} + local status=$? + if [ ${status} != 0 ]; then + log_error "Command Failed \"${@}\"" + exit ${status} + fi +} + + +get_script() { + local url="https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/${2}" + local curlopt="--location --silent --show-error --output ${1}/${2}" + if [ -e ${1}/${2} ]; then + rm -f ${1}/${2} + fi + run_cmd curl ${curlopt} ${url} +} + + +get_exec() { + get_script ${1} ${2} + run_cmd chmod 755 ${1}/${2} +} + + +run_cmd mkdir -pv ${1} +get_script ${1} functions.sh +get_exec ${1} install.sh +for stage in "build" "test" "docs" "wheels" "deploy"; do + get_exec ${1} before_${stage}.sh + get_exec ${1} ${stage}.sh + get_exec ${1} after_${stage}.sh +done +get_exec ${1} update_feedstock.py diff --git a/ci/test.sh b/ci/test.sh new file mode 100755 index 0000000000000000000000000000000000000000..85ac251a22e2d266cfd12bc0d394d4b91c7051d3 --- /dev/null +++ b/ci/test.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Wed 21 Sep 2016 13:08:05 CEST + +source $(dirname ${0})/functions.sh + +run_cmd cd ${PREFIX} + +# The tests: +run_cmd ${PREFIX}/bin/python ${BOB_PREFIX_PATH}/bin/coverage run --source=${CI_PROJECT_NAME} ${BOB_PREFIX_PATH}/bin/nosetests -sv ${CI_PROJECT_NAME} +run_cmd ${PREFIX}/bin/python ${BOB_PREFIX_PATH}/bin/coverage report +run_cmd ${PREFIX}/bin/python ${BOB_PREFIX_PATH}/bin/sphinx-build -b doctest ${CI_PROJECT_DIR}/doc ${CI_PROJECT_NAME}/sphinx + +run_cmd cd ${CI_PROJECT_DIR} diff --git a/ci/update_feedstock.py b/ci/update_feedstock.py new file mode 100755 index 0000000000000000000000000000000000000000..cbdfdac1cbedc386539ebe6be899f4b09290f208 --- /dev/null +++ b/ci/update_feedstock.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python + +import hashlib +try: + from urllib2 import urlopen +except ImportError: + from urllib.request import urlopen +import requests +import json +try: + from packaging.version import parse +except ImportError: + from pip._vendor.packaging.version import parse +import re +import tempfile +import shutil +import os +import subprocess + +URL_PATTERN = 'https://pypi.python.org/pypi/{package}/json' + + +def run_commands(*calls): + """runs the given commands.""" + # get all calls + for call in calls: + print(' - ' + ' '.join(call)) + # execute call + if subprocess.call(call): + # call failed (has non-zero exit status) + raise ValueError("Command '%s' failed; stopping" % ' '.join(call)) + + +def get_version(package, url_pattern=URL_PATTERN): + """Return version of package on pypi.python.org using json.""" + req = requests.get(url_pattern.format(package=package)) + version = parse('0') + if req.status_code == requests.codes.ok: + j = json.loads(req.text) + if 'releases' in j: + releases = j['releases'] + for release in releases: + ver = parse(release) + if not ver.is_prerelease: + version = max(version, ver) + return str(version) + + +def get_remote_md5_sum(url, max_file_size=100 * 1024 * 1024): + remote = urlopen(url) + hash = hashlib.md5() + + total_read = 0 + while True: + data = remote.read(4096) + total_read += 4096 + + if not data or total_read > max_file_size: + break + hash.update(data) + + return hash.hexdigest() + + +def main(package, direct_push=False): + stable_version = get_version(package) + print('latest stable version for {} is {}'.format(package, stable_version)) + url = 'https://pypi.io/packages/source/{0}/{1}/{1}-{2}.zip'.format( + package[0], package, stable_version) + try: + md5 = get_remote_md5_sum(url) + except Exception: + raise + temp_dir = tempfile.mkdtemp() + try: + print("\nClonning the feedstock") + feedstock = os.path.join(temp_dir, '{}-feedstock'.format(package)) + try: + run_commands( + ['git', 'clone', + 'git@github.com:conda-forge/{}-feedstock.git'.format(package), + feedstock]) + except ValueError: + print("\nThe feedstock does not exist on conda-forge. Exiting ...") + raise + os.chdir(feedstock) + if not direct_push: + run_commands( + ['git', 'remote', 'add', 'bioidiap', + 'git@github.com:bioidiap/{}-feedstock.git'.format(package)], + ['git', 'checkout', '-b', stable_version]) + # update meta.yaml + with open('recipe/meta.yaml') as f: + doc = f.read() + if package == 'bob.math': + build_number = '200' + else: + build_number = '0' + doc = re.sub(r'\{\s?%\s?set\s?version\s?=\s?".*"\s?%\s?\}', + '{% set version = "' + str(stable_version) + '" %}', + doc, count=1) + doc = re.sub(r'\s+number\:\s?[0-9]+', '\n number: ' + build_number, doc, + count=1) + doc = re.sub(r'\{\s?%\s?set\s?build_number\s?=\s?"[0-9]+"\s?%\s?\}', + '{% set build_number = "' + build_number + '" %}', + doc, count=1) + doc = re.sub(r'\s+md5\:.*', '\n md5: {}'.format(md5), doc, count=1) + doc = re.sub(r'\s+url\:.*', + '\n url: {}'.format( + url.replace(stable_version, '{{ version }}')), + doc, count=1) + doc = re.sub(r'\s+home\:.*', + '\n home: https://www.idiap.ch/software/bob/', + doc, count=1) + + if package == 'bob': + requrl = 'https://gitlab.idiap.ch/bob/bob/raw/master/requirements.txt' + remote = requests.get(requrl) + req = remote.content.decode() + req = '\n - '.join(req.replace('== ', '==').strip().split('\n')) + be_id = doc.find('bob.extension') + te_id = doc.find('test:\n', be_id) + template = '''{req} + + run: + - python + - {req} + +'''.format(req=req) + doc = doc[:be_id] + template + doc[te_id:] + + with open('recipe/meta.yaml', 'w') as f: + f.write(doc) + + conda_smithy_path = '' + if os.path.isdir('/local/conda/bin/'): + conda_smithy_path = '/local/conda/bin/' + run_commands([conda_smithy_path+'conda-smithy', 'rerender'], + ['git', '--no-pager', 'diff'], + ['git', 'add', '-A']) + try: + run_commands(['git', 'commit', '-am', + 'Update to version {}'.format(stable_version)]) + except ValueError: + print('Feedstock is already uptodate, skipping.') + return + if direct_push: + print(feedstock) + try: + answer = raw_input('Would you like to push directly to master?').lower() + except Exception: + answer = input('Would you like to push directly to master?').lower() + if answer.startswith('y') or answer == '': + run_commands(['git', 'push']) + print('See the changes at:\n' + 'https://github.com/conda-forge/' + '{}-feedstock/commits/master\n\n'.format(package)) + else: + run_commands(['git', 'push', '--force', '--set-upstream', + 'bioidiap', stable_version], + ['hub', 'pull-request', '-b', 'conda-forge:master', + '-h', 'bioidiap:{}'.format(stable_version), + '-m', 'Update to version {}'.format(stable_version)]) + finally: + shutil.rmtree(temp_dir) + + +if __name__ == '__main__': + import sys + pkg = sys.argv[1] + main(pkg) diff --git a/ci/wheels.sh b/ci/wheels.sh new file mode 100755 index 0000000000000000000000000000000000000000..34fb958f109f24c281b06cc3dcc437f6ed6dfe4c --- /dev/null +++ b/ci/wheels.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# Thu 22 Sep 2016 13:58:56 CEST + +source $(dirname ${0})/functions.sh + +for file in dist/*.whl; do + doc_upload ${file} software/bob/wheels-upload/gitlab/ +done diff --git a/wheels_dependencies.sh b/ci/wheels_dependencies.sh old mode 100755 new mode 100644 similarity index 88% rename from wheels_dependencies.sh rename to ci/wheels_dependencies.sh index f58b807aae1cf2c6c94e86887f5e5be59ebb061e..a2668c3c8fbb8646c61be171c59a316d1d9721ee --- a/wheels_dependencies.sh +++ b/ci/wheels_dependencies.sh @@ -1,8 +1,11 @@ #!/bin/bash ./bin/python setup.py bdist_wheel --dist-dir ./dist --python-tag ${1} + +REQUIREMENTS = ${2} + #Iterate the dependency files to generate the wheels -for f in requirements.txt +for f in REQUIREMENTS do while read -r line do diff --git a/requirements-bob.txt b/requirements-bob.txt new file mode 100644 index 0000000000000000000000000000000000000000..48df22fa17740939c364a01278d478474b33e61c --- /dev/null +++ b/requirements-bob.txt @@ -0,0 +1,32 @@ +bob.extension +bob.blitz +bob.core +bob.io.base +bob.sp +bob.ap +bob.math +bob.measure +bob.db.base +bob.io.image +bob.io.video +bob.io.matlab +bob.io.audio +bob.ip.base +bob.ip.color +bob.ip.draw +bob.ip.gabor +bob.learn.activation +bob.learn.libsvm +bob.learn.linear +bob.learn.mlp +bob.learn.em +bob.learn.boosting +bob.ip.facedetect +bob.ip.flandmark +bob.ip.optflow.hornschunck +bob.ip.optflow.liu +bob.db.iris +bob.db.mnist +bob.db.wine +bob.db.atnt +bob.db.arface \ No newline at end of file diff --git a/requirements.txt b/requirements-satellites.txt similarity index 51% rename from requirements.txt rename to requirements-satellites.txt index 6f4357016f84383eac709e203f238c2dc019c428..9e36e4a7dc86c43962a1c6691f2db2d4ec898430 100644 --- a/requirements.txt +++ b/requirements-satellites.txt @@ -1,36 +1,4 @@ -bob.extension -bob.blitz -bob.core -bob.io.base -bob.sp -bob.ap -bob.math -bob.measure -bob.db.base -bob.io.image -bob.io.video -bob.io.matlab -bob.io.audio -bob.ip.base -bob.ip.color -bob.ip.draw -bob.ip.gabor -bob.learn.activation -bob.learn.libsvm -bob.learn.linear -bob.learn.mlp -bob.learn.em -bob.learn.boosting -bob.ip.facedetect -bob.ip.flandmark -bob.ip.optflow.hornschunck -bob.ip.optflow.liu bob.db.bio_filelist -bob.db.iris -bob.db.mnist -bob.db.wine -bob.db.atnt -bob.db.arface bob.db.banca bob.db.biosecure bob.db.caspeal @@ -62,4 +30,4 @@ bob.db.asvspoof bob.db.kboc16 bob.db.putvein bob.db.ijba -bob.db.cuhk_cufs +bob.db.cuhk_cufs \ No newline at end of file