From 64b542a2125dc38751bdb2092d5b7cddd50711f3 Mon Sep 17 00:00:00 2001 From: Andre Anjos <andre.dos.anjos@gmail.com> Date: Thu, 23 May 2019 08:23:14 +0200 Subject: [PATCH] [ci] Implement use of bob.devtools for builds --- .gitlab-ci.yml | 136 +------------------------------- before_build.sh | 46 ----------- conda/meta.yaml | 2 +- requirements.txt | 196 +++++++++++++++++++++++------------------------ 4 files changed, 100 insertions(+), 280 deletions(-) delete mode 100755 before_build.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 57a5deb..eb063ae 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,135 +1 @@ -# This YAML file contains descriptions for the CI of most of our Bob/BEAT/BATL -# packages - do **not** modify it unless you know what you're doing (and up -# to!) - -# Definition of global variables (all stages) -variables: - PYTHONUNBUFFERED: "1" - CONDA_ROOT: "${CI_PROJECT_DIR}/miniconda" - BOOTSTRAP: "https://gitlab.idiap.ch/bob/bob.devtools/raw/master/bob/devtools/bootstrap.py" - - -# Definition of our build pipeline order -stages: - - build - - deploy - - pypi - - -# Build targets -.build_template: &build_job - stage: build - before_script: - - ./before_build.sh - script: - - curl --silent "${BOOTSTRAP}" --output "bootstrap.py" - - python3 bootstrap.py -vv channel base - - source ${CONDA_ROOT}/etc/profile.d/conda.sh - - conda activate base - - bdt ci build ./conda -vv - - bdt ci clean -vv - cache: &build_caches - paths: - - miniconda.sh - - ${CONDA_ROOT}/pkgs/*.tar.bz2 - - ${CONDA_ROOT}/pkgs/urls.txt - - -.build_linux_template: &linux_build_job - <<: *build_job - tags: - - docker - image: continuumio/conda-concourse-ci - artifacts: - expire_in: 1 week - paths: - - ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2 - cache: - <<: *build_caches - key: "linux-cache" - - -build_linux_36: - <<: *linux_build_job - variables: - GITLAB_CHECKOUT_STRATEGY: "https://gitlab-ci-token:${CI_BUILD_TOKEN}@gitlab.idiap.ch/" - PYTHON_VERSION: "3.6" - BUILD_EGG: "true" - script: - - curl --silent "${BOOTSTRAP}" --output "bootstrap.py" - - python3 bootstrap.py -vv channel base - - source ${CONDA_ROOT}/etc/profile.d/conda.sh - - conda activate base - - bdt ci build -vv - - bdt ci clean -vv - artifacts: - expire_in: 1 week - paths: - - dist/*.zip - - sphinx - - ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2 - - -# Test targets (not normally used) -.test_template: &test_job - stage: test - script: - - curl --silent "${BOOTSTRAP}" --output "bootstrap.py" - - python3 bootstrap.py -vv channel base - - source ${CONDA_ROOT}/etc/profile.d/conda.sh - - conda activate base - - bdt ci test -vv - - bdt ci clean -vv - cache: &test_caches - paths: - - miniconda.sh - - ${CONDA_ROOT}/pkgs/*.tar.bz2 - - ${CONDA_ROOT}/pkgs/urls.txt - - -.test_linux_template: &linux_test_job - <<: *test_job - tags: - - docker - image: continuumio/conda-concourse-ci - cache: - <<: *test_caches - key: "linux-cache" - - -# Deploy targets -.deploy_template: &deploy_job - stage: deploy - script: - - curl --silent "${BOOTSTRAP}" --output "bootstrap.py" - - python3 bootstrap.py channel base - - source ${CONDA_ROOT}/etc/profile.d/conda.sh - - conda activate base - - bdt ci deploy -vv - - bdt ci clean -vv - dependencies: - - build_linux_36 - tags: - - docker - cache: &build_caches - paths: - - miniconda.sh - - ${CONDA_ROOT}/pkgs/*.tar.bz2 - - ${CONDA_ROOT}/pkgs/urls.txt - - -deploy_beta: - <<: *deploy_job - environment: beta - only: - - master - - -deploy_stable: - <<: *deploy_job - environment: stable - only: - - /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags) - except: - - branches - +include: 'https://gitlab.idiap.ch/bob/bob.devtools/raw/master/bob/devtools/data/gitlab-ci/docs.yaml' diff --git a/before_build.sh b/before_build.sh deleted file mode 100755 index e297432..0000000 --- a/before_build.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env bash - -set -ex - -# to fix cloning in docker images -GITLAB_CHECKOUT_STRATEGY="${GITLAB_CHECKOUT_STRATEGY:-git@gitlab.idiap.ch:}" - -# Clone all packages -for pkg in bob `cat requirements.txt | sed -e '/^\s*#.*/d;/^\s*$/d'`; do - git clone --depth 1 ${GITLAB_CHECKOUT_STRATEGY}bob/${pkg}.git doc/$pkg || \ - { git -C doc/$pkg reset --hard HEAD && \ - git -C doc/$pkg checkout master && \ - git -C doc/$pkg pull; } - if [[ -n "${CI_COMMIT_TAG}" ]]; then - git -C doc/$pkg fetch --tags - tag=`git -C doc/$pkg tag --sort='v:refname' | grep -e 'v[0-9]*\.[0-9]*\.[0-9]*$' | tail -n 1` - git -C doc/$pkg checkout $tag - fi -done - -# Create extra-intersphinx.txt -# Add newlines in the end of files -# remove bobs, gridtk -# remove comments -# remove trailing whitespace -sed -e '$s/$/\n/' \ - -e '/^bob/d' \ - -e '/^gridtk/d' \ - -e 's:#.*$::g' \ - -e 's/[[:space:]]*$//' \ - -s \ - doc/*/doc/extra-intersphinx.txt \ - doc/*/requirements.txt \ - doc/*/test-requirements.txt \ - | sort -u > doc/extra-intersphinx.txt - -# Create nitpick-exceptions.txt -# Add newlines in the end of files -# remove comments -# remove trailing whitespace -sed -e '$s/$/\n/' \ - -e 's:#.*$::g' \ - -e 's/[[:space:]]*$//' \ - -s \ - doc/*/doc/nitpick-exceptions.txt \ - | sort -u > doc/nitpick-exceptions.txt diff --git a/conda/meta.yaml b/conda/meta.yaml index ed2de4c..f44dd4e 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -84,7 +84,7 @@ requirements: - bob.ip.base - bob.ip.caffe_extractor - bob.ip.color - - bob.ip.dlib + - bob.ip.dlib - bob.ip.draw - bob.ip.facedetect - bob.ip.facelandmarks diff --git a/requirements.txt b/requirements.txt index 3d04ac2..e3b83b5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,98 +1,98 @@ -bob -bob.ap -bob.bio.base -bob.bio.caffe_face -bob.bio.face -bob.bio.gmm -bob.bio.spear -bob.bio.vein -bob.bio.video -bob.blitz -bob.buildout -bob.core -bob.db.arface -bob.db.asvspoof -bob.db.asvspoof2017 -bob.db.atnt -bob.db.atvskeystroke -bob.db.avspoof -bob.db.banca -bob.db.base -bob.db.biosecure -bob.db.biosecurid.face -bob.db.casme2 -bob.db.caspeal -bob.db.cbsr_nir_vis_2 -bob.db.cohface -bob.db.cuhk_cufs -bob.db.cuhk_cufsf -bob.db.frgc -bob.db.fv3d -bob.db.gbu -bob.db.hci_tagging -bob.db.ijba -bob.db.ijbc -bob.db.iris -bob.db.kboc16 -bob.db.lfw -bob.db.livdet2013 -bob.db.maskattack -bob.db.mnist -bob.db.mobio -bob.db.msu_mfsd_mod -bob.db.multipie -bob.db.nist_sre12 -bob.db.nivl -bob.db.oulunpu -bob.db.pericrosseye -bob.db.pola_thermal -bob.db.putvein -bob.db.replay -bob.db.replaymobile -bob.db.scface -bob.db.utfvp -bob.db.uvad -bob.db.verafinger -bob.db.voicepa -bob.db.voxforge -bob.db.wine -bob.db.xm2vts -bob.db.youtube -bob.db.fargo -bob.extension -bob.fusion.base -bob.io.audio -bob.io.base -bob.io.image -bob.io.matlab -bob.io.video -bob.ip.base -bob.ip.caffe_extractor -bob.ip.color -bob.ip.dlib -bob.ip.draw -bob.ip.facedetect -bob.ip.facelandmarks -bob.ip.flandmark -bob.ip.gabor -bob.ip.mtcnn -bob.ip.optflow.hornschunck -bob.ip.optflow.liu -bob.ip.qualitymeasure -bob.ip.skincolorfilter -bob.ip.tensorflow_extractor -bob.learn.activation -bob.learn.boosting -bob.learn.em -bob.learn.libsvm -bob.learn.linear -bob.learn.mlp -bob.math -bob.measure -bob.pad.base -bob.pad.face -bob.pad.vein -bob.pad.voice -bob.rppg.base -bob.sp -gridtk +bob/bob +bob/bob.ap +bob/bob.bio.base +bob/bob.bio.caffe_face +bob/bob.bio.face +bob/bob.bio.gmm +bob/bob.bio.spear +bob/bob.bio.vein +bob/bob.bio.video +bob/bob.blitz +bob/bob.buildout +bob/bob.core +bob/bob.db.arface +bob/bob.db.asvspoof +bob/bob.db.asvspoof2017 +bob/bob.db.atnt +bob/bob.db.atvskeystroke +bob/bob.db.avspoof +bob/bob.db.banca +bob/bob.db.base +bob/bob.db.biosecure +bob/bob.db.biosecurid.face +bob/bob.db.casme2 +bob/bob.db.caspeal +bob/bob.db.cbsr_nir_vis_2 +bob/bob.db.cohface +bob/bob.db.cuhk_cufs +bob/bob.db.cuhk_cufsf +bob/bob.db.frgc +bob/bob.db.fv3d +bob/bob.db.gbu +bob/bob.db.hci_tagging +bob/bob.db.ijba +bob/bob.db.ijbc +bob/bob.db.iris +bob/bob.db.kboc16 +bob/bob.db.lfw +bob/bob.db.livdet2013 +bob/bob.db.maskattack +bob/bob.db.mnist +bob/bob.db.mobio +bob/bob.db.msu_mfsd_mod +bob/bob.db.multipie +bob/bob.db.nist_sre12 +bob/bob.db.nivl +bob/bob.db.oulunpu +bob/bob.db.pericrosseye +bob/bob.db.pola_thermal +bob/bob.db.putvein +bob/bob.db.replay +bob/bob.db.replaymobile +bob/bob.db.scface +bob/bob.db.utfvp +bob/bob.db.uvad +bob/bob.db.verafinger +bob/bob.db.voicepa +bob/bob.db.voxforge +bob/bob.db.wine +bob/bob.db.xm2vts +bob/bob.db.youtube +bob/bob.db.fargo +bob/bob.extension +bob/bob.fusion.base +bob/bob.io.audio +bob/bob.io.base +bob/bob.io.image +bob/bob.io.matlab +bob/bob.io.video +bob/bob.ip.base +bob/bob.ip.caffe_extractor +bob/bob.ip.color +bob/bob.ip.dlib +bob/bob.ip.draw +bob/bob.ip.facedetect +bob/bob.ip.facelandmarks +bob/bob.ip.flandmark +bob/bob.ip.gabor +bob/bob.ip.mtcnn +bob/bob.ip.optflow.hornschunck +bob/bob.ip.optflow.liu +bob/bob.ip.qualitymeasure +bob/bob.ip.skincolorfilter +bob/bob.ip.tensorflow_extractor +bob/bob.learn.activation +bob/bob.learn.boosting +bob/bob.learn.em +bob/bob.learn.libsvm +bob/bob.learn.linear +bob/bob.learn.mlp +bob/bob.math +bob/bob.measure +bob/bob.pad.base +bob/bob.pad.face +bob/bob.pad.vein +bob/bob.pad.voice +bob/bob.rppg.base +bob/bob.sp +bob/gridtk -- GitLab