diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d06151385c61b358945b7d51987ad34a41750ccf..b3d0024da8ed07e2061cad7aae2417a802da1f5d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,17 +6,16 @@ variables: # Definition of our build pipeline order stages: - build + - deploy # Build targets .build_template: &build_job stage: build before_script: - - ./_ci/before_build.sh + - ./_ci/bootstrap.sh script: - ./_ci/build.sh - after_script: - - ./_ci/after_build.sh cache: &build_caches paths: - miniconda.sh @@ -68,3 +67,38 @@ build_macosx_36: <<: *macosx_build_job variables: PYTHON_VERSION: "3.6" + + +# Deploy targets +.deploy_template: &deploy_job + stage: deploy + before_script: + - ./_ci/bootstrap.sh self + script: + - ./_ci/deploy.sh + dependencies: + - build_linux_36 + - build_macosx_36 + tags: + - deployer + cache: &build_caches + paths: + - miniconda.sh + - ${CONDA_ROOT}/pkgs/*.tar.bz2 + - ${CONDA_ROOT}/pkgs/urls.txt + + +deploy_beta: + <<: *deploy_job + environment: beta + only: + - master + + +deploy_stable: + <<: *deploy_job + environment: stable + only: + - /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags) + except: + - branches diff --git a/_ci/after_build.sh b/_ci/after_build.sh deleted file mode 100755 index 9f0fe54c0c85b4f2e89114abbabde97fb3a90bab..0000000000000000000000000000000000000000 --- a/_ci/after_build.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash - -source $(dirname ${0})/functions.sh - -# delete the bob packages from the cache otherwise the cache keeps increasing -# over and over. See https://gitlab.idiap.ch/bob/bob.admin/issues/65 -run_cmd rm -rf ${CONDA_ROOT}/pkgs/bob*.tar.bz2 - -# run git clean to clean everything that is not needed. This helps to keep the -# disk usage on CI machines to minimum. -run_cmd git clean -ffdx \ - --exclude="miniconda.sh" \ - --exclude="miniconda/pkgs/*.tar.bz2" \ - --exclude="miniconda/pkgs/urls.txt" \ - --exclude="miniconda/conda-bld/${OS_SLUG}/*.tar.bz2" \ - --exclude="_ci" \ - --exclude="dist/*.zip" \ - --exclude="sphinx" diff --git a/_ci/before_build.sh b/_ci/before_build.sh deleted file mode 100755 index d9c5c006ba25f89838ea1fd35a29041c06789309..0000000000000000000000000000000000000000 --- a/_ci/before_build.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash -# Tue 16 Jan 11:15:32 2018 CET - -source $(dirname ${0})/functions.sh - -# checks if a conda installation exists. Otherwise, install one -if [ ! -e ${CONDA_ROOT}/bin/conda ]; then - install_miniconda ${CONDA_ROOT} -fi - -mkdir -p ${CONDA_ROOT}/pkgs -touch ${CONDA_ROOT}/pkgs/urls -touch ${CONDA_ROOT}/pkgs/urls.txt - -create_condarc ${CONDARC} - -set_conda_channels ${CI_COMMIT_TAG} -for k in "${CONDA_CHANNELS[@]}"; do - echo " - ${DOCSERVER}/${k}" >> ${CONDARC} -done -echo " - defaults" >> ${CONDARC} - -# displays contents of our configuration -echo "Contents of \`${CONDARC}':" -cat ${CONDARC} - -# updates conda installation -run_cmd ${CONDA_ROOT}/bin/conda install python conda=4 conda-build=3 - -# cleans up -run_cmd ${CONDA_ROOT}/bin/conda clean --lock - -# print conda information for debugging purposes -run_cmd ${CONDA_ROOT}/bin/conda info diff --git a/_ci/bootstrap.sh b/_ci/bootstrap.sh new file mode 100755 index 0000000000000000000000000000000000000000..86e1b8d5b430648b8bbe2244f1880af1b9b3447b --- /dev/null +++ b/_ci/bootstrap.sh @@ -0,0 +1,131 @@ +#!/urs/bin/env bash + +# Bootstraps a new conda installation and prepares base environment +# if "self" is passed as parameter, then self installs an already built +# version of bob.devtools available on your conda-bld directory. + +# datetime prefix for logging +log_datetime() { + echo "($(date +%T.%3N))" +} + +# Functions for coloring echo commands +log_info() { + echo -e "$(log_datetime) \033[1;34m${@}\033[0m" +} + + +log_error() { + echo -e "$(log_datetime) \033[1;31mError: ${@}\033[0m" >&2 +} + +# Function for running command and echoing results +run_cmd() { + log_info "$ ${@}" + ${@} + local status=$? + if [ ${status} != 0 ]; then + log_error "Command Failed \"${@}\"" + exit ${status} + fi +} + + +# merges conda cache folders +# $1: Path to the main cache to keep. The directory must exist. +# $2: Path to the extra cache to be merged into main cache +merge_conda_cache() { + if [ -e ${1} ]; then + _cached_urlstxt="${2}/urls.txt" + _urlstxt="${1}/urls.txt" + if [ -e ${2} ]; then + log_info "Merging urls.txt and packages with cached files..." + mv ${2}/*.tar.bz2 ${1}/ + cat ${_urlstxt} ${_cached_urlstxt} | sort | uniq > ${_urlstxt} + fi + fi +} + + +# Checks just if the variable is defined and has non-zero length +check_defined() { + if [ -z "${!1+abc}" ]; then + log_error "Variable ${1} is undefined - aborting..."; + exit 1 + elif [ -z "${!1}" ]; then + log_error "Variable ${1} is zero-length - aborting..."; + exit 1 + fi +} + + +# installs a miniconda installation. +# $1: Path to where to install miniconda. +install_miniconda() { + log_info "Installing miniconda in ${1} ..." + + # checks if miniconda.sh exists + if [ ! -e miniconda.sh ]; then + log_info "Downloading latest miniconda3 installer..." + # downloads the latest conda installation script + if [ "$(uname -s)" == "Linux" ]; then _os="Linux" else _os="MacOSX"; fi + obj=https://repo.continuum.io/miniconda/Miniconda3-latest-${_os}-x86_64.sh + run_cmd curl --silent --output miniconda.sh ${obj} + else + log_info "Re-using cached miniconda3 installer..." + ls -l miniconda.sh + fi + + # move cache to a different folder if it exists + if [ -e ${1} ]; then + run_cmd mv ${1} ${1}.cached + fi + + # install miniconda + run_cmd bash miniconda.sh -b -p ${1} + + # Put back cache and merge urls.txt + merge_conda_cache ${1}/pkgs ${1}.cached/pkgs + # remove the backup cache folder + rm -rf ${1}.cached + + # List currently available packages on cache + # run_cmd ls -l ${1}/pkgs/ + # run_cmd cat ${1}/pkgs/urls.txt + + hash -r +} + +check_defined CONDA_ROOT +check_defined CI_PROJECT_DIR + +# checks if a conda installation exists. Otherwise, installs one +if [ ! -e ${CONDA_ROOT}/bin/conda ]; then + install_miniconda ${CONDA_ROOT} +fi + +run_cmd mkdir -p ${CONDA_ROOT}/pkgs +run_cmd touch ${CONDA_ROOT}/pkgs/urls +run_cmd touch ${CONDA_ROOT}/pkgs/urls.txt + +cp -fv ${CI_PROJECT_DIR}/bob/devtools/data/base-condarc ${CONDARC} +echo "channels:" >> ${CONDARC} +echo " - http://www.idiap.ch/public/conda" >> ${CONDARC} +echo " - defaults" >> ${CONDARC} + +# displays contents of our configuration +echo "Contents of \`${CONDARC}':" +cat ${CONDARC} + +# updates conda installation, installs just built bob.devtools +if [ "${1}" == "self" ]; then + run_cmd ${CONDA_ROOT}/bin/conda create -n bdt ${CONDA_ROOT}/conda-bld/${OS_SLUG}/bob.devtools-*.tar.bz2 +else + run_cmd ${CONDA_ROOT}/bin/conda install -n base python conda=4 conda-build=3 +fi + +# cleans up +run_cmd ${CONDA_ROOT}/bin/conda clean --lock + +# print conda information for debugging purposes +run_cmd ${CONDA_ROOT}/bin/conda info diff --git a/_ci/build.sh b/_ci/build.sh index 3e14f5fc6df47f97d9280735e521033ae68c9e76..92609b05750bdd6ada8837af57f76f83fbf0c24e 100755 --- a/_ci/build.sh +++ b/_ci/build.sh @@ -1,21 +1,78 @@ #!/usr/bin/env bash -source $(dirname ${0})/functions.sh +# datetime prefix for logging +log_datetime() { + echo "($(date +%T.%3N))" +} + +# Functions for coloring echo commands +log_info() { + echo -e "$(log_datetime) \033[1;34m${@}\033[0m" +} + + +log_error() { + echo -e "$(log_datetime) \033[1;31mError: ${@}\033[0m" >&2 +} + + +# Checks just if the variable is defined and has non-zero length +check_defined() { + if [ -z "${!1+abc}" ]; then + log_error "Variable ${1} is undefined - aborting..."; + exit 1 + elif [ -z "${!1}" ]; then + log_error "Variable ${1} is zero-length - aborting..."; + exit 1 + fi +} + + +# Exports a given environment variable, verbosely +export_env() { + check_defined "${1}" + export ${1} + log_info "export ${1}=${!1}" +} + +check_defined CONDA_ROOT +check_defined CI_PROJECT_DIR +check_defined CI_PROJECT_NAME +check_defined CI_COMMIT_TAG +check_defined PYTHON_VERSION + +BOB_PACKAGE_VERSION=`cat version.txt | tr -d '\n'`; +check_defined BOB_PACKAGE_VERSION # Makes sure we activate the base environment if available run_cmd source ${CONDA_ROOT}/etc/profile.d/conda.sh run_cmd conda activate base export_env PATH -set_conda_channels ${CI_COMMIT_TAG} -log_info "$ ${CONDA_ROOT}/bin/python ${SCRIPTS_DIR}/nextbuild.py ${DOCSERVER}/${CONDA_CHANNELS[0]} ${CI_PROJECT_NAME} ${BOB_PACKAGE_VERSION} ${PYTHON_VERSION}" -BOB_BUILD_NUMBER=$(${CONDA_ROOT}/bin/python ${SCRIPTS_DIR}/nextbuild.py ${DOCSERVER}/${CONDA_CHANNELS[0]} ${CI_PROJECT_NAME} ${BOB_PACKAGE_VERSION} ${PYTHON_VERSION}) +if [ -z "${CI_COMMIT_TAG}" ]; then #building beta + channel="http://www.idiap.ch/public/conda/label/beta" +else + channel="http://www.idiap.ch/public/conda" +fi + +log_info "$ ${CONDA_ROOT}/bin/python ${CI_PROJECT_DIR}/_ci/nextbuild.py ${channel} ${CI_PROJECT_NAME} ${BOB_PACKAGE_VERSION} ${PYTHON_VERSION}" +BOB_BUILD_NUMBER=$(${CONDA_ROOT}/bin/python ${CI_PROJECT_DIR}/_ci/nextbuild.py ${channel} ${CI_PROJECT_NAME} ${BOB_PACKAGE_VERSION} ${PYTHON_VERSION}) export_env BOB_BUILD_NUMBER # copy the recipe_append.yaml over before build run_cmd cp ${CI_PROJECT_DIR}/bob/devtools/data/recipe_append.yaml conda/ run_cmd cp ${CI_PROJECT_DIR}/bob/devtools/data/conda_build_config.yaml conda/ -BLDOPT="--python=${PYTHON_VERSION} --no-anaconda-upload" +run_cmd ${CONDA_ROOT}/bin/conda build "--python=${PYTHON_VERSION} --no-anaconda-upload" conda -run_cmd ${CONDA_ROOT}/bin/conda build ${BLDOPT} conda +# run git clean to clean everything that is not needed. This helps to keep the +# disk usage on CI machines to minimum. +if [ "$(uname -s)" == "Linux" ]; then _os="linux" else _os="osx"; fi +run_cmd git clean -ffdx \ + --exclude="miniconda.sh" \ + --exclude="miniconda/pkgs/*.tar.bz2" \ + --exclude="miniconda/pkgs/urls.txt" \ + --exclude="miniconda/conda-bld/${_os}-64/*.tar.bz2" \ + --exclude="_ci" \ + --exclude="dist/*.zip" \ + --exclude="sphinx" diff --git a/_ci/deploy.sh b/_ci/deploy.sh new file mode 100755 index 0000000000000000000000000000000000000000..f96ea43790d76f0f2597f1a81d2331c3654ac0e7 --- /dev/null +++ b/_ci/deploy.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Wed Jan 9 16:48:57 CET 2019 + +source $(dirname ${0})/functions.sh + +run_cmd cp -fv ${CI_PROJECT_DIR}/bob/devtools/data/base-condarc ${CONDARC} +echo "channels:" >> ${CONDARC} +echo " - ${DOCSERVER}/public/conda" >> ${CONDARC} +echo " - defaults" >> ${CONDARC} + +deploy_conda_packages ${CONDA_CHANNELS[0]} ${CI_PROJECT_NAME} + +# upload the docs from the sphinx folder (usually an artifact of Linux Python +# 3.6 builds) +for folder in "${DOC_UPLOADS[@]}"; do + dav_upload_folder sphinx "${folder}" +done diff --git a/_ci/functions.sh b/_ci/functions.sh deleted file mode 100644 index 1f41563f029a33d57728575ed172b5edfebed713..0000000000000000000000000000000000000000 --- a/_ci/functions.sh +++ /dev/null @@ -1,229 +0,0 @@ -#!/usr/bin/env bash -# Wed 9 Jan 2019 14:33:20 CET - -# Build utilities -SCRIPTS_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) - -# Determines the operating system we're using -osname() { - [[ "$(uname -s)" == "Darwin" ]] && echo "osx" || echo "linux" -} - -# datetime prefix for logging -log_datetime() { - echo "($(date +%T.%3N))" -} - -# Functions for coloring echo commands -log_info() { - echo -e "$(log_datetime) \033[1;34m${@}\033[0m" -} - - -log_error() { - echo -e "$(log_datetime) \033[1;31mError: ${@}\033[0m" >&2 -} - - -# Checks just if the variable is defined and has non-zero length -check_defined() { - if [ -z "${!1+abc}" ]; then - log_error "Variable ${1} is undefined - aborting..."; - exit 1 - elif [ -z "${!1}" ]; then - log_error "Variable ${1} is zero-length - aborting..."; - exit 1 - fi -} - - -# Logs a given environment variable to the screen -log_env() { - log_info "${1}=${!1}" -} - - -# Checks a given environment variable is set (non-zero size) -check_env() { - check_defined "${1}" - log_env "${1}" -} - - -# Checks a given environment variable array is set (non-zero size) -# Then prints all of its components -check_array_env() { - check_defined "${1}" - eval array=\( \${${1}[@]} \) - for i in "${!array[@]}"; do - log_info "${1}[${i}]=${array[${i}]}"; - done -} - - -# Exports a given environment variable, verbosely -export_env() { - check_defined "${1}" - export ${1} - log_info "export ${1}=${!1}" -} - - -# Function for running command and echoing results -run_cmd() { - log_info "$ ${@}" - ${@} - local status=$? - if [ ${status} != 0 ]; then - log_error "Command Failed \"${@}\"" - exit ${status} - fi -} - - -if [ -z "${BOB_PACKAGE_VERSION}" ]; then - if [ ! -r "version.txt" ]; then - log_error "./version.txt does not exist - cannot figure out version number" - exit 1 - fi - BOB_PACKAGE_VERSION=`cat version.txt | tr -d '\n'`; -fi - - -# merges conda cache folders -# $1: Path to the main cache to keep. The directory must exist. -# $2: Path to the extra cache to be merged into main cache -merge_conda_cache() { - if [ -e ${1} ]; then - _cached_urlstxt="${2}/urls.txt" - _urlstxt="${1}/urls.txt" - if [ -e ${2} ]; then - log_info "Merging urls.txt and packages with cached files..." - mv ${2}/*.tar.bz2 ${1}/ - cat ${_urlstxt} ${_cached_urlstxt} | sort | uniq > ${_urlstxt} - fi - fi -} - - -# installs a miniconda installation. -# $1: Path to where to install miniconda. -install_miniconda() { - log_info "Installing miniconda in ${1} ..." - - # downloads the latest conda installation script - if [ "${OSNAME}" == "linux" ]; then - object=https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh - else - object=https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh - fi - - # checks if miniconda.sh exists - if [ ! -e miniconda.sh ]; then - log_info "Downloading latest miniconda3 installer..." - run_cmd curl --silent --output miniconda.sh ${object} - else - log_info "Re-using cached miniconda3 installer..." - ls -l miniconda.sh - fi - - # move cache to a different folder if it exists - if [ -e ${1} ]; then - run_cmd mv ${1} ${1}.cached - fi - - # install miniconda - run_cmd bash miniconda.sh -b -p ${1} - - # Put back cache and merge urls.txt - merge_conda_cache ${1}/pkgs ${1}.cached/pkgs - # remove the backup cache folder - rm -rf ${1}.cached - - # List currently available packages on cache - # run_cmd ls -l ${1}/pkgs/ - # run_cmd cat ${1}/pkgs/urls.txt - - hash -r -} - - -# Creates a sane base condarc file -# $1: Path to where to populate the condarc file -create_condarc() { - log_info "Populating the basic condarc file in ${1} ..." - - cat <<EOF > ${1} -default_channels: #!final - - https://repo.anaconda.com/pkgs/main - - https://repo.anaconda.com/pkgs/free - - https://repo.anaconda.com/pkgs/r - - https://repo.anaconda.com/pkgs/pro -add_pip_as_python_dependency: false #!final -changeps1: false #!final -always_yes: true #!final -quiet: true #!final -show_channel_urls: true #!final -anaconda_upload: false #!final -ssl_verify: false #!final -channels: #!final -EOF - -} - - -# sets CONDA_CHANNELS to the list of conda channels that should be considered -# $2: typically, the value of ${CI_COMMIT_TAG} or empty -# given the current visibility/tagging conditions of the package. -set_conda_channels() { - CONDA_CHANNELS=() #resets bash array - if [ -z "${1}" ]; then #public beta build - CONDA_CHANNELS+=('public/conda/label/beta') - CONDA_CHANNELS+=('public/conda') - else #public (tagged) build - CONDA_CHANNELS+=('public/conda') - fi - check_array_env CONDA_CHANNELS -} - - -log_env PYTHON_VERSION -check_env CI_PROJECT_DIR -check_env CI_PROJECT_NAME -check_env CI_COMMIT_REF_NAME -export_env BOB_PACKAGE_VERSION - -# Sets up variables -OSNAME=`osname` -check_env OSNAME - -if [ -z "${OS_SLUG}" ]; then - OS_SLUG="${OSNAME}-64" -fi -export_env OS_SLUG - -DOCSERVER=http://www.idiap.ch - -# Sets up the location of our rc file for conda -CONDARC=${CONDA_ROOT}/condarc - -export_env DOCSERVER -check_env CONDA_ROOT -export_env CONDARC - -# Sets up certificates for curl and openssl -CURL_CA_BUNDLE="${CI_PROJECT_DIR}/bob/devtools/data/cacert.pem" -export_env CURL_CA_BUNDLE -SSL_CERT_FILE="${CURL_CA_BUNDLE}" -export_env SSL_CERT_FILE -GIT_SSL_CAINFO="${CURL_CA_BUNDLE}" -export_env GIT_SSL_CAINFO - -# Sets up upload folders for documentation (just in case we need them) -# See: https://gitlab.idiap.ch/bob/bob.admin/issues/2 - -# Sets up the language so Unicode filenames are considered correctly -LANG="en_US.UTF-8" -LC_ALL="${LANG}" -export_env LANG -export_env LC_ALL diff --git a/bob/devtools/data/base-condarc b/bob/devtools/data/base-condarc new file mode 100644 index 0000000000000000000000000000000000000000..480c551df17ff2e90f5d07083bb99c07a61b1ce8 --- /dev/null +++ b/bob/devtools/data/base-condarc @@ -0,0 +1,12 @@ +default_channels: + - https://repo.anaconda.com/pkgs/main + - https://repo.anaconda.com/pkgs/free + - https://repo.anaconda.com/pkgs/r + - https://repo.anaconda.com/pkgs/pro +add_pip_as_python_dependency: false #!final +changeps1: false #!final +always_yes: true #!final +quiet: true #!final +show_channel_urls: true #!final +anaconda_upload: false #!final +ssl_verify: false #!final diff --git a/conda/meta.yaml b/conda/meta.yaml index 25f80dbd976e2ef7c73efc7f3dbfb224d2237997..dfc10678c4a1c52343611ffa11ca8e51572c4aff 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -33,7 +33,8 @@ requirements: - setuptools - click - click-plugins - - conda-build + - conda=4 + - conda-build=3 - pytz - python-dateutil - gitpython