diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e4c0e1472456b1a82fd11720543d0570c1bef660..d06151385c61b358945b7d51987ad34a41750ccf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,3 @@ -# This build file uses template features from YAML so it is generic enough for -# any Bob project. Don't modify it unless you know what you're doing. - # Definition of global variables (all stages) variables: CONDA_ROOT: "${CI_PROJECT_DIR}/miniconda" @@ -9,18 +6,12 @@ variables: # Definition of our build pipeline order stages: - build - - deploy - - pypi # Build targets .build_template: &build_job stage: build before_script: - - mkdir _ci - - curl --silent "https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/install.sh" > _ci/install.sh - - chmod 755 _ci/install.sh - - ./_ci/install.sh _ci master #installs ci support scripts - ./_ci/before_build.sh script: - ./_ci/build.sh @@ -41,7 +32,6 @@ stages: artifacts: expire_in: 1 week paths: - - _ci/ - ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2 cache: <<: *build_caches @@ -55,7 +45,6 @@ stages: artifacts: expire_in: 1 week paths: - - _ci/ - ${CONDA_ROOT}/conda-bld/osx-64/*.tar.bz2 cache: <<: *build_caches @@ -70,7 +59,6 @@ build_linux_36: artifacts: expire_in: 1 week paths: - - _ci/ - dist/*.zip - sphinx - ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2 @@ -80,50 +68,3 @@ build_macosx_36: <<: *macosx_build_job variables: PYTHON_VERSION: "3.6" - - -# Deploy targets -.deploy_template: &deploy_job - stage: deploy - before_script: - - ./_ci/install.sh _ci master #updates ci support scripts - script: - - ./_ci/deploy.sh - dependencies: - - build_linux_36 - - build_macosx_36 - tags: - - deployer - - -deploy_beta: - <<: *deploy_job - environment: beta - only: - - master - - -deploy_stable: - <<: *deploy_job - environment: stable - only: - - /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags) - except: - - branches - - -pypi: - stage: pypi - environment: pypi - only: - - /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags) - except: - - branches - before_script: - - ./_ci/install.sh _ci master #updates ci support scripts - script: - - ./_ci/pypi.sh - dependencies: - - build_linux_36 - tags: - - deployer diff --git a/_ci/after_build.sh b/_ci/after_build.sh new file mode 100755 index 0000000000000000000000000000000000000000..d3798e096e356884640877fa4610c8157e3c6dc0 --- /dev/null +++ b/_ci/after_build.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +source $(dirname ${0})/functions.sh + +# delete the bob packages from the cache otherwise the cache keeps increasing +# over and over. See https://gitlab.idiap.ch/bob/bob.admin/issues/65 +run_cmd rm -rf ${CONDA_ROOT}/pkgs/batl*.tar.bz2 +run_cmd rm -rf ${CONDA_ROOT}/pkgs/beat*.tar.bz2 +run_cmd rm -rf ${CONDA_ROOT}/pkgs/bob*.tar.bz2 +run_cmd rm -rf ${CONDA_ROOT}/pkgs/gridtk*.tar.bz2 + +# run git clean to clean everything that is not needed. This helps to keep the +# disk usage on CI machines to minimum. +run_cmd git clean -ffdx \ + --exclude="miniconda.sh" \ + --exclude="miniconda/pkgs/*.tar.bz2" \ + --exclude="miniconda/pkgs/urls.txt" \ + --exclude="miniconda/conda-bld/linux-64/*.tar.bz2" \ + --exclude="miniconda/conda-bld/osx-64/*.tar.bz2" \ + --exclude="_ci" \ + --exclude="dist/*.zip" \ + --exclude="sphinx" diff --git a/_ci/before_build.sh b/_ci/before_build.sh new file mode 100755 index 0000000000000000000000000000000000000000..1061a4014eb8b6cbe245f2ac71eab8374188de74 --- /dev/null +++ b/_ci/before_build.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Tue 16 Jan 11:15:32 2018 CET + +source $(dirname ${0})/functions.sh + +# checks if a conda installation exists. Otherwise, install one +if [ ! -e ${CONDA_ROOT}/bin/conda ]; then + install_miniconda ${CONDA_ROOT} +fi + +mkdir -p ${CONDA_ROOT}/pkgs +touch ${CONDA_ROOT}/pkgs/urls +touch ${CONDA_ROOT}/pkgs/urls.txt + +create_condarc ${CONDARC} + +set_conda_channels ${CI_COMMIT_TAG} +for k in "${CONDA_CHANNELS[@]}"; do + echo " - ${DOCSERVER}/${k}" >> ${CONDARC} +done +echo " - defaults" >> ${CONDARC} + +# displays contents of our configuration +echo "Contents of \`${CONDARC}':" +cat ${CONDARC} + +# updates conda installation +run_cmd ${CONDA_ROOT}/bin/conda install python conda=4 curl conda-build=3 + +# cleans up +run_cmd ${CONDA_ROOT}/bin/conda clean --lock + +# print conda information for debugging purposes +run_cmd ${CONDA_ROOT}/bin/conda info diff --git a/_ci/build.sh b/_ci/build.sh new file mode 100755 index 0000000000000000000000000000000000000000..c60d8e2f4bdab6b4abe2be54e89382ec6118ad72 --- /dev/null +++ b/_ci/build.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +source $(dirname ${0})/functions.sh + +# Makes sure we activate the base environment if available +run_cmd source ${CONDA_ROOT}/etc/profile.d/conda.sh +run_cmd conda activate base +export_env PATH + +set_conda_channels ${CI_COMMIT_TAG} +log_info "$ ${CONDA_ROOT}/bin/python ${SCRIPTS_DIR}/nextbuild.py ${DOCSERVER}/${1} ${CI_PROJECT_NAME} ${BOB_PACKAGE_VERSION} ${PYTHON_VERSION}" +BOB_BUILD_NUMBER=$(${CONDA_ROOT}/bin/python ${SCRIPTS_DIR}/nextbuild.py ${DOCSERVER}/${1} ${CI_PROJECT_NAME} ${BOB_PACKAGE_VERSION} ${PYTHON_VERSION}) +export_env BOB_BUILD_NUMBER + +# copy the recipe_append.yaml over before build +run_cmd cp ${CI_PROJECT_DIR}/bob/devtools/data/recipe_append.yaml conda/ +run_cmd cp ${CI_PROJECT_DIR}/bob/devtools/data/conda_build_config.yaml conda/ + +BLDOPT="--python=${PYTHON_VERSION} --no-anaconda-upload" + +run_cmd ${CONDA_ROOT}/bin/conda build ${BLDOPT} conda diff --git a/_ci/functions.sh b/_ci/functions.sh new file mode 100644 index 0000000000000000000000000000000000000000..62dcf54217cc96431c6e73a19a19eb0ef41c737d --- /dev/null +++ b/_ci/functions.sh @@ -0,0 +1,224 @@ +#!/usr/bin/env bash +# Wed 9 Jan 2019 14:33:20 CET + +# Build utilities +SCRIPTS_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) + +# datetime prefix for logging +log_datetime() { + echo "($(date +%T.%3N))" +} + +# Functions for coloring echo commands +log_info() { + echo -e "$(log_datetime) \033[1;34m${@}\033[0m" +} + + +log_error() { + echo -e "$(log_datetime) \033[1;31mError: ${@}\033[0m" >&2 +} + + +# Checks just if the variable is defined and has non-zero length +check_defined() { + if [ -z "${!1+abc}" ]; then + log_error "Variable ${1} is undefined - aborting..."; + exit 1 + elif [ -z "${!1}" ]; then + log_error "Variable ${1} is zero-length - aborting..."; + exit 1 + fi +} + + +# Logs a given environment variable to the screen +log_env() { + log_info "${1}=${!1}" +} + + +# Checks a given environment variable is set (non-zero size) +check_env() { + check_defined "${1}" + log_env "${1}" +} + + +# Checks a given environment variable array is set (non-zero size) +# Then prints all of its components +check_array_env() { + check_defined "${1}" + eval array=\( \${${1}[@]} \) + for i in "${!array[@]}"; do + log_info "${1}[${i}]=${array[${i}]}"; + done +} + + +# Exports a given environment variable, verbosely +export_env() { + check_defined "${1}" + export ${1} + log_info "export ${1}=${!1}" +} + + +# Function for running command and echoing results +run_cmd() { + log_info "$ ${@}" + ${@} + local status=$? + if [ ${status} != 0 ]; then + log_error "Command Failed \"${@}\"" + exit ${status} + fi +} + + +if [ -z "${BOB_PACKAGE_VERSION}" ]; then + if [ ! -r "version.txt" ]; then + log_error "./version.txt does not exist - cannot figure out version number" + exit 1 + fi + BOB_PACKAGE_VERSION=`cat version.txt | tr -d '\n'`; +fi + + +# merges conda cache folders +# $1: Path to the main cache to keep. The directory must exist. +# $2: Path to the extra cache to be merged into main cache +merge_conda_cache() { + if [ -e ${1} ]; then + _cached_urlstxt="${2}/urls.txt" + _urlstxt="${1}/urls.txt" + if [ -e ${2} ]; then + log_info "Merging urls.txt and packages with cached files..." + mv ${2}/*.tar.bz2 ${1}/ + cat ${_urlstxt} ${_cached_urlstxt} | sort | uniq > ${_urlstxt} + fi + fi +} + + +# installs a miniconda installation. +# $1: Path to where to install miniconda. +install_miniconda() { + log_info "Installing miniconda in ${1} ..." + + # downloads the latest conda installation script + if [ "${OSNAME}" == "linux" ]; then + object=https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh + else + object=https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh + fi + + # checks if miniconda.sh exists + if [ ! -e miniconda.sh ]; then + log_info "Downloading latest miniconda3 installer..." + run_cmd curl --silent --output miniconda.sh ${object} + else + log_info "Re-using cached miniconda3 installer..." + ls -l miniconda.sh + fi + + # move cache to a different folder if it exists + if [ -e ${1} ]; then + run_cmd mv ${1} ${1}.cached + fi + + # install miniconda + run_cmd bash miniconda.sh -b -p ${1} + + # Put back cache and merge urls.txt + merge_conda_cache ${1}/pkgs ${1}.cached/pkgs + # remove the backup cache folder + rm -rf ${1}.cached + + # List currently available packages on cache + # run_cmd ls -l ${1}/pkgs/ + # run_cmd cat ${1}/pkgs/urls.txt + + hash -r +} + + +# Creates a sane base condarc file +# $1: Path to where to populate the condarc file +create_condarc() { + log_info "Populating the basic condarc file in ${1} ..." + + cat <<EOF > ${1} +default_channels: #!final + - https://repo.anaconda.com/pkgs/main + - https://repo.anaconda.com/pkgs/free + - https://repo.anaconda.com/pkgs/r + - https://repo.anaconda.com/pkgs/pro +add_pip_as_python_dependency: false #!final +changeps1: false #!final +always_yes: true #!final +quiet: true #!final +show_channel_urls: true #!final +anaconda_upload: false #!final +ssl_verify: false #!final +channels: #!final +EOF + +} + + +# sets CONDA_CHANNELS to the list of conda channels that should be considered +# $2: typically, the value of ${CI_COMMIT_TAG} or empty +# given the current visibility/tagging conditions of the package. +set_conda_channels() { + CONDA_CHANNELS=() #resets bash array + if [ -z "${1}" ]; then #public beta build + CONDA_CHANNELS+=('public/conda/label/beta') + CONDA_CHANNELS+=('public/conda') + else #public (tagged) build + CONDA_CHANNELS+=('public/conda') + fi + check_array_env CONDA_CHANNELS +} + + +log_env PYTHON_VERSION +check_env CI_PROJECT_DIR +check_env CI_PROJECT_NAME +check_env CI_COMMIT_REF_NAME +export_env BOB_PACKAGE_VERSION + +# Sets up variables +OSNAME=`osname` +check_env OSNAME + +DOCSERVER=http://www.idiap.ch + +# Sets up the location of our rc file for conda +CONDARC=${CONDA_ROOT}/condarc + +if [ -z "${OS_SLUG}" ]; then + OS_SLUG="${OSNAME}-64" +fi + +export_env OS_SLUG +export_env DOCSERVER +check_env CONDA_ROOT +export_env CONDARC + +# Sets up certificates for curl and openssl +CURL_CA_BUNDLE="${CI_PROJECT_DIR}/bob/devtools/data/cacert.pem" +export_env CURL_CA_BUNDLE +SSL_CERT_FILE="${CURL_CA_BUNDLE}" +export_env SSL_CERT_FILE +GIT_SSL_CAINFO="${CURL_CA_BUNDLE}" +export_env GIT_SSL_CAINFO + +# Sets up upload folders for documentation (just in case we need them) +# See: https://gitlab.idiap.ch/bob/bob.admin/issues/2 + +# Sets up the language so Unicode filenames are considered correctly +LANG="en_US.UTF-8" +LC_ALL="${LANG}" +export_env LANG +export_env LC_ALL diff --git a/_ci/nextbuild.py b/_ci/nextbuild.py new file mode 100644 index 0000000000000000000000000000000000000000..8559ec4e173299d013fc731952070f6f0a105c39 --- /dev/null +++ b/_ci/nextbuild.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +'''This is a copy of bob/devtools/conda:next_build_number with a CLI''' + +import os +import re +import sys + +from conda.exports import get_index + + +if __name__ == '__main__': + + channel_url = sys.argv[1] + name = sys.argv[2] + version = sys.argv[3] + python = sys.argv[4] + + # no dot in py_ver + py_ver = python.replace('.', '') + + # get the channel index + index = get_index(channel_urls=[channel_url], prepend=False) + + # search if package with the same version exists + build_number = 0 + urls = [] + for dist in index: + + if dist.name == name and dist.version == version: + match = re.match('py[2-9][0-9]+', dist.build_string) + + if match and match.group() == 'py{}'.format(py_ver): + build_number = max(build_number, dist.build_number + 1) + urls.append(index[dist].url) + + urls = [url.replace(channel_url, '') for url in urls] + + print(build_number) diff --git a/conda/conda_build_config.yaml b/conda/conda_build_config.yaml deleted file mode 120000 index dbdbc0f110d96e2e6a1cdf7c1c1ae3c2bc9b0445..0000000000000000000000000000000000000000 --- a/conda/conda_build_config.yaml +++ /dev/null @@ -1 +0,0 @@ -../bob/devtools/data/conda_build_config.yaml \ No newline at end of file