[ci] Largely simplified

.gitlab-ci.yml

@@ -6,17 +6,16 @@ variables:
 # Definition of our build pipeline order
 stages:
   - build
+  - deploy
 
 
 # Build targets
 .build_template: &build_job
   stage: build
   before_script:
-    - ./_ci/before_build.sh
+    - ./_ci/bootstrap.sh
   script:
     - ./_ci/build.sh
-  after_script:
-    - ./_ci/after_build.sh
   cache: &build_caches
     paths:
       - miniconda.sh
@@ -68,3 +67,38 @@ build_macosx_36:
   <<: *macosx_build_job
   variables:
     PYTHON_VERSION: "3.6"
+
+
+# Deploy targets
+.deploy_template: &deploy_job
+  stage: deploy
+  before_script:
+    - ./_ci/bootstrap.sh self
+  script:
+    - ./_ci/deploy.sh
+  dependencies:
+    - build_linux_36
+    - build_macosx_36
+  tags:
+    - deployer
+  cache: &build_caches
+    paths:
+      - miniconda.sh
+      - ${CONDA_ROOT}/pkgs/*.tar.bz2
+      - ${CONDA_ROOT}/pkgs/urls.txt
+
+
+deploy_beta:
+  <<: *deploy_job
+  environment: beta
+  only:
+    - master
+
+
+deploy_stable:
+  <<: *deploy_job
+  environment: stable
+  only:
+    - /^v\d+\.\d+\.\d+([abc]\d*)?$/  # PEP-440 compliant version (tags)
+  except:
+    - branches

_ci/after_build.sh

-#!/usr/bin/env bash
-
-source $(dirname ${0})/functions.sh
-
-# delete the bob packages from the cache otherwise the cache keeps increasing
-# over and over. See https://gitlab.idiap.ch/bob/bob.admin/issues/65
-run_cmd rm -rf ${CONDA_ROOT}/pkgs/bob*.tar.bz2
-
-# run git clean to clean everything that is not needed. This helps to keep the
-# disk usage on CI machines to minimum.
-run_cmd git clean -ffdx \
-    --exclude="miniconda.sh" \
-    --exclude="miniconda/pkgs/*.tar.bz2" \
-    --exclude="miniconda/pkgs/urls.txt" \
-    --exclude="miniconda/conda-bld/${OS_SLUG}/*.tar.bz2" \
-    --exclude="_ci" \
-    --exclude="dist/*.zip" \
-    --exclude="sphinx"

_ci/before_build.sh

-#!/usr/bin/env bash
-# Tue 16 Jan 11:15:32 2018 CET
-
-source $(dirname ${0})/functions.sh
-
-# checks if a conda installation exists. Otherwise, install one
-if [ ! -e ${CONDA_ROOT}/bin/conda ]; then
-  install_miniconda ${CONDA_ROOT}
-fi
-
-mkdir -p ${CONDA_ROOT}/pkgs
-touch ${CONDA_ROOT}/pkgs/urls
-touch ${CONDA_ROOT}/pkgs/urls.txt
-
-create_condarc ${CONDARC}
-
-set_conda_channels ${CI_COMMIT_TAG}
-for k in "${CONDA_CHANNELS[@]}"; do
-  echo "  - ${DOCSERVER}/${k}" >> ${CONDARC}
-done
-echo "  - defaults" >> ${CONDARC}
-
-# displays contents of our configuration
-echo "Contents of \`${CONDARC}':"
-cat ${CONDARC}
-
-# updates conda installation
-run_cmd ${CONDA_ROOT}/bin/conda install python conda=4 conda-build=3
-
-# cleans up
-run_cmd ${CONDA_ROOT}/bin/conda clean --lock
-
-# print conda information for debugging purposes
-run_cmd ${CONDA_ROOT}/bin/conda info

_ci/functions.sh → _ci/bootstrap.sh

-#!/usr/bin/env bash
-# Wed  9 Jan 2019 14:33:20 CET
+#!/urs/bin/env bash
 
-# Build utilities
-SCRIPTS_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-
-# Determines the operating system we're using
-osname() {
-  [[ "$(uname -s)" == "Darwin" ]] && echo "osx" || echo "linux"
-}
+# Bootstraps a new conda installation and prepares base environment
+# if "self" is passed as parameter, then self installs an already built
+# version of bob.devtools available on your conda-bld directory.
 
 # datetime prefix for logging
 log_datetime() {
@@ -24,51 +19,6 @@ log_error() {
   echo -e "$(log_datetime) \033[1;31mError: ${@}\033[0m" >&2
 }
 
-
-# Checks just if the variable is defined and has non-zero length
-check_defined() {
-  if [ -z "${!1+abc}" ]; then
-    log_error "Variable ${1} is undefined - aborting...";
-    exit 1
-  elif [ -z "${!1}" ]; then
-    log_error "Variable ${1} is zero-length - aborting...";
-    exit 1
-  fi
-}
-
-
-# Logs a given environment variable to the screen
-log_env() {
-  log_info "${1}=${!1}"
-}
-
-
-# Checks a given environment variable is set (non-zero size)
-check_env() {
-  check_defined "${1}"
-  log_env "${1}"
-}
-
-
-# Checks a given environment variable array is set (non-zero size)
-# Then prints all of its components
-check_array_env() {
-  check_defined "${1}"
-  eval array=\( \${${1}[@]} \)
-  for i in "${!array[@]}"; do
-    log_info "${1}[${i}]=${array[${i}]}";
-  done
-}
-
-
-# Exports a given environment variable, verbosely
-export_env() {
-  check_defined "${1}"
-  export ${1}
-  log_info "export ${1}=${!1}"
-}
-
-
 # Function for running command and echoing results
 run_cmd() {
   log_info "$ ${@}"
@@ -81,15 +31,6 @@ run_cmd() {
 }
 
 
-if [ -z "${BOB_PACKAGE_VERSION}" ]; then
-  if [ ! -r "version.txt" ]; then
-    log_error "./version.txt does not exist - cannot figure out version number"
-    exit 1
-  fi
-  BOB_PACKAGE_VERSION=`cat version.txt | tr -d '\n'`;
-fi
-
-
 # merges conda cache folders
 # $1: Path to the main cache to keep. The directory must exist.
 # $2: Path to the extra cache to be merged into main cache
@@ -106,22 +47,30 @@ merge_conda_cache() {
 }
 
 
+# Checks just if the variable is defined and has non-zero length
+check_defined() {
+  if [ -z "${!1+abc}" ]; then
+    log_error "Variable ${1} is undefined - aborting...";
+    exit 1
+  elif [ -z "${!1}" ]; then
+    log_error "Variable ${1} is zero-length - aborting...";
+    exit 1
+  fi
+}
+
+
 # installs a miniconda installation.
 # $1: Path to where to install miniconda.
 install_miniconda() {
   log_info "Installing miniconda in ${1} ..."
 
-  # downloads the latest conda installation script
-  if [ "${OSNAME}" == "linux" ]; then
-    object=https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
-  else
-    object=https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
-  fi
-
   # checks if miniconda.sh exists
   if [ ! -e miniconda.sh ]; then
     log_info "Downloading latest miniconda3 installer..."
-    run_cmd curl --silent --output miniconda.sh ${object}
+    # downloads the latest conda installation script
+    if [ "$(uname -s)" == "Linux" ]; then _os="Linux" else _os="MacOSX"; fi
+    obj=https://repo.continuum.io/miniconda/Miniconda3-latest-${_os}-x86_64.sh
+    run_cmd curl --silent --output miniconda.sh ${obj}
   else
     log_info "Re-using cached miniconda3 installer..."
     ls -l miniconda.sh
@@ -147,83 +96,36 @@ install_miniconda() {
   hash -r
 }
 
+check_defined CONDA_ROOT
+check_defined CI_PROJECT_DIR
 
-# Creates a sane base condarc file
-# $1: Path to where to populate the condarc file
-create_condarc() {
-  log_info "Populating the basic condarc file in ${1} ..."
-
-  cat <<EOF > ${1}
-default_channels: #!final
-  - https://repo.anaconda.com/pkgs/main
-  - https://repo.anaconda.com/pkgs/free
-  - https://repo.anaconda.com/pkgs/r
-  - https://repo.anaconda.com/pkgs/pro
-add_pip_as_python_dependency: false #!final
-changeps1: false #!final
-always_yes: true #!final
-quiet: true #!final
-show_channel_urls: true #!final
-anaconda_upload: false #!final
-ssl_verify: false #!final
-channels: #!final
-EOF
-
-}
-
-
-# sets CONDA_CHANNELS to the list of conda channels that should be considered
-# $2: typically, the value of ${CI_COMMIT_TAG} or empty
-# given the current visibility/tagging conditions of the package.
-set_conda_channels() {
-  CONDA_CHANNELS=() #resets bash array
-  if [ -z "${1}" ]; then #public beta build
-    CONDA_CHANNELS+=('public/conda/label/beta')
-    CONDA_CHANNELS+=('public/conda')
-  else #public (tagged) build
-    CONDA_CHANNELS+=('public/conda')
-  fi
-  check_array_env CONDA_CHANNELS
-}
-
-
-log_env PYTHON_VERSION
-check_env CI_PROJECT_DIR
-check_env CI_PROJECT_NAME
-check_env CI_COMMIT_REF_NAME
-export_env BOB_PACKAGE_VERSION
-
-# Sets up variables
-OSNAME=`osname`
-check_env OSNAME
-
-if [ -z "${OS_SLUG}" ]; then
-  OS_SLUG="${OSNAME}-64"
+# checks if a conda installation exists. Otherwise, installs one
+if [ ! -e ${CONDA_ROOT}/bin/conda ]; then
+  install_miniconda ${CONDA_ROOT}
 fi
-export_env OS_SLUG
 
-DOCSERVER=http://www.idiap.ch
+run_cmd mkdir -p ${CONDA_ROOT}/pkgs
+run_cmd touch ${CONDA_ROOT}/pkgs/urls
+run_cmd touch ${CONDA_ROOT}/pkgs/urls.txt
 
-# Sets up the location of our rc file for conda
-CONDARC=${CONDA_ROOT}/condarc
+cp -fv ${CI_PROJECT_DIR}/bob/devtools/data/base-condarc ${CONDARC}
+echo "channels:" >> ${CONDARC}
+echo "  - http://www.idiap.ch/public/conda" >> ${CONDARC}
+echo "  - defaults" >> ${CONDARC}
 
-export_env DOCSERVER
-check_env CONDA_ROOT
-export_env CONDARC
+# displays contents of our configuration
+echo "Contents of \`${CONDARC}':"
+cat ${CONDARC}
 
-# Sets up certificates for curl and openssl
-CURL_CA_BUNDLE="${CI_PROJECT_DIR}/bob/devtools/data/cacert.pem"
-export_env CURL_CA_BUNDLE
-SSL_CERT_FILE="${CURL_CA_BUNDLE}"
-export_env SSL_CERT_FILE
-GIT_SSL_CAINFO="${CURL_CA_BUNDLE}"
-export_env GIT_SSL_CAINFO
+# updates conda installation, installs just built bob.devtools
+if [ "${1}" == "self" ]; then
+  run_cmd ${CONDA_ROOT}/bin/conda create -n bdt ${CONDA_ROOT}/conda-bld/${OS_SLUG}/bob.devtools-*.tar.bz2
+else
+  run_cmd ${CONDA_ROOT}/bin/conda install -n base python conda=4 conda-build=3
+fi
 
-# Sets up upload folders for documentation (just in case we need them)
-# See: https://gitlab.idiap.ch/bob/bob.admin/issues/2
+# cleans up
+run_cmd ${CONDA_ROOT}/bin/conda clean --lock
 
-# Sets up the language so Unicode filenames are considered correctly
-LANG="en_US.UTF-8"
-LC_ALL="${LANG}"
-export_env LANG
-export_env LC_ALL
+# print conda information for debugging purposes
+run_cmd ${CONDA_ROOT}/bin/conda info

_ci/build.sh

 #!/usr/bin/env bash
 
-source $(dirname ${0})/functions.sh
+# datetime prefix for logging
+log_datetime() {
+	echo "($(date +%T.%3N))"
+}
+
+# Functions for coloring echo commands
+log_info() {
+  echo -e "$(log_datetime) \033[1;34m${@}\033[0m"
+}
+
+
+log_error() {
+  echo -e "$(log_datetime) \033[1;31mError: ${@}\033[0m" >&2
+}
+
+
+# Checks just if the variable is defined and has non-zero length
+check_defined() {
+  if [ -z "${!1+abc}" ]; then
+    log_error "Variable ${1} is undefined - aborting...";
+    exit 1
+  elif [ -z "${!1}" ]; then
+    log_error "Variable ${1} is zero-length - aborting...";
+    exit 1
+  fi
+}
+
+
+# Exports a given environment variable, verbosely
+export_env() {
+  check_defined "${1}"
+  export ${1}
+  log_info "export ${1}=${!1}"
+}
+
+check_defined CONDA_ROOT
+check_defined CI_PROJECT_DIR
+check_defined CI_PROJECT_NAME
+check_defined CI_COMMIT_TAG
+check_defined PYTHON_VERSION
+
+BOB_PACKAGE_VERSION=`cat version.txt | tr -d '\n'`;
+check_defined BOB_PACKAGE_VERSION
 
 # Makes sure we activate the base environment if available
 run_cmd source ${CONDA_ROOT}/etc/profile.d/conda.sh
 run_cmd conda activate base
 export_env PATH
 
-set_conda_channels ${CI_COMMIT_TAG}
-log_info "$ ${CONDA_ROOT}/bin/python ${SCRIPTS_DIR}/nextbuild.py ${DOCSERVER}/${CONDA_CHANNELS[0]} ${CI_PROJECT_NAME} ${BOB_PACKAGE_VERSION} ${PYTHON_VERSION}"
-BOB_BUILD_NUMBER=$(${CONDA_ROOT}/bin/python ${SCRIPTS_DIR}/nextbuild.py ${DOCSERVER}/${CONDA_CHANNELS[0]} ${CI_PROJECT_NAME} ${BOB_PACKAGE_VERSION} ${PYTHON_VERSION})
+if [ -z "${CI_COMMIT_TAG}" ]; then #building beta
+  channel="http://www.idiap.ch/public/conda/label/beta"
+else
+  channel="http://www.idiap.ch/public/conda"
+fi
+
+log_info "$ ${CONDA_ROOT}/bin/python ${CI_PROJECT_DIR}/_ci/nextbuild.py ${channel} ${CI_PROJECT_NAME} ${BOB_PACKAGE_VERSION} ${PYTHON_VERSION}"
+BOB_BUILD_NUMBER=$(${CONDA_ROOT}/bin/python ${CI_PROJECT_DIR}/_ci/nextbuild.py ${channel} ${CI_PROJECT_NAME} ${BOB_PACKAGE_VERSION} ${PYTHON_VERSION})
 export_env BOB_BUILD_NUMBER
 
 # copy the recipe_append.yaml over before build
 run_cmd cp ${CI_PROJECT_DIR}/bob/devtools/data/recipe_append.yaml conda/
 run_cmd cp ${CI_PROJECT_DIR}/bob/devtools/data/conda_build_config.yaml conda/
 
-BLDOPT="--python=${PYTHON_VERSION} --no-anaconda-upload"
+run_cmd ${CONDA_ROOT}/bin/conda build "--python=${PYTHON_VERSION} --no-anaconda-upload" conda
 
-run_cmd ${CONDA_ROOT}/bin/conda build ${BLDOPT} conda
+# run git clean to clean everything that is not needed. This helps to keep the
+# disk usage on CI machines to minimum.
+if [ "$(uname -s)" == "Linux" ]; then _os="linux" else _os="osx"; fi
+run_cmd git clean -ffdx \
+    --exclude="miniconda.sh" \
+    --exclude="miniconda/pkgs/*.tar.bz2" \
+    --exclude="miniconda/pkgs/urls.txt" \
+    --exclude="miniconda/conda-bld/${_os}-64/*.tar.bz2" \
+    --exclude="_ci" \
+    --exclude="dist/*.zip" \
+    --exclude="sphinx"

_ci/deploy.sh

+#!/usr/bin/env bash
+# Wed Jan  9 16:48:57 CET 2019
+
+source $(dirname ${0})/functions.sh
+
+run_cmd cp -fv ${CI_PROJECT_DIR}/bob/devtools/data/base-condarc ${CONDARC}
+echo "channels:" >> ${CONDARC}
+echo "  - ${DOCSERVER}/public/conda" >> ${CONDARC}
+echo "  - defaults" >> ${CONDARC}
+
+deploy_conda_packages ${CONDA_CHANNELS[0]} ${CI_PROJECT_NAME}
+
+# upload the docs from the sphinx folder (usually an artifact of Linux Python
+# 3.6 builds)
+for folder in "${DOC_UPLOADS[@]}"; do
+  dav_upload_folder sphinx "${folder}"
+done

bob/devtools/data/base-condarc

+default_channels:
+  - https://repo.anaconda.com/pkgs/main
+  - https://repo.anaconda.com/pkgs/free
+  - https://repo.anaconda.com/pkgs/r
+  - https://repo.anaconda.com/pkgs/pro
+add_pip_as_python_dependency: false #!final
+changeps1: false #!final
+always_yes: true #!final
+quiet: true #!final
+show_channel_urls: true #!final
+anaconda_upload: false #!final
+ssl_verify: false #!final

conda/meta.yaml

@@ -33,7 +33,8 @@ requirements:
     - setuptools
     - click
     - click-plugins
-    - conda-build
+    - conda=4
+    - conda-build=3
     - pytz
     - python-dateutil
     - gitpython