[ci] Implement core build strategy for package

.gitlab-ci.yml

-# This build file uses template features from YAML so it is generic enough for
-# any Bob project. Don't modify it unless you know what you're doing.
-
 # Definition of global variables (all stages)
 variables:
   CONDA_ROOT: "${CI_PROJECT_DIR}/miniconda"
@@ -9,18 +6,12 @@ variables:
 # Definition of our build pipeline order
 stages:
   - build
-  - deploy
-  - pypi
 
 
 # Build targets
 .build_template: &build_job
   stage: build
   before_script:
-    - mkdir _ci
-    - curl --silent "https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/install.sh" > _ci/install.sh
-    - chmod 755 _ci/install.sh
-    - ./_ci/install.sh _ci master #installs ci support scripts
     - ./_ci/before_build.sh
   script:
     - ./_ci/build.sh
@@ -41,7 +32,6 @@ stages:
   artifacts:
     expire_in: 1 week
     paths:
-      - _ci/
       - ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
   cache:
     <<: *build_caches
@@ -55,7 +45,6 @@ stages:
   artifacts:
     expire_in: 1 week
     paths:
-      - _ci/
       - ${CONDA_ROOT}/conda-bld/osx-64/*.tar.bz2
   cache:
     <<: *build_caches
@@ -70,7 +59,6 @@ build_linux_36:
   artifacts:
     expire_in: 1 week
     paths:
-      - _ci/
       - dist/*.zip
       - sphinx
       - ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
@@ -80,50 +68,3 @@ build_macosx_36:
   <<: *macosx_build_job
   variables:
     PYTHON_VERSION: "3.6"
-
-
-# Deploy targets
-.deploy_template: &deploy_job
-  stage: deploy
-  before_script:
-    - ./_ci/install.sh _ci master #updates ci support scripts
-  script:
-    - ./_ci/deploy.sh
-  dependencies:
-    - build_linux_36
-    - build_macosx_36
-  tags:
-    - deployer
-
-
-deploy_beta:
-  <<: *deploy_job
-  environment: beta
-  only:
-    - master
-
-
-deploy_stable:
-  <<: *deploy_job
-  environment: stable
-  only:
-    - /^v\d+\.\d+\.\d+([abc]\d*)?$/  # PEP-440 compliant version (tags)
-  except:
-    - branches
-
-
-pypi:
-  stage: pypi
-  environment: pypi
-  only:
-    - /^v\d+\.\d+\.\d+([abc]\d*)?$/  # PEP-440 compliant version (tags)
-  except:
-    - branches
-  before_script:
-    - ./_ci/install.sh _ci master #updates ci support scripts
-  script:
-    - ./_ci/pypi.sh
-  dependencies:
-    - build_linux_36
-  tags:
-    - deployer

_ci/after_build.sh

+#!/usr/bin/env bash
+
+source $(dirname ${0})/functions.sh
+
+# delete the bob packages from the cache otherwise the cache keeps increasing
+# over and over. See https://gitlab.idiap.ch/bob/bob.admin/issues/65
+run_cmd rm -rf ${CONDA_ROOT}/pkgs/batl*.tar.bz2
+run_cmd rm -rf ${CONDA_ROOT}/pkgs/beat*.tar.bz2
+run_cmd rm -rf ${CONDA_ROOT}/pkgs/bob*.tar.bz2
+run_cmd rm -rf ${CONDA_ROOT}/pkgs/gridtk*.tar.bz2
+
+# run git clean to clean everything that is not needed. This helps to keep the
+# disk usage on CI machines to minimum.
+run_cmd git clean -ffdx \
+    --exclude="miniconda.sh" \
+    --exclude="miniconda/pkgs/*.tar.bz2" \
+    --exclude="miniconda/pkgs/urls.txt" \
+    --exclude="miniconda/conda-bld/linux-64/*.tar.bz2" \
+    --exclude="miniconda/conda-bld/osx-64/*.tar.bz2" \
+    --exclude="_ci" \
+    --exclude="dist/*.zip" \
+    --exclude="sphinx"

_ci/before_build.sh

+#!/usr/bin/env bash
+# Tue 16 Jan 11:15:32 2018 CET
+
+source $(dirname ${0})/functions.sh
+
+# checks if a conda installation exists. Otherwise, install one
+if [ ! -e ${CONDA_ROOT}/bin/conda ]; then
+  install_miniconda ${CONDA_ROOT}
+fi
+
+mkdir -p ${CONDA_ROOT}/pkgs
+touch ${CONDA_ROOT}/pkgs/urls
+touch ${CONDA_ROOT}/pkgs/urls.txt
+
+create_condarc ${CONDARC}
+
+set_conda_channels ${CI_COMMIT_TAG}
+for k in "${CONDA_CHANNELS[@]}"; do
+  echo "  - ${DOCSERVER}/${k}" >> ${CONDARC}
+done
+echo "  - defaults" >> ${CONDARC}
+
+# displays contents of our configuration
+echo "Contents of \`${CONDARC}':"
+cat ${CONDARC}
+
+# updates conda installation
+run_cmd ${CONDA_ROOT}/bin/conda install python conda=4 curl conda-build=3
+
+# cleans up
+run_cmd ${CONDA_ROOT}/bin/conda clean --lock
+
+# print conda information for debugging purposes
+run_cmd ${CONDA_ROOT}/bin/conda info

_ci/build.sh

+#!/usr/bin/env bash
+
+source $(dirname ${0})/functions.sh
+
+# Makes sure we activate the base environment if available
+run_cmd source ${CONDA_ROOT}/etc/profile.d/conda.sh
+run_cmd conda activate base
+export_env PATH
+
+set_conda_channels ${CI_COMMIT_TAG}
+log_info "$ ${CONDA_ROOT}/bin/python ${SCRIPTS_DIR}/nextbuild.py ${DOCSERVER}/${1} ${CI_PROJECT_NAME} ${BOB_PACKAGE_VERSION} ${PYTHON_VERSION}"
+BOB_BUILD_NUMBER=$(${CONDA_ROOT}/bin/python ${SCRIPTS_DIR}/nextbuild.py ${DOCSERVER}/${1} ${CI_PROJECT_NAME} ${BOB_PACKAGE_VERSION} ${PYTHON_VERSION})
+export_env BOB_BUILD_NUMBER
+
+# copy the recipe_append.yaml over before build
+run_cmd cp ${CI_PROJECT_DIR}/bob/devtools/data/recipe_append.yaml conda/
+run_cmd cp ${CI_PROJECT_DIR}/bob/devtools/data/conda_build_config.yaml conda/
+
+BLDOPT="--python=${PYTHON_VERSION} --no-anaconda-upload"
+
+run_cmd ${CONDA_ROOT}/bin/conda build ${BLDOPT} conda

_ci/functions.sh

+#!/usr/bin/env bash
+# Wed  9 Jan 2019 14:33:20 CET
+
+# Build utilities
+SCRIPTS_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+
+# datetime prefix for logging
+log_datetime() {
+	echo "($(date +%T.%3N))"
+}
+
+# Functions for coloring echo commands
+log_info() {
+  echo -e "$(log_datetime) \033[1;34m${@}\033[0m"
+}
+
+
+log_error() {
+  echo -e "$(log_datetime) \033[1;31mError: ${@}\033[0m" >&2
+}
+
+
+# Checks just if the variable is defined and has non-zero length
+check_defined() {
+  if [ -z "${!1+abc}" ]; then
+    log_error "Variable ${1} is undefined - aborting...";
+    exit 1
+  elif [ -z "${!1}" ]; then
+    log_error "Variable ${1} is zero-length - aborting...";
+    exit 1
+  fi
+}
+
+
+# Logs a given environment variable to the screen
+log_env() {
+  log_info "${1}=${!1}"
+}
+
+
+# Checks a given environment variable is set (non-zero size)
+check_env() {
+  check_defined "${1}"
+  log_env "${1}"
+}
+
+
+# Checks a given environment variable array is set (non-zero size)
+# Then prints all of its components
+check_array_env() {
+  check_defined "${1}"
+  eval array=\( \${${1}[@]} \)
+  for i in "${!array[@]}"; do
+    log_info "${1}[${i}]=${array[${i}]}";
+  done
+}
+
+
+# Exports a given environment variable, verbosely
+export_env() {
+  check_defined "${1}"
+  export ${1}
+  log_info "export ${1}=${!1}"
+}
+
+
+# Function for running command and echoing results
+run_cmd() {
+  log_info "$ ${@}"
+  ${@}
+  local status=$?
+  if [ ${status} != 0 ]; then
+    log_error "Command Failed \"${@}\""
+    exit ${status}
+  fi
+}
+
+
+if [ -z "${BOB_PACKAGE_VERSION}" ]; then
+  if [ ! -r "version.txt" ]; then
+    log_error "./version.txt does not exist - cannot figure out version number"
+    exit 1
+  fi
+  BOB_PACKAGE_VERSION=`cat version.txt | tr -d '\n'`;
+fi
+
+
+# merges conda cache folders
+# $1: Path to the main cache to keep. The directory must exist.
+# $2: Path to the extra cache to be merged into main cache
+merge_conda_cache() {
+  if [ -e ${1} ]; then
+    _cached_urlstxt="${2}/urls.txt"
+    _urlstxt="${1}/urls.txt"
+    if [ -e ${2} ]; then
+      log_info "Merging urls.txt and packages with cached files..."
+      mv ${2}/*.tar.bz2 ${1}/
+      cat ${_urlstxt} ${_cached_urlstxt} | sort | uniq > ${_urlstxt}
+    fi
+  fi
+}
+
+
+# installs a miniconda installation.
+# $1: Path to where to install miniconda.
+install_miniconda() {
+  log_info "Installing miniconda in ${1} ..."
+
+  # downloads the latest conda installation script
+  if [ "${OSNAME}" == "linux" ]; then
+    object=https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
+  else
+    object=https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
+  fi
+
+  # checks if miniconda.sh exists
+  if [ ! -e miniconda.sh ]; then
+    log_info "Downloading latest miniconda3 installer..."
+    run_cmd curl --silent --output miniconda.sh ${object}
+  else
+    log_info "Re-using cached miniconda3 installer..."
+    ls -l miniconda.sh
+  fi
+
+  # move cache to a different folder if it exists
+  if [ -e ${1} ]; then
+    run_cmd mv ${1} ${1}.cached
+  fi
+
+  # install miniconda
+  run_cmd bash miniconda.sh -b -p ${1}
+
+  # Put back cache and merge urls.txt
+  merge_conda_cache ${1}/pkgs ${1}.cached/pkgs
+  # remove the backup cache folder
+  rm -rf ${1}.cached
+
+  # List currently available packages on cache
+  # run_cmd ls -l ${1}/pkgs/
+  # run_cmd cat ${1}/pkgs/urls.txt
+
+  hash -r
+}
+
+
+# Creates a sane base condarc file
+# $1: Path to where to populate the condarc file
+create_condarc() {
+  log_info "Populating the basic condarc file in ${1} ..."
+
+  cat <<EOF > ${1}
+default_channels: #!final
+  - https://repo.anaconda.com/pkgs/main
+  - https://repo.anaconda.com/pkgs/free
+  - https://repo.anaconda.com/pkgs/r
+  - https://repo.anaconda.com/pkgs/pro
+add_pip_as_python_dependency: false #!final
+changeps1: false #!final
+always_yes: true #!final
+quiet: true #!final
+show_channel_urls: true #!final
+anaconda_upload: false #!final
+ssl_verify: false #!final
+channels: #!final
+EOF
+
+}
+
+
+# sets CONDA_CHANNELS to the list of conda channels that should be considered
+# $2: typically, the value of ${CI_COMMIT_TAG} or empty
+# given the current visibility/tagging conditions of the package.
+set_conda_channels() {
+  CONDA_CHANNELS=() #resets bash array
+  if [ -z "${1}" ]; then #public beta build
+    CONDA_CHANNELS+=('public/conda/label/beta')
+    CONDA_CHANNELS+=('public/conda')
+  else #public (tagged) build
+    CONDA_CHANNELS+=('public/conda')
+  fi
+  check_array_env CONDA_CHANNELS
+}
+
+
+log_env PYTHON_VERSION
+check_env CI_PROJECT_DIR
+check_env CI_PROJECT_NAME
+check_env CI_COMMIT_REF_NAME
+export_env BOB_PACKAGE_VERSION
+
+# Sets up variables
+OSNAME=`osname`
+check_env OSNAME
+
+DOCSERVER=http://www.idiap.ch
+
+# Sets up the location of our rc file for conda
+CONDARC=${CONDA_ROOT}/condarc
+
+if [ -z "${OS_SLUG}" ]; then
+  OS_SLUG="${OSNAME}-64"
+fi
+
+export_env OS_SLUG
+export_env DOCSERVER
+check_env CONDA_ROOT
+export_env CONDARC
+
+# Sets up certificates for curl and openssl
+CURL_CA_BUNDLE="${CI_PROJECT_DIR}/bob/devtools/data/cacert.pem"
+export_env CURL_CA_BUNDLE
+SSL_CERT_FILE="${CURL_CA_BUNDLE}"
+export_env SSL_CERT_FILE
+GIT_SSL_CAINFO="${CURL_CA_BUNDLE}"
+export_env GIT_SSL_CAINFO
+
+# Sets up upload folders for documentation (just in case we need them)
+# See: https://gitlab.idiap.ch/bob/bob.admin/issues/2
+
+# Sets up the language so Unicode filenames are considered correctly
+LANG="en_US.UTF-8"
+LC_ALL="${LANG}"
+export_env LANG
+export_env LC_ALL

_ci/nextbuild.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+'''This is a copy of bob/devtools/conda:next_build_number with a CLI'''
+
+import os
+import re
+import sys
+
+from conda.exports import get_index
+
+
+if __name__ == '__main__':
+
+  channel_url = sys.argv[1]
+  name = sys.argv[2]
+  version = sys.argv[3]
+  python = sys.argv[4]
+
+  # no dot in py_ver
+  py_ver = python.replace('.', '')
+
+  # get the channel index
+  index = get_index(channel_urls=[channel_url], prepend=False)
+
+  # search if package with the same version exists
+  build_number = 0
+  urls = []
+  for dist in index:
+
+    if dist.name == name and dist.version == version:
+      match = re.match('py[2-9][0-9]+', dist.build_string)
+
+      if match and match.group() == 'py{}'.format(py_ver):
+        build_number = max(build_number, dist.build_number + 1)
+        urls.append(index[dist].url)
+
+  urls = [url.replace(channel_url, '') for url in urls]
+
+  print(build_number)

conda/conda_build_config.yaml

-../bob/devtools/data/conda_build_config.yaml
\ No newline at end of file