Try to organize build scripts a bit more cleanly

gitlab/before_build.sh

+#!/usr/bin/env bash
+# Tue 16 Jan 11:15:32 2018 CET
+
+source $(dirname ${0})/functions.sh
+
+# checks if a conda installation exists. Otherwise, install one
+if [ ! -e ${CONDA_FOLDER}/bin/conda ]; then
+  install_miniconda ${CONDA_FOLDER}
+fi
+
+# moves the cache to conda folder
+mkdir -p ${CONDA_FOLDER}/pkgs
+touch ${CONDA_FOLDER}/pkgs/urls
+touch ${CONDA_FOLDER}/pkgs/urls.txt
+
+merge_conda_cache ${CONDA_FOLDER}/pkgs ${CONDA_ENVS_PATH}/.pkgs
+
+cat <<EOF > ${CONDARC}
+default_channels:
+  - https://repo.continuum.io/pkgs/main
+  - https://repo.continuum.io/pkgs/free
+  - https://repo.continuum.io/pkgs/r
+  - https://repo.continuum.io/pkgs/pro
+EOF
+echo "Contents of \`${CONDARC}':"
+cat ${CONDARC}
+
+run_cmd ${CONDA_FOLDER}/bin/conda config --set add_pip_as_python_dependency false
+run_cmd ${CONDA_FOLDER}/bin/conda config --set always_yes yes --set changeps1 no
+run_cmd ${CONDA_FOLDER}/bin/conda config --set always_yes true
+run_cmd ${CONDA_FOLDER}/bin/conda config --set show_channel_urls true
+run_cmd ${CONDA_FOLDER}/bin/conda config --set anaconda_upload no
+run_cmd ${CONDA_FOLDER}/bin/conda config --add channels ${CHANNEL_URL}
+run_cmd ${CONDA_FOLDER}/bin/conda config --set ssl_verify false
+run_cmd ${CONDA_FOLDER}/bin/conda install --quiet --yes -c defaults python conda=4.4 curl conda-build=3
+
+# copies our central build configuration to the conda folder of the package
+run_cmd cp _ci/conda_build_config.yaml conda/
+
+# cleans up
+run_cmd ${CONDA_FOLDER}/bin/conda clean --lock
+
+# print conda information for debugging purposes
+run_cmd ${CONDA_FOLDER}/bin/conda info

gitlab/build.sh

@@ -3,22 +3,9 @@
 
 source $(dirname ${0})/functions.sh
 
-# Move the cache to conda folder
-if [ "${OSNAME}" == "linux" ]; then
-    mkdir -p ${CONDA_FOLDER}/pkgs
-    touch ${CONDA_FOLDER}/pkgs/urls
-    touch ${CONDA_FOLDER}/pkgs/urls.txt
-fi
-merge_conda_cache ${CONDA_FOLDER}/pkgs ${CONDA_ENVS_PATH}/.pkgs
-
-run_cmd ${CONDA_FOLDER}/bin/conda install --override-channels -c https://repo.continuum.io/pkgs/main -c https://repo.continuum.io/pkgs/free -c https://repo.continuum.io/pkgs/r -c https://repo.continuum.io/pkgs/pro -n root --yes --quiet conda=4.4 conda-build=3
-run_cmd ${CONDA_FOLDER}/bin/conda config --set always_yes yes --set changeps1 no
-run_cmd ${CONDA_FOLDER}/bin/conda config --set show_channel_urls true
-run_cmd ${CONDA_FOLDER}/bin/conda config --set add_pip_as_python_dependency False
-run_cmd ${CONDA_FOLDER}/bin/conda config --set anaconda_upload no
-run_cmd ${CONDA_FOLDER}/bin/conda clean --lock
-run_cmd cp _ci/conda_build_config.yaml conda/
-run_cmd ${CONDA_FOLDER}/bin/conda info
+# Makes sure we activate the base environment if available
+run_cmd source ${CONDA_FOLDER}/etc/profile.d/conda.sh
+run_cmd conda activate base
 
 run_cmd mkdir -p ./_ci/${OS_SLUG}/${PYTHON_VERSION}
 

gitlab/functions.sh

@@ -288,14 +288,14 @@ merge_conda_cache() {
 install_miniconda() {
   log_info "Installing miniconda in ${1} ..."
 
-  # Download the latest conda installation script
+  # downloads the latest conda installation script
   if [ "${OSNAME}" == "linux" ]; then
     object=https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
   else
     object=https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
   fi
 
-  # check if miniconda.sh exists
+  # checks if miniconda.sh exists
   if [ ! -e miniconda.sh ]; then
     log_info "Downloading latest miniconda3 installer..."
     run_cmd curl --silent --output miniconda.sh ${object}
@@ -312,7 +312,7 @@ install_miniconda() {
   # install miniconda
   bash miniconda.sh -b -p ${1}
 
-  # Put back cache and merge urls.txt
+  # Put back cache and merge urls.txt (meaningful only on macosx)
   merge_conda_cache ${1}/pkgs ${1}.cached/pkgs
   # remove the backup cache folder
   rm -rf ${1}.cached
@@ -350,14 +350,6 @@ if [ -z "${CONDA_FOLDER}" ]; then
   CONDA_FOLDER=/opt/miniconda
 fi
 
-# check if a conda installation exists. Otherwise, install one (only in build
-# stage):
-if [ ! -e ${CONDA_FOLDER}/bin/conda ]; then
-  if [ "${CI_JOB_STAGE}" == "build" ]; then
-    install_miniconda ${CONDA_FOLDER}
-  fi
-fi
-
 PYVER=py$(echo ${PYTHON_VERSION} | tr -d '.')
 
 if [ -z "${DOCSERVER}" ]; then
@@ -447,3 +439,7 @@ LANG="en_US.UTF-8"
 LC_ALL="${LANG}"
 export_env LANG
 export_env LC_ALL
+
+# Sets up the location of our rc file for conda
+CONDARC=${CONDA_FOLDER}/condarc
+export_env CONDARC

gitlab/install.sh

@@ -59,8 +59,9 @@ get_exec() {
 run_cmd mkdir -pv ${1}
 get_script ${1} cacert.pem
 get_script ${1} functions.sh
+get_script ${1} conda_build_config.yaml
+get_exec ${1} before_build.sh
 for stage in "build" "deploy" "pypi"; do
   get_exec ${1} ${stage}.sh
 done
 get_exec ${1} channel_support.py
-get_exec ${1} conda_build_config.yaml

templates/gitlab-ci.yml

@@ -22,7 +22,10 @@ stages:
     - mkdir _ci
     - curl --silent "https://gitlab.idiap.ch/bob/bob.admin/raw/condapackage/gitlab/install.sh" > _ci/install.sh
     - chmod 755 _ci/install.sh
-    - ./_ci/install.sh _ci condapackage #updates installation
+    - ./_ci/install.sh _ci condapackage #installs ci support scripts
+    - ./_ci/before_build.sh
+  variables: &build_variables
+    CONDA_FOLDER: "${CI_PROJECT_DIR}/${CONDA_ENVS_PATH}"
   script:
     - ./_ci/build.sh
 
@@ -36,8 +39,6 @@ stages:
   tags:
     - docker
   image: continuumio/conda-concourse-ci
-  variables: &linux_variables
-    CONDA_FOLDER: "/opt/miniconda"
   cache:
     key: "linux-cache"
     paths:
@@ -53,8 +54,6 @@ stages:
       - ${CONDA_ENVS_PATH}/osx-64/*.tar.bz2
   tags:
     - macosx
-  variables: &macosx_variables
-    CONDA_FOLDER: "${CI_PROJECT_DIR}/${CONDA_ENVS_PATH}"
   cache:
     key: "macosx-cache"
     paths:
@@ -86,13 +85,13 @@ build_linux_36:
 build_macosx_27:
   <<: *macosx_build_job
   variables:
-    <<: *macosx_variables
+    <<: *build_variables
     PYTHON_VERSION: "2.7"
 
 build_macosx_36:
   <<: *macosx_build_job
   variables:
-    <<: *macosx_variables
+    <<: *build_variables
     PYTHON_VERSION: "3.6"
 
 
@@ -100,7 +99,7 @@ build_macosx_36:
 .deploy_template: &deploy_job
   stage: deploy
   before_script:
-    - ./_ci/install.sh _ci condapackage #updates
+    - ./_ci/install.sh _ci condapackage #updates ci support scripts
   script:
     - ./_ci/deploy.sh
   dependencies:
@@ -135,7 +134,7 @@ pypi:
   except:
     - branches
   before_script:
-    - ./_ci/install.sh _ci condapackage #updates
+    - ./_ci/install.sh _ci condapackage #updates ci support scripts
   script:
     - ./_ci/pypi.sh
   dependencies: