diff --git a/gitlab/build.sh b/gitlab/build.sh index 3a6ea46696753fb482505b7c40d7cc8b1da007a0..52cdb191dce1412ffc97f4d3c2dc3f2fcbdf3041 100755 --- a/gitlab/build.sh +++ b/gitlab/build.sh @@ -4,7 +4,7 @@ source $(dirname ${0})/functions.sh run_cmd ${CONDA_FOLDER}/bin/conda install -n root --yes --quiet conda=4 conda-build=3 -run_cmd ${CONDA_FOLDER}/bin/conda config --set always_yes true +run_cmd ${CONDA_FOLDER}/bin/conda config --set always_yes yes --set changeps1 no run_cmd ${CONDA_FOLDER}/bin/conda config --set show_channel_urls true run_cmd ${CONDA_FOLDER}/bin/conda clean --lock run_cmd cp _ci/conda_build_config.yaml conda/ diff --git a/gitlab/deploy.sh b/gitlab/deploy.sh index 039b2bcb68bad88c685d0d4f88d4062ab7185c89..a69da38c54f9fb5f44029df24b431147b6071c99 100755 --- a/gitlab/deploy.sh +++ b/gitlab/deploy.sh @@ -5,7 +5,7 @@ source $(dirname ${0})/functions.sh # Uploads all the built packages for os in "osx-64" "noarch" "linux-64"; do - for f in conda-env/${os}/*.tar.bz2; do + for f in ${CONDA_BLD_PATH}/${os}/*.tar.bz2; do if [[ -f $f ]]; then if [ -z "${CI_COMMIT_TAG}" ]; then #beta url="private" @@ -17,6 +17,7 @@ for os in "osx-64" "noarch" "linux-64"; do done done +# If a package does not build for Linux, $f does not exist. # Uploads docs for the last treated package run_cmd tar xvfj "${f}" docs/${CI_PROJECT_NAME} diff --git a/gitlab/functions.sh b/gitlab/functions.sh index 155d887836b105e22c201e2f3adb5afcae7a0a51..6ab06be69f310db506faffdf23ca37b0588551af 100644 --- a/gitlab/functions.sh +++ b/gitlab/functions.sh @@ -263,6 +263,44 @@ if [ -z "${BOB_PACKAGE_VERSION}" ]; then BOB_PACKAGE_VERSION=`cat version.txt | tr -d '\n'`; fi + +# installs a miniconda installation. +# $1: Path to where to install miniconda. The path should not exist. +install_miniconda() { + log_info "Installing miniconda in ${1} ..." + # Download the latest conda installation script + if [ "${OSNAME}" == "macosx" ]; then + object=https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh + else + object=https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh + fi + # check if miniconda.sh exists + if [ ! -e miniconda.sh ]; then + log_info "Downloading latest miniconda3 installer..." + run_cmd curl --silent --output miniconda.sh ${object} + else + log_info "Re-using cached miniconda3 installer..." + ls -l miniconda.sh + fi + # Check if cache exists and save urls.txt to create a valid cache after conda + # installation override + _urls="${1}/pkgs/urls.txt" + if [ -e ${_urls} ]; then + run_cmd cp ${_urls} ${_urls}.cached + fi + bash miniconda.sh -b -p ${1} + # Reset urls.txt + if [ -e ${_urls}.cached ]; then + log_info "Merging urls.txt files with cached values..." + cat ${_urls} ${_urls}.cached | sort | uniq > ${_urls} + fi + # List currently available packages on cache + run_cmd ls -l ${1}/pkgs/ + run_cmd cat ${1}/pkgs/urls.txt + hash -r +} + + check_env PYTHON_VERSION check_env CI_PROJECT_URL check_env CI_PROJECT_DIR @@ -284,6 +322,11 @@ if [ -z "${CONDA_FOLDER}" ]; then CONDA_FOLDER=/opt/miniconda fi +# check if a conda installation exists. Otherwise, install one: +if [ ! -e $1 ]; then + install_miniconda ${CONDA_FOLDER} +fi + PYVER=py$(echo ${PYTHON_VERSION} | tr -d '.') if [ -z "${DOCSERVER}" ]; then diff --git a/templates/gitlab-ci.yml b/templates/gitlab-ci.yml index 26662a1bb5b96c1cfb92de1302bedaf565451748..9fdc0cc9a0e2686b18483f2687808a727e3d6459 100644 --- a/templates/gitlab-ci.yml +++ b/templates/gitlab-ci.yml @@ -1,6 +1,11 @@ # This build file uses template features from YAML so it is generic enough for # any Bob project. Don't modify it unless you know what you're doing. +# global variables +variables: + CONDA_ENVS_PATH: "conda-env" + CONDA_BLD_PATH: "conda-env" + # Definition of our build pipeline order stages: @@ -12,7 +17,13 @@ stages: # Build targets .build_template: &build_job stage: build + script: + - ./_ci/build.sh + +.build_linux_template: &linux_build_job + <<: *build_job before_script: + - export CONDA_FOLDER=/opt/miniconda - export PATH=/opt/miniconda/bin:$PATH - mkdir _ci - curl https://curl.haxx.se/ca/cacert.pem > _ci/cacert.pem @@ -21,34 +32,44 @@ stages: - curl "https://gitlab.idiap.ch/bob/bob.admin/raw/condapackage/gitlab/install.sh" > _ci/install.sh - chmod 755 _ci/install.sh - ./_ci/install.sh _ci condapackage - script: - - ./_ci/build.sh - -.build_linux_template: &linux_build_job - <<: *build_job artifacts: expire_in: 1 week paths: - _ci/ - - conda-env/linux-64/*.tar.bz2 + - ${CONDA_ENVS_PATH}/linux-64/*.tar.bz2 tags: - docker image: continuumio/conda_builder_linux cache: key: "$CI_JOB_NAME" paths: - - conda-env/.pkgs/*.tar.bz2 - - conda-env/.pkgs/urls.txt + - ${CONDA_ENVS_PATH}/.pkgs/*.tar.bz2 + - ${CONDA_ENVS_PATH}/.pkgs/urls.txt .build_macosx_template: &macosx_build_job <<: *build_job + before_script: + - export CONDA_FOLDER=$CI_PROJECT_DIR/${CONDA_ENVS_PATH} + - export PATH=$CI_PROJECT_DIR/${CONDA_ENVS_PATH}/bin:$PATH + - mkdir _ci + - curl https://curl.haxx.se/ca/cacert.pem > _ci/cacert.pem + - export CURL_CA_BUNDLE=`pwd`/_ci/cacert.pem + - export SSL_CERT_FILE=`pwd`/_ci/cacert.pem + - curl "https://gitlab.idiap.ch/bob/bob.admin/raw/condapackage/gitlab/install.sh" > _ci/install.sh + - chmod 755 _ci/install.sh + - ./_ci/install.sh _ci condapackage artifacts: expire_in: 1 week paths: - _ci/ - - conda-env/osx-64/*.tar.bz2 + - ${CONDA_ENVS_PATH}/osx-64/*.tar.bz2 tags: - conda-macosx + cache: + key: "$CI_JOB_NAME" + paths: + - miniconda.sh + - ${CONDA_ENVS_PATH}/pkgs/ build_linux_27: @@ -71,7 +92,7 @@ build_linux_36: paths: - _ci/ - dist/ - - conda-env/linux-64/*.tar.bz2 + - ${CONDA_ENVS_PATH}/linux-64/*.tar.bz2 build_macosx_27: <<: *macosx_build_job