Commit 48608ad6 authored by André Anjos's avatar André Anjos 💬

[ci] Largely simplified

parent 87d312b6
Pipeline #25778 failed with stages
in 27 seconds
......@@ -6,17 +6,16 @@ variables:
# Definition of our build pipeline order
stages:
- build
- deploy
# Build targets
.build_template: &build_job
stage: build
before_script:
- ./_ci/before_build.sh
- ./_ci/bootstrap.sh
script:
- ./_ci/build.sh
after_script:
- ./_ci/after_build.sh
cache: &build_caches
paths:
- miniconda.sh
......@@ -68,3 +67,38 @@ build_macosx_36:
<<: *macosx_build_job
variables:
PYTHON_VERSION: "3.6"
# Deploy targets
.deploy_template: &deploy_job
stage: deploy
before_script:
- ./_ci/bootstrap.sh self
script:
- ./_ci/deploy.sh
dependencies:
- build_linux_36
- build_macosx_36
tags:
- deployer
cache: &build_caches
paths:
- miniconda.sh
- ${CONDA_ROOT}/pkgs/*.tar.bz2
- ${CONDA_ROOT}/pkgs/urls.txt
deploy_beta:
<<: *deploy_job
environment: beta
only:
- master
deploy_stable:
<<: *deploy_job
environment: stable
only:
- /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags)
except:
- branches
#!/usr/bin/env bash
source $(dirname ${0})/functions.sh
# delete the bob packages from the cache otherwise the cache keeps increasing
# over and over. See https://gitlab.idiap.ch/bob/bob.admin/issues/65
run_cmd rm -rf ${CONDA_ROOT}/pkgs/bob*.tar.bz2
# run git clean to clean everything that is not needed. This helps to keep the
# disk usage on CI machines to minimum.
run_cmd git clean -ffdx \
--exclude="miniconda.sh" \
--exclude="miniconda/pkgs/*.tar.bz2" \
--exclude="miniconda/pkgs/urls.txt" \
--exclude="miniconda/conda-bld/${OS_SLUG}/*.tar.bz2" \
--exclude="_ci" \
--exclude="dist/*.zip" \
--exclude="sphinx"
#!/usr/bin/env bash
# Tue 16 Jan 11:15:32 2018 CET
source $(dirname ${0})/functions.sh
# checks if a conda installation exists. Otherwise, install one
if [ ! -e ${CONDA_ROOT}/bin/conda ]; then
install_miniconda ${CONDA_ROOT}
fi
mkdir -p ${CONDA_ROOT}/pkgs
touch ${CONDA_ROOT}/pkgs/urls
touch ${CONDA_ROOT}/pkgs/urls.txt
create_condarc ${CONDARC}
set_conda_channels ${CI_COMMIT_TAG}
for k in "${CONDA_CHANNELS[@]}"; do
echo " - ${DOCSERVER}/${k}" >> ${CONDARC}
done
echo " - defaults" >> ${CONDARC}
# displays contents of our configuration
echo "Contents of \`${CONDARC}':"
cat ${CONDARC}
# updates conda installation
run_cmd ${CONDA_ROOT}/bin/conda install python conda=4 conda-build=3
# cleans up
run_cmd ${CONDA_ROOT}/bin/conda clean --lock
# print conda information for debugging purposes
run_cmd ${CONDA_ROOT}/bin/conda info
#!/usr/bin/env bash
# Wed 9 Jan 2019 14:33:20 CET
#!/urs/bin/env bash
# Build utilities
SCRIPTS_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# Determines the operating system we're using
osname() {
[[ "$(uname -s)" == "Darwin" ]] && echo "osx" || echo "linux"
}
# Bootstraps a new conda installation and prepares base environment
# if "self" is passed as parameter, then self installs an already built
# version of bob.devtools available on your conda-bld directory.
# datetime prefix for logging
log_datetime() {
......@@ -24,51 +19,6 @@ log_error() {
echo -e "$(log_datetime) \033[1;31mError: ${@}\033[0m" >&2
}
# Checks just if the variable is defined and has non-zero length
check_defined() {
if [ -z "${!1+abc}" ]; then
log_error "Variable ${1} is undefined - aborting...";
exit 1
elif [ -z "${!1}" ]; then
log_error "Variable ${1} is zero-length - aborting...";
exit 1
fi
}
# Logs a given environment variable to the screen
log_env() {
log_info "${1}=${!1}"
}
# Checks a given environment variable is set (non-zero size)
check_env() {
check_defined "${1}"
log_env "${1}"
}
# Checks a given environment variable array is set (non-zero size)
# Then prints all of its components
check_array_env() {
check_defined "${1}"
eval array=\( \${${1}[@]} \)
for i in "${!array[@]}"; do
log_info "${1}[${i}]=${array[${i}]}";
done
}
# Exports a given environment variable, verbosely
export_env() {
check_defined "${1}"
export ${1}
log_info "export ${1}=${!1}"
}
# Function for running command and echoing results
run_cmd() {
log_info "$ ${@}"
......@@ -81,15 +31,6 @@ run_cmd() {
}
if [ -z "${BOB_PACKAGE_VERSION}" ]; then
if [ ! -r "version.txt" ]; then
log_error "./version.txt does not exist - cannot figure out version number"
exit 1
fi
BOB_PACKAGE_VERSION=`cat version.txt | tr -d '\n'`;
fi
# merges conda cache folders
# $1: Path to the main cache to keep. The directory must exist.
# $2: Path to the extra cache to be merged into main cache
......@@ -106,22 +47,30 @@ merge_conda_cache() {
}
# Checks just if the variable is defined and has non-zero length
check_defined() {
if [ -z "${!1+abc}" ]; then
log_error "Variable ${1} is undefined - aborting...";
exit 1
elif [ -z "${!1}" ]; then
log_error "Variable ${1} is zero-length - aborting...";
exit 1
fi
}
# installs a miniconda installation.
# $1: Path to where to install miniconda.
install_miniconda() {
log_info "Installing miniconda in ${1} ..."
# downloads the latest conda installation script
if [ "${OSNAME}" == "linux" ]; then
object=https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
else
object=https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
fi
# checks if miniconda.sh exists
if [ ! -e miniconda.sh ]; then
log_info "Downloading latest miniconda3 installer..."
run_cmd curl --silent --output miniconda.sh ${object}
# downloads the latest conda installation script
if [ "$(uname -s)" == "Linux" ]; then _os="Linux" else _os="MacOSX"; fi
obj=https://repo.continuum.io/miniconda/Miniconda3-latest-${_os}-x86_64.sh
run_cmd curl --silent --output miniconda.sh ${obj}
else
log_info "Re-using cached miniconda3 installer..."
ls -l miniconda.sh
......@@ -147,83 +96,36 @@ install_miniconda() {
hash -r
}
check_defined CONDA_ROOT
check_defined CI_PROJECT_DIR
# Creates a sane base condarc file
# $1: Path to where to populate the condarc file
create_condarc() {
log_info "Populating the basic condarc file in ${1} ..."
cat <<EOF > ${1}
default_channels: #!final
- https://repo.anaconda.com/pkgs/main
- https://repo.anaconda.com/pkgs/free
- https://repo.anaconda.com/pkgs/r
- https://repo.anaconda.com/pkgs/pro
add_pip_as_python_dependency: false #!final
changeps1: false #!final
always_yes: true #!final
quiet: true #!final
show_channel_urls: true #!final
anaconda_upload: false #!final
ssl_verify: false #!final
channels: #!final
EOF
}
# sets CONDA_CHANNELS to the list of conda channels that should be considered
# $2: typically, the value of ${CI_COMMIT_TAG} or empty
# given the current visibility/tagging conditions of the package.
set_conda_channels() {
CONDA_CHANNELS=() #resets bash array
if [ -z "${1}" ]; then #public beta build
CONDA_CHANNELS+=('public/conda/label/beta')
CONDA_CHANNELS+=('public/conda')
else #public (tagged) build
CONDA_CHANNELS+=('public/conda')
fi
check_array_env CONDA_CHANNELS
}
log_env PYTHON_VERSION
check_env CI_PROJECT_DIR
check_env CI_PROJECT_NAME
check_env CI_COMMIT_REF_NAME
export_env BOB_PACKAGE_VERSION
# Sets up variables
OSNAME=`osname`
check_env OSNAME
if [ -z "${OS_SLUG}" ]; then
OS_SLUG="${OSNAME}-64"
# checks if a conda installation exists. Otherwise, installs one
if [ ! -e ${CONDA_ROOT}/bin/conda ]; then
install_miniconda ${CONDA_ROOT}
fi
export_env OS_SLUG
DOCSERVER=http://www.idiap.ch
run_cmd mkdir -p ${CONDA_ROOT}/pkgs
run_cmd touch ${CONDA_ROOT}/pkgs/urls
run_cmd touch ${CONDA_ROOT}/pkgs/urls.txt
# Sets up the location of our rc file for conda
CONDARC=${CONDA_ROOT}/condarc
cp -fv ${CI_PROJECT_DIR}/bob/devtools/data/base-condarc ${CONDARC}
echo "channels:" >> ${CONDARC}
echo " - http://www.idiap.ch/public/conda" >> ${CONDARC}
echo " - defaults" >> ${CONDARC}
export_env DOCSERVER
check_env CONDA_ROOT
export_env CONDARC
# displays contents of our configuration
echo "Contents of \`${CONDARC}':"
cat ${CONDARC}
# Sets up certificates for curl and openssl
CURL_CA_BUNDLE="${CI_PROJECT_DIR}/bob/devtools/data/cacert.pem"
export_env CURL_CA_BUNDLE
SSL_CERT_FILE="${CURL_CA_BUNDLE}"
export_env SSL_CERT_FILE
GIT_SSL_CAINFO="${CURL_CA_BUNDLE}"
export_env GIT_SSL_CAINFO
# updates conda installation, installs just built bob.devtools
if [ "${1}" == "self" ]; then
run_cmd ${CONDA_ROOT}/bin/conda create -n bdt ${CONDA_ROOT}/conda-bld/${OS_SLUG}/bob.devtools-*.tar.bz2
else
run_cmd ${CONDA_ROOT}/bin/conda install -n base python conda=4 conda-build=3
fi
# Sets up upload folders for documentation (just in case we need them)
# See: https://gitlab.idiap.ch/bob/bob.admin/issues/2
# cleans up
run_cmd ${CONDA_ROOT}/bin/conda clean --lock
# Sets up the language so Unicode filenames are considered correctly
LANG="en_US.UTF-8"
LC_ALL="${LANG}"
export_env LANG
export_env LC_ALL
# print conda information for debugging purposes
run_cmd ${CONDA_ROOT}/bin/conda info
#!/usr/bin/env bash
source $(dirname ${0})/functions.sh
# datetime prefix for logging
log_datetime() {
echo "($(date +%T.%3N))"
}
# Functions for coloring echo commands
log_info() {
echo -e "$(log_datetime) \033[1;34m${@}\033[0m"
}
log_error() {
echo -e "$(log_datetime) \033[1;31mError: ${@}\033[0m" >&2
}
# Checks just if the variable is defined and has non-zero length
check_defined() {
if [ -z "${!1+abc}" ]; then
log_error "Variable ${1} is undefined - aborting...";
exit 1
elif [ -z "${!1}" ]; then
log_error "Variable ${1} is zero-length - aborting...";
exit 1
fi
}
# Exports a given environment variable, verbosely
export_env() {
check_defined "${1}"
export ${1}
log_info "export ${1}=${!1}"
}
check_defined CONDA_ROOT
check_defined CI_PROJECT_DIR
check_defined CI_PROJECT_NAME
check_defined CI_COMMIT_TAG
check_defined PYTHON_VERSION
BOB_PACKAGE_VERSION=`cat version.txt | tr -d '\n'`;
check_defined BOB_PACKAGE_VERSION
# Makes sure we activate the base environment if available
run_cmd source ${CONDA_ROOT}/etc/profile.d/conda.sh
run_cmd conda activate base
export_env PATH
set_conda_channels ${CI_COMMIT_TAG}
log_info "$ ${CONDA_ROOT}/bin/python ${SCRIPTS_DIR}/nextbuild.py ${DOCSERVER}/${CONDA_CHANNELS[0]} ${CI_PROJECT_NAME} ${BOB_PACKAGE_VERSION} ${PYTHON_VERSION}"
BOB_BUILD_NUMBER=$(${CONDA_ROOT}/bin/python ${SCRIPTS_DIR}/nextbuild.py ${DOCSERVER}/${CONDA_CHANNELS[0]} ${CI_PROJECT_NAME} ${BOB_PACKAGE_VERSION} ${PYTHON_VERSION})
if [ -z "${CI_COMMIT_TAG}" ]; then #building beta
channel="http://www.idiap.ch/public/conda/label/beta"
else
channel="http://www.idiap.ch/public/conda"
fi
log_info "$ ${CONDA_ROOT}/bin/python ${CI_PROJECT_DIR}/_ci/nextbuild.py ${channel} ${CI_PROJECT_NAME} ${BOB_PACKAGE_VERSION} ${PYTHON_VERSION}"
BOB_BUILD_NUMBER=$(${CONDA_ROOT}/bin/python ${CI_PROJECT_DIR}/_ci/nextbuild.py ${channel} ${CI_PROJECT_NAME} ${BOB_PACKAGE_VERSION} ${PYTHON_VERSION})
export_env BOB_BUILD_NUMBER
# copy the recipe_append.yaml over before build
run_cmd cp ${CI_PROJECT_DIR}/bob/devtools/data/recipe_append.yaml conda/
run_cmd cp ${CI_PROJECT_DIR}/bob/devtools/data/conda_build_config.yaml conda/
BLDOPT="--python=${PYTHON_VERSION} --no-anaconda-upload"
run_cmd ${CONDA_ROOT}/bin/conda build "--python=${PYTHON_VERSION} --no-anaconda-upload" conda
run_cmd ${CONDA_ROOT}/bin/conda build ${BLDOPT} conda
# run git clean to clean everything that is not needed. This helps to keep the
# disk usage on CI machines to minimum.
if [ "$(uname -s)" == "Linux" ]; then _os="linux" else _os="osx"; fi
run_cmd git clean -ffdx \
--exclude="miniconda.sh" \
--exclude="miniconda/pkgs/*.tar.bz2" \
--exclude="miniconda/pkgs/urls.txt" \
--exclude="miniconda/conda-bld/${_os}-64/*.tar.bz2" \
--exclude="_ci" \
--exclude="dist/*.zip" \
--exclude="sphinx"
#!/usr/bin/env bash
# Wed Jan 9 16:48:57 CET 2019
source $(dirname ${0})/functions.sh
run_cmd cp -fv ${CI_PROJECT_DIR}/bob/devtools/data/base-condarc ${CONDARC}
echo "channels:" >> ${CONDARC}
echo " - ${DOCSERVER}/public/conda" >> ${CONDARC}
echo " - defaults" >> ${CONDARC}
deploy_conda_packages ${CONDA_CHANNELS[0]} ${CI_PROJECT_NAME}
# upload the docs from the sphinx folder (usually an artifact of Linux Python
# 3.6 builds)
for folder in "${DOC_UPLOADS[@]}"; do
dav_upload_folder sphinx "${folder}"
done
default_channels:
- https://repo.anaconda.com/pkgs/main
- https://repo.anaconda.com/pkgs/free
- https://repo.anaconda.com/pkgs/r
- https://repo.anaconda.com/pkgs/pro
add_pip_as_python_dependency: false #!final
changeps1: false #!final
always_yes: true #!final
quiet: true #!final
show_channel_urls: true #!final
anaconda_upload: false #!final
ssl_verify: false #!final
......@@ -33,7 +33,8 @@ requirements:
- setuptools
- click
- click-plugins
- conda-build
- conda=4
- conda-build=3
- pytz
- python-dateutil
- gitpython
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment