Skip to content
Snippets Groups Projects
Commit 48608ad6 authored by André Anjos's avatar André Anjos :speech_balloon:
Browse files

[ci] Largely simplified

parent 87d312b6
No related branches found
No related tags found
No related merge requests found
Pipeline #25778 failed
Stage: build
Stage: deploy
Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yml
+ 37
− 3
View file @ 48608ad6
...@@ -6,17 +6,16 @@ variables: ...@@ -6,17 +6,16 @@ variables:
# Definition of our build pipeline order # Definition of our build pipeline order
stages: stages:
- build - build
- deploy
# Build targets # Build targets
.build_template: &build_job .build_template: &build_job
stage: build stage: build
before_script: before_script:
- ./_ci/before_build.sh - ./_ci/bootstrap.sh
script: script:
- ./_ci/build.sh - ./_ci/build.sh
after_script:
- ./_ci/after_build.sh
cache: &build_caches cache: &build_caches
paths: paths:
- miniconda.sh - miniconda.sh
...@@ -68,3 +67,38 @@ build_macosx_36: ...@@ -68,3 +67,38 @@ build_macosx_36:
<<: *macosx_build_job <<: *macosx_build_job
variables: variables:
PYTHON_VERSION: "3.6" PYTHON_VERSION: "3.6"
# Deploy targets
.deploy_template: &deploy_job
stage: deploy
before_script:
- ./_ci/bootstrap.sh self
script:
- ./_ci/deploy.sh
dependencies:
- build_linux_36
- build_macosx_36
tags:
- deployer
cache: &build_caches
paths:
- miniconda.sh
- ${CONDA_ROOT}/pkgs/*.tar.bz2
- ${CONDA_ROOT}/pkgs/urls.txt
deploy_beta:
<<: *deploy_job
environment: beta
only:
- master
deploy_stable:
<<: *deploy_job
environment: stable
only:
- /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags)
except:
- branches
_ci/after_build.sh deleted 100755 → 0
+ 0
− 18
View file @ 87d312b6
#!/usr/bin/env bash
source $(dirname ${0})/functions.sh
# delete the bob packages from the cache otherwise the cache keeps increasing
# over and over. See https://gitlab.idiap.ch/bob/bob.admin/issues/65
run_cmd rm -rf ${CONDA_ROOT}/pkgs/bob*.tar.bz2
# run git clean to clean everything that is not needed. This helps to keep the
# disk usage on CI machines to minimum.
run_cmd git clean -ffdx \
--exclude="miniconda.sh" \
--exclude="miniconda/pkgs/*.tar.bz2" \
--exclude="miniconda/pkgs/urls.txt" \
--exclude="miniconda/conda-bld/${OS_SLUG}/*.tar.bz2" \
--exclude="_ci" \
--exclude="dist/*.zip" \
--exclude="sphinx"
_ci/before_build.sh deleted 100755 → 0
+ 0
− 34
View file @ 87d312b6
#!/usr/bin/env bash
# Tue 16 Jan 11:15:32 2018 CET
source $(dirname ${0})/functions.sh
# checks if a conda installation exists. Otherwise, install one
if [ ! -e ${CONDA_ROOT}/bin/conda ]; then
install_miniconda ${CONDA_ROOT}
fi
mkdir -p ${CONDA_ROOT}/pkgs
touch ${CONDA_ROOT}/pkgs/urls
touch ${CONDA_ROOT}/pkgs/urls.txt
create_condarc ${CONDARC}
set_conda_channels ${CI_COMMIT_TAG}
for k in "${CONDA_CHANNELS[@]}"; do
echo " - ${DOCSERVER}/${k}" >> ${CONDARC}
done
echo " - defaults" >> ${CONDARC}
# displays contents of our configuration
echo "Contents of \`${CONDARC}':"
cat ${CONDARC}
# updates conda installation
run_cmd ${CONDA_ROOT}/bin/conda install python conda=4 conda-build=3
# cleans up
run_cmd ${CONDA_ROOT}/bin/conda clean --lock
# print conda information for debugging purposes
run_cmd ${CONDA_ROOT}/bin/conda info
_ci/functions.sh → _ci/bootstrap.sh 100644 → 100755
+ 131
− 0
View file @ 48608ad6
#!/usr/bin/env bash #!/urs/bin/env bash
# Wed 9 Jan 2019 14:33:20 CET
# Build utilities # Bootstraps a new conda installation and prepares base environment
SCRIPTS_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # if "self" is passed as parameter, then self installs an already built
# version of bob.devtools available on your conda-bld directory.
# Determines the operating system we're using
osname() {
[[ "$(uname -s)" == "Darwin" ]] && echo "osx" || echo "linux"
}
# datetime prefix for logging # datetime prefix for logging
log_datetime() { log_datetime() {
...@@ -24,51 +19,6 @@ log_error() { ...@@ -24,51 +19,6 @@ log_error() {
echo -e "$(log_datetime) \033[1;31mError: ${@}\033[0m" >&2 echo -e "$(log_datetime) \033[1;31mError: ${@}\033[0m" >&2
} }
# Checks just if the variable is defined and has non-zero length
check_defined() {
if [ -z "${!1+abc}" ]; then
log_error "Variable ${1} is undefined - aborting...";
exit 1
elif [ -z "${!1}" ]; then
log_error "Variable ${1} is zero-length - aborting...";
exit 1
fi
}
# Logs a given environment variable to the screen
log_env() {
log_info "${1}=${!1}"
}
# Checks a given environment variable is set (non-zero size)
check_env() {
check_defined "${1}"
log_env "${1}"
}
# Checks a given environment variable array is set (non-zero size)
# Then prints all of its components
check_array_env() {
check_defined "${1}"
eval array=\( \${${1}[@]} \)
for i in "${!array[@]}"; do
log_info "${1}[${i}]=${array[${i}]}";
done
}
# Exports a given environment variable, verbosely
export_env() {
check_defined "${1}"
export ${1}
log_info "export ${1}=${!1}"
}
# Function for running command and echoing results # Function for running command and echoing results
run_cmd() { run_cmd() {
log_info "$ ${@}" log_info "$ ${@}"
...@@ -81,15 +31,6 @@ run_cmd() { ...@@ -81,15 +31,6 @@ run_cmd() {
} }
if [ -z "${BOB_PACKAGE_VERSION}" ]; then
if [ ! -r "version.txt" ]; then
log_error "./version.txt does not exist - cannot figure out version number"
exit 1
fi
BOB_PACKAGE_VERSION=`cat version.txt | tr -d '\n'`;
fi
# merges conda cache folders # merges conda cache folders
# $1: Path to the main cache to keep. The directory must exist. # $1: Path to the main cache to keep. The directory must exist.
# $2: Path to the extra cache to be merged into main cache # $2: Path to the extra cache to be merged into main cache
...@@ -106,22 +47,30 @@ merge_conda_cache() { ...@@ -106,22 +47,30 @@ merge_conda_cache() {
} }
# Checks just if the variable is defined and has non-zero length
check_defined() {
if [ -z "${!1+abc}" ]; then
log_error "Variable ${1} is undefined - aborting...";
exit 1
elif [ -z "${!1}" ]; then
log_error "Variable ${1} is zero-length - aborting...";
exit 1
fi
}
# installs a miniconda installation. # installs a miniconda installation.
# $1: Path to where to install miniconda. # $1: Path to where to install miniconda.
install_miniconda() { install_miniconda() {
log_info "Installing miniconda in ${1} ..." log_info "Installing miniconda in ${1} ..."
# downloads the latest conda installation script
if [ "${OSNAME}" == "linux" ]; then
object=https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
else
object=https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
fi
# checks if miniconda.sh exists # checks if miniconda.sh exists
if [ ! -e miniconda.sh ]; then if [ ! -e miniconda.sh ]; then
log_info "Downloading latest miniconda3 installer..." log_info "Downloading latest miniconda3 installer..."
run_cmd curl --silent --output miniconda.sh ${object} # downloads the latest conda installation script
if [ "$(uname -s)" == "Linux" ]; then _os="Linux" else _os="MacOSX"; fi
obj=https://repo.continuum.io/miniconda/Miniconda3-latest-${_os}-x86_64.sh
run_cmd curl --silent --output miniconda.sh ${obj}
else else
log_info "Re-using cached miniconda3 installer..." log_info "Re-using cached miniconda3 installer..."
ls -l miniconda.sh ls -l miniconda.sh
...@@ -147,83 +96,36 @@ install_miniconda() { ...@@ -147,83 +96,36 @@ install_miniconda() {
hash -r hash -r
} }
check_defined CONDA_ROOT
check_defined CI_PROJECT_DIR
# Creates a sane base condarc file # checks if a conda installation exists. Otherwise, installs one
# $1: Path to where to populate the condarc file if [ ! -e ${CONDA_ROOT}/bin/conda ]; then
create_condarc() { install_miniconda ${CONDA_ROOT}
log_info "Populating the basic condarc file in ${1} ..."
cat <<EOF > ${1}
default_channels: #!final
- https://repo.anaconda.com/pkgs/main
- https://repo.anaconda.com/pkgs/free
- https://repo.anaconda.com/pkgs/r
- https://repo.anaconda.com/pkgs/pro
add_pip_as_python_dependency: false #!final
changeps1: false #!final
always_yes: true #!final
quiet: true #!final
show_channel_urls: true #!final
anaconda_upload: false #!final
ssl_verify: false #!final
channels: #!final
EOF
}
# sets CONDA_CHANNELS to the list of conda channels that should be considered
# $2: typically, the value of ${CI_COMMIT_TAG} or empty
# given the current visibility/tagging conditions of the package.
set_conda_channels() {
CONDA_CHANNELS=() #resets bash array
if [ -z "${1}" ]; then #public beta build
CONDA_CHANNELS+=('public/conda/label/beta')
CONDA_CHANNELS+=('public/conda')
else #public (tagged) build
CONDA_CHANNELS+=('public/conda')
fi
check_array_env CONDA_CHANNELS
}
log_env PYTHON_VERSION
check_env CI_PROJECT_DIR
check_env CI_PROJECT_NAME
check_env CI_COMMIT_REF_NAME
export_env BOB_PACKAGE_VERSION
# Sets up variables
OSNAME=`osname`
check_env OSNAME
if [ -z "${OS_SLUG}" ]; then
OS_SLUG="${OSNAME}-64"
fi fi
export_env OS_SLUG
DOCSERVER=http://www.idiap.ch run_cmd mkdir -p ${CONDA_ROOT}/pkgs
run_cmd touch ${CONDA_ROOT}/pkgs/urls
run_cmd touch ${CONDA_ROOT}/pkgs/urls.txt
# Sets up the location of our rc file for conda cp -fv ${CI_PROJECT_DIR}/bob/devtools/data/base-condarc ${CONDARC}
CONDARC=${CONDA_ROOT}/condarc echo "channels:" >> ${CONDARC}
echo " - http://www.idiap.ch/public/conda" >> ${CONDARC}
echo " - defaults" >> ${CONDARC}
export_env DOCSERVER # displays contents of our configuration
check_env CONDA_ROOT echo "Contents of \`${CONDARC}':"
export_env CONDARC cat ${CONDARC}
# Sets up certificates for curl and openssl # updates conda installation, installs just built bob.devtools
CURL_CA_BUNDLE="${CI_PROJECT_DIR}/bob/devtools/data/cacert.pem" if [ "${1}" == "self" ]; then
export_env CURL_CA_BUNDLE run_cmd ${CONDA_ROOT}/bin/conda create -n bdt ${CONDA_ROOT}/conda-bld/${OS_SLUG}/bob.devtools-*.tar.bz2
SSL_CERT_FILE="${CURL_CA_BUNDLE}" else
export_env SSL_CERT_FILE run_cmd ${CONDA_ROOT}/bin/conda install -n base python conda=4 conda-build=3
GIT_SSL_CAINFO="${CURL_CA_BUNDLE}" fi
export_env GIT_SSL_CAINFO
# Sets up upload folders for documentation (just in case we need them) # cleans up
# See: https://gitlab.idiap.ch/bob/bob.admin/issues/2 run_cmd ${CONDA_ROOT}/bin/conda clean --lock
# Sets up the language so Unicode filenames are considered correctly # print conda information for debugging purposes
LANG="en_US.UTF-8" run_cmd ${CONDA_ROOT}/bin/conda info
LC_ALL="${LANG}"
export_env LANG
export_env LC_ALL
_ci/build.sh
+ 63
− 6
View file @ 48608ad6
#!/usr/bin/env bash #!/usr/bin/env bash
source $(dirname ${0})/functions.sh # datetime prefix for logging
log_datetime() {
echo "($(date +%T.%3N))"
}
# Functions for coloring echo commands
log_info() {
echo -e "$(log_datetime) \033[1;34m${@}\033[0m"
}
log_error() {
echo -e "$(log_datetime) \033[1;31mError: ${@}\033[0m" >&2
}
# Checks just if the variable is defined and has non-zero length
check_defined() {
if [ -z "${!1+abc}" ]; then
log_error "Variable ${1} is undefined - aborting...";
exit 1
elif [ -z "${!1}" ]; then
log_error "Variable ${1} is zero-length - aborting...";
exit 1
fi
}
# Exports a given environment variable, verbosely
export_env() {
check_defined "${1}"
export ${1}
log_info "export ${1}=${!1}"
}
check_defined CONDA_ROOT
check_defined CI_PROJECT_DIR
check_defined CI_PROJECT_NAME
check_defined CI_COMMIT_TAG
check_defined PYTHON_VERSION
BOB_PACKAGE_VERSION=`cat version.txt | tr -d '\n'`;
check_defined BOB_PACKAGE_VERSION
# Makes sure we activate the base environment if available # Makes sure we activate the base environment if available
run_cmd source ${CONDA_ROOT}/etc/profile.d/conda.sh run_cmd source ${CONDA_ROOT}/etc/profile.d/conda.sh
run_cmd conda activate base run_cmd conda activate base
export_env PATH export_env PATH
set_conda_channels ${CI_COMMIT_TAG} if [ -z "${CI_COMMIT_TAG}" ]; then #building beta
log_info "$ ${CONDA_ROOT}/bin/python ${SCRIPTS_DIR}/nextbuild.py ${DOCSERVER}/${CONDA_CHANNELS[0]} ${CI_PROJECT_NAME} ${BOB_PACKAGE_VERSION} ${PYTHON_VERSION}" channel="http://www.idiap.ch/public/conda/label/beta"
BOB_BUILD_NUMBER=$(${CONDA_ROOT}/bin/python ${SCRIPTS_DIR}/nextbuild.py ${DOCSERVER}/${CONDA_CHANNELS[0]} ${CI_PROJECT_NAME} ${BOB_PACKAGE_VERSION} ${PYTHON_VERSION}) else
channel="http://www.idiap.ch/public/conda"
fi
log_info "$ ${CONDA_ROOT}/bin/python ${CI_PROJECT_DIR}/_ci/nextbuild.py ${channel} ${CI_PROJECT_NAME} ${BOB_PACKAGE_VERSION} ${PYTHON_VERSION}"
BOB_BUILD_NUMBER=$(${CONDA_ROOT}/bin/python ${CI_PROJECT_DIR}/_ci/nextbuild.py ${channel} ${CI_PROJECT_NAME} ${BOB_PACKAGE_VERSION} ${PYTHON_VERSION})
export_env BOB_BUILD_NUMBER export_env BOB_BUILD_NUMBER
# copy the recipe_append.yaml over before build # copy the recipe_append.yaml over before build
run_cmd cp ${CI_PROJECT_DIR}/bob/devtools/data/recipe_append.yaml conda/ run_cmd cp ${CI_PROJECT_DIR}/bob/devtools/data/recipe_append.yaml conda/
run_cmd cp ${CI_PROJECT_DIR}/bob/devtools/data/conda_build_config.yaml conda/ run_cmd cp ${CI_PROJECT_DIR}/bob/devtools/data/conda_build_config.yaml conda/
BLDOPT="--python=${PYTHON_VERSION} --no-anaconda-upload" run_cmd ${CONDA_ROOT}/bin/conda build "--python=${PYTHON_VERSION} --no-anaconda-upload" conda
run_cmd ${CONDA_ROOT}/bin/conda build ${BLDOPT} conda # run git clean to clean everything that is not needed. This helps to keep the
# disk usage on CI machines to minimum.
if [ "$(uname -s)" == "Linux" ]; then _os="linux" else _os="osx"; fi
run_cmd git clean -ffdx \
--exclude="miniconda.sh" \
--exclude="miniconda/pkgs/*.tar.bz2" \
--exclude="miniconda/pkgs/urls.txt" \
--exclude="miniconda/conda-bld/${_os}-64/*.tar.bz2" \
--exclude="_ci" \
--exclude="dist/*.zip" \
--exclude="sphinx"
_ci/deploy.sh 0 → 100755
+ 17
− 0
View file @ 48608ad6
#!/usr/bin/env bash
# Wed Jan 9 16:48:57 CET 2019
source $(dirname ${0})/functions.sh
run_cmd cp -fv ${CI_PROJECT_DIR}/bob/devtools/data/base-condarc ${CONDARC}
echo "channels:" >> ${CONDARC}
echo " - ${DOCSERVER}/public/conda" >> ${CONDARC}
echo " - defaults" >> ${CONDARC}
deploy_conda_packages ${CONDA_CHANNELS[0]} ${CI_PROJECT_NAME}
# upload the docs from the sphinx folder (usually an artifact of Linux Python
# 3.6 builds)
for folder in "${DOC_UPLOADS[@]}"; do
dav_upload_folder sphinx "${folder}"
done
bob/devtools/data/base-condarc 0 → 100644
+ 12
− 0
View file @ 48608ad6
default_channels:
- https://repo.anaconda.com/pkgs/main
- https://repo.anaconda.com/pkgs/free
- https://repo.anaconda.com/pkgs/r
- https://repo.anaconda.com/pkgs/pro
add_pip_as_python_dependency: false #!final
changeps1: false #!final
always_yes: true #!final
quiet: true #!final
show_channel_urls: true #!final
anaconda_upload: false #!final
ssl_verify: false #!final
conda/meta.yaml
+ 2
− 1
View file @ 48608ad6
...@@ -33,7 +33,8 @@ requirements: ...@@ -33,7 +33,8 @@ requirements:
- setuptools - setuptools
- click - click
- click-plugins - click-plugins
- conda-build - conda=4
- conda-build=3
- pytz - pytz
- python-dateutil - python-dateutil
- gitpython - gitpython
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment