Commit 9b402197 authored by André Anjos's avatar André Anjos 💬
Browse files

[ci] Implement core build strategy for package

parent 5729e750
Pipeline #25767 failed with stage
in 23 seconds
# This build file uses template features from YAML so it is generic enough for
# any Bob project. Don't modify it unless you know what you're doing.
# Definition of global variables (all stages)
variables:
CONDA_ROOT: "${CI_PROJECT_DIR}/miniconda"
......@@ -9,18 +6,12 @@ variables:
# Definition of our build pipeline order
stages:
- build
- deploy
- pypi
# Build targets
.build_template: &build_job
stage: build
before_script:
- mkdir _ci
- curl --silent "https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/install.sh" > _ci/install.sh
- chmod 755 _ci/install.sh
- ./_ci/install.sh _ci master #installs ci support scripts
- ./_ci/before_build.sh
script:
- ./_ci/build.sh
......@@ -41,7 +32,6 @@ stages:
artifacts:
expire_in: 1 week
paths:
- _ci/
- ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
cache:
<<: *build_caches
......@@ -55,7 +45,6 @@ stages:
artifacts:
expire_in: 1 week
paths:
- _ci/
- ${CONDA_ROOT}/conda-bld/osx-64/*.tar.bz2
cache:
<<: *build_caches
......@@ -70,7 +59,6 @@ build_linux_36:
artifacts:
expire_in: 1 week
paths:
- _ci/
- dist/*.zip
- sphinx
- ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
......@@ -80,50 +68,3 @@ build_macosx_36:
<<: *macosx_build_job
variables:
PYTHON_VERSION: "3.6"
# Deploy targets
.deploy_template: &deploy_job
stage: deploy
before_script:
- ./_ci/install.sh _ci master #updates ci support scripts
script:
- ./_ci/deploy.sh
dependencies:
- build_linux_36
- build_macosx_36
tags:
- deployer
deploy_beta:
<<: *deploy_job
environment: beta
only:
- master
deploy_stable:
<<: *deploy_job
environment: stable
only:
- /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags)
except:
- branches
pypi:
stage: pypi
environment: pypi
only:
- /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags)
except:
- branches
before_script:
- ./_ci/install.sh _ci master #updates ci support scripts
script:
- ./_ci/pypi.sh
dependencies:
- build_linux_36
tags:
- deployer
#!/usr/bin/env bash
source $(dirname ${0})/functions.sh
# delete the bob packages from the cache otherwise the cache keeps increasing
# over and over. See https://gitlab.idiap.ch/bob/bob.admin/issues/65
run_cmd rm -rf ${CONDA_ROOT}/pkgs/batl*.tar.bz2
run_cmd rm -rf ${CONDA_ROOT}/pkgs/beat*.tar.bz2
run_cmd rm -rf ${CONDA_ROOT}/pkgs/bob*.tar.bz2
run_cmd rm -rf ${CONDA_ROOT}/pkgs/gridtk*.tar.bz2
# run git clean to clean everything that is not needed. This helps to keep the
# disk usage on CI machines to minimum.
run_cmd git clean -ffdx \
--exclude="miniconda.sh" \
--exclude="miniconda/pkgs/*.tar.bz2" \
--exclude="miniconda/pkgs/urls.txt" \
--exclude="miniconda/conda-bld/linux-64/*.tar.bz2" \
--exclude="miniconda/conda-bld/osx-64/*.tar.bz2" \
--exclude="_ci" \
--exclude="dist/*.zip" \
--exclude="sphinx"
#!/usr/bin/env bash
# Tue 16 Jan 11:15:32 2018 CET
source $(dirname ${0})/functions.sh
# checks if a conda installation exists. Otherwise, install one
if [ ! -e ${CONDA_ROOT}/bin/conda ]; then
install_miniconda ${CONDA_ROOT}
fi
mkdir -p ${CONDA_ROOT}/pkgs
touch ${CONDA_ROOT}/pkgs/urls
touch ${CONDA_ROOT}/pkgs/urls.txt
create_condarc ${CONDARC}
set_conda_channels ${CI_COMMIT_TAG}
for k in "${CONDA_CHANNELS[@]}"; do
echo " - ${DOCSERVER}/${k}" >> ${CONDARC}
done
echo " - defaults" >> ${CONDARC}
# displays contents of our configuration
echo "Contents of \`${CONDARC}':"
cat ${CONDARC}
# updates conda installation
run_cmd ${CONDA_ROOT}/bin/conda install python conda=4 curl conda-build=3
# cleans up
run_cmd ${CONDA_ROOT}/bin/conda clean --lock
# print conda information for debugging purposes
run_cmd ${CONDA_ROOT}/bin/conda info
#!/usr/bin/env bash
source $(dirname ${0})/functions.sh
# Makes sure we activate the base environment if available
run_cmd source ${CONDA_ROOT}/etc/profile.d/conda.sh
run_cmd conda activate base
export_env PATH
set_conda_channels ${CI_COMMIT_TAG}
log_info "$ ${CONDA_ROOT}/bin/python ${SCRIPTS_DIR}/nextbuild.py ${DOCSERVER}/${1} ${CI_PROJECT_NAME} ${BOB_PACKAGE_VERSION} ${PYTHON_VERSION}"
BOB_BUILD_NUMBER=$(${CONDA_ROOT}/bin/python ${SCRIPTS_DIR}/nextbuild.py ${DOCSERVER}/${1} ${CI_PROJECT_NAME} ${BOB_PACKAGE_VERSION} ${PYTHON_VERSION})
export_env BOB_BUILD_NUMBER
# copy the recipe_append.yaml over before build
run_cmd cp ${CI_PROJECT_DIR}/bob/devtools/data/recipe_append.yaml conda/
run_cmd cp ${CI_PROJECT_DIR}/bob/devtools/data/conda_build_config.yaml conda/
BLDOPT="--python=${PYTHON_VERSION} --no-anaconda-upload"
run_cmd ${CONDA_ROOT}/bin/conda build ${BLDOPT} conda
#!/usr/bin/env bash
# Wed 9 Jan 2019 14:33:20 CET
# Build utilities
SCRIPTS_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# datetime prefix for logging
log_datetime() {
echo "($(date +%T.%3N))"
}
# Functions for coloring echo commands
log_info() {
echo -e "$(log_datetime) \033[1;34m${@}\033[0m"
}
log_error() {
echo -e "$(log_datetime) \033[1;31mError: ${@}\033[0m" >&2
}
# Checks just if the variable is defined and has non-zero length
check_defined() {
if [ -z "${!1+abc}" ]; then
log_error "Variable ${1} is undefined - aborting...";
exit 1
elif [ -z "${!1}" ]; then
log_error "Variable ${1} is zero-length - aborting...";
exit 1
fi
}
# Logs a given environment variable to the screen
log_env() {
log_info "${1}=${!1}"
}
# Checks a given environment variable is set (non-zero size)
check_env() {
check_defined "${1}"
log_env "${1}"
}
# Checks a given environment variable array is set (non-zero size)
# Then prints all of its components
check_array_env() {
check_defined "${1}"
eval array=\( \${${1}[@]} \)
for i in "${!array[@]}"; do
log_info "${1}[${i}]=${array[${i}]}";
done
}
# Exports a given environment variable, verbosely
export_env() {
check_defined "${1}"
export ${1}
log_info "export ${1}=${!1}"
}
# Function for running command and echoing results
run_cmd() {
log_info "$ ${@}"
${@}
local status=$?
if [ ${status} != 0 ]; then
log_error "Command Failed \"${@}\""
exit ${status}
fi
}
if [ -z "${BOB_PACKAGE_VERSION}" ]; then
if [ ! -r "version.txt" ]; then
log_error "./version.txt does not exist - cannot figure out version number"
exit 1
fi
BOB_PACKAGE_VERSION=`cat version.txt | tr -d '\n'`;
fi
# merges conda cache folders
# $1: Path to the main cache to keep. The directory must exist.
# $2: Path to the extra cache to be merged into main cache
merge_conda_cache() {
if [ -e ${1} ]; then
_cached_urlstxt="${2}/urls.txt"
_urlstxt="${1}/urls.txt"
if [ -e ${2} ]; then
log_info "Merging urls.txt and packages with cached files..."
mv ${2}/*.tar.bz2 ${1}/
cat ${_urlstxt} ${_cached_urlstxt} | sort | uniq > ${_urlstxt}
fi
fi
}
# installs a miniconda installation.
# $1: Path to where to install miniconda.
install_miniconda() {
log_info "Installing miniconda in ${1} ..."
# downloads the latest conda installation script
if [ "${OSNAME}" == "linux" ]; then
object=https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
else
object=https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
fi
# checks if miniconda.sh exists
if [ ! -e miniconda.sh ]; then
log_info "Downloading latest miniconda3 installer..."
run_cmd curl --silent --output miniconda.sh ${object}
else
log_info "Re-using cached miniconda3 installer..."
ls -l miniconda.sh
fi
# move cache to a different folder if it exists
if [ -e ${1} ]; then
run_cmd mv ${1} ${1}.cached
fi
# install miniconda
run_cmd bash miniconda.sh -b -p ${1}
# Put back cache and merge urls.txt
merge_conda_cache ${1}/pkgs ${1}.cached/pkgs
# remove the backup cache folder
rm -rf ${1}.cached
# List currently available packages on cache
# run_cmd ls -l ${1}/pkgs/
# run_cmd cat ${1}/pkgs/urls.txt
hash -r
}
# Creates a sane base condarc file
# $1: Path to where to populate the condarc file
create_condarc() {
log_info "Populating the basic condarc file in ${1} ..."
cat <<EOF > ${1}
default_channels: #!final
- https://repo.anaconda.com/pkgs/main
- https://repo.anaconda.com/pkgs/free
- https://repo.anaconda.com/pkgs/r
- https://repo.anaconda.com/pkgs/pro
add_pip_as_python_dependency: false #!final
changeps1: false #!final
always_yes: true #!final
quiet: true #!final
show_channel_urls: true #!final
anaconda_upload: false #!final
ssl_verify: false #!final
channels: #!final
EOF
}
# sets CONDA_CHANNELS to the list of conda channels that should be considered
# $2: typically, the value of ${CI_COMMIT_TAG} or empty
# given the current visibility/tagging conditions of the package.
set_conda_channels() {
CONDA_CHANNELS=() #resets bash array
if [ -z "${1}" ]; then #public beta build
CONDA_CHANNELS+=('public/conda/label/beta')
CONDA_CHANNELS+=('public/conda')
else #public (tagged) build
CONDA_CHANNELS+=('public/conda')
fi
check_array_env CONDA_CHANNELS
}
log_env PYTHON_VERSION
check_env CI_PROJECT_DIR
check_env CI_PROJECT_NAME
check_env CI_COMMIT_REF_NAME
export_env BOB_PACKAGE_VERSION
# Sets up variables
OSNAME=`osname`
check_env OSNAME
DOCSERVER=http://www.idiap.ch
# Sets up the location of our rc file for conda
CONDARC=${CONDA_ROOT}/condarc
if [ -z "${OS_SLUG}" ]; then
OS_SLUG="${OSNAME}-64"
fi
export_env OS_SLUG
export_env DOCSERVER
check_env CONDA_ROOT
export_env CONDARC
# Sets up certificates for curl and openssl
CURL_CA_BUNDLE="${CI_PROJECT_DIR}/bob/devtools/data/cacert.pem"
export_env CURL_CA_BUNDLE
SSL_CERT_FILE="${CURL_CA_BUNDLE}"
export_env SSL_CERT_FILE
GIT_SSL_CAINFO="${CURL_CA_BUNDLE}"
export_env GIT_SSL_CAINFO
# Sets up upload folders for documentation (just in case we need them)
# See: https://gitlab.idiap.ch/bob/bob.admin/issues/2
# Sets up the language so Unicode filenames are considered correctly
LANG="en_US.UTF-8"
LC_ALL="${LANG}"
export_env LANG
export_env LC_ALL
#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''This is a copy of bob/devtools/conda:next_build_number with a CLI'''
import os
import re
import sys
from conda.exports import get_index
if __name__ == '__main__':
channel_url = sys.argv[1]
name = sys.argv[2]
version = sys.argv[3]
python = sys.argv[4]
# no dot in py_ver
py_ver = python.replace('.', '')
# get the channel index
index = get_index(channel_urls=[channel_url], prepend=False)
# search if package with the same version exists
build_number = 0
urls = []
for dist in index:
if dist.name == name and dist.version == version:
match = re.match('py[2-9][0-9]+', dist.build_string)
if match and match.group() == 'py{}'.format(py_ver):
build_number = max(build_number, dist.build_number + 1)
urls.append(index[dist].url)
urls = [url.replace(channel_url, '') for url in urls]
print(build_number)
../bob/devtools/data/conda_build_config.yaml
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment