Skip to content
Snippets Groups Projects
Commit d14b5a6d authored by Amir MOHAMMADI's avatar Amir MOHAMMADI
Browse files

Nightlies now runs packages pipelines

parent f93cdf2b
No related branches found
No related tags found
No related merge requests found
Pipeline #
Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yml
+ 15
− 250
View file @ d14b5a6d
stages: stages:
- core-build - core
- core-wheels - extra
- extra-build
- extra-wheels
core_build:
# --------- stage: core
# Templates
# ---------
.core_build_template: &core_build_job
stage: core-build
before_script:
- git clean -ffdx
script:
- ./build.sh core.txt ${PYTHON_VERSION}
artifacts:
expire_in: 1 week
paths:
- _ci/
- dist/
.core_wheels_template: &core_wheels_job
stage: core-wheels
only:
- master
script:
- ./_ci/wheels.sh
.extra_build_template: &extra_build_job
stage: extra-build
script:
- ./build.sh extra.txt ${PYTHON_VERSION}
artifacts:
expire_in: 1 week
paths:
- _ci/
- dist/
.extra_wheels_template: &extra_wheels_job
stage: extra-wheels
only:
- master
script: script:
- ./_ci/wheels.sh - pip install --upgrade python-gitlab
- python build.py core.txt 1
image: python
# -------------
# Build Targets
# -------------
# Python 2.7 on Linux
core_build_linux_27:
<<: *core_build_job
variables: &core_build_linux_27_variables
PYTHON_VERSION: "2.7"
tags:
- conda-linux
core_wheels_linux_27:
<<: *core_wheels_job
variables: *core_build_linux_27_variables
dependencies:
- core_build_linux_27
tags:
- conda-linux
extra_build_linux_27:
<<: *extra_build_job
variables: *core_build_linux_27_variables
dependencies:
- core_build_linux_27
tags: tags:
- conda-linux - docker
extra_wheels_linux_27: extra_build:
<<: *extra_wheels_job stage: core
variables: *core_build_linux_27_variables script:
dependencies: - pip install --upgrade python-gitlab
- extra_build_linux_27 - python build.py extra.txt 0
tags: image: python
- conda-linux
# Python 3.5 on Linux
core_build_linux_35:
<<: *core_build_job
variables: &core_build_linux_35_variables
PYTHON_VERSION: "3.5"
tags:
- conda-linux
core_wheels_linux_35:
<<: *core_wheels_job
variables: *core_build_linux_35_variables
dependencies:
- core_build_linux_35
tags:
- conda-linux
extra_build_linux_35:
<<: *extra_build_job
variables: *core_build_linux_35_variables
dependencies:
- core_build_linux_35
tags:
- conda-linux
extra_wheels_linux_35:
<<: *extra_wheels_job
variables: *core_build_linux_35_variables
dependencies:
- extra_build_linux_35
tags:
- conda-linux
# Python 3.6 on Linux
core_build_linux_36:
<<: *core_build_job
variables: &core_build_linux_36_variables
PYTHON_VERSION: "3.6"
only:
- master
tags:
- conda-linux
core_wheels_linux_36:
<<: *core_wheels_job
variables: *core_build_linux_36_variables
dependencies:
- core_build_linux_36
tags:
- conda-linux
extra_build_linux_36:
<<: *extra_build_job
variables: *core_build_linux_36_variables
dependencies:
- core_build_linux_36
only:
- master
tags:
- conda-linux
extra_wheels_linux_36:
<<: *extra_wheels_job
variables: *core_build_linux_36_variables
dependencies:
- extra_build_linux_36
tags:
- conda-linux
# Python 2.7 on MacOSX
core_build_macosx_27:
<<: *core_build_job
variables: &core_build_macosx_27_variables
PYTHON_VERSION: "2.7"
tags:
- conda-macosx
core_wheels_macosx_27:
<<: *core_wheels_job
variables: *core_build_macosx_27_variables
dependencies:
- core_build_macosx_27
tags:
- conda-macosx
extra_build_macosx_27:
<<: *extra_build_job
variables: *core_build_macosx_27_variables
dependencies:
- core_build_macosx_27
tags:
- conda-macosx
extra_wheels_macosx_27:
<<: *extra_wheels_job
variables: *core_build_macosx_27_variables
dependencies:
- extra_build_macosx_27
tags:
- conda-macosx
# Python 3.5 on MacOSX
core_build_macosx_35:
<<: *core_build_job
variables: &core_build_macosx_35_variables
PYTHON_VERSION: "3.5"
tags:
- conda-macosx
core_wheels_macosx_35:
<<: *core_wheels_job
variables: *core_build_macosx_35_variables
dependencies:
- core_build_macosx_35
tags:
- conda-macosx
extra_build_macosx_35:
<<: *extra_build_job
variables: *core_build_macosx_35_variables
dependencies:
- core_build_macosx_35
tags:
- conda-macosx
extra_wheels_macosx_35:
<<: *extra_wheels_job
variables: *core_build_macosx_35_variables
dependencies:
- extra_build_macosx_35
tags:
- conda-macosx
# Python 3.6 on MacOSX
core_build_macosx_36:
<<: *core_build_job
variables: &core_build_macosx_36_variables
PYTHON_VERSION: "3.6"
only:
- master
tags:
- conda-macosx
core_wheels_macosx_36:
<<: *core_wheels_job
variables: *core_build_macosx_36_variables
dependencies:
- core_build_macosx_36
tags:
- conda-macosx
extra_build_macosx_36:
<<: *extra_build_job
variables: *core_build_macosx_36_variables
dependencies:
- core_build_macosx_36
only:
- master
tags:
- conda-macosx
extra_wheels_macosx_36:
<<: *extra_wheels_job
variables: *core_build_macosx_36_variables
dependencies:
- extra_build_macosx_36
tags: tags:
- conda-macosx - docker
build.py 0 → 100644
+ 37
− 0
View file @ d14b5a6d
#!/usr/bin/env python
import os
import gitlab
import time
def main(pkg_list_file, fail=True):
private_token = os.environ['GITLAB_API_TOKEN']
gl = gitlab.Gitlab('https://gitlab.idiap.ch',
private_token=private_token, api_version=4)
with open(pkg_list_file) as f:
for line in f:
line = line.strip()
if not line or line.startswith('#'):
continue
pkg = line.split('#')[0]
project = gl.projects.get('bob/{}'.format(pkg))
pipeline = project.pipelines.create({'ref': 'master'})
while pipeline.status in ('pending', 'running'):
time.sleep(10)
pipeline = project.pipelines.get(pipeline.id)
if pipeline.status != 'success':
message = "Pipeline {} for {} with status {}".format(
pipeline.id, pkg, pipeline.status)
if fail:
raise RuntimeError(message)
else:
print(message)
if __name__ == '__main__':
import sys
pkg_list_file = sys.argv[1]
pkg_list_file = sys.argv[1]
fail = bool(sys.argv[2])
main(pkg_list_file, fail)
build.sh deleted 100755 → 0
+ 0
− 214
View file @ f93cdf2b
#!/usr/bin/env bash
# Andre Anjos <andre.anjos@idiap.ch>
# Thu 22 Dec 2016 09:31:34 CET
if [ $# -lt 2 ] || [ $# -gt 4 ]; then
echo "Runs the nightly builds by setting environment variables and"
echo "proceeding in the order establish in the provided text file"
echo "usage: $0 <order.txt> <python-version> [<conda-root> [<env-name>]]"
echo "example: $0 core.txt 2.7 #ci paths and structure"
echo "example: $0 core.txt 2.7 /opt/conda #changes conda root"
echo "example: $0 core.txt 2.7 /opt/conda bob-devel-py27 #be specific"
exit 1
fi
# Some required variables
PYTHON_VERSION=$2
basedir=`pwd`
# Download stock build scripts
if [ ! -d _ci ]; then
mkdir _ci
curl --silent "https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/install.sh" > ${basedir}/_ci/install.sh
chmod 755 ${basedir}/_ci/install.sh
${basedir}/_ci/install.sh _ci
fi
# Defines some required defaults, if they are not set by the CI
CI_PROJECT_DIR="${CI_PROJECT_DIR:-${basedir}}"
CI_PROJECT_NAME="${CI_PROJECT_NAME:-$(basename ${basedir})}"
CI_PROJECT_NAMESPACE="${CI_PROJECT_NAMESPACE:-bob}"
CI_PROJECT_PATH="${CI_PROJECT_PATH:-${CI_PROJECT_NAMESPACE}/${CI_PROJECT_NAME}}"
CI_BUILD_REF="${CI_BUILD_REF:-12345678}"
CI_BUILD_REF_NAME="${CI_BUILD_REF_NAME:-@local}"
CI_PROJECT_URL="${CI_PROJECT_URL:-https://gitlab.idiap.ch/${CI_PROJECT_PATH}}"
PYPIUSER="${PYPIUSER:-unknown}"
PYPIPASS="${PYPIUSER:-unknown}"
DOCUSER="${PYPIUSER:-unknown}"
DOCPASS="${PYPIUSER:-unknown}"
# The way to clone the repositories for this build
GITLAB_CHECKOUT_STRATEGY="git@gitlab.idiap.ch:"
if [ -n "$3" ]; then
CONDA_FOLDER=$3;
else
CONDA_FOLDER=`which conda`
if [ ! -z "${CONDA_FOLDER}" ]; then
CONDA_FOLDER=$(dirname $(dirname ${CONDA_FOLDER}))
else
# defaults to /opt/conda if nothing is obviously set
CONDA_FOLDER=/opt/conda
fi
fi
if [ -n "$4" ]; then
CONDA_ENV=$4;
export CONDA_ENV
fi
# Source useful functions
source ${basedir}/_ci/functions.sh
# Gets the latest stable version of a given package or returns the empty string
# $1: work directory for the package checkout
latest_stable () {
git -C $1 fetch --tags
echo `git -C $1 tag --sort='v:refname' | grep -e 'v[0-9]*\.[0-9]*\.[0-9]*$' | tail -n 1`
}
PREFIX=${basedir}/build-prefix
export_env PREFIX
if [ -e "${PREFIX}" ]; then
run_cmd rm -rf ${PREFIX}
fi
# Does base environment installation
run_cmd ${CONDA_FOLDER}/bin/conda create --yes -p ${PREFIX} --clone ${CONDA_ENV}
# Checks some extra settings
check_env CI_PROJECT_URL
# Overrides some stuff
if [ "${CI_BUILD_REF_NAME}" == "@local" ]; then
export_env CONDA_FOLDER
fi
log_info "Exporting variables for subsequent builds..."
export_env CI_PROJECT_DIR
export_env CI_PROJECT_NAME
export_env CI_PROJECT_NAMESPACE
export_env CI_PROJECT_PATH
export_env CI_BUILD_REF
export_env CI_BUILD_REF_NAME
export_env CI_PROJECT_URL
export_env PYPIUSER
export_env PYPIPASS
export_env DOCUSER
export_env DOCPASS
export_env PYTHON_VERSION
# Prepares dist container
DIST_DIR=${basedir}/dist
if [ ! -d ${DIST_DIR} ]; then
run_cmd mkdir -pv ${DIST_DIR}
run_cmd rm -fv ${DIST_DIR}/*
fi
# Prefer locally built indexes for documentation to avoid pinging the
# remote server (faster and more reliable)
run_cmd mkdir -p ${DIST_DIR}/docs
BOB_DOCUMENTATION_SERVER="file://${DIST_DIR}/docs/%s/"
export_env BOB_DOCUMENTATION_SERVER
# Place all sources under src
run_cmd mkdir -p ${basedir}/src
total=`cat $1 | sed -e '/^\s*#.*/d;/^\s*$/d' | wc -l | xargs echo -n`
log_info "Running builds for all ${total} packages in \`$1'..."
current=0
# Reads input file, avoids empty lines or lines starting with #
for f in `cat $1 | sed -e '/^\s*#.*/d;/^\s*$/d'`; do
((current++))
if [ -e ${DIST_DIR}/${f}-*.whl ]; then
echo "";
log_debug " ==========================================================";
log_debug " Installing $f (${current}/${total})...";
log_debug " ==========================================================";
echo "";
run_cmd ${PREFIX}/bin/pip install --find-links ${DIST_DIR} --use-wheel --no-index ${f}
continue;
fi
run_cmd cd ${basedir}
# Prints a clear marker to distinguish different packages
echo "";
log_debug " ==========================================================";
log_debug " Building $f (${current}/${total})...";
log_debug " ==========================================================";
echo "";
# Setup project variables
CI_PROJECT_DIR=${basedir}/src/${f}
CI_PROJECT_PATH=${CI_PROJECT_NAMESPACE}/${f}
CI_PROJECT_NAME=${f}
WHEELS_REPOSITORY=${DIST_DIR}
export_env CI_PROJECT_DIR
export_env CI_PROJECT_PATH
export_env CI_PROJECT_NAME
export_env WHEELS_REPOSITORY
if [ -d "${CI_PROJECT_DIR}" ]; then
run_cmd git -C ${CI_PROJECT_DIR} pull #updates
run_cmd git -C ${CI_PROJECT_DIR} clean -ffdx #rebuilds
else
run_cmd git clone --depth 1 ${GITLAB_CHECKOUT_STRATEGY}${CI_PROJECT_PATH} ${CI_PROJECT_DIR}
fi
if [ -n "${STABLE}" ]; then
CI_BUILD_TAG=`latest_stable ${CI_PROJECT_DIR}`
if [ -z "${CI_BUILD_TAG}" ]; then
echo "";
log_debug " ==========================================================";
log_debug " Skipping $f (${current}/${total}). No stable release";
log_debug " ==========================================================";
echo "";
continue;
fi
export_env CI_BUILD_TAG
run_cmd git -C ${CI_PROJECT_DIR} checkout ${CI_BUILD_TAG}
fi
run_cmd cd ${CI_PROJECT_DIR}
run_cmd ln -s ${basedir}/_ci .
# Define WHEEL_TAG if defined on the package's CI yaml file
if [ $(grep -c WHEEL_TAG ${CI_PROJECT_DIR}/.gitlab-ci.yml) -gt 0 ]; then
# Python-only package, need to define the WHEEL_TAG
case ${PYTHON_VERSION} in
2.7)
WHEEL_TAG=py27
;;
3.*)
WHEEL_TAG=py3
;;
*)
log_error "Python version ${PYTHON_VERSION} is unsupported";
exit 1
;;
esac
export_env WHEEL_TAG
else
unset WHEEL_TAG
fi
# Runs the whole shebang
run_cmd ./_ci/before_build.sh
run_cmd ./_ci/build.sh
run_cmd ./_ci/before_test.sh
run_cmd ./_ci/test.sh
# Copies the generated wheel to our provisional wheels repository
run_cmd cp -fv dist/*.whl ${DIST_DIR}/
# Copies the documentation so it can be subsequently used
if [ -d ${CI_PROJECT_DIR}/sphinx ]; then
run_cmd cp -r ${CI_PROJECT_DIR}/sphinx ${DIST_DIR}/docs/${f}
fi
done
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment