diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 1f4b37e16985089c19695804237e8512abb2720e..b581c40f48600f2f5bc35558d7a98215fc4890b8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,256 +1,21 @@
stages:
- - core-build
- - core-wheels
- - extra-build
- - extra-wheels
+ - core
+ - extra
-
-# ---------
-# Templates
-# ---------
-
-.core_build_template: &core_build_job
- stage: core-build
- before_script:
- - git clean -ffdx
- script:
- - ./build.sh core.txt ${PYTHON_VERSION}
- artifacts:
- expire_in: 1 week
- paths:
- - _ci/
- - dist/
-
-.core_wheels_template: &core_wheels_job
- stage: core-wheels
- only:
- - master
- script:
- - ./_ci/wheels.sh
-
-.extra_build_template: &extra_build_job
- stage: extra-build
- script:
- - ./build.sh extra.txt ${PYTHON_VERSION}
- artifacts:
- expire_in: 1 week
- paths:
- - _ci/
- - dist/
-
-.extra_wheels_template: &extra_wheels_job
- stage: extra-wheels
- only:
- - master
+core_build:
+ stage: core
script:
- - ./_ci/wheels.sh
-
-
-# -------------
-# Build Targets
-# -------------
-
-
-# Python 2.7 on Linux
-core_build_linux_27:
- <<: *core_build_job
- variables: &core_build_linux_27_variables
- PYTHON_VERSION: "2.7"
- tags:
- - conda-linux
-
-core_wheels_linux_27:
- <<: *core_wheels_job
- variables: *core_build_linux_27_variables
- dependencies:
- - core_build_linux_27
- tags:
- - conda-linux
-
-extra_build_linux_27:
- <<: *extra_build_job
- variables: *core_build_linux_27_variables
- dependencies:
- - core_build_linux_27
+ - pip install --upgrade python-gitlab
+ - python build.py core.txt 1
+ image: python
tags:
- - conda-linux
+ - docker
-extra_wheels_linux_27:
- <<: *extra_wheels_job
- variables: *core_build_linux_27_variables
- dependencies:
- - extra_build_linux_27
- tags:
- - conda-linux
-
-
-# Python 3.5 on Linux
-core_build_linux_35:
- <<: *core_build_job
- variables: &core_build_linux_35_variables
- PYTHON_VERSION: "3.5"
- tags:
- - conda-linux
-
-core_wheels_linux_35:
- <<: *core_wheels_job
- variables: *core_build_linux_35_variables
- dependencies:
- - core_build_linux_35
- tags:
- - conda-linux
-
-extra_build_linux_35:
- <<: *extra_build_job
- variables: *core_build_linux_35_variables
- dependencies:
- - core_build_linux_35
- tags:
- - conda-linux
-
-extra_wheels_linux_35:
- <<: *extra_wheels_job
- variables: *core_build_linux_35_variables
- dependencies:
- - extra_build_linux_35
- tags:
- - conda-linux
-
-
-# Python 3.6 on Linux
-core_build_linux_36:
- <<: *core_build_job
- variables: &core_build_linux_36_variables
- PYTHON_VERSION: "3.6"
- only:
- - master
- tags:
- - conda-linux
-
-core_wheels_linux_36:
- <<: *core_wheels_job
- variables: *core_build_linux_36_variables
- dependencies:
- - core_build_linux_36
- tags:
- - conda-linux
-
-extra_build_linux_36:
- <<: *extra_build_job
- variables: *core_build_linux_36_variables
- dependencies:
- - core_build_linux_36
- only:
- - master
- tags:
- - conda-linux
-
-extra_wheels_linux_36:
- <<: *extra_wheels_job
- variables: *core_build_linux_36_variables
- dependencies:
- - extra_build_linux_36
- tags:
- - conda-linux
-
-
-# Python 2.7 on MacOSX
-core_build_macosx_27:
- <<: *core_build_job
- variables: &core_build_macosx_27_variables
- PYTHON_VERSION: "2.7"
- tags:
- - conda-macosx
-
-core_wheels_macosx_27:
- <<: *core_wheels_job
- variables: *core_build_macosx_27_variables
- dependencies:
- - core_build_macosx_27
- tags:
- - conda-macosx
-
-extra_build_macosx_27:
- <<: *extra_build_job
- variables: *core_build_macosx_27_variables
- dependencies:
- - core_build_macosx_27
- tags:
- - conda-macosx
-
-extra_wheels_macosx_27:
- <<: *extra_wheels_job
- variables: *core_build_macosx_27_variables
- dependencies:
- - extra_build_macosx_27
- tags:
- - conda-macosx
-
-
-# Python 3.5 on MacOSX
-core_build_macosx_35:
- <<: *core_build_job
- variables: &core_build_macosx_35_variables
- PYTHON_VERSION: "3.5"
- tags:
- - conda-macosx
-
-core_wheels_macosx_35:
- <<: *core_wheels_job
- variables: *core_build_macosx_35_variables
- dependencies:
- - core_build_macosx_35
- tags:
- - conda-macosx
-
-extra_build_macosx_35:
- <<: *extra_build_job
- variables: *core_build_macosx_35_variables
- dependencies:
- - core_build_macosx_35
- tags:
- - conda-macosx
-
-extra_wheels_macosx_35:
- <<: *extra_wheels_job
- variables: *core_build_macosx_35_variables
- dependencies:
- - extra_build_macosx_35
- tags:
- - conda-macosx
-
-# Python 3.6 on MacOSX
-core_build_macosx_36:
- <<: *core_build_job
- variables: &core_build_macosx_36_variables
- PYTHON_VERSION: "3.6"
- only:
- - master
- tags:
- - conda-macosx
-
-core_wheels_macosx_36:
- <<: *core_wheels_job
- variables: *core_build_macosx_36_variables
- dependencies:
- - core_build_macosx_36
- tags:
- - conda-macosx
-
-extra_build_macosx_36:
- <<: *extra_build_job
- variables: *core_build_macosx_36_variables
- dependencies:
- - core_build_macosx_36
- only:
- - master
- tags:
- - conda-macosx
-
-extra_wheels_macosx_36:
- <<: *extra_wheels_job
- variables: *core_build_macosx_36_variables
- dependencies:
- - extra_build_macosx_36
+extra_build:
+ stage: core
+ script:
+ - pip install --upgrade python-gitlab
+ - python build.py extra.txt 0
+ image: python
tags:
- - conda-macosx
+ - docker
diff --git a/build.py b/build.py
new file mode 100644
index 0000000000000000000000000000000000000000..8c22b7d5d0b4094cc3c7d4eacc9944ee1f9175ef
--- /dev/null
+++ b/build.py
@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+import os
+import gitlab
+import time
+
+
+def main(pkg_list_file, fail=True):
+
+ private_token = os.environ['GITLAB_API_TOKEN']
+ gl = gitlab.Gitlab('https://gitlab.idiap.ch',
+ private_token=private_token, api_version=4)
+ with open(pkg_list_file) as f:
+ for line in f:
+ line = line.strip()
+ if not line or line.startswith('#'):
+ continue
+ pkg = line.split('#')[0]
+ project = gl.projects.get('bob/{}'.format(pkg))
+ pipeline = project.pipelines.create({'ref': 'master'})
+ while pipeline.status in ('pending', 'running'):
+ time.sleep(10)
+ pipeline = project.pipelines.get(pipeline.id)
+ if pipeline.status != 'success':
+ message = "Pipeline {} for {} with status {}".format(
+ pipeline.id, pkg, pipeline.status)
+ if fail:
+ raise RuntimeError(message)
+ else:
+ print(message)
+
+
+if __name__ == '__main__':
+ import sys
+ pkg_list_file = sys.argv[1]
+ pkg_list_file = sys.argv[1]
+ fail = bool(sys.argv[2])
+ main(pkg_list_file, fail)
diff --git a/build.sh b/build.sh
deleted file mode 100755
index d726d3638ddde4684b0db20eb59750e053e2ef75..0000000000000000000000000000000000000000
--- a/build.sh
+++ /dev/null
@@ -1,214 +0,0 @@
-#!/usr/bin/env bash
-# Andre Anjos <andre.anjos@idiap.ch>
-# Thu 22 Dec 2016 09:31:34 CET
-
-if [ $# -lt 2 ] || [ $# -gt 4 ]; then
- echo "Runs the nightly builds by setting environment variables and"
- echo "proceeding in the order establish in the provided text file"
- echo "usage: $0 <order.txt> <python-version> [<conda-root> [<env-name>]]"
- echo "example: $0 core.txt 2.7 #ci paths and structure"
- echo "example: $0 core.txt 2.7 /opt/conda #changes conda root"
- echo "example: $0 core.txt 2.7 /opt/conda bob-devel-py27 #be specific"
- exit 1
-fi
-
-# Some required variables
-PYTHON_VERSION=$2
-basedir=`pwd`
-
-# Download stock build scripts
-if [ ! -d _ci ]; then
- mkdir _ci
- curl --silent "https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/install.sh" > ${basedir}/_ci/install.sh
- chmod 755 ${basedir}/_ci/install.sh
- ${basedir}/_ci/install.sh _ci
-fi
-
-# Defines some required defaults, if they are not set by the CI
-CI_PROJECT_DIR="${CI_PROJECT_DIR:-${basedir}}"
-CI_PROJECT_NAME="${CI_PROJECT_NAME:-$(basename ${basedir})}"
-CI_PROJECT_NAMESPACE="${CI_PROJECT_NAMESPACE:-bob}"
-CI_PROJECT_PATH="${CI_PROJECT_PATH:-${CI_PROJECT_NAMESPACE}/${CI_PROJECT_NAME}}"
-CI_BUILD_REF="${CI_BUILD_REF:-12345678}"
-CI_BUILD_REF_NAME="${CI_BUILD_REF_NAME:-@local}"
-CI_PROJECT_URL="${CI_PROJECT_URL:-https://gitlab.idiap.ch/${CI_PROJECT_PATH}}"
-PYPIUSER="${PYPIUSER:-unknown}"
-PYPIPASS="${PYPIUSER:-unknown}"
-DOCUSER="${PYPIUSER:-unknown}"
-DOCPASS="${PYPIUSER:-unknown}"
-
-# The way to clone the repositories for this build
-GITLAB_CHECKOUT_STRATEGY="git@gitlab.idiap.ch:"
-
-if [ -n "$3" ]; then
- CONDA_FOLDER=$3;
-else
- CONDA_FOLDER=`which conda`
- if [ ! -z "${CONDA_FOLDER}" ]; then
- CONDA_FOLDER=$(dirname $(dirname ${CONDA_FOLDER}))
- else
- # defaults to /opt/conda if nothing is obviously set
- CONDA_FOLDER=/opt/conda
- fi
-fi
-
-if [ -n "$4" ]; then
- CONDA_ENV=$4;
- export CONDA_ENV
-fi
-
-# Source useful functions
-source ${basedir}/_ci/functions.sh
-
-# Gets the latest stable version of a given package or returns the empty string
-# $1: work directory for the package checkout
-latest_stable () {
- git -C $1 fetch --tags
- echo `git -C $1 tag --sort='v:refname' | grep -e 'v[0-9]*\.[0-9]*\.[0-9]*$' | tail -n 1`
-}
-
-PREFIX=${basedir}/build-prefix
-export_env PREFIX
-if [ -e "${PREFIX}" ]; then
- run_cmd rm -rf ${PREFIX}
-fi
-# Does base environment installation
-run_cmd ${CONDA_FOLDER}/bin/conda create --yes -p ${PREFIX} --clone ${CONDA_ENV}
-
-# Checks some extra settings
-check_env CI_PROJECT_URL
-
-# Overrides some stuff
-if [ "${CI_BUILD_REF_NAME}" == "@local" ]; then
- export_env CONDA_FOLDER
-fi
-
-log_info "Exporting variables for subsequent builds..."
-export_env CI_PROJECT_DIR
-export_env CI_PROJECT_NAME
-export_env CI_PROJECT_NAMESPACE
-export_env CI_PROJECT_PATH
-export_env CI_BUILD_REF
-export_env CI_BUILD_REF_NAME
-export_env CI_PROJECT_URL
-export_env PYPIUSER
-export_env PYPIPASS
-export_env DOCUSER
-export_env DOCPASS
-export_env PYTHON_VERSION
-
-# Prepares dist container
-DIST_DIR=${basedir}/dist
-if [ ! -d ${DIST_DIR} ]; then
- run_cmd mkdir -pv ${DIST_DIR}
- run_cmd rm -fv ${DIST_DIR}/*
-fi
-
-# Prefer locally built indexes for documentation to avoid pinging the
-# remote server (faster and more reliable)
-run_cmd mkdir -p ${DIST_DIR}/docs
-BOB_DOCUMENTATION_SERVER="file://${DIST_DIR}/docs/%s/"
-export_env BOB_DOCUMENTATION_SERVER
-
-# Place all sources under src
-run_cmd mkdir -p ${basedir}/src
-
-total=`cat $1 | sed -e '/^\s*#.*/d;/^\s*$/d' | wc -l | xargs echo -n`
-
-log_info "Running builds for all ${total} packages in \`$1'..."
-
-current=0
-
-# Reads input file, avoids empty lines or lines starting with #
-for f in `cat $1 | sed -e '/^\s*#.*/d;/^\s*$/d'`; do
-
- ((current++))
-
- if [ -e ${DIST_DIR}/${f}-*.whl ]; then
- echo "";
- log_debug " ==========================================================";
- log_debug " Installing $f (${current}/${total})...";
- log_debug " ==========================================================";
- echo "";
- run_cmd ${PREFIX}/bin/pip install --find-links ${DIST_DIR} --use-wheel --no-index ${f}
- continue;
- fi
-
- run_cmd cd ${basedir}
-
- # Prints a clear marker to distinguish different packages
- echo "";
- log_debug " ==========================================================";
- log_debug " Building $f (${current}/${total})...";
- log_debug " ==========================================================";
- echo "";
-
- # Setup project variables
- CI_PROJECT_DIR=${basedir}/src/${f}
- CI_PROJECT_PATH=${CI_PROJECT_NAMESPACE}/${f}
- CI_PROJECT_NAME=${f}
- WHEELS_REPOSITORY=${DIST_DIR}
-
- export_env CI_PROJECT_DIR
- export_env CI_PROJECT_PATH
- export_env CI_PROJECT_NAME
- export_env WHEELS_REPOSITORY
-
- if [ -d "${CI_PROJECT_DIR}" ]; then
- run_cmd git -C ${CI_PROJECT_DIR} pull #updates
- run_cmd git -C ${CI_PROJECT_DIR} clean -ffdx #rebuilds
- else
- run_cmd git clone --depth 1 ${GITLAB_CHECKOUT_STRATEGY}${CI_PROJECT_PATH} ${CI_PROJECT_DIR}
- fi
-
- if [ -n "${STABLE}" ]; then
- CI_BUILD_TAG=`latest_stable ${CI_PROJECT_DIR}`
- if [ -z "${CI_BUILD_TAG}" ]; then
- echo "";
- log_debug " ==========================================================";
- log_debug " Skipping $f (${current}/${total}). No stable release";
- log_debug " ==========================================================";
- echo "";
- continue;
- fi
- export_env CI_BUILD_TAG
- run_cmd git -C ${CI_PROJECT_DIR} checkout ${CI_BUILD_TAG}
- fi
-
- run_cmd cd ${CI_PROJECT_DIR}
- run_cmd ln -s ${basedir}/_ci .
-
- # Define WHEEL_TAG if defined on the package's CI yaml file
- if [ $(grep -c WHEEL_TAG ${CI_PROJECT_DIR}/.gitlab-ci.yml) -gt 0 ]; then
- # Python-only package, need to define the WHEEL_TAG
- case ${PYTHON_VERSION} in
- 2.7)
- WHEEL_TAG=py27
- ;;
- 3.*)
- WHEEL_TAG=py3
- ;;
- *)
- log_error "Python version ${PYTHON_VERSION} is unsupported";
- exit 1
- ;;
- esac
- export_env WHEEL_TAG
- else
- unset WHEEL_TAG
- fi
-
- # Runs the whole shebang
- run_cmd ./_ci/before_build.sh
- run_cmd ./_ci/build.sh
- run_cmd ./_ci/before_test.sh
- run_cmd ./_ci/test.sh
-
- # Copies the generated wheel to our provisional wheels repository
- run_cmd cp -fv dist/*.whl ${DIST_DIR}/
- # Copies the documentation so it can be subsequently used
- if [ -d ${CI_PROJECT_DIR}/sphinx ]; then
- run_cmd cp -r ${CI_PROJECT_DIR}/sphinx ${DIST_DIR}/docs/${f}
- fi
-
-done