diff --git a/.gitignore b/.gitignore
index 0117b7c35d7db5aa24e245317a384e465d8f26ec..ac2f3277d9129aa2caf76606814f1457007f8c9f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,5 +2,5 @@
*.swp
_ci/
src/
-dist/
-build-prefix/
+miniconda.sh
+miniconda/
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ccd8867baf5c3f89d87df6f38c466f76d943280e..c0ffa4c3b4b18e537214253480d45c814e2e4811 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,24 +1,57 @@
-stages:
- - core
- - extra
-
variables:
+ CONDA_ROOT: "${CI_PROJECT_DIR}/miniconda"
+ BOB_PACKAGE_VERSION: "unknown"
PYTHONUNBUFFERED: 1
-core_build:
- stage: core
+stages:
+ - build
+
+.build_template: &build_job
+ stage: build
+ before_script:
+ - mkdir _ci
+ - curl --silent "https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/install.sh" > _ci/install.sh
+ - chmod 755 _ci/install.sh
+ - ./_ci/install.sh _ci master #installs ci support scripts
+ - ./_ci/before_build.sh
script:
- - pip install --upgrade python-gitlab
- - python build.py core.txt 1
- image: python
+ - ./scripts/build.sh
+ after_script:
+ - ./_ci/after_build.sh
+ cache: &build_caches
+ key: "$CI_BUILD_NAME"
+ paths:
+ - miniconda.sh
+ - ${CONDA_ROOT}/pkgs/*.tar.bz2
+ - ${CONDA_ROOT}/pkgs/urls.txt
+ - ${CONDA_ROOT}/conda-bld/src_cache
+
+linux_27:
+ <<: *build_job
+ variables:
+ PYTHON_VERSION: "2.7"
tags:
- docker
+ image: continuumio/conda-concourse-ci
-extra_build:
- stage: extra
- script:
- - pip install --upgrade python-gitlab
- - python build.py extra.txt 0
- image: python
+linux_36:
+ <<: *build_job
+ variables:
+ PYTHON_VERSION: "3.6"
tags:
- docker
+ image: continuumio/conda-concourse-ci
+
+macosx_27:
+ <<: *build_job
+ variables:
+ PYTHON_VERSION: "2.7"
+ tags:
+ - macosx
+
+macosx_36:
+ <<: *build_job
+ variables:
+ PYTHON_VERSION: "3.6"
+ tags:
+ - macosx
diff --git a/README.rst b/README.rst
index 5bd03847af4563683299ff4db52b2743e10d1a3c..75e0679f1189711b4712174abade0b690a12d3d4 100644
--- a/README.rst
+++ b/README.rst
@@ -1,6 +1,6 @@
.. vim: set fileencoding=utf-8 :
.. Andre Anjos <andre.anjos@idiap.ch>
-.. Thu 22 Dec 2016 08:47:14 CET
+.. Sat Feb 17 14:26:47 2018 CET
.. image:: https://gitlab.idiap.ch/bob/bob.nightlies/badges/master/build.svg
:target: https://gitlab.idiap.ch/bob/bob.nightlies/commits/master
@@ -18,20 +18,14 @@ latest environment and tested together under that light.
Installation
------------
-This package can be built starting from a base development environment, as
-provided by our `from-scratch.sh`_ script. First run it to create a conda-based
-environment for development. Once the environment is ready, follow the
-instructions on the ``.gitlab-ci.yml`` file to complete the development cycle.
+This package can be built completely from scratch by following the instructions
+on the ``.gitlab-ci.yml`` file.
The script ``build.sh`` will try to execute the steps from the CI in the right
order and can be used to locally test your modifications **before** pushing a
branch into the nightlies, which will trigger a complete set of builds in
multiple platforms.
-The script ``build.sh`` takes several parameters (run it for a help print out)::
-
- $ ./build.sh
-
Updating a Package
------------------
@@ -40,7 +34,7 @@ Updating a Package
Before adding a package to this prototype, please ensure that the package
contains all standard components of a Bob_ package (README, unit tests, CI
- integration and documentation among others).
+ integration, documentation and conda build recipe among others).
If you don't know how to do this, ask for information on our `mailing
list`_.
@@ -55,8 +49,8 @@ The new package sits in one of two categories:
to the order in which packages should be compiled.
Once the file is modified, commit the changes to a **new branch**, with a
-suggestive name. Push the branch and, once it is green, you may merge it to the
-master (or ask it to be merged), via a standard merge request.
+suggestive name. Push the branch and merge it to the master (or ask it to be
+merged), via a standard merge request.
Contact
diff --git a/build.py b/build.py
deleted file mode 100644
index 7791109d781bc870c0172ef7457c2fe335632632..0000000000000000000000000000000000000000
--- a/build.py
+++ /dev/null
@@ -1,61 +0,0 @@
-#!/usr/bin/env python
-import os
-import gitlab
-import time
-from datetime import datetime
-
-
-def wait_for_pipeline(project, pipeline):
- time.sleep(15)
- pipeline = project.pipelines.get(pipeline.id)
- while pipeline.status in ('pending', 'running'):
- time.sleep(15)
- pipeline = project.pipelines.get(pipeline.id)
- return pipeline
-
-
-def main(pkg_list_file, fail=True):
-
- private_token = os.environ['GITLAB_API_TOKEN']
- gl = gitlab.Gitlab('https://gitlab.idiap.ch',
- private_token=private_token, api_version=4)
- any_failed = False
- with open(pkg_list_file) as f:
- for line in f:
- line = line.strip()
- if not line or line.startswith('#'):
- continue
- pkg = line.split('#')[0]
- project = gl.projects.get('bob/{}'.format(pkg))
- pipeline = project.pipelines.create({'ref': 'master'})
- message = ("{date} Triggered pipeline: " +
- "https://gitlab.idiap.ch/bob/{pkg}/pipelines/{id}")
- print(message.format(date=datetime.now(), pkg=pkg, id=pipeline.id))
-
- pipeline = wait_for_pipeline(project, pipeline)
-
- # re-try once if the pipeline fails
- if pipeline.status != 'success':
- print("Retrying the pipeline.")
- pipeline.retry()
- pipeline = wait_for_pipeline(project, pipeline)
-
- if pipeline.status != 'success':
- any_failed = True
- message = ("Pipeline: https://gitlab.idiap.ch/bob/{pkg}/" +
- "pipelines/{id} failed with status {status}")
- message = message.format(
- id=pipeline.id, pkg=pkg, status=pipeline.status)
- if fail:
- raise RuntimeError(message)
- else:
- print(message)
- if any_failed:
- raise RuntimeError("Failed to build some of the packages.")
-
-
-if __name__ == '__main__':
- import sys
- pkg_list_file = sys.argv[1]
- fail = bool(int(sys.argv[2]))
- main(pkg_list_file, fail)
diff --git a/buildout.cfg.template b/buildout.cfg.template
deleted file mode 100644
index 5925fda712bd453b15d87d56998b4e1b30a2bf1d..0000000000000000000000000000000000000000
--- a/buildout.cfg.template
+++ /dev/null
@@ -1,18 +0,0 @@
-[buildout]
-parts = scripts
-extensions = bob.buildout
- mr.developer
-auto-checkout = *
-debug = true
-newest = false
-verbose = true
-eggs = {eggs}
-
-develop = {develop}
-
-[scripts]
-recipe = bob.buildout:scripts
-dependent-scripts = true
-
-[sources]
-{sources}
diff --git a/core.txt b/core.txt
deleted file mode 100644
index 1f64da0418a07a2af9302edf877c40c14343c5c5..0000000000000000000000000000000000000000
--- a/core.txt
+++ /dev/null
@@ -1,33 +0,0 @@
-bob.buildout
-bob.extension
-bob.blitz
-bob.core
-bob.io.base
-bob.math
-bob.measure
-bob.io.image
-bob.db.base
-bob.io.video
-bob.io.matlab
-bob.io.audio
-bob.sp
-bob.ap
-bob.ip.base
-bob.ip.color
-bob.ip.draw
-bob.ip.gabor
-bob.learn.activation
-bob.learn.libsvm
-bob.learn.linear
-bob.learn.mlp
-bob.learn.boosting
-bob.db.iris
-bob.learn.em
-bob.db.wine
-bob.db.mnist
-bob.db.atnt
-bob.ip.facedetect
-bob.ip.optflow.hornschunck
-bob.ip.optflow.liu
-bob.ip.flandmark
-bob
diff --git a/generate_buildout_config b/generate_buildout_config
deleted file mode 100755
index 3a836703f36ce6b5e97f14a289a579d59cec2a9c..0000000000000000000000000000000000000000
--- a/generate_buildout_config
+++ /dev/null
@@ -1,55 +0,0 @@
-#!/usr/bin/env python
-"""Generates a buildout configuration file based on the list of the packages
-
-Usage:
- %(prog)s <package_list>...
- %(prog)s --help
-
-Arguments:
- <package_list> The files containing the package list. Each line should
- contain only the name of one package. Empty lines and
- comments starting with # are allowed.
-
-Options:
- -h --help Show this help message and exit
-"""
-
-
-def format_list(package_list, prepend, left_fill):
- text = prepend + package_list[0] + '\n'
- package_list = package_list[1:]
- for pkg in package_list:
- text += ' ' * left_fill + prepend + pkg + '\n'
- return text
-
-
-def main(argv=None):
- from docopt import docopt
- import sys
- import os
- docs = __doc__ % {'prog': os.path.basename(sys.argv[0])}
- args = docopt(docs, argv=argv)
- package_list_files = args['<package_list>']
- package_list = []
- for path in package_list_files:
- for line in open(path):
- line = line.partition('#')[0].strip()
- if not line:
- continue
- package_list.append(line)
- print(package_list)
- # read the buildout config template
- with open("buildout.cfg.template") as f:
- template = f.read()
- # eggs
- eggs = format_list(package_list, '', 7)
- develop = format_list(package_list, 'src/', 10)
- source_list = ['{0} = git git@gitlab.idiap.ch:bob/{0}'.format(p)
- for p in package_list]
- sources = format_list(source_list, '', 0)
- with open('buildout.cfg', 'wt') as f:
- f.write(template.format(eggs=eggs, develop=develop, sources=sources))
-
-
-if __name__ == '__main__':
- main()
diff --git a/extra.txt b/order.txt
similarity index 69%
rename from extra.txt
rename to order.txt
index b586a988e7e9e0eb28e4445a0d06393c83fd2473..c90c37360a721a9a47cb41f704ecedd2d670d82d 100644
--- a/extra.txt
+++ b/order.txt
@@ -1,3 +1,36 @@
+bob.buildout
+bob.extension
+bob.blitz
+bob.core
+bob.io.base
+bob.math
+bob.measure
+bob.io.image
+bob.db.base
+bob.io.video
+bob.io.matlab
+bob.io.audio
+bob.sp
+bob.ap
+bob.ip.base
+bob.ip.color
+bob.ip.draw
+bob.ip.gabor
+bob.learn.activation
+bob.learn.libsvm
+bob.learn.linear
+bob.learn.mlp
+bob.learn.boosting
+bob.db.iris
+bob.learn.em
+bob.db.wine
+bob.db.mnist
+bob.db.atnt
+bob.ip.facedetect
+bob.ip.optflow.hornschunck
+bob.ip.optflow.liu
+bob.ip.flandmark
+bob
gridtk
bob.ip.qualitymeasure
bob.ip.skincolorfilter
diff --git a/scripts/build.sh b/scripts/build.sh
new file mode 100755
index 0000000000000000000000000000000000000000..f75fc1a6bd459febbba5a931734dce2b08976dcc
--- /dev/null
+++ b/scripts/build.sh
@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+# Thu 15 Feb 15:36:30 2018 CET
+
+export PYTHON_VERSION="${PYTHON_VERSION:-${1}}"
+if [ -z "${PYTHON_VERSION}" ]; then
+ echo "usage: $0 <python-version>"
+ exit 1
+fi
+
+scriptdir="$(cd $(dirname ${BASH_SOURCE[0]}) && pwd)"
+basedir=`pwd`
+
+# These should be set by the CI - resetting in case of running from cmdline
+export CONDA_ROOT="${CONDA_ROOT:-${basedir}/miniconda}"
+export BOB_PACKAGE_VERSION="${BOB_PACKAGE_VERSION:-unknown}"
+export PYTHONUNBUFFERED="${PYTHONUNBUFFERED:-1}"
+export CI_PROJECT_URL="${CI_PROJECT_URL:-https://gitlab.idiap.ch/bob/bob.nightlies}"
+
+# This should be installed by CI, but installing it here in case running from
+# cmdline
+if [ ! -d "_ci" ]; then
+ mkdir _ci
+ curl --silent "https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/install.sh" > _ci/install.sh
+ chmod 755 _ci/install.sh
+ ./_ci/install.sh _ci master #installs ci support scripts
+fi
+
+# If not running on the CI, set docserver
+if [ -z "${DOCPASS}" ]; then
+ export DOCSERVER=https://www.idiap.ch
+ export CONDA_CHANNEL=https://www.idiap.ch/software/bob/conda/label/main
+ export CONDA_BETA_CHANNEL=http://beatubulatest.lab.idiap.ch/private/conda
+fi
+
+if [ ! -d "${CONDA_ROOT}" ]; then
+ ./_ci/before_build.sh
+fi
+
+source ${basedir}/_ci/functions.sh
+
+PACKAGES=($(sed -e "/^\s*#.*/d;/^\s*$/d" ${basedir}/order.txt))
+
+current=0
+
+for f in "${PACKAGES[@]}"; do
+
+ ((current++))
+
+ echo "";
+ log_debug " ==========================================================";
+ log_debug " Checking ${f} (${current}/${#PACKAGES[@]})...";
+ log_debug " ==========================================================";
+ echo "";
+
+ # First checks if all package tests pass
+ export BOB_TEST_ONLY="true"
+ ${scriptdir}/rebuild.sh ${CONDA_ROOT} ${PYTHON_VERSION} ${f}
+ unset BOB_TEST_ONLY
+
+ if [ $? -ne 0 ]; then
+ log_warn "Package ${f} check FAILED - rebuilding it..."
+ echo "";
+ log_info " ==========================================================";
+ log_info " Building ${f} (${current}/${#PACKAGES[@]})...";
+ log_info " ==========================================================";
+ echo "";
+ ${scriptdir}/rebuild.sh ${CONDA_ROOT} ${PYTHON_VERSION} ${f}
+ else
+ log_info "Package ${f} check SUCCESS - skipping rebuild"
+ fi
+
+done
diff --git a/scripts/rebuild.sh b/scripts/rebuild.sh
new file mode 100755
index 0000000000000000000000000000000000000000..a5ff6e4c47231b425726236dd16d0b67f011693a
--- /dev/null
+++ b/scripts/rebuild.sh
@@ -0,0 +1,128 @@
+#!/usr/bin/env bash
+# Mon 19 Feb 10:42:26 2018 CET
+
+if [ $# -ne 3 ]; then
+ echo "Completely rebuilds the given package"
+ echo "usage: $0 <conda> <python-version> <package-name>"
+ echo "example: $0 \$\{CONDA_ROOT\} 2.7 bob.blitz"
+ exit 1
+fi
+
+CONDA_ROOT=${1}
+PYTHON_VERSION=${2}
+CI_PROJECT_NAME=${3}
+
+# Defines some required defaults, if they are not set by the CI (these allow
+# local builds)
+basedir=`pwd`
+CI_PROJECT_NAMESPACE="${CI_PROJECT_NAMESPACE:-bob}"
+CI_PROJECT_PATH=${CI_PROJECT_NAMESPACE}/${CI_PROJECT_NAME}
+CI_PROJECT_DIR=${basedir}/src/${CI_PROJECT_NAME}
+CI_PROJECT_URL="https://gitlab.idiap.ch/${CI_PROJECT_PATH}"
+CI_COMMIT_REF="${CI_COMMIT_REF:-12345678}"
+CI_COMMIT_REF_NAME="${CI_COMMIT_REF_NAME:-@local}"
+
+# The way to clone the repositories for this build
+GITLAB_CHECKOUT_STRATEGY="git@gitlab.idiap.ch:"
+
+# Overrides some stuff
+if [ "${CI_COMMIT_REF_NAME}" == "@local" ]; then
+ BOB_PACKAGE_VERSION="unknown"
+ PYTHONUNBUFFERED=1
+fi
+
+source ${basedir}/_ci/functions.sh
+unset BOB_PACKAGE_VERSION
+
+# Setup project variables
+export_env CI_PROJECT_DIR
+export_env CI_PROJECT_NAME
+export_env CI_PROJECT_NAMESPACE
+export_env CI_PROJECT_PATH
+export_env CI_COMMIT_REF
+export_env CI_COMMIT_REF_NAME
+export_env CI_PROJECT_URL
+export_env CONDA_ROOT
+export_env PYTHONUNBUFFERED
+export_env PYTHON_VERSION
+
+export_env CI_PROJECT_DIR
+export_env CI_PROJECT_PATH
+export_env CI_PROJECT_NAME
+
+if [ -d "${CI_PROJECT_DIR}" ]; then
+ run_cmd git -C ${CI_PROJECT_DIR} pull #updates
+ run_cmd git -C ${CI_PROJECT_DIR} clean -ffdx #rebuilds
+else
+ if [ ! -d ${basedir}/src ]; then
+ run_cmd mkdir -p ${basedir}/src
+ fi
+ run_cmd git clone --depth 1 ${GITLAB_CHECKOUT_STRATEGY}${CI_PROJECT_PATH} ${CI_PROJECT_DIR}
+fi
+
+# Gets the latest stable version of a given package or returns the empty string
+# $1: work directory for the package checkout
+latest_stable () {
+ git -C $1 fetch --tags
+ echo `git -C $1 tag --sort='v:refname' | grep -e 'v[0-9]*\.[0-9]*\.[0-9]*$' | tail -n 1`
+}
+
+if [ -n "${STABLE}" ]; then
+ CI_COMMIT_TAG=`latest_stable ${CI_PROJECT_DIR}`
+ if [ -z "${CI_COMMIT_TAG}" ]; then
+ echo "";
+ log_debug " ==========================================================";
+ log_debug " Skipping $f (${current}/${total}). No stable release";
+ log_debug " ==========================================================";
+ echo "";
+ continue;
+ fi
+ export_env CI_COMMIT_TAG
+ run_cmd git -C ${CI_PROJECT_DIR} checkout ${CI_COMMIT_TAG}
+fi
+
+run_cmd cd ${CI_PROJECT_DIR}
+run_cmd ln -s ${basedir}/_ci .
+
+# Calculates package version
+if [ ! -r "version.txt" ]; then
+ log_error "./version.txt does not exist - cannot figure out version number"
+ exit 1
+fi
+BOB_PACKAGE_VERSION=`cat version.txt | tr -d '\n'`;
+export_env BOB_PACKAGE_VERSION
+
+# Calculates the current build-number available if in test-only mode
+if [ -n "${BOB_TEST_ONLY}" ]; then
+ run_cmd mkdir -p ./_ci/${OS_SLUG}/${PYTHON_VERSION}
+ if [ -z "${CI_COMMIT_TAG}" ]; then
+ run_cmd ${CONDA_ROOT}/bin/python _ci/channel_support.py ${CONDA_BETA_CHANNEL} ${CI_PROJECT_NAME} ${BOB_PACKAGE_VERSION} ${PYTHON_VERSION} -u --log ./_ci/${OS_SLUG}/${PYTHON_VERSION}/build_number.txt
+ else
+ run_cmd ${CONDA_ROOT}/bin/python _ci/channel_support.py ${CONDA_CHANNEL} ${CI_PROJECT_NAME} ${BOB_PACKAGE_VERSION} ${PYTHON_VERSION} -u --log ./_ci/${OS_SLUG}/${PYTHON_VERSION}/build_number.txt
+ fi
+
+ BOB_BUILD_NUMBER=`head -n 1 ./_ci/${OS_SLUG}/${PYTHON_VERSION}/build_number.txt | tr -d '\n'`
+ ((BOB_BUILD_NUMBER--)) #uses the latest available
+ export_env BOB_BUILD_NUMBER
+fi
+
+run_cmd ./_ci/build.sh
+
+if [ -n "${BOB_TEST_ONLY}" ]; then
+ # we don't keep stuff while testing only
+ run_cmd ${CONDA_ROOT}/bin/conda build purge
+else
+ # consider uploading...
+ if [[ "${IS_MASTER}" == "true" ]]; then
+ # Uploads all the built packages
+ for os in "osx-64" "noarch" "linux-64"; do
+ for f in ${CONDA_ROOT}/conda-bld/${os}/*.tar.bz2; do
+ if [[ -f $f ]]; then
+ dav_check_upload "${f}" "public-upload/conda/label/main/${os}/"
+ fi
+ done
+ done
+ else
+ log_info 'Skipping package upload (not on master branch)'
+ fi
+fi