New build strategy (closes #31)

.gitignore

@@ -2,5 +2,5 @@
 *.swp
 _ci/
 src/
-dist/
-build-prefix/
+miniconda.sh
+miniconda/

.gitlab-ci.yml

-stages:
-  - core
-  - extra
-
 variables:
+  CONDA_ROOT: "${CI_PROJECT_DIR}/miniconda"
+  BOB_PACKAGE_VERSION: "unknown"
   PYTHONUNBUFFERED: 1
 
-core_build:
-  stage: core
+stages:
+  - build
+
+.build_template: &build_job
+  stage: build
+  before_script:
+    - mkdir _ci
+    - curl --silent "https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/install.sh" > _ci/install.sh
+    - chmod 755 _ci/install.sh
+    - ./_ci/install.sh _ci master #installs ci support scripts
+    - ./_ci/before_build.sh
   script:
-    - pip install --upgrade python-gitlab
-    - python build.py core.txt 1
-  image: python
+    - ./scripts/build.sh
+  after_script:
+    - ./_ci/after_build.sh
+  cache: &build_caches
+    key: "$CI_BUILD_NAME"
+    paths:
+      - miniconda.sh
+      - ${CONDA_ROOT}/pkgs/*.tar.bz2
+      - ${CONDA_ROOT}/pkgs/urls.txt
+      - ${CONDA_ROOT}/conda-bld/src_cache
+
+linux_27:
+  <<: *build_job
+  variables:
+    PYTHON_VERSION: "2.7"
   tags:
     - docker
+  image: continuumio/conda-concourse-ci
 
-extra_build:
-  stage: extra
-  script:
-    - pip install --upgrade python-gitlab
-    - python build.py extra.txt 0
-  image: python
+linux_36:
+  <<: *build_job
+  variables:
+    PYTHON_VERSION: "3.6"
   tags:
     - docker
+  image: continuumio/conda-concourse-ci
+
+macosx_27:
+  <<: *build_job
+  variables:
+    PYTHON_VERSION: "2.7"
+  tags:
+    - macosx
+
+macosx_36:
+  <<: *build_job
+  variables:
+    PYTHON_VERSION: "3.6"
+  tags:
+    - macosx

README.rst

 .. vim: set fileencoding=utf-8 :
 .. Andre Anjos <andre.anjos@idiap.ch>
-.. Thu 22 Dec 2016 08:47:14 CET
+.. Sat Feb 17 14:26:47 2018 CET
 
 .. image:: https://gitlab.idiap.ch/bob/bob.nightlies/badges/master/build.svg
    :target: https://gitlab.idiap.ch/bob/bob.nightlies/commits/master
@@ -18,20 +18,14 @@ latest environment and tested together under that light.
 Installation
 ------------
 
-This package can be built starting from a base development environment, as
-provided by our `from-scratch.sh`_ script. First run it to create a conda-based
-environment for development. Once the environment is ready, follow the
-instructions on the ``.gitlab-ci.yml`` file to complete the development cycle.
+This package can be built completely from scratch by following the instructions
+on the ``.gitlab-ci.yml`` file.
 
 The script ``build.sh`` will try to execute the steps from the CI in the right
 order and can be used to locally test your modifications **before** pushing a
 branch into the nightlies, which will trigger a complete set of builds in
 multiple platforms.
 
-The script ``build.sh`` takes several parameters (run it for a help print out)::
-
-  $ ./build.sh
-
 
 Updating a Package
 ------------------
@@ -40,7 +34,7 @@ Updating a Package
 
    Before adding a package to this prototype, please ensure that the package
    contains all standard components of a Bob_ package (README, unit tests, CI
-   integration and documentation among others).
+   integration, documentation and conda build recipe among others).
 
    If you don't know how to do this, ask for information on our `mailing
    list`_.
@@ -55,8 +49,8 @@ The new package sits in one of two categories:
    to the order in which packages should be compiled.
 
 Once the file is modified, commit the changes to a **new branch**, with a
-suggestive name. Push the branch and, once it is green, you may merge it to the
-master (or ask it to be merged), via a standard merge request.
+suggestive name. Push the branch and merge it to the master (or ask it to be
+merged), via a standard merge request.
 
 
 Contact

build.py

-#!/usr/bin/env python
-import os
-import gitlab
-import time
-from datetime import datetime
-
-
-def wait_for_pipeline(project, pipeline):
-    time.sleep(15)
-    pipeline = project.pipelines.get(pipeline.id)
-    while pipeline.status in ('pending', 'running'):
-        time.sleep(15)
-        pipeline = project.pipelines.get(pipeline.id)
-    return pipeline
-
-
-def main(pkg_list_file, fail=True):
-
-    private_token = os.environ['GITLAB_API_TOKEN']
-    gl = gitlab.Gitlab('https://gitlab.idiap.ch',
-                       private_token=private_token, api_version=4)
-    any_failed = False
-    with open(pkg_list_file) as f:
-        for line in f:
-            line = line.strip()
-            if not line or line.startswith('#'):
-                continue
-            pkg = line.split('#')[0]
-            project = gl.projects.get('bob/{}'.format(pkg))
-            pipeline = project.pipelines.create({'ref': 'master'})
-            message = ("{date} Triggered pipeline: " +
-                       "https://gitlab.idiap.ch/bob/{pkg}/pipelines/{id}")
-            print(message.format(date=datetime.now(), pkg=pkg, id=pipeline.id))
-
-            pipeline = wait_for_pipeline(project, pipeline)
-
-            # re-try once if the pipeline fails
-            if pipeline.status != 'success':
-                print("Retrying the pipeline.")
-                pipeline.retry()
-                pipeline = wait_for_pipeline(project, pipeline)
-
-            if pipeline.status != 'success':
-                any_failed = True
-                message = ("Pipeline: https://gitlab.idiap.ch/bob/{pkg}/" +
-                           "pipelines/{id} failed with status {status}")
-                message = message.format(
-                    id=pipeline.id, pkg=pkg, status=pipeline.status)
-                if fail:
-                    raise RuntimeError(message)
-                else:
-                    print(message)
-    if any_failed:
-        raise RuntimeError("Failed to build some of the packages.")
-
-
-if __name__ == '__main__':
-    import sys
-    pkg_list_file = sys.argv[1]
-    fail = bool(int(sys.argv[2]))
-    main(pkg_list_file, fail)

buildout.cfg.template

-[buildout]
-parts = scripts
-extensions = bob.buildout
-             mr.developer
-auto-checkout = *
-debug = true
-newest = false
-verbose = true
-eggs = {eggs}
-
-develop = {develop}
-
-[scripts]
-recipe = bob.buildout:scripts
-dependent-scripts = true
-
-[sources]
-{sources}

core.txt

-bob.buildout
-bob.extension
-bob.blitz
-bob.core
-bob.io.base
-bob.math
-bob.measure
-bob.io.image
-bob.db.base
-bob.io.video
-bob.io.matlab
-bob.io.audio
-bob.sp
-bob.ap
-bob.ip.base
-bob.ip.color
-bob.ip.draw
-bob.ip.gabor
-bob.learn.activation
-bob.learn.libsvm
-bob.learn.linear
-bob.learn.mlp
-bob.learn.boosting
-bob.db.iris
-bob.learn.em
-bob.db.wine
-bob.db.mnist
-bob.db.atnt
-bob.ip.facedetect
-bob.ip.optflow.hornschunck
-bob.ip.optflow.liu
-bob.ip.flandmark
-bob

generate_buildout_config

-#!/usr/bin/env python
-"""Generates a buildout configuration file based on the list of the packages
-
-Usage:
-    %(prog)s <package_list>...
-    %(prog)s --help
-
-Arguments:
-    <package_list>  The files containing the package list. Each line should
-                    contain only the name of one package. Empty lines and
-                    comments starting with # are allowed.
-
-Options:
-    -h --help       Show this help message and exit
-"""
-
-
-def format_list(package_list, prepend, left_fill):
-    text = prepend + package_list[0] + '\n'
-    package_list = package_list[1:]
-    for pkg in package_list:
-        text += ' ' * left_fill + prepend + pkg + '\n'
-    return text
-
-
-def main(argv=None):
-    from docopt import docopt
-    import sys
-    import os
-    docs = __doc__ % {'prog': os.path.basename(sys.argv[0])}
-    args = docopt(docs, argv=argv)
-    package_list_files = args['<package_list>']
-    package_list = []
-    for path in package_list_files:
-        for line in open(path):
-            line = line.partition('#')[0].strip()
-            if not line:
-                continue
-            package_list.append(line)
-    print(package_list)
-    # read the buildout config template
-    with open("buildout.cfg.template") as f:
-        template = f.read()
-    # eggs
-    eggs = format_list(package_list, '', 7)
-    develop = format_list(package_list, 'src/', 10)
-    source_list = ['{0} = git git@gitlab.idiap.ch:bob/{0}'.format(p)
-                   for p in package_list]
-    sources = format_list(source_list, '', 0)
-    with open('buildout.cfg', 'wt') as f:
-        f.write(template.format(eggs=eggs, develop=develop, sources=sources))
-
-
-if __name__ == '__main__':
-    main()

extra.txt → order.txt

+bob.buildout
+bob.extension
+bob.blitz
+bob.core
+bob.io.base
+bob.math
+bob.measure
+bob.io.image
+bob.db.base
+bob.io.video
+bob.io.matlab
+bob.io.audio
+bob.sp
+bob.ap
+bob.ip.base
+bob.ip.color
+bob.ip.draw
+bob.ip.gabor
+bob.learn.activation
+bob.learn.libsvm
+bob.learn.linear
+bob.learn.mlp
+bob.learn.boosting
+bob.db.iris
+bob.learn.em
+bob.db.wine
+bob.db.mnist
+bob.db.atnt
+bob.ip.facedetect
+bob.ip.optflow.hornschunck
+bob.ip.optflow.liu
+bob.ip.flandmark
+bob
 gridtk
 bob.ip.qualitymeasure
 bob.ip.skincolorfilter

scripts/build.sh

+#!/usr/bin/env bash
+# Thu 15 Feb 15:36:30 2018 CET
+
+export PYTHON_VERSION="${PYTHON_VERSION:-${1}}"
+if [ -z "${PYTHON_VERSION}" ]; then
+  echo "usage: $0 <python-version>"
+  exit 1
+fi
+
+scriptdir="$(cd $(dirname ${BASH_SOURCE[0]}) && pwd)"
+basedir=`pwd`
+
+# These should be set by the CI - resetting in case of running from cmdline
+export CONDA_ROOT="${CONDA_ROOT:-${basedir}/miniconda}"
+export BOB_PACKAGE_VERSION="${BOB_PACKAGE_VERSION:-unknown}"
+export PYTHONUNBUFFERED="${PYTHONUNBUFFERED:-1}"
+export CI_PROJECT_URL="${CI_PROJECT_URL:-https://gitlab.idiap.ch/bob/bob.nightlies}"
+
+# This should be installed by CI, but installing it here in case running from
+# cmdline
+if [ ! -d "_ci" ]; then
+  mkdir _ci
+  curl --silent "https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/install.sh" > _ci/install.sh
+  chmod 755 _ci/install.sh
+  ./_ci/install.sh _ci master #installs ci support scripts
+fi
+
+# If not running on the CI, set docserver
+if [ -z "${DOCPASS}" ]; then
+  export DOCSERVER=https://www.idiap.ch
+  export CONDA_CHANNEL=https://www.idiap.ch/software/bob/conda/label/main
+  export CONDA_BETA_CHANNEL=http://beatubulatest.lab.idiap.ch/private/conda
+fi
+
+if [ ! -d "${CONDA_ROOT}" ]; then
+  ./_ci/before_build.sh
+fi
+
+source ${basedir}/_ci/functions.sh
+
+PACKAGES=($(sed -e "/^\s*#.*/d;/^\s*$/d" ${basedir}/order.txt))
+
+current=0
+
+for f in "${PACKAGES[@]}"; do
+
+  ((current++))
+
+  echo "";
+  log_debug " ==========================================================";
+  log_debug "  Checking ${f} (${current}/${#PACKAGES[@]})...";
+  log_debug " ==========================================================";
+  echo "";
+
+  # First checks if all package tests pass
+  export BOB_TEST_ONLY="true"
+  ${scriptdir}/rebuild.sh ${CONDA_ROOT} ${PYTHON_VERSION} ${f}
+  unset BOB_TEST_ONLY
+
+  if [ $? -ne 0 ]; then
+    log_warn "Package ${f} check FAILED - rebuilding it..."
+    echo "";
+    log_info " ==========================================================";
+    log_info "  Building ${f} (${current}/${#PACKAGES[@]})...";
+    log_info " ==========================================================";
+    echo "";
+    ${scriptdir}/rebuild.sh ${CONDA_ROOT} ${PYTHON_VERSION} ${f}
+  else
+    log_info "Package ${f} check SUCCESS - skipping rebuild"
+  fi
+
+done

scripts/rebuild.sh

+#!/usr/bin/env bash
+# Mon 19 Feb 10:42:26 2018 CET
+
+if [ $# -ne 3 ]; then
+  echo "Completely rebuilds the given package"
+  echo "usage: $0 <conda> <python-version> <package-name>"
+  echo "example: $0 \$\{CONDA_ROOT\} 2.7 bob.blitz"
+  exit 1
+fi
+
+CONDA_ROOT=${1}
+PYTHON_VERSION=${2}
+CI_PROJECT_NAME=${3}
+
+# Defines some required defaults, if they are not set by the CI (these allow
+# local builds)
+basedir=`pwd`
+CI_PROJECT_NAMESPACE="${CI_PROJECT_NAMESPACE:-bob}"
+CI_PROJECT_PATH=${CI_PROJECT_NAMESPACE}/${CI_PROJECT_NAME}
+CI_PROJECT_DIR=${basedir}/src/${CI_PROJECT_NAME}
+CI_PROJECT_URL="https://gitlab.idiap.ch/${CI_PROJECT_PATH}"
+CI_COMMIT_REF="${CI_COMMIT_REF:-12345678}"
+CI_COMMIT_REF_NAME="${CI_COMMIT_REF_NAME:-@local}"
+
+# The way to clone the repositories for this build
+GITLAB_CHECKOUT_STRATEGY="git@gitlab.idiap.ch:"
+
+# Overrides some stuff
+if [ "${CI_COMMIT_REF_NAME}" == "@local" ]; then
+  BOB_PACKAGE_VERSION="unknown"
+  PYTHONUNBUFFERED=1
+fi
+
+source ${basedir}/_ci/functions.sh
+unset BOB_PACKAGE_VERSION
+
+# Setup project variables
+export_env CI_PROJECT_DIR
+export_env CI_PROJECT_NAME
+export_env CI_PROJECT_NAMESPACE
+export_env CI_PROJECT_PATH
+export_env CI_COMMIT_REF
+export_env CI_COMMIT_REF_NAME
+export_env CI_PROJECT_URL
+export_env CONDA_ROOT
+export_env PYTHONUNBUFFERED
+export_env PYTHON_VERSION
+
+export_env CI_PROJECT_DIR
+export_env CI_PROJECT_PATH
+export_env CI_PROJECT_NAME
+
+if [ -d "${CI_PROJECT_DIR}" ]; then
+  run_cmd git -C ${CI_PROJECT_DIR} pull #updates
+  run_cmd git -C ${CI_PROJECT_DIR} clean -ffdx #rebuilds
+else
+  if [ ! -d ${basedir}/src ]; then
+    run_cmd mkdir -p ${basedir}/src
+  fi
+  run_cmd git clone --depth 1 ${GITLAB_CHECKOUT_STRATEGY}${CI_PROJECT_PATH} ${CI_PROJECT_DIR}
+fi
+
+# Gets the latest stable version of a given package or returns the empty string
+# $1: work directory for the package checkout
+latest_stable () {
+  git -C $1 fetch --tags
+  echo `git -C $1 tag --sort='v:refname' | grep -e 'v[0-9]*\.[0-9]*\.[0-9]*$' | tail -n 1`
+}
+
+if [ -n "${STABLE}" ]; then
+  CI_COMMIT_TAG=`latest_stable ${CI_PROJECT_DIR}`
+  if [ -z "${CI_COMMIT_TAG}" ]; then
+    echo "";
+    log_debug " ==========================================================";
+    log_debug "  Skipping $f (${current}/${total}). No stable release";
+    log_debug " ==========================================================";
+    echo "";
+    continue;
+  fi
+  export_env CI_COMMIT_TAG
+  run_cmd git -C ${CI_PROJECT_DIR} checkout ${CI_COMMIT_TAG}
+fi
+
+run_cmd cd ${CI_PROJECT_DIR}
+run_cmd ln -s ${basedir}/_ci .
+
+# Calculates package version
+if [ ! -r "version.txt" ]; then
+  log_error "./version.txt does not exist - cannot figure out version number"
+  exit 1
+fi
+BOB_PACKAGE_VERSION=`cat version.txt | tr -d '\n'`;
+export_env BOB_PACKAGE_VERSION
+
+# Calculates the current build-number available if in test-only mode
+if [ -n "${BOB_TEST_ONLY}" ]; then
+  run_cmd mkdir -p ./_ci/${OS_SLUG}/${PYTHON_VERSION}
+  if [ -z "${CI_COMMIT_TAG}" ]; then
+    run_cmd ${CONDA_ROOT}/bin/python _ci/channel_support.py ${CONDA_BETA_CHANNEL} ${CI_PROJECT_NAME} ${BOB_PACKAGE_VERSION} ${PYTHON_VERSION} -u --log ./_ci/${OS_SLUG}/${PYTHON_VERSION}/build_number.txt
+  else
+    run_cmd ${CONDA_ROOT}/bin/python _ci/channel_support.py ${CONDA_CHANNEL} ${CI_PROJECT_NAME} ${BOB_PACKAGE_VERSION} ${PYTHON_VERSION} -u --log ./_ci/${OS_SLUG}/${PYTHON_VERSION}/build_number.txt
+  fi
+
+  BOB_BUILD_NUMBER=`head -n 1 ./_ci/${OS_SLUG}/${PYTHON_VERSION}/build_number.txt | tr -d '\n'`
+  ((BOB_BUILD_NUMBER--)) #uses the latest available
+  export_env BOB_BUILD_NUMBER
+fi
+
+run_cmd ./_ci/build.sh
+
+if [ -n "${BOB_TEST_ONLY}" ]; then
+  # we don't keep stuff while testing only
+  run_cmd ${CONDA_ROOT}/bin/conda build purge
+else
+  # consider uploading...
+  if [[ "${IS_MASTER}" == "true" ]]; then
+    # Uploads all the built packages
+    for os in "osx-64" "noarch" "linux-64"; do
+      for f in ${CONDA_ROOT}/conda-bld/${os}/*.tar.bz2; do
+        if [[ -f $f ]]; then
+          dav_check_upload "${f}" "public-upload/conda/label/main/${os}/"
+        fi
+      done
+    done
+  else
+    log_info 'Skipping package upload (not on master branch)'
+  fi
+fi