Commit 4b48bae5 authored by Philip ABBET's avatar Philip ABBET

Use docker caching mechanism to speed-up the build

parent 64d0fae4
Pipeline #6699 passed with stage
in 71 minutes and 7 seconds
......@@ -5,43 +5,46 @@ stages:
variables:
DOCKER_REGISTRY: docker.idiap.ch
CONTAINER_NAME: $DOCKER_REGISTRY/$CI_PROJECT_PATH:$CI_BUILD_REF_NAME
DEPLOY_NAME: $DOCKER_REGISTRY/$CI_PROJECT_PATH:latest
before_script:
- docker info
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN docker.idiap.ch
- sed -i "s/git@gitlab.idiap.ch:/https:\/\/gitlab-ci-token:${CI_BUILD_TOKEN}@gitlab.idiap.ch\//" .gitmodules
- git submodule sync
- git submodule update --init --recursive
- sed -i "s/https:\/\/gitlab-ci-token:${CI_BUILD_TOKEN}@gitlab.idiap.ch\//git@gitlab.idiap.ch:/" .gitmodules
build:
stage: build
only:
- /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags)
variables:
CONTAINER_NAME: $DOCKER_REGISTRY/$CI_PROJECT_PATH:$CI_BUILD_REF_NAME
script:
- cp /home/gitlab-runner/.ssh/id_rsa* .
- CONTAINER_NAME=${CONTAINER_NAME/:v/:};
- sed -i "s/git@gitlab.idiap.ch:/https:\/\/gitlab-ci-token:${CI_BUILD_TOKEN}@gitlab.idiap.ch\//" .gitmodules
- git submodule sync
- git submodule update --init --recursive
- ./build.sh $CONTAINER_NAME "gitlab-ci-token:${CI_BUILD_TOKEN}"
# - docker push $CONTAINER_NAME # No automatic upload at the moment
- docker images $CONTAINER_NAME
after_script:
- rm -f id_rsa*
environment:
name: registry
tags:
- docker-build
# Deploys `latest` tag
deploy:
# Deploys 'push*' tags, using an existing 'vX.Y.Z' tag on the same commit
deploy_version:
stage: deploy
only:
- /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags)
- /^push/ # Only for tags starting with 'push'
script:
- docker pull $CONTAINER_NAME
- docker tag $CONTAINER_NAME $DEPLOY_NAME
# - docker push $DEPLOY_NAME # No automatic upload at the moment
- ALL_TAGS=`git tag --contains $CI_BUILD_REF`
- for TAG in $ALL_TAGS;
do
if [[ $TAG =~ ^v[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+([abc][[:digit:]]*)?$ ]]; then
CONTAINER_NAME=$DOCKER_REGISTRY/$CI_PROJECT_PATH:$TAG;
break;
fi;
done
- CONTAINER_NAME=${CONTAINER_NAME/:v/:}
- docker push $CONTAINER_NAME
environment:
name: registry
tags:
......
......@@ -22,7 +22,7 @@
# #
###############################################################################
FROM debian:8.4
FROM beat.env.python:0.1.0__pre_build
MAINTAINER Andre Anjos <andre.anjos@idiap.ch>
# the destination directory that will contain the 'execute' environment
......@@ -33,41 +33,7 @@ ARG tarball
ADD $tarball $destdir
# this is the bare minimum to install on debian base
RUN set -ex \
&& build_deps=' \
cmake \
dh-autoreconf \
gcc \
gfortran \
libatlas-dev \
libc-dev \
libfreetype6-dev \
libgomp1 \
libjpeg-dev \
liblapack-dev \
libpq-dev \
libreadline-dev \
make \
pkg-config \
unzip \
wget \
' \
&& runtime_deps=' \
libatlas3-base \
libffi6 \
libfreetype6 \
libjpeg62-turbo \
libreadline6 \
' \
&& apt-get update \
&& apt-get install -y $build_deps \
&& make -C $destdir download \
# building of the content of the tarball
RUN make -C $destdir download \
&& make -C $destdir \
&& rm -rf $destdir/pkg $destdir/log \
&& apt-get purge -y --auto-remove $build_deps \
&& apt-get install -y $runtime_deps \
&& rm -rf /var/lib/apt/lists/*
# at this point, the image is pretty much built
ENV PATH $destdir/bin:$PATH
&& rm -rf $destdir/pkg $destdir/log
###############################################################################
# #
# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# This file is part of the beat.backend.python module of the BEAT platform. #
# #
# Commercial License Usage #
# Licensees holding valid commercial BEAT licenses may use this file in #
# accordance with the terms contained in a written agreement between you #
# and Idiap. For further information contact tto@idiap.ch #
# #
# Alternatively, this file may be used under the terms of the GNU Affero #
# Public License version 3 as published by the Free Software and appearing #
# in the file LICENSE.AGPL included in the packaging of this file. #
# The BEAT platform is distributed in the hope that it will be useful, but #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
# or FITNESS FOR A PARTICULAR PURPOSE. #
# #
# You should have received a copy of the GNU Affero Public License along #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
# #
###############################################################################
FROM beat.env.python:0.1.0__build
MAINTAINER Andre Anjos <andre.anjos@idiap.ch>
# the destination directory that will contain the 'execute' environment
ARG destdir=/usr/local/beat
# remove the packages no longer needed, and and some runtime-only ones
RUN set -ex \
&& build_deps=' \
cmake \
dh-autoreconf \
gcc \
gfortran \
libatlas-dev \
libc-dev \
libfreetype6-dev \
libgomp1 \
libjpeg-dev \
liblapack-dev \
libpq-dev \
libreadline-dev \
make \
pkg-config \
unzip \
wget \
' \
&& runtime_deps=' \
libatlas3-base \
libffi6 \
libfreetype6 \
libjpeg62-turbo \
libreadline6 \
' \
&& apt-get purge -y --auto-remove $build_deps \
&& apt-get install -y $runtime_deps \
&& rm -rf /var/lib/apt/lists/*
# change the PATH, so we can execute the scripts easily
ENV PATH $destdir/bin:$PATH
\ No newline at end of file
###############################################################################
# #
# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# This file is part of the beat.backend.python module of the BEAT platform. #
# #
# Commercial License Usage #
# Licensees holding valid commercial BEAT licenses may use this file in #
# accordance with the terms contained in a written agreement between you #
# and Idiap. For further information contact tto@idiap.ch #
# #
# Alternatively, this file may be used under the terms of the GNU Affero #
# Public License version 3 as published by the Free Software and appearing #
# in the file LICENSE.AGPL included in the packaging of this file. #
# The BEAT platform is distributed in the hope that it will be useful, but #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
# or FITNESS FOR A PARTICULAR PURPOSE. #
# #
# You should have received a copy of the GNU Affero Public License along #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
# #
###############################################################################
FROM debian:8.4
MAINTAINER Andre Anjos <andre.anjos@idiap.ch>
RUN set -ex \
&& build_deps=' \
cmake \
dh-autoreconf \
gcc \
gfortran \
libatlas-dev \
libc-dev \
libfreetype6-dev \
libgomp1 \
libjpeg-dev \
liblapack-dev \
libpq-dev \
libreadline-dev \
make \
pkg-config \
unzip \
wget \
' \
&& apt-get update \
&& apt-get install -y $build_deps
......@@ -3,10 +3,6 @@
This package contains docker instructions for building a container for the Python
backend of the BEAT platform.
It is tightly integrated to Gitlab CI. By default, it will build and update a
new container on the registry, named after the branch or release tag. Upon a
tag, the container will also be uploaded as `latest`.
## Usage
......@@ -15,3 +11,19 @@ To test the creation of a container on your machine, run the following command:
```sh
$ ./build.sh
```
## Integration with Gitlab CI
By default, nothing is done on the CI when commits are pushed to the Gitlab
repository.
To trigger the build of a docker image, push a tag of the form *vX.Y.Z*. This
will create an image with the name *docker.idiap.ch/beat/beat.env.python:X.Y.Z*.
To trigger the upload of the image to the *docker.idiap.ch* registry, you can push
a special tag (see below). Note that the commit you are tagging **must** also have
a *vX.Y.Z* tag, and that this build must have successfully passed.
The special tag must start with *push* (examples: *push*, *push_thisversion*).
This will upload the image with the name *docker.idiap.ch/beat/beat.env.python:X.Y.Z*
......@@ -8,7 +8,7 @@ IMAGE_TAG=0.1.0
REGISTRY=docker.idiap.ch/beat
# Customisation from the command-line, used mainly by the CI
if [ $# > 0 ]; then
if [ $# != 0 ]; then
IFS=:
read image_fullname IMAGE_TAG <<< "$1"
REGISTRY=${image_fullname%/*}
......@@ -19,5 +19,7 @@ if [ $# == 2 ]; then
PROJECT_REPOSITORY="https://${2}@gitlab.idiap.ch/${PROJECT_REPOSITORY##*:}"
fi
# Build the image
./scripts/build.sh "${PROJECT_REPOSITORY}" $PROJECT_BRANCH $IMAGE_NAME $IMAGE_TAG $REGISTRY
# Build the image (in three steps)
docker build --rm=true -t "${IMAGE_NAME}:${IMAGE_TAG}__pre_build" -f ./Dockerfile_pre_build .
./scripts/build.sh "${PROJECT_REPOSITORY}" $PROJECT_BRANCH $IMAGE_NAME "${IMAGE_TAG}__build"
docker build --rm=true -t "${REGISTRY}/${IMAGE_NAME}:${IMAGE_TAG}" -f ./Dockerfile_post_build .
Subproject commit 76b97c9d80f92021a879df7dbeab48196ed2c076
Subproject commit 1b54797b95ee83e679f1715d6d1619f573c91d2f
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment