Merge branch 'issue-63' into 'master'

Add automatic test image pulling if not already available (closes #63)

Closes #63

See merge request !51

.gitlab-ci.yml

-# This build file uses template features from YAML so it is generic enough for
-# any Bob project. Don't modify it unless you know what you're doing.
-
-# Definition of global variables (all stages)
-variables:
-  CONDA_ROOT: "${CI_PROJECT_DIR}/miniconda"
-  DOCKER_REGISTRY: docker.idiap.ch
-
-
-# Definition of our build pipeline order
-stages:
-  - build
-  - deploy
-  - pypi
-
-
-# Build targets
-.build_template: &build_job
-  stage: build
-  before_script:
-    - mkdir _ci
-    - curl --silent "https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/install.sh" > _ci/install.sh
-    - chmod 755 _ci/install.sh
-    - ./_ci/install.sh _ci master #installs ci support scripts
-    - ./_ci/before_build.sh
-    - docker info
-    - ./scripts/docker_pull.sh #pulls required test images
-  script:
-    - ./_ci/build.sh
-  after_script:
-    - ./_ci/after_build.sh
-  cache: &build_caches
-    paths:
-      - miniconda.sh
-      - ${CONDA_ROOT}/pkgs/*.tar.bz2
-      - ${CONDA_ROOT}/pkgs/urls.txt
-
-
-.build_linux_template: &linux_build_job
-  <<: *build_job
-  tags:
-    - docker-build
-  artifacts:
-    expire_in: 1 week
-    paths:
-      - _ci/
-      - ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
-  cache:
-    <<: *build_caches
-    key: "linux-cache"
-
-
-.build_macosx_template: &macosx_build_job
-  <<: *build_job
-  tags:
-    - macosx
-  artifacts:
-    expire_in: 1 week
-    paths:
-      - _ci/
-      - ${CONDA_ROOT}/conda-bld/osx-64/*.tar.bz2
-  cache:
-    <<: *build_caches
-    key: "macosx-cache"
-
-
-build_linux_36:
-  <<: *linux_build_job
-  variables:
-    PYTHON_VERSION: "3.6"
-    BUILD_EGG: "true"
-  artifacts:
-    expire_in: 1 week
-    paths:
-      - _ci/
-      - dist/*.zip
-      - sphinx
-      - ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
-
-
-build_macosx_36:
-  <<: *macosx_build_job
-  variables:
-    PYTHON_VERSION: "3.6"
-
-
-# Deploy targets
-.deploy_template: &deploy_job
-  stage: deploy
-  before_script:
-    - ./_ci/install.sh _ci master #updates ci support scripts
-  script:
-    - ./_ci/deploy.sh
-  dependencies:
-    - build_linux_36
-    - build_macosx_36
-  tags:
-    - deployer
-
-
-deploy_beta:
-  <<: *deploy_job
-  environment: beta
-  only:
-    - master
-
-
-deploy_stable:
-  <<: *deploy_job
-  environment: stable
-  only:
-    - /^v\d+\.\d+\.\d+([abc]\d*)?$/  # PEP-440 compliant version (tags)
-  except:
-    - branches
-
-
-pypi:
-  stage: pypi
-  environment: pypi
-  only:
-    - /^v\d+\.\d+\.\d+([abc]\d*)?$/  # PEP-440 compliant version (tags)
-  except:
-    - branches
-  before_script:
-    - ./_ci/install.sh _ci master #updates ci support scripts
-  script:
-    - ./_ci/pypi.sh
-  dependencies:
-    - build_linux_36
-  tags:
-    - deployer
+include:
+  - 'https://gitlab.idiap.ch/bob/bob.devtools/raw/master/bob/devtools/data/gitlab-ci/single-package.yaml'
+  - 'https://gitlab.idiap.ch/bob/bob.devtools/raw/master/bob/devtools/data/gitlab-ci/beat-docker-package.yaml'

beat/core/test/utils.py

@@ -34,6 +34,49 @@ import shutil
 import nose
 import docker
 
+
+#----------------------------------------------------------
+
+
+DOCKER_TEST_IMAGES = {
+    "docker.idiap.ch/beat/beat.env.system.python": "1.3.0r4",
+    "docker.idiap.ch/beat/beat.env.db.examples": "1.4.0r4",
+    "docker.idiap.ch/beat/beat.env.cxx": "2.0.0r1",
+    "docker.idiap.ch/beat/beat.env.client": "2.0.0r1",
+    }
+"""Images used for docker-enabled tests within this and other BEAT packages
+"""
+
+
+def pull_docker_test_images():
+    """To be called when you need to set up tests using ``DOCKER_TEST_IMAGES``
+
+    This function will pull images that are not locally available yet
+    This technique prevents errors if docker.idiap.ch is not available,
+    e.g. when running outside the Idiap network
+    """
+
+    import docker
+    client = docker.from_env()
+
+    for image, tag in DOCKER_TEST_IMAGES.items():
+        has_image = False
+
+        for installed_image in client.images.list():
+            for installed_tag in installed_image.tags:
+                if installed_tag == ('%s:%s' % (image, tag)):
+                    has_image = True
+
+        if not has_image:  #must pull (network connection required)
+
+            token = os.environ.get('CI_BUILD_TOKEN')
+            params = (image, tag)
+            if token is not None:  #running on CI, setup
+                auth_config = dict(username='gitlab-ci-token', password=token)
+                params += (auth_config,)
+            client.images.pull(*params)
+
+
 #----------------------------------------------------------
 
 

buildout.cfg

@@ -9,6 +9,6 @@ recipe = bob.buildout:scripts
 
 [docker]
 recipe = collective.recipe.cmd
-cmds = BUILDOUT=true ./scripts/docker_pull.sh
+cmds = ./bin/python -c 'from beat.core.test.utils import pull_docker_test_images as f; f()'
 on_install = true
 on_update = true

conda/meta.yaml

@@ -52,6 +52,8 @@ test:
     - {{ name }}
 
   commands:
+    # pulls required images once before running the tests
+    - python -c "from beat.core.test.utils import pull_docker_test_images as f; f()"
     - worker --help
     - nosetests --with-coverage --cover-package={{ name }} -sv {{ name }}
     - sphinx-build -aEW {{ project_dir }}/doc {{ project_dir }}/sphinx

develop.cfg

@@ -16,6 +16,6 @@ recipe = bob.buildout:scripts
 
 [docker]
 recipe = collective.recipe.cmd
-cmds = BUILDOUT=true ./scripts/docker_pull.sh
+cmds = ./bin/python -c 'from beat.core.test.utils import pull_docker_test_images as f; f()'
 on_install = true
 on_update = true

scripts/docker_pull.sh

-#!/usr/bin/env bash
-
-basedir=`pwd`
-
-if [ ! -e ${basedir}/_ci/functions.sh ]; then
-  # this will happen when executing buildout for the first time
-  mkdir ${basedir}/_ci
-  curl --silent "https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/install.sh" > ${basedir}/_ci/install.sh
-  chmod 755 ${basedir}/_ci/install.sh
-  ${basedir}/_ci/install.sh ${basedir}/_ci master #installs ci support scripts
-fi
-
-if [ "${BUILDOUT}" == "true" ]; then
-  # when executing in the context of buildout, define dummies
-  export CI_PROJECT_URL=https://gitlab.idiap.ch/beat/beat.core
-  export CI_PROJECT_DIR=beat
-  export CI_PROJECT_PATH=beat/beat.core
-  export CI_PROJECT_NAME=beat.core
-  export CI_COMMIT_REF_NAME=master
-  export PYPIUSER=pypiuser
-  export PYPIPASS=pypipass
-  export DOCUSER=docuser
-  export DOCPASS=docpass
-  export CONDA_ROOT=${basedir}/miniconda
-  export DOCKER_REGISTRY=docker.idiap.ch
-  export DOCSERVER=https://www.idiap.ch/software/bob
-fi
-
-source ${basedir}/_ci/functions.sh
-
-check_env DOCKER_REGISTRY
-
-# Select here images that are required for minimal operation (or tests)
-IMAGES=(
-    "${DOCKER_REGISTRY}/beat/beat.env.system.python:1.3.0r4"
-    "${DOCKER_REGISTRY}/beat/beat.env.db.examples:1.4.0r4"
-    "${DOCKER_REGISTRY}/beat/beat.env.cxx:2.0.0r1"
-    "${DOCKER_REGISTRY}/beat/beat.env.client:2.0.0r1"
-    )
-check_array_env IMAGES
-
-missing=()
-log_info "Checking for minimal set of docker images..."
-for k in ${IMAGES[@]}; do
-    hash_=$(docker images -q ${k})
-    if [ -z "${hash_}" ]; then
-        missing+=(${k})
-        log_warn "${k} NOT available - downloading...";
-    else
-        log_info "${k} already available - not re-downloading";
-    fi
-done
-
-# If there are some images missing, download them
-if [[ "${#missing[@]}" > 0 ]]; then
-  # Log in the registry if needed
-  if [ -z "$CI_SERVER" ]; then
-    if ! grep -q "${DOCKER_REGISTRY}" ~/.docker/config.json ; then
-      run_cmd docker login "${DOCKER_REGISTRY}"
-    fi
-  else
-    check_pass CI_BUILD_TOKEN
-    log_info docker login -u gitlab-ci-token -p "*************" "${DOCKER_REGISTRY}"
-    docker login -u gitlab-ci-token -p "${CI_BUILD_TOKEN}" "${DOCKER_REGISTRY}"
-  fi
-
-  # Pull the images
-  for k in "${missing[@]}"; do
-    run_cmd docker pull ${k}
-  done
-fi