From 8eb5c22a1323c9664ab925a69b431d9c80038c22 Mon Sep 17 00:00:00 2001
From: Andre Anjos <andre.dos.anjos@gmail.com>
Date: Mon, 18 Feb 2019 14:13:14 +0100
Subject: [PATCH] [test] Pull docker images once before begin of tests (during
buildout or conda-build)
---
beat/core/dock.py | 39 +++++++++++++++++++
beat/core/test/__init__.py | 39 -------------------
beat/core/test/test_docker.py | 3 --
.../test/test_docker_databases_provider.py | 3 --
beat/core/test/test_docker_environments.py | 3 --
beat/core/test/test_docker_execution.py | 3 --
beat/core/test/test_docker_worker.py | 3 --
buildout.cfg | 8 +++-
conda/meta.yaml | 2 +
develop.cfg | 8 +++-
10 files changed, 55 insertions(+), 56 deletions(-)
diff --git a/beat/core/dock.py b/beat/core/dock.py
index 05296802..b13105e2 100644
--- a/beat/core/dock.py
+++ b/beat/core/dock.py
@@ -48,6 +48,45 @@ from beat.core import stats
logger = logging.getLogger(__name__)
+DOCKER_TEST_IMAGES = {
+ "docker.idiap.ch/beat/beat.env.system.python": "1.3.0r4",
+ "docker.idiap.ch/beat/beat.env.db.examples": "1.4.0r4",
+ "docker.idiap.ch/beat/beat.env.cxx": "2.0.0r1",
+ "docker.idiap.ch/beat/beat.env.client": "2.0.0r1",
+ }
+"""Images used for docker-enabled tests within this and other BEAT packages
+"""
+
+
+def pull_docker_test_images():
+ """To be called when you need to set up tests using ``DOCKER_TEST_IMAGES``
+
+ This function will pull images that are not locally available yet
+ This technique prevents errors if docker.idiap.ch is not available,
+ e.g. when running outside the Idiap network
+ """
+
+ import docker
+ client = docker.from_env()
+
+ for image, tag in DOCKER_TEST_IMAGES.items():
+ has_image = False
+
+ for installed_image in client.images.list():
+ for installed_tag in installed_image.tags:
+ if installed_tag == ('%s:%s' % (image, tag)):
+ has_image = True
+
+ if not has_image: #must pull (network connection required)
+
+ token = os.environ.get('CI_BUILD_TOKEN')
+ params = (image, tag)
+ if token is not None: #running on CI, setup
+ auth_config = dict(username='gitlab-ci-token', password=token)
+ params += (auth_config,)
+ client.images.pull(*params)
+
+
class Host(object):
"""An object of this class can connect to the docker host and resolve stuff
"""
diff --git a/beat/core/test/__init__.py b/beat/core/test/__init__.py
index 8d3832bf..73c6ca5a 100644
--- a/beat/core/test/__init__.py
+++ b/beat/core/test/__init__.py
@@ -73,45 +73,6 @@ if VERBOSE_TEST_LOGGING:
logger.addHandler(handler)
-TEST_IMAGES = {
- "docker.idiap.ch/beat/beat.env.system.python": "1.3.0r4",
- "docker.idiap.ch/beat/beat.env.db.examples": "1.4.0r4",
- "docker.idiap.ch/beat/beat.env.cxx": "2.0.0r1",
- "docker.idiap.ch/beat/beat.env.client": "2.0.0r1",
- }
-"""Images used for docker-enabled tests
-"""
-
-
-def setup_docker_test_images():
- """To be called when you need to set those up. Check ``TEST_IMAGES``
-
- This function will pull images that are not locally available yet
- This technique prevents errors if docker.idiap.ch is not available,
- e.g. when running outside the Idiap network
- """
-
- import docker
- client = docker.from_env()
-
- for image, tag in TEST_IMAGES.items():
- has_image = False
-
- for installed_image in client.images.list():
- for installed_tag in installed_image.tags:
- if installed_tag == ('%s:%s' % (image, tag)):
- has_image = True
-
- if not has_image: #must pull (network connection required)
-
- token = os.environ.get('CI_BUILD_TOKEN')
- params = (image, tag)
- if token is not None: #running on CI, setup
- auth_config = dict(username='gitlab-ci-token', password=token)
- params += (auth_config,)
- client.images.pull(*params)
-
-
def setup_package():
prefixes = [
pkg_resources.resource_filename('beat.backend.python.test', 'prefix'),
diff --git a/beat/core/test/test_docker.py b/beat/core/test/test_docker.py
index 4e52fab0..a9cf1331 100644
--- a/beat/core/test/test_docker.py
+++ b/beat/core/test/test_docker.py
@@ -50,9 +50,6 @@ from .utils import skipif
from . import network_name
from . import DOCKER_NETWORK_TEST_ENABLED
-# this will ensure we pull the required images for the tests
-from . import setup_docker_test_images as setup_module
-
class NoDiscoveryTests(unittest.TestCase):
"""Test cases that don't require the discovery of database and runtime
diff --git a/beat/core/test/test_docker_databases_provider.py b/beat/core/test/test_docker_databases_provider.py
index 9aa013af..152edfd8 100644
--- a/beat/core/test/test_docker_databases_provider.py
+++ b/beat/core/test/test_docker_databases_provider.py
@@ -54,9 +54,6 @@ from ..utils import find_free_port
from . import prefix
from . import tmp_prefix
-# this will ensure we pull the required images for the tests
-from . import setup_docker_test_images as setup_module
-
#----------------------------------------------------------
diff --git a/beat/core/test/test_docker_environments.py b/beat/core/test/test_docker_environments.py
index 7411b8f0..6d121d99 100644
--- a/beat/core/test/test_docker_environments.py
+++ b/beat/core/test/test_docker_environments.py
@@ -33,9 +33,6 @@ from .. import environments
from .utils import slow
-# this will ensure we pull the required images for the tests
-from . import setup_docker_test_images as setup_module
-
class EnvironmentTest(unittest.TestCase):
diff --git a/beat/core/test/test_docker_execution.py b/beat/core/test/test_docker_execution.py
index 61976485..ac358b10 100644
--- a/beat/core/test/test_docker_execution.py
+++ b/beat/core/test/test_docker_execution.py
@@ -44,9 +44,6 @@ from . import network_name
from . import prefix_folder
from . import DOCKER_NETWORK_TEST_ENABLED
-# this will ensure we pull the required images for the tests
-from . import setup_docker_test_images as setup_module
-
BUILDER_IMAGE = "docker.idiap.ch/beat/beat.env.client:2.0.0r0"
#----------------------------------------------------------
diff --git a/beat/core/test/test_docker_worker.py b/beat/core/test/test_docker_worker.py
index 3feb2ce0..1938967f 100644
--- a/beat/core/test/test_docker_worker.py
+++ b/beat/core/test/test_docker_worker.py
@@ -32,9 +32,6 @@ from ..dock import Host
from .test_worker import TestOneWorker, TestTwoWorkers
-# this will ensure we pull the required images for the tests
-from . import setup_docker_test_images as setup_module
-
#----------------------------------------------------------
diff --git a/buildout.cfg b/buildout.cfg
index f4dd8303..17564769 100644
--- a/buildout.cfg
+++ b/buildout.cfg
@@ -1,8 +1,14 @@
[buildout]
-parts = scripts
+parts = scripts docker
eggs = beat.core
develop = .
newest = false
[scripts]
recipe = bob.buildout:scripts
+
+[docker]
+recipe = collective.recipe.cmd
+cmds = ./bin/python -c 'from beat.core.dock import pull_docker_test_images as f; f()'
+on_install = true
+on_update = true
diff --git a/conda/meta.yaml b/conda/meta.yaml
index 41dad405..1414a48a 100644
--- a/conda/meta.yaml
+++ b/conda/meta.yaml
@@ -52,6 +52,8 @@ test:
- {{ name }}
commands:
+ # pulls required images once before running the tests
+ - python -c "from beat.core.dock import pull_docker_test_images as f; f()"
- worker --help
- nosetests --with-coverage --cover-package={{ name }} -sv {{ name }}
- sphinx-build -aEW {{ project_dir }}/doc {{ project_dir }}/sphinx
diff --git a/develop.cfg b/develop.cfg
index a93af33a..c2b4af0c 100644
--- a/develop.cfg
+++ b/develop.cfg
@@ -1,5 +1,5 @@
[buildout]
-parts = scripts
+parts = scripts docker
extensions = mr.developer
auto-checkout = *
develop = src/beat.backend.python
@@ -13,3 +13,9 @@ beat.backend.python = git https://gitlab.idiap.ch/beat/beat.backend.python
[scripts]
recipe = bob.buildout:scripts
+
+[docker]
+recipe = collective.recipe.cmd
+cmds = ./bin/python -c 'from beat.core.dock import pull_docker_test_images as f; f()'
+on_install = true
+on_update = true
--
GitLab