[test] Pull docker images once before begin of tests (during buildout or conda-build)

beat/core/dock.py

@@ -48,6 +48,45 @@ from beat.core import stats
 logger = logging.getLogger(__name__)
 
 
+DOCKER_TEST_IMAGES = {
+    "docker.idiap.ch/beat/beat.env.system.python": "1.3.0r4",
+    "docker.idiap.ch/beat/beat.env.db.examples": "1.4.0r4",
+    "docker.idiap.ch/beat/beat.env.cxx": "2.0.0r1",
+    "docker.idiap.ch/beat/beat.env.client": "2.0.0r1",
+    }
+"""Images used for docker-enabled tests within this and other BEAT packages
+"""
+
+
+def pull_docker_test_images():
+    """To be called when you need to set up tests using ``DOCKER_TEST_IMAGES``
+
+    This function will pull images that are not locally available yet
+    This technique prevents errors if docker.idiap.ch is not available,
+    e.g. when running outside the Idiap network
+    """
+
+    import docker
+    client = docker.from_env()
+
+    for image, tag in DOCKER_TEST_IMAGES.items():
+        has_image = False
+
+        for installed_image in client.images.list():
+            for installed_tag in installed_image.tags:
+                if installed_tag == ('%s:%s' % (image, tag)):
+                    has_image = True
+
+        if not has_image:  #must pull (network connection required)
+
+            token = os.environ.get('CI_BUILD_TOKEN')
+            params = (image, tag)
+            if token is not None:  #running on CI, setup
+                auth_config = dict(username='gitlab-ci-token', password=token)
+                params += (auth_config,)
+            client.images.pull(*params)
+
+
 class Host(object):
     """An object of this class can connect to the docker host and resolve stuff
     """

beat/core/test/__init__.py

@@ -73,45 +73,6 @@ if VERBOSE_TEST_LOGGING:
       logger.addHandler(handler)
 
 
-TEST_IMAGES = {
-    "docker.idiap.ch/beat/beat.env.system.python": "1.3.0r4",
-    "docker.idiap.ch/beat/beat.env.db.examples": "1.4.0r4",
-    "docker.idiap.ch/beat/beat.env.cxx": "2.0.0r1",
-    "docker.idiap.ch/beat/beat.env.client": "2.0.0r1",
-    }
-"""Images used for docker-enabled tests
-"""
-
-
-def setup_docker_test_images():
-    """To be called when you need to set those up. Check ``TEST_IMAGES``
-
-    This function will pull images that are not locally available yet
-    This technique prevents errors if docker.idiap.ch is not available,
-    e.g. when running outside the Idiap network
-    """
-
-    import docker
-    client = docker.from_env()
-
-    for image, tag in TEST_IMAGES.items():
-        has_image = False
-
-        for installed_image in client.images.list():
-            for installed_tag in installed_image.tags:
-                if installed_tag == ('%s:%s' % (image, tag)):
-                    has_image = True
-
-        if not has_image:  #must pull (network connection required)
-
-            token = os.environ.get('CI_BUILD_TOKEN')
-            params = (image, tag)
-            if token is not None:  #running on CI, setup
-                auth_config = dict(username='gitlab-ci-token', password=token)
-                params += (auth_config,)
-            client.images.pull(*params)
-
-
 def setup_package():
     prefixes = [
         pkg_resources.resource_filename('beat.backend.python.test', 'prefix'),

beat/core/test/test_docker.py

@@ -50,9 +50,6 @@ from .utils import skipif
 from . import network_name
 from . import DOCKER_NETWORK_TEST_ENABLED
 
-# this will ensure we pull the required images for the tests
-from . import setup_docker_test_images as setup_module
-
 
 class NoDiscoveryTests(unittest.TestCase):
     """Test cases that don't require the discovery of database and runtime

beat/core/test/test_docker_databases_provider.py

@@ -54,9 +54,6 @@ from ..utils import find_free_port
 from . import prefix
 from . import tmp_prefix
 
-# this will ensure we pull the required images for the tests
-from . import setup_docker_test_images as setup_module
-
 
 #----------------------------------------------------------
 

beat/core/test/test_docker_environments.py

@@ -33,9 +33,6 @@ from .. import environments
 
 from .utils import slow
 
-# this will ensure we pull the required images for the tests
-from . import setup_docker_test_images as setup_module
-
 
 class EnvironmentTest(unittest.TestCase):
 

beat/core/test/test_docker_execution.py

@@ -44,9 +44,6 @@ from . import network_name
 from . import prefix_folder
 from . import DOCKER_NETWORK_TEST_ENABLED
 
-# this will ensure we pull the required images for the tests
-from . import setup_docker_test_images as setup_module
-
 BUILDER_IMAGE = "docker.idiap.ch/beat/beat.env.client:2.0.0r0"
 
 #----------------------------------------------------------

beat/core/test/test_docker_worker.py

@@ -32,9 +32,6 @@ from ..dock import Host
 
 from .test_worker import TestOneWorker, TestTwoWorkers
 
-# this will ensure we pull the required images for the tests
-from . import setup_docker_test_images as setup_module
-
 
 
 #----------------------------------------------------------

buildout.cfg

 [buildout]
-parts = scripts
+parts = scripts docker
 eggs = beat.core
 develop = .
 newest = false
 
 [scripts]
 recipe = bob.buildout:scripts
+
+[docker]
+recipe = collective.recipe.cmd
+cmds = ./bin/python -c 'from beat.core.dock import pull_docker_test_images as f; f()'
+on_install = true
+on_update = true

conda/meta.yaml

@@ -52,6 +52,8 @@ test:
     - {{ name }}
 
   commands:
+    # pulls required images once before running the tests
+    - python -c "from beat.core.dock import pull_docker_test_images as f; f()"
     - worker --help
     - nosetests --with-coverage --cover-package={{ name }} -sv {{ name }}
     - sphinx-build -aEW {{ project_dir }}/doc {{ project_dir }}/sphinx

develop.cfg

 [buildout]
-parts = scripts
+parts = scripts docker
 extensions = mr.developer
 auto-checkout = *
 develop = src/beat.backend.python
@@ -13,3 +13,9 @@ beat.backend.python = git https://gitlab.idiap.ch/beat/beat.backend.python
 
 [scripts]
 recipe = bob.buildout:scripts
+
+[docker]
+recipe = collective.recipe.cmd
+cmds = ./bin/python -c 'from beat.core.dock import pull_docker_test_images as f; f()'
+on_install = true
+on_update = true