diff --git a/beat/core/dock.py b/beat/core/dock.py index 05296802ae79f2e3d956e2d25e0da6c8126b3ef2..b13105e2b8737bf160b4fd2d492e666fc320e095 100644 --- a/beat/core/dock.py +++ b/beat/core/dock.py @@ -48,6 +48,45 @@ from beat.core import stats logger = logging.getLogger(__name__) +DOCKER_TEST_IMAGES = { + "docker.idiap.ch/beat/beat.env.system.python": "1.3.0r4", + "docker.idiap.ch/beat/beat.env.db.examples": "1.4.0r4", + "docker.idiap.ch/beat/beat.env.cxx": "2.0.0r1", + "docker.idiap.ch/beat/beat.env.client": "2.0.0r1", + } +"""Images used for docker-enabled tests within this and other BEAT packages +""" + + +def pull_docker_test_images(): + """To be called when you need to set up tests using ``DOCKER_TEST_IMAGES`` + + This function will pull images that are not locally available yet + This technique prevents errors if docker.idiap.ch is not available, + e.g. when running outside the Idiap network + """ + + import docker + client = docker.from_env() + + for image, tag in DOCKER_TEST_IMAGES.items(): + has_image = False + + for installed_image in client.images.list(): + for installed_tag in installed_image.tags: + if installed_tag == ('%s:%s' % (image, tag)): + has_image = True + + if not has_image: #must pull (network connection required) + + token = os.environ.get('CI_BUILD_TOKEN') + params = (image, tag) + if token is not None: #running on CI, setup + auth_config = dict(username='gitlab-ci-token', password=token) + params += (auth_config,) + client.images.pull(*params) + + class Host(object): """An object of this class can connect to the docker host and resolve stuff """ diff --git a/beat/core/test/__init__.py b/beat/core/test/__init__.py index 8d3832bf03f9797a3513f5584a90dbabddec2416..73c6ca5a4c30ad68992e5d7867ec6fabbcc421ef 100644 --- a/beat/core/test/__init__.py +++ b/beat/core/test/__init__.py @@ -73,45 +73,6 @@ if VERBOSE_TEST_LOGGING: logger.addHandler(handler) -TEST_IMAGES = { - "docker.idiap.ch/beat/beat.env.system.python": "1.3.0r4", - "docker.idiap.ch/beat/beat.env.db.examples": "1.4.0r4", - "docker.idiap.ch/beat/beat.env.cxx": "2.0.0r1", - "docker.idiap.ch/beat/beat.env.client": "2.0.0r1", - } -"""Images used for docker-enabled tests -""" - - -def setup_docker_test_images(): - """To be called when you need to set those up. Check ``TEST_IMAGES`` - - This function will pull images that are not locally available yet - This technique prevents errors if docker.idiap.ch is not available, - e.g. when running outside the Idiap network - """ - - import docker - client = docker.from_env() - - for image, tag in TEST_IMAGES.items(): - has_image = False - - for installed_image in client.images.list(): - for installed_tag in installed_image.tags: - if installed_tag == ('%s:%s' % (image, tag)): - has_image = True - - if not has_image: #must pull (network connection required) - - token = os.environ.get('CI_BUILD_TOKEN') - params = (image, tag) - if token is not None: #running on CI, setup - auth_config = dict(username='gitlab-ci-token', password=token) - params += (auth_config,) - client.images.pull(*params) - - def setup_package(): prefixes = [ pkg_resources.resource_filename('beat.backend.python.test', 'prefix'), diff --git a/beat/core/test/test_docker.py b/beat/core/test/test_docker.py index 4e52fab0214c7bfe75c8410864ba70f0b8f30d3f..a9cf1331b59b886725624a28f0254cdf2d1d30cf 100644 --- a/beat/core/test/test_docker.py +++ b/beat/core/test/test_docker.py @@ -50,9 +50,6 @@ from .utils import skipif from . import network_name from . import DOCKER_NETWORK_TEST_ENABLED -# this will ensure we pull the required images for the tests -from . import setup_docker_test_images as setup_module - class NoDiscoveryTests(unittest.TestCase): """Test cases that don't require the discovery of database and runtime diff --git a/beat/core/test/test_docker_databases_provider.py b/beat/core/test/test_docker_databases_provider.py index 9aa013af2dc2b632a1d5a579137093ab7db2d53d..152edfd849e1a7d65d8790c866a7f2083b1b354e 100644 --- a/beat/core/test/test_docker_databases_provider.py +++ b/beat/core/test/test_docker_databases_provider.py @@ -54,9 +54,6 @@ from ..utils import find_free_port from . import prefix from . import tmp_prefix -# this will ensure we pull the required images for the tests -from . import setup_docker_test_images as setup_module - #---------------------------------------------------------- diff --git a/beat/core/test/test_docker_environments.py b/beat/core/test/test_docker_environments.py index 7411b8f0a1cf9affed1b076d1c070c75cfefbd8d..6d121d993f0c9dc8e1461f8a1b6befd8851da6bf 100644 --- a/beat/core/test/test_docker_environments.py +++ b/beat/core/test/test_docker_environments.py @@ -33,9 +33,6 @@ from .. import environments from .utils import slow -# this will ensure we pull the required images for the tests -from . import setup_docker_test_images as setup_module - class EnvironmentTest(unittest.TestCase): diff --git a/beat/core/test/test_docker_execution.py b/beat/core/test/test_docker_execution.py index 6197648553a0b143ae7d0b93fcb0cdeef9c3a4e6..ac358b1060f1467b1d47132deb0ce833f1218d81 100644 --- a/beat/core/test/test_docker_execution.py +++ b/beat/core/test/test_docker_execution.py @@ -44,9 +44,6 @@ from . import network_name from . import prefix_folder from . import DOCKER_NETWORK_TEST_ENABLED -# this will ensure we pull the required images for the tests -from . import setup_docker_test_images as setup_module - BUILDER_IMAGE = "docker.idiap.ch/beat/beat.env.client:2.0.0r0" #---------------------------------------------------------- diff --git a/beat/core/test/test_docker_worker.py b/beat/core/test/test_docker_worker.py index 3feb2ce09a4b9da04b2dd096e5e16be0a13eab4c..1938967f605ffca0f00b00919ef13eb7f77c0c9e 100644 --- a/beat/core/test/test_docker_worker.py +++ b/beat/core/test/test_docker_worker.py @@ -32,9 +32,6 @@ from ..dock import Host from .test_worker import TestOneWorker, TestTwoWorkers -# this will ensure we pull the required images for the tests -from . import setup_docker_test_images as setup_module - #---------------------------------------------------------- diff --git a/buildout.cfg b/buildout.cfg index f4dd83034743049f04ae98f007b84f941d12386b..17564769e87ed0620e70c52481e627770991ede2 100644 --- a/buildout.cfg +++ b/buildout.cfg @@ -1,8 +1,14 @@ [buildout] -parts = scripts +parts = scripts docker eggs = beat.core develop = . newest = false [scripts] recipe = bob.buildout:scripts + +[docker] +recipe = collective.recipe.cmd +cmds = ./bin/python -c 'from beat.core.dock import pull_docker_test_images as f; f()' +on_install = true +on_update = true diff --git a/conda/meta.yaml b/conda/meta.yaml index 41dad405f83681d76cbc4ce1475a0df4de8774b9..1414a48a28851e7cfd34e26db13ac4ba79872784 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -52,6 +52,8 @@ test: - {{ name }} commands: + # pulls required images once before running the tests + - python -c "from beat.core.dock import pull_docker_test_images as f; f()" - worker --help - nosetests --with-coverage --cover-package={{ name }} -sv {{ name }} - sphinx-build -aEW {{ project_dir }}/doc {{ project_dir }}/sphinx diff --git a/develop.cfg b/develop.cfg index a93af33a156e62413e3a678f47baae551b46c240..c2b4af0c2c7240e1627584c0f4136f84da49e2a4 100644 --- a/develop.cfg +++ b/develop.cfg @@ -1,5 +1,5 @@ [buildout] -parts = scripts +parts = scripts docker extensions = mr.developer auto-checkout = * develop = src/beat.backend.python @@ -13,3 +13,9 @@ beat.backend.python = git https://gitlab.idiap.ch/beat/beat.backend.python [scripts] recipe = bob.buildout:scripts + +[docker] +recipe = collective.recipe.cmd +cmds = ./bin/python -c 'from beat.core.dock import pull_docker_test_images as f; f()' +on_install = true +on_update = true