From fa7de4f14fedfdb4d1f169340eb535c220d9aa80 Mon Sep 17 00:00:00 2001
From: Philip ABBET <philip.abbet@idiap.ch>
Date: Mon, 21 Aug 2017 13:46:19 +0200
Subject: [PATCH] Optimization: the discovery of docker images can be cached
---
beat/core/dock.py | 22 ++++++++++-
beat/core/test/test_docker.py | 73 +++++++++++++++++++++++++++++++++++
2 files changed, 94 insertions(+), 1 deletion(-)
mode change 100644 => 100755 beat/core/dock.py
diff --git a/beat/core/dock.py b/beat/core/dock.py
old mode 100644
new mode 100755
index 4deba1a1..a36e680f
--- a/beat/core/dock.py
+++ b/beat/core/dock.py
@@ -47,6 +47,8 @@ from . import stats
class Host(object):
'''An object of this class can connect to the docker host and resolve stuff'''
+ images_cache = {}
+
def __init__(self, **kwargs):
@@ -60,6 +62,11 @@ class Host(object):
del kwargs['port']
kwargs['base_url'] = "http://%s:%s" % (host, port)
+ self.images_cache_filename = None
+ if 'images_cache' in kwargs:
+ self.images_cache_filename = kwargs.get('images_cache')
+ del kwargs['images_cache']
+
self.kwargs = kwargs
self.environments = {}
self.db_environments = {}
@@ -70,8 +77,16 @@ class Host(object):
self.client = docker.Client(**self.kwargs)
+ if (self.images_cache_filename is not None) and os.path.exists(self.images_cache_filename):
+ with open(self.images_cache_filename, 'r') as f:
+ Host.images_cache = simplejson.load(f)
+
(self.environments, self.db_environments) = self._discover_environments(raise_on_errors)
+ if self.images_cache_filename is not None:
+ with open(self.images_cache_filename, 'w') as f:
+ simplejson.dump(Host.images_cache, f, indent=4)
+
def __contains__(self, key):
return (key in self.environments) or (key in self.db_environments)
@@ -149,10 +164,15 @@ class Host(object):
def _describe(image):
'''Tries to run the "describe" app on the image, collect results'''
+ if Host.images_cache.has_key(image):
+ return Host.images_cache[image]
+
status, output = self.get_statusoutput(image, ['describe'])
if status == 0:
try:
- return simplejson.loads(output)
+ infos = simplejson.loads(output)
+ Host.images_cache[image] = infos
+ return infos
except Exception as e:
logger.warn("Ignoring potential environment at `%s' since " \
"`describe' output cannot be parsed: %s", image, str(e))
diff --git a/beat/core/test/test_docker.py b/beat/core/test/test_docker.py
index 96237c12..2679a33e 100644
--- a/beat/core/test/test_docker.py
+++ b/beat/core/test/test_docker.py
@@ -34,11 +34,13 @@ import sys
import time
import unittest
import pkg_resources
+import time
import docker
import requests
from ..dock import Popen, Host
+from . import tmp_prefix
# in case you want to see the printouts dynamically, set to ``True``
if False:
@@ -275,3 +277,74 @@ class AsyncTest(unittest.TestCase):
def test_cpulimit_at_100percent(self):
# runs 4 processes that should consume 50% of the host CPU
self._run_cpulimit(4, 100, 3)
+
+
+
+class HostTest(unittest.TestCase):
+
+ def setUp(self):
+ Host.images_cache = {}
+
+
+ def test_images_cache(self):
+ self.assertEqual(len(Host.images_cache), 0)
+
+ # Might take some time
+ start = time.time()
+
+ host = Host()
+ host.setup(raise_on_errors=False)
+ host.teardown()
+
+ stop = time.time()
+
+ nb_images = len(Host.images_cache)
+ self.assertTrue(nb_images > 0)
+
+ self.assertTrue(stop - start > 2.0)
+
+ # Should be instantaneous
+ start = time.time()
+
+ host = Host()
+ host.setup(raise_on_errors=False)
+ host.teardown()
+
+ stop = time.time()
+
+ self.assertEqual(len(Host.images_cache), nb_images)
+
+ self.assertTrue(stop - start < 1.0)
+
+
+ def test_images_cache_file(self):
+ self.assertEqual(len(Host.images_cache), 0)
+
+ # Might take some time
+ start = time.time()
+
+ host = Host(images_cache=os.path.join(tmp_prefix, 'images_cache.json'))
+ host.setup(raise_on_errors=False)
+ host.teardown()
+
+ stop = time.time()
+
+ nb_images = len(Host.images_cache)
+ self.assertTrue(nb_images > 0)
+
+ self.assertTrue(stop - start > 2.0)
+
+ Host.images_cache = {}
+
+ # Should be instantaneous
+ start = time.time()
+
+ host = Host(images_cache=os.path.join(tmp_prefix, 'images_cache.json'))
+ host.setup(raise_on_errors=False)
+ host.teardown()
+
+ stop = time.time()
+
+ self.assertEqual(len(Host.images_cache), nb_images)
+
+ self.assertTrue(stop - start < 1.0)
--
GitLab