Optimization: the discovery of docker images can be cached

beat/core/dock.py

@@ -47,6 +47,8 @@ from . import stats
 class Host(object):
   '''An object of this class can connect to the docker host and resolve stuff'''
 
+  images_cache = {}
+
 
   def __init__(self, **kwargs):
 
@@ -60,6 +62,11 @@ class Host(object):
       del kwargs['port']
       kwargs['base_url'] = "http://%s:%s" % (host, port)
 
+    self.images_cache_filename = None
+    if 'images_cache' in kwargs:
+      self.images_cache_filename = kwargs.get('images_cache')
+      del kwargs['images_cache']
+
     self.kwargs = kwargs
     self.environments = {}
     self.db_environments = {}
@@ -70,8 +77,16 @@ class Host(object):
 
     self.client = docker.Client(**self.kwargs)
 
+    if (self.images_cache_filename is not None) and os.path.exists(self.images_cache_filename):
+      with open(self.images_cache_filename, 'r') as f:
+        Host.images_cache = simplejson.load(f)
+
     (self.environments, self.db_environments) = self._discover_environments(raise_on_errors)
 
+    if self.images_cache_filename is not None:
+      with open(self.images_cache_filename, 'w') as f:
+        simplejson.dump(Host.images_cache, f, indent=4)
+
 
   def __contains__(self, key):
     return (key in self.environments) or (key in self.db_environments)
@@ -149,10 +164,15 @@ class Host(object):
     def _describe(image):
       '''Tries to run the "describe" app on the image, collect results'''
 
+      if Host.images_cache.has_key(image):
+        return Host.images_cache[image]
+
       status, output = self.get_statusoutput(image, ['describe'])
       if status == 0:
         try:
-          return simplejson.loads(output)
+          infos = simplejson.loads(output)
+          Host.images_cache[image] = infos
+          return infos
         except Exception as e:
           logger.warn("Ignoring potential environment at `%s' since " \
                   "`describe' output cannot be parsed: %s", image, str(e))

beat/core/test/test_docker.py

@@ -34,11 +34,13 @@ import sys
 import time
 import unittest
 import pkg_resources
+import time
 
 import docker
 import requests
 
 from ..dock import Popen, Host
+from . import tmp_prefix
 
 # in case you want to see the printouts dynamically, set to ``True``
 if False:
@@ -275,3 +277,74 @@ class AsyncTest(unittest.TestCase):
   def test_cpulimit_at_100percent(self):
     # runs 4 processes that should consume 50% of the host CPU
     self._run_cpulimit(4, 100, 3)
+
+
+
+class HostTest(unittest.TestCase):
+
+  def setUp(self):
+    Host.images_cache = {}
+
+
+  def test_images_cache(self):
+    self.assertEqual(len(Host.images_cache), 0)
+
+    # Might take some time
+    start = time.time()
+
+    host = Host()
+    host.setup(raise_on_errors=False)
+    host.teardown()
+
+    stop = time.time()
+
+    nb_images = len(Host.images_cache)
+    self.assertTrue(nb_images > 0)
+
+    self.assertTrue(stop - start > 2.0)
+
+    # Should be instantaneous
+    start = time.time()
+
+    host = Host()
+    host.setup(raise_on_errors=False)
+    host.teardown()
+
+    stop = time.time()
+
+    self.assertEqual(len(Host.images_cache), nb_images)
+
+    self.assertTrue(stop - start < 1.0)
+
+
+  def test_images_cache_file(self):
+    self.assertEqual(len(Host.images_cache), 0)
+
+    # Might take some time
+    start = time.time()
+
+    host = Host(images_cache=os.path.join(tmp_prefix, 'images_cache.json'))
+    host.setup(raise_on_errors=False)
+    host.teardown()
+
+    stop = time.time()
+
+    nb_images = len(Host.images_cache)
+    self.assertTrue(nb_images > 0)
+
+    self.assertTrue(stop - start > 2.0)
+
+    Host.images_cache = {}
+
+    # Should be instantaneous
+    start = time.time()
+
+    host = Host(images_cache=os.path.join(tmp_prefix, 'images_cache.json'))
+    host.setup(raise_on_errors=False)
+    host.teardown()
+
+    stop = time.time()
+
+    self.assertEqual(len(Host.images_cache), nb_images)
+
+    self.assertTrue(stop - start < 1.0)