Merge branch '77_implement_label_based_search' into 'master'

Implement environment discovery using docker labels

See merge request !106

beat/core/dock.py

@@ -43,6 +43,7 @@ Docker helper classes
 """
 
 
+import ast
 import os
 import simplejson as json
 import socket
@@ -52,6 +53,8 @@ import docker
 import subprocess as sp  # nosec
 import logging
 
+from packaging import version
+
 from beat.core import stats
 
 logger = logging.getLogger(__name__)
@@ -85,7 +88,12 @@ class Host(object):
             (
                 self.processing_environments,
                 self.db_environments,
-            ) = self._discover_environments()
+            ) = self._discover_environments_using_labels()
+
+            if not self.db_environments and not self.processing_environments:
+                self.processing_environments, self.db_environments = (
+                    self._discover_environments_using_describe()
+                )
 
         # (If necessary) Save the known infos about the images
         if self.images_cache_filename is not None:
@@ -157,7 +165,7 @@ class Host(object):
         s.connect(("8.8.8.8", 1))  # connecting to a UDP address doesn't send packets
         return s.getsockname()[0]
 
-    def _discover_environments(self):
+    def _discover_environments_using_describe(self):
         """Returns a dictionary containing information about docker environments
 
         Raises:
@@ -299,7 +307,9 @@ class Host(object):
         for image in images:
             # Call the "describe" application on each existing image
             description = _describe(image)
+
             if not description:
+                logger.debug("Description not found for", image)
                 continue
 
             key = description["name"] + " (" + description["version"] + ")"
@@ -331,6 +341,85 @@ class Host(object):
 
         return (environments, db_environments)
 
+    def _discover_environments_using_labels(self):
+        """Search BEAT runtime environments using docker labels"""
+
+        def _must_replace(key, image, environments):
+            environment = environments[key]
+            if environment["image"] not in image.tags:
+                logger.warn(
+                    "Different images providing the same environment: {} VS {}".format(
+                        environment["image"], image.tags
+                    )
+                )
+                if self.raise_on_errors:
+                    raise RuntimeError(
+                        "Environments at '%s' and '%s' have the "
+                        "same name ('%s'). Distinct environments must be "
+                        "uniquely named. Fix this and re-start."
+                        % (image.tags[0], environments[key]["image"], key)
+                    )
+                else:
+                    logger.debug("Keeping more recent")
+
+            current_version = "{}{}".format(
+                environment["version"], environment["revision"]
+            )
+            new_version = "{}{}".format(
+                image.labels["beat.env.version"], image.labels["beat.env.revision"]
+            )
+            current_version = version.parse(current_version)
+            new_version = version.parse(new_version)
+            return new_version > current_version
+
+        def _parse_image_info(image):
+            labels = image.labels
+            data = {
+                "image": image.tags[0],
+                "name": labels["beat.env.name"],
+                "version": labels["beat.env.version"],
+                "revision": labels["beat.env.revision"],
+            }
+
+            database_list = labels.get("beat.env.databases")
+            if database_list:
+                data["databases"] = ast.literal_eval(database_list)
+
+            capabilities = labels.get("beat.env.capabilities")
+            if capabilities:
+                data["capabilities"] = ast.literal_eval(capabilities)
+
+            return data
+
+        def _process_image_list(image_list):
+            environments = {}
+
+            for image in image_list:
+                if not len(image.tags):
+                    logger.warn("Untagged image, skipping")
+                    continue
+
+                image_info = _parse_image_info(image)
+                key = "{} {}".format(image_info["name"], image_info["version"])
+
+                if key in environments:
+                    if _must_replace(key, image, environments):
+                        environments[key] = image_info
+                else:
+                    environments[key] = image_info
+                    Host.images_cache[image_info["image"]] = environments[key]
+            return environments
+
+        client = docker.from_env()
+
+        databases = client.images.list(filters={"label": ["beat.env.type=database"]})
+        db_environments = _process_image_list(databases)
+
+        executors = client.images.list(filters={"label": ["beat.env.type=execution"]})
+        environments = _process_image_list(executors)
+
+        return environments, db_environments
+
     def create_container(self, image, command):
 
         if image in self:  # Replace by a real image name