Commit 9f19dc94 authored by Samuel GAIST's avatar Samuel GAIST Committed by Samuel GAIST

[dock] Implement environment discover using docker labels

The next release of BEAT environments will make use of docker
labels to provide the information the current describe command
returns (minus the os entry which is not currently used).

This new implementation of discovery uses these labels and thus
speeds up the process.

The original code is kept as a fallback method in case a node only
has images using the old build process.
parent c645666d
Pipeline #36432 passed with stage
in 19 minutes and 48 seconds
......@@ -43,6 +43,7 @@ Docker helper classes
import ast
import os
import simplejson as json
import socket
......@@ -52,6 +53,8 @@ import docker
import subprocess as sp # nosec
import logging
from packaging import version
from beat.core import stats
logger = logging.getLogger(__name__)
......@@ -85,7 +88,12 @@ class Host(object):
) = self._discover_environments()
) = self._discover_environments_using_labels()
if not self.db_environments and not self.processing_environments:
self.processing_environments, self.db_environments = (
# (If necessary) Save the known infos about the images
if self.images_cache_filename is not None:
......@@ -157,7 +165,7 @@ class Host(object):
s.connect(("", 1)) # connecting to a UDP address doesn't send packets
return s.getsockname()[0]
def _discover_environments(self):
def _discover_environments_using_describe(self):
"""Returns a dictionary containing information about docker environments
......@@ -299,7 +307,9 @@ class Host(object):
for image in images:
# Call the "describe" application on each existing image
description = _describe(image)
if not description:
logger.debug("Description not found for", image)
key = description["name"] + " (" + description["version"] + ")"
......@@ -331,6 +341,85 @@ class Host(object):
return (environments, db_environments)
def _discover_environments_using_labels(self):
"""Search BEAT runtime environments using docker labels"""
def _must_replace(key, image, environments):
environment = environments[key]
if environment["image"] not in image.tags:
"Different images providing the same environment: {} VS {}".format(
environment["image"], image.tags
if self.raise_on_errors:
raise RuntimeError(
"Environments at '%s' and '%s' have the "
"same name ('%s'). Distinct environments must be "
"uniquely named. Fix this and re-start."
% (image.tags[0], environments[key]["image"], key)
logger.debug("Keeping more recent")
current_version = "{}{}".format(
environment["version"], environment["revision"]
new_version = "{}{}".format(
image.labels["beat.env.version"], image.labels["beat.env.revision"]
current_version = version.parse(current_version)
new_version = version.parse(new_version)
return new_version > current_version
def _parse_image_info(image):
labels = image.labels
data = {
"image": image.tags[0],
"name": labels[""],
"version": labels["beat.env.version"],
"revision": labels["beat.env.revision"],
database_list = labels.get("beat.env.databases")
if database_list:
data["databases"] = ast.literal_eval(database_list)
capabilities = labels.get("beat.env.capabilities")
if capabilities:
data["capabilities"] = ast.literal_eval(capabilities)
return data
def _process_image_list(image_list):
environments = {}
for image in image_list:
if not len(image.tags):
logger.warn("Untagged image, skipping")
image_info = _parse_image_info(image)
key = "{} {}".format(image_info["name"], image_info["version"])
if key in environments:
if _must_replace(key, image, environments):
environments[key] = image_info
environments[key] = image_info
Host.images_cache[image_info["image"]] = environments[key]
return environments
client = docker.from_env()
databases = client.images.list(filters={"label": ["beat.env.type=database"]})
db_environments = _process_image_list(databases)
executors = client.images.list(filters={"label": ["beat.env.type=execution"]})
environments = _process_image_list(executors)
return environments, db_environments
def create_container(self, image, command):
if image in self: # Replace by a real image name
Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment