Commit df54e749 authored by André Anjos's avatar André Anjos 💬

Merge branch '77_implement_label_based_search' into 'master'

Implement environment discovery using docker labels

See merge request !106
parents c645666d 9f19dc94
Pipeline #36459 passed with stages
in 17 minutes and 17 seconds
......@@ -43,6 +43,7 @@ Docker helper classes
"""
import ast
import os
import simplejson as json
import socket
......@@ -52,6 +53,8 @@ import docker
import subprocess as sp # nosec
import logging
from packaging import version
from beat.core import stats
logger = logging.getLogger(__name__)
......@@ -85,7 +88,12 @@ class Host(object):
(
self.processing_environments,
self.db_environments,
) = self._discover_environments()
) = self._discover_environments_using_labels()
if not self.db_environments and not self.processing_environments:
self.processing_environments, self.db_environments = (
self._discover_environments_using_describe()
)
# (If necessary) Save the known infos about the images
if self.images_cache_filename is not None:
......@@ -157,7 +165,7 @@ class Host(object):
s.connect(("8.8.8.8", 1)) # connecting to a UDP address doesn't send packets
return s.getsockname()[0]
def _discover_environments(self):
def _discover_environments_using_describe(self):
"""Returns a dictionary containing information about docker environments
Raises:
......@@ -299,7 +307,9 @@ class Host(object):
for image in images:
# Call the "describe" application on each existing image
description = _describe(image)
if not description:
logger.debug("Description not found for", image)
continue
key = description["name"] + " (" + description["version"] + ")"
......@@ -331,6 +341,85 @@ class Host(object):
return (environments, db_environments)
def _discover_environments_using_labels(self):
"""Search BEAT runtime environments using docker labels"""
def _must_replace(key, image, environments):
environment = environments[key]
if environment["image"] not in image.tags:
logger.warn(
"Different images providing the same environment: {} VS {}".format(
environment["image"], image.tags
)
)
if self.raise_on_errors:
raise RuntimeError(
"Environments at '%s' and '%s' have the "
"same name ('%s'). Distinct environments must be "
"uniquely named. Fix this and re-start."
% (image.tags[0], environments[key]["image"], key)
)
else:
logger.debug("Keeping more recent")
current_version = "{}{}".format(
environment["version"], environment["revision"]
)
new_version = "{}{}".format(
image.labels["beat.env.version"], image.labels["beat.env.revision"]
)
current_version = version.parse(current_version)
new_version = version.parse(new_version)
return new_version > current_version
def _parse_image_info(image):
labels = image.labels
data = {
"image": image.tags[0],
"name": labels["beat.env.name"],
"version": labels["beat.env.version"],
"revision": labels["beat.env.revision"],
}
database_list = labels.get("beat.env.databases")
if database_list:
data["databases"] = ast.literal_eval(database_list)
capabilities = labels.get("beat.env.capabilities")
if capabilities:
data["capabilities"] = ast.literal_eval(capabilities)
return data
def _process_image_list(image_list):
environments = {}
for image in image_list:
if not len(image.tags):
logger.warn("Untagged image, skipping")
continue
image_info = _parse_image_info(image)
key = "{} {}".format(image_info["name"], image_info["version"])
if key in environments:
if _must_replace(key, image, environments):
environments[key] = image_info
else:
environments[key] = image_info
Host.images_cache[image_info["image"]] = environments[key]
return environments
client = docker.from_env()
databases = client.images.list(filters={"label": ["beat.env.type=database"]})
db_environments = _process_image_list(databases)
executors = client.images.list(filters={"label": ["beat.env.type=execution"]})
environments = _process_image_list(executors)
return environments, db_environments
def create_container(self, image, command):
if image in self: # Replace by a real image name
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment