Commit 057d0cc9 authored by Samuel GAIST's avatar Samuel GAIST

[execution][docker] Make volume creation more precise

This will only mount what is needed by the various containers.
parent f0d79b05
......@@ -49,6 +49,7 @@ import requests
import simplejson as json
from beat.backend.python.execution import MessageHandler
from beat.backend.python.data import getAllFilenames
from .. import stats
from .. import utils
......@@ -278,6 +279,54 @@ class DockerExecutor(RemoteExecutor):
)
return retval
def __setup_io_volumes(self, algorithm_container, configuration):
"""Setup all the volumes for input and output files.
Parameters:
algorithm_container: container that will execute an algorithm
configuration: json object containing the algorithm parameters
"""
for item in configuration["inputs"].values():
file_path = item["path"]
source_path = os.path.join(self.cache, file_path)
if os.path.isfile(source_path):
algorithm_container.add_volume(
source_path, os.path.join(self.CONTAINER_CACHE_PATH, file_path)
)
else:
all_files = getAllFilenames(source_path)
for file_list in all_files:
for file_ in file_list:
target_path = file_[len(self.cache) + 1 :]
cache_path = os.path.join(
self.CONTAINER_CACHE_PATH, target_path
)
algorithm_container.add_volume(file_, cache_path)
def __add_writable_volume(file_path):
output_folder = file_path[: file_path.rfind("/")]
source_folder = os.path.join(self.cache, output_folder)
if not os.path.exists(source_folder):
os.makedirs(source_folder)
algorithm_container.add_volume(
source_folder,
os.path.join(self.CONTAINER_CACHE_PATH, output_folder),
read_only=False,
)
for item in configuration.get("outputs", {}).values():
file_path = item["path"]
__add_writable_volume(file_path)
result = configuration.get("result")
if result:
file_path = result["path"]
__add_writable_volume(file_path)
def process(
self, virtual_memory_in_megabytes=0, max_cpu_percent=0, timeout_in_minutes=0
):
......@@ -414,9 +463,7 @@ class DockerExecutor(RemoteExecutor):
loop_algorithm_container.add_volume(
configuration_path, self.CONTAINER_PREFIX_PATH
)
loop_algorithm_container.add_volume(
self.cache, self.CONTAINER_CACHE_PATH, read_only=False
)
self.__setup_io_volumes(loop_algorithm_container, self.data["loop"])
# Start the container
self.host.start(
......@@ -458,9 +505,7 @@ class DockerExecutor(RemoteExecutor):
# Volumes
algorithm_container.add_volume(configuration_path, self.CONTAINER_PREFIX_PATH)
algorithm_container.add_volume(
self.cache, self.CONTAINER_CACHE_PATH, read_only=False
)
self.__setup_io_volumes(algorithm_container, self.data)
# Start the container
self.host.start(
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment