Commit e6583a16 authored by Samuel GAIST's avatar Samuel GAIST
Browse files

[execution] Implement temporary resources cleanup

The Docker and subprocess executors use one or more
temp folders during processing. This patch implements
proper removal of them when they are done.

There's a debug settings that can be used to switch
that cleanup off.
parent 5cc7c066
Pipeline #24132 canceled with stages
in 4 minutes and 19 seconds
......@@ -152,6 +152,7 @@ class BaseExecutor(object):
self.data_sinks = []
self.errors = []
self.data = data
self.debug = False
# Check that the cache path exists
if not os.path.exists(self.cache):
......
......@@ -34,6 +34,7 @@ Execution utilities
"""
import os
import shutil
import requests
import simplejson
import zmq
......@@ -230,6 +231,7 @@ class DockerExecutor(RemoteExecutor):
#----- (If necessary) Instantiate the docker container that provide the databases
databases_container = None
databases_configuration_path = None
datasets_uid = self.data.pop('datasets_uid', None)
network_name = self.data.pop('network_name', 'bridge')
......@@ -423,4 +425,9 @@ class DockerExecutor(RemoteExecutor):
self.message_handler.destroy()
self.message_handler = None
if not self.debug:
if databases_configuration_path:
shutil.rmtree(databases_configuration_path)
shutil.rmtree(configuration_path)
return retval
......@@ -37,6 +37,7 @@ Execution utilities
from __future__ import absolute_import
import os
import shutil
import requests
import zmq
import sys
......@@ -245,6 +246,7 @@ class SubprocessExecutor(RemoteExecutor):
#----- (If necessary) Instantiate the subprocess that provide the databases
databases_process = None
databases_configuration_path = None
database_port = utils.find_free_port()
if len(self.databases) > 0:
......@@ -370,4 +372,9 @@ class SubprocessExecutor(RemoteExecutor):
self.message_handler.destroy()
self.message_handler = None
if not self.debug:
if databases_configuration_path:
shutil.rmtree(databases_configuration_path)
shutil.rmtree(configuration_path)
return retval
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment