[execution] Implement temporary resources cleanup

The Docker and subprocess executors use one or more
temp folders during processing. This patch implements
proper removal of them when they are done.

There's a debug settings that can be used to switch
that cleanup off.

beat/core/execution/base.py

@@ -152,6 +152,7 @@ class BaseExecutor(object):
         self.data_sinks = []
         self.errors = []
         self.data = data
+        self.debug = False
 
         # Check that the cache path exists
         if not os.path.exists(self.cache):

beat/core/execution/docker.py

@@ -34,6 +34,7 @@ Execution utilities
 """
 
 import os
+import shutil
 import requests
 import simplejson
 import zmq
@@ -230,6 +231,7 @@ class DockerExecutor(RemoteExecutor):
         #----- (If necessary) Instantiate the docker container that provide the databases
 
         databases_container = None
+        databases_configuration_path = None
         datasets_uid =  self.data.pop('datasets_uid', None)
         network_name = self.data.pop('network_name', 'bridge')
 
@@ -423,4 +425,9 @@ class DockerExecutor(RemoteExecutor):
         self.message_handler.destroy()
         self.message_handler = None
 
+        if not self.debug:
+            if databases_configuration_path:
+                shutil.rmtree(databases_configuration_path)
+            shutil.rmtree(configuration_path)
+
         return retval

beat/core/execution/subprocess.py

@@ -37,6 +37,7 @@ Execution utilities
 from __future__ import absolute_import
 
 import os
+import shutil
 import requests
 import zmq
 import sys
@@ -245,6 +246,7 @@ class SubprocessExecutor(RemoteExecutor):
         #----- (If necessary) Instantiate the subprocess that provide the databases
 
         databases_process = None
+        databases_configuration_path = None
         database_port = utils.find_free_port()
 
         if len(self.databases) > 0:
@@ -370,4 +372,9 @@ class SubprocessExecutor(RemoteExecutor):
         self.message_handler.destroy()
         self.message_handler = None
 
+        if not self.debug:
+            if databases_configuration_path:
+                shutil.rmtree(databases_configuration_path)
+            shutil.rmtree(configuration_path)
+
         return retval