Commit 2ebf20f7 authored by Samuel GAIST's avatar Samuel GAIST

[execution][local] Improve error handling on algorithm execution

Currently, when an error occurred in one of the scripts, it
could just hang or the error itself wasn't properly processed.

This patch improves exception handling as well as error
message returned.
parent 099ef01e
Pipeline #33686 passed with stage
in 23 minutes and 12 seconds
...@@ -188,8 +188,12 @@ class LocalExecutor(BaseExecutor): ...@@ -188,8 +188,12 @@ class LocalExecutor(BaseExecutor):
self.zmq_context = None self.zmq_context = None
def __cleanup(self): def __cleanup(self, early=False):
if self.loop_executor: if self.loop_executor:
if early:
self.loop_socket.send_string("don")
self.loop_socket.recv() # ack
self.loop_executor.wait() self.loop_executor.wait()
self.loop_executor.close() self.loop_executor.close()
...@@ -311,19 +315,35 @@ class LocalExecutor(BaseExecutor): ...@@ -311,19 +315,35 @@ class LocalExecutor(BaseExecutor):
cache_root=self.cache, cache_root=self.cache,
) )
retval = self.loop_executor.setup() try:
retval = self.loop_executor.setup()
except Exception as e:
message = _process_exception(e, self.prefix, "algorithms")
retval = False
else:
message = None
if not retval: if not retval:
self.__cleanup() self.__cleanup()
raise RuntimeError( error = "Loop algorithm {} setup failed".format(self.algorithm.name)
"Loop algorithm {} setup failed".format(self.algorithm.name) if message:
) error += ": {}".format(message)
raise RuntimeError(error)
try:
prepared = self.loop_executor.prepare()
except Exception as e:
message = _process_exception(e, self.prefix, "algorithms")
prepared = False
else:
message = None
prepared = self.loop_executor.prepare()
if not prepared: if not prepared:
self.__cleanup() self.__cleanup()
raise RuntimeError( error = "Loop algorithm {} prepare failed".format(self.algorithm.name)
"Loop algorithm {} prepare failed".format(self.algorithm.name) if message:
) error += ": {}".format(message)
raise RuntimeError(error)
self.loop_executor.process() self.loop_executor.process()
...@@ -337,26 +357,40 @@ class LocalExecutor(BaseExecutor): ...@@ -337,26 +357,40 @@ class LocalExecutor(BaseExecutor):
try: try:
status = self.executor.setup() status = self.executor.setup()
except Exception: except Exception as e:
message = _process_exception(e, self.prefix, "algorithms")
status = 0 status = 0
else:
message = None
if not status: if not status:
self.__cleanup() self.__cleanup(early=True)
raise RuntimeError("Algorithm {} setup failed".format(self.algorithm.name)) error = "Algorithm {} setup failed".format(self.algorithm.name)
if message:
error += ": {}".format(message)
raise RuntimeError(error)
try:
prepared = self.executor.prepare()
except Exception as e:
message = _process_exception(e, self.prefix, "algorithms")
prepared = 0
else:
message = None
prepared = self.executor.prepare()
if not prepared: if not prepared:
self.__cleanup() self.__cleanup(early=True)
raise RuntimeError( error = "Algorithm {} prepare failed".format(self.algorithm.name)
"Algorithm {} prepare failed".format(self.algorithm.name) if message:
) error += ": {}".format(message)
raise RuntimeError(error)
_start = time.time() _start = time.time()
try: try:
processed = self.executor.process() processed = self.executor.process()
except Exception as e: except Exception as e:
message = _process_exception(e, self.prefix, "databases") message = _process_exception(e, self.prefix, "algorithms")
self.__cleanup() self.__cleanup()
return _create_result(1, message) return _create_result(1, message)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment