Make exex wrap exceptions, and cope with exceptions even when they ar…

…en't wrapped (#993) * port commit a3c773b to extreme scale workers * Handle the case where what I was expecting to be a wrapped exception is actually just an exception * add missing remote exception wrapper
Parsl · Jun 4, 2019 · 4c32c46a442cb3603c6b77583daafe87b40d7b23 · 4c32c46
1 parent da4702d
commit 4c32c46a442cb3603c6b77583daafe87b40d7b23
Unified Split

Showing with 11 additions and 5 deletions.

+2 −1 parsl/executors/extreme_scale/mpi_worker_pool.py

+9 −4 parsl/executors/high_throughput/executor.py
diff --git a/parsl/executors/extreme_scale/mpi_worker_pool.py b/parsl/executors/extreme_scale/mpi_worker_pool.py
@@ -16,6 +16,7 @@
 from mpi4py import MPI
+  
+from parsl.app.errors import RemoteExceptionWrapper
 from parsl.version import VERSION as PARSL_VERSION
 from ipyparallel.serialize import unpack_apply_message  # pack_apply_message,
 from ipyparallel.serialize import serialize_object
@@ -402,7 +403,7 @@ def worker(comm, rank):
        try:
            result = execute_task(req['buffer'])
        except Exception as e:
-  
-            result_package = {'task_id': tid, 'exception': serialize_object(e)}
+  
+            result_package = {'task_id': tid, 'exception': serialize_object(RemoteExceptionWrapper(*sys.exc_info()))}
            logger.debug("No result due to exception: {} with result package {}".format(e, result_package))
        else:
            result_package = {'task_id': tid, 'result': serialize_object(result)}

diff --git a/parsl/executors/high_throughput/executor.py b/parsl/executors/high_throughput/executor.py
@@ -362,10 +362,15 @@ def _queue_management_worker(self):
                            try:
                                s, _ = deserialize_object(msg['exception'])
                                # s should be a RemoteExceptionWrapper... so we can reraise it
-  
-                                try:
-  
-                                    s.reraise()
-  
-                                except Exception as e:
-  
-                                    task_fut.set_exception(e)
+  
+                                if isinstance(s, RemoteExceptionWapper):
+  
+                                    try:
+  
+                                        s.reraise()
+  
+                                    except Exception as e:
+  
+                                        task_fut.set_exception(e)
+  
+                                elif isinstance(s, Exception):
+  
+                                    task_fut.set_exception(s)
+  
+                                else:
+  
+                                    raise ValueError("Unknown exception-like type received: {}".format(type(s)))
                            except Exception as e:
                                # TODO could be a proper wrapped exception?
                                task_fut.set_exception(