Merge branch 'rickard/non-smp-trace-port-exit-bug/OTP-12889' into maint-18

* rickard/non-smp-trace-port-exit-bug/OTP-12889:
  Teach non-smp VM how to deal with trace port crash
  Test case testing crash in tracer port

2 files changed, 59 insertions, 1 deletions

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 8b409e139b..38def5d89f 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -2163,6 +2163,22 @@ void process_main(void)
 	 OpCase(wait_f):
 
 	 wait2: {
+#ifndef ERTS_SMP
+	     if (ERTS_PROC_IS_EXITING(c_p)) {
+		 /*
+		  * I non smp case:
+		  *
+		  * Currently executing process might be sent an exit
+		  * signal if it is traced by a port that it also is
+		  * linked to, and the port terminates during the
+		  * trace. In this case we do *not* want to clear
+		  * the active flag, which will make the process hang
+		  * in limbo forever.
+		  */
+		 SWAPOUT;
+		 goto do_schedule;
+	     }
+#endif
 	     c_p->i = (BeamInstr *) Arg(0); /* L1 */
 	     SWAPOUT;
 	     c_p->arity = 0;
@@ -6219,6 +6235,23 @@ erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* re
     int arity;
     Eterm tmp;
 
+#ifndef ERTS_SMP
+    if (ERTS_PROC_IS_EXITING(c_p)) {
+	/*
+	 * I non smp case:
+	 *
+	 * Currently executing process might be sent an exit
+	 * signal if it is traced by a port that it also is
+	 * linked to, and the port terminates during the
+	 * trace. In this case we do *not* want to clear
+	 * the active flag, which will make the process hang
+	 * in limbo forever. Get out of here and terminate
+	 * the process...
+	 */
+	return -1;
+    }
+#endif
+
     if (is_not_atom(module) || is_not_atom(function)) {
 	/*
 	 * No need to test args here -- done below.
@@ -6295,7 +6328,16 @@ erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* re
 	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
-#ifdef ERTS_SMP
+#ifndef ERTS_SMP
+	if (ERTS_PROC_IS_EXITING(c_p)) {
+	    /*
+	     * See comment in the begining of the function...
+	     *
+	     * This second test is needed since gc might be traced.
+	     */
+	    return -1;
+	}
+#else /* ERTS_SMP */
 	ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p);
 	if (!c_p->msg.len)
 #endif
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 7b3d12ce09..ee1dd36d48 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -11409,6 +11409,22 @@ set_proc_exiting(Process *p,
     KILL_CATCHES(p);
     p->i = (BeamInstr *) beam_exit;
 
+#ifndef ERTS_SMP
+    if (state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)) {
+	/*
+	 * I non smp case:
+	 *
+	 * Currently executing process might be sent an exit
+	 * signal if it is traced by a port that it also is
+	 * linked to, and the port terminates during the
+	 * trace. In this case we want schedule out the
+	 * process as quickly as possible in order to detect
+	 * the event as fast as possible.
+	 */
+	ERTS_VBUMP_ALL_REDS(p);
+    }
+#endif
+
     if (enqueue)
 	add2runq(enqueue > 0 ? p : make_proxy_proc(NULL, p, enq_prio),
 		 state,