erts: Fix seq_trace:reset_trace dirty gc bug

When seq_trace:reset_trace could be called while a
process was doing a dirty GC. This triggered a race
where all signals was moved to the internal signal
queue during the GC which in turn caused the a heap
overrun problem.

This fix makes it so that the main and msgq lock are
taken before the clear. This will make sure that we
are allowed to do the clear.

6 files changed, 12 insertions, 11 deletions

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 45b7540aeb..fdce4f7b95 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -542,6 +542,7 @@ atom reload
 atom rem
 atom report_errors
 atom reset
+atom reset_seq_trace
 atom restart
 atom return_from
 atom return_to
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index f18af8bcd7..79ed5a0c18 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -4486,11 +4486,12 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
         ERTS_TRACER_CLEAR(&old_seq_tracer);
 
         BIF_RET(ret);
-    } else if (BIF_ARG_1 == make_small(1)) {
+    } else if (BIF_ARG_1 == am_reset_seq_trace) {
 	int i, max;
-	erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-	erts_thr_progress_block();
 
+        erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
+        erts_thr_progress_block();
+        
 	max = erts_ptab_max(&erts_proc);
 	for (i = 0; i < max; i++) {
 	    Process *p = erts_pix2proc(i);
@@ -4502,13 +4503,14 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
 #endif
 		p->seq_trace_clock = 0;
 		p->seq_trace_lastcnt = 0;
-
+                erts_proc_lock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_MSGQ);
                 erts_proc_sig_clear_seq_trace_tokens(p);
+                erts_proc_unlock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_MSGQ);
 	    }
 	}
 
-	erts_thr_progress_unblock();
-	erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+        erts_thr_progress_unblock();
+        erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
 
 	BIF_RET(am_true);
     } else if (BIF_ARG_1 == am_scheduler_wall_time) {
diff --git a/erts/emulator/beam/erl_proc_sig_queue.c b/erts/emulator/beam/erl_proc_sig_queue.c
index f343e984f7..18418a76e1 100644
--- a/erts/emulator/beam/erl_proc_sig_queue.c
+++ b/erts/emulator/beam/erl_proc_sig_queue.c
@@ -3812,7 +3812,6 @@ clear_seq_trace_token(ErtsMessage *sig)
 void
 erts_proc_sig_clear_seq_trace_tokens(Process *c_p)
 {
-    ASSERT(erts_thr_progress_is_blocking());
     erts_proc_sig_fetch(c_p);
     ERTS_FOREACH_SIG_PRIVQS(c_p, sig, clear_seq_trace_token(sig));
 }
diff --git a/erts/emulator/beam/erl_proc_sig_queue.h b/erts/emulator/beam/erl_proc_sig_queue.h
index 3fc2d06b2d..6b065a7add 100644
--- a/erts/emulator/beam/erl_proc_sig_queue.h
+++ b/erts/emulator/beam/erl_proc_sig_queue.h
@@ -989,8 +989,7 @@ erts_proc_sig_fetch(Process *proc)
     Sint res = 0;
     ErtsSignal *sig;
 
-    ERTS_LC_ASSERT(erts_thr_progress_is_blocking()
-                   || ERTS_PROC_IS_EXITING(proc)
+    ERTS_LC_ASSERT(ERTS_PROC_IS_EXITING(proc)
                    || ((erts_proc_lc_my_proc_locks(proc)
                         & (ERTS_PROC_LOCK_MAIN
                            | ERTS_PROC_LOCK_MSGQ))
diff --git a/erts/preloaded/src/erlang.erl b/erts/preloaded/src/erlang.erl
index 1ed6b6b284..f37a3fc5db 100644
--- a/erts/preloaded/src/erlang.erl
+++ b/erts/preloaded/src/erlang.erl
@@ -2534,7 +2534,7 @@ subtract(_,_) ->
                         %% These are deliberately not documented
 			(internal_cpu_topology, term()) -> term();
                         (sequential_tracer, pid() | port() | {module(), term()} | false) -> pid() | port() | false;
-                        (1,0) -> true.
+                        (reset_seq_trace,true) -> true.
 
 system_flag(_Flag, _Value) ->
     erlang:nif_error(undefined).
diff --git a/lib/kernel/src/seq_trace.erl b/lib/kernel/src/seq_trace.erl
index 14fe21e9de..4f9d7b3e5c 100644
--- a/lib/kernel/src/seq_trace.erl
+++ b/lib/kernel/src/seq_trace.erl
@@ -98,7 +98,7 @@ print(Label, Term) ->
 -spec reset_trace() -> 'true'.
 
 reset_trace() ->
-    erlang:system_flag(1, 0).
+    erlang:system_flag(reset_seq_trace, true).
 
 %% reset_trace(Pid) -> % this might be a useful function too