erts: Implement tracer modules

Add the possibility to use modules as trace data receivers. The functions
in the module have to be nifs as otherwise complex trace probes will be
very hard to handle (complex means trace probes for ports for example).

This commit changes the way that the ptab->tracer field works from always
being an immediate, to now be NIL if no tracer is present or else be
the tuple {TracerModule, TracerState} where TracerModule is an atom that
is later used to lookup the appropriate tracer callbacks to call and
TracerState is just passed to the tracer callback. The default process and
port tracers have been rewritten to use the new API.

This commit also changes the order which trace messages are delivered to the
potential tracer process. Any enif_send done in a tracer module may be delayed
indefinitely because of lock order issues. If a message is delayed any other
trace message send from that process is also delayed so that order is preserved
for each traced entity. This means that for some trace events (i.e. send/receive)
the events may come in an unintuitive order (receive before send) to the
trace receiver. Timestamps are taken when the trace message is generated so
trace messages from differented processes may arrive with the timestamp
out of order.

Both the erlang:trace and seq_trace:set_system_tracer accept the new tracer
module tracers and also the backwards compatible arguments.

OTP-10267

1 files changed, 157 insertions, 34 deletions

diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c
index e1f470d381..a69185bc5c 100644
--- a/erts/emulator/beam/erl_process_lock.c
+++ b/erts/emulator/beam/erl_process_lock.c
@@ -106,6 +106,7 @@ static struct {
     Sint16 proc_lock_msgq;
     Sint16 proc_lock_btm;
     Sint16 proc_lock_status;
+    Sint16 proc_lock_trace;
 } lc_id;
 #endif
 
@@ -152,6 +153,7 @@ erts_init_proc_lock(int cpus)
     lc_id.proc_lock_msgq	= erts_lc_get_lock_order_id("proc_msgq");
     lc_id.proc_lock_btm		= erts_lc_get_lock_order_id("proc_btm");
     lc_id.proc_lock_status	= erts_lc_get_lock_order_id("proc_status");
+    lc_id.proc_lock_trace	= erts_lc_get_lock_order_id("proc_trace");
 #endif
 }
 
@@ -1057,6 +1059,11 @@ erts_proc_lock_init(Process *p)
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_trylock(1, &p->lock.status.lc);
 #endif
+    erts_mtx_init_x(&p->lock.trace, "proc_trace", p->common.id, do_lock_count);
+    ethr_mutex_lock(&p->lock.trace.mtx);
+#ifdef ERTS_ENABLE_LOCK_CHECK
+    erts_lc_trylock(1, &p->lock.trace.lc);
+#endif
 #endif
 #ifdef ERTS_PROC_LOCK_DEBUG
     for (i = 0; i <= ERTS_PROC_LOCK_MAX_BIT; i++)
@@ -1078,6 +1085,7 @@ erts_proc_lock_fin(Process *p)
     erts_mtx_destroy(&p->lock.msgq);
     erts_mtx_destroy(&p->lock.btm);
     erts_mtx_destroy(&p->lock.status);
+    erts_mtx_destroy(&p->lock.trace);
 #endif
 #if defined(ERTS_ENABLE_LOCK_COUNT) && defined(ERTS_SMP)
     erts_lcnt_proc_lock_destroy(p);
@@ -1100,26 +1108,29 @@ void erts_lcnt_enable_proc_lock_count(int enable) {
 void erts_lcnt_proc_lock_init(Process *p) {
     if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) {
         erts_lcnt_init_lock_empty(&(p->lock.lcnt_main));
+        erts_lcnt_init_lock_empty(&(p->lock.lcnt_link));
         erts_lcnt_init_lock_empty(&(p->lock.lcnt_msgq));
         erts_lcnt_init_lock_empty(&(p->lock.lcnt_btm));
-        erts_lcnt_init_lock_empty(&(p->lock.lcnt_link));
         erts_lcnt_init_lock_empty(&(p->lock.lcnt_status));
+        erts_lcnt_init_lock_empty(&(p->lock.lcnt_trace));
     } else { /* now the common case */
         Eterm pid = (p->common.id != ERTS_INVALID_PID) ? p->common.id : NIL;
         erts_lcnt_init_lock_x(&(p->lock.lcnt_main),  "proc_main",  ERTS_LCNT_LT_PROCLOCK, pid);
+        erts_lcnt_init_lock_x(&(p->lock.lcnt_link),  "proc_link",  ERTS_LCNT_LT_PROCLOCK, pid);
         erts_lcnt_init_lock_x(&(p->lock.lcnt_msgq),  "proc_msgq",  ERTS_LCNT_LT_PROCLOCK, pid);
         erts_lcnt_init_lock_x(&(p->lock.lcnt_btm),   "proc_btm",   ERTS_LCNT_LT_PROCLOCK, pid);
-        erts_lcnt_init_lock_x(&(p->lock.lcnt_link),  "proc_link",  ERTS_LCNT_LT_PROCLOCK, pid);
         erts_lcnt_init_lock_x(&(p->lock.lcnt_status),"proc_status",ERTS_LCNT_LT_PROCLOCK, pid);
+        erts_lcnt_init_lock_x(&(p->lock.lcnt_trace), "proc_trace", ERTS_LCNT_LT_PROCLOCK, pid);
     } /* the lock names should really be aligned to four characters */
 } /* logic reversed */
 
 void erts_lcnt_proc_lock_destroy(Process *p) {
     erts_lcnt_destroy_lock(&(p->lock.lcnt_main));
+    erts_lcnt_destroy_lock(&(p->lock.lcnt_link));
     erts_lcnt_destroy_lock(&(p->lock.lcnt_msgq));
     erts_lcnt_destroy_lock(&(p->lock.lcnt_btm));
-    erts_lcnt_destroy_lock(&(p->lock.lcnt_link));
     erts_lcnt_destroy_lock(&(p->lock.lcnt_status));
+    erts_lcnt_destroy_lock(&(p->lock.lcnt_trace));
 }
 
 static void lcnt_enable_proc_lock_count(Process *proc, int enable) {
@@ -1138,10 +1149,11 @@ static void lcnt_enable_proc_lock_count(Process *proc, int enable) {
 void erts_lcnt_proc_lock(erts_proc_lock_t *lock, ErtsProcLocks locks) {
     if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return;
     if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_lock(&(lock->lcnt_main)); }
+    if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_lock(&(lock->lcnt_link)); }
     if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_lock(&(lock->lcnt_msgq)); }
     if (locks & ERTS_PROC_LOCK_BTM) { erts_lcnt_lock(&(lock->lcnt_btm)); }
-    if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_lock(&(lock->lcnt_link)); }
     if (locks & ERTS_PROC_LOCK_STATUS) { erts_lcnt_lock(&(lock->lcnt_status)); }
+    if (locks & ERTS_PROC_LOCK_TRACE) { erts_lcnt_lock(&(lock->lcnt_trace)); }
 }
 
 void erts_lcnt_proc_lock_post_x(erts_proc_lock_t *lock, ErtsProcLocks locks,
@@ -1150,44 +1162,50 @@ void erts_lcnt_proc_lock_post_x(erts_proc_lock_t *lock, ErtsProcLocks locks,
     if (locks & ERTS_PROC_LOCK_MAIN) {
 	erts_lcnt_lock_post_x(&(lock->lcnt_main), file, line);
     }
+    if (locks & ERTS_PROC_LOCK_LINK) {
+	erts_lcnt_lock_post_x(&(lock->lcnt_link), file, line);
+    }
     if (locks & ERTS_PROC_LOCK_MSGQ) {
         erts_lcnt_lock_post_x(&(lock->lcnt_msgq), file, line);
     }
     if (locks & ERTS_PROC_LOCK_BTM) {
         erts_lcnt_lock_post_x(&(lock->lcnt_btm), file, line);
     }
-    if (locks & ERTS_PROC_LOCK_LINK) {
-	erts_lcnt_lock_post_x(&(lock->lcnt_link), file, line);
-    }
     if (locks & ERTS_PROC_LOCK_STATUS) {
 	erts_lcnt_lock_post_x(&(lock->lcnt_status), file, line);
     }
+    if (locks & ERTS_PROC_LOCK_TRACE) {
+        erts_lcnt_lock_post_x(&(lock->lcnt_trace), file, line);
+    }
 }
 
 void erts_lcnt_proc_lock_unaquire(erts_proc_lock_t *lock, ErtsProcLocks locks) {
     if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return;
     if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_lock_unaquire(&(lock->lcnt_main)); }
+    if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_lock_unaquire(&(lock->lcnt_link)); }
     if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_lock_unaquire(&(lock->lcnt_msgq)); }
     if (locks & ERTS_PROC_LOCK_BTM) { erts_lcnt_lock_unaquire(&(lock->lcnt_btm)); }
-    if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_lock_unaquire(&(lock->lcnt_link)); }
     if (locks & ERTS_PROC_LOCK_STATUS) { erts_lcnt_lock_unaquire(&(lock->lcnt_status)); }
+    if (locks & ERTS_PROC_LOCK_TRACE) { erts_lcnt_lock_unaquire(&(lock->lcnt_trace)); }
 }
 
 void erts_lcnt_proc_unlock(erts_proc_lock_t *lock, ErtsProcLocks locks) {
     if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return;
     if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_unlock(&(lock->lcnt_main)); }
+    if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_unlock(&(lock->lcnt_link)); }
     if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_unlock(&(lock->lcnt_msgq)); }
     if (locks & ERTS_PROC_LOCK_BTM) { erts_lcnt_unlock(&(lock->lcnt_btm)); }
-    if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_unlock(&(lock->lcnt_link)); }
     if (locks & ERTS_PROC_LOCK_STATUS) { erts_lcnt_unlock(&(lock->lcnt_status)); }
+    if (locks & ERTS_PROC_LOCK_TRACE) { erts_lcnt_unlock(&(lock->lcnt_trace)); }
 }
 void erts_lcnt_proc_trylock(erts_proc_lock_t *lock, ErtsProcLocks locks, int res) {
     if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return;
     if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_trylock(&(lock->lcnt_main), res); }
+    if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_trylock(&(lock->lcnt_link), res); }
     if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_trylock(&(lock->lcnt_msgq), res); }
     if (locks & ERTS_PROC_LOCK_BTM) { erts_lcnt_trylock(&(lock->lcnt_btm), res); }
-    if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_trylock(&(lock->lcnt_link), res); }
     if (locks & ERTS_PROC_LOCK_STATUS) { erts_lcnt_trylock(&(lock->lcnt_status), res); }
+    if (locks & ERTS_PROC_LOCK_TRACE) { erts_lcnt_trylock(&(lock->lcnt_trace), res); }
 } /* reversed logic */
 #endif /* ERTS_ENABLE_LOCK_COUNT */
 
@@ -1224,6 +1242,10 @@ erts_proc_lc_lock(Process *p, ErtsProcLocks locks, char *file, unsigned int line
 	lck.id = lc_id.proc_lock_status;
 	erts_lc_lock_x(&lck,file,line);
     }
+    if (locks & ERTS_PROC_LOCK_TRACE) {
+	lck.id = lc_id.proc_lock_trace;
+	erts_lc_lock_x(&lck,file,line);
+    }
 }
 
 void
@@ -1253,6 +1275,10 @@ erts_proc_lc_trylock(Process *p, ErtsProcLocks locks, int locked,
 	lck.id = lc_id.proc_lock_status;
 	erts_lc_trylock_x(locked, &lck, file, line);
     }
+    if (locks & ERTS_PROC_LOCK_TRACE) {
+	lck.id = lc_id.proc_lock_trace;
+	erts_lc_trylock_x(locked, &lck, file, line);
+    }
 }
 
 void
@@ -1261,6 +1287,10 @@ erts_proc_lc_unlock(Process *p, ErtsProcLocks locks)
     erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1,
 					   p->common.id,
 					   ERTS_LC_FLG_LT_PROCLOCK);
+    if (locks & ERTS_PROC_LOCK_TRACE) {
+	lck.id = lc_id.proc_lock_trace;
+	erts_lc_unlock(&lck);
+    }
     if (locks & ERTS_PROC_LOCK_STATUS) {
 	lck.id = lc_id.proc_lock_status;
 	erts_lc_unlock(&lck);
@@ -1292,6 +1322,10 @@ erts_proc_lc_might_unlock(Process *p, ErtsProcLocks locks)
     erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1,
 					   p->common.id,
 					   ERTS_LC_FLG_LT_PROCLOCK);
+    if (locks & ERTS_PROC_LOCK_TRACE) {
+	lck.id = lc_id.proc_lock_trace;
+	erts_lc_might_unlock(&lck);
+    }
     if (locks & ERTS_PROC_LOCK_STATUS) {
 	lck.id = lc_id.proc_lock_status;
 	erts_lc_might_unlock(&lck);
@@ -1323,6 +1357,8 @@ erts_proc_lc_might_unlock(Process *p, ErtsProcLocks locks)
 	erts_lc_might_unlock(&p->lock.btm.lc);
     if (locks & ERTS_PROC_LOCK_STATUS)
 	erts_lc_might_unlock(&p->lock.status.lc);
+    if (locks & ERTS_PROC_LOCK_TRACE)
+	erts_lc_might_unlock(&p->lock.trace.lc);
 #endif
 }
 
@@ -1354,6 +1390,10 @@ erts_proc_lc_require_lock(Process *p, ErtsProcLocks locks, char *file,
 	lck.id = lc_id.proc_lock_status;
 	erts_lc_require_lock(&lck, file, line);
     }
+    if (locks & ERTS_PROC_LOCK_TRACE) {
+	lck.id = lc_id.proc_lock_trace;
+	erts_lc_require_lock(&lck, file, line);
+    }
 #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
     if (locks & ERTS_PROC_LOCK_MAIN)
 	erts_lc_require_lock(&p->lock.main.lc, file, line);
@@ -1365,6 +1405,8 @@ erts_proc_lc_require_lock(Process *p, ErtsProcLocks locks, char *file,
 	erts_lc_require_lock(&p->lock.btm.lc, file, line);
     if (locks & ERTS_PROC_LOCK_STATUS)
 	erts_lc_require_lock(&p->lock.status.lc, file, line);
+    if (locks & ERTS_PROC_LOCK_TRACE)
+	erts_lc_require_lock(&p->lock.trace.lc, file, line);
 #endif
 }
 
@@ -1375,6 +1417,10 @@ erts_proc_lc_unrequire_lock(Process *p, ErtsProcLocks locks)
     erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1,
 					   p->common.id,
 					   ERTS_LC_FLG_LT_PROCLOCK);
+    if (locks & ERTS_PROC_LOCK_TRACE) {
+	lck.id = lc_id.proc_lock_trace;
+	erts_lc_unrequire_lock(&lck);
+    }
     if (locks & ERTS_PROC_LOCK_STATUS) {
 	lck.id = lc_id.proc_lock_status;
 	erts_lc_unrequire_lock(&lck);
@@ -1406,6 +1452,8 @@ erts_proc_lc_unrequire_lock(Process *p, ErtsProcLocks locks)
 	erts_lc_unrequire_lock(&p->lock.btm.lc);
     if (locks & ERTS_PROC_LOCK_STATUS)
 	erts_lc_unrequire_lock(&p->lock.status.lc);
+    if (locks & ERTS_PROC_LOCK_TRACE)
+	erts_lc_unrequire_lock(&p->lock.trace.lc);
 #endif
 }
 
@@ -1429,6 +1477,8 @@ erts_proc_lc_trylock_force_busy(Process *p, ErtsProcLocks locks)
 	    lck.id = lc_id.proc_lock_btm;
 	else if (locks & ERTS_PROC_LOCK_STATUS)
 	    lck.id = lc_id.proc_lock_status;
+	else if (locks & ERTS_PROC_LOCK_TRACE)
+	    lck.id = lc_id.proc_lock_trace;
 	else
 	    erts_lc_fail("Unknown proc lock found");
 
@@ -1441,14 +1491,7 @@ erts_proc_lc_trylock_force_busy(Process *p, ErtsProcLocks locks)
 
 void erts_proc_lc_chk_only_proc_main(Process *p)
 {
-#if ERTS_PROC_LOCK_OWN_IMPL
-    erts_lc_lock_t proc_main = ERTS_LC_LOCK_INIT(lc_id.proc_lock_main,
-						 p->common.id,
-						 ERTS_LC_FLG_LT_PROCLOCK);
-    erts_lc_check_exact(&proc_main, 1);
-#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
-    erts_lc_check_exact(&p->lock.main.lc, 1);
-#endif
+    erts_proc_lc_chk_only_proc(p, ERTS_PROC_LOCK_MAIN);
 }
 
 #if ERTS_PROC_LOCK_OWN_IMPL
@@ -1456,14 +1499,67 @@ void erts_proc_lc_chk_only_proc_main(Process *p)
   ERTS_LC_LOCK_INIT(-1, THE_NON_VALUE, ERTS_LC_FLG_LT_PROCLOCK)
 #endif /* ERTS_PROC_LOCK_OWN_IMPL */
 
+void erts_proc_lc_chk_only_proc(Process *p, ErtsProcLocks locks)
+{
+    int have_locks_len = 0;
+#if ERTS_PROC_LOCK_OWN_IMPL
+    erts_lc_lock_t have_locks[6] = {ERTS_PROC_LC_EMPTY_LOCK_INIT,
+				    ERTS_PROC_LC_EMPTY_LOCK_INIT,
+				    ERTS_PROC_LC_EMPTY_LOCK_INIT,
+				    ERTS_PROC_LC_EMPTY_LOCK_INIT,
+				    ERTS_PROC_LC_EMPTY_LOCK_INIT,
+                                    ERTS_PROC_LC_EMPTY_LOCK_INIT};
+    if (locks & ERTS_PROC_LOCK_MAIN) {
+	have_locks[have_locks_len].id = lc_id.proc_lock_main;
+	have_locks[have_locks_len++].extra = p->common.id;
+    }
+    if (locks & ERTS_PROC_LOCK_LINK) {
+	have_locks[have_locks_len].id = lc_id.proc_lock_link;
+	have_locks[have_locks_len++].extra = p->common.id;
+    }
+    if (locks & ERTS_PROC_LOCK_MSGQ) {
+	have_locks[have_locks_len].id = lc_id.proc_lock_msgq;
+	have_locks[have_locks_len++].extra = p->common.id;
+    }
+    if (locks & ERTS_PROC_LOCK_BTM) {
+	have_locks[have_locks_len].id = lc_id.proc_lock_btm;
+	have_locks[have_locks_len++].extra = p->common.id;
+    }
+    if (locks & ERTS_PROC_LOCK_STATUS) {
+	have_locks[have_locks_len].id = lc_id.proc_lock_status;
+	have_locks[have_locks_len++].extra = p->common.id;
+    }
+    if (locks & ERTS_PROC_LOCK_TRACE) {
+	have_locks[have_locks_len].id = lc_id.proc_lock_trace;
+	have_locks[have_locks_len++].extra = p->common.id;
+    }
+#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
+    erts_lc_lock_t have_locks[6];
+    if (locks & ERTS_PROC_LOCK_MAIN)
+	have_locks[have_locks_len++] = p->lock.main.lc;
+    if (locks & ERTS_PROC_LOCK_LINK)
+	have_locks[have_locks_len++] = p->lock.link.lc;
+    if (locks & ERTS_PROC_LOCK_MSGQ)
+	have_locks[have_locks_len++] = p->lock.msgq.lc;
+    if (locks & ERTS_PROC_LOCK_BTM)
+	have_locks[have_locks_len++] = p->lock.btm.lc;
+    if (locks & ERTS_PROC_LOCK_STATUS)
+	have_locks[have_locks_len++] = p->lock.status.lc;
+    if (locks & ERTS_PROC_LOCK_TRACE)
+	have_locks[have_locks_len++] = p->lock.trace.lc;
+#endif
+    erts_lc_check_exact(have_locks, have_locks_len);
+}
+
 void
 erts_proc_lc_chk_have_proc_locks(Process *p, ErtsProcLocks locks)
 {
     int have_locks_len = 0;
 #if ERTS_PROC_LOCK_OWN_IMPL
-    erts_lc_lock_t have_locks[5] = {ERTS_PROC_LC_EMPTY_LOCK_INIT,
+    erts_lc_lock_t have_locks[6] = {ERTS_PROC_LC_EMPTY_LOCK_INIT,
 				    ERTS_PROC_LC_EMPTY_LOCK_INIT,
 				    ERTS_PROC_LC_EMPTY_LOCK_INIT,
+                                    ERTS_PROC_LC_EMPTY_LOCK_INIT,
 				    ERTS_PROC_LC_EMPTY_LOCK_INIT,
 				    ERTS_PROC_LC_EMPTY_LOCK_INIT};
     if (locks & ERTS_PROC_LOCK_MAIN) {
@@ -1486,8 +1582,12 @@ erts_proc_lc_chk_have_proc_locks(Process *p, ErtsProcLocks locks)
 	have_locks[have_locks_len].id = lc_id.proc_lock_status;
 	have_locks[have_locks_len++].extra = p->common.id;
     }
+    if (locks & ERTS_PROC_LOCK_TRACE) {
+	have_locks[have_locks_len].id = lc_id.proc_lock_trace;
+	have_locks[have_locks_len++].extra = p->common.id;
+    }
 #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
-    erts_lc_lock_t have_locks[5];
+    erts_lc_lock_t have_locks[6];
     if (locks & ERTS_PROC_LOCK_MAIN)
 	have_locks[have_locks_len++] = p->lock.main.lc;
     if (locks & ERTS_PROC_LOCK_LINK)
@@ -1498,6 +1598,8 @@ erts_proc_lc_chk_have_proc_locks(Process *p, ErtsProcLocks locks)
 	have_locks[have_locks_len++] = p->lock.btm.lc;
     if (locks & ERTS_PROC_LOCK_STATUS)
 	have_locks[have_locks_len++] = p->lock.status.lc;
+    if (locks & ERTS_PROC_LOCK_TRACE)
+	have_locks[have_locks_len++] = p->lock.trace.lc;
 #endif
     erts_lc_check(have_locks, have_locks_len, NULL, 0);
 }
@@ -1508,12 +1610,14 @@ erts_proc_lc_chk_proc_locks(Process *p, ErtsProcLocks locks)
     int have_locks_len = 0;
     int have_not_locks_len = 0;
 #if ERTS_PROC_LOCK_OWN_IMPL
-    erts_lc_lock_t have_locks[5] = {ERTS_PROC_LC_EMPTY_LOCK_INIT,
+    erts_lc_lock_t have_locks[6] = {ERTS_PROC_LC_EMPTY_LOCK_INIT,
 				    ERTS_PROC_LC_EMPTY_LOCK_INIT,
+                                    ERTS_PROC_LC_EMPTY_LOCK_INIT,
 				    ERTS_PROC_LC_EMPTY_LOCK_INIT,
 				    ERTS_PROC_LC_EMPTY_LOCK_INIT};
-    erts_lc_lock_t have_not_locks[5] = {ERTS_PROC_LC_EMPTY_LOCK_INIT,
+    erts_lc_lock_t have_not_locks[6] = {ERTS_PROC_LC_EMPTY_LOCK_INIT,
 					ERTS_PROC_LC_EMPTY_LOCK_INIT,
+                                        ERTS_PROC_LC_EMPTY_LOCK_INIT,
 					ERTS_PROC_LC_EMPTY_LOCK_INIT,
 					ERTS_PROC_LC_EMPTY_LOCK_INIT};
 
@@ -1557,9 +1661,17 @@ erts_proc_lc_chk_proc_locks(Process *p, ErtsProcLocks locks)
 	have_not_locks[have_not_locks_len].id = lc_id.proc_lock_status;
 	have_not_locks[have_not_locks_len++].extra = p->common.id;
     }
+    if (locks & ERTS_PROC_LOCK_TRACE) {
+	have_locks[have_locks_len].id = lc_id.proc_lock_trace;
+	have_locks[have_locks_len++].extra = p->common.id;
+    }
+    else {
+	have_not_locks[have_not_locks_len].id = lc_id.proc_lock_trace;
+	have_not_locks[have_not_locks_len++].extra = p->common.id;
+    }
 #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
-    erts_lc_lock_t have_locks[5];
-    erts_lc_lock_t have_not_locks[5];
+    erts_lc_lock_t have_locks[6];
+    erts_lc_lock_t have_not_locks[6];
 
     if (locks & ERTS_PROC_LOCK_MAIN)
 	have_locks[have_locks_len++] = p->lock.main.lc;
@@ -1581,6 +1693,10 @@ erts_proc_lc_chk_proc_locks(Process *p, ErtsProcLocks locks)
 	have_locks[have_locks_len++] = p->lock.status.lc;
     else
 	have_not_locks[have_not_locks_len++] = p->lock.status.lc;
+    if (locks & ERTS_PROC_LOCK_TRACE)
+	have_locks[have_locks_len++] = p->lock.trace.lc;
+    else
+	have_not_locks[have_not_locks_len++] = p->lock.trace.lc;
 #endif
 
     erts_lc_check(have_locks, have_locks_len,
@@ -1590,10 +1706,10 @@ erts_proc_lc_chk_proc_locks(Process *p, ErtsProcLocks locks)
 ErtsProcLocks
 erts_proc_lc_my_proc_locks(Process *p)
 {
-    int resv[5];
+    int resv[6];
     ErtsProcLocks res = 0;
 #if ERTS_PROC_LOCK_OWN_IMPL
-    erts_lc_lock_t locks[5] = {ERTS_LC_LOCK_INIT(lc_id.proc_lock_main,
+    erts_lc_lock_t locks[6] = {ERTS_LC_LOCK_INIT(lc_id.proc_lock_main,
 						 p->common.id,
 						 ERTS_LC_FLG_LT_PROCLOCK),
 			       ERTS_LC_LOCK_INIT(lc_id.proc_lock_link,
@@ -1607,16 +1723,20 @@ erts_proc_lc_my_proc_locks(Process *p)
 						 ERTS_LC_FLG_LT_PROCLOCK),
 			       ERTS_LC_LOCK_INIT(lc_id.proc_lock_status,
 						 p->common.id,
+						 ERTS_LC_FLG_LT_PROCLOCK),
+			       ERTS_LC_LOCK_INIT(lc_id.proc_lock_trace,
+						 p->common.id,
 						 ERTS_LC_FLG_LT_PROCLOCK)};
 #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
-    erts_lc_lock_t locks[5] = {p->lock.main.lc,
+    erts_lc_lock_t locks[6] = {p->lock.main.lc,
 			       p->lock.link.lc,
 			       p->lock.msgq.lc,
 			       p->lock.btm.lc,
-			       p->lock.status.lc};
+			       p->lock.status.lc,
+			       p->lock.trace.lc};
 #endif
 
-    erts_lc_have_locks(resv, locks, 5);
+    erts_lc_have_locks(resv, locks, 6);
     if (resv[0])
 	res |= ERTS_PROC_LOCK_MAIN;
     if (resv[1])
@@ -1627,6 +1747,8 @@ erts_proc_lc_my_proc_locks(Process *p)
 	res |= ERTS_PROC_LOCK_BTM;
     if (resv[4])
 	res |= ERTS_PROC_LOCK_STATUS;
+    if (resv[5])
+	res |= ERTS_PROC_LOCK_TRACE;
 
     return res;
 }
@@ -1634,14 +1756,15 @@ erts_proc_lc_my_proc_locks(Process *p)
 void
 erts_proc_lc_chk_no_proc_locks(char *file, int line)
 {
-    int resv[5];
-    int ids[5] = {lc_id.proc_lock_main,
+    int resv[6];
+    int ids[6] = {lc_id.proc_lock_main,
 		  lc_id.proc_lock_link,
 		  lc_id.proc_lock_msgq,
 		  lc_id.proc_lock_btm,
-		  lc_id.proc_lock_status};
-    erts_lc_have_lock_ids(resv, ids, 5);
-    if (!ERTS_IS_CRASH_DUMPING && (resv[0] || resv[1] || resv[2] || resv[3] || resv[4])) {
+		  lc_id.proc_lock_status,
+		  lc_id.proc_lock_trace};
+    erts_lc_have_lock_ids(resv, ids, 6);
+    if (!ERTS_IS_CRASH_DUMPING && (resv[0] || resv[1] || resv[2] || resv[3] || resv[4] || resv[5])) {
 	erts_lc_fail("%s:%d: Thread has process locks locked when expected "
 		     "not to have any process locks locked",
 		     file, line);