erts: Implement tracer modules

Add the possibility to use modules as trace data receivers. The functions
in the module have to be nifs as otherwise complex trace probes will be
very hard to handle (complex means trace probes for ports for example).

This commit changes the way that the ptab->tracer field works from always
being an immediate, to now be NIL if no tracer is present or else be
the tuple {TracerModule, TracerState} where TracerModule is an atom that
is later used to lookup the appropriate tracer callbacks to call and
TracerState is just passed to the tracer callback. The default process and
port tracers have been rewritten to use the new API.

This commit also changes the order which trace messages are delivered to the
potential tracer process. Any enif_send done in a tracer module may be delayed
indefinitely because of lock order issues. If a message is delayed any other
trace message send from that process is also delayed so that order is preserved
for each traced entity. This means that for some trace events (i.e. send/receive)
the events may come in an unintuitive order (receive before send) to the
trace receiver. Timestamps are taken when the trace message is generated so
trace messages from differented processes may arrive with the timestamp
out of order.

Both the erlang:trace and seq_trace:set_system_tracer accept the new tracer
module tracers and also the backwards compatible arguments.

OTP-10267

1 files changed, 212 insertions, 331 deletions

diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index a9443ee8df..049899211f 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -52,7 +52,7 @@ static int                         erts_default_trace_pattern_is_on;
 static Binary                     *erts_default_match_spec;
 static Binary                     *erts_default_meta_match_spec;
 static struct trace_pattern_flags  erts_default_trace_pattern_flags;
-static Eterm                       erts_default_meta_tracer_pid;
+static ErtsTracer                  erts_default_meta_tracer;
 
 static struct {			/* Protected by code write permission */
     int current;
@@ -75,8 +75,6 @@ static BIF_RETTYPE
 system_monitor(Process *p, Eterm monitor_pid, Eterm list);
 
 static void new_seq_trace_token(Process* p); /* help func for seq_trace_2*/
-static int already_traced(Process *p, Process *tracee_p, Eterm tracer);
-static int port_already_traced(Process *p, Port *tracee_port, Eterm tracer);
 static Eterm trace_info_pid(Process* p, Eterm pid_spec, Eterm key);
 static Eterm trace_info_func(Process* p, Eterm pid_spec, Eterm key);
 static Eterm trace_info_on_load(Process* p, Eterm key);
@@ -94,21 +92,15 @@ erts_bif_trace_init(void)
     erts_default_match_spec = NULL;
     erts_default_meta_match_spec = NULL;
     erts_default_trace_pattern_flags = erts_trace_pattern_flags_off;
-    erts_default_meta_tracer_pid = NIL;
+    erts_default_meta_tracer = erts_tracer_nil;
 }
 
 /*
  * Turn on/off call tracing for the given function(s).
  */
-  
-Eterm
-trace_pattern_2(BIF_ALIST_2)
-{
-    return trace_pattern(BIF_P, BIF_ARG_1, BIF_ARG_2, NIL);
-}
 
 Eterm
-trace_pattern_3(BIF_ALIST_3)
+erts_internal_trace_pattern_3(BIF_ALIST_3)
 {
     return trace_pattern(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
 }
@@ -125,11 +117,10 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
     Eterm l;
     struct trace_pattern_flags flags = erts_trace_pattern_flags_off;
     int is_global;
-    Process *meta_tracer_proc = p;
-    Eterm meta_tracer_pid = p->common.id;
+    ErtsTracer meta_tracer = erts_tracer_nil;
 
     if (!erts_try_seize_code_write_permission(p)) {
-	ERTS_BIF_YIELD3(bif_export[BIF_trace_pattern_3], p, MFA, Pattern, flaglist);
+	ERTS_BIF_YIELD3(bif_export[BIF_erts_internal_trace_pattern_3], p, MFA, Pattern, flaglist);
     }
     finish_bp.current = -1;
 
@@ -160,31 +151,11 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
     is_global = 0;
     for(l = flaglist; is_list(l); l = CDR(list_val(l))) {
 	if (is_tuple(CAR(list_val(l)))) {
-	    Eterm *tp = tuple_val(CAR(list_val(l)));
-	    
-	    if (arityval(tp[0]) != 2 || tp[1] != am_meta) {
-		goto error;
-	    }
-	    meta_tracer_pid = tp[2];
-	    if (is_internal_pid(meta_tracer_pid)) {
-		meta_tracer_proc = erts_pid2proc(NULL, 0, meta_tracer_pid, 0);
-		if (!meta_tracer_proc) {
-		    goto error;
-		}
-	    } else if (is_internal_port(meta_tracer_pid)) {
-		Port *meta_tracer_port;
-		meta_tracer_proc = NULL;
-		meta_tracer_port = (erts_port_lookup(
-					meta_tracer_pid,
-					ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP));
-		if (!meta_tracer_port)
-		    goto error;
-	    } else {
-		goto error;
-	    }
-	    if (is_global) {
-		goto error;
-	    }
+            meta_tracer = erts_term_to_tracer(am_meta, CAR(list_val(l)));
+            if (meta_tracer == THE_NON_VALUE) {
+                meta_tracer = erts_tracer_nil;
+                goto error;
+            }
 	    flags.breakpoint = 1;
 	    flags.meta       = 1;
 	} else {
@@ -202,6 +173,8 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
 		}
 		flags.breakpoint = 1;
 		flags.meta       = 1;
+                if (ERTS_TRACER_IS_NIL(meta_tracer))
+                    meta_tracer = erts_term_to_tracer(THE_NON_VALUE, p->common.id);
 		break;
 	    case am_global:
 		if (flags.breakpoint) {
@@ -252,14 +225,11 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
 	    MatchSetUnref(erts_default_meta_match_spec);
 	    erts_default_meta_match_spec = match_prog_set;
 	    MatchSetRef(erts_default_meta_match_spec);
-	    erts_default_meta_tracer_pid = meta_tracer_pid;
-	    if (meta_tracer_proc) {
-		ERTS_TRACE_FLAGS(meta_tracer_proc) |= F_TRACER;
-	    }
+            erts_tracer_update(&erts_default_meta_tracer, meta_tracer);
 	} else if (! flags.breakpoint) {
 	    MatchSetUnref(erts_default_meta_match_spec);
 	    erts_default_meta_match_spec = NULL;
-	    erts_default_meta_tracer_pid = NIL;
+	    ERTS_TRACER_CLEAR(&erts_default_meta_tracer);
 	}
 	if (erts_default_trace_pattern_flags.breakpoint &&
 	    flags.breakpoint) { 
@@ -340,20 +310,18 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
 	if (is_small(mfa[2])) {
 	    mfa[2] = signed_val(mfa[2]);
 	}
-    
-	if (meta_tracer_proc) {
-	    ERTS_TRACE_FLAGS(meta_tracer_proc) |= F_TRACER;
-	}
 
 	matches = erts_set_trace_pattern(p, mfa, specified,
 					 match_prog_set, match_prog_set,
-					 on, flags, meta_tracer_pid, 0);
+					 on, flags, meta_tracer, 0);
     }
 
  error:
     MatchSetUnref(match_prog_set);
     UnUseTmpHeap(3,p);
 
+    ERTS_TRACER_CLEAR(&meta_tracer);
+
 #ifdef ERTS_SMP
     if (finish_bp.current >= 0) {
 	ASSERT(matches >= 0);
@@ -404,7 +372,7 @@ erts_get_default_trace_pattern(int *trace_pattern_is_on,
 			       Binary **match_spec,
 			       Binary **meta_match_spec,
 			       struct trace_pattern_flags *trace_pattern_flags,
-			       Eterm *meta_tracer_pid)
+			       ErtsTracer *meta_tracer)
 {
     ERTS_SMP_LC_ASSERT(erts_has_code_write_permission() ||
 		       erts_smp_thr_progress_is_blocking());
@@ -416,8 +384,8 @@ erts_get_default_trace_pattern(int *trace_pattern_is_on,
 	*meta_match_spec = erts_default_meta_match_spec;
     if (trace_pattern_flags)
 	*trace_pattern_flags = erts_default_trace_pattern_flags;
-    if (meta_tracer_pid)
-	*meta_tracer_pid = erts_default_meta_tracer_pid;
+    if (meta_tracer)
+	*meta_tracer = erts_default_meta_tracer;
 }
 
 int erts_is_default_trace_enabled(void)
@@ -465,12 +433,12 @@ erts_trace_flag2bit(Eterm flag)
 ** occurred in the argument list.
 */
 int
-erts_trace_flags(Eterm List, 
-		 Uint *pMask, Eterm *pTracer, int *pCpuTimestamp)
+erts_trace_flags(Eterm List,
+                 Uint *pMask, ErtsTracer *pTracer, int *pCpuTimestamp)
 {
     Eterm list = List;
     Uint mask = 0;
-    Eterm tracer = NIL;
+    ErtsTracer tracer = erts_tracer_nil;
     int cpu_timestamp = 0;
     
     while (is_list(list)) {
@@ -483,33 +451,72 @@ erts_trace_flags(Eterm List,
 	    cpu_timestamp = !0;
 #endif
 	} else if (is_tuple(item)) {
-	    Eterm* tp = tuple_val(item);
-	    
-	    if (arityval(tp[0]) != 2 || tp[1] != am_tracer) goto error;
-	    if (is_internal_pid(tp[2]) || is_internal_port(tp[2])) {
-		tracer = tp[2];
-	    } else goto error;
+            tracer = erts_term_to_tracer(am_tracer, item);
+            if (tracer == THE_NON_VALUE)
+                goto error;
 	} else goto error;
 	list = CDR(list_val(list));
     }
     if (is_not_nil(list)) goto error;
     
-    if (pMask && mask)                  *pMask         = mask;
-    if (pTracer && tracer != NIL)       *pTracer       = tracer;
-    if (pCpuTimestamp && cpu_timestamp) *pCpuTimestamp = cpu_timestamp;
+    if (pMask && mask)                           *pMask         = mask;
+    if (pTracer && !ERTS_TRACER_IS_NIL(tracer))  *pTracer       = tracer;
+    if (pCpuTimestamp && cpu_timestamp)          *pCpuTimestamp = cpu_timestamp;
     return !0;
  error:
     return 0;
 }
 
-Eterm trace_3(BIF_ALIST_3)
+static ERTS_INLINE int
+start_trace(Process *c_p, ErtsTracer tracer,
+            ErtsPTabElementCommon *common,
+            int on, int mask)
+{
+    /* We can use the common part of both port+proc without checking what it is
+       In the code below port is used for both proc and port */
+    Port *port = (Port*)common;
+
+    /*
+     * SMP build assumes that either system is blocked or:
+     * * main lock is held on c_p
+     * * all locks are held on port common
+     */
+
+    if (!ERTS_TRACER_IS_NIL(tracer)) {
+        if ((ERTS_TRACE_FLAGS(port) & TRACEE_FLAGS)
+            && !ERTS_TRACER_COMPARE(ERTS_TRACER(port), tracer)) {
+            /* This tracee is already being traced, and not by the
+             * tracer to be */
+            if (erts_is_tracer_proc_enabled(c_p, ERTS_PROC_LOCKS_ALL, common)) {
+                /* The tracer is still in use */
+                return 1;
+            }
+            /* Current tracer now invalid */
+        }
+    }
+
+    if (on)
+        ERTS_TRACE_FLAGS(port) |= mask;
+    else
+        ERTS_TRACE_FLAGS(port) &= ~mask;
+
+    if ((ERTS_TRACE_FLAGS(port) & TRACEE_FLAGS) == 0) {
+        tracer = erts_tracer_nil;
+        erts_tracer_replace(common, erts_tracer_nil);
+    } else if (!ERTS_TRACER_IS_NIL(tracer))
+        erts_tracer_replace(common, tracer);
+
+    return 0;
+}
+
+Eterm erts_internal_trace_3(BIF_ALIST_3)
 {
     Process* p = BIF_P;
     Eterm pid_spec = BIF_ARG_1;
     Eterm how = BIF_ARG_2;
     Eterm list = BIF_ARG_3;
     int on;
-    Eterm tracer = NIL;
+    ErtsTracer tracer = erts_tracer_nil;
     int matches = 0;
     Uint mask = 0;
     int cpu_ts = 0;
@@ -522,41 +529,24 @@ Eterm trace_3(BIF_ALIST_3)
     }
 
     if (!erts_try_seize_code_write_permission(BIF_P)) {
-	ERTS_BIF_YIELD3(bif_export[BIF_trace_3], BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
+	ERTS_BIF_YIELD3(bif_export[BIF_erts_internal_trace_3],
+                        BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
     }
 
-    if (is_nil(tracer) || is_internal_pid(tracer)) {
-	Process *tracer_proc = erts_pid2proc(p,
-					     ERTS_PROC_LOCK_MAIN,
-					     is_nil(tracer) ? p->common.id : tracer,
-					     ERTS_PROC_LOCKS_ALL);
-	if (!tracer_proc)
-	    goto error;
-	ERTS_TRACE_FLAGS(tracer_proc) |= F_TRACER;
-	erts_smp_proc_unlock(tracer_proc,
-			     (tracer_proc == p
-			      ? ERTS_PROC_LOCKS_ALL_MINOR
-			      : ERTS_PROC_LOCKS_ALL));
-    } else if (is_internal_port(tracer)) {
-	Port *tracer_port = erts_id2port_sflgs(tracer,
-					       p,
-					       ERTS_PROC_LOCK_MAIN,
-					       ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP);
-	if (!tracer_port)
-	    goto error;
-	ERTS_TRACE_FLAGS(tracer_port) |= F_TRACER;
-	erts_port_release(tracer_port);
-    } else
-	goto error;
-
     switch (how) {
     case am_false: 
 	on = 0; 
 	break;
     case am_true: 
 	on = 1;
-	if (is_nil(tracer))
-	    tracer = p->common.id;
+        if (ERTS_TRACER_IS_NIL(tracer))
+            tracer = erts_term_to_tracer(am_tracer, p->common.id);
+
+        if (tracer == THE_NON_VALUE) {
+            tracer = erts_tracer_nil;
+            goto error;
+        }
+
 	break;
     default: 
 	goto error;
@@ -575,34 +565,20 @@ Eterm trace_3(BIF_ALIST_3)
 	}
 #endif
 
-	if (pid_spec == tracer)
-	    goto error;
-
 	tracee_port = erts_id2port_sflgs(pid_spec,
 					 p,
 					 ERTS_PROC_LOCK_MAIN,
 					 ERTS_PORT_SFLGS_INVALID_LOOKUP);
+
 	if (!tracee_port)
 	    goto error;
-	
-	if (tracer != NIL && port_already_traced(p, tracee_port, tracer)) {
+
+        if (start_trace(p, tracer, &tracee_port->common, on, mask)) {
 	    erts_port_release(tracee_port);
 	    goto already_traced;
-	}
-
-	if (on)
-	    ERTS_TRACE_FLAGS(tracee_port) |= mask;
-	else
-	    ERTS_TRACE_FLAGS(tracee_port) &= ~mask;
-
-	if (!ERTS_TRACE_FLAGS(tracee_port))
-	    ERTS_TRACER_PROC(tracee_port) = NIL;
-	else if (tracer != NIL)
-	    ERTS_TRACER_PROC(tracee_port) = tracer;
-
-	erts_port_release(tracee_port);
-
-	matches = 1;
+        }
+        erts_port_release(tracee_port);
+        matches = 1;
     } else if (is_pid(pid_spec)) {
 	Process *tracee_p;
 
@@ -615,33 +591,19 @@ Eterm trace_3(BIF_ALIST_3)
 	 * and not about to be tracing.
 	 */
 
-	if (pid_spec == tracer)
-	    goto error;
-
 	tracee_p = erts_pid2proc(p, ERTS_PROC_LOCK_MAIN,
 				 pid_spec, ERTS_PROC_LOCKS_ALL);
 	if (!tracee_p)
 	    goto error;
 
-	if (tracer != NIL && already_traced(p, tracee_p, tracer)) {
+        if (start_trace(tracee_p, tracer, &tracee_p->common, on, mask)) {
 	    erts_smp_proc_unlock(tracee_p,
 				 (tracee_p == p
 				  ? ERTS_PROC_LOCKS_ALL_MINOR
 				  : ERTS_PROC_LOCKS_ALL));
 	    goto already_traced;
-	}
-
-	if (on)
-	    ERTS_TRACE_FLAGS(tracee_p) |= mask;
-	else
-	    ERTS_TRACE_FLAGS(tracee_p) &= ~mask;
-
-	if ((ERTS_TRACE_FLAGS(tracee_p) & TRACEE_FLAGS) == 0)
-	    ERTS_TRACER_PROC(tracee_p) = NIL;
-	else if (tracer != NIL)
-	    ERTS_TRACER_PROC(tracee_p) = tracer;
-
-	erts_smp_proc_unlock(tracee_p,
+        }
+        erts_smp_proc_unlock(tracee_p,
 			     (tracee_p == p
 			      ? ERTS_PROC_LOCKS_ALL_MINOR
 			      : ERTS_PROC_LOCKS_ALL));
@@ -719,22 +681,7 @@ Eterm trace_3(BIF_ALIST_3)
 		    Process* tracee_p = erts_pix2proc(i);
 		    if (! tracee_p) 
 			continue;
-		    if (tracer != NIL) {
-			if (tracee_p->common.id == tracer)
-			    continue;
-			if (already_traced(NULL, tracee_p, tracer))
-			    continue;
-		    }
-		    if (on) {
-			ERTS_TRACE_FLAGS(tracee_p) |= mask;
-		    } else {
-			ERTS_TRACE_FLAGS(tracee_p) &= ~mask;
-		    }
-		    if(!(ERTS_TRACE_FLAGS(tracee_p) & TRACEE_FLAGS)) {
-			ERTS_TRACER_PROC(tracee_p) = NIL;
-		    } else if (tracer != NIL) {
-			ERTS_TRACER_PROC(tracee_p) = tracer;
-		    }
+                    start_trace(p, tracer, &tracee_p->common, on, mask);
 		    matches++;
 		}
 	    }
@@ -749,21 +696,7 @@ Eterm trace_3(BIF_ALIST_3)
 		    state = erts_atomic32_read_nob(&tracee_port->state);
 		    if (state & ERTS_PORT_SFLGS_DEAD)
 			continue;
-		    if (tracer != NIL) {
-			if (tracee_port->common.id == tracer)
-			    continue;
-			if (port_already_traced(NULL, tracee_port, tracer))
-			    continue;
-		    }
-
-		    if (on) ERTS_TRACE_FLAGS(tracee_port) |= mask;
-		    else ERTS_TRACE_FLAGS(tracee_port) &= ~mask;
-		
-		    if (!(ERTS_TRACE_FLAGS(tracee_port) & TRACEE_FLAGS)) {
-			ERTS_TRACER_PROC(tracee_port) = NIL;
-		    } else if (tracer != NIL) {
-			ERTS_TRACER_PROC(tracee_port) = tracer;
-		    }
+                    start_trace(p, tracer, &tracee_port->common, on, mask);
 		    /* matches are not counted for ports since it would violate compatibility */
 		    /* This could be a reason to modify this function or make a new one. */
 		}
@@ -772,10 +705,9 @@ Eterm trace_3(BIF_ALIST_3)
 
 	if (pid_spec == am_all || pid_spec == am_new) {
 	    Uint def_flags = mask;
-	    Eterm def_tracer = tracer;
 
 	    ok = 1;
-	    erts_change_default_tracing(on, &def_flags, &def_tracer);
+	    erts_change_default_tracing(on, &def_flags, tracer);
 
 #ifdef HAVE_ERTS_NOW_CPU
 	    if (cpu_ts && !on) {
@@ -801,6 +733,7 @@ Eterm trace_3(BIF_ALIST_3)
     }
 #endif
     erts_release_code_write_permission();
+    ERTS_TRACER_CLEAR(&tracer);
 
     BIF_RET(make_small(matches));
 
@@ -810,6 +743,8 @@ Eterm trace_3(BIF_ALIST_3)
 
  error:
 
+    ERTS_TRACER_CLEAR(&tracer);
+
 #ifdef ERTS_SMP
     if (system_blocked) {
 	erts_smp_thr_progress_unblock();
@@ -821,88 +756,6 @@ Eterm trace_3(BIF_ALIST_3)
     BIF_ERROR(p, BADARG);
 }
 
-/* Check that the process to be traced is not already traced
- * by a valid other tracer than the tracer to be.
- */
-static int port_already_traced(Process *c_p, Port *tracee_port, Eterm tracer)
-{
-    /*
-     * SMP build assumes that either system is blocked or:
-     * * main lock is held on c_p
-     * * all locks are held on port tracee_p
-     */
-    if ((ERTS_TRACE_FLAGS(tracee_port) & TRACEE_FLAGS)
-	&& ERTS_TRACER_PROC(tracee_port) != tracer) {
-	/* This tracee is already being traced, and not by the 
-	 * tracer to be */
-	if (is_internal_port(ERTS_TRACER_PROC(tracee_port))) {
-	    if (!erts_is_valid_tracer_port(ERTS_TRACER_PROC(tracee_port))) {
-		/* Current trace port now invalid 
-		 * - discard it and approve the new. */
-		goto remove_tracer;
-	    } else
-		return 1;
-	}
-	else if(is_internal_pid(ERTS_TRACER_PROC(tracee_port))) {
-	    Process *tracer_p = erts_proc_lookup(ERTS_TRACER_PROC(tracee_port));
-	    if (!tracer_p) {
-		/* Current trace process now invalid
-		 * - discard it and approve the new. */
-		goto remove_tracer;
-	    } else
-		return 1;
-	}
-	else {
-	remove_tracer:
-	    ERTS_TRACE_FLAGS(tracee_port) &= ~TRACEE_FLAGS;
-	    ERTS_TRACER_PROC(tracee_port) = NIL;
-	}
-    }
-    return 0;
-}
-
-/* Check that the process to be traced is not already traced
- * by a valid other tracer than the tracer to be.
- */
-static int already_traced(Process *c_p, Process *tracee_p, Eterm tracer)
-{
-    /*
-     * SMP build assumes that either system is blocked or:
-     * * main lock is held on c_p
-     * * all locks multiple are held on tracee_p
-     */
-    if ((ERTS_TRACE_FLAGS(tracee_p) & TRACEE_FLAGS)
-	&& ERTS_TRACER_PROC(tracee_p) != tracer) {
-	/* This tracee is already being traced, and not by the 
-	 * tracer to be */
-	if (is_internal_port(ERTS_TRACER_PROC(tracee_p))) {
-	    if (!erts_is_valid_tracer_port(ERTS_TRACER_PROC(tracee_p))) {
-		/* Current trace port now invalid 
-		 * - discard it and approve the new. */
-		goto remove_tracer;
-	    } else
-		return 1;
-	}
-	else if(is_internal_pid(ERTS_TRACER_PROC(tracee_p))) {
-	    Process *tracer_p;
-
-	    tracer_p = erts_proc_lookup(ERTS_TRACER_PROC(tracee_p));
-	    if (!tracer_p) {
-		/* Current trace process now invalid
-		 * - discard it and approve the new. */
-		goto remove_tracer;
-	    } else
-		return 1;
-	}
-	else {
-	remove_tracer:
-	    ERTS_TRACE_FLAGS(tracee_p) &= ~TRACEE_FLAGS;
-	    ERTS_TRACER_PROC(tracee_p) = NIL;
-	}
-    }
-    return 0;
-}
-
 /*
  * Return information about a process or an external function being traced.
  */
@@ -936,41 +789,30 @@ static Eterm
 trace_info_pid(Process* p, Eterm pid_spec, Eterm key)
 {
     Eterm tracer;
-    Uint trace_flags;
+    Uint trace_flags = am_false;
     Eterm* hp;
 
     if (pid_spec == am_new) {
-	erts_get_default_tracing(&trace_flags, &tracer);
+        ErtsTracer def_tracer;
+	erts_get_default_tracing(&trace_flags, &def_tracer);
+        tracer = erts_tracer_to_term(p, def_tracer);
+        ERTS_TRACER_CLEAR(&def_tracer);
     } else if (is_internal_pid(pid_spec)) {
 	Process *tracee;
 	tracee = erts_pid2proc(p, ERTS_PROC_LOCK_MAIN,
-			       pid_spec, ERTS_PROC_LOCKS_ALL);
+			       pid_spec, ERTS_PROC_LOCK_MAIN);
 
-	if (!tracee) {
+	if (!tracee)
 	    return am_undefined;
-	} else {
-	    tracer = ERTS_TRACER_PROC(tracee);
-	    trace_flags = ERTS_TRACE_FLAGS(tracee);
-	}
 
-	if (is_internal_pid(tracer)) {
-	    if (!erts_proc_lookup(tracer)) {
-	    reset_tracer:
-		ERTS_TRACE_FLAGS(tracee) &= ~TRACEE_FLAGS;
-		trace_flags = ERTS_TRACE_FLAGS(tracee);
-		tracer = ERTS_TRACER_PROC(tracee) = NIL;
-	    }
-	}
-	else if (is_internal_port(tracer)) {
-	    if (!erts_is_valid_tracer_port(tracer))
-		goto reset_tracer;
-	}
-#ifdef ERTS_SMP
-	erts_smp_proc_unlock(tracee,
-			     (tracee == p
-			      ? ERTS_PROC_LOCKS_ALL_MINOR
-			      : ERTS_PROC_LOCKS_ALL));
-#endif
+        if (!ERTS_TRACER_IS_NIL(ERTS_TRACER(tracee)))
+            erts_is_tracer_proc_enabled(tracee, ERTS_PROC_LOCK_MAIN, &tracee->common);
+
+        tracer = erts_tracer_to_term(p, ERTS_TRACER(tracee));
+        trace_flags = ERTS_TRACE_FLAGS(tracee);
+
+        if (tracee != p)
+            erts_smp_proc_unlock(tracee, ERTS_PROC_LOCK_MAIN);
     } else if (is_external_pid(pid_spec)
 	       && external_pid_dist_entry(pid_spec) == erts_this_dist_entry) {
 	    return am_undefined;
@@ -1024,8 +866,10 @@ trace_info_pid(Process* p, Eterm pid_spec, Eterm key)
 	HRelease(p,limit,hp+3);
 	return TUPLE2(hp, key, flag_list);
     } else if (key == am_tracer) {
-	hp = HAlloc(p, 3);
-	return TUPLE2(hp, key, tracer); /* Local pid or port */
+        if (tracer == am_false)
+            tracer = NIL;
+        hp = HAlloc(p, 3);
+        return TUPLE2(hp, key, tracer);
     } else {
 	goto error;
     }
@@ -1054,11 +898,11 @@ trace_info_pid(Process* p, Eterm pid_spec, Eterm key)
  */
 static int function_is_traced(Process *p,
 			      Eterm mfa[3],
-			      Binary **ms,              /* out */
-			      Binary **ms_meta,         /* out */
-			      Eterm   *tracer_pid_meta, /* out */
-			      Uint    *count,           /* out */
-			      Eterm   *call_time)       /* out */
+			      Binary    **ms,              /* out */
+			      Binary    **ms_meta,         /* out */
+			      ErtsTracer *tracer_pid_meta, /* out */
+			      Uint       *count,           /* out */
+			      Eterm      *call_time)       /* out */
 {
     Export e;
     Export* ep;
@@ -1123,7 +967,7 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key)
     Eterm traced = am_false;
     Eterm match_spec = am_false;
     Eterm retval = am_false;
-    Eterm meta = am_false;
+    ErtsTracer meta = erts_tracer_nil;
     Eterm call_time = NIL;
     int r;
 
@@ -1193,7 +1037,10 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key)
 	retval = match_spec;
 	break;
     case am_meta:
-	retval = meta;
+        retval = erts_tracer_to_term(p, meta);
+        if (retval == am_false)
+            /* backwards compatibility */
+            retval = NIL;
 	break;
     case am_meta_match_spec:
 	if (r & FUNC_TRACE_META_TRACE) {
@@ -1216,7 +1063,8 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key)
 	}
 	break;
     case am_all: {
-	Eterm match_spec_meta = am_false, c = am_false, t, ct = am_false;
+	Eterm match_spec_meta = am_false, c = am_false, t, ct = am_false,
+            m = am_false;
 	
 	if (ms) {
 	    match_spec = MatchSetGetSource(ms);
@@ -1235,6 +1083,9 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key)
 	if (r & FUNC_TRACE_TIME_TRACE) {
 	    ct = call_time;
 	}
+
+        m = erts_tracer_to_term(p, meta);
+
 	hp = HAlloc(p, (3+2)*6);
 	retval = NIL;
 	t = TUPLE2(hp, am_call_count, c); hp += 3;
@@ -1243,7 +1094,7 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key)
 	retval = CONS(hp, t, retval); hp += 2;
 	t = TUPLE2(hp, am_meta_match_spec, match_spec_meta); hp += 3;
 	retval = CONS(hp, t, retval); hp += 2;
-	t = TUPLE2(hp, am_meta, meta); hp += 3;
+	t = TUPLE2(hp, am_meta, m); hp += 3;
 	retval = CONS(hp, t, retval); hp += 2;
 	t = TUPLE2(hp, am_match_spec, match_spec); hp += 3;
 	retval = CONS(hp, t, retval); hp += 2;
@@ -1306,7 +1157,8 @@ trace_info_on_load(Process* p, Eterm key)
     case am_meta:
 	hp = HAlloc(p, 3);
 	if (erts_default_trace_pattern_flags.meta) {
-	    return TUPLE2(hp, key, erts_default_meta_tracer_pid);
+            ASSERT(!ERTS_TRACER_IS_NIL(erts_default_meta_tracer));
+	    return TUPLE2(hp, key, erts_tracer_to_term(p, erts_default_meta_tracer));
 	} else {
 	    return TUPLE2(hp, key, am_false);
 	}
@@ -1345,7 +1197,7 @@ trace_info_on_load(Process* p, Eterm key)
 	}
     case am_all:
 	{
-	    Eterm match_spec = am_false, meta_match_spec = am_false, r = NIL, t;
+	    Eterm match_spec = am_false, meta_match_spec = am_false, r = NIL, t, m;
 	    
 	    if (erts_default_trace_pattern_flags.local ||
 		(! erts_default_trace_pattern_flags.breakpoint)) {
@@ -1363,6 +1215,8 @@ trace_info_on_load(Process* p, Eterm key)
 		    MatchSetGetSource(erts_default_meta_match_spec);
 		meta_match_spec = copy_object(meta_match_spec, p);
 	    }
+            m = (erts_default_trace_pattern_flags.meta
+                 ? erts_tracer_to_term(p, erts_default_meta_tracer) : am_false);
 	    hp = HAlloc(p, (3+2)*5 + 3);
 	    t = TUPLE2(hp, am_call_count, 
 		       (erts_default_trace_pattern_flags.call_count
@@ -1370,9 +1224,7 @@ trace_info_on_load(Process* p, Eterm key)
 	    r = CONS(hp, t, r); hp += 2;
 	    t = TUPLE2(hp, am_meta_match_spec, meta_match_spec); hp += 3;
 	    r = CONS(hp, t, r); hp += 2;
-	    t = TUPLE2(hp, am_meta, 
-		       (erts_default_trace_pattern_flags.meta
-			? erts_default_meta_tracer_pid : am_false)); hp += 3;
+	    t = TUPLE2(hp, am_meta, m); hp += 3;
 	    r = CONS(hp, t, r); hp += 2;
 	    t = TUPLE2(hp, am_match_spec, match_spec); hp += 3;
 	    r = CONS(hp, t, r); hp += 2;
@@ -1397,7 +1249,7 @@ int
 erts_set_trace_pattern(Process*p, Eterm* mfa, int specified,
 		       Binary* match_prog_set, Binary *meta_match_prog_set,
 		       int on, struct trace_pattern_flags flags,
-		       Eterm meta_tracer_pid, int is_blocking)
+		       ErtsTracer meta_tracer, int is_blocking)
 {
     const ErtsCodeIndex code_ix = erts_active_code_ix();
     int matches = 0;
@@ -1487,7 +1339,7 @@ erts_set_trace_pattern(Process*p, Eterm* mfa, int specified,
 		    }
 		    if (flags.meta) {
 			erts_set_mtrace_bif(pc, meta_match_prog_set,
-					    meta_tracer_pid);
+					    meta_tracer);
 			m = 1;
 		    }
 		    if (flags.call_time) {
@@ -1527,7 +1379,7 @@ erts_set_trace_pattern(Process*p, Eterm* mfa, int specified,
 	    }
 	    if (flags.meta) {
 		erts_set_mtrace_break(&finish_bp.f, meta_match_prog_set,
-				      meta_tracer_pid);
+				      meta_tracer);
 	    }
 	    if (flags.call_count) {
 		erts_set_count_break(&finish_bp.f, on);
@@ -2336,50 +2188,79 @@ BIF_RETTYPE system_profile_2(BIF_ALIST_2)
 }
 /* End: Trace for System Profiling */
 
-BIF_RETTYPE
-trace_delivered_1(BIF_ALIST_1)
+/* Trace delivered send an aux work message to all schedulers
+   and when all schedulers have acknowledged that they have seen
+   the message the message is sent to the requesting process.
+
+   IMPORTANT: We have to make sure that the all messages sent
+   using enif_send have been delivered before we send the message
+   to the caller.
+
+   There used to be a separate implementation for when only a pid
+   is passed in, but since this is not performance critical code
+   we now use the same approach for both.
+*/
+
+typedef struct {
+    Process *proc;
+    Eterm ref;
+    Eterm ref_heap[REF_THING_SIZE];
+    Eterm target;
+    erts_smp_atomic32_t refc;
+} ErtsTraceDeliveredAll;
+
+static void
+reply_trace_delivered_all(void *vtdarp)
 {
-    DECL_AM(trace_delivered);
-#ifdef ERTS_SMP
-    ErlHeapFragment *bp;
-#else
-    ErtsProcLocks locks = 0;
-#endif
-    Eterm *hp;
-    Eterm msg, ref, msg_ref;
-    Process *p;
-    if (BIF_ARG_1 == am_all) {
-	p = NULL;
-    } else if (! (p = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN,
-				    BIF_ARG_1, ERTS_PROC_LOCKS_ALL))) {
-	if (is_not_internal_pid(BIF_ARG_1)) {
-	    BIF_ERROR(BIF_P, BADARG);
-	}
-    }
-    
-    ref = erts_make_ref(BIF_P);
+    ErtsTraceDeliveredAll *tdarp = (ErtsTraceDeliveredAll *) vtdarp;
+    ErtsProcLocks rp_locks = 0;
 
-#ifdef ERTS_SMP
-    bp = new_message_buffer(REF_THING_SIZE + 4);
-    hp = &bp->mem[0];
-    msg_ref = STORE_NC(&hp, &bp->off_heap, ref);
-#else
-    hp = HAlloc(BIF_P, 4);
-    msg_ref = ref;
-#endif
+    if (erts_smp_atomic32_dec_read_nob(&tdarp->refc) == 0) {
+        Process *rp = tdarp->proc;
+        Eterm *hp = NULL;
+        ErlOffHeap *ohp = NULL;
+        ErtsMessage *mp = NULL;
+        Eterm ref_copy, msg;
 
-    msg = TUPLE3(hp, AM_trace_delivered, BIF_ARG_1, msg_ref);
+        mp = erts_alloc_message_heap(
+            rp, &rp_locks, 4 + NC_HEAP_SIZE(tdarp->ref), &hp, &ohp);
 
-#ifdef ERTS_SMP
-    erts_send_sys_msg_proc(BIF_P->common.id, BIF_P->common.id, msg, bp);
-    if (p)
-	erts_smp_proc_unlock(p,
-			     (BIF_P == p
-			      ? ERTS_PROC_LOCKS_ALL_MINOR
-			      : ERTS_PROC_LOCKS_ALL));
-#else
-    erts_send_message(BIF_P, BIF_P, &locks, msg, ERTS_SND_FLG_NO_SEQ_TRACE);
-#endif
+        ref_copy = STORE_NC(&hp, ohp, tdarp->ref);
+        msg = TUPLE3(hp, am_trace_delivered, tdarp->target, ref_copy);
+
+        erts_queue_message(rp, &rp_locks, mp, msg);
+
+        if (rp_locks)
+            erts_smp_proc_unlock(rp, rp_locks);
 
-    BIF_RET(ref);
+	erts_free(ERTS_ALC_T_MISC_AUX_WORK, vtdarp);
+        erts_proc_dec_refc(rp);
+    }
+}
+
+BIF_RETTYPE
+trace_delivered_1(BIF_ALIST_1)
+{
+
+    if (BIF_ARG_1 == am_all || is_internal_pid(BIF_ARG_1)) {
+        Eterm *hp, ref;
+        ErtsTraceDeliveredAll *tdarp =
+            erts_alloc(ERTS_ALC_T_MISC_AUX_WORK, sizeof(ErtsTraceDeliveredAll));
+
+        tdarp->proc = BIF_P;
+        ref = erts_make_ref(BIF_P);
+        hp = &tdarp->ref_heap[0];
+        tdarp->ref = STORE_NC(&hp, NULL, ref);
+        tdarp->target = BIF_ARG_1;
+        erts_smp_atomic32_init_nob(&tdarp->refc,
+                                   (erts_aint32_t) erts_no_schedulers);
+        erts_proc_add_refc(BIF_P, 1);
+        erts_schedule_multi_misc_aux_work(0,
+                                          erts_no_schedulers,
+                                          reply_trace_delivered_all,
+                                          (void *) tdarp);
+        BIF_RET(ref);
+    } else {
+        BIF_ERROR(BIF_P, BADARG);
+    }
 }