aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator/beam/erl_trace.c
diff options
context:
space:
mode:
Diffstat (limited to 'erts/emulator/beam/erl_trace.c')
-rw-r--r--erts/emulator/beam/erl_trace.c3260
1 files changed, 3260 insertions, 0 deletions
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
new file mode 100644
index 0000000000..2afb16fc52
--- /dev/null
+++ b/erts/emulator/beam/erl_trace.c
@@ -0,0 +1,3260 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 1999-2009. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Support functions for tracing.
+ */
+
+#ifdef HAVE_CONFIG_H
+# include "config.h"
+#endif
+
+#include "sys.h"
+#include "erl_vm.h"
+#include "global.h"
+#include "erl_process.h"
+#include "big.h"
+#include "bif.h"
+#include "dist.h"
+#include "beam_bp.h"
+#include "error.h"
+#include "erl_binary.h"
+#include "erl_bits.h"
+
+#if 0
+#define DEBUG_PRINTOUTS
+#else
+#undef DEBUG_PRINTOUTS
+#endif
+
+extern Eterm beam_return_to_trace[1]; /* OpCode(i_return_to_trace) */
+extern Eterm beam_return_trace[1]; /* OpCode(i_return_trace) */
+
+/* Pseudo export entries. Never filled in with data, only used to
+ yield unique pointers of the correct type. */
+Export exp_send, exp_receive, exp_timeout;
+
+static Eterm system_seq_tracer;
+static Uint default_trace_flags;
+static Eterm default_tracer;
+
+static Eterm system_monitor;
+static Eterm system_profile;
+
+#ifdef HAVE_ERTS_NOW_CPU
+int erts_cpu_timestamp;
+#endif
+
+static erts_smp_mtx_t smq_mtx;
+static erts_smp_mtx_t sys_trace_mtx;
+
+enum ErtsSysMsgType {
+ SYS_MSG_TYPE_UNDEFINED,
+ SYS_MSG_TYPE_TRACE,
+ SYS_MSG_TYPE_SEQTRACE,
+ SYS_MSG_TYPE_SYSMON,
+ SYS_MSG_TYPE_ERRLGR,
+ SYS_MSG_TYPE_PROC_MSG,
+ SYS_MSG_TYPE_SYSPROF
+};
+
+#ifdef ERTS_SMP
+static void enqueue_sys_msg_unlocked(enum ErtsSysMsgType type,
+ Eterm from,
+ Eterm to,
+ Eterm msg,
+ ErlHeapFragment *bp);
+static void enqueue_sys_msg(enum ErtsSysMsgType type,
+ Eterm from,
+ Eterm to,
+ Eterm msg,
+ ErlHeapFragment *bp);
+static void init_sys_msg_dispatcher(void);
+#endif
+
+void erts_init_trace(void) {
+ erts_smp_mtx_init(&sys_trace_mtx, "sys_tracers");
+#ifdef HAVE_ERTS_NOW_CPU
+ erts_cpu_timestamp = 0;
+#endif
+ erts_bif_trace_init();
+ erts_system_monitor_clear(NULL);
+ erts_system_profile_clear(NULL);
+ default_trace_flags = F_INITIAL_TRACE_FLAGS;
+ default_tracer = NIL;
+ system_seq_tracer = am_false;
+#ifdef ERTS_SMP
+ init_sys_msg_dispatcher();
+#endif
+}
+
+static Eterm system_seq_tracer;
+
+#ifdef ERTS_SMP
+#define ERTS_ALLOC_SYSMSG_HEAP(SZ, BPP, OHPP, UNUSED) \
+ (*(BPP) = new_message_buffer((SZ)), \
+ *(OHPP) = &(*(BPP))->off_heap, \
+ (*(BPP))->mem)
+#else
+#define ERTS_ALLOC_SYSMSG_HEAP(SZ, BPP, OHPP, RPP) \
+ erts_alloc_message_heap((SZ), (BPP), (OHPP), (RPP), 0)
+#endif
+
+#ifdef ERTS_SMP
+#define ERTS_ENQ_TRACE_MSG(FPID, TPID, MSG, BP) \
+do { \
+ ERTS_LC_ASSERT(erts_smp_lc_mtx_is_locked(&smq_mtx)); \
+ enqueue_sys_msg_unlocked(SYS_MSG_TYPE_TRACE, (FPID), (TPID), (MSG), (BP)); \
+} while(0)
+#else
+#define ERTS_ENQ_TRACE_MSG(FPID, TPROC, MSG, BP) \
+ erts_queue_message((TPROC), NULL, (BP), (MSG), NIL)
+#endif
+
+/*
+ * NOTE that the ERTS_GET_TRACER_REF() returns from the function (!!!)
+ * using it, and resets the parameters used if the tracer is invalid, i.e.,
+ * use it with extreme care!
+ */
+#ifdef ERTS_SMP
+#define ERTS_NULL_TRACER_REF NIL
+#define ERTS_TRACER_REF_TYPE Eterm
+ /* In the smp case, we never find the tracer invalid here (the sys
+ message dispatcher thread takes care of that). */
+#define ERTS_GET_TRACER_REF(RES, TPID, TRACEE_FLGS) \
+do { (RES) = (TPID); } while(0)
+#else
+#define ERTS_NULL_TRACER_REF NULL
+#define ERTS_TRACER_REF_TYPE Process *
+#define ERTS_GET_TRACER_REF(RES, TPID, TRACEE_FLGS) \
+do { \
+ (RES) = process_tab[internal_pid_index((TPID))]; \
+ if (INVALID_PID((RES), (TPID)) || !((RES)->trace_flags & F_TRACER)) { \
+ (TPID) = NIL; \
+ (TRACEE_FLGS) &= ~TRACEE_FLAGS; \
+ return; \
+ } \
+} while (0)
+#endif
+
+static Uint active_sched;
+
+void
+erts_system_profile_setup_active_schedulers(void)
+{
+ ERTS_SMP_LC_ASSERT(erts_is_system_blocked(0));
+ active_sched = erts_active_schedulers();
+}
+
+void
+erts_trace_check_exiting(Eterm exiting)
+{
+ erts_smp_mtx_lock(&sys_trace_mtx);
+ if (exiting == default_tracer) {
+ default_tracer = NIL;
+ default_trace_flags &= TRACEE_FLAGS;
+#ifdef DEBUG
+ default_trace_flags |= F_INITIAL_TRACE_FLAGS;
+#endif
+ }
+ if (exiting == system_seq_tracer) {
+#ifdef DEBUG_PRINTOUTS
+ erts_fprintf(stderr, "seq tracer %T exited\n", exiting);
+#endif
+ system_seq_tracer = am_false;
+ }
+ if (exiting == system_monitor) {
+#ifdef ERTS_SMP
+ system_monitor = NIL;
+ /* Let the trace message dispatcher clear flags, etc */
+#else
+ erts_system_monitor_clear(NULL);
+#endif
+ }
+ if (exiting == system_profile) {
+#ifdef ERTS_SMP
+ system_profile = NIL;
+ /* Let the trace message dispatcher clear flags, etc */
+#else
+ erts_system_profile_clear(NULL);
+#endif
+ }
+ erts_smp_mtx_unlock(&sys_trace_mtx);
+}
+
+Eterm
+erts_set_system_seq_tracer(Process *c_p, ErtsProcLocks c_p_locks, Eterm new)
+{
+ Eterm old = THE_NON_VALUE;
+
+ if (new != am_false) {
+ if (!erts_pid2proc(c_p, c_p_locks, new, 0)
+ && !erts_is_valid_tracer_port(new)) {
+ return old;
+ }
+ }
+
+ erts_smp_mtx_lock(&sys_trace_mtx);
+ old = system_seq_tracer;
+ system_seq_tracer = new;
+
+#ifdef DEBUG_PRINTOUTS
+ erts_fprintf(stderr, "set seq tracer new=%T old=%T\n", new, old);
+#endif
+ erts_smp_mtx_unlock(&sys_trace_mtx);
+ return old;
+}
+
+Eterm
+erts_get_system_seq_tracer(void)
+{
+ Eterm st;
+ erts_smp_mtx_lock(&sys_trace_mtx);
+ st = system_seq_tracer;
+#ifdef DEBUG_PRINTOUTS
+ erts_fprintf(stderr, "get seq tracer %T\n", st);
+#endif
+ erts_smp_mtx_unlock(&sys_trace_mtx);
+ return st;
+}
+
+static ERTS_INLINE void
+get_default_tracing(Uint *flagsp, Eterm *tracerp)
+{
+ if (!(default_trace_flags & TRACEE_FLAGS))
+ default_tracer = NIL;
+
+ if (is_nil(default_tracer)) {
+ default_trace_flags &= ~TRACEE_FLAGS;
+ } else if (is_internal_pid(default_tracer)) {
+ if (!erts_pid2proc(NULL, 0, default_tracer, 0)) {
+ reset_tracer:
+ default_trace_flags &= ~TRACEE_FLAGS;
+ default_tracer = NIL;
+ }
+ } else {
+ ASSERT(is_internal_port(default_tracer));
+ if (!erts_is_valid_tracer_port(default_tracer))
+ goto reset_tracer;
+ }
+
+ if (flagsp)
+ *flagsp = default_trace_flags;
+ if (tracerp)
+ *tracerp = default_tracer;
+}
+
+void
+erts_change_default_tracing(int setflags, Uint *flagsp, Eterm *tracerp)
+{
+ erts_smp_mtx_lock(&sys_trace_mtx);
+ if (flagsp) {
+ if (setflags)
+ default_trace_flags |= *flagsp;
+ else
+ default_trace_flags &= ~(*flagsp);
+ }
+ if (tracerp)
+ default_tracer = *tracerp;
+ get_default_tracing(flagsp, tracerp);
+ erts_smp_mtx_unlock(&sys_trace_mtx);
+}
+
+void
+erts_get_default_tracing(Uint *flagsp, Eterm *tracerp)
+{
+ erts_smp_mtx_lock(&sys_trace_mtx);
+ get_default_tracing(flagsp, tracerp);
+ erts_smp_mtx_unlock(&sys_trace_mtx);
+}
+
+void
+erts_set_system_monitor(Eterm monitor)
+{
+ erts_smp_mtx_lock(&sys_trace_mtx);
+ system_monitor = monitor;
+ erts_smp_mtx_unlock(&sys_trace_mtx);
+}
+
+Eterm
+erts_get_system_monitor(void)
+{
+ Eterm monitor;
+ erts_smp_mtx_lock(&sys_trace_mtx);
+ monitor = system_monitor;
+ erts_smp_mtx_unlock(&sys_trace_mtx);
+ return monitor;
+}
+
+/* Performance monitoring */
+void erts_set_system_profile(Eterm profile) {
+ erts_smp_mtx_lock(&sys_trace_mtx);
+ system_profile = profile;
+ erts_smp_mtx_unlock(&sys_trace_mtx);
+}
+
+Eterm
+erts_get_system_profile(void) {
+ Eterm profile;
+ erts_smp_mtx_lock(&sys_trace_mtx);
+ profile = system_profile;
+ erts_smp_mtx_unlock(&sys_trace_mtx);
+ return profile;
+}
+
+
+#ifdef HAVE_ERTS_NOW_CPU
+# define GET_NOW(m, s, u) \
+do { \
+ if (erts_cpu_timestamp) \
+ erts_get_now_cpu(m, s, u); \
+ else \
+ get_now(m, s, u); \
+} while (0)
+#else
+# define GET_NOW(m, s, u) do {get_now(m, s, u);} while (0)
+#endif
+
+
+
+static Eterm* patch_ts(Eterm tuple4, Eterm* hp);
+
+#ifdef ERTS_SMP
+static void
+do_send_to_port(Eterm to,
+ Port* unused_port,
+ Eterm from,
+ enum ErtsSysMsgType type,
+ Eterm message)
+{
+ Uint sz = size_object(message);
+ ErlHeapFragment *bp = new_message_buffer(sz);
+ Uint *hp = bp->mem;
+ Eterm msg = copy_struct(message, sz, &hp, &bp->off_heap);
+
+ enqueue_sys_msg_unlocked(type, from, to, msg, bp);
+}
+
+#define WRITE_SYS_MSG_TO_PORT write_sys_msg_to_port
+#else
+#define WRITE_SYS_MSG_TO_PORT do_send_to_port
+#endif
+
+static void
+WRITE_SYS_MSG_TO_PORT(Eterm unused_to,
+ Port* trace_port,
+ Eterm unused_from,
+ enum ErtsSysMsgType unused_type,
+ Eterm message) {
+ byte *buffer;
+ byte *ptr;
+ unsigned size;
+
+ size = erts_encode_ext_size(message);
+ buffer = (byte *) erts_alloc(ERTS_ALC_T_TMP, size);
+
+ ptr = buffer;
+
+ erts_encode_ext(message, &ptr);
+ if (!(ptr <= buffer+size)) {
+ erl_exit(1, "Internal error in do_send_to_port: %d\n", ptr-buffer);
+ }
+
+#ifndef ERTS_SMP
+ if (!INVALID_TRACER_PORT(trace_port, trace_port->id)) {
+#endif
+ erts_raw_port_command(trace_port, buffer, ptr-buffer);
+#ifndef ERTS_SMP
+ erts_port_release(trace_port);
+ }
+#endif
+
+ erts_free(ERTS_ALC_T_TMP, (void *) buffer);
+}
+
+
+#ifndef ERTS_SMP
+/* Send {trace_ts, Pid, out, 0, Timestamp}
+ * followed by {trace_ts, Pid, in, 0, NewTimestamp}
+ *
+ * 'NewTimestamp' is fetched from GET_NOW() through patch_ts().
+ */
+static void
+do_send_schedfix_to_port(Port *trace_port, Eterm pid, Eterm timestamp) {
+ Eterm local_heap[4+5+5];
+ Eterm message;
+ Eterm *hp;
+ Eterm mfarity;
+
+ ASSERT(is_pid(pid));
+ ASSERT(is_tuple(timestamp));
+ ASSERT(*tuple_val(timestamp) == make_arityval(3));
+
+ hp = local_heap;
+ mfarity = make_small(0);
+ message = TUPLE5(hp, am_trace_ts, pid, am_out, mfarity, timestamp);
+ /* Note, hp is deliberately NOT incremented since it will be reused */
+
+ do_send_to_port(trace_port->id,
+ trace_port,
+ pid,
+ SYS_MSG_TYPE_UNDEFINED,
+ message);
+
+ message = TUPLE4(hp, am_trace_ts, pid, am_in, mfarity);
+ hp += 5;
+ hp = patch_ts(message, hp);
+
+ do_send_to_port(trace_port->id,
+ trace_port,
+ pid,
+ SYS_MSG_TYPE_UNDEFINED,
+ message);
+}
+#endif
+
+/* If (c_p != NULL), a fake schedule out/in message pair will be sent,
+ * if the driver so requests.
+ * It is assumed that 'message' is not an 'out' message.
+ *
+ * 'c_p' is the currently executing process, "tracee" is the traced process
+ * which 'message' concerns => if (*tracee_flags & F_TIMESTAMP),
+ * 'message' must contain a timestamp.
+ */
+static void
+send_to_port(Process *c_p, Eterm message,
+ Eterm *tracer_pid, Uint *tracee_flags) {
+ Port* trace_port;
+#ifndef ERTS_SMP
+ Eterm ts, local_heap[4], *hp;
+#endif
+
+ ASSERT(is_internal_port(*tracer_pid));
+#ifdef ERTS_SMP
+ if (is_not_internal_port(*tracer_pid))
+ return;
+
+ trace_port = NULL;
+#else
+ if (is_not_internal_port(*tracer_pid))
+ goto invalid_tracer_port;
+
+ trace_port = &erts_port[internal_port_index(*tracer_pid)];
+
+ if (INVALID_TRACER_PORT(trace_port, *tracer_pid)) {
+ invalid_tracer_port:
+ *tracee_flags &= ~TRACEE_FLAGS;
+ *tracer_pid = NIL;
+ return;
+ }
+
+ /*
+ * Make a fake schedule only if the current process is traced
+ * with 'running' and 'timestamp'.
+ */
+
+ if ( c_p == NULL ||
+ (! IS_TRACED_FL(c_p, F_TRACE_SCHED | F_TIMESTAMP))) {
+#endif
+ do_send_to_port(*tracer_pid,
+ trace_port,
+ c_p ? c_p->id : NIL,
+ SYS_MSG_TYPE_TRACE,
+ message);
+#ifndef ERTS_SMP
+ return;
+ }
+
+ /*
+ * Note that the process being traced for some type of trace messages
+ * (e.g. getting_linked) need not be the current process. That other
+ * process might not have timestamps enabled.
+ */
+ if (*tracee_flags & F_TIMESTAMP) {
+ ASSERT(is_tuple(message));
+ hp = tuple_val(message);
+ ts = hp[arityval(hp[0])];
+ } else {
+ /* A fake schedule might be needed,
+ * but this message does not contain a timestamp.
+ * Create a dummy trace message with timestamp to be
+ * passed to do_send_schedfix_to_port().
+ */
+ Uint ms,s,us;
+ GET_NOW(&ms, &s, &us);
+ hp = local_heap;
+ ts = TUPLE3(hp, make_small(ms), make_small(s), make_small(us));
+ hp += 4;
+ }
+
+ trace_port->control_flags &= ~PORT_CONTROL_FLAG_HEAVY;
+ do_send_to_port(*tracer_pid,
+ trace_port,
+ c_p ? c_p->id : NIL,
+ SYS_MSG_TYPE_TRACE,
+ message);
+
+ if (trace_port->control_flags & PORT_CONTROL_FLAG_HEAVY) {
+ /* The driver has just informed us that the last write took a
+ * non-neglectible amount of time.
+ *
+ * We need to fake some trace messages to compensate for the time the
+ * current process had to sacrifice for the writing of the previous
+ * trace message. We pretend that the process got scheduled out
+ * just after writning the real trace message, and now gets scheduled
+ * in again.
+ */
+ do_send_schedfix_to_port(trace_port, c_p->id, ts);
+ }
+#endif
+}
+
+#ifndef ERTS_SMP
+/* Profile send
+ * Checks if profiler is port or process
+ * Eterm msg is local, need copying.
+ */
+
+static void
+profile_send(Eterm message) {
+ Uint sz = 0;
+ ErlHeapFragment *bp = NULL;
+ Uint *hp = NULL;
+ Eterm msg = NIL;
+ Process *profile_p = NULL;
+ ErlOffHeap *off_heap = NULL;
+
+ Eterm profiler = erts_get_system_profile();
+
+ if (is_internal_port(profiler)) {
+ Port *profiler_port = NULL;
+
+ /* not smp */
+
+
+ profiler_port = &erts_port[internal_port_index(profiler)];
+
+ do_send_to_port(profiler,
+ profiler_port,
+ NIL, /* or current process->id */
+ SYS_MSG_TYPE_SYSPROF,
+ message);
+
+ } else {
+ ASSERT(is_internal_pid(profiler)
+ && internal_pid_index(profiler) < erts_max_processes);
+
+ profile_p = process_tab[internal_pid_index(profiler)];
+
+ if (INVALID_PID(profile_p, profiler)) return;
+
+ sz = size_object(message);
+ hp = erts_alloc_message_heap(sz, &bp, &off_heap, profile_p, 0);
+ msg = copy_struct(message, sz, &hp, &bp->off_heap);
+
+ erts_queue_message(profile_p, NULL, bp, msg, NIL);
+ }
+}
+
+#endif
+
+
+/* A fake schedule out/in message pair will be sent,
+ * if the driver so requests.
+ * If (timestamp == NIL), one is fetched from GET_NOW().
+ *
+ * 'c_p' is the currently executing process, may be NULL.
+ */
+static void
+seq_trace_send_to_port(Process *c_p,
+ Eterm seq_tracer,
+ Eterm message,
+ Eterm timestamp)
+{
+ Port* trace_port;
+#ifndef ERTS_SMP
+ Eterm ts, local_heap[4], *hp;
+#endif
+
+ ASSERT(is_internal_port(seq_tracer));
+#ifdef ERTS_SMP
+ if (is_not_internal_port(seq_tracer))
+ return;
+
+ trace_port = NULL;
+#else
+ if (is_not_internal_port(seq_tracer))
+ goto invalid_tracer_port;
+
+ trace_port = &erts_port[internal_port_index(seq_tracer)];
+
+ if (INVALID_TRACER_PORT(trace_port, seq_tracer)) {
+ invalid_tracer_port:
+ system_seq_tracer = am_false;
+ return;
+ }
+
+ if (c_p == NULL
+ || (! IS_TRACED_FL(c_p, F_TRACE_SCHED | F_TIMESTAMP))) {
+#endif
+ do_send_to_port(seq_tracer,
+ trace_port,
+ c_p ? c_p->id : NIL,
+ SYS_MSG_TYPE_SEQTRACE,
+ message);
+
+#ifndef ERTS_SMP
+ return;
+ }
+ /* Make a fake schedule only if the current process is traced
+ * with 'running' and 'timestamp'.
+ */
+
+ if (timestamp != NIL) {
+ ts = timestamp;
+ } else {
+ /* A fake schedule might be needed,
+ * but this message does not contain a timestamp.
+ * Create a dummy trace message with timestamp to be
+ * passed to do_send_schedfix_to_port().
+ */
+ Uint ms,s,us;
+ GET_NOW(&ms, &s, &us);
+ hp = local_heap;
+ ts = TUPLE3(hp, make_small(ms), make_small(s), make_small(us));
+ hp += 4;
+ }
+
+ trace_port->control_flags &= ~PORT_CONTROL_FLAG_HEAVY;
+ do_send_to_port(seq_tracer,
+ trace_port,
+ c_p ? c_p->id : NIL,
+ SYS_MSG_TYPE_SEQTRACE,
+ message);
+
+ if (trace_port->control_flags & PORT_CONTROL_FLAG_HEAVY) {
+ /* The driver has just informed us that the last write took a
+ * non-neglectible amount of time.
+ *
+ * We need to fake some trace messages to compensate for the time the
+ * current process had to sacrifice for the writing of the previous
+ * trace message. We pretend that the process got scheduled out
+ * just after writing the real trace message, and now gets scheduled
+ * in again.
+ */
+ do_send_schedfix_to_port(trace_port, c_p->id, ts);
+ }
+#endif
+}
+
+#define TS_HEAP_WORDS 5
+#define TS_SIZE(p) (((p)->trace_flags & F_TIMESTAMP) ? TS_HEAP_WORDS : 0)
+
+/*
+ * Patch a timestamp into a tuple. The tuple must be the last thing
+ * built on the heap.
+ *
+ * Returns the new hp pointer.
+*/
+static Eterm*
+patch_ts(Eterm tuple, Eterm* hp)
+{
+ Uint ms, s, us;
+ Eterm* ptr = tuple_val(tuple);
+ int arity = arityval(*ptr);
+
+ ASSERT((ptr+arity+1) == hp);
+ ptr[0] = make_arityval(arity+1);
+ ptr[1] = am_trace_ts;
+ GET_NOW(&ms, &s, &us);
+ *hp = TUPLE3(hp+1, make_small(ms), make_small(s), make_small(us));
+ return hp+5;
+}
+
+static ERTS_INLINE void
+send_to_tracer(Process *tracee,
+ ERTS_TRACER_REF_TYPE tracer_ref,
+ Eterm msg,
+ Eterm **hpp,
+ ErlHeapFragment *bp,
+ int no_fake_sched)
+{
+ ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(tracee));
+
+ erts_smp_mtx_lock(&smq_mtx);
+
+ if (tracee->trace_flags & F_TIMESTAMP)
+ *hpp = patch_ts(msg, *hpp);
+
+ if (is_internal_pid(tracee->tracer_proc))
+ ERTS_ENQ_TRACE_MSG(tracee->id, tracer_ref, msg, bp);
+ else {
+ ASSERT(is_internal_port(tracee->tracer_proc));
+ send_to_port(no_fake_sched ? NULL : tracee,
+ msg,
+ &tracee->tracer_proc,
+ &tracee->trace_flags);
+ }
+
+ erts_smp_mtx_unlock(&smq_mtx);
+
+}
+
+static void
+trace_sched_aux(Process *p, Eterm what, int never_fake_sched)
+{
+ Eterm local_heap[5+4+1+TS_HEAP_WORDS];
+ Eterm tmp, mess, *hp;
+ ErlHeapFragment *bp = NULL;
+ ErlOffHeap *off_heap;
+ ERTS_TRACER_REF_TYPE tracer_ref = ERTS_NULL_TRACER_REF;
+ int sched_no, curr_func, to_port, no_fake_sched;
+
+ if (is_nil(p->tracer_proc))
+ return;
+
+ no_fake_sched = never_fake_sched;
+
+ switch (what) {
+ case am_out:
+ case am_out_exiting:
+ case am_out_exited:
+ no_fake_sched = 1;
+ break;
+ case am_in:
+ case am_in_exiting:
+ break;
+ default:
+ ASSERT(0);
+ break;
+ }
+
+ sched_no = IS_TRACED_FL(p, F_TRACE_SCHED_NO);
+ to_port = is_internal_port(p->tracer_proc);
+
+ if (!to_port) {
+ ASSERT(is_internal_pid(p->tracer_proc)
+ && internal_pid_index(p->tracer_proc) < erts_max_processes);
+
+ ERTS_GET_TRACER_REF(tracer_ref, p->tracer_proc, p->trace_flags);
+ }
+
+ if (ERTS_PROC_IS_EXITING(p)
+#ifndef ERTS_SMP
+ || p->status == P_FREE
+#endif
+ ) {
+ curr_func = 0;
+ }
+ else {
+ if (!p->current)
+ p->current = find_function_from_pc(p->i);
+ curr_func = p->current != NULL;
+ }
+
+ if (to_port)
+ hp = &local_heap[0];
+ else {
+ Uint size = 5;
+ if (curr_func)
+ size += 4;
+ if (sched_no)
+ size += 1;
+ size += TS_SIZE(p);
+ hp = ERTS_ALLOC_SYSMSG_HEAP(size, &bp, &off_heap, tracer_ref);
+ }
+
+ if (!curr_func) {
+ tmp = make_small(0);
+ } else {
+ tmp = TUPLE3(hp,p->current[0],p->current[1],make_small(p->current[2]));
+ hp += 4;
+ }
+
+ if (!sched_no) {
+ mess = TUPLE4(hp, am_trace, p->id, what, tmp);
+ hp += 5;
+ }
+ else {
+#ifdef ERTS_SMP
+ Eterm sched_id = make_small(p->scheduler_data->no);
+#else
+ Eterm sched_id = make_small(1);
+#endif
+ mess = TUPLE5(hp, am_trace, p->id, what, sched_id, tmp);
+ hp += 6;
+ }
+
+ send_to_tracer(p, tracer_ref, mess, &hp, bp, no_fake_sched);
+}
+
+/* Send {trace_ts, Pid, What, {Mod, Func, Arity}, Timestamp}
+ * or {trace, Pid, What, {Mod, Func, Arity}}
+ *
+ * where 'What' is supposed to be 'in', 'out', 'in_exiting',
+ * 'out_exiting', or 'out_exited'.
+ */
+void
+trace_sched(Process *p, Eterm what)
+{
+ trace_sched_aux(p, what, 0);
+}
+
+/* Send {trace_ts, Pid, Send, Msg, DestPid, Timestamp}
+ * or {trace, Pid, Send, Msg, DestPid}
+ *
+ * where 'Send' is 'send' or 'send_to_non_existing_process'.
+ */
+void
+trace_send(Process *p, Eterm to, Eterm msg)
+{
+ Eterm operation;
+ unsigned sz_msg;
+ unsigned sz_to;
+ Eterm* hp;
+ Eterm mess;
+
+ if (!ARE_TRACE_FLAGS_ON(p, F_TRACE_SEND)) {
+ return;
+ }
+
+ operation = am_send;
+ if (is_internal_pid(to)) {
+ if (!erts_pid2proc(p, ERTS_PROC_LOCK_MAIN, to, 0))
+ goto send_to_non_existing_process;
+ }
+ else if(is_external_pid(to)
+ && external_pid_dist_entry(to) == erts_this_dist_entry) {
+ char *s;
+ send_to_non_existing_process:
+ s = "send_to_non_existing_process";
+ operation = am_atom_put(s, sys_strlen(s));
+ }
+
+ if (is_internal_port(p->tracer_proc)) {
+ Eterm local_heap[11];
+ hp = local_heap;
+ mess = TUPLE5(hp, am_trace, p->id, operation, msg, to);
+ hp += 6;
+ erts_smp_mtx_lock(&smq_mtx);
+ if (p->trace_flags & F_TIMESTAMP) {
+ hp = patch_ts(mess, hp);
+ }
+ send_to_port(p, mess, &p->tracer_proc, &p->trace_flags);
+ erts_smp_mtx_unlock(&smq_mtx);
+ } else {
+ Uint need;
+ ErlHeapFragment *bp;
+ ErlOffHeap *off_heap;
+ ERTS_TRACER_REF_TYPE tracer_ref;
+
+ ASSERT(is_internal_pid(p->tracer_proc)
+ && internal_pid_index(p->tracer_proc) < erts_max_processes);
+
+ ERTS_GET_TRACER_REF(tracer_ref, p->tracer_proc, p->trace_flags);
+
+ sz_msg = size_object(msg);
+ sz_to = size_object(to);
+ need = sz_msg + sz_to + 6 + TS_SIZE(p);
+
+ hp = ERTS_ALLOC_SYSMSG_HEAP(need, &bp, &off_heap, tracer_ref);
+
+ to = copy_struct(to,
+ sz_to,
+ &hp,
+ off_heap);
+ msg = copy_struct(msg,
+ sz_msg,
+ &hp,
+ off_heap);
+ mess = TUPLE5(hp, am_trace, p->id/* Local pid */, operation, msg, to);
+ hp += 6;
+
+ erts_smp_mtx_lock(&smq_mtx);
+
+ if (p->trace_flags & F_TIMESTAMP) {
+ patch_ts(mess, hp);
+ }
+
+ ERTS_ENQ_TRACE_MSG(p->id, tracer_ref, mess, bp);
+ erts_smp_mtx_unlock(&smq_mtx);
+ }
+}
+
+/* Send {trace_ts, Pid, receive, Msg, Timestamp}
+ * or {trace, Pid, receive, Msg}
+ */
+void
+trace_receive(Process *rp, Eterm msg)
+{
+ Eterm mess;
+ size_t sz_msg;
+ Eterm* hp;
+
+ if (is_internal_port(rp->tracer_proc)) {
+ Eterm local_heap[10];
+ hp = local_heap;
+ mess = TUPLE4(hp, am_trace, rp->id, am_receive, msg);
+ hp += 5;
+ erts_smp_mtx_lock(&smq_mtx);
+ if (rp->trace_flags & F_TIMESTAMP) {
+ hp = patch_ts(mess, hp);
+ }
+ send_to_port(rp, mess, &rp->tracer_proc, &rp->trace_flags);
+ erts_smp_mtx_unlock(&smq_mtx);
+ } else {
+ Uint hsz;
+ ErlHeapFragment *bp;
+ ErlOffHeap *off_heap;
+ ERTS_TRACER_REF_TYPE tracer_ref;
+
+ ASSERT(is_internal_pid(rp->tracer_proc)
+ && internal_pid_index(rp->tracer_proc) < erts_max_processes);
+
+ ERTS_GET_TRACER_REF(tracer_ref, rp->tracer_proc, rp->trace_flags);
+
+ sz_msg = size_object(msg);
+
+ hsz = sz_msg + 5 + TS_SIZE(rp);
+
+ hp = ERTS_ALLOC_SYSMSG_HEAP(hsz, &bp, &off_heap, tracer_ref);
+
+ msg = copy_struct(msg, sz_msg, &hp, off_heap);
+ mess = TUPLE4(hp, am_trace, rp->id/* Local pid */, am_receive, msg);
+ hp += 5;
+
+ erts_smp_mtx_lock(&smq_mtx);
+
+ if (rp->trace_flags & F_TIMESTAMP) {
+ patch_ts(mess, hp);
+ }
+
+ ERTS_ENQ_TRACE_MSG(rp->id, tracer_ref, mess, bp);
+ erts_smp_mtx_unlock(&smq_mtx);
+ }
+}
+
+int
+seq_trace_update_send(Process *p)
+{
+ Eterm seq_tracer = erts_get_system_seq_tracer();
+ ASSERT((is_tuple(SEQ_TRACE_TOKEN(p)) || is_nil(SEQ_TRACE_TOKEN(p))));
+ if ( (p->id == seq_tracer) || (SEQ_TRACE_TOKEN(p) == NIL))
+ return 0;
+
+ SEQ_TRACE_TOKEN_SENDER(p) = p->id; /* Internal pid */
+ SEQ_TRACE_TOKEN_SERIAL(p) =
+ make_small(++(p -> seq_trace_clock));
+ SEQ_TRACE_TOKEN_LASTCNT(p) =
+ make_small(p -> seq_trace_lastcnt);
+ return 1;
+}
+
+
+/* Send a sequential trace message to the sequential tracer.
+ * p is the caller (which contains the trace token),
+ * msg is the original message, type is trace type (SEQ_TRACE_SEND etc),
+ * and receiver is the receiver of the message.
+ *
+ * The message to be received by the sequential tracer is:
+ *
+ * TraceMsg =
+ * {seq_trace, Label, {Type, {Lastcnt, Serial}, Sender, Receiver, Msg} [,Timestamp] }
+ *
+ */
+void
+seq_trace_output_generic(Eterm token, Eterm msg, Uint type,
+ Eterm receiver, Process *process, Eterm exitfrom)
+{
+ Eterm mess;
+ ErlHeapFragment* bp;
+ Eterm* hp;
+ Eterm label;
+ Eterm lastcnt_serial;
+ Eterm type_atom;
+ int sz_exit;
+ Eterm seq_tracer;
+
+ seq_tracer = erts_get_system_seq_tracer();
+
+ ASSERT(is_tuple(token) || is_nil(token));
+ if (SEQ_TRACE_T_SENDER(token) == seq_tracer || token == NIL ||
+ (process && process->trace_flags & F_SENSITIVE)) {
+ return;
+ }
+
+ switch (type) {
+ case SEQ_TRACE_SEND: type_atom = am_send; break;
+ case SEQ_TRACE_PRINT: type_atom = am_print; break;
+ case SEQ_TRACE_RECEIVE: type_atom = am_receive; break;
+ default:
+ erl_exit(1, "invalid type in seq_trace_output_generic: %d:\n", type);
+ return; /* To avoid warning */
+ }
+
+ if ((unsigned_val(SEQ_TRACE_T_FLAGS(token)) & type) == 0) {
+ /* No flags set, nothing to do */
+ return;
+ }
+
+ if (seq_tracer == am_false) {
+ return; /* no need to send anything */
+ }
+
+ if (is_internal_port(seq_tracer)) {
+ Eterm local_heap[64];
+ hp = local_heap;
+ label = SEQ_TRACE_T_LABEL(token);
+ lastcnt_serial = TUPLE2(hp, SEQ_TRACE_T_LASTCNT(token),
+ SEQ_TRACE_T_SERIAL(token));
+ hp += 3;
+ if (exitfrom != NIL) {
+ msg = TUPLE3(hp, am_EXIT, exitfrom, msg);
+ hp += 4;
+ }
+ mess = TUPLE5(hp, type_atom, lastcnt_serial, SEQ_TRACE_T_SENDER(token),
+ receiver, msg);
+ hp += 6;
+ erts_smp_mtx_lock(&smq_mtx);
+ if ((unsigned_val(SEQ_TRACE_T_FLAGS(token)) & SEQ_TRACE_TIMESTAMP) == 0) {
+ mess = TUPLE3(hp, am_seq_trace, label, mess);
+ seq_trace_send_to_port(NULL, seq_tracer, mess, NIL);
+ } else {
+ Uint ms,s,us,ts;
+ GET_NOW(&ms, &s, &us);
+ ts = TUPLE3(hp, make_small(ms),make_small(s), make_small(us));
+ hp += 4;
+ mess = TUPLE4(hp, am_seq_trace, label, mess, ts);
+ seq_trace_send_to_port(process, seq_tracer, mess, ts);
+ }
+ erts_smp_mtx_unlock(&smq_mtx);
+ } else {
+#ifndef ERTS_SMP
+ Process* tracer;
+#endif
+ Eterm sender_copy;
+ Eterm receiver_copy;
+ Eterm m2;
+ Uint sz_label, sz_lastcnt_serial, sz_msg, sz_ts, sz_sender,
+ sz_exitfrom, sz_receiver;
+
+ ASSERT(is_internal_pid(seq_tracer)
+ && internal_pid_index(seq_tracer) < erts_max_processes);
+
+#ifndef ERTS_SMP
+
+ tracer = process_tab[internal_pid_index(seq_tracer)];
+ if (INVALID_PID(tracer, tracer->id)) {
+ system_seq_tracer = am_false;
+ return; /* no need to send anything */
+ }
+#endif
+ if (receiver == seq_tracer) {
+ return; /* no need to send anything */
+ }
+
+ sz_label = size_object(SEQ_TRACE_T_LABEL(token));
+ sz_sender = size_object(SEQ_TRACE_T_SENDER(token));
+ sz_receiver = size_object(receiver);
+ sz_lastcnt_serial = 3; /* TUPLE2 */
+ sz_msg = size_object(msg);
+
+ sz_ts = ((unsigned_val(SEQ_TRACE_T_FLAGS(token)) & SEQ_TRACE_TIMESTAMP) ?
+ 5 : 0);
+ if (exitfrom != NIL) {
+ sz_exit = 4; /* create {'EXIT',exitfrom,msg} */
+ sz_exitfrom = size_object(exitfrom);
+ }
+ else {
+ sz_exit = 0;
+ sz_exitfrom = 0;
+ }
+ bp = new_message_buffer(4 /* TUPLE3 */ + sz_ts + 6 /* TUPLE5 */
+ + sz_lastcnt_serial + sz_label + sz_msg
+ + sz_exit + sz_exitfrom
+ + sz_sender + sz_receiver);
+ hp = bp->mem;
+ label = copy_struct(SEQ_TRACE_T_LABEL(token), sz_label, &hp, &bp->off_heap);
+ lastcnt_serial = TUPLE2(hp,SEQ_TRACE_T_LASTCNT(token),SEQ_TRACE_T_SERIAL(token));
+ hp += 3;
+ m2 = copy_struct(msg, sz_msg, &hp, &bp->off_heap);
+ if (sz_exit) {
+ Eterm exitfrom_copy = copy_struct(exitfrom,
+ sz_exitfrom,
+ &hp,
+ &bp->off_heap);
+ m2 = TUPLE3(hp, am_EXIT, exitfrom_copy, m2);
+ hp += 4;
+ }
+ sender_copy = copy_struct(SEQ_TRACE_T_SENDER(token),
+ sz_sender,
+ &hp,
+ &bp->off_heap);
+ receiver_copy = copy_struct(receiver,
+ sz_receiver,
+ &hp,
+ &bp->off_heap);
+ mess = TUPLE5(hp,
+ type_atom,
+ lastcnt_serial,
+ sender_copy,
+ receiver_copy,
+ m2);
+ hp += 6;
+
+ erts_smp_mtx_lock(&smq_mtx);
+
+ if (sz_ts) {/* timestamp should be included */
+ Uint ms,s,us,ts;
+ GET_NOW(&ms, &s, &us);
+ ts = TUPLE3(hp, make_small(ms),make_small(s), make_small(us));
+ hp += 4;
+ mess = TUPLE4(hp, am_seq_trace, label, mess, ts);
+ } else {
+ mess = TUPLE3(hp, am_seq_trace, label, mess);
+ }
+
+#ifdef ERTS_SMP
+ enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SEQTRACE, NIL, NIL, mess, bp);
+ erts_smp_mtx_unlock(&smq_mtx);
+#else
+ erts_queue_message(tracer, NULL, bp, mess, NIL); /* trace_token must be NIL here */
+#endif
+ }
+}
+
+/* Send {trace_ts, Pid, return_to, {Mod, Func, Arity}, Timestamp}
+ * or {trace, Pid, return_to, {Mod, Func, Arity}}
+ */
+void
+erts_trace_return_to(Process *p, Uint *pc)
+{
+ Eterm* hp;
+ Eterm mfa;
+ Eterm mess;
+ Eterm local_heap[4+5+5];
+
+ Eterm *code_ptr = find_function_from_pc(pc);
+
+ hp = local_heap;
+
+ if (!code_ptr) {
+ mfa = am_undefined;
+ } else {
+ mfa = TUPLE3(hp, code_ptr[0], code_ptr[1], make_small(code_ptr[2]));
+ hp += 4;
+ }
+
+ mess = TUPLE4(hp, am_trace, p->id, am_return_to, mfa);
+ hp += 5;
+
+ erts_smp_mtx_lock(&smq_mtx);
+
+ if (p->trace_flags & F_TIMESTAMP) {
+ hp = patch_ts(mess, hp);
+ }
+
+ if (is_internal_port(p->tracer_proc)) {
+ send_to_port(p, mess, &p->tracer_proc, &p->trace_flags);
+ } else {
+ ErlHeapFragment *bp;
+ ErlOffHeap *off_heap;
+ ERTS_TRACER_REF_TYPE tracer_ref;
+ unsigned size;
+
+ /*
+ * Find the tracer.
+ */
+ ASSERT(is_internal_pid(p->tracer_proc)
+ && internal_pid_index(p->tracer_proc) < erts_max_processes);
+
+ ERTS_GET_TRACER_REF(tracer_ref, p->tracer_proc, p->trace_flags);
+
+ size = size_object(mess);
+
+ hp = ERTS_ALLOC_SYSMSG_HEAP(size, &bp, &off_heap, tracer_ref);
+
+ /*
+ * Copy the trace message into the buffer and enqueue it.
+ */
+ mess = copy_struct(mess, size, &hp, off_heap);
+ ERTS_ENQ_TRACE_MSG(p->id, tracer_ref, mess, bp);
+ }
+ erts_smp_mtx_unlock(&smq_mtx);
+}
+
+
+/* Send {trace_ts, Pid, return_from, {Mod, Name, Arity}, Retval, Timestamp}
+ * or {trace, Pid, return_from, {Mod, Name, Arity}, Retval}
+ */
+void
+erts_trace_return(Process* p, Eterm* fi, Eterm retval, Eterm *tracer_pid)
+{
+ Eterm* hp;
+ Eterm mfa;
+ Eterm mess;
+ Eterm mod, name;
+ int arity;
+ Uint meta_flags, *tracee_flags;
+#ifdef ERTS_SMP
+ Eterm tracee;
+#endif
+
+ ASSERT(tracer_pid);
+ if (*tracer_pid == am_true) {
+ /* Breakpoint trace enabled without specifying tracer =>
+ * use process tracer and flags
+ */
+ tracer_pid = &p->tracer_proc;
+ }
+ if (is_nil(*tracer_pid)) {
+ /* Trace disabled */
+ return;
+ }
+ ASSERT(is_internal_pid(*tracer_pid) || is_internal_port(*tracer_pid));
+ if (*tracer_pid == p->id) {
+ /* Do not generate trace messages to oneself */
+ return;
+ }
+ if (tracer_pid == &p->tracer_proc) {
+ /* Tracer specified in process structure =>
+ * non-breakpoint trace =>
+ * use process flags
+ */
+ tracee_flags = &p->trace_flags;
+#ifdef ERTS_SMP
+ tracee = p->id;
+#endif
+ } else {
+ /* Tracer not specified in process structure =>
+ * tracer specified in breakpoint =>
+ * meta trace =>
+ * use fixed flag set instead of process flags
+ */
+ meta_flags = F_TRACE_CALLS | F_TIMESTAMP;
+ tracee_flags = &meta_flags;
+#ifdef ERTS_SMP
+ tracee = NIL;
+#endif
+ }
+ if (! (*tracee_flags & F_TRACE_CALLS)) {
+ return;
+ }
+
+ mod = fi[0];
+ name = fi[1];
+ arity = fi[2];
+
+ if (is_internal_port(*tracer_pid)) {
+ Eterm local_heap[4+6+5];
+ hp = local_heap;
+ mfa = TUPLE3(hp, mod, name, make_small(arity));
+ hp += 4;
+ mess = TUPLE5(hp, am_trace, p->id, am_return_from, mfa, retval);
+ hp += 6;
+ erts_smp_mtx_lock(&smq_mtx);
+ if (*tracee_flags & F_TIMESTAMP) {
+ hp = patch_ts(mess, hp);
+ }
+ send_to_port(p, mess, tracer_pid, tracee_flags);
+ erts_smp_mtx_unlock(&smq_mtx);
+ } else {
+ ErlHeapFragment *bp;
+ ErlOffHeap *off_heap;
+ ERTS_TRACER_REF_TYPE tracer_ref;
+ unsigned size;
+ unsigned retval_size;
+#ifdef DEBUG
+ Eterm* limit;
+#endif
+
+ ASSERT(is_internal_pid(*tracer_pid)
+ && internal_pid_index(*tracer_pid) < erts_max_processes);
+
+ ERTS_GET_TRACER_REF(tracer_ref, *tracer_pid, *tracee_flags);
+
+ retval_size = size_object(retval);
+ size = 6 + 4 + retval_size;
+ if (*tracee_flags & F_TIMESTAMP) {
+ size += 1+4;
+ }
+
+ hp = ERTS_ALLOC_SYSMSG_HEAP(size, &bp, &off_heap, tracer_ref);
+#ifdef DEBUG
+ limit = hp + size;
+#endif
+
+ /*
+ * Build the trace tuple and put it into receive queue of the tracer process.
+ */
+
+ mfa = TUPLE3(hp, mod, name, make_small(arity));
+ hp += 4;
+ retval = copy_struct(retval, retval_size, &hp, off_heap);
+ mess = TUPLE5(hp, am_trace, p->id/* Local pid */, am_return_from, mfa, retval);
+ hp += 6;
+
+ erts_smp_mtx_lock(&smq_mtx);
+
+ if (*tracee_flags & F_TIMESTAMP) {
+ hp = patch_ts(mess, hp);
+ }
+
+ ASSERT(hp == limit);
+
+ ERTS_ENQ_TRACE_MSG(tracee, tracer_ref, mess, bp);
+ erts_smp_mtx_unlock(&smq_mtx);
+ }
+}
+
+/* Send {trace_ts, Pid, exception_from, {Mod, Name, Arity}, {Class,Value},
+ * Timestamp}
+ * or {trace, Pid, exception_from, {Mod, Name, Arity}, {Class,Value},
+ * Timestamp}
+ *
+ * Where Class is atomic but Value is any term.
+ */
+void
+erts_trace_exception(Process* p, Eterm mfa[3], Eterm class, Eterm value,
+ Eterm *tracer_pid)
+{
+ Eterm* hp;
+ Eterm mfa_tuple;
+ Eterm cv;
+ Eterm mess;
+ Uint meta_flags, *tracee_flags;
+#ifdef ERTS_SMP
+ Eterm tracee;
+#endif
+
+ ASSERT(tracer_pid);
+ if (*tracer_pid == am_true) {
+ /* Breakpoint trace enabled without specifying tracer =>
+ * use process tracer and flags
+ */
+ tracer_pid = &p->tracer_proc;
+ }
+ if (is_nil(*tracer_pid)) {
+ /* Trace disabled */
+ return;
+ }
+ ASSERT(is_internal_pid(*tracer_pid) || is_internal_port(*tracer_pid));
+ if (*tracer_pid == p->id) {
+ /* Do not generate trace messages to oneself */
+ return;
+ }
+ if (tracer_pid == &p->tracer_proc) {
+ /* Tracer specified in process structure =>
+ * non-breakpoint trace =>
+ * use process flags
+ */
+ tracee_flags = &p->trace_flags;
+#ifdef ERTS_SMP
+ tracee = p->id;
+#endif
+ if (! (*tracee_flags & F_TRACE_CALLS)) {
+ return;
+ }
+ } else {
+ /* Tracer not specified in process structure =>
+ * tracer specified in breakpoint =>
+ * meta trace =>
+ * use fixed flag set instead of process flags
+ */
+ meta_flags = F_TRACE_CALLS | F_TIMESTAMP;
+ tracee_flags = &meta_flags;
+#ifdef ERTS_SMP
+ tracee = NIL;
+#endif
+ }
+
+ if (is_internal_port(*tracer_pid)) {
+ Eterm local_heap[4+3+6+5];
+ hp = local_heap;
+ mfa_tuple = TUPLE3(hp, mfa[0], mfa[1], make_small(mfa[2]));
+ hp += 4;
+ cv = TUPLE2(hp, class, value);
+ hp += 3;
+ mess = TUPLE5(hp, am_trace, p->id, am_exception_from, mfa_tuple, cv);
+ hp += 6;
+ ASSERT((hp - local_heap)*sizeof(*hp) <= sizeof(local_heap));
+ erts_smp_mtx_lock(&smq_mtx);
+ if (*tracee_flags & F_TIMESTAMP) {
+ hp = patch_ts(mess, hp); /* hp += 5 */
+ ASSERT((hp - local_heap)*sizeof(*hp) == sizeof(local_heap));
+ }
+ send_to_port(p, mess, tracer_pid, tracee_flags);
+ erts_smp_mtx_unlock(&smq_mtx);
+ } else {
+ ErlHeapFragment *bp;
+ ErlOffHeap *off_heap;
+ ERTS_TRACER_REF_TYPE tracer_ref;
+ unsigned size;
+ unsigned value_size;
+#ifdef DEBUG
+ Eterm* limit;
+#endif
+
+ ASSERT(is_internal_pid(*tracer_pid)
+ && internal_pid_index(*tracer_pid) < erts_max_processes);
+
+ ERTS_GET_TRACER_REF(tracer_ref, *tracer_pid, *tracee_flags);
+
+ value_size = size_object(value);
+ size = 6 + 4 + 3 + value_size;
+ if (*tracee_flags & F_TIMESTAMP) {
+ size += 1+4;
+ }
+
+ hp = ERTS_ALLOC_SYSMSG_HEAP(size, &bp, &off_heap, tracer_ref);
+#ifdef DEBUG
+ limit = hp + size;
+#endif
+
+ /*
+ * Build the trace tuple and put it into receive queue of the tracer process.
+ */
+
+ mfa_tuple = TUPLE3(hp, mfa[0], mfa[1], make_small(mfa[2]));
+ hp += 4;
+ value = copy_struct(value, value_size, &hp, off_heap);
+ cv = TUPLE2(hp, class, value);
+ hp += 3;
+ mess = TUPLE5(hp, am_trace, p->id/* Local pid */,
+ am_exception_from, mfa_tuple, cv);
+ hp += 6;
+
+ erts_smp_mtx_lock(&smq_mtx);
+
+ if (*tracee_flags & F_TIMESTAMP) {
+ hp = patch_ts(mess, hp);
+ }
+
+ ASSERT(hp == limit);
+
+ ERTS_ENQ_TRACE_MSG(tracee, tracer_ref, mess, bp);
+ erts_smp_mtx_unlock(&smq_mtx);
+ }
+}
+
+/*
+ * This function implements the new call trace.
+ *
+ * Send {trace_ts, Pid, call, {Mod, Func, A}, PamResult, Timestamp}
+ * or {trace_ts, Pid, call, {Mod, Func, A}, Timestamp}
+ * or {trace, Pid, call, {Mod, Func, A}, PamResult}
+ * or {trace, Pid, call, {Mod, Func, A}
+ *
+ * where 'A' is arity or argument list depending on trace flag 'arity'.
+ *
+ * If *tracer_pid is am_true, it is a breakpoint trace that shall use
+ * the process tracer, if it is NIL no trace message is generated,
+ * if it is a pid or port we do a meta trace.
+ */
+Uint32
+erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec,
+ Eterm* args, int local, Eterm *tracer_pid)
+{
+ Eterm* hp;
+ Eterm mfa_tuple;
+ int arity;
+ int i;
+ Uint32 return_flags;
+ Eterm pam_result = am_true;
+ Eterm mess;
+ Uint meta_flags, *tracee_flags;
+#ifdef ERTS_SMP
+ Eterm tracee;
+#endif
+ Eterm transformed_args[MAX_ARG];
+ ErlSubBin sub_bin_heap;
+
+ ASSERT(tracer_pid);
+ if (*tracer_pid == am_true) {
+ /* Breakpoint trace enabled without specifying tracer =>
+ * use process tracer and flags
+ */
+ tracer_pid = &p->tracer_proc;
+ }
+ if (is_nil(*tracer_pid)) {
+ /* Trace disabled */
+ return 0;
+ }
+ ASSERT(is_internal_pid(*tracer_pid) || is_internal_port(*tracer_pid));
+ if (*tracer_pid == p->id) {
+ /* Do not generate trace messages to oneself */
+ return 0;
+ }
+ if (tracer_pid == &p->tracer_proc) {
+ /* Tracer specified in process structure =>
+ * non-breakpoint trace =>
+ * use process flags
+ */
+ tracee_flags = &p->trace_flags;
+#ifdef ERTS_SMP
+ tracee = p->id;
+#endif
+ } else {
+ /* Tracer not specified in process structure =>
+ * tracer specified in breakpoint =>
+ * meta trace =>
+ * use fixed flag set instead of process flags
+ */
+ if (p->trace_flags & F_SENSITIVE) {
+ /* No trace messages for sensitive processes. */
+ return 0;
+ }
+ meta_flags = F_TRACE_CALLS | F_TIMESTAMP;
+ tracee_flags = &meta_flags;
+#ifdef ERTS_SMP
+ tracee = NIL;
+#endif
+ }
+
+ /*
+ * Because of the delayed sub-binary creation optimization introduced in
+ * R12B, (at most) one of arguments can be a match context instead of
+ * a binary. Since we don't want to handle match contexts in utility functions
+ * such as size_object() and copy_struct(), we must make sure that we
+ * temporarily convert any match contexts to sub binaries.
+ */
+ arity = mfa[2];
+#ifdef DEBUG
+ sub_bin_heap.thing_word = 0;
+#endif
+ for (i = 0; i < arity; i++) {
+ Eterm arg = args[i];
+ if (is_boxed(arg) && header_is_bin_matchstate(*boxed_val(arg))) {
+ ErlBinMatchState* ms = (ErlBinMatchState *) boxed_val(arg);
+ ErlBinMatchBuffer* mb = &ms->mb;
+ ErlSubBin* sb = &sub_bin_heap;
+ Uint bit_size;
+
+ ASSERT(sub_bin_heap.thing_word == 0); /* At most one of match context */
+
+ bit_size = mb->size - mb->offset;
+ sb->thing_word = HEADER_SUB_BIN;
+ sb->size = BYTE_OFFSET(bit_size);
+ sb->bitsize = BIT_OFFSET(bit_size);
+ sb->offs = BYTE_OFFSET(mb->offset);
+ sb->bitoffs = BIT_OFFSET(mb->offset);
+ sb->is_writable = 0;
+ sb->orig = mb->orig;
+
+ arg = make_binary(sb);
+ }
+ transformed_args[i] = arg;
+ }
+ args = transformed_args;
+
+ if (is_internal_port(*tracer_pid)) {
+ Eterm local_heap[64+MAX_ARG];
+ hp = local_heap;
+
+ if (!erts_is_valid_tracer_port(*tracer_pid)) {
+#ifdef ERTS_SMP
+ ASSERT(is_nil(tracee) || tracer_pid == &p->tracer_proc);
+ if (is_not_nil(tracee))
+ erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR);
+#endif
+ *tracee_flags &= ~TRACEE_FLAGS;
+ *tracer_pid = NIL;
+#ifdef ERTS_SMP
+ if (is_not_nil(tracee))
+ erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR);
+#endif
+ return 0;
+ }
+
+ /*
+ * If there is a PAM program, run it. Return if it fails.
+ *
+ * Some precedence rules:
+ *
+ * - No proc flags, e.g 'silent' or 'return_to'
+ * has any effect on meta trace.
+ * - The 'silent' process trace flag silences all call
+ * related messages, e.g 'call', 'return_to' and 'return_from'.
+ * - The {message,_} PAM function does not affect {return_trace}.
+ * - The {message,false} PAM function shall give the same
+ * 'call' trace message as no PAM match.
+ * - The {message,true} PAM function shall give the same
+ * 'call' trace message as a nonexistent PAM program.
+ */
+
+ /* BEGIN this code should be the same for port and pid trace */
+ return_flags = 0;
+ if (match_spec) {
+ pam_result = erts_match_set_run(p, match_spec, args, arity,
+ &return_flags);
+ if (is_non_value(pam_result)) {
+ erts_match_set_release_result(p);
+ return 0;
+ }
+ }
+ if (tracee_flags == &meta_flags) {
+ /* Meta trace */
+ if (pam_result == am_false) {
+ erts_match_set_release_result(p);
+ return return_flags;
+ }
+ } else {
+ /* Non-meta trace */
+ if (*tracee_flags & F_TRACE_SILENT) {
+ erts_match_set_release_result(p);
+ return 0;
+ }
+ if (pam_result == am_false) {
+ erts_match_set_release_result(p);
+ return return_flags;
+ }
+ if (local && (*tracee_flags & F_TRACE_RETURN_TO)) {
+ return_flags |= MATCH_SET_RETURN_TO_TRACE;
+ }
+ }
+ /* END this code should be the same for port and pid trace */
+
+ /*
+ * Build the the {M,F,A} tuple in the local heap.
+ * (A is arguments or arity.)
+ */
+
+ if (*tracee_flags & F_TRACE_ARITY_ONLY) {
+ mfa_tuple = make_small(arity);
+ } else {
+ mfa_tuple = NIL;
+ for (i = arity-1; i >= 0; i--) {
+ mfa_tuple = CONS(hp, args[i], mfa_tuple);
+ hp += 2;
+ }
+ }
+ mfa_tuple = TUPLE3(hp, mfa[0], mfa[1], mfa_tuple);
+ hp += 4;
+
+ /*
+ * Build the trace tuple and send it to the port.
+ */
+
+ mess = TUPLE4(hp, am_trace, p->id, am_call, mfa_tuple);
+ hp += 5;
+ if (pam_result != am_true) {
+ hp[-5] = make_arityval(5);
+ *hp++ = pam_result;
+ }
+ erts_smp_mtx_lock(&smq_mtx);
+ if (*tracee_flags & F_TIMESTAMP) {
+ hp = patch_ts(mess, hp);
+ }
+ send_to_port(p, mess, tracer_pid, tracee_flags);
+ erts_smp_mtx_unlock(&smq_mtx);
+ erts_match_set_release_result(p);
+ return *tracer_pid == NIL ? 0 : return_flags;
+
+ } else {
+ ErlHeapFragment *bp;
+ ErlOffHeap *off_heap;
+ Process *tracer;
+ ERTS_TRACER_REF_TYPE tracer_ref;
+#ifdef ERTS_SMP
+ Eterm tpid;
+#endif
+ unsigned size;
+ unsigned sizes[MAX_ARG];
+ unsigned pam_result_size = 0;
+ int invalid_tracer;
+#ifdef DEBUG
+ Eterm* limit;
+#endif
+
+ ASSERT(is_internal_pid(*tracer_pid)
+ && internal_pid_index(*tracer_pid) < erts_max_processes);
+
+ tracer = erts_pid2proc(p, ERTS_PROC_LOCK_MAIN,
+ *tracer_pid, ERTS_PROC_LOCK_STATUS);
+ if (!tracer)
+ invalid_tracer = 1;
+ else {
+ invalid_tracer = (tracer->trace_flags & F_TRACER) == 0;
+ erts_smp_proc_unlock(tracer, ERTS_PROC_LOCK_STATUS);
+ }
+
+ if (invalid_tracer) {
+#ifdef ERTS_SMP
+ ASSERT(is_nil(tracee) || tracer_pid == &p->tracer_proc);
+ if (is_not_nil(tracee))
+ erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR);
+#endif
+ *tracee_flags &= ~TRACEE_FLAGS;
+ *tracer_pid = NIL;
+#ifdef ERTS_SMP
+ if (is_not_nil(tracee))
+ erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR);
+#endif
+ return 0;
+ }
+
+#ifdef ERTS_SMP
+ tpid = *tracer_pid; /* Need to save tracer pid,
+ since *tracer_pid might
+ be reset by erts_match_set_run() */
+ tracer_ref = tpid;
+#else
+ tracer_ref = tracer;
+#endif
+
+ /*
+ * If there is a PAM program, run it. Return if it fails.
+ *
+ * See the rules above in the port trace code.
+ */
+
+ /* BEGIN this code should be the same for port and pid trace */
+ return_flags = 0;
+ if (match_spec) {
+ pam_result = erts_match_set_run(p, match_spec, args, arity,
+ &return_flags);
+ if (is_non_value(pam_result)) {
+ erts_match_set_release_result(p);
+ return 0;
+ }
+ }
+ if (tracee_flags == &meta_flags) {
+ /* Meta trace */
+ if (pam_result == am_false) {
+ erts_match_set_release_result(p);
+ return return_flags;
+ }
+ } else {
+ /* Non-meta trace */
+ if (*tracee_flags & F_TRACE_SILENT) {
+ erts_match_set_release_result(p);
+ return 0;
+ }
+ if (pam_result == am_false) {
+ erts_match_set_release_result(p);
+ return return_flags;
+ }
+ if (local && (*tracee_flags & F_TRACE_RETURN_TO)) {
+ return_flags |= MATCH_SET_RETURN_TO_TRACE;
+ }
+ }
+ /* END this code should be the same for port and pid trace */
+
+ /*
+ * Calculate number of words needed on heap.
+ */
+
+ size = 4 + 5; /* Trace tuple + MFA tuple. */
+ if (! (*tracee_flags & F_TRACE_ARITY_ONLY)) {
+ size += 2*arity;
+ for (i = arity-1; i >= 0; i--) {
+ sizes[i] = size_object(args[i]);
+ size += sizes[i];
+ }
+ }
+ if (*tracee_flags & F_TIMESTAMP) {
+ size += 1 + 4;
+ /* One element in trace tuple + timestamp tuple. */
+ }
+ if (pam_result != am_true) {
+ pam_result_size = size_object(pam_result);
+ size += 1 + pam_result_size;
+ /* One element in trace tuple + term size. */
+ }
+
+ hp = ERTS_ALLOC_SYSMSG_HEAP(size, &bp, &off_heap, tracer_ref);
+#ifdef DEBUG
+ limit = hp + size;
+#endif
+
+ /*
+ * Build the the {M,F,A} tuple in the message buffer.
+ * (A is arguments or arity.)
+ */
+
+ if (*tracee_flags & F_TRACE_ARITY_ONLY) {
+ mfa_tuple = make_small(arity);
+ } else {
+ mfa_tuple = NIL;
+ for (i = arity-1; i >= 0; i--) {
+ Eterm term = copy_struct(args[i], sizes[i], &hp, off_heap);
+ mfa_tuple = CONS(hp, term, mfa_tuple);
+ hp += 2;
+ }
+ }
+ mfa_tuple = TUPLE3(hp, mfa[0], mfa[1], mfa_tuple);
+ hp += 4;
+
+ /*
+ * Copy the PAM result (if any) onto the heap.
+ */
+
+ if (pam_result != am_true) {
+ pam_result = copy_struct(pam_result, pam_result_size, &hp, off_heap);
+ }
+
+ erts_match_set_release_result(p);
+
+ /*
+ * Build the trace tuple and enqueue it.
+ */
+
+ mess = TUPLE4(hp, am_trace, p->id/* Local pid */, am_call, mfa_tuple);
+ hp += 5;
+ if (pam_result != am_true) {
+ hp[-5] = make_arityval(5);
+ *hp++ = pam_result;
+ }
+
+ erts_smp_mtx_lock(&smq_mtx);
+
+ if (*tracee_flags & F_TIMESTAMP) {
+ hp = patch_ts(mess, hp);
+ }
+
+ ASSERT(hp == limit);
+ ERTS_ENQ_TRACE_MSG(tracee, tracer_ref, mess, bp);
+ erts_smp_mtx_unlock(&smq_mtx);
+ return return_flags;
+ }
+}
+
+/* Sends trace message:
+ * {trace_ts, ProcessPid, What, Data, Timestamp}
+ * or {trace, ProcessPid, What, Data}
+ *
+ * 'what' must be atomic, 'data' may be a deep term.
+ * 'c_p' is the currently executing process, may be NULL.
+ * 't_p' is the traced process.
+ */
+void
+trace_proc(Process *c_p, Process *t_p, Eterm what, Eterm data)
+{
+ Eterm mess;
+ Eterm* hp;
+ int need;
+
+ if (is_internal_port(t_p->tracer_proc)) {
+ Eterm local_heap[5+5];
+ hp = local_heap;
+ mess = TUPLE4(hp, am_trace, t_p->id, what, data);
+ hp += 5;
+ erts_smp_mtx_lock(&smq_mtx);
+ if (t_p->trace_flags & F_TIMESTAMP) {
+ hp = patch_ts(mess, hp);
+ }
+ send_to_port(
+#ifndef ERTS_SMP
+ /* No fake schedule out and in again after an exit */
+ what == am_exit ? NULL : c_p,
+#else
+ /* Fake schedule out and in are never sent when smp enabled */
+ c_p,
+#endif
+ mess, &t_p->tracer_proc, &t_p->trace_flags);
+ erts_smp_mtx_unlock(&smq_mtx);
+ } else {
+ Eterm tmp;
+ ErlHeapFragment *bp;
+ ErlOffHeap *off_heap;
+ ERTS_TRACER_REF_TYPE tracer_ref;
+ size_t sz_data;
+
+ ASSERT(is_internal_pid(t_p->tracer_proc)
+ && internal_pid_index(t_p->tracer_proc) < erts_max_processes);
+
+ ERTS_GET_TRACER_REF(tracer_ref, t_p->tracer_proc, t_p->trace_flags);
+
+ sz_data = size_object(data);
+
+ need = sz_data + 5 + TS_SIZE(t_p);
+
+ hp = ERTS_ALLOC_SYSMSG_HEAP(need, &bp, &off_heap, tracer_ref);
+
+ tmp = copy_struct(data, sz_data, &hp, off_heap);
+ mess = TUPLE4(hp, am_trace, t_p->id/* Local pid */, what, tmp);
+ hp += 5;
+
+ erts_smp_mtx_lock(&smq_mtx);
+
+ if (t_p->trace_flags & F_TIMESTAMP) {
+ hp = patch_ts(mess, hp);
+ }
+
+ ERTS_ENQ_TRACE_MSG(t_p->id, tracer_ref, mess, bp);
+ erts_smp_mtx_unlock(&smq_mtx);
+ }
+}
+
+
+/* Sends trace message:
+ * {trace_ts, ParentPid, spawn, ChildPid, {Mod, Func, Args}, Timestamp}
+ * or {trace, ParentPid, spawn, ChildPid, {Mod, Func, Args}}
+ *
+ * 'pid' is the ChildPid, 'mod' and 'func' must be atomic,
+ * and 'args' may be a deep term.
+ */
+void
+trace_proc_spawn(Process *p, Eterm pid,
+ Eterm mod, Eterm func, Eterm args)
+{
+ Eterm mfa;
+ Eterm mess;
+ Eterm* hp;
+
+ if (is_internal_port(p->tracer_proc)) {
+ Eterm local_heap[4+6+5];
+ hp = local_heap;
+ mfa = TUPLE3(hp, mod, func, args);
+ hp += 4;
+ mess = TUPLE5(hp, am_trace, p->id, am_spawn, pid, mfa);
+ hp += 6;
+ erts_smp_mtx_lock(&smq_mtx);
+ if (p->trace_flags & F_TIMESTAMP) {
+ hp = patch_ts(mess, hp);
+ }
+ send_to_port(p, mess, &p->tracer_proc, &p->trace_flags);
+ erts_smp_mtx_unlock(&smq_mtx);
+ } else {
+ Eterm tmp;
+ ErlHeapFragment *bp;
+ ErlOffHeap *off_heap;
+ ERTS_TRACER_REF_TYPE tracer_ref;
+ size_t sz_args, sz_pid;
+ Uint need;
+
+ ASSERT(is_internal_pid(p->tracer_proc)
+ && internal_pid_index(p->tracer_proc) < erts_max_processes);
+
+ ERTS_GET_TRACER_REF(tracer_ref, p->tracer_proc, p->trace_flags);
+
+ sz_args = size_object(args);
+ sz_pid = size_object(pid);
+ need = sz_args + 4 + 6 + TS_SIZE(p);
+
+ hp = ERTS_ALLOC_SYSMSG_HEAP(need, &bp, &off_heap, tracer_ref);
+
+ tmp = copy_struct(args, sz_args, &hp, off_heap);
+ mfa = TUPLE3(hp, mod, func, tmp);
+ hp += 4;
+ tmp = copy_struct(pid, sz_pid, &hp, off_heap);
+ mess = TUPLE5(hp, am_trace, p->id, am_spawn, tmp, mfa);
+ hp += 6;
+
+ erts_smp_mtx_lock(&smq_mtx);
+
+ if (p->trace_flags & F_TIMESTAMP) {
+ hp = patch_ts(mess, hp);
+ }
+
+ ERTS_ENQ_TRACE_MSG(p->id, tracer_ref, mess, bp);
+ erts_smp_mtx_unlock(&smq_mtx);
+ }
+}
+
+void save_calls(Process *p, Export *e)
+{
+ struct saved_calls *scb = ERTS_PROC_GET_SAVED_CALLS_BUF(p);
+ if (scb) {
+ Export **ct = &scb->ct[0];
+ int len = scb->len;
+
+ ct[scb->cur] = e;
+ if (++scb->cur >= len)
+ scb->cur = 0;
+ if (scb->n < len)
+ scb->n++;
+ }
+}
+
+/*
+ * Entry point called by the trace wrap functions in erl_bif_wrap.c
+ *
+ * The trace wrap functions are themselves called through the export
+ * entries instead of the original BIF functions.
+ */
+Eterm
+erts_bif_trace(int bif_index, Process* p,
+ Eterm arg1, Eterm arg2, Eterm arg3, Uint *I)
+{
+ Eterm result;
+ int meta = !!(erts_bif_trace_flags[bif_index] & BIF_TRACE_AS_META);
+
+ ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
+
+ if (!ARE_TRACE_FLAGS_ON(p, F_TRACE_CALLS) && (! meta)) {
+ /* Warning! This is an Optimization.
+ *
+ * If neither meta trace is active nor process trace flags then
+ * no tracing will occur. Doing the whole else branch will
+ * also do nothing, only slower.
+ */
+ Eterm (*func)(Process*, Eterm, Eterm, Eterm, Uint*) = bif_table[bif_index].f;
+ result = func(p, arg1, arg2, arg3, I);
+ } else {
+ Eterm (*func)(Process*, Eterm, Eterm, Eterm, Uint*);
+ Export* ep = bif_export[bif_index];
+ Uint32 flags = 0, flags_meta = 0;
+ int global = !!(erts_bif_trace_flags[bif_index] & BIF_TRACE_AS_GLOBAL);
+ int local = !!(erts_bif_trace_flags[bif_index] & BIF_TRACE_AS_LOCAL);
+ Eterm meta_tracer_pid = NIL;
+ int applying = (I == &(ep->code[3])); /* Yup, the apply code for a bif
+ * is actually in the
+ * export entry */
+ Eterm *cp = p->cp;
+
+#ifndef _OSE_
+ Eterm args[3] = {arg1, arg2, arg3};
+#else
+ Eterm args[3];
+ args[0] = arg1;
+ args[1] = arg2;
+ args[2] = arg3;
+#endif
+
+ /*
+ * Make continuation pointer OK, it is not during direct BIF calls,
+ * but it is correct during apply of bif.
+ */
+ if (!applying) {
+ p->cp = I;
+ }
+ if (global || local) {
+ flags = erts_call_trace(p, ep->code, ep->match_prog_set, args,
+ local, &p->tracer_proc);
+ }
+ if (meta) {
+ flags_meta = erts_bif_mtrace(p, ep->code+3, args, local,
+ &meta_tracer_pid);
+ }
+ /* Restore original continuation pointer (if changed). */
+ p->cp = cp;
+
+ func = bif_table[bif_index].f;
+
+ result = func(p, arg1, arg2, arg3, I);
+
+ if (applying && (flags & MATCH_SET_RETURN_TO_TRACE)) {
+ Uint i_return_trace = beam_return_trace[0];
+ Uint i_return_to_trace = beam_return_to_trace[0];
+ Eterm *cpp;
+ /* Maybe advance cp to skip trace stack frames */
+ for (cpp = p->stop; ; cp = cp_val(*cpp++)) {
+ ASSERT(is_CP((Eterm) cp));
+ if (*cp_val((Eterm) cp) == i_return_trace) {
+ /* Skip stack frame variables */
+ while (is_not_CP(*cpp)) cpp++;
+ cpp += 2; /* Skip return_trace parameters */
+ } else if (*cp_val((Eterm) cp) == i_return_to_trace) {
+ /* A return_to trace message is going to be generated
+ * by normal means, so we do not have to.
+ */
+ cp = NULL;
+ break;
+ } else break;
+ }
+ }
+
+ /* Try to get these in the order
+ * they usually appear in normal code... */
+ if (is_non_value(result)) {
+ Uint reason = p->freason;
+ if (reason != TRAP) {
+ Eterm class;
+ Eterm value = p->fvalue;
+ Eterm nocatch[3];
+ /* Expand error value like in handle_error() */
+ if (reason & EXF_ARGLIST) {
+ Eterm *tp;
+ ASSERT(is_tuple(value));
+ tp = tuple_val(value);
+ value = tp[1];
+ }
+ if ((reason & EXF_THROWN) && (p->catches <= 0)) {
+ value = TUPLE2(nocatch, am_nocatch, value);
+ reason = EXC_ERROR;
+ }
+ /* Note: expand_error_value() could theoretically
+ * allocate on the heap, but not for any error
+ * returned by a BIF, and it would do no harm,
+ * just be annoying.
+ */
+ value = expand_error_value(p, reason, value);
+ class = exception_tag[GET_EXC_CLASS(reason)];
+
+ if (flags_meta & MATCH_SET_EXCEPTION_TRACE) {
+ erts_trace_exception(p, ep->code, class, value,
+ &meta_tracer_pid);
+ }
+ if (flags & MATCH_SET_EXCEPTION_TRACE) {
+ erts_trace_exception(p, ep->code, class, value,
+ &p->tracer_proc);
+ }
+ if ((flags & MATCH_SET_RETURN_TO_TRACE) && p->catches > 0) {
+ /* can only happen if(local)*/
+ Eterm *ptr = p->stop;
+ ASSERT(is_CP(*ptr));
+ ASSERT(ptr <= STACK_START(p));
+ /* Search the nearest stack frame for a catch */
+ while (++ptr < STACK_START(p)) {
+ if (is_CP(*ptr)) break;
+ if (is_catch(*ptr)) {
+ if (applying) {
+ /* Apply of BIF, cp is in calling function */
+ if (cp) erts_trace_return_to(p, cp);
+ } else {
+ /* Direct bif call, I points into
+ * calling function */
+ erts_trace_return_to(p, I);
+ }
+ }
+ }
+ }
+ if ((flags_meta|flags) & MATCH_SET_EXCEPTION_TRACE) {
+ erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR);
+ p->trace_flags |= F_EXCEPTION_TRACE;
+ erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR);
+ }
+ }
+ } else {
+ if (flags_meta & MATCH_SET_RX_TRACE) {
+ erts_trace_return(p, ep->code, result, &meta_tracer_pid);
+ }
+ /* MATCH_SET_RETURN_TO_TRACE cannot occur if(meta) */
+ if (flags & MATCH_SET_RX_TRACE) {
+ erts_trace_return(p, ep->code, result, &p->tracer_proc);
+ }
+ if (flags & MATCH_SET_RETURN_TO_TRACE) {
+ /* can only happen if(local)*/
+ if (applying) {
+ /* Apply of BIF, cp is in calling function */
+ if (cp) erts_trace_return_to(p, cp);
+ } else {
+ /* Direct bif call, I points into calling function */
+ erts_trace_return_to(p, I);
+ }
+ }
+ }
+ }
+ ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
+ return result;
+}
+
+/* Sends trace message:
+ * {trace_ts, Pid, What, Msg, Timestamp}
+ * or {trace, Pid, What, Msg}
+ *
+ * where 'What' must be atomic and 'Msg' is:
+ * [{heap_size, HeapSize}, {old_heap_size, OldHeapSize},
+ * {stack_size, StackSize}, {recent_size, RecentSize},
+ * {mbuf_size, MbufSize}]
+ *
+ * where 'HeapSize', 'OldHeapSize', 'StackSize', 'RecentSize and 'MbufSize'
+ * are all small (atomic) integers.
+ */
+void
+trace_gc(Process *p, Eterm what)
+{
+ ErlHeapFragment *bp = NULL;
+ ErlOffHeap *off_heap;
+ ERTS_TRACER_REF_TYPE tracer_ref = ERTS_NULL_TRACER_REF; /* Initialized
+ to eliminate
+ compiler
+ warning */
+ Eterm* hp;
+ Eterm msg = NIL;
+ Uint size;
+ Eterm tags[] = {
+ am_old_heap_block_size,
+ am_heap_block_size,
+ am_mbuf_size,
+ am_recent_size,
+ am_stack_size,
+ am_old_heap_size,
+ am_heap_size
+ };
+ Uint values[] = {
+ OLD_HEAP(p) ? OLD_HEND(p) - OLD_HEAP(p) : 0,
+ HEAP_SIZE(p),
+ MBUF_SIZE(p),
+ HIGH_WATER(p) - HEAP_START(p),
+ STACK_START(p) - p->stop,
+ OLD_HEAP(p) ? OLD_HTOP(p) - OLD_HEAP(p) : 0,
+ HEAP_TOP(p) - HEAP_START(p)
+ };
+ Eterm local_heap[(sizeof(values)/sizeof(Uint))
+ *(2/*cons*/ + 3/*2-tuple*/ + BIG_UINT_HEAP_SIZE)
+ + 5/*4-tuple */ + TS_HEAP_WORDS];
+#ifdef DEBUG
+ Eterm* limit;
+#endif
+
+ ASSERT(sizeof(values)/sizeof(Uint) == sizeof(tags)/sizeof(Eterm));
+
+ if (is_internal_port(p->tracer_proc)) {
+ hp = local_heap;
+#ifdef DEBUG
+ size = 0;
+ (void) erts_bld_atom_uint_2tup_list(NULL,
+ &size,
+ sizeof(values)/sizeof(Uint),
+ tags,
+ values);
+ size += 5/*4-tuple*/ + TS_SIZE(p);
+#endif
+ } else {
+ ASSERT(is_internal_pid(p->tracer_proc)
+ && internal_pid_index(p->tracer_proc) < erts_max_processes);
+
+ ERTS_GET_TRACER_REF(tracer_ref, p->tracer_proc, p->trace_flags);
+
+ size = 0;
+ (void) erts_bld_atom_uint_2tup_list(NULL,
+ &size,
+ sizeof(values)/sizeof(Uint),
+ tags,
+ values);
+ size += 5/*4-tuple*/ + TS_SIZE(p);
+
+ hp = ERTS_ALLOC_SYSMSG_HEAP(size, &bp, &off_heap, tracer_ref);
+ }
+
+#ifdef DEBUG
+ limit = hp + size;
+ ASSERT(size <= sizeof(local_heap)/sizeof(Eterm));
+#endif
+
+ msg = erts_bld_atom_uint_2tup_list(&hp,
+ NULL,
+ sizeof(values)/sizeof(Uint),
+ tags,
+ values);
+
+ msg = TUPLE4(hp, am_trace, p->id/* Local pid */, what, msg);
+ hp += 5;
+
+ erts_smp_mtx_lock(&smq_mtx);
+
+ if (p->trace_flags & F_TIMESTAMP) {
+ hp = patch_ts(msg, hp);
+ }
+ ASSERT(hp == limit);
+ if (is_internal_port(p->tracer_proc))
+ send_to_port(p, msg, &p->tracer_proc, &p->trace_flags);
+ else
+ ERTS_ENQ_TRACE_MSG(p->id, tracer_ref, msg, bp);
+ erts_smp_mtx_unlock(&smq_mtx);
+}
+
+
+
+void
+monitor_long_gc(Process *p, Uint time) {
+ ErlHeapFragment *bp;
+ ErlOffHeap *off_heap;
+#ifndef ERTS_SMP
+ Process *monitor_p;
+#endif
+ Uint hsz;
+ Eterm *hp, list, msg;
+ Eterm tags[] = {
+ am_timeout,
+ am_old_heap_block_size,
+ am_heap_block_size,
+ am_mbuf_size,
+ am_stack_size,
+ am_old_heap_size,
+ am_heap_size
+ };
+ Eterm values[] = {
+ time,
+ OLD_HEAP(p) ? OLD_HEND(p) - OLD_HEAP(p) : 0,
+ HEAP_SIZE(p),
+ MBUF_SIZE(p),
+ STACK_START(p) - p->stop,
+ OLD_HEAP(p) ? OLD_HTOP(p) - OLD_HEAP(p) : 0,
+ HEAP_TOP(p) - HEAP_START(p)
+ };
+#ifdef DEBUG
+ Eterm *hp_end;
+#endif
+
+#ifndef ERTS_SMP
+ ASSERT(is_internal_pid(system_monitor)
+ && internal_pid_index(system_monitor) < erts_max_processes);
+ monitor_p = process_tab[internal_pid_index(system_monitor)];
+ if (INVALID_PID(monitor_p, system_monitor) || p == monitor_p) {
+ return;
+ }
+#endif
+
+ hsz = 0;
+ (void) erts_bld_atom_uint_2tup_list(NULL,
+ &hsz,
+ sizeof(values)/sizeof(Uint),
+ tags,
+ values);
+ hsz += 5 /* 4-tuple */;
+
+ hp = ERTS_ALLOC_SYSMSG_HEAP(hsz, &bp, &off_heap, monitor_p);
+
+#ifdef DEBUG
+ hp_end = hp + hsz;
+#endif
+
+ list = erts_bld_atom_uint_2tup_list(&hp,
+ NULL,
+ sizeof(values)/sizeof(Uint),
+ tags,
+ values);
+ msg = TUPLE4(hp, am_monitor, p->id/* Local pid */, am_long_gc, list);
+
+#ifdef DEBUG
+ hp += 5 /* 4-tuple */;
+ ASSERT(hp == hp_end);
+#endif
+
+#ifdef ERTS_SMP
+ enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->id, NIL, msg, bp);
+#else
+ erts_queue_message(monitor_p, NULL, bp, msg, NIL);
+#endif
+}
+
+void
+monitor_large_heap(Process *p) {
+ ErlHeapFragment *bp;
+ ErlOffHeap *off_heap;
+#ifndef ERTS_SMP
+ Process *monitor_p;
+#endif
+ Uint hsz;
+ Eterm *hp, list, msg;
+ Eterm tags[] = {
+ am_old_heap_block_size,
+ am_heap_block_size,
+ am_mbuf_size,
+ am_stack_size,
+ am_old_heap_size,
+ am_heap_size
+ };
+ Uint values[] = {
+ OLD_HEAP(p) ? OLD_HEND(p) - OLD_HEAP(p) : 0,
+ HEAP_SIZE(p),
+ MBUF_SIZE(p),
+ STACK_START(p) - p->stop,
+ OLD_HEAP(p) ? OLD_HTOP(p) - OLD_HEAP(p) : 0,
+ HEAP_TOP(p) - HEAP_START(p)
+ };
+#ifdef DEBUG
+ Eterm *hp_end;
+#endif
+
+
+#ifndef ERTS_SMP
+ ASSERT(is_internal_pid(system_monitor)
+ && internal_pid_index(system_monitor) < erts_max_processes);
+ monitor_p = process_tab[internal_pid_index(system_monitor)];
+ if (INVALID_PID(monitor_p, system_monitor) || p == monitor_p) {
+ return;
+ }
+#endif
+
+ hsz = 0;
+ (void) erts_bld_atom_uint_2tup_list(NULL,
+ &hsz,
+ sizeof(values)/sizeof(Uint),
+ tags,
+ values);
+ hsz += 5 /* 4-tuple */;
+
+ hp = ERTS_ALLOC_SYSMSG_HEAP(hsz, &bp, &off_heap, monitor_p);
+
+#ifdef DEBUG
+ hp_end = hp + hsz;
+#endif
+
+ list = erts_bld_atom_uint_2tup_list(&hp,
+ NULL,
+ sizeof(values)/sizeof(Uint),
+ tags,
+ values);
+ msg = TUPLE4(hp, am_monitor, p->id/* Local pid */, am_large_heap, list);
+
+#ifdef DEBUG
+ hp += 5 /* 4-tuple */;
+ ASSERT(hp == hp_end);
+#endif
+
+#ifdef ERTS_SMP
+ enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->id, NIL, msg, bp);
+#else
+ erts_queue_message(monitor_p, NULL, bp, msg, NIL);
+#endif
+}
+
+void
+monitor_generic(Process *p, Eterm type, Eterm spec) {
+ ErlHeapFragment *bp;
+ ErlOffHeap *off_heap;
+#ifndef ERTS_SMP
+ Process *monitor_p;
+#endif
+ Eterm *hp, msg;
+
+#ifndef ERTS_SMP
+ ASSERT(is_internal_pid(system_monitor)
+ && internal_pid_index(system_monitor) < erts_max_processes);
+ monitor_p = process_tab[internal_pid_index(system_monitor)];
+ if (INVALID_PID(monitor_p, system_monitor) || p == monitor_p) {
+ return;
+ }
+#endif
+
+ hp = ERTS_ALLOC_SYSMSG_HEAP(5, &bp, &off_heap, monitor_p);
+
+ msg = TUPLE4(hp, am_monitor, p->id/* Local pid */, type, spec);
+ hp += 5;
+
+#ifdef ERTS_SMP
+ enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->id, NIL, msg, bp);
+#else
+ erts_queue_message(monitor_p, NULL, bp, msg, NIL);
+#endif
+
+}
+
+
+/* Begin system_profile tracing */
+/* Scheduler profiling */
+
+void
+profile_scheduler(Eterm scheduler_id, Eterm state) {
+ Eterm *hp, msg, timestamp;
+ Uint Ms, s, us;
+
+#ifndef ERTS_SMP
+ Eterm local_heap[4 + 7];
+ hp = local_heap;
+#else
+ ErlHeapFragment *bp;
+ Uint hsz;
+
+ hsz = 4 + 7;
+
+ bp = new_message_buffer(hsz);
+ hp = bp->mem;
+#endif
+
+ erts_smp_mtx_lock(&smq_mtx);
+
+ switch (state) {
+ case am_active:
+ active_sched++;
+ break;
+ case am_inactive:
+ active_sched--;
+ break;
+ default:
+ ASSERT(!"Invalid state");
+ break;
+ }
+
+ GET_NOW(&Ms, &s, &us);
+ timestamp = TUPLE3(hp, make_small(Ms), make_small(s), make_small(us)); hp += 4;
+ msg = TUPLE6(hp, am_profile, am_scheduler, scheduler_id, state,
+ make_small(active_sched), timestamp); hp += 7;
+
+#ifndef ERTS_SMP
+ profile_send(msg);
+#else
+ enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, NIL, NIL, msg, bp);
+#endif
+ erts_smp_mtx_unlock(&smq_mtx);
+
+}
+
+void
+profile_scheduler_q(Eterm scheduler_id, Eterm state, Eterm no_schedulers, Uint Ms, Uint s, Uint us) {
+ Eterm *hp, msg, timestamp;
+
+#ifndef ERTS_SMP
+ Eterm local_heap[4 + 7];
+ hp = local_heap;
+#else
+ ErlHeapFragment *bp;
+ Uint hsz;
+
+ hsz = 4 + 7;
+
+ bp = new_message_buffer(hsz);
+ hp = bp->mem;
+#endif
+
+ erts_smp_mtx_lock(&smq_mtx);
+
+ timestamp = TUPLE3(hp, make_small(Ms), make_small(s), make_small(us)); hp += 4;
+ msg = TUPLE6(hp, am_profile, am_scheduler, scheduler_id, state, no_schedulers, timestamp); hp += 7;
+#ifndef ERTS_SMP
+ profile_send(msg);
+#else
+ enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, NIL, NIL, msg, bp);
+#endif
+ erts_smp_mtx_unlock(&smq_mtx);
+
+}
+
+
+/* Send {trace_ts, Pid, What, {Mod, Func, Arity}, Timestamp}
+ * or {trace, Pid, What, {Mod, Func, Arity}}
+ *
+ * where 'What' is supposed to be 'in' or 'out'.
+ *
+ * Virtual scheduling do not fake scheduling for ports.
+ */
+
+
+void trace_virtual_sched(Process *p, Eterm what)
+{
+ trace_sched_aux(p, what, 1);
+}
+
+/* Port profiling */
+
+void
+trace_port_open(Port *p, Eterm calling_pid, Eterm drv_name) {
+ Eterm mess;
+ Eterm* hp;
+
+ if (is_internal_port(p->tracer_proc)) {
+ Eterm local_heap[5+6];
+ hp = local_heap;
+
+ mess = TUPLE5(hp, am_trace, calling_pid, am_open, p->id, drv_name);
+ hp += 6;
+ erts_smp_mtx_lock(&smq_mtx);
+ if (p->trace_flags & F_TIMESTAMP) {
+ hp = patch_ts(mess, hp);
+ }
+ /* No fake schedule */
+ send_to_port(NULL, mess, &p->tracer_proc, &p->trace_flags);
+ erts_smp_mtx_unlock(&smq_mtx);
+ } else {
+ ErlHeapFragment *bp;
+ ErlOffHeap *off_heap;
+ size_t sz_data;
+ ERTS_TRACER_REF_TYPE tracer_ref;
+
+ ASSERT(is_internal_pid(p->tracer_proc)
+ && internal_pid_index(p->tracer_proc) < erts_max_processes);
+
+ sz_data = 6 + TS_SIZE(p);
+
+ ERTS_GET_TRACER_REF(tracer_ref, p->tracer_proc, p->trace_flags);
+
+ hp = ERTS_ALLOC_SYSMSG_HEAP(sz_data, &bp, &off_heap, tracer_ref);
+
+ mess = TUPLE5(hp, am_trace, calling_pid, am_open, p->id, drv_name);
+ hp += 6;
+
+ erts_smp_mtx_lock(&smq_mtx);
+
+ if (p->trace_flags & F_TIMESTAMP) {
+ hp = patch_ts(mess, hp);
+ }
+
+ ERTS_ENQ_TRACE_MSG(p->id, tracer_ref, mess, bp);
+ erts_smp_mtx_unlock(&smq_mtx);
+ }
+
+}
+
+/* Sends trace message:
+ * {trace_ts, PortPid, What, Data, Timestamp}
+ * or {trace, PortPid, What, Data}
+ *
+ * 'what' must be atomic, 'data' must be atomic.
+ * 't_p' is the traced port.
+ */
+void
+trace_port(Port *t_p, Eterm what, Eterm data) {
+ Eterm mess;
+ Eterm* hp;
+
+ if (is_internal_port(t_p->tracer_proc)) {
+ Eterm local_heap[5+5];
+ hp = local_heap;
+ mess = TUPLE4(hp, am_trace, t_p->id, what, data);
+ hp += 5;
+ erts_smp_mtx_lock(&smq_mtx);
+ if (t_p->trace_flags & F_TIMESTAMP) {
+ hp = patch_ts(mess, hp);
+ }
+ /* No fake schedule */
+ send_to_port(NULL, mess, &t_p->tracer_proc, &t_p->trace_flags);
+ erts_smp_mtx_unlock(&smq_mtx);
+ } else {
+ ErlHeapFragment *bp;
+ ErlOffHeap *off_heap;
+ size_t sz_data;
+ ERTS_TRACER_REF_TYPE tracer_ref;
+
+ ASSERT(is_internal_pid(t_p->tracer_proc)
+ && internal_pid_index(t_p->tracer_proc) < erts_max_processes);
+
+ sz_data = 5 + TS_SIZE(t_p);
+
+ ERTS_GET_TRACER_REF(tracer_ref, t_p->tracer_proc, t_p->trace_flags);
+
+ hp = ERTS_ALLOC_SYSMSG_HEAP(sz_data, &bp, &off_heap, tracer_ref);
+
+ mess = TUPLE4(hp, am_trace, t_p->id, what, data);
+ hp += 5;
+
+ erts_smp_mtx_lock(&smq_mtx);
+
+ if (t_p->trace_flags & F_TIMESTAMP) {
+ hp = patch_ts(mess, hp);
+ }
+
+ ERTS_ENQ_TRACE_MSG(t_p->id, tracer_ref, mess, bp);
+ erts_smp_mtx_unlock(&smq_mtx);
+ }
+}
+
+/* Send {trace_ts, Pid, What, {Mod, Func, Arity}, Timestamp}
+ * or {trace, Pid, What, {Mod, Func, Arity}}
+ *
+ * where 'What' is supposed to be 'in' or 'out' and
+ * where 'where' is supposed to be location (callback)
+ * for the port.
+ */
+
+void
+trace_sched_ports(Port *p, Eterm what) {
+ trace_sched_ports_where(p,what, make_small(0));
+}
+
+void
+trace_sched_ports_where(Port *p, Eterm what, Eterm where) {
+ Eterm mess;
+ Eterm* hp;
+ int ws = 5;
+ Eterm sched_id = am_undefined;
+
+ if (is_internal_port(p->tracer_proc)) {
+ Eterm local_heap[5+6];
+ hp = local_heap;
+
+ if (IS_TRACED_FL(p, F_TRACE_SCHED_NO)) {
+#ifdef ERTS_SMP
+ ErtsSchedulerData *esd = erts_get_scheduler_data();
+ if (esd) sched_id = make_small(esd->no);
+ else sched_id = am_undefined;
+#else
+ sched_id = make_small(1);
+#endif
+ mess = TUPLE5(hp, am_trace, p->id, what, sched_id, where);
+ ws = 6;
+ } else {
+ mess = TUPLE4(hp, am_trace, p->id, what, where);
+ ws = 5;
+ }
+ hp += ws;
+
+ erts_smp_mtx_lock(&smq_mtx);
+ if (p->trace_flags & F_TIMESTAMP) {
+ hp = patch_ts(mess, hp);
+ }
+
+ /* No fake scheduling */
+ send_to_port(NULL, mess, &p->tracer_proc, &p->trace_flags);
+ erts_smp_mtx_unlock(&smq_mtx);
+ } else {
+ ErlHeapFragment *bp;
+ ErlOffHeap *off_heap;
+ ERTS_TRACER_REF_TYPE tracer_ref;
+
+ ASSERT(is_internal_pid(p->tracer_proc)
+ && internal_pid_index(p->tracer_proc) < erts_max_processes);
+
+ if (IS_TRACED_FL(p, F_TRACE_SCHED_NO)) ws = 6; /* Make place for scheduler id */
+
+ ERTS_GET_TRACER_REF(tracer_ref, p->tracer_proc, p->trace_flags);
+
+ hp = ERTS_ALLOC_SYSMSG_HEAP(ws+TS_SIZE(p), &bp, &off_heap, tracer_ref);
+
+ if (IS_TRACED_FL(p, F_TRACE_SCHED_NO)) {
+#ifdef ERTS_SMP
+ ErtsSchedulerData *esd = erts_get_scheduler_data();
+ if (esd) sched_id = make_small(esd->no);
+ else sched_id = am_undefined;
+#else
+ sched_id = make_small(1);
+#endif
+ mess = TUPLE5(hp, am_trace, p->id, what, sched_id, where);
+ } else {
+ mess = TUPLE4(hp, am_trace, p->id, what, where);
+ }
+ hp += ws;
+
+ erts_smp_mtx_lock(&smq_mtx);
+
+ if (p->trace_flags & F_TIMESTAMP) {
+ hp = patch_ts(mess, hp);
+ }
+
+ ERTS_ENQ_TRACE_MSG(p->id, tracer_ref, mess, bp);
+ erts_smp_mtx_unlock(&smq_mtx);
+ }
+}
+
+/* Port profiling */
+
+void
+profile_runnable_port(Port *p, Eterm status) {
+ Uint Ms, s, us;
+ Eterm *hp, msg, timestamp;
+
+ Eterm count = make_small(0);
+
+#ifndef ERTS_SMP
+ Eterm local_heap[4 + 6];
+ hp = local_heap;
+
+#else
+ ErlHeapFragment *bp;
+ Uint hsz;
+
+ hsz = 4 + 6;
+
+ bp = new_message_buffer(hsz);
+ hp = bp->mem;
+#endif
+
+ erts_smp_mtx_lock(&smq_mtx);
+
+ GET_NOW(&Ms, &s, &us);
+ timestamp = TUPLE3(hp, make_small(Ms), make_small(s), make_small(us)); hp += 4;
+ msg = TUPLE5(hp, am_profile, p->id, status, count, timestamp); hp += 6;
+
+#ifndef ERTS_SMP
+ profile_send(msg);
+#else
+ enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, NIL, NIL, msg, bp);
+#endif
+ erts_smp_mtx_unlock(&smq_mtx);
+}
+
+/* Process profiling */
+void
+profile_runnable_proc(Process *p, Eterm status){
+ Uint Ms, s, us;
+ Eterm *hp, msg, timestamp;
+ Eterm where = am_undefined;
+
+#ifndef ERTS_SMP
+ Eterm local_heap[4 + 6 + 4];
+ hp = local_heap;
+#else
+ ErlHeapFragment *bp;
+ Uint hsz = 4 + 6 + 4;
+#endif
+
+ if (!p->current) {
+ p->current = find_function_from_pc(p->i);
+ }
+
+#ifdef ERTS_SMP
+ if (!p->current) {
+ hsz = 4 + 6;
+ }
+
+ bp = new_message_buffer(hsz);
+ hp = bp->mem;
+#endif
+
+ if (p->current) {
+ where = TUPLE3(hp, p->current[0], p->current[1], make_small(p->current[2])); hp += 4;
+ } else {
+ where = make_small(0);
+ }
+
+ erts_smp_mtx_lock(&smq_mtx);
+
+ GET_NOW(&Ms, &s, &us);
+ timestamp = TUPLE3(hp, make_small(Ms), make_small(s), make_small(us)); hp += 4;
+ msg = TUPLE5(hp, am_profile, p->id, status, where, timestamp); hp += 6;
+#ifndef ERTS_SMP
+ profile_send(msg);
+#else
+ enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, NIL, NIL, msg, bp);
+#endif
+ erts_smp_mtx_unlock(&smq_mtx);
+}
+/* End system_profile tracing */
+
+
+
+#ifdef ERTS_SMP
+
+void
+erts_check_my_tracer_proc(Process *p)
+{
+ if (is_internal_pid(p->tracer_proc)) {
+ Process *tracer = erts_pid2proc(p, ERTS_PROC_LOCK_MAIN,
+ p->tracer_proc, ERTS_PROC_LOCK_STATUS);
+ int invalid_tracer = !tracer || !(tracer->trace_flags & F_TRACER);
+ if (tracer)
+ erts_smp_proc_unlock(tracer, ERTS_PROC_LOCK_STATUS);
+ if (invalid_tracer) {
+ erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR);
+ p->trace_flags &= ~TRACEE_FLAGS;
+ p->tracer_proc = NIL;
+ erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR);
+ }
+ }
+}
+
+
+typedef struct ErtsSysMsgQ_ ErtsSysMsgQ;
+struct ErtsSysMsgQ_ {
+ ErtsSysMsgQ *next;
+ enum ErtsSysMsgType type;
+ Eterm from;
+ Eterm to;
+ Eterm msg;
+ ErlHeapFragment *bp;
+};
+
+static ErtsSysMsgQ *sys_message_queue;
+static ErtsSysMsgQ *sys_message_queue_end;
+
+static erts_tid_t sys_msg_dispatcher_tid;
+static erts_cnd_t smq_cnd;
+
+static int dispatcher_waiting;
+
+ERTS_QUALLOC_IMPL(smq_element, ErtsSysMsgQ, 20, ERTS_ALC_T_SYS_MSG_Q)
+
+static void
+enqueue_sys_msg_unlocked(enum ErtsSysMsgType type,
+ Eterm from,
+ Eterm to,
+ Eterm msg,
+ ErlHeapFragment *bp)
+{
+ ErtsSysMsgQ *smqp;
+
+ smqp = smq_element_alloc();
+ smqp->next = NULL;
+ smqp->type = type;
+ smqp->from = from;
+ smqp->to = to;
+ smqp->msg = msg;
+ smqp->bp = bp;
+
+ if (sys_message_queue_end) {
+ ASSERT(sys_message_queue);
+ sys_message_queue_end->next = smqp;
+ }
+ else {
+ ASSERT(!sys_message_queue);
+ sys_message_queue = smqp;
+ }
+ sys_message_queue_end = smqp;
+ erts_smp_cnd_signal(&smq_cnd);
+}
+
+static void
+enqueue_sys_msg(enum ErtsSysMsgType type,
+ Eterm from,
+ Eterm to,
+ Eterm msg,
+ ErlHeapFragment *bp)
+{
+ erts_smp_mtx_lock(&smq_mtx);
+ enqueue_sys_msg_unlocked(type, from, to, msg, bp);
+ erts_smp_mtx_unlock(&smq_mtx);
+}
+
+static void
+prepare_for_block(void *unused)
+{
+ erts_smp_mtx_unlock(&smq_mtx);
+}
+
+static void
+resume_after_block(void *unused)
+{
+ erts_smp_mtx_lock(&smq_mtx);
+}
+
+void
+erts_queue_error_logger_message(Eterm from, Eterm msg, ErlHeapFragment *bp)
+{
+ enqueue_sys_msg(SYS_MSG_TYPE_ERRLGR, from, am_error_logger, msg, bp);
+}
+
+void
+erts_send_sys_msg_proc(Eterm from, Eterm to, Eterm msg, ErlHeapFragment *bp)
+{
+ ASSERT(is_internal_pid(to));
+ enqueue_sys_msg(SYS_MSG_TYPE_PROC_MSG, from, to, msg, bp);
+}
+
+#ifdef DEBUG_PRINTOUTS
+static void
+print_msg_type(ErtsSysMsgQ *smqp)
+{
+ switch (smqp->type) {
+ case SYS_MSG_TYPE_TRACE:
+ erts_fprintf(stderr, "TRACE ");
+ break;
+ case SYS_MSG_TYPE_SEQTRACE:
+ erts_fprintf(stderr, "SEQTRACE ");
+ break;
+ case SYS_MSG_TYPE_SYSMON:
+ erts_fprintf(stderr, "SYSMON ");
+ break;
+ case SYS_MSG_TYPE_SYSPROF:
+ erts_fprintf(stderr, "SYSPROF ");
+ break;
+ case SYS_MSG_TYPE_ERRLGR:
+ erts_fprintf(stderr, "ERRLGR ");
+ break;
+ case SYS_MSG_TYPE_PROC_MSG:
+ erts_fprintf(stderr, "PROC_MSG ");
+ break;
+ default:
+ erts_fprintf(stderr, "??? ");
+ break;
+ }
+}
+#endif
+
+static void
+sys_msg_disp_failure(ErtsSysMsgQ *smqp, Eterm receiver)
+{
+ switch (smqp->type) {
+ case SYS_MSG_TYPE_TRACE:
+ /* Invalid tracer_proc's are removed when processes
+ are scheduled in. */
+ break;
+ case SYS_MSG_TYPE_SEQTRACE:
+ /* Reset seq_tracer if it hasn't changed */
+ erts_smp_mtx_lock(&sys_trace_mtx);
+ if (system_seq_tracer == receiver)
+ system_seq_tracer = am_false;
+ erts_smp_mtx_unlock(&sys_trace_mtx);
+ break;
+ case SYS_MSG_TYPE_SYSMON:
+ if (receiver == NIL
+ && !erts_system_monitor_long_gc
+ && !erts_system_monitor_large_heap
+ && !erts_system_monitor_flags.busy_port
+ && !erts_system_monitor_flags.busy_dist_port)
+ break; /* Everything is disabled */
+ erts_smp_block_system(ERTS_BS_FLG_ALLOW_GC);
+ if (system_monitor == receiver || receiver == NIL)
+ erts_system_monitor_clear(NULL);
+ erts_smp_release_system();
+ break;
+ case SYS_MSG_TYPE_SYSPROF:
+ if (receiver == NIL
+ && !erts_system_profile_flags.runnable_procs
+ && !erts_system_profile_flags.runnable_ports
+ && !erts_system_profile_flags.exclusive
+ && !erts_system_profile_flags.scheduler)
+ break;
+ /* Block system to clear flags */
+ erts_smp_block_system(0);
+ if (system_profile == receiver || receiver == NIL) {
+ erts_system_profile_clear(NULL);
+ }
+ erts_smp_release_system();
+ break;
+ case SYS_MSG_TYPE_ERRLGR: {
+ char *no_elgger = "(no error logger present)";
+ Eterm *tp;
+ Eterm tag;
+ if (is_not_tuple(smqp->msg)) {
+ unexpected_elmsg:
+ erts_fprintf(stderr,
+ "%s unexpected error logger message: %T\n",
+ no_elgger,
+ smqp->msg);
+ }
+
+ tp = tuple_val(smqp->msg);
+ if (arityval(tp[0]) != 2)
+ goto unexpected_elmsg;
+ if (is_not_tuple(tp[2]))
+ goto unexpected_elmsg;
+ tp = tuple_val(tp[2]);
+ if (arityval(tp[0]) != 3)
+ goto unexpected_elmsg;
+ tag = tp[1];
+ if (is_not_tuple(tp[3]))
+ goto unexpected_elmsg;
+ tp = tuple_val(tp[3]);
+ if (arityval(tp[0]) != 3)
+ goto unexpected_elmsg;
+ if (is_not_list(tp[3]))
+ goto unexpected_elmsg;
+ erts_fprintf(stderr, "%s %T: %T\n",
+ no_elgger, tag, CAR(list_val(tp[3])));
+ break;
+ }
+ case SYS_MSG_TYPE_PROC_MSG:
+ break;
+ default:
+ ASSERT(0);
+ }
+}
+
+static void *
+sys_msg_dispatcher_func(void *unused)
+{
+ ErtsSysMsgQ *local_sys_message_queue = NULL;
+
+#ifdef ERTS_ENABLE_LOCK_CHECK
+ erts_lc_set_thread_name("system message dispatcher");
+#endif
+
+ erts_register_blockable_thread();
+ erts_smp_activity_begin(ERTS_ACTIVITY_IO, NULL, NULL, NULL);
+
+ while (1) {
+ ErtsSysMsgQ *smqp;
+
+ ERTS_SMP_LC_ASSERT(!ERTS_LC_IS_BLOCKING);
+
+ erts_smp_mtx_lock(&smq_mtx);
+
+ /* Free previously used queue ... */
+ while (local_sys_message_queue) {
+ smqp = local_sys_message_queue;
+ local_sys_message_queue = smqp->next;
+ smq_element_free(smqp);
+ }
+
+ /* Fetch current trace message queue ... */
+ erts_smp_activity_change(ERTS_ACTIVITY_IO,
+ ERTS_ACTIVITY_WAIT,
+ prepare_for_block,
+ resume_after_block,
+ NULL);
+ dispatcher_waiting = 1;
+ while (!sys_message_queue)
+ erts_smp_cnd_wait(&smq_cnd, &smq_mtx);
+ dispatcher_waiting = 0;
+ erts_smp_activity_change(ERTS_ACTIVITY_WAIT,
+ ERTS_ACTIVITY_IO,
+ prepare_for_block,
+ resume_after_block,
+ NULL);
+
+ local_sys_message_queue = sys_message_queue;
+ sys_message_queue = NULL;
+ sys_message_queue_end = NULL;
+
+ erts_smp_mtx_unlock(&smq_mtx);
+
+ /* Send trace messages ... */
+
+ ASSERT(local_sys_message_queue);
+
+ for (smqp = local_sys_message_queue; smqp; smqp = smqp->next) {
+ Eterm receiver;
+ ErtsProcLocks proc_locks = ERTS_PROC_LOCKS_MSG_SEND;
+ Process *proc = NULL;
+ Port *port = NULL;
+
+#ifdef DEBUG_PRINTOUTS
+ print_msg_type(smqp);
+#endif
+ switch (smqp->type) {
+ case SYS_MSG_TYPE_TRACE:
+ case SYS_MSG_TYPE_PROC_MSG:
+ receiver = smqp->to;
+ break;
+ case SYS_MSG_TYPE_SEQTRACE:
+ receiver = erts_get_system_seq_tracer();
+ break;
+ case SYS_MSG_TYPE_SYSMON:
+ receiver = erts_get_system_monitor();
+ if (smqp->from == receiver) {
+#ifdef DEBUG_PRINTOUTS
+ erts_fprintf(stderr, "MSG=%T to %T... ",
+ smqp->msg, receiver);
+#endif
+ goto drop_sys_msg;
+ }
+ break;
+ case SYS_MSG_TYPE_SYSPROF:
+ receiver = erts_get_system_profile();
+ if (smqp->from == receiver) {
+#ifdef DEBUG_PRINTOUTS
+ erts_fprintf(stderr, "MSG=%T to %T... ",
+ smqp->msg, receiver);
+#endif
+ goto drop_sys_msg;
+ }
+ break;
+ case SYS_MSG_TYPE_ERRLGR:
+ receiver = am_error_logger;
+ break;
+ default:
+ receiver = NIL;
+ break;
+ }
+
+#ifdef DEBUG_PRINTOUTS
+ erts_fprintf(stderr, "MSG=%T to %T... ", smqp->msg, receiver);
+#endif
+
+ if (is_internal_pid(receiver)) {
+ proc = erts_pid2proc(NULL, 0, receiver, proc_locks);
+ if (!proc
+ || (smqp->type == SYS_MSG_TYPE_TRACE
+ && !(proc->trace_flags & F_TRACER))) {
+ /* Bad tracer */
+#ifdef DEBUG_PRINTOUTS
+ if (smqp->type == SYS_MSG_TYPE_TRACE && proc)
+ erts_fprintf(stderr,
+ "<tracer alive but missing "
+ "F_TRACER flag> ");
+#endif
+ goto failure;
+ }
+ else {
+ queue_proc_msg:
+ erts_queue_message(proc,&proc_locks,smqp->bp,smqp->msg,NIL);
+#ifdef DEBUG_PRINTOUTS
+ erts_fprintf(stderr, "delivered\n");
+#endif
+ erts_smp_proc_unlock(proc, proc_locks);
+ }
+ }
+ else if (receiver == am_error_logger) {
+ proc = erts_whereis_process(NULL,0,receiver,proc_locks,0);
+ if (!proc)
+ goto failure;
+ else if (smqp->from == proc->id)
+ goto drop_sys_msg;
+ else
+ goto queue_proc_msg;
+ }
+ else if (is_internal_port(receiver)) {
+ port = erts_id2port(receiver, NULL, 0);
+ if (INVALID_TRACER_PORT(port, receiver)) {
+ if (port)
+ erts_port_release(port);
+ goto failure;
+ }
+ else {
+ write_sys_msg_to_port(receiver,
+ port,
+ smqp->from,
+ smqp->type,
+ smqp->msg);
+ if (port->control_flags & PORT_CONTROL_FLAG_HEAVY)
+ port->control_flags &= ~PORT_CONTROL_FLAG_HEAVY;
+#ifdef DEBUG_PRINTOUTS
+ erts_fprintf(stderr, "delivered\n");
+#endif
+ erts_port_release(port);
+ if (smqp->bp)
+ free_message_buffer(smqp->bp);
+ }
+ }
+ else {
+ failure:
+ sys_msg_disp_failure(smqp, receiver);
+ drop_sys_msg:
+ if (proc)
+ erts_smp_proc_unlock(proc, proc_locks);
+ if (smqp->bp)
+ free_message_buffer(smqp->bp);
+#ifdef DEBUG_PRINTOUTS
+ erts_fprintf(stderr, "dropped\n");
+#endif
+ }
+ }
+ }
+
+ erts_smp_activity_end(ERTS_ACTIVITY_IO, NULL, NULL, NULL);
+ return NULL;
+}
+
+void
+erts_foreach_sys_msg_in_q(void (*func)(Eterm,
+ Eterm,
+ Eterm,
+ ErlHeapFragment *))
+{
+ ErtsSysMsgQ *sm;
+ erts_smp_mtx_lock(&smq_mtx);
+ for (sm = sys_message_queue; sm; sm = sm->next) {
+ Eterm to;
+ switch (sm->type) {
+ case SYS_MSG_TYPE_TRACE:
+ to = sm->to;
+ break;
+ case SYS_MSG_TYPE_SEQTRACE:
+ to = erts_get_system_seq_tracer();
+ break;
+ case SYS_MSG_TYPE_SYSMON:
+ to = erts_get_system_monitor();
+ break;
+ case SYS_MSG_TYPE_SYSPROF:
+ to = erts_get_system_profile();
+ break;
+ case SYS_MSG_TYPE_ERRLGR:
+ to = am_error_logger;
+ break;
+ default:
+ to = NIL;
+ break;
+ }
+ (*func)(sm->from, to, sm->msg, sm->bp);
+ }
+ erts_smp_mtx_unlock(&smq_mtx);
+}
+
+
+static void
+init_sys_msg_dispatcher(void)
+{
+ erts_smp_thr_opts_t thr_opts = ERTS_SMP_THR_OPTS_DEFAULT_INITER;
+ thr_opts.detached = 1;
+ init_smq_element_alloc();
+ sys_message_queue = NULL;
+ sys_message_queue_end = NULL;
+ erts_smp_cnd_init(&smq_cnd);
+ erts_smp_mtx_init(&smq_mtx, "sys_msg_q");
+ dispatcher_waiting = 0;
+ erts_smp_thr_create(&sys_msg_dispatcher_tid,
+ sys_msg_dispatcher_func,
+ NULL,
+ &thr_opts);
+}
+
+#endif