erts: Implement tracer modules

Add the possibility to use modules as trace data receivers. The functions
in the module have to be nifs as otherwise complex trace probes will be
very hard to handle (complex means trace probes for ports for example).

This commit changes the way that the ptab->tracer field works from always
being an immediate, to now be NIL if no tracer is present or else be
the tuple {TracerModule, TracerState} where TracerModule is an atom that
is later used to lookup the appropriate tracer callbacks to call and
TracerState is just passed to the tracer callback. The default process and
port tracers have been rewritten to use the new API.

This commit also changes the order which trace messages are delivered to the
potential tracer process. Any enif_send done in a tracer module may be delayed
indefinitely because of lock order issues. If a message is delayed any other
trace message send from that process is also delayed so that order is preserved
for each traced entity. This means that for some trace events (i.e. send/receive)
the events may come in an unintuitive order (receive before send) to the
trace receiver. Timestamps are taken when the trace message is generated so
trace messages from differented processes may arrive with the timestamp
out of order.

Both the erlang:trace and seq_trace:set_system_tracer accept the new tracer
module tracers and also the backwards compatible arguments.

OTP-10267

6 files changed, 45 insertions, 69 deletions

diff --git a/lib/kernel/doc/src/seq_trace.xml b/lib/kernel/doc/src/seq_trace.xml
index 1feb3fcb07..5ac199b6a7 100644
--- a/lib/kernel/doc/src/seq_trace.xml
+++ b/lib/kernel/doc/src/seq_trace.xml
@@ -213,9 +213,10 @@ seq_trace:set_token(OldToken), % activate the trace token again
       <type name="tracer"/>
       <desc>
         <p>Sets the system tracer. The system tracer can be either a
-          process or port denoted by <c><anno>Tracer</anno></c>. Returns the previous
-          value (which can be <c>false</c> if no system tracer is
-          active).</p>
+          process, port or <seealso marker="erts:erl_tracer">tracer module</seealso>
+          denoted by <c><anno>Tracer</anno></c>.
+          Returns the previous value (which can be <c>false</c> if no system
+          tracer is active).</p>
         <p>Failure: <c>{badarg, Info}}</c> if <c><anno>Pid</anno></c> is not an
           existing local pid.</p>
       </desc>
@@ -225,7 +226,7 @@ seq_trace:set_token(OldToken), % activate the trace token again
       <fsummary>Return the pid() or port() of the current system tracer.</fsummary>
       <type name="tracer"/>
       <desc>
-        <p>Returns the pid or port identifier of the current system
+        <p>Returns the pid, port identifier or tracer module of the current system
           tracer or <c>false</c> if no system tracer is activated.</p>
       </desc>
     </func>
@@ -298,7 +299,7 @@ TimeStamp = {Seconds, Milliseconds, Microseconds}
       matches a message in a receive statement, according to the trace
       token carried by the received message, empty or not.</p>
     <p>On each Erlang node, a process can be set as the <em>system tracer</em>.
-      This process receives trace messages each time
+      This process will receive trace messages each time
       a message with a trace token is sent or received (if the trace
       token flag <c>send</c> or <c>'receive'</c> is set). The system
       tracer can then print each trace event, write it to a file, or
diff --git a/lib/kernel/src/seq_trace.erl b/lib/kernel/src/seq_trace.erl
index 01ac024c88..cc0c10909b 100644
--- a/lib/kernel/src/seq_trace.erl
+++ b/lib/kernel/src/seq_trace.erl
@@ -106,14 +106,24 @@ reset_trace() ->
 
 %% reset_trace(Pid) -> % this might be a useful function too
 
--type tracer() :: (Pid :: pid()) | port() | 'false'.
+-type tracer() :: (Pid :: pid()) | port() |
+                  (TracerModule :: {module(), term()}) |
+                  'false'.
 
 -spec set_system_tracer(Tracer) -> OldTracer when
       Tracer :: tracer(),
       OldTracer :: tracer().
 
-set_system_tracer(Pid) ->
-    erlang:system_flag(sequential_tracer, Pid).
+set_system_tracer({Module, State} = Tracer) ->
+    case erlang:module_loaded(Module) of
+        false ->
+            Module:enabled(trace_status, erlang:self(), State);
+        true ->
+            ok
+    end,
+    erlang:system_flag(sequential_tracer, Tracer);
+set_system_tracer(Tracer) ->
+    erlang:system_flag(sequential_tracer, Tracer).
 
 -spec get_system_tracer() -> Tracer when
       Tracer :: tracer().
diff --git a/lib/kernel/test/seq_trace_SUITE.erl b/lib/kernel/test/seq_trace_SUITE.erl
index 0a03503163..000994751f 100644
--- a/lib/kernel/test/seq_trace_SUITE.erl
+++ b/lib/kernel/test/seq_trace_SUITE.erl
@@ -787,7 +787,6 @@ start_tracer() ->
     seq_trace:set_system_tracer(Pid),
     Pid.
 
-
 set_token_flags([]) ->
     ok;
 set_token_flags([no_timestamp|Flags]) ->
@@ -836,7 +835,7 @@ check_ts(strict_monotonic_timestamp, Ts) ->
 	    ct:fail({unexpected_timestamp, Ts})
     end,
     ok.
-	
+
 start_node(Name, Param) ->
     test_server:start_node(Name, slave, [{args, Param}]).
 
diff --git a/lib/runtime_tools/c_src/Makefile.in b/lib/runtime_tools/c_src/Makefile.in
index 58c12e0d88..70b48daf97 100644
--- a/lib/runtime_tools/c_src/Makefile.in
+++ b/lib/runtime_tools/c_src/Makefile.in
@@ -33,21 +33,18 @@ VSN=$(RUNTIME_TOOLS_VSN)
 # be set for that system only.
 # ----------------------------------------------------
 CC = $(DED_CC)
-CFLAGS = $(DED_CFLAGS)
+CFLAGS = $(DED_CFLAGS) -I./
 LD = $(DED_LD)
 SHELL = /bin/sh
 LIBS = $(DED_LIBS)
 LDFLAGS += $(DED_LDFLAGS)
 
-DTRACE_LIBNAME = dyntrace
+TRACE_LIBNAME = dyntrace trace_file_drv trace_ip_drv
 
 SYSINCLUDE = $(DED_SYS_INCLUDE)
 
 TRACE_DRV_INCLUDES = $(SYSINCLUDE)
 
-ALL_CFLAGS = $(CFLAGS) @DEFS@ $(TYPE_FLAGS) $(TRACE_DRV_INCLUDES) \
-	-I$(OBJDIR) -I$(ERL_TOP)/erts/emulator/$(TARGET)
-
 ifeq ($(TYPE),debug)
 TYPEMARKER = .debug
 TYPE_FLAGS = $(subst -O3,,$(subst -O2,,$(CFLAGS))) -DDEBUG @DEBUG_FLAGS@
@@ -61,6 +58,9 @@ TYPE_FLAGS = $(CFLAGS)
 endif
 endif
 
+ALL_CFLAGS = @DEFS@ $(TYPE_FLAGS) $(TRACE_DRV_INCLUDES) \
+	-I$(OBJDIR) -I$(ERL_TOP)/erts/emulator/$(TARGET)
+
 ROOTDIR = $(ERL_TOP)/lib
 PRIVDIR = ../priv
 LIBDIR = $(PRIVDIR)/lib/$(TARGET)
@@ -74,37 +74,16 @@ RELSYSDIR = $(RELEASE_PATH)/lib/runtime_tools-$(VSN)
 # ----------------------------------------------------
 # Misc Macros
 # ----------------------------------------------------
-before_DTrace_OBJS = $(OBJDIR)/dyntrace$(TYPEMARKER).o
-## NIF_MAKEFILE = $(PRIVDIR)/Makefile
 
-# Higher-level makefiles says that we can only compile on UNIX flavors
-NIF_LIB = $(LIBDIR)/dyntrace$(TYPEMARKER).@DED_EXT@
+TRACE_LIBS = $(foreach LIB, $(TRACE_LIBNAME), $(LIBDIR)/$(LIB)$(TYPEMARKER).@DED_EXT@)
 
-ifeq ($(HOST_OS),)
-HOST_OS := $(shell $(ERL_TOP)/erts/autoconf/config.guess)
-endif
-
-TRACE_IP_DRV_OBJS = \
-	$(OBJDIR)/trace_ip_drv.o
-
-TRACE_FILE_DRV_OBJS = \
-	$(OBJDIR)/trace_file_drv.o
-
-ifeq ($(findstring win32,$(TARGET)), win32)
-SOLIBS = $(LIBDIR)/trace_ip_drv.dll $(LIBDIR)/trace_file_drv.dll
-LN=cp
-else
-SOLIBS = $(LIBDIR)/trace_ip_drv.so $(LIBDIR)/trace_file_drv.so
-endif
 # ----------------------------------------------------
 # Targets
 # ----------------------------------------------------
 
 _create_dirs := $(shell mkdir -p $(OBJDIR) $(LIBDIR))
 
-debug opt valgrind: $(SOLIBS) $(OBJDIR) $(LIBDIR) $(NIF_LIB)
-
-DYNTRACE_OBJS = $(before_DTrace_OBJS)
+debug opt valgrind: $(SOLIBS) $(OBJDIR) $(LIBDIR) $(TRACE_LIBS)
 
 $(OBJDIR):
 	-@mkdir -p $(OBJDIR)
@@ -112,50 +91,26 @@ $(OBJDIR):
 $(LIBDIR):
 	-@mkdir -p $(LIBDIR)
 
-$(OBJDIR)/dyntrace$(TYPEMARKER).o: dyntrace.c
-	$(V_at)$(INSTALL_DIR) $(OBJDIR)
-	$(V_CC) -c -o $@ $(ALL_CFLAGS) $<
-
-$(NIF_LIB): $(DYNTRACE_OBJS)
-	$(V_at)$(INSTALL_DIR) $(LIBDIR)
-	$(V_LD) $(LDFLAGS) -o $@ $^ $(LDLIBS)
-
-$(OBJDIR)/%.o: %.c
+$(OBJDIR)/%$(TYPEMARKER).o: %.c
 	$(V_CC) -c -o $@ $(ALL_CFLAGS) $<
 
-$(LIBDIR)/trace_ip_drv.so: $(TRACE_IP_DRV_OBJS)
-	$(V_LD) $(LDFLAGS) -o $@ $^ -lc $(LIBS)
-
-$(LIBDIR)/trace_file_drv.so: $(TRACE_FILE_DRV_OBJS)
-	$(V_LD) $(LDFLAGS) -o $@ $^ -lc $(LIBS)
-
-$(LIBDIR)/trace_ip_drv.dll: $(TRACE_IP_DRV_OBJS)
-	$(V_LD) $(LDFLAGS) -o $@ $^ $(LIBS)
-$(LIBDIR)/trace_file_drv.dll: $(TRACE_FILE_DRV_OBJS)
+$(LIBDIR)/%$(TYPEMARKER).@DED_EXT@: $(OBJDIR)/%$(TYPEMARKER).o
 	$(V_LD) $(LDFLAGS) -o $@ $^ $(LIBS)
 
 clean:
-	rm -f $(SOLIBS) $(TRACE_IP_DRV_OBJS) $(TRACE_FILE_DRV_OBJS)
-	rm -f $(LIBDIR)/dyntrace.@DED_EXT@
-	rm -f $(LIBDIR)/dyntrace.debug.@DED_EXT@
-	rm -f $(LIBDIR)/dyntrace.valgrind.@DED_EXT@
-	rm -f $(OBJDIR)/dyntrace.o
-	rm -f $(OBJDIR)/dyntrace.debug.o
-	rm -f $(OBJDIR)/dyntrace.valgrind.o
+	rm -f $(TRACE_LIBS)
 	rm -f core *~
 
 docs:
 
 # ----------------------------------------------------
 # Release Target
-# ---------------------------------------------------- 
+# ----------------------------------------------------
 include $(ERL_TOP)/make/otp_release_targets.mk
 
 release_spec: opt
-	$(INSTALL_DIR) "$(RELSYSDIR)/priv/obj"
 	$(INSTALL_DIR) "$(RELSYSDIR)/priv/lib"
-	$(INSTALL_PROGRAM) $(DYNTRACE_OBJS) "$(RELSYSDIR)/priv/obj"
-	$(INSTALL_PROGRAM) $(NIF_LIB) $(SOLIBS) "$(RELSYSDIR)/priv/lib"
+	$(INSTALL_PROGRAM) $(TRACE_LIBS) "$(RELSYSDIR)/priv/lib"
 
 release_docs_spec:
 
diff --git a/lib/runtime_tools/c_src/trace_file_drv.c b/lib/runtime_tools/c_src/trace_file_drv.c
index 703766c188..e7fd5968c1 100644
--- a/lib/runtime_tools/c_src/trace_file_drv.c
+++ b/lib/runtime_tools/c_src/trace_file_drv.c
@@ -172,6 +172,7 @@ static ErlDrvData trace_file_start(ErlDrvPort port, char *buff);
 static void trace_file_stop(ErlDrvData handle);
 static void trace_file_output(ErlDrvData handle, char *buff,
 			      ErlDrvSizeT bufflen);
+static void trace_file_outputv(ErlDrvData handle, ErlIOVec *ev);
 static void trace_file_finish(void);
 static ErlDrvSSizeT trace_file_control(ErlDrvData handle,
 				      unsigned int command, 
@@ -214,7 +215,7 @@ ErlDrvEntry trace_file_driver_entry = {
     NULL,                  /* void * that is not used (BC) */
     trace_file_control,    /* F_PTR control, port_control callback */
     trace_file_timeout,    /* F_PTR timeout, driver_set_timer callback */
-    NULL,                  /* F_PTR outputv, reserved */
+    trace_file_outputv,    /* F_PTR outputv, reserved */
     NULL, /* ready_async */
     NULL, /* flush */
     NULL, /* call */
@@ -363,6 +364,16 @@ static void trace_file_stop(ErlDrvData handle)
 /*
 ** Data sent from erlang to port.
 */
+static void trace_file_outputv(ErlDrvData handle, ErlIOVec *ev)
+{
+    int i;
+    for (i = 0; i < ev->vsize; i++) {
+        if (ev->iov[i].iov_len)
+            trace_file_output(handle, ev->iov[i].iov_base,
+                              ev->iov[i].iov_len);
+    }
+}
+
 static void trace_file_output(ErlDrvData handle, char *buff,
 			      ErlDrvSizeT bufflen)
 {
diff --git a/lib/sasl/src/systools_make.erl b/lib/sasl/src/systools_make.erl
index c23607f8a1..352e4984df 100644
--- a/lib/sasl/src/systools_make.erl
+++ b/lib/sasl/src/systools_make.erl
@@ -1490,7 +1490,7 @@ mandatory_modules() ->
 
 preloaded() ->
     %% Sorted
-    [erl_prim_loader,erlang,
+    [erl_prim_loader,erl_tracer,erlang,
      erts_code_purger,
      erts_internal,init,otp_ring0,prim_eval,prim_file,
      prim_inet,prim_zip,zlib].