erts: Implement tracer modules

Add the possibility to use modules as trace data receivers. The functions
in the module have to be nifs as otherwise complex trace probes will be
very hard to handle (complex means trace probes for ports for example).

This commit changes the way that the ptab->tracer field works from always
being an immediate, to now be NIL if no tracer is present or else be
the tuple {TracerModule, TracerState} where TracerModule is an atom that
is later used to lookup the appropriate tracer callbacks to call and
TracerState is just passed to the tracer callback. The default process and
port tracers have been rewritten to use the new API.

This commit also changes the order which trace messages are delivered to the
potential tracer process. Any enif_send done in a tracer module may be delayed
indefinitely because of lock order issues. If a message is delayed any other
trace message send from that process is also delayed so that order is preserved
for each traced entity. This means that for some trace events (i.e. send/receive)
the events may come in an unintuitive order (receive before send) to the
trace receiver. Timestamps are taken when the trace message is generated so
trace messages from differented processes may arrive with the timestamp
out of order.

Both the erlang:trace and seq_trace:set_system_tracer accept the new tracer
module tracers and also the backwards compatible arguments.

OTP-10267

5 files changed, 134 insertions, 10 deletions

diff --git a/erts/preloaded/src/Makefile b/erts/preloaded/src/Makefile
index 1e3de9f1d7..4a447d3a09 100644
--- a/erts/preloaded/src/Makefile
+++ b/erts/preloaded/src/Makefile
@@ -43,7 +43,8 @@ PRE_LOADED_ERL_MODULES = \
 	otp_ring0 \
 	erts_code_purger \
 	erlang \
-	erts_internal
+	erts_internal \
+	erl_tracer
 
 PRE_LOADED_BEAM_MODULES = \
 	prim_eval
diff --git a/erts/preloaded/src/erl_tracer.erl b/erts/preloaded/src/erl_tracer.erl
new file mode 100644
index 0000000000..3415ff3135
--- /dev/null
+++ b/erts/preloaded/src/erl_tracer.erl
@@ -0,0 +1,41 @@
+-module(erl_tracer).
+
+-export([enabled/3, trace/6, on_load/0]).
+
+-type tracee() :: port() | pid() | undefined.
+-type trace_tag() :: send | send_to_non_existing_process | 'receive' |
+                     call | return_to | return_from | exception_from |
+                     spawn | exit | link | unlink | getting_linked |
+                     getting_unlinked | register | unregister | in | out |
+                     gc_start | gc_end.
+-type trace_opts() :: #{ match_spec_result => true | term(),
+                         scheduler_id => undefined | non_neg_integer(),
+                         timestamp => undefined | timestamp | cpu_timestamp |
+                                      monotonic | strict_monotonic }.
+-type tracer_state() :: term().
+
+on_load() ->
+    case erlang:load_nif(atom_to_list(?MODULE), 0) of
+        ok -> ok
+    end.
+
+%%%
+%%% NIF placeholders
+%%%
+
+-spec enabled(Tag :: trace_tag() | seq_trace | trace_status,
+              TracerState :: tracer_state(),
+              Tracee :: tracee()) ->
+   trace | discard | remove.
+enabled(_, _, _) ->
+    erlang:nif_error(nif_not_loaded).
+
+-spec trace(Tag :: trace_tag() | seq_trace,
+            TracerState :: tracer_state(),
+            Tracee :: tracee(),
+            Msg :: term(),
+            Extra :: term(),
+            Opts :: trace_opts()) -> any().
+
+trace(_, _, _, _, _, _) ->
+    erlang:nif_error(nif_not_loaded).
diff --git a/erts/preloaded/src/erlang.erl b/erts/preloaded/src/erlang.erl
index 4374bdcd89..8d6d17b043 100644
--- a/erts/preloaded/src/erlang.erl
+++ b/erts/preloaded/src/erlang.erl
@@ -245,12 +245,14 @@
       set_on_first_spawn |
       set_on_link |
       set_on_first_link |
-      {tracer, pid() | port()}.
+      {tracer, pid() | port()} |
+      {tracer, module(), term()}.
 
 -type trace_info_item_result() ::
        {traced, global | local | false | undefined} |
        {match_spec, trace_match_spec() | false | undefined} |
        {meta, pid() | port() | false | undefined | []} |
+       {meta, module(), term() } |
        {meta_match_spec, trace_match_spec() | false | undefined} |
        {call_count, non_neg_integer() | boolean() | undefined} |
        {call_time, [{pid(), non_neg_integer(),
@@ -276,6 +278,7 @@
       undefined |
       {flags, [trace_info_flag()]} |
       {tracer, pid() | port() | []} |
+      {tracer, module(), term()} |
       trace_info_item_result() |
       {all, [ trace_info_item_result() ] | false | undefined}.
 
@@ -1709,8 +1712,28 @@ time() ->
       PidSpec :: pid() | existing | new | all,
       How :: boolean(),
       FlagList :: [trace_flag()].
-trace(_PidSpec, _How, _FlagList) ->
-    erlang:nif_error(undefined).
+trace(PidSpec, How, FlagList) ->
+    %% Make sure that we have loaded the tracer module
+    case lists:keyfind(tracer, 1, FlagList) of
+        {tracer, Module, State} when erlang:is_atom(Module) ->
+            case erlang:module_loaded(Module) of
+                false ->
+                    Module:enabled(trace_status, erlang:self(), State);
+                true ->
+                    ok
+            end;
+        _ ->
+            ignore
+    end,
+
+    try erts_internal:trace(PidSpec, How, FlagList) of
+        Res -> Res
+    catch E:R ->
+            {_, [_ | CST]} = erlang:process_info(
+                               erlang:self(), current_stacktrace),
+            erlang:raise(
+              E, R, [{?MODULE, trace, [PidSpec, How, FlagList], []} | CST])
+    end.
 
 %% trace_delivered/1
 -spec erlang:trace_delivered(Tracee) -> Ref when
@@ -2319,7 +2342,7 @@ subtract(_,_) ->
       OldState :: preliminary | final | volatile;
                         %% These are deliberately not documented
 			(internal_cpu_topology, term()) -> term();
-                        (sequential_tracer, pid() | port() | false) -> pid() | port() | false;
+                        (sequential_tracer, pid() | port() | {module(), term()} | false) -> pid() | port() | false;
                         (1,0) -> true.
 
 system_flag(_Flag, _Value) ->
@@ -2355,12 +2378,20 @@ tl(_List) ->
                  | boolean()
                  | restart
                  | pause.
-trace_pattern(_MFA, _MatchSpec) ->
-    erlang:nif_error(undefined).
+trace_pattern(MFA, MatchSpec) ->
+    try erts_internal:trace_pattern(MFA, MatchSpec, []) of
+        Res -> Res
+    catch E:R ->
+            {_, [_ | CST]} = erlang:process_info(
+                               erlang:self(), current_stacktrace),
+            erlang:raise(
+              E, R, [{?MODULE, trace_pattern, [MFA, MatchSpec], []} | CST])
+    end.
 
 -type trace_pattern_flag() ::
       global | local |
       meta | {meta, Pid :: pid()} |
+      {meta, TracerModule :: module(), TracerState :: term()} |
       call_count |
       call_time.
 
@@ -2371,8 +2402,28 @@ trace_pattern(_MFA, _MatchSpec) ->
                  | restart
                  | pause,
       FlagList :: [ trace_pattern_flag() ].
-trace_pattern(_MFA, _MatchSpec, _FlagList) ->
-    erlang:nif_error(undefined).
+trace_pattern(MFA, MatchSpec, FlagList) ->
+    %% Make sure that we have loaded the tracer module
+    case lists:keyfind(meta, 1, FlagList) of
+        {meta, Module, State} when erlang:is_atom(Module) ->
+            case erlang:module_loaded(Module) of
+                false ->
+                    Module:enabled(trace_status, erlang:self(), State);
+                true ->
+                    ok
+            end;
+        _ ->
+            ignore
+    end,
+
+    try erts_internal:trace_pattern(MFA, MatchSpec, FlagList) of
+        Res -> Res
+    catch E:R ->
+            {_, [_ | CST]} = erlang:process_info(
+                               erlang:self(), current_stacktrace),
+            erlang:raise(
+              E, R, [{?MODULE, trace_pattern, [MFA, MatchSpec, FlagList], []} | CST])
+    end.
 
 %% Shadowed by erl_bif_types: erlang:tuple_to_list/1
 -spec tuple_to_list(Tuple) -> [term()] when
diff --git a/erts/preloaded/src/erts_internal.erl b/erts/preloaded/src/erts_internal.erl
index 769757ba75..2459ea2a2c 100644
--- a/erts/preloaded/src/erts_internal.erl
+++ b/erts/preloaded/src/erts_internal.erl
@@ -55,7 +55,9 @@
 -export([await_microstate_accounting_modifications/3,
 	 gather_microstate_accounting_result/2]).
 
-%% Auto-import name clash
+-export([trace/3, trace_pattern/3]).
+
+%% Auto import name clash
 -export([check_process_code/2]).
 
 %%
@@ -403,3 +405,28 @@ microstate_accounting(Ref, Threads) ->
         {Ref, Res} ->
 	    [Res | microstate_accounting(Ref, Threads - 1)]
     end.
+
+-spec trace(PidPortSpec, How, FlagList) -> integer() when
+      PidPortSpec :: pid() | port()
+                   | all | processes | ports
+                   | existing | existing_processes | existing_ports
+                   | new | new_processes | new_ports,
+      How :: boolean(),
+      FlagList :: [].
+trace(_PidSpec, _How, _FlagList) ->
+    erlang:nif_error(undefined).
+
+-type trace_pattern_mfa() ::
+      {atom(),atom(),arity() | '_'} | on_load.
+-type trace_match_spec() ::
+      [{[term()] | '_' ,[term()],[term()]}].
+
+-spec trace_pattern(MFA, MatchSpec, FlagList) -> non_neg_integer() when
+      MFA :: trace_pattern_mfa(),
+      MatchSpec :: (MatchSpecList :: trace_match_spec())
+                 | boolean()
+                 | restart
+                 | pause,
+      FlagList :: [ ].
+trace_pattern(_MFA, _MatchSpec, _FlagList) ->
+    erlang:nif_error(undefined).
diff --git a/erts/preloaded/src/init.erl b/erts/preloaded/src/init.erl
index 5d5d2f8012..77684751c8 100644
--- a/erts/preloaded/src/init.erl
+++ b/erts/preloaded/src/init.erl
@@ -179,6 +179,10 @@ stop(Status) -> init ! {stop,{stop,Status}}, ok.
 boot(BootArgs) ->
     register(init, self()),
     process_flag(trap_exit, true),
+
+    %% Load the tracer nif
+    erl_tracer:on_load(),
+
     {Start0,Flags,Args} = parse_boot_args(BootArgs),
     Start = map(fun prepare_run_args/1, Start0),
     boot(Start, Flags, Args).