Merge branch 'lukas/erts/tracer-module-and-more/PR-1009/OTP-10268'

* lukas/erts/tracer-module-and-more/PR-1009/OTP-10268: (26 commits)
  erts: Don't trace on link events when port is dead
  erts: more logging in trace_bif_SUITE trace_bif_return
  erts: Make trace_delivered go via sys msg dispatcher again
  erts: Add comment about future trace optimizations
  erts: Optimize tracer reload test
  erts: Document erlang:match_spec_test/3
  runtime_tools: Make dbg work with erl_tracer modules
  runtime_rools: Allow new timestamp trace flags
  runtime_tools: Lots of dbg docs updates
  erts: Deallocate heap fragments from trace nif calls
  eprof: Fix tests after tracer module incompatabilities
  fprof: update to work with new spawned trace event
  erts: Add 'spawned' trace event to 'procs' trace flag
  erts: send and receive no longer need status lock
  erts: Do 'unregister' as "self-tracing"
  erts: Silence harmless valgrind warning in dec_term
  erts: Fix end_per_testcase crash in local_trace_SUITE
  observer: Update ttb to work with tracing on ports
  runtime_tools: Update dbg to work with tracing on ports
  erts: Add tracing examples in match spec docs
  ...

6 files changed, 155 insertions, 22 deletions

diff --git a/erts/preloaded/ebin/erl_tracer.beam b/erts/preloaded/ebin/erl_tracer.beam
new file mode 100644
index 0000000000..dcb74c3bf7
--- /dev/null
+++ b/erts/preloaded/ebin/erl_tracer.beam
diff --git a/erts/preloaded/src/Makefile b/erts/preloaded/src/Makefile
index 1e3de9f1d7..4a447d3a09 100644
--- a/erts/preloaded/src/Makefile
+++ b/erts/preloaded/src/Makefile
@@ -43,7 +43,8 @@ PRE_LOADED_ERL_MODULES = \
 	otp_ring0 \
 	erts_code_purger \
 	erlang \
-	erts_internal
+	erts_internal \
+	erl_tracer
 
 PRE_LOADED_BEAM_MODULES = \
 	prim_eval
diff --git a/erts/preloaded/src/erl_tracer.erl b/erts/preloaded/src/erl_tracer.erl
new file mode 100644
index 0000000000..2177e48f60
--- /dev/null
+++ b/erts/preloaded/src/erl_tracer.erl
@@ -0,0 +1,42 @@
+-module(erl_tracer).
+
+-export([enabled/3, trace/6, on_load/0]).
+
+-type tracee() :: port() | pid() | undefined.
+-type trace_tag() :: send | send_to_non_existing_process | 'receive' |
+                     call | return_to | return_from | exception_from |
+                     spawn | spawned | exit | link | unlink | getting_linked |
+                     getting_unlinked | register | unregister | in | out |
+                     in_exiting | out_exiting | out_exited |
+                     open | closed | gc_start | gc_end.
+-type trace_opts() :: #{ match_spec_result => true | term(),
+                         scheduler_id => undefined | non_neg_integer(),
+                         timestamp => undefined | timestamp | cpu_timestamp |
+                                      monotonic | strict_monotonic }.
+-type tracer_state() :: term().
+
+on_load() ->
+    case erlang:load_nif(atom_to_list(?MODULE), 0) of
+        ok -> ok
+    end.
+
+%%%
+%%% NIF placeholders
+%%%
+
+-spec enabled(Tag :: trace_tag() | seq_trace | trace_status,
+              TracerState :: tracer_state(),
+              Tracee :: tracee()) ->
+   trace | discard | remove.
+enabled(_, _, _) ->
+    erlang:nif_error(nif_not_loaded).
+
+-spec trace(Tag :: trace_tag() | seq_trace,
+            TracerState :: tracer_state(),
+            Tracee :: tracee(),
+            Msg :: term(),
+            Extra :: term(),
+            Opts :: trace_opts()) -> any().
+
+trace(_, _, _, _, _, _) ->
+    erlang:nif_error(nif_not_loaded).
diff --git a/erts/preloaded/src/erlang.erl b/erts/preloaded/src/erlang.erl
index 4374bdcd89..3cc17014ff 100644
--- a/erts/preloaded/src/erlang.erl
+++ b/erts/preloaded/src/erlang.erl
@@ -230,27 +230,32 @@
       send |
       'receive' |
       procs |
+      ports |
       call |
-      silent |
+      arity |
       return_to |
+      silent |
       running |
       exiting |
+      running_procs |
+      running_ports |
       garbage_collection |
       timestamp |
       cpu_timestamp |
       monotonic_timestamp |
       strict_monotonic_timestamp |
-      arity |
       set_on_spawn |
       set_on_first_spawn |
       set_on_link |
       set_on_first_link |
-      {tracer, pid() | port()}.
+      {tracer, pid() | port()} |
+      {tracer, module(), term()}.
 
 -type trace_info_item_result() ::
        {traced, global | local | false | undefined} |
        {match_spec, trace_match_spec() | false | undefined} |
        {meta, pid() | port() | false | undefined | []} |
+       {meta, module(), term() } |
        {meta_match_spec, trace_match_spec() | false | undefined} |
        {call_count, non_neg_integer() | boolean() | undefined} |
        {call_time, [{pid(), non_neg_integer(),
@@ -276,6 +281,7 @@
       undefined |
       {flags, [trace_info_flag()]} |
       {tracer, pid() | port() | []} |
+      {tracer, module(), term()} |
       trace_info_item_result() |
       {all, [ trace_info_item_result() ] | false | undefined}.
 
@@ -1157,10 +1163,10 @@ map_size(_Map) ->
     erlang:nif_error(undefined).
 
 %% match_spec_test/3
--spec erlang:match_spec_test(P1, P2, P3) -> TestResult when
-      P1 :: [term()] | tuple(),
-      P2 :: term(),
-      P3 :: table | trace,
+-spec erlang:match_spec_test(MatchAgainst, MatchSpec, Type) -> TestResult when
+      MatchAgainst :: [term()] | tuple(),
+      MatchSpec :: term(),
+      Type :: table | trace,
       TestResult :: {ok, term(), [return_trace], [ {error | warning, string()} ]} | {error, [ {error | warning, string()} ]}.
 match_spec_test(_P1, _P2, _P3) ->
     erlang:nif_error(undefined).
@@ -1705,12 +1711,35 @@ time() ->
     erlang:nif_error(undefined).
 
 %% trace/3
--spec erlang:trace(PidSpec, How, FlagList) -> integer() when
-      PidSpec :: pid() | existing | new | all,
+-spec erlang:trace(PidPortSpec, How, FlagList) -> integer() when
+      PidPortSpec :: pid() | port()
+                   | all | processes | ports
+                   | existing | existing_processes | existing_ports
+                   | new | new_processes | new_ports,
       How :: boolean(),
       FlagList :: [trace_flag()].
-trace(_PidSpec, _How, _FlagList) ->
-    erlang:nif_error(undefined).
+trace(PidPortSpec, How, FlagList) ->
+    %% Make sure that we have loaded the tracer module
+    case lists:keyfind(tracer, 1, FlagList) of
+        {tracer, Module, State} when erlang:is_atom(Module) ->
+            case erlang:module_loaded(Module) of
+                false ->
+                    Module:enabled(trace_status, erlang:self(), State);
+                true ->
+                    ok
+            end;
+        _ ->
+            ignore
+    end,
+
+    try erts_internal:trace(PidPortSpec, How, FlagList) of
+        Res -> Res
+    catch E:R ->
+            {_, [_ | CST]} = erlang:process_info(
+                               erlang:self(), current_stacktrace),
+            erlang:raise(
+              E, R, [{?MODULE, trace, [PidPortSpec, How, FlagList], []} | CST])
+    end.
 
 %% trace_delivered/1
 -spec erlang:trace_delivered(Tracee) -> Ref when
@@ -1720,14 +1749,16 @@ trace_delivered(_Tracee) ->
     erlang:nif_error(undefined).
 
 %% trace_info/2
--spec erlang:trace_info(PidOrFunc, Item) -> Res when
-      PidOrFunc :: pid() | new | {Module, Function, Arity} | on_load,
+-spec erlang:trace_info(PidPortOrFunc, Item) -> Res when
+      PidPortOrFunc :: pid() | port() | new | new_processes | new_ports
+                     | {Module, Function, Arity} | on_load,
       Module :: module(),
       Function :: atom(),
       Arity :: arity(),
-      Item :: flags | tracer | traced | match_spec | meta | meta_match_spec | call_count | call_time | all,
+      Item :: flags | tracer | traced | match_spec
+            | meta | meta_match_spec | call_count | call_time | all,
       Res :: trace_info_return().
-trace_info(_PidOrFunc, _Item) ->
+trace_info(_PidPortOrFunc, _Item) ->
     erlang:nif_error(undefined).
 
 %% trunc/1
@@ -2319,7 +2350,7 @@ subtract(_,_) ->
       OldState :: preliminary | final | volatile;
                         %% These are deliberately not documented
 			(internal_cpu_topology, term()) -> term();
-                        (sequential_tracer, pid() | port() | false) -> pid() | port() | false;
+                        (sequential_tracer, pid() | port() | {module(), term()} | false) -> pid() | port() | false;
                         (1,0) -> true.
 
 system_flag(_Flag, _Value) ->
@@ -2355,12 +2386,20 @@ tl(_List) ->
                  | boolean()
                  | restart
                  | pause.
-trace_pattern(_MFA, _MatchSpec) ->
-    erlang:nif_error(undefined).
+trace_pattern(MFA, MatchSpec) ->
+    try erts_internal:trace_pattern(MFA, MatchSpec, []) of
+        Res -> Res
+    catch E:R ->
+            {_, [_ | CST]} = erlang:process_info(
+                               erlang:self(), current_stacktrace),
+            erlang:raise(
+              E, R, [{?MODULE, trace_pattern, [MFA, MatchSpec], []} | CST])
+    end.
 
 -type trace_pattern_flag() ::
       global | local |
       meta | {meta, Pid :: pid()} |
+      {meta, TracerModule :: module(), TracerState :: term()} |
       call_count |
       call_time.
 
@@ -2371,8 +2410,28 @@ trace_pattern(_MFA, _MatchSpec) ->
                  | restart
                  | pause,
       FlagList :: [ trace_pattern_flag() ].
-trace_pattern(_MFA, _MatchSpec, _FlagList) ->
-    erlang:nif_error(undefined).
+trace_pattern(MFA, MatchSpec, FlagList) ->
+    %% Make sure that we have loaded the tracer module
+    case lists:keyfind(meta, 1, FlagList) of
+        {meta, Module, State} when erlang:is_atom(Module) ->
+            case erlang:module_loaded(Module) of
+                false ->
+                    Module:enabled(trace_status, erlang:self(), State);
+                true ->
+                    ok
+            end;
+        _ ->
+            ignore
+    end,
+
+    try erts_internal:trace_pattern(MFA, MatchSpec, FlagList) of
+        Res -> Res
+    catch E:R ->
+            {_, [_ | CST]} = erlang:process_info(
+                               erlang:self(), current_stacktrace),
+            erlang:raise(
+              E, R, [{?MODULE, trace_pattern, [MFA, MatchSpec, FlagList], []} | CST])
+    end.
 
 %% Shadowed by erl_bif_types: erlang:tuple_to_list/1
 -spec tuple_to_list(Tuple) -> [term()] when
diff --git a/erts/preloaded/src/erts_internal.erl b/erts/preloaded/src/erts_internal.erl
index 769757ba75..2459ea2a2c 100644
--- a/erts/preloaded/src/erts_internal.erl
+++ b/erts/preloaded/src/erts_internal.erl
@@ -55,7 +55,9 @@
 -export([await_microstate_accounting_modifications/3,
 	 gather_microstate_accounting_result/2]).
 
-%% Auto-import name clash
+-export([trace/3, trace_pattern/3]).
+
+%% Auto import name clash
 -export([check_process_code/2]).
 
 %%
@@ -403,3 +405,28 @@ microstate_accounting(Ref, Threads) ->
         {Ref, Res} ->
 	    [Res | microstate_accounting(Ref, Threads - 1)]
     end.
+
+-spec trace(PidPortSpec, How, FlagList) -> integer() when
+      PidPortSpec :: pid() | port()
+                   | all | processes | ports
+                   | existing | existing_processes | existing_ports
+                   | new | new_processes | new_ports,
+      How :: boolean(),
+      FlagList :: [].
+trace(_PidSpec, _How, _FlagList) ->
+    erlang:nif_error(undefined).
+
+-type trace_pattern_mfa() ::
+      {atom(),atom(),arity() | '_'} | on_load.
+-type trace_match_spec() ::
+      [{[term()] | '_' ,[term()],[term()]}].
+
+-spec trace_pattern(MFA, MatchSpec, FlagList) -> non_neg_integer() when
+      MFA :: trace_pattern_mfa(),
+      MatchSpec :: (MatchSpecList :: trace_match_spec())
+                 | boolean()
+                 | restart
+                 | pause,
+      FlagList :: [ ].
+trace_pattern(_MFA, _MatchSpec, _FlagList) ->
+    erlang:nif_error(undefined).
diff --git a/erts/preloaded/src/init.erl b/erts/preloaded/src/init.erl
index 5d5d2f8012..77684751c8 100644
--- a/erts/preloaded/src/init.erl
+++ b/erts/preloaded/src/init.erl
@@ -179,6 +179,10 @@ stop(Status) -> init ! {stop,{stop,Status}}, ok.
 boot(BootArgs) ->
     register(init, self()),
     process_flag(trap_exit, true),
+
+    %% Load the tracer nif
+    erl_tracer:on_load(),
+
     {Start0,Flags,Args} = parse_boot_args(BootArgs),
     Start = map(fun prepare_run_args/1, Start0),
     boot(Start, Flags, Args).