Merge branch 'lukas/erts/tracer-module-and-more/PR-1009/OTP-10268'

* lukas/erts/tracer-module-and-more/PR-1009/OTP-10268: (26 commits)
  erts: Don't trace on link events when port is dead
  erts: more logging in trace_bif_SUITE trace_bif_return
  erts: Make trace_delivered go via sys msg dispatcher again
  erts: Add comment about future trace optimizations
  erts: Optimize tracer reload test
  erts: Document erlang:match_spec_test/3
  runtime_tools: Make dbg work with erl_tracer modules
  runtime_rools: Allow new timestamp trace flags
  runtime_tools: Lots of dbg docs updates
  erts: Deallocate heap fragments from trace nif calls
  eprof: Fix tests after tracer module incompatabilities
  fprof: update to work with new spawned trace event
  erts: Add 'spawned' trace event to 'procs' trace flag
  erts: send and receive no longer need status lock
  erts: Do 'unregister' as "self-tracing"
  erts: Silence harmless valgrind warning in dec_term
  erts: Fix end_per_testcase crash in local_trace_SUITE
  observer: Update ttb to work with tracing on ports
  runtime_tools: Update dbg to work with tracing on ports
  erts: Add tracing examples in match spec docs
  ...

7 files changed, 719 insertions, 118 deletions

diff --git a/erts/doc/src/Makefile b/erts/doc/src/Makefile
index e02e89238e..b96cbbce40 100644
--- a/erts/doc/src/Makefile
+++ b/erts/doc/src/Makefile
@@ -50,12 +50,14 @@ XML_REF1_FILES = epmd.xml \
 XML_REF3_EFILES = \
 	erl_prim_loader.xml \
 	erlang.xml \
+	erl_tracer.xml \
 	init.xml \
 	zlib.xml
 
 XML_REF3_FILES = \
 	driver_entry.xml \
 	erl_nif.xml \
+	erl_tracer.xml \
 	erl_driver.xml \
 	erl_prim_loader.xml \
 	erlang.xml \
@@ -154,18 +156,9 @@ clean:
 	rm -f $(SPECDIR)/*
 	rm -f errs core *~ 
 
-$(SPECDIR)/specs_driver_entry.xml:
+$(SPECDIR)/specs_%.xml:
 	escript $(SPECS_EXTRACTOR) $(SPECS_FLAGS) \
-                -o$(dir $@) -module driver_entry
-$(SPECDIR)/specs_erl_nif.xml:
-	escript $(SPECS_EXTRACTOR) $(SPECS_FLAGS) \
-                -o$(dir $@) -module erl_nif
-$(SPECDIR)/specs_erl_driver.xml:
-	escript $(SPECS_EXTRACTOR) $(SPECS_FLAGS) \
-                -o$(dir $@) -module erl_driver
-$(SPECDIR)/specs_erts_alloc.xml:
-	escript $(SPECS_EXTRACTOR) $(SPECS_FLAGS) \
-                -o$(dir $@) -module erts_alloc
+                -o$(dir $@) -module $(patsubst $(SPECDIR)/specs_%.xml,%,$@)
 
 # ----------------------------------------------------
 # Release Target
diff --git a/erts/doc/src/erl_nif.xml b/erts/doc/src/erl_nif.xml
index 5f56d8b595..7546f7ef81 100644
--- a/erts/doc/src/erl_nif.xml
+++ b/erts/doc/src/erl_nif.xml
@@ -566,8 +566,7 @@ typedef struct {
            <code type="none">
 typedef void ErlNifResourceDtor(ErlNifEnv* env, void* obj);
 </code>
-            <p>The function prototype of a resource destructor function.
-            A destructor function is not allowed to call any term-making functions.</p>
+            <p>The function prototype of a resource destructor function.</p>
           </item>
           <tag><marker id="ErlNifCharEncoding"/>ErlNifCharEncoding</tag>
            <item>
diff --git a/erts/doc/src/erl_tracer.xml b/erts/doc/src/erl_tracer.xml
new file mode 100644
index 0000000000..1e8e78b25f
--- /dev/null
+++ b/erts/doc/src/erl_tracer.xml
@@ -0,0 +1,324 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<!DOCTYPE erlref SYSTEM "erlref.dtd">
+
+<erlref>
+  <header>
+    <copyright>
+      <year>2016</year><year>2016</year>
+      <holder>Ericsson AB. All Rights Reserved.</holder>
+    </copyright>
+    <legalnotice>
+      Licensed under the Apache License, Version 2.0 (the "License");
+      you may not use this file except in compliance with the License.
+      You may obtain a copy of the License at
+
+          http://www.apache.org/licenses/LICENSE-2.0
+
+      Unless required by applicable law or agreed to in writing, software
+      distributed under the License is distributed on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+      See the License for the specific language governing permissions and
+      limitations under the License.
+
+    </legalnotice>
+
+    <title>erl_tracer</title>
+    <prepared></prepared>
+    <docno></docno>
+    <date></date>
+    <rev></rev>
+  </header>
+  <module>erl_tracer</module>
+  <modulesummary>Erlang Tracer Behaviour</modulesummary>
+  <description>
+    <p>A behaviour module for implementing the back end of the erlang
+    tracing system. The functions in this module will be called whenever
+    a trace probe is triggered. Both the <c>enabled</c> and <c>trace</c>
+    functions are called in the context of the entity that triggered the
+    trace probe.
+    This means that the overhead by having the tracing enabled will be
+    greatly effected by how much time is spent in these functions. So do as
+    little work as possible in these functions.</p>
+    <note>
+      <p>All functions in this behaviour have to be implemented as NIF's.
+      This is a limitation that may the lifted in the future.
+      There is an <seealso marker="#example">example tracer module nif</seealso>
+      implementation at the end of this page.</p>
+    </note>
+    <warning>
+      <p>Do not send messages or issue port commands to the <c>Tracee</c>
+      in any of the callbacks. Doing so is not allowed and can cause all
+      sorts of strange behaviour, including but not limited to infinite
+      recursions.</p>
+    </warning>
+  </description>
+
+  <datatypes>
+    <datatype>
+      <name name="trace_tag" />
+      <desc>
+	<p>The different trace tags that the tracer will be called with.
+        Each trace tag is described in greater detail in
+        <seealso marker="#trace">Module:trace/6</seealso>
+        </p>
+      </desc>
+    </datatype>
+    <datatype>
+      <name name="tracee" />
+      <desc>
+	<p>The process or port that the trace belongs to.
+        </p>
+      </desc>
+    </datatype>
+    <datatype>
+      <name name="trace_opts" />
+      <desc>
+	<p>The options for the tracee.
+        <taglist>
+          <tag><c>timestamp</c></tag>
+          <item>If not set to <c>undefined</c>, the tracer has been requested to
+          include a timestamp.</item>
+          <tag><c>match_spec_result</c></tag>
+          <item>If not set to <c>true</c>, the tracer has been requested to
+          include the output of a match specification that was run.</item>
+          <tag><c>scheduler_id</c></tag>
+          <item>Set to a number of the scheduler id is to be included by the tracer.
+          Otherwise it is set to <c>undefined</c>.</item>
+        </taglist>
+        </p>
+      </desc>
+    </datatype>
+    <datatype>
+      <name name="tracer_state" />
+      <desc>
+        <p>
+          The state which is given when calling
+          <seealso marker="erlang#trace-3"><c>erlang:trace(PidPortSpec,true,[{tracer,Module,TracerState}])</c></seealso>.
+          The tracer state is an immutable value that is passed to erl_tracer callbacks and should
+          contain all the data that is needed to generate the trace event.
+        </p>
+      </desc>
+    </datatype>
+  </datatypes>
+
+  <section>
+    <title>CALLBACK FUNCTIONS</title>
+    <p>The following functions
+      should be exported from a <c>erl_tracer</c> callback module.</p>
+  </section>
+  <marker id="enabled"></marker>
+  <funcs>
+    <func>
+      <name>Module:enabled(TraceTag, TracerState, Tracee) -> Result</name>
+      <fsummary>Check if a trace event should be generated.</fsummary>
+      <type>
+        <v>TraceTag = <seealso marker="#type-trace_tag">trace_tag()</seealso> | trace_status</v>
+        <v>TracerState = term()</v>
+        <v>Tracee = <seealso marker="#type-trace_tag">tracee()</seealso></v>
+        <v>Result =  trace | discard | remove</v>
+      </type>
+      <desc>
+        <p>This callback will be called whenever a trace point is triggered. It
+        allows the tracer to decide whether a trace should be generated or not.
+        This check is made as early as possible in order to limit the amount of
+        overhead associated with tracing. If <c>trace</c> is returned the
+        necessary trace data will be created and the trace call-back of the tracer
+        will be called. If <c>discard</c> is returned, this trace call
+        will be discarded and no call to trace will be done. If
+        <c>remove</c> is returned, the VM will attempt to remove this tracer
+        from the tracee, together with any trace flags set on the tracee.
+        </p>
+        <p><c>trace_status</c> is a special type of <c>TraceTag</c> which is used
+        to check if the tracer should still be active. It is called in multiple
+        scenarios, but most significantly it is used when tracing is started
+        using this tracer.</p>
+        <p>This function may be called multiple times per trace point, so it
+        is important that it is both fast and side effect free.</p>
+      </desc>
+    </func>
+    <marker id="trace"></marker>
+    <func>
+      <name>Module:trace(TraceTag, TracerState, Tracee, FirstTraceTerm, SecondTraceTerm, Opts) -> Result</name>
+      <fsummary>Check if a trace event should be generated.</fsummary>
+      <type>
+        <v>TraceTag = <seealso marker="#type-trace_tag">trace_tag()</seealso></v>
+        <v>TracerState = term()</v>
+        <v>Tracee = <seealso marker="#type-trace_tag">tracee()</seealso></v>
+        <v>FirstTraceTerm = term()</v>
+        <v>SecondTraceTerm = term() | undefined</v>
+        <v>Opts = <seealso marker="#type-trace_opts">trace_opts()</seealso></v>
+        <v>Result =  ok</v>
+      </type>
+      <desc>
+        <p>This callback will be called when a trace point is triggered and
+        the <seealso marker="#enabled">Module:enabled/3</seealso>
+        callback returned <c>trace</c>. In it any side effects needed by
+        the tracer should be done. The trace point payload is located in
+        the <c>FirstTraceTerm</c> and <c>SecondTraceTerm</c>. The content
+        of the TraceTerms depends on which <c>TraceTag</c> has been triggered.
+        The <c>FirstTraceTerm</c> and <c>SecondTraceTerm</c> correspond to the
+        fourth and fifth slot in the trace tuples described in
+        <seealso marker="erlang#trace_3_trace_messages">erlang:trace/3</seealso>.
+        If the tuple only has four elements, <c>SecondTraceTerm</c> will be
+        <c>undefined</c>.</p>
+      </desc>
+    </func>
+    <func>
+      <name name="trace">Module:trace(seq_trace, TracerState, Label, SeqTraceInfo, undefined, Opts) -> Result</name>
+      <fsummary>Check if a sequence trace event should be generated.</fsummary>
+      <type>
+        <v>TracerState = term()</v>
+        <v>Label = term()</v>
+        <v>SeqTraceInfo = term()</v>
+        <v>Opts = <seealso marker="#type-trace_opts">trace_opts()</seealso></v>
+        <v>Result =  ok</v>
+      </type>
+      <desc>
+        <p>The <c>TraceTag</c> <c>seq_trace</c> is handled a little bit
+        differently. There is not <c>Tracee</c> for seq_trace, instead the
+        <c>Label</c> associated with the seq_trace event is given.
+        For more info on what <c>Label</c> and <c>SeqTraceInfo</c> can be
+        see the <seealso marker="kernel:seq_trace">seq_trace</seealso> manual.</p>
+      </desc>
+    </func>
+  </funcs>
+  <section>
+    <marker id="example"></marker>
+    <title>Erl Tracer Module example</title>
+    <p>In the example below a tracer module with a nif backend sends a message
+    for each <c>send</c> trace tag containing only the sender and receiver.
+    Using this tracer module, a much more lightweight message tracer is
+    used that only records who sent messages to who.</p>
+    <p>Here is an example session using it on Linux.</p>
+    <pre>
+$ gcc -I erts-8.0/include/ -fPIC -shared -o erl_msg_tracer.so erl_msg_tracer.c
+$ erl
+Erlang/OTP 19 [DEVELOPMENT] [erts-8.0] [source-ed2b56b] [64-bit] [smp:8:8] [async-threads:10] [hipe] [kernel-poll:false]
+
+Eshell V8.0  (abort with ^G)
+1&gt; c(erl_msg_tracer), erl_msg_tracer:load().
+ok
+2&gt; Tracer = spawn(fun F() -&gt; receive M -&gt; io:format("~p~n",[M]), F() end end).
+&lt;0.37.0&gt;
+3&gt; erlang:trace(new, true, [send,{tracer, erl_msg_tracer, Tracer}]).
+0
+{&lt;0.39.0&gt;,&lt;0.27.0&gt;}
+4&gt; {ok, D} = file:open("/tmp/tmp.data",[write]).
+{trace,#Port&lt;0.486&gt;,&lt;0.40.0&gt;}
+{trace,&lt;0.40.0&gt;,&lt;0.21.0&gt;}
+{trace,#Port&lt;0.487&gt;,&lt;0.4.0&gt;}
+{trace,#Port&lt;0.488&gt;,&lt;0.4.0&gt;}
+{trace,#Port&lt;0.489&gt;,&lt;0.4.0&gt;}
+{trace,#Port&lt;0.490&gt;,&lt;0.4.0&gt;}
+{ok,&lt;0.40.0&gt;}
+{trace,&lt;0.41.0&gt;,&lt;0.27.0&gt;}
+5&gt;
+    </pre>
+    <p>erl_msg_tracer.erl</p>
+    <pre>
+-module(erl_msg_tracer).
+
+-export([enabled/3, trace/6, load/0]).
+
+load() ->
+    erlang:load_nif("erl_msg_tracer", []).
+
+enabled(_, _, _) ->
+    error.
+
+trace(_, _, _,_, _, _) ->
+    error.
+    </pre>
+    <p>erl_msg_tracer.c</p>
+    <pre>
+#include "erl_nif.h"
+
+/* NIF interface declarations */
+static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info);
+static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info);
+static void unload(ErlNifEnv* env, void* priv_data);
+
+/* The NIFs: */
+static ERL_NIF_TERM enabled(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]);
+static ERL_NIF_TERM trace(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]);
+
+static ErlNifFunc nif_funcs[] = {
+    {"enabled", 3, enabled},
+    {"trace", 6, trace}
+};
+
+ERL_NIF_INIT(erl_msg_tracer, nif_funcs, load, NULL, upgrade, unload)
+
+static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
+{
+    *priv_data = NULL;
+    return 0;
+}
+
+static void unload(ErlNifEnv* env, void* priv_data)
+{
+
+}
+
+static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data,
+		   ERL_NIF_TERM load_info)
+{
+    if (*old_priv_data != NULL || *priv_data != NULL) {
+	return -1; /* Don't know how to do that */
+    }
+    if (load(env, priv_data, load_info)) {
+	return -1;
+    }
+    return 0;
+}
+
+/*
+ * argv[0]: Trace Tag
+ * argv[1]: TracerState
+ * argv[2]: Tracee
+ */
+static ERL_NIF_TERM enabled(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    ErlNifPid to_pid;
+    if (enif_get_local_pid(env, argv[1], &amp;to_pid))
+        if (!enif_is_process_alive(env, &amp;to_pid))
+            /* tracer is dead so we should remove this trace point */
+            return enif_make_atom(env, "remove");
+
+    /* Only generate trace for when tracer != tracee */
+    if (enif_is_identical(argv[1], argv[2]))
+        return enif_make_atom(env, "discard");
+
+    /* Only trigger trace messages on 'send' */
+    if (enif_is_identical(enif_make_atom(env, "send"), argv[0]))
+        return enif_make_atom(env, "trace");
+
+    /* Have to answer trace_status */
+    if (enif_is_identical(enif_make_atom(env, "trace_status"), argv[0]))
+        return enif_make_atom(env, "trace");
+
+    return enif_make_atom(env, "discard");
+}
+
+/*
+ * argv[0]: Trace Tag, should only be 'send'
+ * argv[1]: TracerState, process to send {argv[2], argv[4]} to
+ * argv[2]: Tracee
+ * argv[3]: Message, ignored
+ * argv[4]: Recipient
+ * argv[5]: Options, ignored
+ */
+static ERL_NIF_TERM trace(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    ErlNifPid to_pid;
+
+    if (enif_get_local_pid(env, argv[1], &amp;to_pid)) {
+        ERL_NIF_TERM msg = enif_make_tuple3(env, enif_make_atom(env, "trace"), argv[2], argv[4]);
+        enif_send(env, &amp;to_pid, NULL, msg);
+    }
+
+    return enif_make_atom(env, "ok");
+}
+    </pre>
+  </section>
+</erlref>
diff --git a/erts/doc/src/erlang.xml b/erts/doc/src/erlang.xml
index ef577c82bf..423ccdf98f 100644
--- a/erts/doc/src/erlang.xml
+++ b/erts/doc/src/erlang.xml
@@ -2613,6 +2613,48 @@ os_prompt% </pre>
     </func>
 
     <func>
+      <name name="match_spec_test" arity="3"/>
+      <fsummary>Test that a match specification works</fsummary>
+      <desc>
+        <p>
+          This function is a utility to test a match_spec used in calls to
+          <seealso marker="stdlib:ets#select/2">ets:select/2</seealso> and
+          <seealso marker="#trace_pattern/3">erlang:trace_pattern/3</seealso>.
+          The function both tests MatchSpec for "syntactic" correctness and
+          runs the match_spec against the object. If the match_spec contains
+          errors, the tuple {error, Errors} is returned where Errors is a list
+          of natural language descriptions of what was wrong with the match_spec.
+        </p>
+        <p>
+          If the <c><anno>Type</anno></c> is <c>table</c> the object to match
+          against should be a tuple. The function then returns
+          {ok,Result,[],Warnings} where Result is what would have been the
+          result in a real ets:select/2 call or false if the match_spec does
+          not match the object tuple.
+        </p>
+
+        <p>
+          If <c><anno>Type</anno></c> is <c>trace</c> the object to match
+          against should be a list. The function returns
+          {ok, Result, Flags, Warnings} where Result is <c>true</c> if a trace
+          message should be emitted, <c>false</c> if a trace message should not
+          be emitted or the message term to be appended to the trace message.
+          Flags is a list containing all the trace flags that will be enabled,
+          at the moment this is only <c>return_trace</c>.
+        </p>
+
+        <p>
+          This is a useful debugging and test tool, especially when writing complicated
+          match specifications.
+        </p>
+        <p>
+          See also
+          <seealso marker="stdlib:ets#test_ms/2">ets:test_ms/2</seealso>.
+        </p>
+      </desc>
+    </func>
+
+    <func>
       <name name="max" arity="2"/>
       <fsummary>Returns the largest of two terms.</fsummary>
       <desc>
@@ -8347,22 +8389,47 @@ timestamp() ->
           <c><anno>How</anno> == false</c>) the trace flags in
           <c><anno>FlagList</anno></c> for
           the process or processes represented by
-          <c><anno>PidSpec</anno></c>.</p>
-        <p><c><anno>PidSpec</anno></c> is either a process identifier
-          (pid) for a local process, or one of the following atoms:</p>
+          <c><anno>PidPortSpec</anno></c>.</p>
+        <p><c><anno>PidPortSpec</anno></c> is either a process identifier
+          (pid) for a local process, a port identifier,
+        or one of the following atoms:</p>
         <taglist>
+          <tag><c>all</c></tag>
+          <item>
+            <p>All currently existing processes and ports and all that
+              will be created in the future.</p>
+          </item>
+          <tag><c>processes</c></tag>
+          <item>
+            <p>All currently existing processes and all that will be created in the future.</p>
+          </item>
+          <tag><c>ports</c></tag>
+          <item>
+            <p>All currently existing ports and all that will be created in the future.</p>
+          </item>
           <tag><c>existing</c></tag>
           <item>
+            <p>All currently existing processes and ports.</p>
+          </item>
+          <tag><c>existing_processes</c></tag>
+          <item>
             <p>All currently existing processes.</p>
           </item>
+          <tag><c>existing_ports</c></tag>
+          <item>
+            <p>All currently existing ports.</p>
+          </item>
           <tag><c>new</c></tag>
           <item>
-            <p>All processes that are created in the future.</p>
+            <p>All processes and ports that will be created in the future.</p>
           </item>
-          <tag><c>all</c></tag>
+          <tag><c>new_processes</c></tag>
+          <item>
+            <p>All processes that will be created in the future.</p>
+          </item>
+          <tag><c>new_ports</c></tag>
           <item>
-            <p>All currently existing processes and all processes that
-              are created in the future.</p>
+            <p>All ports that will be created in the future.</p>
           </item>
         </taglist>
         <p><c><anno>FlagList</anno></c> can contain any number of the
@@ -8371,35 +8438,28 @@ timestamp() ->
         <taglist>
           <tag><c>all</c></tag>
           <item>
-            <p>Sets all trace flags except <c>{tracer, Tracer}</c> and
+            <p>Sets all trace flags except <c>tracer</c> and
               <c>cpu_timestamp</c>, which are in their nature different
               than the others.</p>
           </item>
           <tag><c>send</c></tag>
           <item>
             <p>Traces sending of messages.</p>
-            <p>Message tags: <c>send</c> and
-              <c>send_to_non_existing_process</c>.</p>
+            <p>Message tags: <c><seealso marker="#trace_3_trace_messages_send">send</seealso></c> and
+            <c><seealso marker="#trace_3_trace_messages_send_to_non_existing_process">send_to_non_existing_process</seealso></c>.</p>
           </item>
           <tag><c>'receive'</c></tag>
           <item>
             <p>Traces receiving of messages.</p>
-            <p>Message tags: <c>'receive'</c>.</p>
+            <p>Message tags: <c><seealso marker="#trace_3_trace_messages_receive">'receive'</seealso></c>.</p>
           </item>
-          <tag><c>procs</c></tag>
-          <item>
-            <p>Traces process-related events.</p>
-            <p>Message tags: <c>spawn</c>, <c>exit</c>,
-              <c>register</c>, <c>unregister</c>, <c>link</c>,
-              <c>unlink</c>, <c>getting_linked</c>, and
-              <c>getting_unlinked</c>.</p>
-          </item>
-          <tag><c>call</c></tag>
+<tag><c>call</c></tag>
           <item>
             <p>Traces certain function calls. Specify which function
               calls to trace by calling
               <seealso marker="#trace_pattern/3">erlang:trace_pattern/3</seealso>.</p>
-            <p>Message tags: <c>call</c> and <c>return_from</c>.</p>
+            <p>Message tags: <c><seealso marker="#trace_3_trace_messages_call">call</seealso></c> and
+            <c><seealso marker="#trace_3_trace_messages_return_from">return_from</seealso></c>.</p>
           </item>
           <tag><c>silent</c></tag>
           <item>
@@ -8417,8 +8477,9 @@ timestamp() ->
               specification function <c>{silent,Bool}</c>, giving
               a high degree of control of which functions with which 
               arguments that trigger the trace.</p>
-            <p>Message tags: <c>call</c>, <c>return_from</c>, and 
-              <c>return_to</c>. Or rather, the absence of.</p>
+              <p>Message tags: <c><seealso marker="#trace_3_trace_messages_call">call</seealso></c>,
+              <c><seealso marker="#trace_3_trace_messages_return_from">return_from</seealso></c>, and
+              <c><seealso marker="#trace_3_trace_messages_return_to">return_to</seealso></c>. Or rather, the absence of.</p>
           </item>
           <tag><c>return_to</c></tag>
           <item>
@@ -8439,23 +8500,63 @@ timestamp() ->
             <p>To get trace messages containing return values from
               functions, use the <c>{return_trace}</c> match
               specification action instead.</p>
-            <p>Message tags: <c>return_to</c>.</p>
+            <p>Message tags: <c><seealso marker="#trace_3_trace_messages_return_to">return_to</seealso></c>.</p>
+          </item>
+          <tag><c>procs</c></tag>
+          <item>
+            <p>Traces process-related events.</p>
+            <p>Message tags: <c><seealso marker="#trace_3_trace_messages_spawn">spawn</seealso></c>,
+            <c><seealso marker="#trace_3_trace_messages_spawned">spawned</seealso></c>,
+            <c><seealso marker="#trace_3_trace_messages_exit">exit</seealso></c>,
+            <c><seealso marker="#trace_3_trace_messages_register">register</seealso></c>,
+            <c><seealso marker="#trace_3_trace_messages_unregister">unregister</seealso></c>,
+            <c><seealso marker="#trace_3_trace_messages_link">link</seealso></c>,
+            <c><seealso marker="#trace_3_trace_messages_unlink">unlink</seealso></c>,
+            <c><seealso marker="#trace_3_trace_messages_getting_linked">getting_linked</seealso></c>, and
+            <c><seealso marker="#trace_3_trace_messages_getting_unlinked">getting_unlinked</seealso></c>.</p>
+          </item>
+          <tag><c>ports</c></tag>
+          <item>
+            <p>Traces port-related events.</p>
+            <p>Message tags: <c><seealso marker="#trace_3_trace_messages_open">open</seealso></c>,
+            <c><seealso marker="#trace_3_trace_messages_closed">closed</seealso></c>,
+            <c><seealso marker="#trace_3_trace_messages_register">register</seealso></c>,
+            <c><seealso marker="#trace_3_trace_messages_unregister">unregister</seealso></c>,
+            <c><seealso marker="#trace_3_trace_messages_getting_linked">getting_linked</seealso></c>, and
+            <c><seealso marker="#trace_3_trace_messages_getting_unlinked">getting_unlinked</seealso></c>.</p>
           </item>
           <tag><c>running</c></tag>
           <item>
             <p>Traces scheduling of processes.</p>
-            <p>Message tags: <c>in</c> and <c>out</c>.</p>
+            <p>Message tags: <c><seealso marker="#trace_3_trace_messages_in_proc">in</seealso></c> and
+            <c><seealso marker="#trace_3_trace_messages_out_proc">out</seealso></c>.</p>
           </item>
           <tag><c>exiting</c></tag>
           <item>
             <p>Traces scheduling of exiting processes.</p>
-            <p>Message tags: <c>in_exiting</c>, <c>out_exiting</c>, and
-	    <c>out_exited</c>.</p>
+            <p>Message tags: <c><seealso marker="#trace_3_trace_messages_in_exiting_proc">in_exiting</seealso></c>,
+            <c><seealso marker="#trace_3_trace_messages_out_exiting_proc">out_exiting</seealso></c>, and
+	    <c><seealso marker="#trace_3_trace_messages_out_exited_proc">out_exited</seealso></c>.</p>
+          </item>
+          <tag><c>running_procs</c></tag>
+          <item>
+            <p>Traces scheduling of processes just like <c>running</c>.
+            However this option also includes schedule events when the
+            process executes within the context of a port without
+            being scheduled out itself.</p>
+            <p>Message tags: <c><seealso marker="#trace_3_trace_messages_in_proc">in</seealso></c> and
+            <c><seealso marker="#trace_3_trace_messages_out_proc">out</seealso></c>.</p>
+          </item>
+          <tag><c>running_ports</c></tag>
+          <item>
+            <p>Traces scheduling of ports.</p>
+            <p>Message tags: <c><seealso marker="#trace_3_trace_messages_in_port">in</seealso></c> and
+            <c><seealso marker="#trace_3_trace_messages_out_port">out</seealso></c>.</p>
           </item>
           <tag><c>garbage_collection</c></tag>
           <item>
             <p>Traces garbage collections of processes.</p>
-            <p>Message tags: <c>gc_start</c> and <c>gc_end</c>.</p>
+            <p>Message tags: <c><seealso marker="#trace_3_trace_messages_gc_start">gc_start</seealso></c> and <c><seealso marker="#trace_3_trace_messages_gc_end">gc_end</seealso></c>.</p>
           </item>
           <tag><c>timestamp</c></tag>
           <item>
@@ -8470,8 +8571,8 @@ timestamp() ->
 	      in CPU time, not wall clock time. That is, <c>cpu_timestamp</c>
 	      will not be used if <c>monotonic_timestamp</c>, or
 	      <c>strict_monotonic_timestamp</c> is enabled.
-              Only allowed with <c>PidSpec==all</c>. If the host
-              machine OS does not support high-resolution
+              Only allowed with <c><anno>PidPortSpec</anno>==all</c>. If the
+              host machine OS does not support high-resolution
               CPU time measurements, <c>trace/3</c> exits with
               <c>badarg</c>. Notice that most OS do
               not synchronize this value across cores, so be prepared
@@ -8483,8 +8584,8 @@ timestamp() ->
 	    <seealso marker="time_correction#Erlang_Monotonic_Time">Erlang
 	    monotonic time</seealso> time-stamp in all trace messages. The
             time-stamp (Ts) has the same format and value as produced by
-            <c>erlang:monotonic_time(nano_seconds)</c>. This flag overrides
-	    the <c>cpu_timestamp</c> flag.</p>
+            <c><seealso marker="#monotonic_time-1">erlang:monotonic_time(nano_seconds)</seealso></c>.
+            This flag overrides the <c>cpu_timestamp</c> flag.</p>
           </item>
           <tag><c>strict_monotonic_timestamp</c></tag>
           <item>
@@ -8493,9 +8594,9 @@ timestamp() ->
 	    monotonic time</seealso> and a monotonically increasing
 	    integer in all trace messages. The time-stamp (Ts) has the
 	    same format and value as produced by
-            <c>{erlang:monotonic_time(nano_seconds),
-	    erlang:unique_integer([monotonic])}</c>. This flag overrides
-	    the <c>cpu_timestamp</c> flag.</p>
+            <c>{<seealso marker="#monotonic_time-1">erlang:monotonic_time(nano_seconds)</seealso>,
+	    <seealso marker="#unique_integer-1">erlang:unique_integer([monotonic])</seealso>}</c>.
+            This flag overrides the <c>cpu_timestamp</c> flag.</p>
           </item>
           <tag><c>arity</c></tag>
           <item>
@@ -8529,12 +8630,20 @@ timestamp() ->
           <item>
             <p>Specifies where to send the trace messages. <c>Tracer</c>
               must be the process identifier of a local process
-              or the port identifier
-              of a local port. If this flag is not given, trace
-              messages are sent to the process that called
-              <c>erlang:trace/3</c>.</p>
+              or the port identifier of a local port.</p>
+          </item>
+          <tag><c>{tracer, TracerModule, TracerState}</c></tag>
+          <item>
+            <p>Specifies that a tracer module should be called
+            instead of sending a trace message. The tracer module
+            can then ignore or change the trace message. For more details
+            on how to write a tracer module see <seealso marker="erl_tracer">
+            erl_tracer</seealso>
+          </p>
           </item>
         </taglist>
+        <p>If no <c>tracer</c> is given, the calling process
+        will be receiving all of the trace messages</p>
         <p>The effect of combining <c>set_on_first_link</c> with
           <c>set_on_link</c> is the same as having
           <c>set_on_first_link</c> alone. Likewise for
@@ -8555,21 +8664,36 @@ timestamp() ->
 	the other one will become active.</p>
         <marker id="trace_3_trace_messages"></marker>
         <taglist>
-          <tag><c>{trace, Pid, 'receive', Msg}</c></tag>
-          <item>
-            <p>When <c>Pid</c> receives message <c>Msg</c>.</p>
-          </item>
-          <tag><c>{trace, Pid, send, Msg, To}</c></tag>
+          <tag>
+            <marker id="trace_3_trace_messages_send"></marker>
+            <c>{trace, PidPort, send, Msg, To}</c>
+          </tag>
           <item>
-            <p>When <c>Pid</c> sends message <c>Msg</c> to
+            <p>When <c>PidPort</c> sends message <c>Msg</c> to
               process <c>To</c>.</p>
           </item>
-          <tag><c>{trace, Pid, send_to_non_existing_process, Msg, To}</c></tag>
+          <tag>
+            <marker id="trace_3_trace_messages_send_to_non_existing_process"></marker>
+            <c>{trace, PidPort, send_to_non_existing_process, Msg, To}</c>
+          </tag>
           <item>
-            <p>When <c>Pid</c> sends message <c>Msg</c> to
+            <p>When <c>PidPort</c> sends message <c>Msg</c> to
               the non-existing process <c>To</c>.</p>
           </item>
-          <tag><c>{trace, Pid, call, {M, F, Args}}</c></tag>
+          <tag>
+            <marker id="trace_3_trace_messages_receive"></marker>
+            <c>{trace, PidPort, 'receive', Msg}</c>
+          </tag>
+          <item>
+            <p>When <c>PidPort</c> receives message <c>Msg</c>.
+            If <c>Msg</c> is set to timeout, then a receive
+            statement may have timedout, or the process received
+            a message with the payload <c>timeout</c>.</p>
+          </item>
+          <tag>
+            <marker id="trace_3_trace_messages_call"></marker>
+            <c>{trace, Pid, call, {M, F, Args}}</c>
+          </tag>
           <item>
             <p>When <c>Pid</c> calls a traced function. The return
               values of calls are never supplied, only the call and its
@@ -8578,7 +8702,10 @@ timestamp() ->
               change the contents of this message, so that <c>Arity</c>
               is specified instead of <c>Args</c>.</p>
           </item>
-          <tag><c>{trace, Pid, return_to, {M, F, Arity}}</c></tag>
+          <tag>
+            <marker id="trace_3_trace_messages_return_to"></marker>
+            <c>{trace, Pid, return_to, {M, F, Arity}}</c>
+          </tag>
           <item>
             <p>When <c>Pid</c> returns <em>to</em> the specified
               function. This trace message is sent if both
@@ -8590,73 +8717,172 @@ timestamp() ->
               (that is, the functions match specification matched, and
               <c>{message, false}</c> was not an action).</p>
           </item>
-          <tag><c>{trace, Pid, return_from, {M, F, Arity}, ReturnValue}</c></tag>
+          <tag>
+            <marker id="trace_3_trace_messages_return_from"></marker>
+            <c>{trace, Pid, return_from, {M, F, Arity}, ReturnValue}</c>
+          </tag>
           <item>
             <p>When <c>Pid</c> returns <em>from</em> the specified
               function. This trace message is sent if flag <c>call</c>
               is set, and the function has a match specification
               with a <c>return_trace</c> or <c>exception_trace</c> action.</p>
           </item>
-          <tag><c>{trace, Pid, exception_from, {M, F, Arity}, {Class, Value}}</c></tag>
+          <tag>
+            <marker id="trace_3_trace_messages_exception_from"></marker>
+            <c>{trace, Pid, exception_from, {M, F, Arity}, {Class, Value}}</c>
+          </tag>
           <item>
             <p>When <c>Pid</c> exits <em>from</em> the specified
               function because of an exception. This trace message is
               sent if flag <c>call</c> is set, and the function has 
               a match specification with an <c>exception_trace</c> action.</p>
           </item>
-          <tag><c>{trace, Pid, spawn, Pid2, {M, F, Args}}</c></tag>
+          <tag>
+            <marker id="trace_3_trace_messages_spawn"></marker>
+            <c>{trace, Pid, spawn, Pid2, {M, F, Args}}</c>
+          </tag>
           <item>
             <p>When <c>Pid</c> spawns a new process <c>Pid2</c> with
               the specified function call as entry point.</p>
             <p><c>Args</c> is supposed to be the argument list,
               but can be any term if the spawn is erroneous.</p>
           </item>
-          <tag><c>{trace, Pid, exit, Reason}</c></tag>
+          <tag>
+            <marker id="trace_3_trace_messages_spawned"></marker>
+            <c>{trace, Pid, spawned, Pid2, {M, F, Args}}</c>
+          </tag>
+          <item>
+            <p>When <c>Pid</c> is spawned by process <c>Pid2</c> with
+              the specified function call as entry point.</p>
+            <p><c>Args</c> is supposed to be the argument list,
+              but can be any term if the spawn is erroneous.</p>
+          </item>
+          <tag>
+            <marker id="trace_3_trace_messages_exit"></marker>
+            <c>{trace, Pid, exit, Reason}</c>
+          </tag>
           <item>
             <p>When <c>Pid</c> exits with reason <c>Reason</c>.</p>
           </item>
-          <tag><c>{trace, Pid, link, Pid2}</c></tag>
+          <tag>
+            <marker id="trace_3_trace_messages_register"></marker>
+            <c>{trace, PidPort, register, RegName}</c>
+          </tag>
+          <item>
+            <p>When <c>PidPort</c> gets the name <c>RegName</c> registered.</p>
+          </item>
+          <tag>
+            <marker id="trace_3_trace_messages_unregister"></marker>
+            <c>{trace, PidPort, unregister, RegName}</c>
+          </tag>
+          <item>
+            <p>When <c>PidPort</c> gets the name <c>RegName</c> unregistered.
+              This is done automatically when a registered
+              process or port exits.</p>
+          </item>
+          <tag>
+            <marker id="trace_3_trace_messages_link"></marker>
+            <c>{trace, Pid, link, Pid2}</c>
+          </tag>
           <item>
             <p>When <c>Pid</c> links to a process <c>Pid2</c>.</p>
           </item>
-          <tag><c>{trace, Pid, unlink, Pid2}</c></tag>
+          <tag>
+            <marker id="trace_3_trace_messages_unlink"></marker>
+            <c>{trace, Pid, unlink, Pid2}</c>
+          </tag>
           <item>
             <p>When <c>Pid</c> removes the link from a process
               <c>Pid2</c>.</p>
           </item>
-          <tag><c>{trace, Pid, getting_linked, Pid2}</c></tag>
+          <tag>
+            <marker id="trace_3_trace_messages_getting_linked"></marker>
+            <c>{trace, PidPort, getting_linked, Pid2}</c>
+          </tag>
           <item>
-            <p>When <c>Pid</c> gets linked to a process <c>Pid2</c>.</p>
+            <p>When <c>PidPort</c> gets linked to a process <c>Pid2</c>.</p>
           </item>
-          <tag><c>{trace, Pid, getting_unlinked, Pid2}</c></tag>
+          <tag>
+            <marker id="trace_3_trace_messages_getting_unlinked"></marker>
+            <c>{trace, PidPort, getting_unlinked, Pid2}</c>
+          </tag>
           <item>
-            <p>When <c>Pid</c> gets unlinked from a process <c>Pid2</c>.</p>
+            <p>When <c>PidPort</c> gets unlinked from a process <c>Pid2</c>.</p>
           </item>
-          <tag><c>{trace, Pid, register, RegName}</c></tag>
+          <tag>
+            <marker id="trace_3_trace_messages_exit"></marker>
+            <c>{trace, Pid, exit, Reason}</c>
+          </tag>
           <item>
-            <p>When <c>Pid</c> gets the name <c>RegName</c> registered.</p>
+            <p>When <c>Pid</c> exits with reason <c>Reason</c>.</p>
           </item>
-          <tag><c>{trace, Pid, unregister, RegName}</c></tag>
+          <tag>
+            <marker id="trace_3_trace_messages_open"></marker>
+            <c>{trace, Port, open, Pid, Driver}</c>
+          </tag>
           <item>
-            <p>When <c>Pid</c> gets the name <c>RegName</c> unregistered.
-              This is done automatically when a registered
-              process exits.</p>
+            <p>When <c>Pid</c> opens a new port <c>Port</c> with
+              the running the <c>Driver</c>.</p>
+            <p><c>Driver</c> is the name of the driver as an atom.</p>
+          </item>
+          <tag>
+            <marker id="trace_3_trace_messages_closed"></marker>
+            <c>{trace, Port, closed, Reason}</c>
+          </tag>
+          <item>
+            <p>When <c>Port</c> closed with <c>Reason</c>.</p>
           </item>
-          <tag><c>{trace, Pid, in, {M, F, Arity} | 0}</c></tag>
+          <tag>
+            <marker id="trace_3_trace_messages_in_proc"></marker>
+            <marker id="trace_3_trace_messages_in_exiting_proc"></marker>
+            <c>{trace, Pid, in | in_exiting, {M, F, Arity} | 0}</c>
+          </tag>
           <item>
             <p>When <c>Pid</c> is scheduled to run. The process
               runs in function <c>{M, F, Arity}</c>. On some rare
               occasions, the current function cannot be determined,
               then the last element is <c>0</c>.</p>
           </item>
-          <tag><c>{trace, Pid, out, {M, F, Arity} | 0}</c></tag>
+          <tag>
+            <marker id="trace_3_trace_messages_out_proc"></marker>
+            <marker id="trace_3_trace_messages_out_exiting_proc"></marker>
+            <marker id="trace_3_trace_messages_out_exited_proc"></marker>
+            <c>{trace, Pid, out | out_exiting | out_exited, {M, F, Arity} | 0}</c>
+          </tag>
           <item>
             <p>When <c>Pid</c> is scheduled out. The process was
               running in function {M, F, Arity}. On some rare occasions,
               the current function cannot be determined, then the last
               element is <c>0</c>.</p>
           </item>
-          <tag><c>{trace, Pid, gc_start, Info}</c></tag>
+          <tag>
+            <marker id="trace_3_trace_messages_in_port"></marker>
+            <c>{trace, Port, in, Command | 0}</c>
+          </tag>
+          <item>
+            <p>When <c>Port</c> is scheduled to run. <c>Command</c> is the
+            first thing the port will execute, it may however run several
+            commands before being scheduled out. On some rare
+            occasions, the current function cannot be determined,
+            then the last element is <c>0</c>.</p>
+            <p>The possible commands are: <c>call | close | command | connect | control | flush | info | link | open | unlink</c></p>
+          </item>
+          <tag>
+            <marker id="trace_3_trace_messages_out_port"></marker>
+            <c>{trace, Port, out, Command | 0}</c>
+          </tag>
+          <item>
+            <p>When <c>Port</c> is scheduled out. The last command run
+              was <c>Command</c>. On some rare occasions,
+              the current function cannot be determined, then the last
+              element is <c>0</c>. <c>Command</c> can contain the same
+              commands as <c>in</c>
+            </p>
+          </item>
+          <tag>
+            <marker id="trace_3_trace_messages_gc_start"></marker>
+            <c>{trace, Pid, gc_start, Info}</c>
+          </tag>
           <item>
           <marker id="gc_start"></marker>
             <p>Sent when garbage collection is about to be started.
@@ -8698,7 +8924,10 @@ timestamp() ->
             </taglist>
             <p>All sizes are in words.</p>
           </item>
-          <tag><c>{trace, Pid, gc_end, Info}</c></tag>
+          <tag>
+            <marker id="trace_3_trace_messages_gc_end"></marker>
+            <c>{trace, Pid, gc_end, Info}</c>
+          </tag>
           <item>
             <p>Sent when garbage collection is finished. <c>Info</c>
               contains the same kind of list as in message <c>gc_start</c>,
@@ -8706,18 +8935,18 @@ timestamp() ->
               garbage collection.</p>
           </item>
         </taglist>
-        <p>If the tracing process dies, the flags are silently
-          removed.</p>
-        <p>Only one process can trace a particular process. Therefore,
+        <p>If the tracing process/port dies or the tracer module returns
+        <c>remove</c>, the flags are silently removed.</p>
+        <p>Each process can only be traced by one tracer. Therefore,
           attempts to trace an already traced process fail.</p>
         <p>Returns: A number indicating the number of processes that
-          matched <c><anno>PidSpec</anno></c>.
-          If <c><anno>PidSpec</anno></c> is a process
+          matched <c><anno>PidPortSpec</anno></c>.
+          If <c><anno>PidPortSpec</anno></c> is a process
           identifier, the return value is <c>1</c>.
-          If <c><anno>PidSpec</anno></c>
+          If <c><anno>PidPortSpec</anno></c>
           is <c>all</c> or <c>existing</c>, the return value is
-          the number of processes running, excluding tracer processes.
-          If <c><anno>PidSpec</anno></c> is <c>new</c>, the return value is
+          the number of processes running.
+          If <c><anno>PidPortSpec</anno></c> is <c>new</c>, the return value is
           <c>0</c>.</p>
         <p>Failure: <c>badarg</c> if the specified arguments are
           not supported. For example, <c>cpu_timestamp</c> is not
@@ -8729,7 +8958,11 @@ timestamp() ->
       <name name="trace_delivered" arity="1"/>
       <fsummary>Notification when trace has been delivered.</fsummary>
       <desc>
-        <p>The delivery of trace messages is dislocated on the time-line
+        <p>The delivery of trace messages (generated by
+         <seealso marker="#trace/3"><c>erlang:trace/3</c></seealso>,
+         <seealso marker="kernel:seq_trace"><c>seq_trace</c></seealso> or
+         <seealso marker="#system_profile/2"><c>erlang:system_profile/2</c></seealso>)
+          is dislocated on the time-line
           compared to other events in the system. If you know that
           <c><anno>Tracee</anno></c> has passed some specific point
           in its execution,
@@ -8750,13 +8983,16 @@ timestamp() ->
           has not been traced by someone, but if this is the case,
           <em>no</em> trace messages have been delivered when the
           <c>trace_delivered</c> message arrives.</p>
-        <p>Notice that that <c><anno>Tracee</anno></c> must refer
+        <p>Notice that <c><anno>Tracee</anno></c> must refer
           to a process currently,
           or previously existing on the same node as the caller of
           <c>erlang:trace_delivered(<anno>Tracee</anno>)</c> resides on.
           The special <c><anno>Tracee</anno></c> atom <c>all</c>
-          denotes all processes
-          that currently are traced in the node.</p>
+          denotes all processes that currently are traced in the node.</p>
+        <p>When used together with an <seealso marker="#erl_tracer">
+          Tracer Module</seealso> any message sent in the trace callback
+          is guaranteed to have reached it's recipient before the
+          <c>trace_delivered</c> message is sent.</p>
         <p>Example: Process <c>A</c> is <c><anno>Tracee</anno></c>,
 	  port <c>B</c> is tracer, and process <c>C</c> is the port
 	  owner of <c>B</c>. <c>C</c> wants to close <c>B</c> when
@@ -8779,12 +9015,15 @@ timestamp() ->
       <type name="trace_info_flag"/>
       <type name="trace_match_spec"/>
       <desc>
-        <p>Returns trace information about a process or function.</p>
-        <p>To get information about a process,
-         <c><anno>PidOrFunc</anno></c> is to
-          be a process identifier (pid) or the atom <c>new</c>.
-          The atom <c>new</c> means that the default trace state for
-          processes to be created is returned.</p>
+        <p>Returns trace information about a port, process or function.</p>
+        <p>To get information about a port or process,
+         <c><anno>PidPortOrFunc</anno></c> is to
+          be a process identifier (pid), port identifier or one of
+          the atoms <c>new</c>, <c>new_processes</c>, <c>new_ports</c>.
+          The atom <c>new</c> or <c>new_processes</c> means that the default trace
+          state for processes to be created is returned. The atom <c>new_ports</c>
+          means that the default trace state for ports to be created is returned.
+        </p>
         <p>The following <c>Item</c>s are valid:</p>
         <taglist>
           <tag><c>flags</c></tag>
@@ -8794,19 +9033,22 @@ timestamp() ->
               traces are enabled, and one or more of the followings
               atoms if traces are enabled: <c>send</c>,
               <c>'receive'</c>, <c>set_on_spawn</c>, <c>call</c>,
-              <c>return_to</c>, <c>procs</c>, <c>set_on_first_spawn</c>,
-              <c>set_on_link</c>, <c>running</c>,
+              <c>return_to</c>, <c>procs</c>, <c>ports</c>, <c>set_on_first_spawn</c>,
+              <c>set_on_link</c>, <c>running</c>, <c>running_procs</c>,
+              <c>running_ports</c>, <c>silent</c>, <c>exiting</c>
+              <c>monotonic_timestamp</c>, <c>strict_monotonic_timestamp</c>,
               <c>garbage_collection</c>, <c>timestamp</c>, and
               <c>arity</c>. The order is arbitrary.</p>
           </item>
           <tag><c>tracer</c></tag>
           <item>
-            <p>Returns the identifier for process or port tracing this
+            <p>Returns the identifier for process, port or a tuple containing
+              the tracer module and tracer state tracing this
               process. If this process is not being traced, the return
               value is <c>[]</c>.</p>
           </item>
         </taglist>
-        <p>To get information about a function, <c>PidOrFunc</c> is to
+        <p>To get information about a function, <c><anno>PidPortOrFunc</anno></c> is to
           be the three-element tuple <c>{Module, Function, Arity}</c> or
           the atom <c>on_load</c>. No wild cards are allowed. Returns
           <c>undefined</c> if the function does not exist, or
@@ -8830,8 +9072,8 @@ timestamp() ->
           </item>
           <tag><c>meta</c></tag>
           <item>
-            <p>Returns the meta-trace tracer process or port for this
-              function, if it has one. If the function is not
+            <p>Returns the meta-trace tracer process, port or trace module
+              for this function, if it has one. If the function is not
               meta-traced, the returned value is <c>false</c>. If
               the function is meta-traced but has once detected that
               the tracer process is invalid, the returned value is [].</p>
@@ -8874,7 +9116,7 @@ timestamp() ->
           <c>Value</c> is the requested information as described earlier.
           If a pid for a dead process was given, or the name of a
           non-existing function, <c>Value</c> is <c>undefined</c>.</p>
-        <p>If <c><anno>PidOrFunc</anno></c> is <c>on_load</c>, the information
+        <p>If <c><anno>PidPortOrFunc</anno></c> is <c>on_load</c>, the information
           returned refers to the default value for code that will be
           loaded.</p>
       </desc>
@@ -8999,13 +9241,12 @@ timestamp() ->
               the process, a <c>return_to</c> message is also sent
               when this function returns to its caller.</p>
           </item>
-          <tag><c>meta | {meta, <anno>Pid</anno>}</c></tag>
+          <tag><c>meta | {meta, <anno>Pid</anno>} | {meta, <anno>TracerModule</anno>, <anno>TracerState</anno>}</c>
+          </tag>
           <item>
             <p>Turns on or off meta-tracing for all types of function
-              calls. Trace messages are sent to the tracer process
-              or port <c><anno>Pid</anno></c> whenever any of the specified
-              functions are called, regardless of how they are called.
-              If no <c><anno>Pid</anno></c> is specified,
+              calls. Trace messages are sent to the tracer whenever any of
+              the specified functions are called. If no tracer is specified,
               <c>self()</c> is used as a default tracer process.</p>
             <p>Meta-tracing traces all processes and does not care
               about the process trace flags set by <c>trace/3</c>,
@@ -9013,7 +9254,7 @@ timestamp() ->
               <c>[call, timestamp]</c>.</p>
             <p>The match specification function <c>{return_trace}</c>
               works with meta-trace and sends its trace message to the
-              same tracer process.</p>
+              same tracer.</p>
           </item>
           <tag><c>call_count</c></tag>
           <item>
diff --git a/erts/doc/src/match_spec.xml b/erts/doc/src/match_spec.xml
index 975f01cf2c..3944f24f84 100644
--- a/erts/doc/src/match_spec.xml
+++ b/erts/doc/src/match_spec.xml
@@ -287,7 +287,7 @@
         can <em>not</em> be one of the atoms <c><![CDATA[all]]></c>, <c><![CDATA[new]]></c> or 
         <c><![CDATA[existing]]></c> (unless, of course, they are registered names).
         <c><![CDATA[P2]]></c> can <em>not</em> be <c><![CDATA[cpu_timestamp]]></c> nor
-        <c><![CDATA[{tracer,_}]]></c>.
+        <c><![CDATA[tracer]]></c>.
         Returns <c><![CDATA[true]]></c> and may only be used in
         the <c><![CDATA[MatchBody]]></c> part when tracing.
         </p>
@@ -298,7 +298,7 @@
         be either a process identifier or a registered name and is given
         as the first argument to the match_spec function. 
         <c><![CDATA[P2]]></c> can <em>not</em> be <c><![CDATA[cpu_timestamp]]></c> nor
-        <c><![CDATA[{tracer,_}]]></c>. Returns
+        <c><![CDATA[tracer]]></c>. Returns
         <c><![CDATA[true]]></c> and may only be used in the <c><![CDATA[MatchBody]]></c> part
         when tracing.
         </p>
@@ -308,11 +308,14 @@
         disable list is applied first, but effectively all changes
         are applied atomically. The trace flags
         are the same as for <c><![CDATA[erlang:trace/3]]></c> not including
-        <c><![CDATA[cpu_timestamp]]></c> but including <c><![CDATA[{tracer,_}]]></c>. If a
+        <c><![CDATA[cpu_timestamp]]></c> but including <c><![CDATA[tracer]]></c>. If a
         tracer is specified in both lists, the tracer in the 
         enable list takes precedence. If no tracer is specified the
         same tracer as the process executing the match spec is 
-        used. With three parameters to this function the first is 
+        used. When using a <seealso marker="erl_tracer">tracer module</seealso>
+        the module has to be loaded before the match specification is executed.
+        If it is not loaded the match will fail.
+        With three parameters to this function the first is
         either a process identifier or the registered name of a
         process to set trace flags on, the second is the disable
         list, and the third is the enable list. Returns
@@ -525,7 +528,7 @@
   </section>
 
   <section>
-    <title>Examples</title>
+    <title>ETS Examples</title>
     <p>Match an argument list of three where the first and third arguments 
       are equal:</p>
     <code type="none"><![CDATA[
@@ -616,5 +619,44 @@
     <p>The function <c><![CDATA[ets:test_ms/2]]></c> can be useful for testing
       complicated ets matches.</p>
   </section>
+  <section>
+    <title>Tracing Examples</title>
+    <p>Only generate trace message if trace control word is set to 1:</p>
+    <code type="none"><![CDATA[
+[{'_',
+  [{'==',{get_tcw},{const, 1}}],
+  []}]
+    ]]></code>
+    <p>Only generate trace message if there is a seq trace token:</p>
+    <code type="none"><![CDATA[
+[{'_',
+  [{'==',{is_seq_trace},{const, 1}}],
+  []}]
+    ]]></code>
+    <p>Remove 'silent' trace flag when first argument is 'verbose'
+    and add it when it is 'silent':</p>
+    <code type="none"><![CDATA[
+[{'$1',
+  [{'==',{hd, '$1'},verbose}],
+  [{trace, [silent],[]}]},
+ {'$1',
+  [{'==',{hd, '$1'},silent}],
+  [{trace, [],[silent]}]}]
+    ]]></code>
+    <p>Add return_trace message if function is of arity 3:</p>
+    <code type="none"><![CDATA[
+[{'$1',
+  [{'==',{length, '$1'},3}],
+  [{return_trace}]},
+ {'_',[],[]}]
+    ]]></code>
+    <p>Only generate trace message if function is of arity 3 and first argument is 'trace':</p>
+    <code type="none"><![CDATA[
+[{['trace','$2','$3'],
+  [],
+  []},
+ {'_',[],[]}]
+    ]]></code>
+  </section>
 </chapter>
 
diff --git a/erts/doc/src/ref_man.xml b/erts/doc/src/ref_man.xml
index aa245ec08a..e45402a397 100644
--- a/erts/doc/src/ref_man.xml
+++ b/erts/doc/src/ref_man.xml
@@ -56,5 +56,6 @@
   <xi:include href="driver_entry.xml"/>
   <xi:include href="erts_alloc.xml"/>
   <xi:include href="erl_nif.xml"/>
+  <xi:include href="erl_tracer.xml"/>
 </application>
 
diff --git a/erts/doc/src/specs.xml b/erts/doc/src/specs.xml
index 41a3984659..ed6be650e5 100644
--- a/erts/doc/src/specs.xml
+++ b/erts/doc/src/specs.xml
@@ -2,6 +2,7 @@
 <specs xmlns:xi="http://www.w3.org/2001/XInclude">
   <xi:include href="../specs/specs_erl_prim_loader.xml"/>
   <xi:include href="../specs/specs_erlang.xml"/>
+  <xi:include href="../specs/specs_erl_tracer.xml"/>
   <xi:include href="../specs/specs_init.xml"/>
   <xi:include href="../specs/specs_zlib.xml"/>
 </specs>