Merge branch 'pan/dtrace' into maint

* pan/dtrace: (22 commits)
  Use distinct function-entry probes for local and global calls
  Correct calculation of stack depth in call/return probes
  Add probes for all kind of calls
  Don't try to "clean up" generated fun names
  erl_process.c: Fix probe for process exit
  Remove code causing dialyzer warning from prim_file
  Add documentation for dyntrace and system_info changes
  Update slogan and add system_info for dynamic trace
  Update README's for dtrace and systemtap
  Rename dyntrace BIFs to more suiting names
  If VM probes are not enabled, short-circuit calls to probe BIFs
  beam_makeops: Add a simple preprocessor
  Ifdef all dynamic trace code
  Move dtrace erlang code and NIF into runtime_tools
  Correct some errors in the user tag spreading
  Change to more specific configure options for dtrace
  Add user tag spreading functionality to VM and use in file
  Update dtrace for changes in R15
  Add DTrace support for OS X, Solaris, and Linux (via SystemTap), 4/4
  Add DTrace support for OS X, Solaris, and Linux (via SystemTap), 3/4
  ...

OTP-10017

31 files changed, 2424 insertions, 9 deletions

diff --git a/lib/runtime_tools/c_src/Makefile.in b/lib/runtime_tools/c_src/Makefile.in
index 3d9a7ed69d..120f9b913f 100644
--- a/lib/runtime_tools/c_src/Makefile.in
+++ b/lib/runtime_tools/c_src/Makefile.in
@@ -21,6 +21,11 @@ include $(ERL_TOP)/make/$(TARGET)/otp.mk
 include $(ERL_TOP)/make/$(TARGET)/otp_ded.mk
 
 # ----------------------------------------------------
+# Items from top-level configure
+# ----------------------------------------------------
+DTRACE_ENABLED=@DTRACE_ENABLED@
+DTRACE_ENABLED_2STEP=@DTRACE_ENABLED_2STEP@
+# ----------------------------------------------------
 # Application version
 # ----------------------------------------------------
 include ../vsn.mk
@@ -38,6 +43,8 @@ SHELL = /bin/sh
 LIBS = $(DED_LIBS)
 LDFLAGS += $(DED_LDFLAGS)
 
+DTRACE_LIBNAME = dyntrace
+
 SYSINCLUDE = $(DED_SYS_INCLUDE)
 ifeq ($(findstring vxworks,$(TARGET)),vxworks)
 	SYSINCLUDE += -I$(ERL_TOP)/erts/etc/vxworks
@@ -45,15 +52,20 @@ endif
 
 TRACE_DRV_INCLUDES = $(SYSINCLUDE)
 
-ALL_CFLAGS = $(CFLAGS) @DEFS@ $(TYPE_FLAGS) $(TRACE_DRV_INCLUDES)
-
+ALL_CFLAGS = $(CFLAGS) @DEFS@ $(TYPE_FLAGS) $(TRACE_DRV_INCLUDES) \
+	-I$(OBJDIR) -I$(ERL_TOP)/erts/emulator/$(TARGET)
 
 ifeq ($(TYPE),debug)
 TYPEMARKER = .debug
-TYPE_FLAGS = -g -DDEBUG @DEBUG_FLAGS@
+TYPE_FLAGS = $(subst -O3,,$(subst -O2,,$(CFLAGS))) -DDEBUG @DEBUG_FLAGS@
+else
+ifeq ($(TYPE),valgrind)
+TYPEMARKER = .valgrind
+TYPE_FLAGS = $(subst -O3,,$(subst -O2,,$(CFLAGS))) -DVALGRIND
 else
 TYPEMARKER =
-TYPE_FLAGS = -O2
+TYPE_FLAGS = $(CFLAGS)
+endif
 endif
 
 ROOTDIR = $(ERL_TOP)/lib
@@ -69,6 +81,16 @@ RELSYSDIR = $(RELEASE_PATH)/lib/runtime_tools-$(VSN)
 # ----------------------------------------------------
 # Misc Macros
 # ----------------------------------------------------
+before_DTrace_OBJS = $(OBJDIR)/dyntrace$(TYPEMARKER).o
+## NIF_MAKEFILE = $(PRIVDIR)/Makefile
+
+# Higher-level makefiles says that we can only compile on UNIX flavors
+NIF_LIB = $(LIBDIR)/dyntrace$(TYPEMARKER).@DED_EXT@
+
+ifeq ($(HOST_OS),)
+HOST_OS := $(shell $(ERL_TOP)/erts/autoconf/config.guess)
+endif
+
 TRACE_IP_DRV_OBJS = \
 	$(OBJDIR)/trace_ip_drv.o
 
@@ -91,7 +113,44 @@ endif
 
 _create_dirs := $(shell mkdir -p $(OBJDIR) $(LIBDIR))
 
-debug opt: $(SOLIBS)
+debug opt valgrind: $(SOLIBS) $(OBJDIR) $(LIBDIR) $(NIF_LIB)
+
+ifdef DTRACE_ENABLED
+DTRACE_USER_HEADER=$(OBJDIR)/dtrace_user.h
+$(OBJDIR)/dtrace_user.h: ./dtrace_user.d
+	dtrace -h -C $(INCLUDES) \
+	  -s ./dtrace_user.d \
+	  -o ./dtrace_user.tmp
+	sed -e '/^#define[ 	]*ERLANG_[A-Z0-9_]*(.*)/y/ABCDEFGHIJKLMNOPQRSTUVWXYZ/abcdefghijklmnopqrstuvwxyz/' ./dtrace_user.tmp > $@
+	rm ./dtrace_user.tmp
+else
+DTRACE_USER_HEADER=
+endif
+
+DTRACE_OBJS =
+ifdef DTRACE_ENABLED_2STEP
+DTRACE_OBJS += $(OBJDIR)/dtrace_user.o
+$(OBJDIR)/dtrace_user.o: $(before_DTrace_OBJS) $(OBJDIR)/dtrace_user.h
+	dtrace -G -C \
+	  -s ./dtrace_user.d \
+	  -o $@ $(before_DTrace_OBJS)
+endif
+
+DYNTRACE_OBJS = $(before_DTrace_OBJS) $(DTRACE_OBJS)
+
+$(OBJDIR):
+	-@mkdir -p $(OBJDIR)
+
+$(LIBDIR):
+	-@mkdir -p $(LIBDIR)
+
+$(OBJDIR)/dyntrace$(TYPEMARKER).o: dyntrace.c $(DTRACE_USER_HEADER)
+	$(INSTALL_DIR) $(OBJDIR)
+	$(CC) -c -o $@ $(ALL_CFLAGS) $<
+
+$(NIF_LIB): $(DYNTRACE_OBJS)
+	$(INSTALL_DIR) $(LIBDIR)
+	$(LD) $(LDFLAGS) -o $@ $^ $(LDLIBS)
 
 $(OBJDIR)/%.o: %.c
 	$(CC) -c -o $@ $(ALL_CFLAGS) $<
@@ -118,6 +177,12 @@ $(LIBDIR)/trace_file_drv.eld: $(TRACE_FILE_DRV_OBJS)
 
 clean:
 	rm -f $(SOLIBS) $(TRACE_IP_DRV_OBJS) $(TRACE_FILE_DRV_OBJS)
+	rm -f $(LIBDIR)/dyntrace.@DED_EXT@
+	rm -f $(LIBDIR)/dyntrace.debug.@DED_EXT@
+	rm -f $(LIBDIR)/dyntrace.valgrind.@DED_EXT@
+	rm -f $(OBJDIR)/dyntrace.o
+	rm -f $(OBJDIR)/dyntrace.debug.o
+	rm -f $(OBJDIR)/dyntrace.valgrind.o
 	rm -f core *~
 
 docs:
@@ -128,8 +193,10 @@ docs:
 include $(ERL_TOP)/make/otp_release_targets.mk
 
 release_spec: opt
+	$(INSTALL_DIR) $(RELSYSDIR)/priv/obj
 	$(INSTALL_DIR) $(RELSYSDIR)/priv/lib
-	$(INSTALL_PROGRAM) $(SOLIBS) $(RELSYSDIR)/priv/lib
+	$(INSTALL_PROGRAM) $(DYNTRACE_OBJS) $(RELSYSDIR)/priv/obj
+	$(INSTALL_PROGRAM) $(NIF_LIB) $(SOLIBS) $(RELSYSDIR)/priv/lib
 
 release_docs_spec:
 
diff --git a/lib/runtime_tools/c_src/dtrace_user.d b/lib/runtime_tools/c_src/dtrace_user.d
new file mode 100644
index 0000000000..45d3ef3b66
--- /dev/null
+++ b/lib/runtime_tools/c_src/dtrace_user.d
@@ -0,0 +1,53 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Scott Lystig Fritchie 2011.
+ * All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+provider erlang {
+    /**
+     * Send a single string to a probe.
+     *
+     * @param NUL-terminated string
+     */
+    probe user_trace__s1(char* message);
+
+    /**
+     * Multi-purpose probe: up to 4 NUL-terminated strings and 4
+     * 64-bit integer arguments.
+     *
+     * @param proc, the PID (string form) of the sending process
+     * @param user_tag, the user tag of the sender
+     * @param i1, integer
+     * @param i2, integer
+     * @param i3, integer
+     * @param i4, integer
+     * @param s1, string/iolist. D's arg6 is NULL if not given by Erlang
+     * @param s2, string/iolist. D's arg7 is NULL if not given by Erlang
+     * @param s3, string/iolist. D's arg8 is NULL if not given by Erlang
+     * @param s4, string/iolist. D's arg9 is NULL if not given by Erlang
+     */
+    probe user_trace__i4s4(char *proc, char *user_tag,
+                           int i1, int i2, int i3, int i4,
+                           char *s1, char *s2, char *s3, char *s4);
+};
+
+#pragma D attributes Evolving/Evolving/Common provider erlang provider
+#pragma D attributes Private/Private/Common provider erlang module
+#pragma D attributes Private/Private/Common provider erlang function
+#pragma D attributes Evolving/Evolving/Common provider erlang name
+#pragma D attributes Evolving/Evolving/Common provider erlang args
diff --git a/lib/runtime_tools/c_src/dyntrace.c b/lib/runtime_tools/c_src/dyntrace.c
new file mode 100644
index 0000000000..d94014559d
--- /dev/null
+++ b/lib/runtime_tools/c_src/dyntrace.c
@@ -0,0 +1,172 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Scott Lystig Fritchie 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Purpose:  Dynamically loadable NIF library for DTrace
+ */
+
+
+
+#include "erl_nif.h"
+#include "config.h"
+#include "sys.h"
+#include "dtrace-wrapper.h"
+#if defined(USE_DYNAMIC_TRACE) && (defined(USE_DTRACE) || defined(USE_SYSTEMTAP))
+#define HAVE_USE_DTRACE 1
+#endif
+#ifdef  HAVE_USE_DTRACE
+#include "dtrace_user.h"
+#endif
+
+void dtrace_nifenv_str(ErlNifEnv *env, char *process_buf);
+void get_string_maybe(ErlNifEnv *env, const ERL_NIF_TERM term, char **ptr, char *buf, int bufsiz);
+
+#ifdef VALGRIND
+    #  include <valgrind/memcheck.h>
+#endif
+
+#ifdef __GNUC__
+    #  define INLINE __inline__
+#else
+    #  define INLINE
+#endif
+
+#define MESSAGE_BUFSIZ 1024
+
+/* NIF interface declarations */
+static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info);
+
+/* The NIFs: */
+static ERL_NIF_TERM available(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]);
+static ERL_NIF_TERM user_trace_s1(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]);
+static ERL_NIF_TERM user_trace_i4s4(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]);
+
+static ErlNifFunc nif_funcs[] = {
+    {"available", 0, available},
+    {"user_trace_s1", 1, user_trace_s1},
+    {"user_trace_i4s4", 9, user_trace_i4s4}
+};
+
+ERL_NIF_INIT(dyntrace, nif_funcs, load, NULL, NULL, NULL)
+
+static ERL_NIF_TERM atom_true;
+static ERL_NIF_TERM atom_false;
+static ERL_NIF_TERM atom_error;
+static ERL_NIF_TERM atom_not_available;
+static ERL_NIF_TERM atom_badarg;
+static ERL_NIF_TERM atom_ok;
+
+static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
+{
+    atom_true = enif_make_atom(env,"true");
+    atom_false = enif_make_atom(env,"false");
+    atom_error = enif_make_atom(env,"error");
+    atom_not_available = enif_make_atom(env,"not_available");
+    atom_badarg = enif_make_atom(env,"badarg");
+    atom_ok = enif_make_atom(env,"ok");
+
+    return 0;
+}
+
+static ERL_NIF_TERM available(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+#ifdef	HAVE_USE_DTRACE
+    return atom_true;
+#else
+    return atom_false;
+#endif
+}
+
+static ERL_NIF_TERM user_trace_s1(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+#ifdef	HAVE_USE_DTRACE
+    ErlNifBinary message_bin;
+    DTRACE_CHARBUF(messagebuf, MESSAGE_BUFSIZ + 1);
+
+    if (DTRACE_ENABLED(user_trace_s1)) {
+	if (!enif_inspect_iolist_as_binary(env, argv[0], &message_bin) ||
+	    message_bin.size > MESSAGE_BUFSIZ) {
+	    return atom_badarg;
+	}
+	memcpy(messagebuf, (char *) message_bin.data, message_bin.size);
+        messagebuf[message_bin.size] = '\0';
+	DTRACE1(user_trace_s1, messagebuf);
+	return atom_true;
+    } else {
+	return atom_false;
+    }
+#else
+    return atom_error;
+#endif
+}
+
+void
+get_string_maybe(ErlNifEnv *env,
+                 const ERL_NIF_TERM term, char **ptr, char *buf, int bufsiz)
+{
+    ErlNifBinary str_bin;
+
+    if (!enif_inspect_iolist_as_binary(env, term, &str_bin) ||
+        str_bin.size > bufsiz) {
+        *ptr = NULL;
+    } else {
+        memcpy(buf, (char *) str_bin.data, str_bin.size);
+        buf[str_bin.size] = '\0';
+        *ptr = buf;
+    }
+}
+
+static ERL_NIF_TERM user_trace_i4s4(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+#ifdef	HAVE_USE_DTRACE
+    DTRACE_CHARBUF(procbuf, 32 + 1);
+    DTRACE_CHARBUF(user_tagbuf, MESSAGE_BUFSIZ + 1);
+    char *utbuf = NULL;
+    ErlNifSInt64 i1, i2, i3, i4;
+    DTRACE_CHARBUF(messagebuf1, MESSAGE_BUFSIZ + 1);
+    DTRACE_CHARBUF(messagebuf2, MESSAGE_BUFSIZ + 1);
+    DTRACE_CHARBUF(messagebuf3, MESSAGE_BUFSIZ + 1);
+    DTRACE_CHARBUF(messagebuf4, MESSAGE_BUFSIZ + 1);
+    char *mbuf1 = NULL, *mbuf2 = NULL, *mbuf3 = NULL, *mbuf4 = NULL;
+
+    if (DTRACE_ENABLED(user_trace_i4s4)) {
+	dtrace_nifenv_str(env, procbuf);
+        get_string_maybe(env, argv[0], &utbuf, user_tagbuf, MESSAGE_BUFSIZ);
+        if (! enif_get_int64(env, argv[1], &i1))
+            i1 = 0;
+        if (! enif_get_int64(env, argv[2], &i2))
+            i2 = 0;
+        if (! enif_get_int64(env, argv[3], &i3))
+            i3 = 0;
+        if (! enif_get_int64(env, argv[4], &i4))
+            i4 = 0;
+        get_string_maybe(env, argv[5], &mbuf1, messagebuf1, MESSAGE_BUFSIZ);
+        get_string_maybe(env, argv[6], &mbuf2, messagebuf2, MESSAGE_BUFSIZ);
+        get_string_maybe(env, argv[7], &mbuf3, messagebuf3, MESSAGE_BUFSIZ);
+        get_string_maybe(env, argv[8], &mbuf4, messagebuf4, MESSAGE_BUFSIZ);
+	DTRACE10(user_trace_i4s4, procbuf, utbuf,
+		 i1, i2, i3, i4, mbuf1, mbuf2, mbuf3, mbuf4);
+	return atom_true;
+    } else {
+	return atom_false;
+    }
+#else
+    return atom_error;
+#endif
+}
diff --git a/lib/runtime_tools/doc/src/Makefile b/lib/runtime_tools/doc/src/Makefile
index dbbae81cfe..f1dd788f80 100644
--- a/lib/runtime_tools/doc/src/Makefile
+++ b/lib/runtime_tools/doc/src/Makefile
@@ -40,7 +40,7 @@ RELSYSDIR = $(RELEASE_PATH)/lib/$(APPLICATION)-$(VSN)
 # Target Specs
 # ----------------------------------------------------
 XML_APPLICATION_FILES = ref_man.xml
-XML_REF3_FILES = dbg.xml erts_alloc_config.xml
+XML_REF3_FILES = dbg.xml dyntrace.xml erts_alloc_config.xml
 XML_REF6_FILES = runtime_tools_app.xml
 
 XML_PART_FILES = part_notes.xml part_notes_history.xml
diff --git a/lib/runtime_tools/doc/src/dyntrace.xml b/lib/runtime_tools/doc/src/dyntrace.xml
new file mode 100644
index 0000000000..5fc5530f25
--- /dev/null
+++ b/lib/runtime_tools/doc/src/dyntrace.xml
@@ -0,0 +1,209 @@
+<?xml version="1.0" encoding="latin1" ?>
+<!DOCTYPE erlref SYSTEM "erlref.dtd">
+
+<erlref>
+  <header>
+    <copyright>
+      <year>1996</year><year>2012</year>
+      <holder>Ericsson AB. All Rights Reserved.</holder>
+    </copyright>
+    <legalnotice>
+      The contents of this file are subject to the Erlang Public License,
+      Version 1.1, (the "License"); you may not use this file except in
+      compliance with the License. You should have received a copy of the
+      Erlang Public License along with this software. If not, it can be
+      retrieved online at http://www.erlang.org/.
+    
+      Software distributed under the License is distributed on an "AS IS"
+      basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+      the License for the specific language governing rights and limitations
+      under the License.
+    
+    </legalnotice>
+
+    <title>dyntrace</title>
+    <prepared>Patrik Nyblom</prepared>
+    <responsible></responsible>
+    <docno>1</docno>
+    <approved>ETX/B/SFP (Kenneth Lundin)</approved>
+    <checked></checked>
+    <date>12-03-20</date>
+    <rev>A</rev>
+    <file>dyntrace.xml</file>
+  </header>
+  <module>dyntrace</module>
+  <modulesummary>Interface to dynamic tracing</modulesummary>
+  <description>
+    <p>This module implements interfaces to dynamic tracing, should such be compiled into the virtual machine. For a standard and/or commercial build, no dynamic tracing is available, in which case none of the functions in this module is usable or give any effect.</p>
+    <p>Should dynamic tracing be enabled in the current build, either by configuring with <c>./configure --with-dynamic-trace=dtrace</c> or with  <c>./configure --with-dynamic-trace=systemtap</c>, the module can be used for two things:</p>
+    <list type="bulleted">
+      <item>Trigger the user-probe <c>user_trace_i4s4</c> in the NIF library <c>dyntrace.so</c> by calling <c>dyntrace:p/{1,2,3,4,5,6,7,8}</c>.</item>
+      <item>Set a user specified tag that will be present in the trace messages of both the <c>efile_drv</c> and the user-probe mentioned above.</item>
+    </list>
+      <p>Both building with dynamic trace probes and using them is experimental and unsupported by Erlang/OTP. It is included as an option for the developer to trace and debug performance issues in their systems.</p>
+      <p>The original implementation is mostly done by Scott Lystiger Fritchie as an Open Source Contribution and it should be viewed as such even though the source for dynamic tracing as well as this module is included in the main distribution. However, the ability to use dynamic tracing of the virtual machine is a very valuable contribution which OTP has every intention to maintain as a tool for the developer.</p>
+      <p>How to write <c>d</c> programs or <c>systemtap</c> scripts can be learned from books and from a lot of pages on the Internet. This manual page does not include any documentation about using the dynamic trace tools of respective platform. The <c>examples</c> directory of the <c>runtime_tools</c> application however contains comprehensive examples of both <c>d</c> and <c>systemtap</c> programs that will help you get started. Another source of information is the <c>README.dtrace(.md)</c> and <c>README.systemtap(.md)</c> files in the Erlang source top directory.</p> 
+  </description>
+  <funcs>
+    <func>
+      <name>available() -> boolean()</name>
+      <fsummary>Check if dynamic tracing is available</fsummary>
+      <desc>
+	<p>This function uses the NIF library to determine if dynamic
+	tracing is available. Usually calling <seealso
+	marker="erts:erlang#system_info/1">erlang:system_info/1</seealso>
+	is a better indicator of the availability of dynamic
+	tracing.</p>
+	<p>The function will throw an exception if the <c>dyntrace</c> NIF library could not be loaded by the on_load function of this module.</p>
+      </desc>
+    </func>
+    <func>
+      <name>p() -> true | false | error | badarg</name>
+      <fsummary>Trigger the user trace probe.</fsummary>
+      <desc>
+        <p>Calling this function will trigger the "user" trace probe user_trace_i4s4 in the dyntrace NIF module, sending a trace message only containing the user tag and zeroes/empty strings in all other fields.</p>
+      </desc>
+    </func>
+    <func>
+      <name>p(integer() | string()) -> true | false | error | badarg</name>
+      <fsummary>Trigger the user trace probe.</fsummary>
+      <desc>
+        <p>Calling this function will trigger the "user" trace probe user_trace_i4s4 in the dyntrace NIF module, sending a trace message containing the user tag and the integer or string parameter in the first integer/string field.</p>
+      </desc>
+    </func>
+    <func>
+      <name>p(integer() | string(), integer() | string()) -> true | false | error | badarg</name>
+      <fsummary>Trigger the user trace probe.</fsummary>
+      <desc>
+        <p>Calling this function will trigger the "user" trace probe user_trace_i4s4 in the dyntrace NIF module, sending a trace message containing the user tag and the integer() or string() parameters as the first fields of respective type. integer() parameters should be put before any string() parameters. I.e. <c>p(1,"Hello")</c> is ok, as is <c>p(1,1)</c> and <c>p("Hello","Again")</c>, but not <c>p("Hello",1)</c>.</p>
+      </desc>
+    </func>
+    <func>
+      <name>p(integer() | string(), integer() | string(), integer() | string()) -> true | false | error | badarg</name>
+      <fsummary>Trigger the user trace probe.</fsummary>
+      <desc>
+        <p>Calling this function will trigger the "user" trace probe user_trace_i4s4 in the dyntrace NIF module, sending a trace message containing the user tag and the integer() or string() parameters as the first fields of respective type. integer() parameters should be put before any string() parameters, as in <seealso marker="#p/2">p/2</seealso>.</p>
+      </desc>
+    </func>
+    <func>
+      <name>p(integer() | string(), integer() | string(), integer() | string(), integer() | string()) -> true | false | error | badarg</name>
+      <fsummary>Trigger the user trace probe.</fsummary>
+      <desc>
+        <p>Calling this function will trigger the "user" trace probe user_trace_i4s4 in the dyntrace NIF module, sending a trace message containing the user tag and the integer() or string() parameters as the first fields of respective type. integer() parameters should be put before any string() parameters, as in <seealso marker="#p/2">p/2</seealso>.</p>
+      </desc>
+    </func>
+    <func>
+      <name>p(integer(), integer() | string(), integer() | string(), integer() | string(), string()) -> true | false | error | badarg</name>
+      <fsummary>Trigger the user trace probe.</fsummary>
+      <desc>
+        <p>Calling this function will trigger the "user" trace probe user_trace_i4s4 in the dyntrace NIF module, sending a trace message containing the user tag and the integer() or string() parameters as the first fields of respective type. integer() parameters should be put before any string() parameters, as in <seealso marker="#p/2">p/2</seealso>.</p>
+	<p>There can be no more than four parameters of any type (integer() or string()), so the first parameter has to be an integer() and the last a string().</p> 
+      </desc>
+    </func>
+     <func>
+      <name>p(integer(), integer(), integer() | string(), integer() | string(), string(), string()) -> true | false | error | badarg</name>
+      <fsummary>Trigger the user trace probe.</fsummary>
+      <desc>
+        <p>Calling this function will trigger the "user" trace probe user_trace_i4s4 in the dyntrace NIF module, sending a trace message containing the user tag and the integer() or string() parameters as the first fields of respective type. integer() parameters should be put before any string() parameters, as in <seealso marker="#p/2">p/2</seealso>.</p>
+	<p>There can be no more than four parameters of any type (integer() or string()), so the first two parameters has to be integer()'s and the last two string()'s.</p> 
+      </desc>
+    </func>
+     <func>
+      <name>p(integer(), integer(), integer(), integer() | string(), string(), string(), string()) -> true | false | error | badarg</name>
+      <fsummary>Trigger the user trace probe.</fsummary>
+      <desc>
+        <p>Calling this function will trigger the "user" trace probe user_trace_i4s4 in the dyntrace NIF module, sending a trace message containing the user tag and the integer() or string() parameters as the first fields of respective type. integer() parameters should be put before any string() parameters, as in <seealso marker="#p/2">p/2</seealso>.</p>
+	<p>There can be no more than four parameters of any type (integer() or string()), so the first three parameters has to be integer()'s and the last three string()'s.</p> 
+      </desc>
+    </func>
+     <func>
+      <name>p(integer(), integer(), integer(), integer(), string(), string(), string(), string()) -> true | false | error | badarg</name>
+      <fsummary>Trigger the user trace probe.</fsummary>
+      <desc>
+        <p>Calling this function will trigger the "user" trace probe user_trace_i4s4 in the dyntrace NIF module, sending a trace message containing all the integer()'s and string()'s provided, as well as any user tag set in the current process.</p> 
+      </desc>
+    </func>
+    <func>
+      <name>get_tag() -> binary() | undefined</name>
+      <fsummary>Get the user tag set in the process.</fsummary>
+      <desc>
+	<p>This function returns the user tag set in the current
+	process. If no tag is set or dynamic tracing is not available,
+	it returns <c>undefined</c></p>
+      </desc>
+    </func>
+    <func>
+      <name>get_tag() -> binary() | undefined</name>
+      <fsummary>Get the user tag set in the process or sent to the process.</fsummary>
+      <desc>
+	<p>This function returns the user tag set in the current
+	process or, if no user tag is present, the last user tag sent
+	to the process together with a message (in the same way as
+	<seealso marker="kernel:seq_trace">sequential trace
+	tokens</seealso> are spread to other processes together with
+	messages. For an explanation of how user tags can be spread
+	together with messages, see <seealso
+	marker="#spread_tag/1">spread_tag/1</seealso>. If no tag is
+	found or dynamic tracing is not available, it returns
+	<c>undefined</c></p>
+      </desc>
+    </func>
+    
+    <func>
+      <name>put_tag(Item) -> binary() | undefined </name>
+      <fsummary>Set the user tag of the current process.</fsummary>
+      <type>
+        <v>Item = iodata()</v>
+      </type>
+      <desc>
+        <p>This function sets the user tag of the current process. The user tag is a binary(), but can be specified as any iodata(), which is automatically converted to a binary by this function.</p>
+	<p>The user tag is provided to the user probes triggered by calls top <c>dyntrace:p/{1,2,3,4,5,6,7,8}</c> as well as probes in the efile_driver. In the future, user tags might be added to more probes.</p>
+	<p>The old user tag (if any) is returned, or <c>undefined</c> if no user tag was present or dynamic tracing is not enabled.</p>
+      </desc>
+    </func>
+    <func>
+      <name>spread_tag(boolean()) -> TagData</name>
+      <fsummary>Start or stop spreading dynamic trace user tags with the next message.</fsummary>
+      <type>
+        <v>TagData = opaque data that can be used as parameter to <seealso marker="#restore_tag/1">restore_tag/1</seealso></v>
+      </type>
+      <desc>
+        <p>This function controls if user tags are to be spread to other processes with the next message. Spreading of user tags work like spreading of sequential trace tokens, so that a received user tag will be active in the process until the next message arrives (if that message does not also contain the user tag.</p>
+	<p>This functionality is used when a client process communicates with a file i/o-server to spread the user tag to the I/O-server and then down to the efile_drv driver. By using <c>spread_tag/1</c> and <c>restore_tag/1</c>, one can enable or disable spreading of user tags to other processes and then restore the previous state of the user tag. The TagData returned from this call contains all previous information so the state (including any previously spread user tags) will be completely restored by a later call to <c>restore_tag/1</c>.</p> 
+	<p>The <seealso marker="kernel:file">file</seealso> module already spread's tags, so there is noo need to manually call these function to get user tags spread to the efile driver through that module.</p>
+	<p>The most use of this function would be if one for example uses the <seealso marker="stdlib:io">io</seealso> module to communicate with an I/O-server for a regular file, like in the following example:</p>
+<pre>
+f() ->
+   {ok, F} = file:open("test.tst",[write]),
+   Saved = dyntrace:spread_tag(true),
+   io:format(F,"Hello world!",[]),
+   dyntrace:restore_tag(Saved),
+   file:close(F).
+</pre>
+          <p>In this example, any user tag set in the calling process will be spread to the I/O-server when the io:format call is done.</p>
+      </desc>
+    </func>
+    <func>
+    <name>restore_tag(TagData) -> true</name>
+      <fsummary>Restore to a previous state of user tag spreading.</fsummary>
+      <type>
+        <v>TagData = opaque data returned by <seealso marker="#spread_tag/1">spread_tag/1</seealso></v>
+      </type>
+      <desc>
+	<p>Restores the previous state of user tags and their spreading as it was before a call to <seealso marker="#spread_tag/1">spread_tag/1</seealso>. Note that the restoring is not limited to the same process, one can utilize this to turn off spreding in one process and restore it in a newly created, the one that actually is going to send messages:</p>
+<pre>
+f() ->
+    TagData=dyntrace:spread_tag(false),
+    spawn(fun() ->
+             dyntrace:restore_tag(TagData),
+             do_something()
+          end),
+    do_something_else(),
+    dyntrace:restore_tag(TagData).
+</pre>
+        <p>Correctly handling user tags and their spreading might take some effort, as Erlang programs tend to send and receive messages so that sometimes the user tag gets lost due to various things, like double receives or communication with a port (ports do not handle user tags, in the same way as they do not handle regular sequential trace tokens).</p>
+      </desc>
+    </func>
+  </funcs>
+ </erlref>
+
diff --git a/lib/runtime_tools/doc/src/ref_man.xml b/lib/runtime_tools/doc/src/ref_man.xml
index 579efcd969..41fa5f17c4 100644
--- a/lib/runtime_tools/doc/src/ref_man.xml
+++ b/lib/runtime_tools/doc/src/ref_man.xml
@@ -33,6 +33,7 @@
   </description>
   <xi:include href="runtime_tools_app.xml"/>
   <xi:include href="dbg.xml"/>
+  <xi:include href="dyntrace.xml"/>
   <xi:include href="erts_alloc_config.xml"/>
 </application>
 
diff --git a/lib/runtime_tools/examples/dist.d b/lib/runtime_tools/examples/dist.d
new file mode 100644
index 0000000000..550e10d363
--- /dev/null
+++ b/lib/runtime_tools/examples/dist.d
@@ -0,0 +1,62 @@
+/* example usage: dtrace -q -s /path/to/dist.d */
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Scott Lystig Fritchie 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+erlang*:::dist-monitor
+{
+    printf("monitor: pid %d, who %s, what %s, node %s, type %s, reason %s\n",
+           pid,
+           copyinstr(arg0), copyinstr(arg1), copyinstr(arg2), copyinstr(arg3),
+           copyinstr(arg4));
+}
+
+erlang*:::dist-port_busy
+{
+    printf("dist port_busy: node %s, port %s, remote_node %s, blocked pid %s\n",
+           copyinstr(arg0), copyinstr(arg1), copyinstr(arg2), copyinstr(arg3));
+    /*
+     * For variable use advice, see:
+     *    http://dtrace.org/blogs/brendan/2011/11/25/dtrace-variable-types/
+     *
+     * Howevever, it's quite possible for the blocked events to span
+     * threads, so we'll use globals.
+     */
+    blocked_procs[copyinstr(arg3)] = timestamp;
+}
+
+erlang*:::dist-output
+{
+    printf("dist output: node %s, port %s, remote_node %s bytes %d\n",
+           copyinstr(arg0), copyinstr(arg1), copyinstr(arg2), arg3);
+}
+
+erlang*:::dist-outputv
+{
+    printf("port outputv: node %s, port %s, remote_node %s bytes %d\n",
+           copyinstr(arg0), copyinstr(arg1), copyinstr(arg2), arg3);
+}
+
+erlang*:::process-scheduled
+/blocked_procs[copyinstr(arg0)]/
+{
+    pidstr = copyinstr(arg0);
+    printf("blocked pid %s scheduled now, waited %d microseconds\n",
+           pidstr, (timestamp - blocked_procs[pidstr]) / 1000);
+    blocked_procs[pidstr] = 0;
+}
diff --git a/lib/runtime_tools/examples/dist.systemtap b/lib/runtime_tools/examples/dist.systemtap
new file mode 100644
index 0000000000..af27b2cab6
--- /dev/null
+++ b/lib/runtime_tools/examples/dist.systemtap
@@ -0,0 +1,76 @@
+/* example usage: stap /path/to/dist.systemtap -x <pid> */
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Scott Lystig Fritchie and Andreas Schultz, 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+/*
+ * Note: This file assumes that you're using the non-SMP-enabled Erlang
+ *       virtual machine, "beam".  The SMP-enabled VM is called "beam.smp".
+ *       Note that other variations of the virtual machine also have
+ *       different names, e.g. the debug build of the SMP-enabled VM
+ *       is "beam.debug.smp".
+ *
+ *       To use a different virtual machine, replace each instance of
+ *       "beam" with "beam.smp" or the VM name appropriate to your
+ *       environment.
+ */
+
+probe process("beam").mark("dist-monitor")
+{
+    printf("monitor: pid %d, who %s, what %s, node %s, type %s, reason %s\n",
+           pid(),
+           user_string($arg1), user_string($arg2), user_string($arg3), user_string($arg4),
+           user_string($arg5));
+}
+
+probe process("beam").mark("dist-port_busy")
+{
+    printf("dist port_busy: node %s, port %s, remote_node %s, blocked pid %s\n",
+           user_string($arg1), user_string($arg2), user_string($arg3), user_string($arg4));
+    blocked_procs[user_string($arg4)] = timestamp;
+}
+
+probe process("beam").mark("dist-port_busy")
+{
+    printf("dist port_busy: node %s, port %s, remote_node %s, blocked pid %s\n",
+           user_string($arg1), user_string($arg2), user_string($arg3), user_string($arg4));
+    blocked_procs[user_string($arg4)] = timestamp;
+}
+
+probe process("beam").mark("dist-output")
+{
+    printf("dist output: node %s, port %s, remote_node %s bytes %d\n",
+           user_string($arg1), user_string($arg2), user_string($arg3), $arg4);
+}
+
+probe process("beam").mark("dist-outputv")
+{
+    printf("port outputv: node %s, port %s, remote_node %s bytes %d\n",
+           user_string($arg1), user_string($arg2), user_string($arg3), $arg4);
+}
+
+probe process("beam").mark("process-scheduled")
+{
+    pidstr = user_string($arg1);
+    if (pidstr in blocked_procs) {
+	printf("blocked pid %s scheduled now, waited %d microseconds\n",
+		pidstr, (timestamp - blocked_procs[pidstr]) / 1000);
+	delete blocked_procs[pidstr];
+    }
+}
+
+global blocked_procs;
diff --git a/lib/runtime_tools/examples/driver1.d b/lib/runtime_tools/examples/driver1.d
new file mode 100644
index 0000000000..9f53ffeb2a
--- /dev/null
+++ b/lib/runtime_tools/examples/driver1.d
@@ -0,0 +1,114 @@
+/* example usage: dtrace -q -s /path/to/driver1.d */
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Scott Lystig Fritchie 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+erlang*:::driver-init
+{
+    printf("driver init name %s major %d minor %d flags %d\n",
+	   copyinstr(arg0), arg1, arg2, arg3);
+}
+
+erlang*:::driver-start
+{
+    printf("driver start pid %s driver name %s port %s\n",
+	   copyinstr(arg0), copyinstr(arg1), copyinstr(arg2));
+}
+
+erlang*:::driver-stop
+{
+    printf("driver stop pid %s driver name %s port %s\n",
+	   copyinstr(arg0), copyinstr(arg1), copyinstr(arg2));
+}
+
+erlang*:::driver-finish
+{
+    printf("driver finish driver name %s port %s\n",
+           copyinstr(arg0), copyinstr(arg1));
+}
+
+erlang*:::driver-flush
+{
+    printf("driver flush pid %s port %s port name %s\n",
+	   copyinstr(arg0), copyinstr(arg1), copyinstr(arg2));
+}
+
+erlang*:::driver-output
+{
+    printf("driver output pid %s port %s port name %s bytes %d\n",
+	   copyinstr(arg0), copyinstr(arg1), copyinstr(arg2), arg3);
+}
+
+erlang*:::driver-outputv
+{
+    printf("driver outputv pid %s port %s port name %s bytes %d\n",
+	   copyinstr(arg0), copyinstr(arg1), copyinstr(arg2), arg3);
+}
+
+erlang*:::driver-control
+{
+    printf("driver control pid %s port %s port name %s command %d bytes %d\n",
+	   copyinstr(arg0), copyinstr(arg1), copyinstr(arg2), arg3, arg4);
+}
+
+erlang*:::driver-call
+{
+    printf("driver call pid %s port %s port name %s command %d bytes %d\n",
+	   copyinstr(arg0), copyinstr(arg1), copyinstr(arg2), arg3, arg4);
+}
+
+erlang*:::driver-event
+{
+    printf("driver event pid %s port %s port name %s\n",
+	   copyinstr(arg0), copyinstr(arg1), copyinstr(arg2));
+}
+
+erlang*:::driver-ready_input
+{
+    printf("driver ready_input pid %s port %s port name %s\n",
+	   copyinstr(arg0), copyinstr(arg1), copyinstr(arg2));
+}
+
+erlang*:::driver-ready_output
+{
+    printf("driver ready_output pid %s port %s port name %s\n",
+	   copyinstr(arg0), copyinstr(arg1), copyinstr(arg2));
+}
+
+erlang*:::driver-timeout
+{
+    printf("driver timeout pid %s port %s port name %s\n",
+	   copyinstr(arg0), copyinstr(arg1), copyinstr(arg2));
+}
+
+erlang*:::driver-ready_async
+{
+    printf("driver ready_async pid %s port %s port name %s\n",
+	   copyinstr(arg0), copyinstr(arg1), copyinstr(arg2));
+}
+
+erlang*:::driver-process_exit
+{
+    printf("driver process_exit pid %s port %s port name %s\n",
+	   copyinstr(arg0), copyinstr(arg1), copyinstr(arg2));
+}
+
+erlang*:::driver-stop_select
+{
+    printf("driver stop_select driver name %s\n", copyinstr(arg0));
+}
diff --git a/lib/runtime_tools/examples/driver1.systemtap b/lib/runtime_tools/examples/driver1.systemtap
new file mode 100644
index 0000000000..8b99e465b7
--- /dev/null
+++ b/lib/runtime_tools/examples/driver1.systemtap
@@ -0,0 +1,125 @@
+/* example usage: stap /path/to/driver1.systemtap -x <pid> */
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Scott Lystig Fritchie and Andreas Schultz, 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+/*
+ * Note: This file assumes that you're using the non-SMP-enabled Erlang
+ *       virtual machine, "beam".  The SMP-enabled VM is called "beam.smp".
+ *       Note that other variations of the virtual machine also have
+ *       different names, e.g. the debug build of the SMP-enabled VM
+ *       is "beam.debug.smp".
+ *
+ *       To use a different virtual machine, replace each instance of
+ *       "beam" with "beam.smp" or the VM name appropriate to your
+ *       environment.
+ */
+
+probe process("beam").mark("driver-init")
+{
+    printf("driver init name %s major %d minor %d flags %d\n",
+	   user_string($arg1), $arg2, $arg3, $arg4);
+}
+
+probe process("beam").mark("driver-start")
+{
+    printf("driver start pid %s driver name %s port %s\n",
+	   user_string($arg1), user_string($arg2), user_string($arg3));
+}
+
+probe process("beam").mark("driver-stop")
+{
+    printf("driver stop pid %s driver name %s port %s\n",
+	   user_string($arg1), user_string($arg2), user_string($arg3));
+}
+
+probe process("beam").mark("driver-finish")
+{
+    printf("driver finish driver name %s\n",
+           user_string($arg1));
+}
+
+probe process("beam").mark("driver-flush")
+{
+    printf("driver flush pid %s port %s port name %s\n",
+	   user_string($arg1), user_string($arg2), user_string($arg3));
+}
+
+probe process("beam").mark("driver-output")
+{
+    printf("driver output pid %s port %s port name %s bytes %d\n",
+	   user_string($arg1), user_string($arg2), user_string($arg3), $arg4);
+}
+
+probe process("beam").mark("driver-outputv")
+{
+    printf("driver outputv pid %s port %s port name %s bytes %d\n",
+	   user_string($arg1), user_string($arg2), user_string($arg3), $arg4);
+}
+
+probe process("beam").mark("driver-control")
+{
+    printf("driver control pid %s port %s port name %s command %d bytes %d\n",
+	   user_string($arg1), user_string($arg2), user_string($arg3), $arg4, $arg5);
+}
+
+probe process("beam").mark("driver-call")
+{
+    printf("driver call pid %s port %s port name %s command %d bytes %d\n",
+	   user_string($arg1), user_string($arg2), user_string($arg3), $arg4, $arg5);
+}
+
+probe process("beam").mark("driver-event")
+{
+    printf("driver event pid %s port %s port name %s\n",
+	   user_string($arg1), user_string($arg2), user_string($arg3));
+}
+
+probe process("beam").mark("driver-ready_input")
+{
+    printf("driver ready_input pid %s port %s port name %s\n",
+	   user_string($arg1), user_string($arg2), user_string($arg3));
+}
+
+probe process("beam").mark("driver-ready_output")
+{
+    printf("driver ready_output pid %s port %s port name %s\n",
+	   user_string($arg1), user_string($arg2), user_string($arg3));
+}
+
+probe process("beam").mark("driver-timeout")
+{
+    printf("driver timeout pid %s port %s port name %s\n",
+	   user_string($arg1), user_string($arg2), user_string($arg3));
+}
+
+probe process("beam").mark("driver-ready_async")
+{
+    printf("driver ready_async pid %s port %s port name %s\n",
+	   user_string($arg1), user_string($arg2), user_string($arg3));
+}
+
+probe process("beam").mark("driver-process_exit")
+{
+    printf("driver process_exit pid %s port %s port name %s\n",
+	   user_string($arg1), user_string($arg2), user_string($arg3));
+}
+
+probe process("beam").mark("driver-stop_select")
+{
+    printf("driver stop_select driver name %s\n", user_string($arg1));
+}
diff --git a/lib/runtime_tools/examples/efile_drv.d b/lib/runtime_tools/examples/efile_drv.d
new file mode 100644
index 0000000000..085995ce58
--- /dev/null
+++ b/lib/runtime_tools/examples/efile_drv.d
@@ -0,0 +1,104 @@
+/* example usage: dtrace -q -s /path/to/efile_drv.d */
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Scott Lystig Fritchie 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+BEGIN
+{
+    op_map[1] = "OPEN";
+    op_map[2] = "READ";
+    op_map[3] = "LSEEK";
+    op_map[4] = "WRITE";
+    op_map[5] = "FSTAT";
+    op_map[6] = "PWD";
+    op_map[7] = "READDIR";
+    op_map[8] = "CHDIR";
+    op_map[9] = "FSYNC";
+    op_map[10] = "MKDIR";
+    op_map[11] = "DELETE";
+    op_map[12] = "RENAME";
+    op_map[13] = "RMDIR";
+    op_map[14] = "TRUNCATE";
+    op_map[15] = "READ_FILE";
+    op_map[16] = "WRITE_INFO";
+    op_map[19] = "LSTAT";
+    op_map[20] = "READLINK";
+    op_map[21] = "LINK";
+    op_map[22] = "SYMLINK";
+    op_map[23] = "CLOSE";
+    op_map[24] = "PWRITEV";
+    op_map[25] = "PREADV";
+    op_map[26] = "SETOPT";
+    op_map[27] = "IPREAD";
+    op_map[28] = "ALTNAME";
+    op_map[29] = "READ_LINE";
+    op_map[30] = "FDATASYNC";
+    op_map[31] = "FADVISE";
+}
+
+erlang*:::aio_pool-add
+{
+    printf("async I/O pool port %s queue len %d\n", copyinstr(arg0), arg1);
+}
+
+erlang*:::aio_pool-get
+{
+    printf("async I/O pool port %s queue len %d\n", copyinstr(arg0), arg1);
+}
+
+erlang*:::efile_drv-entry
+{
+    printf("efile_drv enter tag={%d,%d} %s%s | %s (%d) | args: %s %s , %d %d (port %s)\n",
+	   arg0, arg1,
+           arg2 == NULL ? "" : "user tag ",
+           arg2 == NULL ? "" : copyinstr(arg2),
+           op_map[arg3], arg3,
+	   arg4 == NULL ? "" : copyinstr(arg4),
+	   arg5 == NULL ? "" : copyinstr(arg5), arg6, arg7,
+           /* NOTE: port name in args[10] is experimental */
+           (args[10] == NULL) ? 
+	       "?" : copyinstr((user_addr_t) args[10]));
+}
+
+erlang*:::efile_drv-int*
+{
+    printf("async I/O worker tag={%d,%d} | %s (%d) | %s\n",
+           arg0, arg1, op_map[arg2], arg2, probename);
+}
+
+/* efile_drv-return error case */
+erlang*:::efile_drv-return
+/arg4 == 0/
+{
+    printf("efile_drv return tag={%d,%d} %s%s | %s (%d) | errno %d\n",
+           arg0, arg1,
+           arg2 == NULL ? "" : "user tag ",
+           arg2 == NULL ? "" : copyinstr(arg2),
+           op_map[arg3], arg3,
+           arg5);
+}
+
+/* efile_drv-return success case */
+erlang*:::efile_drv-return
+/arg4 != 0/
+{
+    printf("efile_drv return tag={%d,%d} %s | %s (%d) ok\n",
+           arg0, arg1,
+           arg2 == NULL ? "" : copyinstr(arg2),
+           op_map[arg3], arg3);
+}
diff --git a/lib/runtime_tools/examples/efile_drv.systemtap b/lib/runtime_tools/examples/efile_drv.systemtap
new file mode 100644
index 0000000000..5a47b3e22b
--- /dev/null
+++ b/lib/runtime_tools/examples/efile_drv.systemtap
@@ -0,0 +1,112 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Scott Lystig Fritchie and Andreas Schultz, 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+/*
+ * Note: This file assumes that you're using the non-SMP-enabled Erlang
+ *       virtual machine, "beam".  The SMP-enabled VM is called "beam.smp".
+ *       Note that other variations of the virtual machine also have
+ *       different names, e.g. the debug build of the SMP-enabled VM
+ *       is "beam.debug.smp".
+ *
+ *       To use a different virtual machine, replace each instance of
+ *       "beam" with "beam.smp" or the VM name appropriate to your
+ *       environment.
+ */
+
+probe begin
+{
+    op_map[1] = "OPEN";
+    op_map[2] = "READ";
+    op_map[3] = "LSEEK";
+    op_map[4] = "WRITE";
+    op_map[5] = "FSTAT";
+    op_map[6] = "PWD";
+    op_map[7] = "READDIR";
+    op_map[8] = "CHDIR";
+    op_map[9] = "FSYNC";
+    op_map[10] = "MKDIR";
+    op_map[11] = "DELETE";
+    op_map[12] = "RENAME";
+    op_map[13] = "RMDIR";
+    op_map[14] = "TRUNCATE";
+    op_map[15] = "READ_FILE";
+    op_map[16] = "WRITE_INFO";
+    op_map[19] = "LSTAT";
+    op_map[20] = "READLINK";
+    op_map[21] = "LINK";
+    op_map[22] = "SYMLINK";
+    op_map[23] = "CLOSE";
+    op_map[24] = "PWRITEV";
+    op_map[25] = "PREADV";
+    op_map[26] = "SETOPT";
+    op_map[27] = "IPREAD";
+    op_map[28] = "ALTNAME";
+    op_map[29] = "READ_LINE";
+    op_map[30] = "FDATASYNC";
+    op_map[31] = "FADVISE";
+}
+
+probe process("beam").mark("aio_pool-add")
+{
+    printf("async I/O pool port %s queue len %d\n", user_string($arg1), $arg2);
+}
+
+probe process("beam").mark("aio_pool-get")
+{
+    printf("async I/O pool port %s queue len %d\n", user_string($arg1), $arg2);
+}
+
+probe process("beam").mark("efile_drv-entry")
+{
+    printf("efile_drv enter tag={%d,%d} %s%s | %s (%d) | args: %s %s , %d %d (port %s)\n",
+	   $arg1, $arg2,
+           $arg3 == NULL ? "" : "user tag ",
+           $arg3 == NULL ? "" : user_string($arg3),
+           op_map[$arg4], $arg4,
+	   $arg5 == NULL ? "" : user_string($arg5),
+	   $arg6 == NULL ? "" : user_string($arg6), $arg7, $arg8,
+           /* NOTE: port name in $arg[11] is experimental */
+           user_string($arg11))
+}
+
+probe process("beam").mark("efile_drv-int*")
+{
+    printf("async I/O worker tag={%d,%d} | %s (%d) | %s\n",
+           $arg1, $arg2, op_map[$arg3], $arg3, probefunc());
+}
+
+probe process("beam").mark("efile_drv-return")
+{
+    if ($arg5 == 0) {
+       /* efile_drv-return error case */
+       printf("efile_drv return tag={%d,%d} %s%s | %s (%d) | errno %d\n",
+           $arg1, $arg2,
+           $arg3 == NULL ? "" : "user tag ",
+           $arg3 == NULL ? "" : user_string($arg3),
+           op_map[$arg4], $arg4,
+           $arg6);
+    } else {
+       /* efile_drv-return success case */
+       printf("efile_drv return tag={%d,%d} %s | %s (%d) ok\n",
+           $arg1, $arg2,
+           $arg3 == NULL ? "" : user_string($arg3),
+           op_map[$arg4], $arg4);
+    }
+}
+
+global op_map;
diff --git a/lib/runtime_tools/examples/function-calls.d b/lib/runtime_tools/examples/function-calls.d
new file mode 100644
index 0000000000..3e015dc2d2
--- /dev/null
+++ b/lib/runtime_tools/examples/function-calls.d
@@ -0,0 +1,57 @@
+/* example usage: dtrace -q -s /path/to/function-calls.d */
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Scott Lystig Fritchie 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+erlang*:::local-function-entry
+{
+    printf("pid %s enter (local)  %s depth %d\n",
+	   copyinstr(arg0), copyinstr(arg1), arg2);
+}
+
+erlang*:::global-function-entry
+{
+    printf("pid %s enter (global)  %s depth %d\n",
+	   copyinstr(arg0), copyinstr(arg1), arg2);
+}
+
+erlang*:::function-return
+{
+    printf("pid %s return %s depth %d\n",
+	   copyinstr(arg0), copyinstr(arg1), arg2);
+}
+
+erlang*:::bif-entry
+{
+    printf("pid %s BIF entry  mfa %s\n", copyinstr(arg0), copyinstr(arg1));
+}
+
+erlang*:::bif-return
+{
+    printf("pid %s BIF return mfa %s\n", copyinstr(arg0), copyinstr(arg1));
+}
+
+erlang*:::nif-entry
+{
+    printf("pid %s NIF entry  mfa %s\n", copyinstr(arg0), copyinstr(arg1));
+}
+
+erlang*:::nif-return
+{
+    printf("pid %s NIF return mfa %s\n", copyinstr(arg0), copyinstr(arg1));
+}
diff --git a/lib/runtime_tools/examples/function-calls.systemtap b/lib/runtime_tools/examples/function-calls.systemtap
new file mode 100644
index 0000000000..16c9c04918
--- /dev/null
+++ b/lib/runtime_tools/examples/function-calls.systemtap
@@ -0,0 +1,67 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Scott Lystig Fritchie and Andreas Schultz, 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+/*
+ * Note: This file assumes that you're using the non-SMP-enabled Erlang
+ *       virtual machine, "beam".  The SMP-enabled VM is called "beam.smp".
+ *       Note that other variations of the virtual machine also have
+ *       different names, e.g. the debug build of the SMP-enabled VM
+ *       is "beam.debug.smp".
+ *
+ *       To use a different virtual machine, replace each instance of
+ *       "beam" with "beam.smp" or the VM name appropriate to your
+ *       environment.
+ */
+
+probe process("beam").mark("local-function-entry")
+{
+    printf("pid %s enter (local) %s depth %d\n",
+	   user_string($arg1), user_string($arg2), $arg3);
+}
+
+probe process("beam").mark("global-function-entry")
+{
+    printf("pid %s enter (global)  %s depth %d\n",
+	   user_string($arg1), user_string($arg2), $arg3);
+}
+
+probe process("beam").mark("function-return")
+{
+    printf("pid %s return %s depth %d\n",
+	   user_string($arg1), user_string($arg2), $arg3);
+}
+
+probe process("beam").mark("bif-entry")
+{
+    printf("pid %s BIF entry  mfa %s\n", user_string($arg1), user_string($arg2));
+}
+
+probe process("beam").mark("bif-return")
+{
+    printf("pid %s BIF return mfa %s\n", user_string($arg1), user_string($arg2));
+}
+
+probe process("beam").mark("nif-entry")
+{
+    printf("pid %s NIF entry  mfa %s\n", user_string($arg1), user_string($arg2));
+}
+
+probe process("beam").mark("nif-return")
+{
+    printf("pid %s NIF return mfa %s\n", user_string($arg1), user_string($arg2));
+}
diff --git a/lib/runtime_tools/examples/garbage-collection.d b/lib/runtime_tools/examples/garbage-collection.d
new file mode 100644
index 0000000000..f234e7d4db
--- /dev/null
+++ b/lib/runtime_tools/examples/garbage-collection.d
@@ -0,0 +1,39 @@
+/* example usage: dtrace -q -s /path/to/garbage-collection.d */
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Scott Lystig Fritchie 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+erlang*:::gc_major-start
+{
+    printf("GC major start pid %s need %d words\n", copyinstr(arg0), arg1);
+}
+
+erlang*:::gc_minor-start
+{
+    printf("GC minor start pid %s need %d words\n", copyinstr(arg0), arg1);
+}
+
+erlang*:::gc_major-end
+{
+    printf("GC major end pid %s reclaimed %d words\n", copyinstr(arg0), arg1);
+}
+
+erlang*:::gc_minor-start
+{
+    printf("GC minor end pid %s reclaimed %d words\n", copyinstr(arg0), arg1);
+}
diff --git a/lib/runtime_tools/examples/garbage-collection.systemtap b/lib/runtime_tools/examples/garbage-collection.systemtap
new file mode 100644
index 0000000000..64d69c6fbd
--- /dev/null
+++ b/lib/runtime_tools/examples/garbage-collection.systemtap
@@ -0,0 +1,49 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Scott Lystig Fritchie and Andreas Schultz, 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+/*
+ * Note: This file assumes that you're using the non-SMP-enabled Erlang
+ *       virtual machine, "beam".  The SMP-enabled VM is called "beam.smp".
+ *       Note that other variations of the virtual machine also have
+ *       different names, e.g. the debug build of the SMP-enabled VM
+ *       is "beam.debug.smp".
+ *
+ *       To use a different virtual machine, replace each instance of
+ *       "beam" with "beam.smp" or the VM name appropriate to your
+ *       environment.
+ */
+
+probe process("beam").mark("gc_major-start")
+{
+    printf("GC major start pid %s need %d words\n", user_string($arg1), $arg2);
+}
+
+probe process("beam").mark("gc_minor-start")
+{
+    printf("GC minor start pid %s need %d words\n", user_string($arg1), $arg2);
+}
+
+probe process("beam").mark("gc_major-end")
+{
+    printf("GC major end pid %s reclaimed %d words\n", user_string($arg1), $arg2);
+}
+
+probe process("beam").mark("gc_minor-start")
+{
+    printf("GC minor end pid %s reclaimed %d words\n", user_string($arg1), $arg2);
+}
diff --git a/lib/runtime_tools/examples/memory1.d b/lib/runtime_tools/examples/memory1.d
new file mode 100644
index 0000000000..c2e16e0779
--- /dev/null
+++ b/lib/runtime_tools/examples/memory1.d
@@ -0,0 +1,41 @@
+/* example usage: dtrace -q -s /path/to/memory1.d */
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Scott Lystig Fritchie 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+erlang*:::copy-struct
+{
+    printf("copy_struct %d bytes\n", arg0);
+}
+
+erlang*:::copy-object
+{
+    printf("copy_object pid %s %d bytes\n", copyinstr(arg0), arg1);
+}
+
+erlang*:::process-heap_grow
+{
+    printf("proc heap grow pid %s %d -> %d bytes\n", copyinstr(arg0),
+	   arg1, arg2);
+}
+
+erlang*:::process-heap_shrink
+{
+    printf("proc heap shrink pid %s %d -> %d bytes\n", copyinstr(arg0),
+	   arg1, arg2);
+}
diff --git a/lib/runtime_tools/examples/memory1.systemtap b/lib/runtime_tools/examples/memory1.systemtap
new file mode 100644
index 0000000000..9723f2d02d
--- /dev/null
+++ b/lib/runtime_tools/examples/memory1.systemtap
@@ -0,0 +1,51 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Scott Lystig Fritchie and Andreas Schultz, 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+/*
+ * Note: This file assumes that you're using the non-SMP-enabled Erlang
+ *       virtual machine, "beam".  The SMP-enabled VM is called "beam.smp".
+ *       Note that other variations of the virtual machine also have
+ *       different names, e.g. the debug build of the SMP-enabled VM
+ *       is "beam.debug.smp".
+ *
+ *       To use a different virtual machine, replace each instance of
+ *       "beam" with "beam.smp" or the VM name appropriate to your
+ *       environment.
+ */
+
+probe process("beam").mark("copy-struct")
+{
+    printf("copy_struct %d bytes\n", $arg1);
+}
+
+probe process("beam").mark("copy-object")
+{
+    printf("copy_object pid %s %d bytes\n", user_string($arg1), $arg2);
+}
+
+probe process("beam").mark("process-heap_grow")
+{
+    printf("proc heap grow pid %s %d -> %d bytes\n", user_string($arg1),
+	   $arg2, $arg3);
+}
+
+probe process("beam").mark("process-heap_shrink")
+{
+    printf("proc heap shrink pid %s %d -> %d bytes\n", user_string($arg1),
+	   $arg2, $arg3);
+}
diff --git a/lib/runtime_tools/examples/messages.d b/lib/runtime_tools/examples/messages.d
new file mode 100644
index 0000000000..6361f3a220
--- /dev/null
+++ b/lib/runtime_tools/examples/messages.d
@@ -0,0 +1,94 @@
+/* example usage: dtrace -q -s /path/to/messages.d */
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Scott Lystig Fritchie 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+BEGIN
+{
+    printf("\n");
+    printf("NOTE: message-queue message size 4294967295 means an external\n");
+    printf("      message that the code isn't smart enough to determine\n");
+    printf("      the actual size.\n");
+    printf("\n");
+}
+
+erlang*:::message-send
+/arg3 == 0 && arg4 == 0 && arg5 == 0/
+{
+    printf("send:   %s -> %s: %d words\n",
+           copyinstr(arg0), copyinstr(arg1), arg2);
+}
+
+erlang*:::message-send
+/arg3 != 0 || arg4 != 0 || arg5 != 0/
+{
+    printf("send:   %s label %d token {%d,%d} -> %s: %d words\n",
+           copyinstr(arg0),
+           arg3, arg4, arg5,
+           copyinstr(arg1), arg2);
+}
+
+/*
+ * TODO:
+ * Weird, on my OS X box, beam says arg6 = 0 but this script says 4294967296.
+ */
+
+erlang*:::message-send-remote
+/arg4 == 0 && arg5 == 0 && (arg6 == 0 || arg6 >= 4294967296)/
+{
+    printf("send :  %s -> %s %s: %d words\n",
+           copyinstr(arg0), copyinstr(arg1), copyinstr(arg2), arg3);
+}
+
+erlang*:::message-send-remote
+/arg4 != 0 || arg5 != 0 || arg6 < 4294967296/
+{
+    printf("send :  %s label %d token {%d,%d} -> %s %s: %d words\n",
+           copyinstr(arg0),
+           arg4, arg5, arg6,
+           copyinstr(arg1), copyinstr(arg2), arg3);
+}
+
+erlang*:::message-queued
+/arg3 == 0 && arg4 == 0 && arg5 == 0/
+{
+    printf("queued: %s: %d words, queue len %d\n", copyinstr(arg0), arg1, arg2);
+}
+
+erlang*:::message-queued
+/arg3 != 0 || arg4 != 0 || arg5 != 0/
+{
+    printf("queued: %s label %d token {%d,%d}: %d words, queue len %d\n",
+           copyinstr(arg0), arg3, arg4, arg5,
+           arg1, arg2);
+}
+
+erlang*:::message-receive
+/arg3 == 0 && arg4 == 0 && arg5 == 0/
+{
+    printf("receive: %s: %d words, queue len %d\n",
+           copyinstr(arg0), arg1, arg2);
+}
+
+erlang*:::message-receive
+/arg3 != 0 || arg4 != 0 || arg5 != 0/
+{
+    printf("receive: %s label %d token {%d,%d}: %d words, queue len %d\n",
+           copyinstr(arg0), arg3, arg4, arg5,
+           arg1, arg2);
+}
diff --git a/lib/runtime_tools/examples/messages.systemtap b/lib/runtime_tools/examples/messages.systemtap
new file mode 100644
index 0000000000..ff8f4076b1
--- /dev/null
+++ b/lib/runtime_tools/examples/messages.systemtap
@@ -0,0 +1,87 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Scott Lystig Fritchie and Andreas Schultz, 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+/*
+ * Note: This file assumes that you're using the non-SMP-enabled Erlang
+ *       virtual machine, "beam".  The SMP-enabled VM is called "beam.smp".
+ *       Note that other variations of the virtual machine also have
+ *       different names, e.g. the debug build of the SMP-enabled VM
+ *       is "beam.debug.smp".
+ *
+ *       To use a different virtual machine, replace each instance of
+ *       "beam" with "beam.smp" or the VM name appropriate to your
+ *       environment.
+ */
+
+probe begin
+{
+    printf("\n");
+    printf("NOTE: message-queue message size 4294967295 means an external\n");
+    printf("      message that the code isn't smart enough to determine\n");
+    printf("      the actual size.\n");
+    printf("\n");
+}
+
+probe process("beam").mark("message-send")
+{
+    if ($arg4 == 0 && $arg5 == 0 && $arg6 == 0) {
+        printf("send:   %s -> %s: %d words\n",
+	       user_string($arg1), user_string($arg2), $arg3);
+    } else {
+        printf("send:   %s label %d token {%d,%d} -> %s: %d words\n",
+	       user_string($arg1),
+               $arg4, $arg5, $arg6,
+	       user_string($arg2), $arg3);
+    }
+}
+
+probe process("beam").mark("message-send-remote")
+{
+    if ($arg5 == 0 && $arg6 == 0 && $arg7 == 0) {
+        printf("send :  %s -> %s %s: %d words\n",
+               user_string($arg1), user_string($arg2), user_string($arg3), $arg4);
+    } else {
+        printf("send :  %s label %d token {%d,%d} -> %s %s: %d words\n",
+               user_string($arg1),
+               $arg5, $arg6, $arg7,
+               user_string($arg2), user_string($arg3), $arg4);
+    }
+}
+
+probe process("beam").mark("message-queued")
+{
+    if ($arg4 == 0 && $arg5 == 0 && $arg6 == 0) {
+        printf("queued: %s: %d words, queue len %d\n", user_string($arg1), $arg2, $arg3);
+    } else {
+        printf("queued: %s label %d token {%d,%d}: %d words, queue len %d\n",
+               user_string($arg1), $arg4, $arg5, $arg6,
+               $arg2, $arg3);
+    }
+}
+
+probe process("beam").mark("message-receive")
+{
+    if ($arg4 == 0 && $arg5 == 0 && $arg6 == 0) {
+        printf("receive: %s: %d words, queue len %d\n",
+               user_string($arg1), $arg2, $arg3);
+    } else {
+        printf("receive: %s label %d token {%d,%d}: %d words, queue len %d\n",
+               user_string($arg1), $arg4, $arg5, $arg6,
+               $arg2, $arg3);
+    }
+}
diff --git a/lib/runtime_tools/examples/port1.d b/lib/runtime_tools/examples/port1.d
new file mode 100644
index 0000000000..204abbd3b8
--- /dev/null
+++ b/lib/runtime_tools/examples/port1.d
@@ -0,0 +1,142 @@
+/* example usage: dtrace -q -s /path/to/port1.d */
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Scott Lystig Fritchie 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+BEGIN
+{
+        driver_map["tcp_inet", 1] = "OPEN";
+        driver_map["tcp_inet", 2] = "CLOSE";
+        driver_map["tcp_inet", 3] = "CONNECT";
+        driver_map["tcp_inet", 4] = "PEER";
+        driver_map["tcp_inet", 5] = "NAME";
+        driver_map["tcp_inet", 6] = "BIND";
+        driver_map["tcp_inet", 7] = "SETOPTS";
+        driver_map["tcp_inet", 8] = "GETOPTS";
+        driver_map["tcp_inet", 11] = "GETSTAT";
+        driver_map["tcp_inet", 12] = "GETHOSTNAME";
+        driver_map["tcp_inet", 13] = "FDOPEN";
+        driver_map["tcp_inet", 14] = "GETFD";
+        driver_map["tcp_inet", 15] = "GETTYPE";
+        driver_map["tcp_inet", 16] = "GETSTATUS";
+        driver_map["tcp_inet", 17] = "GETSERVBYNAME";
+        driver_map["tcp_inet", 18] = "GETSERVBYPORT";
+        driver_map["tcp_inet", 19] = "SETNAME";
+        driver_map["tcp_inet", 20] = "SETPEER";
+        driver_map["tcp_inet", 21] = "GETIFLIST";
+        driver_map["tcp_inet", 22] = "IFGET";
+        driver_map["tcp_inet", 23] = "IFSET";
+        driver_map["tcp_inet", 24] = "SUBSCRIBE";
+        driver_map["tcp_inet", 25] = "GETIFADDRS";
+        driver_map["tcp_inet", 40] = "ACCEPT";
+        driver_map["tcp_inet", 41] = "LISTEN";
+        driver_map["tcp_inet", 42] = "RECV";
+        driver_map["tcp_inet", 43] = "UNRECV";
+        driver_map["tcp_inet", 44] = "SHUTDOWN";
+        driver_map["tcp_inet", 60] = "RECV";
+        driver_map["tcp_inet", 61] = "LISTEN";
+        driver_map["tcp_inet", 62] = "BINDX";
+        /* No looping constructs, so repeat for udp_inet */
+        driver_map["udp_inet", 1] = "OPEN";
+        driver_map["udp_inet", 2] = "CLOSE";
+        driver_map["udp_inet", 3] = "CONNECT";
+        driver_map["udp_inet", 4] = "PEER";
+        driver_map["udp_inet", 5] = "NAME";
+        driver_map["udp_inet", 6] = "BIND";
+        driver_map["udp_inet", 7] = "SETOPTS";
+        driver_map["udp_inet", 8] = "GETOPTS";
+        driver_map["udp_inet", 11] = "GETSTAT";
+        driver_map["udp_inet", 12] = "GETHOSTNAME";
+        driver_map["udp_inet", 13] = "FDOPEN";
+        driver_map["udp_inet", 14] = "GETFD";
+        driver_map["udp_inet", 15] = "GETTYPE";
+        driver_map["udp_inet", 16] = "GETSTATUS";
+        driver_map["udp_inet", 17] = "GETSERVBYNAME";
+        driver_map["udp_inet", 18] = "GETSERVBYPORT";
+        driver_map["udp_inet", 19] = "SETNAME";
+        driver_map["udp_inet", 20] = "SETPEER";
+        driver_map["udp_inet", 21] = "GETIFLIST";
+        driver_map["udp_inet", 22] = "IFGET";
+        driver_map["udp_inet", 23] = "IFSET";
+        driver_map["udp_inet", 24] = "SUBSCRIBE";
+        driver_map["udp_inet", 25] = "GETIFADDRS";
+        driver_map["udp_inet", 40] = "ACCEPT";
+        driver_map["udp_inet", 41] = "LISTEN";
+        driver_map["udp_inet", 42] = "RECV";
+        driver_map["udp_inet", 43] = "UNRECV";
+        driver_map["udp_inet", 44] = "SHUTDOWN";
+        driver_map["udp_inet", 60] = "RECV";
+        driver_map["udp_inet", 61] = "LISTEN";
+        driver_map["udp_inet", 62] = "BINDX";
+}
+
+erlang*:::port-open
+{
+    printf("port open pid %s port name %s port %s\n",
+           copyinstr(arg0), copyinstr(arg1), copyinstr(arg2));
+}
+
+erlang*:::port-command
+{
+    printf("port command pid %s port %s port name %s command type %s\n",
+	   copyinstr(arg0), copyinstr(arg1), copyinstr(arg2), copyinstr(arg3));
+}
+
+erlang*:::port-control
+{
+    /* http://dtrace.org/blogs/brendan/2011/11/25/dtrace-variable-types/ */
+    this->cmd = driver_map[copyinstr(arg2), arg3];
+    this->cmd_str = (this->cmd == 0) ? "unknown" : this->cmd;
+    printf("port control pid %s port %s port name %s command %d %s\n",
+	   copyinstr(arg0), copyinstr(arg1), copyinstr(arg2), arg3,
+           this->cmd_str);
+}
+
+/* port-exit is fired as a result of port_close() or exit signal */
+
+erlang*:::port-exit
+{
+    printf("port exit pid %s port %s port name %s reason %s\n",
+	   copyinstr(arg0), copyinstr(arg1), copyinstr(arg2), copyinstr(arg3));
+}
+
+erlang*:::port-connect
+{
+    printf("port connect pid %s port %s port name %s new pid %s\n",
+	   copyinstr(arg0), copyinstr(arg1), copyinstr(arg2), copyinstr(arg3));
+}
+
+erlang*:::port-busy
+{
+    printf("port busy %s\n", copyinstr(arg0));
+}
+
+erlang*:::port-not_busy
+{
+    printf("port not busy %s\n", copyinstr(arg0));
+}
+
+erlang*:::aio_pool-add
+{
+    printf("async I/O pool add thread %d queue len %d\n", arg0, arg1);
+}
+
+erlang*:::aio_pool-get
+{
+    printf("async I/O pool get thread %d queue len %d\n", arg0, arg1);
+}
diff --git a/lib/runtime_tools/examples/port1.systemtap b/lib/runtime_tools/examples/port1.systemtap
new file mode 100644
index 0000000000..a63d9b670c
--- /dev/null
+++ b/lib/runtime_tools/examples/port1.systemtap
@@ -0,0 +1,152 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Scott Lystig Fritchie and Andreas Schultz, 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+/*
+ * Note: This file assumes that you're using the non-SMP-enabled Erlang
+ *       virtual machine, "beam".  The SMP-enabled VM is called "beam.smp".
+ *       Note that other variations of the virtual machine also have
+ *       different names, e.g. the debug build of the SMP-enabled VM
+ *       is "beam.debug.smp".
+ *
+ *       To use a different virtual machine, replace each instance of
+ *       "beam" with "beam.smp" or the VM name appropriate to your
+ *       environment.
+ */
+
+probe begin
+{
+        driver_map["tcp_inet", 1] = "OPEN";
+        driver_map["tcp_inet", 2] = "CLOSE";
+        driver_map["tcp_inet", 3] = "CONNECT";
+        driver_map["tcp_inet", 4] = "PEER";
+        driver_map["tcp_inet", 5] = "NAME";
+        driver_map["tcp_inet", 6] = "BIND";
+        driver_map["tcp_inet", 7] = "SETOPTS";
+        driver_map["tcp_inet", 8] = "GETOPTS";
+        driver_map["tcp_inet", 11] = "GETSTAT";
+        driver_map["tcp_inet", 12] = "GETHOSTNAME";
+        driver_map["tcp_inet", 13] = "FDOPEN";
+        driver_map["tcp_inet", 14] = "GETFD";
+        driver_map["tcp_inet", 15] = "GETTYPE";
+        driver_map["tcp_inet", 16] = "GETSTATUS";
+        driver_map["tcp_inet", 17] = "GETSERVBYNAME";
+        driver_map["tcp_inet", 18] = "GETSERVBYPORT";
+        driver_map["tcp_inet", 19] = "SETNAME";
+        driver_map["tcp_inet", 20] = "SETPEER";
+        driver_map["tcp_inet", 21] = "GETIFLIST";
+        driver_map["tcp_inet", 22] = "IFGET";
+        driver_map["tcp_inet", 23] = "IFSET";
+        driver_map["tcp_inet", 24] = "SUBSCRIBE";
+        driver_map["tcp_inet", 25] = "GETIFADDRS";
+        driver_map["tcp_inet", 40] = "ACCEPT";
+        driver_map["tcp_inet", 41] = "LISTEN";
+        driver_map["tcp_inet", 42] = "RECV";
+        driver_map["tcp_inet", 43] = "UNRECV";
+        driver_map["tcp_inet", 44] = "SHUTDOWN";
+        driver_map["tcp_inet", 60] = "RECV";
+        driver_map["tcp_inet", 61] = "LISTEN";
+        driver_map["tcp_inet", 62] = "BINDX";
+        /* No looping constructs, so repeat for udp_inet */
+        driver_map["udp_inet", 1] = "OPEN";
+        driver_map["udp_inet", 2] = "CLOSE";
+        driver_map["udp_inet", 3] = "CONNECT";
+        driver_map["udp_inet", 4] = "PEER";
+        driver_map["udp_inet", 5] = "NAME";
+        driver_map["udp_inet", 6] = "BIND";
+        driver_map["udp_inet", 7] = "SETOPTS";
+        driver_map["udp_inet", 8] = "GETOPTS";
+        driver_map["udp_inet", 11] = "GETSTAT";
+        driver_map["udp_inet", 12] = "GETHOSTNAME";
+        driver_map["udp_inet", 13] = "FDOPEN";
+        driver_map["udp_inet", 14] = "GETFD";
+        driver_map["udp_inet", 15] = "GETTYPE";
+        driver_map["udp_inet", 16] = "GETSTATUS";
+        driver_map["udp_inet", 17] = "GETSERVBYNAME";
+        driver_map["udp_inet", 18] = "GETSERVBYPORT";
+        driver_map["udp_inet", 19] = "SETNAME";
+        driver_map["udp_inet", 20] = "SETPEER";
+        driver_map["udp_inet", 21] = "GETIFLIST";
+        driver_map["udp_inet", 22] = "IFGET";
+        driver_map["udp_inet", 23] = "IFSET";
+        driver_map["udp_inet", 24] = "SUBSCRIBE";
+        driver_map["udp_inet", 25] = "GETIFADDRS";
+        driver_map["udp_inet", 40] = "ACCEPT";
+        driver_map["udp_inet", 41] = "LISTEN";
+        driver_map["udp_inet", 42] = "RECV";
+        driver_map["udp_inet", 43] = "UNRECV";
+        driver_map["udp_inet", 44] = "SHUTDOWN";
+        driver_map["udp_inet", 60] = "RECV";
+        driver_map["udp_inet", 61] = "LISTEN";
+        driver_map["udp_inet", 62] = "BINDX";
+}
+
+probe process("beam").mark("port-open")
+{
+    printf("port open pid %s port name %s port %s\n",
+           user_string($arg1), user_string($arg2), user_string($arg3));
+}
+
+probe process("beam").mark("port-command")
+{
+    printf("port command pid %s port %s port name %s command type %s\n",
+	   user_string($arg1), user_string($arg2), user_string($arg3), user_string($arg4));
+}
+
+probe process("beam").mark("port-control")
+{
+    cmd = driver_map[user_string($arg3), $arg4];
+    cmd_str = (cmd == "") ? "unknown" : cmd;
+    printf("port control pid %s port %s port name %s command %d %s\n",
+	   user_string($arg1), user_string($arg2), user_string($arg3), $arg4, cmd_str);
+}
+
+/* port-exit is fired as a result of port_close() or exit signal */
+
+probe process("beam").mark("port-exit")
+{
+    printf("port exit pid %s port %s port name %s reason %s\n",
+	   user_string($arg1), user_string($arg2), user_string($arg3), user_string($arg4));
+}
+
+probe process("beam").mark("port-connect")
+{
+    printf("port connect pid %s port %s port name %s new pid %s\n",
+	   user_string($arg1), user_string($arg2), user_string($arg3), user_string($arg4));
+}
+
+probe process("beam").mark("port-busy")
+{
+    printf("port busy %s\n", user_string($arg1));
+}
+
+probe process("beam").mark("port-not_busy")
+{
+    printf("port not busy %s\n", user_string($arg1));
+}
+
+probe process("beam").mark("aio_pool-add")
+{
+    printf("async I/O pool add thread %d queue len %d\n", $arg1, $arg2);
+}
+
+probe process("beam").mark("aio_pool-get")
+{
+    printf("async I/O pool get thread %d queue len %d\n", $arg1, $arg2);
+}
+
+global driver_map;
\ No newline at end of file
diff --git a/lib/runtime_tools/examples/process-scheduling.d b/lib/runtime_tools/examples/process-scheduling.d
new file mode 100644
index 0000000000..79e9cc598c
--- /dev/null
+++ b/lib/runtime_tools/examples/process-scheduling.d
@@ -0,0 +1,35 @@
+/* example usage: dtrace -q -s /path/to/process-scheduling.d */
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Scott Lystig Fritchie 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+erlang*:::process-scheduled
+{
+    printf("  Schedule pid %s mfa %s\n", copyinstr(arg0), copyinstr(arg1));
+}
+
+erlang*:::process-unscheduled
+{
+    printf("Unschedule pid %s\n", copyinstr(arg0));
+}
+
+erlang*:::process-hibernate
+{
+    printf("  Hibernate pid %s resume mfa %s\n",
+           copyinstr(arg0), copyinstr(arg1));
+}
diff --git a/lib/runtime_tools/examples/process-scheduling.systemtap b/lib/runtime_tools/examples/process-scheduling.systemtap
new file mode 100644
index 0000000000..c8cee60a07
--- /dev/null
+++ b/lib/runtime_tools/examples/process-scheduling.systemtap
@@ -0,0 +1,45 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Scott Lystig Fritchie and Andreas Schultz, 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+/*
+ * Note: This file assumes that you're using the non-SMP-enabled Erlang
+ *       virtual machine, "beam".  The SMP-enabled VM is called "beam.smp".
+ *       Note that other variations of the virtual machine also have
+ *       different names, e.g. the debug build of the SMP-enabled VM
+ *       is "beam.debug.smp".
+ *
+ *       To use a different virtual machine, replace each instance of
+ *       "beam" with "beam.smp" or the VM name appropriate to your
+ *       environment.
+ */
+
+probe process("beam").mark("process-scheduled")
+{
+    printf("  Schedule pid %s mfa %s\n", user_string($arg1), user_string($arg2));
+}
+
+probe process("beam").mark("process-unscheduled")
+{
+    printf("Unschedule pid %s\n", user_string($arg1));
+}
+
+probe process("beam").mark("process-hibernate")
+{
+    printf("  Hibernate pid %s resume mfa %s\n",
+     user_string($arg1), user_string($arg2));
+}
diff --git a/lib/runtime_tools/examples/spawn-exit.d b/lib/runtime_tools/examples/spawn-exit.d
new file mode 100644
index 0000000000..7310f3343d
--- /dev/null
+++ b/lib/runtime_tools/examples/spawn-exit.d
@@ -0,0 +1,41 @@
+/* example usage: dtrace -q -s /path/to/spawn-exit.d */
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Scott Lystig Fritchie 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+erlang*:::process-spawn
+{
+    printf("pid %s mfa %s\n", copyinstr(arg0), copyinstr(arg1));
+}
+
+erlang*:::process-exit
+{
+    printf("pid %s reason %s\n", copyinstr(arg0), copyinstr(arg1));
+}
+
+erlang*:::process-exit_signal
+{
+    printf("sender %s -> pid %s reason %s\n",
+	   copyinstr(arg0), copyinstr(arg1), copyinstr(arg2));
+}
+
+erlang*:::process-exit_signal-remote
+{
+    printf("sender %s -> node %s pid %s reason %s\n",
+	   copyinstr(arg0), copyinstr(arg1), copyinstr(arg2), copyinstr(arg3));
+}
diff --git a/lib/runtime_tools/examples/spawn-exit.systemtap b/lib/runtime_tools/examples/spawn-exit.systemtap
new file mode 100644
index 0000000000..5e3be9fc1b
--- /dev/null
+++ b/lib/runtime_tools/examples/spawn-exit.systemtap
@@ -0,0 +1,51 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Scott Lystig Fritchie and Andreas Schultz, 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+/*
+ * Note: This file assumes that you're using the non-SMP-enabled Erlang
+ *       virtual machine, "beam".  The SMP-enabled VM is called "beam.smp".
+ *       Note that other variations of the virtual machine also have
+ *       different names, e.g. the debug build of the SMP-enabled VM
+ *       is "beam.debug.smp".
+ *
+ *       To use a different virtual machine, replace each instance of
+ *       "beam" with "beam.smp" or the VM name appropriate to your
+ *       environment.
+ */
+
+probe process("beam").mark("process-spawn")
+{
+    printf("pid %s mfa %s\n", user_string($arg1), user_string($arg2));
+}
+
+probe process("beam").mark("process-exit")
+{
+    printf("pid %s reason %s\n", user_string($arg1), user_string($arg2));
+}
+
+probe process("beam").mark("process-exit_signal")
+{
+    printf("sender %s -> pid %s reason %s\n",
+	   user_string($arg1), user_string($arg2), user_string($arg3));
+}
+
+probe process("beam").mark("process-exit_signal-remote")
+{
+    printf("sender %s -> node %s pid %s reason %s\n",
+	   user_string($arg1), user_string($arg2), user_string($arg3), user_string($arg4));
+}
diff --git a/lib/runtime_tools/examples/user-probe.d b/lib/runtime_tools/examples/user-probe.d
new file mode 100644
index 0000000000..13baff6a32
--- /dev/null
+++ b/lib/runtime_tools/examples/user-probe.d
@@ -0,0 +1,36 @@
+/* example usage: dtrace -q -s /path/to/user-probe.d */
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Scott Lystig Fritchie 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+erlang*:::user_trace-s1
+{
+    printf("%s\n", copyinstr(arg0));
+}
+
+erlang*:::user_trace-i4s4
+{
+    printf("%s %s %d %d %d %d '%s' '%s' '%s' '%s'\n",
+           copyinstr(arg0),
+           arg1 == NULL ? "" : copyinstr(arg1),
+           arg2, arg3, arg4, arg5,
+           arg6 == NULL ? "" : copyinstr(arg6),
+           arg7 == NULL ? "" : copyinstr(arg7),
+           arg8 == NULL ? "" : copyinstr(arg8),
+           arg9 == NULL ? "" : copyinstr(arg9));
+}
diff --git a/lib/runtime_tools/examples/user-probe.systemtap b/lib/runtime_tools/examples/user-probe.systemtap
new file mode 100644
index 0000000000..84a45709e8
--- /dev/null
+++ b/lib/runtime_tools/examples/user-probe.systemtap
@@ -0,0 +1,46 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Scott Lystig Fritchie and Andreas Schultz, 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+/*
+ * Note: This file assumes that you're using the non-SMP-enabled Erlang
+ *       virtual machine, "beam".  The SMP-enabled VM is called "beam.smp".
+ *       Note that other variations of the virtual machine also have
+ *       different names, e.g. the debug build of the SMP-enabled VM
+ *       is "beam.debug.smp".
+ *
+ *       To use a different virtual machine, replace each instance of
+ *       "beam" with "beam.smp" or the VM name appropriate to your
+ *       environment.
+ */
+
+probe process("dyntrace.so").mark("user_trace-s1")
+{
+    printf("%s\n", user_string($arg1));
+}
+
+probe process("dyntrace.so").mark("user_trace-i4s4")
+{
+    printf("%s %s %d %d %d %d '%s' '%s' '%s' '%s'\n",
+           user_string($arg1),
+           $arg2 == NULL ? "" : user_string($arg2),
+           $arg3, $arg4, $arg5, $arg6,
+           $arg7 == NULL ? "" : user_string($arg7),
+           $arg8 == NULL ? "" : user_string($arg8),
+           $arg9 == NULL ? "" : user_string($arg9),
+           $arg9 == NULL ? "" : user_string($arg9));
+}
diff --git a/lib/runtime_tools/src/Makefile b/lib/runtime_tools/src/Makefile
index 946409b262..6dc89bea32 100644
--- a/lib/runtime_tools/src/Makefile
+++ b/lib/runtime_tools/src/Makefile
@@ -45,6 +45,7 @@ MODULES= \
 	runtime_tools \
 	runtime_tools_sup \
 	dbg \
+	dyntrace \
 	percept_profile \
 	observer_backend \
 	ttb_autostart
@@ -64,10 +65,15 @@ APPUP_FILE= runtime_tools.appup
 APPUP_SRC= $(APPUP_FILE).src
 APPUP_TARGET= $(EBIN)/$(APPUP_FILE)
 
+EXAMPLE_FILES= \
+	../examples/*
 # ----------------------------------------------------
 # FLAGS
 # ----------------------------------------------------
-ERL_COMPILE_FLAGS += -I../include 
+ERL_COMPILE_FLAGS += \
+	-I../include \
+	-I ../../et/include \
+	-I ../../../libraries/et/include
 
 # ----------------------------------------------------
 # Targets
@@ -97,6 +103,8 @@ release_spec: opt
 	$(INSTALL_DATA) $(ERL_FILES) $(RELSYSDIR)/src
 	$(INSTALL_DIR) $(RELSYSDIR)/include
 	$(INSTALL_DATA) $(HRL_FILES) $(RELSYSDIR)/include
+	$(INSTALL_DIR) $(RELSYSDIR)/examples
+	$(INSTALL_DATA) $(EXAMPLE_FILES) $(RELSYSDIR)/examples
 	$(INSTALL_DIR) $(RELSYSDIR)/ebin
 	$(INSTALL_DATA) $(TARGET_FILES) $(RELSYSDIR)/ebin
 
diff --git a/lib/runtime_tools/src/dyntrace.erl b/lib/runtime_tools/src/dyntrace.erl
new file mode 100644
index 0000000000..388c7679b9
--- /dev/null
+++ b/lib/runtime_tools/src/dyntrace.erl
@@ -0,0 +1,279 @@
+-module(dyntrace).
+
+%%% @doc The Dynamic tracing interface module
+%%%
+%%% This Dynamic tracing interface module, with the corresponding NIFs, should
+%%% work on any operating system platform where user-space DTrace/Systemtap 
+%%% (and in the future LttNG UST) probes are supported.
+%%%
+%%% Use the `dyntrace:init()' function to load the NIF shared library and
+%%% to initialize library's private state.
+%%%
+%%% It is recommended that you use the `dyntrace:p()' function to add
+%%% Dynamic trace probes to your Erlang code.  This function can accept up to
+%%% four integer arguments and four string arguments; the integer
+%%% argument(s) must come before any string argument.  For example:
+%%% ```
+%%% 1> dyntrace:put_tag("GGOOOAAALL!!!!!").
+%%% true
+%%% 2> dyntrace:init().
+%%% ok
+%%%
+%%% % % % If using dtrace, enable the Dynamic trace probe using the 'dtrace' 
+%%% % % % command.
+%%%
+%%% 3> dyntrace:p(7, 8, 9, "one", "four").
+%%% true
+%%% '''
+%%%
+%%% Output from the example D script `user-probe.d' looks like:
+%%% ```
+%%% <0.34.0> GGOOOAAALL!!!!! 7 8 9 0 'one' 'four' '' ''
+%%% '''
+%%%
+%%% If the expected type of variable is not present, e.g. integer when
+%%% integer() is expected, or an I/O list when iolist() is expected,
+%%% then the driver will ignore the user's input and use a default
+%%% value of 0 or NULL, respectively.
+
+-export([available/0,
+         user_trace_s1/1, % TODO: unify with pid & tag args like user_trace_i4s4
+         p/0, p/1, p/2, p/3, p/4, p/5, p/6, p/7, p/8]).
+-export([put_tag/1, get_tag/0, get_tag_data/0, spread_tag/1, restore_tag/1]).
+
+-export([scaff/0]). % Development only
+-export([user_trace_i4s4/9]). % Know what you're doing!
+-on_load(on_load/0).
+
+-type probe_arg() :: integer() | iolist().
+-type int_p_arg() :: integer() | iolist() | undef.
+
+%% The *_maybe() types use atom() instead of a stricter 'undef'
+%% because user_trace_i4s4/9 is exposed to the outside world, and
+%% because the driver will allow any atom to be used as a "not
+%% present" indication, we'll allow any atom in the types.
+
+-type integer_maybe() :: integer() | atom().
+-type iolist_maybe() :: iolist() | atom().
+
+on_load() ->
+    PrivDir = code:priv_dir(runtime_tools),
+    LibName = "dyntrace",
+    Lib = filename:join([PrivDir, "lib", LibName]),
+    Status = case erlang:load_nif(Lib, 0) of
+                 ok -> ok;
+                 {error, {load_failed, _}}=Error1 ->
+                     ArchLibDir = 
+                         filename:join([PrivDir, "lib", 
+                                        erlang:system_info(system_architecture)]),
+                     Candidate =
+                         filelib:wildcard(filename:join([ArchLibDir,LibName ++ "*" ])),
+                     case Candidate of
+                         [] -> Error1;
+                         _ ->
+                             ArchLib = filename:join([ArchLibDir, LibName]),
+                             erlang:load_nif(ArchLib, 0)
+                     end;
+                 Error1 -> Error1
+             end,
+    case Status of
+        ok -> ok;
+        {error, {E, Str}} ->
+	    case erlang:system_info(dynamic_trace) of
+		none ->
+		    ok;
+		_ ->
+		    error_logger:error_msg("Unable to load dyntrace library. Failed with error:~n
+\"~p, ~s\"~n"
+					   "Dynamic tracing is enabled but the driver is not built correctly~n",[
+														 E,Str]),
+		    Status
+	    end
+    end.
+
+%%%
+%%% NIF placeholders
+%%%
+
+-spec available() -> true | false.
+
+available() ->
+    erlang:nif_error(nif_not_loaded).
+
+-spec user_trace_s1(iolist()) -> true | false | error | badarg.
+
+user_trace_s1(_Message) ->
+    erlang:nif_error(nif_not_loaded).
+
+-spec user_trace_i4s4(iolist(),
+                      integer_maybe(), integer_maybe(),
+                          integer_maybe(), integer_maybe(),
+                      iolist_maybe(), iolist_maybe(),
+                          iolist_maybe(), iolist_maybe()) ->
+      true | false | error | badarg.
+
+user_trace_i4s4(_, _, _, _, _, _, _, _, _) ->
+    erlang:nif_error(nif_not_loaded).
+
+%%%
+%%% Erlang support functions
+%%%
+
+-spec p() -> true | false | error | badarg.
+
+p() ->
+    user_trace_int(undef, undef, undef, undef, undef, undef, undef, undef).
+
+-spec p(probe_arg()) -> true | false | error | badarg.
+
+p(I1) when is_integer(I1) ->
+    user_trace_int(I1, undef, undef, undef, undef, undef, undef, undef);
+p(S1) ->
+    user_trace_int(undef, undef, undef, undef, S1, undef, undef, undef).
+
+-spec p(probe_arg(), probe_arg()) -> true | false | error | badarg.
+
+p(I1, I2) when is_integer(I1), is_integer(I2) ->
+    user_trace_int(I1, I2, undef, undef, undef, undef, undef, undef);
+p(I1, S1) when is_integer(I1) ->
+    user_trace_int(I1, undef, undef, undef, S1, undef, undef, undef);
+p(S1, S2) ->
+    user_trace_int(undef, undef, undef, undef, S1, S2, undef, undef).
+
+-spec p(probe_arg(), probe_arg(), probe_arg()) -> true | false | error | badarg.
+
+p(I1, I2, I3) when is_integer(I1), is_integer(I2), is_integer(I3) ->
+    user_trace_int(I1, I2, I3, undef, undef, undef, undef, undef);
+p(I1, I2, S1) when is_integer(I1), is_integer(I2) ->
+    user_trace_int(I1, I2, undef, undef, S1, undef, undef, undef);
+p(I1, S1, S2) when is_integer(I1) ->
+    user_trace_int(I1, undef, undef, undef, S1, S2, undef, undef);
+p(S1, S2, S3) ->
+    user_trace_int(undef, undef, undef, undef, S1, S2, S3, undef).
+
+-spec p(probe_arg(), probe_arg(), probe_arg(), probe_arg()) ->
+      true | false | error | badarg.
+
+p(I1, I2, I3, I4) when is_integer(I1), is_integer(I2), is_integer(I3), is_integer(I4) ->
+    user_trace_int(I1, I2, I3, I4, undef, undef, undef, undef);
+p(I1, I2, I3, S1) when is_integer(I1), is_integer(I2), is_integer(I3) ->
+    user_trace_int(I1, I2, I3, undef, S1, undef, undef, undef);
+p(I1, I2, S1, S2) when is_integer(I1), is_integer(I2) ->
+    user_trace_int(I1, I2, undef, undef, S1, S2, undef, undef);
+p(I1, S1, S2, S3) when is_integer(I1) ->
+    user_trace_int(I1, undef, undef, undef, S1, S2, S3, undef);
+p(S1, S2, S3, S4) ->
+    user_trace_int(undef, undef, undef, undef, S1, S2, S3, S4).
+
+-spec p(probe_arg(), probe_arg(), probe_arg(), probe_arg(),
+        probe_arg()) ->
+      true | false | error | badarg.
+
+p(I1, I2, I3, I4, S1) when is_integer(I1), is_integer(I2), is_integer(I3), is_integer(I4) ->
+    user_trace_int(I1, I2, I3, I4, S1, undef, undef, undef);
+p(I1, I2, I3, S1, S2) when is_integer(I1), is_integer(I2), is_integer(I3) ->
+    user_trace_int(I1, I2, I3, undef, S1, S2, undef, undef);
+p(I1, I2, S1, S2, S3) when is_integer(I1), is_integer(I2) ->
+    user_trace_int(I1, I2, undef, undef, S1, S2, S3, undef);
+p(I1, S1, S2, S3, S4) when is_integer(I1) ->
+    user_trace_int(I1, undef, undef, undef, S1, S2, S3, S4).
+
+-spec p(probe_arg(), probe_arg(), probe_arg(), probe_arg(),
+        probe_arg(), probe_arg()) ->
+      true | false | error | badarg.
+
+p(I1, I2, I3, I4, S1, S2) when is_integer(I1), is_integer(I2), is_integer(I3), is_integer(I4) ->
+    user_trace_int(I1, I2, I3, I4, S1, S2, undef, undef);
+p(I1, I2, I3, S1, S2, S3) when is_integer(I1), is_integer(I2), is_integer(I3) ->
+    user_trace_int(I1, I2, I3, undef, S1, S2, S3, undef);
+p(I1, I2, S1, S2, S3, S4) when is_integer(I1), is_integer(I2) ->
+    user_trace_int(I1, I2, undef, undef, S1, S2, S3, S4).
+
+-spec p(probe_arg(), probe_arg(), probe_arg(), probe_arg(),
+        probe_arg(), probe_arg(), probe_arg()) ->
+      true | false | error | badarg.
+
+p(I1, I2, I3, I4, S1, S2, S3) when is_integer(I1), is_integer(I2), is_integer(I3), is_integer(I4) ->
+    user_trace_int(I1, I2, I3, I4, S1, S2, S3, undef);
+p(I1, I2, I3, S1, S2, S3, S4) when is_integer(I1), is_integer(I2), is_integer(I3) ->
+    user_trace_int(I1, I2, I3, undef, S1, S2, S3, S4).
+
+-spec p(probe_arg(), probe_arg(), probe_arg(), probe_arg(),
+        probe_arg(), probe_arg(), probe_arg(), probe_arg()) ->
+      true | false | error | badarg.
+
+p(I1, I2, I3, I4, S1, S2, S3, S4) when is_integer(I1), is_integer(I2), is_integer(I3), is_integer(I4) ->
+    user_trace_int(I1, I2, I3, I4, S1, S2, S3, S4).
+
+-spec user_trace_int(int_p_arg(), int_p_arg(), int_p_arg(), int_p_arg(),
+                     int_p_arg(), int_p_arg(), int_p_arg(), int_p_arg()) ->
+      true | false | error | badarg.
+
+user_trace_int(I1, I2, I3, I4, S1, S2, S3, S4) ->
+    UTag = get_tag(),
+    try
+        user_trace_i4s4(UTag, I1, I2, I3, I4, S1, S2, S3, S4)
+    catch
+        error:nif_not_loaded ->
+            false
+    end.
+
+-spec put_tag(undefined | iodata()) -> binary() | undefined.
+put_tag(Data) ->
+    erlang:dt_put_tag(unicode:characters_to_binary(Data)).
+
+-spec get_tag() -> binary() | undefined.
+get_tag() ->
+    erlang:dt_get_tag().
+
+-spec get_tag_data() -> binary() | undefined.
+%% Gets tag if set, otherwise the spread tag data from last incoming message
+get_tag_data() ->
+    erlang:dt_get_tag_data().
+
+-spec spread_tag(boolean()) -> true | {non_neg_integer(), binary() | []}.
+%% Makes the tag behave as a sequential trace token, will spread with 
+%% messages to be picked up by someone using get_tag_data 
+spread_tag(B) ->			   
+    erlang:dt_spread_tag(B).
+
+-spec restore_tag(true | {non_neg_integer(), binary() | []}) -> true.
+restore_tag(T) ->
+    erlang:dt_restore_tag(T).
+    
+
+%% Scaffolding to write tedious code: quick brute force and not 100% correct.
+
+scaff_int_args(N) ->
+    L = lists:sublist(["I1", "I2", "I3", "I4"], N),
+    [string:join(L, ", ")].
+
+scaff_int_guards(N) ->
+    L = lists:sublist(["is_integer(I1)", "is_integer(I2)", "is_integer(I3)",
+                       "is_integer(I4)"], N),
+    lists:flatten(string:join(L, ", ")).
+
+scaff_char_args(N) ->
+    L = lists:sublist(["S1", "S2", "S3", "S4"], N),
+    [string:join(L, ", ")].
+
+scaff_fill(N) ->
+    [string:join(lists:duplicate(N, "undef"), ", ")].
+
+scaff() ->
+    L = [begin
+             IntArgs = scaff_int_args(N_int),
+             IntGuards = scaff_int_guards(N_int),
+             IntFill = scaff_fill(4 - N_int),
+             CharArgs = scaff_char_args(N_char),
+             CharFill = scaff_fill(4 - N_char),
+             InArgs = string:join(IntArgs ++ CharArgs, ", "),
+             OutArgs = string:join(IntArgs ++ IntFill ++ CharArgs ++ CharFill,
+                                   ", "),
+             {N_int + N_char,
+              lists:flatten([io_lib:format("p(~s) when ~s ->\n",
+                                           [InArgs, IntGuards]),
+                             io_lib:format("    user_trace_int(~s);\n", [OutArgs])
+                            ])}
+         end || N_int <- [0,1,2,3,4], N_char <- [0,1,2,3,4]],
+    [io:format("%%~p\n~s", [N, Str]) || {N, Str} <- lists:sort(L)].
diff --git a/lib/runtime_tools/src/runtime_tools.app.src b/lib/runtime_tools/src/runtime_tools.app.src
index 76fd998530..a9d2d68857 100644
--- a/lib/runtime_tools/src/runtime_tools.app.src
+++ b/lib/runtime_tools/src/runtime_tools.app.src
@@ -23,7 +23,7 @@
 		    inviso_rt,inviso_rt_lib,inviso_rt_meta,
 		    inviso_as_lib,inviso_autostart,inviso_autostart_server,
 		    runtime_tools,runtime_tools_sup,erts_alloc_config,
-		    ttb_autostart]},
+		    ttb_autostart,dyntrace]},
     {registered,   [runtime_tools_sup,inviso_rt,inviso_rt_meta]},
     {applications, [kernel, stdlib]},
 %    {env,          [{inviso_autostart_mod,your_own_autostart_module}]},