Support for stopped and resumed tracing.

4 files changed, 99 insertions, 20 deletions

diff --git a/lib/runtime_tools/src/Makefile b/lib/runtime_tools/src/Makefile
index 4f831f3dd8..46b570210a 100644
--- a/lib/runtime_tools/src/Makefile
+++ b/lib/runtime_tools/src/Makefile
@@ -46,7 +46,8 @@ MODULES= \
 	runtime_tools_sup \
 	dbg \
 	percept_profile \
-	observer_backend
+	observer_backend \
+	ttb_autostart
 HRL_FILES= ../include/observer_backend.hrl
 
 ERL_FILES= $(MODULES:%=%.erl)
diff --git a/lib/runtime_tools/src/observer_backend.erl b/lib/runtime_tools/src/observer_backend.erl
index 39d54bf782..9c1f9da5b1 100644
--- a/lib/runtime_tools/src/observer_backend.erl
+++ b/lib/runtime_tools/src/observer_backend.erl
@@ -31,6 +31,7 @@
 	 ttb_write_binary/2,
 	 ttb_stop/1,
 	 ttb_fetch/2,
+         ttb_resume_trace/0,
 	 ttb_get_filenames/1]).
 -define(CHUNKSIZE,8191). % 8 kbytes - 1 byte
 
@@ -102,8 +103,12 @@ ttb_init_node(MetaFile_0,PI,Traci) ->
 	true -> 				% {local,_,_}
 	    MetaFile = MetaFile_0
     end,
+    case proplists:get_value(resume, Traci) of
+        {true, _} -> (autostart_module()):write_config(Traci);
+        _    -> ok
+    end,
     Self = self(),
-    MetaPid = spawn(fun() -> ttb_meta_tracer(MetaFile,PI,Self) end),
+    MetaPid = spawn(fun() -> ttb_meta_tracer(MetaFile,PI,Self,Traci) end),
     receive {MetaPid,started} -> ok end,
     MetaPid ! {metadata,Traci},
     case PI of
@@ -119,7 +124,8 @@ ttb_write_trace_info(MetaPid,Key,What) ->
     MetaPid ! {metadata,Key,What},
     ok.
 
-ttb_meta_tracer(MetaFile,PI,Parent) ->
+ttb_meta_tracer(MetaFile,PI,Parent,SessionData) ->
+    erlang:monitor(process, proplists:get_value(ttb_control, SessionData)),
     case PI of
 	true ->
 	    ReturnMS = [{'_',[],[{return_trace}]}],
@@ -132,22 +138,29 @@ ttb_meta_tracer(MetaFile,PI,Parent) ->
 	    ok
     end,
     Parent ! {self(),started},
-    ttb_meta_tracer_loop(MetaFile,PI,dict:new()).
+    case proplists:get_value(overload_check, SessionData) of
+        {Ms, M, F} ->
+            catch M:F(init),
+            erlang:send_after(Ms, self(), overload_check);
+        _ ->
+            ok
+    end,
+    ttb_meta_tracer_loop(MetaFile,PI,dict:new(),SessionData).
 
-ttb_meta_tracer_loop(MetaFile,PI,Acc) ->
+ttb_meta_tracer_loop(MetaFile,PI,Acc,State) ->
     receive
 	{trace_ts,_,call,{erlang,register,[Name,Pid]},_} ->
 	    ttb_store_meta({pid,{Pid,Name}},MetaFile),
-	    ttb_meta_tracer_loop(MetaFile,PI,Acc);
+	    ttb_meta_tracer_loop(MetaFile,PI,Acc,State);
 	{trace_ts,_,call,{global,register_name,[Name,Pid]},_} ->
 	    ttb_store_meta({pid,{Pid,{global,Name}}},MetaFile),
-	    ttb_meta_tracer_loop(MetaFile,PI,Acc);
+	    ttb_meta_tracer_loop(MetaFile,PI,Acc,State);
 	{trace_ts,CallingPid,call,{erlang,spawn_opt,[{M,F,Args,_}]},_} ->
 	    MFA = {M,F,length(Args)},
 	    NewAcc = dict:update(CallingPid,
 				 fun(Old) -> [MFA|Old] end, [MFA], 
 				 Acc),
-	    ttb_meta_tracer_loop(MetaFile,PI,NewAcc);
+	    ttb_meta_tracer_loop(MetaFile,PI,NewAcc,State);
 	{trace_ts,CallingPid,return_from,{erlang,spawn_opt,_Arity},Ret,_} ->
 	    case Ret of
 		{NewPid,_Mref} when is_pid(NewPid) -> ok;
@@ -160,14 +173,14 @@ ttb_meta_tracer_loop(MetaFile,PI,Acc) ->
 				    T 
 			    end,
 			    Acc),
-	    ttb_meta_tracer_loop(MetaFile,PI,NewAcc);
+	    ttb_meta_tracer_loop(MetaFile,PI,NewAcc,State);
 	{trace_ts,CallingPid,call,{erlang,Spawn,[M,F,Args]},_} 
 	when Spawn==spawn;Spawn==spawn_link ->
 	    MFA = {M,F,length(Args)},
 	    NewAcc = dict:update(CallingPid,
 				 fun(Old) -> [MFA|Old] end, [MFA], 
 				 Acc),
-	    ttb_meta_tracer_loop(MetaFile,PI,NewAcc);
+	    ttb_meta_tracer_loop(MetaFile,PI,NewAcc,State);
 
 	{trace_ts,CallingPid,return_from,{erlang,Spawn,_Arity},NewPid,_} 
 	when Spawn==spawn;Spawn==spawn_link ->
@@ -178,28 +191,53 @@ ttb_meta_tracer_loop(MetaFile,PI,Acc) ->
 				    T
 			    end,
 			    Acc),
-	    ttb_meta_tracer_loop(MetaFile,PI,NewAcc);
+	    ttb_meta_tracer_loop(MetaFile,PI,NewAcc,State);
 
 	{metadata,Data} when is_list(Data) ->
 	    ttb_store_meta(Data,MetaFile),
-	    ttb_meta_tracer_loop(MetaFile,PI,Acc);
+	    ttb_meta_tracer_loop(MetaFile,PI,Acc,State);
 
 	{metadata,Key,Fun} when is_function(Fun) ->
 	    ttb_store_meta([{Key,Fun()}],MetaFile),
-	    ttb_meta_tracer_loop(MetaFile,PI,Acc);
+	    ttb_meta_tracer_loop(MetaFile,PI,Acc,State);
 
 	{metadata,Key,What} ->
 	    ttb_store_meta([{Key,What}],MetaFile),
-	    ttb_meta_tracer_loop(MetaFile,PI,Acc);
-
-	stop when PI=:=true ->
-	    erlang:trace_pattern({erlang,spawn,3},false,[meta]),
+	    ttb_meta_tracer_loop(MetaFile,PI,Acc,State);
+        overload_check ->
+            {Ms, M, F} = proplists:get_value(overload_check, State),
+            case catch M:F(check) of
+                true ->
+                    erlang:trace(all, false, [all]),
+                    ControlPid = proplists:get_value(ttb_control, State),
+                    ControlPid ! {node_overloaded, node()},
+                    catch M:F(stop),
+                    ttb_meta_tracer_loop(MetaFile,PI,Acc,lists:keydelete(overload_check, 1, State));
+                _ ->
+                    erlang:send_after(Ms, self(), overload_check),
+                    ttb_meta_tracer_loop(MetaFile,PI,Acc, State)
+            end;
+     {'DOWN', _, _, _, _} ->
+            stop_seq_trace(),
+            self() ! stop,
+            ttb_meta_tracer_loop(MetaFile,PI,Acc, State);
+     stop when PI=:=true ->
+            try_stop_resume(State),
+            try_stop_overload_check(State),
+            erlang:trace_pattern({erlang,spawn,3},false,[meta]),
 	    erlang:trace_pattern({erlang,spawn_link,3},false,[meta]),
 	    erlang:trace_pattern({erlang,spawn_opt,1},false,[meta]),
 	    erlang:trace_pattern({erlang,register,2},false,[meta]),
 	    erlang:trace_pattern({global,register_name,2},false,[meta]);
 	stop ->
-	    ok
+            try_stop_resume(State),
+            try_stop_overload_check(State)
+    end.
+
+try_stop_overload_check(State) ->
+    case proplists:get_value(overload, State) of
+        undefined -> ok;
+        {_, M, F} -> catch M:F(stop)
     end.
 
 pnames() ->
@@ -224,6 +262,40 @@ pinfo(P,Globals) ->
 	undefined -> [] % the process has terminated
     end.
 
+autostart_module() ->
+    element(2, application:get_env(runtime_tools, ttb_autostart_module)).
+
+try_stop_resume(State) ->
+    case proplists:get_value(resume, State) of
+        true -> (autostart_module()):delete_config();
+        _    -> ok
+    end.
+
+ttb_resume_trace() ->
+    case (autostart_module()):read_config() of
+        {error, _} ->
+            ok;
+        {ok, Data} ->
+            Pid = proplists:get_value(ttb_control, Data),
+            {_, Timeout} = proplists:get_value(resume, Data),
+            case rpc:call(node(Pid), erlang, whereis, [ttb]) of
+                Pid ->
+                    Pid ! {noderesumed, node(), self()},
+                    wait_for_fetch_ready(Timeout);
+                _ ->
+                    ok
+            end,
+            (autostart_module()):delete_config(),
+            ok
+    end.
+
+wait_for_fetch_ready(Timeout) ->
+    receive
+        trace_resumed ->
+            ok
+    after Timeout ->
+            ok
+    end.
 
 ttb_store_meta(Data,{local,MetaFile,Port}) when is_list(Data) ->
     ttb_send_to_port(Port,MetaFile,Data);
@@ -275,6 +347,9 @@ ttb_stop(MetaPid) ->
     %% returns, and then the Port (in {local,MetaFile,Port})
     %% cannot be accessed any more.
     receive {'DOWN', Ref, process, MetaPid, _Info} -> ok end,
+    stop_seq_trace().
+
+stop_seq_trace() ->
     seq_trace:reset_trace(),
     seq_trace:set_system_tracer(false).
 
diff --git a/lib/runtime_tools/src/runtime_tools.app.src b/lib/runtime_tools/src/runtime_tools.app.src
index e6dc7a21d4..095567b165 100644
--- a/lib/runtime_tools/src/runtime_tools.app.src
+++ b/lib/runtime_tools/src/runtime_tools.app.src
@@ -22,7 +22,8 @@
     {modules,      [dbg,observer_backend,percept_profile,
 		    inviso_rt,inviso_rt_lib,inviso_rt_meta,
 		    inviso_as_lib,inviso_autostart,inviso_autostart_server,
-		    runtime_tools,runtime_tools_sup,erts_alloc_config]},
+		    runtime_tools,runtime_tools_sup,erts_alloc_config,
+		    ttb_autostart]},
     {registered,   [runtime_tools_sup,inviso_rt,inviso_rt_meta]},
     {applications, [kernel, stdlib]},
 %    {env,          [{inviso_autostart_mod,your_own_autostart_module}]},
diff --git a/lib/runtime_tools/src/runtime_tools_sup.erl b/lib/runtime_tools/src/runtime_tools_sup.erl
index 1a872c355d..4fcb2292d0 100644
--- a/lib/runtime_tools/src/runtime_tools_sup.erl
+++ b/lib/runtime_tools/src/runtime_tools_sup.erl
@@ -38,6 +38,8 @@
 init(AutoModArgs) ->
     Flags = {one_for_one, 0, 3600},
     Children = [{inviso_rt, {inviso_rt, start_link_auto, [AutoModArgs]}, 
-		 temporary, 3000, worker, [inviso_rt]}],
+		 temporary, 3000, worker, [inviso_rt]},
+                {ttb_autostart, {ttb_autostart, start_link, []},
+                 temporary, 3000, worker, [ttb_autostart]}],
     {ok, {Flags, Children}}.
 %% -----------------------------------------------------------------------------