Support for stopped and resumed tracing.

6 files changed, 182 insertions, 25 deletions

diff --git a/lib/observer/test/ttb_SUITE.erl b/lib/observer/test/ttb_SUITE.erl
index 4c7010e427..1fd8b4c892 100644
--- a/lib/observer/test/ttb_SUITE.erl
+++ b/lib/observer/test/ttb_SUITE.erl
@@ -71,7 +71,11 @@ all() ->
      only_one_state_for_format_handler, only_one_state_with_default_format_handler,
      only_one_state_with_initial_format_handler, run_trace_with_shortcut1,
      run_trace_with_shortcut2, run_trace_with_shortcut3, run_trace_with_shortcut4,
-     cant_specify_local_and_flush, trace_sorted_by_default,disable_sorting].
+     cant_specify_local_and_flush, trace_sorted_by_default,disable_sorting,
+     trace_resumed_after_node_restart, trace_resumed_after_node_restart_ip,
+     trace_resumed_after_node_restart_wrap,
+     trace_resumed_after_node_restart_wrap_mult
+].
 
 groups() -> 
     [].
@@ -1027,7 +1031,7 @@ format_on_trace_stop(doc) ->
 format_on_trace_stop(Config) when is_list(Config) ->
     ?line {ServerNode, ClientNode} = start_client_and_server(),
     ?line begin_trace(ServerNode, ClientNode, {local, ?FNAME}),
-    ?line ttb_helper:msgs(2),
+    ?line ttb_helper:msgs_ip(2),
     ?line file:delete("HANDLER_OK"),
     ?line {_,_} = ttb:stop([fetch, return_fetch_dir, {format, {handler, marking_call_handler()}}]),
     ?line ?t:stop_node(ServerNode),
@@ -1109,9 +1113,9 @@ changing_cwd_on_control_node_with_local_trace(Config) when is_list(Config) ->
     ?line {ServerNode, ClientNode} = start_client_and_server(),
     ?line begin_trace(ServerNode, ClientNode, {local, ?FNAME}),
     ?line NumMsgs = 3,
-    ?line ttb_helper:msgs(NumMsgs),
+    ?line ttb_helper:msgs_ip(NumMsgs),
     ?line ok = file:set_cwd(".."),
-    ?line ttb_helper:msgs(NumMsgs),
+    ?line ttb_helper:msgs_ip(NumMsgs),
     ?line {_, D} = ttb:stop([fetch, return_fetch_dir]),
     ?line ttb:format(D, [{out, ?OUTPUT}, {handler, simple_call_handler()}]),
     ?line {ok, Ret} = file:consult(?OUTPUT),
@@ -1350,3 +1354,77 @@ disable_sorting(Config) when is_list(Config) ->
     ?line ttb:format(D, [{out, ?OUTPUT}, {handler, node_call_handler()}, {disable_sort, true}]),
     {ok, Ret} = file:consult(?OUTPUT),
     ?line [ClientNode,ClientNode,ServerNode,ServerNode,ServerNode] = Ret.
+
+%% -----------------------------------------------------------------------------
+%% tests for autoresume of tracing
+%% -----------------------------------------------------------------------------
+
+trace_resumed_after_node_restart(suite) ->
+    [];
+trace_resumed_after_node_restart(doc) ->
+    ["Test trace resumed after node restart, trace to files on remote node."];
+trace_resumed_after_node_restart(Config) when is_list(Config) ->
+    ?line {ServerNode, ClientNode} = start_client_and_server(),
+    ?line begin_trace_with_resume(ServerNode, ClientNode, ?FNAME),
+    ?line logic(2,6,file).
+
+trace_resumed_after_node_restart_ip(suite) ->
+    [];
+trace_resumed_after_node_restart_ip(doc) ->
+    ["Test trace resumed after node restart, trace via tcp/ip to local node."];
+trace_resumed_after_node_restart_ip(Config) when is_list(Config) ->
+    ?line {ServerNode, ClientNode} = start_client_and_server(),
+    ?line begin_trace_with_resume(ServerNode, ClientNode, {local, ?FNAME}),
+    ?line logic(2,6,local).
+
+trace_resumed_after_node_restart_wrap(suite) ->
+    [];
+trace_resumed_after_node_restart_wrap(doc) ->
+    ["Test trace resumed after node restart, wrap option."];
+trace_resumed_after_node_restart_wrap(Config) when is_list(Config) ->
+    ?line {ServerNode, ClientNode} = start_client_and_server(),
+    ?line begin_trace_with_resume(ServerNode, ClientNode, {wrap, ?FNAME, 10, 4}),
+    ?line logic(1,4,file).
+
+trace_resumed_after_node_restart_wrap_mult(suite) ->
+    [];
+trace_resumed_after_node_restart_wrap_mult(doc) ->
+    ["Test trace resumed after node restart, wrap option, multiple files."];
+trace_resumed_after_node_restart_wrap_mult(Config) when is_list(Config) ->
+    ?line {ServerNode, ClientNode} = start_client_and_server(),
+    ?line begin_trace_with_resume(ServerNode, ClientNode, {wrap, ?FNAME, 10, 4}),
+    ?line logic(20,8,file).
+
+logic(N, M, TracingType) ->
+    helper_msgs(N, TracingType),
+    ?t:stop_node(ttb_helper:get_node(client)),
+    timer:sleep(2500),
+    ?line {ok,ClientNode} = ?t:start_node(client,slave,[]),
+    ?line ok = ttb_helper:c(code, add_paths, [code:get_path()]),
+    ?line ttb_helper:c(client, init, []),
+    ?line helper_msgs(N, TracingType),
+    ?line {_, D} = ttb:stop([return_fetch_dir]),
+    ?line ?t:stop_node(ttb_helper:get_node(server)),
+    ?line ?t:stop_node(ClientNode),
+    ?line ttb:format(D, [{out, ?OUTPUT}, {handler, ret_caller_call_handler2()}]),
+    ?line {ok, Ret} = file:consult(?OUTPUT),
+    ?line M = length(Ret).
+
+begin_trace_with_resume(ServerNode, ClientNode, Dest) ->
+    ?line {ok, _} = ttb:tracer([ServerNode,ClientNode], [{file, Dest}, resume]),
+    ?line ttb:p(all, [call, timestamp]),
+    ?line ttb:tp(server, received, []),
+    ?line ttb:tp(client, put, []),
+    ?line ttb:tp(client, get, []).
+
+ret_caller_call_handler2() ->
+    {fun(A, {trace_ts, _, call, _, _} ,_,_) -> io:format(A, "ok.~n", []);
+	(_, _, _, _) -> ok end, []}.
+
+helper_msgs(N, TracingType) ->
+    case TracingType of
+        local ->
+            ttb_helper:msgs_ip(N);
+        _ ->
+            ttb_helper:msgs(N)
+    end.
diff --git a/lib/observer/test/ttb_helper.erl b/lib/observer/test/ttb_helper.erl
index 570ce934ab..19fdc0e159 100644
--- a/lib/observer/test/ttb_helper.erl
+++ b/lib/observer/test/ttb_helper.erl
@@ -11,7 +11,7 @@
 
 -define(NODE_CMD(Name),
 	"erl -sname " ++ atom_to_list(Name) ++
-	" -pa .. -pa . -detached -run h send_ok").
+	" -pa .. -pa . -detached -run ttb_helper send_ok").
 -define(REG_NAME, nc_testing).
 
 new_fun() ->
diff --git a/lib/runtime_tools/src/Makefile b/lib/runtime_tools/src/Makefile
index 4f831f3dd8..46b570210a 100644
--- a/lib/runtime_tools/src/Makefile
+++ b/lib/runtime_tools/src/Makefile
@@ -46,7 +46,8 @@ MODULES= \
 	runtime_tools_sup \
 	dbg \
 	percept_profile \
-	observer_backend
+	observer_backend \
+	ttb_autostart
 HRL_FILES= ../include/observer_backend.hrl
 
 ERL_FILES= $(MODULES:%=%.erl)
diff --git a/lib/runtime_tools/src/observer_backend.erl b/lib/runtime_tools/src/observer_backend.erl
index 39d54bf782..9c1f9da5b1 100644
--- a/lib/runtime_tools/src/observer_backend.erl
+++ b/lib/runtime_tools/src/observer_backend.erl
@@ -31,6 +31,7 @@
 	 ttb_write_binary/2,
 	 ttb_stop/1,
 	 ttb_fetch/2,
+         ttb_resume_trace/0,
 	 ttb_get_filenames/1]).
 -define(CHUNKSIZE,8191). % 8 kbytes - 1 byte
 
@@ -102,8 +103,12 @@ ttb_init_node(MetaFile_0,PI,Traci) ->
 	true -> 				% {local,_,_}
 	    MetaFile = MetaFile_0
     end,
+    case proplists:get_value(resume, Traci) of
+        {true, _} -> (autostart_module()):write_config(Traci);
+        _    -> ok
+    end,
     Self = self(),
-    MetaPid = spawn(fun() -> ttb_meta_tracer(MetaFile,PI,Self) end),
+    MetaPid = spawn(fun() -> ttb_meta_tracer(MetaFile,PI,Self,Traci) end),
     receive {MetaPid,started} -> ok end,
     MetaPid ! {metadata,Traci},
     case PI of
@@ -119,7 +124,8 @@ ttb_write_trace_info(MetaPid,Key,What) ->
     MetaPid ! {metadata,Key,What},
     ok.
 
-ttb_meta_tracer(MetaFile,PI,Parent) ->
+ttb_meta_tracer(MetaFile,PI,Parent,SessionData) ->
+    erlang:monitor(process, proplists:get_value(ttb_control, SessionData)),
     case PI of
 	true ->
 	    ReturnMS = [{'_',[],[{return_trace}]}],
@@ -132,22 +138,29 @@ ttb_meta_tracer(MetaFile,PI,Parent) ->
 	    ok
     end,
     Parent ! {self(),started},
-    ttb_meta_tracer_loop(MetaFile,PI,dict:new()).
+    case proplists:get_value(overload_check, SessionData) of
+        {Ms, M, F} ->
+            catch M:F(init),
+            erlang:send_after(Ms, self(), overload_check);
+        _ ->
+            ok
+    end,
+    ttb_meta_tracer_loop(MetaFile,PI,dict:new(),SessionData).
 
-ttb_meta_tracer_loop(MetaFile,PI,Acc) ->
+ttb_meta_tracer_loop(MetaFile,PI,Acc,State) ->
     receive
 	{trace_ts,_,call,{erlang,register,[Name,Pid]},_} ->
 	    ttb_store_meta({pid,{Pid,Name}},MetaFile),
-	    ttb_meta_tracer_loop(MetaFile,PI,Acc);
+	    ttb_meta_tracer_loop(MetaFile,PI,Acc,State);
 	{trace_ts,_,call,{global,register_name,[Name,Pid]},_} ->
 	    ttb_store_meta({pid,{Pid,{global,Name}}},MetaFile),
-	    ttb_meta_tracer_loop(MetaFile,PI,Acc);
+	    ttb_meta_tracer_loop(MetaFile,PI,Acc,State);
 	{trace_ts,CallingPid,call,{erlang,spawn_opt,[{M,F,Args,_}]},_} ->
 	    MFA = {M,F,length(Args)},
 	    NewAcc = dict:update(CallingPid,
 				 fun(Old) -> [MFA|Old] end, [MFA], 
 				 Acc),
-	    ttb_meta_tracer_loop(MetaFile,PI,NewAcc);
+	    ttb_meta_tracer_loop(MetaFile,PI,NewAcc,State);
 	{trace_ts,CallingPid,return_from,{erlang,spawn_opt,_Arity},Ret,_} ->
 	    case Ret of
 		{NewPid,_Mref} when is_pid(NewPid) -> ok;
@@ -160,14 +173,14 @@ ttb_meta_tracer_loop(MetaFile,PI,Acc) ->
 				    T 
 			    end,
 			    Acc),
-	    ttb_meta_tracer_loop(MetaFile,PI,NewAcc);
+	    ttb_meta_tracer_loop(MetaFile,PI,NewAcc,State);
 	{trace_ts,CallingPid,call,{erlang,Spawn,[M,F,Args]},_} 
 	when Spawn==spawn;Spawn==spawn_link ->
 	    MFA = {M,F,length(Args)},
 	    NewAcc = dict:update(CallingPid,
 				 fun(Old) -> [MFA|Old] end, [MFA], 
 				 Acc),
-	    ttb_meta_tracer_loop(MetaFile,PI,NewAcc);
+	    ttb_meta_tracer_loop(MetaFile,PI,NewAcc,State);
 
 	{trace_ts,CallingPid,return_from,{erlang,Spawn,_Arity},NewPid,_} 
 	when Spawn==spawn;Spawn==spawn_link ->
@@ -178,28 +191,53 @@ ttb_meta_tracer_loop(MetaFile,PI,Acc) ->
 				    T
 			    end,
 			    Acc),
-	    ttb_meta_tracer_loop(MetaFile,PI,NewAcc);
+	    ttb_meta_tracer_loop(MetaFile,PI,NewAcc,State);
 
 	{metadata,Data} when is_list(Data) ->
 	    ttb_store_meta(Data,MetaFile),
-	    ttb_meta_tracer_loop(MetaFile,PI,Acc);
+	    ttb_meta_tracer_loop(MetaFile,PI,Acc,State);
 
 	{metadata,Key,Fun} when is_function(Fun) ->
 	    ttb_store_meta([{Key,Fun()}],MetaFile),
-	    ttb_meta_tracer_loop(MetaFile,PI,Acc);
+	    ttb_meta_tracer_loop(MetaFile,PI,Acc,State);
 
 	{metadata,Key,What} ->
 	    ttb_store_meta([{Key,What}],MetaFile),
-	    ttb_meta_tracer_loop(MetaFile,PI,Acc);
-
-	stop when PI=:=true ->
-	    erlang:trace_pattern({erlang,spawn,3},false,[meta]),
+	    ttb_meta_tracer_loop(MetaFile,PI,Acc,State);
+        overload_check ->
+            {Ms, M, F} = proplists:get_value(overload_check, State),
+            case catch M:F(check) of
+                true ->
+                    erlang:trace(all, false, [all]),
+                    ControlPid = proplists:get_value(ttb_control, State),
+                    ControlPid ! {node_overloaded, node()},
+                    catch M:F(stop),
+                    ttb_meta_tracer_loop(MetaFile,PI,Acc,lists:keydelete(overload_check, 1, State));
+                _ ->
+                    erlang:send_after(Ms, self(), overload_check),
+                    ttb_meta_tracer_loop(MetaFile,PI,Acc, State)
+            end;
+     {'DOWN', _, _, _, _} ->
+            stop_seq_trace(),
+            self() ! stop,
+            ttb_meta_tracer_loop(MetaFile,PI,Acc, State);
+     stop when PI=:=true ->
+            try_stop_resume(State),
+            try_stop_overload_check(State),
+            erlang:trace_pattern({erlang,spawn,3},false,[meta]),
 	    erlang:trace_pattern({erlang,spawn_link,3},false,[meta]),
 	    erlang:trace_pattern({erlang,spawn_opt,1},false,[meta]),
 	    erlang:trace_pattern({erlang,register,2},false,[meta]),
 	    erlang:trace_pattern({global,register_name,2},false,[meta]);
 	stop ->
-	    ok
+            try_stop_resume(State),
+            try_stop_overload_check(State)
+    end.
+
+try_stop_overload_check(State) ->
+    case proplists:get_value(overload, State) of
+        undefined -> ok;
+        {_, M, F} -> catch M:F(stop)
     end.
 
 pnames() ->
@@ -224,6 +262,40 @@ pinfo(P,Globals) ->
 	undefined -> [] % the process has terminated
     end.
 
+autostart_module() ->
+    element(2, application:get_env(runtime_tools, ttb_autostart_module)).
+
+try_stop_resume(State) ->
+    case proplists:get_value(resume, State) of
+        true -> (autostart_module()):delete_config();
+        _    -> ok
+    end.
+
+ttb_resume_trace() ->
+    case (autostart_module()):read_config() of
+        {error, _} ->
+            ok;
+        {ok, Data} ->
+            Pid = proplists:get_value(ttb_control, Data),
+            {_, Timeout} = proplists:get_value(resume, Data),
+            case rpc:call(node(Pid), erlang, whereis, [ttb]) of
+                Pid ->
+                    Pid ! {noderesumed, node(), self()},
+                    wait_for_fetch_ready(Timeout);
+                _ ->
+                    ok
+            end,
+            (autostart_module()):delete_config(),
+            ok
+    end.
+
+wait_for_fetch_ready(Timeout) ->
+    receive
+        trace_resumed ->
+            ok
+    after Timeout ->
+            ok
+    end.
 
 ttb_store_meta(Data,{local,MetaFile,Port}) when is_list(Data) ->
     ttb_send_to_port(Port,MetaFile,Data);
@@ -275,6 +347,9 @@ ttb_stop(MetaPid) ->
     %% returns, and then the Port (in {local,MetaFile,Port})
     %% cannot be accessed any more.
     receive {'DOWN', Ref, process, MetaPid, _Info} -> ok end,
+    stop_seq_trace().
+
+stop_seq_trace() ->
     seq_trace:reset_trace(),
     seq_trace:set_system_tracer(false).
 
diff --git a/lib/runtime_tools/src/runtime_tools.app.src b/lib/runtime_tools/src/runtime_tools.app.src
index e6dc7a21d4..095567b165 100644
--- a/lib/runtime_tools/src/runtime_tools.app.src
+++ b/lib/runtime_tools/src/runtime_tools.app.src
@@ -22,7 +22,8 @@
     {modules,      [dbg,observer_backend,percept_profile,
 		    inviso_rt,inviso_rt_lib,inviso_rt_meta,
 		    inviso_as_lib,inviso_autostart,inviso_autostart_server,
-		    runtime_tools,runtime_tools_sup,erts_alloc_config]},
+		    runtime_tools,runtime_tools_sup,erts_alloc_config,
+		    ttb_autostart]},
     {registered,   [runtime_tools_sup,inviso_rt,inviso_rt_meta]},
     {applications, [kernel, stdlib]},
 %    {env,          [{inviso_autostart_mod,your_own_autostart_module}]},
diff --git a/lib/runtime_tools/src/runtime_tools_sup.erl b/lib/runtime_tools/src/runtime_tools_sup.erl
index 1a872c355d..4fcb2292d0 100644
--- a/lib/runtime_tools/src/runtime_tools_sup.erl
+++ b/lib/runtime_tools/src/runtime_tools_sup.erl
@@ -38,6 +38,8 @@
 init(AutoModArgs) ->
     Flags = {one_for_one, 0, 3600},
     Children = [{inviso_rt, {inviso_rt, start_link_auto, [AutoModArgs]}, 
-		 temporary, 3000, worker, [inviso_rt]}],
+		 temporary, 3000, worker, [inviso_rt]},
+                {ttb_autostart, {ttb_autostart, start_link, []},
+                 temporary, 3000, worker, [ttb_autostart]}],
     {ok, {Flags, Children}}.
 %% -----------------------------------------------------------------------------