Autoresume tracing

1 files changed, 86 insertions, 24 deletions

diff --git a/lib/observer/src/ttb.erl b/lib/observer/src/ttb.erl
index 032aecfad2..78c065d2eb 100644
--- a/lib/observer/src/ttb.erl
+++ b/lib/observer/src/ttb.erl
@@ -34,10 +34,12 @@
 
 -include_lib("kernel/include/file.hrl").
 -define(meta_time,5000).
+-define(fetch_time, 10000).
 -define(history_table,ttb_history_table).
 -define(seq_trace_flags,[send,'receive',print,timestamp]).
 -define(upload_dir,"ttb_upload").
 -define(last_config, "ttb_last_config").
+-define(partial_dir, "ttb_partial_result").
 -ifdef(debug).
 -define(get_status,;get_status -> erlang:display(dict:to_list(NodeInfo),loop(NodeInfo, TraceInfo)).
 -else.
@@ -126,6 +128,10 @@ opt([{timer, MSec}|O],{PI,Client,Traci}) ->
     opt(O,{PI,Client,[{timer,{MSec, []}}|Traci]});
 opt([shell|O],{PI,Client,Traci}) ->
     opt(O,{PI,Client,[{shell, true}|Traci]});
+opt([resume|O],{PI,Client,Traci}) ->
+    opt(O,{PI,Client,[{resume, {true, ?fetch_time}}|Traci]});
+opt([{resume,MSec}|O],{PI,Client,Traci}) ->
+    opt(O,{PI,Client,[{resume, {true, MSec}}|Traci]});
 opt([],Opt) ->
     ensure_opt(Opt).
 
@@ -232,17 +238,29 @@ run_history([H|T]) ->
 	ok -> run_history(T);
 	{error,not_found} -> {error,{not_found,H}}
     end;
+
+run_history(all) ->
+    CurrentHist = ets:tab2list(?history_table),
+    ets:delete_all_objects(?history_table),
+    [run_printed(MFA,true) || {_, MFA} <- CurrentHist];
+run_history(all_silent) ->
+    CurrentHist = ets:tab2list(?history_table),
+    ets:delete_all_objects(?history_table),
+    [run_printed(MFA,false) || {_, MFA} <- CurrentHist];
 run_history([]) ->
     ok;
 run_history(N) ->
     case catch ets:lookup(?history_table,N) of
 	[{N,{M,F,A}}] -> 
-	    print_func(M,F,A),
-	    R = apply(M,F,A),
-	    print_result(R);
+            run_printed({M,F,A},true);
 	_ -> 
 	    {error, not_found}
     end.
+
+run_printed({M,F,A},Verbose) ->
+    Verbose andalso print_func(M,F,A),
+    R = apply(M,F,A),
+    Verbose andalso print_result(R).
 	
 write_config(ConfigFile,all) ->
     write_config(ConfigFile,['_']);
@@ -583,12 +601,12 @@ start(SessionInfo) ->
 	    ok
     end.
 
-
 init(Parent, SessionInfo) ->
     register(?MODULE,self()),
     ets:new(?history_table,[ordered_set,named_table,public]),
     Parent ! {started,self()},
-    loop(dict:new(), SessionInfo).
+    NewSessionInfo = [{partials, 0}, {dead_nodes, []} | SessionInfo],
+    loop(dict:new(), NewSessionInfo).
 
 loop(NodeInfo, SessionInfo) ->
     receive 
@@ -618,7 +636,13 @@ loop(NodeInfo, SessionInfo) ->
 		      NodeInfo),
 	    loop(NodeInfo, SessionInfo);
 	{nodedown,Node} ->
-	    loop(dict:erase(Node,NodeInfo), SessionInfo);
+            NewState = make_node_dead(Node, NodeInfo, SessionInfo),
+	    loop(dict:erase(Node,NodeInfo), NewState);
+        {noderesumed,Node,Reporter} ->
+            {MetaFile, CurrentSuffix, NewState} = make_node_alive(Node, SessionInfo),
+            fetch_partial_result(Node, MetaFile, CurrentSuffix),
+            spawn(fun() -> resume_trace(Reporter) end),
+            loop(NodeInfo, NewState);
 	{timeout, StopOpts} ->
 	    spawn(?MODULE, stop, [StopOpts]),
 	    loop(NodeInfo, SessionInfo);
@@ -661,12 +685,13 @@ loop(NodeInfo, SessionInfo) ->
 	    AllNodes = 
 		lists:map(
 		  fun({Node,MetaFile}) ->
-			  spawn(fun() -> fetch(Localhost,Dir,Node,MetaFile) end),
+			  spawn(fun() -> fetch_report(Localhost,Dir,Node,MetaFile) end),
 			  Node
 		  end,
 		  AllNodesAndMeta),
 	    ets:delete(?history_table),
 	    wait_for_fetch(AllNodes),
+            copy_partials(Dir, proplists:get_value(partials, SessionInfo)),
             Absname = filename:absname(Dir),
 	    io:format("Stored logs in ~s~n",[Absname]),
 	    case FetchOrFormat of
@@ -677,9 +702,25 @@ loop(NodeInfo, SessionInfo) ->
          ?get_status
     end.
 
+make_node_dead(Node, NodeInfo, SessionInfo) ->
+    {MetaFile,_} = dict:fetch(Node, NodeInfo),
+    NewDeadNodes = [{Node, MetaFile} | proplists:get_value(dead_nodes, SessionInfo)],
+    [{dead_nodes, NewDeadNodes} | lists:keydelete(dead_nodes, 1, SessionInfo)].
+
+make_node_alive(Node, SessionInfo) ->
+            DeadNodes = proplists:get_value(dead_nodes, SessionInfo),
+            Partials = proplists:get_value(partials, SessionInfo),
+            {value, {_, MetaFile}, Dn2} = lists:keytake(Node, 1, DeadNodes),
+            SessionInfo2 = lists:keyreplace(dead_nodes, 1, SessionInfo, {dead_nodes, Dn2}),
+            {MetaFile, Partials + 1, lists:keyreplace(partials, 1, SessionInfo2, {partials, Partials + 1})}.
+
 get_fetch_dir(undefined) -> ?upload_dir ++ ts();
 get_fetch_dir(Dir) -> Dir.
 
+resume_trace(Reporter) ->
+    ?MODULE:run_history(all_silent),
+    Reporter ! trace_resumed.
+
 get_nodes() ->
     ?MODULE ! {get_nodes,self()},
     receive {?MODULE,Nodes} -> Nodes end.
@@ -689,6 +730,28 @@ ts() ->
     io_lib:format("-~4.4.0w~2.2.0w~2.2.0w-~2.2.0w~2.2.0w~2.2.0w",
 		  [Y,M,D,H,Min,S]).
 
+copy_partials(_, 0) ->
+    ok;
+copy_partials(Dir, Num) ->
+    PartialDir = ?partial_dir ++ integer_to_list(Num),
+    file:rename(PartialDir, filename:join(Dir,PartialDir)),
+    copy_partials(Dir, Num - 1).
+
+fetch_partial_result(Node,MetaFile,Current) ->
+    DirName = ?partial_dir ++ integer_to_list(Current),
+    case file:list_dir(DirName) of
+        {error, enoent} ->
+            ok;
+        {ok, Files} ->
+            [ file:delete(filename:join(DirName, File)) || File <- Files ],
+            file:del_dir(DirName)
+    end,
+    file:make_dir(DirName),
+    fetch(host(node()), DirName, Node, MetaFile).
+
+fetch_report(Localhost, Dir, Node, MetaFile) ->
+    fetch(Localhost,Dir,Node,MetaFile),
+    ?MODULE ! {fetch_complete,Node}.
 
 fetch(Localhost,Dir,Node,MetaFile) ->
     case (host(Node) == Localhost) orelse is_local(MetaFile) of
@@ -718,8 +781,7 @@ fetch(Localhost,Dir,Node,MetaFile) ->
 	    receive_files(Dir,Sock,undefined),
 	    ok = gen_tcp:close(LSock),
 	    ok = gen_tcp:close(Sock)
-    end,
-    ?MODULE ! {fetch_complete,Node}.
+    end.
 
 is_local({local, _, _}) ->
     true;
@@ -749,7 +811,6 @@ host(Node) ->
     [_name,Host] = string:tokens(atom_to_list(Node),"@"),
     Host.
 
-
 wait_for_fetch([]) ->
     ok;
 wait_for_fetch(Nodes) ->
@@ -797,15 +858,8 @@ format(File,Out,Handler) when is_list(File), is_integer(hd(File)) ->
     Files = 
 	case filelib:is_dir(File) of
 	    true ->  % will merge all files in the directory
-		MetaFiles = filelib:wildcard(filename:join(File,"*.ti")),
-		lists:map(fun(M) ->
-				  Sub = string:left(M,length(M)-3),
-				  case filelib:is_file(Sub) of
-				      true -> Sub;
-				      false -> Sub++".*.wrp"
-				  end
-			  end,
-			  MetaFiles);
+                List = filelib:wildcard(filename:join(File, ?partial_dir++"*")),
+                lists:append(collect_files([File | List]));
 	    false -> % format one file
 		[File]
 	end,
@@ -827,6 +881,19 @@ format(Files,Out,Handler) when is_list(Files), is_list(hd(Files)) ->
     end,
     R.
 
+collect_files(Dirs) ->
+    lists:map(fun(Dir) ->
+                      MetaFiles = filelib:wildcard(filename:join(Dir,"*.ti")),
+                      lists:map(fun(M) ->
+                                        Sub = string:left(M,length(M)-3),
+                                        case filelib:is_file(Sub) of
+                                            true -> Sub;
+                                            false -> Sub++".*.wrp"
+                                        end
+                                end,
+                                MetaFiles)
+              end, Dirs).
+
 prepare(File,Handler) ->
     {Traci,Proci} = read_traci(File),
     Node = get_node(Traci),
@@ -855,7 +922,6 @@ format_opt(Opt) when is_list(Opt) ->
 format_opt(Opt) ->
     format_opt([Opt]).
 
-
 read_traci(File) ->
     MetaFile = get_metafile(File),
     case file:read_file(MetaFile) of
@@ -931,7 +997,6 @@ check_exists(File) ->
 	    exit({error,no_file})
     end.
 
-
 get_handler(Handler,Traci) ->
     case Handler of
 	undefined ->
@@ -1060,7 +1125,6 @@ get_next(Client,State) when is_pid(Client) ->
 sort(List) ->
     lists:keysort(1,List).
 
-
 timestamp(Trace) when element(1,Trace) =:= trace_ts;
 		      element(1,Trace) =:= seq_trace, tuple_size(Trace) =:= 4 ->
     element(tuple_size(Trace),Trace);
@@ -1133,5 +1197,3 @@ dump_ti(<<>>,Acc) ->
 dump_ti(B,Acc) ->
     {Term,Rest} = get_term(B),
     dump_ti(Rest,[Term|Acc]).
-
-