aboutsummaryrefslogtreecommitdiffstats
path: root/lib/observer
diff options
context:
space:
mode:
Diffstat (limited to 'lib/observer')
-rw-r--r--lib/observer/src/ttb.erl110
1 files changed, 86 insertions, 24 deletions
diff --git a/lib/observer/src/ttb.erl b/lib/observer/src/ttb.erl
index 032aecfad2..78c065d2eb 100644
--- a/lib/observer/src/ttb.erl
+++ b/lib/observer/src/ttb.erl
@@ -34,10 +34,12 @@
-include_lib("kernel/include/file.hrl").
-define(meta_time,5000).
+-define(fetch_time, 10000).
-define(history_table,ttb_history_table).
-define(seq_trace_flags,[send,'receive',print,timestamp]).
-define(upload_dir,"ttb_upload").
-define(last_config, "ttb_last_config").
+-define(partial_dir, "ttb_partial_result").
-ifdef(debug).
-define(get_status,;get_status -> erlang:display(dict:to_list(NodeInfo),loop(NodeInfo, TraceInfo)).
-else.
@@ -126,6 +128,10 @@ opt([{timer, MSec}|O],{PI,Client,Traci}) ->
opt(O,{PI,Client,[{timer,{MSec, []}}|Traci]});
opt([shell|O],{PI,Client,Traci}) ->
opt(O,{PI,Client,[{shell, true}|Traci]});
+opt([resume|O],{PI,Client,Traci}) ->
+ opt(O,{PI,Client,[{resume, {true, ?fetch_time}}|Traci]});
+opt([{resume,MSec}|O],{PI,Client,Traci}) ->
+ opt(O,{PI,Client,[{resume, {true, MSec}}|Traci]});
opt([],Opt) ->
ensure_opt(Opt).
@@ -232,17 +238,29 @@ run_history([H|T]) ->
ok -> run_history(T);
{error,not_found} -> {error,{not_found,H}}
end;
+
+run_history(all) ->
+ CurrentHist = ets:tab2list(?history_table),
+ ets:delete_all_objects(?history_table),
+ [run_printed(MFA,true) || {_, MFA} <- CurrentHist];
+run_history(all_silent) ->
+ CurrentHist = ets:tab2list(?history_table),
+ ets:delete_all_objects(?history_table),
+ [run_printed(MFA,false) || {_, MFA} <- CurrentHist];
run_history([]) ->
ok;
run_history(N) ->
case catch ets:lookup(?history_table,N) of
[{N,{M,F,A}}] ->
- print_func(M,F,A),
- R = apply(M,F,A),
- print_result(R);
+ run_printed({M,F,A},true);
_ ->
{error, not_found}
end.
+
+run_printed({M,F,A},Verbose) ->
+ Verbose andalso print_func(M,F,A),
+ R = apply(M,F,A),
+ Verbose andalso print_result(R).
write_config(ConfigFile,all) ->
write_config(ConfigFile,['_']);
@@ -583,12 +601,12 @@ start(SessionInfo) ->
ok
end.
-
init(Parent, SessionInfo) ->
register(?MODULE,self()),
ets:new(?history_table,[ordered_set,named_table,public]),
Parent ! {started,self()},
- loop(dict:new(), SessionInfo).
+ NewSessionInfo = [{partials, 0}, {dead_nodes, []} | SessionInfo],
+ loop(dict:new(), NewSessionInfo).
loop(NodeInfo, SessionInfo) ->
receive
@@ -618,7 +636,13 @@ loop(NodeInfo, SessionInfo) ->
NodeInfo),
loop(NodeInfo, SessionInfo);
{nodedown,Node} ->
- loop(dict:erase(Node,NodeInfo), SessionInfo);
+ NewState = make_node_dead(Node, NodeInfo, SessionInfo),
+ loop(dict:erase(Node,NodeInfo), NewState);
+ {noderesumed,Node,Reporter} ->
+ {MetaFile, CurrentSuffix, NewState} = make_node_alive(Node, SessionInfo),
+ fetch_partial_result(Node, MetaFile, CurrentSuffix),
+ spawn(fun() -> resume_trace(Reporter) end),
+ loop(NodeInfo, NewState);
{timeout, StopOpts} ->
spawn(?MODULE, stop, [StopOpts]),
loop(NodeInfo, SessionInfo);
@@ -661,12 +685,13 @@ loop(NodeInfo, SessionInfo) ->
AllNodes =
lists:map(
fun({Node,MetaFile}) ->
- spawn(fun() -> fetch(Localhost,Dir,Node,MetaFile) end),
+ spawn(fun() -> fetch_report(Localhost,Dir,Node,MetaFile) end),
Node
end,
AllNodesAndMeta),
ets:delete(?history_table),
wait_for_fetch(AllNodes),
+ copy_partials(Dir, proplists:get_value(partials, SessionInfo)),
Absname = filename:absname(Dir),
io:format("Stored logs in ~s~n",[Absname]),
case FetchOrFormat of
@@ -677,9 +702,25 @@ loop(NodeInfo, SessionInfo) ->
?get_status
end.
+make_node_dead(Node, NodeInfo, SessionInfo) ->
+ {MetaFile,_} = dict:fetch(Node, NodeInfo),
+ NewDeadNodes = [{Node, MetaFile} | proplists:get_value(dead_nodes, SessionInfo)],
+ [{dead_nodes, NewDeadNodes} | lists:keydelete(dead_nodes, 1, SessionInfo)].
+
+make_node_alive(Node, SessionInfo) ->
+ DeadNodes = proplists:get_value(dead_nodes, SessionInfo),
+ Partials = proplists:get_value(partials, SessionInfo),
+ {value, {_, MetaFile}, Dn2} = lists:keytake(Node, 1, DeadNodes),
+ SessionInfo2 = lists:keyreplace(dead_nodes, 1, SessionInfo, {dead_nodes, Dn2}),
+ {MetaFile, Partials + 1, lists:keyreplace(partials, 1, SessionInfo2, {partials, Partials + 1})}.
+
get_fetch_dir(undefined) -> ?upload_dir ++ ts();
get_fetch_dir(Dir) -> Dir.
+resume_trace(Reporter) ->
+ ?MODULE:run_history(all_silent),
+ Reporter ! trace_resumed.
+
get_nodes() ->
?MODULE ! {get_nodes,self()},
receive {?MODULE,Nodes} -> Nodes end.
@@ -689,6 +730,28 @@ ts() ->
io_lib:format("-~4.4.0w~2.2.0w~2.2.0w-~2.2.0w~2.2.0w~2.2.0w",
[Y,M,D,H,Min,S]).
+copy_partials(_, 0) ->
+ ok;
+copy_partials(Dir, Num) ->
+ PartialDir = ?partial_dir ++ integer_to_list(Num),
+ file:rename(PartialDir, filename:join(Dir,PartialDir)),
+ copy_partials(Dir, Num - 1).
+
+fetch_partial_result(Node,MetaFile,Current) ->
+ DirName = ?partial_dir ++ integer_to_list(Current),
+ case file:list_dir(DirName) of
+ {error, enoent} ->
+ ok;
+ {ok, Files} ->
+ [ file:delete(filename:join(DirName, File)) || File <- Files ],
+ file:del_dir(DirName)
+ end,
+ file:make_dir(DirName),
+ fetch(host(node()), DirName, Node, MetaFile).
+
+fetch_report(Localhost, Dir, Node, MetaFile) ->
+ fetch(Localhost,Dir,Node,MetaFile),
+ ?MODULE ! {fetch_complete,Node}.
fetch(Localhost,Dir,Node,MetaFile) ->
case (host(Node) == Localhost) orelse is_local(MetaFile) of
@@ -718,8 +781,7 @@ fetch(Localhost,Dir,Node,MetaFile) ->
receive_files(Dir,Sock,undefined),
ok = gen_tcp:close(LSock),
ok = gen_tcp:close(Sock)
- end,
- ?MODULE ! {fetch_complete,Node}.
+ end.
is_local({local, _, _}) ->
true;
@@ -749,7 +811,6 @@ host(Node) ->
[_name,Host] = string:tokens(atom_to_list(Node),"@"),
Host.
-
wait_for_fetch([]) ->
ok;
wait_for_fetch(Nodes) ->
@@ -797,15 +858,8 @@ format(File,Out,Handler) when is_list(File), is_integer(hd(File)) ->
Files =
case filelib:is_dir(File) of
true -> % will merge all files in the directory
- MetaFiles = filelib:wildcard(filename:join(File,"*.ti")),
- lists:map(fun(M) ->
- Sub = string:left(M,length(M)-3),
- case filelib:is_file(Sub) of
- true -> Sub;
- false -> Sub++".*.wrp"
- end
- end,
- MetaFiles);
+ List = filelib:wildcard(filename:join(File, ?partial_dir++"*")),
+ lists:append(collect_files([File | List]));
false -> % format one file
[File]
end,
@@ -827,6 +881,19 @@ format(Files,Out,Handler) when is_list(Files), is_list(hd(Files)) ->
end,
R.
+collect_files(Dirs) ->
+ lists:map(fun(Dir) ->
+ MetaFiles = filelib:wildcard(filename:join(Dir,"*.ti")),
+ lists:map(fun(M) ->
+ Sub = string:left(M,length(M)-3),
+ case filelib:is_file(Sub) of
+ true -> Sub;
+ false -> Sub++".*.wrp"
+ end
+ end,
+ MetaFiles)
+ end, Dirs).
+
prepare(File,Handler) ->
{Traci,Proci} = read_traci(File),
Node = get_node(Traci),
@@ -855,7 +922,6 @@ format_opt(Opt) when is_list(Opt) ->
format_opt(Opt) ->
format_opt([Opt]).
-
read_traci(File) ->
MetaFile = get_metafile(File),
case file:read_file(MetaFile) of
@@ -931,7 +997,6 @@ check_exists(File) ->
exit({error,no_file})
end.
-
get_handler(Handler,Traci) ->
case Handler of
undefined ->
@@ -1060,7 +1125,6 @@ get_next(Client,State) when is_pid(Client) ->
sort(List) ->
lists:keysort(1,List).
-
timestamp(Trace) when element(1,Trace) =:= trace_ts;
element(1,Trace) =:= seq_trace, tuple_size(Trace) =:= 4 ->
element(tuple_size(Trace),Trace);
@@ -1133,5 +1197,3 @@ dump_ti(<<>>,Acc) ->
dump_ti(B,Acc) ->
{Term,Rest} = get_term(B),
dump_ti(Rest,[Term|Acc]).
-
-