aboutsummaryrefslogtreecommitdiffstats
path: root/lib/appmon/src/process_info.erl
diff options
context:
space:
mode:
authorErlang/OTP <[email protected]>2009-11-20 14:54:40 +0000
committerErlang/OTP <[email protected]>2009-11-20 14:54:40 +0000
commit84adefa331c4159d432d22840663c38f155cd4c1 (patch)
treebff9a9c66adda4df2106dfd0e5c053ab182a12bd /lib/appmon/src/process_info.erl
downloadotp-84adefa331c4159d432d22840663c38f155cd4c1.tar.gz
otp-84adefa331c4159d432d22840663c38f155cd4c1.tar.bz2
otp-84adefa331c4159d432d22840663c38f155cd4c1.zip
The R13B03 release.OTP_R13B03
Diffstat (limited to 'lib/appmon/src/process_info.erl')
-rw-r--r--lib/appmon/src/process_info.erl662
1 files changed, 662 insertions, 0 deletions
diff --git a/lib/appmon/src/process_info.erl b/lib/appmon/src/process_info.erl
new file mode 100644
index 0000000000..e5d44ae50e
--- /dev/null
+++ b/lib/appmon/src/process_info.erl
@@ -0,0 +1,662 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2001-2009. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+-module(process_info).
+-behavior(gen_server).
+
+-export([start/0, start_link/0, stop/0]).
+-export([is_node/1, get_nodes/0,
+ get_applications/1, get_application_keys/2,
+ get_processes/3, get_process_data/2,
+ send_trace/1]).
+
+%% gen_server callbacks
+-export([init/1, handle_call/3, handle_cast/2, handle_info/2,
+ terminate/2, code_change/3]).
+
+-record(data, {que=undef,
+ procs=undef,
+ links=undef,
+ links2=undef}).
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%% %%
+%% Functions to retrieve information about which application %%
+%% at the node %%
+%% %%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+start() ->
+ gen_server:start({local, proc_info}, process_info, [], []).
+
+start_link() ->
+ gen_server:start_link({local, proc_info}, process_info, [], []).
+
+stop() ->
+ gen_server:call(proc_info, stop, 1000).
+
+%% is_node(NodeS) -> {bool(), Node, NodeS2}
+%% NodeS = NodeS2 = string()
+%% Node = node()
+is_node(NodeS) ->
+ Node = list_to_atom(NodeS),
+ case lists:member(Node, [node()|nodes()]) of
+ true->
+ {true, Node, NodeS};
+ false ->
+ {false, node(), atom_to_list(node())}
+ end.
+
+%% get_nodes() -> [node()]
+get_nodes() ->
+ [node()|nodes()].
+
+%% get_applications(Node) -> [App]
+%% Node = node()
+%% App = atom()
+%% Returns the list of all applications with a supervision tree (that
+%% is, not library applications such as stdlib) at Node.
+get_applications(Node) ->
+ Info = rpc:call(Node, application, info, []),
+ {value, {running, Apps}} = lists:keysearch(running, 1, Info),
+ [App || {App, Pid} <- Apps, is_pid(Pid)].
+
+%% get_application_keys(App, Node) -> {ok, Keys} | {error, Reason}
+%% Node = node()
+%% App = atom()
+%% Keys = [{Key, Val}]
+%% Key = atom()
+%% Val = term()
+%% Reason = badapp | badrpc
+get_application_keys(App, Node) ->
+ case rpc:call(Node, application, get_all_key, [App]) of
+ {ok, Keys} ->
+ {ok, Keys};
+ undefined ->
+ {error, badapp};
+ {badrpc, _} ->
+ {error, badrpc}
+ end.
+
+%% get_processes(App, Mode, Node) -> {Tree, Dict} | unknown
+%% App = atom()
+%% Mode = sup | sup_child | all
+%% Node = node()
+get_processes(App, Mode, Node) ->
+ gen_server:call(proc_info, {get_processes, App, Mode, Node}).
+
+%% get_process_data(Pid, Node) -> ProcData
+%% Pid = pid()
+%% Node = node()
+%% ProcData -- see erlang:process_info/1
+get_process_data(Pid, Node) ->
+ case rpc:call(Node, erlang, process_info, [Pid]) of
+ {badrpc, _} ->
+ [{error,"Please try again"}];
+ Res ->
+ Res
+ end.
+
+%% send_trace(PidL) -> void()
+send_trace(PidL) ->
+ gen_server:call(proc_info, {send_trace, PidL}).
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%% %%
+%% gen_server callbacks %%
+%% %%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+init([]) ->
+ {ok, ets:new(procs, [])}.
+
+handle_call({get_processes, App, Mode, Node}, _From, State) ->
+ case do_get_processes(App, Mode, Node) of
+ unknown ->
+ {reply, unknown, State};
+ Tree ->
+ {reply, {Tree, State}, State}
+ end;
+handle_call({send_trace, PidL}, _From, State) ->
+ do_send_trace(PidL, State),
+ {reply, ok, State};
+handle_call(stop, _From, State) ->
+ {stop, normal, ok, State}.
+
+handle_cast(_, State) ->
+ {noreply, State}.
+
+handle_info(_, State) ->
+ {noreply, State}.
+
+terminate(_Reason, _State) ->
+ ok.
+
+code_change(_, State, _) ->
+ {ok, State}.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%% %%
+%% Internal functions %%
+%% %%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+%% do_get_processes(App, Mode, Node) -> Tree | unknown
+%% App = atom()
+%% Mode = all | sup | sup_childs
+%% Node = node()
+%% Tree = term()
+do_get_processes(App, Mode, Node) ->
+ case rpc:call(Node, application_controller, get_master, [App]) of
+ Pid when is_pid(Pid) ->
+ start_collecting_data(Pid, Mode, Node);
+ undefined ->
+ unknown
+ end.
+
+%% Initiate the database, get the processes and the links.
+%% Then build lists and return them.
+start_collecting_data(Pid, Mode, Node) ->
+ Db = get_database(),
+ {Db2, Tree} = build_graph({master,Pid}, Db, Pid, Mode, Node),
+ delete_database(Db2),
+ Tree.
+
+get_database() ->
+ P = ets:new(procs,[]),
+ L = ets:new(link,[bag]),
+ L2 = ets:new(link2,[bag]),
+ Q = queue:new(),
+ ets:insert(P, {whereis(application_controller), crap}),
+ ets:insert(P, {whereis(gs), crap}),
+ #data{que=Q, procs=P, links=L, links2=L2}.
+
+delete_database(Db) ->
+ ets:delete(Db#data.procs),
+ ets:delete(Db#data.links),
+ ets:delete(Db#data.links2).
+
+%% The thought is
+%% 1. Get the processes that links to Pid.
+%% Pid is the application master the first time.
+%% 2. Add the processes to the database and clear the list of children
+%% from processes which for some resason not should be there.
+%% 3. Queue the children, so we later can se if they have any links.
+%% 4. Add links to the childrens.
+%% 5. When the whole tree is retreived remove the unnecessary processes
+%% depending on the mode.
+%% 6. Take all links that point to the same pid and sort out
+%% the primary and secondary relations.
+%% If more than one process links to the same process, the relation
+%% between a supervisor and a process is primary. The rest is
+%% secondary, there is no different in real world just in logic
+%% between a secondary and a primary relation.
+%% When all processes in the application is collected,
+%% fix secondary links and return the tree.
+build_graph(finish, Db, Grp, Mode, Node) ->
+ Db = fix_links(Db, Grp, Node),
+ delete_unwanted(Db, Mode, Grp),
+ Tree = start_tree(Db, Node),
+ {Db, Tree};
+build_graph(Pid, Db, Grp, Mode, Node) ->
+ Children = get_children(Pid, Mode, Node),
+ Children2 = add_and_remove(Children, Pid, Db, Grp, Node),
+ Q2 = queue_children(Db#data.que, Children2),
+ add_children(Pid, Db, Children2, 1),
+ case queue:out(Q2) of
+ {empty, _}->
+ build_graph(finish, Db, Grp, Mode, Node);
+ {{value,NPid}, Q3}->
+ Db2 = Db#data{que=Q3},
+ build_graph(NPid,Db2,Grp,Mode,Node)
+ end.
+
+%% Collect the processes which the current process has a link to
+%% Pid is now the application_master and the application master's
+%% child is the application supervisor but in reality there are two
+%% application master processes.
+%% Fix this by reordering the processes a little.
+get_children({master,Pid}, _Mode, Node) when is_pid(Pid) ->
+ %% Get the master pid
+ MPid = case application_master:get_child(Pid) of
+ {Pid1, _App} -> Pid1;
+ Pid1 -> Pid1
+ end,
+ %% Get the second appplication master process and order them
+ %% correctly
+ case rpc:call(Node, erlang, process_info, [MPid,links]) of
+ {links, [H|T]} -> [H,MPid|T];
+ {links, []} -> MPid
+ end;
+get_children({Pid, _Name}, _Mode, Node) when is_pid(Pid),
+ Node==node(Pid) ->
+ {links,Links} = rpc:call(Node, erlang, process_info, [Pid,links]),
+ Links;
+get_children(Pid, _Mode, Node) when is_pid(Pid), Node==node(Pid) ->
+ {links,Links} = rpc:call(Node, erlang, process_info, [Pid,links]),
+ Links;
+get_children(Pid, _Mode, Node) when is_pid(Pid), Node/=node(Pid) ->
+ [];
+get_children(Port, _Mode, _Node) when is_port(Port) ->
+ [].
+
+%% Add the links to the database.
+%% The first case -- when it is the application master process -- there
+%% is only one real child even though there are more links.
+add_children({master,Pid}, Db, [Child|_Rest], N) ->
+ add_child(Pid, Db, Child, N);
+add_children(_Pid, _Db, [], _N) ->
+ ok;
+add_children(Pid, Db, [Child|Rest], N) ->
+ add_child(Pid, Db, Child, N),
+ add_children(Pid, Db, Rest, N+1).
+
+add_child(Pid, Db, Child, N) ->
+ case ets:match_object(Db#data.links, {Pid,Child,'_'}) of
+ [] ->
+ ets:insert(Db#data.links, {Pid,Child,N});
+ _ ->
+ ok
+ end.
+
+%% Add the list of processes to the queue.
+queue_children(Queue, []) ->
+ Queue;
+queue_children(Queue, [H|T]) ->
+ Q = queue:in(H, Queue),
+ queue_children(Q, T).
+
+%% The processess that we already has added to the database are
+%% not children to the current process, so we don't need to add them a
+%% second time.
+remove_used_children([], _Db, New_list) ->
+ lists:reverse(New_list);
+remove_used_children([Child|Rest], Db, New) ->
+ case ets:lookup(Db#data.procs, Child) of
+ [] ->
+ remove_used_children(Rest, Db, [Child|New]);
+ _ ->
+ remove_used_children(Rest, Db, New)
+ end.
+
+%% Take the list of links and separate it into a list with ports and a
+%% list with pids.
+separate_ports([], Pids, Ports) ->
+ {Pids, Ports};
+separate_ports([Child|Rest], Pids, Ports) ->
+ if
+ is_port(Child) ->
+ separate_ports(Rest, Pids, [Child|Ports]);
+ is_pid(Child) ->
+ separate_ports(Rest, [Child|Pids], Ports)
+ end.
+
+%% Add the current pid to the ets table with processes and clear
+%% the list of children from processes that should not be there.
+%% In the first case, no children are used so it's not necessary.
+add_and_remove(Children, {master,Pid}, Db, _Grp, Node)
+ when is_pid(Pid), Node==node(Pid) ->
+ ets:insert(Db#data.procs, {Pid, {master,master}, controller}),
+ {_Pids,Ports} = separate_ports(Children, [], []),
+ Ports++Children;
+%% This clause is removable when using only link as retrieving mode .
+add_and_remove(Children, {Pid,_Name}, Db, Grp, Node)
+ when is_pid(Pid), Node==node(Pid) ->
+ ets:insert(Db#data.procs, {Pid,
+ rpc:call(Node,erlang,process_info,
+ [Pid,registered_name])}),
+ {Pids, Ports} = separate_ports(Children, [], []),
+ Children1 = remove_used_children(Pids, Db, []),
+ Children2 = remove_others_children(Children1, Grp, Node),
+ Ports++Children2;
+add_and_remove(Children, Pid, Db, Grp, Node) when is_pid(Pid),
+ Node==node(Pid) ->
+ ets:insert(Db#data.procs, {Pid,
+ rpc:call(Node,erlang,process_info,
+ [Pid,registered_name])}),
+ {Pids, Ports} = separate_ports(Children, [], []),
+ Children1 = remove_used_children(Pids, Db, []),
+ Children2 =remove_others_children(Children1, Grp, Node),
+ Ports++Children2;
+add_and_remove(_Children, Pid, _Db, _Grp, Node) when is_pid(Pid),
+ Node/=node(Pid) ->
+ [];
+%% Take care of the ports, don't add them to the table with processes.
+add_and_remove(_Children, Pid, _Db, _Grp, _Node) when is_port(Pid) ->
+ [].
+
+%% Control that the application's group leader is the group leader of
+%% Pid
+group_leader_check({Pid,_Name}, Grp, Node) ->
+ group_leader_check(Pid, Grp, Node);
+group_leader_check(Pid, Grp, Node) ->
+ case rpc:call(Node, erlang, process_info, [Pid,group_leader]) of
+ {_Item, Grp} -> yes;
+ _ -> no
+ end.
+
+%% Take the list of children and remove the ones with anoother group
+%% leader.
+remove_others_children(Children, Grp, Node) ->
+ lists:filter(fun(Child) ->
+ case group_leader_check(Child, Grp, Node) of
+ yes -> true;
+ no -> false
+ end
+ end,
+ Children).
+
+%% Mark the processes in the procs table as either supervisor or worker.
+fix_links(Db, Leader, Node) ->
+ {Sup,_Work} = mark_supervisors_workers(Db, Leader, Node),
+ ets:match_delete(Db#data.procs, {'_',crap}),
+ [_Pid|Procs] = ets:tab2list(Db#data.procs),
+ N_links = get_n_links(Procs, Db#data.links, []),
+ N_links2 = take_sup_links(Sup, Db#data.links, N_links),
+ add_shared_links(N_links2, Db#data.links2),
+ Db.
+
+%% Add the links that point to the same child to the shared links table
+add_shared_links(N_links, Links2) ->
+ Insert_fun = fun(Link) -> ets:insert(Links2, Link) end,
+ lists:map(fun(List) -> lists:map(Insert_fun, List) end, N_links).
+
+%% Take the list of links that point to the same children and remove
+%% the ones that are children to supervisors.
+%% The first argument is a list of the supervisors.
+%% N_links contains a list of list of links that points to the same
+%% child.
+take_sup_links([], _Db, N_links) ->
+ N_links;
+take_sup_links([H|Supervised], Links_table, N_links) ->
+ N_list_fun = fun(Link) ->
+ insert_sup_links(Link, H, Links_table)
+ end,
+ N_links2 = lists:map(fun(Link_list) ->
+ lists:filter(N_list_fun,Link_list)
+ end,
+ N_links),
+ take_sup_links(Supervised, Links_table, N_links2).
+
+%% Insert the supervised links in the primary links list.
+%% This function should be used as a fun to the filter function in
+%% take_sup_links/3.
+insert_sup_links({From,To,N}, Sup, Links_table) ->
+ case From of
+ Sup ->
+ ets:insert(Links_table, {From,To,N}),
+ false;
+ _ ->
+ true
+ end.
+
+%% Get the links which points to the same children.
+get_n_links([], _Links, N_link) ->
+ N_link;
+get_n_links([{Pid,_,_}|Procs], Links, N_link) ->
+ case ets:match_object(Links, {'_',Pid,'_'}) of
+ L when length(L)>1 ->
+ ets:match_delete(Links, {'_',Pid,'_'}),
+ get_n_links(Procs, Links, [L|N_link]);
+ _L ->
+ get_n_links(Procs, Links, N_link)
+ end;
+get_n_links([{Pid,_}|Procs], Links, N_link) ->
+ case ets:match_object(Links, {'_',Pid,'_'}) of
+ L when length(L)>1 ->
+ ets:match_delete(Links, {'_',Pid,'_'}),
+ get_n_links(Procs, Links, [L|N_link]);
+ _L ->
+ get_n_links(Procs, Links, N_link)
+ end.
+
+%% Mark the processes that are in the supervisor tree as either worker
+%% or supervisor.
+mark_supervisors_workers(Db, Leader, Node) ->
+ %% Get the supervisors and workers.
+ {Sup_list, Worker_list} = get_by_supervisors1(Leader),
+ %% Update the supervisor pids.
+ lists:map(fun(Pid) ->
+ ets:insert(Db#data.procs,
+ {Pid,
+ rpc:call(Node, erlang,process_info,
+ [Pid,registered_name]),
+ supervisor})
+ end,
+ Sup_list),
+ %% Update the worker pids.
+ lists:map(fun(Pid) ->
+ ets:insert(Db#data.procs,
+ {Pid,
+ rpc:call(Node, erlang,process_info,
+ [Pid,registered_name]),
+ worker})
+ end,
+ Worker_list),
+ {lists:reverse(Sup_list), Worker_list}.
+
+%% The second way to retrieve the applications processes is to go by
+%% the supervision tree.
+get_by_supervisors1(Leader) ->
+ case application_master:get_child(Leader) of
+ {Pid, _Name}->
+ get_by_supervisors([{namn,Pid,supervisor,list_of_mods}],
+ [], []);
+ Pid ->
+ get_by_supervisors([{namn,Pid,supervisor,list_of_mods}],
+ [], [])
+ end.
+
+get_by_supervisors([], Sup, Work) ->
+ {Sup, Work};
+get_by_supervisors([{_,Pid,supervisor,_}|Rest], Sup, Work)
+ when is_pid(Pid) ->
+ Children = supervisor:which_children(Pid),
+ Children2 = lists:append(Children, Rest),
+ get_by_supervisors(Children2, [Pid|Sup], Work);
+get_by_supervisors([{_,Pid,_,_}|Rest], Sup, Work) when is_pid(Pid) ->
+ get_by_supervisors(Rest, Sup, [Pid|Work]);
+get_by_supervisors([_Whatever|Rest], Sup, Work) ->
+ get_by_supervisors(Rest, Sup, Work).
+
+%% Use pattern matching to select mode and delete the unneccesary pids
+delete_unwanted(Db, sup_child, App_pid) ->
+ delete_not_in_supervisor_tree(Db),
+ add_main_link(Db, App_pid),
+ Db;
+delete_unwanted(Db, all, _App_pid) ->
+ Db;
+delete_unwanted(Db, sup, App_pid) ->
+ delete_workers(Db),
+ delete_not_in_supervisor_tree(Db),
+ add_main_link(Db, App_pid),
+ Db.
+
+add_main_link(Db, App_pid) ->
+ case application_master:get_child(App_pid) of
+ {Pid, _Name} when is_pid(Pid) ->
+ ets:insert(Db#data.links, {App_pid,Pid,1});
+ Pid when is_pid(Pid) ->
+ ets:insert(Db#data.links, {App_pid,Pid,1});
+ _ ->
+ false
+ end.
+
+%% Delete the processes that are in the supervision tree but are
+%% workers, and their links.
+delete_workers(Db) ->
+ Pids = ets:match_object(Db#data.procs, {'_','_',worker}),
+ Pids2 =
+ lists:map(
+ fun({Pid,_,_}) ->
+ %% Remove the unwanted pids from the process table.
+ ets:match_delete(Db#data.procs, {Pid,'_','_'}),
+ %% Remove the links to and from the pid.
+ ets:match_delete(Db#data.links, {Pid,'_','_'}),
+ ets:match_delete(Db#data.links, {'_',Pid,'_'}),
+ ets:match_delete(Db#data.links2, {Pid,'_','_'}),
+ ets:match_delete(Db#data.links2, {'_',Pid,'_'})
+ end,
+ Pids),
+ Pids2.
+
+%% Delete the processes that are not in the supervision tree.
+delete_not_in_supervisor_tree(Db) ->
+ Pids = ets:match_object(Db#data.procs,{'_','_'}),
+ Pids2 =
+ lists:map(
+ fun({Pid,_}) ->
+ %% Remove the unwanted from the process table.
+ ets:match_delete(Db#data.procs, {Pid,'_'}),
+ %% Remove the links to and from the pid.
+ ets:match_delete(Db#data.links, {Pid,'_','_'}),
+ ets:match_delete(Db#data.links, {'_',Pid,'_'}),
+ ets:match_delete(Db#data.links2, {Pid,'_','_'}),
+ ets:match_delete(Db#data.links2, {'_',Pid,'_'})
+ end,
+ Pids),
+ Pids2.
+
+%% Start generating the tree.
+start_tree(Db, Node) ->
+ case get_master(Db) of
+ no -> false;
+ Pid ->
+ build_node(Pid, Db, Node)
+ end.
+
+%% Build a node and then it runs itself on every child to the current
+%% pid.
+build_node(Pid, Db, Node) when is_pid(Pid), Node==node(Pid) ->
+ Sort_fun = fun sort_order/2,
+ Fix_sec_name_fun = fun(Pid2) -> get_link_name(Pid2, Db) end,
+ Build_tree_fun = fun({_,Pid1,_}) -> build_node(Pid1,Db,Node) end,
+ Children = ets:match_object(Db#data.links, {Pid,'_','_'}),
+ Children1 = lists:sort(Sort_fun, Children),
+ Sec_children = ets:match_object(Db#data.links2, {Pid,'_','_'}),
+ {get_name(Pid,Db),
+ lists:map(Build_tree_fun,Children1),
+ lists:map(Fix_sec_name_fun,Sec_children)};
+build_node(Pid, _Db, Node) when is_pid(Pid), Node/=node(Pid) ->
+ {"Runs on another node:"++erlang:pid_to_list(Pid), [], []};
+build_node(Pid, _Db, _Node) when is_port(Pid) ->
+ {"Port :"++erlang:port_to_list(Pid), [], []}.
+
+%% Select the name of the pid from the database where we previosly
+%% added it.
+get_name(Pid, Db) ->
+ case ets:lookup(Db#data.procs, Pid) of
+ [{_,{_,master},_}] -> pid_to_list(Pid);
+ [{_,{_,Name}}] -> atom_to_list(Name)++" : "++pid_to_list(Pid);
+ [{_,{_,Name},_}] -> atom_to_list(Name)++" : "++pid_to_list(Pid);
+ _ -> pid_to_list(Pid)
+ end.
+
+%% Select the name of the process which we have a link to.
+get_link_name({_,Pid,_}, Db) when is_pid(Pid) ->
+ case ets:lookup(Db#data.procs, Pid) of
+ [{_,{_,Name}}] -> atom_to_list(Name)++" : "++pid_to_list(Pid);
+ [{_,{_,Name},_}] -> atom_to_list(Name)++" : "++pid_to_list(Pid);
+ _ -> pid_to_list(Pid)
+ end;
+get_link_name({_,Port,_}, _Db) when is_port(Port) ->
+ "Port :"++" : ";
+get_link_name(_, _) ->
+ "".
+
+%% Sort the links in the order they where added, in ascending order.
+sort_order({_,_,N1}, {_,_,N2}) when N1>N2 -> true;
+sort_order(_N1, _N2) -> false.
+
+%% Select the pid of the application master.
+get_master(Db) ->
+ case ets:match_object(Db#data.procs,
+ {'_',{master,master},controller}) of
+ [{Pid,_,_}|_Rest] -> Pid;
+ _ -> no
+ end.
+
+%% The main function to handle tracing.
+%% Checks if the process is in the table with traced processes. If so,
+%% it stops the trace, otherwise it starts the trace.
+do_send_trace(PidL, Traced_tab) ->
+ Pid = list_to_pid(PidL),
+ Key = get_key(Pid),
+ case catch ets:lookup(Traced_tab, Key) of
+ [] ->
+ trace_process(Pid, Key, true, Traced_tab);
+ [_Object]->
+ trace_process(Pid, Key, false, Traced_tab)
+ end,
+ filter_procs(Traced_tab, ets:tab2list(Traced_tab)).
+
+get_key(Pid) ->
+ Node = node(Pid),
+ case rpc:call(Node, erlang, process_info, [Pid,registered_name]) of
+ [] -> pid_to_list(Pid);
+ {registered_name, Name} ->
+ atom_to_list(Name)++" : "++pid_to_list(Pid)
+ end.
+
+%% Tries to toggle the trace flag for the process.
+trace_process(Pid, Key, On_or_off, Procs_tab) ->
+ case rpc:call(node(Pid), sys, trace, [Pid,On_or_off,1000]) of
+ timeout ->
+ Node = node(Pid),
+ io:fwrite("timeout node= ~w, Pid= ~w mode= ~w ~n",
+ [Node, Pid, On_or_off]);
+ {badrpc, _} ->
+ Node = node(Pid),
+ io:fwrite("badrpc node= ~w, Pid= ~w mode= ~w ~n",
+ [Node, Pid, On_or_off]);
+ Res ->
+ Node = node(Pid),
+ io:fwrite("anymode ~w node= ~w, Pid= ~w mode= ~w ~n",
+ [Res, Node, Pid,On_or_off]),
+ case On_or_off of
+ true -> ets:insert(Procs_tab, {Key,On_or_off});
+ false -> ets:delete(Procs_tab, Key)
+ end
+ end.
+
+%% Check if the processes in the ets table with traced processes
+%% are alive. If not, remove them.
+filter_procs(Tab, Tab_list) ->
+ lists:foreach(fun({Key,_Val}) -> is_alive(Key, Tab) end, Tab_list).
+
+is_alive(Key, Tab) ->
+ case get_pid(Key) of
+ nopid -> false;
+ Pid -> is_alive2(Pid, Key, Tab)
+ end.
+
+%% Key is either a pid in list form or Pidname:Pid in list form.
+get_pid(Key) ->
+ case catch list_to_pid(string:substr(Key,string:rchr(Key,$<))) of
+ Pid when is_pid(Pid) ->
+ Pid;
+ _ ->
+ nopid
+ end.
+
+is_alive2(Pid, Key, Tab) ->
+ case catch rpc:call(node(Pid), erlang, is_process_alive, [Pid]) of
+ true -> true;
+ false ->
+ catch ets:delete(Tab, Key),
+ ok
+ end.