dbg-type tracing

1 files changed, 262 insertions, 91 deletions

diff --git a/lib/observer/src/ttb.erl b/lib/observer/src/ttb.erl
index d5f4b52b42..dfba0411a0 100644
--- a/lib/observer/src/ttb.erl
+++ b/lib/observer/src/ttb.erl
@@ -35,10 +35,12 @@
 
 -include_lib("kernel/include/file.hrl").
 -define(meta_time,5000).
+-define(fetch_time, 10000).
 -define(history_table,ttb_history_table).
 -define(seq_trace_flags,[send,'receive',print,timestamp]).
 -define(upload_dir,"ttb_upload").
 -define(last_config, "ttb_last_config").
+-define(partial_dir, "ttb_partial_result").
 -ifdef(debug).
 -define(get_status,;get_status -> erlang:display(dict:to_list(NodeInfo),loop(NodeInfo, TraceInfo)).
 -else.
@@ -49,21 +51,22 @@
 %%% Shortcut
 start_trace(Nodes, Patterns, {Procs, Flags}, Options) ->
     {ok, _} = tracer(Nodes, Options),
-    {ok, _} = p(Procs, Flags),
-    [{ok, _} = apply(?MODULE, tpl, tuple_to_list(Args)) || Args <- Patterns].
-
+    [{ok, _} = apply(?MODULE, tpl, tuple_to_list(Args)) || Args <- Patterns],
+    {ok, _} = p(Procs, Flags).
 
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %%% Open a trace port on all given nodes and create the meta data file
 tracer() -> tracer(node()).
+tracer(shell) -> tracer(node(), shell);
+tracer(dbg) -> tracer(node(), {shell, only});
 tracer(Nodes) -> tracer(Nodes,[]).
 tracer(Nodes,Opt) ->
     {PI,Client,Traci} = opt(Opt),
     %%We use initial Traci as SessionInfo for loop/2
-    start(Traci),
+    Pid = start(Traci),
     store(tracer,[Nodes,Opt]),
-    do_tracer(Nodes,PI,Client,Traci).
+    do_tracer(Nodes,PI,Client,[{ttb_control, Pid}|Traci]).
 
 do_tracer(Nodes0,PI,Client,Traci) ->
     Nodes = nods(Nodes0),
@@ -71,9 +74,14 @@ do_tracer(Nodes0,PI,Client,Traci) ->
     do_tracer(Clients,PI,Traci).
 
 do_tracer(Clients,PI,Traci) ->
+    ShellOutput = proplists:get_value(shell, Traci, false),
     {ClientSucc,Succ} = 
 	lists:foldl(
 	  fun({N,{local,File},TF},{CS,S}) -> 
+                  TF2 = case ShellOutput of
+                            only -> none;
+                            _    -> TF
+                        end,
 		  [_Sname,Host] = string:tokens(atom_to_list(N),"@"),
 		  case catch dbg:tracer(N,port,dbg:trace_port(ip,0)) of
 		      {ok,N} ->
@@ -81,8 +89,8 @@ do_tracer(Clients,PI,Traci) ->
 			  {ok,T} = dbg:get_tracer(N),
 			  rpc:call(N,seq_trace,set_system_tracer,[T]),
 			  dbg:trace_client(ip,{Host,Port},
-					   {fun ip_to_file/2,{file,File}}),
-			  {[{N,{local,File,Port},TF}|CS], [N|S]};
+					   {fun ip_to_file/2,{{file,File}, ShellOutput}}),
+			  {[{N,{local,File,Port},TF2}|CS], [N|S]};
 		      Other ->
 			  display_warning(N,{cannot_open_ip_trace_port,
 					     Host,
@@ -125,8 +133,34 @@ opt([{timer, {MSec, StopOpts}}|O],{PI,Client,Traci}) ->
     opt(O,{PI,Client,[{timer,{MSec, StopOpts}}|Traci]});
 opt([{timer, MSec}|O],{PI,Client,Traci}) ->
     opt(O,{PI,Client,[{timer,{MSec, []}}|Traci]});
+opt([{overload_check, {MSec,M,F}}|O],{PI,Client,Traci}) ->
+    opt(O,{PI,Client,[{overload_check,{MSec,M,F}}|Traci]});
+opt([shell|O],{PI,Client,Traci}) ->
+    opt(O,{PI,Client,[{shell, true}|Traci]});
+opt([{shell,Type}|O],{PI,Client,Traci}) ->
+    opt(O,{PI,Client,[{shell, Type}|Traci]});
+opt([resume|O],{PI,Client,Traci}) ->
+    opt(O,{PI,Client,[{resume, {true, ?fetch_time}}|Traci]});
+opt([{resume,MSec}|O],{PI,Client,Traci}) ->
+    opt(O,{PI,Client,[{resume, {true, MSec}}|Traci]});
+opt([{flush,MSec}|O],{PI,Client,Traci}) ->
+    opt(O,{PI,Client,[{flush, MSec}|Traci]});
 opt([],Opt) ->
-    Opt.
+    ensure_opt(Opt).
+
+ensure_opt({PI,Client,Traci}) ->
+    case {proplists:get_value(flush, Traci), Client} of
+        {undefined, _}  -> ok;
+        {_, {local, _}} -> exit(flush_unsupported_with_ip_trace_port);
+        {_,_}           -> ok
+    end,
+    NeedIpTracer = proplists:get_value(shell, Traci, false) /= false,
+    case {NeedIpTracer, Client} of
+        {false, _}        -> {PI, Client, Traci};
+        {true, ?MODULE}       -> {PI, {local, ?MODULE}, Traci};
+        {true, {local, File}} -> {PI, {local, File}, Traci};
+        {true, _}             -> exit(local_client_required_on_shell_tracing)
+    end.
 
 nods(all) ->
     Nodes1 = remove_active([node()|nodes()]),
@@ -223,17 +257,29 @@ run_history([H|T]) ->
 	ok -> run_history(T);
 	{error,not_found} -> {error,{not_found,H}}
     end;
+
+run_history(all) ->
+    CurrentHist = ets:tab2list(?history_table),
+    ets:delete_all_objects(?history_table),
+    [run_printed(MFA,true) || {_, MFA} <- CurrentHist];
+run_history(all_silent) ->
+    CurrentHist = ets:tab2list(?history_table),
+    ets:delete_all_objects(?history_table),
+    [run_printed(MFA,false) || {_, MFA} <- CurrentHist];
 run_history([]) ->
     ok;
 run_history(N) ->
     case catch ets:lookup(?history_table,N) of
 	[{N,{M,F,A}}] -> 
-	    print_func(M,F,A),
-	    R = apply(M,F,A),
-	    print_result(R);
+            run_printed({M,F,A},true);
 	_ -> 
 	    {error, not_found}
     end.
+
+run_printed({M,F,A},Verbose) ->
+    Verbose andalso print_func(M,F,A),
+    R = apply(M,F,A),
+    Verbose andalso print_result(R).
 	
 write_config(ConfigFile,all) ->
     write_config(ConfigFile,['_']);
@@ -331,6 +377,7 @@ arg_list([A1|A],Acc) ->
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %%% Set trace flags on processes
 p(Procs0,Flags0) ->
+    ensure_no_overloaded_nodes(),
     store(p,[Procs0,Flags0]),
     no_store_p(Procs0,Flags0).
 no_store_p(Procs0,Flags0) ->
@@ -384,22 +431,28 @@ proc({global,Name}) ->
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %%% Trace pattern
 tp(A,B) ->
+    ensure_no_overloaded_nodes(),
     store(tp,[A,ms(B)]),
     dbg:tp(A,ms(B)).
 tp(A,B,C) ->
+    ensure_no_overloaded_nodes(),
     store(tp,[A,B,ms(C)]),
     dbg:tp(A,B,ms(C)).
 tp(A,B,C,D) ->
+    ensure_no_overloaded_nodes(),
     store(tp,[A,B,C,ms(D)]),
     dbg:tp(A,B,C,ms(D)).
 
 tpl(A,B) ->
+    ensure_no_overloaded_nodes(),
     store(tpl,[A,ms(B)]),
     dbg:tpl(A,ms(B)).
 tpl(A,B,C) ->
+    ensure_no_overloaded_nodes(),
     store(tpl,[A,B,ms(C)]),
     dbg:tpl(A,B,ms(C)).
 tpl(A,B,C,D) ->
+    ensure_no_overloaded_nodes(),
     store(tpl,[A,B,C,ms(D)]),
     dbg:tpl(A,B,C,ms(D)).
 
@@ -452,6 +505,14 @@ ms({codestr, FunStr}) ->
 ms(Other) ->
     Other.
 
+ensure_no_overloaded_nodes() ->
+    ttb ! {get_overloaded, self()},
+    Overloaded = receive O -> O after 300 -> [] end,
+    case Overloaded of
+        [] -> ok;
+        Overloaded -> exit({error, overload_protection_active, Overloaded})
+    end.
+
 -spec string2ms(string()) -> {ok, list()} | {error, fun_format}.
 string2ms(FunStr) ->
     case erl_scan:string(fix_dot(FunStr)) of
@@ -537,12 +598,12 @@ stop_opts(Opts) ->
     case {FormatData, lists:member(return, Opts)} of
 	{false, true} ->
 	    {fetch, FetchDir}; % if we specify return, the data should be fetched
-        {false, false} ->
-	    case lists:member(fetch,Opts) of
-		true -> {fetch, FetchDir};
-		false -> nofetch
+    {false, false} ->
+	    case lists:member(nofetch,Opts) of
+		    false -> {fetch, FetchDir};
+		    true -> nofetch
 	    end;
-        {FormatData, _} ->
+    {FormatData, _} ->
             {FormatData, FetchDir}
     end.
 
@@ -558,6 +619,8 @@ ensure_fetch_dir(Dir) ->
 stop_return(R,Opts) ->
     case {lists:member(return,Opts),R} of
         {true,_} ->
+            %%Printout moved out of the ttb loop to avoid occasional deadlock
+            io:format("Stored logs in ~s~n",[element(2, R)]),
             R;
         {false,{stopped,_}} ->
             stopped;
@@ -574,16 +637,19 @@ start(SessionInfo) ->
 	undefined ->
 	    Parent = self(),
 	    Pid = spawn(fun() -> init(Parent, SessionInfo) end),
-	    receive {started,Pid} -> ok end;
+	    receive {started,Pid} -> ok end,
+            Pid;
 	Pid when is_pid(Pid) ->
-	    ok
+	    Pid
     end.
 
 init(Parent, SessionInfo) ->
     register(?MODULE,self()),
     ets:new(?history_table,[ordered_set,named_table,public]),
     Parent ! {started,self()},
-    loop(dict:new(), SessionInfo).
+    NewSessionInfo = [{partials, 0}, {dead_nodes, []} | SessionInfo],
+    try_send_flush_tick(NewSessionInfo),
+    loop(dict:new(), NewSessionInfo).
 
 loop(NodeInfo, SessionInfo) ->
     receive 
@@ -613,68 +679,118 @@ loop(NodeInfo, SessionInfo) ->
 		      NodeInfo),
 	    loop(NodeInfo, SessionInfo);
 	{nodedown,Node} ->
-	    loop(dict:erase(Node,NodeInfo), SessionInfo);
+            NewState = make_node_dead(Node, NodeInfo, SessionInfo),
+	    loop(dict:erase(Node,NodeInfo), NewState);
+        {noderesumed,Node,Reporter} ->
+            {MetaFile, CurrentSuffix, NewState} = make_node_alive(Node, SessionInfo),
+            fetch_partial_result(Node, MetaFile, CurrentSuffix),
+            spawn(fun() -> resume_trace(Reporter) end),
+            loop(NodeInfo, NewState);
 	{timeout, StopOpts} ->
 	    spawn(?MODULE, stop, [StopOpts]),
 	    loop(NodeInfo, SessionInfo);
+        {node_overloaded, Node} ->
+            io:format("Overload check activated on node: ~p.~n", [Node]),
+            {Overloaded, SI} = {proplists:get_value(overloaded, SessionInfo, []),
+                                lists:keydelete(overloaded, 1, SessionInfo)},
+	    loop(NodeInfo, [{overloaded, [Node|Overloaded]} | SI]);
+        {get_overloaded, Pid} ->
+            Pid ! proplists:get_value(overloaded, SessionInfo, []),
+            loop(NodeInfo, SessionInfo);
 	trace_started ->
 	    case proplists:get_value(timer, SessionInfo) of
 		undefined -> ok;
 		{MSec, StopOpts}  -> erlang:send_after(MSec, self(), {timeout, StopOpts})
 	    end,
 	    loop(NodeInfo, SessionInfo);
+        flush_timeout ->
+            [ dbg:flush_trace_port(Node) || Node <- dict:fetch_keys(NodeInfo) ],
+            try_send_flush_tick(SessionInfo),
+            loop(NodeInfo, SessionInfo);
 	{stop,nofetch,Sender} ->
-	    write_config(?last_config, all),
-	    dict:fold(
-	      fun(Node,{_,MetaPid},_) -> 
-		      rpc:call(Node,observer_backend,ttb_stop,[MetaPid])
-	      end,
-	      ok,
-	      NodeInfo),
-	    dbg:stop_clear(),
-	    ets:delete(?history_table),
-	    Sender ! {?MODULE,stopped};
-	{stop,{FetchOrFormat, UserDir} ,Sender} ->
-	    write_config(?last_config, all),
-	    Localhost = host(node()),
-	    Dir = get_fetch_dir(UserDir),
-	    file:make_dir(Dir),
-	    %% The nodes are traversed twice here because
-	    %% the meta tracing in observer_backend must be
-	    %% stopped before dbg is stopped, and dbg must
-	    %% be stopped before the trace logs are moved orelse
-	    %% windows complains.
-	    AllNodesAndMeta = 
-		dict:fold(
-		  fun(Node,{MetaFile,MetaPid},Nodes) -> 
-			  rpc:call(Node,observer_backend,ttb_stop,[MetaPid]),
-			  [{Node,MetaFile}|Nodes]
-		  end,
-		  [],
-		  NodeInfo),
-	    dbg:stop_clear(),
-	    AllNodes =
-		lists:map(
-		  fun({Node,MetaFile}) ->
-			  spawn(fun() -> fetch(Localhost,Dir,Node,MetaFile) end),
-			  Node
-		  end,
-		  AllNodesAndMeta),
-	    ets:delete(?history_table),
-	    wait_for_fetch(AllNodes),
-            Absname = filename:absname(Dir),
-	    io:format("Stored logs in ~s~n",[Absname]),
-	    case FetchOrFormat of
-		fetch -> ok;
-		{format, Opts} -> format(Dir, Opts)
-	    end,
-	    Sender ! {?MODULE,{stopped,Absname}}
-         ?get_status
+            do_stop(nofetch, Sender, NodeInfo, SessionInfo);
+	{stop,FetchSpec,Sender} ->
+            case proplists:get_value(shell, SessionInfo, false) of
+                only -> do_stop(nofetch, Sender, NodeInfo, SessionInfo);
+                _    -> do_stop(FetchSpec, Sender, NodeInfo, SessionInfo)
+            end
+    end.
+
+do_stop(nofetch, Sender, NodeInfo, _) ->
+    write_config(?last_config, all),
+    dict:fold(
+      fun(Node,{_,MetaPid},_) ->
+              rpc:call(Node,observer_backend,ttb_stop,[MetaPid])
+      end,
+      ok,
+      NodeInfo),
+    dbg:stop_clear(),
+    ets:delete(?history_table),
+    Sender ! {?MODULE, stopped};
+
+do_stop({FetchOrFormat, UserDir}, Sender, NodeInfo, SessionInfo) ->
+    write_config(?last_config, all),
+    Localhost = host(node()),
+    Dir = get_fetch_dir(UserDir),
+    file:make_dir(Dir),
+    %% The nodes are traversed twice here because
+    %% the meta tracing in observer_backend must be
+    %% stopped before dbg is stopped, and dbg must
+    %% be stopped before the trace logs are moved orelse
+    %% windows complains.
+    AllNodesAndMeta =
+        dict:fold(
+          fun(Node,{MetaFile,MetaPid},Nodes) ->
+                  rpc:call(Node,observer_backend,ttb_stop,[MetaPid]),
+                  [{Node,MetaFile}|Nodes]
+          end,
+          [],
+          NodeInfo),
+    dbg:stop_clear(),
+    AllNodes =
+        lists:map(
+          fun({Node,MetaFile}) ->
+                  spawn(fun() -> fetch_report(Localhost,Dir,Node,MetaFile) end),
+                  Node
+          end,
+          AllNodesAndMeta),
+    ets:delete(?history_table),
+    wait_for_fetch(AllNodes),
+    copy_partials(Dir, proplists:get_value(partials, SessionInfo)),
+    Absname = filename:absname(Dir),
+    case FetchOrFormat of
+        fetch -> ok;
+        {format, Opts} -> format(Dir, Opts)
+    end,
+    Sender ! {?MODULE,{stopped,Absname}}.
+
+make_node_dead(Node, NodeInfo, SessionInfo) ->
+    {MetaFile,_} = dict:fetch(Node, NodeInfo),
+    NewDeadNodes = [{Node, MetaFile} | proplists:get_value(dead_nodes, SessionInfo)],
+    [{dead_nodes, NewDeadNodes} | lists:keydelete(dead_nodes, 1, SessionInfo)].
+
+make_node_alive(Node, SessionInfo) ->
+            DeadNodes = proplists:get_value(dead_nodes, SessionInfo),
+            Partials = proplists:get_value(partials, SessionInfo),
+            {value, {_, MetaFile}, Dn2} = lists:keytake(Node, 1, DeadNodes),
+            SessionInfo2 = lists:keyreplace(dead_nodes, 1, SessionInfo, {dead_nodes, Dn2}),
+            {MetaFile, Partials + 1, lists:keyreplace(partials, 1, SessionInfo2, {partials, Partials + 1})}.
+
+try_send_flush_tick(State) ->
+    case proplists:get_value(flush, State) of
+        undefined ->
+            ok;
+        MSec ->
+            erlang:send_after(MSec, self(), flush_timeout)
     end.
 
 get_fetch_dir(undefined) -> ?upload_dir ++ ts();
 get_fetch_dir(Dir) -> Dir.
 
+resume_trace(Reporter) ->
+    ?MODULE:run_history(all_silent),
+    Reporter ! trace_resumed.
+
 get_nodes() ->
     ?MODULE ! {get_nodes,self()},
     receive {?MODULE,Nodes} -> Nodes end.
@@ -684,17 +800,46 @@ ts() ->
     io_lib:format("-~4.4.0w~2.2.0w~2.2.0w-~2.2.0w~2.2.0w~2.2.0w",
 		  [Y,M,D,H,Min,S]).
 
+copy_partials(_, 0) ->
+    ok;
+copy_partials(Dir, Num) ->
+    PartialDir = ?partial_dir ++ integer_to_list(Num),
+    file:rename(PartialDir, filename:join(Dir,PartialDir)),
+    copy_partials(Dir, Num - 1).
+
+fetch_partial_result(Node,MetaFile,Current) ->
+    DirName = ?partial_dir ++ integer_to_list(Current),
+    case file:list_dir(DirName) of
+        {error, enoent} ->
+            ok;
+        {ok, Files} ->
+            [ file:delete(filename:join(DirName, File)) || File <- Files ],
+            file:del_dir(DirName)
+    end,
+    file:make_dir(DirName),
+    fetch(host(node()), DirName, Node, MetaFile).
+
+fetch_report(Localhost, Dir, Node, MetaFile) ->
+    fetch(Localhost,Dir,Node,MetaFile),
+    ?MODULE ! {fetch_complete,Node}.
+
 fetch(Localhost,Dir,Node,MetaFile) ->
     case (host(Node) == Localhost) orelse is_local(MetaFile) of
         true -> % same host, just move the files
 	    Files = get_filenames(Node,MetaFile),
 	    lists:foreach(
 	      fun(File0) ->
-		      %%Other nodes may still have different CWD
-		      {ok, Cwd} = rpc:call(Node, file, get_cwd, []),
-		      File1 = filename:join(Cwd, File0),
-		      File = filename:join(Dir,filename:basename(File1)),
-		      file:rename(File1,File)
+                      case MetaFile of
+                          {local, _, _} ->
+                              File = filename:join(Dir,filename:basename(File0)),
+                              file:rename(File0, File);
+                          _ ->
+                              %%Other nodes may still have different CWD
+                              {ok, Cwd} = rpc:call(Node, file, get_cwd, []),
+                              File1 = filename:join(Cwd, File0),
+                              File = filename:join(Dir,filename:basename(File1)),
+                              file:rename(File1,File)
+                      end
 	      end,
 	      Files);
 	false ->
@@ -706,8 +851,7 @@ fetch(Localhost,Dir,Node,MetaFile) ->
 	    receive_files(Dir,Sock,undefined),
 	    ok = gen_tcp:close(LSock),
 	    ok = gen_tcp:close(Sock)
-    end,
-    ?MODULE ! {fetch_complete,Node}.
+    end.
 
 is_local({local, _, _}) ->
     true;
@@ -759,9 +903,14 @@ wait_for_fetch(Nodes) ->
 %%% - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 
 write_info(Nodes,PI,Traci) ->
-    lists:foreach(fun({N,{local,C,_},F}) -> 
-			  MetaFile = F ++ ".ti",
-			  file:delete(MetaFile),
+    lists:foreach(fun({N,{local,C,_},F}) ->
+			  MetaFile = case F of
+                                         none ->
+                                             none;
+                                         F ->
+                                             file:delete(F ++ ".ti"),
+                                             F ++ ".ti"
+                                     end,
 			  Traci1 = [{node,N},{file,C}|Traci],
 			  {ok,Port} = dbg:get_tracer(N),
 			  ?MODULE ! 
@@ -788,15 +937,8 @@ format(File,Out,Handler,DisableSort) when is_list(File), is_integer(hd(File)) ->
     Files = 
 	case filelib:is_dir(File) of
 	    true ->  % will merge all files in the directory
-		MetaFiles = filelib:wildcard(filename:join(File,"*.ti")),
-		lists:map(fun(M) ->
-				  Sub = string:left(M,length(M)-3),
-				  case filelib:is_file(Sub) of
-				      true -> Sub;
-				      false -> Sub++".*.wrp"
-				  end
-			  end,
-			  MetaFiles);
+                List = filelib:wildcard(filename:join(File, ?partial_dir++"*")),
+                lists:append(collect_files([File | List]));
 	    false -> % format one file
 		[File]
 	end,
@@ -819,6 +961,19 @@ format(Files,Out,Handler,DisableSort) when is_list(Files), is_list(hd(Files)) ->
     end,
     R.
 
+collect_files(Dirs) ->
+    lists:map(fun(Dir) ->
+                      MetaFiles = filelib:wildcard(filename:join(Dir,"*.ti")),
+                      lists:map(fun(M) ->
+                                        Sub = string:left(M,length(M)-3),
+                                        case filelib:is_file(Sub) of
+                                            true -> Sub;
+                                            false -> Sub++".*.wrp"
+                                        end
+                                end,
+                                MetaFiles)
+              end, Dirs).
+
 get_handler(undefined, Files) ->
     %%We retrieve traci from the first available file
     {Traci, _} = read_traci(hd(Files)),
@@ -933,6 +1088,7 @@ check_exists(File) ->
 	    exit({error,no_file})
     end.
 
+
 do_format(Fd,Details,DisableSort,Handler) ->
     Clients = lists:foldl(fun({FileOrWrap,Traci},Acc) ->
                                   [start_client(FileOrWrap,Traci)|Acc]
@@ -1085,19 +1241,34 @@ display_warning(Item,Warning) ->
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %%% Trace client which reads an IP port and puts data directly to a file.
 %%% This is used when tracing remote nodes with no file system.
-ip_to_file(Trace,{file,File}) ->
+ip_to_file({metadata,_,_},{_, only} = State) ->
+    State;
+ip_to_file(Trace, {_, only} = State) ->
+    dbg:dhandler(Trace, standard_io),
+    State;
+ip_to_file(Trace,{{file,File}, ShellOutput}) ->
     Fun = dbg:trace_port(file,File), %File can be a filename or a wrap spec
     Port = Fun(),
-    ip_to_file(Trace,Port);
-ip_to_file({metadata,MetaFile,MetaData},Port) ->
+    case Trace of
+        {metadata, _, _} -> ok;
+        Trace            -> show_trace(Trace, ShellOutput)
+    end,
+    ip_to_file(Trace,{Port,ShellOutput});
+ip_to_file({metadata,MetaFile,MetaData},State) ->
     {ok,MetaFd} = file:open(MetaFile,[write,raw,append]),
     file:write(MetaFd,MetaData),
     file:close(MetaFd),
-    Port;
-ip_to_file(Trace,Port) ->
+    State;
+ip_to_file(Trace,{Port, ShellOutput}) ->
+    show_trace(Trace, ShellOutput),
     B = term_to_binary(Trace),
     erlang:port_command(Port,B),
-    Port.
+    {Port, ShellOutput}.
+
+show_trace(Trace, true) ->
+    dbg:dhandler(Trace, standard_io);
+show_trace(_,_) ->
+    ok.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %%% For debugging