[observer] Add more trace functionality

Add logging output options, to file, shell or log-window.
Add remove/change process options, trace patterns via right-click.
Show traced nodes.
Add possibility to add named processes, which will be traced on all nodes.
Add menu to log window

1 files changed, 398 insertions, 151 deletions

diff --git a/lib/observer/src/observer_trace_wx.erl b/lib/observer/src/observer_trace_wx.erl
index 0ab7db121b..31c57bd9d3 100644
--- a/lib/observer/src/observer_trace_wx.erl
+++ b/lib/observer/src/observer_trace_wx.erl
@@ -27,30 +27,43 @@
 -include_lib("wx/include/wx.hrl").
 -include("observer_defs.hrl").
 
--define(OPTIONS, 301).
--define(SAVE_BUFFER, 302).
--define(CLOSE, 303).
--define(CLEAR, 304).
 -define(SAVE_TRACEOPTS, 305).
 -define(LOAD_TRACEOPTS, 306).
 -define(TOGGLE_TRACE, 307).
 -define(ADD_NEW, 308).
 -define(ADD_TP, 309).
--define(PROCESSES, 350).
--define(MODULES, 351).
--define(FUNCTIONS, 352).
--define(TRACERWIN, 353).
+-define(TRACE_OUTPUT, 310).
+-define(TRACE_DEFMS,  311).
+-define(TRACE_DEFPS,  312).
+
+-define(NODES_WIN, 330).
+-define(ADD_NODES, 331).
+-define(REMOVE_NODES, 332).
+
+-define(PROC_WIN, 340).
+-define(EDIT_PROCS, 341).
+-define(REMOVE_PROCS, 342).
+
+-define(MODULES_WIN, 350).
+
+-define(FUNCS_WIN, 360).
+-define(EDIT_FUNCS_MS, 361).
+-define(REMOVE_FUNCS_MS, 362).
+
+-define(LOG_WIN, 370).
+-define(LOG_SAVE, 321).
+-define(LOG_CLEAR, 322).
 
 -record(state,
 	{parent,
 	 panel,
-	 p_view,
-	 m_view,
-	 f_view,
+	 n_view,  p_view, m_view, f_view,  %% The listCtrl's
+	 logwin, %% The latest log window
 	 nodes = [],
 	 toggle_button,
 	 tpids = [],  %% #tpid
 	 def_trace_opts = [],
+	 output = [],
 	 tpatterns = dict:new(), % Key =:= Module::atom, Value =:= {M, F, A, MatchSpec}
 	 match_specs = []}). % [ #match_spec{} ]
 
@@ -72,26 +85,27 @@ create_window(Notebook, ParentPid) ->
     Panel = wxPanel:new(Notebook, [{size, wxWindow:getClientSize(Notebook)}]),
     Sizer = wxBoxSizer:new(?wxVERTICAL),
     Splitter = wxSplitterWindow:new(Panel, [{size, wxWindow:getClientSize(Panel)}]),
-    ProcessView = create_process_view(Splitter),
+    {NodeProcView, NodeView, ProcessView} = create_process_view(Splitter),
     {MatchSpecView,ModView,FuncView} = create_matchspec_view(Splitter),
     wxSplitterWindow:setSashGravity(Splitter, 0.5),
     wxSplitterWindow:setMinimumPaneSize(Splitter,50),
-    wxSplitterWindow:splitHorizontally(Splitter, ProcessView, MatchSpecView),
+    wxSplitterWindow:splitHorizontally(Splitter, NodeProcView, MatchSpecView),
     wxSizer:add(Sizer, Splitter, [{flag, ?wxEXPAND bor ?wxALL}, {border, 5}, {proportion, 1}]),
     %% Buttons
     Buttons = wxBoxSizer:new(?wxHORIZONTAL),
     ToggleButton = wxToggleButton:new(Panel, ?TOGGLE_TRACE, "Start Trace", []),
     wxSizer:add(Buttons, ToggleButton),
-    New = wxButton:new(Panel, ?ADD_NEW, [{label, "Trace New Processes"}]),
-    wxSizer:add(Buttons, New),
-    ATP = wxButton:new(Panel, ?ADD_TP, [{label, "Add Trace Pattern"}]),
-    wxSizer:add(Buttons, ATP),
-    wxMenu:connect(Panel, command_togglebutton_clicked, []),
-    wxMenu:connect(Panel, command_button_clicked, []),
-    wxSizer:add(Sizer, Buttons, [{flag, ?wxALL},{border, 2}, {proportion,0}]),
+    wxSizer:addSpacer(Buttons, 15),
+    wxSizer:add(Buttons, wxButton:new(Panel, ?ADD_NODES, [{label, "Add Nodes"}])),
+    wxSizer:add(Buttons, wxButton:new(Panel, ?ADD_NEW, [{label, "Add 'new' Process"}])),
+    wxSizer:add(Buttons, wxButton:new(Panel, ?ADD_TP, [{label, "Add Trace Pattern"}])),
+    wxMenu:connect(Panel, command_togglebutton_clicked, [{skip, true}]),
+    wxMenu:connect(Panel, command_button_clicked, [{skip, true}]),
+    wxSizer:add(Sizer, Buttons, [{flag, ?wxLEFT bor ?wxRIGHT bor ?wxDOWN},
+				 {border, 5}, {proportion,0}]),
     wxWindow:setSizer(Panel, Sizer),
     {Panel, #state{parent=ParentPid, panel=Panel,
-		   p_view=ProcessView, m_view=ModView, f_view=FuncView,
+		   n_view=NodeView, p_view=ProcessView, m_view=ModView, f_view=FuncView,
 		   toggle_button = ToggleButton,
 		   match_specs=default_matchspecs()}}.
 
@@ -103,36 +117,51 @@ default_matchspecs() ->
     [make_ms(Name,Term,FunStr) || {Name,Term,FunStr} <- Ms].
 
 create_process_view(Parent) ->
-    Style = ?wxLC_REPORT bor ?wxLC_SINGLE_SEL bor ?wxLC_HRULES,
-    Grid = wxListCtrl:new(Parent, [{winid, ?PROCESSES}, {style, Style}]),
+    Panel  = wxPanel:new(Parent),
+    MainSz = wxBoxSizer:new(?wxHORIZONTAL),
+    Style = ?wxLC_REPORT bor ?wxLC_HRULES,
+    Splitter = wxSplitterWindow:new(Panel, []),
+    Nodes = wxListCtrl:new(Splitter, [{winid, ?NODES_WIN}, {style, Style}]),
+    Procs = wxListCtrl:new(Splitter, [{winid, ?PROC_WIN}, {style, Style}]),
     Li = wxListItem:new(),
-    AddListEntry = fun({Name, Align, DefSize}, Col) ->
+    wxListItem:setText(Li, "Nodes"),
+    wxListCtrl:insertColumn(Nodes, 0, Li),
+
+    AddProc = fun({Name, Align, DefSize}, Col) ->
 			   wxListItem:setText(Li, Name),
 			   wxListItem:setAlign(Li, Align),
-			   wxListCtrl:insertColumn(Grid, Col, Li),
-			   wxListCtrl:setColumnWidth(Grid, Col, DefSize),
+			   wxListCtrl:insertColumn(Procs, Col, Li),
+			   wxListCtrl:setColumnWidth(Procs, Col, DefSize),
 			   Col + 1
 		   end,
     ListItems = [{"Process Id",    ?wxLIST_FORMAT_CENTER,  120},
 		 {"Trace Options", ?wxLIST_FORMAT_LEFT, 300}],
-    lists:foldl(AddListEntry, 0, ListItems),
+    lists:foldl(AddProc, 0, ListItems),
     wxListItem:destroy(Li),
 
-    %% wxListCtrl:connect(Grid, command_list_item_activated),
-    %% wxListCtrl:connect(Grid, command_list_item_selected),
-    wxListCtrl:connect(Grid, size, [{skip, true}]),
+    wxSplitterWindow:setSashGravity(Splitter, 0.0),
+    wxSplitterWindow:setMinimumPaneSize(Splitter,50),
+    wxSplitterWindow:splitVertically(Splitter, Nodes, Procs, [{sashPosition, 150}]),
+    wxSizer:add(MainSz, Splitter, [{flag, ?wxEXPAND}, {proportion, 1}]),
 
-    wxWindow:setFocus(Grid),
-    Grid.
+    wxListCtrl:connect(Procs, command_list_item_right_click),
+    wxListCtrl:connect(Nodes, command_list_item_right_click),
+    wxListCtrl:connect(Procs, size, [{skip, true}]),
+    wxListCtrl:connect(Nodes, size, [{skip, true}]),
+
+    wxPanel:setSizer(Panel, MainSz),
+    wxWindow:setFocus(Procs),
+    {Panel, Nodes, Procs}.
 
 create_matchspec_view(Parent) ->
     Panel  = wxPanel:new(Parent),
     MainSz = wxBoxSizer:new(?wxHORIZONTAL),
-    Style = ?wxLC_REPORT bor ?wxLC_SINGLE_SEL bor ?wxLC_HRULES,
+    Style = ?wxLC_REPORT bor ?wxLC_HRULES,
     Splitter = wxSplitterWindow:new(Panel, []),
-    Modules = wxListCtrl:new(Splitter, [{winid, ?MODULES}, {style, Style}]),
-    Funcs   = wxListCtrl:new(Splitter, [{winid, ?FUNCTIONS}, {style, Style}]),
+    Modules = wxListCtrl:new(Splitter, [{winid, ?MODULES_WIN}, {style, Style}]),
+    Funcs   = wxListCtrl:new(Splitter, [{winid, ?FUNCS_WIN}, {style, Style}]),
     Li = wxListItem:new(),
+
     wxListItem:setText(Li, "Modules"),
     wxListCtrl:insertColumn(Modules, 0, Li),
     wxListItem:setText(Li, "Functions"),
@@ -142,6 +171,7 @@ create_matchspec_view(Parent) ->
     wxListCtrl:insertColumn(Funcs, 1, Li),
     wxListCtrl:setColumnWidth(Funcs, 1, 300),
     wxListItem:destroy(Li),
+
     wxSplitterWindow:setSashGravity(Splitter, 0.0),
     wxSplitterWindow:setMinimumPaneSize(Splitter,50),
     wxSplitterWindow:splitVertically(Splitter, Modules, Funcs, [{sashPosition, 150}]),
@@ -150,14 +180,19 @@ create_matchspec_view(Parent) ->
     wxListCtrl:connect(Modules, size, [{skip, true}]),
     wxListCtrl:connect(Funcs,   size, [{skip, true}]),
     wxListCtrl:connect(Modules, command_list_item_selected),
-    %% wxListCtrl:connect(Funcs, command_list_item_selected),
+    wxListCtrl:connect(Funcs, command_list_item_right_click),
     wxPanel:setSizer(Panel, MainSz),
     {Panel, Modules, Funcs}.
 
 create_menues(Parent) ->
-    Menus = [{"File", [#create_menu{id = ?LOAD_TRACEOPTS, text = "Load settings"},
-		       #create_menu{id = ?SAVE_TRACEOPTS, text = "Save settings"}]
-	     }],
+    Menus = [{"File",
+	      [#create_menu{id = ?LOAD_TRACEOPTS, text = "Load settings"},
+	       #create_menu{id = ?SAVE_TRACEOPTS, text = "Save settings"}]},
+	     {"Options",
+	      [#create_menu{id = ?TRACE_OUTPUT, text = "Output"},
+	       #create_menu{id = ?TRACE_DEFMS, text = "Match Specifications"},
+	       #create_menu{id = ?TRACE_DEFPS, text = "Default Process Options"}]}
+	    ],
     observer_wx:create_menus(Parent, Menus).
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -179,12 +214,11 @@ handle_event(#wx{obj=Obj, event=#wxSize{size={W,_}}}, State) ->
     {noreply, State};
 
 handle_event(#wx{id=?ADD_NEW}, State = #state{panel=Parent, def_trace_opts=TraceOpts}) ->
-    case observer_traceoptions_wx:process_trace(Parent, TraceOpts) of
-	{ok, Opts} ->
-	    Process = #tpid{pid=new, opts=Opts},
-	    {noreply, do_add_processes([Process], State#state{def_trace_opts=Opts})};
-	cancel ->
-	    {noreply, State}
+    try
+	Opts = observer_traceoptions_wx:process_trace(Parent, TraceOpts),
+	Process = #tpid{pid=new, opts=Opts},
+	{noreply, do_add_processes([Process], State#state{def_trace_opts=Opts})}
+    catch cancel -> {noreply, State}
     end;
 
 handle_event(#wx{id=?ADD_TP},
@@ -200,98 +234,101 @@ handle_event(#wx{id=?ADD_TP},
 	    {noreply, do_add_patterns(Patterns, State)}
     end;
 
-handle_event(#wx{id=?MODULES, event=#wxList{type=command_list_item_selected, itemIndex=Row}},
+handle_event(#wx{id=?MODULES_WIN, event=#wxList{type=command_list_item_selected, itemIndex=Row}},
 	     State = #state{tpatterns=TPs, m_view=Mview, f_view=Fview}) ->
     Module = list_to_atom(wxListCtrl:getItemText(Mview, Row)),
     update_functions_view(dict:fetch(Module, TPs), Fview),
     {noreply, State};
 
-
 handle_event(#wx{event = #wxCommand{type = command_togglebutton_clicked, commandInt = 1}},
 	     #state{panel = Panel,
 		    nodes = Nodes,
 		    tpids = TProcs,
-		    tpatterns = TPs,
-		    toggle_button = ToggleBtn} = State) ->
-    LogWin = wxFrame:new(Panel, ?TRACERWIN, "Trace Log", [{size, {750, 800}}]),
-    Text   = wxTextCtrl:new(LogWin, ?wxID_ANY,
-			    [{style, ?wxTE_MULTILINE bor ?wxTE_RICH2 bor
-				  ?wxTE_DONTWRAP bor ?wxTE_READONLY}]),
-    Font = observer_wx:get_attrib({font, fixed}),
-    Attr = wxTextAttr:new(?wxBLACK, [{font, Font}]),
-    true = wxTextCtrl:setDefaultStyle(Text, Attr),
-    Env = wx:get_env(),
-    Write = fun(Trace) ->
-		    wx:set_env(Env),
-		    wxTextCtrl:appendText(Text, textformat(Trace))
-	    end,
-    {ok, _} = ttb:tracer(Nodes, [{file, {local,"/tmp/foo"}}, {shell, {only, Write}}]),
-    setup_ttb(dict:to_list(TPs), TProcs),
-    wxFrame:connect(LogWin, close_window, [{skip, true}]),
-    wxFrame:show(LogWin),
-    wxToggleButton:setLabel(ToggleBtn, "Stop Trace"),
-    {noreply, State};
+		    tpatterns = TPs0,
+		    toggle_button = ToggleBtn,
+		    output = Opts
+		   } = State) ->
+    try
+	TPs = dict:to_list(TPs0),
+	(TProcs == []) andalso throw({error, "No processes traced"}),
+	(Nodes == []) andalso throw({error, "No nodes traced"}),
+	HaveCallTrace = fun(#tpid{opts=Os}) -> lists:member(functions,Os) end,
+	WStr = "Call trace actived but no trace patterns used",
+	(TPs == []) andalso lists:any(HaveCallTrace, TProcs) andalso
+	    observer_wx:create_txt_dialog(Panel, WStr, "Warning", ?wxICON_WARNING),
+
+	{TTB, LogWin}  = ttb_output_args(Panel, Opts),
+	{ok, _} = ttb:tracer(Nodes, TTB),
+	setup_ttb(TPs, TProcs),
+	wxToggleButton:setLabel(ToggleBtn, "Stop Trace"),
+	{noreply, State#state{logwin=LogWin}}
+    catch {error, Msg} ->
+	    observer_wx:create_txt_dialog(Panel, Msg, "Error", ?wxICON_ERROR),
+	    wxToggleButton:setValue(ToggleBtn, false),
+	    {noreply, State}
+    end;
 
 handle_event(#wx{event = #wxCommand{type = command_togglebutton_clicked, commandInt = 0}},
 	     #state{toggle_button = ToggleBtn} = State) ->
     %%Stop tracing
     ttb:stop(nofetch),
     wxToggleButton:setLabel(ToggleBtn, "Start Trace"),
+    wxToggleButton:setValue(ToggleBtn, false),
+    {noreply, State#state{logwin=false}};
+
+handle_event(#wx{id=Id, obj=LogWin, event=Ev},
+	     #state{toggle_button = ToggleBtn, logwin=Latest} = State)
+  when Id =:= ?LOG_WIN; is_record(Ev, wxClose) ->
+    case LogWin of
+	Latest ->
+	    %%Stop tracing
+	    ttb:stop(nofetch),
+	    wxToggleButton:setLabel(ToggleBtn, "Start Trace"),
+	    wxToggleButton:setValue(ToggleBtn, false),
+	    {noreply, State#state{logwin=false}};
+	_ ->
+	    {noreply, State}
+    end;
+
+handle_event(Ev = #wx{id=?LOG_CLEAR, userData=TCtrl}, State) ->
+    wxTextCtrl:clear(TCtrl),
     {noreply, State};
 
-handle_event(#wx{id=?TRACERWIN, event=#wxClose{}},
-	     #state{toggle_button = ToggleBtn} = State) ->
-    %%Stop tracing
-    ttb:stop(nofetch),
-    wxToggleButton:setLabel(ToggleBtn, "Start Trace"),
+handle_event(#wx{id=?LOG_SAVE, userData=TCtrl}, #state{panel=Panel} = State) ->
+    Dialog = wxFileDialog:new(Panel, [{style, ?wxFD_SAVE bor ?wxFD_OVERWRITE_PROMPT}]),
+    case wxFileDialog:showModal(Dialog) of
+	?wxID_OK ->
+	    Path = wxFileDialog:getPath(Dialog),
+	    wxDialog:destroy(Dialog),
+	    wxTextCtrl:saveFile(TCtrl, [{file, Path}]);
+	_ ->
+	    wxDialog:destroy(Dialog),
+	    ok
+    end,
     {noreply, State};
 
-%% handle_event(#wx{id = ?CLEAR, event = #wxCommand{type = command_menu_selected}},
-%% 	     #state{text_ctrl = TxtCtrl} = State) ->
-%%     wxTextCtrl:clear(TxtCtrl),
-%%     {noreply, State};
-
-%% handle_event(#wx{id = ?SAVE_BUFFER, event = #wxCommand{type = command_menu_selected}},
-%% 	     #state{frame = Frame, text_ctrl = TxtCtrl} = State) ->
-%%     Dialog = wxFileDialog:new(Frame, [{style, ?wxFD_SAVE bor ?wxFD_OVERWRITE_PROMPT}]),
-%%     case wxFileDialog:showModal(Dialog) of
-%% 	?wxID_OK ->
-%% 	    Path = wxFileDialog:getPath(Dialog),
-%% 	    wxDialog:destroy(Dialog),
-%% 	    case filelib:is_file(Path) of
-%% 		true ->
-%% 		    observer_wx:create_txt_dialog(Frame, "File already exists: " ++ Path ++ "\n",
-%% 						  "Error", ?wxICON_ERROR);
-%% 		false ->
-%% 		    wxTextCtrl:saveFile(TxtCtrl, [{file, Path}])
-%% 	    end;
-%% 	_ ->
-%% 	    wxDialog:destroy(Dialog),
-%% 	    ok
-%%     end,
-%%     {noreply, State};
-
-handle_event(#wx{id = ?SAVE_TRACEOPTS,
-		 event = #wxCommand{type = command_menu_selected}},
+handle_event(#wx{id = ?SAVE_TRACEOPTS},
 	     #state{panel = Panel,
 		    def_trace_opts = TraceOpts,
 		    match_specs = MatchSpecs,
-		    tpatterns = TracePatterns
+		    tpatterns = TracePatterns,
+		    output = Output
 		   } = State) ->
     Dialog = wxFileDialog:new(Panel, [{style, ?wxFD_SAVE bor ?wxFD_OVERWRITE_PROMPT}]),
     case wxFileDialog:showModal(Dialog) of
 	?wxID_OK ->
 	    Path = wxFileDialog:getPath(Dialog),
-	    write_file(Panel, Path, TraceOpts, MatchSpecs, dict:to_list(TracePatterns));
+	    write_file(Panel, Path,
+		       TraceOpts, MatchSpecs, Output,
+		       dict:to_list(TracePatterns)
+		      );
 	_ ->
 	    ok
     end,
     wxDialog:destroy(Dialog),
     {noreply, State};
 
-handle_event(#wx{id = ?LOAD_TRACEOPTS,
-		 event = #wxCommand{type = command_menu_selected}},
-	     #state{panel = Panel} = State) ->
+handle_event(#wx{id = ?LOAD_TRACEOPTS}, #state{panel = Panel} = State) ->
     Dialog = wxFileDialog:new(Panel, [{style, ?wxFD_FILE_MUST_EXIST}]),
     State2 = case wxFileDialog:showModal(Dialog) of
 		 ?wxID_OK ->
@@ -303,27 +340,142 @@ handle_event(#wx{id = ?LOAD_TRACEOPTS,
     wxDialog:destroy(Dialog),
     {noreply, State2};
 
+handle_event(#wx{id=Type, event=#wxList{type=command_list_item_right_click}},
+	     State = #state{panel=Panel}) ->
+    Menus = case Type of
+		?PROC_WIN ->
+		    [{?EDIT_PROCS, "Edit process options"},
+		     {?REMOVE_PROCS, "Remove processes"}];
+		?FUNCS_WIN ->
+		    [{?EDIT_FUNCS_MS, "Edit matchspecs"},
+		     {?REMOVE_FUNCS_MS, "Remove trace patterns"}];
+		?NODES_WIN ->
+		    [{?ADD_NODES, "Trace other nodes"},
+		     {?REMOVE_NODES, "Remove nodes"}]
+	    end,
+    Menu = wxMenu:new(),
+    [wxMenu:append(Menu,Id,Str) || {Id,Str} <- Menus],
+    wxWindow:popupMenu(Panel, Menu),
+    wxMenu:destroy(Menu),
+    {noreply, State};
+
+handle_event(#wx{id=?EDIT_PROCS}, #state{panel=Panel, tpids=Tpids, p_view=Ps} = State) ->
+    try
+	[#tpid{opts=DefOpts}|_] = Selected = get_selected_items(Ps, Tpids),
+	Opts = observer_traceoptions_wx:process_trace(Panel, DefOpts),
+	Changed = [Tpid#tpid{opts=Opts} || Tpid <- Selected],
+	{noreply, do_add_processes(Changed, State#state{def_trace_opts=Opts})}
+    catch _:_ ->
+	    {noreply, State}
+    end;
+
+handle_event(#wx{id=?REMOVE_PROCS}, #state{tpids=Tpids, p_view=LCtrl} = State) ->
+    Selected = get_selected_items(LCtrl, Tpids),
+    Pids = Tpids -- Selected,
+    update_process_view(Pids, LCtrl),
+    {noreply, State#state{tpids=Pids}};
+
+handle_event(#wx{id=?TRACE_DEFPS}, #state{panel=Panel, def_trace_opts=PO} = State) ->
+    try
+	Opts = observer_traceoptions_wx:process_trace(Panel, PO),
+	{noreply, State#state{def_trace_opts=Opts}}
+    catch _:_ ->
+	    {noreply, State}
+    end;
+
+handle_event(#wx{id=?TRACE_DEFMS}, #state{panel=Panel, match_specs=Ms} = State) ->
+    try %% Return selected MS and sends new MS's to us
+	observer_traceoptions_wx:select_matchspec(self(), Panel, Ms)
+    catch _:_ ->
+	    cancel
+    end,
+    {noreply, State};
+
+handle_event(#wx{id=?EDIT_FUNCS_MS}, #state{panel=Panel, tpatterns=TPs,
+					    f_view=LCtrl, m_view=Mview,
+					    match_specs=Mss
+					   } = State) ->
+    try
+	[Module] = get_selected_items(Mview, lists:sort(dict:fetch_keys(TPs))),
+	Selected = get_selected_items(LCtrl, dict:fetch(Module, TPs)),
+	Ms = observer_traceoptions_wx:select_matchspec(self(), Panel, Mss),
+	Changed = [TP#tpattern{ms=Ms} || TP <- Selected],
+	{noreply, do_add_patterns({Module, Changed}, State)}
+    catch _:_ ->
+	    {noreply, State}
+    end;
+
+handle_event(#wx{id=?REMOVE_FUNCS_MS}, #state{tpatterns=TPs0, f_view=LCtrl, m_view=Mview} = State) ->
+    case get_selected_items(Mview, lists:sort(dict:fetch_keys(TPs0))) of
+	[] -> {noreply, State};
+	[Module] ->
+	    FMs0 = dict:fetch(Module, TPs0),
+	    Selected = get_selected_items(LCtrl, FMs0),
+	    FMs = FMs0 -- Selected,
+	    update_functions_view(FMs, LCtrl),
+	    TPs = case FMs of
+		      [] ->
+			  New = dict:erase(Module, TPs0),
+			  update_modules_view(lists:sort(dict:fetch_keys(New)), Module, Mview),
+			  New;
+		      _ ->
+			  dict:store(Module, FMs, TPs0)
+		  end,
+	    {noreply, State#state{tpatterns=TPs}}
+    end;
+
+handle_event(#wx{id=?TRACE_OUTPUT}, #state{panel=Panel, output=Out0} = State) ->
+    try
+	Out = observer_traceoptions_wx:output(Panel, Out0),
+	{noreply, State#state{output=Out}}
+    catch _:_ ->
+	    {noreply, State}
+    end;
+
+handle_event(#wx{id=?ADD_NODES}, #state{panel=Panel, n_view=Nview, nodes=Ns0} = State) ->
+    try
+	Possible = [node()|nodes()] -- Ns0,
+	case Possible of
+	    [] ->
+		Msg = "Already selected all connected nodes\n"
+		    "Use the Nodes menu to connect to new nodes first.",
+		observer_wx:create_txt_dialog(Panel, Msg, "No available nodes", ?wxICON_INFORMATION),
+		throw(cancel);
+	    _ ->
+		Ns = lists:usort(Ns0 ++ observer_traceoptions_wx:select_nodes(Panel, Possible)),
+		update_nodes_view(Ns, Nview),
+		{noreply, State#state{nodes=Ns}}
+	end
+    catch cancel ->
+	    {noreply, State}
+    end;
+
+handle_event(#wx{id=?REMOVE_NODES}, #state{n_view=Nview, nodes=Ns0} = State) ->
+    Sel = get_selected_items(Nview, Ns0),
+    Ns = Ns0 -- Sel,
+    update_nodes_view(Ns, Nview),
+    {noreply, State#state{nodes = Ns}};
+
 handle_event(#wx{id=ID, event = What}, State) ->
-    io:format("~p:~p: Unhandled event: ~p, ~p ~n", [?MODULE, self(), ID, What]),
+    io:format("~p:~p: Unhandled event: ~p, ~p ~n", [?MODULE, ?LINE, ID, What]),
     {noreply, State}.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-handle_call(Msg, _From, State) ->
-    io:format("~p~p: Got Call ~p~n",[?MODULE, ?LINE, Msg]),
-    {reply, ok, State}.
+handle_call(Msg, From, _State) ->
+    error({unhandled_call, Msg, From}).
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 handle_cast({add_processes, Pids}, State = #state{panel=Parent, def_trace_opts=TraceOpts}) ->
-    case observer_traceoptions_wx:process_trace(Parent, TraceOpts) of
-	{ok, Opts} ->
-	    POpts = [#tpid{pid=Pid, opts=Opts} || Pid <- Pids],
-	    {noreply, do_add_processes(POpts, State#state{def_trace_opts=Opts})};
-	cancel ->
+    try
+	Opts = observer_traceoptions_wx:process_trace(Parent, TraceOpts),
+	POpts = [#tpid{pid=Pid, opts=Opts} || Pid <- Pids],
+	S = do_add_processes(POpts, State#state{def_trace_opts=Opts}),
+	{noreply, S}
+    catch cancel ->
 	    {noreply, State}
     end;
-handle_cast(Msg, State) ->
-    io:format("~p ~p: Unhandled cast ~p~n", [?MODULE, ?LINE, Msg]),
-    {noreply, State}.
+handle_cast(Msg, _State) ->
+    error({unhandled_cast, Msg}).
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
@@ -342,18 +494,30 @@ handle_info(Any, State) ->
     {noreply, State}.
 
 terminate(_Reason, #state{nodes=_Nodes}) ->
-    %% case observer_wx:try_rpc(Node, erlang, whereis, [dbg]) of
-    %% 	undefined -> fine;
-    %% 	Pid -> exit(Pid, kill)
-    %% end,
+    ttb:stop(nofetch),
     ok.
 
 code_change(_, _, State) ->
     {stop, not_yet_implemented, State}.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+do_add_patterns({Module, NewPs}, State=#state{tpatterns=TPs0, m_view=Mview, f_view=Fview}) ->
+    Old = case dict:find(Module, TPs0) of
+	      {ok, Prev}  -> Prev;
+	      error -> []
+	  end,
+    case merge_patterns(NewPs, Old) of
+	{Old, [], []} ->
+	    State;
+	{MPatterns, _New, _Changed} ->
+	    %% if dynamicly updates update New and Changed
+	    TPs = dict:store(Module, MPatterns, TPs0),
+	    update_modules_view(lists:sort(dict:fetch_keys(TPs)), Module, Mview),
+	    update_functions_view(dict:fetch(Module, TPs), Fview),
+	    State#state{tpatterns=TPs}
+    end.
 
-do_add_processes(POpts, S0=#state{p_view=LCtrl, tpids=OldPids, nodes=Ns0}) ->
+do_add_processes(POpts, S0=#state{n_view=Nview, p_view=LCtrl, tpids=OldPids, nodes=Ns0}) ->
     case merge_pids(POpts, OldPids) of
 	{OldPids, [], []} ->
 	    S0;
@@ -363,13 +527,15 @@ do_add_processes(POpts, S0=#state{p_view=LCtrl, tpids=OldPids, nodes=Ns0}) ->
 	    Nodes = case ordsets:subtract(Ns1, Ns0) of
 			[] -> Ns0; %% No new Nodes
 			NewNs ->
-			    %% Handle new nodes
-			    %% BUGBUG add trace patterns for new nodes
-			    ordsets:union(NewNs, Ns0)
+			    %% if dynamicly updates add trace patterns for new nodes
+			    All = ordsets:union(NewNs, Ns0),
+			    update_nodes_view(All, Nview),
+			    All
 		    end,
 	    S0#state{tpids=Pids, nodes=Nodes}
     end.
 
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 update_process_view(Pids, LCtrl) ->
     wxListCtrl:deleteAllItems(LCtrl),
     wx:foldl(fun(#tpid{pid=Pid, opts=Opts}, Row) ->
@@ -381,20 +547,15 @@ update_process_view(Pids, LCtrl) ->
 		     Row+1
 	     end, 0, Pids).
 
-do_add_patterns({Module, NewPs}, State=#state{tpatterns=TPs0, m_view=Mview, f_view=Fview}) ->
-    Old = case dict:find(Module, TPs0) of
-	      {ok, Prev}  -> Prev;
-	      error -> []
-	  end,
-    case merge_patterns(NewPs, Old) of
-	{Old, [], []} ->
-	    State;
-	{MPatterns, _New, _Changed} ->
-	    TPs = dict:store(Module, MPatterns, TPs0),
-	    update_modules_view(lists:sort(dict:fetch_keys(TPs)), Module, Mview),
-	    update_functions_view(dict:fetch(Module, TPs), Fview),
-	    State#state{tpatterns=TPs}
-    end.
+update_nodes_view(Nodes, LCtrl) ->
+    wxListCtrl:deleteAllItems(LCtrl),
+    wx:foldl(fun(Node, Row) ->
+		     _Item = wxListCtrl:insertItem(LCtrl, Row, ""),
+		     ?EVEN(Row) andalso
+			 wxListCtrl:setItemBackgroundColour(LCtrl, Row, ?BG_EVEN),
+		     wxListCtrl:setItem(LCtrl, Row, 0, observer_lib:to_str(Node)),
+		     Row+1
+	     end, 0, Nodes).
 
 update_modules_view(Mods, Module, LCtrl) ->
     wxListCtrl:deleteAllItems(LCtrl),
@@ -419,7 +580,6 @@ update_functions_view(Funcs, LCtrl) ->
 	     end, 0, Funcs).
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
 merge_pids([N1=#tpid{pid=new}|Ns], [N2=#tpid{pid=new}|Old]) ->
     {Pids, New, Changed} = merge_pids_1(Ns,Old),
     {[N1|Pids], New, [{N2,N2}|Changed]};
@@ -438,7 +598,6 @@ merge_pids_1(New, Old) ->
 merge_patterns(New, Old) ->
     merge(lists:sort(New), Old, #tpattern.fa, [], [], []).
 
-
 merge([N|Ns], [N|Os], El, New, Ch, All) ->
     merge(Ns, Os, El, New, Ch, [N|All]);
 merge([N|Ns], [O|Os], El, New, Ch, All)
@@ -456,13 +615,78 @@ merge(Ns, [], _El, New, Ch, All) ->
     {lists:reverse(All, Ns), Ns++New, Ch}.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+ttb_output_args(Parent, Opts) ->
+    ToWindow = proplists:get_value(window, Opts, true),
+    ToShell = proplists:get_value(shell, Opts, false),
+    ToFile = proplists:get_value(file, Opts, false),
+    ToWindow orelse ToShell orelse ToShell orelse
+	throw({error, "No output of trace"}),
+    {LogWin,Text} = create_logwindow(Parent, ToWindow),
+    Write = output_fun(Text, ToShell),
+    Shell = output_shell(ToFile, Write),
+    FileOpts = output_file(ToFile, proplists:get_value(wrap, Opts, false), Opts),
+    {[{file, {local,FileOpts}}|Shell], LogWin}.
+
+output_shell(true, false) ->
+    []; %% File only
+output_shell(true, Write) when is_function(Write) ->
+    [{shell, Write}];
+output_shell(false, Write) when is_function(Write) ->
+    [{shell, {only, Write}}].
+
+output_fun(false, false) -> false;
+output_fun(false, true) -> fun(Trace) -> io:put_chars(textformat(Trace)) end;
+output_fun(Text, false) ->
+    Env = wx:get_env(),
+    fun(Trace) ->
+	    wx:set_env(Env),
+	    wxTextCtrl:appendText(Text, textformat(Trace))
+    end;
+output_fun(Text, true) ->
+    Env = wx:get_env(),
+    fun(Trace) ->
+	    wx:set_env(Env),
+	    IoList = textformat(Trace),
+	    wxTextCtrl:appendText(Text, IoList),
+	    io:put_chars(textformat(Trace))
+    end.
+
+output_file(false, _, _Opts) ->
+    "ttb";  %% Will be ignored
+output_file(true, false, Opts) ->
+    proplists:get_value(filename, Opts, "ttb");
+output_file(true, true, Opts) ->
+    Name = proplists:get_value(filename, Opts, "ttb"),
+    Size = proplists:get_value(wrap_sz, Opts, 128),
+    Count = proplists:get_value(wrap_c, Opts, 8),
+    {wrap, Name, Size*1024, Count}.
+
+
+create_logwindow(_Parent, false) -> {false, false};
+create_logwindow(Parent, true) ->
+    LogWin = wxFrame:new(Parent, ?LOG_WIN, "Trace Log", [{size, {750, 800}}]),
+    MB = wxMenuBar:new(),
+    File = wxMenu:new(),
+    wxMenu:append(File, ?LOG_CLEAR, "Clear Log\tCtrl-C"),
+    wxMenu:append(File, ?LOG_SAVE, "Save Log\tCtrl-S"),
+    wxMenu:append(File, ?wxID_CLOSE, "Close"),
+    wxMenuBar:append(MB, File, "File"),
+    wxFrame:setMenuBar(LogWin, MB),
+    Text = wxTextCtrl:new(LogWin, ?wxID_ANY,
+			  [{style, ?wxTE_MULTILINE bor ?wxTE_RICH2 bor
+				?wxTE_DONTWRAP bor ?wxTE_READONLY}]),
+    Font = observer_wx:get_attrib({font, fixed}),
+    Attr = wxTextAttr:new(?wxBLACK, [{font, Font}]),
+    true = wxTextCtrl:setDefaultStyle(Text, Attr),
+    wxFrame:connect(LogWin, close_window, [{skip, true}]),
+    wxFrame:connect(LogWin, command_menu_selected, [{userData, Text}]),
+    wxFrame:show(LogWin),
+    {LogWin, Text}.
 
 setup_ttb(TPs, TPids) ->
     _R1 = [setup_tps(FTP, []) || {_, FTP} <- TPs],
     _R2 = [ttb:p(Pid, dbg_flags(Flags)) || #tpid{pid=Pid, opts=Flags} <- TPids],
     [#tpid{pid=_Pid, opts=_Flags}|_] = TPids,
-    %% io:format("ttb:p(pid(\"~w\", ~w).", [Pid, Flags]),
-    %% io:format("TTB ~w ~w~n",[R2, R1]),
     ok.
 
 %% Sigh order is important
@@ -511,33 +735,33 @@ format_trace(Trace, Size, TS0={_,_,MS}) ->
 	    case element(4, Trace) of
 		{dbg,ok} -> "";
 		Message ->
-		    io_lib:format("~s (~100p) << ~100p ~n", [TS,From,Message])
+		    io_lib:format("~s (~100p) << ~100p~n", [TS,From,Message])
 	    end;
 	'send' ->
 	    Message = element(4, Trace),
 	    To = element(5, Trace),
-	    io_lib:format("~s (~100p) ~100p ! ~100p ~n", [TS,From,To,Message]);
+	    io_lib:format("~s (~100p) ~100p ! ~100p~n", [TS,From,To,Message]);
 	call ->
 	    case element(4, Trace) of
 		MFA when Size == 5 ->
 		    Message = element(5, Trace),
 		    io_lib:format("~s (~100p) call ~s (~100p) ~n", [TS,From,ffunc(MFA),Message]);
 		MFA ->
-		    io_lib:format("~s (~100p) call ~s ~n", [TS,From,ffunc(MFA)])
+		    io_lib:format("~s (~100p) call ~s~n", [TS,From,ffunc(MFA)])
 	    end;
 	return_from ->
 	    MFA = element(4, Trace),
 	    Ret = element(5, Trace),
-	    io_lib:format("~s (~100p) returned from ~s -> ~100p ~n", [TS,From,ffunc(MFA),Ret]);
+	    io_lib:format("~s (~100p) returned from ~s -> ~100p~n", [TS,From,ffunc(MFA),Ret]);
 	return_to ->
 	    MFA = element(4, Trace),
-	    io_lib:format("~s (~100p) returning to ~s ~n", [TS,From,ffunc(MFA)]);
+	    io_lib:format("~s (~100p) returning to ~s~n", [TS,From,ffunc(MFA)]);
 	spawn when Size == 5 ->
 	    Pid = element(4, Trace),
 	    MFA = element(5, Trace),
-	    io_lib:format("~s (~100p) spawn ~100p as ~s ~n", [TS,From,Pid,ffunc(MFA)]);
+	    io_lib:format("~s (~100p) spawn ~100p as ~s~n", [TS,From,Pid,ffunc(MFA)]);
 	Op ->
-	    io_lib:format("~s (~100p) ~100p ~s ~n", [TS,From,Op,ftup(Trace,4,Size)])
+	    io_lib:format("~s (~100p) ~100p ~s~n", [TS,From,Op,ftup(Trace,4,Size)])
     end.
 
 %%% These f* functions returns non-flat strings
@@ -567,7 +791,7 @@ ftup(Trace, Index, Size) ->
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
-write_file(Frame, Filename, TraceOps, MatchSpecs, TPs) ->
+write_file(Frame, Filename, TraceOps, MatchSpecs, Output, TPs) ->
     FormatMS = fun(#match_spec{name=Id, term=T, func=F}) ->
 		       io_lib:format("[{name,\"~s\"}, {term, ~w}, {func, \"~s\"}]",
 				     [Id, T, F])
@@ -581,6 +805,7 @@ write_file(Frame, Filename, TraceOps, MatchSpecs, TPs) ->
 	 "%%%\n%%% DO NOT EDIT!\n%%%\n",
 	 [["{ms, ", FormatMS(Ms), "}.\n"] || Ms <- MatchSpecs],
 	 "{traceopts, ", io_lib:format("~w",[TraceOps]) ,"}.\n",
+	 "{output, ", io_lib:format("~w",[Output]) ,"}.\n",
 	 [FormatTP(TP) || TP <- TPs]
 	],
     case file:write_file(Filename, list_to_binary(Str)) of
@@ -602,8 +827,9 @@ read_settings(Filename, #state{match_specs=Ms0, def_trace_opts=TO0} = State) ->
 	{ok, Terms} ->
 	    Ms  = lists:usort(Ms0 ++ [parse_ms(MsList) || {ms, MsList} <- Terms]),
 	    TOs = lists:usort(TO0 ++ proplists:get_value(traceopts, Terms, [])),
+	    Out = proplists:get_value(output, Terms, []),
 	    lists:foldl(fun parse_tp/2,
-			State#state{match_specs=Ms, def_trace_opts=TOs},
+			State#state{match_specs=Ms, def_trace_opts=TOs, output=Out},
 			Terms);
 	{error, _} ->
 	    observer_wx:create_txt_dialog(State#state.panel, "Could not load settings",
@@ -626,3 +852,24 @@ parse_tp({tp, Mod, FAs}, State) ->
     do_add_patterns({Mod, Patterns}, State);
 parse_tp(_, State) ->
     State.
+
+get_selected_items(Grid, Data) ->
+    get_indecies(get_selected_items(Grid, -1, []), Data).
+get_selected_items(Grid, Index, ItemAcc) ->
+    Item = wxListCtrl:getNextItem(Grid, Index, [{geometry, ?wxLIST_NEXT_ALL},
+						{state, ?wxLIST_STATE_SELECTED}]),
+    case Item of
+	-1 ->
+	    lists:reverse(ItemAcc);
+	_ ->
+	    get_selected_items(Grid, Item, [Item | ItemAcc])
+    end.
+
+get_indecies(Items, Data) ->
+    get_indecies(Items, 0, Data).
+get_indecies([I|Rest], I, [H|T]) ->
+    [H|get_indecies(Rest, I+1, T)];
+get_indecies(Rest = [_|_], I, [_|T]) ->
+    get_indecies(Rest, I+1, T);
+get_indecies(_, _, _) ->
+    [].