[observer] Use new scheduler_wall_time measurment

2 files changed, 123 insertions, 66 deletions

diff --git a/lib/observer/src/observer_perf_wx.erl b/lib/observer/src/observer_perf_wx.erl
index 1f872db3a9..0d18112085 100644
--- a/lib/observer/src/observer_perf_wx.erl
+++ b/lib/observer/src/observer_perf_wx.erl
@@ -23,11 +23,15 @@
 -export([init/1, handle_info/2, terminate/2, code_change/3, handle_call/3,
 	 handle_event/2, handle_sync_event/3, handle_cast/2]).
 
+-export([fetch_stats/2]).
+
+-compile(export_all).
+
 -behaviour(wx_object).
 -include_lib("wx/include/wx.hrl").
 -include("observer_defs.hrl").
 
--compile(export_all).
+%%-compile(export_all).
 
 -record(state,
 	{
@@ -41,7 +45,7 @@
 	  appmon
 	}).
 
--record(paint, {font, pen, pens}).
+-record(paint, {font, small, pen, pen2, pens}).
 
 -define(RQ_W,  1).
 -define(MEM_W, 2).
@@ -53,7 +57,6 @@ start_link(Notebook, Parent) ->
 init([Notebook, Parent]) ->
  try
     Panel = wxPanel:new(Notebook),
-    %% wxWindow:setBackgroundColour(Panel, {222,222,222}),
     Main  = wxBoxSizer:new(?wxVERTICAL),
 
     CPU = wxPanel:new(Panel, [{winid, ?RQ_W}, {style,?wxFULL_REPAINT_ON_RESIZE}]),
@@ -76,17 +79,22 @@ init([Notebook, Parent]) ->
     wxPanel:connect(CPU, paint, [callback]),
     wxPanel:connect(IO, paint, [callback]),
     wxPanel:connect(MEM, paint, [callback]),
-    % wxPanel:connect(DrawingArea, size, [{skip, true}]),
 
     DefFont  = wxSystemSettings:getFont(?wxSYS_DEFAULT_GUI_FONT),
+    DefSize = wxFont:getPointSize(DefFont),
+    DefFamily = wxFont:getFamily(DefFont),
+    Font = wxFont:new(DefSize, DefFamily, ?wxFONTSTYLE_NORMAL, ?wxFONTWEIGHT_BOLD),
+    SmallFont = wxFont:new(10, DefFamily, ?wxFONTSTYLE_NORMAL, ?wxFONTWEIGHT_NORMAL),
+    BlackPen = wxPen:new({0,0,0}, [{width, 2}]),
     Pens = [wxPen:new(Col, [{width, 2}]) || Col <- tuple_to_list(colors())],
-    %%  GC = wxGraphicsContext:create(DrawingArea),
-    %%  _Font = wxGraphicsContext:createFont(GC, DefFont),
+    process_flag(trap_exit, true),
     {Panel, #state{parent=Parent,
 		   panel =Panel,
 		   windows = {CPU, MEM, IO},
-		   paint=#paint{font= DefFont,
-				pen = ?wxBLACK_PEN,
+		   paint=#paint{font = Font,
+				small = SmallFont,
+				pen  = ?wxGREY_PEN,
+				pen2 = BlackPen,
 				pens = list_to_tuple(Pens)
 			       }
 		  }}
@@ -155,7 +163,7 @@ handle_info({active, Node}, State = #state{parent=Parent, appmon=Old}) ->
     catch _:_ ->
 	    catch Old ! exit,
 	    Me = self(),
-	    Pid = spawn_link(Node, fun() -> fetch_stats(Me) end),
+	    Pid = spawn_link(Node, ?MODULE, fetch_stats, [Me, 1000]),
 	    {noreply, State#state{active=true, appmon=Pid, data={0, queue:new()}}}
     end;
 
@@ -163,13 +171,16 @@ handle_info(not_active, State = #state{appmon=_Pid}) ->
     %% Pid ! exit,
     {noreply, State#state{active=false}};
 
+handle_info({'EXIT', Old, _}, State = #state{appmon=Old}) ->
+    {noreply, State#state{active=false, appmon=undefined}};
+
 handle_info(_Event, State) ->
     io:format("~p:~p: ~p~n",[?MODULE,?LINE,_Event]),
     {noreply, State}.
 
 %%%%%%%%%%
 terminate(_Event, #state{appmon=Pid}) ->
-    Pid ! exit,
+    catch Pid ! exit,
     ok.
 code_change(_, _, State) ->
     State.
@@ -179,16 +190,21 @@ add_data(Stats, {N, Q}) when N > 60 ->
 add_data(Stats, {N, Q}) ->
     {N+1, queue:in(Stats, Q)}.
 
-fetch_stats(Parent) ->
+fetch_stats(Parent, Time) ->
+    erlang:system_flag(scheduler_wall_time, true),
+    fetch_stats_loop(Parent, Time),
+    erlang:system_flag(scheduler_wall_time, false).
+
+fetch_stats_loop(Parent, Time) ->
     receive
 	exit -> normal
-    after 1000 ->
+    after Time ->
 	    _M = Parent ! {stats, 1,
-			   erlang:statistics(run_queues),
+			   erlang:statistics(scheduler_wall_time),
 			   erlang:statistics(io),
 			   erlang:memory()},
 	    %% io:format("IO ~p~n",[element(4,_M)]),
-	    fetch_stats(Parent)
+	    fetch_stats(Parent, Time)
     end.
 
 
@@ -204,8 +220,18 @@ create_menus(Parent, _) ->
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 collect_data(?RQ_W, {N, Q}) ->
-    Data = [RQ || {stats, _Ver, RQ, _IO, _Mem} <- queue:to_list(Q)],
-    {N, lmax(Data), Data};
+    case queue:to_list(Q) of
+	[] ->  {0, 0, []};
+	[_] ->  {0, 0, []};
+	[{stats, _Ver, Init0, _IO, _Mem}|Data0] ->
+	    Init = lists:sort(Init0),
+	    [_|Data=[First|_]] = lists:foldl(fun({stats, _, T0, _, _}, [Prev|Acc]) ->
+					   TN = lists:sort(T0),
+					   Delta = calc_delta(TN, Prev),
+					   [TN, list_to_tuple(Delta)|Acc]
+				   end, [Init], Data0),
+	    {N, lmax(Data), lists:reverse([First|Data])}
+    end;
 collect_data(?MEM_W, {N, Q}) ->
     MemT = mem_types(),
     Data = [list_to_tuple([Value || {Type,Value} <- MemInfo,
@@ -231,6 +257,10 @@ lmax([]) -> 0;
 lmax(List) ->
     lists:max([lists:max(tuple_to_list(T)) || T <- List]).
 
+calc_delta([{Id, WN, TN}|Ss], [{Id, WP, TP}|Ps]) ->
+    [100*(WN-WP) div (TN-TP)|calc_delta(Ss, Ps)];
+calc_delta([], []) -> [].
+
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 draw(Offset, Id, DC, Panel, Paint=#paint{pens=Pens}, Data) ->
     {Len, Max, Hs} = collect_data(Id, Data),
@@ -242,7 +272,7 @@ draw(Offset, Id, DC, Panel, Paint=#paint{pens=Pens}, Data) ->
 	[] -> ignore;
 	_ ->
 	    Draw = fun(N) ->
-			   Lines = make_lines(Hs, Start, N, X0, Y0, WS, HS),
+			   Lines = make_lines(Hs, Start, N, {X0,round(Max*HS)}, Y0, WS, HS),
 			   wxDC:setPen(DC, element(1+ (N-1 rem tuple_size(Pens)) , Pens)),
 			   wxDC:drawLines(DC, Lines),
 			   N+1
@@ -251,31 +281,31 @@ draw(Offset, Id, DC, Panel, Paint=#paint{pens=Pens}, Data) ->
     end,
     ok.
 
-make_lines(Ds = [Data|_], PX, N, MinX, MaxY, WS, HS) ->
+make_lines(Ds = [Data|_], PX, N, Clip, ZeroY, WS, HS) ->
     Y = element(N,Data),
-    make_lines(Ds, PX, N, MinX, MaxY, WS, HS, Y, []).
+    make_lines(Ds, PX, N, Clip, ZeroY, WS, HS, Y, []).
 
-make_lines([D1 | Ds = [D2|Rest]], PX, N, MinX, MaxY, WS, HS, Y0, Acc0) ->
+make_lines([D1 | Ds = [D2|Rest]], PX, N, Clip={Cx,Cy}, ZeroY, WS, HS, Y0, Acc0) ->
     Y1 = element(N,D1),
     Y2 = element(N,D2),
     Y3 = case Rest of
 	     [D3|_] -> element(N,D3);
 	     [] -> Y2
 	 end,
-    This = {max(MinX, round(PX)),MaxY-round(Y1*HS)},
-    Acc = make_splines(Y0,Y1,Y2,Y3,PX,MinX,MaxY,WS,HS,[This|Acc0]),
-    make_lines(Ds, PX+WS, N, MinX, MaxY, WS, HS, Y1, Acc);
-make_lines([D1],  PX, N, _MinX, MaxY, _WS, HS, _Y0, Acc) ->
+    This = {max(Cx, round(PX)),ZeroY-min(Cy,round(Y1*HS))},
+    Acc = make_splines(Y0,Y1,Y2,Y3,PX,Clip,ZeroY,WS,HS,[This|Acc0]),
+    make_lines(Ds, PX+WS, N, Clip, ZeroY, WS, HS, Y1, Acc);
+make_lines([D1],  PX, N, _Clip, ZeroY, _WS, HS, _Y0, Acc) ->
     Y1 = element(N,D1),
-    [{round(PX),MaxY-round(Y1*HS)}|Acc].
+    [{round(PX),ZeroY-round(Y1*HS)}|Acc].
 
-make_splines(_Y0,Y1,Y2,_Y3, _PX, _MinX,_MaxY, _WS,HS, Acc)
+make_splines(_Y0,Y1,Y2,_Y3, _PX, _Clip,_ZeroY, _WS,HS, Acc)
   when (abs(Y1-Y2) * HS) < 3.0  ->
     Acc;
-make_splines(_Y0,_Y1,_Y2,_Y3, _PX, _MinX,_MaxY, WS,_HS, Acc)
+make_splines(_Y0,_Y1,_Y2,_Y3, _PX, _Clip,_ZeroY, WS,_HS, Acc)
   when WS < 3.0 ->
     Acc;
-make_splines(Y00,Y10,Y20,Y30,PX,MinX,MaxY,WS,HS,Acc) ->
+make_splines(Y00,Y10,Y20,Y30,PX,Clip,ZeroY,WS,HS,Acc) ->
     Y1 = Y10*HS,
     Y2 = Y20*HS,
     Steps = round(min(abs(Y1-Y2), WS)),
@@ -284,22 +314,23 @@ make_splines(Y00,Y10,Y20,Y30,PX,MinX,MaxY,WS,HS,Acc) ->
 	    Y3 = Y30*HS,
 	    Tan = spline_tan(Y0,Y1,Y2,Y3),
 	    Delta = 1/Steps,
-	    splines(Steps-1, 0.0, Delta, Tan, Y1,Y2, PX, MinX,MaxY, Delta*WS, Acc);
+	    splines(Steps-1, 0.0, Delta, Tan, Y1,Y2, PX, Clip,ZeroY, Delta*WS, Acc);
        true ->
 	    Acc
     end.
 
-splines(N, XD, XD0, Tan, Y1,Y2, PX0, MinX,MaxY, WS, Acc) when N > 0 ->
+splines(N, XD, XD0, Tan, Y1,Y2, PX0, Clip={Cx,Cy},ZeroY, WS, Acc) when N > 0 ->
     PX = PX0+WS,
     Delta = XD+XD0,
-    if PX < MinX ->
-	    splines(N-1, Delta, XD0, Tan, Y1, Y2, PX, MinX,MaxY, WS, Acc);
+    if PX < Cx ->
+	    splines(N-1, Delta, XD0, Tan, Y1, Y2, PX, Clip,ZeroY, WS, Acc);
        true ->
-	    Y = max(0,round(spline(Delta, Tan, Y1,Y2))),
+	    Y = min(Cy, max(0,round(spline(Delta, Tan, Y1,Y2)))),
 	    %% io:format("Y1:~p Y(~p):~p Y2:~p~n",[round(Y1),round(X),round(Y),round(Y2)]),
-	    splines(N-1, Delta, XD0, Tan, Y1, Y2, PX, MinX,MaxY, WS, [{round(PX), MaxY-Y}|Acc])
+	    splines(N-1, Delta, XD0, Tan, Y1, Y2, PX, Clip,ZeroY, WS,
+		    [{round(PX), ZeroY-Y}|Acc])
     end;
-splines(_N, _XD, _XD0, _Tan, _Y1,_Y2, _PX, _MinX,_MaxY, _WS, Acc) -> Acc.
+splines(_N, _XD, _XD0, _Tan, _Y1,_Y2, _PX, _Clip,_ZeroY, _WS, Acc) -> Acc.
 
 spline(T, {M1, M2}, Y1, Y2) ->
     %% Hermite Basis Funcs
@@ -323,15 +354,14 @@ spline_tan(Y0, Y1, Y2, Y3) ->
 -define(BH, 5).
 
 draw_borders(Type, NoGraphs, DC, {W,H}, Max0,
-	     #paint{pen=Pen, font=Font}) ->
+	     #paint{pen=Pen, pen2=Pen2, font=Font, small=Small}) ->
     Max = calc_max(Max0),
-    wxDC:setPen(DC, Pen),
-    wxDC:setFont(DC, Font),
     {Unit, MaxUnit} = bytes(Type, Max),
     Str1 = observer_lib:to_str(MaxUnit),
     Str2 = observer_lib:to_str(MaxUnit div 2),
     Str3 = observer_lib:to_str(0),
     {TW,TH} = wxDC:getTextExtent(DC, Str1),
+    {SpaceW, _} = wxDC:getTextExtent(DC, " "),
 
     GraphX0 = ?BW+TW+?BW,
     GraphX1 = W-?BW*4,
@@ -341,36 +371,49 @@ draw_borders(Type, NoGraphs, DC, {W,H}, Max0,
     SecondsY = BottomTextY - ?BH - TH,
     GraphY0 = MaxTextY + (TH div 2),
     GraphY1 = SecondsY - ?BH,
-    GraphW = max(GraphX1-GraphX0-1, 60),
-    GraphH = max(GraphY1-GraphY0-1, 100),
+    GraphW = GraphX1-GraphX0-1,
+    GraphH = GraphY1-GraphY0-1,
+    GraphY25 = GraphY0 + (GraphY1 - GraphY0) div 4,
+    GraphY50 = GraphY0 + (GraphY1 - GraphY0) div 2,
+    GraphY75 = GraphY0 + 3*(GraphY1 - GraphY0) div 4,
     ScaleW = GraphW / 60,
-    ScaleH = calc_scale(GraphH, Max),
-
-    case Type of
-	?RQ_W ->  wxDC:drawText(DC, "CPU History - Run queue length", {TopTextX,?BH});
-	?MEM_W -> wxDC:drawText(DC, "Memory Usage " ++ Unit, {TopTextX,?BH});
-	?IO_W ->  wxDC:drawText(DC, "IO Usage " ++ Unit, {TopTextX,?BH})
-    end,
+    ScaleH = GraphH / Max,
 
+    wxDC:setFont(DC, Small),
     Align = fun(Str, Y) ->
 		    {StrW, _} = wxDC:getTextExtent(DC, Str),
 		    wxDC:drawText(DC, Str, {GraphX0 - StrW - ?BW, Y})
 	    end,
     Align(Str1, MaxTextY),
-    Align(Str2, MaxTextY - (TW div 2) + (GraphY1 - MaxTextY) div 2),
+    Align(Str2, GraphY50 - (TH div 2)),
     Align(Str3, GraphY1 - (TH div 2) + 1),
 
+    wxDC:setPen(DC, Pen),
     DrawSecs = fun(Secs, Pos) ->
 		       Str = [observer_lib:to_str(Secs)|" s"],
-		       wxDC:drawText(DC, Str,  {round(GraphX0-?BH+Pos), SecondsY}),
+		       X = round(GraphX0+Pos),
+		       wxDC:drawText(DC, Str,  {X-SpaceW, SecondsY}),
+		       wxDC:drawLine(DC, {X, GraphY0}, {X, GraphY1+5}),
 		       Pos + 10*ScaleW
 	       end,
     lists:foldl(DrawSecs, 0, lists:seq(60,0, -10)),
 
-    wxDC:drawLines(DC, [{GraphX0, GraphY0}, {GraphX0, GraphY1},
-			{GraphX1, GraphY1}, {GraphX1, GraphY0}, {GraphX0, GraphY0}]),
+    wxDC:drawLine(DC, {GraphX0-3, GraphY25}, {GraphX1, GraphY25}),
+    wxDC:drawLine(DC, {GraphX0-3, GraphY50}, {GraphX1, GraphY50}),
+    wxDC:drawLine(DC, {GraphX0-3, GraphY75}, {GraphX1, GraphY75}),
+
+    wxDC:setPen(DC, Pen2),
+    wxDC:drawLines(DC, [{GraphX0, GraphY0-1}, {GraphX0, GraphY1+1},
+			{GraphX1, GraphY1+1}, {GraphX1, GraphY0-1},
+			{GraphX0, GraphY0-1}]),
+
+    wxDC:setFont(DC, Font),
+    case Type of
+	?RQ_W ->  wxDC:drawText(DC, "Scheduler Utilization (%) ", {TopTextX,?BH});
+	?MEM_W -> wxDC:drawText(DC, "Memory Usage " ++ Unit, {TopTextX,?BH});
+	?IO_W ->  wxDC:drawText(DC, "IO Usage " ++ Unit, {TopTextX,?BH})
+    end,
 
-    {SpaceW, _} = wxDC:getTextExtent(DC, " "),
     Text = fun(X,Y, Str, PenId) ->
 		   if PenId == 0 ->
 			   wxDC:setTextForeground(DC, {0,0,0});
@@ -411,7 +454,11 @@ calc_max(Max) -> calc_max1(Max).
 calc_max1(Max) ->
     case Max div 10 of
 	X when X < 10 ->
-	    (X+1)*10;
+	    case Max rem 10 of
+		0 -> Max;
+		_ ->
+		    (X+1)*10
+	    end;
 	X ->
 	    10*calc_max1(X)
     end.
@@ -428,15 +475,9 @@ bytes(_, B) ->
 	true    -> {"(B)", B}
     end.
 
-calc_scale(H, {_Type, Max}) -> calc_scale(H,Max);
-calc_scale(Height, Max) when Height > Max ->
-    Height / Max;
-calc_scale(Height, Max) ->
-    Height / Max.
-
 colors() ->
-    {{220, 50, 50}, {50, 220, 50}, {50, 50, 220},
-     {220, 220, 50}, {50, 220, 220}, {220, 50, 220},
-     {220, 100, 100}, {220, 100, 220},
-     {100, 220, 220}, {100, 220, 100}
+    {{200, 50, 50}, {50, 200, 50}, {50, 50, 200},
+     {255, 110, 0}, {50, 200, 200}, {200, 50, 200},
+     {240, 200, 80}, {140, 2, 140},
+     {100, 200, 240}, {100, 240, 100}
     }.
diff --git a/lib/observer/src/observer_wx.erl b/lib/observer/src/observer_wx.erl
index 2403b984e5..e2b256d768 100644
--- a/lib/observer/src/observer_wx.erl
+++ b/lib/observer/src/observer_wx.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2011. All Rights Reserved.
+%% Copyright Ericsson AB 2011-2012. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -328,8 +328,9 @@ handle_info({nodedown, Node},
     create_txt_dialog(Frame, Msg, "Node down", ?wxICON_EXCLAMATION),
     {noreply, State3};
 
-handle_info({'EXIT', _Pid, _Reason}, State) ->
-    io:format("Child crashed exiting:  ~p ~p~n", [_Pid,_Reason]),
+handle_info({'EXIT', Pid, _Reason}, State) ->
+    io:format("Child (~s) crashed exiting:  ~p ~p~n",
+	      [pid2panel(Pid, State), Pid,_Reason]),
     {stop, normal, State};
 
 handle_info(_Info, State) ->
@@ -351,6 +352,7 @@ try_rpc(Node, Mod, Func, Args) ->
 	    error_logger:error_report([{node, Node},
 				       {call, {Mod, Func, Args}},
 				       {reason, {badrpc, Reason}}]),
+	    observer ! {nodedown, Node},
 	    error({badrpc, Reason});
 	Res ->
 	    Res
@@ -423,6 +425,20 @@ get_active_pid(#state{notebook=Notebook, pro_panel=Pro, sys_panel=Sys,
 	    end,
     wx_object:get_pid(Panel).
 
+pid2panel(Pid, #state{pro_panel=Pro, sys_panel=Sys,
+		      tv_panel=Tv, trace_panel=Trace, app_panel=App,
+		      perf_panel=Perf}) ->
+    case Pid of
+	Pro -> "Processes";
+	Sys -> "System";
+	Tv -> "Table Viewer" ;
+	Trace -> ?TRACE_STR;
+	Perf -> "Load Charts";
+	App -> "Applications";
+	_ -> "unknown"
+    end.
+
+
 create_connect_dialog(ping, #state{frame = Frame}) ->
     Dialog = wxTextEntryDialog:new(Frame, "Connect to node"),
     case wxDialog:showModal(Dialog) of