Merge branch 'sverker/erts/more-crash-dump-info/OTP-14820' into maint

* sverker/erts/more-crash-dump-info/OTP-14820:
  erts,observer: Add port-suspended pids to crash dump
  erts,observer: Add port states and flags to crash dump
  erts,observer: Add dirty schedulers to crash dump
  observer: Refactor get_schedulerinfo1
  erts,observer: Add more port info to crash dump
  erts: Cleanup dump_process_info()
  erts: Include failing garbing process in crash dump
  erts: Remove unused args to collect_live_heap_frags
  erts: Add binary vheap sizes to crash dump

5 files changed, 193 insertions, 50 deletions

diff --git a/lib/observer/src/cdv_port_cb.erl b/lib/observer/src/cdv_port_cb.erl
index b5cbe8132d..6bb8f07a74 100644
--- a/lib/observer/src/cdv_port_cb.erl
+++ b/lib/observer/src/cdv_port_cb.erl
@@ -34,7 +34,8 @@
 -define(COL_CONN, ?COL_ID+1).
 -define(COL_NAME, ?COL_CONN+1).
 -define(COL_CTRL, ?COL_NAME+1).
--define(COL_SLOT, ?COL_CTRL+1).
+-define(COL_QUEUE, ?COL_CTRL+1).
+-define(COL_SLOT, ?COL_QUEUE+1).
 
 
 
@@ -44,6 +45,7 @@ col_to_elem(?COL_ID)  -> #port.id;
 col_to_elem(?COL_CONN) -> #port.connected;
 col_to_elem(?COL_NAME)  -> #port.name;
 col_to_elem(?COL_CTRL) -> #port.controls;
+col_to_elem(?COL_QUEUE) -> #port.queue;
 col_to_elem(?COL_SLOT) -> #port.slot.
 
 col_spec() ->
@@ -51,6 +53,7 @@ col_spec() ->
      {"Connected", ?wxLIST_FORMAT_LEFT, 120},
      {"Name", ?wxLIST_FORMAT_LEFT, 150},
      {"Controls", ?wxLIST_FORMAT_LEFT, 200},
+     {"Queue", ?wxLIST_FORMAT_RIGHT, 100},
      {"Slot", ?wxLIST_FORMAT_RIGHT, 50}].
 
 get_info(_) ->
@@ -96,9 +99,17 @@ format(D) ->
 info_fields() ->
     [{"Overview",
       [{"Name",             name},
+       {"State",            state},
+       {"Task Flags",       task_flags},
        {"Connected",        {click,connected}},
        {"Slot",             slot},
-       {"Controls",         controls}]},
+       {"Controls",         controls},
+       {"Input bytes",      input},
+       {"Output bytes",     output},
+       {"Queue bytes",      queue},
+       {"Port data",        port_data}]},
     {scroll_boxes,
      [{"Links",1,{click,links}},
-       {"Monitors",1,{click,monitors}}]}].
+      {"Monitors",1,{click,monitors}},
+      {"Suspended",1,{click,suspended}}
+     ]}].
diff --git a/lib/observer/src/cdv_proc_cb.erl b/lib/observer/src/cdv_proc_cb.erl
index f10650bbb7..0ea23dd7cb 100644
--- a/lib/observer/src/cdv_proc_cb.erl
+++ b/lib/observer/src/cdv_proc_cb.erl
@@ -149,6 +149,10 @@ info_fields() ->
        {"Old Heap",         old_heap},
        {"Heap Unused",      heap_unused},
        {"Old Heap Unused",  old_heap_unused},
+       {"Binary vheap",     bin_vheap},
+       {"Old Binary vheap", old_bin_vheap},
+       {"Binary vheap unused",  bin_vheap_unused},
+       {"Old Binary vheap unused", old_bin_vheap_unused},
        {"Number of Heap Fragements", num_heap_frag},
        {"Heap Fragment Data",heap_frag_data},
        {"New Heap Start",   new_heap_start},
diff --git a/lib/observer/src/cdv_sched_cb.erl b/lib/observer/src/cdv_sched_cb.erl
index 192aaf31a7..d2696a276f 100644
--- a/lib/observer/src/cdv_sched_cb.erl
+++ b/lib/observer/src/cdv_sched_cb.erl
@@ -31,7 +31,8 @@
 
 %% Columns
 -define(COL_ID,  0).
--define(COL_PROC,  ?COL_ID+1).
+-define(COL_TYPE,  ?COL_ID+1).
+-define(COL_PROC,  ?COL_TYPE+1).
 -define(COL_PORT,  ?COL_PROC+1).
 -define(COL_RQL,   ?COL_PORT+1).
 -define(COL_PQL,   ?COL_RQL+1).
@@ -39,6 +40,7 @@
 %% Callbacks for cdv_virtual_list_wx
 col_to_elem(id) -> col_to_elem(?COL_ID);
 col_to_elem(?COL_ID)  -> #sched.name;
+col_to_elem(?COL_TYPE)  -> #sched.type;
 col_to_elem(?COL_PROC)  -> #sched.process;
 col_to_elem(?COL_PORT)  -> #sched.port;
 col_to_elem(?COL_RQL)   -> #sched.run_q;
@@ -46,6 +48,7 @@ col_to_elem(?COL_PQL)   -> #sched.port_q.
 
 col_spec() ->
     [{"Id",                ?wxLIST_FORMAT_RIGHT,   50},
+     {"Type",              ?wxLIST_FORMAT_CENTER, 100},
      {"Current Process",   ?wxLIST_FORMAT_CENTER, 130},
      {"Current Port",      ?wxLIST_FORMAT_CENTER, 130},
      {"Run Queue Length",  ?wxLIST_FORMAT_RIGHT,  180},
@@ -73,7 +76,8 @@ detail_pages() ->
     [{"Scheduler Information", fun init_gen_page/2}].
 
 init_gen_page(Parent, Info0) ->
-    Fields = info_fields(),
+    Type = proplists:get_value(type, Info0),
+    Fields = info_fields(Type),
     Details = proplists:get_value(details, Info0),
     Info = if is_map(Details) -> Info0 ++ maps:to_list(Details);
 	      true -> Info0
@@ -81,15 +85,16 @@ init_gen_page(Parent, Info0) ->
     cdv_info_wx:start_link(Parent,{Fields,Info,[]}).
 
 %%% Internal
-info_fields() ->
+info_fields(Type) ->
     [{"Scheduler Overview",
       [{"Id",             id},
+       {"Type",           type},
        {"Current Process",process},
        {"Current Port",   port},
        {"Sleep Info Flags", sleep_info},
        {"Sleep Aux Work",   sleep_aux}
       ]},
-     {"Run Queues",
+     {run_queues_header(Type),
       [{"Flags",                   runq_flags},
        {"Priority Max Length",     runq_max},
        {"Priority High Length",    runq_high},
@@ -116,3 +121,8 @@ info_fields() ->
        {"     ",           {currp_stack, 11}}
       ]}
     ].
+
+run_queues_header(normal) ->
+    "Run Queues";
+run_queues_header(DirtyX) ->
+    "Run Queues (common for all '" ++ atom_to_list(DirtyX) ++ "' schedulers)".
diff --git a/lib/observer/src/crashdump_viewer.erl b/lib/observer/src/crashdump_viewer.erl
index 861b592439..d2a175d52d 100644
--- a/lib/observer/src/crashdump_viewer.erl
+++ b/lib/observer/src/crashdump_viewer.erl
@@ -116,6 +116,10 @@
 -define(allocator,allocator).
 -define(atoms,atoms).
 -define(binary,binary).
+-define(dirty_cpu_scheduler,dirty_cpu_scheduler).
+-define(dirty_cpu_run_queue,dirty_cpu_run_queue).
+-define(dirty_io_scheduler,dirty_io_scheduler).
+-define(dirty_io_run_queue,dirty_io_run_queue).
 -define(ende,ende).
 -define(erl_crash_dump,erl_crash_dump).
 -define(ets,ets).
@@ -1222,6 +1226,18 @@ all_procinfo(Fd,Fun,Proc,WS,LineHead) ->
 	"OldHeap unused" ->
 	    Bytes = list_to_integer(bytes(Fd))*WS,
 	    get_procinfo(Fd,Fun,Proc#proc{old_heap_unused=Bytes},WS);
+	"BinVHeap" ->
+	    Bytes = list_to_integer(bytes(Fd))*WS,
+	    get_procinfo(Fd,Fun,Proc#proc{bin_vheap=Bytes},WS);
+	"OldBinVHeap" ->
+	    Bytes = list_to_integer(bytes(Fd))*WS,
+	    get_procinfo(Fd,Fun,Proc#proc{old_bin_vheap=Bytes},WS);
+	"BinVHeap unused" ->
+	    Bytes = list_to_integer(bytes(Fd))*WS,
+	    get_procinfo(Fd,Fun,Proc#proc{bin_vheap_unused=Bytes},WS);
+	"OldBinVHeap unused" ->
+	    Bytes = list_to_integer(bytes(Fd))*WS,
+	    get_procinfo(Fd,Fun,Proc#proc{old_bin_vheap_unused=Bytes},WS);
 	"New heap start" ->
 	    get_procinfo(Fd,Fun,Proc#proc{new_heap_start=bytes(Fd)},WS);
 	"New heap top" ->
@@ -1632,6 +1648,10 @@ port_to_tuple("#Port<"++Port) ->
 
 get_portinfo(Fd,Port) ->
     case line_head(Fd) of
+        "State" ->
+	    get_portinfo(Fd,Port#port{state=bytes(Fd)});
+        "Task Flags" ->
+	    get_portinfo(Fd,Port#port{task_flags=bytes(Fd)});
 	"Slot" ->
 	    %% stored as integer so we can sort on it
 	    get_portinfo(Fd,Port#port{slot=list_to_integer(bytes(Fd))});
@@ -1656,6 +1676,10 @@ get_portinfo(Fd,Port) ->
 			    {Pid,Pid++" ("++Ref++")"}
 			end || Mon <- Monitors0],
 	    get_portinfo(Fd,Port#port{monitors=Monitors});
+        "Suspended" ->
+	    Pids = split_pid_list_no_space(bytes(Fd)),
+	    Suspended = [{Pid,Pid} || Pid <- Pids],
+	    get_portinfo(Fd,Port#port{suspended=Suspended});
 	"Port controls linked-in driver" ->
 	    Str = lists:flatten(["Linked in driver: " | string(Fd)]),
 	    get_portinfo(Fd,Port#port{controls=Str});
@@ -1671,6 +1695,15 @@ get_portinfo(Fd,Port) ->
 	"Port is UNIX fd not opened by emulator" ->
 	    Str = lists:flatten(["UNIX fd not opened by emulator: "| string(Fd)]),
 	    get_portinfo(Fd,Port#port{controls=Str});
+        "Input" ->
+	    get_portinfo(Fd,Port#port{input=list_to_integer(bytes(Fd))});
+        "Output" ->
+	    get_portinfo(Fd,Port#port{output=list_to_integer(bytes(Fd))});
+        "Queue" ->
+	    get_portinfo(Fd,Port#port{queue=list_to_integer(bytes(Fd))});
+        "Port Data" ->
+	    get_portinfo(Fd,Port#port{port_data=string(Fd)});
+
 	"=" ++ _next_tag ->
 	    Port;
 	Other ->
@@ -2501,73 +2534,142 @@ get_indextableinfo1(Fd,IndexTable) ->
 %%-----------------------------------------------------------------
 %% Page with scheduler table information
 schedulers(File) ->
-    case lookup_index(?scheduler) of
-	[] ->
-	    [];
-	Schedulers ->
-	    Fd = open(File),
-	    R = lists:map(fun({Name,Start}) ->
-				  get_schedulerinfo(Fd,Name,Start)
-			  end,
-			  Schedulers),
-	    close(Fd),
-	    R
-    end.
+    Fd = open(File),
 
-get_schedulerinfo(Fd,Name,Start) ->
+    Schds0 = case lookup_index(?scheduler) of
+                 [] ->
+                     [];
+                 Normals ->
+                     [{Normals, #sched{type=normal}}]
+             end,
+    Schds1 = case lookup_index(?dirty_cpu_scheduler) of
+                 [] ->
+                     Schds0;
+                 DirtyCpus ->
+                     [{DirtyCpus, get_dirty_runqueue(Fd, ?dirty_cpu_run_queue)}
+                      | Schds0]
+             end,
+    Schds2 = case lookup_index(?dirty_io_scheduler) of
+                 [] ->
+                     Schds1;
+                 DirtyIos ->
+                     [{DirtyIos, get_dirty_runqueue(Fd, ?dirty_io_run_queue)}
+                      | Schds1]
+             end,
+
+    R = schedulers1(Fd, Schds2, []),
+    close(Fd),
+    R.
+
+schedulers1(_Fd, [], Acc) ->
+    Acc;
+schedulers1(Fd, [{Scheds,Sched0} | Tail], Acc0) ->
+    Acc1 = lists:foldl(fun({Name,Start}, AccIn) ->
+                               [get_schedulerinfo(Fd,Name,Start,Sched0) | AccIn]
+                       end,
+                       Acc0,
+                       Scheds),
+    schedulers1(Fd, Tail, Acc1).
+
+get_schedulerinfo(Fd,Name,Start,Sched0) ->
     pos_bof(Fd,Start),
-    get_schedulerinfo1(Fd,#sched{name=Name}).
+    get_schedulerinfo1(Fd,Sched0#sched{name=list_to_integer(Name)}).
 
-get_schedulerinfo1(Fd,Sched=#sched{details=Ds}) ->
+sched_type(?dirty_cpu_run_queue) -> dirty_cpu;
+sched_type(?dirty_io_run_queue) ->  dirty_io.
+
+get_schedulerinfo1(Fd, Sched) ->
+    case get_schedulerinfo2(Fd, Sched) of
+        {more, Sched2} ->
+            get_schedulerinfo1(Fd, Sched2);
+        {done, Sched2} ->
+            Sched2
+    end.
+
+get_schedulerinfo2(Fd, Sched=#sched{details=Ds}) ->
     case line_head(Fd) of
 	"Current Process" ->
-	    get_schedulerinfo1(Fd,Sched#sched{process=bytes(Fd, "None")});
+	    {more, Sched#sched{process=bytes(Fd, "None")}};
 	"Current Port" ->
-	    get_schedulerinfo1(Fd,Sched#sched{port=bytes(Fd, "None")});
+	    {more, Sched#sched{port=bytes(Fd, "None")}};
+
+	"Scheduler Sleep Info Flags" ->
+	    {more, Sched#sched{details=Ds#{sleep_info=>bytes(Fd, "None")}}};
+	"Scheduler Sleep Info Aux Work" ->
+	    {more, Sched#sched{details=Ds#{sleep_aux=>bytes(Fd, "None")}}};
+
+	"Current Process State" ->
+	    {more, Sched#sched{details=Ds#{currp_state=>bytes(Fd)}}};
+	"Current Process Internal State" ->
+	    {more, Sched#sched{details=Ds#{currp_int_state=>bytes(Fd)}}};
+	"Current Process Program counter" ->
+	    {more, Sched#sched{details=Ds#{currp_prg_cnt=>string(Fd)}}};
+	"Current Process CP" ->
+	    {more, Sched#sched{details=Ds#{currp_cp=>string(Fd)}}};
+	"Current Process Limited Stack Trace" ->
+	    %% If there shall be last in scheduler information block
+	    {done, Sched#sched{details=get_limited_stack(Fd, 0, Ds)}};
+
+	"=" ++ _next_tag ->
+            {done, Sched};
+
+	Other ->
+            case Sched#sched.type of
+                normal ->
+                    get_runqueue_info2(Fd, Other, Sched);
+                _ ->
+                    unexpected(Fd,Other,"dirty scheduler information"),
+                    {done, Sched}
+            end
+    end.
+
+get_dirty_runqueue(Fd, Tag) ->
+    case lookup_index(Tag) of
+        [{_, Start}] ->
+            pos_bof(Fd,Start),
+            get_runqueue_info1(Fd,#sched{type=sched_type(Tag)});
+        [] ->
+            #sched{}
+    end.
+
+get_runqueue_info1(Fd, Sched) ->
+    case get_runqueue_info2(Fd, line_head(Fd), Sched) of
+        {more, Sched2} ->
+            get_runqueue_info1(Fd, Sched2);
+        {done, Sched2} ->
+            Sched2
+    end.
+
+get_runqueue_info2(Fd, LineHead, Sched=#sched{details=Ds}) ->
+    case LineHead of
 	"Run Queue Max Length" ->
 	    RQMax = list_to_integer(bytes(Fd)),
 	    RQ = RQMax + Sched#sched.run_q,
-	    get_schedulerinfo1(Fd,Sched#sched{run_q=RQ, details=Ds#{runq_max=>RQMax}});
+	    {more, Sched#sched{run_q=RQ, details=Ds#{runq_max=>RQMax}}};
 	"Run Queue High Length" ->
 	    RQHigh = list_to_integer(bytes(Fd)),
 	    RQ = RQHigh + Sched#sched.run_q,
-	    get_schedulerinfo1(Fd,Sched#sched{run_q=RQ, details=Ds#{runq_high=>RQHigh}});
+	    {more, Sched#sched{run_q=RQ, details=Ds#{runq_high=>RQHigh}}};
 	"Run Queue Normal Length" ->
 	    RQNorm = list_to_integer(bytes(Fd)),
 	    RQ = RQNorm + Sched#sched.run_q,
-	    get_schedulerinfo1(Fd,Sched#sched{run_q=RQ, details=Ds#{runq_norm=>RQNorm}});
+	    {more, Sched#sched{run_q=RQ, details=Ds#{runq_norm=>RQNorm}}};
 	"Run Queue Low Length" ->
 	    RQLow = list_to_integer(bytes(Fd)),
 	    RQ = RQLow + Sched#sched.run_q,
-	    get_schedulerinfo1(Fd,Sched#sched{run_q=RQ, details=Ds#{runq_low=>RQLow}});
+	    {more, Sched#sched{run_q=RQ, details=Ds#{runq_low=>RQLow}}};
 	"Run Queue Port Length" ->
 	    RQ = list_to_integer(bytes(Fd)),
-	    get_schedulerinfo1(Fd,Sched#sched{port_q=RQ});
-
-	"Scheduler Sleep Info Flags" ->
-	    get_schedulerinfo1(Fd,Sched#sched{details=Ds#{sleep_info=>bytes(Fd, "None")}});
-	"Scheduler Sleep Info Aux Work" ->
-	    get_schedulerinfo1(Fd,Sched#sched{details=Ds#{sleep_aux=>bytes(Fd, "None")}});
+	    {more, Sched#sched{port_q=RQ}};
 
 	"Run Queue Flags" ->
-	    get_schedulerinfo1(Fd,Sched#sched{details=Ds#{runq_flags=>bytes(Fd, "None")}});
+	    {more, Sched#sched{details=Ds#{runq_flags=>bytes(Fd, "None")}}};
 
-	"Current Process State" ->
-	    get_schedulerinfo1(Fd,Sched#sched{details=Ds#{currp_state=>bytes(Fd)}});
-	"Current Process Internal State" ->
-	    get_schedulerinfo1(Fd,Sched#sched{details=Ds#{currp_int_state=>bytes(Fd)}});
-	"Current Process Program counter" ->
-	    get_schedulerinfo1(Fd,Sched#sched{details=Ds#{currp_prg_cnt=>string(Fd)}});
-	"Current Process CP" ->
-	    get_schedulerinfo1(Fd,Sched#sched{details=Ds#{currp_cp=>string(Fd)}});
-	"Current Process Limited Stack Trace" ->
-	    %% If there shall be last in scheduler information block
-	    Sched#sched{details=get_limited_stack(Fd, 0, Ds)};
 	"=" ++ _next_tag ->
-	    Sched;
+            {done, Sched};
 	Other ->
 	    unexpected(Fd,Other,"scheduler information"),
-	    Sched
+	    {done, Sched}
     end.
 
 get_limited_stack(Fd, N, Ds) ->
@@ -2998,6 +3100,10 @@ tag_to_atom("allocated_areas") -> ?allocated_areas;
 tag_to_atom("allocator") -> ?allocator;
 tag_to_atom("atoms") -> ?atoms;
 tag_to_atom("binary") -> ?binary;
+tag_to_atom("dirty_cpu_scheduler") -> ?dirty_cpu_scheduler;
+tag_to_atom("dirty_cpu_run_queue") -> ?dirty_cpu_run_queue;
+tag_to_atom("dirty_io_scheduler") -> ?dirty_io_scheduler;
+tag_to_atom("dirty_io_run_queue") -> ?dirty_io_run_queue;
 tag_to_atom("end") -> ?ende;
 tag_to_atom("erl_crash_dump") -> ?erl_crash_dump;
 tag_to_atom("ets") -> ?ets;
diff --git a/lib/observer/src/crashdump_viewer.hrl b/lib/observer/src/crashdump_viewer.hrl
index 6a93a089fd..252e19379d 100644
--- a/lib/observer/src/crashdump_viewer.hrl
+++ b/lib/observer/src/crashdump_viewer.hrl
@@ -80,6 +80,10 @@
 	 old_heap,
 	 heap_unused,
 	 old_heap_unused,
+         bin_vheap,
+         old_bin_vheap,
+         bin_vheap_unused,
+         old_bin_vheap_unused,
 	 new_heap_start,
 	 new_heap_top,
 	 stack_top,
@@ -95,19 +99,27 @@
 
 -record(port,
 	{id,
+         state,
+         task_flags=0,
 	 slot,
 	 connected,
 	 links,
 	 name,
 	 monitors,
-	 controls}).
+         suspended,
+	 controls,
+         input,
+         output,
+         queue,
+         port_data}).
 
 -record(sched,
 	{name,
+         type,
 	 process,
 	 port,
 	 run_q=0,
-	 port_q=0,
+	 port_q,
 	 details=#{}
 	}).