Merge branch 'siri/cdv/crash-on-large-dump/OTP-14386' into maint

* siri/cdv/crash-on-large-dump/OTP-14386:
  observer: Improve debugging code in test suite
  cdv: Add ~tw as formatting option when expanding a term
  observer: Remove compiler warnings in test and set warnings_as_errors
  cdv: Warn about unsafe crashdump
  cdv: Show progress bar while reading big data
  cdv: Optimize reading of crashdump with many binaries
  cdv: Find end of 'Last calls' section
  cdv: Don't mark process as truncated if truncation is after all proc_heaps
  cdv: Set ERL_CRASH_DUMP_SECONDS to 0 in cdv scripts

4 files changed, 138 insertions, 32 deletions

diff --git a/lib/observer/test/Makefile b/lib/observer/test/Makefile
index fcb1b73911..a44e54fc52 100644
--- a/lib/observer/test/Makefile
+++ b/lib/observer/test/Makefile
@@ -47,7 +47,7 @@ RELSYSDIR = $(RELEASE_PATH)/observer_test
 # FLAGS
 # ----------------------------------------------------
 ERL_MAKE_FLAGS +=
-ERL_COMPILE_FLAGS += +nowarn_export_all
+ERL_COMPILE_FLAGS += +warnings_as_errors +nowarn_export_all
 
 EBIN = .
 
diff --git a/lib/observer/test/crashdump_viewer_SUITE.erl b/lib/observer/test/crashdump_viewer_SUITE.erl
index 77cf086d4b..f9ac884743 100644
--- a/lib/observer/test/crashdump_viewer_SUITE.erl
+++ b/lib/observer/test/crashdump_viewer_SUITE.erl
@@ -76,7 +76,7 @@ end_per_testcase(Case, Config) ->
     end,
     ok.
 
-suite() -> [{ct_hooks,[ts_install_cth]}].
+suite() -> [].
 
 all() -> 
     [start_stop,
@@ -364,6 +364,9 @@ special(File,Procs) ->
 		crashdump_viewer:expand_binary({SOffset,SSize,SPos}),
 	    io:format("  expand binary ok",[]),
 
+	    #proc{last_calls=LastCalls} = ProcDetails,
+            true = length(LastCalls) =< 4,
+
 	    ['#CDVPid',X1,Y1,Z1] = proplists:get_value(ext_pid,Dict),
 	    ChannelStr1 = integer_to_list(X1),
 	    ExtPid =
@@ -413,19 +416,90 @@ special(File,Procs) ->
 			old_attrib=undefined,
 			old_comp_info=undefined}=Mod2,
 	    ok;
-	%% ".strangemodname" ->
-	%%     {ok,Mods,[]} = crashdump_viewer:loaded_modules(),
-	%%     lookat_all_mods(Mods),
-	%%     ok;
-	%% ".sort" ->
-	%%     %% sort ports, atoms and modules ????
-	%%     ok;
-	%% ".trunc" ->
-	%%     %% ????
-	%%     ok;
-        ".trunc.bytes" ->
+	".trunc_bin1" ->
+            %% This is 'full_dist' truncated after the first
+            %% "=binary:"
+            %% i.e. no binary exist in the dump
+	    [#proc{pid=Pid0}|_Rest] = lists:keysort(#proc.name,Procs),
+	    Pid = pid_to_list(Pid0),
+	    {ok,ProcDetails=#proc{},[]} = crashdump_viewer:proc_details(Pid),
+	    io:format("  process details ok",[]),
+
+	    #proc{dict=Dict} = ProcDetails,
+
+	    '#CDVNonexistingBinary' = proplists:get_value(bin,Dict),
+	    '#CDVNonexistingBinary' = proplists:get_value(sub_bin,Dict),
+
+	    io:format("  nonexisting binaries ok",[]),
+            ok;
+	".trunc_bin2" ->
+            %% This is 'full_dist' truncated after the first
+            %% "=binary:Addr\n
+            %%  Size"
+            %% i.e. binaries are truncated
+	    [#proc{pid=Pid0}|_Rest] = lists:keysort(#proc.name,Procs),
+	    Pid = pid_to_list(Pid0),
+	    {ok,ProcDetails=#proc{},[]} = crashdump_viewer:proc_details(Pid),
+	    io:format("  process details ok",[]),
+
+	    #proc{dict=Dict} = ProcDetails,
+
+	    ['#CDVBin',Offset,Size,Pos] = proplists:get_value(bin,Dict),
+            {ok,'#CDVTruncatedBinary'} =
+		crashdump_viewer:expand_binary({Offset,Size,Pos}),
+	    ['#CDVBin',SOffset,SSize,SPos] = proplists:get_value(sub_bin,Dict),
+            {ok,'#CDVTruncatedBinary'} =
+		crashdump_viewer:expand_binary({SOffset,SSize,SPos}),
+
+	    io:format("  expand truncated binary ok",[]),
+            ok;
+	".trunc_bin3" ->
+            %% This is 'full_dist' truncated after the first
+            %% "=binary:Addr\n
+            %%  Size:"
+            %% i.e. same as 'trunc_bin2', except the colon exists also
+	    [#proc{pid=Pid0}|_Rest] = lists:keysort(#proc.name,Procs),
+	    Pid = pid_to_list(Pid0),
+	    {ok,ProcDetails=#proc{},[]} = crashdump_viewer:proc_details(Pid),
+	    io:format("  process details ok",[]),
+
+	    #proc{dict=Dict} = ProcDetails,
+
+	    ['#CDVBin',Offset,Size,Pos] = proplists:get_value(bin,Dict),
+            {ok,'#CDVTruncatedBinary'} =
+		crashdump_viewer:expand_binary({Offset,Size,Pos}),
+	    ['#CDVBin',SOffset,SSize,SPos] = proplists:get_value(sub_bin,Dict),
+            {ok,'#CDVTruncatedBinary'} =
+		crashdump_viewer:expand_binary({SOffset,SSize,SPos}),
+
+	    io:format("  expand truncated binary ok",[]),
+            ok;
+	".trunc_bin4" ->
+            %% This is 'full_dist' truncated after the first
+            %% "=binary:Addr\n
+            %%  Size:BinaryMissinOneByte"
+            %% i.e. the full binary is truncated, but the sub binary is complete
+	    [#proc{pid=Pid0}|_Rest] = lists:keysort(#proc.name,Procs),
+	    Pid = pid_to_list(Pid0),
+	    {ok,ProcDetails=#proc{},[]} = crashdump_viewer:proc_details(Pid),
+	    io:format("  process details ok",[]),
+
+	    #proc{dict=Dict} = ProcDetails,
+
+	    ['#CDVBin',Offset,Size,Pos] = proplists:get_value(bin,Dict),
+            {ok,'#CDVTruncatedBinary'} =
+		crashdump_viewer:expand_binary({Offset,Size,Pos}),
+	    io:format("  expand truncated binary ok",[]),
+	    ['#CDVBin',SOffset,SSize,SPos] = proplists:get_value(sub_bin,Dict),
+            {ok,<<_:SSize/binary>>} =
+		crashdump_viewer:expand_binary({SOffset,SSize,SPos}),
+	    io:format("  expand complete sub binary ok",[]),
+
+            ok;
+        ".trunc_bytes" ->
             {ok,_,[TW]} = crashdump_viewer:general_info(),
             {match,_} = re:run(TW,"CRASH DUMP SIZE LIMIT REACHED"),
+	    io:format("  size limit information ok",[]),
             ok;
         ".unicode" ->
             #proc{pid=Pid0} =
@@ -500,16 +574,45 @@ do_create_dumps(DataDir,Rel) ->
 	current ->
 	    CD3 = dump_with_args(DataDir,Rel,"instr","+Mim true"),
 	    CD4 = dump_with_strange_module_name(DataDir,Rel,"strangemodname"),
-            Bytes = rand:uniform(300000) + 100,
-            CD5 = dump_with_args(DataDir,Rel,"trunc.bytes",
+	    Tmp = dump_with_args(DataDir,Rel,"trunc_bytes",""),
+            {ok,#file_info{size=Max}} = file:read_file_info(Tmp),
+            ok = file:delete(Tmp),
+            Bytes = max(15,rand:uniform(Max)),
+            CD5 = dump_with_args(DataDir,Rel,"trunc_bytes",
                                  "-env ERL_CRASH_DUMP_BYTES " ++
                                      integer_to_list(Bytes)),
             CD6 = dump_with_unicode_atoms(DataDir,Rel,"unicode"),
-	    {[CD1,CD2,CD3,CD4,CD5,CD6], DosDump};
+            TruncatedDumps = truncate_dump(CD1),
+	    {[CD1,CD2,CD3,CD4,CD5,CD6|TruncatedDumps], DosDump};
 	_ ->
 	    {[CD1,CD2], DosDump}
     end.
 
+truncate_dump(File) ->
+    {ok,Bin} = file:read_file(File),
+    BinTag = <<"\n=binary:">>,
+    Colon = <<":">>,
+    NewLine = case os:type() of
+                  {win32,_} -> <<"\r\n">>;
+                  _ -> <<"\n">>
+              end,
+    [StartBin,AfterTag] = binary:split(Bin,BinTag),
+    [AddrAndSize,BinaryAndRest] = binary:split(AfterTag,Colon),
+    [Binary,_Rest] = binary:split(BinaryAndRest,NewLine),
+    TruncSize = byte_size(Binary) - 2,
+    <<TruncBinary:TruncSize/binary,_/binary>> = Binary,
+    TruncName = filename:rootname(File) ++ ".trunc_bin",
+    write_trunc_files(TruncName,StartBin,
+                      [BinTag,AddrAndSize,Colon,TruncBinary],1).
+
+write_trunc_files(TruncName0,Bin,[Part|Parts],N) ->
+    TruncName = TruncName0++integer_to_list(N),
+    Bin1 = <<Bin/binary,Part/binary>>,
+    ok = file:write_file(TruncName,Bin1),
+    [TruncName|write_trunc_files(TruncName0,Bin1,Parts,N+1)];
+write_trunc_files(_,_,[],_) ->
+    [].
+
 
 %% Create a dump which has three visible nodes, one hidden and one
 %% not connected node, and with monitors and links between nodes.
diff --git a/lib/observer/test/observer_SUITE.erl b/lib/observer/test/observer_SUITE.erl
index 41726b1521..0db2c1ea77 100644
--- a/lib/observer/test/observer_SUITE.erl
+++ b/lib/observer/test/observer_SUITE.erl
@@ -115,6 +115,7 @@ basic(doc) -> [""];
 basic(Config) when is_list(Config) ->
     timer:send_after(100, "foobar"), %% Otherwise the timer server gets added to procs
     ProcsBefore = processes(),
+    ProcInfoBefore = [{P,process_info(P)} || P <- ProcsBefore],
     NumProcsBefore = length(ProcsBefore),
 
     ok = observer:start(),
@@ -145,8 +146,10 @@ basic(Config) when is_list(Config) ->
     ProcsAfter = processes(),
     NumProcsAfter = length(ProcsAfter),
     if NumProcsAfter=/=NumProcsBefore ->
+            BeforeNotAfter = ProcsBefore -- ProcsAfter,
 	    ct:log("Before but not after:~n~p~n",
-		   [[{P,process_info(P)} || P <- ProcsBefore -- ProcsAfter]]),
+		   [[{P,I} || {P,I} <- ProcInfoBefore,
+                              lists:member(P,BeforeNotAfter)]]),
 	    ct:log("After but not before:~n~p~n",
 		   [[{P,process_info(P)} || P <- ProcsAfter -- ProcsBefore]]),
 	    ct:fail("leaking processes");
@@ -304,10 +307,10 @@ table_win(Config) when is_list(Config) ->
 %% Test PR-1296/OTP-14151
 %% Clicking a link to a port before the port tab has been activated the
 %% first time crashes observer.
-port_win_when_tab_not_initiated(Config) ->
+port_win_when_tab_not_initiated(_Config) ->
     {ok,Port} = gen_tcp:listen(0,[]),
     ok = observer:start(),
-    Notebook = setup_whitebox_testing(),
+    _Notebook = setup_whitebox_testing(),
     observer ! {open_link,erlang:port_to_list(Port)},
     timer:sleep(1000),
     observer:stop(),
diff --git a/lib/observer/test/ttb_SUITE.erl b/lib/observer/test/ttb_SUITE.erl
index c06ec21f36..e8c2f9e37d 100644
--- a/lib/observer/test/ttb_SUITE.erl
+++ b/lib/observer/test/ttb_SUITE.erl
@@ -778,37 +778,37 @@ otp_4967_2(suite) ->
 otp_4967_2(doc) ->
     ["OTP-4967: Trace message sent to {Name, Node}"];
 otp_4967_2(Config) when is_list(Config) ->
-    io:format("1: ~p",[now()]),
+    io:format("1: ~p",[erlang:timestamp()]),
     ?line Privdir = priv_dir(Config),
-    io:format("2: ~p",[now()]),
+    io:format("2: ~p",[erlang:timestamp()]),
     ?line File = filename:join(Privdir,"otp_4967"),
-    io:format("3: ~p",[now()]),
+    io:format("3: ~p",[erlang:timestamp()]),
     ?line S = self(),
-    io:format("4: ~p",[now()]),
+    io:format("4: ~p",[erlang:timestamp()]),
     ?line {ok,[Node]} =
 	ttb:tracer(node(),[{file, File},
 			   {handler,{fun myhandler/4, S}}]),
 
-    io:format("5: ~p",[now()]),
+    io:format("5: ~p",[erlang:timestamp()]),
     %% Test that delayed registration of a process works.
     receive after 200 -> ok end,
     ?line register(otp_4967,self()),
-    io:format("6: ~p",[now()]),
+    io:format("6: ~p",[erlang:timestamp()]),
     ?line {ok,[{S,[{matched,Node,1}]}]} =  ttb:p(self(),s),
-    io:format("7: ~p",[now()]),
+    io:format("7: ~p",[erlang:timestamp()]),
     ?line {otp_4967,node()} ! heihopp,
-    io:format("8: ~p",[now()]),
+    io:format("8: ~p",[erlang:timestamp()]),
     ?line stopped = ttb:stop([format]),
-    io:format("9: ~p",[now()]),
+    io:format("9: ~p",[erlang:timestamp()]),
     ?line Msgs = flush(),
-    io:format("10: ~p",[now()]),
+    io:format("10: ~p",[erlang:timestamp()]),
     ?line io:format("Messages received: \n~p\n",[Msgs]),
-    io:format("11: ~p",[now()]),
+    io:format("11: ~p",[erlang:timestamp()]),
     ?line true = lists:member(heihopp,Msgs), % the heihopp message itself
-    io:format("13: ~p",[now()]),
+    io:format("13: ~p",[erlang:timestamp()]),
     ?line {value,{trace_ts,_,send,heihopp,{_,otp_4967,Node},{_,_,_}}} =
 	lists:keysearch(heihopp,4,Msgs), % trace trace of the heihopp message
-    io:format("14: ~p",[now()]),
+    io:format("14: ~p",[erlang:timestamp()]),
     ?line end_of_trace = lists:last(Msgs), % end of the trace
     ok.