aboutsummaryrefslogtreecommitdiffstats
path: root/lib
diff options
context:
space:
mode:
authorBjörn Gustavsson <[email protected]>2017-10-20 14:23:26 +0200
committerBjörn Gustavsson <[email protected]>2017-10-20 14:23:26 +0200
commit4b8255e9217c293f84a1e96b3ae8034d089e815b (patch)
treed65914ea3e8c939a5a5a1d97a1a8f9c3b07f0cdb /lib
parent3ea750a53874d61ef7d4d806656ca63c7759a8a4 (diff)
parent806c2de5ca3806ad8110531c99749063e5603ac0 (diff)
downloadotp-4b8255e9217c293f84a1e96b3ae8034d089e815b.tar.gz
otp-4b8255e9217c293f84a1e96b3ae8034d089e815b.tar.bz2
otp-4b8255e9217c293f84a1e96b3ae8034d089e815b.zip
Merge branch 'maint'
* maint: Bump version of crash dumps to 0.4 Verify that binaries of different sizes are dumped correctly Don't dump literal areas that are not referenced at all Dump literals separately to avoid incomplete heap data Implement dumping of maps in crash dumps Buffer writing of crash dumps Conflicts: erts/emulator/beam/erl_alloc.types
Diffstat (limited to 'lib')
-rw-r--r--lib/observer/src/crashdump_viewer.erl93
-rw-r--r--lib/observer/test/crashdump_helper.erl40
-rw-r--r--lib/observer/test/crashdump_viewer_SUITE.erl47
3 files changed, 172 insertions, 8 deletions
diff --git a/lib/observer/src/crashdump_viewer.erl b/lib/observer/src/crashdump_viewer.erl
index 51ff69fd69..7f6bb9bdcd 100644
--- a/lib/observer/src/crashdump_viewer.erl
+++ b/lib/observer/src/crashdump_viewer.erl
@@ -104,6 +104,7 @@
-define(index_table,index_table).
-define(instr_data,instr_data).
-define(internal_ets,internal_ets).
+-define(literals,literals).
-define(loaded_modules,loaded_modules).
-define(memory,memory).
-define(memory_map,memory_map).
@@ -785,6 +786,9 @@ parse_vsn_str(Str,WS) ->
%%% Progress is reported during the time and MUST be checked with
%%% crashdump_viewer:get_progress/0 until it returns {ok,done}.
do_read_file(File) ->
+ erase(?literals), %Clear literal cache.
+ put(truncated,false), %Not truncated (yet).
+ erase(truncated_reason), %Not truncated (yet).
case file:read_file_info(File) of
{ok,#file_info{type=regular,
access=FileA,
@@ -856,6 +860,19 @@ indexify(Fd,AddrAdj,Bin,N) ->
{?proc_heap,LastId} ->
[{_,LastPos}] = lookup_index(?proc_heap,LastId),
ets:insert(cdv_heap_file_chars,{LastId,N+Start+1-LastPos});
+ {?literals,[]} ->
+ case get(truncated_reason) of
+ undefined ->
+ [{_,LastPos}] = lookup_index(?literals,[]),
+ ets:insert(cdv_heap_file_chars,
+ {literals,N+Start+1-LastPos});
+ _ ->
+ %% Literals are truncated. Make sure we never
+ %% attempt to read in the literals. (Heaps that
+ %% references literals will show markers for
+ %% incomplete heaps, but will otherwise work.)
+ delete_index(?literals, [])
+ end;
_ -> ok
end,
indexify(Fd,AddrAdj,Rest,N1);
@@ -908,6 +925,7 @@ check_if_truncated() ->
find_truncated_proc({Tag,_Id}) when Tag==?atoms;
Tag==?binary;
Tag==?instr_data;
+ Tag==?literals;
Tag==?memory_status;
Tag==?memory_map ->
put(truncated_proc,false);
@@ -1386,13 +1404,33 @@ maybe_other_node2(Channel) ->
expand_memory(Fd,Pid,DumpVsn) ->
BinAddrAdj = get_bin_addr_adj(DumpVsn),
put(fd,Fd),
- Dict = read_heap(Fd,Pid,BinAddrAdj,gb_trees:empty()),
+ Dict0 = case get(?literals) of
+ undefined ->
+ Literals = read_literals(Fd),
+ put(?literals,Literals),
+ put(fd,Fd),
+ Literals;
+ Literals ->
+ Literals
+ end,
+ Dict = read_heap(Fd,Pid,BinAddrAdj,Dict0),
Expanded = {read_stack_dump(Fd,Pid,BinAddrAdj,Dict),
read_messages(Fd,Pid,BinAddrAdj,Dict),
read_dictionary(Fd,Pid,BinAddrAdj,Dict)},
erase(fd),
Expanded.
+read_literals(Fd) ->
+ case lookup_index(?literals,[]) of
+ [{_,Start}] ->
+ [{_,Chars}] = ets:lookup(cdv_heap_file_chars,literals),
+ init_progress("Reading literals",Chars),
+ pos_bof(Fd,Start),
+ read_heap(0,gb_trees:empty());
+ [] ->
+ gb_trees:empty()
+ end.
+
%%%-----------------------------------------------------------------
%%% This is a workaround for a bug in dump versions prior to 0.3:
%%% Addresses were truncated to 32 bits. This could cause binaries to
@@ -2591,8 +2629,26 @@ parse_heap_term("Ys"++Line0, Addr, BinAddrAdj, D0) -> %Sub binary.
end
end,
D = gb_trees:insert(Addr, Term, D0),
- {Term,Line,D}.
-
+ {Term,Line,D};
+parse_heap_term("Mf"++Line0, Addr, BinAddrAdj, D0) -> %Flatmap.
+ {Size,":"++Line1} = get_hex(Line0),
+ {Keys,":"++Line2,D1} = parse_term(Line1, BinAddrAdj, D0),
+ {Values,Line,D2} = parse_tuple(Size, Line2, Addr,BinAddrAdj, D1, []),
+ Pairs = zip_tuples(tuple_size(Keys), Keys, Values, []),
+ Map = maps:from_list(Pairs),
+ D = gb_trees:update(Addr, Map, D2),
+ {Map,Line,D};
+parse_heap_term("Mh"++Line0, Addr, BinAddrAdj, D0) -> %Head node in a hashmap.
+ {MapSize,":"++Line1} = get_hex(Line0),
+ {N,":"++Line2} = get_hex(Line1),
+ {Nodes,Line,D1} = parse_tuple(N, Line2, Addr, BinAddrAdj, D0, []),
+ Map = maps:from_list(flatten_hashmap_nodes(Nodes)),
+ MapSize = maps:size(Map), %Assertion.
+ D = gb_trees:update(Addr, Map, D1),
+ {Map,Line,D};
+parse_heap_term("Mn"++Line0, Addr, BinAddrAdj, D) -> %Interior node in a hashmap.
+ {N,":"++Line} = get_hex(Line0),
+ parse_tuple(N, Line, Addr, BinAddrAdj, D, []).
parse_tuple(0, Line, Addr, _, D0, Acc) ->
Tuple = list_to_tuple(lists:reverse(Acc)),
@@ -2606,6 +2662,25 @@ parse_tuple(N, Line0, Addr, BinAddrAdj, D0, Acc) ->
parse_tuple(N-1, Line, Addr, BinAddrAdj, D, [Term|Acc])
end.
+zip_tuples(0, _T1, _T2, Acc) ->
+ Acc;
+zip_tuples(N, T1, T2, Acc) when N =< tuple_size(T1) ->
+ zip_tuples(N-1, T1, T2, [{element(N, T1),element(N, T2)}|Acc]).
+
+flatten_hashmap_nodes(Tuple) ->
+ flatten_hashmap_nodes_1(tuple_size(Tuple), Tuple, []).
+
+flatten_hashmap_nodes_1(0, _Tuple, Acc) ->
+ Acc;
+flatten_hashmap_nodes_1(N, Tuple0, Acc0) ->
+ case element(N, Tuple0) of
+ [K|V] ->
+ flatten_hashmap_nodes_1(N-1, Tuple0, [{K,V}|Acc0]);
+ Tuple when is_tuple(Tuple) ->
+ Acc = flatten_hashmap_nodes_1(N-1, Tuple0, Acc0),
+ flatten_hashmap_nodes_1(tuple_size(Tuple), Tuple, Acc)
+ end.
+
parse_term([$H|Line0], BinAddrAdj, D) -> %Pointer to heap term.
{Ptr,Line} = get_hex(Line0),
deref_ptr(Ptr, Line, BinAddrAdj, D);
@@ -2794,6 +2869,10 @@ reset_tables() ->
insert_index(Tag,Id,Pos) ->
ets:insert(cdv_dump_index_table,{{Tag,Pos},Id}).
+delete_index(Tag,Id) ->
+ Ms = [{{{Tag,'$1'},Id},[],[true]}],
+ ets:select_delete(cdv_dump_index_table, Ms).
+
lookup_index({Tag,Id}) ->
lookup_index(Tag,Id);
lookup_index(Tag) ->
@@ -2810,6 +2889,7 @@ insert_binary_index(Addr,Pos) ->
lookup_binary_index(Addr) ->
ets:lookup(cdv_binary_index_table,Addr).
+
%%-----------------------------------------------------------------
%% Convert tags read from crashdump to atoms used as first part of key
%% in cdv_dump_index_table
@@ -2827,6 +2907,7 @@ tag_to_atom("hidden_node") -> ?hidden_node;
tag_to_atom("index_table") -> ?index_table;
tag_to_atom("instr_data") -> ?instr_data;
tag_to_atom("internal_ets") -> ?internal_ets;
+tag_to_atom("literals") -> ?literals;
tag_to_atom("loaded_modules") -> ?loaded_modules;
tag_to_atom("memory") -> ?memory;
tag_to_atom("mod") -> ?mod;
@@ -2850,8 +2931,10 @@ tag_to_atom(UnknownTag) ->
%%%-----------------------------------------------------------------
%%% Store last tag for use when truncated, and reason if aborted
put_last_tag(?abort,Reason) ->
- %% Don't overwrite the real last tag
- put(truncated_reason,Reason);
+ %% Don't overwrite the real last tag, and make sure to return
+ %% the previous last tag.
+ put(truncated_reason,Reason),
+ get(last_tag);
put_last_tag(Tag,Id) ->
put(last_tag,{Tag,Id}).
diff --git a/lib/observer/test/crashdump_helper.erl b/lib/observer/test/crashdump_helper.erl
index f37d9057cb..41041682c2 100644
--- a/lib/observer/test/crashdump_helper.erl
+++ b/lib/observer/test/crashdump_helper.erl
@@ -19,7 +19,9 @@
%%
-module(crashdump_helper).
--export([n1_proc/2,remote_proc/2]).
+-export([n1_proc/2,remote_proc/2,
+ dump_maps/0,create_maps/0,
+ create_binaries/0]).
-compile(r18).
-include_lib("common_test/include/ct.hrl").
@@ -60,6 +62,7 @@ n1_proc(Creator,_N2,Pid2,Port2,_L) ->
put(ref,Ref),
put(pid,Pid),
put(bin,Bin),
+ put(bins,create_binaries()),
put(sub_bin,SubBin),
put(bignum,83974938738373873),
put(neg_bignum,-38748762783736367),
@@ -92,3 +95,38 @@ remote_proc(P1,Creator) ->
Creator ! {self(),done},
receive after infinity -> ok end
end).
+
+create_binaries() ->
+ Sizes = lists:seq(60, 70) ++ lists:seq(120, 140),
+ [begin
+ <<H:16/unit:8>> = erlang:md5(<<Size:32>>),
+ Data = ((H bsl (8*150)) div (H+7919)),
+ <<Data:Size/unit:8>>
+ end || Size <- Sizes].
+
+%%%
+%%% Test dumping of maps. Dumping of maps only from OTP 20.2.
+%%%
+
+dump_maps() ->
+ Parent = self(),
+ F = fun() ->
+ register(aaaaaaaa_maps, self()),
+ put(maps, create_maps()),
+ Parent ! {self(),done},
+ receive _ -> ok end
+ end,
+ Pid = spawn_link(F),
+ receive
+ {Pid,done} ->
+ {ok,Pid}
+ end.
+
+create_maps() ->
+ Map0 = maps:from_list([{I,[I,I+1]} || I <- lists:seq(1, 40)]),
+ Map1 = maps:from_list([{I,{a,[I,I*I],{}}} || I <- lists:seq(1, 100)]),
+ Map2 = maps:from_list([{{I},(I*I) bsl 24} || I <- lists:seq(1, 10000)]),
+ Map3 = lists:foldl(fun(I, A) ->
+ A#{I=>I*I}
+ end, Map2, lists:seq(-10, 0)),
+ #{a=>Map0,b=>Map1,c=>Map2,d=>Map3,e=>#{}}.
diff --git a/lib/observer/test/crashdump_viewer_SUITE.erl b/lib/observer/test/crashdump_viewer_SUITE.erl
index f9ac884743..86a60e15f4 100644
--- a/lib/observer/test/crashdump_viewer_SUITE.erl
+++ b/lib/observer/test/crashdump_viewer_SUITE.erl
@@ -364,6 +364,10 @@ special(File,Procs) ->
crashdump_viewer:expand_binary({SOffset,SSize,SPos}),
io:format(" expand binary ok",[]),
+ Binaries = crashdump_helper:create_binaries(),
+ verify_binaries(Binaries, proplists:get_value(bins,Dict)),
+ io:format(" binaries ok",[]),
+
#proc{last_calls=LastCalls} = ProcDetails,
true = length(LastCalls) =< 4,
@@ -513,11 +517,36 @@ special(File,Procs) ->
io:format(" unicode table name ok",[]),
ok;
+ ".maps" ->
+ %% I registered a process as aaaaaaaa_maps in the map dump
+ %% to make sure it will be the first in the list when sorted
+ %% on names.
+ [#proc{pid=Pid0,name=Name}|_Rest] = lists:keysort(#proc.name,Procs),
+ "aaaaaaaa_maps" = Name,
+ Pid = pid_to_list(Pid0),
+ {ok,ProcDetails=#proc{},[]} = crashdump_viewer:proc_details(Pid),
+ io:format(" process details ok",[]),
+
+ #proc{dict=Dict} = ProcDetails,
+ io:format("~p\n", [Dict]),
+ Maps = crashdump_helper:create_maps(),
+ Maps = proplists:get_value(maps,Dict),
+ io:format(" maps ok",[]),
+ ok;
_ ->
ok
end,
ok.
+verify_binaries([H|T1], [H|T2]) ->
+ %% Heap binary.
+ verify_binaries(T1, T2);
+verify_binaries([Bin|T1], [['#CDVBin',Offset,Size,Pos]|T2]) ->
+ %% Refc binary.
+ {ok,<<Bin:Size/binary>>} = crashdump_viewer:expand_binary({Offset,Size,Pos}),
+ verify_binaries(T1, T2);
+verify_binaries([], []) ->
+ ok.
lookat_all_pids([]) ->
ok;
@@ -582,8 +611,9 @@ do_create_dumps(DataDir,Rel) ->
"-env ERL_CRASH_DUMP_BYTES " ++
integer_to_list(Bytes)),
CD6 = dump_with_unicode_atoms(DataDir,Rel,"unicode"),
+ CD7 = dump_with_maps(DataDir,Rel,"maps"),
TruncatedDumps = truncate_dump(CD1),
- {[CD1,CD2,CD3,CD4,CD5,CD6|TruncatedDumps], DosDump};
+ {[CD1,CD2,CD3,CD4,CD5,CD6,CD7|TruncatedDumps], DosDump};
_ ->
{[CD1,CD2], DosDump}
end.
@@ -596,7 +626,10 @@ truncate_dump(File) ->
{win32,_} -> <<"\r\n">>;
_ -> <<"\n">>
end,
- [StartBin,AfterTag] = binary:split(Bin,BinTag),
+ %% Split after "our binary" created by crashdump_helper
+ %% (it may not be the first binary).
+ RE = <<"\n=binary:(?=[0-9A-Z]+",NewLine/binary,"FF:010203)">>,
+ [StartBin,AfterTag] = re:split(Bin,RE,[{parts,2}]),
[AddrAndSize,BinaryAndRest] = binary:split(AfterTag,Colon),
[Binary,_Rest] = binary:split(BinaryAndRest,NewLine),
TruncSize = byte_size(Binary) - 2,
@@ -699,6 +732,16 @@ dump_with_unicode_atoms(DataDir,Rel,DumpName) ->
?t:stop_node(n1),
CD.
+dump_with_maps(DataDir,Rel,DumpName) ->
+ Opt = rel_opt(Rel),
+ Pz = "-pz \"" ++ filename:dirname(code:which(?MODULE)) ++ "\"",
+ PzOpt = [{args,Pz}],
+ {ok,N1} = ?t:start_node(n1,peer,Opt ++ PzOpt),
+ {ok,_Pid} = rpc:call(N1,crashdump_helper,dump_maps,[]),
+ CD = dump(N1,DataDir,Rel,DumpName),
+ ?t:stop_node(n1),
+ CD.
+
dump(Node,DataDir,Rel,DumpName) ->
Crashdump = filename:join(DataDir, dump_prefix(Rel)++DumpName),
rpc:call(Node,os,putenv,["ERL_CRASH_DUMP",Crashdump]),