Merge branch 'bjorn/erts/persistent_terms/OTP-14669' into maint

* bjorn/erts/persistent_terms/OTP-14669:
  Implement a tab for persistent terms in crashdump viewer
  Add tests of persistent terms for crashdump_viewer
  Add a persistent term storage
  Refactor releasing of literals
  Extend the sharing-preserving routines to optionally copy literals

10 files changed, 206 insertions, 17 deletions

diff --git a/lib/observer/src/Makefile b/lib/observer/src/Makefile
index ff2bcbdb99..f9f239db37 100644
--- a/lib/observer/src/Makefile
+++ b/lib/observer/src/Makefile
@@ -50,6 +50,7 @@ MODULES= \
 	cdv_mem_cb \
 	cdv_mod_cb \
 	cdv_multi_wx \
+	cdv_persistent_cb \
 	cdv_port_cb \
 	cdv_proc_cb \
 	cdv_sched_cb \
diff --git a/lib/observer/src/cdv_html_wx.erl b/lib/observer/src/cdv_html_wx.erl
index d9efa7fc2f..ffef83227c 100644
--- a/lib/observer/src/cdv_html_wx.erl
+++ b/lib/observer/src/cdv_html_wx.erl
@@ -33,13 +33,17 @@
 	{panel,
 	 app,         %% which tool is the user
 	 expand_table,
-	 expand_wins=[]}).
+         expand_wins=[],
+         delayed_fetch,
+         trunc_warn=[]}).
 
 start_link(ParentWin, Info) ->
     wx_object:start_link(?MODULE, [ParentWin, Info], []).
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
+init([ParentWin, Callback]) when is_atom(Callback) ->
+    init(ParentWin, Callback);
 init([ParentWin, {App, Fun}]) when is_function(Fun) ->
     init([ParentWin, {App, Fun()}]);
 init([ParentWin, {expand,HtmlText,Tab}]) ->
@@ -60,9 +64,29 @@ init(ParentWin, HtmlText, Tab, App) ->
     wx_misc:endBusyCursor(),
     {HtmlWin, #state{panel=HtmlWin,expand_table=Tab,app=App}}.
 
+init(ParentWin, Callback) ->
+    {HtmlWin, State} = init(ParentWin, "", undefined, cdv),
+    {HtmlWin, State#state{delayed_fetch=Callback}}.
+
 %%%%%%%%%%%%%%%%%%%%%%% Callbacks %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
+handle_info(active, #state{panel=HtmlWin,delayed_fetch=Callback}=State)
+  when Callback=/=undefined ->
+    observer_lib:display_progress_dialog(HtmlWin,
+                                         "Crashdump Viewer",
+                                         "Reading data"),
+    {{expand,HtmlText,Tab},TW} = Callback:get_info(),
+    observer_lib:sync_destroy_progress_dialog(),
+    wx_misc:beginBusyCursor(),
+    wxHtmlWindow:setPage(HtmlWin,HtmlText),
+    cdv_wx:set_status(TW),
+    wx_misc:endBusyCursor(),
+    {noreply, State#state{expand_table=Tab,
+                          delayed_fetch=undefined,
+                          trunc_warn=TW}};
+
 handle_info(active, State) ->
+    cdv_wx:set_status(State#state.trunc_warn),
     {noreply, State};
 
 handle_info(Info, State) ->
diff --git a/lib/observer/src/cdv_persistent_cb.erl b/lib/observer/src/cdv_persistent_cb.erl
new file mode 100644
index 0000000000..d5da18f7fc
--- /dev/null
+++ b/lib/observer/src/cdv_persistent_cb.erl
@@ -0,0 +1,32 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2018. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+
+-module(cdv_persistent_cb).
+
+-export([get_info/0]).
+
+-include_lib("wx/include/wx.hrl").
+
+get_info() ->
+    Tab = ets:new(pt_expand,[set,public]),
+    {ok,PT,TW} = crashdump_viewer:persistent_terms(),
+    {{expand,
+      observer_html_lib:expandable_term("Persistent Terms",PT,Tab),
+      Tab},
+     TW}.
diff --git a/lib/observer/src/cdv_wx.erl b/lib/observer/src/cdv_wx.erl
index 78a897111c..1e9cef8952 100644
--- a/lib/observer/src/cdv_wx.erl
+++ b/lib/observer/src/cdv_wx.erl
@@ -51,6 +51,7 @@
 -define(DIST_STR,  "Nodes").
 -define(MOD_STR,   "Modules").
 -define(MEM_STR,   "Memory").
+-define(PERSISTENT_STR, "Persistent Terms").
 -define(INT_STR,   "Internal Tables").
 
 %% Records
@@ -74,6 +75,7 @@
 	 dist_panel,
 	 mod_panel,
 	 mem_panel,
+         persistent_panel,
 	 int_panel,
 	 active_tab
 	}).
@@ -193,6 +195,10 @@ setup(#state{frame=Frame, notebook=Notebook}=State) ->
     %% Memory Panel
     MemPanel = add_page(Notebook, ?MEM_STR, cdv_multi_wx, cdv_mem_cb),
 
+    %% Persistent Terms Panel
+    PersistentPanel = add_page(Notebook, ?PERSISTENT_STR,
+                               cdv_html_wx, cdv_persistent_cb),
+
     %% Memory Panel
     IntPanel = add_page(Notebook, ?INT_STR, cdv_multi_wx, cdv_int_tab_cb),
 
@@ -215,6 +221,7 @@ setup(#state{frame=Frame, notebook=Notebook}=State) ->
 			dist_panel = DistPanel,
 			mod_panel = ModPanel,
 			mem_panel = MemPanel,
+                        persistent_panel = PersistentPanel,
 			int_panel = IntPanel,
 			active_tab = GenPid
 		       }}.
@@ -250,6 +257,7 @@ handle_event(#wx{id = ?wxID_OPEN,
 			  State#state.dist_panel,
 			  State#state.mod_panel,
 			  State#state.mem_panel,
+			  State#state.persistent_panel,
 			  State#state.int_panel],
 		_ = [wx_object:call(Panel,new_dump) || Panel<-Panels],
 		wxNotebook:setSelection(State#state.notebook,0),
@@ -343,8 +351,8 @@ check_page_title(Notebook) ->
 get_active_pid(#state{notebook=Notebook, gen_panel=Gen, pro_panel=Pro,
 		      port_panel=Ports, ets_panel=Ets, timer_panel=Timers,
 		      fun_panel=Funs, atom_panel=Atoms, dist_panel=Dist,
-		      mod_panel=Mods, mem_panel=Mem, int_panel=Int,
-		      sched_panel=Sched
+		      mod_panel=Mods, mem_panel=Mem, persistent_panel=Persistent,
+                      int_panel=Int, sched_panel=Sched
 		     }) ->
     Panel = case check_page_title(Notebook) of
 		?GEN_STR -> Gen;
@@ -358,6 +366,7 @@ get_active_pid(#state{notebook=Notebook, gen_panel=Gen, pro_panel=Pro,
 		?DIST_STR -> Dist;
 		?MOD_STR -> Mods;
 		?MEM_STR -> Mem;
+		?PERSISTENT_STR -> Persistent;
 		?INT_STR -> Int
 	    end,
     wx_object:get_pid(Panel).
@@ -365,7 +374,7 @@ get_active_pid(#state{notebook=Notebook, gen_panel=Gen, pro_panel=Pro,
 pid2panel(Pid, #state{gen_panel=Gen, pro_panel=Pro, port_panel=Ports,
 		      ets_panel=Ets, timer_panel=Timers, fun_panel=Funs,
 		      atom_panel=Atoms, dist_panel=Dist, mod_panel=Mods,
-		      mem_panel=Mem, int_panel=Int}) ->
+		      mem_panel=Mem, persistent_panel=Persistent, int_panel=Int}) ->
     case Pid of
 	Gen -> ?GEN_STR;
 	Pro -> ?PRO_STR;
@@ -377,6 +386,7 @@ pid2panel(Pid, #state{gen_panel=Gen, pro_panel=Pro, port_panel=Ports,
 	Dist -> ?DIST_STR;
 	Mods -> ?MOD_STR;
 	Mem -> ?MEM_STR;
+        ?PERSISTENT_STR -> Persistent;
 	Int -> ?INT_STR;
 	_ -> "unknown"
     end.
diff --git a/lib/observer/src/crashdump_viewer.erl b/lib/observer/src/crashdump_viewer.erl
index 276b209ce5..0627c15b1c 100644
--- a/lib/observer/src/crashdump_viewer.erl
+++ b/lib/observer/src/crashdump_viewer.erl
@@ -74,6 +74,7 @@
 	 loaded_modules/0,
 	 loaded_mod_details/1,
 	 memory/0,
+         persistent_terms/0,
 	 allocated_areas/0,
 	 allocator_info/0,
 	 hash_tables/0,
@@ -139,6 +140,7 @@
 -define(node,node).
 -define(not_connected,not_connected).
 -define(old_instr_data,old_instr_data).
+-define(persistent_terms,persistent_terms).
 -define(port,port).
 -define(proc,proc).
 -define(proc_dictionary,proc_dictionary).
@@ -293,6 +295,8 @@ loaded_mod_details(Mod) ->
     call({loaded_mod_details,Mod}).
 memory() ->
     call(memory).
+persistent_terms() ->
+    call(persistent_terms).
 allocated_areas() ->
     call(allocated_areas).
 allocator_info() ->
@@ -471,6 +475,11 @@ handle_call(memory,_From,State=#state{file=File}) ->
     Memory=memory(File),
     TW = truncated_warning([?memory]),
     {reply,{ok,Memory,TW},State};
+handle_call(persistent_terms,_From,State=#state{file=File,dump_vsn=DumpVsn}) ->
+    TW = truncated_warning([?persistent_terms,?literals]),
+    DecodeOpts = get_decode_opts(DumpVsn),
+    Terms = persistent_terms(File, DecodeOpts),
+    {reply,{ok,Terms,TW},State};
 handle_call(allocated_areas,_From,State=#state{file=File}) ->
     AllocatedAreas=allocated_areas(File),
     TW = truncated_warning([?allocated_areas]),
@@ -1444,15 +1453,7 @@ maybe_other_node2(Channel) ->
 expand_memory(Fd,Pid,DumpVsn) ->
     DecodeOpts = get_decode_opts(DumpVsn),
     put(fd,Fd),
-    Dict0 = case get(?literals) of
-                undefined ->
-                    Literals = read_literals(Fd,DecodeOpts),
-                    put(?literals,Literals),
-                    put(fd,Fd),
-                    Literals;
-                Literals ->
-                    Literals
-            end,
+    Dict0 = get_literals(Fd,DecodeOpts),
     Dict = read_heap(Fd,Pid,DecodeOpts,Dict0),
     Expanded = {read_stack_dump(Fd,Pid,DecodeOpts,Dict),
 		read_messages(Fd,Pid,DecodeOpts,Dict),
@@ -1468,6 +1469,18 @@ expand_memory(Fd,Pid,DumpVsn) ->
         end,
     {Expanded,IncompleteWarning}.
 
+get_literals(Fd,DecodeOpts) ->
+    case get(?literals) of
+        undefined ->
+            OldFd = put(fd,Fd),
+            Literals = read_literals(Fd,DecodeOpts),
+            put(fd,OldFd),
+            put(?literals,Literals),
+            Literals;
+        Literals ->
+            Literals
+    end.
+
 read_literals(Fd,DecodeOpts) ->
     case lookup_index(?literals,[]) of
 	[{_,Start}] ->
@@ -2203,6 +2216,56 @@ get_atom(Atom) when is_binary(Atom) ->
     {Atom,nq}. % not quoted
 
 %%-----------------------------------------------------------------
+%% Page with list of all persistent terms
+persistent_terms(File, DecodeOpts) ->
+    case lookup_index(?persistent_terms) of
+	[{_Id,Start}] ->
+	    Fd = open(File),
+	    pos_bof(Fd,Start),
+	    Terms = get_persistent_terms(Fd),
+            Dict = get_literals(Fd,DecodeOpts),
+            parse_persistent_terms(Terms,DecodeOpts,Dict);
+	_ ->
+	    []
+    end.
+
+parse_persistent_terms([[Name0,Val0]|Terms],DecodeOpts,Dict) ->
+    {Name,_,_} = parse_term(binary_to_list(Name0),DecodeOpts,Dict),
+    {Val,_,_} = parse_term(binary_to_list(Val0),DecodeOpts,Dict),
+    [{Name,Val}|parse_persistent_terms(Terms,DecodeOpts,Dict)];
+parse_persistent_terms([],_,_) -> [].
+
+get_persistent_terms(Fd) ->
+    case get_chunk(Fd) of
+	{ok,Bin} ->
+	    get_persistent_terms(Fd,Bin,[]);
+	eof ->
+	    []
+    end.
+
+
+%% Persistent_Terms are written one per line in the crash dump.
+get_persistent_terms(Fd,Bin,PersistentTerms) ->
+    Bins = binary:split(Bin,<<"\n">>,[global]),
+    get_persistent_terms1(Fd,Bins,PersistentTerms).
+
+get_persistent_terms1(_Fd,[<<"=",_/binary>>|_],PersistentTerms) ->
+    PersistentTerms;
+get_persistent_terms1(Fd,[LastBin],PersistentTerms) ->
+    case get_chunk(Fd) of
+	{ok,Bin0} ->
+	    get_persistent_terms(Fd,<<LastBin/binary,Bin0/binary>>,PersistentTerms);
+	eof ->
+	    [get_persistent_term(LastBin)|PersistentTerms]
+    end;
+get_persistent_terms1(Fd,[Bin|Bins],Persistent_Terms) ->
+    get_persistent_terms1(Fd,Bins,[get_persistent_term(Bin)|Persistent_Terms]).
+
+get_persistent_term(Bin) ->
+    binary:split(Bin,<<"|">>).
+
+
+%%-----------------------------------------------------------------
 %% Page with memory information
 memory(File) ->
     case lookup_index(?memory) of
@@ -3192,6 +3255,7 @@ tag_to_atom("literals") -> ?literals;
 tag_to_atom("loaded_modules") -> ?loaded_modules;
 tag_to_atom("memory") -> ?memory;
 tag_to_atom("mod") -> ?mod;
+tag_to_atom("persistent_terms") -> ?persistent_terms;
 tag_to_atom("no_distribution") -> ?no_distribution;
 tag_to_atom("node") -> ?node;
 tag_to_atom("not_connected") -> ?not_connected;
diff --git a/lib/observer/src/observer.app.src b/lib/observer/src/observer.app.src
index d73293a5f9..d48b846ad2 100644
--- a/lib/observer/src/observer.app.src
+++ b/lib/observer/src/observer.app.src
@@ -34,6 +34,7 @@
 	       cdv_mem_cb,
 	       cdv_mod_cb,
 	       cdv_multi_wx,
+               cdv_persistent_cb,
 	       cdv_port_cb,
 	       cdv_proc_cb,
 	       cdv_table_wx,
diff --git a/lib/observer/src/observer_html_lib.erl b/lib/observer/src/observer_html_lib.erl
index 0c4e32af49..c67fa28c6d 100644
--- a/lib/observer/src/observer_html_lib.erl
+++ b/lib/observer/src/observer_html_lib.erl
@@ -62,7 +62,8 @@ expandable_term_body(Heading,[],_Tab) ->
 	 "Dictionary" -> "No dictionary was found";
 	 "ProcState"  -> "Information could not be retrieved,"
 			     " system messages may not be handled by this process.";
-         "SaslLog"    -> "No log entry was found"
+         "SaslLog"    -> "No log entry was found";
+         "Persistent Terms" -> "No persistent terms were found"
      end];
 expandable_term_body(Heading,Expanded,Tab) ->
     Attr = "BORDER=0 CELLPADDING=0 CELLSPACING=1 WIDTH=100%",
diff --git a/lib/observer/test/crashdump_helper.erl b/lib/observer/test/crashdump_helper.erl
index d8f4e046ae..576d112154 100644
--- a/lib/observer/test/crashdump_helper.erl
+++ b/lib/observer/test/crashdump_helper.erl
@@ -21,7 +21,9 @@
 -module(crashdump_helper).
 -export([n1_proc/2,remote_proc/2,
          dump_maps/0,create_maps/0,
-         create_binaries/0,create_sub_binaries/1]).
+         create_binaries/0,create_sub_binaries/1,
+         dump_persistent_terms/0,
+         create_persistent_terms/0]).
 -compile(r18).
 -include_lib("common_test/include/ct.hrl").
 
@@ -162,3 +164,27 @@ literal_map() ->
     %% Adress1, then the map at Address2.
 
     #{"one"=>1,"two"=>2,"three"=>3,"four"=>4}.
+
+%%%
+%%% Test dumping of persistent terms (from OTP 21.2).
+%%%
+
+dump_persistent_terms() ->
+    Parent = self(),
+    F = fun() ->
+                register(aaaaaaaa_persistent_terms, self()),
+                put(pts, create_persistent_terms()),
+                Parent ! {self(),done},
+                receive _ -> ok end
+        end,
+    Pid = spawn_link(F),
+    receive
+        {Pid,done} ->
+            {ok,Pid}
+    end.
+
+create_persistent_terms() ->
+    persistent_term:put({?MODULE,first}, {pid,42.0}),
+    persistent_term:put({?MODULE,second}, [1,2,3]),
+    persistent_term:get().
+
diff --git a/lib/observer/test/crashdump_viewer_SUITE.erl b/lib/observer/test/crashdump_viewer_SUITE.erl
index 864454cdff..ed53c46a0d 100644
--- a/lib/observer/test/crashdump_viewer_SUITE.erl
+++ b/lib/observer/test/crashdump_viewer_SUITE.erl
@@ -345,6 +345,7 @@ browse_file(File) ->
     {ok,_AllocINfo,_AllocInfoTW} = crashdump_viewer:allocator_info(),
     {ok,_HashTabs,_HashTabsTW} = crashdump_viewer:hash_tables(),
     {ok,_IndexTabs,_IndexTabsTW} = crashdump_viewer:index_tables(),
+    {ok,_PTs,_PTsTW} = crashdump_viewer:persistent_terms(),
 
     io:format("  info read",[]),
 
@@ -595,6 +596,23 @@ special(File,Procs) ->
             Maps = proplists:get_value(maps,Dict),
             io:format("  maps ok",[]),
             ok;
+        ".persistent_terms" ->
+	    %% I registered a process as aaaaaaaa_persistent_term in
+	    %% the dump to make sure it will be the first in the list
+	    %% when sorted on names.
+	    [#proc{pid=Pid0,name=Name}|_Rest] = lists:keysort(#proc.name,Procs),
+            "aaaaaaaa_persistent_terms" = Name,
+	    Pid = pid_to_list(Pid0),
+	    {ok,ProcDetails=#proc{},[]} = crashdump_viewer:proc_details(Pid),
+	    io:format("  process details ok",[]),
+
+	    #proc{dict=Dict} = ProcDetails,
+            %% io:format("~p\n", [Dict]),
+            Pts1 = crashdump_helper:create_persistent_terms(),
+            Pts2 = proplists:get_value(pts,Dict),
+            true = lists:sort(Pts1) =:= lists:sort(Pts2),
+            io:format("  persistent terms ok",[]),
+            ok;
 	_ ->
 	    ok
     end,
@@ -679,9 +697,11 @@ do_create_dumps(DataDir,Rel) ->
             CD5 = dump_with_size_limit_reached(DataDir,Rel,"trunc_bytes"),
             CD6 = dump_with_unicode_atoms(DataDir,Rel,"unicode"),
             CD7 = dump_with_maps(DataDir,Rel,"maps"),
+            CD8 = dump_with_persistent_terms(DataDir,Rel,"persistent_terms"),
             TruncDumpMod = truncate_dump_mod(CD1),
             TruncatedDumpsBinary = truncate_dump_binary(CD1),
-	    {[CD1,CD2,CD3,CD4,CD5,CD6,CD7,TruncDumpMod|TruncatedDumpsBinary],
+	    {[CD1,CD2,CD3,CD4,CD5,CD6,CD7,CD8,
+              TruncDumpMod|TruncatedDumpsBinary],
              DosDump};
 	_ ->
 	    {[CD1,CD2], DosDump}
@@ -850,6 +870,16 @@ dump_with_maps(DataDir,Rel,DumpName) ->
     ?t:stop_node(n1),
     CD.
 
+dump_with_persistent_terms(DataDir,Rel,DumpName) ->
+    Opt = rel_opt(Rel),
+    Pz = "-pz \"" ++ filename:dirname(code:which(?MODULE)) ++ "\"",
+    PzOpt = [{args,Pz}],
+    {ok,N1} = ?t:start_node(n1,peer,Opt ++ PzOpt),
+    {ok,_Pid} = rpc:call(N1,crashdump_helper,dump_persistent_terms,[]),
+    CD = dump(N1,DataDir,Rel,DumpName),
+    ?t:stop_node(n1),
+    CD.
+
 dump(Node,DataDir,Rel,DumpName) ->
     Crashdump = filename:join(DataDir, dump_prefix(Rel)++DumpName),
     rpc:call(Node,os,putenv,["ERL_CRASH_DUMP",Crashdump]),
diff --git a/lib/sasl/src/systools_make.erl b/lib/sasl/src/systools_make.erl
index 6916107623..983df58ef7 100644
--- a/lib/sasl/src/systools_make.erl
+++ b/lib/sasl/src/systools_make.erl
@@ -1565,7 +1565,7 @@ preloaded() ->
     [erl_prim_loader,erl_tracer,erlang,
      erts_code_purger,erts_dirty_process_signal_handler,
      erts_internal,erts_literal_area_collector,
-     init,otp_ring0,prim_buffer,prim_eval,prim_file,
+     init,otp_ring0,persistent_term,prim_buffer,prim_eval,prim_file,
      prim_inet,prim_zip,zlib].
 
 %%______________________________________________________________________