Merge branch 'sverker/ets-table-identifiers/OTP-14094'

* sverker/ets-table-identifiers:
  observer: Polish crashdump viewer for ETS
  observer: Polish Table Viewer tab
  stdlib: Remove ets_SUITE:memory_check_summary
  erts: Improve reduction count during table cleanup
  erts: Cleanup table status bits
  erts: Remove now redundant 'id' from DbTableCommon
  erts: Remove meta_main_tab
  erts: Pass tid argument down to trapping functions
  erts: Print table id as ref in crashdump and break menu
  erts: Replace meta_pid_to{_fixed}_tab with linked lists
  erts: Correct erl_rbtree comments about yielding
  erts: Add ERTS_RBT_YIELD_STAT_INIT to erl_rbtree
  Fix node_container_SUITE
  list_to_ref/1
  Implement ets:all() using scheduler specific data
  Rename fixation count in ets table to avoid confusion
  Introduce references as table identifiers

6 files changed, 111 insertions, 77 deletions

diff --git a/lib/observer/src/cdv_ets_cb.erl b/lib/observer/src/cdv_ets_cb.erl
index ddd2d42df6..18f0c86fd3 100644
--- a/lib/observer/src/cdv_ets_cb.erl
+++ b/lib/observer/src/cdv_ets_cb.erl
@@ -30,26 +30,23 @@
 -include("crashdump_viewer.hrl").
 
 %% Defines
--define(COL_ID,    0).
--define(COL_NAME,  ?COL_ID+1).
--define(COL_SLOT,  ?COL_NAME+1).
--define(COL_OWNER, ?COL_SLOT+1).
+-define(COL_NAME,  0).
+-define(COL_IS_NAMED,  ?COL_NAME+1).
+-define(COL_OWNER, ?COL_IS_NAMED+1).
 -define(COL_OBJ,   ?COL_OWNER+1).
 -define(COL_MEM,   ?COL_OBJ+1).
 
 %% Callbacks for cdv_virtual_list_wx
-col_to_elem(id) -> col_to_elem(?COL_ID);
-col_to_elem(?COL_ID)    -> #ets_table.id;
+col_to_elem(id) -> col_to_elem(?COL_NAME);
+col_to_elem(?COL_IS_NAMED) -> #ets_table.is_named;
 col_to_elem(?COL_NAME)  -> #ets_table.name;
-col_to_elem(?COL_SLOT)  -> #ets_table.slot;
 col_to_elem(?COL_OWNER) -> #ets_table.pid;
 col_to_elem(?COL_OBJ)   -> #ets_table.size;
 col_to_elem(?COL_MEM)   -> #ets_table.memory.
 
 col_spec() ->
-    [{"Id",      ?wxLIST_FORMAT_LEFT,   200},
-     {"Name",    ?wxLIST_FORMAT_LEFT,   200},
-     {"Slot",    ?wxLIST_FORMAT_RIGHT,  50},
+    [{"Name",    ?wxLIST_FORMAT_LEFT,   200},
+     {"Is Named", ?wxLIST_FORMAT_CENTRE, 70},
      {"Owner",   ?wxLIST_FORMAT_CENTRE, 120},
      {"Objects", ?wxLIST_FORMAT_RIGHT,  80},
      {"Memory",  ?wxLIST_FORMAT_RIGHT,  80}
@@ -68,7 +65,7 @@ get_details(Id, Data) ->
     {ok,{"Table:" ++ Id,Proplist,""}}.
 
 get_detail_cols(all) ->
-    {[{ets, ?COL_ID}, {process, ?COL_OWNER}],true};
+    {[{ets, ?COL_NAME}, {process, ?COL_OWNER}],true};
 get_detail_cols(_W) ->
     {[],true}.
 
diff --git a/lib/observer/src/crashdump_viewer.erl b/lib/observer/src/crashdump_viewer.erl
index 13e73f027d..e21f1c501b 100644
--- a/lib/observer/src/crashdump_viewer.erl
+++ b/lib/observer/src/crashdump_viewer.erl
@@ -1555,10 +1555,14 @@ split_pid_list_no_space([],[],Pids) ->
 %% Page with external ets tables
 get_ets_tables(File,Pid,WS) ->
     ParseFun = fun(Fd,Id) ->
-		       get_etsinfo(Fd,#ets_table{pid=list_to_pid(Id)},WS)
+		       ET = get_etsinfo(Fd,#ets_table{pid=list_to_pid(Id)},WS),
+                       ET#ets_table{is_named=tab_is_named(ET)}
 	       end,
     lookup_and_parse_index(File,{?ets,Pid},ParseFun,"ets").
 
+tab_is_named(#ets_table{id=Name,name=Name}) -> "yes";
+tab_is_named(#ets_table{}) -> "no".
+
 get_etsinfo(Fd,EtsTable = #ets_table{details=Ds},WS) ->
     case line_head(Fd) of
 	"Slot" ->
diff --git a/lib/observer/src/crashdump_viewer.hrl b/lib/observer/src/crashdump_viewer.hrl
index a08659efd6..742e145641 100644
--- a/lib/observer/src/crashdump_viewer.hrl
+++ b/lib/observer/src/crashdump_viewer.hrl
@@ -118,6 +118,7 @@
 	 slot,
 	 id,
 	 name,
+         is_named,
 	 data_type="hash",
 	 buckets="-",
 	 size,
diff --git a/lib/observer/src/observer_tv_wx.erl b/lib/observer/src/observer_tv_wx.erl
index 4356cb890c..d04fb839c8 100644
--- a/lib/observer/src/observer_tv_wx.erl
+++ b/lib/observer/src/observer_tv_wx.erl
@@ -78,11 +78,11 @@ init([Notebook, Parent]) ->
 			   Col + 1
 		   end,
     ListItems = [{"Table Name", ?wxLIST_FORMAT_LEFT,  200},
-		 {"Table Id",   ?wxLIST_FORMAT_RIGHT, 100},
 		 {"Objects",    ?wxLIST_FORMAT_RIGHT, 100},
 		 {"Size (kB)",  ?wxLIST_FORMAT_RIGHT, 100},
 		 {"Owner Pid",  ?wxLIST_FORMAT_CENTER, 150},
-		 {"Owner Name", ?wxLIST_FORMAT_LEFT,  200}
+		 {"Owner Name", ?wxLIST_FORMAT_LEFT,  200},
+		 {"Table Id",   ?wxLIST_FORMAT_LEFT, 250}
 		],
     lists:foldl(AddListEntry, 0, ListItems),
     wxListItem:destroy(Li),
@@ -387,8 +387,8 @@ update_grid2(Grid, #opt{sort_key=Sort,sort_incr=Dir}, Tables) ->
 				 ({Col, Val}) ->
 				      wxListCtrl:setItem(Grid, Row, Col, observer_lib:to_str(Val))
 			      end,
-			      [{0,Name}, {1,Id}, {2,Size}, {3, Memory div 1024},
-			       {4,Owner}, {5,RegName}]),
+			      [{0,Name}, {1,Size}, {2, Memory div 1024},
+			       {3,Owner}, {4,RegName}, {5,Id}]),
 		Row + 1
 	end,
     ProcInfo = case Dir of
diff --git a/lib/stdlib/src/ets.erl b/lib/stdlib/src/ets.erl
index d6fd1e3ea1..90e19e6b9f 100644
--- a/lib/stdlib/src/ets.erl
+++ b/lib/stdlib/src/ets.erl
@@ -75,10 +75,28 @@
          take/2,
          update_counter/3, update_counter/4, update_element/3]).
 
+%% internal exports
+-export([internal_request_all/0]).
+
 -spec all() -> [Tab] when
       Tab :: tab().
 
 all() ->
+    receive_all(ets:internal_request_all(),
+                erlang:system_info(schedulers),
+                []).
+
+receive_all(_Ref, 0, All) ->
+    All;
+receive_all(Ref, N, All) ->
+    receive
+        {Ref, SchedAll} ->
+            receive_all(Ref, N-1, SchedAll ++ All)
+    end.
+
+-spec internal_request_all() -> reference().
+
+internal_request_all() ->
     erlang:nif_error(undef).
 
 -spec delete(Tab) -> true when
diff --git a/lib/stdlib/test/ets_SUITE.erl b/lib/stdlib/test/ets_SUITE.erl
index 8581440d58..ebf7dbff62 100644
--- a/lib/stdlib/test/ets_SUITE.erl
+++ b/lib/stdlib/test/ets_SUITE.erl
@@ -75,7 +75,7 @@
 -export([otp_9423/1]).
 -export([otp_10182/1]).
 -export([ets_all/1]).
--export([memory_check_summary/1]).
+-export([massive_ets_all/1]).
 -export([take/1]).
 
 -export([init_per_testcase/2, end_per_testcase/2]).
@@ -93,7 +93,6 @@ init_per_testcase(Case, Config) ->
     io:format("*** SEED: ~p ***\n", [rand:export_seed()]),
     start_spawn_logger(),
     wait_for_test_procs(), %% Ensure previous case cleaned up
-    put('__ETS_TEST_CASE__', Case),
     [{test_case, Case} | Config].
 
 end_per_testcase(_Func, _Config) ->
@@ -134,9 +133,8 @@ all() ->
      otp_9932,
      otp_9423,
      ets_all,
-     take,
-
-     memory_check_summary]. % MUST BE LAST
+     massive_ets_all,
+     take].
 
 groups() ->
     [{new, [],
@@ -181,27 +179,6 @@ init_per_group(_GroupName, Config) ->
 end_per_group(_GroupName, Config) ->
     Config.
 
-%% Test that we did not have "too many" failed verify_etsmem()'s
-%% in the test suite.
-%% verify_etsmem() may give a low number of false positives
-%% as concurrent activities, such as lingering processes
-%% from earlier test suites, may do unrelated ets (de)allocations.
-memory_check_summary(_Config) ->
-    case whereis(ets_test_spawn_logger) of
-	undefined ->
-	    ct:fail("No spawn logger exist");
-	_ ->
-	    ets_test_spawn_logger ! {self(), get_failed_memchecks},
-	    receive {get_failed_memchecks, FailedMemchecks} -> ok end,
-	    io:format("Failed memchecks: ~p\n",[FailedMemchecks]),
-	    NoFailedMemchecks = length(FailedMemchecks),
-	    if NoFailedMemchecks > 1 ->
-		    ct:fail("Too many failed (~p) memchecks", [NoFailedMemchecks]);
-	       true ->
-		    ok
-	    end
-    end.
-
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
@@ -5545,6 +5522,68 @@ ets_all_run() ->
     false = lists:member(Table, ets:all()),
     ets_all_run().
 
+create_tables(N) ->
+    create_tables(N, []).
+
+create_tables(0, Ts) ->
+    Ts;
+create_tables(N, Ts) ->
+    create_tables(N-1, [ets:new(tjo, [])|Ts]).
+
+massive_ets_all(Config) when is_list(Config) ->
+    Me = self(),
+    InitTables = lists:sort(ets:all()),
+    io:format("InitTables=~p~n", [InitTables]),
+    PMs0 = lists:map(fun (Sid) ->
+                             my_spawn_opt(fun () ->
+                                                  Ts = create_tables(250),
+                                                  Me ! {self(), up, Ts},
+                                                  receive {Me, die} -> ok end
+                                          end,
+                                          [link, monitor, {scheduler, Sid}])
+                     end,
+                     lists:seq(1, erlang:system_info(schedulers_online))),
+    AllRes = lists:sort(lists:foldl(fun ({P, _M}, Ts) ->
+                                            receive
+                                                {P, up, PTs} ->
+                                                    PTs ++ Ts
+                                            end
+                                    end,
+                                    InitTables,
+                                    PMs0)),
+    AllRes = lists:sort(ets:all()),
+    PMs1 = lists:map(fun (_) ->
+                             my_spawn_opt(fun () ->
+                                                  AllRes = lists:sort(ets:all())
+                                          end,
+                                          [link, monitor])
+                     end, lists:seq(1, 50)),
+    lists:foreach(fun ({P, M}) ->
+                          receive
+                              {'DOWN', M, process, P, _} ->
+                                  ok
+                          end
+                  end, PMs1),
+    PMs2 = lists:map(fun (_) ->
+                             my_spawn_opt(fun () ->
+                                                  _ = ets:all()
+                                          end,
+                                          [link, monitor])
+                     end, lists:seq(1, 50)),
+    lists:foreach(fun ({P, _M}) ->
+                          P ! {Me, die}
+                  end, PMs0),
+    lists:foreach(fun ({P, M}) ->
+                          receive
+                              {'DOWN', M, process, P, _} ->
+                                  ok
+                          end
+                  end, PMs0 ++ PMs2),
+    EndTables = lists:sort(ets:all()),
+    io:format("EndTables=~p~n", [EndTables]),
+    InitTables = EndTables,
+    ok.
+
 
 take(Config) when is_list(Config) ->
     %% Simple test for set tables.
@@ -5712,45 +5751,27 @@ etsmem() ->
 			   {Bl0+Bl,BlSz0+BlSz}
 		   end, {0,0}, CS)
 	 end},
-    {Mem,AllTabs, erts_debug:get_internal_state('DbTable_meta')}.
+    {Mem,AllTabs}.
 
-verify_etsmem(EtsMem) ->
+verify_etsmem({MemInfo,AllTabs}) ->
     wait_for_test_procs(),
-    verify_etsmem(EtsMem, false).
-
-verify_etsmem({MemInfo,AllTabs,MetaState}=EtsMem, Adjusted) ->
     case etsmem() of
-	{MemInfo,_,_} ->
+	{MemInfo,_} ->
 	    io:format("Ets mem info: ~p", [MemInfo]),
 	    case MemInfo of
 		{ErlMem,EtsAlloc} when ErlMem == notsup; EtsAlloc == undefined ->
 		    %% Use 'erl +Mea max' to do more complete memory leak testing.
 		    {comment,"Incomplete or no mem leak testing"};
 		_ ->
-		    case Adjusted of
-			true ->
-			    {comment, "Meta state adjusted"};
-			false ->
-			    ok
-		    end
+                    ok
 	    end;
 
-	{MemInfo2, AllTabs2, MetaState2} ->
+	{MemInfo2, AllTabs2} ->
 	    io:format("Expected: ~p", [MemInfo]),
 	    io:format("Actual:   ~p", [MemInfo2]),
 	    io:format("Changed tables before: ~p\n",[AllTabs -- AllTabs2]),
 	    io:format("Changed tables after: ~p\n", [AllTabs2 -- AllTabs]),
-	    io:format("Meta state before: ~p\n", [MetaState]),
-	    io:format("Meta state after:  ~p\n", [MetaState2]),
-	    case {MetaState =:= MetaState2, Adjusted} of
-		{false, false} ->
-		    io:format("Adjust meta state and retry...\n\n",[]),
-		    {ok,ok} = erts_debug:set_internal_state('DbTable_meta', MetaState),
-		    verify_etsmem(EtsMem, true);
-		_ ->
-		    ets_test_spawn_logger ! {failed_memcheck, get('__ETS_TEST_CASE__')},
-		    {comment, "Failed memory check"}
-	    end
+            ct:fail("Failed memory check")
     end.
 
 
@@ -5772,10 +5793,10 @@ stop_loopers(Loopers) ->
 looper(Fun, State) ->
     looper(Fun, Fun(State)).
 
-spawn_logger(Procs, FailedMemchecks) ->
+spawn_logger(Procs) ->
     receive
 	{new_test_proc, Proc} ->
-	    spawn_logger([Proc|Procs], FailedMemchecks);
+	    spawn_logger([Proc|Procs]);
 	{sync_test_procs, Kill, From} ->
 	    lists:foreach(fun (Proc) when From == Proc ->
 				  ok;
@@ -5799,14 +5820,7 @@ spawn_logger(Procs, FailedMemchecks) ->
 				  end
 			  end, Procs),
 	    From ! test_procs_synced,
-	    spawn_logger([From], FailedMemchecks);
-
-	{failed_memcheck, TestCase} ->
-	    spawn_logger(Procs, [TestCase|FailedMemchecks]);
-
-	{Pid, get_failed_memchecks} ->
-	    Pid ! {get_failed_memchecks, FailedMemchecks},
-	    spawn_logger(Procs, FailedMemchecks)
+	    spawn_logger([From])
     end.
 
 pid_status(Pid) ->
@@ -5822,7 +5836,7 @@ start_spawn_logger() ->
     case whereis(ets_test_spawn_logger) of
 	Pid when is_pid(Pid) -> true;
 	_ -> register(ets_test_spawn_logger,
-		      spawn_opt(fun () -> spawn_logger([], []) end,
+		      spawn_opt(fun () -> spawn_logger([]) end,
 				[{priority, max}]))
     end.