erts: Rewrite memory instrumentation

This commit replaces the old memory instrumentation with a new
implementation that scans carriers instead of wrapping
erts_alloc/erts_free. The old implementation could not extract
information without halting the emulator, had considerable runtime
overhead, and the memory maps it produced were noisy and lacked
critical information.

Since the new implementation walks through existing data structures
there's no longer a need to start the emulator with special flags to
get information about carrier utilization/fragmentation. Memory
fragmentation is also easier to diagnose as it's presented on a
per-carrier basis which eliminates the need to account for "holes"
between mmap segments.

To help track allocations, each allocation can now be tagged with
what it is and who allocated it at the cost of one extra word per
allocation. This is controlled on a per-allocator basis with the
+M<S>atags option, and is enabled by default for binary_alloc and
driver_alloc (which is also used by NIFs).

3 files changed, 29 insertions, 49 deletions

diff --git a/erts/preloaded/ebin/erts_internal.beam b/erts/preloaded/ebin/erts_internal.beam
index f5967780ad..d1bc4ecd2c 100644
--- a/erts/preloaded/ebin/erts_internal.beam
+++ b/erts/preloaded/ebin/erts_internal.beam
diff --git a/erts/preloaded/src/erlang.erl b/erts/preloaded/src/erlang.erl
index bffa59338e..9c2f8ef0ae 100644
--- a/erts/preloaded/src/erlang.erl
+++ b/erts/preloaded/src/erlang.erl
@@ -3552,11 +3552,9 @@ max(A, _) -> A.
 %%
 
 -type memory_type() :: 'total' | 'processes' | 'processes_used' | 'system'
-                     | 'atom' | 'atom_used' | 'binary' | 'code' | 'ets'
-                     | 'low' | 'maximum'.
+                     | 'atom' | 'atom_used' | 'binary' | 'code' | 'ets'.
 
 -define(CARRIER_ALLOCS, [mseg_alloc]).
--define(LOW_ALLOCS, [ll_low_alloc, std_low_alloc]).
 -define(ALL_NEEDED_ALLOCS, (erlang:system_info(alloc_util_allocators)
 			    -- ?CARRIER_ALLOCS)).
 
@@ -3568,9 +3566,7 @@ max(A, _) -> A.
 		 atom_used = 0,
 		 binary = 0,
 		 code = 0,
-		 ets = 0,
-		 low = 0,
-		 maximum = 0}).
+		 ets = 0}).
 
 -spec erlang:memory() -> [{Type, Size}] when
       Type :: memory_type(),
@@ -3580,14 +3576,6 @@ memory() ->
 	notsup ->
 	    erlang:error(notsup);
 	Mem ->
-	    InstrTail = case Mem#memory.maximum of
-			    0 -> [];
-			    _ -> [{maximum, Mem#memory.maximum}]
-			end,
-	    Tail = case Mem#memory.low of
-		       0 -> InstrTail;
-		       _ -> [{low, Mem#memory.low} | InstrTail]
-		   end,
 	    [{total, Mem#memory.total},
 	     {processes, Mem#memory.processes},
 	     {processes_used, Mem#memory.processes_used},
@@ -3596,7 +3584,7 @@ memory() ->
 	     {atom_used, Mem#memory.atom_used},
 	     {binary, Mem#memory.binary},
 	     {code, Mem#memory.code},
-	     {ets, Mem#memory.ets} | Tail]
+	     {ets, Mem#memory.ets}]
     end.
 
 -spec erlang:memory(Type :: memory_type()) -> non_neg_integer();
@@ -3684,16 +3672,6 @@ need_mem_info(binary) ->
     {false, [binary_alloc], true, false};
 need_mem_info(ets) ->
     {true, [ets_alloc], true, false};
-need_mem_info(low) ->
-    LowAllocs = ?LOW_ALLOCS -- ?CARRIER_ALLOCS,
-    {_, _, FeatureList, _} = erlang:system_info(allocator),
-    AlcUAllocs = case LowAllocs -- FeatureList of
-		     [] -> LowAllocs;
-		     _ -> []
-		 end,
-    {false, AlcUAllocs, true, true};
-need_mem_info(maximum) ->
-    {true, [], true, true};
 need_mem_info(_) ->
     {false, [], false, true}.
 
@@ -3706,8 +3684,6 @@ get_memval(atom_used, #memory{atom_used = V}) -> V;
 get_memval(binary, #memory{binary = V}) -> V;
 get_memval(code, #memory{code = V}) -> V;
 get_memval(ets, #memory{ets = V}) -> V;
-get_memval(low, #memory{low = V}) -> V;
-get_memval(maximum, #memory{maximum = V}) -> V;
 get_memval(_, #memory{}) -> 0.
 
 memory_is_supported() ->
@@ -3762,16 +3738,6 @@ fix_proc([_ | Rest], Acc) ->
 fix_proc([], Acc) ->
     Acc.
 
-is_low_alloc(_A, []) ->
-    false;
-is_low_alloc(A, [A|_As]) ->
-    true;
-is_low_alloc(A, [_A|As]) ->
-    is_low_alloc(A, As).
-
-is_low_alloc(A) ->
-    is_low_alloc(A, ?LOW_ALLOCS).
-
 au_mem_data(notsup, _) ->
     notsup;
 au_mem_data(_, [{_, false} | _]) ->
@@ -3824,16 +3790,11 @@ au_mem_data(#memory{total = Tot,
 			Rest)
     end;
 au_mem_data(#memory{total = Tot,
-		    system = Sys,
-		    low = Low} = Mem,
-	    [{A, _, Data} | Rest]) ->
+		    system = Sys} = Mem,
+	    [{_, _, Data} | Rest]) ->
     Sz = blocks_size(Data, 0),
     au_mem_data(Mem#memory{total = Tot+Sz,
-			   system = Sys+Sz,
-			   low = case is_low_alloc(A) of
-				     true -> Low+Sz;
-				     false -> Low
-				 end},
+			   system = Sys+Sz},
 		Rest);
 au_mem_data(EMD, []) ->
     EMD.
@@ -3855,10 +3816,6 @@ receive_emd(Ref) ->
     receive_emd(Ref, #memory{}, erlang:system_info(schedulers)).
 
 aa_mem_data(#memory{} = Mem,
-	    [{maximum, Max} | Rest]) ->
-    aa_mem_data(Mem#memory{maximum = Max},
-		Rest);
-aa_mem_data(#memory{} = Mem,
 	    [{total, Tot} | Rest]) ->
     aa_mem_data(Mem#memory{total = Tot,
 			   system = 0}, % system will be adjusted later
diff --git a/erts/preloaded/src/erts_internal.erl b/erts/preloaded/src/erts_internal.erl
index da5c9c68ed..79169b7d23 100644
--- a/erts/preloaded/src/erts_internal.erl
+++ b/erts/preloaded/src/erts_internal.erl
@@ -80,6 +80,8 @@
 
 -export([is_process_alive/1, is_process_alive/2]).
 
+-export([gather_alloc_histograms/1, gather_carrier_info/1]).
+
 %%
 %% Await result of send to port
 %%
@@ -621,3 +623,24 @@ is_process_alive(Pid) ->
             Res
     end.
 
+-spec gather_alloc_histograms({Type, SchedId, HistWidth, HistStart, Ref}) -> MsgCount when
+    Type :: atom(),
+    SchedId :: non_neg_integer(),
+    HistWidth :: non_neg_integer(),
+    HistStart :: non_neg_integer(),
+    Ref :: reference(),
+    MsgCount :: non_neg_integer().
+
+gather_alloc_histograms(_) ->
+    erlang:nif_error(undef).
+
+-spec gather_carrier_info({Type, SchedId, HistWidth, HistStart, Ref}) -> MsgCount when
+    Type :: atom(),
+    SchedId :: non_neg_integer(),
+    HistWidth :: non_neg_integer(),
+    HistStart :: non_neg_integer(),
+    Ref :: reference(),
+    MsgCount :: non_neg_integer().
+
+gather_carrier_info(_) ->
+    erlang:nif_error(undef).