aboutsummaryrefslogtreecommitdiffstats
path: root/lib
diff options
context:
space:
mode:
authorJohn Högberg <[email protected]>2018-04-24 16:45:18 +0200
committerGitHub <[email protected]>2018-04-24 16:45:18 +0200
commite2a760e691d1b08a2396233fb50cefd98199ba12 (patch)
tree3b7d1f2d3ad7b8934dbac263704bc95fc7a7f1e7 /lib
parent685cd438232282ee0fa790517e7fe193110ff265 (diff)
parent573a5abd9d6b1668b49376b489b187780c7125c7 (diff)
downloadotp-e2a760e691d1b08a2396233fb50cefd98199ba12.tar.gz
otp-e2a760e691d1b08a2396233fb50cefd98199ba12.tar.bz2
otp-e2a760e691d1b08a2396233fb50cefd98199ba12.zip
Merge pull request #1790 from jhogberg/john/erts/more-alloc-info/OTP-14961
Improve memory instrumentation OTP-15024 OTP-14961
Diffstat (limited to 'lib')
-rw-r--r--lib/observer/test/crashdump_viewer_SUITE.erl2
-rw-r--r--lib/tools/doc/specs/.gitignore1
-rw-r--r--lib/tools/doc/src/Makefile14
-rw-r--r--lib/tools/doc/src/instrument.xml527
-rw-r--r--lib/tools/doc/src/specs.xml12
-rw-r--r--lib/tools/src/instrument.erl538
-rw-r--r--lib/tools/test/instrument_SUITE.erl391
7 files changed, 642 insertions, 843 deletions
diff --git a/lib/observer/test/crashdump_viewer_SUITE.erl b/lib/observer/test/crashdump_viewer_SUITE.erl
index 41ca3f3ce9..8bd7ad387b 100644
--- a/lib/observer/test/crashdump_viewer_SUITE.erl
+++ b/lib/observer/test/crashdump_viewer_SUITE.erl
@@ -674,7 +674,7 @@ do_create_dumps(DataDir,Rel) ->
end,
case Rel of
current ->
- CD3 = dump_with_args(DataDir,Rel,"instr","+Mim true"),
+ CD3 = dump_with_args(DataDir,Rel,"instr","+Muatags true"),
CD4 = dump_with_strange_module_name(DataDir,Rel,"strangemodname"),
CD5 = dump_with_size_limit_reached(DataDir,Rel,"trunc_bytes"),
CD6 = dump_with_unicode_atoms(DataDir,Rel,"unicode"),
diff --git a/lib/tools/doc/specs/.gitignore b/lib/tools/doc/specs/.gitignore
new file mode 100644
index 0000000000..322eebcb06
--- /dev/null
+++ b/lib/tools/doc/specs/.gitignore
@@ -0,0 +1 @@
+specs_*.xml
diff --git a/lib/tools/doc/src/Makefile b/lib/tools/doc/src/Makefile
index b554781382..4b663106a0 100644
--- a/lib/tools/doc/src/Makefile
+++ b/lib/tools/doc/src/Makefile
@@ -84,11 +84,20 @@ HTML_REF_MAN_FILE = $(HTMLDIR)/index.html
TOP_PDF_FILE = $(PDFDIR)/$(APPLICATION)-$(VSN).pdf
+SPECS_FILES = $(XML_REF3_FILES:%.xml=$(SPECDIR)/specs_%.xml)
+
+TOP_SPECS_FILE = specs.xml
+
# ----------------------------------------------------
# FLAGS
# ----------------------------------------------------
XML_FLAGS +=
+TOOLS_SRC=$(ERL_TOP)/lib/tools/src
+TOOLS_INCLUDE=$(ERL_TOP)/lib/tools/include
+
+SPECS_FLAGS = -I$(TOOLS_SRC) -I$(TOOLS_INCLUDE)
+
# ----------------------------------------------------
# Targets
# ----------------------------------------------------
@@ -113,8 +122,13 @@ clean clean_docs:
rm -rf $(HTMLDIR)/*
rm -f $(MAN3DIR)/*
rm -f $(TOP_PDF_FILE) $(TOP_PDF_FILE:%.pdf=%.fo)
+ rm -f $(SPECDIR)/*
rm -f errs core *~
+# erlang_mode doesn't have erlang source so we generate a dummy file for it.
+$(SPECDIR)/specs_erlang_mode.xml:
+ echo '<module name="erlang_mode"/>' > $(SPECDIR)/specs_erlang_mode.xml
+
# ----------------------------------------------------
# Release Target
# ----------------------------------------------------
diff --git a/lib/tools/doc/src/instrument.xml b/lib/tools/doc/src/instrument.xml
index bb6f9b6100..9fd9332373 100644
--- a/lib/tools/doc/src/instrument.xml
+++ b/lib/tools/doc/src/instrument.xml
@@ -4,7 +4,7 @@
<erlref>
<header>
<copyright>
- <year>1998</year><year>2016</year>
+ <year>1998</year><year>2018</year>
<holder>Ericsson AB. All Rights Reserved.</holder>
</copyright>
<legalnotice>
@@ -41,387 +41,190 @@
<note>
<p>Note that this whole module is experimental, and the representations
used as well as the functionality is likely to change in the future.</p>
- <p>The <c>instrument</c> module interface was slightly changed in
- Erlang/OTP R9C.</p>
</note>
- <p>To start an Erlang runtime system with instrumentation, use the
- <c>+Mi*</c> set of command-line arguments to the <c>erl</c> command (see
- the erts_alloc(3) and erl(1) man pages).</p>
- <p>The basic object of study in the case of memory allocation is a memory
- allocation map. A memory allocation map contains a list of descriptors
- for each allocated memory block. Currently, a descriptor is a 4-tuple</p>
- <pre>
- {TypeNo, Address, Size, PidDesc} </pre>
- <p>where <c>TypeNo</c> is the memory block type number, <c>Address</c>
- is its place in memory, and <c>Size</c> is its size, in bytes.
- <c>PidDesc</c> is either a tuple <c>{X,Y,Z}</c> identifying the
- process which was executing when the block was allocated, or
- <c>undefined</c> if no process was executing. The pid tuple
- <c>{X,Y,Z}</c> can be transformed into a real pid by usage of the
- <c>c:pid/3</c> function.</p>
- <p>Various details about memory allocation:</p>
- <p>Memory blocks are allocated both on the heap segment and on other memory
- segments. This can cause the instrumentation functionality to report
- very large holes. Currently the instrumentation functionality doesn't
- provide any support for distinguishing between holes between memory
- segments, and holes between allocated blocks inside memory segments.
- The current size of the process cannot be obtained from within Erlang,
- but can be seen with one of the system statistics tools, e.g.,
- <c>ps</c> or <c>top</c>. The Solaris utility <c>pmap</c> can be
- useful. It reports currently mapped memory segments. </p>
- <p>Overhead for instrumentation: When the emulator has been started with
- the <seealso marker="erts:erts_alloc#Mim">"+Mim true"</seealso>
- flag, each block is preceded by a 24 bytes large
- header on a 32-bit machine and a 48 bytes large header on a 64-bit
- machine. When the emulator has been started with the
- <seealso marker="erts:erts_alloc#Mis">"+Mis true"</seealso>
- flag, each block is preceded by an 8 bytes large header. These are the header
- sizes used by the Erlang 5.3/OTP R9C emulator. Other versions of the
- emulator may use other header sizes. The function
- <seealso marker="#block_header_size/1">block_header_size/1</seealso>
- can be used for retrieving the header size used for a specific memory
- allocation map. The time overhead for managing the instrumentation
- data is small.</p>
- <p>Sizes presented by the instrumentation functionality are (by the
- emulator) requested sizes, i.e. neither instrumentation headers nor
- headers used by allocators are included.</p>
</description>
+ <datatypes>
+ <datatype>
+ <name name="block_histogram"/>
+ <desc>
+ <p>A histogram of block sizes where each interval's upper bound is
+ twice as high as the one before it.</p>
+ <p>The upper bound of the first interval is provided by the function
+ that returned the histogram, and the last interval has no upper
+ bound.</p>
+ </desc>
+ </datatype>
+ <datatype>
+ <name name="allocation_summary"/>
+ <desc>
+ <p>A summary of allocated block sizes (including their headers) grouped
+ by their <c><anno>Origin</anno></c> and <c><anno>Type</anno></c>.</p>
+ <p><c><anno>Origin</anno></c> is generally which NIF or driver that
+ allocated the blocks, or 'system' if it could not be determined.</p>
+ <p><c><anno>Type</anno></c> is the allocation category that the blocks
+ belong to, e.g. <c>db_term</c>, <c>message</c> or <c>binary</c>.</p>
+ <p>If one or more carriers could not be scanned in full without harming
+ the responsiveness of the system, <c><anno>UnscannedSize</anno></c>
+ is the number of bytes that had to be skipped.</p>
+ </desc>
+ </datatype>
+ <datatype>
+ <name name="carrier_info_list"/>
+ <desc>
+ <p><c><anno>AllocatorType</anno></c> is the type of the allocator that
+ employs this carrier.</p>
+ <p><c><anno>TotalSize</anno></c> is the total size of the carrier,
+ including its header.</p>
+ <p><c><anno>AllocatedSize</anno></c> is the combined size of the
+ carrier's allocated blocks, including their headers.</p>
+ <p><c><anno>AllocatedCount</anno></c> is the number of allocated
+ blocks in the carrier.</p>
+ <p><c><anno>InPool</anno></c> is whether the carrier is in the
+ migration pool.</p>
+ <p><c><anno>FreeBlocks</anno></c> is a histogram of the free block
+ sizes in the carrier.</p>
+ <p>If the carrier could not be scanned in full without harming the
+ responsiveness of the system, <c><anno>UnscannedSize</anno></c> is
+ the number of bytes that had to be skipped.</p>
+ </desc>
+ </datatype>
+ </datatypes>
<funcs>
+
<func>
- <name>allocator_descr(MemoryData, TypeNo) -> AllocDescr | invalid_type | "unknown"</name>
- <fsummary>Returns a allocator description</fsummary>
- <type>
- <v>MemoryData = {term(), AllocList}</v>
- <v>AllocList = [Desc]</v>
- <v>Desc = {int(), int(), int(), PidDesc}</v>
- <v>PidDesc = {int(), int(), int()} | undefined</v>
- <v>TypeNo = int()</v>
- <v>AllocDescr = atom() | string()</v>
- </type>
- <desc>
- <p>Returns the allocator description of the allocator that
- manages memory blocks of type number <c>TypeNo</c> used in
- <c>MemoryData</c>.
- Valid <c>TypeNo</c>s are in the range returned by
- <seealso marker="#type_no_range/1">type_no_range/1</seealso> on
- this specific memory allocation map. If <c>TypeNo</c> is an
- invalid integer, <c>invalid_type</c> is returned.</p>
- </desc>
- </func>
- <func>
- <name>block_header_size(MemoryData) -> int()</name>
- <fsummary>Returns the memory block header size used by the emulator that generated the memory allocation map</fsummary>
- <type>
- <v>MemoryData = {term(), AllocList}</v>
- <v>AllocList = [Desc]</v>
- <v>Desc = {int(), int(), int(), PidDesc}</v>
- <v>PidDesc = {int(), int(), int()} | undefined</v>
- </type>
- <desc>
- <marker id="block_header_size_1"></marker>
- <p>Returns the memory block header size used by the
- emulator that generated the memory allocation map. The block
- header size may differ between different emulators.</p>
- </desc>
- </func>
- <func>
- <name>class_descr(MemoryData, TypeNo) -> ClassDescr | invalid_type | "unknown"</name>
- <fsummary>Returns a allocator description</fsummary>
- <type>
- <v>MemoryData = {term(), AllocList}</v>
- <v>AllocList = [Desc]</v>
- <v>Desc = {int(), int(), int(), PidDesc}</v>
- <v>PidDesc = {int(), int(), int()} | undefined</v>
- <v>TypeNo = int()</v>
- <v>ClassDescr = atom() | string()</v>
- </type>
- <desc>
- <p>Returns the class description of the class that
- the type number <c>TypeNo</c> used in <c>MemoryData</c> belongs
- to.
- Valid <c>TypeNo</c>s are in the range returned by
- <seealso marker="#type_no_range/1">type_no_range/1</seealso> on
- this specific memory allocation map. If <c>TypeNo</c> is an
- invalid integer, <c>invalid_type</c> is returned.</p>
- </desc>
- </func>
- <func>
- <name>descr(MemoryData) -> DescrMemoryData</name>
- <fsummary>Replace type numbers in memory allocation map with type descriptions</fsummary>
- <type>
- <v>MemoryData = {term(), AllocList}</v>
- <v>AllocList = [Desc]</v>
- <v>Desc = {int(), int(), int(), PidDesc}</v>
- <v>PidDesc = {int(), int(), int()} | undefined</v>
- <v>DescrMemoryData = {term(), DescrAllocList}</v>
- <v>DescrAllocList = [DescrDesc]</v>
- <v>DescrDesc = {TypeDescr, int(), int(), DescrPidDesc}</v>
- <v>TypeDescr = atom() | string()</v>
- <v>DescrPidDesc = pid() | undefined</v>
- </type>
- <desc>
- <p>Returns a memory allocation map where the type numbers (first
- element of <c>Desc</c>) have been replaced by type descriptions,
- and pid tuples (fourth element of <c>Desc</c>) have been
- replaced by real pids.</p>
- </desc>
- </func>
- <func>
- <name>holes(MemoryData) -> ok</name>
- <fsummary>Print out the sizes of unused memory blocks</fsummary>
- <type>
- <v>MemoryData = {term(), AllocList}</v>
- <v>AllocList = [Desc]</v>
- <v>Desc = {int(), int(), int(), PidDesc}</v>
- <v>PidDesc = {int(), int(), int()} | undefined</v>
- </type>
- <desc>
- <p>Prints out the size of each hole (i.e., the space between
- allocated blocks) on the terminal. <em>NOTE:</em> Really large holes
- are probably holes between memory segments.
- The memory allocation map has to be sorted (see
- <seealso marker="#sort/1">sort/1</seealso>).</p>
- </desc>
- </func>
- <func>
- <name>mem_limits(MemoryData) -> {Low, High}</name>
- <fsummary>Return lowest and highest memory address used</fsummary>
- <type>
- <v>MemoryData = {term(), AllocList}</v>
- <v>AllocList = [Desc]</v>
- <v>Desc = {int(), int(), int(), PidDesc}</v>
- <v>PidDesc = {int(), int(), int()} | undefined</v>
- <v>Low = High = int()</v>
- </type>
- <desc>
- <p>Returns a tuple <c>{Low, High}</c> indicating
- the lowest and highest address used.
- The memory allocation map has to be sorted (see
- <seealso marker="#sort/1">sort/1</seealso>).</p>
- </desc>
- </func>
- <func>
- <name>memory_data() -> MemoryData | false</name>
- <fsummary>Return the current memory allocation map</fsummary>
- <type>
- <v>MemoryData = {term(), AllocList}</v>
- <v>AllocList = [Desc]</v>
- <v>Desc = {int(), int(), int(), PidDesc}</v>
- <v>PidDesc = {int(), int(), int()} | undefined</v>
- </type>
- <desc>
- <p>Returns <c>MemoryData</c> (a the memory allocation map)
- if the emulator has been started with the "<c>+Mim true</c>"
- command-line argument; otherwise, <c>false</c>. <em>NOTE:</em><c>memory_data/0</c> blocks execution of other processes while
- the data is collected. The time it takes to collect the data can
- be substantial.</p>
- </desc>
- </func>
- <func>
- <name>memory_status(StatusType) -> [StatusInfo] | false</name>
- <fsummary>Return current memory allocation status</fsummary>
- <type>
- <v>StatusType = total | allocators | classes | types</v>
- <v>StatusInfo = {About, [Info]}</v>
- <v>About = atom()</v>
- <v>Info = {InfoName, Current, MaxSinceLast, MaxEver}</v>
- <v>InfoName = sizes|blocks</v>
- <v>Current = int()</v>
- <v>MaxSinceLast = int()</v>
- <v>MaxEver = int()</v>
- </type>
- <desc>
- <p>Returns a list of <c>StatusInfo</c> if the emulator has been
- started with the "<c>+Mis true</c>" or "<c>+Mim true</c>"
- command-line argument; otherwise, <c>false</c>. </p>
- <p>See the
- <seealso marker="#read_memory_status/1">read_memory_status/1</seealso>
- function for a description of the <c>StatusInfo</c> term.</p>
- </desc>
- </func>
- <func>
- <name>read_memory_data(File) -> MemoryData | {error, Reason}</name>
- <fsummary>Read memory allocation map</fsummary>
- <type>
- <v>File = string()</v>
- <v>MemoryData = {term(), AllocList}</v>
- <v>AllocList = [Desc]</v>
- <v>Desc = {int(), int(), int(), PidDesc}</v>
- <v>PidDesc = {int(), int(), int()} | undefined</v>
- </type>
+ <name name="allocations" arity="0"/>
+ <fsummary>Return a summary of all allocations in the system.</fsummary>
<desc>
- <marker id="read_memory_data_1"></marker>
- <p>Reads a memory allocation map from the file <c>File</c> and
- returns it. The file is assumed to have been created by
- <c>store_memory_data/1</c>. The error codes are the same as for
- <c>file:consult/1</c>.</p>
+ <p>Shorthand for
+ <seealso marker="#allocations/1"><c>allocations(#{})</c>.</seealso></p>
</desc>
</func>
+
<func>
- <name>read_memory_status(File) -> MemoryStatus | {error, Reason}</name>
- <fsummary>Read memory allocation status from a file</fsummary>
- <type>
- <v>File = string()</v>
- <v>MemoryStatus = [{StatusType, [StatusInfo]}]</v>
- <v>StatusType = total | allocators | classes | types</v>
- <v>StatusInfo = {About, [Info]}</v>
- <v>About = atom()</v>
- <v>Info = {InfoName, Current, MaxSinceLast, MaxEver}</v>
- <v>InfoName = sizes|blocks</v>
- <v>Current = int()</v>
- <v>MaxSinceLast = int()</v>
- <v>MaxEver = int()</v>
- </type>
- <desc>
- <marker id="read_memory_status_1"></marker>
- <p>Reads memory allocation status from the file <c>File</c> and
- returns it. The file is assumed to have been created by
- <c>store_memory_status/1</c>. The error codes are the same as
- for <c>file:consult/1</c>.</p>
- <p>When <c>StatusType</c> is <c>allocators</c>, <c>About</c> is
- the allocator that the information is about. When
- <c>StatusType</c> is <c>types</c>, <c>About</c> is
- the memory block type that the information is about. Memory
- block types are not described other than by their name and may
- vary between emulators. When <c>StatusType</c> is <c>classes</c>,
- <c>About</c> is the memory block type class that information is
- presented about. Memory block types are classified after their
- use. Currently the following classes exist:</p>
+ <name name="allocations" arity="1"/>
+ <fsummary>Return a summary of all allocations filtered by allocator type
+ and scheduler id.</fsummary>
+ <desc>
+ <p>Returns a summary of all tagged allocations in the system,
+ optionally filtered by allocator type and scheduler id.</p>
+ <p>Only binaries and allocations made by NIFs and drivers are tagged by
+ default, but this can be configured an a per-allocator basis with the
+ <seealso marker="erts:erts_alloc#M_atags"><c>+M&lt;S&gt;atags</c>
+ </seealso> emulator option.</p>
+ <p>If tagged allocations are not enabled on any of the specified
+ allocator types, the call will fail with
+ <c>{error, not_enabled}</c>.</p>
+ <p>The following options can be used:</p>
<taglist>
- <tag><c>process_data</c></tag>
- <item>Erlang process specific data.</item>
- <tag><c>binary_data</c></tag>
- <item>Erlang binaries.</item>
- <tag><c>atom_data</c></tag>
- <item>Erlang atoms.</item>
- <tag><c>code_data</c></tag>
- <item>Erlang code.</item>
- <tag><c>system_data</c></tag>
- <item>Other data used by the system</item>
+ <tag><c>allocator_types</c></tag>
+ <item>
+ <p>The allocator types that will be searched. Defaults to all
+ <c>alloc_util</c> allocators.</p>
+ </item>
+ <tag><c>scheduler_ids</c></tag>
+ <item>
+ <p>The scheduler ids whose allocator instances will be searched. A
+ scheduler id of 0 will refer to the global instance that is not
+ tied to any particular scheduler. Defaults to all schedulers and
+ the global instance.</p>
+ </item>
+ <tag><c>histogram_start</c></tag>
+ <item>
+ <p>The upper bound of the first interval in the allocated block
+ size histograms. Defaults to 128.</p>
+ </item>
+ <tag><c>histogram_width</c></tag>
+ <item>
+ <p>The number of intervals in the allocated block size histograms.
+ Defaults to 18.</p>
+ </item>
</taglist>
- <p>When <c>InfoName</c> is <c>sizes</c>, <c>Current</c>,
- <c>MaxSinceLast</c>, and <c>MaxEver</c> are, respectively, current
- size, maximum size since last call to
- <c>store_memory_status/1</c> or <c>memory_status/1</c> with the
- specific <c>StatusType</c>, and maximum size since the emulator
- was started. When <c>InfoName</c> is <c>blocks</c>, <c>Current</c>,
- <c>MaxSinceLast</c>, and <c>MaxEver</c> are, respectively, current
- number of blocks, maximum number of blocks since last call to
- <c>store_memory_status/1</c> or <c>memory_status/1</c> with the
- specific <c>StatusType</c>, and maximum number of blocks since the
- emulator was started. </p>
- <p><em>NOTE:</em>A memory block is accounted for at
- "the first level" allocator. E.g. <c>fix_alloc</c> allocates its
- memory pools via <c>ll_alloc</c>. When a <c>fix_alloc</c> block
- is allocated, neither the block nor the pool in which it resides
- are accounted for as memory allocated via <c>ll_alloc</c> even
- though it is.</p>
- </desc>
- </func>
- <func>
- <name>sort(MemoryData) -> MemoryData</name>
- <fsummary>Sort the memory allocation list</fsummary>
- <type>
- <v>MemoryData = {term(), AllocList}</v>
- <v>AllocList = [Desc]</v>
- <v>Desc = {int(), int(), int(), PidDesc}</v>
- <v>PidDesc = {int(), int(), int()} | undefined</v>
- </type>
- <desc>
- <marker id="sort_1"></marker>
- <p>Sorts a memory allocation map so that the addresses are in
- ascending order.</p>
- </desc>
- </func>
- <func>
- <name>store_memory_data(File) -> true|false</name>
- <fsummary>Store the current memory allocation map on a file</fsummary>
- <type>
- <v>File = string()</v>
- </type>
- <desc>
- <p>Stores the current memory allocation map on the file
- <c>File</c>. Returns <c>true</c> if the emulator has been
- started with the "<c>+Mim true</c>" command-line argument, and
- the map was successfully stored; otherwise, <c>false</c>. The
- contents of the file can later be read using
- <seealso marker="#read_memory_data/1">read_memory_data/1</seealso>.
- <em>NOTE:</em><c>store_memory_data/0</c> blocks execution of
- other processes while the data is collected. The time it takes
- to collect the data can be substantial.</p>
- </desc>
- </func>
- <func>
- <name>store_memory_status(File) -> true|false</name>
- <fsummary>Store the current memory allocation status on a file</fsummary>
- <type>
- <v>File = string()</v>
- </type>
- <desc>
- <p>Stores the current memory status on the file
- <c>File</c>. Returns <c>true</c> if the emulator has been
- started with the "<c>+Mis true</c>", or "<c>+Mim true</c>"
- command-line arguments, and the data was successfully stored;
- otherwise, <c>false</c>. The contents of the file can later be
- read using
- <seealso marker="#read_memory_status/1">read_memory_status/1</seealso>.</p>
- </desc>
- </func>
- <func>
- <name>sum_blocks(MemoryData) -> int()</name>
- <fsummary>Return the total amount of memory used</fsummary>
- <type>
- <v>MemoryData = {term(), AllocList}</v>
- <v>AllocList = [Desc]</v>
- <v>Desc = {int(), int(), int(), PidDesc}</v>
- <v>PidDesc = {int(), int(), int()} | undefined</v>
- </type>
- <desc>
- <p>Returns the total size of the memory blocks in the list.</p>
+ <p><em>Example:</em></p>
+ <code type="none"><![CDATA[
+> instrument:allocations(#{ histogram_start => 128, histogram_width => 15 }).
+{ok,{128,0,
+ #{udp_inet =>
+ #{driver_event_state => {0,0,0,0,0,0,0,0,0,1,0,0,0,0,0}},
+ system =>
+ #{heap => {0,0,0,0,20,4,2,2,2,3,0,1,0,0,1},
+ db_term => {271,3,1,52,80,1,0,0,0,0,0,0,0,0,0},
+ code => {0,0,0,5,3,6,11,22,19,20,10,2,1,0,0},
+ binary => {18,0,0,0,7,0,0,1,0,0,0,0,0,0,0},
+ message => {0,40,78,2,2,0,0,0,0,0,0,0,0,0,0},
+ ... }
+ spawn_forker =>
+ #{driver_select_data_state =>
+ {1,0,0,0,0,0,0,0,0,0,0,0,0,0,0}},
+ ram_file_drv => #{drv_binary => {0,0,0,0,0,0,1,0,0,0,0,0,0,0,0}},
+ prim_file =>
+ #{process_specific_data => {2,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
+ nif_trap_export_entry => {0,4,0,0,0,0,0,0,0,0,0,0,0,0,0},
+ monitor_extended => {0,1,0,0,0,0,0,0,0,0,0,0,0,0,0},
+ drv_binary => {0,0,0,0,0,0,1,0,3,5,0,0,0,1,0},
+ binary => {0,4,0,0,0,0,0,0,0,0,0,0,0,0,0}},
+ prim_buffer =>
+ #{nif_internal => {0,4,0,0,0,0,0,0,0,0,0,0,0,0,0},
+ binary => {0,4,0,0,0,0,0,0,0,0,0,0,0,0,0}}}}}
+ ]]></code>
</desc>
</func>
+
<func>
- <name>type_descr(MemoryData, TypeNo) -> TypeDescr | invalid_type</name>
- <fsummary>Returns a type description</fsummary>
- <type>
- <v>MemoryData = {term(), AllocList}</v>
- <v>AllocList = [Desc]</v>
- <v>Desc = {int(), int(), int(), PidDesc}</v>
- <v>PidDesc = {int(), int(), int()} | undefined</v>
- <v>TypeNo = int()</v>
- <v>TypeDescr = atom() | string()</v>
- </type>
+ <name name="carriers" arity="0"/>
+ <fsummary>Return a list of all carriers in the system.</fsummary>
<desc>
- <p>Returns the type description of a type number used in
- <c>MemoryData</c>.
- Valid <c>TypeNo</c>s are in the range returned by
- <seealso marker="#type_no_range/1">type_no_range/1</seealso> on
- this specific memory allocation map. If <c>TypeNo</c> is an
- invalid integer, <c>invalid_type</c> is returned.</p>
+ <p>Shorthand for
+ <seealso marker="#carriers/1"><c>carriers(#{})</c>.</seealso></p>
</desc>
</func>
+
<func>
- <name>type_no_range(MemoryData) -> {Min, Max}</name>
- <fsummary>Returns the memory block type numbers</fsummary>
- <type>
- <v>MemoryData = {term(), AllocList}</v>
- <v>AllocList = [Desc]</v>
- <v>Desc = {int(), int(), int(), PidDesc}</v>
- <v>PidDesc = {int(), int(), int()} | undefined</v>
- <v>Min = int()</v>
- <v>Max = int()</v>
- </type>
+ <name name="carriers" arity="1"/>
+ <fsummary>Return a list of all carriers filtered by allocator type and
+ scheduler id.</fsummary>
<desc>
- <marker id="type_no_range_1"></marker>
- <p>Returns the memory block type number range used in
- <c>MemoryData</c>. When the memory allocation map was generated
- by an Erlang 5.3/OTP R9C or newer emulator, all integers <c>T</c>
- that satisfy <c>Min</c> &lt;= <c>T</c> &lt;= <c>Max</c> are
- valid type numbers. When the memory allocation map was generated
- by a pre Erlang 5.3/OTP R9C emulator, all integers in the
- range are <em>not</em> valid type numbers.</p>
+ <p>Returns a summary of all carriers in the system, optionally filtered
+ by allocator type and scheduler id.</p>
+ <p>If the specified allocator types are not enabled, the call will fail
+ with <c>{error, not_enabled}</c>.</p>
+ <p>The following options can be used:</p>
+ <taglist>
+ <tag><c>allocator_types</c></tag>
+ <item>
+ <p>The allocator types that will be searched. Defaults to all
+ <c>alloc_util</c> allocators.</p>
+ </item>
+ <tag><c>scheduler_ids</c></tag>
+ <item>
+ <p>The scheduler ids whose allocator instances will be searched. A
+ scheduler id of 0 will refer to the global instance that is not
+ tied to any particular scheduler. Defaults to all schedulers and
+ the global instance.</p>
+ </item>
+ <tag><c>histogram_start</c></tag>
+ <item>
+ <p>The upper bound of the first interval in the free block size
+ histograms. Defaults to 512.</p>
+ </item>
+ <tag><c>histogram_width</c></tag>
+ <item>
+ <p>The number of intervals in the free block size histograms.
+ Defaults to 14.</p>
+ </item>
+ </taglist>
+ <p><em>Example:</em></p>
+ <code type="none"><![CDATA[
+> instrument:carriers(#{ histogram_start => 512, histogram_width => 8 }).
+{ok,{512,
+ [{ll_alloc,1048576,0,1048344,71,false,{0,0,0,0,0,0,0,0}},
+ {binary_alloc,1048576,0,324640,13,false,{3,0,0,1,0,0,0,2}},
+ {eheap_alloc,2097152,0,1037200,45,false,{2,1,1,3,4,3,2,2}},
+ {fix_alloc,32768,0,29544,82,false,{22,0,0,0,0,0,0,0}},
+ {...}|...]}}
+ ]]></code>
</desc>
</func>
+
</funcs>
<section>
diff --git a/lib/tools/doc/src/specs.xml b/lib/tools/doc/src/specs.xml
new file mode 100644
index 0000000000..0b5b7b171c
--- /dev/null
+++ b/lib/tools/doc/src/specs.xml
@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<specs xmlns:xi="http://www.w3.org/2001/XInclude">
+ <xi:include href="../specs/specs_fprof.xml"/>
+ <xi:include href="../specs/specs_make.xml"/>
+ <xi:include href="../specs/specs_lcnt.xml"/>
+ <xi:include href="../specs/specs_eprof.xml"/>
+ <xi:include href="../specs/specs_tags.xml"/>
+ <xi:include href="../specs/specs_cover.xml"/>
+ <xi:include href="../specs/specs_xref.xml"/>
+ <xi:include href="../specs/specs_instrument.xml"/>
+ <xi:include href="../specs/specs_erlang_mode.xml"/>
+</specs>
diff --git a/lib/tools/src/instrument.erl b/lib/tools/src/instrument.erl
index 055f4a7afb..0203fefe13 100644
--- a/lib/tools/src/instrument.erl
+++ b/lib/tools/src/instrument.erl
@@ -1,7 +1,7 @@
%%
%% %CopyrightBegin%
%%
-%% Copyright Ericsson AB 1998-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1998-2018. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
@@ -19,410 +19,140 @@
%%
-module(instrument).
--export([holes/1, mem_limits/1, memory_data/0, read_memory_data/1,
- sort/1, store_memory_data/1, sum_blocks/1,
- descr/1, type_descr/2, allocator_descr/2, class_descr/2,
- type_no_range/1, block_header_size/1, store_memory_status/1,
- read_memory_status/1, memory_status/1]).
-
-
--define(OLD_INFO_SIZE, 32). %% (sizeof(mem_link) in pre R9C utils.c)
-
--define(IHMARKER(H), element(1, H)).
--define(VSN(H), element(2, H)).
--define(INFO_SIZE(H), element(3, H)).
--define(TYPEMAP(H), element(4, H)).
-
--define(IHDR(H), is_tuple(H), ?IHMARKER(H) =:= instr_hdr).
--define(IHDRVSN(H, V), ?IHDR(H), ?VSN(H) =:= V).
-
-memory_data() ->
- case catch erlang:system_info(allocated) of
- {'EXIT',{Error,_}} ->
- erlang:error(Error, []);
- {'EXIT',Error} ->
- erlang:error(Error, []);
- Res ->
- Res
+-export([allocations/0, allocations/1,
+ carriers/0, carriers/1]).
+
+-type block_histogram() :: tuple().
+
+-type allocation_summary() ::
+ {HistogramStart :: non_neg_integer(),
+ UnscannedSize :: non_neg_integer(),
+ Allocations :: #{ Origin :: atom() =>
+ #{ Type :: atom() => block_histogram() }}}.
+
+-spec allocations() -> {ok, Result} | {error, Reason} when
+ Result :: allocation_summary(),
+ Reason :: not_enabled.
+allocations() ->
+ allocations(#{}).
+
+-spec allocations(Options) -> {ok, Result} | {error, Reason} when
+ Result :: allocation_summary(),
+ Reason :: not_enabled,
+ Options :: #{ scheduler_ids => list(non_neg_integer()),
+ allocator_types => list(atom()),
+ histogram_start => pos_integer(),
+ histogram_width => pos_integer() }.
+allocations(Options) ->
+ Ref = make_ref(),
+
+ Defaults = #{ scheduler_ids => lists:seq(0, erlang:system_info(schedulers)),
+ allocator_types => erlang:system_info(alloc_util_allocators),
+ histogram_start => 128,
+ histogram_width => 18 },
+
+ {HistStart, MsgCount} =
+ dispatch_gather(maps:merge(Defaults, Options), Ref,
+ fun erts_internal:gather_alloc_histograms/1),
+
+ alloc_hist_receive(HistStart, MsgCount, Ref).
+
+alloc_hist_receive(_HistStart, 0, _Ref) ->
+ {error, not_enabled};
+alloc_hist_receive(HistStart, MsgCount, Ref) when MsgCount > 0 ->
+ {Unscanned, Histograms} = alloc_hist_receive_1(MsgCount, Ref, 0, #{}),
+ {ok, {HistStart, Unscanned, Histograms}}.
+
+alloc_hist_receive_1(0, _Ref, Unscanned, Result) ->
+ {Unscanned, Result};
+alloc_hist_receive_1(MsgCount, Ref, Unscanned0, Result0) ->
+ receive
+ {Ref, Unscanned, Tags} ->
+ Result = lists:foldl(fun alloc_hist_fold_result/2, Result0, Tags),
+ alloc_hist_receive_1(MsgCount - 1, Ref, Unscanned0 + Unscanned, Result)
end.
-store_memory_data(File) ->
- case catch erlang:system_info({allocated, File}) of
- {'EXIT',{Error,_}} ->
- erlang:error(Error, [File]);
- {'EXIT',Error} ->
- erlang:error(Error, [File]);
- Res ->
- Res
+alloc_hist_fold_result({Id, Type, BlockHist}, Result0) ->
+ IdAllocs0 = maps:get(Id, Result0, #{}),
+ MergedHists = case maps:find(Type, IdAllocs0) of
+ {ok, PrevHist} ->
+ alloc_hist_merge_hist(tuple_size(BlockHist),
+ BlockHist,
+ PrevHist);
+ error ->
+ BlockHist
+ end,
+ IdAllocs = IdAllocs0#{ Type => MergedHists },
+ Result0#{ Id => IdAllocs }.
+
+alloc_hist_merge_hist(0, A, _B) ->
+ A;
+alloc_hist_merge_hist(Index, A, B) ->
+ Merged = setelement(Index, A, element(Index, A) + element(Index, B)),
+ alloc_hist_merge_hist(Index - 1, Merged, B).
+
+-type carrier_info_list() ::
+ {HistogramStart :: non_neg_integer(),
+ Carriers :: [{AllocatorType :: atom(),
+ TotalSize :: non_neg_integer(),
+ UnscannedSize :: non_neg_integer(),
+ AllocatedSize :: non_neg_integer(),
+ AllocatedCount :: non_neg_integer(),
+ InPool :: boolean(),
+ FreeBlocks :: block_histogram()}]}.
+
+-spec carriers() -> {ok, Result} | {error, Reason} when
+ Result :: carrier_info_list(),
+ Reason :: not_enabled.
+carriers() ->
+ carriers(#{}).
+
+-spec carriers(Options) -> {ok, Result} | {error, Reason} when
+ Result :: carrier_info_list(),
+ Reason :: not_enabled,
+ Options :: #{ scheduler_ids => list(non_neg_integer()),
+ allocator_types => list(atom()),
+ histogram_start => pos_integer(),
+ histogram_width => pos_integer() }.
+carriers(Options) ->
+ Ref = make_ref(),
+
+ Defaults = #{ scheduler_ids => lists:seq(0, erlang:system_info(schedulers)),
+ allocator_types => erlang:system_info(alloc_util_allocators),
+ histogram_start => 512,
+ histogram_width => 14 },
+
+ {HistStart, MsgCount} =
+ dispatch_gather(maps:merge(Defaults, Options), Ref,
+ fun erts_internal:gather_carrier_info/1),
+
+ carrier_info_receive(HistStart, MsgCount, Ref).
+
+carrier_info_receive(_HistStart, 0, _Ref) ->
+ {error, not_enabled};
+carrier_info_receive(HistStart, MsgCount, Ref) ->
+ {ok, {HistStart, carrier_info_receive_1(MsgCount, Ref, [])}}.
+
+carrier_info_receive_1(0, _Ref, Result) ->
+ lists:flatten(Result);
+carrier_info_receive_1(MsgCount, Ref, Result0) ->
+ receive
+ {Ref, Carriers} ->
+ carrier_info_receive_1(MsgCount - 1, Ref, [Carriers, Result0])
end.
-memory_status(Type) when is_atom(Type) ->
- case catch erlang:system_info({allocated, status, Type}) of
- {'EXIT',{Error,_}} ->
- erlang:error(Error, [Type]);
- {'EXIT',Error} ->
- erlang:error(Error, [Type]);
- Res ->
- Res
- end;
-memory_status(Type) ->
- erlang:error(badarg, [Type]).
-
-store_memory_status(File) when is_list(File) ->
- case catch erlang:system_info({allocated, status, File}) of
- {'EXIT',{Error,_}} ->
- erlang:error(Error, [File]);
- {'EXIT',Error} ->
- erlang:error(Error, [File]);
- Res ->
- Res
- end;
-store_memory_status(File) ->
- erlang:error(badarg, [File]).
-
-read_memory_data(File) when is_list(File) ->
- case file:consult(File) of
- {ok, [Hdr|MD]} when ?IHDR(Hdr) ->
- {Hdr, MD};
- {ok, [{T,A,S,undefined}|_] = MD} when is_integer(T),
- is_integer(A),
- is_integer(S) ->
- {{instr_hdr, 1, ?OLD_INFO_SIZE}, MD};
- {ok, [{T,A,S,{X,Y,Z}}|_] = MD} when is_integer(T),
- is_integer(A),
- is_integer(S),
- is_integer(X),
- is_integer(Y),
- is_integer(Z) ->
- {{instr_hdr, 1, ?OLD_INFO_SIZE}, MD};
- {ok, _} ->
- {error, eio};
- Error ->
- Error
- end;
-read_memory_data(File) ->
- erlang:error(badarg, [File]).
-
-read_memory_status(File) when is_list(File) ->
- case file:consult(File) of
- {ok, [{instr_vsn, _}|Stat]} ->
- Stat;
- {ok, _} ->
- {error, eio};
- Error ->
- Error
- end;
-read_memory_status(File) ->
- erlang:error(badarg, [File]).
-
-holes({Hdr, MD}) when ?IHDR(Hdr) ->
- check_holes(?INFO_SIZE(Hdr), MD).
-
-check_holes(_ISz, []) ->
- ok;
-check_holes(ISz, [E | L]) ->
- check_holes(ISz, E, L).
-
-check_holes(_ISz, _E1, []) ->
- io:format("~n");
-check_holes(ISz, E1, [E2 | Rest]) ->
- check_hole(ISz, E1, E2),
- check_holes(ISz, E2, Rest).
-
-check_hole(ISz, {_,P1,S1,_}, {_,P2,_,_}) ->
- End = P1+S1,
- Hole = P2 - (End + ISz),
- if
- Hole =< 7 ->
- ok;
- true ->
- io:format(" ~p", [Hole])
- end.
-
-sum_blocks({Hdr, L}) when ?IHDR(Hdr) ->
- lists:foldl(fun({_,_,S,_}, Sum) -> S+Sum end,
- 0,
- L).
-
-mem_limits({Hdr, L}) when ?IHDR(Hdr) ->
- {_, P1, _, _} = hd(L),
- {_, P2, S2, _} = lists:last(L),
- {P1, P2+S2}.
-
-sort({Hdr, MD}) when ?IHDR(Hdr) ->
- {Hdr, lists:keysort(2, MD)}.
-
-descr({Hdr, MD} = ID) when ?IHDR(Hdr) ->
- {Hdr, lists:map(fun ({TN, Addr, Sz, {0, N, S}}) ->
- {type_descr(ID, TN),
- Addr,
- Sz,
- list_to_pid("<0."
- ++ integer_to_list(N)
- ++ "."
- ++ integer_to_list(S)
- ++ ">")};
- ({TN, Addr, Sz, undefined}) ->
- {type_descr(ID, TN),
- Addr,
- Sz,
- undefined}
- end,
- MD)}.
-
-block_header_size({Hdr, _}) when ?IHDR(Hdr) ->
- ?INFO_SIZE(Hdr).
-
-type_descr({Hdr, _}, TypeNo) when ?IHDRVSN(Hdr, 2),
- is_integer(TypeNo) ->
- case catch element(1, element(TypeNo, ?TYPEMAP(Hdr))) of
- {'EXIT', _} -> invalid_type;
- Type -> Type
- end;
-type_descr({Hdr, _}, TypeNo) when ?IHDRVSN(Hdr, 1),
- is_integer(TypeNo) ->
- type_string(TypeNo).
-
-
-allocator_descr({Hdr, _}, TypeNo) when ?IHDRVSN(Hdr, 2), is_integer(TypeNo) ->
- case catch element(2, element(TypeNo, ?TYPEMAP(Hdr))) of
- {'EXIT', _} -> invalid_type;
- Type -> Type
- end;
-allocator_descr({Hdr, _}, TypeNo) when ?IHDRVSN(Hdr, 1), is_integer(TypeNo) ->
- "unknown".
-
-class_descr({Hdr, _}, TypeNo) when ?IHDRVSN(Hdr, 2), is_integer(TypeNo) ->
- case catch element(3, element(TypeNo, ?TYPEMAP(Hdr))) of
- {'EXIT', _} -> invalid_type;
- Type -> Type
- end;
-class_descr({Hdr, _}, TypeNo) when ?IHDRVSN(Hdr, 1), is_integer(TypeNo) ->
- "unknown".
-
-type_no_range({Hdr, _}) when ?IHDRVSN(Hdr, 2) ->
- {1, tuple_size(?TYPEMAP(Hdr))};
-type_no_range({Hdr, _}) when ?IHDRVSN(Hdr, 1) ->
- {-1, 1000}.
-
-type_string(-1) ->
- "unknown";
-type_string(1) ->
- "atom text";
-type_string(11) ->
- "atom desc";
-type_string(2) ->
- "bignum (big_to_list)";
-type_string(31) ->
- "fixalloc";
-type_string(32) ->
- "unknown fixalloc block";
-type_string(33) ->
- "message buffer";
-type_string(34) ->
- "message link";
-type_string(4) ->
- "estack";
-type_string(40) ->
- "db table vec";
-type_string(41) ->
- "db tree select buffer";
-type_string(43) ->
- "db hash select buffer";
-type_string(44) ->
- "db hash select list";
-type_string(45) ->
- "db match prog stack";
-type_string(46) ->
- "db match prog heap data";
-type_string(47) ->
- "db temp buffer";
-type_string(48) ->
- "db error";
-type_string(49) ->
- "db error info";
-type_string(50) ->
- "db trans tab";
-type_string(51) ->
- "db segment";
-type_string(52) ->
- "db term";
-type_string(53) ->
- "db add_counter";
-type_string(54) ->
- "db segment table";
-type_string(55) ->
- "db table (fix)";
-type_string(56) ->
- "db bindings";
-type_string(57) ->
- "db counter";
-type_string(58) ->
- "db trace vec";
-type_string(59) ->
- "db fixed deletion";
-type_string(60) ->
- "binary (external.c)";
-type_string(61) ->
- "binary";
-type_string(62) ->
- "procbin (fix)";
-type_string(70) ->
- "driver alloc (io.c)";
-type_string(71) ->
- "binary (io.c)";
-type_string(72) ->
- "binary vec (io.c)";
-type_string(73) ->
- "binary vec 2 (io.c)";
-type_string(74) ->
- "io vec (io.c)";
-type_string(75) ->
- "io vec 2 (io.c)";
-type_string(76) ->
- "temp io buffer (io.c)";
-type_string(77) ->
- "temp io buffer 2 (io.c)";
-type_string(78) ->
- "line buffer (io.c)";
-type_string(8) ->
- "heap";
-type_string(801) ->
- "heap (1)";
-type_string(802) ->
- "heap (2)";
-type_string(803) ->
- "heap (3)";
-type_string(804) ->
- "heap (4)";
-type_string(805) ->
- "heap (5)";
-type_string(821) ->
- "heap fragment (1)";
-type_string(822) ->
- "heap fragment (2)";
-type_string(830) ->
- "sequential store buffer (for vectors)";
-type_string(91) ->
- "process table";
-type_string(92) ->
- "process desc";
-type_string(110) ->
- "hash buckets";
-type_string(111) ->
- "hash table";
-type_string(120) ->
- "index init";
-type_string(121) ->
- "index table";
-type_string(130) ->
- "temp buffer";
-type_string(140) ->
- "timer wheel";
-type_string(150) ->
- "distribution cache";
-type_string(151) ->
- "dmem";
-type_string(152) ->
- "distribution table";
-type_string(153) ->
- "distribution table buckets";
-type_string(154) ->
- "distribution table entry";
-type_string(155) ->
- "node table";
-type_string(156) ->
- "node table buckets";
-type_string(157) ->
- "node table entry";
-type_string(160) ->
- "port table";
-type_string(161) ->
- "driver entry";
-type_string(162) ->
- "port setup";
-type_string(163) ->
- "port wait";
-type_string(170) ->
- "module";
-type_string(171) ->
- "fundef";
-type_string(180) ->
- "file table";
-type_string(181) ->
- "driver table";
-type_string(182) ->
- "poll struct";
-type_string(190) ->
- "inet driver";
-type_string(200) ->
- "efile driver";
-type_string(210) ->
- "gc root set";
-type_string(220) ->
- "breakpoint data";
-type_string(230) ->
- "async queue";
-type_string(231) ->
- "async (exit)";
-type_string(232) ->
- "async (driver)";
-type_string(240) ->
- "bits buffer";
-type_string(241) ->
- "bits temp buffer";
-type_string(250) ->
- "modules (loader)";
-type_string(251) ->
- "code (loader)";
-type_string(252) ->
- "atom tab (loader)";
-type_string(253) ->
- "import tab (loader)";
-type_string(254) ->
- "export tab (loader)";
-type_string(255) ->
- "lable tab (loader)";
-type_string(256) ->
- "gen op (loader)";
-type_string(257) ->
- "gen op args (loader)";
-type_string(258) ->
- "gen op args 2 (loader)";
-type_string(259) ->
- "gen op args 3 (loader)";
-type_string(260) ->
- "lambdas (loader)";
-type_string(261) ->
- "temp int buffer (loader)";
-type_string(262) ->
- "temp heap (loader)";
-type_string(280) ->
- "dist ctrl msg buffer";
-type_string(281) ->
- "dist_buf";
-type_string(290) ->
- "call trace buffer";
-type_string(300) ->
- "bif timer rec";
-type_string(310) ->
- "argument registers";
-type_string(320) ->
- "compressed binary temp buffer";
-type_string(330) ->
- "term_to_binary temp buffer";
-type_string(340) ->
- "proc dict";
-type_string(350) ->
- "trace to port temp buffer";
-type_string(360) ->
- "lists subtract temp buffer";
-type_string(370) ->
- "link (lh)";
-type_string(380) ->
- "port call buffer";
-type_string(400) ->
- "definite_alloc block";
-type_string(_) ->
- invalid_type.
-
+dispatch_gather(#{ allocator_types := AllocatorTypes,
+ scheduler_ids := SchedulerIds,
+ histogram_start := HistStart,
+ histogram_width := HistWidth }, Ref, Gather)
+ when is_list(AllocatorTypes),
+ is_list(SchedulerIds),
+ HistStart >= 1, HistStart =< (1 bsl 28),
+ HistWidth >= 1, HistWidth =< 32 ->
+ MsgCount = lists:sum(
+ [Gather({AllocatorType, SchedId, HistWidth, HistStart, Ref}) ||
+ SchedId <- SchedulerIds,
+ AllocatorType <- AllocatorTypes]),
+ {HistStart, MsgCount};
+dispatch_gather(_, _, _) ->
+ error(badarg).
diff --git a/lib/tools/test/instrument_SUITE.erl b/lib/tools/test/instrument_SUITE.erl
index f37d28c277..8c521b2e1a 100644
--- a/lib/tools/test/instrument_SUITE.erl
+++ b/lib/tools/test/instrument_SUITE.erl
@@ -1,7 +1,7 @@
%%
%% %CopyrightBegin%
%%
-%% Copyright Ericsson AB 1998-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1998-2018. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
@@ -20,89 +20,274 @@
-module(instrument_SUITE).
-export([all/0, suite/0]).
--export(['+Mim true'/1, '+Mis true'/1]).
+
+-export([allocations_enabled/1, allocations_disabled/1, allocations_ramv/1,
+ carriers_enabled/1, carriers_disabled/1]).
+
+-export([test_all_alloc/2, test_per_alloc/2, test_format/3, test_abort/1,
+ generate_test_blocks/0, churn_memory/0]).
-include_lib("common_test/include/ct.hrl").
suite() ->
[{ct_hooks,[ts_install_cth]},
- {timetrap,{seconds,10}}].
+ {timetrap,{minutes,5}}].
all() ->
- ['+Mim true', '+Mis true'].
-
-
-%% Check that memory data can be read and processed
-'+Mim true'(Config) when is_list(Config) ->
- Node = start_slave("+Mim true"),
- MD = rpc:call(Node, instrument, memory_data, []),
- [{total,[{sizes,S1,S2,S3},{blocks,B1,B2,B3}]}]
- = rpc:call(Node, instrument, memory_status, [total]),
- stop_slave(Node),
- true = S1 =< S2,
- true = S2 =< S3,
- true = B1 =< B2,
- true = B2 =< B3,
- MDS = instrument:sort(MD),
- {Low, High} = instrument:mem_limits(MDS),
- true = Low < High,
- {_, AL} = MDS,
- SumBlocks = instrument:sum_blocks(MD),
- case SumBlocks of
- N when is_integer(N) ->
- N = lists:foldl(fun ({_,_,Size,_}, Sum) ->
- Size+Sum
- end, 0, AL),
- true = N =< S3;
- Other ->
- ct:fail(Other)
+ [allocations_enabled, allocations_disabled, allocations_ramv,
+ carriers_enabled, carriers_disabled].
+
+-define(GENERATED_SBC_BLOCK_COUNT, 1000).
+-define(GENERATED_MBC_BLOCK_COUNT, ?GENERATED_SBC_BLOCK_COUNT).
+
+-define(GENERATED_BLOCK_COUNT, (?GENERATED_SBC_BLOCK_COUNT +
+ ?GENERATED_MBC_BLOCK_COUNT)).
+-define(GENERATED_CARRIER_COUNT, ?GENERATED_SBC_BLOCK_COUNT).
+
+allocations_test(Args, Plain, PerAlloc) ->
+ run_test(Args, fun(Node) ->
+ ok = rpc:call(Node, ?MODULE, test_all_alloc,
+ [fun instrument:allocations/0, Plain]),
+ ok = rpc:call(Node, ?MODULE, test_per_alloc,
+ [fun instrument:allocations/1, PerAlloc]),
+ ok = rpc:call(Node, ?MODULE, test_format,
+ [#{ histogram_start => 512,
+ histogram_width => 4 },
+ fun instrument:allocations/1,
+ fun verify_allocations_output/2]),
+ ok = rpc:call(Node, ?MODULE, test_abort,
+ [fun erts_internal:gather_alloc_histograms/1])
+ end).
+
+allocations_enabled(Config) when is_list(Config) ->
+ allocations_test("+Meamax +Muatags true",
+ fun verify_allocations_enabled/1,
+ fun verify_allocations_enabled/2).
+
+allocations_disabled(Config) when is_list(Config) ->
+ allocations_test("+Meamax +Muatags false",
+ fun verify_allocations_disabled/1,
+ fun verify_allocations_disabled/2).
+
+allocations_ramv(Config) when is_list(Config) ->
+ allocations_test("+Meamax +Muatags true +Muramv true",
+ fun verify_allocations_enabled/1,
+ fun verify_allocations_enabled/2).
+
+verify_allocations_disabled(_AllocType, Result) ->
+ verify_allocations_disabled(Result).
+
+verify_allocations_disabled({error, not_enabled}) ->
+ ok.
+
+%% Skip types that have unstable results or are unaffected by +Muatags
+verify_allocations_enabled(literal_alloc, _Result) -> ok;
+verify_allocations_enabled(exec_alloc, _Result) -> ok;
+verify_allocations_enabled(temp_alloc, _Result) -> ok;
+verify_allocations_enabled(sl_alloc, _Result) -> ok;
+verify_allocations_enabled(_AllocType, Result) ->
+ verify_allocations_enabled(Result).
+
+verify_allocations_enabled({ok, {_HistStart, _UnscannedBytes, Allocs}}) ->
+ true = Allocs =/= #{}.
+
+verify_allocations_output(#{ histogram_start := HistStart,
+ histogram_width := HistWidth },
+ {ok, {HistStart, _UnscannedBytes, ByOrigin}}) ->
+ AllHistograms = lists:flatten([maps:values(ByType) ||
+ ByType <- maps:values(ByOrigin)]),
+
+ %% Do the histograms look alright?
+ HistogramSet = ordsets:from_list(AllHistograms),
+ Verified = [H || H <- HistogramSet,
+ tuple_size(H) =:= HistWidth,
+ hist_sum(H) >= 1],
+ [] = ordsets:subtract(HistogramSet, Verified),
+
+ %% Do we have at least as many blocks as we've generated?
+ BlockCount = lists:foldl(fun(Hist, Acc) ->
+ hist_sum(Hist) + Acc
+ end, 0, AllHistograms),
+ GenTotalBlockCount = ?GENERATED_BLOCK_COUNT,
+ GenSBCBlockCount = ?GENERATED_SBC_BLOCK_COUNT,
+ if
+ BlockCount < GenSBCBlockCount ->
+ ct:fail("Found ~p blocks, required at least ~p (SB)." ,
+ [BlockCount, GenSBCBlockCount]);
+ BlockCount >= GenTotalBlockCount ->
+ ct:pal("Found ~p blocks, expected at least ~p (SB + MB).",
+ [BlockCount, GenTotalBlockCount]);
+ BlockCount < GenTotalBlockCount ->
+ ct:pal("Found ~p blocks, expected at least ~p (SB + MB), but this "
+ "may be due to MBCs being skipped if they're about to be "
+ "scanned just as they're fetched from the carrier pool.",
+ [BlockCount, GenTotalBlockCount])
+ end,
+
+ ok;
+verify_allocations_output(#{}, {error, not_enabled}) ->
+ ok.
+
+%% %% %% %% %% %%
+
+carriers_test(Args, Plain, PerAlloc) ->
+ run_test(Args, fun(Node) ->
+ ok = rpc:call(Node, ?MODULE, test_all_alloc,
+ [fun instrument:carriers/0, Plain]),
+ ok = rpc:call(Node, ?MODULE, test_per_alloc,
+ [fun instrument:carriers/1, PerAlloc]),
+ ok = rpc:call(Node, ?MODULE, test_format,
+ [#{ histogram_start => 1024,
+ histogram_width => 4 },
+ fun instrument:carriers/1,
+ fun verify_carriers_output/2]),
+ ok = rpc:call(Node, ?MODULE, test_abort,
+ [fun erts_internal:gather_carrier_info/1])
+ end).
+
+carriers_enabled(Config) when is_list(Config) ->
+ carriers_test("+Meamax",
+ fun verify_carriers_enabled/1,
+ fun verify_carriers_enabled/2).
+
+carriers_disabled(Config) when is_list(Config) ->
+ carriers_test("+Meamin",
+ fun verify_carriers_disabled/1,
+ fun verify_carriers_disabled/2).
+
+verify_carriers_disabled(_AllocType, Result) ->
+ verify_carriers_disabled(Result).
+
+verify_carriers_disabled({error, not_enabled}) ->
+ ok;
+verify_carriers_disabled({ok, {_HistStart, Carriers}}) ->
+ verify_carriers_disabled_1(Carriers).
+
+verify_carriers_disabled_1([]) ->
+ ok;
+%% literal_alloc, exec_alloc, and temp_alloc can't be disabled, so we have to
+%% accept their presence in carriers_disabled/test_all_alloc.
+verify_carriers_disabled_1([Carrier | Rest]) when
+ element(1, Carrier) =:= literal_alloc;
+ element(1, Carrier) =:= exec_alloc;
+ element(1, Carrier) =:= temp_alloc ->
+ verify_carriers_disabled_1(Rest).
+
+%% exec_alloc only has a carrier if it's actually used.
+verify_carriers_enabled(exec_alloc, _Result) -> ok;
+verify_carriers_enabled(_AllocType, Result) -> verify_carriers_enabled(Result).
+
+verify_carriers_enabled({ok, {_HistStart, Carriers}}) when Carriers =/= [] ->
+ ok.
+
+verify_carriers_output(#{ histogram_start := HistStart,
+ histogram_width := HistWidth },
+ {ok, {HistStart, AllCarriers}}) ->
+
+ %% Do the carriers look alright?
+ CarrierSet = ordsets:from_list(AllCarriers),
+ Verified = [C || {AllocType,
+ TotalSize,
+ UnscannedSize,
+ AllocatedSize,
+ AllocatedCount,
+ InPool,
+ FreeBlockHist} = C <- CarrierSet,
+ is_atom(AllocType),
+ is_integer(TotalSize), TotalSize >= 1,
+ is_integer(UnscannedSize), UnscannedSize < TotalSize,
+ UnscannedSize >= 0,
+ is_integer(AllocatedSize), AllocatedSize < TotalSize,
+ AllocatedSize >= 0,
+ is_integer(AllocatedCount), AllocatedCount =< AllocatedSize,
+ AllocatedCount >= 0,
+ is_boolean(InPool),
+ tuple_size(FreeBlockHist) =:= HistWidth,
+ carrier_block_check(AllocatedCount, FreeBlockHist)],
+ [] = ordsets:subtract(CarrierSet, Verified),
+
+ %% Do we have at least as many carriers as we've generated?
+ CarrierCount = length(AllCarriers),
+ GenSBCCount = ?GENERATED_SBC_BLOCK_COUNT,
+ if
+ CarrierCount < GenSBCCount ->
+ ct:fail("Carrier count is ~p, expected at least ~p (SBC).",
+ [CarrierCount, GenSBCCount]);
+ CarrierCount >= GenSBCCount ->
+ ok
end,
- lists:foldl(
- fun ({TDescr,Addr,Size,Proc}, MinAddr) ->
- true = TDescr /= invalid_type,
- true = is_integer(TDescr),
- true = is_integer(Addr),
- true = is_integer(Size),
- true = Addr >= MinAddr,
- case Proc of
- {0, Number, Serial} ->
- true = is_integer(Number),
- true = is_integer(Serial);
- undefined ->
- ok;
- BadProc ->
- ct:fail({badproc, BadProc})
- end,
- NextMinAddr = Addr+Size,
- true = NextMinAddr =< High,
- NextMinAddr
- end, Low, AL),
- {_, DAL} = instrument:descr(MDS),
- lists:foreach(
- fun ({TDescr,_,_,Proc}) ->
- true = TDescr /= invalid_type,
- true = is_atom(TDescr) orelse is_list(TDescr),
- true = is_pid(Proc) orelse Proc == undefined
- end, DAL),
- ASL = lists:map(fun ({_,A,S,_}) -> {A,S} end, AL),
- ASL = lists:map(fun ({_,A,S,_}) -> {A,S} end, DAL),
- instrument:holes(MDS),
- {comment, "total status - sum of blocks = " ++ integer_to_list(S1-SumBlocks)}.
-
-%% Check that memory data can be read and processed
-'+Mis true'(Config) when is_list(Config) ->
- Node = start_slave("+Mis true"),
- [{total,[{sizes,S1,S2,S3},{blocks,B1,B2,B3}]}]
- = rpc:call(Node, instrument, memory_status, [total]),
- true = S1 =< S2,
- true = S2 =< S3,
- true = B1 =< B2,
- true = B2 =< B3,
- true = is_list(rpc:call(Node,instrument,memory_status,[allocators])),
- true = is_list(rpc:call(Node,instrument,memory_status,[classes])),
- true = is_list(rpc:call(Node,instrument,memory_status,[types])),
+
+ ok;
+verify_carriers_output(#{}, {error, not_enabled}) ->
+ ok.
+
+carrier_block_check(AllocCount, FreeHist) ->
+ %% A carrier must contain at least one block, and th. number of free blocks
+ %% must not exceed the number of allocated blocks + 1.
+ FreeCount = hist_sum(FreeHist),
+
+ (AllocCount + FreeCount) >= 1 andalso FreeCount =< (AllocCount + 1).
+
+%% %% %% %% %% %%
+
+test_all_alloc(Gather, Verify) ->
+ Verify(Gather()),
ok.
+test_per_alloc(Gather, Verify) ->
+ [begin
+ Verify(T, Gather(#{ allocator_types => [T] }))
+ end || T <- erlang:system_info(alloc_util_allocators)],
+ ok.
+
+test_format(#{ allocator_types := _ }, _, _) ->
+ error(badarg);
+test_format(Options0, Gather, Verify) ->
+ %% We limit format checking to binary_alloc since we generated the test
+ %% vectors there.
+ Options = Options0#{ allocator_types => [binary_alloc] },
+ Verify(Options, Gather(Options)),
+ ok.
+
+test_abort(Gather) ->
+ %% There's no way for us to tell whether this actually aborted or ran to
+ %% completion, but it might catch a few segfaults.
+ Runner = self(),
+ Ref = make_ref(),
+ spawn_opt(fun() ->
+ [Gather({Type, SchedId, 1, 1, Ref}) ||
+ Type <- erlang:system_info(alloc_util_allocators),
+ SchedId <- lists:seq(0, erlang:system_info(schedulers))],
+ Runner ! Ref
+ end, [{priority, max}]),
+ receive
+ Ref -> ok
+ end.
+
+hist_sum(H) -> hist_sum_1(H, tuple_size(H), 0).
+hist_sum_1(_H, 0, A) -> A;
+hist_sum_1(H, N, A) -> hist_sum_1(H, N - 1, element(N, H) + A).
+
+%%
+
+run_test(Args0, Test) ->
+ %% Override single-block carrier threshold for binaries to ensure we have
+ %% coverage for that path. generate_test_blocks builds a few binaries that
+ %% crosses this threshold.
+ %%
+ %% We also set the abandon carrier threshold to 70% to provoke more
+ %% activity in the carrier pool.
+ Args = Args0 ++ " +MBsbct 1 +Muacul 70",
+ Node = start_slave(Args),
+
+ ok = rpc:call(Node, ?MODULE, generate_test_blocks, []),
+ ok = Test(Node),
+
+ ok = rpc:call(Node, ?MODULE, churn_memory, []),
+ ok = Test(Node),
+
+ true = test_server:stop_node(Node).
+
start_slave(Args) ->
MicroSecs = erlang:monotonic_time(),
Name = "instr" ++ integer_to_list(MicroSecs),
@@ -112,6 +297,60 @@ start_slave(Args) ->
[{args, "-pa " ++ Pa ++ " " ++ Args}]),
Node.
+generate_test_blocks() ->
+ Runner = self(),
+ Ref = make_ref(),
+ spawn(fun() ->
+ %% We've set the single-block carrier threshold to 1KB so one
+ %% ought to land in a SBC and the other in a MBC. Both are kept
+ %% alive forever.
+ SBCs = [<<I, 0:(1 bsl 10)/unit:8>> ||
+ I <- lists:seq(1, ?GENERATED_SBC_BLOCK_COUNT)],
+ MBCs = [<<I, 0:64/unit:8>> ||
+ I <- lists:seq(1, ?GENERATED_MBC_BLOCK_COUNT)],
+ Runner ! Ref,
+ receive after infinity -> ok end,
+ unreachable ! {SBCs, MBCs}
+ end),
+ receive
+ Ref -> ok
+ end.
-stop_slave(Node) ->
- true = test_server:stop_node(Node).
+churn_memory() ->
+ %% All processes spawned from here on have 'low' priority to avoid starving
+ %% others (e.g. the rpc process) which could cause the test to time out.
+ [begin
+ churn_list_to_binary(),
+ churn_processes(),
+ churn_ets()
+ end || _ <- lists:seq(1, erlang:system_info(schedulers))],
+ ok.
+
+churn_processes() ->
+ Pid = spawn_opt(fun churn_processes/0, [{priority, low}]),
+ [Pid ! <<I, 0:128/unit:8>> || I <- lists:seq(1, 128)].
+
+%% Nearly all types have a few allocations at all times but sl_alloc is
+%% often empty. list_to_binary on large inputs will yield and spill the
+%% state into an 'estack' which is allocated through sl_alloc.
+%%
+%% This is inherently unstable so we skip the verification step for this
+%% type, but there's still a point to hammering it.
+churn_list_to_binary() ->
+ List = binary_to_list(<<0:(1 bsl 20)/unit:8>>),
+ spawn_opt(fun() -> churn_list_to_binary_1(List) end, [{priority, low}]).
+
+churn_list_to_binary_1(List) ->
+ _ = id(list_to_binary(List)),
+ churn_list_to_binary_1(List).
+
+churn_ets() ->
+ spawn_opt(fun() -> churn_ets_1(ets:new(gurka, [])) end, [{priority, low}]).
+
+churn_ets_1(Tab) ->
+ ets:insert(Tab, {gaffel, lists:seq(1, 16)}),
+ ets:delete_all_objects(Tab),
+ churn_ets_1(Tab).
+
+id(I) ->
+ I.