diff options
author | Lukas Larsson <[email protected]> | 2016-04-21 18:36:45 +0200 |
---|---|---|
committer | Lukas Larsson <[email protected]> | 2016-05-10 08:33:03 +0200 |
commit | e146a3eec5a2d384260aa8829777c89eaab09cbd (patch) | |
tree | 474b3ea6372918215f2b810f0f694d02684d8b58 | |
parent | 5ca152a5ccca92e354188f7b696ca5b9f6b92806 (diff) | |
download | otp-e146a3eec5a2d384260aa8829777c89eaab09cbd.tar.gz otp-e146a3eec5a2d384260aa8829777c89eaab09cbd.tar.bz2 otp-e146a3eec5a2d384260aa8829777c89eaab09cbd.zip |
erts: Implement max_heap_size process flag
The max_heap_size process flag can be used to limit the
growth of a process heap by killing it before it becomes
too large to handle. It is possible to set the maximum
using the `erl +hmax` option, `system_flag(max_heap_size, ...)`,
`spawn_opt(Fun, [{max_heap_size, ...}])` and
`process_flag(max_heap_size, ...)`.
It is possible to configure the behaviour of the process
when the maximum heap size is reached. The process may be
sent an untrappable exit signal with reason kill and/or
send an error_logger message with details on the process
state. A new trace event called gc_max_heap_size is
also triggered for the garbage_collection trace flag
when the heap grows larger than the configured size.
If kill and error_logger are disabled, it is still
possible to see that the maximum has been reached by
doing garbage collection tracing on the process.
The heap size is defined as the sum of the heap memory
that the process is currently using. This includes
all generational heaps, the stack, any messages that
are considered to be part of the heap and any extra
memory the garbage collector may need during collection.
In the current implementation this means that when a process
is set using on_heap message queue data mode, the messages
that are in the internal message queue are counted towards
this value. For off_heap, only matched messages count towards
the size of the heap. For mixed, it depends on race conditions
within the VM whether a message is part of the heap or not.
Below is an example run of the new behaviour:
Eshell V8.0 (abort with ^G)
1> f(P),P = spawn_opt(fun() -> receive ok -> ok end end, [{max_heap_size, 512}]).
<0.60.0>
2> erlang:trace(P, true, [garbage_collection, procs]).
1
3> [P ! lists:duplicate(M,M) || M <- lists:seq(1,15)],ok.
ok
4>
=ERROR REPORT==== 26-Apr-2016::16:25:10 ===
Process: <0.60.0>
Context: maximum heap size reached
Max heap size: 512
Total heap size: 723
Kill: true
Error Logger: true
GC Info: [{old_heap_block_size,0},
{heap_block_size,609},
{mbuf_size,145},
{recent_size,0},
{stack_size,9},
{old_heap_size,0},
{heap_size,211},
{bin_vheap_size,0},
{bin_vheap_block_size,46422},
{bin_old_vheap_size,0},
{bin_old_vheap_block_size,46422}]
flush().
Shell got {trace,<0.60.0>,gc_start,
[{old_heap_block_size,0},
{heap_block_size,233},
{mbuf_size,145},
{recent_size,0},
{stack_size,9},
{old_heap_size,0},
{heap_size,211},
{bin_vheap_size,0},
{bin_vheap_block_size,46422},
{bin_old_vheap_size,0},
{bin_old_vheap_block_size,46422}]}
Shell got {trace,<0.60.0>,gc_max_heap_size,
[{old_heap_block_size,0},
{heap_block_size,609},
{mbuf_size,145},
{recent_size,0},
{stack_size,9},
{old_heap_size,0},
{heap_size,211},
{bin_vheap_size,0},
{bin_vheap_block_size,46422},
{bin_old_vheap_size,0},
{bin_old_vheap_block_size,46422}]}
Shell got {trace,<0.60.0>,exit,killed}
-rw-r--r-- | erts/doc/src/erl.xml | 28 | ||||
-rw-r--r-- | erts/doc/src/erlang.xml | 294 | ||||
-rw-r--r-- | erts/emulator/beam/atom.names | 2 | ||||
-rw-r--r-- | erts/emulator/beam/beam_emu.c | 43 | ||||
-rw-r--r-- | erts/emulator/beam/bif.c | 52 | ||||
-rw-r--r-- | erts/emulator/beam/erl_bif_info.c | 49 | ||||
-rw-r--r-- | erts/emulator/beam/erl_gc.c | 246 | ||||
-rw-r--r-- | erts/emulator/beam/erl_gc.h | 4 | ||||
-rw-r--r-- | erts/emulator/beam/erl_init.c | 54 | ||||
-rw-r--r-- | erts/emulator/beam/erl_process.c | 7 | ||||
-rw-r--r-- | erts/emulator/beam/erl_process.h | 12 | ||||
-rw-r--r-- | erts/emulator/beam/erl_trace.c | 21 | ||||
-rw-r--r-- | erts/emulator/beam/erl_trace.h | 2 | ||||
-rw-r--r-- | erts/emulator/beam/erl_vm.h | 3 | ||||
-rw-r--r-- | erts/emulator/test/process_SUITE.erl | 186 | ||||
-rw-r--r-- | erts/etc/common/erlexec.c | 3 | ||||
-rw-r--r-- | erts/preloaded/src/erlang.erl | 17 | ||||
-rw-r--r-- | lib/stdlib/doc/src/proc_lib.xml | 7 | ||||
-rw-r--r-- | lib/stdlib/src/proc_lib.erl | 5 |
19 files changed, 918 insertions, 117 deletions
diff --git a/erts/doc/src/erl.xml b/erts/doc/src/erl.xml index c499fc8081..1bbde7f1e0 100644 --- a/erts/doc/src/erl.xml +++ b/erts/doc/src/erl.xml @@ -627,6 +627,34 @@ <p>Sets the default binary virtual heap size of processes to the size <c><![CDATA[Size]]></c>.</p> </item> + <marker id="+hmax"/> + <tag><c><![CDATA[+hmax Size]]></c></tag> + <item> + <p>Sets the default maximum heap size of processes to the size + <c><![CDATA[Size]]></c>. If <c>+hmax</c> is not given, the default is <c>0</c> + which means that no maximum heap size is used. + For more information, see the documentation of + <seealso marker="erlang#process_flag_max_heap_size"> + <c>process_flag(max_heap_size, MaxHeapSize)</c></seealso>.</p> + </item> + <marker id="+hmaxel"/> + <tag><c><![CDATA[+hmaxel true|false]]></c></tag> + <item> + <p>Sets whether to send an error logger message for processes that reach + the maximum heap size or not. If <c>+hmaxel</c> is not given, the default is <c>true</c>. + For more information, see the documentation of + <seealso marker="erlang#process_flag_max_heap_size"> + <c>process_flag(max_heap_size, MaxHeapSize)</c></seealso>.</p> + </item> + <marker id="+hmaxk"/> + <tag><c><![CDATA[+hmaxk true|false]]></c></tag> + <item> + <p>Sets whether to kill processes that reach the maximum heap size or not. If + <c>+hmaxk</c> is not given, the default is <c>true</c>. For more information, + see the documentation of + <seealso marker="erlang#process_flag_max_heap_size"> + <c>process_flag(max_heap_size, MaxHeapSize)</c></seealso>.</p> + </item> <tag><c><![CDATA[+hpds Size]]></c></tag> <item> <p>Sets the initial process dictionary size of processes to the size diff --git a/erts/doc/src/erlang.xml b/erts/doc/src/erlang.xml index bf30cc7928..3930e5e11d 100644 --- a/erts/doc/src/erlang.xml +++ b/erts/doc/src/erlang.xml @@ -4322,6 +4322,7 @@ os_prompt% </pre> </desc> </func> + <marker id="process_flag_min_heap_size"/> <func> <name name="process_flag" arity="2" clause_i="3"/> <fsummary>Sets process flag <c>min_heap_size</c> for the calling process.</fsummary> @@ -4340,9 +4341,95 @@ os_prompt% </pre> <p>Returns the old value of the flag.</p> </desc> </func> - <marker id="process_flag_message_queue_data"/> + <marker id="process_flag_max_heap_size"/> <func> <name name="process_flag" arity="2" clause_i="5"/> + <type name="max_heap_size"/> + <fsummary>Sets process flag <c>max_heap_size</c> for the calling process.</fsummary> + <desc> + <p> + This flag sets the maximum heap size for the calling process. + If <c><anno>MaxHeapSize</anno></c> is an integer, the system default + values for <c>kill</c> and <c>error_logger</c> are used. + <taglist> + <tag><c>size</c></tag> + <item> + <p> + The maximum size in words of the process. If set to zero, the + heap size limit is disabled. Badarg will be thrown if the value is + smaller than + <seealso marker="#process_flag_min_heap_size"><c>min_heap_size</c></seealso>. + The size check is only done when a garbage collection is triggered. + </p> + <p> + <c>size</c> is the entire heap of the process when garbage collection + is triggered, this includes all generational heaps, the process stack, + any <seealso marker="process_flag_message_queue_data"> + messages that are considered to be part of the heap</seealso> and any + extra memory that the garbage collector needs during collection. + </p> + <p> + <c>size</c> is the same as can be retrieved using + <seealso marker="#process_info_total_heap_size"> + <c>erlang:process_info(Pid, total_heap_size)</c></seealso>, + or by adding <c>heap_block_size</c>, <c>old_heap_block_size</c> + and <c>mbuf_size</c> from <seealso marker="#process_info_garbage_collection_info"> + <c>erlang:process_info(Pid, garbage_collection_info)</c></seealso>. + </p> + </item> + <tag><c>kill</c></tag> + <item> + <p> + When set to <c>true</c> the runtime system will send an + untrappable exit signal with reason <c>kill</c> to the process + if the maximum heap size is reached. The garbage collection + that triggered the <c>kill</c> will not be completed, instead the + process will exit as soon as is possible. When set to <c>false</c> + no exit signal will be sent to the process, instead it will + continue executing. + </p> + <p> + If <c>kill</c> is not defined in the map + the system default will be used. The default system default + is <c>true</c>. It can be changed by either the erl + <seealso marker="erl#+hmaxk">+hmaxk</seealso> option, + or <seealso marker="#system_flag_max_heap_size"><c> + erlang:system_flag(max_heap_size, MaxHeapSize)</c></seealso>. + </p> + </item> + <tag><c>error_logger</c></tag> + <item> + <p> + When set to <c>true</c> the runtime system will send a + message to the current <seealso marker="kernel:error_logger"><c>error_logger</c></seealso> + containing details about the process when the maximum + heap size is reached. One <c>error_logger</c> report will + be sent each time the limit is reached. + </p> + <p> + If <c>error_logger</c> is not defined in the map the system + default will be used. The default system default is <c>true</c>. + It can be changed by either the erl <seealso marker="erl#+hmaxel">+hmaxel</seealso> + option, or <seealso marker="#system_flag_max_heap_size"><c> + erlang:system_flag(max_heap_size, MaxHeapSize)</c></seealso>. + </p> + </item> + <p> + The heap size of a process is quite hard to predict, especially the + amount of memory that is used during the garbage collection. When + contemplating using this option, it is recommended to first run + it in production with <c>kill</c> set to <c>false</c> and inspect + the <c>error_logger</c> reports to see what the normal peak sizes + of the processes in the system is and then tune the value + accordingly. + </p> + </taglist> + </p> + </desc> + </func> + <marker id="process_flag_message_queue_data"/> + <func> + <name name="process_flag" arity="2" clause_i="6"/> <fsummary>Set process flag <c>message_queue_data</c> for the calling process</fsummary> <type name="message_queue_data"/> <desc> @@ -4392,7 +4479,7 @@ os_prompt% </pre> </desc> </func> <func> - <name name="process_flag" arity="2" clause_i="6"/> + <name name="process_flag" arity="2" clause_i="7"/> <fsummary>Sets process flag <c>priority</c> for the calling process.</fsummary> <type name="priority_level"/> <desc> @@ -4466,7 +4553,7 @@ os_prompt% </pre> </func> <func> - <name name="process_flag" arity="2" clause_i="7"/> + <name name="process_flag" arity="2" clause_i="8"/> <fsummary>Sets process flag <c>save_calls</c> for the calling process.</fsummary> <desc> <p><c><anno>N</anno></c> must be an integer in the interval 0..10000. @@ -4497,7 +4584,7 @@ os_prompt% </pre> </func> <func> - <name name="process_flag" arity="2" clause_i="8"/> + <name name="process_flag" arity="2" clause_i="9"/> <fsummary>Sets process flag <c>sensitive</c> for the calling process.</fsummary> <desc> <p>Sets or clears flag <c>sensitive</c> for the current process. @@ -4551,6 +4638,7 @@ os_prompt% </pre> <type name="process_info_result_item"/> <type name="priority_level"/> <type name="stack_item"/> + <type name="max_heap_size" /> <type name="message_queue_data" /> <desc> <p>Returns a list containing <c><anno>InfoTuple</anno></c>s with @@ -4604,6 +4692,7 @@ os_prompt% </pre> <type name="process_info_result_item"/> <type name="stack_item"/> <type name="priority_level"/> + <type name="max_heap_size" /> <type name="message_queue_data" /> <desc> <p>Returns information about the process identified by @@ -4696,6 +4785,7 @@ os_prompt% </pre> The content of <c><anno>GCInfo</anno></c> can be changed without prior notice.</p> </item> + <marker id="process_info_garbage_collection_info"/> <tag><c>{garbage_collection_info, <anno>GCInfo</anno>}</c></tag> <item> <p><c><anno>GCInfo</anno></c> is a list containing miscellaneous @@ -4875,6 +4965,7 @@ os_prompt% </pre> total suspend count on <c><anno>Suspendee</anno></c>, only the parts contributed by <c><anno>Pid</anno></c>.</p> </item> + <marker id="process_info_total_heap_size"/> <tag><c>{total_heap_size, <anno>Size</anno>}</c></tag> <item> <p><c><anno>Size</anno></c> is the total size, in words, of all heap @@ -5569,6 +5660,7 @@ true</pre> <name name="spawn_opt" arity="2"/> <fsummary>Creates a new process with a fun as entry point.</fsummary> <type name="priority_level"/> + <type name="max_heap_size" /> <type name="message_queue_data" /> <type name="spawn_opt_option" /> <desc> @@ -5586,6 +5678,7 @@ true</pre> <name name="spawn_opt" arity="3"/> <fsummary>Creates a new process with a fun as entry point on a given node.</fsummary> <type name="priority_level"/> + <type name="max_heap_size" /> <type name="message_queue_data" /> <type name="spawn_opt_option" /> <desc> @@ -5602,6 +5695,7 @@ true</pre> <name name="spawn_opt" arity="4"/> <fsummary>Creates a new process with a function as entry point.</fsummary> <type name="priority_level"/> + <type name="max_heap_size" /> <type name="message_queue_data" /> <type name="spawn_opt_option" /> <desc> @@ -5705,6 +5799,16 @@ true</pre> fine-tuning an application and to measure the execution time with various <c><anno>VSize</anno></c> values.</p> </item> + <tag><c>{max_heap_size, <anno>Size</anno>}</c></tag> + <item> + <p>Sets the <c>max_heap_size</c> process flag. The default + <c>max_heap_size</c> is determined by the + <seealso marker="erl#+hmax"><c>+hmax</c></seealso> <c>erl</c> + command line argument. For more information, see the + documentation of + <seealso marker="#process_flag_max_heap_size"><c>process_flag(max_heap_size, + <anno>Size</anno>)</c></seealso>.</p> + </item> <tag><c>{message_queue_data, <anno>MQD</anno>}</c></tag> <item> <p>Sets the state of the <c>message_queue_data</c> process @@ -5725,6 +5829,7 @@ true</pre> <name name="spawn_opt" arity="5"/> <fsummary>Creates a new process with a function as entry point on a given node.</fsummary> <type name="priority_level"/> + <type name="max_heap_size" /> <type name="message_queue_data" /> <type name="spawn_opt_option" /> <desc> @@ -6506,8 +6611,25 @@ ok </desc> </func> + <marker id="system_flag_max_heap_size"></marker> <func> <name name="system_flag" arity="2" clause_i="8"/> + <type name="max_heap_size"/> + <fsummary>Sets system flag <c>max_heap_size</c></fsummary> + <desc> + <p> + Sets the default maximum heap size settings for processes. + The size is given in words. The new <c>max_heap_size</c> + effects only processes spawned efter the change has been made. + <c>max_heap_size</c> can be set for individual processes using + <seealso marker="#spawn_opt/4">spawn_opt/N</seealso> or + <seealso marker="#process_flag_message_queue_data">process_flag/2</seealso>.</p> + <p>Returns the old value of the flag.</p> + </desc> + </func> + + <func> + <name name="system_flag" arity="2" clause_i="9"/> <fsummary>Sets system flag <c>multi_scheduling</c>.</fsummary> <desc> <p><marker id="system_flag_multi_scheduling"></marker> @@ -6557,7 +6679,7 @@ ok </func> <func> - <name name="system_flag" arity="2" clause_i="9"/> + <name name="system_flag" arity="2" clause_i="10"/> <fsummary>Sets system flag <c>scheduler_bind_type</c>.</fsummary> <type name="scheduler_bind_type"/> <desc> @@ -6675,7 +6797,7 @@ ok </func> <func> - <name name="system_flag" arity="2" clause_i="10"/> + <name name="system_flag" arity="2" clause_i="11"/> <fsummary>Sets system flag <c>scheduler_wall_time</c>.</fsummary> <desc><p><marker id="system_flag_scheduler_wall_time"></marker> Turns on or off scheduler wall time measurements.</p> @@ -6685,7 +6807,7 @@ ok </func> <func> - <name name="system_flag" arity="2" clause_i="11"/> + <name name="system_flag" arity="2" clause_i="12"/> <fsummary>Sets system flag <c>schedulers_online</c>.</fsummary> <desc> <p><marker id="system_flag_schedulers_online"></marker> @@ -6710,7 +6832,7 @@ ok </func> <func> - <name name="system_flag" arity="2" clause_i="12"/> + <name name="system_flag" arity="2" clause_i="13"/> <fsummary>Sets system flag <c>trace_control_word</c>.</fsummary> <desc> <p>Sets the value of the node trace control word to @@ -6724,7 +6846,7 @@ ok </func> <func> - <name name="system_flag" arity="2" clause_i="12"/> + <name name="system_flag" arity="2" clause_i="14"/> <fsummary>Finalize the Time Offset</fsummary> <desc> <p><marker id="system_flag_time_offset"></marker> @@ -6991,6 +7113,81 @@ ok </func> <func> + <name name="system_info" arity="1" clause_i="27"/> + <name name="system_info" arity="1" clause_i="28"/> + <name name="system_info" arity="1" clause_i="36"/> + <name name="system_info" arity="1" clause_i="37"/> + <name name="system_info" arity="1" clause_i="38"/> + <name name="system_info" arity="1" clause_i="39"/> + <type name="message_queue_data"/> + <type name="max_heap_size"/> + <fsummary>Information about the default process heap settings.</fsummary> + <desc> + <taglist> + <tag><c>fullsweep_after</c></tag> + <item> + <p>Returns <c>{fullsweep_after, integer() >= 0}</c>, which is + the <c>fullsweep_after</c> garbage collection setting used + by default. For more information, see + <c>garbage_collection</c> described in the following.</p> + </item> + <tag><c>garbage_collection</c></tag> + <item> + <p>Returns a list describing the default garbage collection + settings. A process spawned on the local node by a + <c>spawn</c> or <c>spawn_link</c> uses these + garbage collection settings. The default settings can be + changed by using + <seealso marker="#system_flag/2">system_flag/2</seealso>. + <seealso marker="#spawn_opt/4">spawn_opt/4</seealso> + can spawn a process that does not use the default + settings.</p> + </item> + <tag><c>max_heap_size</c></tag> + <item> + <p>Returns <c>{max_heap_size, <anno>MaxHeapSize</anno>}</c>, + where <c><anno>MaxHeapSize</anno></c> is the current + system-wide max heap size settings for spawned processes. + This setting can be set using the <c>erl</c> command line + flags <seealso marker="erl#+hmax"><c>+hmax</c></seealso>, + <seealso marker="erl#+hmaxk"><c>+hmaxk</c></seealso> and + <seealso marker="erl#+hmaxel"><c>+hmaxel</c></seealso>. It can + also be changed at run-time using + <seealso marker="#system_flag_max_heap_size"> + <c>erlang:system_flag(max_heap_size, MaxHeapSize)</c></seealso>. + For more details about the <c>max_heap_size</c> process flag + see <seealso marker="#process_flag_max_heap_size"> + <c>process_flag(max_heap_size, MaxHeapSize)</c></seealso>. + </p> + </item> + <tag><c>min_heap_size</c></tag> + <item> + <p>Returns <c>{min_heap_size, <anno>MinHeapSize</anno>}</c>, + where <c><anno>MinHeapSize</anno></c> is the current + system-wide minimum heap size for spawned processes.</p> + </item> + <tag><marker id="system_info_message_queue_data"><c>message_queue_data</c></marker></tag> + <item> + <p>Returns the default value of the <c>message_queue_data</c> + process flag which is either <c>off_heap</c>, <c>on_heap</c>, or <c>mixed</c>. + This default is set by the <c>erl</c> command line argument + <seealso marker="erl#+hmqd"><c>+hmqd</c></seealso>. For more information on the + <c>message_queue_data</c> process flag, see documentation of + <seealso marker="#process_flag_message_queue_data"><c>process_flag(message_queue_data, + MQD)</c></seealso>.</p> + </item> + <tag><c>min_bin_vheap_size</c></tag> + <item> + <p>Returns <c>{min_bin_vheap_size, + <anno>MinBinVHeapSize</anno>}</c>, where + <c><anno>MinBinVHeapSize</anno></c> is the current system-wide + minimum binary virtual heap size for spawned processes.</p> + </item> + </taglist> + </desc> + </func> + + <func> <name name="system_info" arity="1" clause_i="6"/> <name name="system_info" arity="1" clause_i="7"/> <name name="system_info" arity="1" clause_i="8"/> @@ -7010,8 +7207,6 @@ ok <name name="system_info" arity="1" clause_i="24"/> <name name="system_info" arity="1" clause_i="25"/> <name name="system_info" arity="1" clause_i="26"/> - <name name="system_info" arity="1" clause_i="27"/> - <name name="system_info" arity="1" clause_i="28"/> <name name="system_info" arity="1" clause_i="29"/> <name name="system_info" arity="1" clause_i="30"/> <name name="system_info" arity="1" clause_i="31"/> @@ -7019,10 +7214,6 @@ ok <name name="system_info" arity="1" clause_i="33"/> <name name="system_info" arity="1" clause_i="34"/> <name name="system_info" arity="1" clause_i="35"/> - <name name="system_info" arity="1" clause_i="36"/> - <name name="system_info" arity="1" clause_i="37"/> - <name name="system_info" arity="1" clause_i="38"/> - <name name="system_info" arity="1" clause_i="39"/> <name name="system_info" arity="1" clause_i="40"/> <name name="system_info" arity="1" clause_i="41"/> <name name="system_info" arity="1" clause_i="42"/> @@ -7052,6 +7243,7 @@ ok <name name="system_info" arity="1" clause_i="66"/> <name name="system_info" arity="1" clause_i="67"/> <name name="system_info" arity="1" clause_i="68"/> + <name name="system_info" arity="1" clause_i="69"/> <fsummary>Information about the system.</fsummary> <desc> <p>Returns various information about the current system @@ -7288,25 +7480,6 @@ ok <c>ERL_MAX_ETS_TABLES</c> before starting the Erlang runtime system.</p> </item> - <tag><c>fullsweep_after</c></tag> - <item> - <p>Returns <c>{fullsweep_after, integer() >= 0}</c>, which is - the <c>fullsweep_after</c> garbage collection setting used - by default. For more information, see - <c>garbage_collection</c> described in the following.</p> - </item> - <tag><c>garbage_collection</c></tag> - <item> - <p>Returns a list describing the default garbage collection - settings. A process spawned on the local node by a - <c>spawn</c> or <c>spawn_link</c> uses these - garbage collection settings. The default settings can be - changed by using - <seealso marker="#system_flag/2">system_flag/2</seealso>. - <seealso marker="#spawn_opt/4">spawn_opt/4</seealso> - can spawn a process that does not use the default - settings.</p> - </item> <tag><c>heap_sizes</c></tag> <item> <p>Returns a list of integers representing valid heap sizes @@ -7381,29 +7554,6 @@ ok <item> <p>Returns a string containing the Erlang machine name.</p> </item> - <tag><c>min_heap_size</c></tag> - <item> - <p>Returns <c>{min_heap_size, <anno>MinHeapSize</anno>}</c>, - where <c><anno>MinHeapSize</anno></c> is the current - system-wide minimum heap size for spawned processes.</p> - </item> - <tag><marker id="system_info_message_queue_data"><c>message_queue_data</c></marker></tag> - <item> - <p>Returns the default value of the <c>message_queue_data</c> - process flag which is either <c>off_heap</c>, <c>on_heap</c>, or <c>mixed</c>. - This default is set by the <c>erl</c> command line argument - <seealso marker="erl#+hmqd"><c>+hmqd</c></seealso>. For more information on the - <c>message_queue_data</c> process flag, see documentation of - <seealso marker="#process_flag_message_queue_data"><c>process_flag(message_queue_data, - MQD)</c></seealso>.</p> - </item> - <tag><c>min_bin_vheap_size</c></tag> - <item> - <p>Returns <c>{min_bin_vheap_size, - <anno>MinBinVHeapSize</anno>}</c>, where - <c><anno>MinBinVHeapSize</anno></c> is the current system-wide - minimum binary virtual heap size for spawned processes.</p> - </item> <tag><c>modified_timing_level</c></tag> <item> <p>Returns the modified timing-level (an integer) if @@ -8028,12 +8178,13 @@ ok <c>GcPid</c> and <c>Info</c> are the same as for <c>long_gc</c> earlier, except that the tuple tagged with <c>timeout</c> is not present.</p> - <p>As of <c>ERTS</c> 5.6, the monitor message is sent - if the sum of the sizes of all memory blocks allocated - for all heap generations is equal to or higher than <c>Size</c>. - Previously the monitor message was sent if the memory block - allocated for the youngest generation was equal to or higher - than <c>Size</c>.</p> + <p>The monitor message is sent if the sum of the sizes of + all memory blocks allocated for all heap generations after + a garbage collection is equal to or higher than <c>Size</c>.</p> + <p>When a process is killed by <seealso marker="#process_flag_max_heap_size"> + <c>max_heap_size</c></seealso>, it is killed before the + garbage collection is complete and thus no large heap message + will be sent.</p> </item> <tag><c>busy_port</c></tag> <item> @@ -8560,7 +8711,9 @@ timestamp() -> <tag><c>garbage_collection</c></tag> <item> <p>Traces garbage collections of processes.</p> - <p>Message tags: <c><seealso marker="#trace_3_trace_messages_gc_minor_start">gc_minor_start</seealso></c> and <c><seealso marker="#trace_3_trace_messages_gc_minor_end">gc_minor_end</seealso></c>.</p> + <p>Message tags: <c><seealso marker="#trace_3_trace_messages_gc_minor_start">gc_minor_start</seealso></c>, + <c><seealso marker="#trace_3_trace_messages_gc_max_heap_size">gc_max_heap_size</seealso></c> and + <c><seealso marker="#trace_3_trace_messages_gc_minor_end">gc_minor_end</seealso></c>.</p> </item> <tag><c>timestamp</c></tag> <item> @@ -8929,6 +9082,19 @@ timestamp() -> <p>All sizes are in words.</p> </item> <tag> + <marker id="trace_3_trace_messages_gc_max_heap_size"></marker> + <c>{trace, Pid, gc_max_heap_size, Info}</c> + </tag> + <item> + <p> + Sent when the <seealso marker="#process_flag_max_heap_size"><c>max_heap_size</c></seealso> + is reached during garbage collection. <c>Info</c> contains the + same kind of list as in message <c>gc_start</c>, + but the sizes reflect the sizes that triggered max_heap_size to + be reached. + </p> + </item> + <tag> <marker id="trace_3_trace_messages_gc_minor_end"></marker> <c>{trace, Pid, gc_minor_end, Info}</c> </tag> diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 3022c0a99a..8f65e71531 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -275,6 +275,7 @@ atom garbage_collection_info atom gc_end atom gc_major_end atom gc_major_start +atom gc_max_heap_size atom gc_minor_end atom gc_minor_start atom gc_start @@ -366,6 +367,7 @@ atom match_spec atom match_spec_result atom max atom maximum +atom max_heap_size atom max_tables max_processes atom mbuf_size atom md5 diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 99eb26bec5..a88fea3802 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -1704,6 +1704,14 @@ void process_main(void) BeamInstr *next; Eterm result; + if (!(FCALLS > 0 || FCALLS > neg_o_reds)) { + /* If we have run out of reductions, we do a context + switch before calling the bif */ + c_p->arity = 2; + c_p->current = NULL; + goto context_switch3; + } + PRE_BIF_SWAPOUT(c_p); c_p->fcalls = FCALLS - 1; result = erl_send(c_p, r(0), x(1)); @@ -2810,6 +2818,15 @@ do { \ BeamInstr *next; ErlHeapFragment *live_hf_end; + + if (!((FCALLS - 1) > 0 || (FCALLS-1) > neg_o_reds)) { + /* If we have run out of reductions, we do a context + switch before calling the bif */ + c_p->arity = ((Export *)Arg(0))->code[2]; + c_p->current = NULL; + goto context_switch3; + } + if (ERTS_MSACC_IS_ENABLED_CACHED_X()) { if (GET_BIF_MODULE(Arg(0)) == am_ets) { ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_ETS); @@ -3338,10 +3355,19 @@ do { \ context_switch2: /* Entry for fun calls. */ c_p->current = I-3; /* Pointer to Mod, Func, Arity */ + context_switch3: + { Eterm* argp; int i; + if (erts_smp_atomic32_read_nob(&c_p->state) & ERTS_PSFLG_EXITING) { + c_p->i = beam_exit; + c_p->arity = 0; + c_p->current = NULL; + goto do_schedule; + } + /* * Make sure that there is enough room for the argument registers to be saved. */ @@ -3509,6 +3535,14 @@ do { \ BifFunction vbf; ErlHeapFragment *live_hf_end; + if (!((FCALLS - 1) > 0 || (FCALLS - 1) > neg_o_reds)) { + /* If we have run out of reductions, we do a context + switch before calling the nif */ + c_p->arity = I[-1]; + c_p->current = NULL; + goto context_switch3; + } + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF); DTRACE_NIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); @@ -3551,6 +3585,15 @@ do { \ * code[3]: &&apply_bif * code[4]: Function pointer to BIF function */ + + if (!((FCALLS - 1) > 0 || (FCALLS - 1) > neg_o_reds)) { + /* If we have run out of reductions, we do a context + switch before calling the bif */ + c_p->arity = I[-1]; + c_p->current = NULL; + goto context_switch3; + } + if (ERTS_MSACC_IS_ENABLED_CACHED_X()) { if ((Eterm)I[-3] == am_ets) { ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_ETS); diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index eed0702688..483c5320d7 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -44,6 +44,7 @@ #include "erl_ptab.h" #include "erl_bits.h" #include "erl_bif_unique.h" +#include "erl_map.h" #include "erl_msacc.h" Export *erts_await_result; @@ -880,6 +881,8 @@ BIF_RETTYPE spawn_opt_1(BIF_ALIST_1) so.flags = erts_default_spo_flags|SPO_USE_ARGS; so.min_heap_size = H_MIN_SIZE; so.min_vheap_size = BIN_VH_MIN_SIZE; + so.max_heap_size = H_MAX_SIZE; + so.max_heap_flags = H_MAX_FLAGS; so.priority = PRIORITY_NORMAL; so.max_gen_gcs = (Uint16) erts_smp_atomic32_read_nob(&erts_max_gen_gcs); so.scheduler = 0; @@ -937,6 +940,9 @@ BIF_RETTYPE spawn_opt_1(BIF_ALIST_1) } else { so.min_heap_size = erts_next_heap_size(min_heap_size, 0); } + } else if (arg == am_max_heap_size) { + if (!erts_max_heap_size(val, &so.max_heap_size, &so.max_heap_flags)) + goto error; } else if (arg == am_min_bin_vheap_size && is_small(val)) { Sint min_vheap_size = signed_val(val); if (min_vheap_size < 0) { @@ -970,6 +976,10 @@ BIF_RETTYPE spawn_opt_1(BIF_ALIST_1) goto error; } + if (so.max_heap_size != 0 && so.max_heap_size < so.min_heap_size) { + goto error; + } + /* * Spawn the process. */ @@ -1731,6 +1741,23 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2) } BIF_RET(old_value); } + else if (BIF_ARG_1 == am_max_heap_size) { + Eterm *hp; + Uint sz = 0, max_heap_size, max_heap_flags; + + if (!erts_max_heap_size(BIF_ARG_2, &max_heap_size, &max_heap_flags)) + goto error; + + if ((max_heap_size < MIN_HEAP_SIZE(BIF_P) && max_heap_size != 0)) + goto error; + + erts_max_heap_size_map(MAX_HEAP_SIZE_GET(BIF_P), MAX_HEAP_SIZE_FLAGS_GET(BIF_P), NULL, &sz); + hp = HAlloc(BIF_P, sz); + old_value = erts_max_heap_size_map(MAX_HEAP_SIZE_GET(BIF_P), MAX_HEAP_SIZE_FLAGS_GET(BIF_P), &hp, NULL); + MAX_HEAP_SIZE_SET(BIF_P, max_heap_size); + MAX_HEAP_SIZE_FLAGS_SET(BIF_P, max_heap_flags); + BIF_RET(old_value); + } else if (BIF_ARG_1 == am_message_queue_data) { old_value = erts_change_message_queue_management(BIF_P, BIF_ARG_2); if (is_non_value(old_value)) @@ -4350,6 +4377,31 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); BIF_RET(make_small(oval)); + } else if (BIF_ARG_1 == am_max_heap_size) { + + Eterm *hp, old_value; + Uint sz = 0, max_heap_size, max_heap_flags; + + if (!erts_max_heap_size(BIF_ARG_2, &max_heap_size, &max_heap_flags)) + goto error; + + if (max_heap_size < H_MIN_SIZE && max_heap_size != 0) + goto error; + + erts_max_heap_size_map(H_MAX_SIZE, H_MAX_FLAGS, NULL, &sz); + hp = HAlloc(BIF_P, sz); + old_value = erts_max_heap_size_map(H_MAX_SIZE, H_MAX_FLAGS, &hp, NULL); + + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_smp_thr_progress_block(); + + H_MAX_SIZE = max_heap_size; + H_MAX_FLAGS = max_heap_flags; + + erts_smp_thr_progress_unblock(); + erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + + BIF_RET(old_value); } else if (BIF_ARG_1 == am_display_items) { int oval = display_items; if (!is_small(BIF_ARG_2) || (n = signed_val(BIF_ARG_2)) < 0) { diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 85cadafebc..da480f8fce 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -45,6 +45,7 @@ #include "erl_async.h" #include "erl_thr_progress.h" #include "erl_bif_unique.h" +#include "erl_map.h" #define ERTS_PTAB_WANT_DEBUG_FUNCS__ #include "erl_ptab.h" #ifdef HIPE @@ -594,6 +595,7 @@ static Eterm pi_args[] = { am_suspending, am_min_heap_size, am_min_bin_vheap_size, + am_max_heap_size, am_current_location, am_current_stacktrace, am_message_queue_data, @@ -643,10 +645,11 @@ pi_arg2ix(Eterm arg) case am_suspending: return 26; case am_min_heap_size: return 27; case am_min_bin_vheap_size: return 28; - case am_current_location: return 29; - case am_current_stacktrace: return 30; - case am_message_queue_data: return 31; - case am_garbage_collection_info: return 32; + case am_max_heap_size: return 29; + case am_current_location: return 30; + case am_current_stacktrace: return 31; + case am_message_queue_data: return 32; + case am_garbage_collection_info: return 33; default: return -1; } } @@ -1348,6 +1351,18 @@ process_info_aux(Process *BIF_P, break; } + case am_max_heap_size: { + Uint hsz = 3; + (void) erts_max_heap_size_map(MAX_HEAP_SIZE_GET(rp), + MAX_HEAP_SIZE_FLAGS_GET(rp), + NULL, &hsz); + hp = HAlloc(BIF_P, hsz); + res = erts_max_heap_size_map(MAX_HEAP_SIZE_GET(rp), + MAX_HEAP_SIZE_FLAGS_GET(rp), + &hp, NULL); + break; + } + case am_total_heap_size: { ErtsMessage *mp; Uint total_heap_size; @@ -1391,8 +1406,12 @@ process_info_aux(Process *BIF_P, case am_garbage_collection: { DECL_AM(minor_gcs); Eterm t; + Uint map_sz = 0; + + erts_max_heap_size_map(MAX_HEAP_SIZE_GET(rp), MAX_HEAP_SIZE_FLAGS_GET(rp), NULL, &map_sz); - hp = HAlloc(BIF_P, 3+2 + 3+2 + 3+2 + 3+2 + 3); /* last "3" is for outside tuple */ + hp = HAlloc(BIF_P, 3+2 + 3+2 + 3+2 + 3+2 + 3+2 + map_sz + 3); + /* last "3" is for outside tuple */ t = TUPLE2(hp, AM_minor_gcs, make_small(GEN_GCS(rp))); hp += 3; res = CONS(hp, t, NIL); hp += 2; @@ -1403,6 +1422,11 @@ process_info_aux(Process *BIF_P, res = CONS(hp, t, res); hp += 2; t = TUPLE2(hp, am_min_bin_vheap_size, make_small(MIN_VHEAP_SIZE(rp))); hp += 3; res = CONS(hp, t, res); hp += 2; + + t = erts_max_heap_size_map(MAX_HEAP_SIZE_GET(rp), MAX_HEAP_SIZE_FLAGS_GET(rp), &hp, NULL); + + t = TUPLE2(hp, am_max_heap_size, t); hp += 3; + res = CONS(hp, t, res); hp += 2; break; } @@ -1412,12 +1436,12 @@ process_info_aux(Process *BIF_P, if (rp == BIF_P) { sz += ERTS_PROCESS_GC_INFO_MAX_SIZE; } else { - erts_process_gc_info(rp, &sz, NULL); + erts_process_gc_info(rp, &sz, NULL, 0, 0); sz += 3; } hp = HAlloc(BIF_P, sz); - res = erts_process_gc_info(rp, &actual_sz, &hp); + res = erts_process_gc_info(rp, &actual_sz, &hp, 0, 0); /* We may have some extra space, fill with 0 tuples */ if (actual_sz <= sz - 3) { @@ -2173,7 +2197,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) } else if (BIF_ARG_1 == am_garbage_collection){ Uint val = (Uint) erts_smp_atomic32_read_nob(&erts_max_gen_gcs); Eterm tup; - hp = HAlloc(BIF_P, 3+2 + 3+2 + 3+2); + hp = HAlloc(BIF_P, 3+2 + 3+2 + 3+2 + 3+2); tup = TUPLE2(hp, am_fullsweep_after, make_small(val)); hp += 3; res = CONS(hp, tup, NIL); hp += 2; @@ -2184,6 +2208,9 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) tup = TUPLE2(hp, am_min_bin_vheap_size, make_small(BIN_VH_MIN_SIZE)); hp += 3; res = CONS(hp, tup, res); hp += 2; + tup = TUPLE2(hp, am_max_heap_size, make_small(H_MAX_SIZE)); hp += 3; + res = CONS(hp, tup, res); hp += 2; + BIF_RET(res); } else if (BIF_ARG_1 == am_fullsweep_after){ Uint val = (Uint) erts_smp_atomic32_read_nob(&erts_max_gen_gcs); @@ -2194,6 +2221,12 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) hp = HAlloc(BIF_P, 3); res = TUPLE2(hp, am_min_heap_size,make_small(H_MIN_SIZE)); BIF_RET(res); + } else if (BIF_ARG_1 == am_max_heap_size) { + Uint sz = 0; + erts_max_heap_size_map(H_MAX_SIZE, H_MAX_FLAGS, NULL, &sz); + hp = HAlloc(BIF_P, sz); + res = erts_max_heap_size_map(H_MAX_SIZE, H_MAX_FLAGS, &hp, NULL); + BIF_RET(res); } else if (BIF_ARG_1 == am_min_bin_vheap_size) { hp = HAlloc(BIF_P, 3); res = TUPLE2(hp, am_min_bin_vheap_size,make_small(BIN_VH_MIN_SIZE)); diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index bed7e668d7..fa0782867d 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -41,6 +41,7 @@ #endif #include "dtrace-wrapper.h" #include "erl_bif_unique.h" +#include "dist.h" #define ERTS_INACT_WR_PB_LEAVE_MUCH_LIMIT 1 #define ERTS_INACT_WR_PB_LEAVE_MUCH_PERCENTAGE 20 @@ -146,7 +147,8 @@ static void offset_rootset(Process *p, Sint offs, char* area, Uint area_size, static void offset_off_heap(Process* p, Sint offs, char* area, Uint area_size); static void offset_mqueue(Process *p, Sint offs, char* area, Uint area_size); static void move_msgq_to_heap(Process *p); - +static int reached_max_heap_size(Process *p, Uint total_heap_size, + Uint extra_heap_size, Uint extra_old_heap_size); static void init_gc_info(ErtsGCInfo *gcip); #ifdef HARDDEBUG @@ -577,9 +579,11 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end, int need, Eterm* objv, int nobj, int fcalls) { Uint reclaimed_now = 0; + Eterm gc_trace_end_tag; int reds; ErtsMonotonicTime start_time = 0; /* Shut up faulty warning... */ ErtsSchedulerData *esdp; + erts_aint32_t state; ERTS_MSACC_PUSH_STATE_M(); #ifdef USE_VM_PROBES DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE); @@ -588,7 +592,9 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end, ASSERT(CONTEXT_REDS - ERTS_REDS_LEFT(p, fcalls) >= ERTS_PROC_GET_SCHDATA(p)->virtual_reds); - if (p->flags & (F_DISABLE_GC|F_DELAY_GC)) + state = erts_smp_atomic32_read_nob(&p->state); + + if (p->flags & (F_DISABLE_GC|F_DELAY_GC) || state & ERTS_PSFLG_EXITING) return delay_garbage_collection(p, live_hf_end, need, fcalls); if (p->abandoned_heap) @@ -623,28 +629,28 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end, if (GEN_GCS(p) < MAX_GEN_GCS(p) && !(FLAGS(p) & F_NEED_FULLSWEEP)) { if (IS_TRACED_FL(p, F_TRACE_GC)) { - trace_gc(p, am_gc_minor_start, need); + trace_gc(p, am_gc_minor_start, need, THE_NON_VALUE); } DTRACE2(gc_minor_start, pidbuf, need); reds = minor_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now); DTRACE2(gc_minor_end, pidbuf, reclaimed_now); - if (IS_TRACED_FL(p, F_TRACE_GC)) { - trace_gc(p, am_gc_minor_end, reclaimed_now); - } - if (reds < 0) + if (reds == -1) { + if (IS_TRACED_FL(p, F_TRACE_GC)) { + trace_gc(p, am_gc_minor_end, reclaimed_now, THE_NON_VALUE); + } goto do_major_collection; + } + gc_trace_end_tag = am_gc_minor_end; } else { do_major_collection: ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_GC_FULL); if (IS_TRACED_FL(p, F_TRACE_GC)) { - trace_gc(p, am_gc_major_start, need); + trace_gc(p, am_gc_major_start, need, THE_NON_VALUE); } DTRACE2(gc_major_start, pidbuf, need); reds = major_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now); DTRACE2(gc_major_end, pidbuf, reclaimed_now); - if (IS_TRACED_FL(p, F_TRACE_GC)) { - trace_gc(p, am_gc_major_end, reclaimed_now); - } + gc_trace_end_tag = am_gc_major_end; ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_GC); } @@ -660,6 +666,33 @@ do_major_collection: erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC); + /* Max heap size has been reached and the process was configured + to be killed, so we kill it and set it in a delayed garbage + collecting state. There should be no gc_end trace or + long_gc/large_gc triggers when this happens as process was + killed before a GC could be done. */ + if (reds == -2) { + ErtsProcLocks locks = ERTS_PROC_LOCKS_ALL; + + erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR); + erts_send_exit_signal(p, p->common.id, p, &locks, + am_kill, NIL, NULL, 0); + erts_smp_proc_unlock(p, locks & ERTS_PROC_LOCKS_ALL_MINOR); + + /* erts_send_exit_signal looks for ERTS_PSFLG_GC, so + we have to remove it after the signal is sent */ + erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC); + + /* We have to make sure that we have space for need on the heap */ + return delay_garbage_collection(p, live_hf_end, need, fcalls); + } + + erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC); + + if (IS_TRACED_FL(p, F_TRACE_GC)) { + trace_gc(p, gc_trace_end_tag, reclaimed_now, THE_NON_VALUE); + } + if (erts_system_monitor_long_gc != 0) { ErtsMonotonicTime end_time; Uint gc_time; @@ -761,6 +794,7 @@ erts_garbage_collect_hibernate(Process* p) heap_size = p->heap_sz + (p->old_htop - p->old_heap) + p->mbuf_sz; + heap = (Eterm*) ERTS_HEAP_ALLOC(ERTS_ALC_T_TMP_HEAP, sizeof(Eterm)*heap_size); htop = heap; @@ -1031,6 +1065,34 @@ minor_collection(Process* p, ErlHeapFragment *live_hf_end, Uint size_before = young_gen_usage(p); /* + * Check if we have gone past the max heap size limit + */ + + if (MAX_HEAP_SIZE_GET(p)) { + Uint heap_size = size_before, + /* Note that we also count the un-allocated area + in between the stack and heap */ + stack_size = HEAP_END(p) - HEAP_TOP(p), + extra_heap_size, + extra_old_heap_size = 0; + + /* Add potential old heap size */ + if (OLD_HEAP(p) == NULL && mature_size != 0) { + extra_old_heap_size = erts_next_heap_size(size_before, 1); + heap_size += extra_old_heap_size; + } else if (OLD_HEAP(p)) + heap_size += OLD_HEND(p) - OLD_HEAP(p); + + /* Add potential new young heap size */ + extra_heap_size = next_heap_size(p, stack_size + size_before, 0); + heap_size += extra_heap_size; + + if (heap_size > MAX_HEAP_SIZE_GET(p)) + if (reached_max_heap_size(p, heap_size, extra_heap_size, extra_old_heap_size)) + return -2; + } + + /* * Allocate an old heap if we don't have one and if we'll need one. */ @@ -1139,6 +1201,16 @@ minor_collection(Process* p, ErlHeapFragment *live_hf_end, ASSERT(HEAP_SIZE(p) == next_heap_size(p, HEAP_SIZE(p), 0)); ASSERT(MBUF(p) == NULL); + /* The heap usage during GC should be larger than what we end up + after a GC, even if we grow it. If this assertion is not true + we have to check size in grow_new_heap and potentially kill the + process from there */ + ASSERT(!MAX_HEAP_SIZE_GET(p) || + !(MAX_HEAP_SIZE_FLAGS_GET(p) & MAX_HEAP_SIZE_KILL) || + MAX_HEAP_SIZE_GET(p) > (young_gen_usage(p) + + (OLD_HEND(p) - OLD_HEAP(p)) + + (HEAP_END(p) - HEAP_TOP(p)))); + return gc_cost(size_after, adjust_size); } @@ -1457,6 +1529,25 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end, if (new_sz == HEAP_SIZE(p) && FLAGS(p) & F_HEAP_GROW) { new_sz = next_heap_size(p, HEAP_SIZE(p), 1); } + + + if (MAX_HEAP_SIZE_GET(p)) { + Uint heap_size = size_before; + + /* Add unused space in old heap */ + heap_size += OLD_HEND(p) - OLD_HTOP(p); + + /* Add stack + unused space in young heap */ + heap_size += HEAP_END(p) - HEAP_TOP(p); + + /* Add size of new young heap */ + heap_size += new_sz; + + if (MAX_HEAP_SIZE_GET(p) < heap_size) + if (reached_max_heap_size(p, heap_size, new_sz, 0)) + return -2; + } + FLAGS(p) &= ~(F_HEAP_GROW|F_NEED_FULLSWEEP); n_htop = n_heap = (Eterm *) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP, sizeof(Eterm)*new_sz); @@ -2986,7 +3077,9 @@ erts_gc_info_request(Process *c_p) } Eterm -erts_process_gc_info(Process *p, Uint *sizep, Eterm **hpp) +erts_process_gc_info(Process *p, Uint *sizep, Eterm **hpp, + Uint extra_heap_block, + Uint extra_old_heap_block_size) { ERTS_DECL_AM(bin_vheap_size); ERTS_DECL_AM(bin_vheap_block_size); @@ -3009,8 +3102,9 @@ erts_process_gc_info(Process *p, Uint *sizep, Eterm **hpp) AM_bin_old_vheap_block_size }; UWord values[] = { - OLD_HEAP(p) ? OLD_HEND(p) - OLD_HEAP(p) : 0, - HEAP_SIZE(p), + OLD_HEAP(p) ? OLD_HEND(p) - OLD_HEAP(p) + extra_old_heap_block_size + : extra_old_heap_block_size, + HEAP_SIZE(p) + extra_heap_block, MBUF_SIZE(p), HIGH_WATER(p) - HEAP_START(p), STACK_START(p) - p->stop, @@ -3056,6 +3150,130 @@ erts_process_gc_info(Process *p, Uint *sizep, Eterm **hpp) return res; } +static int +reached_max_heap_size(Process *p, Uint total_heap_size, + Uint extra_heap_size, Uint extra_old_heap_size) +{ + Uint max_heap_flags = MAX_HEAP_SIZE_FLAGS_GET(p); + if (IS_TRACED_FL(p, F_TRACE_GC) || + max_heap_flags & MAX_HEAP_SIZE_LOG) { + Eterm msg; + Uint size = 0; + Eterm *o_hp , *hp; + erts_process_gc_info(p, &size, NULL, extra_heap_size, + extra_old_heap_size); + o_hp = hp = erts_alloc(ERTS_ALC_T_TMP, size * sizeof(Eterm)); + msg = erts_process_gc_info(p, NULL, &hp, extra_heap_size, + extra_old_heap_size); + + if (max_heap_flags & MAX_HEAP_SIZE_LOG) { + int alive = erts_is_alive; + erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); + Eterm *o_hp, *hp, args = NIL; + + /* Build the format message */ + erts_dsprintf(dsbufp, " Process: ~p "); + if (alive) + erts_dsprintf(dsbufp, "on node ~p"); + erts_dsprintf(dsbufp, "~n Context: maximum heap size reached~n"); + erts_dsprintf(dsbufp, " Max heap size: ~p~n"); + erts_dsprintf(dsbufp, " Total heap size: ~p~n"); + erts_dsprintf(dsbufp, " Kill: ~p~n"); + erts_dsprintf(dsbufp, " Error Logger: ~p~n"); + erts_dsprintf(dsbufp, " GC Info: ~p~n"); + + /* Build the args in reverse order */ + o_hp = hp = erts_alloc(ERTS_ALC_T_TMP, 2*(alive ? 7 : 6) * sizeof(Eterm)); + args = CONS(hp, msg, args); hp += 2; + args = CONS(hp, am_true, args); hp += 2; + args = CONS(hp, (max_heap_flags & MAX_HEAP_SIZE_KILL ? am_true : am_false), args); hp += 2; + args = CONS(hp, make_small(total_heap_size), args); hp += 2; + args = CONS(hp, make_small(MAX_HEAP_SIZE_GET(p)), args); hp += 2; + if (alive) { + args = CONS(hp, erts_this_node->sysname, args); hp += 2; + } + args = CONS(hp, p->common.id, args); hp += 2; + + erts_send_error_term_to_logger(p->group_leader, dsbufp, args); + erts_free(ERTS_ALC_T_TMP, o_hp); + } + + if (IS_TRACED_FL(p, F_TRACE_GC)) + trace_gc(p, am_gc_max_heap_size, 0, msg); + + erts_free(ERTS_ALC_T_TMP, o_hp); + } + /* returns true if we should kill the process */ + return max_heap_flags & MAX_HEAP_SIZE_KILL; +} + +Eterm +erts_max_heap_size_map(Sint max_heap_size, Uint max_heap_flags, + Eterm **hpp, Uint *sz) +{ + if (!hpp) { + *sz += (2*3 + 1 + MAP_HEADER_FLATMAP_SZ); + return THE_NON_VALUE; + } else { + Eterm *hp = *hpp; + Eterm keys = TUPLE3(hp, am_error_logger, am_kill, am_size); + flatmap_t *mp; + hp += 4; + mp = (flatmap_t*) hp; + mp->thing_word = MAP_HEADER_FLATMAP; + mp->size = 3; + mp->keys = keys; + hp += MAP_HEADER_FLATMAP_SZ; + *hp++ = max_heap_flags & MAX_HEAP_SIZE_LOG ? am_true : am_false; + *hp++ = max_heap_flags & MAX_HEAP_SIZE_KILL ? am_true : am_false; + *hp++ = make_small(max_heap_size); + *hpp = hp; + return make_flatmap(mp); + } +} + +int +erts_max_heap_size(Eterm arg, Uint *max_heap_size, Uint *max_heap_flags) +{ + Sint sz; + *max_heap_flags = H_MAX_FLAGS; + if (is_small(arg)) { + sz = signed_val(arg); + *max_heap_flags = H_MAX_FLAGS; + } else if (is_map(arg)) { + const Eterm *size = erts_maps_get(am_size, arg); + const Eterm *kill = erts_maps_get(am_kill, arg); + const Eterm *log = erts_maps_get(am_error_logger, arg); + if (size && is_small(*size)) { + sz = signed_val(*size); + } else { + /* size is mandatory */ + return 0; + } + if (kill) { + if (*kill == am_true) + *max_heap_flags |= MAX_HEAP_SIZE_KILL; + else if (*kill == am_false) + *max_heap_flags &= ~MAX_HEAP_SIZE_KILL; + else + return 0; + } + if (log) { + if (*log == am_true) + *max_heap_flags |= MAX_HEAP_SIZE_LOG; + else if (*log == am_false) + *max_heap_flags &= ~MAX_HEAP_SIZE_LOG; + else + return 0; + } + } else + return 0; + if (sz < 0) + return 0; + *max_heap_size = sz; + return 1; +} + #if defined(DEBUG) || defined(ERTS_OFFHEAP_DEBUG) static int diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h index 8a6ff99990..54ea9ca3c0 100644 --- a/erts/emulator/beam/erl_gc.h +++ b/erts/emulator/beam/erl_gc.h @@ -135,7 +135,7 @@ typedef struct { #define ERTS_PROCESS_GC_INFO_MAX_TERMS (11) /* number of elements in process_gc_info*/ #define ERTS_PROCESS_GC_INFO_MAX_SIZE \ (ERTS_PROCESS_GC_INFO_MAX_TERMS * (2/*cons*/ + 3/*2-tuple*/ + BIG_UINT_HEAP_SIZE)) -Eterm erts_process_gc_info(struct process*, Uint *, Eterm **); +Eterm erts_process_gc_info(struct process*, Uint *, Eterm **, Uint, Uint); void erts_gc_info(ErtsGCInfo *gcip); void erts_init_gc(void); @@ -155,5 +155,7 @@ void erts_offset_off_heap(struct erl_off_heap*, Sint, Eterm*, Eterm*); void erts_offset_heap_ptr(Eterm*, Uint, Sint, Eterm*, Eterm*); void erts_offset_heap(Eterm*, Uint, Sint, Eterm*, Eterm*); void erts_free_heap_frags(struct process* p); +Eterm erts_max_heap_size_map(Sint, Uint, Eterm **, Uint *); +int erts_max_heap_size(Eterm, Uint *, Uint *); #endif /* __ERL_GC_H__ */ diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index fcd2739ac3..0649fb68de 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -164,6 +164,8 @@ int erts_use_sender_punish; Uint display_items; /* no of items to display in traces etc */ int H_MIN_SIZE; /* The minimum heap grain */ int BIN_VH_MIN_SIZE; /* The minimum binary virtual*/ +int H_MAX_SIZE; /* The maximum heap size */ +int H_MAX_FLAGS; /* The maximum heap flags */ Uint32 erts_debug_flags; /* Debug flags. */ int erts_backtrace_depth; /* How many functions to show in a backtrace @@ -576,6 +578,10 @@ void erts_usage(void) H_DEFAULT_SIZE); erts_fprintf(stderr, "-hmbs size set minimum binary virtual heap size in words (default %d)\n", VH_DEFAULT_SIZE); + erts_fprintf(stderr, "-hmax size set maximum heap size in words (default %d)\n", + H_DEFAULT_MAX_SIZE); + erts_fprintf(stderr, "-hmaxk bool enable or disable kill at max heap size (default true)\n"); + erts_fprintf(stderr, "-hmaxel bool enable or disable error_logger report at max heap size (default true)\n"); erts_fprintf(stderr, "-hpds size initial process dictionary size (default %d)\n", erts_pd_initial_size); erts_fprintf(stderr, "-hmqd val set default message queue data flag for processes,\n"); @@ -759,6 +765,8 @@ early_init(int *argc, char **argv) /* erts_async_thread_suggested_stack_size = ERTS_ASYNC_THREAD_MIN_STACK_SIZE; H_MIN_SIZE = H_DEFAULT_SIZE; BIN_VH_MIN_SIZE = VH_DEFAULT_SIZE; + H_MAX_SIZE = H_DEFAULT_MAX_SIZE; + H_MAX_FLAGS = MAX_HEAP_SIZE_KILL|MAX_HEAP_SIZE_LOG; erts_initialized = 0; @@ -1484,10 +1492,13 @@ erl_start(int argc, char **argv) char *sub_param = argv[i]+2; /* set default heap size * - * h|ms - min_heap_size - * h|mbs - min_bin_vheap_size - * h|pds - erts_pd_initial_size - * h|mqd - message_queue_data + * h|ms - min_heap_size + * h|mbs - min_bin_vheap_size + * h|pds - erts_pd_initial_size + * h|mqd - message_queue_data + * h|max - max_heap_size + * h|maxk - max_heap_kill + * h|maxel - max_heap_error_logger * */ if (has_prefix("mbs", sub_param)) { @@ -1530,6 +1541,41 @@ erl_start(int argc, char **argv) "Invalid message_queue_data flag: %s\n", arg); erts_usage(); } + } else if (has_prefix("maxk", sub_param)) { + arg = get_arg(sub_param+4, argv[i+1], &i); + if (strcmp(arg,"true") == 0) { + H_MAX_FLAGS |= MAX_HEAP_SIZE_KILL; + } else if (strcmp(arg,"false") == 0) { + H_MAX_FLAGS &= ~MAX_HEAP_SIZE_KILL; + } else { + erts_fprintf(stderr, "bad max heap kill %s\n", arg); + erts_usage(); + } + VERBOSE(DEBUG_SYSTEM, ("using max heap kill %d\n", H_MAX_FLAGS)); + } else if (has_prefix("maxel", sub_param)) { + arg = get_arg(sub_param+5, argv[i+1], &i); + if (strcmp(arg,"true") == 0) { + H_MAX_FLAGS |= MAX_HEAP_SIZE_LOG; + } else if (strcmp(arg,"false") == 0) { + H_MAX_FLAGS &= ~MAX_HEAP_SIZE_LOG; + } else { + erts_fprintf(stderr, "bad max heap error logger %s\n", arg); + erts_usage(); + } + VERBOSE(DEBUG_SYSTEM, ("using max heap log %d\n", H_MAX_FLAGS)); + } else if (has_prefix("max", sub_param)) { + arg = get_arg(sub_param+3, argv[i+1], &i); + if ((H_MAX_SIZE = atoi(arg)) < 0) { + erts_fprintf(stderr, "bad max heap size %s\n", arg); + erts_usage(); + } + if (H_MAX_SIZE < H_MIN_SIZE && H_MAX_SIZE) { + erts_fprintf(stderr, "max heap size (%s) is not allowed to be " + "smaller than min heap size (%d)\n", + arg, H_MIN_SIZE); + erts_usage(); + } + VERBOSE(DEBUG_SYSTEM, ("using max heap size %d\n", H_MAX_SIZE)); } else { /* backward compatibility */ arg = get_arg(argv[i]+2, argv[i+1], &i); diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 36b0af0c1a..e2f14c23bf 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -11021,9 +11021,13 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->min_heap_size = so->min_heap_size; p->min_vheap_size = so->min_vheap_size; p->max_gen_gcs = so->max_gen_gcs; + MAX_HEAP_SIZE_SET(p, so->max_heap_size); + MAX_HEAP_SIZE_FLAGS_SET(p, so->max_heap_flags); } else { p->min_heap_size = H_MIN_SIZE; p->min_vheap_size = BIN_VH_MIN_SIZE; + MAX_HEAP_SIZE_SET(p, H_MAX_SIZE); + MAX_HEAP_SIZE_FLAGS_SET(p, H_MAX_FLAGS); p->max_gen_gcs = (Uint16) erts_smp_atomic32_read_nob(&erts_max_gen_gcs); } p->schedule_count = 0; @@ -11600,7 +11604,8 @@ set_proc_exiting(Process *p, p->i = (BeamInstr *) beam_exit; #ifndef ERTS_SMP - if (state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)) { + if (state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS) + && !(state & ERTS_PSFLG_GC)) { /* * I non smp case: * diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 59da9c1779..12a919bc87 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -918,6 +918,15 @@ struct ErtsPendingSuspend_ { # define BIN_OLD_VHEAP_SZ(p) (p)->bin_old_vheap_sz # define BIN_OLD_VHEAP(p) (p)->bin_old_vheap +# define MAX_HEAP_SIZE_GET(p) ((p)->max_heap_size >> 2) +# define MAX_HEAP_SIZE_SET(p, sz) ((p)->max_heap_size = ((sz) << 2) | \ + MAX_HEAP_SIZE_FLAGS_GET(p)) +# define MAX_HEAP_SIZE_FLAGS_GET(p) ((p)->max_heap_size & 0x3) +# define MAX_HEAP_SIZE_FLAGS_SET(p, flags) ((p)->max_heap_size = flags | \ + ((p)->max_heap_size & ~0x3)) +# define MAX_HEAP_SIZE_KILL 1 +# define MAX_HEAP_SIZE_LOG 2 + struct process { ErtsPTabElementCommon common; /* *Need* to be first in struct */ @@ -935,6 +944,7 @@ struct process { Uint heap_sz; /* Size of heap in words */ Uint min_heap_size; /* Minimum size of heap (in words). */ Uint min_vheap_size; /* Minimum size of virtual heap (in words). */ + Uint max_heap_size; /* Maximum size of heap (in words). */ #if !defined(NO_FPE_SIGNALS) || defined(HIPE) volatile unsigned long fp_exception; @@ -1285,6 +1295,8 @@ typedef struct { Uint min_vheap_size; /* Minimum virtual heap size */ int priority; /* Priority for process. */ Uint16 max_gen_gcs; /* Maximum number of gen GCs before fullsweep. */ + Uint max_heap_size; /* Maximum heap size in words */ + Uint max_heap_flags; /* Maximum heap flags (kill | log) */ int scheduler; } ErlSpawnOpts; diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index 1c0fc0a11f..6d6373b706 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -1409,25 +1409,28 @@ void save_calls(Process *p, Export *e) * are all small (atomic) integers. */ void -trace_gc(Process *p, Eterm what, Uint size) +trace_gc(Process *p, Eterm what, Uint size, Eterm msg) { ErtsTracerNif *tnif = NULL; Eterm* hp; - Eterm msg = NIL; Uint sz = 0; Eterm tup; - if (is_tracer_enabled(p, ERTS_PROC_LOCK_MAIN, &p->common, &tnif, TRACE_FUN_E_GC, what)) { + if (is_tracer_enabled(p, ERTS_PROC_LOCK_MAIN, &p->common, &tnif, + TRACE_FUN_E_GC, what)) { + + if (is_non_value(msg)) { - (void) erts_process_gc_info(p, &sz, NULL); - hp = HAlloc(p, sz + 3 + 2); + (void) erts_process_gc_info(p, &sz, NULL, 0, 0); + hp = HAlloc(p, sz + 3 + 2); - msg = erts_process_gc_info(p, NULL, &hp); - tup = TUPLE2(hp, am_wordsize, make_small(size)); hp += 3; - msg = CONS(hp, tup, msg); hp += 2; + msg = erts_process_gc_info(p, NULL, &hp, 0, 0); + tup = TUPLE2(hp, am_wordsize, make_small(size)); hp += 3; + msg = CONS(hp, tup, msg); hp += 2; + } send_to_tracer_nif(p, &p->common, p->common.id, tnif, TRACE_FUN_T_GC, - what, msg, am_undefined, am_true); + what, msg, THE_NON_VALUE, am_true); } } diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h index de4beadb7e..fd5879bc9d 100644 --- a/erts/emulator/beam/erl_trace.h +++ b/erts/emulator/beam/erl_trace.h @@ -112,7 +112,7 @@ void trace_sched(Process*, ErtsProcLocks, Eterm); void trace_proc(Process*, ErtsProcLocks, Process*, Eterm, Eterm); void trace_proc_spawn(Process*, Eterm what, Eterm pid, Eterm mod, Eterm func, Eterm args); void save_calls(Process *p, Export *); -void trace_gc(Process *p, Eterm what, Uint size); +void trace_gc(Process *p, Eterm what, Uint size, Eterm msg); /* port tracing */ void trace_virtual_sched(Process*, ErtsProcLocks, Eterm); void trace_sched_ports(Port *pp, Eterm); diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h index f3c54de214..f97716d030 100644 --- a/erts/emulator/beam/erl_vm.h +++ b/erts/emulator/beam/erl_vm.h @@ -50,6 +50,7 @@ #define H_DEFAULT_SIZE 233 /* default (heap + stack) min size */ #define VH_DEFAULT_SIZE 32768 /* default virtual (bin) heap min size (words) */ +#define H_DEFAULT_MAX_SIZE 0 /* default max heap size is off */ #define CP_SIZE 1 @@ -160,6 +161,8 @@ extern int num_instructions; /* Number of instruction in opc[]. */ extern int H_MIN_SIZE; /* minimum (heap + stack) */ extern int BIN_VH_MIN_SIZE; /* minimum virtual (bin) heap */ +extern int H_MAX_SIZE; /* maximum (heap + stack) */ +extern int H_MAX_FLAGS; /* maximum heap flags */ extern int erts_atom_table_size;/* Atom table size */ extern int erts_pd_initial_size;/* Initial Process dictionary table size */ diff --git a/erts/emulator/test/process_SUITE.erl b/erts/emulator/test/process_SUITE.erl index 61a68f9759..eaa4026a8a 100644 --- a/erts/emulator/test/process_SUITE.erl +++ b/erts/emulator/test/process_SUITE.erl @@ -46,7 +46,7 @@ process_status_exiting/1, otp_4725/1, bad_register/1, garbage_collect/1, otp_6237/1, process_info_messages/1, process_flag_badarg/1, process_flag_heap_size/1, - spawn_opt_heap_size/1, + spawn_opt_heap_size/1, spawn_opt_max_heap_size/1, processes_large_tab/1, processes_default_tab/1, processes_small_tab/1, processes_this_tab/1, processes_apply_trap/1, processes_last_call_trap/1, processes_gc_trap/1, @@ -60,7 +60,7 @@ system_task_on_suspended/1, gc_request_when_gc_disabled/1, gc_request_blast_when_gc_disabled/1]). --export([prio_server/2, prio_client/2]). +-export([prio_server/2, prio_client/2, init/1, handle_event/2]). -export([init_per_testcase/2, end_per_testcase/2]). @@ -84,7 +84,8 @@ all() -> bump_reductions, low_prio, yield, yield2, otp_4725, bad_register, garbage_collect, process_info_messages, process_flag_badarg, process_flag_heap_size, - spawn_opt_heap_size, otp_6237, {group, processes_bif}, + spawn_opt_heap_size, spawn_opt_max_heap_size, otp_6237, + {group, processes_bif}, {group, otp_7738}, garb_other_running, {group, system_task}]. @@ -425,6 +426,8 @@ t_process_info(Config) when is_list(Config) -> {status, running} = process_info(self(), status), {min_heap_size, 233} = process_info(self(), min_heap_size), {min_bin_vheap_size,46422} = process_info(self(), min_bin_vheap_size), + {max_heap_size, #{ size := 0, kill := true, error_logger := true}} = + process_info(self(), max_heap_size), {current_function,{?MODULE,t_process_info,1}} = process_info(self(), current_function), {current_function,{?MODULE,t_process_info,1}} = @@ -564,6 +567,8 @@ process_info_other_msg(Config) when is_list(Config) -> {min_heap_size, 233} = process_info(Pid, min_heap_size), {min_bin_vheap_size, 46422} = process_info(Pid, min_bin_vheap_size), + {max_heap_size, #{ size := 0, kill := true, error_logger := true}} = + process_info(self(), max_heap_size), Pid ! stop, ok. @@ -914,10 +919,14 @@ process_info_garbage_collection(_Config) -> Parent = self(), Pid = spawn_link( fun() -> + %% We set mqd to off_heap and send an tuple + %% to process in order to force mbuf_size + %% to be used + process_flag(message_queue_data, off_heap), receive go -> ok end, (fun F(0) -> Parent ! deep, - receive ok -> ok end, + receive {ok,_} -> ok end, []; F(N) -> timer:sleep(1), @@ -926,31 +935,52 @@ process_info_garbage_collection(_Config) -> Parent ! shallow, receive done -> ok end end), - {garbage_collection_info, Before} = - erlang:process_info(Pid, garbage_collection_info), + [{garbage_collection_info, Before},{total_heap_size, THSBefore}] = + erlang:process_info(Pid, [garbage_collection_info, total_heap_size]), Pid ! go, receive deep -> ok end, - {_, Deep} = erlang:process_info(Pid, garbage_collection_info), - Pid ! ok, receive shallow -> ok end, - {_, After} = erlang:process_info(Pid, garbage_collection_info), + [{_, Deep},{_,THSDeep}] = + erlang:process_info(Pid, [garbage_collection_info, total_heap_size]), + Pid ! {ok, make_ref()}, receive shallow -> ok end, + [{_, After},{_, THSAfter}] = + erlang:process_info(Pid, [garbage_collection_info, total_heap_size]), Pid ! done, %% Do some general checks to see if everything seems to be roughly correct ct:log("Before: ~p",[Before]), ct:log("Deep: ~p",[Deep]), ct:log("After: ~p",[After]), + ct:log("Before THS: ~p",[THSBefore]), + ct:log("Deep THS: ~p",[THSDeep]), + ct:log("After THS: ~p",[THSAfter]), %% Check stack_size - true = proplists:get_value(stack_size, Before) < proplists:get_value(stack_size, Deep), - true = proplists:get_value(stack_size, After) < proplists:get_value(stack_size, Deep), + true = gv(stack_size, Before) < gv(stack_size, Deep), + true = gv(stack_size, After) < gv(stack_size, Deep), %% Check used heap size - true = proplists:get_value(heap_size, Before) + proplists:get_value(old_heap_size, Before) - < proplists:get_value(heap_size, Deep) + proplists:get_value(old_heap_size, Deep), - true = proplists:get_value(heap_size, Before) + proplists:get_value(old_heap_size, Before) - < proplists:get_value(heap_size, After) + proplists:get_value(old_heap_size, After), + true = gv(heap_size, Before) + gv(old_heap_size, Before) + < gv(heap_size, Deep) + gv(old_heap_size, Deep), + true = gv(heap_size, Before) + gv(old_heap_size, Before) + < gv(heap_size, After) + gv(old_heap_size, After), + + %% Check that total_heap_size == heap_block_size + old_heap_block_size + mbuf_size + THSBefore = gv(heap_block_size, Before) + + gv(old_heap_block_size, Before) + + gv(mbuf_size, Before), + + THSDeep = gv(heap_block_size, Deep) + + gv(old_heap_block_size, Deep) + + gv(mbuf_size, Deep), + + THSAfter = gv(heap_block_size, After) + + gv(old_heap_block_size, After) + + gv(mbuf_size, After), ok. +gv(Key,List) -> + proplists:get_value(Key,List). + %% Tests erlang:bump_reductions/1. bump_reductions(Config) when is_list(Config) -> erlang:garbage_collect(), @@ -1323,6 +1353,28 @@ process_flag_badarg(Config) when is_list(Config) -> chk_badarg(fun () -> process_flag(min_heap_size, gurka) end), chk_badarg(fun () -> process_flag(min_bin_vheap_size, gurka) end), chk_badarg(fun () -> process_flag(min_bin_vheap_size, -1) end), + + chk_badarg(fun () -> process_flag(max_heap_size, gurka) end), + chk_badarg(fun () -> process_flag(max_heap_size, -1) end), + chk_badarg(fun () -> + {_,Min} = process_info(self(), min_heap_size), + process_flag(max_heap_size, Min - 1) + end), + chk_badarg(fun () -> + {_,Min} = process_info(self(), min_heap_size), + process_flag(max_heap_size, #{size => Min - 1}) + end), + chk_badarg(fun () -> process_flag(max_heap_size, #{}) end), + chk_badarg(fun () -> process_flag(max_heap_size, #{ kill => true }) end), + chk_badarg(fun () -> process_flag(max_heap_size, #{ size => 233, + kill => gurka }) end), + chk_badarg(fun () -> process_flag(max_heap_size, #{ size => 233, + error_logger => gurka }) end), + chk_badarg(fun () -> process_flag(max_heap_size, #{ size => 233, + kill => true, + error_logger => gurka }) end), + chk_badarg(fun () -> process_flag(max_heap_size, #{ size => 1 bsl 64 }) end), + chk_badarg(fun () -> process_flag(priority, 4711) end), chk_badarg(fun () -> process_flag(save_calls, hmmm) end), P= spawn_link(fun () -> receive die -> ok end end), @@ -1923,6 +1975,110 @@ spawn_opt_heap_size(Config) when is_list(Config) -> Pid ! stop, ok. +spawn_opt_max_heap_size(_Config) -> + + error_logger:add_report_handler(?MODULE, self()), + + %% Test that numerical limit works + max_heap_size_test(1024, 1024, true, true), + + %% Test that map limit works + max_heap_size_test(#{ size => 1024 }, 1024, true, true), + + %% Test that no kill is sent + max_heap_size_test(#{ size => 1024, kill => false }, 1024, false, true), + + %% Test that no error_logger report is sent + max_heap_size_test(#{ size => 1024, error_logger => false }, 1024, true, false), + + %% Test that system_flag works + erlang:system_flag(max_heap_size, #{ size => 0, kill => false, + error_logger => true}), + max_heap_size_test(#{ size => 1024 }, 1024, false, true), + max_heap_size_test(#{ size => 1024, kill => true }, 1024, true, true), + + erlang:system_flag(max_heap_size, #{ size => 0, kill => true, + error_logger => false}), + max_heap_size_test(#{ size => 1024 }, 1024, true, false), + max_heap_size_test(#{ size => 1024, error_logger => true }, 1024, true, true), + + erlang:system_flag(max_heap_size, #{ size => 1 bsl 20, kill => true, + error_logger => true}), + max_heap_size_test(#{ }, 1 bsl 20, true, true), + + erlang:system_flag(max_heap_size, #{ size => 0, kill => true, + error_logger => true}), + + %% Test that ordinary case works as expected again + max_heap_size_test(1024, 1024, true, true), + + ok. + +max_heap_size_test(Option, Size, Kill, ErrorLogger) + when map_size(Option) == 0 -> + max_heap_size_test([], Size, Kill, ErrorLogger); +max_heap_size_test(Option, Size, Kill, ErrorLogger) + when is_map(Option); is_integer(Option) -> + max_heap_size_test([{max_heap_size, Option}], Size, Kill, ErrorLogger); +max_heap_size_test(Option, Size, Kill, ErrorLogger) -> + OomFun = fun F() -> timer:sleep(5),[lists:seq(1,1000)|F()] end, + Pid = spawn_opt(OomFun, Option), + {max_heap_size, MHSz} = erlang:process_info(Pid, max_heap_size), + ct:log("Default: ~p~nOption: ~p~nProc: ~p~n", + [erlang:system_info(max_heap_size), Option, MHSz]), + + #{ size := Size} = MHSz, + + Ref = erlang:monitor(process, Pid), + if Kill -> + receive + {'DOWN', Ref, process, Pid, killed} -> + ok + end; + true -> + ok + end, + if ErrorLogger -> + receive + {error, _, {emulator, _, [Pid|_]}} -> + ok + end; + true -> + ok + end, + if not Kill -> + exit(Pid, die), + receive + {'DOWN', Ref, process, Pid, die} -> + ok + end, + flush(); + true -> + ok + end, + receive + M -> + ct:fail({unexpected_message, M}) + after 10 -> + ok + end. + +flush() -> + receive + _M -> + flush() + after 1000 -> + ok + end. + +%% error_logger report handler proxy +init(Pid) -> + {ok, Pid}. + +handle_event(Event, Pid) -> + Pid ! Event, + {ok, Pid}. + processes_term_proc_list(Config) when is_list(Config) -> Tester = self(), as_expected = processes_term_proc_list_test(false), diff --git a/erts/etc/common/erlexec.c b/erts/etc/common/erlexec.c index 5a5021b003..7f4f78b5fe 100644 --- a/erts/etc/common/erlexec.c +++ b/erts/etc/common/erlexec.c @@ -150,6 +150,9 @@ static char *plush_val_switches[] = { "ms", "mbs", "pds", + "max", + "maxk", + "maxel", "mqd", "", NULL diff --git a/erts/preloaded/src/erlang.erl b/erts/preloaded/src/erlang.erl index 90fd536b15..3d152c4e92 100644 --- a/erts/preloaded/src/erlang.erl +++ b/erts/preloaded/src/erlang.erl @@ -2073,6 +2073,9 @@ open_port(PortName, PortSettings) -> (min_bin_vheap_size, MinBinVHeapSize) -> OldMinBinVHeapSize when MinBinVHeapSize :: non_neg_integer(), OldMinBinVHeapSize :: non_neg_integer(); + (max_heap_size, MaxHeapSize) -> OldMaxHeapSize when + MaxHeapSize :: max_heap_size(), + OldMaxHeapSize :: max_heap_size(); (message_queue_data, MQD) -> OldMQD when MQD :: message_queue_data(), OldMQD :: message_queue_data(); @@ -2154,6 +2157,7 @@ process_flag(_Flag, _Value) -> {messages, MessageQueue :: [term()]} | {min_heap_size, MinHeapSize :: non_neg_integer()} | {min_bin_vheap_size, MinBinVHeapSize :: non_neg_integer()} | + {max_heap_size, MaxHeapSize :: max_heap_size()} | {monitored_by, Pids :: [pid()]} | {monitors, Monitors :: [{process, Pid :: pid() | @@ -2238,6 +2242,7 @@ setelement(_Index, _Tuple1, _Value) -> | {priority, Level :: priority_level()} | {fullsweep_after, Number :: non_neg_integer()} | {min_heap_size, Size :: non_neg_integer()} + | {max_heap_size, Size :: max_heap_size()} | {min_bin_vheap_size, VSize :: non_neg_integer()}. spawn_opt(_Tuple) -> erlang:nif_error(undefined). @@ -2330,6 +2335,9 @@ subtract(_,_) -> OldMinBinVHeapSize when MinBinVHeapSize :: non_neg_integer(), OldMinBinVHeapSize :: non_neg_integer(); + (max_heap_size, MaxHeapSize) -> OldMaxHeapSize when + MaxHeapSize :: max_heap_size(), + OldMaxHeapSize :: max_heap_size(); (multi_scheduling, BlockState) -> OldBlockState when BlockState :: block | unblock | block_normal | unblock_normal, OldBlockState :: blocked | disabled | enabled; @@ -2511,6 +2519,7 @@ tuple_to_list(_Tuple) -> logical_processors_available | logical_processors_online) -> unknown | pos_integer(); (machine) -> string(); + (max_heap_size) -> {max_heap_size, MaxHeapSize :: max_heap_size()}; (message_queue_data) -> message_queue_data(); (min_heap_size) -> {min_heap_size, MinHeapSize :: pos_integer()}; (min_bin_vheap_size) -> {min_bin_vheap_size, @@ -2648,6 +2657,13 @@ spawn_monitor(M, F, A) -> erlang:error(badarg, [M,F,A]). +-type max_heap_size() :: + Size :: non_neg_integer() + %% TODO change size => to := when -type maps support is finalized + | #{ size => non_neg_integer(), + kill => boolean(), + error_logger => boolean() }. + -type spawn_opt_option() :: link | monitor @@ -2655,6 +2671,7 @@ spawn_monitor(M, F, A) -> | {fullsweep_after, Number :: non_neg_integer()} | {min_heap_size, Size :: non_neg_integer()} | {min_bin_vheap_size, VSize :: non_neg_integer()} + | {max_heap_size, Size :: max_heap_size()} | {message_queue_data, MQD :: message_queue_data()}. -spec spawn_opt(Fun, Options) -> pid() | {pid(), reference()} when diff --git a/lib/stdlib/doc/src/proc_lib.xml b/lib/stdlib/doc/src/proc_lib.xml index 245580b1ba..f02b1f0651 100644 --- a/lib/stdlib/doc/src/proc_lib.xml +++ b/lib/stdlib/doc/src/proc_lib.xml @@ -73,6 +73,13 @@ <name name="priority_level"/> </datatype> <datatype> + <name name="max_heap_size"/> + <desc> + <p>See <seealso marker="erts:erlang#process_flag_max_heap_size"> + erlang:process_flag(max_heap_size, MaxHeapSize)</seealso>.</p> + </desc> + </datatype> + <datatype> <name name="dict_or_pid"/> </datatype> </datatypes> diff --git a/lib/stdlib/src/proc_lib.erl b/lib/stdlib/src/proc_lib.erl index fdc2ae4070..4a19603ec2 100644 --- a/lib/stdlib/src/proc_lib.erl +++ b/lib/stdlib/src/proc_lib.erl @@ -43,9 +43,14 @@ %%----------------------------------------------------------------------------- -type priority_level() :: 'high' | 'low' | 'max' | 'normal'. +-type max_heap_size() :: non_neg_integer() | + #{ size => non_neg_integer(), + kill => true, + error_logger => true}. -type spawn_option() :: 'link' | 'monitor' | {'priority', priority_level()} + | {'max_heap_size', max_heap_size()} | {'min_heap_size', non_neg_integer()} | {'min_bin_vheap_size', non_neg_integer()} | {'fullsweep_after', non_neg_integer()} |