diff options
Diffstat (limited to 'erts')
56 files changed, 4803 insertions, 319 deletions
diff --git a/erts/doc/src/Makefile b/erts/doc/src/Makefile index 21aa3db864..40f74b78ff 100644 --- a/erts/doc/src/Makefile +++ b/erts/doc/src/Makefile @@ -52,18 +52,17 @@ XML_REF3_EFILES = \ erlang.xml \ erl_tracer.xml \ init.xml \ + persistent_term.xml \ + atomics.xml \ + counters.xml \ zlib.xml XML_REF3_FILES = \ + $(XML_REF3_EFILES) \ driver_entry.xml \ erl_nif.xml \ - erl_tracer.xml \ erl_driver.xml \ - erl_prim_loader.xml \ - erlang.xml \ - erts_alloc.xml \ - init.xml \ - zlib.xml + erts_alloc.xml XML_PART_FILES = \ part.xml diff --git a/erts/doc/src/atomics.xml b/erts/doc/src/atomics.xml new file mode 100644 index 0000000000..3fca92fb97 --- /dev/null +++ b/erts/doc/src/atomics.xml @@ -0,0 +1,183 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE erlref SYSTEM "erlref.dtd"> + +<erlref> + <header> + <copyright> + <year>2018</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + </legalnotice> + + <title>atomics</title> + </header> + <module>atomics</module> + <modulesummary>Atomic Functions</modulesummary> + <description> + <p>This module provides a set of functions to do atomic operations towards + mutable atomic variables. The implementation utilizes only + atomic hardware instructions without any software level locking, which makes + it very efficient for concurrent access. The atomics are organized into + arrays with the follwing semantics:</p> + <list type="bulleted"> + <item> + <p>Atomics are 64 bit integers.</p> + </item> + <item> + <p>Atomics can be represented as either signed or unsigned.</p> + </item> + <item> + <p>Atomics wrap around at overflow and underflow operations.</p> + </item> + <item> + <p>All operations guarantee atomicity. No intermediate results can be + seen. The result of one mutation can only be the input to one + following mutation.</p> + </item> + <item> + <p>All atomic operations are mutually ordered. If atomic B is updated + <em>after</em> atomic A, then that is how it will appear to any + concurrent readers. No one can read the new value of B and then read the + old value of A.</p> + </item> + <item> + <p>Indexes into atomic arrays are one-based. An atomic array of + arity N contains N atomics with index from 1 to N.</p> + </item> + </list> + </description> + + <datatypes> + <datatype> + <name name="atomics_ref"/> + <desc><p>Identifies an atomic array returned from + <seealso marker="#new/2"><c>new/2</c></seealso>.</p> + </desc> + </datatype> + </datatypes> + + <funcs> + <func> + <name name="new" arity="2"/> + <fsummary>Create atomic array</fsummary> + <desc> + <p>Create a new atomic array of <c><anno>Arity</anno></c> atomics.</p> + <p>Argument <c><anno>Opts</anno></c> is a list of the following possible + options:</p> + <taglist> + <tag><c>{signed, boolean()}</c></tag> + <item><p>Indicate if the elements of the array will be treated + as signed or unsigned integers. Default is <c>true</c> (signed).</p> + <p>The integer interval for signed atomics are from <c>-(1 bsl 63)</c> + to <c>(1 bsl 63)-1</c> and for unsigned atomics from <c>0</c> to <c>(1 + bsl 64)-1</c>.</p> + </item> + </taglist> + </desc> + </func> + + <func> + <name name="put" arity="3"/> + <fsummary>Set atomic value</fsummary> + <desc> + <p>Set atomic to <c><anno>Value</anno></c>.</p> + </desc> + </func> + + <func> + <name name="get" arity="2"/> + <fsummary>Read atomic value</fsummary> + <desc> + <p>Read atomic value.</p> + </desc> + </func> + + <func> + <name name="add" arity="3"/> + <fsummary>Add to atomic</fsummary> + <desc> + <p>Add <c><anno>Incr</anno></c> to atomic.</p> + </desc> + </func> + + <func> + <name name="add_get" arity="3"/> + <fsummary>Atomic add and get</fsummary> + <desc> + <p>Atomic addition and return of the result.</p> + </desc> + </func> + + <func> + <name name="sub" arity="3"/> + <fsummary>Subtract from atomic</fsummary> + <desc> + <p>Subtract <c><anno>Decr</anno></c> from atomic.</p> + </desc> + </func> + + <func> + <name name="sub_get" arity="3"/> + <fsummary>Atomic sub and get</fsummary> + <desc> + <p>Atomic subtraction and return of the result.</p> + </desc> + </func> + + <func> + <name name="exchange" arity="3"/> + <fsummary>Atomic exchange.</fsummary> + <desc> + <p>Atomically replaces the value of the atomic with + <c><anno>Desired</anno></c> and returns the value it held + previously.</p> + </desc> + </func> + + <func> + <name name="compare_exchange" arity="4"/> + <fsummary>Atomic compare and exchange.</fsummary> + <desc> + <p>Atomically compares the atomic with <c><anno>Expected</anno></c>, + and if those are equal, set atomic to <c><anno>Desired</anno></c>. + Returns <c>ok</c> if <c><anno>Desired</anno></c> was written. Returns + the actual atomic value if not equal to <c><anno>Expected</anno></c>.</p> + </desc> + </func> + + <func> + <name name="info" arity="1"/> + <fsummary>Get information about atomic array.</fsummary> + <desc> + <p>Return information about an atomic array in a map. The map + has the following keys:</p> + <taglist> + <tag><c>size</c></tag> + <item><p>The number of atomics in the array.</p></item> + <tag><c>max</c></tag> + <item><p>The highest possible value an atomic in this array can + hold.</p></item> + <tag><c>min</c></tag> + <item><p>The lowest possible value an atomic in this array can + hold.</p></item> + <tag><c>memory</c></tag> + <item><p>Approximate memory consumption for the array in + bytes.</p></item> + </taglist> + </desc> + </func> + + </funcs> +</erlref> diff --git a/erts/doc/src/counters.xml b/erts/doc/src/counters.xml new file mode 100644 index 0000000000..85eedfdadc --- /dev/null +++ b/erts/doc/src/counters.xml @@ -0,0 +1,142 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE erlref SYSTEM "erlref.dtd"> + +<erlref> + <header> + <copyright> + <year>2018</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + </legalnotice> + + <title>counters</title> + </header> + <module>counters</module> + <modulesummary>Counter Functions</modulesummary> + <description> + <p>This module provides a set of functions to do operations towards + shared mutable counter variables. The implementation does not utilize any + software level locking, which makes it very efficient for concurrent + access. The counters are organized into arrays with the follwing + semantics:</p> + <list type="bulleted"> + <item> + <p>Counters are 64 bit signed integers.</p> + </item> + <item> + <p>Counters wrap around at overflow and underflow operations.</p> + </item> + <item><p>Counters are initialized to zero and can then only be written to + by adding or subtracting.</p> + </item> + <item> + <p>Write operations guarantee atomicity. No intermediate results can be + seen from a single write operation.</p> + </item> + <item> + <p>Two types of counter arrays can be created with options <c>atomics</c> or + <c>write_concurrency</c>. The <c>atomics</c> counters have good allround + performance with nice consistent semantics while + <c>write_concurrency</c> counters offers even better concurrent + write performance at the expense of some potential read + inconsistencies. See <seealso marker="#new/2"><c>new/2</c></seealso>.</p> + </item> + <item> + <p>Indexes into counter arrays are one-based. A counter array of + size N contains N counters with index from 1 to N.</p> + </item> + </list> + </description> + + <datatypes> + <datatype> + <name name="counters_ref"/> + <desc><p>Identifies a counter array returned from + <seealso marker="#new/2"><c>new/2</c></seealso>.</p> + </desc> + </datatype> + </datatypes> + + <funcs> + <func> + <name name="new" arity="2"/> + <fsummary>Create counter array</fsummary> + <desc> + <p>Create a new counter array of <c><anno>Size</anno></c> counters.</p> + <p>Argument <c><anno>Opts</anno></c> is a list of the following possible + options:</p> + <taglist> + <tag><c>atomics</c> (Default)</tag> + <item><p>Counters will be sequentially consistent. If write + operation A is done sequencially before write operation B, then a concurrent reader + may see none of them, only A, or both A and B. It cannot see only B.</p> + </item> + <tag><c>write_concurrency</c></tag> + <item><p>This is an optimization to achieve very efficient concurrent + write operations at the expense of potential read inconsistency and memory + consumption per counter.</p> + <p>Read operations may see sequentially inconsistent results with + regard to concurrent write operations. Even if write operation A is done + sequencially before write operation B, a concurrent reader may see any + combination of A and B, including only B. A read operation is only + guaranteed to see all writes done sequentially before the read. No writes + are ever lost, but will eventually all be seen.</p> + </item> + </taglist> + </desc> + </func> + + <func> + <name name="get" arity="2"/> + <fsummary>Read counter value</fsummary> + <desc> + <p>Read counter value.</p> + </desc> + </func> + + <func> + <name name="add" arity="3"/> + <fsummary>Add to counter</fsummary> + <desc> + <p>Add <c><anno>Incr</anno></c> to counter.</p> + </desc> + </func> + + <func> + <name name="sub" arity="3"/> + <fsummary>Subtract from counter</fsummary> + <desc> + <p>Subtract <c><anno>Decr</anno></c> from counter.</p> + </desc> + </func> + + <func> + <name name="info" arity="1"/> + <fsummary>Get information about counter array.</fsummary> + <desc> + <p>Return information about a counter array in a map. The map + has the following keys (at least):</p> + <taglist> + <tag><c>size</c></tag> + <item><p>The number of counters in the array.</p></item> + <tag><c>memory</c></tag> + <item><p>Approximate memory consumption for the array in + bytes.</p></item> + </taglist> + </desc> + </func> + + </funcs> +</erlref> diff --git a/erts/doc/src/erl_nif.xml b/erts/doc/src/erl_nif.xml index a20b8ee884..190ec12d0e 100644 --- a/erts/doc/src/erl_nif.xml +++ b/erts/doc/src/erl_nif.xml @@ -1252,7 +1252,7 @@ typedef struct { <p>Similar to <c>fprintf</c> but this format string also accepts <c>"%T"</c>, which formats Erlang terms of type <seealso marker="#ERL_NIF_TERM"><c>ERL_NIF_TERM</c></seealso>.</p> - <p>This function is primarily intenden for debugging purpose. It is not + <p>This function is primarily intended for debugging purpose. It is not recommended to print very large terms with <c>%T</c>. The function may change <c>errno</c>, even if successful.</p> </desc> @@ -3194,7 +3194,7 @@ if (retval & ERL_NIF_SELECT_STOP_CALLED) { <p>Similar to <c>snprintf</c> but this format string also accepts <c>"%T"</c>, which formats Erlang terms of type <seealso marker="#ERL_NIF_TERM"><c>ERL_NIF_TERM</c></seealso>.</p> - <p>This function is primarily intenden for debugging purpose. It is not + <p>This function is primarily intended for debugging purpose. It is not recommended to print very large terms with <c>%T</c>. The function may change <c>errno</c>, even if successful.</p> </desc> diff --git a/erts/doc/src/match_spec.xml b/erts/doc/src/match_spec.xml index 5cd6dc1750..48e502739a 100644 --- a/erts/doc/src/match_spec.xml +++ b/erts/doc/src/match_spec.xml @@ -113,6 +113,7 @@ <c><![CDATA[length]]></c> | <c><![CDATA[map_get]]></c> | <c><![CDATA[map_size]]></c> | <c><![CDATA[node]]></c> | <c><![CDATA[round]]></c> | <c><![CDATA[size]]></c> | + <c><![CDATA[bit_size]]></c> | <c><![CDATA[tl]]></c> | <c><![CDATA[trunc]]></c> | <c><![CDATA['+']]></c> | <c><![CDATA['-']]></c> | <c><![CDATA['*']]></c> | <c><![CDATA['div']]></c> | @@ -194,6 +195,7 @@ <c><![CDATA[length]]></c> | <c><![CDATA[map_get]]></c> | <c><![CDATA[map_size]]></c> | <c><![CDATA[node]]></c> | <c><![CDATA[round]]></c> | <c><![CDATA[size]]></c> | + <c><![CDATA[bit_size]]></c> | <c><![CDATA[tl]]></c> | <c><![CDATA[trunc]]></c> | <c><![CDATA['+']]></c> | <c><![CDATA['-']]></c> | <c><![CDATA['*']]></c> | <c><![CDATA['div']]></c> | @@ -269,8 +271,9 @@ other <c>false</c> to return <c><![CDATA[true]]></c>; otherwise <c><![CDATA['xor']]></c> returns false.</p> </item> - <tag><c>abs</c>, <c>element</c>, <c>hd</c>, <c>length</c>, <c>node</c>, - <c>round</c>, <c>size</c>, <c>tl</c>, <c>trunc</c>, <c>'+'</c>, + <tag><c>abs</c>, <c>element</c>, <c>hd</c>, <c>length</c>, + <c>map_get</c>, <c>map_size</c>, <c>node</c>, <c>round</c>, + <c>size</c>, <c>bit_size</c>, <c>tl</c>, <c>trunc</c>, <c>'+'</c>, <c>'-'</c>, <c>'*'</c>, <c>'div'</c>, <c>'rem'</c>, <c>'band'</c>, <c>'bor'</c>, <c>'bxor'</c>, <c>'bnot'</c>, <c>'bsl'</c>, <c>'bsr'</c>, <c>'>'</c>, <c>'>='</c>, <c>'<'</c>, <c>'=<'</c>, diff --git a/erts/doc/src/notes.xml b/erts/doc/src/notes.xml index 8a377a4d48..1b165e885c 100644 --- a/erts/doc/src/notes.xml +++ b/erts/doc/src/notes.xml @@ -1355,6 +1355,63 @@ </section> +<section><title>Erts 9.3.3.6</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p>List subtraction (The <c>--</c> operator) will now + yield properly on large inputs.</p> + <p> + Own Id: OTP-15371</p> + </item> + </list> + </section> + +</section> + +<section><title>Erts 9.3.3.5</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + ERTS internal trees of monitor structures could get into + an inconsistent state. This could cause <c>'DOWN'</c> + messages not to be delivered when they should, as well as + delivery of <c>'DOWN'</c> messages that should not be + delivered.</p> + <p> + This bug was introduced in ERTS version 9.0 (OTP 20.0) + and was fixed in ERTS version 10.0 (OTP 21.0) due to a + rewrite of the monitor code. That is, this bug only exist + in the OTP 20 release.</p> + <p> + Own Id: OTP-15399 Aux Id: ERL-751, ERIERL-262, OTP-14205 </p> + </item> + </list> + </section> + +</section> + +<section><title>Erts 9.3.3.4</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fixed bug in <c>ets:select_replace</c> when called with a + fully bound key could cause a following call to + <c>ets:next</c> or <c>ets:prev</c> to crash the emulator + or return invalid result.</p> + <p> + Own Id: OTP-15346</p> + </item> + </list> + </section> + +</section> + <section><title>Erts 9.3.3.3</title> <section><title>Fixed Bugs and Malfunctions</title> @@ -4757,6 +4814,37 @@ </section> +<section><title>Erts 7.3.1.6</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p>List subtraction (The <c>--</c> operator) will now + yield properly on large inputs.</p> + <p> + Own Id: OTP-15371</p> + </item> + </list> + </section> + +</section> + +<section><title>Erts 7.3.1.5</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fixed small memory leak that could occur when sending to + a terminating port.</p> + <p> + Own Id: OTP-14609 Aux Id: ERIERL-238 </p> + </item> + </list> + </section> + +</section> + <section><title>Erts 7.3.1.4</title> <section><title>Fixed Bugs and Malfunctions</title> <list> @@ -6249,6 +6337,103 @@ </section> +<section><title>Erts 6.4.1.7</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + A process communicating with a port via one of the + <c>erlang:port_*</c> BIFs could potentially end up in an + inconsistent state if the port terminated during the + communication. When this occurred the process could later + block in a <c>receive</c> even though it had messages + matching in its message queue.</p> + <p> + This bug was introduced in erts version 5.10 (OTP R16A).</p> + <p> + Own Id: OTP-13424 Aux Id: OTP-10336 </p> + </item> + <item> + <p> + Calls to <c>erl_drv_send_term()</c> or + <c>erl_drv_output_term()</c> from a non-scheduler thread + while the corresponding port was invalid caused the + emulator to enter an inconsistent state which eventually + caused an emulator crash.</p> + <p> + Own Id: OTP-13866</p> + </item> + <item> + <p>Driver and NIF operations accessing processes or ports + could cause an emulator crash when used from + non-scheduler threads. Those operations are:</p> <list> + <item><c>erl_drv_send_term()</c></item> + <item><c>driver_send_term()</c></item> + <item><c>erl_drv_output_term()</c></item> + <item><c>driver_output_term()</c></item> + <item><c>enif_send()</c></item> + <item><c>enif_port_command()</c></item> </list> + <p> + Own Id: OTP-13869</p> + </item> + <item> + <p> + Fix bug in <c>binary_to_term</c> for binaries created by + <c>term_to_binary </c> with option <c>compressed</c>. The + bug can cause <c>badarg</c> exception for a valid binary + when Erlang VM is linked against a <c>zlib</c> library of + version 1.2.9 or newer. Bug exists since OTP 17.0.</p> + <p> + Own Id: OTP-14159 Aux Id: ERL-340 </p> + </item> + <item> + <p> + Fixed bug in operator <c>bxor</c> causing erroneuos + result when one operand is a big <em>negative</em> + integer with the lowest <c>N*W</c> bits as zero and the + other operand not larger than <c>N*W</c> bits. <c>N</c> + is an integer of 1 or larger and <c>W</c> is 32 or 64 + depending on word size.</p> + <p> + Own Id: OTP-14514</p> + </item> + <item> + <p> + Fixed bug in <c>binary_to_term</c> and + <c>binary_to_atom</c> that could cause VM crash. + Typically happens when the last character of an UTF8 + string is in the range 128 to 255, but truncated to only + one byte. Bug exists in <c>binary_to_term</c> since ERTS + version 5.10.2 (OTP_R16B01) and <c>binary_to_atom</c> + since ERTS version 9.0 (OTP-20.0).</p> + <p> + Own Id: OTP-14590 Aux Id: ERL-474 </p> + </item> + </list> + </section> + +</section> + +<section><title>Erts 6.4.1.6</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + When calling <c>garbage_collect/[1,2]</c> or + <c>check_process_code/[2,3]</c> from a process with a + higher priority than the priority of the process operated + on, the run queues could end up in an inconsistent state. + This bug has now been fixed.</p> + <p> + Own Id: OTP-13298 Aux Id: OTP-11388 </p> + </item> + </list> + </section> + +</section> + <section><title>Erts 6.4.1.5</title> <section><title>Fixed Bugs and Malfunctions</title> @@ -19075,4 +19260,3 @@ </section> </section> </chapter> - diff --git a/erts/doc/src/persistent_term.xml b/erts/doc/src/persistent_term.xml new file mode 100644 index 0000000000..d2a138d65f --- /dev/null +++ b/erts/doc/src/persistent_term.xml @@ -0,0 +1,290 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE erlref SYSTEM "erlref.dtd"> + +<erlref> + <header> + <copyright> + <year>2018</year><year>2018</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + </legalnotice> + + <title>persistent_term</title> + <prepared></prepared> + <docno></docno> + <date></date> + <rev></rev> + <file>persistent_term.xml</file> + </header> + <module>persistent_term</module> + <modulesummary>Persistent terms.</modulesummary> + <description> + <p>This module is similar to <seealso + marker="stdlib:ets"><c>ets</c></seealso> in that it provides a + storage for Erlang terms that can be accessed in constant time, + but with the difference that <c>persistent_term</c> has been + highly optimized for reading terms at the expense of writing and + updating terms. When a persistent term is updated or deleted, a + global garbage collection pass is run to scan all processes for + the deleted term, and to copy it into each process that still uses + it. Therefore, <c>persistent_term</c> is suitable for storing + Erlang terms that are frequently accessed but never or + infrequently updated.</p> + + <warning><p>Persistent terms is an advanced feature and is not a + general replacement for ETS tables. Before using persistent terms, + make sure to fully understand the consequence to system + performance when updating or deleting persistent terms.</p></warning> + + <p>Term lookup (using <seealso + marker="#get/1"><c>get/1</c></seealso>), is done in constant time + and without taking any locks, and the term is <strong>not</strong> + copied to the heap (as is the case with terms stored in ETS + tables).</p> + + <p>Storing or updating a term (using <seealso + marker="#put/2"><c>put/2</c></seealso>) is proportional to the + number of already created persistent terms because the hash table + holding the keys will be copied. In addition, the term itself will + be copied.</p> + + <p>When a (complex) term is deleted (using <seealso + marker="#erase/1"><c>erase/1</c></seealso>) or replaced by another + (using <seealso marker="#put/2"><c>put/2</c></seealso>), a global + garbage collection is initiated. It works like this:</p> + + <list> + <item><p>All processes in the system will be scheduled to run a + scan of their heaps for the term that has been deleted. While + such scan is relatively light-weight, if there are many + processes, the system can become less responsive until all + process have scanned theirs heaps.</p></item> + + <item><p>If the deleted term (or any part of it) is still used + by a process, that process will do a major (fullsweep) garbage + collection and copy the term into the process. However, at most + two processes at a time will be scheduled to do that kind of + garbage collection.</p></item> + </list> + + <p>Deletion of atoms and other terms that fit in one machine word + is specially optimized to avoid doing a global GC. It is still not + recommended to update persistent terms with such values too + frequently because the hash table holding the keys is copied every + time a persistent term is updated.</p> + + <p>Some examples are suitable uses for persistent terms are:</p> + + <list> + <item><p>Storing of configuration data that must be easily + accessible by all processes.</p></item> + + <item><p>Storing of references for NIF resources.</p></item> + + <item><p>Storing of references for efficient counters.</p></item> + + <item><p>Storing an atom to indicate a logging level or whether debugging + is turned on.</p></item> + </list> + + </description> + + <section> + <title>Storing Huge Persistent Terms</title> + <p>The current implementation of persistent terms uses the literal + <seealso marker="erts_alloc">allocator</seealso> also used for + literals (constant terms) in BEAM code. By default, 1 GB of + virtual address space is reserved for literals in BEAM code and + persistent terms. The amount of virtual address space reserved for + literals can be changed by using the <seealso + marker="erts_alloc#MIscs"><c>+MIscs option</c></seealso> when + starting the emulator.</p> + + <p>Here is an example how the reserved virtual address space for literals + can be raised to 2 GB (2048 MB):</p> + + <pre> + erl +MIscs 2048</pre> + </section> + + <section> + <title>Warning For Many Persistent Terms</title> + <p>The runtime system will send a warning report to the + error logger if more than 20000 persistent terms have been + created. It will look like this:</p> + +<pre> +More than 20000 persistent terms have been created. +It is recommended to avoid creating an excessive number of +persistent terms, as creation and deletion of persistent terms +will be slower as the number of persistent terms increases.</pre> + </section> + + <section> + <title>Best Practices for Using Persistent Terms</title> + + <p>It is recommended to use keys like <c>?MODULE</c> or + <c>{?MODULE,SubKey}</c> to avoid name collisions.</p> + + <p>Prefer creating a few large persistent terms to creating many + small persistent terms. The execution time for storing a + persistent term is proportional to the number of already existing + terms.</p> + + <p>Updating a persistent term with the same value as it already + has is specially optimized to do nothing quickly; thus, there is + no need compare the old and new values and avoid calling + <seealso marker="#put/2"><c>put/2</c></seealso> if the values + are equal.</p> + + <p>When atoms or other terms that fit in one machine word are + deleted, no global GC is needed. Therefore, persistent terms that + have atoms as their values can be updated more frequently, but + note that updating such persistent terms is still much more + expensive than reading them.</p> + + <p>Updating or deleting a persistent term will trigger a global GC + if the term does not fit in one machine word. Processes will be + scheduled as usual, but all processes will be made runnable at + once, which will make the system less responsive until all process + have run and scanned their heaps for the deleted terms. One way to + minimize the effects on responsiveness could be to minimize the + number of processes on the node before updating or deleting a + persistent term. It would also be wise to avoid updating terms + when the system is at peak load.</p> + + <p>Avoid storing a retrieved persistent term in a process if that + persistent term could be deleted or updated in the future. If a + process holds a reference to a persistent term when the term is + deleted, the process will be garbage collected and the term copied + to process.</p> + + <p>Avoid updating or deleting more than one persistent term at a + time. Each deleted term will trigger its own global GC. That + means that deleting N terms will make the system less responsive N + times longer than deleting a single persistent term. Therefore, + terms that are to be updated at the same time should be collected + into a larger term, for example, a map or a tuple.</p> + </section> + + <section> + <title>Example</title> + + <p>The following example shows how lock contention for ETS tables + can be minimized by having one ETS table for each scheduler. The + table identifiers for the ETS tables are stored as a single + persistent term:</p> + +<pre> + %% There is one ETS table for each scheduler. + Sid = erlang:system_info(scheduler_id), + Tid = element(Sid, persistent_term:get(?MODULE)), + ets:update_counter(Tid, Key, 1).</pre> + + </section> + + <datatypes> + <datatype> + <name name="key"/> + <desc> + <p>Any Erlang term.</p> + </desc> + </datatype> + <datatype> + <name name="value"/> + <desc> + <p>Any Erlang term.</p> + </desc> + </datatype> + </datatypes> + + <funcs> + <func> + <name name="erase" arity="1"/> + <fsummary>Erase the name for a persistent term.</fsummary> + <desc> + <p>Erase the name for the persistent term with key + <c><anno>Key</anno></c>. The return value will be <c>true</c> + if there was a persistent term with the key + <c><anno>Key</anno></c>, and <c>false</c> if there was no + persistent term associated with the key.</p> + <p>If there existed a previous persistent term associated with + key <c><anno>Key</anno></c>, a global GC has been initiated + when <c>erase/1</c> returns. See <seealso + marker="#description">Description</seealso>.</p> + </desc> + </func> + + <func> + <name name="get" arity="0"/> + <fsummary>Get all persistent terms.</fsummary> + <desc> + <p>Retrieve the keys and values for all persistent terms. + The keys will be copied to the heap for the process calling + <c>get/0</c>, but the values will not.</p> + </desc> + </func> + + <func> + <name name="get" arity="1"/> + <fsummary>Get the value for a persistent term.</fsummary> + <desc> + <p>Retrieve the value for the persistent term associated with + the key <c><anno>Key</anno></c>. The lookup will be made in + constant time and the value will not be copied to the heap + of the calling process.</p> + <p>This function fails with a <c>badarg</c> exception if no + term has been stored with the key + <c><anno>Key</anno></c>.</p> + <p>If the calling process holds on to the value of the + persistent term and the persistent term is deleted in the future, + the term will be copied to the process.</p> + </desc> + </func> + + <func> + <name name="info" arity="0"/> + <fsummary>Get information about persistent terms.</fsummary> + <desc> + <p>Return information about persistent terms in a map. The map + has the following keys:</p> + <taglist> + <tag><c>count</c></tag> + <item><p>The number of persistent terms.</p></item> + <tag><c>memory</c></tag> + <item><p>The total amount of memory (measured in bytes) + used by all persistent terms.</p></item> + </taglist> + </desc> + </func> + + <func> + <name name="put" arity="2"/> + <fsummary>Store a term.</fsummary> + <desc> + <p>Store the value <c><anno>Value</anno></c> as a persistent term and + associate it with the key <c><anno>Key</anno></c>.</p> + <p>If the value <c><anno>Value</anno></c> is equal to the value + previously stored for the key, <c>put/2</c> will do nothing and return + quickly.</p> + <p>If there existed a previous persistent term associated with + key <c><anno>Key</anno></c>, a global GC has been initiated + when <c>put/2</c> returns. See <seealso + marker="#description">Description</seealso>.</p> + </desc> + </func> + </funcs> +</erlref> diff --git a/erts/doc/src/ref_man.xml b/erts/doc/src/ref_man.xml index 0617463a7b..a78aaa449e 100644 --- a/erts/doc/src/ref_man.xml +++ b/erts/doc/src/ref_man.xml @@ -34,6 +34,7 @@ <xi:include href="erl_prim_loader.xml"/> <xi:include href="erlang.xml"/> <xi:include href="init.xml"/> + <xi:include href="persistent_term.xml"/> <xi:include href="zlib.xml"/> <xi:include href="epmd.xml"/> <xi:include href="erl.xml"/> @@ -49,5 +50,7 @@ <xi:include href="erts_alloc.xml"/> <xi:include href="erl_nif.xml"/> <xi:include href="erl_tracer.xml"/> + <xi:include href="atomics.xml"/> + <xi:include href="counters.xml"/> </application> diff --git a/erts/doc/src/specs.xml b/erts/doc/src/specs.xml index ed6be650e5..0b943e6295 100644 --- a/erts/doc/src/specs.xml +++ b/erts/doc/src/specs.xml @@ -4,5 +4,8 @@ <xi:include href="../specs/specs_erlang.xml"/> <xi:include href="../specs/specs_erl_tracer.xml"/> <xi:include href="../specs/specs_init.xml"/> + <xi:include href="../specs/specs_persistent_term.xml"/> <xi:include href="../specs/specs_zlib.xml"/> + <xi:include href="../specs/specs_atomics.xml"/> + <xi:include href="../specs/specs_counters.xml"/> </specs> diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index 054692819e..57a9d45887 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -633,21 +633,24 @@ GENERATE += $(TTF_DIR)/driver_tab.c # This list must be consistent with PRE_LOADED_MODULES in # erts/preloaded/src/Makefile. -PRELOAD_BEAM = $(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \ - $(ERL_TOP)/erts/preloaded/ebin/erts_code_purger.beam \ - $(ERL_TOP)/erts/preloaded/ebin/init.beam \ - $(ERL_TOP)/erts/preloaded/ebin/prim_buffer.beam \ - $(ERL_TOP)/erts/preloaded/ebin/prim_eval.beam \ - $(ERL_TOP)/erts/preloaded/ebin/prim_inet.beam \ - $(ERL_TOP)/erts/preloaded/ebin/prim_file.beam \ - $(ERL_TOP)/erts/preloaded/ebin/zlib.beam \ - $(ERL_TOP)/erts/preloaded/ebin/prim_zip.beam \ - $(ERL_TOP)/erts/preloaded/ebin/erl_prim_loader.beam \ - $(ERL_TOP)/erts/preloaded/ebin/erlang.beam \ - $(ERL_TOP)/erts/preloaded/ebin/erts_internal.beam \ - $(ERL_TOP)/erts/preloaded/ebin/erl_tracer.beam \ - $(ERL_TOP)/erts/preloaded/ebin/erts_literal_area_collector.beam \ - $(ERL_TOP)/erts/preloaded/ebin/erts_dirty_process_signal_handler.beam +PRELOAD_BEAM = $(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \ + $(ERL_TOP)/erts/preloaded/ebin/erts_code_purger.beam \ + $(ERL_TOP)/erts/preloaded/ebin/init.beam \ + $(ERL_TOP)/erts/preloaded/ebin/prim_buffer.beam \ + $(ERL_TOP)/erts/preloaded/ebin/prim_eval.beam \ + $(ERL_TOP)/erts/preloaded/ebin/prim_inet.beam \ + $(ERL_TOP)/erts/preloaded/ebin/prim_file.beam \ + $(ERL_TOP)/erts/preloaded/ebin/zlib.beam \ + $(ERL_TOP)/erts/preloaded/ebin/prim_zip.beam \ + $(ERL_TOP)/erts/preloaded/ebin/erl_prim_loader.beam \ + $(ERL_TOP)/erts/preloaded/ebin/erlang.beam \ + $(ERL_TOP)/erts/preloaded/ebin/erts_internal.beam \ + $(ERL_TOP)/erts/preloaded/ebin/erl_tracer.beam \ + $(ERL_TOP)/erts/preloaded/ebin/erts_literal_area_collector.beam \ + $(ERL_TOP)/erts/preloaded/ebin/erts_dirty_process_signal_handler.beam \ + $(ERL_TOP)/erts/preloaded/ebin/atomics.beam \ + $(ERL_TOP)/erts/preloaded/ebin/counters.beam \ + $(ERL_TOP)/erts/preloaded/ebin/persistent_term.beam ifeq ($(TARGET),win32) # On windows the preloaded objects are in a resource object. @@ -839,6 +842,8 @@ RUN_OBJS += \ $(OBJDIR)/erl_bif_ddll.o $(OBJDIR)/erl_bif_guard.o \ $(OBJDIR)/erl_bif_info.o $(OBJDIR)/erl_bif_op.o \ $(OBJDIR)/erl_bif_os.o $(OBJDIR)/erl_bif_lists.o \ + $(OBJDIR)/erl_bif_persistent.o \ + $(OBJDIR)/erl_bif_atomics.o $(OBJDIR)/erl_bif_counters.o \ $(OBJDIR)/erl_bif_trace.o $(OBJDIR)/erl_bif_unique.o \ $(OBJDIR)/erl_bif_wrap.o $(OBJDIR)/erl_nfunc_sched.o \ $(OBJDIR)/erl_guard_bifs.o $(OBJDIR)/erl_dirty_bif_wrap.o \ diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c index 5381611fab..59b51fd15e 100644 --- a/erts/emulator/beam/atom.c +++ b/erts/emulator/beam/atom.c @@ -174,7 +174,7 @@ atom_alloc(Atom* tmpl) /* * Precompute ordinal value of first 3 bytes + 7 bits. - * This is used by utils.c:erts_cmp_atoms(). + * This is used by erl_utils.h:erts_cmp_atoms(). * We cannot use the full 32 bits of the first 4 bytes, * since we use the sign of the difference between two * ordinal values to represent their relative order. diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 45b7540aeb..a14f22b19e 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -182,6 +182,7 @@ atom control atom copy atom copy_literals atom counters +atom count atom cpu atom cpu_timestamp atom cr @@ -287,6 +288,7 @@ atom gc_minor_end atom gc_minor_start atom Ge='>=' atom generational +atom get_all_trap atom get_seq_token atom get_tcw atom gather_gc_info_result @@ -325,6 +327,7 @@ atom index atom infinity atom info atom info_msg +atom info_trap atom init atom initial_call atom input @@ -393,6 +396,7 @@ atom microsecond atom microstate_accounting atom milli_seconds atom millisecond +atom min atom min_heap_size atom min_bin_vheap_size atom minor diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index d221e6aea6..bb1b2e5b27 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -1752,29 +1752,7 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) finalize_purge_operation(BIF_P, ret == am_true); if (literals) { - ErtsLiteralAreaRef *ref; - ErtsMessage *mp; - ref = erts_alloc(ERTS_ALC_T_LITERAL_REF, - sizeof(ErtsLiteralAreaRef)); - ref->literal_area = literals; - ref->next = NULL; - erts_mtx_lock(&release_literal_areas.mtx); - if (release_literal_areas.last) { - release_literal_areas.last->next = ref; - release_literal_areas.last = ref; - } - else { - release_literal_areas.first = ref; - release_literal_areas.last = ref; - } - erts_mtx_unlock(&release_literal_areas.mtx); - mp = erts_alloc_message(0, NULL); - ERL_MESSAGE_TOKEN(mp) = am_undefined; - erts_queue_proc_message(BIF_P, - erts_literal_area_collector, - 0, - mp, - am_copy_literals); + erts_queue_release_literals(BIF_P, literals); } return ret; @@ -1786,6 +1764,41 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) } } +void +erts_queue_release_literals(Process* c_p, ErtsLiteralArea* literals) +{ + ErtsLiteralAreaRef *ref; + ErtsMessage *mp; + ref = erts_alloc(ERTS_ALC_T_LITERAL_REF, + sizeof(ErtsLiteralAreaRef)); + ref->literal_area = literals; + ref->next = NULL; + erts_mtx_lock(&release_literal_areas.mtx); + if (release_literal_areas.last) { + release_literal_areas.last->next = ref; + release_literal_areas.last = ref; + } else { + release_literal_areas.first = ref; + release_literal_areas.last = ref; + } + erts_mtx_unlock(&release_literal_areas.mtx); + mp = erts_alloc_message(0, NULL); + ERL_MESSAGE_TOKEN(mp) = am_undefined; + if (c_p == NULL) { + erts_queue_message(erts_literal_area_collector, + 0, + mp, + am_copy_literals, + am_system); + } else { + erts_queue_proc_message(c_p, + erts_literal_area_collector, + 0, + mp, + am_copy_literals); + } +} + /* * Move code from current to old and null all export entries for the module */ diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index f18af8bcd7..015c051cc1 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -3622,6 +3622,10 @@ erts_internal_garbage_collect_1(BIF_ALIST_1) default: BIF_ERROR(BIF_P, BADARG); } erts_garbage_collect(BIF_P, 0, NULL, 0); + if (ERTS_PROC_IS_EXITING(BIF_P)) { + /* The max heap size limit was reached. */ + return THE_NON_VALUE; + } return am_true; } diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 7548924178..aa3c3acd9f 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -40,6 +40,7 @@ # Note: Guards BIFs usually require special support in the compiler. # + gcbif erlang:abs/1 bif erlang:adler32/1 bif erlang:adler32/2 @@ -698,3 +699,28 @@ ubif erlang:map_get/2 ubif erlang:is_map_key/2 bif ets:internal_delete_all/2 bif ets:internal_select_delete/2 + +# +# New in 21.2 +# + +bif persistent_term:put/2 +bif persistent_term:get/1 +bif persistent_term:get/0 +bif persistent_term:erase/1 +bif persistent_term:info/0 +bif erts_internal:erase_persistent_terms/0 + +bif erts_internal:atomics_new/2 +bif atomics:get/2 +bif atomics:put/3 +bif atomics:add/3 +bif atomics:add_get/3 +bif atomics:exchange/3 +bif atomics:compare_exchange/4 +bif atomics:info/1 + +bif erts_internal:counters_new/1 +bif erts_internal:counters_get/2 +bif erts_internal:counters_add/3 +bif erts_internal:counters_info/1 diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index e7bfd04b73..e7bd046e18 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -1074,6 +1074,7 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info) Eterm* ptr; Eterm *lit_purge_ptr = info->lit_purge_ptr; Uint lit_purge_sz = info->lit_purge_sz; + int copy_literals = info->copy_literals; #ifdef DEBUG Eterm mypid = erts_get_current_pid(); #endif @@ -1119,7 +1120,7 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info) /* off heap list pointers are copied verbatim */ if (erts_is_literal(obj,ptr)) { VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", mypid, ptr, obj)); - if (in_literal_purge_area(ptr)) + if (copy_literals || in_literal_purge_area(ptr)) info->literal_size += size_object(obj); goto pop_next; } @@ -1170,7 +1171,7 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info) /* off heap pointers to boxes are copied verbatim */ if (erts_is_literal(obj,ptr)) { VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", mypid, ptr, obj)); - if (in_literal_purge_area(ptr)) + if (copy_literals || in_literal_purge_area(ptr)) info->literal_size += size_object(obj); goto pop_next; } @@ -1338,6 +1339,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, unsigned remaining; Eterm *lit_purge_ptr = info->lit_purge_ptr; Uint lit_purge_sz = info->lit_purge_sz; + int copy_literals = info->copy_literals; #ifdef DEBUG Eterm mypid = erts_get_current_pid(); Eterm saved_obj = obj; @@ -1387,7 +1389,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, ptr = list_val(obj); /* off heap list pointers are copied verbatim */ if (erts_is_literal(obj,ptr)) { - if (!in_literal_purge_area(ptr)) { + if (!(copy_literals || in_literal_purge_area(ptr))) { *resp = obj; } else { Uint bsz = 0; @@ -1455,7 +1457,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, ptr = boxed_val(obj); /* off heap pointers to boxes are copied verbatim */ if (erts_is_literal(obj,ptr)) { - if (!in_literal_purge_area(ptr)) { + if (!(copy_literals || in_literal_purge_area(ptr))) { *resp = obj; } else { Uint bsz = 0; diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 8fe1ccb758..9e36d5e0d1 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -4030,6 +4030,9 @@ debug_free(ErtsAlcType_t n, void *extra, void *ptr) ASSERT(ERTS_ALC_N_MIN <= n && n <= ERTS_ALC_N_MAX); + if (!ptr) + return; + dptr = check_memory_fence(ptr, &size, n, ERTS_ALC_O_FREE); #ifdef ERTS_ALC_A_EXEC diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 5409b89bab..4f03a34390 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -274,9 +274,13 @@ type ML_YIELD_STATE SHORT_LIVED SYSTEM monitor_link_yield_state type ML_DIST STANDARD SYSTEM monitor_link_dist type PF3_ARGS SHORT_LIVED PROCESSES process_flag_3_arguments type SETUP_CONN_ARG SHORT_LIVED PROCESSES setup_connection_argument +type LIST_TRAP SHORT_LIVED PROCESSES list_bif_trap_state type ENVIRONMENT SYSTEM SYSTEM environment +type PERSISTENT_TERM LONG_LIVED CODE persisten_term +type PERSISTENT_LOCK_Q SHORT_LIVED SYSTEM persistent_lock_q + # # Types used for special emulators # @@ -334,6 +338,8 @@ type GC_INFO_REQ SHORT_LIVED SYSTEM gc_info_request type PORT_DATA_HEAP STANDARD SYSTEM port_data_heap type MSACC DRIVER SYSTEM microstate_accounting type SYS_CHECK_REQ SHORT_LIVED SYSTEM system_check_request +type ATOMICS STANDARD SYSTEM erl_bif_atomics +type COUNTERS STANDARD SYSTEM erl_bif_counters # # Types used by system specific code diff --git a/erts/emulator/beam/erl_bif_atomics.c b/erts/emulator/beam/erl_bif_atomics.c new file mode 100644 index 0000000000..092dbb3bd3 --- /dev/null +++ b/erts/emulator/beam/erl_bif_atomics.c @@ -0,0 +1,256 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +/* + * Purpose: High performance atomics. + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include <stddef.h> /* offsetof */ + +#include "sys.h" +#include "export.h" +#include "bif.h" +#include "erl_threads.h" +#include "big.h" +#include "erl_binary.h" +#include "erl_bif_unique.h" +#include "erl_map.h" + +typedef struct +{ + int is_signed; + UWord vlen; + erts_atomic64_t v[1]; +}AtomicsRef; + +static int atomics_destructor(Binary *unused) +{ + return 1; +} + +#define OPT_SIGNED (1 << 0) + +BIF_RETTYPE erts_internal_atomics_new_2(BIF_ALIST_2) +{ + AtomicsRef* p; + Binary* mbin; + UWord i, cnt, opts; + Uint bytes; + Eterm* hp; + + if (!term_to_UWord(BIF_ARG_1, &cnt) + || cnt == 0 + || !term_to_UWord(BIF_ARG_2, &opts)) { + + BIF_ERROR(BIF_P, BADARG); + } + + if (cnt > (ERTS_UWORD_MAX / sizeof(p->v[0]))) + BIF_ERROR(BIF_P, SYSTEM_LIMIT); + + bytes = offsetof(AtomicsRef, v) + cnt*sizeof(p->v[0]); + mbin = erts_create_magic_binary_x(bytes, + atomics_destructor, + ERTS_ALC_T_ATOMICS, + 0); + p = ERTS_MAGIC_BIN_DATA(mbin); + p->is_signed = opts & OPT_SIGNED; + p->vlen = cnt; + for (i=0; i < cnt; i++) + erts_atomic64_init_nob(&p->v[i], 0); + hp = HAlloc(BIF_P, ERTS_MAGIC_REF_THING_SIZE); + return erts_mk_magic_ref(&hp, &MSO(BIF_P), mbin); +} + +static ERTS_INLINE int get_ref(Eterm ref, AtomicsRef** pp) +{ + Binary* mbin; + if (!is_internal_magic_ref(ref)) + return 0; + + mbin = erts_magic_ref2bin(ref); + if (ERTS_MAGIC_BIN_DESTRUCTOR(mbin) != atomics_destructor) + return 0; + *pp = ERTS_MAGIC_BIN_DATA(mbin); + return 1; +} + +static ERTS_INLINE int get_ref_ix(Eterm ref, Eterm ix, + AtomicsRef** pp, UWord* ixp) +{ + return (get_ref(ref, pp) + && term_to_UWord(ix, ixp) + && --(*ixp) < (*pp)->vlen); +} + +static ERTS_INLINE int get_value(AtomicsRef* p, Eterm term, erts_aint64_t *valp) +{ + return (p->is_signed ? + term_to_Sint64(term, (Sint64*)valp) : + term_to_Uint64(term, (Uint64*)valp)); +} + +static ERTS_INLINE int get_incr(AtomicsRef* p, Eterm term, erts_aint64_t *valp) +{ + return (term_to_Sint64(term, (Sint64*)valp) + || term_to_Uint64(term, (Uint64*)valp)); +} + +static ERTS_INLINE Eterm bld_atomic(Process* proc, AtomicsRef* p, + erts_aint64_t val) +{ + if (p->is_signed) { + if (IS_SSMALL(val)) + return make_small((Sint) val); + else { + Uint hsz = ERTS_SINT64_HEAP_SIZE(val); + Eterm* hp = HAlloc(proc, hsz); + return erts_sint64_to_big(val, &hp); + } + } + else { + if ((Uint64)val <= MAX_SMALL) + return make_small((Sint) val); + else { + Uint hsz = ERTS_UINT64_HEAP_SIZE(val); + Eterm* hp = HAlloc(proc, hsz); + return erts_uint64_to_big(val, &hp); + } + } +} + +BIF_RETTYPE atomics_put_3(BIF_ALIST_3) +{ + AtomicsRef* p; + UWord ix; + erts_aint64_t val; + + if (!get_ref_ix(BIF_ARG_1, BIF_ARG_2, &p, &ix) + || !get_value(p, BIF_ARG_3, &val)) { + BIF_ERROR(BIF_P, BADARG); + } + erts_atomic64_set_mb(&p->v[ix], val); + return am_ok; +} + +BIF_RETTYPE atomics_get_2(BIF_ALIST_2) +{ + AtomicsRef* p; + UWord ix; + + if (!get_ref_ix(BIF_ARG_1, BIF_ARG_2, &p, &ix)) { + BIF_ERROR(BIF_P, BADARG); + } + return bld_atomic(BIF_P, p, erts_atomic64_read_mb(&p->v[ix])); +} + +BIF_RETTYPE atomics_add_3(BIF_ALIST_3) +{ + AtomicsRef* p; + UWord ix; + erts_aint64_t incr; + + if (!get_ref_ix(BIF_ARG_1, BIF_ARG_2, &p, &ix) + || !get_incr(p, BIF_ARG_3, &incr)) { + BIF_ERROR(BIF_P, BADARG); + } + erts_atomic64_add_mb(&p->v[ix], incr); + return am_ok; +} + +BIF_RETTYPE atomics_add_get_3(BIF_ALIST_3) +{ + AtomicsRef* p; + UWord ix; + erts_aint64_t incr; + + if (!get_ref_ix(BIF_ARG_1, BIF_ARG_2, &p, &ix) + || !get_incr(p, BIF_ARG_3, &incr)) { + BIF_ERROR(BIF_P, BADARG); + } + return bld_atomic(BIF_P, p, erts_atomic64_add_read_mb(&p->v[ix], incr)); +} + +BIF_RETTYPE atomics_exchange_3(BIF_ALIST_3) +{ + AtomicsRef* p; + UWord ix; + erts_aint64_t desired, was; + + if (!get_ref_ix(BIF_ARG_1, BIF_ARG_2, &p, &ix) + || !get_value(p, BIF_ARG_3, &desired)) { + BIF_ERROR(BIF_P, BADARG); + } + was = erts_atomic64_xchg_mb(&p->v[ix], desired); + return bld_atomic(BIF_P, p, was); +} + +BIF_RETTYPE atomics_compare_exchange_4(BIF_ALIST_4) +{ + AtomicsRef* p; + UWord ix; + erts_aint64_t expected, desired, was; + + if (!get_ref_ix(BIF_ARG_1, BIF_ARG_2, &p, &ix) + || !get_value(p, BIF_ARG_3, &expected) + || !get_value(p, BIF_ARG_4, &desired)) { + BIF_ERROR(BIF_P, BADARG); + } + was = erts_atomic64_cmpxchg_mb(&p->v[ix], desired, expected); + return was == expected ? am_ok : bld_atomic(BIF_P, p, was); +} + +BIF_RETTYPE atomics_info_1(BIF_ALIST_1) +{ + AtomicsRef* p; + Uint hsz = MAP4_SZ; + Eterm *hp; + Uint64 max; + Sint64 min; + UWord memory; + Eterm max_val, min_val, sz_val, mem_val; + + if (!get_ref(BIF_ARG_1, &p)) + BIF_ERROR(BIF_P, BADARG); + + max = p->is_signed ? ERTS_SINT64_MAX : ERTS_UINT64_MAX; + min = p->is_signed ? ERTS_SINT64_MIN : 0; + memory = erts_magic_ref2bin(BIF_ARG_1)->orig_size; + + erts_bld_uint64(NULL, &hsz, max); + erts_bld_sint64(NULL, &hsz, min); + erts_bld_uword(NULL, &hsz, p->vlen); + erts_bld_uword(NULL, &hsz, memory); + + hp = HAlloc(BIF_P, hsz); + max_val = erts_bld_uint64(&hp, NULL, max); + min_val = erts_bld_sint64(&hp, NULL, min); + sz_val = erts_bld_uword(&hp, NULL, p->vlen); + mem_val = erts_bld_uword(&hp, NULL, memory); + + return MAP4(hp, am_max, max_val, + am_memory, mem_val, + am_min, min_val, + am_size, sz_val); +} diff --git a/erts/emulator/beam/erl_bif_counters.c b/erts/emulator/beam/erl_bif_counters.c new file mode 100644 index 0000000000..a46b462225 --- /dev/null +++ b/erts/emulator/beam/erl_bif_counters.c @@ -0,0 +1,219 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +/* + * Purpose: High performance atomics. + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include <stddef.h> /* offsetof */ + +#include "sys.h" +#include "export.h" +#include "bif.h" +#include "erl_threads.h" +#include "big.h" +#include "erl_binary.h" +#include "erl_bif_unique.h" +#include "erl_map.h" + + +#define COUNTERS_PER_CACHE_LINE (ERTS_CACHE_LINE_SIZE / sizeof(erts_atomic64_t)) + +typedef struct +{ + UWord arity; +#ifdef DEBUG + UWord ulen; +#endif + union { + erts_atomic64_t v[COUNTERS_PER_CACHE_LINE]; + byte cache_line__[ERTS_CACHE_LINE_SIZE]; + } u[1]; +}CountersRef; + +static int counters_destructor(Binary *unused) +{ + return 1; +} + + +static UWord ERTS_INLINE div_ceil(UWord dividend, UWord divisor) +{ + return (dividend + divisor - 1) / divisor; +} + +BIF_RETTYPE erts_internal_counters_new_1(BIF_ALIST_1) +{ + CountersRef* p; + Binary* mbin; + UWord ui, vi, cnt; + Uint bytes, cache_lines; + Eterm* hp; + + if (!term_to_UWord(BIF_ARG_1, &cnt) + || cnt == 0) { + BIF_ERROR(BIF_P, BADARG); + } + + if (cnt > (ERTS_UWORD_MAX / (sizeof(erts_atomic64_t)*2*erts_no_schedulers))) + BIF_ERROR(BIF_P, SYSTEM_LIMIT); + + cache_lines = erts_no_schedulers * div_ceil(cnt, COUNTERS_PER_CACHE_LINE); + bytes = offsetof(CountersRef, u) + cache_lines * ERTS_CACHE_LINE_SIZE; + mbin = erts_create_magic_binary_x(bytes, + counters_destructor, + ERTS_ALC_T_ATOMICS, + 0); + p = ERTS_MAGIC_BIN_DATA(mbin); + p->arity = cnt; +#ifdef DEBUG + p->ulen = cache_lines; +#endif + ASSERT((byte*)&p->u[cache_lines] <= ((byte*)p + bytes)); + for (ui=0; ui < cache_lines; ui++) + for (vi=0; vi < COUNTERS_PER_CACHE_LINE; vi++) + erts_atomic64_init_nob(&p->u[ui].v[vi], 0); + hp = HAlloc(BIF_P, ERTS_MAGIC_REF_THING_SIZE); + return erts_mk_magic_ref(&hp, &MSO(BIF_P), mbin); +} + +static ERTS_INLINE int get_ref(Eterm ref, CountersRef** pp) +{ + Binary* mbin; + if (!is_internal_magic_ref(ref)) + return 0; + + mbin = erts_magic_ref2bin(ref); + if (ERTS_MAGIC_BIN_DESTRUCTOR(mbin) != counters_destructor) + return 0; + *pp = ERTS_MAGIC_BIN_DATA(mbin); + return 1; +} + +static ERTS_INLINE int get_ref_cnt(Eterm ref, Eterm index, + CountersRef** pp, + erts_atomic64_t** app, + UWord sched_ix) +{ + CountersRef* p; + UWord ix, ui, vi; + if (!get_ref(ref, &p) || !term_to_UWord(index, &ix) || --ix >= p->arity) + return 0; + ui = (ix / COUNTERS_PER_CACHE_LINE) * erts_no_schedulers + sched_ix; + vi = ix % COUNTERS_PER_CACHE_LINE; + ASSERT(ui < p->ulen); + *pp = p; + *app = &p->u[ui].v[vi]; + return 1; +} + +static ERTS_INLINE int get_ref_my_cnt(Eterm ref, Eterm index, + CountersRef** pp, + erts_atomic64_t** app) +{ + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + ASSERT(esdp && !ERTS_SCHEDULER_IS_DIRTY(esdp)); + return get_ref_cnt(ref, index, pp, app, esdp->no - 1); +} + +static ERTS_INLINE int get_ref_first_cnt(Eterm ref, Eterm index, + CountersRef** pp, + erts_atomic64_t** app) +{ + return get_ref_cnt(ref, index, pp, app, 0); +} + +static ERTS_INLINE int get_incr(CountersRef* p, Eterm term, erts_aint64_t *valp) +{ + return (term_to_Sint64(term, (Sint64*)valp) + || term_to_Uint64(term, (Uint64*)valp)); +} + +static ERTS_INLINE Eterm bld_counter(Process* proc, CountersRef* p, + erts_aint64_t val) +{ + if (IS_SSMALL(val)) + return make_small((Sint) val); + else { + Uint hsz = ERTS_SINT64_HEAP_SIZE(val); + Eterm* hp = HAlloc(proc, hsz); + return erts_sint64_to_big(val, &hp); + } +} + +BIF_RETTYPE erts_internal_counters_get_2(BIF_ALIST_2) +{ + CountersRef* p; + erts_atomic64_t* ap; + erts_aint64_t acc = 0; + int j; + + if (!get_ref_first_cnt(BIF_ARG_1, BIF_ARG_2, &p, &ap)) { + BIF_ERROR(BIF_P, BADARG); + } + for (j = erts_no_schedulers; j ; --j) { + acc += erts_atomic64_read_nob(ap); + ap = (erts_atomic64_t*) ((byte*)ap + ERTS_CACHE_LINE_SIZE); + } + return bld_counter(BIF_P, p, acc); +} + +BIF_RETTYPE erts_internal_counters_add_3(BIF_ALIST_3) +{ + CountersRef* p; + erts_atomic64_t* ap; + erts_aint64_t incr, sum; + + if (!get_ref_my_cnt(BIF_ARG_1, BIF_ARG_2, &p, &ap) + || !get_incr(p, BIF_ARG_3, &incr)) { + BIF_ERROR(BIF_P, BADARG); + } + sum = incr + erts_atomic64_read_nob(ap); + erts_atomic64_set_nob(ap, sum); + return am_ok; +} + + +BIF_RETTYPE erts_internal_counters_info_1(BIF_ALIST_1) +{ + CountersRef* p; + Uint hsz = MAP2_SZ; + Eterm *hp; + UWord memory; + Eterm sz_val, mem_val; + + if (!get_ref(BIF_ARG_1, &p)) + BIF_ERROR(BIF_P, BADARG); + + memory = erts_magic_ref2bin(BIF_ARG_1)->orig_size; + erts_bld_uword(NULL, &hsz, p->arity); + erts_bld_uword(NULL, &hsz, memory); + + hp = HAlloc(BIF_P, hsz); + sz_val = erts_bld_uword(&hp, NULL, p->arity); + mem_val = erts_bld_uword(&hp, NULL, memory); + + return MAP2(hp, am_memory, mem_val, + am_size, sz_val); +} diff --git a/erts/emulator/beam/erl_bif_lists.c b/erts/emulator/beam/erl_bif_lists.c index 395be67a90..a793b34852 100644 --- a/erts/emulator/beam/erl_bif_lists.c +++ b/erts/emulator/beam/erl_bif_lists.c @@ -29,12 +29,13 @@ #include "sys.h" #include "erl_vm.h" #include "global.h" -#include "erl_process.h" -#include "error.h" #include "bif.h" +#include "erl_binary.h" + static Eterm keyfind(int Bif, Process* p, Eterm Key, Eterm Pos, Eterm List); + static BIF_RETTYPE append(Process* p, Eterm A, Eterm B) { Eterm list; @@ -146,103 +147,724 @@ BIF_RETTYPE append_2(BIF_ALIST_2) return append(BIF_P, BIF_ARG_1, BIF_ARG_2); } -/* - * erlang:'--'/2 - */ +/* erlang:'--'/2 + * + * Subtracts a list from another (LHS -- RHS), removing the first occurrence of + * each element in LHS from RHS. There is no type coercion so the elements must + * match exactly. + * + * The BIF is broken into several stages that can all trap individually, and it + * chooses its algorithm based on input size. If either input is small it will + * use a linear scan tuned to which side it's on, and if both inputs are large + * enough it will convert RHS into a multiset to provide good asymptotic + * behavior. */ + +#define SUBTRACT_LHS_THRESHOLD 16 +#define SUBTRACT_RHS_THRESHOLD 16 + +typedef enum { + SUBTRACT_STAGE_START, + SUBTRACT_STAGE_LEN_LHS, + + /* Naive linear scan that's efficient when + * LEN_LHS <= SUBTRACT_LHS_THRESHOLD. */ + SUBTRACT_STAGE_NAIVE_LHS, + + SUBTRACT_STAGE_LEN_RHS, + + /* As SUBTRACT_STAGE_NAIVE_LHS but for RHS. */ + SUBTRACT_STAGE_NAIVE_RHS, + + /* Creates a multiset from RHS for faster lookups before sweeping through + * LHS. The set is implemented as a red-black tree and duplicate elements + * are handled by a counter on each node. */ + SUBTRACT_STAGE_SET_BUILD, + SUBTRACT_STAGE_SET_FINISH +} ErtsSubtractCtxStage; + +typedef struct subtract_node__ { + struct subtract_node__ *parent; + struct subtract_node__ *left; + struct subtract_node__ *right; + int is_red; + + Eterm key; + Uint count; +} subtract_tree_t; + +typedef struct { + ErtsSubtractCtxStage stage; + + Eterm lhs_original; + Eterm rhs_original; + + Uint lhs_remaining; + Uint rhs_remaining; + + Eterm iterator; + + Eterm *result_cdr; + Eterm result; + + union { + Eterm lhs_elements[SUBTRACT_LHS_THRESHOLD]; + Eterm rhs_elements[SUBTRACT_RHS_THRESHOLD]; + + struct { + subtract_tree_t *tree; + + /* A memory area for the tree's nodes, saving us the need to have + * one allocation per node. */ + subtract_tree_t *alloc_start; + subtract_tree_t *alloc; + } rhs_set; + } u; +} ErtsSubtractContext; + +#define ERTS_RBT_PREFIX subtract +#define ERTS_RBT_T subtract_tree_t +#define ERTS_RBT_KEY_T Eterm +#define ERTS_RBT_FLAGS_T int +#define ERTS_RBT_INIT_EMPTY_TNODE(T) \ + do { \ + (T)->parent = NULL; \ + (T)->left = NULL; \ + (T)->right = NULL; \ + } while(0) +#define ERTS_RBT_IS_RED(T) ((T)->is_red) +#define ERTS_RBT_SET_RED(T) ((T)->is_red = 1) +#define ERTS_RBT_IS_BLACK(T) (!ERTS_RBT_IS_RED(T)) +#define ERTS_RBT_SET_BLACK(T) ((T)->is_red = 0) +#define ERTS_RBT_GET_FLAGS(T) ((T)->is_red) +#define ERTS_RBT_SET_FLAGS(T, F) ((T)->is_red = F) +#define ERTS_RBT_GET_PARENT(T) ((T)->parent) +#define ERTS_RBT_SET_PARENT(T, P) ((T)->parent = P) +#define ERTS_RBT_GET_RIGHT(T) ((T)->right) +#define ERTS_RBT_SET_RIGHT(T, R) ((T)->right = (R)) +#define ERTS_RBT_GET_LEFT(T) ((T)->left) +#define ERTS_RBT_SET_LEFT(T, L) ((T)->left = (L)) +#define ERTS_RBT_GET_KEY(T) ((T)->key) +#define ERTS_RBT_CMP_KEYS(KX, KY) CMP_TERM(KX, KY) +#define ERTS_RBT_WANT_LOOKUP_INSERT +#define ERTS_RBT_WANT_LOOKUP +#define ERTS_RBT_WANT_DELETE +#define ERTS_RBT_UNDEF + +#include "erl_rbtree.h" + +static int subtract_continue(Process *p, ErtsSubtractContext *context); + +static void subtract_ctx_dtor(ErtsSubtractContext *context) { + switch (context->stage) { + case SUBTRACT_STAGE_SET_BUILD: + case SUBTRACT_STAGE_SET_FINISH: + erts_free(ERTS_ALC_T_LIST_TRAP, context->u.rhs_set.alloc_start); + break; + default: + break; + } +} -#define SMALL_VEC_SIZE 10 -static Eterm subtract(Process* p, Eterm A, Eterm B) -{ - Eterm list; - Eterm* hp; - Uint need; - Eterm res; - Eterm small_vec[SMALL_VEC_SIZE]; /* Preallocated memory for small lists */ - Eterm* vec_p; - Eterm* vp; - Sint i; - Sint n; - Sint m; - - if ((n = erts_list_length(A)) < 0) { - BIF_ERROR(p, BADARG); +static int subtract_ctx_bin_dtor(Binary *context_bin) { + ErtsSubtractContext *context = ERTS_MAGIC_BIN_DATA(context_bin); + subtract_ctx_dtor(context); + return 1; +} + +static void subtract_ctx_move(ErtsSubtractContext *from, + ErtsSubtractContext *to) { + int uses_result_cdr = 0; + + to->stage = from->stage; + + to->lhs_original = from->lhs_original; + to->rhs_original = from->rhs_original; + + to->lhs_remaining = from->lhs_remaining; + to->rhs_remaining = from->rhs_remaining; + + to->iterator = from->iterator; + to->result = from->result; + + switch (to->stage) { + case SUBTRACT_STAGE_NAIVE_LHS: + sys_memcpy(to->u.lhs_elements, + from->u.lhs_elements, + sizeof(Eterm) * to->lhs_remaining); + break; + case SUBTRACT_STAGE_NAIVE_RHS: + sys_memcpy(to->u.rhs_elements, + from->u.rhs_elements, + sizeof(Eterm) * to->rhs_remaining); + + uses_result_cdr = 1; + break; + case SUBTRACT_STAGE_SET_FINISH: + uses_result_cdr = 1; + /* FALL THROUGH */ + case SUBTRACT_STAGE_SET_BUILD: + to->u.rhs_set.alloc_start = from->u.rhs_set.alloc_start; + to->u.rhs_set.alloc = from->u.rhs_set.alloc; + to->u.rhs_set.tree = from->u.rhs_set.tree; + break; + default: + break; } - if ((m = erts_list_length(B)) < 0) { - BIF_ERROR(p, BADARG); + + if (uses_result_cdr) { + if (from->result_cdr == &from->result) { + to->result_cdr = &to->result; + } else { + to->result_cdr = from->result_cdr; + } } - - if (n == 0) - BIF_RET(NIL); - if (m == 0) - BIF_RET(A); - - /* allocate element vector */ - if (n <= SMALL_VEC_SIZE) - vec_p = small_vec; - else - vec_p = (Eterm*) erts_alloc(ERTS_ALC_T_TMP, n * sizeof(Eterm)); - - /* PUT ALL ELEMENTS IN VP */ - vp = vec_p; - list = A; - i = n; - while(i--) { - Eterm* listp = list_val(list); - *vp++ = CAR(listp); - list = CDR(listp); +} + +static Eterm subtract_create_trap_state(Process *p, + ErtsSubtractContext *context) { + Binary *state_bin; + Eterm *hp; + + state_bin = erts_create_magic_binary(sizeof(ErtsSubtractContext), + subtract_ctx_bin_dtor); + + subtract_ctx_move(context, ERTS_MAGIC_BIN_DATA(state_bin)); + + hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE); + + return erts_mk_magic_ref(&hp, &MSO(p), state_bin); +} + +static int subtract_enter_len_lhs(Process *p, ErtsSubtractContext *context) { + context->stage = SUBTRACT_STAGE_LEN_LHS; + + context->iterator = context->lhs_original; + context->lhs_remaining = 0; + + return subtract_continue(p, context); +} + +static int subtract_enter_len_rhs(Process *p, ErtsSubtractContext *context) { + context->stage = SUBTRACT_STAGE_LEN_RHS; + + context->iterator = context->rhs_original; + context->rhs_remaining = 0; + + return subtract_continue(p, context); +} + +static int subtract_get_length(Process *p, Eterm *iterator_p, Uint *count_p) { + static const Sint ELEMENTS_PER_RED = 32; + + Sint budget, count; + Eterm iterator; + + budget = ELEMENTS_PER_RED * ERTS_BIF_REDS_LEFT(p); + iterator = *iterator_p; + +#ifdef DEBUG + budget = budget / 10 + 1; +#endif + + for (count = 0; count < budget && is_list(iterator); count++) { + iterator = CDR(list_val(iterator)); } - - /* UNMARK ALL DELETED CELLS */ - list = B; - m = 0; /* number of deleted elements */ - while(is_list(list)) { - Eterm* listp = list_val(list); - Eterm elem = CAR(listp); - i = n; - vp = vec_p; - while(i--) { - if (is_value(*vp) && eq(*vp, elem)) { - *vp = THE_NON_VALUE; - m++; - break; - } - vp++; - } - list = CDR(listp); + + if (!is_list(iterator) && !is_nil(iterator)) { + return -1; } - - if (m == n) /* All deleted ? */ - res = NIL; - else if (m == 0) /* None deleted ? */ - res = A; - else { /* REBUILD LIST */ - res = NIL; - need = 2*(n - m); - hp = HAlloc(p, need); - vp = vec_p + n - 1; - while(vp >= vec_p) { - if (is_value(*vp)) { - res = CONS(hp, *vp, res); - hp += 2; - } - vp--; - } + + BUMP_REDS(p, count / ELEMENTS_PER_RED); + + *iterator_p = iterator; + *count_p += count; + + if (is_nil(iterator)) { + return 1; } - if (vec_p != small_vec) - erts_free(ERTS_ALC_T_TMP, (void *) vec_p); - BIF_RET(res); + + return 0; } -BIF_RETTYPE ebif_minusminus_2(BIF_ALIST_2) -{ - return subtract(BIF_P, BIF_ARG_1, BIF_ARG_2); +static int subtract_enter_naive_lhs(Process *p, ErtsSubtractContext *context) { + Eterm iterator; + int i = 0; + + context->stage = SUBTRACT_STAGE_NAIVE_LHS; + + context->iterator = context->rhs_original; + context->result = NIL; + + iterator = context->lhs_original; + + while (is_list(iterator)) { + const Eterm *cell = list_val(iterator); + + ASSERT(i < SUBTRACT_LHS_THRESHOLD); + + context->u.lhs_elements[i++] = CAR(cell); + iterator = CDR(cell); + } + + ASSERT(i == context->lhs_remaining); + + return subtract_continue(p, context); } -BIF_RETTYPE subtract_2(BIF_ALIST_2) -{ - return subtract(BIF_P, BIF_ARG_1, BIF_ARG_2); +static int subtract_naive_lhs(Process *p, ErtsSubtractContext *context) { + const Sint CHECKS_PER_RED = 16; + Sint checks, budget; + + budget = CHECKS_PER_RED * ERTS_BIF_REDS_LEFT(p); + checks = 0; + + while (checks < budget && is_list(context->iterator)) { + const Eterm *cell; + Eterm value, next; + int found_at; + + cell = list_val(context->iterator); + + value = CAR(cell); + next = CDR(cell); + + for (found_at = 0; found_at < context->lhs_remaining; found_at++) { + if (EQ(value, context->u.lhs_elements[found_at])) { + /* We shift the array one step down as we have to preserve + * order. + * + * Note that we can't exit early as that would suppress errors + * in the right-hand side (this runs prior to determining the + * length of RHS). */ + + context->lhs_remaining--; + sys_memmove(&context->u.lhs_elements[found_at], + &context->u.lhs_elements[found_at + 1], + (context->lhs_remaining - found_at) * sizeof(Eterm)); + break; + } + } + + checks += MAX(1, context->lhs_remaining); + context->iterator = next; + } + + BUMP_REDS(p, MIN(checks, budget) / CHECKS_PER_RED); + + if (is_list(context->iterator)) { + return 0; + } else if (!is_nil(context->iterator)) { + return -1; + } + + if (context->lhs_remaining > 0) { + Eterm *hp; + int i; + + hp = HAlloc(p, context->lhs_remaining * 2); + + for (i = context->lhs_remaining - 1; i >= 0; i--) { + Eterm value = context->u.lhs_elements[i]; + + context->result = CONS(hp, value, context->result); + hp += 2; + } + } + + ASSERT(context->lhs_remaining > 0 || context->result == NIL); + + return 1; +} + +static int subtract_enter_naive_rhs(Process *p, ErtsSubtractContext *context) { + Eterm iterator; + int i = 0; + + context->stage = SUBTRACT_STAGE_NAIVE_RHS; + + context->iterator = context->lhs_original; + context->result_cdr = &context->result; + context->result = NIL; + + iterator = context->rhs_original; + + while (is_list(iterator)) { + const Eterm *cell = list_val(iterator); + + ASSERT(i < SUBTRACT_RHS_THRESHOLD); + + context->u.rhs_elements[i++] = CAR(cell); + iterator = CDR(cell); + } + + ASSERT(i == context->rhs_remaining); + + return subtract_continue(p, context); +} + +static int subtract_naive_rhs(Process *p, ErtsSubtractContext *context) { + const Sint CHECKS_PER_RED = 16; + Sint checks, budget; + + budget = CHECKS_PER_RED * ERTS_BIF_REDS_LEFT(p); + checks = 0; + +#ifdef DEBUG + budget = budget / 10 + 1; +#endif + + while (checks < budget && is_list(context->iterator)) { + const Eterm *cell; + Eterm value, next; + int found_at; + + cell = list_val(context->iterator); + value = CAR(cell); + next = CDR(cell); + + for (found_at = context->rhs_remaining - 1; found_at >= 0; found_at--) { + if (EQ(value, context->u.rhs_elements[found_at])) { + break; + } + } + + if (found_at < 0) { + /* Destructively add the value to the result. This is safe + * since the GC is disabled and the unfinished term is never + * leaked to the outside world. */ + Eterm *hp = HAllocX(p, 2, context->lhs_remaining * 2); + + *context->result_cdr = make_list(hp); + context->result_cdr = &CDR(hp); + + CAR(hp) = value; + } else if (found_at >= 0) { + Eterm swap; + + if (context->rhs_remaining-- == 1) { + /* We've run out of items to remove, so the rest of the + * result will be equal to the remainder of the input. We know + * that LHS is well-formed as any errors would've been reported + * during length determination. */ + *context->result_cdr = next; + + BUMP_REDS(p, MIN(budget, checks) / CHECKS_PER_RED); + + return 1; + } + + swap = context->u.rhs_elements[context->rhs_remaining]; + context->u.rhs_elements[found_at] = swap; + } + + checks += context->rhs_remaining; + context->iterator = next; + context->lhs_remaining--; + } + + /* The result only has to be terminated when returning it to the user, but + * we're doing it when trapping as well to prevent headaches when + * debugging. */ + *context->result_cdr = NIL; + + BUMP_REDS(p, MIN(budget, checks) / CHECKS_PER_RED); + + if (is_list(context->iterator)) { + ASSERT(context->lhs_remaining > 0 && context->rhs_remaining > 0); + return 0; + } + + return 1; } +static int subtract_enter_set_build(Process *p, ErtsSubtractContext *context) { + context->stage = SUBTRACT_STAGE_SET_BUILD; + + context->u.rhs_set.alloc_start = + erts_alloc(ERTS_ALC_T_LIST_TRAP, + context->rhs_remaining * sizeof(subtract_tree_t)); + + context->u.rhs_set.alloc = context->u.rhs_set.alloc_start; + context->u.rhs_set.tree = NULL; + + context->iterator = context->rhs_original; + + return subtract_continue(p, context); +} + +static int subtract_set_build(Process *p, ErtsSubtractContext *context) { + const static Sint INSERTIONS_PER_RED = 16; + Sint budget, insertions; + + budget = INSERTIONS_PER_RED * ERTS_BIF_REDS_LEFT(p); + insertions = 0; + +#ifdef DEBUG + budget = budget / 10 + 1; +#endif + + while (insertions < budget && is_list(context->iterator)) { + subtract_tree_t *existing_node, *new_node; + const Eterm *cell; + Eterm value, next; + + cell = list_val(context->iterator); + value = CAR(cell); + next = CDR(cell); + + new_node = context->u.rhs_set.alloc; + new_node->key = value; + new_node->count = 1; + + existing_node = subtract_rbt_lookup_insert(&context->u.rhs_set.tree, + new_node); + + if (existing_node != NULL) { + existing_node->count++; + } else { + context->u.rhs_set.alloc++; + } + + context->iterator = next; + insertions++; + } + + BUMP_REDS(p, insertions / INSERTIONS_PER_RED); + + ASSERT(is_list(context->iterator) || is_nil(context->iterator)); + ASSERT(context->u.rhs_set.tree != NULL); + + return is_nil(context->iterator); +} + +static int subtract_enter_set_finish(Process *p, ErtsSubtractContext *context) { + context->stage = SUBTRACT_STAGE_SET_FINISH; + + context->result_cdr = &context->result; + context->result = NIL; + + context->iterator = context->lhs_original; + + return subtract_continue(p, context); +} + +static int subtract_set_finish(Process *p, ErtsSubtractContext *context) { + const Sint CHECKS_PER_RED = 8; + Sint checks, budget; + + budget = CHECKS_PER_RED * ERTS_BIF_REDS_LEFT(p); + checks = 0; + +#ifdef DEBUG + budget = budget / 10 + 1; +#endif + + while (checks < budget && is_list(context->iterator)) { + subtract_tree_t *node; + const Eterm *cell; + Eterm value, next; + + cell = list_val(context->iterator); + value = CAR(cell); + next = CDR(cell); + + ASSERT(context->rhs_remaining > 0); + + node = subtract_rbt_lookup(context->u.rhs_set.tree, value); + + if (node == NULL) { + Eterm *hp = HAllocX(p, 2, context->lhs_remaining * 2); + + *context->result_cdr = make_list(hp); + context->result_cdr = &CDR(hp); + + CAR(hp) = value; + } else { + if (context->rhs_remaining-- == 1) { + *context->result_cdr = next; + + BUMP_REDS(p, checks / CHECKS_PER_RED); + + return 1; + } + + if (node->count-- == 1) { + subtract_rbt_delete(&context->u.rhs_set.tree, node); + } + } + + context->iterator = next; + context->lhs_remaining--; + checks++; + } + + *context->result_cdr = NIL; + + BUMP_REDS(p, checks / CHECKS_PER_RED); + + if (is_list(context->iterator)) { + ASSERT(context->lhs_remaining > 0 && context->rhs_remaining > 0); + return 0; + } + + return 1; +} + +static int subtract_continue(Process *p, ErtsSubtractContext *context) { + switch (context->stage) { + case SUBTRACT_STAGE_START: { + return subtract_enter_len_lhs(p, context); + } + + case SUBTRACT_STAGE_LEN_LHS: { + int res = subtract_get_length(p, + &context->iterator, + &context->lhs_remaining); + + if (res != 1) { + return res; + } + + if (context->lhs_remaining <= SUBTRACT_LHS_THRESHOLD) { + return subtract_enter_naive_lhs(p, context); + } + + return subtract_enter_len_rhs(p, context); + } + + case SUBTRACT_STAGE_NAIVE_LHS: { + return subtract_naive_lhs(p, context); + } + + case SUBTRACT_STAGE_LEN_RHS: { + int res = subtract_get_length(p, + &context->iterator, + &context->rhs_remaining); + + if (res != 1) { + return res; + } + + /* We've walked through both lists fully now so we no longer need + * to check for errors past this point. */ + + if (context->rhs_remaining <= SUBTRACT_RHS_THRESHOLD) { + return subtract_enter_naive_rhs(p, context); + } + + return subtract_enter_set_build(p, context); + } + + case SUBTRACT_STAGE_NAIVE_RHS: { + return subtract_naive_rhs(p, context); + } + + case SUBTRACT_STAGE_SET_BUILD: { + int res = subtract_set_build(p, context); + + if (res != 1) { + return res; + } + + return subtract_enter_set_finish(p, context); + } + + case SUBTRACT_STAGE_SET_FINISH: { + return subtract_set_finish(p, context); + } + + default: + ERTS_ASSERT(!"unreachable"); + } +} + +static int subtract_start(Process *p, Eterm lhs, Eterm rhs, + ErtsSubtractContext *context) { + context->stage = SUBTRACT_STAGE_START; + + context->lhs_original = lhs; + context->rhs_original = rhs; + + return subtract_continue(p, context); +} + +/* erlang:'--'/2 */ +static Eterm subtract(Export *bif_entry, BIF_ALIST_2) { + Eterm lhs = BIF_ARG_1, rhs = BIF_ARG_2; + + if ((is_list(lhs) || is_nil(lhs)) && (is_list(rhs) || is_nil(rhs))) { + /* We start with the context on the stack in the hopes that we won't + * have to trap. */ + ErtsSubtractContext context; + int res; + + res = subtract_start(BIF_P, lhs, rhs, &context); + + if (res == 0) { + Eterm state_mref; + + state_mref = subtract_create_trap_state(BIF_P, &context); + erts_set_gc_state(BIF_P, 0); + + BIF_TRAP2(bif_entry, BIF_P, state_mref, NIL); + } + + subtract_ctx_dtor(&context); + + if (res < 0) { + BIF_ERROR(BIF_P, BADARG); + } + + BIF_RET(context.result); + } else if (is_internal_magic_ref(lhs)) { + ErtsSubtractContext *context; + int (*dtor)(Binary*); + Binary *magic_bin; + + int res; + + magic_bin = erts_magic_ref2bin(lhs); + dtor = ERTS_MAGIC_BIN_DESTRUCTOR(magic_bin); + + if (dtor != subtract_ctx_bin_dtor) { + BIF_ERROR(BIF_P, BADARG); + } + + ASSERT(BIF_P->flags & F_DISABLE_GC); + ASSERT(rhs == NIL); + + context = ERTS_MAGIC_BIN_DATA(magic_bin); + res = subtract_continue(BIF_P, context); + + if (res == 0) { + BIF_TRAP2(bif_entry, BIF_P, lhs, NIL); + } + + erts_set_gc_state(BIF_P, 1); + + if (res < 0) { + ERTS_BIF_ERROR_TRAPPED2(BIF_P, BADARG, bif_entry, + context->lhs_original, + context->rhs_original); + } + + BIF_RET(context->result); + } + + ASSERT(!(BIF_P->flags & F_DISABLE_GC)); + + BIF_ERROR(BIF_P, BADARG); +} + +BIF_RETTYPE ebif_minusminus_2(BIF_ALIST_2) { + return subtract(bif_export[BIF_ebif_minusminus_2], BIF_CALL_ARGS); +} + +BIF_RETTYPE subtract_2(BIF_ALIST_2) { + return subtract(bif_export[BIF_subtract_2], BIF_CALL_ARGS); +} + + BIF_RETTYPE lists_member_2(BIF_ALIST_2) { Eterm term; @@ -283,7 +905,7 @@ static BIF_RETTYPE lists_reverse_alloc(Process *c_p, { static const Uint CELLS_PER_RED = 40; - Eterm *heap_top, *heap_end; + Eterm *alloc_top, *alloc_end; Uint cells_left, max_cells; Eterm list, tail; Eterm lookahead; @@ -305,18 +927,18 @@ static BIF_RETTYPE lists_reverse_alloc(Process *c_p, BIF_ERROR(c_p, BADARG); } - heap_top = HAlloc(c_p, 2 * (max_cells - cells_left)); - heap_end = heap_top + 2 * (max_cells - cells_left); + alloc_top = HAlloc(c_p, 2 * (max_cells - cells_left)); + alloc_end = alloc_top + 2 * (max_cells - cells_left); - while (heap_top < heap_end) { + while (alloc_top < alloc_end) { Eterm *pair = list_val(list); - tail = CONS(heap_top, CAR(pair), tail); + tail = CONS(alloc_top, CAR(pair), tail); list = CDR(pair); ASSERT(is_list(list) || is_nil(list)); - heap_top += 2; + alloc_top += 2; } if (is_nil(list)) { @@ -333,7 +955,7 @@ static BIF_RETTYPE lists_reverse_onheap(Process *c_p, { static const Uint CELLS_PER_RED = 60; - Eterm *heap_top, *heap_end; + Eterm *alloc_start, *alloc_top, *alloc_end; Uint cells_left, max_cells; Eterm list, tail; @@ -343,21 +965,27 @@ static BIF_RETTYPE lists_reverse_onheap(Process *c_p, cells_left = max_cells = CELLS_PER_RED * (1 + ERTS_BIF_REDS_LEFT(c_p)); ASSERT(HEAP_LIMIT(c_p) >= HEAP_TOP(c_p) + 2); - heap_end = HEAP_LIMIT(c_p) - 2; - heap_top = HEAP_TOP(c_p); + alloc_start = HEAP_TOP(c_p); + alloc_end = HEAP_LIMIT(c_p) - 2; + alloc_top = alloc_start; + + /* Don't process more cells than we have reductions for. */ + alloc_end = MIN(alloc_top + (cells_left * 2), alloc_end); - while (heap_top < heap_end && is_list(list)) { + while (alloc_top < alloc_end && is_list(list)) { Eterm *pair = list_val(list); - tail = CONS(heap_top, CAR(pair), tail); + tail = CONS(alloc_top, CAR(pair), tail); list = CDR(pair); - heap_top += 2; + alloc_top += 2; } - cells_left -= (heap_top - heap_end) / 2; + cells_left -= (alloc_top - alloc_start) / 2; + HEAP_TOP(c_p) = alloc_top; + + ASSERT(cells_left >= 0 && cells_left <= max_cells); BUMP_REDS(c_p, (max_cells - cells_left) / CELLS_PER_RED); - HEAP_TOP(c_p) = heap_top; if (is_nil(list)) { BIF_RET(tail); diff --git a/erts/emulator/beam/erl_bif_persistent.c b/erts/emulator/beam/erl_bif_persistent.c new file mode 100644 index 0000000000..9dca768a18 --- /dev/null +++ b/erts/emulator/beam/erl_bif_persistent.c @@ -0,0 +1,983 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +/* + * Purpose: Implement persistent term storage. + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "sys.h" +#include "erl_vm.h" +#include "global.h" +#include "erl_process.h" +#include "error.h" +#include "erl_driver.h" +#include "bif.h" +#include "erl_map.h" +#include "erl_binary.h" + +/* + * The limit for the number of persistent terms before + * a warning is issued. + */ + +#define WARNING_LIMIT 20000 +#define XSTR(s) STR(s) +#define STR(s) #s + +/* + * Parameters for the hash table. + */ +#define INITIAL_SIZE 8 +#define LOAD_FACTOR ((Uint)50) +#define MUST_GROW(t) (((Uint)100) * t->num_entries >= LOAD_FACTOR * t->allocated) +#define MUST_SHRINK(t) (((Uint)200) * t->num_entries <= LOAD_FACTOR * t->allocated && \ + t->allocated > INITIAL_SIZE) + +typedef struct hash_table { + Uint allocated; + Uint num_entries; + Uint mask; + Uint first_to_delete; + Uint num_to_delete; + erts_atomic_t refc; + struct hash_table* delete_next; + ErtsThrPrgrLaterOp thr_prog_op; + Eterm term[1]; +} HashTable; + +typedef struct trap_data { + HashTable* table; + Uint idx; + Uint remaining; + Uint memory; /* Used by info/0 to count used memory */ +} TrapData; + +/* + * Declarations of local functions. + */ + +static HashTable* create_initial_table(void); +static Uint lookup(HashTable* hash_table, Eterm key); +static HashTable* copy_table(HashTable* old_table, Uint new_size, int rehash); +static HashTable* tmp_table_copy(HashTable* old_table); +static int try_seize_update_permission(Process* c_p); +static void release_update_permission(int release_updater); +static void table_updater(void* table); +static void table_deleter(void* hash_table); +static void dec_table_refc(Process* c_p, HashTable* old_table); +static void delete_table(Process* c_p, HashTable* table); +static void mark_for_deletion(HashTable* hash_table, Uint entry_index); +static ErtsLiteralArea* term_to_area(Eterm tuple); +static void suspend_updater(Process* c_p); +static Eterm do_get_all(Process* c_p, TrapData* trap_data, Eterm res); +static Eterm do_info(Process* c_p, TrapData* trap_data); +static void append_to_delete_queue(HashTable* table); +static HashTable* next_to_delete(void); +static Eterm alloc_trap_data(Process* c_p); +static int cleanup_trap_data(Binary *bp); + +/* + * Traps + */ + +static Export persistent_term_get_all_export; +static BIF_RETTYPE persistent_term_get_all_trap(BIF_ALIST_2); +static Export persistent_term_info_export; +static BIF_RETTYPE persistent_term_info_trap(BIF_ALIST_1); + +/* + * Pointer to the current hash table. + */ + +static erts_atomic_t the_hash_table; + +/* + * Queue of processes waiting to update the hash table. + */ + +struct update_queue_item { + Process *p; + struct update_queue_item* next; +}; + +static erts_mtx_t update_table_permission_mtx; +static struct update_queue_item* update_queue = NULL; +static Process* updater_process = NULL; + +/* Protected by update_table_permission_mtx */ +static ErtsThrPrgrLaterOp thr_prog_op; +static int issued_warning = 0; + +/* + * Queue of hash tables to be deleted. + */ + +static erts_mtx_t delete_queue_mtx; +static HashTable* delete_queue_head = NULL; +static HashTable** delete_queue_tail = &delete_queue_head; + +/* + * The following variables are only used during crash dumping. They + * are intialized by erts_init_persistent_dumping(). + */ + +ErtsLiteralArea** erts_persistent_areas; +Uint erts_num_persistent_areas; + +void erts_init_bif_persistent_term(void) +{ + HashTable* hash_table; + + /* + * Initialize the mutex protecting updates. + */ + + erts_mtx_init(&update_table_permission_mtx, + "update_persistent_term_permission", + NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | + ERTS_LOCK_FLAGS_CATEGORY_GENERIC); + + /* + * Initialize delete queue. + */ + + erts_mtx_init(&delete_queue_mtx, + "persistent_term_delete_permission", + NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | + ERTS_LOCK_FLAGS_CATEGORY_GENERIC); + + /* + * Allocate a small initial hash table. + */ + + hash_table = create_initial_table(); + erts_atomic_init_nob(&the_hash_table, (erts_aint_t)hash_table); + + /* + * Initialize export entry for traps + */ + + erts_init_trap_export(&persistent_term_get_all_export, + am_persistent_term, am_get_all_trap, 2, + &persistent_term_get_all_trap); + erts_init_trap_export(&persistent_term_info_export, + am_persistent_term, am_info_trap, 1, + &persistent_term_info_trap); +} + +BIF_RETTYPE persistent_term_put_2(BIF_ALIST_2) +{ + Eterm key; + Eterm term; + Eterm heap[3]; + Eterm tuple; + HashTable* hash_table; + Uint term_size; + Uint lit_area_size; + ErlOffHeap code_off_heap; + ErtsLiteralArea* literal_area; + erts_shcopy_t info; + Eterm* ptr; + Uint entry_index; + + if (!try_seize_update_permission(BIF_P)) { + ERTS_BIF_YIELD2(bif_export[BIF_persistent_term_put_2], + BIF_P, BIF_ARG_1, BIF_ARG_2); + } + + hash_table = (HashTable *) erts_atomic_read_nob(&the_hash_table); + + key = BIF_ARG_1; + term = BIF_ARG_2; + + entry_index = lookup(hash_table, key); + + heap[0] = make_arityval(2); + heap[1] = key; + heap[2] = term; + tuple = make_tuple(heap); + + if (is_nil(hash_table->term[entry_index])) { + Uint size = hash_table->allocated; + if (MUST_GROW(hash_table)) { + size *= 2; + } + hash_table = copy_table(hash_table, size, 0); + entry_index = lookup(hash_table, key); + hash_table->num_entries++; + } else { + Eterm tuple = hash_table->term[entry_index]; + Eterm old_term; + + ASSERT(is_tuple_arity(tuple, 2)); + old_term = boxed_val(tuple)[2]; + if (EQ(term, old_term)) { + /* Same value. No need to update anything. */ + release_update_permission(0); + BIF_RET(am_ok); + } else { + /* Mark the old term for deletion. */ + mark_for_deletion(hash_table, entry_index); + hash_table = copy_table(hash_table, hash_table->allocated, 0); + } + } + + /* + * Preserve internal sharing in the term by using the + * sharing-preserving functions. However, literals must + * be copied in case the module holding them are unloaded. + */ + INITIALIZE_SHCOPY(info); + info.copy_literals = 1; + term_size = copy_shared_calculate(tuple, &info); + ERTS_INIT_OFF_HEAP(&code_off_heap); + lit_area_size = ERTS_LITERAL_AREA_ALLOC_SIZE(term_size); + literal_area = erts_alloc(ERTS_ALC_T_LITERAL, lit_area_size); + ptr = &literal_area->start[0]; + literal_area->end = ptr + term_size; + tuple = copy_shared_perform(tuple, term_size, &info, &ptr, &code_off_heap); + ASSERT(tuple_val(tuple) == literal_area->start); + literal_area->off_heap = code_off_heap.first; + DESTROY_SHCOPY(info); + erts_set_literal_tag(&tuple, literal_area->start, term_size); + hash_table->term[entry_index] = tuple; + + erts_schedule_thr_prgr_later_op(table_updater, hash_table, &thr_prog_op); + suspend_updater(BIF_P); + + /* + * Issue a warning once if the warning limit has been exceeded. + */ + + if (hash_table->num_entries > WARNING_LIMIT && issued_warning == 0) { + static char w[] = + "More than " XSTR(WARNING_LIMIT) " persistent terms " + "have been created.\n" + "It is recommended to avoid creating an excessive number of\n" + "persistent terms, as creation and deletion of persistent terms\n" + "will be slower as the number of persistent terms increases.\n"; + issued_warning = 1; + erts_send_warning_to_logger_str(BIF_P->group_leader, w); + } + + ERTS_BIF_YIELD_RETURN(BIF_P, am_ok); +} + +BIF_RETTYPE persistent_term_get_0(BIF_ALIST_0) +{ + HashTable* hash_table = (HashTable *) erts_atomic_read_nob(&the_hash_table); + TrapData* trap_data; + Eterm res = NIL; + Eterm magic_ref; + Binary* mbp; + + magic_ref = alloc_trap_data(BIF_P); + mbp = erts_magic_ref2bin(magic_ref); + trap_data = ERTS_MAGIC_BIN_DATA(mbp); + trap_data->table = hash_table; + trap_data->idx = 0; + trap_data->remaining = hash_table->num_entries; + res = do_get_all(BIF_P, trap_data, res); + if (trap_data->remaining == 0) { + BUMP_REDS(BIF_P, hash_table->num_entries); + trap_data->table = NULL; /* Prevent refc decrement */ + BIF_RET(res); + } else { + /* + * Increment the ref counter to prevent an update operation (by put/2 + * or erase/1) to delete this hash table. + */ + erts_atomic_inc_nob(&hash_table->refc); + BUMP_ALL_REDS(BIF_P); + BIF_TRAP2(&persistent_term_get_all_export, BIF_P, magic_ref, res); + } +} + +BIF_RETTYPE persistent_term_get_1(BIF_ALIST_1) +{ + Eterm key = BIF_ARG_1; + HashTable* hash_table = (HashTable *) erts_atomic_read_nob(&the_hash_table); + Uint entry_index; + Eterm term; + + entry_index = lookup(hash_table, key); + term = hash_table->term[entry_index]; + if (is_boxed(term)) { + ASSERT(is_tuple_arity(term, 2)); + BIF_RET(tuple_val(term)[2]); + } + BIF_ERROR(BIF_P, BADARG); +} + +BIF_RETTYPE persistent_term_erase_1(BIF_ALIST_1) +{ + Eterm key = BIF_ARG_1; + HashTable* old_table; + HashTable* new_table; + Uint entry_index; + Eterm old_term; + + if (!try_seize_update_permission(BIF_P)) { + ERTS_BIF_YIELD1(bif_export[BIF_persistent_term_erase_1], + BIF_P, BIF_ARG_1); + } + + old_table = (HashTable *) erts_atomic_read_nob(&the_hash_table); + entry_index = lookup(old_table, key); + old_term = old_table->term[entry_index]; + if (is_boxed(old_term)) { + Uint new_size; + HashTable* tmp_table; + + /* + * Since we don't use any delete markers, we must rehash + * the table when deleting terms to ensure that all terms + * can still be reached if there are hash collisions. + * We can't rehash in place and it would not be safe to modify + * the old table yet, so we will first need a new + * temporary table copy of the same size as the old one. + */ + + ASSERT(is_tuple_arity(old_term, 2)); + tmp_table = tmp_table_copy(old_table); + + /* + * Delete the term from the temporary table. Then copy the + * temporary table to a new table, rehashing the entries + * while copying. + */ + + tmp_table->term[entry_index] = NIL; + tmp_table->num_entries--; + new_size = tmp_table->allocated; + if (MUST_SHRINK(tmp_table)) { + new_size /= 2; + } + new_table = copy_table(tmp_table, new_size, 1); + erts_free(ERTS_ALC_T_TMP, tmp_table); + + mark_for_deletion(old_table, entry_index); + erts_schedule_thr_prgr_later_op(table_updater, new_table, &thr_prog_op); + suspend_updater(BIF_P); + ERTS_BIF_YIELD_RETURN(BIF_P, am_true); + } + + /* + * Key is not present. Nothing to do. + */ + + ASSERT(is_nil(old_term)); + release_update_permission(0); + BIF_RET(am_false); +} + +BIF_RETTYPE erts_internal_erase_persistent_terms_0(BIF_ALIST_0) +{ + HashTable* old_table; + HashTable* new_table; + + if (!try_seize_update_permission(BIF_P)) { + ERTS_BIF_YIELD0(bif_export[BIF_erts_internal_erase_persistent_terms_0], + BIF_P); + } + old_table = (HashTable *) erts_atomic_read_nob(&the_hash_table); + old_table->first_to_delete = 0; + old_table->num_to_delete = old_table->allocated; + new_table = create_initial_table(); + erts_schedule_thr_prgr_later_op(table_updater, new_table, &thr_prog_op); + suspend_updater(BIF_P); + ERTS_BIF_YIELD_RETURN(BIF_P, am_true); +} + +BIF_RETTYPE persistent_term_info_0(BIF_ALIST_0) +{ + HashTable* hash_table = (HashTable *) erts_atomic_read_nob(&the_hash_table); + TrapData* trap_data; + Eterm res = NIL; + Eterm magic_ref; + Binary* mbp; + + magic_ref = alloc_trap_data(BIF_P); + mbp = erts_magic_ref2bin(magic_ref); + trap_data = ERTS_MAGIC_BIN_DATA(mbp); + trap_data->table = hash_table; + trap_data->idx = 0; + trap_data->remaining = hash_table->num_entries; + trap_data->memory = 0; + res = do_info(BIF_P, trap_data); + if (trap_data->remaining == 0) { + BUMP_REDS(BIF_P, hash_table->num_entries); + trap_data->table = NULL; /* Prevent refc decrement */ + BIF_RET(res); + } else { + /* + * Increment the ref counter to prevent an update operation (by put/2 + * or erase/1) to delete this hash table. + */ + erts_atomic_inc_nob(&hash_table->refc); + BUMP_ALL_REDS(BIF_P); + BIF_TRAP2(&persistent_term_info_export, BIF_P, magic_ref, res); + } +} + +Uint +erts_persistent_term_count(void) +{ + HashTable* hash_table = (HashTable *) erts_atomic_read_nob(&the_hash_table); + return hash_table->num_entries; +} + +void +erts_init_persistent_dumping(void) +{ + HashTable* hash_table = (HashTable *) erts_atomic_read_nob(&the_hash_table); + ErtsLiteralArea** area_p; + Uint i; + + /* + * Overwrite the array of Eterms in the current hash table + * with pointers to literal areas. + */ + + erts_persistent_areas = (ErtsLiteralArea **) hash_table->term; + erts_num_persistent_areas = hash_table->num_entries; + area_p = erts_persistent_areas; + for (i = 0; i < hash_table->allocated; i++) { + Eterm term = hash_table->term[i]; + + if (is_boxed(term)) { + *area_p++ = term_to_area(term); + } + } +} + +/* + * Local functions. + */ + +static HashTable* +create_initial_table(void) +{ + HashTable* hash_table; + int i; + + hash_table = (HashTable *) erts_alloc(ERTS_ALC_T_PERSISTENT_TERM, + sizeof(HashTable)+sizeof(Eterm) * + (INITIAL_SIZE-1)); + hash_table->allocated = INITIAL_SIZE; + hash_table->num_entries = 0; + hash_table->mask = INITIAL_SIZE-1; + hash_table->first_to_delete = 0; + hash_table->num_to_delete = 0; + erts_atomic_init_nob(&hash_table->refc, (erts_aint_t)1); + for (i = 0; i < INITIAL_SIZE; i++) { + hash_table->term[i] = NIL; + } + return hash_table; +} + +static BIF_RETTYPE +persistent_term_get_all_trap(BIF_ALIST_2) +{ + TrapData* trap_data; + Eterm res = BIF_ARG_2; + Uint bump_reds; + Binary* mbp; + + ASSERT(is_list(BIF_ARG_2)); + mbp = erts_magic_ref2bin(BIF_ARG_1); + trap_data = ERTS_MAGIC_BIN_DATA(mbp); + bump_reds = trap_data->remaining; + res = do_get_all(BIF_P, trap_data, res); + ASSERT(is_list(res)); + if (trap_data->remaining > 0) { + BUMP_ALL_REDS(BIF_P); + BIF_TRAP2(&persistent_term_get_all_export, BIF_P, BIF_ARG_1, res); + } else { + /* + * Decrement ref count (and possibly delete the hash table + * and associated literal area). + */ + dec_table_refc(BIF_P, trap_data->table); + trap_data->table = NULL; /* Prevent refc decrement */ + BUMP_REDS(BIF_P, bump_reds); + BIF_RET(res); + } +} + +static Eterm +do_get_all(Process* c_p, TrapData* trap_data, Eterm res) +{ + HashTable* hash_table; + Uint remaining; + Uint idx; + Uint max_iter; + Uint i; + Eterm* hp; + Uint heap_size; + struct copy_term { + Uint key_size; + Eterm* tuple_ptr; + } *copy_data; + + hash_table = trap_data->table; + idx = trap_data->idx; +#if defined(DEBUG) || defined(VALGRIND) + max_iter = 50; +#else + max_iter = ERTS_BIF_REDS_LEFT(c_p); +#endif + remaining = trap_data->remaining < max_iter ? + trap_data->remaining : max_iter; + trap_data->remaining -= remaining; + + copy_data = (struct copy_term *) erts_alloc(ERTS_ALC_T_TMP, + remaining * + sizeof(struct copy_term)); + i = 0; + heap_size = (2 + 3) * remaining; + while (remaining != 0) { + Eterm term = hash_table->term[idx]; + if (is_tuple(term)) { + Uint key_size; + Eterm* tup_val; + + ASSERT(is_tuple_arity(term, 2)); + tup_val = tuple_val(term); + key_size = size_object(tup_val[1]); + copy_data[i].key_size = key_size; + copy_data[i].tuple_ptr = tup_val; + heap_size += key_size; + i++; + remaining--; + } + idx++; + } + trap_data->idx = idx; + + hp = HAlloc(c_p, heap_size); + remaining = i; + for (i = 0; i < remaining; i++) { + Eterm* tuple_ptr; + Uint key_size; + Eterm key; + Eterm tup; + + tuple_ptr = copy_data[i].tuple_ptr; + key_size = copy_data[i].key_size; + key = copy_struct(tuple_ptr[1], key_size, &hp, &c_p->off_heap); + tup = TUPLE2(hp, key, tuple_ptr[2]); + hp += 3; + res = CONS(hp, tup, res); + hp += 2; + } + erts_free(ERTS_ALC_T_TMP, copy_data); + return res; +} + +static BIF_RETTYPE +persistent_term_info_trap(BIF_ALIST_1) +{ + TrapData* trap_data = (TrapData *) BIF_ARG_1; + Eterm res; + Uint bump_reds; + Binary* mbp; + + mbp = erts_magic_ref2bin(BIF_ARG_1); + trap_data = ERTS_MAGIC_BIN_DATA(mbp); + bump_reds = trap_data->remaining; + res = do_info(BIF_P, trap_data); + if (trap_data->remaining > 0) { + ASSERT(res == am_ok); + BUMP_ALL_REDS(BIF_P); + BIF_TRAP1(&persistent_term_info_export, BIF_P, BIF_ARG_1); + } else { + /* + * Decrement ref count (and possibly delete the hash table + * and associated literal area). + */ + dec_table_refc(BIF_P, trap_data->table); + trap_data->table = NULL; /* Prevent refc decrement */ + BUMP_REDS(BIF_P, bump_reds); + ASSERT(is_map(res)); + BIF_RET(res); + } +} + +#define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1) + +static Eterm +do_info(Process* c_p, TrapData* trap_data) +{ + HashTable* hash_table; + Uint remaining; + Uint idx; + Uint max_iter; + + hash_table = trap_data->table; + idx = trap_data->idx; +#if defined(DEBUG) || defined(VALGRIND) + max_iter = 50; +#else + max_iter = ERTS_BIF_REDS_LEFT(c_p); +#endif + remaining = trap_data->remaining < max_iter ? trap_data->remaining : max_iter; + trap_data->remaining -= remaining; + while (remaining != 0) { + if (is_boxed(hash_table->term[idx])) { + ErtsLiteralArea* area; + area = term_to_area(hash_table->term[idx]); + trap_data->memory += sizeof(ErtsLiteralArea) + + sizeof(Eterm) * (area->end - area->start - 1); + remaining--; + } + idx++; + } + trap_data->idx = idx; + if (trap_data->remaining > 0) { + return am_ok; /* Dummy return value */ + } else { + Eterm* hp; + Eterm count_term; + Eterm memory_term; + Eterm res; + Uint memory; + Uint hsz = MAP_SZ(2); + + memory = sizeof(HashTable) + (trap_data->table->allocated-1) * + sizeof(Eterm) + trap_data->memory; + (void) erts_bld_uint(NULL, &hsz, hash_table->num_entries); + (void) erts_bld_uint(NULL, &hsz, memory); + hp = HAlloc(c_p, hsz); + count_term = erts_bld_uint(&hp, NULL, hash_table->num_entries); + memory_term = erts_bld_uint(&hp, NULL, memory); + res = MAP2(hp, am_count, count_term, am_memory, memory_term); + return res; + } +} + +#undef DECL_AM + +static Eterm +alloc_trap_data(Process* c_p) +{ + Binary* mbp = erts_create_magic_binary(sizeof(TrapData), + cleanup_trap_data); + Eterm* hp; + + hp = HAlloc(c_p, ERTS_MAGIC_REF_THING_SIZE); + return erts_mk_magic_ref(&hp, &MSO(c_p), mbp); +} + +static int +cleanup_trap_data(Binary *bp) +{ + TrapData* trap_data = ERTS_MAGIC_BIN_DATA(bp); + + if (trap_data->table) { + /* + * The process has been killed and is now exiting. + * Decrement the reference counter for the table. + */ + dec_table_refc(NULL, trap_data->table); + } + return 1; +} + +static Uint +lookup(HashTable* hash_table, Eterm key) +{ + Uint mask = hash_table->mask; + Eterm* table = hash_table->term; + Uint32 idx = make_internal_hash(key, 0); + Eterm term; + + do { + idx++; + term = table[idx & mask]; + } while (is_boxed(term) && !EQ(key, (tuple_val(term))[1])); + return idx & mask; +} + +static HashTable* +tmp_table_copy(HashTable* old_table) +{ + Uint size = old_table->allocated; + HashTable* tmp_table; + Uint i; + + tmp_table = (HashTable *) erts_alloc(ERTS_ALC_T_TMP, + sizeof(HashTable) + + sizeof(Eterm) * (size-1)); + *tmp_table = *old_table; + for (i = 0; i < size; i++) { + tmp_table->term[i] = old_table->term[i]; + } + return tmp_table; +} + +static HashTable* +copy_table(HashTable* old_table, Uint new_size, int rehash) +{ + HashTable* new_table; + Uint old_size = old_table->allocated; + Uint i; + + new_table = (HashTable *) erts_alloc(ERTS_ALC_T_PERSISTENT_TERM, + sizeof(HashTable) + + sizeof(Eterm) * (new_size-1)); + if (old_table->allocated == new_size && !rehash) { + /* + * Same size and no key deleted. Make an exact copy of the table. + */ + *new_table = *old_table; + for (i = 0; i < new_size; i++) { + new_table->term[i] = old_table->term[i]; + } + } else { + /* + * The size of the table has changed or an element has been + * deleted. Must rehash, by inserting all old terms into the + * new (empty) table. + */ + new_table->allocated = new_size; + new_table->num_entries = old_table->num_entries; + new_table->mask = new_size - 1; + for (i = 0; i < new_size; i++) { + new_table->term[i] = NIL; + } + for (i = 0; i < old_size; i++) { + if (is_tuple(old_table->term[i])) { + Eterm key = tuple_val(old_table->term[i])[1]; + Uint entry_index = lookup(new_table, key); + ASSERT(is_nil(new_table->term[entry_index])); + new_table->term[entry_index] = old_table->term[i]; + } + } + } + new_table->first_to_delete = 0; + new_table->num_to_delete = 0; + erts_atomic_init_nob(&new_table->refc, (erts_aint_t)1); + return new_table; +} + +static void +mark_for_deletion(HashTable* hash_table, Uint entry_index) +{ + hash_table->first_to_delete = entry_index; + hash_table->num_to_delete = 1; +} + +static ErtsLiteralArea* +term_to_area(Eterm tuple) +{ + ASSERT(is_tuple_arity(tuple, 2)); + return (ErtsLiteralArea *) (((char *) tuple_val(tuple)) - + offsetof(ErtsLiteralArea, start)); +} + +static void +table_updater(void* data) +{ + HashTable* old_table; + HashTable* new_table; + + old_table = (HashTable *) erts_atomic_read_nob(&the_hash_table); + new_table = (HashTable *) data; + ASSERT(new_table->num_to_delete == 0); + erts_atomic_set_nob(&the_hash_table, (erts_aint_t)new_table); + append_to_delete_queue(old_table); + erts_schedule_thr_prgr_later_op(table_deleter, + old_table, + &old_table->thr_prog_op); + release_update_permission(1); +} + +static void +table_deleter(void* data) +{ + HashTable* old_table = (HashTable *) data; + + dec_table_refc(NULL, old_table); +} + +static void +dec_table_refc(Process* c_p, HashTable* old_table) +{ + erts_aint_t refc = erts_atomic_dec_read_nob(&old_table->refc); + + if (refc == 0) { + HashTable* to_delete; + + while ((to_delete = next_to_delete()) != NULL) { + delete_table(c_p, to_delete); + } + } +} + +static void +delete_table(Process* c_p, HashTable* table) +{ + Uint idx = table->first_to_delete; + Uint n = table->num_to_delete; + + /* + * There are no longer any references to this hash table. + * + * Any literals pointed for deletion can be queued for + * deletion and the table itself can be deallocated. + */ + +#ifdef DEBUG + if (n == 1) { + ASSERT(is_tuple_arity(table->term[idx], 2)); + } +#endif + + while (n > 0) { + Eterm term = table->term[idx]; + + if (is_tuple_arity(term, 2)) { + if (is_immed(tuple_val(term)[2])) { + erts_release_literal_area(term_to_area(term)); + } else { + erts_queue_release_literals(c_p, term_to_area(term)); + } + } + idx++, n--; + } + erts_free(ERTS_ALC_T_PERSISTENT_TERM, table); +} + +/* + * Caller *must* yield if this function returns 0. + */ + +static int +try_seize_update_permission(Process* c_p) +{ + int success; + + ASSERT(!erts_thr_progress_is_blocking()); /* to avoid deadlock */ + ASSERT(c_p != NULL); + + erts_mtx_lock(&update_table_permission_mtx); + ASSERT(updater_process != c_p); + success = (updater_process == NULL); + if (success) { + updater_process = c_p; + } else { + struct update_queue_item* qitem; + qitem = erts_alloc(ERTS_ALC_T_PERSISTENT_LOCK_Q, sizeof(*qitem)); + qitem->p = c_p; + erts_proc_inc_refc(c_p); + qitem->next = update_queue; + update_queue = qitem; + erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL); + } + erts_mtx_unlock(&update_table_permission_mtx); + return success; +} + +static void +release_update_permission(int release_updater) +{ + erts_mtx_lock(&update_table_permission_mtx); + ASSERT(updater_process != NULL); + + if (release_updater) { + erts_proc_lock(updater_process, ERTS_PROC_LOCK_STATUS); + if (!ERTS_PROC_IS_EXITING(updater_process)) { + erts_resume(updater_process, ERTS_PROC_LOCK_STATUS); + } + erts_proc_unlock(updater_process, ERTS_PROC_LOCK_STATUS); + } + updater_process = NULL; + + while (update_queue != NULL) { /* Unleash the entire herd */ + struct update_queue_item* qitem = update_queue; + erts_proc_lock(qitem->p, ERTS_PROC_LOCK_STATUS); + if (!ERTS_PROC_IS_EXITING(qitem->p)) { + erts_resume(qitem->p, ERTS_PROC_LOCK_STATUS); + } + erts_proc_unlock(qitem->p, ERTS_PROC_LOCK_STATUS); + update_queue = qitem->next; + erts_proc_dec_refc(qitem->p); + erts_free(ERTS_ALC_T_PERSISTENT_LOCK_Q, qitem); + } + erts_mtx_unlock(&update_table_permission_mtx); +} + +static void +suspend_updater(Process* c_p) +{ +#ifdef DEBUG + ASSERT(c_p != NULL); + erts_mtx_lock(&update_table_permission_mtx); + ASSERT(updater_process == c_p); + erts_mtx_unlock(&update_table_permission_mtx); +#endif + erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL); +} + +static void +append_to_delete_queue(HashTable* table) +{ + erts_mtx_lock(&delete_queue_mtx); + table->delete_next = NULL; + *delete_queue_tail = table; + delete_queue_tail = &table->delete_next; + erts_mtx_unlock(&delete_queue_mtx); +} + +static HashTable* +next_to_delete(void) +{ + HashTable* table; + + erts_mtx_lock(&delete_queue_mtx); + table = delete_queue_head; + if (table) { + if (erts_atomic_read_nob(&table->refc)) { + /* + * This hash table is still referenced. Hash tables + * must be deleted in order, so we return a NULL + * pointer. + */ + table = NULL; + } else { + /* + * Remove the first hash table from the queue. + */ + delete_queue_head = table->delete_next; + if (delete_queue_head == NULL) { + delete_queue_tail = &delete_queue_head; + } + } + } + erts_mtx_unlock(&delete_queue_mtx); + return table; +} diff --git a/erts/emulator/beam/erl_bif_unique.h b/erts/emulator/beam/erl_bif_unique.h index 40b70667c0..944788c67c 100644 --- a/erts/emulator/beam/erl_bif_unique.h +++ b/erts/emulator/beam/erl_bif_unique.h @@ -242,11 +242,11 @@ erts_internal_ref_number_cmp(Uint32 num1[ERTS_REF_NUMBERS], Uint32 num2[ERTS_REF_NUMBERS]) { if (num1[2] != num2[2]) - return (int) ((Sint64) num1[2] - (Sint64) num2[2]); + return num1[2] > num2[2] ? 1 : -1; if (num1[1] != num2[1]) - return (int) ((Sint64) num1[1] - (Sint64) num2[1]); + return num1[1] > num2[1] ? 1 : -1; if (num1[0] != num2[0]) - return (int) ((Sint64) num1[0] - (Sint64) num2[0]); + return num1[0] > num2[0] ? 1 : -1; return 0; } diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index 788718ab09..45e4be2426 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -1860,22 +1860,14 @@ static int db_select_replace_tree(Process *p, DbTable *tbl, Eterm tid, sc.mp = mpi.mp; - stack = get_static_stack(tb); if (!mpi.got_partial && mpi.some_limitation && CMP_EQ(mpi.least,mpi.most)) { - TreeDbTerm* term = *(mpi.save_term); doit_select_replace(tb,mpi.save_term,&sc,0 /* dummy */); - if (stack != NULL) { - if (TOP_NODE(stack) == term) - // throw away potentially invalid reference - REPLACE_TOP_NODE(stack, *(mpi.save_term)); - release_stack(tb, stack); - } + reset_static_stack(tb); /* may refer replaced term */ RET_TO_BIF(erts_make_integer(sc.replaced,p),DB_ERROR_NONE); } - if (stack == NULL) - stack = get_any_stack(tb); + stack = get_any_stack(tb); if (mpi.some_limitation) { if ((this = find_next_from_pb_key(tb, stack, mpi.most)) != NULL) { diff --git a/erts/emulator/beam/erl_dirty_bif.tab b/erts/emulator/beam/erl_dirty_bif.tab index 086275fbe5..20299ff604 100644 --- a/erts/emulator/beam/erl_dirty_bif.tab +++ b/erts/emulator/beam/erl_dirty_bif.tab @@ -59,8 +59,6 @@ dirty-cpu erts_debug:lcnt_clear/0 dirty-cpu-test erlang:'++'/2 dirty-cpu-test erlang:append/2 -dirty-cpu-test erlang:'--'/2 -dirty-cpu-test erlang:subtract/2 dirty-cpu-test erlang:iolist_size/1 dirty-cpu-test erlang:make_tuple/2 dirty-cpu-test erlang:make_tuple/3 diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index a65dbbf42b..47dd115c82 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -1133,9 +1133,28 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, reds = (Sint64) garbage_collect(p, ERTS_INVALID_HFRAG_PTR, 0, p->arg_reg, p->arity, fcalls, ygen_usage); + if (ERTS_PROC_IS_EXITING(p)) { + return 0; + } ASSERT(!(p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC))); + if (MAX_HEAP_SIZE_GET(p)) { + Uint new_heap_size; + Uint old_heap_size; + Uint total_heap_size; + + new_heap_size = HEAP_END(p) - HEAP_START(p); + old_heap_size = erts_next_heap_size(lit_size, 0); + total_heap_size = new_heap_size + old_heap_size; + if (MAX_HEAP_SIZE_GET(p) < total_heap_size && + reached_max_heap_size(p, total_heap_size, + new_heap_size, old_heap_size)) { + erts_set_self_exiting(p, am_killed); + return 0; + } + } + /* * Set GC state. */ diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 57c6c10c7f..f687dcf335 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -354,6 +354,7 @@ erl_init(int ncpu, erts_init_bif(); erts_init_bif_chksum(); erts_init_bif_binary(); + erts_init_bif_persistent_term(); erts_init_bif_re(); erts_init_unicode(); /* after RE to get access to PCRE unicode */ erts_init_external(); diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 463ae898a3..1416c5f96c 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -97,6 +97,8 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "proc_btm", "pid" }, { "dist_entry", "address" }, { "dist_entry_links", "address" }, + { "update_persistent_term_permission", NULL }, + { "persistent_term_delete_permission", NULL }, { "code_write_permission", NULL }, { "purge_state", NULL }, { "proc_status", "pid" }, diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c index 243db4c734..ac5054ea10 100644 --- a/erts/emulator/beam/erl_process_dump.c +++ b/erts/emulator/beam/erl_process_dump.c @@ -58,6 +58,7 @@ static void dump_externally(fmtfn_t to, void *to_arg, Eterm term); static void mark_literal(Eterm* ptr); static void init_literal_areas(void); static void dump_literals(fmtfn_t to, void *to_arg); +static void dump_persistent_terms(fmtfn_t to, void *to_arg); static void dump_module_literals(fmtfn_t to, void *to_arg, ErtsLiteralArea* lit_area); @@ -74,6 +75,7 @@ erts_deep_process_dump(fmtfn_t to, void *to_arg) all_binaries = NULL; init_literal_areas(); + erts_init_persistent_dumping(); for (i = 0; i < max; i++) { Process *p = erts_pix2proc(i); @@ -93,6 +95,7 @@ erts_deep_process_dump(fmtfn_t to, void *to_arg) } } + dump_persistent_terms(to, to_arg); dump_literals(to, to_arg); dump_binaries(to, to_arg, all_binaries); } @@ -775,6 +778,9 @@ init_literal_areas(void) qsort(lit_areas, num_lit_areas, sizeof(ErtsLiteralArea *), compare_areas); + qsort(erts_persistent_areas, erts_num_persistent_areas, + sizeof(ErtsLiteralArea *), compare_areas); + erts_runlock_old_code(code_ix); } @@ -796,6 +802,13 @@ static void mark_literal(Eterm* ptr) ap = bsearch(ptr, lit_areas, num_lit_areas, sizeof(ErtsLiteralArea*), search_areas); + if (ap == 0) { + ap = bsearch(ptr, erts_persistent_areas, + erts_num_persistent_areas, + sizeof(ErtsLiteralArea*), + search_areas); + } + /* * If the literal was created by native code, this search will not @@ -807,12 +820,12 @@ static void mark_literal(Eterm* ptr) } } - static void dump_literals(fmtfn_t to, void *to_arg) { ErtsCodeIndex code_ix; int i; + Uint idx; code_ix = erts_active_code_ix(); erts_rlock_old_code(code_ix); @@ -825,6 +838,28 @@ dump_literals(fmtfn_t to, void *to_arg) } erts_runlock_old_code(code_ix); + + for (idx = 0; idx < erts_num_persistent_areas; idx++) { + dump_module_literals(to, to_arg, erts_persistent_areas[idx]); + } +} + +static void +dump_persistent_terms(fmtfn_t to, void *to_arg) +{ + Uint idx; + + erts_print(to, to_arg, "=persistent_terms\n"); + + for (idx = 0; idx < erts_num_persistent_areas; idx++) { + ErtsLiteralArea* ap = erts_persistent_areas[idx]; + Eterm tuple = make_tuple(ap->start); + Eterm* tup_val = tuple_val(tuple); + + dump_element(to, to_arg, tup_val[1]); + erts_putc(to, to_arg, '|'); + dump_element_nl(to, to_arg, tup_val[2]); + } } static void @@ -963,7 +998,8 @@ dump_module_literals(fmtfn_t to, void *to_arg, ErtsLiteralArea* lit_area) } erts_putc(to, to_arg, '\n'); } - } else if (is_export_header(w)) { + } else { + /* Dump everything else in the external format */ dump_externally(to, to_arg, term); erts_putc(to, to_arg, '\n'); } diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h index b3bfa69052..880febba8b 100644 --- a/erts/emulator/beam/erl_utils.h +++ b/erts/emulator/beam/erl_utils.h @@ -22,6 +22,7 @@ #define ERL_UTILS_H__ #include "sys.h" +#include "atom.h" #include "erl_printf.h" struct process; @@ -112,10 +113,12 @@ int eq(Eterm, Eterm); #define EQ(x,y) (((x) == (y)) || (is_not_both_immed((x),(y)) && eq((x),(y)))) -int erts_cmp_atoms(Eterm a, Eterm b); -Sint erts_cmp(Eterm, Eterm, int, int); -Sint erts_cmp_compound(Eterm, Eterm, int, int); +ERTS_GLB_INLINE Sint erts_cmp(Eterm, Eterm, int, int); +ERTS_GLB_INLINE int erts_cmp_atoms(Eterm a, Eterm b); + Sint cmp(Eterm a, Eterm b); +Sint erts_cmp_compound(Eterm, Eterm, int, int); + #define CMP(A,B) erts_cmp(A,B,0,0) #define CMP_TERM(A,B) erts_cmp(A,B,1,0) #define CMP_EQ_ONLY(A,B) erts_cmp(A,B,0,1) @@ -150,4 +153,56 @@ Sint cmp(Eterm a, Eterm b); if (erts_cmp_compound(X,Y,0,EqOnly) Op 0) { Action; }; \ } +#define erts_float_comp(x,y) (((x)<(y)) ? -1 : (((x)==(y)) ? 0 : 1)) + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE int erts_cmp_atoms(Eterm a, Eterm b) { + Atom *aa = atom_tab(atom_val(a)); + Atom *bb = atom_tab(atom_val(b)); + + byte *name_a, *name_b; + int len_a, len_b, diff; + + diff = aa->ord0 - bb->ord0; + + if (diff != 0) { + return diff; + } + + name_a = &aa->name[3]; + name_b = &bb->name[3]; + len_a = aa->len-3; + len_b = bb->len-3; + + if (len_a > 0 && len_b > 0) { + diff = sys_memcmp(name_a, name_b, MIN(len_a, len_b)); + + if (diff != 0) { + return diff; + } + } + + return len_a - len_b; +} + +ERTS_GLB_INLINE Sint erts_cmp(Eterm a, Eterm b, int exact, int eq_only) { + if (is_atom(a) && is_atom(b)) { + return erts_cmp_atoms(a, b); + } else if (is_both_small(a, b)) { + return (signed_val(a) - signed_val(b)); + } else if (is_float(a) && is_float(b)) { + FloatDef af, bf; + + GET_DOUBLE(a, af); + GET_DOUBLE(b, bf); + + return erts_float_comp(af.fd, bf.fd); + } + + return erts_cmp_compound(a,b,exact,eq_only); +} + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + #endif diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 621ba108ba..9a66e491f3 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1953,7 +1953,8 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla #define RETURN_STATE() \ do { \ - hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE+3); \ + static const int TUPLE2_SIZE = 2 + 1; \ + hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE + TUPLE2_SIZE); \ c_term = erts_mk_magic_ref(&hp, &MSO(p), context_b); \ res = TUPLE2(hp, Term, c_term); \ BUMP_ALL_REDS(p); \ diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 21ae205237..0631404599 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -906,6 +906,8 @@ typedef struct ErtsLiteralArea_ { Eterm start[1]; /* beginning of area */ } ErtsLiteralArea; +void erts_queue_release_literals(Process *c_p, ErtsLiteralArea* literals); + #define ERTS_LITERAL_AREA_ALLOC_SIZE(N) \ (sizeof(ErtsLiteralArea) + sizeof(Eterm)*((N) - 1)) @@ -1001,6 +1003,7 @@ typedef struct { Uint literal_size; Eterm *lit_purge_ptr; Uint lit_purge_sz; + int copy_literals; } erts_shcopy_t; #define INITIALIZE_SHCOPY(info) \ @@ -1010,6 +1013,7 @@ typedef struct { info.bitstore_start = info.bitstore_default; \ info.shtable_start = info.shtable_default; \ info.literal_size = 0; \ + info.copy_literals = 0; \ if (larea__) { \ info.lit_purge_ptr = &larea__->start[0]; \ info.lit_purge_sz = larea__->end - info.lit_purge_ptr; \ @@ -1238,6 +1242,13 @@ Sint erts_re_set_loop_limit(Sint limit); void erts_init_bif_binary(void); Sint erts_binary_set_loop_limit(Sint limit); +/* erl_bif_persistent.c */ +void erts_init_bif_persistent_term(void); +Uint erts_persistent_term_count(void); +void erts_init_persistent_dumping(void); +extern ErtsLiteralArea** erts_persistent_areas; +extern Uint erts_num_persistent_areas; + /* external.c */ void erts_init_external(void); diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index bb22548587..869a575cb4 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -325,6 +325,7 @@ typedef long Sint erts_align_attribute(sizeof(long)); #define UWORD_CONSTANT(Const) Const##UL #define ERTS_UWORD_MAX ULONG_MAX #define ERTS_SWORD_MAX LONG_MAX +#define ERTS_SWORD_MIN LONG_MIN #define ERTS_SIZEOF_ETERM SIZEOF_LONG #define ErtsStrToSint strtol #elif SIZEOF_VOID_P == SIZEOF_INT @@ -335,6 +336,7 @@ typedef int Sint erts_align_attribute(sizeof(int)); #define UWORD_CONSTANT(Const) Const##U #define ERTS_UWORD_MAX UINT_MAX #define ERTS_SWORD_MAX INT_MAX +#define ERTS_SWORD_MIN INT_MIN #define ERTS_SIZEOF_ETERM SIZEOF_INT #define ErtsStrToSint strtol #elif SIZEOF_VOID_P == SIZEOF_LONG_LONG @@ -345,6 +347,7 @@ typedef long long Sint erts_align_attribute(sizeof(long long)); #define UWORD_CONSTANT(Const) Const##ULL #define ERTS_UWORD_MAX ULLONG_MAX #define ERTS_SWORD_MAX LLONG_MAX +#define ERTS_SWORD_MIN LLONG_MIN #define ERTS_SIZEOF_ETERM SIZEOF_LONG_LONG #if defined(__WIN32__) #define ErtsStrToSint _strtoi64 diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 08f8ca9788..d81bd89a48 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -2615,27 +2615,6 @@ not_equal: } -/* - * Lexically compare two strings of bytes (string s1 length l1 and s2 l2). - * - * s1 < s2 return -1 - * s1 = s2 return 0 - * s1 > s2 return +1 - */ -static int cmpbytes(byte *s1, int l1, byte *s2, int l2) -{ - int i; - i = 0; - while((i < l1) && (i < l2)) { - if (s1[i] < s2[i]) return(-1); - if (s1[i] > s2[i]) return(1); - i++; - } - if (l1 < l2) return(-1); - if (l1 > l2) return(1); - return(0); -} - /* * Compare objects. @@ -2649,20 +2628,6 @@ static int cmpbytes(byte *s1, int l1, byte *s2, int l2) * */ - -#define float_comp(x,y) (((x)<(y)) ? -1 : (((x)==(y)) ? 0 : 1)) - -int erts_cmp_atoms(Eterm a, Eterm b) -{ - Atom *aa = atom_tab(atom_val(a)); - Atom *bb = atom_tab(atom_val(b)); - int diff = aa->ord0 - bb->ord0; - if (diff) - return diff; - return cmpbytes(aa->name+3, aa->len-3, - bb->name+3, bb->len-3); -} - /* cmp(Eterm a, Eterm b) * For compatibility with HiPE - arith-based compare. */ @@ -2673,22 +2638,6 @@ Sint cmp(Eterm a, Eterm b) Sint erts_cmp_compound(Eterm a, Eterm b, int exact, int eq_only); -Sint erts_cmp(Eterm a, Eterm b, int exact, int eq_only) -{ - if (is_atom(a) && is_atom(b)) { - return erts_cmp_atoms(a, b); - } else if (is_both_small(a, b)) { - return (signed_val(a) - signed_val(b)); - } else if (is_float(a) && is_float(b)) { - FloatDef af, bf; - GET_DOUBLE(a, af); - GET_DOUBLE(b, bf); - return float_comp(af.fd, bf.fd); - } - return erts_cmp_compound(a,b,exact,eq_only); -} - - /* erts_cmp(Eterm a, Eterm b, int exact) * exact = 1 -> term-based compare * exact = 0 -> arith-based compare @@ -2985,7 +2934,7 @@ tailrecur_ne: GET_DOUBLE(a, af); GET_DOUBLE(b, bf); - ON_CMP_GOTO(float_comp(af.fd, bf.fd)); + ON_CMP_GOTO(erts_float_comp(af.fd, bf.fd)); } case (_TAG_HEADER_POS_BIG >> _TAG_PRIMARY_SIZE): case (_TAG_HEADER_NEG_BIG >> _TAG_PRIMARY_SIZE): @@ -3022,10 +2971,7 @@ tailrecur_ne: ErlFunThing* f2 = (ErlFunThing *) fun_val(b); Sint diff; - diff = cmpbytes(atom_tab(atom_val(f1->fe->module))->name, - atom_tab(atom_val(f1->fe->module))->len, - atom_tab(atom_val(f2->fe->module))->name, - atom_tab(atom_val(f2->fe->module))->len); + diff = erts_cmp_atoms((f1->fe)->module, (f2->fe)->module); if (diff != 0) { RETURN_NEQ(diff); } @@ -3219,7 +3165,7 @@ tailrecur_ne: if (f2.fd < MAX_LOSSLESS_FLOAT && f2.fd > MIN_LOSSLESS_FLOAT) { /* Float is within the no loss limit */ f1.fd = signed_val(aw); - j = float_comp(f1.fd, f2.fd); + j = erts_float_comp(f1.fd, f2.fd); } #if ERTS_SIZEOF_ETERM == 8 else if (f2.fd > (double) (MAX_SMALL + 1)) { @@ -3266,7 +3212,7 @@ tailrecur_ne: if (big_to_double(aw, &f1.fd) < 0) { j = big_sign(aw) ? -1 : 1; } else { - j = float_comp(f1.fd, f2.fd); + j = erts_float_comp(f1.fd, f2.fd); } } else { big = double_to_big(f2.fd, big_buf, sizeof(big_buf)/sizeof(Eterm)); @@ -3282,7 +3228,7 @@ tailrecur_ne: if (f1.fd < MAX_LOSSLESS_FLOAT && f1.fd > MIN_LOSSLESS_FLOAT) { /* Float is within the no loss limit */ f2.fd = signed_val(bw); - j = float_comp(f1.fd, f2.fd); + j = erts_float_comp(f1.fd, f2.fd); } #if ERTS_SIZEOF_ETERM == 8 else if (f1.fd > (double) (MAX_SMALL + 1)) { diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c index 259a27cf57..3195ca3874 100644 --- a/erts/emulator/drivers/common/inet_drv.c +++ b/erts/emulator/drivers/common/inet_drv.c @@ -812,6 +812,7 @@ static size_t my_strnlen(const char *s, size_t maxlen) #define INET_OPT_PKTOPTIONS 45 /* IP(V6)_PKTOPTIONS get ancillary data */ #define INET_OPT_TTL 46 /* IP_TTL */ #define INET_OPT_RECVTTL 47 /* IP_RECVTTL ancillary data */ +#define TCP_OPT_NOPUSH 48 /* super-Nagle, aka TCP_CORK */ /* SCTP options: a separate range, from 100: */ #define SCTP_OPT_RTOINFO 100 #define SCTP_OPT_ASSOCINFO 101 @@ -955,6 +956,12 @@ static size_t my_strnlen(const char *s, size_t maxlen) #endif +#if defined(TCP_CORK) +#define INET_TCP_NOPUSH TCP_CORK +#elif defined(TCP_NOPUSH) && !defined(__DARWIN__) +#define INET_TCP_NOPUSH TCP_NOPUSH +#endif + #define BIN_REALLOC_MARGIN(x) ((x)/4) /* 25% */ /* The general purpose sockaddr */ @@ -5178,6 +5185,71 @@ static int hwaddr_libdlpi_lookup(const char *ifnm, } #endif +#ifdef HAVE_GETIFADDRS +/* Returns 0 for success and errno() for failure */ +static int call_getifaddrs(inet_descriptor* desc_p, struct ifaddrs **ifa_pp) +{ + int result, save_errno; +#ifdef HAVE_SETNS + int current_ns; + + current_ns = 0; + if (desc_p->netns != NULL) { + int new_ns; + /* Temporarily change network namespace for this thread + * over the getifaddrs() call + */ + current_ns = open("/proc/self/ns/net", O_RDONLY); + if (current_ns == INVALID_SOCKET) + return sock_errno(); + new_ns = open(desc_p->netns, O_RDONLY); + if (new_ns == INVALID_SOCKET) { + save_errno = sock_errno(); + while (close(current_ns) == INVALID_SOCKET && + sock_errno() == EINTR); + return save_errno; + } + if (setns(new_ns, CLONE_NEWNET) != 0) { + save_errno = sock_errno(); + while (close(new_ns) == INVALID_SOCKET && + sock_errno() == EINTR); + while (close(current_ns) == INVALID_SOCKET && + sock_errno() == EINTR); + return save_errno; + } + else { + while (close(new_ns) == INVALID_SOCKET && + sock_errno() == EINTR); + } + } +#endif + save_errno = 0; + result = getifaddrs(ifa_pp); + if (result < 0) + save_errno = sock_errno(); +#ifdef HAVE_SETNS + if (desc_p->netns != NULL) { + /* Restore network namespace */ + if (setns(current_ns, CLONE_NEWNET) != 0) { + /* XXX Failed to restore network namespace. + * What to do? Tidy up and return an error... + * Note that the thread now might still be in the set namespace. + * Can this even happen? Should the emulator be aborted? + */ + if (result >= 0) { + /* We got a result but have to waste it */ + save_errno = sock_errno(); + freeifaddrs(*ifa_pp); + } + } + while (close(current_ns) == INVALID_SOCKET && + sock_errno() == EINTR); + } +#endif + return save_errno; +} +#endif /* #ifdef HAVE_GETIFADDRS */ + /* FIXME: temporary hack */ #ifndef IFHWADDRLEN #define IFHWADDRLEN 6 @@ -5255,8 +5327,8 @@ static ErlDrvSSizeT inet_ctl_ifget(inet_descriptor* desc, struct sockaddr_dl *sdlp; int found = 0; - if (getifaddrs(&ifa) == -1) - goto error; + if (call_getifaddrs(desc, &ifa) != 0) + goto error; for (ifp = ifa; ifp; ifp = ifp->ifa_next) { if ((ifp->ifa_addr->sa_family == AF_LINK) && @@ -5974,6 +6046,7 @@ static ErlDrvSSizeT inet_ctl_getifaddrs(inet_descriptor* desc_p, ErlDrvSizeT buf_size; char *buf_p; char *buf_alloc_p; + int save_errno; buf_size = GETIFADDRS_BUFSZ; buf_alloc_p = ALLOC(GETIFADDRS_BUFSZ); @@ -6008,9 +6081,9 @@ static ErlDrvSSizeT inet_ctl_getifaddrs(inet_descriptor* desc_p, } \ } while (0) - if (getifaddrs(&ifa_p) < 0) { - return ctl_error(sock_errno(), rbuf_pp, rsize); - } + if ((save_errno = call_getifaddrs(desc_p, &ifa_p)) != 0) + return ctl_error(save_errno, rbuf_pp, rsize); + ifa_free_p = ifa_p; *buf_p++ = INET_REP_OK; for (; ifa_p; ifa_p = ifa_p->ifa_next) { @@ -6532,6 +6605,19 @@ static int inet_set_opts(inet_descriptor* desc, char* ptr, int len) (long)desc->port, desc->s, ival)); break; + case TCP_OPT_NOPUSH: +#if defined(INET_TCP_NOPUSH) + proto = IPPROTO_TCP; + type = INET_TCP_NOPUSH; + DEBUGF(("inet_set_opts(%ld): s=%d, t=%d TCP_NOPUSH=%d\r\n", + (long)desc->port, desc->s, type, ival)); + break; +#else + /* inet_fill_opts always returns a value for this option, + * so we need to ignore it if not implemented, just in case */ + continue; +#endif + #if defined(HAVE_MULTICAST_SUPPORT) && defined(IPPROTO_IP) case UDP_OPT_MULTICAST_TTL: @@ -7693,6 +7779,16 @@ static ErlDrvSSizeT inet_fill_opts(inet_descriptor* desc, proto = IPPROTO_TCP; type = TCP_NODELAY; break; + case TCP_OPT_NOPUSH: +#if defined(INET_TCP_NOPUSH) + proto = IPPROTO_TCP; + type = INET_TCP_NOPUSH; + break; +#else + *ptr++ = opt; + put_int32(0, ptr); + continue; +#endif #if defined(HAVE_MULTICAST_SUPPORT) && defined(IPPROTO_IP) case UDP_OPT_MULTICAST_TTL: diff --git a/erts/emulator/sys/unix/sys_uds.c b/erts/emulator/sys/unix/sys_uds.c index 39a4866065..c9f73622ba 100644 --- a/erts/emulator/sys/unix/sys_uds.c +++ b/erts/emulator/sys/unix/sys_uds.c @@ -88,8 +88,9 @@ sys_uds_readv(int fd, struct iovec *iov, size_t iov_len, if((msg.msg_flags & MSG_CTRUNC) == MSG_CTRUNC) { /* We assume that we have given enough space for any header - that are sent to us. So the only remaining reason to get - this flag set is if the caller has run out of file descriptors. + that are sent to us. So the only remaining reasons to get + this flag set is if the caller has run out of file descriptors + or an SELinux policy prunes the response (eg. O_APPEND on STDERR). */ errno = EMFILE; return -1; diff --git a/erts/emulator/test/Makefile b/erts/emulator/test/Makefile index bf00de2204..6a064ec8d4 100644 --- a/erts/emulator/test/Makefile +++ b/erts/emulator/test/Makefile @@ -33,6 +33,7 @@ MODULES= \ after_SUITE \ alloc_SUITE \ async_ports_SUITE \ + atomics_SUITE \ beam_SUITE \ beam_literals_SUITE \ bif_SUITE \ @@ -50,6 +51,7 @@ MODULES= \ call_trace_SUITE \ code_SUITE \ code_parallel_load_SUITE \ + counters_SUITE \ crypto_SUITE \ ddll_SUITE \ decode_packet_SUITE \ @@ -92,6 +94,7 @@ MODULES= \ port_SUITE \ port_bif_SUITE \ prim_eval_SUITE \ + persistent_term_SUITE \ process_SUITE \ pseudoknot_SUITE \ receive_SUITE \ diff --git a/erts/emulator/test/atomics_SUITE.erl b/erts/emulator/test/atomics_SUITE.erl new file mode 100644 index 0000000000..8c42354770 --- /dev/null +++ b/erts/emulator/test/atomics_SUITE.erl @@ -0,0 +1,147 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(atomics_SUITE). + +-include_lib("common_test/include/ct.hrl"). + +-compile(export_all). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [signed, unsigned, bad, signed_limits, unsigned_limits]. + +signed(Config) when is_list(Config) -> + Size = 10, + Ref = atomics:new(Size,[]), + #{size:=Size, memory:=Memory} = atomics:info(Ref), + {_,true} = {Memory, Memory > Size*8}, + {_,true} = {Memory, Memory < Size*max_atomic_sz() + 100}, + [signed_do(Ref, Ix) || Ix <- lists:seq(1, Size)], + ok. + +signed_do(Ref, Ix) -> + 0 = atomics:get(Ref, Ix), + ok = atomics:put(Ref, Ix, 3), + ok = atomics:add(Ref, Ix, 14), + 17 = atomics:get(Ref, Ix), + 20 = atomics:add_get(Ref, Ix, 3), + -3 = atomics:add_get(Ref, Ix, -23), + 17 = atomics:add_get(Ref, Ix, 20), + ok = atomics:sub(Ref, Ix, 4), + 13 = atomics:get(Ref, Ix), + -7 = atomics:sub_get(Ref, Ix, 20), + 3 = atomics:sub_get(Ref, Ix, -10), + 3 = atomics:exchange(Ref, Ix, 666), + ok = atomics:compare_exchange(Ref, Ix, 666, 777), + 777 = atomics:compare_exchange(Ref, Ix, 666, -666), + ok. + +unsigned(Config) when is_list(Config) -> + Size = 10, + Ref = atomics:new(Size,[{signed, false}]), + #{size:=Size, memory:=Memory} = atomics:info(Ref), + true = Memory > Size*8, + true = Memory < Size*max_atomic_sz() + 100, + [unsigned_do(Ref, Ix) || Ix <- lists:seq(1, Size)], + ok. + +unsigned_do(Ref, Ix) -> + 0 = atomics:get(Ref, Ix), + ok = atomics:put(Ref, Ix, 3), + ok = atomics:add(Ref, Ix, 14), + 17 = atomics:get(Ref, Ix), + 20 = atomics:add_get(Ref, Ix, 3), + ok = atomics:sub(Ref, Ix, 7), + 13 = atomics:get(Ref, Ix), + 3 = atomics:sub_get(Ref, Ix, 10), + 3 = atomics:exchange(Ref, Ix, 666), + ok = atomics:compare_exchange(Ref, Ix, 666, 777), + 777 = atomics:compare_exchange(Ref, Ix, 666, 888), + ok. + +bad(Config) when is_list(Config) -> + {'EXIT',{badarg,_}} = (catch atomics:new(0,[])), + {'EXIT',{badarg,_}} = (catch atomics:new(10,[bad])), + {'EXIT',{badarg,_}} = (catch atomics:new(10,[{signed,bad}])), + {'EXIT',{badarg,_}} = (catch atomics:new(10,[{signed,true}, bad])), + {'EXIT',{badarg,_}} = (catch atomics:new(10,[{signed,false} | bad])), + Ref = atomics:new(10,[]), + {'EXIT',{badarg,_}} = (catch atomics:get(1742, 7)), + {'EXIT',{badarg,_}} = (catch atomics:get(make_ref(), 7)), + {'EXIT',{badarg,_}} = (catch atomics:get(Ref, -1)), + {'EXIT',{badarg,_}} = (catch atomics:get(Ref, 0)), + {'EXIT',{badarg,_}} = (catch atomics:get(Ref, 11)), + {'EXIT',{badarg,_}} = (catch atomics:get(Ref, 7.0)), + ok. + + +signed_limits(Config) when is_list(Config) -> + Bits = 64, + Max = (1 bsl (Bits-1)) - 1, + Min = -(1 bsl (Bits-1)), + + Ref = atomics:new(1,[{signed, true}]), + #{max:=Max, min:=Min} = atomics:info(Ref), + 0 = atomics:get(Ref, 1), + ok = atomics:add(Ref, 1, Max), + Min = atomics:add_get(Ref, 1, 1), + Max = atomics:sub_get(Ref, 1, 1), + + IncrMax = (Max bsl 1) bor 1, + ok = atomics:put(Ref, 1, 0), + ok = atomics:add(Ref, 1, IncrMax), + -1 = atomics:get(Ref, 1), + {'EXIT',{badarg,_}} = (catch atomics:add(Ref, 1, IncrMax+1)), + {'EXIT',{badarg,_}} = (catch atomics:add(Ref, 1, Min-1)), + + ok. + +unsigned_limits(Config) when is_list(Config) -> + Bits = 64, + Max = (1 bsl Bits) - 1, + Min = 0, + + Ref = atomics:new(1,[{signed,false}]), + #{max:=Max, min:=Min} = atomics:info(Ref), + 0 = atomics:get(Ref, 1), + ok = atomics:add(Ref, 1, Max), + Min = atomics:add_get(Ref, 1, 1), + Max = atomics:sub_get(Ref, 1, 1), + + {'EXIT',{badarg,_}} = (catch atomics:add(Ref, 1, Max+1)), + IncrMin = -(1 bsl (Bits-1)), + ok = atomics:put(Ref, 1, -IncrMin), + ok = atomics:add(Ref, 1, IncrMin), + 0 = atomics:get(Ref, 1), + {'EXIT',{badarg,_}} = (catch atomics:add(Ref, 1, IncrMin-1)), + + ok. + +max_atomic_sz() -> + case erlang:system_info({wordsize, external}) of + 4 -> 16; + 8 -> + EI = erlang:system_info(ethread_info), + case lists:keyfind("64-bit native atomics", 1, EI) of + {_, "no", _} -> 16; + _ -> 8 + end + end. diff --git a/erts/emulator/test/code_SUITE.erl b/erts/emulator/test/code_SUITE.erl index 9c6dc3ff83..0444ba4f89 100644 --- a/erts/emulator/test/code_SUITE.erl +++ b/erts/emulator/test/code_SUITE.erl @@ -28,7 +28,7 @@ fake_literals/1, false_dependency/1,coverage/1,fun_confusion/1, t_copy_literals/1, t_copy_literals_frags/1, - erl_544/1]). + erl_544/1, max_heap_size/1]). -define(line_trace, 1). -include_lib("common_test/include/ct.hrl"). @@ -43,7 +43,7 @@ all() -> constant_pools, constant_refc_binaries, fake_literals, false_dependency, coverage, fun_confusion, t_copy_literals, t_copy_literals_frags, - erl_544]. + erl_544, max_heap_size]. init_per_suite(Config) -> erts_debug:set_internal_state(available_internal_state, true), @@ -968,6 +968,39 @@ erl_544(Config) when is_list(Config) -> {skipped, "Only run when native file name encoding is utf8"} end. +%% Test that the copying of literals to a process during purging of +%% literals will cause the process to be killed if the max heap size +%% is exceeded. +max_heap_size(_Config) -> + Mod = ?FUNCTION_NAME, + Value = [I || I <- lists:seq(1, 5000)], + Code = gen_lit(Mod, [{term,Value}]), + {module,Mod} = erlang:load_module(Mod, Code), + SpawnOpts = [monitor, + {max_heap_size, + #{size=>1024, + kill=>true, + error_logger=>true}}], + {Pid,Ref} = spawn_opt(fun() -> + max_heap_size_proc(Mod) + end, SpawnOpts), + receive + {'DOWN',Ref,process,Pid,Reason} -> + killed = Reason; + Other -> + ct:fail({unexpected_message,Other}) + after 10000 -> + ct:fail({process_did_not_die, Pid, erlang:process_info(Pid)}) + end. + +max_heap_size_proc(Mod) -> + Value = Mod:term(), + code:delete(Mod), + code:purge(Mod), + receive + _ -> Value + end. + %% Utilities. make_sub_binary(Bin) when is_binary(Bin) -> diff --git a/erts/emulator/test/counters_SUITE.erl b/erts/emulator/test/counters_SUITE.erl new file mode 100644 index 0000000000..7de164096b --- /dev/null +++ b/erts/emulator/test/counters_SUITE.erl @@ -0,0 +1,112 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(counters_SUITE). + +-include_lib("common_test/include/ct.hrl"). + +-compile(export_all). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [basic, bad, limits]. + +basic(Config) when is_list(Config) -> + Size = 10, + [begin + Ref = counters:new(Size,[Type]), + #{size:=Size, memory:=Memory} = counters:info(Ref), + check_memory(Type, Memory, Size), + [basic_do(Ref, Ix) || Ix <- lists:seq(1, Size)] + end + || Type <- [atomics, write_concurrency]], + ok. + +basic_do(Ref, Ix) -> + 0 = counters:get(Ref, Ix), + ok = counters:add(Ref, Ix, 3), + 3 = counters:get(Ref, Ix), + ok = counters:add(Ref, Ix, 14), + 17 = counters:get(Ref, Ix), + ok = counters:add(Ref, Ix, -20), + -3 = counters:get(Ref, Ix), + ok = counters:add(Ref, Ix, 100), + 97 = counters:get(Ref, Ix), + ok = counters:sub(Ref, Ix, 20), + 77 = counters:get(Ref, Ix), + ok = counters:sub(Ref, Ix, -10), + 87 = counters:get(Ref, Ix), + ok. + +check_memory(atomics, Memory, Size) -> + {_,true} = {Memory, Memory > Size*8}, + {_,true} = {Memory, Memory < Size*max_atomic_sz() + 100}; +check_memory(write_concurrency, Memory, Size) -> + NScheds = erlang:system_info(schedulers), + {_,true} = {Memory, Memory > NScheds*Size*8}, + {_,true} = {Memory, Memory < NScheds*(Size+7)*max_atomic_sz() + 100}. + +max_atomic_sz() -> + case erlang:system_info({wordsize, external}) of + 4 -> 16; + 8 -> + EI = erlang:system_info(ethread_info), + case lists:keyfind("64-bit native atomics", 1, EI) of + {_, "no", _} -> 16; + _ -> 8 + end + end. + +bad(Config) when is_list(Config) -> + {'EXIT',{badarg,_}} = (catch counters:new(0,[])), + {'EXIT',{badarg,_}} = (catch counters:new(10,[bad])), + {'EXIT',{badarg,_}} = (catch counters:new(10,[atomic, bad])), + {'EXIT',{badarg,_}} = (catch counters:new(10,[write_concurrency | bad])), + Ref = counters:new(10,[]), + {'EXIT',{badarg,_}} = (catch counters:get(1742, 7)), + {'EXIT',{badarg,_}} = (catch counters:get(make_ref(), 7)), + {'EXIT',{badarg,_}} = (catch counters:get(Ref, -1)), + {'EXIT',{badarg,_}} = (catch counters:get(Ref, 0)), + {'EXIT',{badarg,_}} = (catch counters:get(Ref, 11)), + {'EXIT',{badarg,_}} = (catch counters:get(Ref, 7.0)), + ok. + + +limits(Config) when is_list(Config) -> + Bits = 64, + Max = (1 bsl (Bits-1)) - 1, + Min = -(1 bsl (Bits-1)), + + Ref = counters:new(1,[]), + 0 = counters:get(Ref, 1), + ok = counters:add(Ref, 1, Max), + ok = counters:add(Ref, 1, 1), + Min = counters:get(Ref, 1), + ok = counters:sub(Ref, 1, 1), + Max = counters:get(Ref, 1), + + IncrMax = (Max bsl 1) bor 1, + ok = counters:sub(Ref, 1, counters:get(Ref, 1)), + ok = counters:add(Ref, 1, IncrMax), + -1 = counters:get(Ref, 1), + {'EXIT',{badarg,_}} = (catch counters:add(Ref, 1, IncrMax+1)), + {'EXIT',{badarg,_}} = (catch counters:add(Ref, 1, Min-1)), + + ok. diff --git a/erts/emulator/test/persistent_term_SUITE.erl b/erts/emulator/test/persistent_term_SUITE.erl new file mode 100644 index 0000000000..58cd3276b0 --- /dev/null +++ b/erts/emulator/test/persistent_term_SUITE.erl @@ -0,0 +1,614 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%5 +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(persistent_term_SUITE). +-include_lib("common_test/include/ct.hrl"). + +-export([all/0,suite/0, + basic/1,purging/1,sharing/1,get_trapping/1, + info/1,info_trapping/1,killed_while_trapping/1, + off_heap_values/1,keys/1,collisions/1, + init_restart/1]). + +%% +-export([test_init_restart_cmd/1]). + +suite() -> + [{ct_hooks,[ts_install_cth]}, + {timetrap,{minutes,10}}]. + +all() -> + [basic,purging,sharing,get_trapping,info,info_trapping, + killed_while_trapping,off_heap_values,keys,collisions, + init_restart]. + +basic(_Config) -> + Chk = chk(), + N = 777, + Seq = lists:seq(1, N), + par(2, N, Seq), + seq(3, Seq), + seq(3, Seq), %Same values. + _ = [begin + Key = {?MODULE,{key,I}}, + true = persistent_term:erase(Key), + false = persistent_term:erase(Key), + {'EXIT',{badarg,_}} = (catch persistent_term:get(Key)) + end || I <- Seq], + [] = [P || {{?MODULE,_},_}=P <- persistent_term:get()], + chk(Chk). + +par(C, N, Seq) -> + _ = [spawn_link(fun() -> + ok = persistent_term:put({?MODULE,{key,I}}, + {value,C*I}) + end) || I <- Seq], + Result = wait(N), + _ = [begin + Double = C*I, + {{?MODULE,{key,I}},{value,Double}} = Res + end || {I,Res} <- lists:zip(Seq, Result)], + ok. + +seq(C, Seq) -> + _ = [ok = persistent_term:put({?MODULE,{key,I}}, {value,C*I}) || + I <- Seq], + All = persistent_term:get(), + All = [P || {{?MODULE,_},_}=P <- persistent_term:get()], + All = [{Key,persistent_term:get(Key)} || {Key,_} <- All], + Result = lists:sort(All), + _ = [begin + Double = C*I, + {{?MODULE,{key,I}},{value,Double}} = Res + end || {I,Res} <- lists:zip(Seq, Result)], + ok. + +wait(N) -> + All = [P || {{?MODULE,_},_}=P <- persistent_term:get()], + case length(All) of + N -> + All = [{Key,persistent_term:get(Key)} || {Key,_} <- All], + lists:sort(All); + _ -> + receive after 10 -> ok end, + wait(N) + end. + +%% Make sure that terms that have been erased are copied into all +%% processes that still hold a pointer to them. + +purging(_Config) -> + Chk = chk(), + do_purging(fun(K) -> persistent_term:put(K, {?MODULE,new}) end, + replaced), + do_purging(fun persistent_term:erase/1, erased), + chk(Chk). + +do_purging(Eraser, Type) -> + Parent = self(), + Key = {?MODULE,?FUNCTION_NAME}, + ok = persistent_term:put(Key, {term,[<<"abc",0:777/unit:8>>]}), + Ps0 = [spawn_monitor(fun() -> purging_tester(Parent, Key) end) || + _ <- lists:seq(1, 50)], + Ps = maps:from_list(Ps0), + purging_recv(gotten, Ps), + Eraser(Key), + _ = [P ! {Parent,Type} || P <- maps:keys(Ps)], + purging_wait(Ps). + +purging_recv(Tag, Ps) when map_size(Ps) > 0 -> + receive + {Pid,Tag} -> + true = is_map_key(Pid, Ps), + purging_recv(Tag, maps:remove(Pid, Ps)) + end; +purging_recv(_, _) -> ok. + +purging_wait(Ps) when map_size(Ps) > 0 -> + receive + {'DOWN',Ref,process,Pid,Reason} -> + normal = Reason, + Ref = map_get(Pid, Ps), + purging_wait(maps:remove(Pid, Ps)) + end; +purging_wait(_) -> ok. + +purging_tester(Parent, Key) -> + Term = persistent_term:get(Key), + purging_check_term(Term), + 0 = erts_debug:size_shared(Term), + Parent ! {self(),gotten}, + receive + {Parent,erased} -> + {'EXIT',{badarg,_}} = (catch persistent_term:get(Key)), + purging_tester_1(Term); + {Parent,replaced} -> + {?MODULE,new} = persistent_term:get(Key), + purging_tester_1(Term) + end. + +%% Wait for the term to be copied into this process. +purging_tester_1(Term) -> + purging_check_term(Term), + receive after 1 -> ok end, + case erts_debug:size_shared(Term) of + 0 -> + purging_tester_1(Term); + Size -> + %% The term has been copied into this process. + purging_check_term(Term), + Size = erts_debug:size(Term) + end. + +purging_check_term({term,[<<"abc",0:777/unit:8>>]}) -> + ok. + +%% Test that sharing is preserved when storing terms. + +sharing(_Config) -> + Chk = chk(), + Depth = 10, + Size = 2*Depth, + Shared = lists:foldl(fun(_, A) -> [A|A] end, + [], lists:seq(1, Depth)), + Size = erts_debug:size(Shared), + Key = {?MODULE,?FUNCTION_NAME}, + ok = persistent_term:put(Key, Shared), + SharedStored = persistent_term:get(Key), + Size = erts_debug:size(SharedStored), + 0 = erts_debug:size_shared(SharedStored), + + {Pid,Ref} = spawn_monitor(fun() -> + Term = persistent_term:get(Key), + Size = erts_debug:size(Term), + 0 = erts_debug:size_shared(Term), + true = Term =:= SharedStored + end), + receive + {'DOWN',Ref,process,Pid,normal} -> + true = persistent_term:erase(Key), + Size = erts_debug:size(SharedStored), + chk(Chk) + end. + +%% Test trapping of persistent_term:get/0. + +get_trapping(_Config) -> + Chk = chk(), + + %% Assume that the get/0 traps after 4000 iterations + %% in a non-debug emulator. + N = case test_server:timetrap_scale_factor() of + 1 -> 10000; + _ -> 1000 + end, + spawn_link(fun() -> get_trapping_create(N) end), + All = do_get_trapping(N, []), + N = get_trapping_check_result(lists:sort(All), 1), + erlang:garbage_collect(), + get_trapping_erase(N), + chk(Chk). + +do_get_trapping(N, Prev) -> + case persistent_term:get() of + Prev when length(Prev) >= N -> + All = [P || {{?MODULE,{get_trapping,_}},_}=P <- Prev], + case length(All) of + N -> All; + _ -> do_get_trapping(N, Prev) + end; + New -> + receive after 1 -> ok end, + do_get_trapping(N, New) + end. + +get_trapping_create(0) -> + ok; +get_trapping_create(N) -> + ok = persistent_term:put({?MODULE,{get_trapping,N}}, N), + get_trapping_create(N-1). + +get_trapping_check_result([{{?MODULE,{get_trapping,N}},N}|T], N) -> + get_trapping_check_result(T, N+1); +get_trapping_check_result([], N) -> N-1. + +get_trapping_erase(0) -> + ok; +get_trapping_erase(N) -> + true = persistent_term:erase({?MODULE,{get_trapping,N}}), + get_trapping_erase(N-1). + +%% Test retrieving information about persistent terms. + +info(_Config) -> + Chk = chk(), + + %% White box test of info/0. + N = 100, + try + Overhead = info_literal_area_overhead(), + io:format("Overhead = ~p\n", [Overhead]), + info_wb(N, Overhead, info_info()) + after + _ = [_ = persistent_term:erase({?MODULE,I}) || + I <- lists:seq(1, N)] + end, + + chk(Chk). + +%% White box test of persistent_term:info/0. We take into account +%% that there might already exist persistent terms (created by the +%% OTP standard libraries), but we assume that they are not +%% changed during the execution of this test case. + +info_wb(0, _, _) -> + ok; +info_wb(N, Overhead, {BaseCount,BaseMemory}) -> + Key = {?MODULE,N}, + Value = lists:seq(1, N), + ok = persistent_term:put(Key, Value), + + %% Calculate the extra memory needed for this term. + WordSize = erlang:system_info(wordsize), + ExtraMemory = Overhead + 2 * N * WordSize, + + %% Call persistent_term:info/0. + {Count,Memory} = info_info(), + + %% There should be one more persistent term. + Count = BaseCount + 1, + + %% Verify that the amount of memory is correct. + case BaseMemory + ExtraMemory of + Memory -> + %% Exactly right. The size of the hash table was not changed. + ok; + Expected -> + %% The size of the hash table has been doubled to avoid filling + %% the table to more than 50 percent. The previous number + %% of entries must have been exactly half the size of the + %% hash table. The expected number of extra words added by + %% the resizing will be twice that number. + ExtraWords = BaseCount * 2, + true = ExtraWords * WordSize =:= (Memory - Expected) + end, + info_wb(N-1, Overhead, {Count,Memory}). + +info_info() -> + #{count:=Count,memory:=Memory} = persistent_term:info(), + true = is_integer(Count) andalso Count >= 0, + true = is_integer(Memory) andalso Memory >= 0, + {Count,Memory}. + +%% Calculate the number of extra bytes needed for storing each term in +%% the literal, assuming that the key is a tuple of size 2 with +%% immediate elements. The calculated number is the size of the +%% ErtsLiteralArea struct excluding the storage for the literal term +%% itself. + +info_literal_area_overhead() -> + Key1 = {?MODULE,1}, + Key2 = {?MODULE,2}, + #{memory:=Mem0} = persistent_term:info(), + ok = persistent_term:put(Key1, literal), + #{memory:=Mem1} = persistent_term:info(), + ok = persistent_term:put(Key2, literal), + #{memory:=Mem2} = persistent_term:info(), + true = persistent_term:erase(Key1), + true = persistent_term:erase(Key2), + + %% The size of the hash table may have doubled when inserting + %% one of the keys. To avoiding counting the change in the hash + %% table size, take the smaller size increase. + min(Mem2-Mem1, Mem1-Mem0). + +%% Test trapping of persistent_term:info/0. + +info_trapping(_Config) -> + Chk = chk(), + + %% Assume that the info/0 traps after 4000 iterations + %% in a non-debug emulator. + N = case test_server:timetrap_scale_factor() of + 1 -> 10000; + _ -> 1000 + end, + spawn_link(fun() -> info_trapping_create(N) end), + All = do_info_trapping(N, 0), + N = info_trapping_check_result(lists:sort(All), 1), + erlang:garbage_collect(), + info_trapping_erase(N), + chk(Chk). + +do_info_trapping(N, PrevMem) -> + case info_info() of + {N,Mem} -> + true = Mem >= PrevMem, + All = [P || {{?MODULE,{info_trapping,_}},_}=P <- persistent_term:get()], + case length(All) of + N -> All; + _ -> do_info_trapping(N, PrevMem) + end; + {_,Mem} -> + true = Mem >= PrevMem, + receive after 1 -> ok end, + do_info_trapping(N, Mem) + end. + +info_trapping_create(0) -> + ok; +info_trapping_create(N) -> + ok = persistent_term:put({?MODULE,{info_trapping,N}}, N), + info_trapping_create(N-1). + +info_trapping_check_result([{{?MODULE,{info_trapping,N}},N}|T], N) -> + info_trapping_check_result(T, N+1); +info_trapping_check_result([], N) -> N-1. + +info_trapping_erase(0) -> + ok; +info_trapping_erase(N) -> + true = persistent_term:erase({?MODULE,{info_trapping,N}}), + info_trapping_erase(N-1). + +%% Test that hash tables are deallocated if a process running +%% persistent_term:get/0 is killed. + +killed_while_trapping(_Config) -> + Chk = chk(), + N = case test_server:timetrap_scale_factor() of + 1 -> 20000; + _ -> 2000 + end, + kwt_put(N), + kwt_spawn(10), + kwt_erase(N), + chk(Chk). + +kwt_put(0) -> + ok; +kwt_put(N) -> + ok = persistent_term:put({?MODULE,{kwt,N}}, N), + kwt_put(N-1). + +kwt_spawn(0) -> + ok; +kwt_spawn(N) -> + Pids = [spawn(fun kwt_getter/0) || _ <- lists:seq(1, 20)], + erlang:yield(), + _ = [exit(Pid, kill) || Pid <- Pids], + kwt_spawn(N-1). + +kwt_getter() -> + _ = persistent_term:get(), + kwt_getter(). + +kwt_erase(0) -> + ok; +kwt_erase(N) -> + true = persistent_term:erase({?MODULE,{kwt,N}}), + kwt_erase(N-1). + +%% Test storing off heap values (such as ref-counted binaries). + +off_heap_values(_Config) -> + Chk = chk(), + Key = {?MODULE,?FUNCTION_NAME}, + Val = {a,list_to_binary(lists:seq(0, 255)),make_ref(),fun() -> ok end}, + ok = persistent_term:put(Key, Val), + FetchedVal = persistent_term:get(Key), + Val = FetchedVal, + true = persistent_term:erase(Key), + off_heap_values_wait(FetchedVal, Val), + chk(Chk). + +off_heap_values_wait(FetchedVal, Val) -> + case erts_debug:size_shared(FetchedVal) of + 0 -> + Val = FetchedVal, + ok; + _ -> + erlang:yield(), + off_heap_values_wait(FetchedVal, Val) + end. + +%% Test some more data types as keys. Use the module name as a key +%% to minimize the risk of collision with any key used +%% by the OTP libraries. + +keys(_Config) -> + Chk = chk(), + do_key(?MODULE), + do_key([?MODULE]), + do_key(?MODULE_STRING), + do_key(list_to_binary(?MODULE_STRING)), + chk(Chk). + +do_key(Key) -> + Val = term_to_binary(Key), + ok = persistent_term:put(Key, Val), + StoredVal = persistent_term:get(Key), + Val = StoredVal, + true = persistent_term:erase(Key). + +%% Create persistent terms with keys that are known to collide. +%% Delete them in random order, making sure that all others +%% terms can still be found. + +collisions(_Config) -> + Chk = chk(), + + %% Create persistent terms with random keys. + Keys = lists:flatten(colliding_keys()), + Kvs = [{K,rand:uniform(1000)} || K <- Keys], + _ = [ok = persistent_term:put(K, V) || {K,V} <- Kvs], + _ = [V = persistent_term:get(K) || {K,V} <- Kvs], + + %% Now delete the persistent terms in random order. + collisions_delete(lists:keysort(2, Kvs)), + + chk(Chk). + +collisions_delete([{Key,Val}|Kvs]) -> + Val = persistent_term:get(Key), + true = persistent_term:erase(Key), + true = lists:sort(persistent_term:get()) =:= lists:sort(Kvs), + _ = [V = persistent_term:get(K) || {K,V} <- Kvs], + collisions_delete(Kvs); +collisions_delete([]) -> + ok. + +colliding_keys() -> + %% Collisions found by Jesper L. Andersen for breaking maps. + L = [[764492191,2361333849], + [49527266765044,90940896816021,20062927283041,267080852079651], + [249858369443708,206247021789428,20287304470696,25847120931175], + [10645228898670,224705626119556,267405565521452,258214397180678], + [264783762221048,166955943492306,98802957003141,102012488332476], + [69425677456944,177142907243411,137138950917722,228865047699598], + [116031213307147,29203342183358,37406949328742,255198080174323], + [200358182338308,235207156008390,120922906095920,116215987197289], + [58728890318426,68877471005069,176496507286088,221041411345780], + [91094120814795,50665258299931,256093108116737,19777509566621], + [74646746200247,98350487270564,154448261001199,39881047281135], + [23408943649483,164410325820923,248161749770122,274558342231648], + [169531547115055,213630535746863,235098262267796,200508473898303], + [235098564415817,85039146398174,51721575960328,173069189684390], + [176136386396069,155368359051606,147817099696487,265419485459634], + [137542881551462,40028925519736,70525669519846,63445773516557], + [173854695142814,114282444507812,149945832627054,99605565798831], + [177686773562184,127158716984798,132495543008547], + [227073396444896,139667311071766,158915951283562], + [26212438434289,94902985796531,198145776057315], + [266279278943923,58550737262493,74297973216378], + [32373606512065,131854353044428,184642643042326], + [34335377662439,85341895822066,273492717750246]], + + %% Verify that the keys still collide (this will fail if the + %% internal hash function has been changed). + erts_debug:set_internal_state(available_internal_state, true), + try + case erlang:system_info(wordsize) of + 8 -> + verify_colliding_keys(L); + 4 -> + %% Not guaranteed to collide on a 32-bit system. + ok + end + after + erts_debug:set_internal_state(available_internal_state, false) + end, + + L. + +verify_colliding_keys([[K|Ks]|Gs]) -> + Hash = internal_hash(K), + [Hash] = lists:usort([internal_hash(Key) || Key <- Ks]), + verify_colliding_keys(Gs); +verify_colliding_keys([]) -> + ok. + +internal_hash(Term) -> + erts_debug:get_internal_state({internal_hash,Term}). + +%% Test that all persistent terms are erased by init:restart/0. + +init_restart(_Config) -> + File = "command_file", + ok = file:write_file(File, term_to_binary(restart)), + {ok,[[Erl]]} = init:get_argument(progname), + ModPath = filename:dirname(code:which(?MODULE)), + Cmd = Erl ++ " -pa " ++ ModPath ++ " -noshell " + "-run " ++ ?MODULE_STRING ++ " test_init_restart_cmd " ++ + File, + io:format("~s\n", [Cmd]), + Expected = "12ok", + case os:cmd(Cmd) of + Expected -> + ok; + Actual -> + io:format("Expected: ~s", [Expected]), + io:format("Actual: ~s\n", [Actual]), + ct:fail(unexpected_output) + end. + +test_init_restart_cmd([File]) -> + try + do_test_init_restart_cmd(File) + catch + C:R -> + io:format("\n~p ~p\n", [C,R]), + halt() + end, + receive + _ -> ok + end. + +do_test_init_restart_cmd(File) -> + {ok,Bin} = file:read_file(File), + Seq = lists:seq(1, 50), + case binary_to_term(Bin) of + restart -> + _ = [persistent_term:put({?MODULE,I}, {value,I}) || + I <- Seq], + ok = file:write_file(File, term_to_binary(was_restarted)), + io:put_chars("1"), + init:restart(), + receive + _ -> ok + end; + was_restarted -> + io:put_chars("2"), + ok = file:delete(File), + _ = [begin + Key = {?MODULE,I}, + {'EXIT',{badarg,_}} = (catch persistent_term:get(Key)) + end || I <- Seq], + io:put_chars("ok"), + init:stop() + end. + +%% Check that there is the same number of persistents terms before +%% and after each test case. + +chk() -> + persistent_term:info(). + +chk(Chk) -> + Chk = persistent_term:info(), + Key = {?MODULE,?FUNCTION_NAME}, + ok = persistent_term:put(Key, {term,Chk}), + Term = persistent_term:get(Key), + true = persistent_term:erase(Key), + chk_not_stuck(Term), + ok. + +chk_not_stuck(Term) -> + %% Hash tables to be deleted are put onto a queue. + %% Make sure that the queue isn't stuck by a table with + %% a non-zero ref count. + + case erts_debug:size_shared(Term) of + 0 -> + erlang:yield(), + chk_not_stuck(Term); + _ -> + ok + end. diff --git a/erts/etc/unix/run_erl.c b/erts/etc/unix/run_erl.c index 725343d701..bfb3e1bd2c 100644 --- a/erts/etc/unix/run_erl.c +++ b/erts/etc/unix/run_erl.c @@ -43,10 +43,10 @@ #endif #ifdef HAVE_WORKING_POSIX_OPENPT # ifndef _XOPEN_SOURCE - /* On OS X and BSD, we must leave _XOPEN_SOURCE undefined in order for - * the prototype of vsyslog() to be included. + /* On OS X, BSD and Solaris, we must leave _XOPEN_SOURCE undefined in order + * for the prototype of vsyslog() to be included. */ -# if !(defined(__APPLE__) || defined(__FreeBSD__) || defined(__DragonFly__)) +# if !(defined(__APPLE__) || defined(__FreeBSD__) || defined(__DragonFly__) || defined(__sun)) # define _XOPEN_SOURCE 600 # endif # endif diff --git a/erts/preloaded/ebin/atomics.beam b/erts/preloaded/ebin/atomics.beam Binary files differnew file mode 100644 index 0000000000..1de97fa668 --- /dev/null +++ b/erts/preloaded/ebin/atomics.beam diff --git a/erts/preloaded/ebin/counters.beam b/erts/preloaded/ebin/counters.beam Binary files differnew file mode 100644 index 0000000000..caaa6167e1 --- /dev/null +++ b/erts/preloaded/ebin/counters.beam diff --git a/erts/preloaded/ebin/erts_internal.beam b/erts/preloaded/ebin/erts_internal.beam Binary files differindex 15c59de80a..e174f71966 100644 --- a/erts/preloaded/ebin/erts_internal.beam +++ b/erts/preloaded/ebin/erts_internal.beam diff --git a/erts/preloaded/ebin/init.beam b/erts/preloaded/ebin/init.beam Binary files differindex 858a9dc63e..1e60ef7e88 100644 --- a/erts/preloaded/ebin/init.beam +++ b/erts/preloaded/ebin/init.beam diff --git a/erts/preloaded/ebin/persistent_term.beam b/erts/preloaded/ebin/persistent_term.beam Binary files differnew file mode 100644 index 0000000000..79ef03b9a6 --- /dev/null +++ b/erts/preloaded/ebin/persistent_term.beam diff --git a/erts/preloaded/ebin/prim_inet.beam b/erts/preloaded/ebin/prim_inet.beam Binary files differindex 4a345f8152..eaa1e2cdf8 100644 --- a/erts/preloaded/ebin/prim_inet.beam +++ b/erts/preloaded/ebin/prim_inet.beam diff --git a/erts/preloaded/src/Makefile b/erts/preloaded/src/Makefile index 4333f6643a..e1bd5bc295 100644 --- a/erts/preloaded/src/Makefile +++ b/erts/preloaded/src/Makefile @@ -47,7 +47,10 @@ PRE_LOADED_ERL_MODULES = \ erts_internal \ erl_tracer \ erts_literal_area_collector \ - erts_dirty_process_signal_handler + erts_dirty_process_signal_handler \ + atomics \ + counters \ + persistent_term PRE_LOADED_BEAM_MODULES = \ prim_eval diff --git a/erts/preloaded/src/atomics.erl b/erts/preloaded/src/atomics.erl new file mode 100644 index 0000000000..d1fe5e65cf --- /dev/null +++ b/erts/preloaded/src/atomics.erl @@ -0,0 +1,119 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% Purpose : Main atomics API module. + +-module(atomics). + +-export([new/2, + put/3, get/2, + add/3, add_get/3, + sub/3, sub_get/3, + exchange/3, compare_exchange/4, + info/1]). + +-export_type([atomics_ref/0]). + +-opaque atomics_ref() :: reference(). + +-define(OPT_SIGNED, (1 bsl 0)). +-define(OPT_DEFAULT, ?OPT_SIGNED). + +-spec new(Arity, Opts) -> atomics_ref() when + Arity :: pos_integer(), + Opts :: [Opt], + Opt :: {signed, boolean()}. +new(Arity, Opts) -> + erts_internal:atomics_new(Arity, encode_opts(Opts, ?OPT_DEFAULT)). + +encode_opts([{signed, true}|T], Acc) -> + encode_opts(T, Acc bor ?OPT_SIGNED); +encode_opts([{signed, false}|T], Acc) -> + encode_opts(T, Acc band (bnot ?OPT_SIGNED)); +encode_opts([], Acc) -> + Acc; +encode_opts(_, _) -> + erlang:error(badarg). + +-spec put(Ref, Ix, Value) -> ok when + Ref :: atomics_ref(), + Ix :: integer(), + Value :: integer(). +put(_Ref, _Ix, _Value) -> + erlang:nif_error(undef). + +-spec get(Ref, Ix) -> integer() when + Ref :: atomics_ref(), + Ix :: integer(). +get(_Ref, _Ix) -> + erlang:nif_error(undef). + +-spec add(Ref, Ix, Incr) -> ok when + Ref :: atomics_ref(), + Ix :: integer(), + Incr :: integer(). +add(_Ref, _Ix, _Incr) -> + erlang:nif_error(undef). + +-spec add_get(Ref, Ix, Incr) -> integer() when + Ref :: atomics_ref(), + Ix :: integer(), + Incr :: integer(). +add_get(_Ref, _Ix, _Incr) -> + erlang:nif_error(undef). + +-spec sub(Ref, Ix, Decr) -> ok when + Ref :: atomics_ref(), + Ix :: integer(), + Decr :: integer(). +sub(Ref, Ix, Decr) -> + ?MODULE:add(Ref, Ix, -Decr). + +-spec sub_get(Ref, Ix, Decr) -> integer() when + Ref :: atomics_ref(), + Ix :: integer(), + Decr :: integer(). +sub_get(Ref, Ix, Decr) -> + ?MODULE:add_get(Ref, Ix, -Decr). + +-spec exchange(Ref, Ix, Desired) -> integer() when + Ref :: atomics_ref(), + Ix :: integer(), + Desired :: integer(). +exchange(_Ref, _Ix, _Desired) -> + erlang:nif_error(undef). + +-spec compare_exchange(Ref, Ix, Expected, Desired) -> ok | integer() when + Ref :: atomics_ref(), + Ix :: integer(), + Expected :: integer(), + Desired :: integer(). +compare_exchange(_Ref, _Ix, _Expected, _Desired) -> + erlang:nif_error(undef). + +-spec info(Ref) -> Info when + Ref :: atomics_ref(), + Info :: #{'size':=Size,'max':=Max,'min':=Min,'memory':=Memory}, + Size :: non_neg_integer(), + Max :: integer(), + Min :: integer(), + Memory :: non_neg_integer(). +info(_Ref) -> + erlang:nif_error(undef). diff --git a/erts/preloaded/src/counters.erl b/erts/preloaded/src/counters.erl new file mode 100644 index 0000000000..67354f648d --- /dev/null +++ b/erts/preloaded/src/counters.erl @@ -0,0 +1,90 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% Purpose : Main atomics API module. + +-module(counters). + +-export([new/2, + get/2, + add/3, + sub/3, + info/1]). + +-export_type([counters_ref/0]). + +-opaque counters_ref() :: {atomics, reference()} | {write_concurrency, reference()}. + +-spec new(Size, Opts) -> counters_ref() when + Size :: pos_integer(), + Opts :: [Opt], + Opt :: atomics | write_concurrency. +new(Size, [atomics]) -> + {atomics, atomics:new(Size, [{signed, true}])}; +new(Size, [write_concurrency]) -> + {write_concurrency, erts_internal:counters_new(Size)}; +new(Size, []) -> + new(Size, [atomics]); +new(_, _) -> + erlang:error(badarg). + +-spec get(Ref, Ix) -> integer() when + Ref :: counters_ref(), + Ix :: integer(). +get({atomics,Ref}, Ix) -> + atomics:get(Ref, Ix); +get({write_concurrency, Ref}, Ix) -> + erts_internal:counters_get(Ref, Ix); +get(_, _) -> + erlang:error(badarg). + + + +-spec add(Ref, Ix, Incr) -> ok when + Ref :: counters_ref(), + Ix :: integer(), + Incr :: integer(). +add({atomics, Ref}, Ix, Incr) -> + atomics:add(Ref, Ix, Incr); +add({write_concurrency, Ref}, Ix, Incr) -> + erts_internal:counters_add(Ref, Ix, Incr); +add(_, _, _) -> + erlang:error(badarg). + + +-spec sub(Ref, Ix, Decr) -> ok when + Ref :: counters_ref(), + Ix :: integer(), + Decr :: integer(). +sub(Ref, Ix, Decr) -> + add(Ref, Ix, -Decr). + +-spec info(Ref) -> Info when + Ref :: counters_ref(), + Info :: #{'size':=Size, 'memory':=Memory}, + Size :: non_neg_integer(), + Memory :: non_neg_integer(). +info({atomics, Ref}) -> + atomics:info(Ref); +info({write_concurrency, Ref}) -> + erts_internal:counters_info(Ref); +info(_) -> + erlang:error(badarg). + diff --git a/erts/preloaded/src/erts.app.src b/erts/preloaded/src/erts.app.src index 8c34c99a98..ed645d1191 100644 --- a/erts/preloaded/src/erts.app.src +++ b/erts/preloaded/src/erts.app.src @@ -33,12 +33,14 @@ prim_file, prim_inet, prim_zip, + atomics, + counters, zlib ]}, {registered, []}, {applications, []}, {env, []}, - {runtime_dependencies, ["stdlib-3.5", "kernel-6.1", "sasl-3.0.1"]} + {runtime_dependencies, ["stdlib-3.5", "kernel-6.1", "sasl-@OTP-13468@"]} ]}. %% vim: ft=erlang diff --git a/erts/preloaded/src/erts_internal.erl b/erts/preloaded/src/erts_internal.erl index 88f47e917b..d491a505c6 100644 --- a/erts/preloaded/src/erts_internal.erl +++ b/erts/preloaded/src/erts_internal.erl @@ -90,6 +90,13 @@ -export([create_dist_channel/4]). +-export([erase_persistent_terms/0]). + +-export([atomics_new/2]). + +-export([counters_new/1, counters_get/2, counters_add/3, + counters_info/1]). + %% %% Await result of send to port %% @@ -691,3 +698,27 @@ process_flag(_Pid, _Flag, _Value) -> create_dist_channel(_Node, _DistCtrlr, _Flags, _Ver) -> erlang:nif_error(undefined). + +-spec erase_persistent_terms() -> 'ok'. +erase_persistent_terms() -> + erlang:nif_error(undefined). + +-spec atomics_new(pos_integer(), pos_integer()) -> reference(). +atomics_new(_Arity, _EncOpts) -> + erlang:nif_error(undef). + +-spec counters_new(pos_integer()) -> reference(). +counters_new(_Size) -> + erlang:nif_error(undef). + +-spec counters_get(reference(), pos_integer()) -> integer(). +counters_get(_Ref, _Ix) -> + erlang:nif_error(undef). + +-spec counters_add(reference(), pos_integer(), integer()) -> ok. +counters_add(_Ref, _Ix, _Incr) -> + erlang:nif_error(undef). + +-spec counters_info(reference()) -> #{}. +counters_info(_Ref) -> + erlang:nif_error(undef). diff --git a/erts/preloaded/src/init.erl b/erts/preloaded/src/init.erl index 253fcf7a1f..b4b8b3bf9b 100644 --- a/erts/preloaded/src/init.erl +++ b/erts/preloaded/src/init.erl @@ -552,6 +552,7 @@ stop(Reason,State) -> do_stop(restart,#state{start = Start, flags = Flags, args = Args}) -> %% Make sure we don't have any outstanding messages before doing the restart. flush(), + erts_internal:erase_persistent_terms(), boot(Start,Flags,Args); do_stop(reboot,_) -> halt(); diff --git a/erts/preloaded/src/persistent_term.erl b/erts/preloaded/src/persistent_term.erl new file mode 100644 index 0000000000..5d0c266127 --- /dev/null +++ b/erts/preloaded/src/persistent_term.erl @@ -0,0 +1,55 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(persistent_term). + +-export([erase/1,get/0,get/1,info/0,put/2]). + +-type key() :: term(). +-type value() :: term(). + +-spec erase(Key) -> Result when + Key :: key(), + Result :: boolean(). +erase(_Key) -> + erlang:nif_error(undef). + +-spec get() -> List when + List :: [{key(),value()}]. +get() -> + erlang:nif_error(undef). + +-spec get(Key) -> Value when + Key :: key(), + Value :: value(). +get(_Key) -> + erlang:nif_error(undef). + +-spec info() -> Info when + Info :: #{'count':=Count,'memory':=Memory}, + Count :: non_neg_integer(), + Memory :: non_neg_integer(). +info() -> + erlang:nif_error(undef). + +-spec put(Key, Value) -> 'ok' when + Key :: key(), + Value :: value(). +put(_Key, _Value) -> + erlang:nif_error(undef). diff --git a/erts/preloaded/src/prim_inet.erl b/erts/preloaded/src/prim_inet.erl index 8169943dde..f1d938c9a4 100644 --- a/erts/preloaded/src/prim_inet.erl +++ b/erts/preloaded/src/prim_inet.erl @@ -49,9 +49,15 @@ -include("inet_sctp.hrl"). -include("inet_int.hrl"). -%-define(DEBUG, 1). +%%%-define(DEBUG, 1). -ifdef(DEBUG). --define(DBG_FORMAT(Format, Args), (io:format((Format), (Args)))). +-define( + DBG_FORMAT(Format, Args), + begin + %% io:format((Format), (Args)), + erlang:display(lists:flatten(io_lib:format((Format), (Args)))), + ok + end). -else. -define(DBG_FORMAT(Format, Args), ok). -endif. @@ -150,39 +156,96 @@ shutdown_1(S, How) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% close(S) when is_port(S) -> + ?DBG_FORMAT("prim_inet:close(~p)~n", [S]), case getopt(S, linger) of {ok,{true,0}} -> close_port(S); - _ -> - case subscribe(S, [subs_empty_out_q]) of - {ok, [{subs_empty_out_q,N}]} when N > 0 -> - close_pend_loop(S, N); %% wait for pending output to be sent - _ -> - close_port(S) - end + {ok,{true,T}} -> + %% Wait for T seconds for pending output to be sent + %% + %% Note that this handling of Linger may look ok, + %% but sweeps some problems under the rug since + %% there are OS buffers that may have remaining data + %% after the inet driver has emptied its buffers. + %% But Linger for nonblocking sockets is broken + %% anyway on all OS:es, according to hearsay, + %% and is a contradiction in itself. + %% We have hereby done our best... + %% + Tref = erlang:start_timer(T * 1000, self(), close_port), + close_pend_loop(S, Tref, undefined); + _ -> % Regard this as {ok,{false,_}} + case subscribe(S, [subs_empty_out_q]) of + {ok, [{subs_empty_out_q,N}]} when N > 0 -> + %% Wait for pending output to be sent + DefaultT = 180000, % Arbitrary system timeout 3 min + Tref = erlang:start_timer(DefaultT, self(), close_port), + close_pend_loop(S, Tref, N); + _ -> + %% Subscribe failed or empty out q - give up or done + close_port(S) + end end. -close_pend_loop(S, N) -> +close_pend_loop(S, Tref, N) -> + ?DBG_FORMAT("prim_inet:close_pend_loop(~p, _, ~p)~n", [S,N]), receive - {empty_out_q,S} -> - close_port(S) + {timeout,Tref,_} -> % Linger timeout + ?DBG_FORMAT("prim_inet:close_pend_loop(~p, _, _) timeout~n", [S]), + close_port(S); + {empty_out_q,S} when N =/= undefined -> + ?DBG_FORMAT( + "prim_inet:close_pend_loop(~p, _, _) empty_out_q~n", [S]), + close_port(S, Tref) after ?INET_CLOSE_TIMEOUT -> case getstat(S, [send_pend]) of {ok, [{send_pend,N1}]} -> + ?DBG_FORMAT( + "prim_inet:close_pend_loop(~p, _, _) send_pend ~p~n", + [S,N1]), if - N1 =:= N -> - close_port(S); - true -> - close_pend_loop(S, N1) + N1 =:= 0 -> + %% Empty outq - done + close_port(S, Tref); + N =:= undefined -> + %% Within linger time - wait some more + close_pend_loop(S, Tref, N); + N1 =:= N -> + %% Inactivity - give up + close_port(S, Tref); + true -> + %% Still moving - wait some more + close_pend_loop(S, Tref, N) end; - _ -> - close_port(S) - end + _Stat -> + %% Failed getstat - give up + ?DBG_FORMAT( + "prim_inet:close_pend_loop(~p, _, _) getstat ~p~n", + [S,_Stat]), + close_port(S, Tref) + end end. + +close_port(S, Tref) -> + ?DBG_FORMAT("prim_inet:close_port(~p, _)~n", [S]), + case erlang:cancel_timer(Tref) of + false -> + receive + {timeout,Tref,_} -> + ok + end; + _N -> + ok + end, + close_port(S). +%% close_port(S) -> - catch erlang:port_close(S), - receive {'EXIT',S,_} -> ok after 0 -> ok end. + ?DBG_FORMAT("prim_inet:close_port(~p)~n", [S]), + _Closed = (catch erlang:port_close(S)), + receive {'EXIT',S,_} -> ok after 0 -> ok end, + ?DBG_FORMAT("prim_inet:close_port(~p) ~p~n", [S,_Closed]), + ok. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% @@ -424,23 +487,49 @@ peeloff(S, AssocId) -> %% be called directly -- use "sendmsg" instead: %% send(S, Data, OptList) when is_port(S), is_list(OptList) -> - ?DBG_FORMAT("prim_inet:send(~p, ~p)~n", [S,Data]), + ?DBG_FORMAT("prim_inet:send(~p, _, ~p)~n", [S,OptList]), try erlang:port_command(S, Data, OptList) of false -> % Port busy and nosuspend option passed ?DBG_FORMAT("prim_inet:send() -> {error,busy}~n", []), {error,busy}; true -> - receive - {inet_reply,S,Status} -> - ?DBG_FORMAT("prim_inet:send() -> ~p~n", [Status]), - Status - end + send_recv_reply(S, undefined) catch error:_Error -> ?DBG_FORMAT("prim_inet:send() -> {error,einval}~n", []), {error,einval} end. +send_recv_reply(S, Mref) -> + ReplyTimeout = + case Mref of + undefined -> + ?INET_CLOSE_TIMEOUT; + _ -> + infinity + end, + receive + {inet_reply,S,Status} -> + ?DBG_FORMAT( + "prim_inet:send_recv_reply(~p, _): inet_reply ~p~n", + [S,Status]), + case Mref of + undefined -> ok; + _ -> + demonitor(Mref, [flush]), + ok + end, + Status; + {'DOWN',Mref,_,_,_Reason} when Mref =/= undefined -> + ?DBG_FORMAT( + "prim_inet:send_recv_reply(~p, _) 'DOWN' ~p~n", + [S,_Reason]), + {error,closed} + after ReplyTimeout -> + send_recv_reply(S, monitor(port, S)) + end. + + send(S, Data) -> send(S, Data, []). @@ -520,13 +609,35 @@ sendfile(S, FileHandle, Offset, Length) sendfile(S, FileHandle, Offset, Length) -> case erlang:port_info(S, connected) of {connected, Pid} when Pid =:= self() -> - sendfile_1(S, FileHandle, Offset, Length); + Uncork = sendfile_maybe_cork(S), + Result = sendfile_1(S, FileHandle, Offset, Length), + sendfile_maybe_uncork(S, Uncork), + Result; {connected, Pid} when Pid =/= self() -> {error, not_owner}; _Other -> {error, einval} end. +sendfile_maybe_cork(S) -> + case getprotocol(S) of + tcp -> + case getopts(S, [nopush]) of + {ok, [{nopush,false}]} -> + _ = setopts(S, [{nopush,true}]), + true; + _ -> + false + end; + _ -> false + end. + +sendfile_maybe_uncork(S, true) -> + _ = setopts(S, [{nopush,false}]), + ok; +sendfile_maybe_uncork(_, false) -> + ok. + sendfile_1(S, FileHandle, Offset, 0) -> sendfile_1(S, FileHandle, Offset, (1 bsl 63) - 1); sendfile_1(_S, _FileHandle, Offset, Length) when @@ -870,9 +981,9 @@ chgopts(S, Opts) when is_port(S), is_list(Opts) -> getifaddrs(S) when is_port(S) -> case ctl_cmd(S, ?INET_REQ_GETIFADDRS, []) of - {ok, Data} -> - {ok, comp_ifaddrs(build_ifaddrs(Data), ktree_empty())}; - {error,enotsup} -> + {ok, Data} -> + {ok, comp_ifaddrs(build_ifaddrs(Data))}; + {error,enotsup} -> case getiflist(S) of {ok, IFs} -> {ok, getifaddrs_ifget(S, IFs)}; @@ -881,30 +992,75 @@ getifaddrs(S) when is_port(S) -> Err2 -> Err2 end. -%% Restructure interface properties per interface and remove duplicates - -comp_ifaddrs([{If,Opts}|IfOpts], T) -> - case ktree_is_defined(If, T) of - true -> - OptSet = comp_ifaddrs_add(ktree_get(If, T), Opts), - comp_ifaddrs(IfOpts, ktree_update(If, OptSet, T)); - false -> - OptSet = comp_ifaddrs_add(ktree_empty(), Opts), - comp_ifaddrs(IfOpts, ktree_insert(If, OptSet, T)) - end; -comp_ifaddrs([], T) -> - [{If,ktree_keys(ktree_get(If, T))} || If <- ktree_keys(T)]. - -comp_ifaddrs_add(OptSet, [Opt|Opts]) -> - case ktree_is_defined(Opt, OptSet) of - true - when element(1, Opt) =:= flags; - element(1, Opt) =:= hwaddr -> - comp_ifaddrs_add(OptSet, Opts); - _ -> - comp_ifaddrs_add(ktree_insert(Opt, undefined, OptSet), Opts) +%% Restructure interface properties per interface + +comp_ifaddrs(IfOpts) -> + comp_ifaddrs(IfOpts, ktree_empty()). +%% +comp_ifaddrs([{If,[{flags,Flags}|Opts]}|IfOpts], IfT) -> + case ktree_is_defined(If, IfT) of + true -> + comp_ifaddrs( + IfOpts, + ktree_update( + If, + comp_ifaddrs_flags(Flags, Opts, ktree_get(If, IfT)), + IfT)); + false -> + comp_ifaddrs( + IfOpts, + ktree_insert( + If, + comp_ifaddrs_flags(Flags, Opts, ktree_empty()), + IfT)) end; -comp_ifaddrs_add(OptSet, []) -> OptSet. +comp_ifaddrs([], IfT) -> + comp_ifaddrs_2(ktree_keys(IfT), IfT). + +comp_ifaddrs_flags(Flags, Opts, FlagsT) -> + case ktree_is_defined(Flags, FlagsT) of + true -> + ktree_update( + Flags, + rev(Opts, ktree_get(Flags, FlagsT)), + FlagsT); + false -> + ktree_insert(Flags, rev(Opts), FlagsT) + end. + +comp_ifaddrs_2([If|Ifs], IfT) -> + FlagsT = ktree_get(If, IfT), + [{If,comp_ifaddrs_3(ktree_keys(FlagsT), FlagsT)} + | comp_ifaddrs_2(Ifs, IfT)]; +comp_ifaddrs_2([], _IfT) -> + []. +%% +comp_ifaddrs_3([Flags|FlagsL], FlagsT) -> + [{flags,Flags}|hwaddr_last(rev(ktree_get(Flags, FlagsT)))] + ++ hwaddr_last(comp_ifaddrs_3(FlagsL, FlagsT)); +comp_ifaddrs_3([], _FlagsT) -> + []. + +%% Place hwaddr last to look more like legacy emulation +hwaddr_last(Opts) -> + hwaddr_last(Opts, Opts, []). +%% +hwaddr_last([{hwaddr,_} = Opt|Opts], L, R) -> + hwaddr_last(Opts, L, [Opt|R]); +hwaddr_last([_|Opts], L, R) -> + hwaddr_last(Opts, L, R); +hwaddr_last([], L, []) -> + L; +hwaddr_last([], L, R) -> + rev(hwaddr_last(L, []), rev(R)). +%% +hwaddr_last([{hwaddr,_}|Opts], R) -> + hwaddr_last(Opts, R); +hwaddr_last([Opt|Opts], R) -> + hwaddr_last(Opts, [Opt|R]); +hwaddr_last([], R) -> + R. + %% Legacy emulation of getifaddrs @@ -912,21 +1068,19 @@ getifaddrs_ifget(_, []) -> []; getifaddrs_ifget(S, [IF|IFs]) -> case ifget(S, IF, [flags]) of {ok,[{flags,Flags}]=FlagsVals} -> - BroadOpts = - case member(broadcast, Flags) of - true -> - [broadaddr,hwaddr]; - false -> - [hwaddr] - end, - P2POpts = - case member(pointtopoint, Flags) of - true -> - [dstaddr|BroadOpts]; - false -> - BroadOpts - end, - getifaddrs_ifget(S, IFs, IF, FlagsVals, [addr,netmask|P2POpts]); + GetOpts = + case member(pointtopoint, Flags) of + true -> + [dstaddr,hwaddr]; + false -> + case member(broadcast, Flags) of + true -> + [broadaddr,hwaddr]; + false -> + [hwaddr] + end + end, + getifaddrs_ifget(S, IFs, IF, FlagsVals, [addr,netmask|GetOpts]); _ -> getifaddrs_ifget(S, IFs, IF, [], [addr,netmask,hwaddr]) end. @@ -1275,6 +1429,7 @@ enc_opt(pktoptions) -> ?INET_OPT_PKTOPTIONS; enc_opt(ttl) -> ?INET_OPT_TTL; enc_opt(recvttl) -> ?INET_OPT_RECVTTL; enc_opt(nodelay) -> ?TCP_OPT_NODELAY; +enc_opt(nopush) -> ?TCP_OPT_NOPUSH; enc_opt(multicast_if) -> ?UDP_OPT_MULTICAST_IF; enc_opt(multicast_ttl) -> ?UDP_OPT_MULTICAST_TTL; enc_opt(multicast_loop) -> ?UDP_OPT_MULTICAST_LOOP; @@ -1336,6 +1491,7 @@ dec_opt(?INET_OPT_PRIORITY) -> priority; dec_opt(?INET_OPT_TOS) -> tos; dec_opt(?INET_OPT_TCLASS) -> tclass; dec_opt(?TCP_OPT_NODELAY) -> nodelay; +dec_opt(?TCP_OPT_NOPUSH) -> nopush; dec_opt(?INET_OPT_RECVTOS) -> recvtos; dec_opt(?INET_OPT_RECVTCLASS) -> recvtclass; dec_opt(?INET_OPT_PKTOPTIONS) -> pktoptions; @@ -1422,6 +1578,7 @@ type_opt_1(pktoptions) -> opts; type_opt_1(ttl) -> int; type_opt_1(recvttl) -> bool; type_opt_1(nodelay) -> bool; +type_opt_1(nopush) -> bool; type_opt_1(ipv6_v6only) -> bool; %% multicast type_opt_1(multicast_ttl) -> int; @@ -2500,7 +2657,7 @@ get_addrs([F|Addrs]) -> [Addr|get_addrs(Rest)]. get_addr(?INET_AF_LOCAL, [N|Addr]) -> - {A,Rest} = lists:split(N, Addr), + {A,Rest} = split(N, Addr), {{local,iolist_to_binary(A)},Rest}; get_addr(?INET_AF_UNSPEC, Rest) -> {{unspec,<<>>},Rest}; |