diff options
141 files changed, 5702 insertions, 407 deletions
diff --git a/Makefile.in b/Makefile.in index 9f053963c4..fa7c128379 100644 --- a/Makefile.in +++ b/Makefile.in @@ -359,7 +359,8 @@ erlang_inst_libdir_configured: bootstrap: depend all_bootstraps check_dev_rt_dep: - @if `grep DEVELOPMENT "$(ERL_TOP)/make/otp_version_tickets" 1>/dev/null 2>&1`; then \ + @if `grep DEVELOPMENT "$(ERL_TOP)/make/otp_version_tickets" 1>/dev/null 2>&1 \ + || grep TESTING "$(ERL_TOP)/make/otp_version_tickets" 1>/dev/null 2>&1`; then \ LANG=C "$(PERL)" "$(ERL_TOP)/make/fixup_development_runtime_dependencies" "$(ERL_TOP)"; \ fi diff --git a/OTP_VERSION b/OTP_VERSION index 2d978e312b..0ecacc9d29 100644 --- a/OTP_VERSION +++ b/OTP_VERSION @@ -1 +1 @@ -21.1.1 +21.1.3 diff --git a/erts/configure.in b/erts/configure.in index bcdc6cd083..4a27b532de 100644 --- a/erts/configure.in +++ b/erts/configure.in @@ -388,6 +388,56 @@ if test X"$with_ets_write_concurrency_locks" != X""; then [Define to override the default number of write_concurrency locks]) fi +AC_ARG_WITH(spectre-mitigation, + AS_HELP_STRING([--with-spectre-mitigation={yes|incomplete}], + [enable spectre mitigation, either fully or with mitigations + disabled in a handful places like the interpreter]) + AS_HELP_STRING([--without-spectre-mitigation], + [build without spectre mitigation]), + [],[with_spectre_mitigation=no]) + +case "$with_spectre_mitigation" in + no) ;; + yes) ;; + incomplete) ;; + *) AC_MSG_ERROR([Invalid spectre mitigation setting]) ;; +esac + +i_noretpoline_attr="" + +if test X"$with_spectre_mitigation" != X"no"; then + CFLAGS="$CFLAGS -mindirect-branch=thunk" + + AC_MSG_CHECKING([for spectre mitigation]) + AC_COMPILE_IFELSE( + [AC_LANG_PROGRAM([],[return 0;])], + [AC_MSG_RESULT([yes])], + [AC_MSG_ERROR([no])]) + + if test X"$with_spectre_mitigation" = X"incomplete"; then + # gcc and clang support this attribute if they're recent enough. Note + # that we must compile with -Werror to check for actual support as they + # warn rather than error out on unsupported attributes. + + i_noretpoline_attr='__attribute__((__indirect_branch__("keep")))' + i_preserve_cflags="$CFLAGS" + CFLAGS="$CFLAGS -Werror" + + AC_MSG_CHECKING([whether spectre mitigation can be disabled on a per-function basis]) + AC_COMPILE_IFELSE( + [AC_LANG_PROGRAM([$i_noretpoline_attr],[return 0;])], + [AC_MSG_RESULT([yes])], + [AC_MSG_ERROR([no])]) + + CFLAGS="$i_preserve_cflags" + fi +fi + +AC_DEFINE_UNQUOTED(ERTS_NO_RETPOLINE, $i_noretpoline_attr, + [Per-function attribute for disabling retpoline. This is + *only* defined when --with-spectre-mitigation=incomplete + and has no effects otherwise]) + dnl ---------------------------------------------------------------------- dnl Checks for programs. dnl ---------------------------------------------------------------------- diff --git a/erts/doc/src/Makefile b/erts/doc/src/Makefile index 21aa3db864..40f74b78ff 100644 --- a/erts/doc/src/Makefile +++ b/erts/doc/src/Makefile @@ -52,18 +52,17 @@ XML_REF3_EFILES = \ erlang.xml \ erl_tracer.xml \ init.xml \ + persistent_term.xml \ + atomics.xml \ + counters.xml \ zlib.xml XML_REF3_FILES = \ + $(XML_REF3_EFILES) \ driver_entry.xml \ erl_nif.xml \ - erl_tracer.xml \ erl_driver.xml \ - erl_prim_loader.xml \ - erlang.xml \ - erts_alloc.xml \ - init.xml \ - zlib.xml + erts_alloc.xml XML_PART_FILES = \ part.xml diff --git a/erts/doc/src/atomics.xml b/erts/doc/src/atomics.xml new file mode 100644 index 0000000000..3fca92fb97 --- /dev/null +++ b/erts/doc/src/atomics.xml @@ -0,0 +1,183 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE erlref SYSTEM "erlref.dtd"> + +<erlref> + <header> + <copyright> + <year>2018</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + </legalnotice> + + <title>atomics</title> + </header> + <module>atomics</module> + <modulesummary>Atomic Functions</modulesummary> + <description> + <p>This module provides a set of functions to do atomic operations towards + mutable atomic variables. The implementation utilizes only + atomic hardware instructions without any software level locking, which makes + it very efficient for concurrent access. The atomics are organized into + arrays with the follwing semantics:</p> + <list type="bulleted"> + <item> + <p>Atomics are 64 bit integers.</p> + </item> + <item> + <p>Atomics can be represented as either signed or unsigned.</p> + </item> + <item> + <p>Atomics wrap around at overflow and underflow operations.</p> + </item> + <item> + <p>All operations guarantee atomicity. No intermediate results can be + seen. The result of one mutation can only be the input to one + following mutation.</p> + </item> + <item> + <p>All atomic operations are mutually ordered. If atomic B is updated + <em>after</em> atomic A, then that is how it will appear to any + concurrent readers. No one can read the new value of B and then read the + old value of A.</p> + </item> + <item> + <p>Indexes into atomic arrays are one-based. An atomic array of + arity N contains N atomics with index from 1 to N.</p> + </item> + </list> + </description> + + <datatypes> + <datatype> + <name name="atomics_ref"/> + <desc><p>Identifies an atomic array returned from + <seealso marker="#new/2"><c>new/2</c></seealso>.</p> + </desc> + </datatype> + </datatypes> + + <funcs> + <func> + <name name="new" arity="2"/> + <fsummary>Create atomic array</fsummary> + <desc> + <p>Create a new atomic array of <c><anno>Arity</anno></c> atomics.</p> + <p>Argument <c><anno>Opts</anno></c> is a list of the following possible + options:</p> + <taglist> + <tag><c>{signed, boolean()}</c></tag> + <item><p>Indicate if the elements of the array will be treated + as signed or unsigned integers. Default is <c>true</c> (signed).</p> + <p>The integer interval for signed atomics are from <c>-(1 bsl 63)</c> + to <c>(1 bsl 63)-1</c> and for unsigned atomics from <c>0</c> to <c>(1 + bsl 64)-1</c>.</p> + </item> + </taglist> + </desc> + </func> + + <func> + <name name="put" arity="3"/> + <fsummary>Set atomic value</fsummary> + <desc> + <p>Set atomic to <c><anno>Value</anno></c>.</p> + </desc> + </func> + + <func> + <name name="get" arity="2"/> + <fsummary>Read atomic value</fsummary> + <desc> + <p>Read atomic value.</p> + </desc> + </func> + + <func> + <name name="add" arity="3"/> + <fsummary>Add to atomic</fsummary> + <desc> + <p>Add <c><anno>Incr</anno></c> to atomic.</p> + </desc> + </func> + + <func> + <name name="add_get" arity="3"/> + <fsummary>Atomic add and get</fsummary> + <desc> + <p>Atomic addition and return of the result.</p> + </desc> + </func> + + <func> + <name name="sub" arity="3"/> + <fsummary>Subtract from atomic</fsummary> + <desc> + <p>Subtract <c><anno>Decr</anno></c> from atomic.</p> + </desc> + </func> + + <func> + <name name="sub_get" arity="3"/> + <fsummary>Atomic sub and get</fsummary> + <desc> + <p>Atomic subtraction and return of the result.</p> + </desc> + </func> + + <func> + <name name="exchange" arity="3"/> + <fsummary>Atomic exchange.</fsummary> + <desc> + <p>Atomically replaces the value of the atomic with + <c><anno>Desired</anno></c> and returns the value it held + previously.</p> + </desc> + </func> + + <func> + <name name="compare_exchange" arity="4"/> + <fsummary>Atomic compare and exchange.</fsummary> + <desc> + <p>Atomically compares the atomic with <c><anno>Expected</anno></c>, + and if those are equal, set atomic to <c><anno>Desired</anno></c>. + Returns <c>ok</c> if <c><anno>Desired</anno></c> was written. Returns + the actual atomic value if not equal to <c><anno>Expected</anno></c>.</p> + </desc> + </func> + + <func> + <name name="info" arity="1"/> + <fsummary>Get information about atomic array.</fsummary> + <desc> + <p>Return information about an atomic array in a map. The map + has the following keys:</p> + <taglist> + <tag><c>size</c></tag> + <item><p>The number of atomics in the array.</p></item> + <tag><c>max</c></tag> + <item><p>The highest possible value an atomic in this array can + hold.</p></item> + <tag><c>min</c></tag> + <item><p>The lowest possible value an atomic in this array can + hold.</p></item> + <tag><c>memory</c></tag> + <item><p>Approximate memory consumption for the array in + bytes.</p></item> + </taglist> + </desc> + </func> + + </funcs> +</erlref> diff --git a/erts/doc/src/counters.xml b/erts/doc/src/counters.xml new file mode 100644 index 0000000000..ba4a22759f --- /dev/null +++ b/erts/doc/src/counters.xml @@ -0,0 +1,170 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE erlref SYSTEM "erlref.dtd"> + +<erlref> + <header> + <copyright> + <year>2018</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + </legalnotice> + + <title>counters</title> + </header> + <module>counters</module> + <modulesummary>Counter Functions</modulesummary> + <description> + <p>This module provides a set of functions to do operations towards + shared mutable counter variables. The implementation does not utilize any + software level locking, which makes it very efficient for concurrent + access. The counters are organized into arrays with the follwing + semantics:</p> + <list type="bulleted"> + <item> + <p>Counters are 64 bit signed integers.</p> + </item> + <item> + <p>Counters wrap around at overflow and underflow operations.</p> + </item> + <item><p>Counters are initialized to zero and can then only be written to + by adding or subtracting.</p> + </item> + <item> + <p>Write operations guarantee atomicity. No intermediate results can be + seen from a single write operation.</p> + </item> + <item> + <p>Two types of counter arrays can be created with options <c>atomics</c> or + <c>write_concurrency</c>. The <c>atomics</c> counters have good allround + performance with nice consistent semantics while + <c>write_concurrency</c> counters offers even better concurrent + write performance at the expense of some potential read + inconsistencies. See <seealso marker="#new/2"><c>new/2</c></seealso>.</p> + </item> + <item> + <p>Indexes into counter arrays are one-based. A counter array of + size N contains N counters with index from 1 to N.</p> + </item> + </list> + </description> + + <datatypes> + <datatype> + <name name="counters_ref"/> + <desc><p>Identifies a counter array returned from + <seealso marker="#new/2"><c>new/2</c></seealso>.</p> + </desc> + </datatype> + </datatypes> + + <funcs> + <func> + <name name="new" arity="2"/> + <fsummary>Create counter array</fsummary> + <desc> + <p>Create a new counter array of <c><anno>Size</anno></c> counters.</p> + <p>Argument <c><anno>Opts</anno></c> is a list of the following possible + options:</p> + <taglist> + <tag><c>atomics</c> (Default)</tag> + <item><p>Counters will be sequentially consistent. If write + operation A is done sequencially before write operation B, then a concurrent reader + may see none of them, only A, or both A and B. It cannot see only B.</p> + </item> + <tag><c>write_concurrency</c></tag> + <item><p>This is an optimization to achieve very efficient concurrent + <seealso marker="#add/3"><c>add</c></seealso> and <seealso + marker="#sub/3"><c>sub</c></seealso> operations at the expense of potential read + inconsistency and memory consumption per counter.</p> + <p>Read operations may see sequentially inconsistent results with + regard to concurrent write operations. Even if write operation A is done + sequencially before write operation B, a concurrent reader may see any + combination of A and B, including only B. A read operation is only + guaranteed to see all writes done sequentially before the read. No writes + are ever lost, but will eventually all be seen.</p> + <p>The typical use case for <c>write_concurrency</c> is when + concurrent calls to <seealso marker="#add/3"><c>add</c></seealso> and + <seealso marker="#sub/3"><c>sub</c></seealso> toward the same counters + are very frequent, while calls to <seealso marker="#get/2"><c>get</c> + </seealso> and <seealso marker="#put/3"><c>put</c></seealso> are much + less frequent. The lack of absolute read consistency must also be + acceptable.</p> + </item> + </taglist> + </desc> + </func> + + <func> + <name name="get" arity="2"/> + <fsummary>Read counter value</fsummary> + <desc> + <p>Read counter value.</p> + </desc> + </func> + + <func> + <name name="add" arity="3"/> + <fsummary>Add to counter</fsummary> + <desc> + <p>Add <c><anno>Incr</anno></c> to counter at index + <c><anno>Ix</anno></c>.</p> + </desc> + </func> + + <func> + <name name="sub" arity="3"/> + <fsummary>Subtract from counter</fsummary> + <desc> + <p>Subtract <c><anno>Decr</anno></c> from counter at index + <c><anno>Ix</anno></c>.</p> + </desc> + </func> + + <func> + <name name="put" arity="3"/> + <fsummary>Set counter to value</fsummary> + <desc> + <p>Write <c><anno>Value</anno></c> to counter at index + <c><anno>Ix</anno></c>.</p> + <note> + <p>Despite its name, the <c>write_concurrency</c> optimization does not + improve <c>put</c>. A call to <c>put</c> is a relative heavy + operation compared to the very lightweight and scalable <seealso + marker="#add/3"><c>add</c></seealso> and <seealso marker="#sub/3"> + <c>sub</c></seealso>. The cost for a <c>put</c> with + <c>write_concurrency</c> is lika a <seealso marker="#get/2"><c>get</c> + </seealso> plus a <c>put</c> without <c>write_concurrency</c>.</p> + </note> + </desc> + </func> + + <func> + <name name="info" arity="1"/> + <fsummary>Get information about counter array.</fsummary> + <desc> + <p>Return information about a counter array in a map. The map + has the following keys (at least):</p> + <taglist> + <tag><c>size</c></tag> + <item><p>The number of counters in the array.</p></item> + <tag><c>memory</c></tag> + <item><p>Approximate memory consumption for the array in + bytes.</p></item> + </taglist> + </desc> + </func> + + </funcs> +</erlref> diff --git a/erts/doc/src/erl_nif.xml b/erts/doc/src/erl_nif.xml index 190ec12d0e..095fc79bdf 100644 --- a/erts/doc/src/erl_nif.xml +++ b/erts/doc/src/erl_nif.xml @@ -293,7 +293,7 @@ return term;</code> arguments. When you write to a shared state either through static variables or <seealso marker="#enif_priv_data"> <c>enif_priv_data</c></seealso>, you need to supply your own explicit - synchronization. This includes terms in process-independent + synchronization. This includes terms in process independent environments that are shared between threads. Resource objects also require synchronization if you treat them as mutable.</p> <p>The library initialization callbacks <c>load</c> and @@ -596,7 +596,7 @@ int writeiovec(ErlNifEnv *env, ERL_NIF_TERM term, ERL_NIF_TERM *tail, <c>--enable-static-nifs</c>, you must define <c>STATIC_ERLANG_NIF</c> before the <c>ERL_NIF_INIT</c> declaration.</p> </item> - <tag><marker id="load"/><c>int (*load)(ErlNifEnv* env, void** priv_data, + <tag><marker id="load"/><c>int (*load)(ErlNifEnv* caller_env, void** priv_data, ERL_NIF_TERM load_info)</c></tag> <item> <p><c>load</c> is called when the NIF library is loaded @@ -612,7 +612,7 @@ int writeiovec(ErlNifEnv *env, ERL_NIF_TERM term, ERL_NIF_TERM *tail, anything other than <c>0</c>. <c>load</c> can be <c>NULL</c> if initialization is not needed.</p> </item> - <tag><marker id="upgrade"/><c>int (*upgrade)(ErlNifEnv* env, void** + <tag><marker id="upgrade"/><c>int (*upgrade)(ErlNifEnv* caller_env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info)</c></tag> <item> <p><c>upgrade</c> is called when the NIF library is loaded @@ -626,7 +626,7 @@ int writeiovec(ErlNifEnv *env, ERL_NIF_TERM term, ERL_NIF_TERM *tail, <p>The library fails to load if <c>upgrade</c> returns anything other than <c>0</c> or if <c>upgrade</c> is <c>NULL</c>.</p> </item> - <tag><marker id="unload"/><c>void (*unload)(ErlNifEnv* env, void* + <tag><marker id="unload"/><c>void (*unload)(ErlNifEnv* caller_env, void* priv_data)</c></tag> <item> <p><c>unload</c> is called when the module code that @@ -654,27 +654,41 @@ int writeiovec(ErlNifEnv *env, ERL_NIF_TERM term, ERL_NIF_TERM *tail, <p><c>ErlNifEnv</c> represents an environment that can host Erlang terms. All terms in an environment are valid as long as the environment is valid. <c>ErlNifEnv</c> is an opaque type; pointers to - it can only be passed on to API functions. Two types of environments + it can only be passed on to API functions. Three types of environments exist:</p> <taglist> - <tag>Process-bound environment</tag> + <tag>Process bound environment</tag> <item> <p>Passed as the first argument to all NIFs. All function arguments passed to a NIF belong to that environment. The return value from a NIF must also be a term belonging to the same environment.</p> - <p>A process-bound environment contains transient information + <p>A process bound environment contains transient information about the calling Erlang process. The environment is only valid in the thread where it was supplied as argument until the NIF returns. It is thus useless and dangerous to store pointers to - process-bound environments between NIF calls.</p> + process bound environments between NIF calls.</p> </item> - <tag>Process-independent environment</tag> + <tag>Callback environment</tag> + <item> + <p>Passed as the first argument to all the non-NIF callback functions + (<seealso marker="#load"><c>load</c></seealso>, + <seealso marker="#upgrade"><c>upgrade</c></seealso>, + <seealso marker="#unload"><c>unload</c></seealso>, + <seealso marker="#ErlNifResourceDtor"><c>dtor</c></seealso>, + <seealso marker="#ErlNifResourceDown"><c>down</c></seealso> and + <seealso marker="#ErlNifResourceStop"><c>stop</c></seealso>). + Works like a process bound environment but with a temporary + pseudo process that "terminates" when the callback has + returned. Terms may be created in this environment but they will + only be accessible during the callback.</p> + </item> + <tag>Process independent environment</tag> <item> <p>Created by calling <seealso marker="#enif_alloc_env"> <c>enif_alloc_env</c></seealso>. This environment can be used to store terms between NIF calls and to send terms with <seealso marker="#enif_send"><c>enif_send</c></seealso>. A - process-independent environment with all its terms is valid until + process independent environment with all its terms is valid until you explicitly invalidate it with <seealso marker="#enif_free_env"><c>enif_free_env</c></seealso> or <c>enif_send</c>.</p> @@ -799,7 +813,7 @@ typedef struct { <tag><marker id="ErlNifResourceDtor"/><c>ErlNifResourceDtor</c></tag> <item> <code type="none"> -typedef void ErlNifResourceDtor(ErlNifEnv* env, void* obj);</code> +typedef void ErlNifResourceDtor(ErlNifEnv* caller_env, void* obj);</code> <p>The function prototype of a resource destructor function.</p> <p>The <c>obj</c> argument is a pointer to the resource. The only allowed use for the resource in the destructor is to access its @@ -809,7 +823,7 @@ typedef void ErlNifResourceDtor(ErlNifEnv* env, void* obj);</code> <tag><marker id="ErlNifResourceDown"/><c>ErlNifResourceDown</c></tag> <item> <code type="none"> -typedef void ErlNifResourceDown(ErlNifEnv* env, void* obj, ErlNifPid* pid, ErlNifMonitor* mon);</code> +typedef void ErlNifResourceDown(ErlNifEnv* caller_env, void* obj, ErlNifPid* pid, ErlNifMonitor* mon);</code> <p>The function prototype of a resource down function, called on the behalf of <seealso marker="#enif_monitor_process"> enif_monitor_process</seealso>. <c>obj</c> is the resource, <c>pid</c> @@ -820,7 +834,7 @@ typedef void ErlNifResourceDown(ErlNifEnv* env, void* obj, ErlNifPid* pid, ErlNi <tag><marker id="ErlNifResourceStop"/><c>ErlNifResourceStop</c></tag> <item> <code type="none"> -typedef void ErlNifResourceStop(ErlNifEnv* env, void* obj, ErlNifEvent event, int is_direct_call);</code> +typedef void ErlNifResourceStop(ErlNifEnv* caller_env, void* obj, ErlNifEvent event, int is_direct_call);</code> <p>The function prototype of a resource stop function, called on the behalf of <seealso marker="#enif_select"> enif_select</seealso>. <c>obj</c> is the resource, <c>event</c> is OS event, @@ -987,7 +1001,7 @@ typedef struct { <name><ret>ErlNifEnv *</ret><nametext>enif_alloc_env()</nametext></name> <fsummary>Create a new environment.</fsummary> <desc> - <p>Allocates a new process-independent environment. The environment can + <p>Allocates a new process independent environment. The environment can be used to hold terms that are not bound to any process. Such terms can later be copied to a process environment with <seealso marker="#enif_make_copy"><c>enif_make_copy</c></seealso> or @@ -1211,14 +1225,17 @@ typedef struct { </func> <func> - <name><ret>int</ret><nametext>enif_demonitor_process(ErlNifEnv* env, void* obj, + <name><ret>int</ret><nametext>enif_demonitor_process(ErlNifEnv* caller_env, void* obj, const ErlNifMonitor* mon)</nametext></name> <fsummary>Cancel a process monitor.</fsummary> <desc> <marker id="enif_demonitor_process"></marker> <p>Cancels a monitor created earlier with <seealso marker="#enif_monitor_process"> <c>enif_monitor_process</c></seealso>. Argument <c>obj</c> is a pointer - to the resource holding the monitor and <c>*mon</c> identifies the monitor.</p> + to the resource holding the monitor and <c>*mon</c> identifies the + monitor.</p> + <p>Argument <c>caller_env</c> is the environment of the calling process + or callback. Must only be NULL if calling from a custom thread.</p> <p>Returns <c>0</c> if the monitor was successfully identified and removed. Returns a non-zero value if the monitor could not be identified, which means it was either</p> @@ -2572,7 +2589,7 @@ enif_map_iterator_destroy(env, &iter);</code> </func> <func> - <name><ret>int</ret><nametext>enif_monitor_process(ErlNifEnv* env, void* obj, + <name><ret>int</ret><nametext>enif_monitor_process(ErlNifEnv* caller_env, void* obj, const ErlNifPid* target_pid, ErlNifMonitor* mon)</nametext></name> <fsummary>Monitor a process from a resource.</fsummary> <desc> @@ -2593,6 +2610,8 @@ enif_map_iterator_destroy(env, &iter);</code> <seealso marker="#enif_compare_monitors"><c>enif_compare_monitors</c></seealso>. A monitor is automatically removed when it triggers or when the resource is deallocated.</p> + <p>Argument <c>caller_env</c> is the environment of the calling process + or callback. Must only be NULL if calling from a custom thread.</p> <p>Returns <c>0</c> on success, < 0 if no <c>down</c> callback is provided, and > 0 if the process is no longer alive.</p> <p>This function is only thread-safe when the emulator with SMP support @@ -2768,7 +2787,7 @@ enif_map_iterator_destroy(env, &iter);</code> <item>The port ID of the receiving port. The port ID is to refer to a port on the local node.</item> <tag><c>msg_env</c></tag> - <item>The environment of the message term. Can be a process-independent + <item>The environment of the message term. Can be a process independent environment allocated with <seealso marker="#enif_alloc_env"> <c>enif_alloc_env</c></seealso> or <c>NULL</c>.</item> <tag><c>msg</c></tag> @@ -3124,26 +3143,26 @@ if (retval & ERL_NIF_SELECT_STOP_CALLED) { <p>Initializes the <seealso marker="#ErlNifPid"><c>ErlNifPid</c></seealso> variable at <c>*pid</c> to represent the calling process.</p> <p>Returns <c>pid</c> if successful, or NULL if <c>caller_env</c> is not - a <seealso marker="#ErlNifEnv">process-bound environment</seealso>.</p> + a <seealso marker="#ErlNifEnv">process bound environment</seealso>.</p> </desc> </func> <func> - <name><ret>int</ret><nametext>enif_send(ErlNifEnv* env, ErlNifPid* to_pid, + <name><ret>int</ret><nametext>enif_send(ErlNifEnv* caller_env, ErlNifPid* to_pid, ErlNifEnv* msg_env, ERL_NIF_TERM msg)</nametext></name> <fsummary>Send a message to a process.</fsummary> <desc> <p>Sends a message to a process.</p> <taglist> - <tag><c>env</c></tag> - <item>The environment of the calling process. Must be <c>NULL</c> - only if calling from a created thread.</item> + <tag><c>caller_env</c></tag> + <item>The environment of the calling process or callback. Must be <c>NULL</c> + only if calling from a custom thread not spawned by ERTS.</item> <tag><c>*to_pid</c></tag> <item>The pid of the receiving process. The pid is to refer to a process on the local node.</item> <tag><c>msg_env</c></tag> <item>The environment of the message term. Must be a - process-independent environment allocated with + process independent environment allocated with <seealso marker="#enif_alloc_env"><c>enif_alloc_env</c></seealso> or NULL.</item> <tag><c>msg</c></tag> diff --git a/erts/doc/src/notes.xml b/erts/doc/src/notes.xml index 533e1d5eb7..2a823d9fe7 100644 --- a/erts/doc/src/notes.xml +++ b/erts/doc/src/notes.xml @@ -31,6 +31,42 @@ </header> <p>This document describes the changes made to the ERTS application.</p> +<section><title>Erts 10.1.3</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p>Added an optional <c>./configure</c> flag to compile + the emulator with spectre mitigation: + <c>--with-spectre-mitigation</c></p> + <p>Note that this requires a recent version of GCC with + support for spectre mitigation and the + <c>--mindirect-branch=thunk</c> flag, such as + <c>8.1</c>.</p> + <p> + Own Id: OTP-15430 Aux Id: ERIERL-237 </p> + </item> + </list> + </section> + +</section> + +<section><title>Erts 10.1.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p>Fixed a rare bug where files could be closed on a + normal instead of an IO scheduler, resulting in system + instability if the operation blocked.</p> + <p> + Own Id: OTP-15421</p> + </item> + </list> + </section> + +</section> + <section><title>Erts 10.1.1</title> <section><title>Fixed Bugs and Malfunctions</title> @@ -1355,6 +1391,21 @@ </section> +<section><title>Erts 9.3.3.6</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p>List subtraction (The <c>--</c> operator) will now + yield properly on large inputs.</p> + <p> + Own Id: OTP-15371</p> + </item> + </list> + </section> + +</section> + <section><title>Erts 9.3.3.5</title> <section><title>Fixed Bugs and Malfunctions</title> @@ -4799,6 +4850,21 @@ </section> +<section><title>Erts 7.3.1.6</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p>List subtraction (The <c>--</c> operator) will now + yield properly on large inputs.</p> + <p> + Own Id: OTP-15371</p> + </item> + </list> + </section> + +</section> + <section><title>Erts 7.3.1.5</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/erts/doc/src/persistent_term.xml b/erts/doc/src/persistent_term.xml new file mode 100644 index 0000000000..29a6c67051 --- /dev/null +++ b/erts/doc/src/persistent_term.xml @@ -0,0 +1,290 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE erlref SYSTEM "erlref.dtd"> + +<erlref> + <header> + <copyright> + <year>2018</year><year>2018</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + </legalnotice> + + <title>persistent_term</title> + <prepared></prepared> + <docno></docno> + <date></date> + <rev></rev> + <file>persistent_term.xml</file> + </header> + <module>persistent_term</module> + <modulesummary>Persistent terms.</modulesummary> + <description> + <p>This module is similar to <seealso + marker="stdlib:ets"><c>ets</c></seealso> in that it provides a + storage for Erlang terms that can be accessed in constant time, + but with the difference that <c>persistent_term</c> has been + highly optimized for reading terms at the expense of writing and + updating terms. When a persistent term is updated or deleted, a + global garbage collection pass is run to scan all processes for + the deleted term, and to copy it into each process that still uses + it. Therefore, <c>persistent_term</c> is suitable for storing + Erlang terms that are frequently accessed but never or + infrequently updated.</p> + + <warning><p>Persistent terms is an advanced feature and is not a + general replacement for ETS tables. Before using persistent terms, + make sure to fully understand the consequence to system + performance when updating or deleting persistent terms.</p></warning> + + <p>Term lookup (using <seealso + marker="#get/1"><c>get/1</c></seealso>), is done in constant time + and without taking any locks, and the term is <strong>not</strong> + copied to the heap (as is the case with terms stored in ETS + tables).</p> + + <p>Storing or updating a term (using <seealso + marker="#put/2"><c>put/2</c></seealso>) is proportional to the + number of already created persistent terms because the hash table + holding the keys will be copied. In addition, the term itself will + be copied.</p> + + <p>When a (complex) term is deleted (using <seealso + marker="#erase/1"><c>erase/1</c></seealso>) or replaced by another + (using <seealso marker="#put/2"><c>put/2</c></seealso>), a global + garbage collection is initiated. It works like this:</p> + + <list> + <item><p>All processes in the system will be scheduled to run a + scan of their heaps for the term that has been deleted. While + such scan is relatively light-weight, if there are many + processes, the system can become less responsive until all + process have scanned their heaps.</p></item> + + <item><p>If the deleted term (or any part of it) is still used + by a process, that process will do a major (fullsweep) garbage + collection and copy the term into the process. However, at most + two processes at a time will be scheduled to do that kind of + garbage collection.</p></item> + </list> + + <p>Deletion of atoms and other terms that fit in one machine word + is specially optimized to avoid doing a global GC. It is still not + recommended to update persistent terms with such values too + frequently because the hash table holding the keys is copied every + time a persistent term is updated.</p> + + <p>Some examples are suitable uses for persistent terms are:</p> + + <list> + <item><p>Storing of configuration data that must be easily + accessible by all processes.</p></item> + + <item><p>Storing of references for NIF resources.</p></item> + + <item><p>Storing of references for efficient counters.</p></item> + + <item><p>Storing an atom to indicate a logging level or whether debugging + is turned on.</p></item> + </list> + + </description> + + <section> + <title>Storing Huge Persistent Terms</title> + <p>The current implementation of persistent terms uses the literal + <seealso marker="erts_alloc">allocator</seealso> also used for + literals (constant terms) in BEAM code. By default, 1 GB of + virtual address space is reserved for literals in BEAM code and + persistent terms. The amount of virtual address space reserved for + literals can be changed by using the <seealso + marker="erts_alloc#MIscs"><c>+MIscs option</c></seealso> when + starting the emulator.</p> + + <p>Here is an example how the reserved virtual address space for literals + can be raised to 2 GB (2048 MB):</p> + + <pre> + erl +MIscs 2048</pre> + </section> + + <section> + <title>Warning For Many Persistent Terms</title> + <p>The runtime system will send a warning report to the + error logger if more than 20000 persistent terms have been + created. It will look like this:</p> + +<pre> +More than 20000 persistent terms have been created. +It is recommended to avoid creating an excessive number of +persistent terms, as creation and deletion of persistent terms +will be slower as the number of persistent terms increases.</pre> + </section> + + <section> + <title>Best Practices for Using Persistent Terms</title> + + <p>It is recommended to use keys like <c>?MODULE</c> or + <c>{?MODULE,SubKey}</c> to avoid name collisions.</p> + + <p>Prefer creating a few large persistent terms to creating many + small persistent terms. The execution time for storing a + persistent term is proportional to the number of already existing + terms.</p> + + <p>Updating a persistent term with the same value as it already + has is specially optimized to do nothing quickly; thus, there is + no need compare the old and new values and avoid calling + <seealso marker="#put/2"><c>put/2</c></seealso> if the values + are equal.</p> + + <p>When atoms or other terms that fit in one machine word are + deleted, no global GC is needed. Therefore, persistent terms that + have atoms as their values can be updated more frequently, but + note that updating such persistent terms is still much more + expensive than reading them.</p> + + <p>Updating or deleting a persistent term will trigger a global GC + if the term does not fit in one machine word. Processes will be + scheduled as usual, but all processes will be made runnable at + once, which will make the system less responsive until all process + have run and scanned their heaps for the deleted terms. One way to + minimize the effects on responsiveness could be to minimize the + number of processes on the node before updating or deleting a + persistent term. It would also be wise to avoid updating terms + when the system is at peak load.</p> + + <p>Avoid storing a retrieved persistent term in a process if that + persistent term could be deleted or updated in the future. If a + process holds a reference to a persistent term when the term is + deleted, the process will be garbage collected and the term copied + to process.</p> + + <p>Avoid updating or deleting more than one persistent term at a + time. Each deleted term will trigger its own global GC. That + means that deleting N terms will make the system less responsive N + times longer than deleting a single persistent term. Therefore, + terms that are to be updated at the same time should be collected + into a larger term, for example, a map or a tuple.</p> + </section> + + <section> + <title>Example</title> + + <p>The following example shows how lock contention for ETS tables + can be minimized by having one ETS table for each scheduler. The + table identifiers for the ETS tables are stored as a single + persistent term:</p> + +<pre> + %% There is one ETS table for each scheduler. + Sid = erlang:system_info(scheduler_id), + Tid = element(Sid, persistent_term:get(?MODULE)), + ets:update_counter(Tid, Key, 1).</pre> + + </section> + + <datatypes> + <datatype> + <name name="key"/> + <desc> + <p>Any Erlang term.</p> + </desc> + </datatype> + <datatype> + <name name="value"/> + <desc> + <p>Any Erlang term.</p> + </desc> + </datatype> + </datatypes> + + <funcs> + <func> + <name name="erase" arity="1"/> + <fsummary>Erase the name for a persistent term.</fsummary> + <desc> + <p>Erase the name for the persistent term with key + <c><anno>Key</anno></c>. The return value will be <c>true</c> + if there was a persistent term with the key + <c><anno>Key</anno></c>, and <c>false</c> if there was no + persistent term associated with the key.</p> + <p>If there existed a previous persistent term associated with + key <c><anno>Key</anno></c>, a global GC has been initiated + when <c>erase/1</c> returns. See <seealso + marker="#description">Description</seealso>.</p> + </desc> + </func> + + <func> + <name name="get" arity="0"/> + <fsummary>Get all persistent terms.</fsummary> + <desc> + <p>Retrieve the keys and values for all persistent terms. + The keys will be copied to the heap for the process calling + <c>get/0</c>, but the values will not.</p> + </desc> + </func> + + <func> + <name name="get" arity="1"/> + <fsummary>Get the value for a persistent term.</fsummary> + <desc> + <p>Retrieve the value for the persistent term associated with + the key <c><anno>Key</anno></c>. The lookup will be made in + constant time and the value will not be copied to the heap + of the calling process.</p> + <p>This function fails with a <c>badarg</c> exception if no + term has been stored with the key + <c><anno>Key</anno></c>.</p> + <p>If the calling process holds on to the value of the + persistent term and the persistent term is deleted in the future, + the term will be copied to the process.</p> + </desc> + </func> + + <func> + <name name="info" arity="0"/> + <fsummary>Get information about persistent terms.</fsummary> + <desc> + <p>Return information about persistent terms in a map. The map + has the following keys:</p> + <taglist> + <tag><c>count</c></tag> + <item><p>The number of persistent terms.</p></item> + <tag><c>memory</c></tag> + <item><p>The total amount of memory (measured in bytes) + used by all persistent terms.</p></item> + </taglist> + </desc> + </func> + + <func> + <name name="put" arity="2"/> + <fsummary>Store a term.</fsummary> + <desc> + <p>Store the value <c><anno>Value</anno></c> as a persistent term and + associate it with the key <c><anno>Key</anno></c>.</p> + <p>If the value <c><anno>Value</anno></c> is equal to the value + previously stored for the key, <c>put/2</c> will do nothing and return + quickly.</p> + <p>If there existed a previous persistent term associated with + key <c><anno>Key</anno></c>, a global GC has been initiated + when <c>put/2</c> returns. See <seealso + marker="#description">Description</seealso>.</p> + </desc> + </func> + </funcs> +</erlref> diff --git a/erts/doc/src/ref_man.xml b/erts/doc/src/ref_man.xml index 0617463a7b..a78aaa449e 100644 --- a/erts/doc/src/ref_man.xml +++ b/erts/doc/src/ref_man.xml @@ -34,6 +34,7 @@ <xi:include href="erl_prim_loader.xml"/> <xi:include href="erlang.xml"/> <xi:include href="init.xml"/> + <xi:include href="persistent_term.xml"/> <xi:include href="zlib.xml"/> <xi:include href="epmd.xml"/> <xi:include href="erl.xml"/> @@ -49,5 +50,7 @@ <xi:include href="erts_alloc.xml"/> <xi:include href="erl_nif.xml"/> <xi:include href="erl_tracer.xml"/> + <xi:include href="atomics.xml"/> + <xi:include href="counters.xml"/> </application> diff --git a/erts/doc/src/specs.xml b/erts/doc/src/specs.xml index ed6be650e5..0b943e6295 100644 --- a/erts/doc/src/specs.xml +++ b/erts/doc/src/specs.xml @@ -4,5 +4,8 @@ <xi:include href="../specs/specs_erlang.xml"/> <xi:include href="../specs/specs_erl_tracer.xml"/> <xi:include href="../specs/specs_init.xml"/> + <xi:include href="../specs/specs_persistent_term.xml"/> <xi:include href="../specs/specs_zlib.xml"/> + <xi:include href="../specs/specs_atomics.xml"/> + <xi:include href="../specs/specs_counters.xml"/> </specs> diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index 054692819e..57a9d45887 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -633,21 +633,24 @@ GENERATE += $(TTF_DIR)/driver_tab.c # This list must be consistent with PRE_LOADED_MODULES in # erts/preloaded/src/Makefile. -PRELOAD_BEAM = $(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \ - $(ERL_TOP)/erts/preloaded/ebin/erts_code_purger.beam \ - $(ERL_TOP)/erts/preloaded/ebin/init.beam \ - $(ERL_TOP)/erts/preloaded/ebin/prim_buffer.beam \ - $(ERL_TOP)/erts/preloaded/ebin/prim_eval.beam \ - $(ERL_TOP)/erts/preloaded/ebin/prim_inet.beam \ - $(ERL_TOP)/erts/preloaded/ebin/prim_file.beam \ - $(ERL_TOP)/erts/preloaded/ebin/zlib.beam \ - $(ERL_TOP)/erts/preloaded/ebin/prim_zip.beam \ - $(ERL_TOP)/erts/preloaded/ebin/erl_prim_loader.beam \ - $(ERL_TOP)/erts/preloaded/ebin/erlang.beam \ - $(ERL_TOP)/erts/preloaded/ebin/erts_internal.beam \ - $(ERL_TOP)/erts/preloaded/ebin/erl_tracer.beam \ - $(ERL_TOP)/erts/preloaded/ebin/erts_literal_area_collector.beam \ - $(ERL_TOP)/erts/preloaded/ebin/erts_dirty_process_signal_handler.beam +PRELOAD_BEAM = $(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \ + $(ERL_TOP)/erts/preloaded/ebin/erts_code_purger.beam \ + $(ERL_TOP)/erts/preloaded/ebin/init.beam \ + $(ERL_TOP)/erts/preloaded/ebin/prim_buffer.beam \ + $(ERL_TOP)/erts/preloaded/ebin/prim_eval.beam \ + $(ERL_TOP)/erts/preloaded/ebin/prim_inet.beam \ + $(ERL_TOP)/erts/preloaded/ebin/prim_file.beam \ + $(ERL_TOP)/erts/preloaded/ebin/zlib.beam \ + $(ERL_TOP)/erts/preloaded/ebin/prim_zip.beam \ + $(ERL_TOP)/erts/preloaded/ebin/erl_prim_loader.beam \ + $(ERL_TOP)/erts/preloaded/ebin/erlang.beam \ + $(ERL_TOP)/erts/preloaded/ebin/erts_internal.beam \ + $(ERL_TOP)/erts/preloaded/ebin/erl_tracer.beam \ + $(ERL_TOP)/erts/preloaded/ebin/erts_literal_area_collector.beam \ + $(ERL_TOP)/erts/preloaded/ebin/erts_dirty_process_signal_handler.beam \ + $(ERL_TOP)/erts/preloaded/ebin/atomics.beam \ + $(ERL_TOP)/erts/preloaded/ebin/counters.beam \ + $(ERL_TOP)/erts/preloaded/ebin/persistent_term.beam ifeq ($(TARGET),win32) # On windows the preloaded objects are in a resource object. @@ -839,6 +842,8 @@ RUN_OBJS += \ $(OBJDIR)/erl_bif_ddll.o $(OBJDIR)/erl_bif_guard.o \ $(OBJDIR)/erl_bif_info.o $(OBJDIR)/erl_bif_op.o \ $(OBJDIR)/erl_bif_os.o $(OBJDIR)/erl_bif_lists.o \ + $(OBJDIR)/erl_bif_persistent.o \ + $(OBJDIR)/erl_bif_atomics.o $(OBJDIR)/erl_bif_counters.o \ $(OBJDIR)/erl_bif_trace.o $(OBJDIR)/erl_bif_unique.o \ $(OBJDIR)/erl_bif_wrap.o $(OBJDIR)/erl_nfunc_sched.o \ $(OBJDIR)/erl_guard_bifs.o $(OBJDIR)/erl_dirty_bif_wrap.o \ diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 45b7540aeb..a14f22b19e 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -182,6 +182,7 @@ atom control atom copy atom copy_literals atom counters +atom count atom cpu atom cpu_timestamp atom cr @@ -287,6 +288,7 @@ atom gc_minor_end atom gc_minor_start atom Ge='>=' atom generational +atom get_all_trap atom get_seq_token atom get_tcw atom gather_gc_info_result @@ -325,6 +327,7 @@ atom index atom infinity atom info atom info_msg +atom info_trap atom init atom initial_call atom input @@ -393,6 +396,7 @@ atom microsecond atom microstate_accounting atom milli_seconds atom millisecond +atom min atom min_heap_size atom min_bin_vheap_size atom minor diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index d221e6aea6..bb1b2e5b27 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -1752,29 +1752,7 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) finalize_purge_operation(BIF_P, ret == am_true); if (literals) { - ErtsLiteralAreaRef *ref; - ErtsMessage *mp; - ref = erts_alloc(ERTS_ALC_T_LITERAL_REF, - sizeof(ErtsLiteralAreaRef)); - ref->literal_area = literals; - ref->next = NULL; - erts_mtx_lock(&release_literal_areas.mtx); - if (release_literal_areas.last) { - release_literal_areas.last->next = ref; - release_literal_areas.last = ref; - } - else { - release_literal_areas.first = ref; - release_literal_areas.last = ref; - } - erts_mtx_unlock(&release_literal_areas.mtx); - mp = erts_alloc_message(0, NULL); - ERL_MESSAGE_TOKEN(mp) = am_undefined; - erts_queue_proc_message(BIF_P, - erts_literal_area_collector, - 0, - mp, - am_copy_literals); + erts_queue_release_literals(BIF_P, literals); } return ret; @@ -1786,6 +1764,41 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) } } +void +erts_queue_release_literals(Process* c_p, ErtsLiteralArea* literals) +{ + ErtsLiteralAreaRef *ref; + ErtsMessage *mp; + ref = erts_alloc(ERTS_ALC_T_LITERAL_REF, + sizeof(ErtsLiteralAreaRef)); + ref->literal_area = literals; + ref->next = NULL; + erts_mtx_lock(&release_literal_areas.mtx); + if (release_literal_areas.last) { + release_literal_areas.last->next = ref; + release_literal_areas.last = ref; + } else { + release_literal_areas.first = ref; + release_literal_areas.last = ref; + } + erts_mtx_unlock(&release_literal_areas.mtx); + mp = erts_alloc_message(0, NULL); + ERL_MESSAGE_TOKEN(mp) = am_undefined; + if (c_p == NULL) { + erts_queue_message(erts_literal_area_collector, + 0, + mp, + am_copy_literals, + am_system); + } else { + erts_queue_proc_message(c_p, + erts_literal_area_collector, + 0, + mp, + am_copy_literals); + } +} + /* * Move code from current to old and null all export entries for the module */ diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index ab5920a67e..e909a0b4da 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -579,6 +579,7 @@ init_emulator(void) * the instructions' C labels to the loader. * The second call starts execution of BEAM code. This call never returns. */ +ERTS_NO_RETPOLINE void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) { static int init_done = 0; diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 7548924178..d4ba90a61a 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -40,6 +40,7 @@ # Note: Guards BIFs usually require special support in the compiler. # + gcbif erlang:abs/1 bif erlang:adler32/1 bif erlang:adler32/2 @@ -698,3 +699,29 @@ ubif erlang:map_get/2 ubif erlang:is_map_key/2 bif ets:internal_delete_all/2 bif ets:internal_select_delete/2 + +# +# New in 21.2 +# + +bif persistent_term:put/2 +bif persistent_term:get/1 +bif persistent_term:get/0 +bif persistent_term:erase/1 +bif persistent_term:info/0 +bif erts_internal:erase_persistent_terms/0 + +bif erts_internal:atomics_new/2 +bif atomics:get/2 +bif atomics:put/3 +bif atomics:add/3 +bif atomics:add_get/3 +bif atomics:exchange/3 +bif atomics:compare_exchange/4 +bif atomics:info/1 + +bif erts_internal:counters_new/1 +bif erts_internal:counters_get/2 +bif erts_internal:counters_add/3 +bif erts_internal:counters_put/3 +bif erts_internal:counters_info/1 diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index e7bfd04b73..e7bd046e18 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -1074,6 +1074,7 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info) Eterm* ptr; Eterm *lit_purge_ptr = info->lit_purge_ptr; Uint lit_purge_sz = info->lit_purge_sz; + int copy_literals = info->copy_literals; #ifdef DEBUG Eterm mypid = erts_get_current_pid(); #endif @@ -1119,7 +1120,7 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info) /* off heap list pointers are copied verbatim */ if (erts_is_literal(obj,ptr)) { VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", mypid, ptr, obj)); - if (in_literal_purge_area(ptr)) + if (copy_literals || in_literal_purge_area(ptr)) info->literal_size += size_object(obj); goto pop_next; } @@ -1170,7 +1171,7 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info) /* off heap pointers to boxes are copied verbatim */ if (erts_is_literal(obj,ptr)) { VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", mypid, ptr, obj)); - if (in_literal_purge_area(ptr)) + if (copy_literals || in_literal_purge_area(ptr)) info->literal_size += size_object(obj); goto pop_next; } @@ -1338,6 +1339,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, unsigned remaining; Eterm *lit_purge_ptr = info->lit_purge_ptr; Uint lit_purge_sz = info->lit_purge_sz; + int copy_literals = info->copy_literals; #ifdef DEBUG Eterm mypid = erts_get_current_pid(); Eterm saved_obj = obj; @@ -1387,7 +1389,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, ptr = list_val(obj); /* off heap list pointers are copied verbatim */ if (erts_is_literal(obj,ptr)) { - if (!in_literal_purge_area(ptr)) { + if (!(copy_literals || in_literal_purge_area(ptr))) { *resp = obj; } else { Uint bsz = 0; @@ -1455,7 +1457,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, ptr = boxed_val(obj); /* off heap pointers to boxes are copied verbatim */ if (erts_is_literal(obj,ptr)) { - if (!in_literal_purge_area(ptr)) { + if (!(copy_literals || in_literal_purge_area(ptr))) { *resp = obj; } else { Uint bsz = 0; diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 98ba0e5f07..4f03a34390 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -278,6 +278,9 @@ type LIST_TRAP SHORT_LIVED PROCESSES list_bif_trap_state type ENVIRONMENT SYSTEM SYSTEM environment +type PERSISTENT_TERM LONG_LIVED CODE persisten_term +type PERSISTENT_LOCK_Q SHORT_LIVED SYSTEM persistent_lock_q + # # Types used for special emulators # @@ -335,6 +338,8 @@ type GC_INFO_REQ SHORT_LIVED SYSTEM gc_info_request type PORT_DATA_HEAP STANDARD SYSTEM port_data_heap type MSACC DRIVER SYSTEM microstate_accounting type SYS_CHECK_REQ SHORT_LIVED SYSTEM system_check_request +type ATOMICS STANDARD SYSTEM erl_bif_atomics +type COUNTERS STANDARD SYSTEM erl_bif_counters # # Types used by system specific code diff --git a/erts/emulator/beam/erl_bif_atomics.c b/erts/emulator/beam/erl_bif_atomics.c new file mode 100644 index 0000000000..092dbb3bd3 --- /dev/null +++ b/erts/emulator/beam/erl_bif_atomics.c @@ -0,0 +1,256 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +/* + * Purpose: High performance atomics. + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include <stddef.h> /* offsetof */ + +#include "sys.h" +#include "export.h" +#include "bif.h" +#include "erl_threads.h" +#include "big.h" +#include "erl_binary.h" +#include "erl_bif_unique.h" +#include "erl_map.h" + +typedef struct +{ + int is_signed; + UWord vlen; + erts_atomic64_t v[1]; +}AtomicsRef; + +static int atomics_destructor(Binary *unused) +{ + return 1; +} + +#define OPT_SIGNED (1 << 0) + +BIF_RETTYPE erts_internal_atomics_new_2(BIF_ALIST_2) +{ + AtomicsRef* p; + Binary* mbin; + UWord i, cnt, opts; + Uint bytes; + Eterm* hp; + + if (!term_to_UWord(BIF_ARG_1, &cnt) + || cnt == 0 + || !term_to_UWord(BIF_ARG_2, &opts)) { + + BIF_ERROR(BIF_P, BADARG); + } + + if (cnt > (ERTS_UWORD_MAX / sizeof(p->v[0]))) + BIF_ERROR(BIF_P, SYSTEM_LIMIT); + + bytes = offsetof(AtomicsRef, v) + cnt*sizeof(p->v[0]); + mbin = erts_create_magic_binary_x(bytes, + atomics_destructor, + ERTS_ALC_T_ATOMICS, + 0); + p = ERTS_MAGIC_BIN_DATA(mbin); + p->is_signed = opts & OPT_SIGNED; + p->vlen = cnt; + for (i=0; i < cnt; i++) + erts_atomic64_init_nob(&p->v[i], 0); + hp = HAlloc(BIF_P, ERTS_MAGIC_REF_THING_SIZE); + return erts_mk_magic_ref(&hp, &MSO(BIF_P), mbin); +} + +static ERTS_INLINE int get_ref(Eterm ref, AtomicsRef** pp) +{ + Binary* mbin; + if (!is_internal_magic_ref(ref)) + return 0; + + mbin = erts_magic_ref2bin(ref); + if (ERTS_MAGIC_BIN_DESTRUCTOR(mbin) != atomics_destructor) + return 0; + *pp = ERTS_MAGIC_BIN_DATA(mbin); + return 1; +} + +static ERTS_INLINE int get_ref_ix(Eterm ref, Eterm ix, + AtomicsRef** pp, UWord* ixp) +{ + return (get_ref(ref, pp) + && term_to_UWord(ix, ixp) + && --(*ixp) < (*pp)->vlen); +} + +static ERTS_INLINE int get_value(AtomicsRef* p, Eterm term, erts_aint64_t *valp) +{ + return (p->is_signed ? + term_to_Sint64(term, (Sint64*)valp) : + term_to_Uint64(term, (Uint64*)valp)); +} + +static ERTS_INLINE int get_incr(AtomicsRef* p, Eterm term, erts_aint64_t *valp) +{ + return (term_to_Sint64(term, (Sint64*)valp) + || term_to_Uint64(term, (Uint64*)valp)); +} + +static ERTS_INLINE Eterm bld_atomic(Process* proc, AtomicsRef* p, + erts_aint64_t val) +{ + if (p->is_signed) { + if (IS_SSMALL(val)) + return make_small((Sint) val); + else { + Uint hsz = ERTS_SINT64_HEAP_SIZE(val); + Eterm* hp = HAlloc(proc, hsz); + return erts_sint64_to_big(val, &hp); + } + } + else { + if ((Uint64)val <= MAX_SMALL) + return make_small((Sint) val); + else { + Uint hsz = ERTS_UINT64_HEAP_SIZE(val); + Eterm* hp = HAlloc(proc, hsz); + return erts_uint64_to_big(val, &hp); + } + } +} + +BIF_RETTYPE atomics_put_3(BIF_ALIST_3) +{ + AtomicsRef* p; + UWord ix; + erts_aint64_t val; + + if (!get_ref_ix(BIF_ARG_1, BIF_ARG_2, &p, &ix) + || !get_value(p, BIF_ARG_3, &val)) { + BIF_ERROR(BIF_P, BADARG); + } + erts_atomic64_set_mb(&p->v[ix], val); + return am_ok; +} + +BIF_RETTYPE atomics_get_2(BIF_ALIST_2) +{ + AtomicsRef* p; + UWord ix; + + if (!get_ref_ix(BIF_ARG_1, BIF_ARG_2, &p, &ix)) { + BIF_ERROR(BIF_P, BADARG); + } + return bld_atomic(BIF_P, p, erts_atomic64_read_mb(&p->v[ix])); +} + +BIF_RETTYPE atomics_add_3(BIF_ALIST_3) +{ + AtomicsRef* p; + UWord ix; + erts_aint64_t incr; + + if (!get_ref_ix(BIF_ARG_1, BIF_ARG_2, &p, &ix) + || !get_incr(p, BIF_ARG_3, &incr)) { + BIF_ERROR(BIF_P, BADARG); + } + erts_atomic64_add_mb(&p->v[ix], incr); + return am_ok; +} + +BIF_RETTYPE atomics_add_get_3(BIF_ALIST_3) +{ + AtomicsRef* p; + UWord ix; + erts_aint64_t incr; + + if (!get_ref_ix(BIF_ARG_1, BIF_ARG_2, &p, &ix) + || !get_incr(p, BIF_ARG_3, &incr)) { + BIF_ERROR(BIF_P, BADARG); + } + return bld_atomic(BIF_P, p, erts_atomic64_add_read_mb(&p->v[ix], incr)); +} + +BIF_RETTYPE atomics_exchange_3(BIF_ALIST_3) +{ + AtomicsRef* p; + UWord ix; + erts_aint64_t desired, was; + + if (!get_ref_ix(BIF_ARG_1, BIF_ARG_2, &p, &ix) + || !get_value(p, BIF_ARG_3, &desired)) { + BIF_ERROR(BIF_P, BADARG); + } + was = erts_atomic64_xchg_mb(&p->v[ix], desired); + return bld_atomic(BIF_P, p, was); +} + +BIF_RETTYPE atomics_compare_exchange_4(BIF_ALIST_4) +{ + AtomicsRef* p; + UWord ix; + erts_aint64_t expected, desired, was; + + if (!get_ref_ix(BIF_ARG_1, BIF_ARG_2, &p, &ix) + || !get_value(p, BIF_ARG_3, &expected) + || !get_value(p, BIF_ARG_4, &desired)) { + BIF_ERROR(BIF_P, BADARG); + } + was = erts_atomic64_cmpxchg_mb(&p->v[ix], desired, expected); + return was == expected ? am_ok : bld_atomic(BIF_P, p, was); +} + +BIF_RETTYPE atomics_info_1(BIF_ALIST_1) +{ + AtomicsRef* p; + Uint hsz = MAP4_SZ; + Eterm *hp; + Uint64 max; + Sint64 min; + UWord memory; + Eterm max_val, min_val, sz_val, mem_val; + + if (!get_ref(BIF_ARG_1, &p)) + BIF_ERROR(BIF_P, BADARG); + + max = p->is_signed ? ERTS_SINT64_MAX : ERTS_UINT64_MAX; + min = p->is_signed ? ERTS_SINT64_MIN : 0; + memory = erts_magic_ref2bin(BIF_ARG_1)->orig_size; + + erts_bld_uint64(NULL, &hsz, max); + erts_bld_sint64(NULL, &hsz, min); + erts_bld_uword(NULL, &hsz, p->vlen); + erts_bld_uword(NULL, &hsz, memory); + + hp = HAlloc(BIF_P, hsz); + max_val = erts_bld_uint64(&hp, NULL, max); + min_val = erts_bld_sint64(&hp, NULL, min); + sz_val = erts_bld_uword(&hp, NULL, p->vlen); + mem_val = erts_bld_uword(&hp, NULL, memory); + + return MAP4(hp, am_max, max_val, + am_memory, mem_val, + am_min, min_val, + am_size, sz_val); +} diff --git a/erts/emulator/beam/erl_bif_counters.c b/erts/emulator/beam/erl_bif_counters.c new file mode 100644 index 0000000000..7c8884ba32 --- /dev/null +++ b/erts/emulator/beam/erl_bif_counters.c @@ -0,0 +1,255 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +/* + * Purpose: The implementation for 'counters' with 'write_concurrency'. + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include <stddef.h> /* offsetof */ + +#include "sys.h" +#include "export.h" +#include "bif.h" +#include "erl_threads.h" +#include "big.h" +#include "erl_binary.h" +#include "erl_bif_unique.h" +#include "erl_map.h" + +/* + * Each logical counter consists of one 64-bit atomic instance per scheduler + * plus one instance for the "base value". + * + * get() reads all atomics for the counter and returns the sum. + * add() reads and writes only its own scheduler specific atomic instance. + * put() reads all scheduler specific atomics and writes a new base value. + */ +#define ATOMICS_PER_COUNTER (erts_no_schedulers + 1) + +#define ATOMICS_PER_CACHE_LINE (ERTS_CACHE_LINE_SIZE / sizeof(erts_atomic64_t)) + +typedef struct +{ + UWord arity; +#ifdef DEBUG + UWord ulen; +#endif + union { + erts_atomic64_t v[ATOMICS_PER_CACHE_LINE]; + byte cache_line__[ERTS_CACHE_LINE_SIZE]; + } u[1]; +}CountersRef; + +static int counters_destructor(Binary *mbin) +{ + return 1; +} + + +static UWord ERTS_INLINE div_ceil(UWord dividend, UWord divisor) +{ + return (dividend + divisor - 1) / divisor; +} + +BIF_RETTYPE erts_internal_counters_new_1(BIF_ALIST_1) +{ + CountersRef* p; + Binary* mbin; + UWord ui, vi, cnt; + Uint bytes, cache_lines; + Eterm* hp; + + if (!term_to_UWord(BIF_ARG_1, &cnt) + || cnt == 0) { + BIF_ERROR(BIF_P, BADARG); + } + + if (cnt > (ERTS_UWORD_MAX / (sizeof(erts_atomic64_t)*2*ATOMICS_PER_COUNTER))) + BIF_ERROR(BIF_P, SYSTEM_LIMIT); + + cache_lines = ATOMICS_PER_COUNTER * div_ceil(cnt, ATOMICS_PER_CACHE_LINE); + bytes = offsetof(CountersRef, u) + cache_lines * ERTS_CACHE_LINE_SIZE; + mbin = erts_create_magic_binary_x(bytes, + counters_destructor, + ERTS_ALC_T_ATOMICS, + 0); + p = ERTS_MAGIC_BIN_DATA(mbin); + p->arity = cnt; + +#ifdef DEBUG + p->ulen = cache_lines; +#endif + ASSERT((byte*)&p->u[cache_lines] <= ((byte*)p + bytes)); + for (ui=0; ui < cache_lines; ui++) + for (vi=0; vi < ATOMICS_PER_CACHE_LINE; vi++) + erts_atomic64_init_nob(&p->u[ui].v[vi], 0); + hp = HAlloc(BIF_P, ERTS_MAGIC_REF_THING_SIZE); + return erts_mk_magic_ref(&hp, &MSO(BIF_P), mbin); +} + +static ERTS_INLINE int get_ref(Eterm ref, CountersRef** pp) +{ + Binary* mbin; + if (!is_internal_magic_ref(ref)) + return 0; + + mbin = erts_magic_ref2bin(ref); + if (ERTS_MAGIC_BIN_DESTRUCTOR(mbin) != counters_destructor) + return 0; + *pp = ERTS_MAGIC_BIN_DATA(mbin); + return 1; +} + +static ERTS_INLINE int get_ref_cnt(Eterm ref, Eterm index, + CountersRef** pp, + erts_atomic64_t** app, + UWord sched_ix) +{ + CountersRef* p; + UWord ix, ui, vi; + if (!get_ref(ref, &p) || !term_to_UWord(index, &ix) || --ix >= p->arity) + return 0; + ui = (ix / ATOMICS_PER_CACHE_LINE) * ATOMICS_PER_COUNTER + sched_ix; + vi = ix % ATOMICS_PER_CACHE_LINE; + ASSERT(ui < p->ulen); + *pp = p; + *app = &p->u[ui].v[vi]; + return 1; +} + +static ERTS_INLINE int get_ref_my_cnt(Eterm ref, Eterm index, + CountersRef** pp, + erts_atomic64_t** app) +{ + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + ASSERT(esdp && !ERTS_SCHEDULER_IS_DIRTY(esdp)); + ASSERT(esdp->no > 0 && esdp->no < ATOMICS_PER_COUNTER); + return get_ref_cnt(ref, index, pp, app, esdp->no); +} + +static ERTS_INLINE int get_ref_first_cnt(Eterm ref, Eterm index, + CountersRef** pp, + erts_atomic64_t** app) +{ + return get_ref_cnt(ref, index, pp, app, 0); +} + +static ERTS_INLINE int get_incr(CountersRef* p, Eterm term, erts_aint64_t *valp) +{ + return (term_to_Sint64(term, (Sint64*)valp) + || term_to_Uint64(term, (Uint64*)valp)); +} + +static ERTS_INLINE Eterm bld_counter(Process* proc, CountersRef* p, + erts_aint64_t val) +{ + if (IS_SSMALL(val)) + return make_small((Sint) val); + else { + Uint hsz = ERTS_SINT64_HEAP_SIZE(val); + Eterm* hp = HAlloc(proc, hsz); + return erts_sint64_to_big(val, &hp); + } +} + +BIF_RETTYPE erts_internal_counters_get_2(BIF_ALIST_2) +{ + CountersRef* p; + erts_atomic64_t* ap; + erts_aint64_t acc = 0; + int j; + + if (!get_ref_first_cnt(BIF_ARG_1, BIF_ARG_2, &p, &ap)) { + BIF_ERROR(BIF_P, BADARG); + } + for (j = ATOMICS_PER_COUNTER; j ; --j) { + acc += erts_atomic64_read_nob(ap); + ap = (erts_atomic64_t*) ((byte*)ap + ERTS_CACHE_LINE_SIZE); + } + return bld_counter(BIF_P, p, acc); +} + +BIF_RETTYPE erts_internal_counters_add_3(BIF_ALIST_3) +{ + CountersRef* p; + erts_atomic64_t* ap; + erts_aint64_t incr, sum; + + if (!get_ref_my_cnt(BIF_ARG_1, BIF_ARG_2, &p, &ap) + || !get_incr(p, BIF_ARG_3, &incr)) { + BIF_ERROR(BIF_P, BADARG); + } + sum = incr + erts_atomic64_read_nob(ap); + erts_atomic64_set_nob(ap, sum); + return am_ok; +} + +BIF_RETTYPE erts_internal_counters_put_3(BIF_ALIST_3) +{ + CountersRef* p; + erts_atomic64_t* first_ap; + erts_atomic64_t* ap; + erts_aint64_t acc; + erts_aint64_t val; + int j; + + if (!get_ref_first_cnt(BIF_ARG_1, BIF_ARG_2, &p, &first_ap) + || !term_to_Sint64(BIF_ARG_3, &val)) { + BIF_ERROR(BIF_P, BADARG); + } + + ap = first_ap; + acc = 0; + j = ATOMICS_PER_COUNTER - 1; + do { + ap = (erts_atomic64_t*) ((byte*)ap + ERTS_CACHE_LINE_SIZE); + acc += erts_atomic64_read_nob(ap); + } while (--j); + erts_atomic64_set_nob(first_ap, val-acc); + + return am_ok; +} + +BIF_RETTYPE erts_internal_counters_info_1(BIF_ALIST_1) +{ + CountersRef* p; + Uint hsz = MAP2_SZ; + Eterm *hp; + UWord memory; + Eterm sz_val, mem_val; + + if (!get_ref(BIF_ARG_1, &p)) + BIF_ERROR(BIF_P, BADARG); + + memory = erts_magic_ref2bin(BIF_ARG_1)->orig_size; + erts_bld_uword(NULL, &hsz, p->arity); + erts_bld_uword(NULL, &hsz, memory); + + hp = HAlloc(BIF_P, hsz); + sz_val = erts_bld_uword(&hp, NULL, p->arity); + mem_val = erts_bld_uword(&hp, NULL, memory); + + return MAP2(hp, am_memory, mem_val, + am_size, sz_val); +} diff --git a/erts/emulator/beam/erl_bif_persistent.c b/erts/emulator/beam/erl_bif_persistent.c new file mode 100644 index 0000000000..9dca768a18 --- /dev/null +++ b/erts/emulator/beam/erl_bif_persistent.c @@ -0,0 +1,983 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +/* + * Purpose: Implement persistent term storage. + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "sys.h" +#include "erl_vm.h" +#include "global.h" +#include "erl_process.h" +#include "error.h" +#include "erl_driver.h" +#include "bif.h" +#include "erl_map.h" +#include "erl_binary.h" + +/* + * The limit for the number of persistent terms before + * a warning is issued. + */ + +#define WARNING_LIMIT 20000 +#define XSTR(s) STR(s) +#define STR(s) #s + +/* + * Parameters for the hash table. + */ +#define INITIAL_SIZE 8 +#define LOAD_FACTOR ((Uint)50) +#define MUST_GROW(t) (((Uint)100) * t->num_entries >= LOAD_FACTOR * t->allocated) +#define MUST_SHRINK(t) (((Uint)200) * t->num_entries <= LOAD_FACTOR * t->allocated && \ + t->allocated > INITIAL_SIZE) + +typedef struct hash_table { + Uint allocated; + Uint num_entries; + Uint mask; + Uint first_to_delete; + Uint num_to_delete; + erts_atomic_t refc; + struct hash_table* delete_next; + ErtsThrPrgrLaterOp thr_prog_op; + Eterm term[1]; +} HashTable; + +typedef struct trap_data { + HashTable* table; + Uint idx; + Uint remaining; + Uint memory; /* Used by info/0 to count used memory */ +} TrapData; + +/* + * Declarations of local functions. + */ + +static HashTable* create_initial_table(void); +static Uint lookup(HashTable* hash_table, Eterm key); +static HashTable* copy_table(HashTable* old_table, Uint new_size, int rehash); +static HashTable* tmp_table_copy(HashTable* old_table); +static int try_seize_update_permission(Process* c_p); +static void release_update_permission(int release_updater); +static void table_updater(void* table); +static void table_deleter(void* hash_table); +static void dec_table_refc(Process* c_p, HashTable* old_table); +static void delete_table(Process* c_p, HashTable* table); +static void mark_for_deletion(HashTable* hash_table, Uint entry_index); +static ErtsLiteralArea* term_to_area(Eterm tuple); +static void suspend_updater(Process* c_p); +static Eterm do_get_all(Process* c_p, TrapData* trap_data, Eterm res); +static Eterm do_info(Process* c_p, TrapData* trap_data); +static void append_to_delete_queue(HashTable* table); +static HashTable* next_to_delete(void); +static Eterm alloc_trap_data(Process* c_p); +static int cleanup_trap_data(Binary *bp); + +/* + * Traps + */ + +static Export persistent_term_get_all_export; +static BIF_RETTYPE persistent_term_get_all_trap(BIF_ALIST_2); +static Export persistent_term_info_export; +static BIF_RETTYPE persistent_term_info_trap(BIF_ALIST_1); + +/* + * Pointer to the current hash table. + */ + +static erts_atomic_t the_hash_table; + +/* + * Queue of processes waiting to update the hash table. + */ + +struct update_queue_item { + Process *p; + struct update_queue_item* next; +}; + +static erts_mtx_t update_table_permission_mtx; +static struct update_queue_item* update_queue = NULL; +static Process* updater_process = NULL; + +/* Protected by update_table_permission_mtx */ +static ErtsThrPrgrLaterOp thr_prog_op; +static int issued_warning = 0; + +/* + * Queue of hash tables to be deleted. + */ + +static erts_mtx_t delete_queue_mtx; +static HashTable* delete_queue_head = NULL; +static HashTable** delete_queue_tail = &delete_queue_head; + +/* + * The following variables are only used during crash dumping. They + * are intialized by erts_init_persistent_dumping(). + */ + +ErtsLiteralArea** erts_persistent_areas; +Uint erts_num_persistent_areas; + +void erts_init_bif_persistent_term(void) +{ + HashTable* hash_table; + + /* + * Initialize the mutex protecting updates. + */ + + erts_mtx_init(&update_table_permission_mtx, + "update_persistent_term_permission", + NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | + ERTS_LOCK_FLAGS_CATEGORY_GENERIC); + + /* + * Initialize delete queue. + */ + + erts_mtx_init(&delete_queue_mtx, + "persistent_term_delete_permission", + NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | + ERTS_LOCK_FLAGS_CATEGORY_GENERIC); + + /* + * Allocate a small initial hash table. + */ + + hash_table = create_initial_table(); + erts_atomic_init_nob(&the_hash_table, (erts_aint_t)hash_table); + + /* + * Initialize export entry for traps + */ + + erts_init_trap_export(&persistent_term_get_all_export, + am_persistent_term, am_get_all_trap, 2, + &persistent_term_get_all_trap); + erts_init_trap_export(&persistent_term_info_export, + am_persistent_term, am_info_trap, 1, + &persistent_term_info_trap); +} + +BIF_RETTYPE persistent_term_put_2(BIF_ALIST_2) +{ + Eterm key; + Eterm term; + Eterm heap[3]; + Eterm tuple; + HashTable* hash_table; + Uint term_size; + Uint lit_area_size; + ErlOffHeap code_off_heap; + ErtsLiteralArea* literal_area; + erts_shcopy_t info; + Eterm* ptr; + Uint entry_index; + + if (!try_seize_update_permission(BIF_P)) { + ERTS_BIF_YIELD2(bif_export[BIF_persistent_term_put_2], + BIF_P, BIF_ARG_1, BIF_ARG_2); + } + + hash_table = (HashTable *) erts_atomic_read_nob(&the_hash_table); + + key = BIF_ARG_1; + term = BIF_ARG_2; + + entry_index = lookup(hash_table, key); + + heap[0] = make_arityval(2); + heap[1] = key; + heap[2] = term; + tuple = make_tuple(heap); + + if (is_nil(hash_table->term[entry_index])) { + Uint size = hash_table->allocated; + if (MUST_GROW(hash_table)) { + size *= 2; + } + hash_table = copy_table(hash_table, size, 0); + entry_index = lookup(hash_table, key); + hash_table->num_entries++; + } else { + Eterm tuple = hash_table->term[entry_index]; + Eterm old_term; + + ASSERT(is_tuple_arity(tuple, 2)); + old_term = boxed_val(tuple)[2]; + if (EQ(term, old_term)) { + /* Same value. No need to update anything. */ + release_update_permission(0); + BIF_RET(am_ok); + } else { + /* Mark the old term for deletion. */ + mark_for_deletion(hash_table, entry_index); + hash_table = copy_table(hash_table, hash_table->allocated, 0); + } + } + + /* + * Preserve internal sharing in the term by using the + * sharing-preserving functions. However, literals must + * be copied in case the module holding them are unloaded. + */ + INITIALIZE_SHCOPY(info); + info.copy_literals = 1; + term_size = copy_shared_calculate(tuple, &info); + ERTS_INIT_OFF_HEAP(&code_off_heap); + lit_area_size = ERTS_LITERAL_AREA_ALLOC_SIZE(term_size); + literal_area = erts_alloc(ERTS_ALC_T_LITERAL, lit_area_size); + ptr = &literal_area->start[0]; + literal_area->end = ptr + term_size; + tuple = copy_shared_perform(tuple, term_size, &info, &ptr, &code_off_heap); + ASSERT(tuple_val(tuple) == literal_area->start); + literal_area->off_heap = code_off_heap.first; + DESTROY_SHCOPY(info); + erts_set_literal_tag(&tuple, literal_area->start, term_size); + hash_table->term[entry_index] = tuple; + + erts_schedule_thr_prgr_later_op(table_updater, hash_table, &thr_prog_op); + suspend_updater(BIF_P); + + /* + * Issue a warning once if the warning limit has been exceeded. + */ + + if (hash_table->num_entries > WARNING_LIMIT && issued_warning == 0) { + static char w[] = + "More than " XSTR(WARNING_LIMIT) " persistent terms " + "have been created.\n" + "It is recommended to avoid creating an excessive number of\n" + "persistent terms, as creation and deletion of persistent terms\n" + "will be slower as the number of persistent terms increases.\n"; + issued_warning = 1; + erts_send_warning_to_logger_str(BIF_P->group_leader, w); + } + + ERTS_BIF_YIELD_RETURN(BIF_P, am_ok); +} + +BIF_RETTYPE persistent_term_get_0(BIF_ALIST_0) +{ + HashTable* hash_table = (HashTable *) erts_atomic_read_nob(&the_hash_table); + TrapData* trap_data; + Eterm res = NIL; + Eterm magic_ref; + Binary* mbp; + + magic_ref = alloc_trap_data(BIF_P); + mbp = erts_magic_ref2bin(magic_ref); + trap_data = ERTS_MAGIC_BIN_DATA(mbp); + trap_data->table = hash_table; + trap_data->idx = 0; + trap_data->remaining = hash_table->num_entries; + res = do_get_all(BIF_P, trap_data, res); + if (trap_data->remaining == 0) { + BUMP_REDS(BIF_P, hash_table->num_entries); + trap_data->table = NULL; /* Prevent refc decrement */ + BIF_RET(res); + } else { + /* + * Increment the ref counter to prevent an update operation (by put/2 + * or erase/1) to delete this hash table. + */ + erts_atomic_inc_nob(&hash_table->refc); + BUMP_ALL_REDS(BIF_P); + BIF_TRAP2(&persistent_term_get_all_export, BIF_P, magic_ref, res); + } +} + +BIF_RETTYPE persistent_term_get_1(BIF_ALIST_1) +{ + Eterm key = BIF_ARG_1; + HashTable* hash_table = (HashTable *) erts_atomic_read_nob(&the_hash_table); + Uint entry_index; + Eterm term; + + entry_index = lookup(hash_table, key); + term = hash_table->term[entry_index]; + if (is_boxed(term)) { + ASSERT(is_tuple_arity(term, 2)); + BIF_RET(tuple_val(term)[2]); + } + BIF_ERROR(BIF_P, BADARG); +} + +BIF_RETTYPE persistent_term_erase_1(BIF_ALIST_1) +{ + Eterm key = BIF_ARG_1; + HashTable* old_table; + HashTable* new_table; + Uint entry_index; + Eterm old_term; + + if (!try_seize_update_permission(BIF_P)) { + ERTS_BIF_YIELD1(bif_export[BIF_persistent_term_erase_1], + BIF_P, BIF_ARG_1); + } + + old_table = (HashTable *) erts_atomic_read_nob(&the_hash_table); + entry_index = lookup(old_table, key); + old_term = old_table->term[entry_index]; + if (is_boxed(old_term)) { + Uint new_size; + HashTable* tmp_table; + + /* + * Since we don't use any delete markers, we must rehash + * the table when deleting terms to ensure that all terms + * can still be reached if there are hash collisions. + * We can't rehash in place and it would not be safe to modify + * the old table yet, so we will first need a new + * temporary table copy of the same size as the old one. + */ + + ASSERT(is_tuple_arity(old_term, 2)); + tmp_table = tmp_table_copy(old_table); + + /* + * Delete the term from the temporary table. Then copy the + * temporary table to a new table, rehashing the entries + * while copying. + */ + + tmp_table->term[entry_index] = NIL; + tmp_table->num_entries--; + new_size = tmp_table->allocated; + if (MUST_SHRINK(tmp_table)) { + new_size /= 2; + } + new_table = copy_table(tmp_table, new_size, 1); + erts_free(ERTS_ALC_T_TMP, tmp_table); + + mark_for_deletion(old_table, entry_index); + erts_schedule_thr_prgr_later_op(table_updater, new_table, &thr_prog_op); + suspend_updater(BIF_P); + ERTS_BIF_YIELD_RETURN(BIF_P, am_true); + } + + /* + * Key is not present. Nothing to do. + */ + + ASSERT(is_nil(old_term)); + release_update_permission(0); + BIF_RET(am_false); +} + +BIF_RETTYPE erts_internal_erase_persistent_terms_0(BIF_ALIST_0) +{ + HashTable* old_table; + HashTable* new_table; + + if (!try_seize_update_permission(BIF_P)) { + ERTS_BIF_YIELD0(bif_export[BIF_erts_internal_erase_persistent_terms_0], + BIF_P); + } + old_table = (HashTable *) erts_atomic_read_nob(&the_hash_table); + old_table->first_to_delete = 0; + old_table->num_to_delete = old_table->allocated; + new_table = create_initial_table(); + erts_schedule_thr_prgr_later_op(table_updater, new_table, &thr_prog_op); + suspend_updater(BIF_P); + ERTS_BIF_YIELD_RETURN(BIF_P, am_true); +} + +BIF_RETTYPE persistent_term_info_0(BIF_ALIST_0) +{ + HashTable* hash_table = (HashTable *) erts_atomic_read_nob(&the_hash_table); + TrapData* trap_data; + Eterm res = NIL; + Eterm magic_ref; + Binary* mbp; + + magic_ref = alloc_trap_data(BIF_P); + mbp = erts_magic_ref2bin(magic_ref); + trap_data = ERTS_MAGIC_BIN_DATA(mbp); + trap_data->table = hash_table; + trap_data->idx = 0; + trap_data->remaining = hash_table->num_entries; + trap_data->memory = 0; + res = do_info(BIF_P, trap_data); + if (trap_data->remaining == 0) { + BUMP_REDS(BIF_P, hash_table->num_entries); + trap_data->table = NULL; /* Prevent refc decrement */ + BIF_RET(res); + } else { + /* + * Increment the ref counter to prevent an update operation (by put/2 + * or erase/1) to delete this hash table. + */ + erts_atomic_inc_nob(&hash_table->refc); + BUMP_ALL_REDS(BIF_P); + BIF_TRAP2(&persistent_term_info_export, BIF_P, magic_ref, res); + } +} + +Uint +erts_persistent_term_count(void) +{ + HashTable* hash_table = (HashTable *) erts_atomic_read_nob(&the_hash_table); + return hash_table->num_entries; +} + +void +erts_init_persistent_dumping(void) +{ + HashTable* hash_table = (HashTable *) erts_atomic_read_nob(&the_hash_table); + ErtsLiteralArea** area_p; + Uint i; + + /* + * Overwrite the array of Eterms in the current hash table + * with pointers to literal areas. + */ + + erts_persistent_areas = (ErtsLiteralArea **) hash_table->term; + erts_num_persistent_areas = hash_table->num_entries; + area_p = erts_persistent_areas; + for (i = 0; i < hash_table->allocated; i++) { + Eterm term = hash_table->term[i]; + + if (is_boxed(term)) { + *area_p++ = term_to_area(term); + } + } +} + +/* + * Local functions. + */ + +static HashTable* +create_initial_table(void) +{ + HashTable* hash_table; + int i; + + hash_table = (HashTable *) erts_alloc(ERTS_ALC_T_PERSISTENT_TERM, + sizeof(HashTable)+sizeof(Eterm) * + (INITIAL_SIZE-1)); + hash_table->allocated = INITIAL_SIZE; + hash_table->num_entries = 0; + hash_table->mask = INITIAL_SIZE-1; + hash_table->first_to_delete = 0; + hash_table->num_to_delete = 0; + erts_atomic_init_nob(&hash_table->refc, (erts_aint_t)1); + for (i = 0; i < INITIAL_SIZE; i++) { + hash_table->term[i] = NIL; + } + return hash_table; +} + +static BIF_RETTYPE +persistent_term_get_all_trap(BIF_ALIST_2) +{ + TrapData* trap_data; + Eterm res = BIF_ARG_2; + Uint bump_reds; + Binary* mbp; + + ASSERT(is_list(BIF_ARG_2)); + mbp = erts_magic_ref2bin(BIF_ARG_1); + trap_data = ERTS_MAGIC_BIN_DATA(mbp); + bump_reds = trap_data->remaining; + res = do_get_all(BIF_P, trap_data, res); + ASSERT(is_list(res)); + if (trap_data->remaining > 0) { + BUMP_ALL_REDS(BIF_P); + BIF_TRAP2(&persistent_term_get_all_export, BIF_P, BIF_ARG_1, res); + } else { + /* + * Decrement ref count (and possibly delete the hash table + * and associated literal area). + */ + dec_table_refc(BIF_P, trap_data->table); + trap_data->table = NULL; /* Prevent refc decrement */ + BUMP_REDS(BIF_P, bump_reds); + BIF_RET(res); + } +} + +static Eterm +do_get_all(Process* c_p, TrapData* trap_data, Eterm res) +{ + HashTable* hash_table; + Uint remaining; + Uint idx; + Uint max_iter; + Uint i; + Eterm* hp; + Uint heap_size; + struct copy_term { + Uint key_size; + Eterm* tuple_ptr; + } *copy_data; + + hash_table = trap_data->table; + idx = trap_data->idx; +#if defined(DEBUG) || defined(VALGRIND) + max_iter = 50; +#else + max_iter = ERTS_BIF_REDS_LEFT(c_p); +#endif + remaining = trap_data->remaining < max_iter ? + trap_data->remaining : max_iter; + trap_data->remaining -= remaining; + + copy_data = (struct copy_term *) erts_alloc(ERTS_ALC_T_TMP, + remaining * + sizeof(struct copy_term)); + i = 0; + heap_size = (2 + 3) * remaining; + while (remaining != 0) { + Eterm term = hash_table->term[idx]; + if (is_tuple(term)) { + Uint key_size; + Eterm* tup_val; + + ASSERT(is_tuple_arity(term, 2)); + tup_val = tuple_val(term); + key_size = size_object(tup_val[1]); + copy_data[i].key_size = key_size; + copy_data[i].tuple_ptr = tup_val; + heap_size += key_size; + i++; + remaining--; + } + idx++; + } + trap_data->idx = idx; + + hp = HAlloc(c_p, heap_size); + remaining = i; + for (i = 0; i < remaining; i++) { + Eterm* tuple_ptr; + Uint key_size; + Eterm key; + Eterm tup; + + tuple_ptr = copy_data[i].tuple_ptr; + key_size = copy_data[i].key_size; + key = copy_struct(tuple_ptr[1], key_size, &hp, &c_p->off_heap); + tup = TUPLE2(hp, key, tuple_ptr[2]); + hp += 3; + res = CONS(hp, tup, res); + hp += 2; + } + erts_free(ERTS_ALC_T_TMP, copy_data); + return res; +} + +static BIF_RETTYPE +persistent_term_info_trap(BIF_ALIST_1) +{ + TrapData* trap_data = (TrapData *) BIF_ARG_1; + Eterm res; + Uint bump_reds; + Binary* mbp; + + mbp = erts_magic_ref2bin(BIF_ARG_1); + trap_data = ERTS_MAGIC_BIN_DATA(mbp); + bump_reds = trap_data->remaining; + res = do_info(BIF_P, trap_data); + if (trap_data->remaining > 0) { + ASSERT(res == am_ok); + BUMP_ALL_REDS(BIF_P); + BIF_TRAP1(&persistent_term_info_export, BIF_P, BIF_ARG_1); + } else { + /* + * Decrement ref count (and possibly delete the hash table + * and associated literal area). + */ + dec_table_refc(BIF_P, trap_data->table); + trap_data->table = NULL; /* Prevent refc decrement */ + BUMP_REDS(BIF_P, bump_reds); + ASSERT(is_map(res)); + BIF_RET(res); + } +} + +#define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1) + +static Eterm +do_info(Process* c_p, TrapData* trap_data) +{ + HashTable* hash_table; + Uint remaining; + Uint idx; + Uint max_iter; + + hash_table = trap_data->table; + idx = trap_data->idx; +#if defined(DEBUG) || defined(VALGRIND) + max_iter = 50; +#else + max_iter = ERTS_BIF_REDS_LEFT(c_p); +#endif + remaining = trap_data->remaining < max_iter ? trap_data->remaining : max_iter; + trap_data->remaining -= remaining; + while (remaining != 0) { + if (is_boxed(hash_table->term[idx])) { + ErtsLiteralArea* area; + area = term_to_area(hash_table->term[idx]); + trap_data->memory += sizeof(ErtsLiteralArea) + + sizeof(Eterm) * (area->end - area->start - 1); + remaining--; + } + idx++; + } + trap_data->idx = idx; + if (trap_data->remaining > 0) { + return am_ok; /* Dummy return value */ + } else { + Eterm* hp; + Eterm count_term; + Eterm memory_term; + Eterm res; + Uint memory; + Uint hsz = MAP_SZ(2); + + memory = sizeof(HashTable) + (trap_data->table->allocated-1) * + sizeof(Eterm) + trap_data->memory; + (void) erts_bld_uint(NULL, &hsz, hash_table->num_entries); + (void) erts_bld_uint(NULL, &hsz, memory); + hp = HAlloc(c_p, hsz); + count_term = erts_bld_uint(&hp, NULL, hash_table->num_entries); + memory_term = erts_bld_uint(&hp, NULL, memory); + res = MAP2(hp, am_count, count_term, am_memory, memory_term); + return res; + } +} + +#undef DECL_AM + +static Eterm +alloc_trap_data(Process* c_p) +{ + Binary* mbp = erts_create_magic_binary(sizeof(TrapData), + cleanup_trap_data); + Eterm* hp; + + hp = HAlloc(c_p, ERTS_MAGIC_REF_THING_SIZE); + return erts_mk_magic_ref(&hp, &MSO(c_p), mbp); +} + +static int +cleanup_trap_data(Binary *bp) +{ + TrapData* trap_data = ERTS_MAGIC_BIN_DATA(bp); + + if (trap_data->table) { + /* + * The process has been killed and is now exiting. + * Decrement the reference counter for the table. + */ + dec_table_refc(NULL, trap_data->table); + } + return 1; +} + +static Uint +lookup(HashTable* hash_table, Eterm key) +{ + Uint mask = hash_table->mask; + Eterm* table = hash_table->term; + Uint32 idx = make_internal_hash(key, 0); + Eterm term; + + do { + idx++; + term = table[idx & mask]; + } while (is_boxed(term) && !EQ(key, (tuple_val(term))[1])); + return idx & mask; +} + +static HashTable* +tmp_table_copy(HashTable* old_table) +{ + Uint size = old_table->allocated; + HashTable* tmp_table; + Uint i; + + tmp_table = (HashTable *) erts_alloc(ERTS_ALC_T_TMP, + sizeof(HashTable) + + sizeof(Eterm) * (size-1)); + *tmp_table = *old_table; + for (i = 0; i < size; i++) { + tmp_table->term[i] = old_table->term[i]; + } + return tmp_table; +} + +static HashTable* +copy_table(HashTable* old_table, Uint new_size, int rehash) +{ + HashTable* new_table; + Uint old_size = old_table->allocated; + Uint i; + + new_table = (HashTable *) erts_alloc(ERTS_ALC_T_PERSISTENT_TERM, + sizeof(HashTable) + + sizeof(Eterm) * (new_size-1)); + if (old_table->allocated == new_size && !rehash) { + /* + * Same size and no key deleted. Make an exact copy of the table. + */ + *new_table = *old_table; + for (i = 0; i < new_size; i++) { + new_table->term[i] = old_table->term[i]; + } + } else { + /* + * The size of the table has changed or an element has been + * deleted. Must rehash, by inserting all old terms into the + * new (empty) table. + */ + new_table->allocated = new_size; + new_table->num_entries = old_table->num_entries; + new_table->mask = new_size - 1; + for (i = 0; i < new_size; i++) { + new_table->term[i] = NIL; + } + for (i = 0; i < old_size; i++) { + if (is_tuple(old_table->term[i])) { + Eterm key = tuple_val(old_table->term[i])[1]; + Uint entry_index = lookup(new_table, key); + ASSERT(is_nil(new_table->term[entry_index])); + new_table->term[entry_index] = old_table->term[i]; + } + } + } + new_table->first_to_delete = 0; + new_table->num_to_delete = 0; + erts_atomic_init_nob(&new_table->refc, (erts_aint_t)1); + return new_table; +} + +static void +mark_for_deletion(HashTable* hash_table, Uint entry_index) +{ + hash_table->first_to_delete = entry_index; + hash_table->num_to_delete = 1; +} + +static ErtsLiteralArea* +term_to_area(Eterm tuple) +{ + ASSERT(is_tuple_arity(tuple, 2)); + return (ErtsLiteralArea *) (((char *) tuple_val(tuple)) - + offsetof(ErtsLiteralArea, start)); +} + +static void +table_updater(void* data) +{ + HashTable* old_table; + HashTable* new_table; + + old_table = (HashTable *) erts_atomic_read_nob(&the_hash_table); + new_table = (HashTable *) data; + ASSERT(new_table->num_to_delete == 0); + erts_atomic_set_nob(&the_hash_table, (erts_aint_t)new_table); + append_to_delete_queue(old_table); + erts_schedule_thr_prgr_later_op(table_deleter, + old_table, + &old_table->thr_prog_op); + release_update_permission(1); +} + +static void +table_deleter(void* data) +{ + HashTable* old_table = (HashTable *) data; + + dec_table_refc(NULL, old_table); +} + +static void +dec_table_refc(Process* c_p, HashTable* old_table) +{ + erts_aint_t refc = erts_atomic_dec_read_nob(&old_table->refc); + + if (refc == 0) { + HashTable* to_delete; + + while ((to_delete = next_to_delete()) != NULL) { + delete_table(c_p, to_delete); + } + } +} + +static void +delete_table(Process* c_p, HashTable* table) +{ + Uint idx = table->first_to_delete; + Uint n = table->num_to_delete; + + /* + * There are no longer any references to this hash table. + * + * Any literals pointed for deletion can be queued for + * deletion and the table itself can be deallocated. + */ + +#ifdef DEBUG + if (n == 1) { + ASSERT(is_tuple_arity(table->term[idx], 2)); + } +#endif + + while (n > 0) { + Eterm term = table->term[idx]; + + if (is_tuple_arity(term, 2)) { + if (is_immed(tuple_val(term)[2])) { + erts_release_literal_area(term_to_area(term)); + } else { + erts_queue_release_literals(c_p, term_to_area(term)); + } + } + idx++, n--; + } + erts_free(ERTS_ALC_T_PERSISTENT_TERM, table); +} + +/* + * Caller *must* yield if this function returns 0. + */ + +static int +try_seize_update_permission(Process* c_p) +{ + int success; + + ASSERT(!erts_thr_progress_is_blocking()); /* to avoid deadlock */ + ASSERT(c_p != NULL); + + erts_mtx_lock(&update_table_permission_mtx); + ASSERT(updater_process != c_p); + success = (updater_process == NULL); + if (success) { + updater_process = c_p; + } else { + struct update_queue_item* qitem; + qitem = erts_alloc(ERTS_ALC_T_PERSISTENT_LOCK_Q, sizeof(*qitem)); + qitem->p = c_p; + erts_proc_inc_refc(c_p); + qitem->next = update_queue; + update_queue = qitem; + erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL); + } + erts_mtx_unlock(&update_table_permission_mtx); + return success; +} + +static void +release_update_permission(int release_updater) +{ + erts_mtx_lock(&update_table_permission_mtx); + ASSERT(updater_process != NULL); + + if (release_updater) { + erts_proc_lock(updater_process, ERTS_PROC_LOCK_STATUS); + if (!ERTS_PROC_IS_EXITING(updater_process)) { + erts_resume(updater_process, ERTS_PROC_LOCK_STATUS); + } + erts_proc_unlock(updater_process, ERTS_PROC_LOCK_STATUS); + } + updater_process = NULL; + + while (update_queue != NULL) { /* Unleash the entire herd */ + struct update_queue_item* qitem = update_queue; + erts_proc_lock(qitem->p, ERTS_PROC_LOCK_STATUS); + if (!ERTS_PROC_IS_EXITING(qitem->p)) { + erts_resume(qitem->p, ERTS_PROC_LOCK_STATUS); + } + erts_proc_unlock(qitem->p, ERTS_PROC_LOCK_STATUS); + update_queue = qitem->next; + erts_proc_dec_refc(qitem->p); + erts_free(ERTS_ALC_T_PERSISTENT_LOCK_Q, qitem); + } + erts_mtx_unlock(&update_table_permission_mtx); +} + +static void +suspend_updater(Process* c_p) +{ +#ifdef DEBUG + ASSERT(c_p != NULL); + erts_mtx_lock(&update_table_permission_mtx); + ASSERT(updater_process == c_p); + erts_mtx_unlock(&update_table_permission_mtx); +#endif + erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL); +} + +static void +append_to_delete_queue(HashTable* table) +{ + erts_mtx_lock(&delete_queue_mtx); + table->delete_next = NULL; + *delete_queue_tail = table; + delete_queue_tail = &table->delete_next; + erts_mtx_unlock(&delete_queue_mtx); +} + +static HashTable* +next_to_delete(void) +{ + HashTable* table; + + erts_mtx_lock(&delete_queue_mtx); + table = delete_queue_head; + if (table) { + if (erts_atomic_read_nob(&table->refc)) { + /* + * This hash table is still referenced. Hash tables + * must be deleted in order, so we return a NULL + * pointer. + */ + table = NULL; + } else { + /* + * Remove the first hash table from the queue. + */ + delete_queue_head = table->delete_next; + if (delete_queue_head == NULL) { + delete_queue_tail = &delete_queue_head; + } + } + } + erts_mtx_unlock(&delete_queue_mtx); + return table; +} diff --git a/erts/emulator/beam/erl_dirty_bif.tab b/erts/emulator/beam/erl_dirty_bif.tab index 086275fbe5..20299ff604 100644 --- a/erts/emulator/beam/erl_dirty_bif.tab +++ b/erts/emulator/beam/erl_dirty_bif.tab @@ -59,8 +59,6 @@ dirty-cpu erts_debug:lcnt_clear/0 dirty-cpu-test erlang:'++'/2 dirty-cpu-test erlang:append/2 -dirty-cpu-test erlang:'--'/2 -dirty-cpu-test erlang:subtract/2 dirty-cpu-test erlang:iolist_size/1 dirty-cpu-test erlang:make_tuple/2 dirty-cpu-test erlang:make_tuple/3 diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 57c6c10c7f..f687dcf335 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -354,6 +354,7 @@ erl_init(int ncpu, erts_init_bif(); erts_init_bif_chksum(); erts_init_bif_binary(); + erts_init_bif_persistent_term(); erts_init_bif_re(); erts_init_unicode(); /* after RE to get access to PCRE unicode */ erts_init_external(); diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 463ae898a3..1416c5f96c 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -97,6 +97,8 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "proc_btm", "pid" }, { "dist_entry", "address" }, { "dist_entry_links", "address" }, + { "update_persistent_term_permission", NULL }, + { "persistent_term_delete_permission", NULL }, { "code_write_permission", NULL }, { "purge_state", NULL }, { "proc_status", "pid" }, diff --git a/erts/emulator/beam/erl_monitor_link.h b/erts/emulator/beam/erl_monitor_link.h index 9ff8aa509a..ed7bf7d54a 100644 --- a/erts/emulator/beam/erl_monitor_link.h +++ b/erts/emulator/beam/erl_monitor_link.h @@ -1387,11 +1387,14 @@ ERTS_GLB_INLINE void erts_monitor_release(ErtsMonitor *mon) { ErtsMonitorData *mdp = erts_monitor_to_data(mon); - ERTS_ML_ASSERT(!(mon->flags & ERTS_ML_FLG_IN_TABLE)); ERTS_ML_ASSERT(erts_atomic32_read_nob(&mdp->refc) > 0); - if (erts_atomic32_dec_read_nob(&mdp->refc) == 0) + if (erts_atomic32_dec_read_nob(&mdp->refc) == 0) { + ERTS_ML_ASSERT(!(mdp->origin.flags & ERTS_ML_FLG_IN_TABLE)); + ERTS_ML_ASSERT(!(mdp->target.flags & ERTS_ML_FLG_IN_TABLE)); + erts_monitor_destroy__(mdp); + } } ERTS_GLB_INLINE void @@ -1399,12 +1402,14 @@ erts_monitor_release_both(ErtsMonitorData *mdp) { ERTS_ML_ASSERT((mdp->origin.flags & ERTS_ML_FLGS_SAME) == (mdp->target.flags & ERTS_ML_FLGS_SAME)); - ERTS_ML_ASSERT(!(mdp->origin.flags & ERTS_ML_FLG_IN_TABLE)); - ERTS_ML_ASSERT(!(mdp->target.flags & ERTS_ML_FLG_IN_TABLE)); ERTS_ML_ASSERT(erts_atomic32_read_nob(&mdp->refc) >= 2); - if (erts_atomic32_add_read_nob(&mdp->refc, (erts_aint32_t) -2) == 0) + if (erts_atomic32_add_read_nob(&mdp->refc, (erts_aint32_t) -2) == 0) { + ERTS_ML_ASSERT(!(mdp->origin.flags & ERTS_ML_FLG_IN_TABLE)); + ERTS_ML_ASSERT(!(mdp->target.flags & ERTS_ML_FLG_IN_TABLE)); + erts_monitor_destroy__(mdp); + } } ERTS_GLB_INLINE int diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index f4dc60941a..18ed782ae3 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -421,8 +421,25 @@ static void schedule_delete_dist_entry(DistEntry* dep) * * Note that timeouts do not guarantee thread progress. */ - erts_schedule_thr_prgr_later_op(start_timer_delete_dist_entry, - dep, &dep->later_op); + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + if (esdp && !ERTS_SCHEDULER_IS_DIRTY(esdp)) { + erts_schedule_thr_prgr_later_op(start_timer_delete_dist_entry, + dep, &dep->later_op); + } else { + /* + * Since OTP 20, it's possible that destructor is executed on + * a dirty scheduler. Aux work cannot be done on a dirty + * scheduler, and scheduling any aux work on a dirty scheduler + * makes the scheduler to loop infinitely. + * To avoid this, make a spot jump: schedule this function again + * on a first normal scheduler. It is guaranteed to be always + * online. Since it's a rare event, this shall not pose a big + * utilisation hit. + */ + erts_schedule_misc_aux_work(1, + (void (*)(void *))schedule_delete_dist_entry, + (void *) dep); + } } static void diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c index 243db4c734..ac5054ea10 100644 --- a/erts/emulator/beam/erl_process_dump.c +++ b/erts/emulator/beam/erl_process_dump.c @@ -58,6 +58,7 @@ static void dump_externally(fmtfn_t to, void *to_arg, Eterm term); static void mark_literal(Eterm* ptr); static void init_literal_areas(void); static void dump_literals(fmtfn_t to, void *to_arg); +static void dump_persistent_terms(fmtfn_t to, void *to_arg); static void dump_module_literals(fmtfn_t to, void *to_arg, ErtsLiteralArea* lit_area); @@ -74,6 +75,7 @@ erts_deep_process_dump(fmtfn_t to, void *to_arg) all_binaries = NULL; init_literal_areas(); + erts_init_persistent_dumping(); for (i = 0; i < max; i++) { Process *p = erts_pix2proc(i); @@ -93,6 +95,7 @@ erts_deep_process_dump(fmtfn_t to, void *to_arg) } } + dump_persistent_terms(to, to_arg); dump_literals(to, to_arg); dump_binaries(to, to_arg, all_binaries); } @@ -775,6 +778,9 @@ init_literal_areas(void) qsort(lit_areas, num_lit_areas, sizeof(ErtsLiteralArea *), compare_areas); + qsort(erts_persistent_areas, erts_num_persistent_areas, + sizeof(ErtsLiteralArea *), compare_areas); + erts_runlock_old_code(code_ix); } @@ -796,6 +802,13 @@ static void mark_literal(Eterm* ptr) ap = bsearch(ptr, lit_areas, num_lit_areas, sizeof(ErtsLiteralArea*), search_areas); + if (ap == 0) { + ap = bsearch(ptr, erts_persistent_areas, + erts_num_persistent_areas, + sizeof(ErtsLiteralArea*), + search_areas); + } + /* * If the literal was created by native code, this search will not @@ -807,12 +820,12 @@ static void mark_literal(Eterm* ptr) } } - static void dump_literals(fmtfn_t to, void *to_arg) { ErtsCodeIndex code_ix; int i; + Uint idx; code_ix = erts_active_code_ix(); erts_rlock_old_code(code_ix); @@ -825,6 +838,28 @@ dump_literals(fmtfn_t to, void *to_arg) } erts_runlock_old_code(code_ix); + + for (idx = 0; idx < erts_num_persistent_areas; idx++) { + dump_module_literals(to, to_arg, erts_persistent_areas[idx]); + } +} + +static void +dump_persistent_terms(fmtfn_t to, void *to_arg) +{ + Uint idx; + + erts_print(to, to_arg, "=persistent_terms\n"); + + for (idx = 0; idx < erts_num_persistent_areas; idx++) { + ErtsLiteralArea* ap = erts_persistent_areas[idx]; + Eterm tuple = make_tuple(ap->start); + Eterm* tup_val = tuple_val(tuple); + + dump_element(to, to_arg, tup_val[1]); + erts_putc(to, to_arg, '|'); + dump_element_nl(to, to_arg, tup_val[2]); + } } static void @@ -963,7 +998,8 @@ dump_module_literals(fmtfn_t to, void *to_arg, ErtsLiteralArea* lit_area) } erts_putc(to, to_arg, '\n'); } - } else if (is_export_header(w)) { + } else { + /* Dump everything else in the external format */ dump_externally(to, to_arg, term); erts_putc(to, to_arg, '\n'); } diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 21ae205237..0631404599 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -906,6 +906,8 @@ typedef struct ErtsLiteralArea_ { Eterm start[1]; /* beginning of area */ } ErtsLiteralArea; +void erts_queue_release_literals(Process *c_p, ErtsLiteralArea* literals); + #define ERTS_LITERAL_AREA_ALLOC_SIZE(N) \ (sizeof(ErtsLiteralArea) + sizeof(Eterm)*((N) - 1)) @@ -1001,6 +1003,7 @@ typedef struct { Uint literal_size; Eterm *lit_purge_ptr; Uint lit_purge_sz; + int copy_literals; } erts_shcopy_t; #define INITIALIZE_SHCOPY(info) \ @@ -1010,6 +1013,7 @@ typedef struct { info.bitstore_start = info.bitstore_default; \ info.shtable_start = info.shtable_default; \ info.literal_size = 0; \ + info.copy_literals = 0; \ if (larea__) { \ info.lit_purge_ptr = &larea__->start[0]; \ info.lit_purge_sz = larea__->end - info.lit_purge_ptr; \ @@ -1238,6 +1242,13 @@ Sint erts_re_set_loop_limit(Sint limit); void erts_init_bif_binary(void); Sint erts_binary_set_loop_limit(Sint limit); +/* erl_bif_persistent.c */ +void erts_init_bif_persistent_term(void); +Uint erts_persistent_term_count(void); +void erts_init_persistent_dumping(void); +extern ErtsLiteralArea** erts_persistent_areas; +extern Uint erts_num_persistent_areas; + /* external.c */ void erts_init_external(void); diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index bb22548587..869a575cb4 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -325,6 +325,7 @@ typedef long Sint erts_align_attribute(sizeof(long)); #define UWORD_CONSTANT(Const) Const##UL #define ERTS_UWORD_MAX ULONG_MAX #define ERTS_SWORD_MAX LONG_MAX +#define ERTS_SWORD_MIN LONG_MIN #define ERTS_SIZEOF_ETERM SIZEOF_LONG #define ErtsStrToSint strtol #elif SIZEOF_VOID_P == SIZEOF_INT @@ -335,6 +336,7 @@ typedef int Sint erts_align_attribute(sizeof(int)); #define UWORD_CONSTANT(Const) Const##U #define ERTS_UWORD_MAX UINT_MAX #define ERTS_SWORD_MAX INT_MAX +#define ERTS_SWORD_MIN INT_MIN #define ERTS_SIZEOF_ETERM SIZEOF_INT #define ErtsStrToSint strtol #elif SIZEOF_VOID_P == SIZEOF_LONG_LONG @@ -345,6 +347,7 @@ typedef long long Sint erts_align_attribute(sizeof(long long)); #define UWORD_CONSTANT(Const) Const##ULL #define ERTS_UWORD_MAX ULLONG_MAX #define ERTS_SWORD_MAX LLONG_MAX +#define ERTS_SWORD_MIN LLONG_MIN #define ERTS_SIZEOF_ETERM SIZEOF_LONG_LONG #if defined(__WIN32__) #define ErtsStrToSint _strtoi64 diff --git a/erts/emulator/nifs/common/prim_file_nif.c b/erts/emulator/nifs/common/prim_file_nif.c index fd6aaa61f6..ba36a33458 100644 --- a/erts/emulator/nifs/common/prim_file_nif.c +++ b/erts/emulator/nifs/common/prim_file_nif.c @@ -38,6 +38,9 @@ static void unload(ErlNifEnv *env, void* priv_data); static ErlNifResourceType *efile_resource_type; +static ERL_NIF_TERM am_erts_prim_file; +static ERL_NIF_TERM am_close; + static ERL_NIF_TERM am_ok; static ERL_NIF_TERM am_error; static ERL_NIF_TERM am_continue; @@ -96,11 +99,14 @@ static ERL_NIF_TERM set_cwd_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM arg static ERL_NIF_TERM read_file_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM open_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM close_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); + +/* Internal ops */ +static ERL_NIF_TERM delayed_close_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM get_handle_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM altname_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM open_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); - /* All file handle operations are passed through a wrapper that handles state * transitions, marking it as busy during the course of the operation, and * closing on completion if the owner died in the middle of an operation. @@ -128,7 +134,11 @@ static ERL_NIF_TERM open_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[] * * CLOSE_PENDING -> * CLOSED (file_handle_wrapper) - */ + * + * Should the owner of a file die, we can't close it immediately as that could + * potentially block a normal scheduler. When entering the CLOSED state from + * owner_death_callback, we will instead send a message to the erts_prim_file + * process that will then close the file through delayed_close_nif. */ typedef ERL_NIF_TERM (*file_op_impl_t)(efile_data_t *d, ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); @@ -142,7 +152,6 @@ static ERL_NIF_TERM file_handle_wrapper(file_op_impl_t operation, ErlNifEnv *env return file_handle_wrapper( name ## _impl , env, argc, argv); \ } -WRAP_FILE_HANDLE_EXPORT(close_nif) WRAP_FILE_HANDLE_EXPORT(read_nif) WRAP_FILE_HANDLE_EXPORT(write_nif) WRAP_FILE_HANDLE_EXPORT(pread_nif) @@ -193,18 +202,27 @@ static ErlNifFunc nif_funcs[] = { /* Internal ops. */ {"get_handle_nif", 1, get_handle_nif}, + {"delayed_close_nif", 1, delayed_close_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, {"altname_nif", 1, altname_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, }; ERL_NIF_INIT(prim_file, nif_funcs, load, NULL, upgrade, unload) +static ErlNifPid erts_prim_file_pid; + static void owner_death_callback(ErlNifEnv* env, void* obj, ErlNifPid* pid, ErlNifMonitor* mon); -static void gc_callback(ErlNifEnv *env, void* data); -static int load(ErlNifEnv *env, void** priv_data, ERL_NIF_TERM load_info) +static int load(ErlNifEnv *env, void** priv_data, ERL_NIF_TERM prim_file_pid) { ErlNifResourceTypeInit callbacks; + if(!enif_get_local_pid(env, prim_file_pid, &erts_prim_file_pid)) { + ASSERT(!"bad pid passed to prim_file_nif"); + } + + am_erts_prim_file = enif_make_atom(env, "erts_prim_file"); + am_close = enif_make_atom(env, "close"); + am_ok = enif_make_atom(env, "ok"); am_error = enif_make_atom(env, "error"); am_continue = enif_make_atom(env, "continue"); @@ -239,7 +257,7 @@ static int load(ErlNifEnv *env, void** priv_data, ERL_NIF_TERM load_info) am_eof = enif_make_atom(env, "eof"); callbacks.down = owner_death_callback; - callbacks.dtor = gc_callback; + callbacks.dtor = NULL; callbacks.stop = NULL; efile_resource_type = enif_open_resource_type_x(env, "efile", &callbacks, @@ -305,8 +323,10 @@ static ERL_NIF_TERM file_handle_wrapper(file_op_impl_t operation, ErlNifEnv *env /* This is the only point where a change from CLOSE_PENDING is * possible, and we're running synchronously, so we can't race with * anything else here. */ + posix_errno_t ignored; + erts_atomic32_set_acqb(&d->state, EFILE_STATE_CLOSED); - efile_close(d); + efile_close(d, &ignored); } } else { /* CLOSE_PENDING should be impossible at this point since it requires @@ -319,6 +339,24 @@ static ERL_NIF_TERM file_handle_wrapper(file_op_impl_t operation, ErlNifEnv *env return result; } +/* This is a special close operation used by the erts_prim_file process for + * cleaning up orphaned files. It differs from the ordinary close_nif in that + * it only works for files that have already entered the CLOSED state. */ +static ERL_NIF_TERM delayed_close_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + posix_errno_t ignored; + efile_data_t *d; + + ASSERT(argc == 1); + if(!get_file_data(env, argv[0], &d)) { + return enif_make_badarg(env); + } + + ASSERT(erts_atomic32_read_acqb(&d->state) == EFILE_STATE_CLOSED); + efile_close(d, &ignored); + + return am_ok; +} + static void owner_death_callback(ErlNifEnv* env, void* obj, ErlNifPid* pid, ErlNifMonitor* mon) { efile_data_t *d = (efile_data_t*)obj; @@ -334,8 +372,24 @@ static void owner_death_callback(ErlNifEnv* env, void* obj, ErlNifPid* pid, ErlN switch(previous_state) { case EFILE_STATE_IDLE: - efile_close(d); - return; + { + /* We cannot close the file here as that could block a normal + * scheduler, so we tell erts_prim_file to do it for us. + * + * This can in turn become a bottleneck (especially in cases + * like NFS failure), but it's less problematic than blocking + * thread progress. */ + ERL_NIF_TERM message, file_ref; + + file_ref = enif_make_resource(env, d); + message = enif_make_tuple2(env, am_close, file_ref); + + if(!enif_send(env, &erts_prim_file_pid, NULL, message)) { + ERTS_INTERNAL_ERROR("Failed to defer prim_file close."); + } + + return; + } case EFILE_STATE_CLOSE_PENDING: case EFILE_STATE_CLOSED: /* We're either already closed or managed to mark ourselves for @@ -352,24 +406,6 @@ static void owner_death_callback(ErlNifEnv* env, void* obj, ErlNifPid* pid, ErlN } } -static void gc_callback(ErlNifEnv *env, void* data) { - efile_data_t *d = (efile_data_t*)data; - - enum efile_state_t previous_state; - - (void)env; - - previous_state = erts_atomic32_cmpxchg_acqb(&d->state, - EFILE_STATE_CLOSED, EFILE_STATE_IDLE); - - ASSERT(previous_state != EFILE_STATE_CLOSE_PENDING && - previous_state != EFILE_STATE_BUSY); - - if(previous_state == EFILE_STATE_IDLE) { - efile_close(d); - } -} - static ERL_NIF_TERM efile_filetype_to_atom(enum efile_filetype_t type) { switch(type) { case EFILE_FILETYPE_DEVICE: return am_device; @@ -454,40 +490,62 @@ static ERL_NIF_TERM open_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[] return posix_error_to_tuple(env, posix_errno); } - result = enif_make_resource(env, d); - enif_release_resource(d); - enif_self(env, &controlling_process); if(enif_monitor_process(env, d, &controlling_process, &d->monitor)) { + /* We need to close the file manually as we haven't registered a + * destructor. */ + posix_errno_t ignored; + + erts_atomic32_set_acqb(&d->state, EFILE_STATE_CLOSED); + efile_close(d, &ignored); + return posix_error_to_tuple(env, EINVAL); } + /* Note that we do not call enif_release_resource at this point. While it's + * normally safe to leave resource management to the GC, efile_close is a + * blocking operation which must not be done in the GC callback, and we + * can't defer it as the resource is gone as soon as it returns. + * + * We instead keep the resource alive until efile_close is called, after + * which it's safe to leave things to the GC. If the controlling process + * were to die before the user had a chance to close their file, the above + * monitor will tell the erts_prim_file process to close it for them. */ + result = enif_make_resource(env, d); + return enif_make_tuple2(env, am_ok, result); } -static ERL_NIF_TERM close_nif_impl(efile_data_t *d, ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { +static ERL_NIF_TERM close_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { enum efile_state_t previous_state; + efile_data_t *d; - if(argc != 0) { + ASSERT(argc == 1); + if(!get_file_data(env, argv[0], &d)) { return enif_make_badarg(env); } previous_state = erts_atomic32_cmpxchg_acqb(&d->state, - EFILE_STATE_CLOSED, EFILE_STATE_BUSY); + EFILE_STATE_CLOSED, EFILE_STATE_IDLE); - ASSERT(previous_state == EFILE_STATE_CLOSE_PENDING || - previous_state == EFILE_STATE_BUSY); + if(previous_state == EFILE_STATE_IDLE) { + posix_errno_t error; - if(previous_state == EFILE_STATE_BUSY) { enif_demonitor_process(env, d, &d->monitor); - if(!efile_close(d)) { - return posix_error_to_tuple(env, d->posix_errno); + if(!efile_close(d, &error)) { + return posix_error_to_tuple(env, error); } - } - return am_ok; + return am_ok; + } else { + /* CLOSE_PENDING should be impossible at this point since it requires + * a transition from BUSY; the only valid state here is CLOSED. */ + ASSERT(previous_state == EFILE_STATE_CLOSED); + + return posix_error_to_tuple(env, EINVAL); + } } static ERL_NIF_TERM read_nif_impl(efile_data_t *d, ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { @@ -1190,7 +1248,7 @@ static posix_errno_t read_file(efile_data_t *d, size_t size, ErlNifBinary *resul } static ERL_NIF_TERM read_file_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { - posix_errno_t posix_errno; + posix_errno_t posix_errno, ignored; efile_fileinfo_t info = {0}; efile_path_t path; @@ -1211,7 +1269,9 @@ static ERL_NIF_TERM read_file_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM a } posix_errno = read_file(d, info.size, &result); - enif_release_resource(d); + + erts_atomic32_set_acqb(&d->state, EFILE_STATE_CLOSED); + efile_close(d, &ignored); if(posix_errno) { return posix_error_to_tuple(env, posix_errno); diff --git a/erts/emulator/nifs/common/prim_file_nif.h b/erts/emulator/nifs/common/prim_file_nif.h index 099c06c48c..b2e30c59dd 100644 --- a/erts/emulator/nifs/common/prim_file_nif.h +++ b/erts/emulator/nifs/common/prim_file_nif.h @@ -159,8 +159,11 @@ posix_errno_t efile_open(const efile_path_t *path, enum efile_modes_t modes, ErlNifResourceType *nif_type, efile_data_t **d); /** @brief Closes a file. The file must have entered the CLOSED state prior to - * calling this to prevent double close. */ -int efile_close(efile_data_t *d); + * calling this to prevent double close. + * + * Note that the file is completely invalid after this point, so the error code + * is provided in \c error rather than d->posix_errno. */ +int efile_close(efile_data_t *d, posix_errno_t *error); /* **** **** **** **** **** **** **** **** **** **** **** **** **** **** **** */ diff --git a/erts/emulator/nifs/unix/unix_prim_file.c b/erts/emulator/nifs/unix/unix_prim_file.c index dea73db18a..169b193993 100644 --- a/erts/emulator/nifs/unix/unix_prim_file.c +++ b/erts/emulator/nifs/unix/unix_prim_file.c @@ -202,21 +202,24 @@ posix_errno_t efile_open(const efile_path_t *path, enum efile_modes_t modes, return errno; } -int efile_close(efile_data_t *d) { +int efile_close(efile_data_t *d, posix_errno_t *error) { efile_unix_t *u = (efile_unix_t*)d; int fd; + ASSERT(enif_thread_type() == ERL_NIF_THR_DIRTY_IO_SCHEDULER); ASSERT(erts_atomic32_read_nob(&d->state) == EFILE_STATE_CLOSED); ASSERT(u->fd != -1); fd = u->fd; u->fd = -1; + enif_release_resource(d); + /* close(2) either always closes (*BSD, Linux) or leaves the fd in an * undefined state (POSIX 2008, Solaris), so we must not retry on EINTR. */ if(close(fd) < 0) { - u->common.posix_errno = errno; + *error = errno; return 0; } diff --git a/erts/emulator/nifs/win32/win_prim_file.c b/erts/emulator/nifs/win32/win_prim_file.c index 602a282dd1..d0aa70542f 100644 --- a/erts/emulator/nifs/win32/win_prim_file.c +++ b/erts/emulator/nifs/win32/win_prim_file.c @@ -466,18 +466,21 @@ posix_errno_t efile_open(const efile_path_t *path, enum efile_modes_t modes, } } -int efile_close(efile_data_t *d) { +int efile_close(efile_data_t *d, posix_errno_t *error) { efile_win_t *w = (efile_win_t*)d; HANDLE handle; + ASSERT(enif_thread_type() == ERL_NIF_THR_DIRTY_IO_SCHEDULER); ASSERT(erts_atomic32_read_nob(&d->state) == EFILE_STATE_CLOSED); ASSERT(w->handle != INVALID_HANDLE_VALUE); handle = w->handle; w->handle = INVALID_HANDLE_VALUE; + enif_release_resource(d); + if(!CloseHandle(handle)) { - w->common.posix_errno = windows_to_posix_errno(GetLastError()); + *error = windows_to_posix_errno(GetLastError()); return 0; } diff --git a/erts/emulator/test/Makefile b/erts/emulator/test/Makefile index bf00de2204..6a064ec8d4 100644 --- a/erts/emulator/test/Makefile +++ b/erts/emulator/test/Makefile @@ -33,6 +33,7 @@ MODULES= \ after_SUITE \ alloc_SUITE \ async_ports_SUITE \ + atomics_SUITE \ beam_SUITE \ beam_literals_SUITE \ bif_SUITE \ @@ -50,6 +51,7 @@ MODULES= \ call_trace_SUITE \ code_SUITE \ code_parallel_load_SUITE \ + counters_SUITE \ crypto_SUITE \ ddll_SUITE \ decode_packet_SUITE \ @@ -92,6 +94,7 @@ MODULES= \ port_SUITE \ port_bif_SUITE \ prim_eval_SUITE \ + persistent_term_SUITE \ process_SUITE \ pseudoknot_SUITE \ receive_SUITE \ diff --git a/erts/emulator/test/atomics_SUITE.erl b/erts/emulator/test/atomics_SUITE.erl new file mode 100644 index 0000000000..8c42354770 --- /dev/null +++ b/erts/emulator/test/atomics_SUITE.erl @@ -0,0 +1,147 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(atomics_SUITE). + +-include_lib("common_test/include/ct.hrl"). + +-compile(export_all). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [signed, unsigned, bad, signed_limits, unsigned_limits]. + +signed(Config) when is_list(Config) -> + Size = 10, + Ref = atomics:new(Size,[]), + #{size:=Size, memory:=Memory} = atomics:info(Ref), + {_,true} = {Memory, Memory > Size*8}, + {_,true} = {Memory, Memory < Size*max_atomic_sz() + 100}, + [signed_do(Ref, Ix) || Ix <- lists:seq(1, Size)], + ok. + +signed_do(Ref, Ix) -> + 0 = atomics:get(Ref, Ix), + ok = atomics:put(Ref, Ix, 3), + ok = atomics:add(Ref, Ix, 14), + 17 = atomics:get(Ref, Ix), + 20 = atomics:add_get(Ref, Ix, 3), + -3 = atomics:add_get(Ref, Ix, -23), + 17 = atomics:add_get(Ref, Ix, 20), + ok = atomics:sub(Ref, Ix, 4), + 13 = atomics:get(Ref, Ix), + -7 = atomics:sub_get(Ref, Ix, 20), + 3 = atomics:sub_get(Ref, Ix, -10), + 3 = atomics:exchange(Ref, Ix, 666), + ok = atomics:compare_exchange(Ref, Ix, 666, 777), + 777 = atomics:compare_exchange(Ref, Ix, 666, -666), + ok. + +unsigned(Config) when is_list(Config) -> + Size = 10, + Ref = atomics:new(Size,[{signed, false}]), + #{size:=Size, memory:=Memory} = atomics:info(Ref), + true = Memory > Size*8, + true = Memory < Size*max_atomic_sz() + 100, + [unsigned_do(Ref, Ix) || Ix <- lists:seq(1, Size)], + ok. + +unsigned_do(Ref, Ix) -> + 0 = atomics:get(Ref, Ix), + ok = atomics:put(Ref, Ix, 3), + ok = atomics:add(Ref, Ix, 14), + 17 = atomics:get(Ref, Ix), + 20 = atomics:add_get(Ref, Ix, 3), + ok = atomics:sub(Ref, Ix, 7), + 13 = atomics:get(Ref, Ix), + 3 = atomics:sub_get(Ref, Ix, 10), + 3 = atomics:exchange(Ref, Ix, 666), + ok = atomics:compare_exchange(Ref, Ix, 666, 777), + 777 = atomics:compare_exchange(Ref, Ix, 666, 888), + ok. + +bad(Config) when is_list(Config) -> + {'EXIT',{badarg,_}} = (catch atomics:new(0,[])), + {'EXIT',{badarg,_}} = (catch atomics:new(10,[bad])), + {'EXIT',{badarg,_}} = (catch atomics:new(10,[{signed,bad}])), + {'EXIT',{badarg,_}} = (catch atomics:new(10,[{signed,true}, bad])), + {'EXIT',{badarg,_}} = (catch atomics:new(10,[{signed,false} | bad])), + Ref = atomics:new(10,[]), + {'EXIT',{badarg,_}} = (catch atomics:get(1742, 7)), + {'EXIT',{badarg,_}} = (catch atomics:get(make_ref(), 7)), + {'EXIT',{badarg,_}} = (catch atomics:get(Ref, -1)), + {'EXIT',{badarg,_}} = (catch atomics:get(Ref, 0)), + {'EXIT',{badarg,_}} = (catch atomics:get(Ref, 11)), + {'EXIT',{badarg,_}} = (catch atomics:get(Ref, 7.0)), + ok. + + +signed_limits(Config) when is_list(Config) -> + Bits = 64, + Max = (1 bsl (Bits-1)) - 1, + Min = -(1 bsl (Bits-1)), + + Ref = atomics:new(1,[{signed, true}]), + #{max:=Max, min:=Min} = atomics:info(Ref), + 0 = atomics:get(Ref, 1), + ok = atomics:add(Ref, 1, Max), + Min = atomics:add_get(Ref, 1, 1), + Max = atomics:sub_get(Ref, 1, 1), + + IncrMax = (Max bsl 1) bor 1, + ok = atomics:put(Ref, 1, 0), + ok = atomics:add(Ref, 1, IncrMax), + -1 = atomics:get(Ref, 1), + {'EXIT',{badarg,_}} = (catch atomics:add(Ref, 1, IncrMax+1)), + {'EXIT',{badarg,_}} = (catch atomics:add(Ref, 1, Min-1)), + + ok. + +unsigned_limits(Config) when is_list(Config) -> + Bits = 64, + Max = (1 bsl Bits) - 1, + Min = 0, + + Ref = atomics:new(1,[{signed,false}]), + #{max:=Max, min:=Min} = atomics:info(Ref), + 0 = atomics:get(Ref, 1), + ok = atomics:add(Ref, 1, Max), + Min = atomics:add_get(Ref, 1, 1), + Max = atomics:sub_get(Ref, 1, 1), + + {'EXIT',{badarg,_}} = (catch atomics:add(Ref, 1, Max+1)), + IncrMin = -(1 bsl (Bits-1)), + ok = atomics:put(Ref, 1, -IncrMin), + ok = atomics:add(Ref, 1, IncrMin), + 0 = atomics:get(Ref, 1), + {'EXIT',{badarg,_}} = (catch atomics:add(Ref, 1, IncrMin-1)), + + ok. + +max_atomic_sz() -> + case erlang:system_info({wordsize, external}) of + 4 -> 16; + 8 -> + EI = erlang:system_info(ethread_info), + case lists:keyfind("64-bit native atomics", 1, EI) of + {_, "no", _} -> 16; + _ -> 8 + end + end. diff --git a/erts/emulator/test/counters_SUITE.erl b/erts/emulator/test/counters_SUITE.erl new file mode 100644 index 0000000000..b3f0358c1e --- /dev/null +++ b/erts/emulator/test/counters_SUITE.erl @@ -0,0 +1,234 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(counters_SUITE). + +-include_lib("common_test/include/ct.hrl"). + +-export([suite/0, all/0]). +-export([basic/1, bad/1, limits/1, indep/1, write_concurrency/1]). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [basic, bad, limits, indep, write_concurrency]. + +basic(Config) when is_list(Config) -> + Size = 10, + [begin + Ref = counters:new(Size,[Type]), + #{size:=Size, memory:=Memory} = counters:info(Ref), + check_memory(Type, Memory, Size), + [basic_do(Ref, Ix) || Ix <- lists:seq(1, Size)] + end + || Type <- [atomics, write_concurrency]], + ok. + +basic_do(Ref, Ix) -> + 0 = counters:get(Ref, Ix), + ok = counters:add(Ref, Ix, 3), + 3 = counters:get(Ref, Ix), + ok = counters:add(Ref, Ix, 14), + 17 = counters:get(Ref, Ix), + ok = counters:add(Ref, Ix, -20), + -3 = counters:get(Ref, Ix), + ok = counters:add(Ref, Ix, 100), + 97 = counters:get(Ref, Ix), + ok = counters:sub(Ref, Ix, 20), + 77 = counters:get(Ref, Ix), + ok = counters:sub(Ref, Ix, -10), + 87 = counters:get(Ref, Ix), + ok = counters:put(Ref, Ix, 0), + 0 = counters:get(Ref, Ix), + ok = counters:put(Ref, Ix, 123), + 123 = counters:get(Ref, Ix), + ok = counters:put(Ref, Ix, -321), + -321 = counters:get(Ref, Ix), + ok. + +check_memory(atomics, Memory, Size) -> + {_,true} = {Memory, Memory > Size*8}, + {_,true} = {Memory, Memory < Size*max_atomic_sz() + 100}; +check_memory(write_concurrency, Memory, Size) -> + NWords = erlang:system_info(schedulers) + 1, + {_,true} = {Memory, Memory > NWords*Size*8}, + {_,true} = {Memory, Memory < NWords*(Size+7)*max_atomic_sz() + 100}. + +max_atomic_sz() -> + case erlang:system_info({wordsize, external}) of + 4 -> 16; + 8 -> + EI = erlang:system_info(ethread_info), + case lists:keyfind("64-bit native atomics", 1, EI) of + {_, "no", _} -> 16; + _ -> 8 + end + end. + +bad(Config) when is_list(Config) -> + {'EXIT',{badarg,_}} = (catch counters:new(0,[])), + {'EXIT',{badarg,_}} = (catch counters:new(10,[bad])), + {'EXIT',{badarg,_}} = (catch counters:new(10,[atomic, bad])), + {'EXIT',{badarg,_}} = (catch counters:new(10,[write_concurrency | bad])), + Ref = counters:new(10,[]), + {'EXIT',{badarg,_}} = (catch counters:get(1742, 7)), + {'EXIT',{badarg,_}} = (catch counters:get(make_ref(), 7)), + {'EXIT',{badarg,_}} = (catch counters:get(Ref, -1)), + {'EXIT',{badarg,_}} = (catch counters:get(Ref, 0)), + {'EXIT',{badarg,_}} = (catch counters:get(Ref, 11)), + {'EXIT',{badarg,_}} = (catch counters:get(Ref, 7.0)), + ok. + + +limits(Config) when is_list(Config) -> + limits_do(counters:new(1,[atomics])), + limits_do(counters:new(1,[write_concurrency])), + ok. + +limits_do(Ref) -> + Bits = 64, + Max = (1 bsl (Bits-1)) - 1, + Min = -(1 bsl (Bits-1)), + + 0 = counters:get(Ref, 1), + ok = counters:put(Ref, 1, Max), + Max = counters:get(Ref, 1), + ok = counters:add(Ref, 1, 1), + Min = counters:get(Ref, 1), + ok = counters:sub(Ref, 1, 1), + Max = counters:get(Ref, 1), + ok = counters:put(Ref, 1, Min), + Min = counters:get(Ref, 1), + + IncrMax = (Max bsl 1) bor 1, + ok = counters:put(Ref, 1, 0), + ok = counters:add(Ref, 1, IncrMax), + -1 = counters:get(Ref, 1), + {'EXIT',{badarg,_}} = (catch counters:add(Ref, 1, IncrMax+1)), + {'EXIT',{badarg,_}} = (catch counters:add(Ref, 1, Min-1)), + {'EXIT',{badarg,_}} = (catch counters:put(Ref, 1, Max+1)), + {'EXIT',{badarg,_}} = (catch counters:add(Ref, 1, Min-1)), + ok. + + +%% Verify that independent workers, using different counters +%% within the same array, do not interfere with each other. +indep(Config) when is_list(Config) -> + NScheds = erlang:system_info(schedulers), + Ref = counters:new(NScheds,[write_concurrency]), + Rounds = 100, + Papa = self(), + Pids = [spawn_opt(fun () -> + Val = I*197, + counters:put(Ref, I, Val), + indep_looper(Rounds, Ref, I, Val), + Papa ! {self(), done} + end, + [link, {scheduler, I}]) + || I <- lists:seq(1, NScheds)], + [receive {P,done} -> ok end || P <- Pids], + ok. + +indep_looper(0, _, _ , _) -> + ok; +indep_looper(N, Ref, I, Val0) -> + %%io:format("Val0 = ~p\n", [Val0]), + Val0 = counters:get(Ref, I), + Val1 = indep_adder(Ref, I, Val0), + indep_subber(Ref, I, Val1), + Val2 = N*7 + I, + counters:put(Ref, I, Val2), + indep_looper(N-1, Ref, I, Val2). + +indep_adder(Ref, I, Val) when Val < (1 bsl 62) -> + %%io:format("adder Val = ~p\n", [Val]), + Incr = abs(Val div 2) + I + 984735, + counters:add(Ref, I, Incr), + Res = Val + Incr, + Res = counters:get(Ref, I), + indep_adder(Ref, I, Res); +indep_adder(_Ref, _I, Val) -> + Val. + +indep_subber(Ref, I, Val) when Val > -(1 bsl 62) -> + %%io:format("subber Val = ~p\n", [Val]), + Decr = (abs(Val div 2) + I + 725634), + counters:sub(Ref, I, Decr), + Res = Val - Decr, + Res = counters:get(Ref, I), + indep_subber(Ref, I, Res); +indep_subber(_Ref, _I, Val) -> + Val. + + + +%% Verify write_concurrency yields correct results. +write_concurrency(Config) when is_list(Config) -> + rand:seed(exs1024s), + io:format("*** SEED: ~p ***\n", [rand:export_seed()]), + NScheds = erlang:system_info(schedulers), + Size = 100, + Ref = counters:new(Size,[write_concurrency]), + Rounds = 1000, + Papa = self(), + Pids = [spawn_opt(fun Worker() -> + receive + {go, Ix, Incr} -> + wc_looper(Rounds, Ref, Ix, Incr), + Papa ! {self(), done, Rounds*Incr}, + Worker(); + stop -> + ok + end + end, + [link, {scheduler, N}]) + || N <- lists:seq(1, NScheds)], + [begin + Base = rand_log64(), + counters:put(Ref, Index, Base), + SendList = [{P,{go, Index, rand_log64()}} || P <- Pids], + [P ! Msg || {P,Msg} <- SendList], + Added = lists:sum([receive {P,done,Contrib} -> Contrib end || P <- Pids]), + Result = mask_sint64(Base+Added), + {_,Result} = {Result, counters:get(Ref, Index)} + end + || Index <- lists:seq(1, Size)], + + [begin unlink(P), P ! stop end || P <- Pids], + ok. + +wc_looper(0, _, _, _) -> + ok; +wc_looper(N, Ref, Ix, Incr) -> + counters:add(Ref, Ix, Incr), + wc_looper(N-1, Ref, Ix, Incr). + +mask_sint64(X) -> + SMask = 1 bsl 63, + UMask = SMask - 1, + (X band UMask) - (X band SMask). + +%% A random signed 64-bit integer +%% with a uniformly distributed number of significant bits. +rand_log64() -> + Uint = round(math:pow(2, rand:uniform()*63)), + case rand:uniform(2) of + 1 -> -Uint; + 2 -> Uint + end. diff --git a/erts/emulator/test/persistent_term_SUITE.erl b/erts/emulator/test/persistent_term_SUITE.erl new file mode 100644 index 0000000000..58cd3276b0 --- /dev/null +++ b/erts/emulator/test/persistent_term_SUITE.erl @@ -0,0 +1,614 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%5 +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(persistent_term_SUITE). +-include_lib("common_test/include/ct.hrl"). + +-export([all/0,suite/0, + basic/1,purging/1,sharing/1,get_trapping/1, + info/1,info_trapping/1,killed_while_trapping/1, + off_heap_values/1,keys/1,collisions/1, + init_restart/1]). + +%% +-export([test_init_restart_cmd/1]). + +suite() -> + [{ct_hooks,[ts_install_cth]}, + {timetrap,{minutes,10}}]. + +all() -> + [basic,purging,sharing,get_trapping,info,info_trapping, + killed_while_trapping,off_heap_values,keys,collisions, + init_restart]. + +basic(_Config) -> + Chk = chk(), + N = 777, + Seq = lists:seq(1, N), + par(2, N, Seq), + seq(3, Seq), + seq(3, Seq), %Same values. + _ = [begin + Key = {?MODULE,{key,I}}, + true = persistent_term:erase(Key), + false = persistent_term:erase(Key), + {'EXIT',{badarg,_}} = (catch persistent_term:get(Key)) + end || I <- Seq], + [] = [P || {{?MODULE,_},_}=P <- persistent_term:get()], + chk(Chk). + +par(C, N, Seq) -> + _ = [spawn_link(fun() -> + ok = persistent_term:put({?MODULE,{key,I}}, + {value,C*I}) + end) || I <- Seq], + Result = wait(N), + _ = [begin + Double = C*I, + {{?MODULE,{key,I}},{value,Double}} = Res + end || {I,Res} <- lists:zip(Seq, Result)], + ok. + +seq(C, Seq) -> + _ = [ok = persistent_term:put({?MODULE,{key,I}}, {value,C*I}) || + I <- Seq], + All = persistent_term:get(), + All = [P || {{?MODULE,_},_}=P <- persistent_term:get()], + All = [{Key,persistent_term:get(Key)} || {Key,_} <- All], + Result = lists:sort(All), + _ = [begin + Double = C*I, + {{?MODULE,{key,I}},{value,Double}} = Res + end || {I,Res} <- lists:zip(Seq, Result)], + ok. + +wait(N) -> + All = [P || {{?MODULE,_},_}=P <- persistent_term:get()], + case length(All) of + N -> + All = [{Key,persistent_term:get(Key)} || {Key,_} <- All], + lists:sort(All); + _ -> + receive after 10 -> ok end, + wait(N) + end. + +%% Make sure that terms that have been erased are copied into all +%% processes that still hold a pointer to them. + +purging(_Config) -> + Chk = chk(), + do_purging(fun(K) -> persistent_term:put(K, {?MODULE,new}) end, + replaced), + do_purging(fun persistent_term:erase/1, erased), + chk(Chk). + +do_purging(Eraser, Type) -> + Parent = self(), + Key = {?MODULE,?FUNCTION_NAME}, + ok = persistent_term:put(Key, {term,[<<"abc",0:777/unit:8>>]}), + Ps0 = [spawn_monitor(fun() -> purging_tester(Parent, Key) end) || + _ <- lists:seq(1, 50)], + Ps = maps:from_list(Ps0), + purging_recv(gotten, Ps), + Eraser(Key), + _ = [P ! {Parent,Type} || P <- maps:keys(Ps)], + purging_wait(Ps). + +purging_recv(Tag, Ps) when map_size(Ps) > 0 -> + receive + {Pid,Tag} -> + true = is_map_key(Pid, Ps), + purging_recv(Tag, maps:remove(Pid, Ps)) + end; +purging_recv(_, _) -> ok. + +purging_wait(Ps) when map_size(Ps) > 0 -> + receive + {'DOWN',Ref,process,Pid,Reason} -> + normal = Reason, + Ref = map_get(Pid, Ps), + purging_wait(maps:remove(Pid, Ps)) + end; +purging_wait(_) -> ok. + +purging_tester(Parent, Key) -> + Term = persistent_term:get(Key), + purging_check_term(Term), + 0 = erts_debug:size_shared(Term), + Parent ! {self(),gotten}, + receive + {Parent,erased} -> + {'EXIT',{badarg,_}} = (catch persistent_term:get(Key)), + purging_tester_1(Term); + {Parent,replaced} -> + {?MODULE,new} = persistent_term:get(Key), + purging_tester_1(Term) + end. + +%% Wait for the term to be copied into this process. +purging_tester_1(Term) -> + purging_check_term(Term), + receive after 1 -> ok end, + case erts_debug:size_shared(Term) of + 0 -> + purging_tester_1(Term); + Size -> + %% The term has been copied into this process. + purging_check_term(Term), + Size = erts_debug:size(Term) + end. + +purging_check_term({term,[<<"abc",0:777/unit:8>>]}) -> + ok. + +%% Test that sharing is preserved when storing terms. + +sharing(_Config) -> + Chk = chk(), + Depth = 10, + Size = 2*Depth, + Shared = lists:foldl(fun(_, A) -> [A|A] end, + [], lists:seq(1, Depth)), + Size = erts_debug:size(Shared), + Key = {?MODULE,?FUNCTION_NAME}, + ok = persistent_term:put(Key, Shared), + SharedStored = persistent_term:get(Key), + Size = erts_debug:size(SharedStored), + 0 = erts_debug:size_shared(SharedStored), + + {Pid,Ref} = spawn_monitor(fun() -> + Term = persistent_term:get(Key), + Size = erts_debug:size(Term), + 0 = erts_debug:size_shared(Term), + true = Term =:= SharedStored + end), + receive + {'DOWN',Ref,process,Pid,normal} -> + true = persistent_term:erase(Key), + Size = erts_debug:size(SharedStored), + chk(Chk) + end. + +%% Test trapping of persistent_term:get/0. + +get_trapping(_Config) -> + Chk = chk(), + + %% Assume that the get/0 traps after 4000 iterations + %% in a non-debug emulator. + N = case test_server:timetrap_scale_factor() of + 1 -> 10000; + _ -> 1000 + end, + spawn_link(fun() -> get_trapping_create(N) end), + All = do_get_trapping(N, []), + N = get_trapping_check_result(lists:sort(All), 1), + erlang:garbage_collect(), + get_trapping_erase(N), + chk(Chk). + +do_get_trapping(N, Prev) -> + case persistent_term:get() of + Prev when length(Prev) >= N -> + All = [P || {{?MODULE,{get_trapping,_}},_}=P <- Prev], + case length(All) of + N -> All; + _ -> do_get_trapping(N, Prev) + end; + New -> + receive after 1 -> ok end, + do_get_trapping(N, New) + end. + +get_trapping_create(0) -> + ok; +get_trapping_create(N) -> + ok = persistent_term:put({?MODULE,{get_trapping,N}}, N), + get_trapping_create(N-1). + +get_trapping_check_result([{{?MODULE,{get_trapping,N}},N}|T], N) -> + get_trapping_check_result(T, N+1); +get_trapping_check_result([], N) -> N-1. + +get_trapping_erase(0) -> + ok; +get_trapping_erase(N) -> + true = persistent_term:erase({?MODULE,{get_trapping,N}}), + get_trapping_erase(N-1). + +%% Test retrieving information about persistent terms. + +info(_Config) -> + Chk = chk(), + + %% White box test of info/0. + N = 100, + try + Overhead = info_literal_area_overhead(), + io:format("Overhead = ~p\n", [Overhead]), + info_wb(N, Overhead, info_info()) + after + _ = [_ = persistent_term:erase({?MODULE,I}) || + I <- lists:seq(1, N)] + end, + + chk(Chk). + +%% White box test of persistent_term:info/0. We take into account +%% that there might already exist persistent terms (created by the +%% OTP standard libraries), but we assume that they are not +%% changed during the execution of this test case. + +info_wb(0, _, _) -> + ok; +info_wb(N, Overhead, {BaseCount,BaseMemory}) -> + Key = {?MODULE,N}, + Value = lists:seq(1, N), + ok = persistent_term:put(Key, Value), + + %% Calculate the extra memory needed for this term. + WordSize = erlang:system_info(wordsize), + ExtraMemory = Overhead + 2 * N * WordSize, + + %% Call persistent_term:info/0. + {Count,Memory} = info_info(), + + %% There should be one more persistent term. + Count = BaseCount + 1, + + %% Verify that the amount of memory is correct. + case BaseMemory + ExtraMemory of + Memory -> + %% Exactly right. The size of the hash table was not changed. + ok; + Expected -> + %% The size of the hash table has been doubled to avoid filling + %% the table to more than 50 percent. The previous number + %% of entries must have been exactly half the size of the + %% hash table. The expected number of extra words added by + %% the resizing will be twice that number. + ExtraWords = BaseCount * 2, + true = ExtraWords * WordSize =:= (Memory - Expected) + end, + info_wb(N-1, Overhead, {Count,Memory}). + +info_info() -> + #{count:=Count,memory:=Memory} = persistent_term:info(), + true = is_integer(Count) andalso Count >= 0, + true = is_integer(Memory) andalso Memory >= 0, + {Count,Memory}. + +%% Calculate the number of extra bytes needed for storing each term in +%% the literal, assuming that the key is a tuple of size 2 with +%% immediate elements. The calculated number is the size of the +%% ErtsLiteralArea struct excluding the storage for the literal term +%% itself. + +info_literal_area_overhead() -> + Key1 = {?MODULE,1}, + Key2 = {?MODULE,2}, + #{memory:=Mem0} = persistent_term:info(), + ok = persistent_term:put(Key1, literal), + #{memory:=Mem1} = persistent_term:info(), + ok = persistent_term:put(Key2, literal), + #{memory:=Mem2} = persistent_term:info(), + true = persistent_term:erase(Key1), + true = persistent_term:erase(Key2), + + %% The size of the hash table may have doubled when inserting + %% one of the keys. To avoiding counting the change in the hash + %% table size, take the smaller size increase. + min(Mem2-Mem1, Mem1-Mem0). + +%% Test trapping of persistent_term:info/0. + +info_trapping(_Config) -> + Chk = chk(), + + %% Assume that the info/0 traps after 4000 iterations + %% in a non-debug emulator. + N = case test_server:timetrap_scale_factor() of + 1 -> 10000; + _ -> 1000 + end, + spawn_link(fun() -> info_trapping_create(N) end), + All = do_info_trapping(N, 0), + N = info_trapping_check_result(lists:sort(All), 1), + erlang:garbage_collect(), + info_trapping_erase(N), + chk(Chk). + +do_info_trapping(N, PrevMem) -> + case info_info() of + {N,Mem} -> + true = Mem >= PrevMem, + All = [P || {{?MODULE,{info_trapping,_}},_}=P <- persistent_term:get()], + case length(All) of + N -> All; + _ -> do_info_trapping(N, PrevMem) + end; + {_,Mem} -> + true = Mem >= PrevMem, + receive after 1 -> ok end, + do_info_trapping(N, Mem) + end. + +info_trapping_create(0) -> + ok; +info_trapping_create(N) -> + ok = persistent_term:put({?MODULE,{info_trapping,N}}, N), + info_trapping_create(N-1). + +info_trapping_check_result([{{?MODULE,{info_trapping,N}},N}|T], N) -> + info_trapping_check_result(T, N+1); +info_trapping_check_result([], N) -> N-1. + +info_trapping_erase(0) -> + ok; +info_trapping_erase(N) -> + true = persistent_term:erase({?MODULE,{info_trapping,N}}), + info_trapping_erase(N-1). + +%% Test that hash tables are deallocated if a process running +%% persistent_term:get/0 is killed. + +killed_while_trapping(_Config) -> + Chk = chk(), + N = case test_server:timetrap_scale_factor() of + 1 -> 20000; + _ -> 2000 + end, + kwt_put(N), + kwt_spawn(10), + kwt_erase(N), + chk(Chk). + +kwt_put(0) -> + ok; +kwt_put(N) -> + ok = persistent_term:put({?MODULE,{kwt,N}}, N), + kwt_put(N-1). + +kwt_spawn(0) -> + ok; +kwt_spawn(N) -> + Pids = [spawn(fun kwt_getter/0) || _ <- lists:seq(1, 20)], + erlang:yield(), + _ = [exit(Pid, kill) || Pid <- Pids], + kwt_spawn(N-1). + +kwt_getter() -> + _ = persistent_term:get(), + kwt_getter(). + +kwt_erase(0) -> + ok; +kwt_erase(N) -> + true = persistent_term:erase({?MODULE,{kwt,N}}), + kwt_erase(N-1). + +%% Test storing off heap values (such as ref-counted binaries). + +off_heap_values(_Config) -> + Chk = chk(), + Key = {?MODULE,?FUNCTION_NAME}, + Val = {a,list_to_binary(lists:seq(0, 255)),make_ref(),fun() -> ok end}, + ok = persistent_term:put(Key, Val), + FetchedVal = persistent_term:get(Key), + Val = FetchedVal, + true = persistent_term:erase(Key), + off_heap_values_wait(FetchedVal, Val), + chk(Chk). + +off_heap_values_wait(FetchedVal, Val) -> + case erts_debug:size_shared(FetchedVal) of + 0 -> + Val = FetchedVal, + ok; + _ -> + erlang:yield(), + off_heap_values_wait(FetchedVal, Val) + end. + +%% Test some more data types as keys. Use the module name as a key +%% to minimize the risk of collision with any key used +%% by the OTP libraries. + +keys(_Config) -> + Chk = chk(), + do_key(?MODULE), + do_key([?MODULE]), + do_key(?MODULE_STRING), + do_key(list_to_binary(?MODULE_STRING)), + chk(Chk). + +do_key(Key) -> + Val = term_to_binary(Key), + ok = persistent_term:put(Key, Val), + StoredVal = persistent_term:get(Key), + Val = StoredVal, + true = persistent_term:erase(Key). + +%% Create persistent terms with keys that are known to collide. +%% Delete them in random order, making sure that all others +%% terms can still be found. + +collisions(_Config) -> + Chk = chk(), + + %% Create persistent terms with random keys. + Keys = lists:flatten(colliding_keys()), + Kvs = [{K,rand:uniform(1000)} || K <- Keys], + _ = [ok = persistent_term:put(K, V) || {K,V} <- Kvs], + _ = [V = persistent_term:get(K) || {K,V} <- Kvs], + + %% Now delete the persistent terms in random order. + collisions_delete(lists:keysort(2, Kvs)), + + chk(Chk). + +collisions_delete([{Key,Val}|Kvs]) -> + Val = persistent_term:get(Key), + true = persistent_term:erase(Key), + true = lists:sort(persistent_term:get()) =:= lists:sort(Kvs), + _ = [V = persistent_term:get(K) || {K,V} <- Kvs], + collisions_delete(Kvs); +collisions_delete([]) -> + ok. + +colliding_keys() -> + %% Collisions found by Jesper L. Andersen for breaking maps. + L = [[764492191,2361333849], + [49527266765044,90940896816021,20062927283041,267080852079651], + [249858369443708,206247021789428,20287304470696,25847120931175], + [10645228898670,224705626119556,267405565521452,258214397180678], + [264783762221048,166955943492306,98802957003141,102012488332476], + [69425677456944,177142907243411,137138950917722,228865047699598], + [116031213307147,29203342183358,37406949328742,255198080174323], + [200358182338308,235207156008390,120922906095920,116215987197289], + [58728890318426,68877471005069,176496507286088,221041411345780], + [91094120814795,50665258299931,256093108116737,19777509566621], + [74646746200247,98350487270564,154448261001199,39881047281135], + [23408943649483,164410325820923,248161749770122,274558342231648], + [169531547115055,213630535746863,235098262267796,200508473898303], + [235098564415817,85039146398174,51721575960328,173069189684390], + [176136386396069,155368359051606,147817099696487,265419485459634], + [137542881551462,40028925519736,70525669519846,63445773516557], + [173854695142814,114282444507812,149945832627054,99605565798831], + [177686773562184,127158716984798,132495543008547], + [227073396444896,139667311071766,158915951283562], + [26212438434289,94902985796531,198145776057315], + [266279278943923,58550737262493,74297973216378], + [32373606512065,131854353044428,184642643042326], + [34335377662439,85341895822066,273492717750246]], + + %% Verify that the keys still collide (this will fail if the + %% internal hash function has been changed). + erts_debug:set_internal_state(available_internal_state, true), + try + case erlang:system_info(wordsize) of + 8 -> + verify_colliding_keys(L); + 4 -> + %% Not guaranteed to collide on a 32-bit system. + ok + end + after + erts_debug:set_internal_state(available_internal_state, false) + end, + + L. + +verify_colliding_keys([[K|Ks]|Gs]) -> + Hash = internal_hash(K), + [Hash] = lists:usort([internal_hash(Key) || Key <- Ks]), + verify_colliding_keys(Gs); +verify_colliding_keys([]) -> + ok. + +internal_hash(Term) -> + erts_debug:get_internal_state({internal_hash,Term}). + +%% Test that all persistent terms are erased by init:restart/0. + +init_restart(_Config) -> + File = "command_file", + ok = file:write_file(File, term_to_binary(restart)), + {ok,[[Erl]]} = init:get_argument(progname), + ModPath = filename:dirname(code:which(?MODULE)), + Cmd = Erl ++ " -pa " ++ ModPath ++ " -noshell " + "-run " ++ ?MODULE_STRING ++ " test_init_restart_cmd " ++ + File, + io:format("~s\n", [Cmd]), + Expected = "12ok", + case os:cmd(Cmd) of + Expected -> + ok; + Actual -> + io:format("Expected: ~s", [Expected]), + io:format("Actual: ~s\n", [Actual]), + ct:fail(unexpected_output) + end. + +test_init_restart_cmd([File]) -> + try + do_test_init_restart_cmd(File) + catch + C:R -> + io:format("\n~p ~p\n", [C,R]), + halt() + end, + receive + _ -> ok + end. + +do_test_init_restart_cmd(File) -> + {ok,Bin} = file:read_file(File), + Seq = lists:seq(1, 50), + case binary_to_term(Bin) of + restart -> + _ = [persistent_term:put({?MODULE,I}, {value,I}) || + I <- Seq], + ok = file:write_file(File, term_to_binary(was_restarted)), + io:put_chars("1"), + init:restart(), + receive + _ -> ok + end; + was_restarted -> + io:put_chars("2"), + ok = file:delete(File), + _ = [begin + Key = {?MODULE,I}, + {'EXIT',{badarg,_}} = (catch persistent_term:get(Key)) + end || I <- Seq], + io:put_chars("ok"), + init:stop() + end. + +%% Check that there is the same number of persistents terms before +%% and after each test case. + +chk() -> + persistent_term:info(). + +chk(Chk) -> + Chk = persistent_term:info(), + Key = {?MODULE,?FUNCTION_NAME}, + ok = persistent_term:put(Key, {term,Chk}), + Term = persistent_term:get(Key), + true = persistent_term:erase(Key), + chk_not_stuck(Term), + ok. + +chk_not_stuck(Term) -> + %% Hash tables to be deleted are put onto a queue. + %% Make sure that the queue isn't stuck by a table with + %% a non-zero ref count. + + case erts_debug:size_shared(Term) of + 0 -> + erlang:yield(), + chk_not_stuck(Term); + _ -> + ok + end. diff --git a/erts/preloaded/ebin/atomics.beam b/erts/preloaded/ebin/atomics.beam Binary files differnew file mode 100644 index 0000000000..1de97fa668 --- /dev/null +++ b/erts/preloaded/ebin/atomics.beam diff --git a/erts/preloaded/ebin/counters.beam b/erts/preloaded/ebin/counters.beam Binary files differnew file mode 100644 index 0000000000..4e1a3566f7 --- /dev/null +++ b/erts/preloaded/ebin/counters.beam diff --git a/erts/preloaded/ebin/erts_internal.beam b/erts/preloaded/ebin/erts_internal.beam Binary files differindex 15c59de80a..651d5e9d05 100644 --- a/erts/preloaded/ebin/erts_internal.beam +++ b/erts/preloaded/ebin/erts_internal.beam diff --git a/erts/preloaded/ebin/init.beam b/erts/preloaded/ebin/init.beam Binary files differindex 858a9dc63e..1e60ef7e88 100644 --- a/erts/preloaded/ebin/init.beam +++ b/erts/preloaded/ebin/init.beam diff --git a/erts/preloaded/ebin/persistent_term.beam b/erts/preloaded/ebin/persistent_term.beam Binary files differnew file mode 100644 index 0000000000..79ef03b9a6 --- /dev/null +++ b/erts/preloaded/ebin/persistent_term.beam diff --git a/erts/preloaded/ebin/prim_file.beam b/erts/preloaded/ebin/prim_file.beam Binary files differindex df611f2bb0..d0435a10ef 100644 --- a/erts/preloaded/ebin/prim_file.beam +++ b/erts/preloaded/ebin/prim_file.beam diff --git a/erts/preloaded/src/Makefile b/erts/preloaded/src/Makefile index 4333f6643a..e1bd5bc295 100644 --- a/erts/preloaded/src/Makefile +++ b/erts/preloaded/src/Makefile @@ -47,7 +47,10 @@ PRE_LOADED_ERL_MODULES = \ erts_internal \ erl_tracer \ erts_literal_area_collector \ - erts_dirty_process_signal_handler + erts_dirty_process_signal_handler \ + atomics \ + counters \ + persistent_term PRE_LOADED_BEAM_MODULES = \ prim_eval diff --git a/erts/preloaded/src/atomics.erl b/erts/preloaded/src/atomics.erl new file mode 100644 index 0000000000..d1fe5e65cf --- /dev/null +++ b/erts/preloaded/src/atomics.erl @@ -0,0 +1,119 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% Purpose : Main atomics API module. + +-module(atomics). + +-export([new/2, + put/3, get/2, + add/3, add_get/3, + sub/3, sub_get/3, + exchange/3, compare_exchange/4, + info/1]). + +-export_type([atomics_ref/0]). + +-opaque atomics_ref() :: reference(). + +-define(OPT_SIGNED, (1 bsl 0)). +-define(OPT_DEFAULT, ?OPT_SIGNED). + +-spec new(Arity, Opts) -> atomics_ref() when + Arity :: pos_integer(), + Opts :: [Opt], + Opt :: {signed, boolean()}. +new(Arity, Opts) -> + erts_internal:atomics_new(Arity, encode_opts(Opts, ?OPT_DEFAULT)). + +encode_opts([{signed, true}|T], Acc) -> + encode_opts(T, Acc bor ?OPT_SIGNED); +encode_opts([{signed, false}|T], Acc) -> + encode_opts(T, Acc band (bnot ?OPT_SIGNED)); +encode_opts([], Acc) -> + Acc; +encode_opts(_, _) -> + erlang:error(badarg). + +-spec put(Ref, Ix, Value) -> ok when + Ref :: atomics_ref(), + Ix :: integer(), + Value :: integer(). +put(_Ref, _Ix, _Value) -> + erlang:nif_error(undef). + +-spec get(Ref, Ix) -> integer() when + Ref :: atomics_ref(), + Ix :: integer(). +get(_Ref, _Ix) -> + erlang:nif_error(undef). + +-spec add(Ref, Ix, Incr) -> ok when + Ref :: atomics_ref(), + Ix :: integer(), + Incr :: integer(). +add(_Ref, _Ix, _Incr) -> + erlang:nif_error(undef). + +-spec add_get(Ref, Ix, Incr) -> integer() when + Ref :: atomics_ref(), + Ix :: integer(), + Incr :: integer(). +add_get(_Ref, _Ix, _Incr) -> + erlang:nif_error(undef). + +-spec sub(Ref, Ix, Decr) -> ok when + Ref :: atomics_ref(), + Ix :: integer(), + Decr :: integer(). +sub(Ref, Ix, Decr) -> + ?MODULE:add(Ref, Ix, -Decr). + +-spec sub_get(Ref, Ix, Decr) -> integer() when + Ref :: atomics_ref(), + Ix :: integer(), + Decr :: integer(). +sub_get(Ref, Ix, Decr) -> + ?MODULE:add_get(Ref, Ix, -Decr). + +-spec exchange(Ref, Ix, Desired) -> integer() when + Ref :: atomics_ref(), + Ix :: integer(), + Desired :: integer(). +exchange(_Ref, _Ix, _Desired) -> + erlang:nif_error(undef). + +-spec compare_exchange(Ref, Ix, Expected, Desired) -> ok | integer() when + Ref :: atomics_ref(), + Ix :: integer(), + Expected :: integer(), + Desired :: integer(). +compare_exchange(_Ref, _Ix, _Expected, _Desired) -> + erlang:nif_error(undef). + +-spec info(Ref) -> Info when + Ref :: atomics_ref(), + Info :: #{'size':=Size,'max':=Max,'min':=Min,'memory':=Memory}, + Size :: non_neg_integer(), + Max :: integer(), + Min :: integer(), + Memory :: non_neg_integer(). +info(_Ref) -> + erlang:nif_error(undef). diff --git a/erts/preloaded/src/counters.erl b/erts/preloaded/src/counters.erl new file mode 100644 index 0000000000..a0e3035e0f --- /dev/null +++ b/erts/preloaded/src/counters.erl @@ -0,0 +1,104 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% Purpose : Main atomics API module. + +-module(counters). + +-export([new/2, + get/2, + add/3, + sub/3, + put/3, + info/1]). + +-export_type([counters_ref/0]). + +-opaque counters_ref() :: {atomics, reference()} | {write_concurrency, reference()}. + +-spec new(Size, Opts) -> counters_ref() when + Size :: pos_integer(), + Opts :: [Opt], + Opt :: atomics | write_concurrency. +new(Size, [atomics]) -> + {atomics, atomics:new(Size, [{signed, true}])}; +new(Size, [write_concurrency]) -> + {write_concurrency, erts_internal:counters_new(Size)}; +new(Size, []) -> + new(Size, [atomics]); +new(_, _) -> + erlang:error(badarg). + +-spec get(Ref, Ix) -> integer() when + Ref :: counters_ref(), + Ix :: integer(). +get({atomics,Ref}, Ix) -> + atomics:get(Ref, Ix); +get({write_concurrency, Ref}, Ix) -> + erts_internal:counters_get(Ref, Ix); +get(_, _) -> + erlang:error(badarg). + + + +-spec add(Ref, Ix, Incr) -> ok when + Ref :: counters_ref(), + Ix :: integer(), + Incr :: integer(). +add({atomics, Ref}, Ix, Incr) -> + atomics:add(Ref, Ix, Incr); +add({write_concurrency, Ref}, Ix, Incr) -> + erts_internal:counters_add(Ref, Ix, Incr); +add(_, _, _) -> + erlang:error(badarg). + + +-spec sub(Ref, Ix, Decr) -> ok when + Ref :: counters_ref(), + Ix :: integer(), + Decr :: integer(). +sub(Ref, Ix, Decr) -> + add(Ref, Ix, -Decr). + + +-spec put(Ref, Ix, Value) -> ok when + Ref :: counters_ref(), + Ix :: integer(), + Value :: integer(). +put({atomics, Ref}, Ix, Value) -> + atomics:put(Ref, Ix, Value); +put({write_concurrency, Ref}, Ix, Value) -> + erts_internal:counters_put(Ref, Ix, Value); +put(_, _, _) -> + erlang:error(badarg). + + +-spec info(Ref) -> Info when + Ref :: counters_ref(), + Info :: #{'size':=Size, 'memory':=Memory}, + Size :: non_neg_integer(), + Memory :: non_neg_integer(). +info({atomics, Ref}) -> + atomics:info(Ref); +info({write_concurrency, Ref}) -> + erts_internal:counters_info(Ref); +info(_) -> + erlang:error(badarg). + diff --git a/erts/preloaded/src/erts.app.src b/erts/preloaded/src/erts.app.src index 8c34c99a98..ed645d1191 100644 --- a/erts/preloaded/src/erts.app.src +++ b/erts/preloaded/src/erts.app.src @@ -33,12 +33,14 @@ prim_file, prim_inet, prim_zip, + atomics, + counters, zlib ]}, {registered, []}, {applications, []}, {env, []}, - {runtime_dependencies, ["stdlib-3.5", "kernel-6.1", "sasl-3.0.1"]} + {runtime_dependencies, ["stdlib-3.5", "kernel-6.1", "sasl-@OTP-13468@"]} ]}. %% vim: ft=erlang diff --git a/erts/preloaded/src/erts_internal.erl b/erts/preloaded/src/erts_internal.erl index 88f47e917b..8f29a569f2 100644 --- a/erts/preloaded/src/erts_internal.erl +++ b/erts/preloaded/src/erts_internal.erl @@ -90,6 +90,13 @@ -export([create_dist_channel/4]). +-export([erase_persistent_terms/0]). + +-export([atomics_new/2]). + +-export([counters_new/1, counters_get/2, counters_add/3, + counters_put/3, counters_info/1]). + %% %% Await result of send to port %% @@ -691,3 +698,31 @@ process_flag(_Pid, _Flag, _Value) -> create_dist_channel(_Node, _DistCtrlr, _Flags, _Ver) -> erlang:nif_error(undefined). + +-spec erase_persistent_terms() -> 'ok'. +erase_persistent_terms() -> + erlang:nif_error(undefined). + +-spec atomics_new(pos_integer(), pos_integer()) -> reference(). +atomics_new(_Arity, _EncOpts) -> + erlang:nif_error(undef). + +-spec counters_new(pos_integer()) -> reference(). +counters_new(_Size) -> + erlang:nif_error(undef). + +-spec counters_get(reference(), pos_integer()) -> integer(). +counters_get(_Ref, _Ix) -> + erlang:nif_error(undef). + +-spec counters_add(reference(), pos_integer(), integer()) -> ok. +counters_add(_Ref, _Ix, _Incr) -> + erlang:nif_error(undef). + +-spec counters_put(reference(), pos_integer(), integer()) -> ok. +counters_put(_Ref, _Ix, _Value) -> + erlang:nif_error(undef). + +-spec counters_info(reference()) -> #{}. +counters_info(_Ref) -> + erlang:nif_error(undef). diff --git a/erts/preloaded/src/init.erl b/erts/preloaded/src/init.erl index 253fcf7a1f..b4b8b3bf9b 100644 --- a/erts/preloaded/src/init.erl +++ b/erts/preloaded/src/init.erl @@ -552,6 +552,7 @@ stop(Reason,State) -> do_stop(restart,#state{start = Start, flags = Flags, args = Args}) -> %% Make sure we don't have any outstanding messages before doing the restart. flush(), + erts_internal:erase_persistent_terms(), boot(Start,Flags,Args); do_stop(reboot,_) -> halt(); diff --git a/erts/preloaded/src/persistent_term.erl b/erts/preloaded/src/persistent_term.erl new file mode 100644 index 0000000000..5d0c266127 --- /dev/null +++ b/erts/preloaded/src/persistent_term.erl @@ -0,0 +1,55 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(persistent_term). + +-export([erase/1,get/0,get/1,info/0,put/2]). + +-type key() :: term(). +-type value() :: term(). + +-spec erase(Key) -> Result when + Key :: key(), + Result :: boolean(). +erase(_Key) -> + erlang:nif_error(undef). + +-spec get() -> List when + List :: [{key(),value()}]. +get() -> + erlang:nif_error(undef). + +-spec get(Key) -> Value when + Key :: key(), + Value :: value(). +get(_Key) -> + erlang:nif_error(undef). + +-spec info() -> Info when + Info :: #{'count':=Count,'memory':=Memory}, + Count :: non_neg_integer(), + Memory :: non_neg_integer(). +info() -> + erlang:nif_error(undef). + +-spec put(Key, Value) -> 'ok' when + Key :: key(), + Value :: value(). +put(_Key, _Value) -> + erlang:nif_error(undef). diff --git a/erts/preloaded/src/prim_file.erl b/erts/preloaded/src/prim_file.erl index 6d85868183..5fc22bc582 100644 --- a/erts/preloaded/src/prim_file.erl +++ b/erts/preloaded/src/prim_file.erl @@ -83,6 +83,15 @@ internal_normalize_utf8(_) -> is_translatable(_) -> erlang:nif_error(undefined). +%% This is a janitor process used to close files whose controlling process has +%% died. The emulator will be torn down if this is killed. +delayed_close_loop() -> + receive + {close, FRef} when is_reference(FRef) -> delayed_close_nif(FRef); + _ -> ok + end, + delayed_close_loop(). + %% %% Returns {error, Reason} | {ok, BytesCopied} @@ -95,7 +104,12 @@ copy(#file_descriptor{module = ?MODULE} = Source, file:copy_opened(Source, Dest, Length). on_load() -> - ok = erlang:load_nif(atom_to_list(?MODULE), 0). + Pid = spawn(fun() -> + process_flag(trap_exit, true), + delayed_close_loop() + end), + true = register(erts_prim_file, Pid), + ok = erlang:load_nif(atom_to_list(?MODULE), Pid). open(Name, Modes) -> %% The try/catch pattern seen here is used throughout the file to adhere to @@ -482,6 +496,8 @@ truncate_nif(_FileRef) -> erlang:nif_error(undef). get_handle_nif(_FileRef) -> erlang:nif_error(undef). +delayed_close_nif(_FileRef) -> + erlang:nif_error(undef). %% %% Quality-of-life helpers diff --git a/erts/vsn.mk b/erts/vsn.mk index 01c19aff2f..643a8a2e76 100644 --- a/erts/vsn.mk +++ b/erts/vsn.mk @@ -18,7 +18,7 @@ # %CopyrightEnd% # -VSN = 10.1.1 +VSN = 10.1.3 # Port number 4365 in 4.2 # Port number 4366 in 4.3 diff --git a/lib/compiler/doc/src/notes.xml b/lib/compiler/doc/src/notes.xml index 5024310788..e0e5bc832b 100644 --- a/lib/compiler/doc/src/notes.xml +++ b/lib/compiler/doc/src/notes.xml @@ -32,6 +32,21 @@ <p>This document describes the changes made to the Compiler application.</p> +<section><title>Compiler 7.2.7</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p>Fixed a bug where incorrect code was generated + following a binary match guard.</p> + <p> + Own Id: OTP-15353 Aux Id: ERL-753 </p> + </item> + </list> + </section> + +</section> + <section><title>Compiler 7.2.6</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/compiler/src/beam_jump.erl b/lib/compiler/src/beam_jump.erl index 9eee56d604..22974da398 100644 --- a/lib/compiler/src/beam_jump.erl +++ b/lib/compiler/src/beam_jump.erl @@ -128,7 +128,7 @@ %%% on the program state. %%% --import(lists, [reverse/1,reverse/2,foldl/3]). +-import(lists, [dropwhile/2,reverse/1,reverse/2,foldl/3]). -type instruction() :: beam_utils:instruction(). @@ -411,14 +411,19 @@ opt_useless_loads([{test,_,{f,L},_}=I|Is], L, St) -> opt_useless_loads(Is, _L, St) -> {Is,St}. -opt_useless_block_loads([{set,[Dst],_,_}=I|Is], L, Index) -> - BlockJump = [{block,Is},{jump,{f,L}}], +opt_useless_block_loads([{set,[Dst],_,_}=I|Is0], L, Index) -> + BlockJump = [{block,Is0},{jump,{f,L}}], case beam_utils:is_killed(Dst, BlockJump, Index) of true -> - %% The register is killed and not used, we can remove the load + %% The register is killed and not used, we can remove the load. + %% Remove any `put` instructions in case we just + %% removed a `put_tuple` instruction. + Is = dropwhile(fun({set,_,_,put}) -> true; + (_) -> false + end, Is0), opt_useless_block_loads(Is, L, Index); false -> - [I|opt_useless_block_loads(Is, L, Index)] + [I|opt_useless_block_loads(Is0, L, Index)] end; opt_useless_block_loads([I|Is], L, Index) -> [I|opt_useless_block_loads(Is, L, Index)]; diff --git a/lib/compiler/src/beam_utils.erl b/lib/compiler/src/beam_utils.erl index 6e23003fc7..6b2ab5a2a4 100644 --- a/lib/compiler/src/beam_utils.erl +++ b/lib/compiler/src/beam_utils.erl @@ -745,8 +745,11 @@ check_liveness_block_2(R, {gc_bif,Op,{f,Lbl}}, Ss, St) -> check_liveness_block_3(R, Lbl, {Op,length(Ss)}, St); check_liveness_block_2(R, {bif,Op,{f,Lbl}}, Ss, St) -> Arity = length(Ss), + + %% Note that is_function/2 is a type test but is not safe. case erl_internal:comp_op(Op, Arity) orelse - erl_internal:new_type_test(Op, Arity) of + (erl_internal:new_type_test(Op, Arity) andalso + erl_bifs:is_safe(erlang, Op, Arity)) of true -> {killed,St}; false -> diff --git a/lib/compiler/test/beam_jump_SUITE.erl b/lib/compiler/test/beam_jump_SUITE.erl index c61e4ab65c..faedc0c1f1 100644 --- a/lib/compiler/test/beam_jump_SUITE.erl +++ b/lib/compiler/test/beam_jump_SUITE.erl @@ -21,7 +21,8 @@ -export([all/0,suite/0,groups/0,init_per_suite/1,end_per_suite/1, init_per_group/2,end_per_group/2, - undefined_label/1,ambiguous_catch_try_state/1]). + undefined_label/1,ambiguous_catch_try_state/1, + build_tuple/1]). suite() -> [{ct_hooks,[ts_install_cth]}]. @@ -32,7 +33,8 @@ all() -> groups() -> [{p,[parallel], [undefined_label, - ambiguous_catch_try_state + ambiguous_catch_try_state, + build_tuple ]}]. init_per_suite(Config) -> @@ -72,3 +74,16 @@ river() -> song. checks(Wanted) -> %% Must be one line to cause the unsafe optimization. {catch case river() of sheet -> begin +Wanted, if "da" -> Wanted end end end, catch case river() of sheet -> begin + Wanted, if "da" -> Wanted end end end}. + +-record(message2, {id, p1}). +-record(message3, {id, p1, p2}). + +build_tuple(_Config) -> + {'EXIT',{{badrecord,message3},_}} = (catch do_build_tuple(#message2{})), + ok. + +do_build_tuple(Message) -> + if is_record(Message, message2) -> + Res = {res, rand:uniform(100)}, + {Message#message3.id, Res} + end. diff --git a/lib/compiler/test/beam_utils_SUITE.erl b/lib/compiler/test/beam_utils_SUITE.erl index ac19305d69..ff0f72d519 100644 --- a/lib/compiler/test/beam_utils_SUITE.erl +++ b/lib/compiler/test/beam_utils_SUITE.erl @@ -26,7 +26,7 @@ select/1,y_catch/1,otp_8949_b/1,liveopt/1,coverage/1, y_registers/1,user_predef/1,scan_f/1,cafu/1, receive_label/1,read_size_file_version/1,not_used/1, - is_used_fr/1]). + is_used_fr/1,unsafe_is_function/1]). -export([id/1]). suite() -> [{ct_hooks,[ts_install_cth]}]. @@ -53,7 +53,8 @@ groups() -> cafu, read_size_file_version, not_used, - is_used_fr + is_used_fr, + unsafe_is_function ]}]. init_per_suite(Config) -> @@ -570,6 +571,24 @@ is_used_fr(X, Y) -> end, X ! 1. +%% ERL-778. +unsafe_is_function(Config) -> + {undefined,any} = unsafe_is_function(undefined, any), + {ok,any} = unsafe_is_function(fun() -> ok end, any), + {'EXIT',{{case_clause,_},_}} = (catch unsafe_is_function(fun(_) -> ok end, any)), + ok. + +unsafe_is_function(F, M) -> + %% There would be an internal consistency failure: + %% Instruction: {bif,is_function,{f,0},[{x,0},{integer,0}],{x,2}} + %% Error: {uninitialized_reg,{y,0}}: + + NewValue = case is_function(F, 0) of + true -> F(); + false when F =:= undefined -> undefined + end, + {NewValue,M}. + %% The identity function. id(I) -> I. diff --git a/lib/compiler/vsn.mk b/lib/compiler/vsn.mk index ab707885f4..92f8aec424 100644 --- a/lib/compiler/vsn.mk +++ b/lib/compiler/vsn.mk @@ -1 +1 @@ -COMPILER_VSN = 7.2.6 +COMPILER_VSN = 7.2.7 diff --git a/lib/crypto/c_src/crypto.c b/lib/crypto/c_src/crypto.c index b2d8123f00..ecb716c98c 100644 --- a/lib/crypto/c_src/crypto.c +++ b/lib/crypto/c_src/crypto.c @@ -173,10 +173,13 @@ #endif // (test for >= 1.1.1pre8) -#if OPENSSL_VERSION_NUMBER >= (PACKED_OPENSSL_VERSION_PLAIN(1,1,1) - 7) \ +#if OPENSSL_VERSION_NUMBER >= (PACKED_OPENSSL_VERSION_PLAIN(1,1,1) -7) \ && !defined(HAS_LIBRESSL) \ && defined(HAVE_EC) # define HAVE_ED_CURVE_DH +# if OPENSSL_VERSION_NUMBER >= (PACKED_OPENSSL_VERSION_PLAIN(1,1,1)) +# define HAVE_EDDSA +# endif #endif #if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION(0,9,8,'c') @@ -632,10 +635,8 @@ static ErlNifFunc nif_funcs[] = { {"rsa_generate_key_nif", 2, rsa_generate_key_nif}, {"dh_generate_key_nif", 4, dh_generate_key_nif}, {"dh_compute_key_nif", 3, dh_compute_key_nif}, - {"evp_compute_key_nif", 3, evp_compute_key_nif}, {"evp_generate_key_nif", 1, evp_generate_key_nif}, - {"privkey_to_pubkey_nif", 2, privkey_to_pubkey_nif}, {"srp_value_B_nif", 5, srp_value_B_nif}, {"srp_user_secret_nif", 7, srp_user_secret_nif}, @@ -742,6 +743,12 @@ static ERL_NIF_TERM atom_x25519; static ERL_NIF_TERM atom_x448; #endif +static ERL_NIF_TERM atom_eddsa; +#ifdef HAVE_EDDSA +static ERL_NIF_TERM atom_ed25519; +static ERL_NIF_TERM atom_ed448; +#endif + static ERL_NIF_TERM atom_rsa_mgf1_md; static ERL_NIF_TERM atom_rsa_oaep_label; static ERL_NIF_TERM atom_rsa_oaep_md; @@ -1165,6 +1172,7 @@ static int initialize(ErlNifEnv* env, ERL_NIF_TERM load_info) atom_ppbasis = enif_make_atom(env,"ppbasis"); atom_onbasis = enif_make_atom(env,"onbasis"); #endif + atom_aes_cfb8 = enif_make_atom(env, "aes_cfb8"); atom_aes_cfb128 = enif_make_atom(env, "aes_cfb128"); #ifdef HAVE_GCM @@ -1195,6 +1203,11 @@ static int initialize(ErlNifEnv* env, ERL_NIF_TERM load_info) atom_x25519 = enif_make_atom(env,"x25519"); atom_x448 = enif_make_atom(env,"x448"); #endif + atom_eddsa = enif_make_atom(env,"eddsa"); +#ifdef HAVE_EDDSA + atom_ed25519 = enif_make_atom(env,"ed25519"); + atom_ed448 = enif_make_atom(env,"ed448"); +#endif atom_rsa_mgf1_md = enif_make_atom(env,"rsa_mgf1_md"); atom_rsa_oaep_label = enif_make_atom(env,"rsa_oaep_label"); atom_rsa_oaep_md = enif_make_atom(env,"rsa_oaep_md"); @@ -1336,13 +1349,13 @@ static void unload(ErlNifEnv* env, void* priv_data) static int algo_hash_cnt, algo_hash_fips_cnt; static ERL_NIF_TERM algo_hash[12]; /* increase when extending the list */ static int algo_pubkey_cnt, algo_pubkey_fips_cnt; -static ERL_NIF_TERM algo_pubkey[11]; /* increase when extending the list */ +static ERL_NIF_TERM algo_pubkey[12]; /* increase when extending the list */ static int algo_cipher_cnt, algo_cipher_fips_cnt; static ERL_NIF_TERM algo_cipher[25]; /* increase when extending the list */ static int algo_mac_cnt, algo_mac_fips_cnt; static ERL_NIF_TERM algo_mac[3]; /* increase when extending the list */ static int algo_curve_cnt, algo_curve_fips_cnt; -static ERL_NIF_TERM algo_curve[87]; /* increase when extending the list */ +static ERL_NIF_TERM algo_curve[89]; /* increase when extending the list */ static int algo_rsa_opts_cnt, algo_rsa_opts_fips_cnt; static ERL_NIF_TERM algo_rsa_opts[11]; /* increase when extending the list */ @@ -1394,6 +1407,10 @@ static void init_algorithms_types(ErlNifEnv* env) #endif // Non-validated algorithms follow algo_pubkey_fips_cnt = algo_pubkey_cnt; + // Don't know if Edward curves are fips validated +#if defined(HAVE_EDDSA) + algo_pubkey[algo_pubkey_cnt++] = enif_make_atom(env, "eddsa"); +#endif algo_pubkey[algo_pubkey_cnt++] = enif_make_atom(env, "srp"); // Validated algorithms first @@ -1554,6 +1571,10 @@ static void init_algorithms_types(ErlNifEnv* env) #endif #endif //-- +#ifdef HAVE_EDDSA + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"ed25519"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"ed448"); +#endif #ifdef HAVE_ED_CURVE_DH algo_curve[algo_curve_cnt++] = enif_make_atom(env,"x25519"); algo_curve[algo_curve_cnt++] = enif_make_atom(env,"x448"); @@ -3215,6 +3236,36 @@ static int get_rsa_public_key(ErlNifEnv* env, ERL_NIF_TERM key, RSA *rsa) return 1; } +#ifdef HAVE_EDDSA + static int get_eddsa_key(ErlNifEnv* env, int public, ERL_NIF_TERM key, EVP_PKEY **pkey) +{ + /* key=[K] */ + ERL_NIF_TERM head, tail, tail2, algo; + ErlNifBinary bin; + int type; + + if (!enif_get_list_cell(env, key, &head, &tail) + || !enif_inspect_binary(env, head, &bin) + || !enif_get_list_cell(env, tail, &algo, &tail2) + || !enif_is_empty_list(env, tail2)) { + return 0; + } + if (algo == atom_ed25519) type = EVP_PKEY_ED25519; + else if (algo == atom_ed448) type = EVP_PKEY_ED448; + else + return 0; + + if (public) + *pkey = EVP_PKEY_new_raw_public_key(type, NULL, bin.data, bin.size); + else + *pkey = EVP_PKEY_new_raw_private_key(type, NULL, bin.data, bin.size); + + if (!pkey) + return 0; + return 1; +} +#endif + static int get_dss_private_key(ErlNifEnv* env, ERL_NIF_TERM key, DSA *dsa) { /* key=[P,Q,G,KEY] */ @@ -4296,7 +4347,9 @@ static int get_pkey_digest_type(ErlNifEnv *env, ERL_NIF_TERM algorithm, ERL_NIF_ *md = NULL; if (type == atom_none && algorithm == atom_rsa) return PKEY_OK; - +#ifdef HAVE_EDDSA + if (algorithm == atom_eddsa) return PKEY_OK; +#endif digp = get_digest_type(type); if (!digp) return PKEY_BADARG; if (!digp->md.p) return PKEY_NOTSUP; @@ -4546,6 +4599,14 @@ static int get_pkey_private_key(ErlNifEnv *env, ERL_NIF_TERM algorithm, ERL_NIF_ #else return PKEY_NOTSUP; #endif + } else if (algorithm == atom_eddsa) { +#if defined(HAVE_EDDSA) + if (!get_eddsa_key(env, 0, key, pkey)) { + return PKEY_BADARG; + } +#else + return PKEY_NOTSUP; +#endif } else if (algorithm == atom_dss) { DSA *dsa = DSA_new(); @@ -4624,6 +4685,14 @@ static int get_pkey_public_key(ErlNifEnv *env, ERL_NIF_TERM algorithm, ERL_NIF_T #else return PKEY_NOTSUP; #endif + } else if (algorithm == atom_eddsa) { +#if defined(HAVE_EDDSA) + if (!get_eddsa_key(env, 1, key, pkey)) { + return PKEY_BADARG; + } +#else + return PKEY_NOTSUP; +#endif } else if (algorithm == atom_dss) { DSA *dsa = DSA_new(); @@ -4697,8 +4766,10 @@ printf("\r\n"); ctx = EVP_PKEY_CTX_new(pkey, NULL); if (!ctx) goto badarg; - if (EVP_PKEY_sign_init(ctx) <= 0) goto badarg; - if (md != NULL && EVP_PKEY_CTX_set_signature_md(ctx, md) <= 0) goto badarg; + if (argv[0] != atom_eddsa) { + if (EVP_PKEY_sign_init(ctx) <= 0) goto badarg; + if (md != NULL && EVP_PKEY_CTX_set_signature_md(ctx, md) <= 0) goto badarg; + } if (argv[0] == atom_rsa) { if (EVP_PKEY_CTX_set_rsa_padding(ctx, sig_opt.rsa_padding) <= 0) goto badarg; @@ -4720,14 +4791,39 @@ printf("\r\n"); #endif } - if (EVP_PKEY_sign(ctx, NULL, &siglen, tbs, tbslen) <= 0) goto badarg; - enif_alloc_binary(siglen, &sig_bin); + if (argv[0] == atom_eddsa) { +#ifdef HAVE_EDDSA + EVP_MD_CTX* mdctx = EVP_MD_CTX_new(); + if (!EVP_DigestSignInit(mdctx, NULL, NULL, NULL, pkey)) { + if (mdctx) EVP_MD_CTX_free(mdctx); + goto badarg; + } - if (md != NULL) { - ERL_VALGRIND_ASSERT_MEM_DEFINED(tbs, EVP_MD_size(md)); + if (!EVP_DigestSign(mdctx, NULL, &siglen, tbs, tbslen)) { + EVP_MD_CTX_free(mdctx); + goto badarg; + } + enif_alloc_binary(siglen, &sig_bin); + + if (!EVP_DigestSign(mdctx, sig_bin.data, &siglen, tbs, tbslen)) { + EVP_MD_CTX_free(mdctx); + goto badarg; + } +#else + goto badarg; +#endif } - i = EVP_PKEY_sign(ctx, sig_bin.data, &siglen, tbs, tbslen); + else + { + if (EVP_PKEY_sign(ctx, NULL, &siglen, tbs, tbslen) <= 0) goto badarg; + enif_alloc_binary(siglen, &sig_bin); + if (md != NULL) { + ERL_VALGRIND_ASSERT_MEM_DEFINED(tbs, EVP_MD_size(md)); + } + i = EVP_PKEY_sign(ctx, sig_bin.data, &siglen, tbs, tbslen); + } + EVP_PKEY_CTX_free(ctx); #else /*printf("Old interface\r\n"); @@ -4835,8 +4931,11 @@ static ERL_NIF_TERM pkey_verify_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM */ ctx = EVP_PKEY_CTX_new(pkey, NULL); if (!ctx) goto badarg; - if (EVP_PKEY_verify_init(ctx) <= 0) goto badarg; - if (md != NULL && EVP_PKEY_CTX_set_signature_md(ctx, md) <= 0) goto badarg; + + if (argv[0] != atom_eddsa) { + if (EVP_PKEY_verify_init(ctx) <= 0) goto badarg; + if (md != NULL && EVP_PKEY_CTX_set_signature_md(ctx, md) <= 0) goto badarg; + } if (argv[0] == atom_rsa) { if (EVP_PKEY_CTX_set_rsa_padding(ctx, sig_opt.rsa_padding) <= 0) goto badarg; @@ -4856,10 +4955,28 @@ static ERL_NIF_TERM pkey_verify_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM } } - if (md != NULL) { - ERL_VALGRIND_ASSERT_MEM_DEFINED(tbs, EVP_MD_size(md)); - } - i = EVP_PKEY_verify(ctx, sig_bin.data, sig_bin.size, tbs, tbslen); + if (argv[0] == atom_eddsa) { +#ifdef HAVE_EDDSA + EVP_MD_CTX* mdctx = EVP_MD_CTX_create(); + + if (!EVP_DigestVerifyInit(mdctx, NULL, NULL, NULL, pkey)) { + if (mdctx) EVP_MD_CTX_destroy(mdctx); + goto badarg; + } + + i = EVP_DigestVerify(mdctx, sig_bin.data, sig_bin.size, tbs, tbslen); + EVP_MD_CTX_destroy(mdctx); +#else + goto badarg; +#endif + } + else + { + if (md != NULL) { + ERL_VALGRIND_ASSERT_MEM_DEFINED(tbs, EVP_MD_size(md)); + } + i = EVP_PKEY_verify(ctx, sig_bin.data, sig_bin.size, tbs, tbslen); + } EVP_PKEY_CTX_free(ctx); #else diff --git a/lib/crypto/c_src/crypto_callback.c b/lib/crypto/c_src/crypto_callback.c index 23d2bed057..0cc7dd609d 100644 --- a/lib/crypto/c_src/crypto_callback.c +++ b/lib/crypto/c_src/crypto_callback.c @@ -179,6 +179,10 @@ DLLEXPORT struct crypto_callbacks* get_crypto_callbacks(int nlocks) /* This is not really a NIF library, but we use ERL_NIF_INIT in order to * get access to the erl_nif API (on Windows). */ -ERL_NIF_INIT(dummy, (ErlNifFunc*)NULL , NULL, NULL, NULL, NULL) +static struct { + int dummy__; + ErlNifFunc funcv[0]; +} empty; +ERL_NIF_INIT(dummy, empty.funcv, NULL, NULL, NULL, NULL) #endif diff --git a/lib/crypto/doc/src/algorithm_details.xml b/lib/crypto/doc/src/algorithm_details.xml index 4d58d26970..854bfbb4b1 100644 --- a/lib/crypto/doc/src/algorithm_details.xml +++ b/lib/crypto/doc/src/algorithm_details.xml @@ -303,6 +303,20 @@ </section> <section> + <title>EdDSA</title> + <p>EdDSA is available with OpenSSL 1.1.1 or later if not disabled by configuration. + To dynamically check availability, check that the atom <c>eddsa</c> is present in the + list with the <c>public_keys</c> tag in the return value of + <seealso marker="crypto#supports-0">crypto:supports()</seealso>. + </p> + <p>Support for the curves ed25519 and ed448 is implemented. + The actual supported named curves could be checked by examining the list with the + <c>curves</c> tag in the return value of + <seealso marker="crypto#supports-0">crypto:supports()</seealso>. + </p> + </section> + + <section> <title>Diffie-Hellman</title> <p>Diffie-Hellman computations are available with OpenSSL versions compatible with Erlang CRYPTO if not disabled by configuration. diff --git a/lib/crypto/doc/src/crypto.xml b/lib/crypto/doc/src/crypto.xml index 651b647e1c..b33db0d6e4 100644 --- a/lib/crypto/doc/src/crypto.xml +++ b/lib/crypto/doc/src/crypto.xml @@ -268,7 +268,8 @@ <datatype_title>Elliptic Curves</datatype_title> <datatype> <name name="ec_named_curve"/> - <name name="edwards_curve"/> + <name name="edwards_curve_dh"/> + <name name="edwards_curve_ed"/> <desc> <p>Note that some curves are disabled if FIPS is enabled.</p> </desc> @@ -348,6 +349,14 @@ </datatype> <datatype> + <name name="eddsa_public"/> + <name name="eddsa_private"/> + <name name="eddsa_params"/> + <desc> + </desc> + </datatype> + + <datatype> <name name="srp_public"/> <name name="srp_private"/> <desc> diff --git a/lib/crypto/src/crypto.erl b/lib/crypto/src/crypto.erl index c2ab88417e..af53a72e16 100644 --- a/lib/crypto/src/crypto.erl +++ b/lib/crypto/src/crypto.erl @@ -118,7 +118,11 @@ -type ecdsa_public() :: key_integer() . -type ecdsa_private() :: key_integer() . --type ecdsa_params() :: ec_named_curve() | edwards_curve() | ec_explicit_curve() . +-type ecdsa_params() :: ec_named_curve() | ec_explicit_curve() . + +-type eddsa_public() :: key_integer() . +-type eddsa_private() :: key_integer() . +-type eddsa_params() :: edwards_curve_ed() . -type srp_public() :: key_integer() . -type srp_private() :: key_integer() . @@ -135,7 +139,7 @@ -type ecdh_public() :: key_integer() . -type ecdh_private() :: key_integer() . --type ecdh_params() :: ec_named_curve() | edwards_curve() | ec_explicit_curve() . +-type ecdh_params() :: ec_named_curve() | edwards_curve_dh() | ec_explicit_curve() . %%% Curves @@ -247,8 +251,9 @@ | wtls9 . --type edwards_curve() :: x25519 - | x448 . +-type edwards_curve_dh() :: x25519 | x448 . + +-type edwards_curve_ed() :: ed25519 | ed448 . %%% -type block_cipher_with_iv() :: cbc_cipher() @@ -328,7 +333,7 @@ stop() -> ], PKs :: [rsa | dss | ecdsa | dh | ecdh | ec_gf2m], Macs :: [hmac | cmac | poly1305], - Curves :: [ec_named_curve() | edwards_curve()], + Curves :: [ec_named_curve() | edwards_curve_dh() | edwards_curve_ed()], RSAopts :: [rsa_sign_verify_opt() | rsa_opt()] . supports()-> {Hashs, PubKeys, Ciphers, Macs, Curves, RsaOpts} = algorithms(), @@ -777,7 +782,7 @@ rand_seed_nif(_Seed) -> ?nif_stub. %%% Sign/verify %%% %%%================================================================ --type pk_sign_verify_algs() :: rsa | dss | ecdsa . +-type pk_sign_verify_algs() :: rsa | dss | ecdsa | eddsa . -type pk_sign_verify_opts() :: [ rsa_sign_verify_opt() ] . @@ -801,7 +806,8 @@ rand_seed_nif(_Seed) -> ?nif_stub. Msg :: binary() | {digest,binary()}, Key :: rsa_private() | dss_private() - | [ecdsa_private()|ecdsa_params()] + | [ecdsa_private() | ecdsa_params()] + | [eddsa_private() | eddsa_params()] | engine_key_ref(), Signature :: binary() . @@ -820,6 +826,7 @@ sign(Algorithm, Type, Data, Key) -> Key :: rsa_private() | dss_private() | [ecdsa_private() | ecdsa_params()] + | [eddsa_private() | eddsa_params()] | engine_key_ref(), Options :: pk_sign_verify_opts(), Signature :: binary() . @@ -842,12 +849,14 @@ pkey_sign_nif(_Algorithm, _Type, _Digest, _Key, _Options) -> ?nif_stub. when Algorithm :: pk_sign_verify_algs(), DigestType :: rsa_digest_type() | dss_digest_type() - | ecdsa_digest_type(), + | ecdsa_digest_type() + | none, Msg :: binary() | {digest,binary()}, Signature :: binary(), - Key :: rsa_private() - | dss_private() - | [ecdsa_private() | ecdsa_params()] + Key :: rsa_public() + | dss_public() + | [ecdsa_public() | ecdsa_params()] + | [eddsa_public() | eddsa_params()] | engine_key_ref(), Result :: boolean(). @@ -865,6 +874,7 @@ verify(Algorithm, Type, Data, Signature, Key) -> Key :: rsa_public() | dss_public() | [ecdsa_public() | ecdsa_params()] + | [eddsa_public() | eddsa_params()] | engine_key_ref(), Options :: pk_sign_verify_opts(), Result :: boolean(). @@ -1762,7 +1772,9 @@ ec_key_generate(_Curve, _Key) -> ?nif_stub. ecdh_compute_key_nif(_Others, _Curve, _My) -> ?nif_stub. --spec ec_curves() -> [EllipticCurve] when EllipticCurve :: ec_named_curve() | edwards_curve() . +-spec ec_curves() -> [EllipticCurve] when EllipticCurve :: ec_named_curve() + | edwards_curve_dh() + | edwards_curve_ed() . ec_curves() -> crypto_ec_curves:curves(). diff --git a/lib/crypto/test/crypto_SUITE.erl b/lib/crypto/test/crypto_SUITE.erl index 025c46aab0..e1dd3f2ace 100644 --- a/lib/crypto/test/crypto_SUITE.erl +++ b/lib/crypto/test/crypto_SUITE.erl @@ -1,4 +1,4 @@ -%% +% %% %CopyrightBegin% %% %% Copyright Ericsson AB 1999-2018. All Rights Reserved. @@ -58,6 +58,8 @@ groups() -> {group, rsa}, {group, dss}, {group, ecdsa}, + {group, ed25519}, + {group, ed448}, {group, dh}, {group, ecdh}, {group, srp}, @@ -145,6 +147,12 @@ groups() -> {ecdsa, [], [sign_verify %% Does not work yet: ,public_encrypt, private_encrypt ]}, + {ed25519, [], [sign_verify + %% Does not work yet: ,public_encrypt, private_encrypt + ]}, + {ed448, [], [sign_verify + %% Does not work yet: ,public_encrypt, private_encrypt + ]}, {dh, [], [generate_compute, compute_bug]}, {ecdh, [], [generate_all_supported, compute, generate]}, @@ -572,7 +580,8 @@ generate_all_supported(_Config) -> ct:log("ERROR: Curve ~p exception ~p:~p~n~p", [C,Cls,Err,Stack]), {error,{C,{Cls,Err}}} end - || C <- crypto:ec_curves() + || C <- crypto:ec_curves(), + not lists:member(C, [ed25519, ed448]) ], OK = [C || {ok,C} <- Results], ct:log("Ok (len=~p): ~p", [length(OK), OK]), @@ -884,6 +893,29 @@ aead_cipher({Type, Key, PlainText, IV, AAD, CipherText, CipherTag, TagLen, Info} {got, Other1}}) end. +do_sign_verify({Type, undefined=Hash, Private, Public, Msg, Signature}) -> + case crypto:sign(eddsa, Hash, Msg, [Private,Type]) of + Signature -> + ct:log("OK crypto:sign(eddsa, ~p, Msg, [Private,~p])", [Hash,Type]), + case crypto:verify(eddsa, Hash, Msg, Signature, [Public,Type]) of + true -> + ct:log("OK crypto:verify(eddsa, ~p, Msg, Signature, [Public,~p])", [Hash,Type]), + negative_verify(eddsa, Hash, Msg, <<10,20>>, [Public,Type]); + false -> + ct:log("ERROR crypto:verify(eddsa, ~p, Msg= ~p, Signature= ~p, [Public= ~p,~p])", + [Hash,Msg,Signature,Public,Type]), + ct:fail({{crypto, verify, [eddsa, Hash, Msg, Signature, [Public,Type]]}}) + end; + ErrorSig -> + ct:log("ERROR crypto:sign(~p, ~p, ..., [Private= ~p,~p])", [eddsa,Hash,Private,Type]), + ct:log("ERROR crypto:verify(eddsa, ~p, Msg= ~p, [Public= ~p,~p])~n" + "ErrorSig = ~p~n" + "CorrectSig = ~p~n" + , + [Hash,Msg,Public,Type,ErrorSig,Signature]), + ct:fail({{crypto, sign, [Type, Hash, Msg, ErrorSig, [Private]]}}) + end; + do_sign_verify({Type, Hash, Public, Private, Msg}) -> Signature = crypto:sign(Type, Hash, Msg, Private), case crypto:verify(Type, Hash, Msg, Signature, Public) of @@ -1443,6 +1475,12 @@ group_config(ecdsa = Type, Config) -> MsgPubEnc = <<"7896345786348 Asldi">>, PubPrivEnc = [{ecdsa, Public, Private, MsgPubEnc, []}], [{sign_verify, SignVerify}, {pub_priv_encrypt, PubPrivEnc} | Config]; + +group_config(Type, Config) when Type == ed25519 ; Type == ed448 -> + TestVectors = eddsa(Type), + [{sign_verify,TestVectors} | Config]; + + group_config(srp, Config) -> GenerateCompute = [srp3(), srp6(), srp6a(), srp6a_smaller_prime()], [{generate_compute, GenerateCompute} | Config]; @@ -2702,6 +2740,392 @@ srp(ClientPrivate, Generator, Prime, Version, Verifier, ServerPublic, ServerPriv ServerPublic, ServerPrivate, {host, [Verifier, Generator, Prime, Version]}, {host, [Verifier, Prime, Version, Scrambler]}, SessionKey}. + +eddsa(ed25519) -> + %% https://tools.ietf.org/html/rfc8032#section-7.1 + %% {ALGORITHM, (SHA)}, SECRET KEY, PUBLIC KEY, MESSAGE, SIGNATURE} + [ + %% TEST 1 + {ed25519, undefined, + hexstr2bin("9d61b19deffd5a60ba844af492ec2cc4" + "4449c5697b326919703bac031cae7f60"), + hexstr2bin("d75a980182b10ab7d54bfed3c964073a" + "0ee172f3daa62325af021a68f707511a"), + hexstr2bin(""), + hexstr2bin("e5564300c360ac729086e2cc806e828a" + "84877f1eb8e5d974d873e06522490155" + "5fb8821590a33bacc61e39701cf9b46b" + "d25bf5f0595bbe24655141438e7a100b")}, + %% TEST 2 + {ed25519, undefined, + hexstr2bin("4ccd089b28ff96da9db6c346ec114e0f" + "5b8a319f35aba624da8cf6ed4fb8a6fb"), + hexstr2bin("3d4017c3e843895a92b70aa74d1b7ebc" + "9c982ccf2ec4968cc0cd55f12af4660c"), + hexstr2bin("72"), + hexstr2bin("92a009a9f0d4cab8720e820b5f642540" + "a2b27b5416503f8fb3762223ebdb69da" + "085ac1e43e15996e458f3613d0f11d8c" + "387b2eaeb4302aeeb00d291612bb0c00")}, + %% TEST 3 + {ed25519, undefined, + hexstr2bin("c5aa8df43f9f837bedb7442f31dcb7b1" + "66d38535076f094b85ce3a2e0b4458f7"), + hexstr2bin("fc51cd8e6218a1a38da47ed00230f058" + "0816ed13ba3303ac5deb911548908025"), + hexstr2bin("af82"), + hexstr2bin("6291d657deec24024827e69c3abe01a3" + "0ce548a284743a445e3680d7db5ac3ac" + "18ff9b538d16f290ae67f760984dc659" + "4a7c15e9716ed28dc027beceea1ec40a")}, + %% TEST 1024 + {ed25519, undefined, + hexstr2bin("f5e5767cf153319517630f226876b86c" + "8160cc583bc013744c6bf255f5cc0ee5"), + hexstr2bin("278117fc144c72340f67d0f2316e8386" + "ceffbf2b2428c9c51fef7c597f1d426e"), + hexstr2bin("08b8b2b733424243760fe426a4b54908" + "632110a66c2f6591eabd3345e3e4eb98" + "fa6e264bf09efe12ee50f8f54e9f77b1" + "e355f6c50544e23fb1433ddf73be84d8" + "79de7c0046dc4996d9e773f4bc9efe57" + "38829adb26c81b37c93a1b270b20329d" + "658675fc6ea534e0810a4432826bf58c" + "941efb65d57a338bbd2e26640f89ffbc" + "1a858efcb8550ee3a5e1998bd177e93a" + "7363c344fe6b199ee5d02e82d522c4fe" + "ba15452f80288a821a579116ec6dad2b" + "3b310da903401aa62100ab5d1a36553e" + "06203b33890cc9b832f79ef80560ccb9" + "a39ce767967ed628c6ad573cb116dbef" + "efd75499da96bd68a8a97b928a8bbc10" + "3b6621fcde2beca1231d206be6cd9ec7" + "aff6f6c94fcd7204ed3455c68c83f4a4" + "1da4af2b74ef5c53f1d8ac70bdcb7ed1" + "85ce81bd84359d44254d95629e9855a9" + "4a7c1958d1f8ada5d0532ed8a5aa3fb2" + "d17ba70eb6248e594e1a2297acbbb39d" + "502f1a8c6eb6f1ce22b3de1a1f40cc24" + "554119a831a9aad6079cad88425de6bd" + "e1a9187ebb6092cf67bf2b13fd65f270" + "88d78b7e883c8759d2c4f5c65adb7553" + "878ad575f9fad878e80a0c9ba63bcbcc" + "2732e69485bbc9c90bfbd62481d9089b" + "eccf80cfe2df16a2cf65bd92dd597b07" + "07e0917af48bbb75fed413d238f5555a" + "7a569d80c3414a8d0859dc65a46128ba" + "b27af87a71314f318c782b23ebfe808b" + "82b0ce26401d2e22f04d83d1255dc51a" + "ddd3b75a2b1ae0784504df543af8969b" + "e3ea7082ff7fc9888c144da2af58429e" + "c96031dbcad3dad9af0dcbaaaf268cb8" + "fcffead94f3c7ca495e056a9b47acdb7" + "51fb73e666c6c655ade8297297d07ad1" + "ba5e43f1bca32301651339e22904cc8c" + "42f58c30c04aafdb038dda0847dd988d" + "cda6f3bfd15c4b4c4525004aa06eeff8" + "ca61783aacec57fb3d1f92b0fe2fd1a8" + "5f6724517b65e614ad6808d6f6ee34df" + "f7310fdc82aebfd904b01e1dc54b2927" + "094b2db68d6f903b68401adebf5a7e08" + "d78ff4ef5d63653a65040cf9bfd4aca7" + "984a74d37145986780fc0b16ac451649" + "de6188a7dbdf191f64b5fc5e2ab47b57" + "f7f7276cd419c17a3ca8e1b939ae49e4" + "88acba6b965610b5480109c8b17b80e1" + "b7b750dfc7598d5d5011fd2dcc5600a3" + "2ef5b52a1ecc820e308aa342721aac09" + "43bf6686b64b2579376504ccc493d97e" + "6aed3fb0f9cd71a43dd497f01f17c0e2" + "cb3797aa2a2f256656168e6c496afc5f" + "b93246f6b1116398a346f1a641f3b041" + "e989f7914f90cc2c7fff357876e506b5" + "0d334ba77c225bc307ba537152f3f161" + "0e4eafe595f6d9d90d11faa933a15ef1" + "369546868a7f3a45a96768d40fd9d034" + "12c091c6315cf4fde7cb68606937380d" + "b2eaaa707b4c4185c32eddcdd306705e" + "4dc1ffc872eeee475a64dfac86aba41c" + "0618983f8741c5ef68d3a101e8a3b8ca" + "c60c905c15fc910840b94c00a0b9d0"), + hexstr2bin("0aab4c900501b3e24d7cdf4663326a3a" + "87df5e4843b2cbdb67cbf6e460fec350" + "aa5371b1508f9f4528ecea23c436d94b" + "5e8fcd4f681e30a6ac00a9704a188a03")}, + %% TEST SHA(abc) + {ed25519, undefined, + hexstr2bin("833fe62409237b9d62ec77587520911e" + "9a759cec1d19755b7da901b96dca3d42"), + hexstr2bin("ec172b93ad5e563bf4932c70e1245034" + "c35467ef2efd4d64ebf819683467e2bf"), + hexstr2bin("ddaf35a193617abacc417349ae204131" + "12e6fa4e89a97ea20a9eeee64b55d39a" + "2192992a274fc1a836ba3c23a3feebbd" + "454d4423643ce80e2a9ac94fa54ca49f"), + hexstr2bin("dc2a4459e7369633a52b1bf277839a00" + "201009a3efbf3ecb69bea2186c26b589" + "09351fc9ac90b3ecfdfbc7c66431e030" + "3dca179c138ac17ad9bef1177331a704")} + ]; + +eddsa(ed448) -> + %% https://tools.ietf.org/html/rfc8032#section-7.4 + [{ed448, undefined, + hexstr2bin("6c82a562cb808d10d632be89c8513ebf" + "6c929f34ddfa8c9f63c9960ef6e348a3" + "528c8a3fcc2f044e39a3fc5b94492f8f" + "032e7549a20098f95b"), + hexstr2bin("5fd7449b59b461fd2ce787ec616ad46a" + "1da1342485a70e1f8a0ea75d80e96778" + "edf124769b46c7061bd6783df1e50f6c" + "d1fa1abeafe8256180"), + hexstr2bin(""), + hexstr2bin("533a37f6bbe457251f023c0d88f976ae" + "2dfb504a843e34d2074fd823d41a591f" + "2b233f034f628281f2fd7a22ddd47d78" + "28c59bd0a21bfd3980ff0d2028d4b18a" + "9df63e006c5d1c2d345b925d8dc00b41" + "04852db99ac5c7cdda8530a113a0f4db" + "b61149f05a7363268c71d95808ff2e65" + "2600")}, + %% 1 octet + {ed448, undefined, + hexstr2bin("c4eab05d357007c632f3dbb48489924d" + "552b08fe0c353a0d4a1f00acda2c463a" + "fbea67c5e8d2877c5e3bc397a659949e" + "f8021e954e0a12274e"), + hexstr2bin("43ba28f430cdff456ae531545f7ecd0a" + "c834a55d9358c0372bfa0c6c6798c086" + "6aea01eb00742802b8438ea4cb82169c" + "235160627b4c3a9480"), + hexstr2bin("03"), + hexstr2bin("26b8f91727bd62897af15e41eb43c377" + "efb9c610d48f2335cb0bd0087810f435" + "2541b143c4b981b7e18f62de8ccdf633" + "fc1bf037ab7cd779805e0dbcc0aae1cb" + "cee1afb2e027df36bc04dcecbf154336" + "c19f0af7e0a6472905e799f1953d2a0f" + "f3348ab21aa4adafd1d234441cf807c0" + "3a00")}, + + %% %% 1 octet (with context) + %% {ed448, undefined, + %% hexstr2bin("c4eab05d357007c632f3dbb48489924d" + %% "552b08fe0c353a0d4a1f00acda2c463a" + %% "fbea67c5e8d2877c5e3bc397a659949e" + %% "f8021e954e0a12274e"), + %% hexstr2bin("43ba28f430cdff456ae531545f7ecd0a" + %% "c834a55d9358c0372bfa0c6c6798c086" + %% "6aea01eb00742802b8438ea4cb82169c" + %% "235160627b4c3a9480"), + %% hexstr2bin("03"), + %% hexstr2bin("666f6f"), % Context + %% hexstr2bin("d4f8f6131770dd46f40867d6fd5d5055" + %% "de43541f8c5e35abbcd001b32a89f7d2" + %% "151f7647f11d8ca2ae279fb842d60721" + %% "7fce6e042f6815ea000c85741de5c8da" + %% "1144a6a1aba7f96de42505d7a7298524" + %% "fda538fccbbb754f578c1cad10d54d0d" + %% "5428407e85dcbc98a49155c13764e66c" + %% "3c00")}, + + %% 11 octets + {ed448, undefined, + hexstr2bin("cd23d24f714274e744343237b93290f5" + "11f6425f98e64459ff203e8985083ffd" + "f60500553abc0e05cd02184bdb89c4cc" + "d67e187951267eb328"), + hexstr2bin("dcea9e78f35a1bf3499a831b10b86c90" + "aac01cd84b67a0109b55a36e9328b1e3" + "65fce161d71ce7131a543ea4cb5f7e9f" + "1d8b00696447001400"), + hexstr2bin("0c3e544074ec63b0265e0c"), + hexstr2bin("1f0a8888ce25e8d458a21130879b840a" + "9089d999aaba039eaf3e3afa090a09d3" + "89dba82c4ff2ae8ac5cdfb7c55e94d5d" + "961a29fe0109941e00b8dbdeea6d3b05" + "1068df7254c0cdc129cbe62db2dc957d" + "bb47b51fd3f213fb8698f064774250a5" + "028961c9bf8ffd973fe5d5c206492b14" + "0e00")}, + %% 12 octets + {ed448, undefined, + hexstr2bin("258cdd4ada32ed9c9ff54e63756ae582" + "fb8fab2ac721f2c8e676a72768513d93" + "9f63dddb55609133f29adf86ec9929dc" + "cb52c1c5fd2ff7e21b"), + hexstr2bin("3ba16da0c6f2cc1f30187740756f5e79" + "8d6bc5fc015d7c63cc9510ee3fd44adc" + "24d8e968b6e46e6f94d19b945361726b" + "d75e149ef09817f580"), + hexstr2bin("64a65f3cdedcdd66811e2915"), + hexstr2bin("7eeeab7c4e50fb799b418ee5e3197ff6" + "bf15d43a14c34389b59dd1a7b1b85b4a" + "e90438aca634bea45e3a2695f1270f07" + "fdcdf7c62b8efeaf00b45c2c96ba457e" + "b1a8bf075a3db28e5c24f6b923ed4ad7" + "47c3c9e03c7079efb87cb110d3a99861" + "e72003cbae6d6b8b827e4e6c143064ff" + "3c00")}, + %% 13 octets + {ed448, undefined, + hexstr2bin("7ef4e84544236752fbb56b8f31a23a10" + "e42814f5f55ca037cdcc11c64c9a3b29" + "49c1bb60700314611732a6c2fea98eeb" + "c0266a11a93970100e"), + hexstr2bin("b3da079b0aa493a5772029f0467baebe" + "e5a8112d9d3a22532361da294f7bb381" + "5c5dc59e176b4d9f381ca0938e13c6c0" + "7b174be65dfa578e80"), + hexstr2bin("64a65f3cdedcdd66811e2915e7"), + hexstr2bin("6a12066f55331b6c22acd5d5bfc5d712" + "28fbda80ae8dec26bdd306743c5027cb" + "4890810c162c027468675ecf645a8317" + "6c0d7323a2ccde2d80efe5a1268e8aca" + "1d6fbc194d3f77c44986eb4ab4177919" + "ad8bec33eb47bbb5fc6e28196fd1caf5" + "6b4e7e0ba5519234d047155ac727a105" + "3100")}, + %% 64 octets + {ed448, undefined, + hexstr2bin("d65df341ad13e008567688baedda8e9d" + "cdc17dc024974ea5b4227b6530e339bf" + "f21f99e68ca6968f3cca6dfe0fb9f4fa" + "b4fa135d5542ea3f01"), + hexstr2bin("df9705f58edbab802c7f8363cfe5560a" + "b1c6132c20a9f1dd163483a26f8ac53a" + "39d6808bf4a1dfbd261b099bb03b3fb5" + "0906cb28bd8a081f00"), + hexstr2bin("bd0f6a3747cd561bdddf4640a332461a" + "4a30a12a434cd0bf40d766d9c6d458e5" + "512204a30c17d1f50b5079631f64eb31" + "12182da3005835461113718d1a5ef944"), + hexstr2bin("554bc2480860b49eab8532d2a533b7d5" + "78ef473eeb58c98bb2d0e1ce488a98b1" + "8dfde9b9b90775e67f47d4a1c3482058" + "efc9f40d2ca033a0801b63d45b3b722e" + "f552bad3b4ccb667da350192b61c508c" + "f7b6b5adadc2c8d9a446ef003fb05cba" + "5f30e88e36ec2703b349ca229c267083" + "3900")}, + %% 256 octets + {ed448, undefined, + hexstr2bin("2ec5fe3c17045abdb136a5e6a913e32a" + "b75ae68b53d2fc149b77e504132d3756" + "9b7e766ba74a19bd6162343a21c8590a" + "a9cebca9014c636df5"), + hexstr2bin("79756f014dcfe2079f5dd9e718be4171" + "e2ef2486a08f25186f6bff43a9936b9b" + "fe12402b08ae65798a3d81e22e9ec80e" + "7690862ef3d4ed3a00"), + hexstr2bin("15777532b0bdd0d1389f636c5f6b9ba7" + "34c90af572877e2d272dd078aa1e567c" + "fa80e12928bb542330e8409f31745041" + "07ecd5efac61ae7504dabe2a602ede89" + "e5cca6257a7c77e27a702b3ae39fc769" + "fc54f2395ae6a1178cab4738e543072f" + "c1c177fe71e92e25bf03e4ecb72f47b6" + "4d0465aaea4c7fad372536c8ba516a60" + "39c3c2a39f0e4d832be432dfa9a706a6" + "e5c7e19f397964ca4258002f7c0541b5" + "90316dbc5622b6b2a6fe7a4abffd9610" + "5eca76ea7b98816af0748c10df048ce0" + "12d901015a51f189f3888145c03650aa" + "23ce894c3bd889e030d565071c59f409" + "a9981b51878fd6fc110624dcbcde0bf7" + "a69ccce38fabdf86f3bef6044819de11"), + hexstr2bin("c650ddbb0601c19ca11439e1640dd931" + "f43c518ea5bea70d3dcde5f4191fe53f" + "00cf966546b72bcc7d58be2b9badef28" + "743954e3a44a23f880e8d4f1cfce2d7a" + "61452d26da05896f0a50da66a239a8a1" + "88b6d825b3305ad77b73fbac0836ecc6" + "0987fd08527c1a8e80d5823e65cafe2a" + "3d00")}, + %% 1023 octets + {ed448, undefined, + hexstr2bin("872d093780f5d3730df7c212664b37b8" + "a0f24f56810daa8382cd4fa3f77634ec" + "44dc54f1c2ed9bea86fafb7632d8be19" + "9ea165f5ad55dd9ce8"), + hexstr2bin("a81b2e8a70a5ac94ffdbcc9badfc3feb" + "0801f258578bb114ad44ece1ec0e799d" + "a08effb81c5d685c0c56f64eecaef8cd" + "f11cc38737838cf400"), + hexstr2bin("6ddf802e1aae4986935f7f981ba3f035" + "1d6273c0a0c22c9c0e8339168e675412" + "a3debfaf435ed651558007db4384b650" + "fcc07e3b586a27a4f7a00ac8a6fec2cd" + "86ae4bf1570c41e6a40c931db27b2faa" + "15a8cedd52cff7362c4e6e23daec0fbc" + "3a79b6806e316efcc7b68119bf46bc76" + "a26067a53f296dafdbdc11c77f7777e9" + "72660cf4b6a9b369a6665f02e0cc9b6e" + "dfad136b4fabe723d2813db3136cfde9" + "b6d044322fee2947952e031b73ab5c60" + "3349b307bdc27bc6cb8b8bbd7bd32321" + "9b8033a581b59eadebb09b3c4f3d2277" + "d4f0343624acc817804728b25ab79717" + "2b4c5c21a22f9c7839d64300232eb66e" + "53f31c723fa37fe387c7d3e50bdf9813" + "a30e5bb12cf4cd930c40cfb4e1fc6225" + "92a49588794494d56d24ea4b40c89fc0" + "596cc9ebb961c8cb10adde976a5d602b" + "1c3f85b9b9a001ed3c6a4d3b1437f520" + "96cd1956d042a597d561a596ecd3d173" + "5a8d570ea0ec27225a2c4aaff26306d1" + "526c1af3ca6d9cf5a2c98f47e1c46db9" + "a33234cfd4d81f2c98538a09ebe76998" + "d0d8fd25997c7d255c6d66ece6fa56f1" + "1144950f027795e653008f4bd7ca2dee" + "85d8e90f3dc315130ce2a00375a318c7" + "c3d97be2c8ce5b6db41a6254ff264fa6" + "155baee3b0773c0f497c573f19bb4f42" + "40281f0b1f4f7be857a4e59d416c06b4" + "c50fa09e1810ddc6b1467baeac5a3668" + "d11b6ecaa901440016f389f80acc4db9" + "77025e7f5924388c7e340a732e554440" + "e76570f8dd71b7d640b3450d1fd5f041" + "0a18f9a3494f707c717b79b4bf75c984" + "00b096b21653b5d217cf3565c9597456" + "f70703497a078763829bc01bb1cbc8fa" + "04eadc9a6e3f6699587a9e75c94e5bab" + "0036e0b2e711392cff0047d0d6b05bd2" + "a588bc109718954259f1d86678a579a3" + "120f19cfb2963f177aeb70f2d4844826" + "262e51b80271272068ef5b3856fa8535" + "aa2a88b2d41f2a0e2fda7624c2850272" + "ac4a2f561f8f2f7a318bfd5caf969614" + "9e4ac824ad3460538fdc25421beec2cc" + "6818162d06bbed0c40a387192349db67" + "a118bada6cd5ab0140ee273204f628aa" + "d1c135f770279a651e24d8c14d75a605" + "9d76b96a6fd857def5e0b354b27ab937" + "a5815d16b5fae407ff18222c6d1ed263" + "be68c95f32d908bd895cd76207ae7264" + "87567f9a67dad79abec316f683b17f2d" + "02bf07e0ac8b5bc6162cf94697b3c27c" + "d1fea49b27f23ba2901871962506520c" + "392da8b6ad0d99f7013fbc06c2c17a56" + "9500c8a7696481c1cd33e9b14e40b82e" + "79a5f5db82571ba97bae3ad3e0479515" + "bb0e2b0f3bfcd1fd33034efc6245eddd" + "7ee2086ddae2600d8ca73e214e8c2b0b" + "db2b047c6a464a562ed77b73d2d841c4" + "b34973551257713b753632efba348169" + "abc90a68f42611a40126d7cb21b58695" + "568186f7e569d2ff0f9e745d0487dd2e" + "b997cafc5abf9dd102e62ff66cba87"), + hexstr2bin("e301345a41a39a4d72fff8df69c98075" + "a0cc082b802fc9b2b6bc503f926b65bd" + "df7f4c8f1cb49f6396afc8a70abe6d8a" + "ef0db478d4c6b2970076c6a0484fe76d" + "76b3a97625d79f1ce240e7c576750d29" + "5528286f719b413de9ada3e8eb78ed57" + "3603ce30d8bb761785dc30dbc320869e" + "1a00")} + ]. + ecdh() -> %% http://csrc.nist.gov/groups/STM/cavp/ Curves = crypto:ec_curves() ++ diff --git a/lib/kernel/src/net_kernel.erl b/lib/kernel/src/net_kernel.erl index 3cf11fd7b1..a9dc77837e 100644 --- a/lib/kernel/src/net_kernel.erl +++ b/lib/kernel/src/net_kernel.erl @@ -808,7 +808,8 @@ handle_info({AcceptPid, {accept_pending,MyNode,Node,Address,Type}}, State) -> _:_ -> error_logger:error_msg("~n** Cannot get connection id for node ~w~n", [Node]), - AcceptPid ! {self(),{accept_pending,nok_pending}} + AcceptPid ! {self(),{accept_pending,nok_pending}}, + {noreply, State} end end; diff --git a/lib/observer/src/Makefile b/lib/observer/src/Makefile index ff2bcbdb99..f9f239db37 100644 --- a/lib/observer/src/Makefile +++ b/lib/observer/src/Makefile @@ -50,6 +50,7 @@ MODULES= \ cdv_mem_cb \ cdv_mod_cb \ cdv_multi_wx \ + cdv_persistent_cb \ cdv_port_cb \ cdv_proc_cb \ cdv_sched_cb \ diff --git a/lib/observer/src/cdv_html_wx.erl b/lib/observer/src/cdv_html_wx.erl index d9efa7fc2f..ffef83227c 100644 --- a/lib/observer/src/cdv_html_wx.erl +++ b/lib/observer/src/cdv_html_wx.erl @@ -33,13 +33,17 @@ {panel, app, %% which tool is the user expand_table, - expand_wins=[]}). + expand_wins=[], + delayed_fetch, + trunc_warn=[]}). start_link(ParentWin, Info) -> wx_object:start_link(?MODULE, [ParentWin, Info], []). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +init([ParentWin, Callback]) when is_atom(Callback) -> + init(ParentWin, Callback); init([ParentWin, {App, Fun}]) when is_function(Fun) -> init([ParentWin, {App, Fun()}]); init([ParentWin, {expand,HtmlText,Tab}]) -> @@ -60,9 +64,29 @@ init(ParentWin, HtmlText, Tab, App) -> wx_misc:endBusyCursor(), {HtmlWin, #state{panel=HtmlWin,expand_table=Tab,app=App}}. +init(ParentWin, Callback) -> + {HtmlWin, State} = init(ParentWin, "", undefined, cdv), + {HtmlWin, State#state{delayed_fetch=Callback}}. + %%%%%%%%%%%%%%%%%%%%%%% Callbacks %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +handle_info(active, #state{panel=HtmlWin,delayed_fetch=Callback}=State) + when Callback=/=undefined -> + observer_lib:display_progress_dialog(HtmlWin, + "Crashdump Viewer", + "Reading data"), + {{expand,HtmlText,Tab},TW} = Callback:get_info(), + observer_lib:sync_destroy_progress_dialog(), + wx_misc:beginBusyCursor(), + wxHtmlWindow:setPage(HtmlWin,HtmlText), + cdv_wx:set_status(TW), + wx_misc:endBusyCursor(), + {noreply, State#state{expand_table=Tab, + delayed_fetch=undefined, + trunc_warn=TW}}; + handle_info(active, State) -> + cdv_wx:set_status(State#state.trunc_warn), {noreply, State}; handle_info(Info, State) -> diff --git a/lib/observer/src/cdv_persistent_cb.erl b/lib/observer/src/cdv_persistent_cb.erl new file mode 100644 index 0000000000..d5da18f7fc --- /dev/null +++ b/lib/observer/src/cdv_persistent_cb.erl @@ -0,0 +1,32 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% + +-module(cdv_persistent_cb). + +-export([get_info/0]). + +-include_lib("wx/include/wx.hrl"). + +get_info() -> + Tab = ets:new(pt_expand,[set,public]), + {ok,PT,TW} = crashdump_viewer:persistent_terms(), + {{expand, + observer_html_lib:expandable_term("Persistent Terms",PT,Tab), + Tab}, + TW}. diff --git a/lib/observer/src/cdv_wx.erl b/lib/observer/src/cdv_wx.erl index 78a897111c..1e9cef8952 100644 --- a/lib/observer/src/cdv_wx.erl +++ b/lib/observer/src/cdv_wx.erl @@ -51,6 +51,7 @@ -define(DIST_STR, "Nodes"). -define(MOD_STR, "Modules"). -define(MEM_STR, "Memory"). +-define(PERSISTENT_STR, "Persistent Terms"). -define(INT_STR, "Internal Tables"). %% Records @@ -74,6 +75,7 @@ dist_panel, mod_panel, mem_panel, + persistent_panel, int_panel, active_tab }). @@ -193,6 +195,10 @@ setup(#state{frame=Frame, notebook=Notebook}=State) -> %% Memory Panel MemPanel = add_page(Notebook, ?MEM_STR, cdv_multi_wx, cdv_mem_cb), + %% Persistent Terms Panel + PersistentPanel = add_page(Notebook, ?PERSISTENT_STR, + cdv_html_wx, cdv_persistent_cb), + %% Memory Panel IntPanel = add_page(Notebook, ?INT_STR, cdv_multi_wx, cdv_int_tab_cb), @@ -215,6 +221,7 @@ setup(#state{frame=Frame, notebook=Notebook}=State) -> dist_panel = DistPanel, mod_panel = ModPanel, mem_panel = MemPanel, + persistent_panel = PersistentPanel, int_panel = IntPanel, active_tab = GenPid }}. @@ -250,6 +257,7 @@ handle_event(#wx{id = ?wxID_OPEN, State#state.dist_panel, State#state.mod_panel, State#state.mem_panel, + State#state.persistent_panel, State#state.int_panel], _ = [wx_object:call(Panel,new_dump) || Panel<-Panels], wxNotebook:setSelection(State#state.notebook,0), @@ -343,8 +351,8 @@ check_page_title(Notebook) -> get_active_pid(#state{notebook=Notebook, gen_panel=Gen, pro_panel=Pro, port_panel=Ports, ets_panel=Ets, timer_panel=Timers, fun_panel=Funs, atom_panel=Atoms, dist_panel=Dist, - mod_panel=Mods, mem_panel=Mem, int_panel=Int, - sched_panel=Sched + mod_panel=Mods, mem_panel=Mem, persistent_panel=Persistent, + int_panel=Int, sched_panel=Sched }) -> Panel = case check_page_title(Notebook) of ?GEN_STR -> Gen; @@ -358,6 +366,7 @@ get_active_pid(#state{notebook=Notebook, gen_panel=Gen, pro_panel=Pro, ?DIST_STR -> Dist; ?MOD_STR -> Mods; ?MEM_STR -> Mem; + ?PERSISTENT_STR -> Persistent; ?INT_STR -> Int end, wx_object:get_pid(Panel). @@ -365,7 +374,7 @@ get_active_pid(#state{notebook=Notebook, gen_panel=Gen, pro_panel=Pro, pid2panel(Pid, #state{gen_panel=Gen, pro_panel=Pro, port_panel=Ports, ets_panel=Ets, timer_panel=Timers, fun_panel=Funs, atom_panel=Atoms, dist_panel=Dist, mod_panel=Mods, - mem_panel=Mem, int_panel=Int}) -> + mem_panel=Mem, persistent_panel=Persistent, int_panel=Int}) -> case Pid of Gen -> ?GEN_STR; Pro -> ?PRO_STR; @@ -377,6 +386,7 @@ pid2panel(Pid, #state{gen_panel=Gen, pro_panel=Pro, port_panel=Ports, Dist -> ?DIST_STR; Mods -> ?MOD_STR; Mem -> ?MEM_STR; + ?PERSISTENT_STR -> Persistent; Int -> ?INT_STR; _ -> "unknown" end. diff --git a/lib/observer/src/crashdump_viewer.erl b/lib/observer/src/crashdump_viewer.erl index 276b209ce5..0627c15b1c 100644 --- a/lib/observer/src/crashdump_viewer.erl +++ b/lib/observer/src/crashdump_viewer.erl @@ -74,6 +74,7 @@ loaded_modules/0, loaded_mod_details/1, memory/0, + persistent_terms/0, allocated_areas/0, allocator_info/0, hash_tables/0, @@ -139,6 +140,7 @@ -define(node,node). -define(not_connected,not_connected). -define(old_instr_data,old_instr_data). +-define(persistent_terms,persistent_terms). -define(port,port). -define(proc,proc). -define(proc_dictionary,proc_dictionary). @@ -293,6 +295,8 @@ loaded_mod_details(Mod) -> call({loaded_mod_details,Mod}). memory() -> call(memory). +persistent_terms() -> + call(persistent_terms). allocated_areas() -> call(allocated_areas). allocator_info() -> @@ -471,6 +475,11 @@ handle_call(memory,_From,State=#state{file=File}) -> Memory=memory(File), TW = truncated_warning([?memory]), {reply,{ok,Memory,TW},State}; +handle_call(persistent_terms,_From,State=#state{file=File,dump_vsn=DumpVsn}) -> + TW = truncated_warning([?persistent_terms,?literals]), + DecodeOpts = get_decode_opts(DumpVsn), + Terms = persistent_terms(File, DecodeOpts), + {reply,{ok,Terms,TW},State}; handle_call(allocated_areas,_From,State=#state{file=File}) -> AllocatedAreas=allocated_areas(File), TW = truncated_warning([?allocated_areas]), @@ -1444,15 +1453,7 @@ maybe_other_node2(Channel) -> expand_memory(Fd,Pid,DumpVsn) -> DecodeOpts = get_decode_opts(DumpVsn), put(fd,Fd), - Dict0 = case get(?literals) of - undefined -> - Literals = read_literals(Fd,DecodeOpts), - put(?literals,Literals), - put(fd,Fd), - Literals; - Literals -> - Literals - end, + Dict0 = get_literals(Fd,DecodeOpts), Dict = read_heap(Fd,Pid,DecodeOpts,Dict0), Expanded = {read_stack_dump(Fd,Pid,DecodeOpts,Dict), read_messages(Fd,Pid,DecodeOpts,Dict), @@ -1468,6 +1469,18 @@ expand_memory(Fd,Pid,DumpVsn) -> end, {Expanded,IncompleteWarning}. +get_literals(Fd,DecodeOpts) -> + case get(?literals) of + undefined -> + OldFd = put(fd,Fd), + Literals = read_literals(Fd,DecodeOpts), + put(fd,OldFd), + put(?literals,Literals), + Literals; + Literals -> + Literals + end. + read_literals(Fd,DecodeOpts) -> case lookup_index(?literals,[]) of [{_,Start}] -> @@ -2203,6 +2216,56 @@ get_atom(Atom) when is_binary(Atom) -> {Atom,nq}. % not quoted %%----------------------------------------------------------------- +%% Page with list of all persistent terms +persistent_terms(File, DecodeOpts) -> + case lookup_index(?persistent_terms) of + [{_Id,Start}] -> + Fd = open(File), + pos_bof(Fd,Start), + Terms = get_persistent_terms(Fd), + Dict = get_literals(Fd,DecodeOpts), + parse_persistent_terms(Terms,DecodeOpts,Dict); + _ -> + [] + end. + +parse_persistent_terms([[Name0,Val0]|Terms],DecodeOpts,Dict) -> + {Name,_,_} = parse_term(binary_to_list(Name0),DecodeOpts,Dict), + {Val,_,_} = parse_term(binary_to_list(Val0),DecodeOpts,Dict), + [{Name,Val}|parse_persistent_terms(Terms,DecodeOpts,Dict)]; +parse_persistent_terms([],_,_) -> []. + +get_persistent_terms(Fd) -> + case get_chunk(Fd) of + {ok,Bin} -> + get_persistent_terms(Fd,Bin,[]); + eof -> + [] + end. + + +%% Persistent_Terms are written one per line in the crash dump. +get_persistent_terms(Fd,Bin,PersistentTerms) -> + Bins = binary:split(Bin,<<"\n">>,[global]), + get_persistent_terms1(Fd,Bins,PersistentTerms). + +get_persistent_terms1(_Fd,[<<"=",_/binary>>|_],PersistentTerms) -> + PersistentTerms; +get_persistent_terms1(Fd,[LastBin],PersistentTerms) -> + case get_chunk(Fd) of + {ok,Bin0} -> + get_persistent_terms(Fd,<<LastBin/binary,Bin0/binary>>,PersistentTerms); + eof -> + [get_persistent_term(LastBin)|PersistentTerms] + end; +get_persistent_terms1(Fd,[Bin|Bins],Persistent_Terms) -> + get_persistent_terms1(Fd,Bins,[get_persistent_term(Bin)|Persistent_Terms]). + +get_persistent_term(Bin) -> + binary:split(Bin,<<"|">>). + + +%%----------------------------------------------------------------- %% Page with memory information memory(File) -> case lookup_index(?memory) of @@ -3192,6 +3255,7 @@ tag_to_atom("literals") -> ?literals; tag_to_atom("loaded_modules") -> ?loaded_modules; tag_to_atom("memory") -> ?memory; tag_to_atom("mod") -> ?mod; +tag_to_atom("persistent_terms") -> ?persistent_terms; tag_to_atom("no_distribution") -> ?no_distribution; tag_to_atom("node") -> ?node; tag_to_atom("not_connected") -> ?not_connected; diff --git a/lib/observer/src/observer.app.src b/lib/observer/src/observer.app.src index d73293a5f9..d48b846ad2 100644 --- a/lib/observer/src/observer.app.src +++ b/lib/observer/src/observer.app.src @@ -34,6 +34,7 @@ cdv_mem_cb, cdv_mod_cb, cdv_multi_wx, + cdv_persistent_cb, cdv_port_cb, cdv_proc_cb, cdv_table_wx, diff --git a/lib/observer/src/observer_html_lib.erl b/lib/observer/src/observer_html_lib.erl index 0c4e32af49..c67fa28c6d 100644 --- a/lib/observer/src/observer_html_lib.erl +++ b/lib/observer/src/observer_html_lib.erl @@ -62,7 +62,8 @@ expandable_term_body(Heading,[],_Tab) -> "Dictionary" -> "No dictionary was found"; "ProcState" -> "Information could not be retrieved," " system messages may not be handled by this process."; - "SaslLog" -> "No log entry was found" + "SaslLog" -> "No log entry was found"; + "Persistent Terms" -> "No persistent terms were found" end]; expandable_term_body(Heading,Expanded,Tab) -> Attr = "BORDER=0 CELLPADDING=0 CELLSPACING=1 WIDTH=100%", diff --git a/lib/observer/test/crashdump_helper.erl b/lib/observer/test/crashdump_helper.erl index d8f4e046ae..576d112154 100644 --- a/lib/observer/test/crashdump_helper.erl +++ b/lib/observer/test/crashdump_helper.erl @@ -21,7 +21,9 @@ -module(crashdump_helper). -export([n1_proc/2,remote_proc/2, dump_maps/0,create_maps/0, - create_binaries/0,create_sub_binaries/1]). + create_binaries/0,create_sub_binaries/1, + dump_persistent_terms/0, + create_persistent_terms/0]). -compile(r18). -include_lib("common_test/include/ct.hrl"). @@ -162,3 +164,27 @@ literal_map() -> %% Adress1, then the map at Address2. #{"one"=>1,"two"=>2,"three"=>3,"four"=>4}. + +%%% +%%% Test dumping of persistent terms (from OTP 21.2). +%%% + +dump_persistent_terms() -> + Parent = self(), + F = fun() -> + register(aaaaaaaa_persistent_terms, self()), + put(pts, create_persistent_terms()), + Parent ! {self(),done}, + receive _ -> ok end + end, + Pid = spawn_link(F), + receive + {Pid,done} -> + {ok,Pid} + end. + +create_persistent_terms() -> + persistent_term:put({?MODULE,first}, {pid,42.0}), + persistent_term:put({?MODULE,second}, [1,2,3]), + persistent_term:get(). + diff --git a/lib/observer/test/crashdump_viewer_SUITE.erl b/lib/observer/test/crashdump_viewer_SUITE.erl index 864454cdff..ed53c46a0d 100644 --- a/lib/observer/test/crashdump_viewer_SUITE.erl +++ b/lib/observer/test/crashdump_viewer_SUITE.erl @@ -345,6 +345,7 @@ browse_file(File) -> {ok,_AllocINfo,_AllocInfoTW} = crashdump_viewer:allocator_info(), {ok,_HashTabs,_HashTabsTW} = crashdump_viewer:hash_tables(), {ok,_IndexTabs,_IndexTabsTW} = crashdump_viewer:index_tables(), + {ok,_PTs,_PTsTW} = crashdump_viewer:persistent_terms(), io:format(" info read",[]), @@ -595,6 +596,23 @@ special(File,Procs) -> Maps = proplists:get_value(maps,Dict), io:format(" maps ok",[]), ok; + ".persistent_terms" -> + %% I registered a process as aaaaaaaa_persistent_term in + %% the dump to make sure it will be the first in the list + %% when sorted on names. + [#proc{pid=Pid0,name=Name}|_Rest] = lists:keysort(#proc.name,Procs), + "aaaaaaaa_persistent_terms" = Name, + Pid = pid_to_list(Pid0), + {ok,ProcDetails=#proc{},[]} = crashdump_viewer:proc_details(Pid), + io:format(" process details ok",[]), + + #proc{dict=Dict} = ProcDetails, + %% io:format("~p\n", [Dict]), + Pts1 = crashdump_helper:create_persistent_terms(), + Pts2 = proplists:get_value(pts,Dict), + true = lists:sort(Pts1) =:= lists:sort(Pts2), + io:format(" persistent terms ok",[]), + ok; _ -> ok end, @@ -679,9 +697,11 @@ do_create_dumps(DataDir,Rel) -> CD5 = dump_with_size_limit_reached(DataDir,Rel,"trunc_bytes"), CD6 = dump_with_unicode_atoms(DataDir,Rel,"unicode"), CD7 = dump_with_maps(DataDir,Rel,"maps"), + CD8 = dump_with_persistent_terms(DataDir,Rel,"persistent_terms"), TruncDumpMod = truncate_dump_mod(CD1), TruncatedDumpsBinary = truncate_dump_binary(CD1), - {[CD1,CD2,CD3,CD4,CD5,CD6,CD7,TruncDumpMod|TruncatedDumpsBinary], + {[CD1,CD2,CD3,CD4,CD5,CD6,CD7,CD8, + TruncDumpMod|TruncatedDumpsBinary], DosDump}; _ -> {[CD1,CD2], DosDump} @@ -850,6 +870,16 @@ dump_with_maps(DataDir,Rel,DumpName) -> ?t:stop_node(n1), CD. +dump_with_persistent_terms(DataDir,Rel,DumpName) -> + Opt = rel_opt(Rel), + Pz = "-pz \"" ++ filename:dirname(code:which(?MODULE)) ++ "\"", + PzOpt = [{args,Pz}], + {ok,N1} = ?t:start_node(n1,peer,Opt ++ PzOpt), + {ok,_Pid} = rpc:call(N1,crashdump_helper,dump_persistent_terms,[]), + CD = dump(N1,DataDir,Rel,DumpName), + ?t:stop_node(n1), + CD. + dump(Node,DataDir,Rel,DumpName) -> Crashdump = filename:join(DataDir, dump_prefix(Rel)++DumpName), rpc:call(Node,os,putenv,["ERL_CRASH_DUMP",Crashdump]), diff --git a/lib/public_key/asn1/OTP-PKIX.asn1 b/lib/public_key/asn1/OTP-PKIX.asn1 index 10a83555af..9bcd99fba3 100644 --- a/lib/public_key/asn1/OTP-PKIX.asn1 +++ b/lib/public_key/asn1/OTP-PKIX.asn1 @@ -326,8 +326,13 @@ PublicKeyAlgorithm ::= SEQUENCE { OPTIONAL } SupportedSignatureAlgorithms SIGNATURE-ALGORITHM-CLASS ::= { - dsa-with-sha1 | dsaWithSHA1 | md2-with-rsa-encryption | - md5-with-rsa-encryption | sha1-with-rsa-encryption | sha-1with-rsa-encryption | + dsa-with-sha1 | dsaWithSHA1 | + dsa-with-sha224 | + dsa-with-sha256 | + md2-with-rsa-encryption | + md5-with-rsa-encryption | + sha1-with-rsa-encryption | + sha-1with-rsa-encryption | sha224-with-rsa-encryption | sha256-with-rsa-encryption | sha384-with-rsa-encryption | @@ -368,6 +373,14 @@ SupportedPublicKeyAlgorithms PUBLIC-KEY-ALGORITHM-CLASS ::= { ID id-dsaWithSHA1 TYPE DSAParams } + dsa-with-sha224 SIGNATURE-ALGORITHM-CLASS ::= { + ID id-dsa-with-sha224 + TYPE DSAParams } + + dsa-with-sha256 SIGNATURE-ALGORITHM-CLASS ::= { + ID id-dsa-with-sha256 + TYPE DSAParams } + id-dsa-with-sha224 OBJECT IDENTIFIER ::= { joint-iso-ccitt(2) country(16) us(840) organization(1) gov(101) csor(3) algorithms(4) id-dsa-with-sha2(3) 1 } diff --git a/lib/public_key/doc/src/notes.xml b/lib/public_key/doc/src/notes.xml index 62b4b4ca1b..7ed60ed3ca 100644 --- a/lib/public_key/doc/src/notes.xml +++ b/lib/public_key/doc/src/notes.xml @@ -35,6 +35,22 @@ <file>notes.xml</file> </header> +<section><title>Public_Key 1.6.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Add DSA SHA2 oids in public_keys ASN1-spec and + public_key:pkix_sign_types/1</p> + <p> + Own Id: OTP-15367</p> + </item> + </list> + </section> + +</section> + <section><title>Public_Key 1.6.2</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/public_key/doc/src/public_key.xml b/lib/public_key/doc/src/public_key.xml index a4d7e4a734..ee3877ddd0 100644 --- a/lib/public_key/doc/src/public_key.xml +++ b/lib/public_key/doc/src/public_key.xml @@ -109,6 +109,13 @@ </datatype> <datatype> + <name name="ed_public_key"/> + <desc> + <warning><p>This format of the EdDSA curves is temporary and may change without prior notice!</p></warning> + </desc> + </datatype> + + <datatype> <name name="private_key"/> <name name="rsa_private_key"/> <name name="dsa_private_key"/> @@ -118,6 +125,14 @@ </datatype> <datatype> + <name name="ed_private_key"/> + <desc> + <warning><p>This format of the EdDSA curves is temporary and may change without prior notice!</p></warning> + </desc> + </datatype> + + + <datatype> <name name="key_params"/> <desc> </desc> diff --git a/lib/public_key/src/pubkey_pem.erl b/lib/public_key/src/pubkey_pem.erl index b92790554f..d7e5bc3ad8 100644 --- a/lib/public_key/src/pubkey_pem.erl +++ b/lib/public_key/src/pubkey_pem.erl @@ -222,7 +222,9 @@ pem_start('CertificateList') -> pem_start('EcpkParameters') -> <<"-----BEGIN EC PARAMETERS-----">>; pem_start('ECPrivateKey') -> - <<"-----BEGIN EC PRIVATE KEY-----">>. + <<"-----BEGIN EC PRIVATE KEY-----">>; +pem_start({no_asn1, new_openssh}) -> %% Temporarily in the prototype of this format + <<"-----BEGIN OPENSSH PRIVATE KEY-----">>. pem_end(<<"-----BEGIN CERTIFICATE-----">>) -> <<"-----END CERTIFICATE-----">>; @@ -250,6 +252,8 @@ pem_end(<<"-----BEGIN EC PARAMETERS-----">>) -> <<"-----END EC PARAMETERS-----">>; pem_end(<<"-----BEGIN EC PRIVATE KEY-----">>) -> <<"-----END EC PRIVATE KEY-----">>; +pem_end(<<"-----BEGIN OPENSSH PRIVATE KEY-----">>) -> + <<"-----END OPENSSH PRIVATE KEY-----">>; pem_end(_) -> undefined. @@ -278,7 +282,10 @@ asn1_type(<<"-----BEGIN X509 CRL-----">>) -> asn1_type(<<"-----BEGIN EC PARAMETERS-----">>) -> 'EcpkParameters'; asn1_type(<<"-----BEGIN EC PRIVATE KEY-----">>) -> - 'ECPrivateKey'. + 'ECPrivateKey'; +asn1_type(<<"-----BEGIN OPENSSH PRIVATE KEY-----">>) -> + {no_asn1, new_openssh}. %% Temporarily in the prototype of this format + pem_decrypt() -> <<"Proc-Type: 4,ENCRYPTED">>. diff --git a/lib/public_key/src/pubkey_ssh.erl b/lib/public_key/src/pubkey_ssh.erl index 02c061efc9..d0ef4abfb1 100644 --- a/lib/public_key/src/pubkey_ssh.erl +++ b/lib/public_key/src/pubkey_ssh.erl @@ -25,7 +25,8 @@ -export([decode/2, encode/2, dh_gex_group/4, - dh_gex_group_sizes/0 + dh_gex_group_sizes/0, +pad/2, new_openssh_encode/1, new_openssh_decode/1 % For test and experiments ]). -define(UINT32(X), X:32/unsigned-big-integer). @@ -67,6 +68,8 @@ decode(Bin, rfc4716_public_key) -> rfc4716_decode(Bin); decode(Bin, ssh2_pubkey) -> ssh2_pubkey_decode(Bin); +decode(Bin, new_openssh) -> + new_openssh_decode(Bin); decode(Bin, Type) -> openssh_decode(Bin, Type). @@ -177,6 +180,70 @@ join_entry([Line | Lines], Entry) -> rfc4716_pubkey_decode(BinKey) -> ssh2_pubkey_decode(BinKey). +%% From https://github.com/openssh/openssh-portable/blob/master/PROTOCOL.key +new_openssh_decode(<<"openssh-key-v1",0, + ?DEC_BIN(CipherName, _L1), + ?DEC_BIN(KdfName, _L2), + ?DEC_BIN(KdfOptions, _L3), + ?UINT32(N), % number of keys + ?DEC_BIN(PublicKey, _L4), + ?DEC_BIN(Encrypted, _L5), + _Rest/binary + >>) -> + %%io:format("CipherName = ~p~nKdfName = ~p~nKdfOptions = ~p~nPublicKey = ~p~nN = ~p~nEncrypted = ~p~nRest = ~p~n", [CipherName, KdfName, KdfOptions, PublicKey, N, Encrypted, _Rest]), + new_openssh_decode(CipherName, KdfName, KdfOptions, PublicKey, N, Encrypted). + +new_openssh_decode(<<"none">>, <<"none">>, <<"">>, _PublicKey, 1, + <<?UINT32(CheckInt), + ?UINT32(CheckInt), + ?DEC_BIN(Type, _Lt), + ?DEC_BIN(PubKey, _Lpu), + ?DEC_BIN(PrivPubKey, _Lpripub), + ?DEC_BIN(_Comment, _C1), + _Pad/binary>>) -> + case {Type,PrivPubKey} of + {<<"ssh-ed25519">>, + <<PrivKey:32/binary, PubKey:32/binary>>} -> + {ed_pri, ed25519, PubKey, PrivKey}; + + {<<"ssh-ed448">>, + <<PrivKey:57/binary, PubKey/binary>>} -> % "Intelligent" guess from + % https://tools.ietf.org/html/draft-ietf-curdle-ssh-ed25519-ed448 + {ed_pri, ed448, PubKey, PrivKey} + end. + + +new_openssh_encode({ed_pri,_,PubKey,PrivKey}=Key) -> + Type = key_type(Key), + CheckInt = 17*256+17, %crypto:strong_rand_bytes(4), + Comment = <<>>, + PublicKey = <<?STRING(Type),?STRING(PubKey)>>, + CipherName = <<"none">>, + KdfName = <<"none">>, + KdfOptions = <<>>, + BlockSize = 8, % Crypto dependent + NumKeys = 1, + Encrypted0 = <<?UINT32(CheckInt), + ?UINT32(CheckInt), + ?STRING(Type), + ?STRING(PubKey), + ?STRING(<<PrivKey/binary,PubKey/binary>>), + ?STRING(Comment) + >>, + Pad = pad(size(Encrypted0), BlockSize), + Encrypted = <<Encrypted0/binary, Pad/binary>>, + <<"openssh-key-v1",0, + ?STRING(CipherName), + ?STRING(KdfName), + ?STRING(KdfOptions), + ?UINT32(NumKeys), + ?STRING(PublicKey), + ?STRING(Encrypted)>>. + +pad(N, BlockSize) when N>BlockSize -> pad(N rem BlockSize, BlockSize); +pad(N, BlockSize) -> list_to_binary(lists:seq(1,BlockSize-N)). + + openssh_decode(Bin, FileType) -> Lines = binary:split(Bin, <<"\n">>, [global]), do_openssh_decode(FileType, Lines, []). @@ -235,6 +302,8 @@ do_openssh_decode(openssh_public_key = FileType, [Line | Lines], Acc) -> <<"ssh-rsa">> -> true; <<"ssh-dss">> -> true; <<"ecdsa-sha2-",Curve/binary>> -> is_ssh_curvename(Curve); + <<"ssh-ed25519">> -> true; + <<"ssh-ed448">> -> true; _ -> false end, @@ -247,7 +316,9 @@ do_openssh_decode(openssh_public_key = FileType, [Line | Lines], Acc) -> Comment = string:strip(string_decode(iolist_to_binary(Comment0)), right, $\n), do_openssh_decode(FileType, Lines, [{openssh_pubkey_decode(KeyType, Base64Enc), - [{comment, Comment}]} | Acc]) + [{comment, Comment}]} | Acc]); + _ when KnownKeyType==false -> + do_openssh_decode(FileType, Lines, Acc) end. @@ -386,6 +457,10 @@ line_end(Comment) -> key_type(#'RSAPublicKey'{}) -> <<"ssh-rsa">>; key_type({_, #'Dss-Parms'{}}) -> <<"ssh-dss">>; +key_type({ed_pub,ed25519,_}) -> <<"ssh-ed25519">>; +key_type({ed_pub,ed448,_}) -> <<"ssh-ed448">>; +key_type({ed_pri,ed25519,_,_}) -> <<"ssh-ed25519">>; +key_type({ed_pri,ed448,_,_}) -> <<"ssh-ed448">>; key_type({#'ECPoint'{}, {namedCurve,Curve}}) -> <<"ecdsa-sha2-", (public_key:oid2ssh_curvename(Curve))/binary>>. comma_list_encode([Option], []) -> @@ -404,7 +479,12 @@ ssh2_pubkey_encode({Y, #'Dss-Parms'{p = P, q = Q, g = G}}) -> <<?STRING(<<"ssh-dss">>), ?Empint(P), ?Empint(Q), ?Empint(G), ?Empint(Y)>>; ssh2_pubkey_encode(Key={#'ECPoint'{point = Q}, {namedCurve,OID}}) -> Curve = public_key:oid2ssh_curvename(OID), - <<?STRING(key_type(Key)), ?Estring(Curve), ?Estring(Q)>>. + <<?STRING(key_type(Key)), ?Estring(Curve), ?Estring(Q)>>; +ssh2_pubkey_encode({ed_pub, ed25519, Key}) -> + <<?STRING(<<"ssh-ed25519">>), ?Estring(Key)>>; +ssh2_pubkey_encode({ed_pub, ed448, Key}) -> + <<?STRING(<<"ssh-ed448">>), ?Estring(Key)>>. + ssh2_pubkey_decode(<<?DEC_BIN(Type,_TL), Bin/binary>>) -> @@ -430,12 +510,23 @@ ssh2_pubkey_decode(<<"ssh-dss">>, ssh2_pubkey_decode(<<"ecdsa-sha2-",Id/binary>>, <<?DEC_BIN(Id, _IL), ?DEC_BIN(Q, _QL)>>) -> - {#'ECPoint'{point = Q}, {namedCurve,public_key:ssh_curvename2oid(Id)}}. + {#'ECPoint'{point = Q}, {namedCurve,public_key:ssh_curvename2oid(Id)}}; + +ssh2_pubkey_decode(<<"ssh-ed25519">>, + <<?DEC_BIN(Key, _L)>>) -> + {ed_pub, ed25519, Key}; + +ssh2_pubkey_decode(<<"ssh-ed448">>, + <<?DEC_BIN(Key, _L)>>) -> + {ed_pub, ed448, Key}. + is_key_field(<<"ssh-dss">>) -> true; is_key_field(<<"ssh-rsa">>) -> true; +is_key_field(<<"ssh-ed25519">>) -> true; +is_key_field(<<"ssh-ed448">>) -> true; is_key_field(<<"ecdsa-sha2-",Id/binary>>) -> is_ssh_curvename(Id); is_key_field(_) -> false. diff --git a/lib/public_key/src/public_key.erl b/lib/public_key/src/public_key.erl index dca1e0766e..75d40d2e8a 100644 --- a/lib/public_key/src/public_key.erl +++ b/lib/public_key/src/public_key.erl @@ -68,8 +68,8 @@ pki_asn1_type/0, asn1_type/0, ssh_file/0, der_encoded/0, key_params/0, digest_type/0]). --type public_key() :: rsa_public_key() | dsa_public_key() | ec_public_key(). --type private_key() :: rsa_private_key() | dsa_private_key() | ec_private_key(). +-type public_key() :: rsa_public_key() | dsa_public_key() | ec_public_key() | ed_public_key() . +-type private_key() :: rsa_private_key() | dsa_private_key() | ec_private_key() | ed_private_key() . -type rsa_public_key() :: #'RSAPublicKey'{}. -type rsa_private_key() :: #'RSAPrivateKey'{}. @@ -79,6 +79,9 @@ -type ecpk_parameters_api() :: ecpk_parameters() | #'ECParameters'{} | {namedCurve, Name::crypto:ec_named_curve()}. -type ec_public_key() :: {#'ECPoint'{}, ecpk_parameters_api()}. -type ec_private_key() :: #'ECPrivateKey'{}. +-type ed_public_key() :: {ed_pub, ed25519|ed448, Key::binary()}. +-type ed_private_key() :: {ed_pri, ed25519|ed448, Pub::binary(), Priv::binary()}. + -type key_params() :: #'DHParameter'{} | {namedCurve, oid()} | #'ECParameters'{} | {rsa, Size::integer(), PubExp::integer()}. -type der_encoded() :: binary(). @@ -166,6 +169,8 @@ pem_entry_decode({'SubjectPublicKeyInfo', Der, _}) -> ECCParams = der_decode('EcpkParameters', Params), {#'ECPoint'{point = Key0}, ECCParams} end; +pem_entry_decode({{no_asn1,new_openssh}, Special, not_encrypted}) -> + ssh_decode(Special, new_openssh); pem_entry_decode({Asn1Type, Der, not_encrypted}) when is_atom(Asn1Type), is_binary(Der) -> der_decode(Asn1Type, Der). @@ -1070,8 +1075,9 @@ pkix_verify_hostname_match_fun(https) -> -spec ssh_decode(SshBin, Type) -> Decoded when SshBin :: binary(), - Type :: ssh2_pubkey | OtherType, + Type :: ssh2_pubkey | OtherType | InternalType, OtherType :: public_key | ssh_file(), + InternalType :: new_openssh, Decoded :: Decoded_ssh2_pubkey | Decoded_OtherType, Decoded_ssh2_pubkey :: public_key(), @@ -1090,7 +1096,8 @@ ssh_decode(SshBin, Type) when is_binary(SshBin), Type == openssh_public_key; Type == auth_keys; Type == known_hosts; - Type == ssh2_pubkey -> + Type == ssh2_pubkey; + Type == new_openssh -> pubkey_ssh:decode(SshBin, Type). %%-------------------------------------------------------------------- @@ -1233,6 +1240,8 @@ format_sign_key(#'DSAPrivateKey'{p = P, q = Q, g = G, x = X}) -> {dss, [P, Q, G, X]}; format_sign_key(#'ECPrivateKey'{privateKey = PrivKey, parameters = Param}) -> {ecdsa, [PrivKey, ec_curve_spec(Param)]}; +format_sign_key({ed_pri, Curve, _Pub, Priv}) -> + {eddsa, [Priv,Curve]}; format_sign_key(_) -> badarg. @@ -1242,6 +1251,8 @@ format_verify_key({#'ECPoint'{point = Point}, Param}) -> {ecdsa, [Point, ec_curve_spec(Param)]}; format_verify_key({Key, #'Dss-Parms'{p = P, q = Q, g = G}}) -> {dss, [P, Q, G, Key]}; +format_verify_key({ed_pub, Curve, Key}) -> + {eddsa, [Key,Curve]}; %% Convert private keys to public keys format_verify_key(#'RSAPrivateKey'{modulus = Mod, publicExponent = Exp}) -> format_verify_key(#'RSAPublicKey'{modulus = Mod, publicExponent = Exp}); diff --git a/lib/public_key/vsn.mk b/lib/public_key/vsn.mk index 4e52028c36..96eaf4f962 100644 --- a/lib/public_key/vsn.mk +++ b/lib/public_key/vsn.mk @@ -1 +1 @@ -PUBLIC_KEY_VSN = 1.6.2 +PUBLIC_KEY_VSN = 1.6.3 diff --git a/lib/sasl/src/sasl.app.src b/lib/sasl/src/sasl.app.src index 688aff16f1..5d45af0b50 100644 --- a/lib/sasl/src/sasl.app.src +++ b/lib/sasl/src/sasl.app.src @@ -43,5 +43,5 @@ {env, []}, {mod, {sasl, []}}, {runtime_dependencies, ["tools-2.6.14","stdlib-3.4","kernel-5.3", - "erts-9.0"]}]}. + "erts-@OTP-13468@"]}]}. diff --git a/lib/sasl/src/systools_make.erl b/lib/sasl/src/systools_make.erl index 6916107623..f085246924 100644 --- a/lib/sasl/src/systools_make.erl +++ b/lib/sasl/src/systools_make.erl @@ -1562,10 +1562,10 @@ mandatory_modules() -> preloaded() -> %% Sorted - [erl_prim_loader,erl_tracer,erlang, + [atomics, counters, erl_prim_loader,erl_tracer,erlang, erts_code_purger,erts_dirty_process_signal_handler, erts_internal,erts_literal_area_collector, - init,otp_ring0,prim_buffer,prim_eval,prim_file, + init,otp_ring0,persistent_term,prim_buffer,prim_eval,prim_file, prim_inet,prim_zip,zlib]. %%______________________________________________________________________ diff --git a/lib/ssh/doc/src/notes.xml b/lib/ssh/doc/src/notes.xml index 7e77c6a457..42bdf667f8 100644 --- a/lib/ssh/doc/src/notes.xml +++ b/lib/ssh/doc/src/notes.xml @@ -230,6 +230,22 @@ </section> </section> +<section><title>Ssh 4.6.9.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Incompatibility with newer OpenSSH fixed. Previously + versions 7.8 and later could cause Erlang SSH to exit.</p> + <p> + Own Id: OTP-15413</p> + </item> + </list> + </section> + +</section> + <section><title>Ssh 4.6.9.1</title> <section><title>Fixed Bugs and Malfunctions</title> <list> @@ -3869,4 +3885,3 @@ </section> </chapter> - diff --git a/lib/ssh/doc/src/ssh.xml b/lib/ssh/doc/src/ssh.xml index b75b4a33c2..8435fced11 100644 --- a/lib/ssh/doc/src/ssh.xml +++ b/lib/ssh/doc/src/ssh.xml @@ -183,22 +183,6 @@ </datatype> <datatype> - <name name="pref_public_key_algs_client_option"/> - <desc> - <p>List of user (client) public key algorithms to try to use.</p> - <p>The default value is the <c>public_key</c> entry in the list returned by - <seealso marker="#default_algorithms/0">ssh:default_algorithms/0</seealso>. - </p> - <p>If there is no public key of a specified type available, the corresponding entry is ignored. - Note that the available set is dependent on the underlying cryptolib and current user's public keys. - </p> - <p>See also the option <seealso marker="ssh_file#type-user_dir_common_option"><c>user_dir</c></seealso> - for specifying the path to the user's keys. - </p> - </desc> - </datatype> - - <datatype> <name name="host_accepting_client_options"/> <name name="accept_hosts"/> <name name="fp_digest_alg"/> @@ -397,9 +381,20 @@ <datatype> <name name="exec_daemon_option"/> + <name name="exec_spec"/> + <desc/> + </datatype> + <datatype> + <name name="exec_fun"/> + <desc/> + </datatype> + <datatype> <name name="'exec_fun/1'"/> <name name="'exec_fun/2'"/> <name name="'exec_fun/3'"/> + <desc/> + </datatype> + <datatype> <name name="exec_result"/> <desc> <p>This option changes how the daemon execute exec-requests from clients. The term in the return value @@ -783,6 +778,22 @@ </datatype> <datatype> + <name name="pref_public_key_algs_common_option"/> + <desc> + <p>List of user (client) public key algorithms to try to use.</p> + <p>The default value is the <c>public_key</c> entry in the list returned by + <seealso marker="#default_algorithms/0">ssh:default_algorithms/0</seealso>. + </p> + <p>If there is no public key of a specified type available, the corresponding entry is ignored. + Note that the available set is dependent on the underlying cryptolib and current user's public keys. + </p> + <p>See also the option <seealso marker="ssh_file#type-user_dir_common_option"><c>user_dir</c></seealso> + for specifying the path to the user's keys. + </p> + </desc> + </datatype> + + <datatype> <name name="disconnectfun_common_option"/> <desc> <p>Provides a fun to implement your own logging when the peer disconnects.</p> diff --git a/lib/ssh/doc/src/ssh_app.xml b/lib/ssh/doc/src/ssh_app.xml index eb804e67dc..0c22a50c3f 100644 --- a/lib/ssh/doc/src/ssh_app.xml +++ b/lib/ssh/doc/src/ssh_app.xml @@ -175,6 +175,8 @@ <item>ecdsa-sha2-nistp384</item> <item>ecdsa-sha2-nistp521</item> <item>ecdsa-sha2-nistp256</item> + <item>ssh-ed25519</item> + <item>ssh-ed448</item> <item>ssh-rsa</item> <item>rsa-sha2-256</item> <item>rsa-sha2-512</item> @@ -378,7 +380,11 @@ <item> <url href="https://tools.ietf.org/html/draft-ietf-curdle-ssh-curves">Secure Shell (SSH) Key Exchange Method using Curve25519 and Curve448 (work in progress)</url> </item> - + + <item> + <url href="https://tools.ietf.org/html/draft-ietf-curdle-ssh-ed25519-ed448">Ed25519 and Ed448 public key algorithms for the Secure Shell (SSH) protocol (work in progress)</url> + </item> + </list> </section> diff --git a/lib/ssh/doc/src/ssh_file.xml b/lib/ssh/doc/src/ssh_file.xml index ae6ba2e1d9..6681d9c306 100644 --- a/lib/ssh/doc/src/ssh_file.xml +++ b/lib/ssh/doc/src/ssh_file.xml @@ -93,6 +93,8 @@ <item><marker id="SYSDIR-ssh_host_dsa_key"/><c>ssh_host_dsa_key</c> - private dss host key (optional)</item> <item><marker id="SYSDIR-ssh_host_rsa_key"/><c>ssh_host_rsa_key</c> - private rsa host key (optional)</item> <item><marker id="SYSDIR-ssh_host_ecdsa_key"/><c>ssh_host_ecdsa_key</c> - private ecdsa host key (optional)</item> + <item><marker id="SYSDIR-ssh_host_ed25519_key"/><c>ssh_host_ed25519_key</c> - private eddsa host key for curve 25519 (optional)</item> + <item><marker id="SYSDIR-ssh_host_ed448_key"/><c>ssh_host_ed448_key</c> - private eddsa host key for curve 448 (optional)</item> </list> <p>At least one host key must be defined. The default value of SYSDIR is <marker id="#/etc/ssh"/><c>/etc/ssh</c>. </p> @@ -115,6 +117,8 @@ <item><marker id="USERDIR-id_dsa"/><c>id_dsa</c> - private dss user key (optional)</item> <item><marker id="USERDIR-id_rsa"/><c>id_rsa</c> - private rsa user key (optional)</item> <item><marker id="USERDIR-id_ecdsa"/><c>id_ecdsa</c> - private ecdsa user key (optional)</item> + <item><marker id="USERDIR-id_ed25519"/><c>id_ed25519</c> - private eddsa user key for curve 25519 (optional)</item> + <item><marker id="USERDIR-id_ed448"/><c>id_ed448</c> - private eddsa user key for curve 448 (optional)</item> </list> <p>The default value of USERDIR is <c>/home/</c><seealso marker="#LOCALUSER"><c>LOCALUSER</c></seealso><c>/.ssh</c>. </p> @@ -157,6 +161,7 @@ <p>If the user's DSA, RSA or ECDSA key is protected by a passphrase, it can be supplied with thoose options. </p> + <p>Note that EdDSA passhrases (Curves 25519 and 448) are not implemented.</p> </desc> </datatype> @@ -183,6 +188,8 @@ <item><seealso marker="#SYSDIR-ssh_host_rsa_key"><c>SYSDIR/ssh_host_rsa_key</c></seealso></item> <item><seealso marker="#SYSDIR-ssh_host_dsa_key"><c>SYSDIR/ssh_host_dsa_key</c></seealso></item> <item><seealso marker="#SYSDIR-ssh_host_ecdsa_key"><c>SYSDIR/ssh_host_ecdsa_key</c></seealso></item> + <item><seealso marker="#SYSDIR-ssh_host_ed25519_key"><c>SYSDIR/ssh_host_ed25519_key</c></seealso></item> + <item><seealso marker="#SYSDIR-ssh_host_ed448_key"><c>SYSDIR/ssh_host_ed448_key</c>c></seealso></item> </list> </desc> </func> @@ -261,11 +268,14 @@ <item><seealso marker="#type-pubkey_passphrase_client_options">rsa_pass_phrase</seealso></item> <item><seealso marker="#type-pubkey_passphrase_client_options">ecdsa_pass_phrase</seealso></item> </list> + <p>Note that EdDSA passhrases (Curves 25519 and 448) are not implemented.</p> <p><strong>Files</strong></p> <list> <item><seealso marker="#USERDIR-id_dsa"><c>USERDIR/id_dsa</c></seealso></item> <item><seealso marker="#USERDIR-id_rsa"><c>USERDIR/id_rsa</c></seealso></item> <item><seealso marker="#USERDIR-id_ecdsa"><c>USERDIR/id_ecdsa</c></seealso></item> + <item><seealso marker="#USERDIR-id_ed25519"><c>USERDIR/id_ed25519</c></seealso></item> + <item><seealso marker="#USERDIR-id_ed448"><c>USERDIR/id_ed448</c></seealso></item> </list> </desc> </func> diff --git a/lib/ssh/src/ssh.hrl b/lib/ssh/src/ssh.hrl index 3ac74c4925..923e9309f4 100644 --- a/lib/ssh/src/ssh.hrl +++ b/lib/ssh/src/ssh.hrl @@ -129,6 +129,8 @@ -type pubkey_alg() :: 'ecdsa-sha2-nistp256' | 'ecdsa-sha2-nistp384' | 'ecdsa-sha2-nistp521' | + 'ssh-ed25519' | + 'ssh-ed448' | 'rsa-sha2-256' | 'rsa-sha2-512' | 'ssh-dss' | @@ -182,6 +184,7 @@ | ssh_msg_debug_fun_common_option() | rekey_limit_common_option() | id_string_common_option() + | pref_public_key_algs_common_option() | preferred_algorithms_common_option() | modify_algorithms_common_option() | auth_methods_common_option() @@ -209,6 +212,7 @@ {ssh_msg_debug_fun, fun((ssh:connection_ref(),AlwaysDisplay::boolean(),Msg::binary(),LanguageTag::binary()) -> any()) } . -type id_string_common_option() :: {id_string, string() | random | {random,Nmin::pos_integer(),Nmax::pos_integer()} }. +-type pref_public_key_algs_common_option() :: {pref_public_key_algs, [pubkey_alg()] } . -type preferred_algorithms_common_option():: {preferred_algorithms, algs_list()}. -type modify_algorithms_common_option() :: {modify_algorithms, modify_algs_list()}. -type auth_methods_common_option() :: {auth_methods, string() }. @@ -227,8 +231,7 @@ -type client_option() :: - pref_public_key_algs_client_option() - | ssh_file:pubkey_passphrase_client_options() + ssh_file:pubkey_passphrase_client_options() | host_accepting_client_options() | authentication_client_options() | diffie_hellman_group_exchange_client_option() @@ -239,11 +242,14 @@ | ?COMMON_OPTION . -type opaque_client_options() :: - {keyboard_interact_fun, fun((term(),term(),term()) -> term())} + {keyboard_interact_fun, fun((Name::iodata(), + Instruction::iodata(), + Prompts::[{Prompt::iodata(),Echo::boolean()}] + ) -> + [Response::iodata()] + )} | opaque_common_options(). --type pref_public_key_algs_client_option() :: {pref_public_key_algs, [pubkey_alg()] } . - -type host_accepting_client_options() :: {silently_accept_hosts, accept_hosts()} | {user_interaction, boolean()} @@ -293,8 +299,9 @@ -type 'shell_fun/1'() :: fun((User::string()) -> pid()) . -type 'shell_fun/2'() :: fun((User::string(), PeerAddr::inet:ip_address()) -> pid()). --type exec_daemon_option() :: {exec, 'exec_fun/1'() | 'exec_fun/2'() | 'exec_fun/3'() }. - +-type exec_daemon_option() :: {exec, exec_spec()} . +-type exec_spec() :: {direct, exec_fun()} . +-type exec_fun() :: 'exec_fun/1'() | 'exec_fun/2'() | 'exec_fun/3'(). -type 'exec_fun/1'() :: fun((Cmd::string()) -> exec_result()) . -type 'exec_fun/2'() :: fun((Cmd::string(), User::string()) -> exec_result()) . -type 'exec_fun/3'() :: fun((Cmd::string(), User::string(), ClientAddr::ip_port()) -> exec_result()) . @@ -380,9 +387,6 @@ algorithms, %% #alg{} - key_cb, %% Private/Public key callback module - io_cb, %% Interaction callback module - send_mac = none, %% send MAC algorithm send_mac_key, %% key used in send MAC algorithm send_mac_size = 0, diff --git a/lib/ssh/src/ssh_auth.erl b/lib/ssh/src/ssh_auth.erl index 4e4aa440de..9632168e65 100644 --- a/lib/ssh/src/ssh_auth.erl +++ b/lib/ssh/src/ssh_auth.erl @@ -91,8 +91,10 @@ unique(L) -> %%%---- userauth_request_msg "callbacks" -password_msg([#ssh{opts = Opts, io_cb = IoCb, - user = User, service = Service} = Ssh0]) -> +password_msg([#ssh{opts = Opts, + user = User, + service = Service} = Ssh0]) -> + IoCb = ?GET_INTERNAL_OPT(io_cb, Opts), {Password,Ssh} = case ?GET_OPT(password, Opts) of undefined when IoCb == ssh_no_io -> @@ -137,9 +139,7 @@ keyboard_interactive_msg([#ssh{user = User, get_public_key(SigAlg, #ssh{opts = Opts}) -> KeyAlg = key_alg(SigAlg), - {KeyCb,KeyCbOpts} = ?GET_OPT(key_cb, Opts), - UserOpts = ?GET_OPT(user_options, Opts), - case KeyCb:user_key(KeyAlg, [{key_cb_private,KeyCbOpts}|UserOpts]) of + case ssh_transport:call_KeyCb(user_key, [KeyAlg], Opts) of {ok, PrivKey} -> try %% Check the key - the KeyCb may be a buggy plugin @@ -387,11 +387,9 @@ handle_userauth_info_request(#ssh_msg_userauth_info_request{name = Name, instruction = Instr, num_prompts = NumPrompts, data = Data}, - #ssh{opts = Opts, - io_cb = IoCb - } = Ssh) -> + #ssh{opts=Opts} = Ssh) -> PromptInfos = decode_keyboard_interactive_prompts(NumPrompts,Data), - case keyboard_interact_get_responses(IoCb, Opts, Name, Instr, PromptInfos) of + case keyboard_interact_get_responses(Opts, Name, Instr, PromptInfos) of not_ok -> not_ok; Responses -> @@ -498,9 +496,7 @@ get_password_option(Opts, User) -> pre_verify_sig(User, KeyBlob, Opts) -> try Key = public_key:ssh_decode(KeyBlob, ssh2_pubkey), % or exception - {KeyCb,KeyCbOpts} = ?GET_OPT(key_cb, Opts), - UserOpts = ?GET_OPT(user_options, Opts), - KeyCb:is_auth_key(Key, User, [{key_cb_private,KeyCbOpts}|UserOpts]) + ssh_transport:call_KeyCb(is_auth_key, [Key, User], Opts) catch _:_ -> false @@ -509,10 +505,8 @@ pre_verify_sig(User, KeyBlob, Opts) -> verify_sig(SessionId, User, Service, AlgBin, KeyBlob, SigWLen, #ssh{opts = Opts} = Ssh) -> try Alg = binary_to_list(AlgBin), - {KeyCb,KeyCbOpts} = ?GET_OPT(key_cb, Opts), - UserOpts = ?GET_OPT(user_options, Opts), Key = public_key:ssh_decode(KeyBlob, ssh2_pubkey), % or exception - true = KeyCb:is_auth_key(Key, User, [{key_cb_private,KeyCbOpts}|UserOpts]), + true = ssh_transport:call_KeyCb(is_auth_key, [Key, User], Opts), PlainText = build_sig_data(SessionId, User, Service, KeyBlob, Alg), <<?UINT32(AlgSigLen), AlgSig:AlgSigLen/binary>> = SigWLen, <<?UINT32(AlgLen), _Alg:AlgLen/binary, @@ -536,56 +530,78 @@ build_sig_data(SessionId, User, Service, KeyBlob, Alg) -> +key_alg('rsa-sha2-256') -> 'ssh-rsa'; +key_alg('rsa-sha2-512') -> 'ssh-rsa'; +key_alg(Alg) -> Alg. + +%%%================================================================ +%%% +%%% Keyboard-interactive +%%% + decode_keyboard_interactive_prompts(_NumPrompts, Data) -> ssh_message:decode_keyboard_interactive_prompts(Data, []). -keyboard_interact_get_responses(IoCb, Opts, Name, Instr, PromptInfos) -> - NumPrompts = length(PromptInfos), +keyboard_interact_get_responses(Opts, Name, Instr, PromptInfos) -> keyboard_interact_get_responses(?GET_OPT(user_interaction, Opts), ?GET_OPT(keyboard_interact_fun, Opts), - ?GET_OPT(password, Opts), IoCb, Name, - Instr, PromptInfos, Opts, NumPrompts). + ?GET_OPT(password, Opts), + Name, + Instr, + PromptInfos, + Opts). -keyboard_interact_get_responses(_, _, not_ok, _, _, _, _, _, _) -> +%% Don't re-try an already rejected password. This could happen if both keyboard-interactive +%% and password methods are tried: +keyboard_interact_get_responses(_, _, not_ok, _, _, _, _) -> not_ok; -keyboard_interact_get_responses(_, undefined, Password, _, _, _, _, _, - 1) when Password =/= undefined -> - [Password]; %% Password auth implemented with keyboard-interaction and passwd is known -keyboard_interact_get_responses(_, _, _, _, _, _, _, _, 0) -> + +%% Only one password requestedm and we have got one via the 'password' option for the daemon: +keyboard_interact_get_responses(_, undefined, Pwd, _, _, [_], _) when Pwd =/= undefined -> + [Pwd]; %% Password auth implemented with keyboard-interaction and passwd is known + +%% No password requested (keyboard-interactive): +keyboard_interact_get_responses(_, _, _, _, _, [], _) -> []; -keyboard_interact_get_responses(false, undefined, undefined, _, _, _, [Prompt|_], Opts, _) -> - ssh_no_io:read_line(Prompt, Opts); %% Throws error as keyboard interaction is not allowed -keyboard_interact_get_responses(true, undefined, _,IoCb, Name, Instr, PromptInfos, Opts, _) -> - keyboard_interact(IoCb, Name, Instr, PromptInfos, Opts); -keyboard_interact_get_responses(true, Fun, _Pwd, _IoCb, Name, Instr, PromptInfos, _Opts, NumPrompts) -> - keyboard_interact_fun(Fun, Name, Instr, PromptInfos, NumPrompts). - -keyboard_interact(IoCb, Name, Instr, Prompts, Opts) -> + +%% user_interaction is forbidden (by option user_interaction) and we have to ask +%% the user for one or more. +%% Throw an error: +keyboard_interact_get_responses(false, undefined, undefined, _, _, [Prompt|_], Opts) -> + ssh_no_io:read_line(Prompt, Opts); + +%% One or more passwords are requested, we may prompt the user and no fun is used +%% to get the responses: +keyboard_interact_get_responses(true, undefined, _, Name, Instr, PromptInfos, Opts) -> + prompt_user_for_passwords(Name, Instr, PromptInfos, Opts); + +%% The passwords are provided with a fun. Use that one! +keyboard_interact_get_responses(true, Fun, _Pwd, Name, Instr, PromptInfos, _Opts) -> + keyboard_interact_fun(Fun, Name, Instr, PromptInfos). + + + +prompt_user_for_passwords(Name, Instr, PromptInfos, Opts) -> + IoCb = ?GET_INTERNAL_OPT(io_cb, Opts), write_if_nonempty(IoCb, Name), write_if_nonempty(IoCb, Instr), lists:map(fun({Prompt, true}) -> IoCb:read_line(Prompt, Opts); ({Prompt, false}) -> IoCb:read_password(Prompt, Opts) end, - Prompts). + PromptInfos). -write_if_nonempty(_, "") -> ok; -write_if_nonempty(_, <<>>) -> ok; -write_if_nonempty(IoCb, Text) -> IoCb:format("~s~n",[Text]). - - -keyboard_interact_fun(KbdInteractFun, Name, Instr, PromptInfos, NumPrompts) -> - Prompts = lists:map(fun({Prompt, _Echo}) -> Prompt end, - PromptInfos), - case KbdInteractFun(Name, Instr, Prompts) of - Rs when length(Rs) == NumPrompts -> - Rs; - _Rs -> +keyboard_interact_fun(KbdInteractFun, Name, Instr, PromptInfos) -> + case KbdInteractFun(Name, Instr, PromptInfos) of + Responses when is_list(Responses), + length(Responses) == length(PromptInfos) -> + Responses; + _ -> nok end. -key_alg('rsa-sha2-256') -> 'ssh-rsa'; -key_alg('rsa-sha2-512') -> 'ssh-rsa'; -key_alg(Alg) -> Alg. +write_if_nonempty(_, "") -> ok; +write_if_nonempty(_, <<>>) -> ok; +write_if_nonempty(IoCb, Text) -> IoCb:format("~s~n",[Text]). diff --git a/lib/ssh/src/ssh_connection_handler.erl b/lib/ssh/src/ssh_connection_handler.erl index 4b41c10cbb..7c87591cf2 100644 --- a/lib/ssh/src/ssh_connection_handler.erl +++ b/lib/ssh/src/ssh_connection_handler.erl @@ -447,7 +447,6 @@ init_ssh_record(Role, Socket, Opts) -> init_ssh_record(Role, Socket, PeerAddr, Opts) -> AuthMethods = ?GET_OPT(auth_methods, Opts), S0 = #ssh{role = Role, - key_cb = ?GET_OPT(key_cb, Opts), opts = Opts, userauth_supported_methods = AuthMethods, available_host_keys = available_hkey_algorithms(Role, Opts), @@ -472,10 +471,11 @@ init_ssh_record(Role, Socket, PeerAddr, Opts) -> S1 = S0#ssh{c_vsn = Vsn, c_version = Version, - io_cb = case ?GET_OPT(user_interaction, Opts) of - true -> ssh_io; - false -> ssh_no_io - end, + opts = ?PUT_INTERNAL_OPT({io_cb, case ?GET_OPT(user_interaction, Opts) of + true -> ssh_io; + false -> ssh_no_io + end}, + Opts), userauth_quiet_mode = ?GET_OPT(quiet_mode, Opts), peer = {PeerName, PeerAddr}, local = LocalName @@ -488,7 +488,6 @@ init_ssh_record(Role, Socket, PeerAddr, Opts) -> server -> S0#ssh{s_vsn = Vsn, s_version = Version, - io_cb = ?GET_INTERNAL_OPT(io_cb, Opts, ssh_io), userauth_methods = string:tokens(AuthMethods, ","), kb_tries_left = 3, peer = {undefined, PeerAddr}, @@ -983,6 +982,10 @@ handle_event(_, #ssh_msg_userauth_info_request{}, {userauth_keyboard_interactive %%% ######## {connected, client|server} #### +%% Skip ext_info messages in connected state (for example from OpenSSH >= 7.7) +handle_event(_, #ssh_msg_ext_info{}, {connected,_Role}, D) -> + {keep_state, D}; + handle_event(_, {#ssh_msg_kexinit{},_}, {connected,Role}, D0) -> {KeyInitMsg, SshPacket, Ssh} = ssh_transport:key_exchange_init_msg(D0#data.ssh_params), D = D0#data{ssh_params = Ssh, @@ -1682,18 +1685,19 @@ peer_role(client) -> server; peer_role(server) -> client. %%-------------------------------------------------------------------- -available_hkey_algorithms(Role, Options) -> - KeyCb = ?GET_OPT(key_cb, Options), - case [A || A <- available_hkey_algos(Options), - (Role==client) orelse available_host_key(KeyCb, A, Options) - ] of - - [] when Role==client -> - error({shutdown, "No public key algs"}); - - [] when Role==server -> - error({shutdown, "No host key available"}); +available_hkey_algorithms(client, Options) -> + case available_hkey_algos(Options) of + [] -> + error({shutdown, "No public key algs"}); + Algs -> + [atom_to_list(A) || A<-Algs] + end; +available_hkey_algorithms(server, Options) -> + case [A || A <- available_hkey_algos(Options), + is_usable_host_key(A, Options)] of + [] -> + error({shutdown, "No host key available"}); Algs -> [atom_to_list(A) || A<-Algs] end. @@ -1709,18 +1713,6 @@ available_hkey_algos(Options) -> AvailableAndSupported. -%% Alg :: atom() -available_host_key({KeyCb,KeyCbOpts}, Alg, Opts) -> - UserOpts = ?GET_OPT(user_options, Opts), - case KeyCb:host_key(Alg, [{key_cb_private,KeyCbOpts}|UserOpts]) of - {ok,Key} -> - %% Check the key - the KeyCb may be a buggy plugin - ssh_transport:valid_key_sha_alg(Key, Alg); - _ -> - false - end. - - send_msg(Msg, State=#data{ssh_params=Ssh0}) when is_tuple(Msg) -> {Bytes, Ssh} = ssh_transport:ssh_packet(Msg, Ssh0), send_bytes(Bytes, State), @@ -1840,10 +1832,21 @@ ext_info(_, D0) -> D0. %%%---------------------------------------------------------------- -is_usable_user_pubkey(A, Ssh) -> - case ssh_auth:get_public_key(A, Ssh) of +is_usable_user_pubkey(Alg, Ssh) -> + try ssh_auth:get_public_key(Alg, Ssh) of {ok,_} -> true; _ -> false + catch + _:_ -> false + end. + +%%%---------------------------------------------------------------- +is_usable_host_key(Alg, Opts) -> + try ssh_transport:get_host_key(Alg, Opts) + of + _PrivHostKey -> true + catch + _:_ -> false end. %%%---------------------------------------------------------------- diff --git a/lib/ssh/src/ssh_file.erl b/lib/ssh/src/ssh_file.erl index 669b0f9be2..510269bbb1 100644 --- a/lib/ssh/src/ssh_file.erl +++ b/lib/ssh/src/ssh_file.erl @@ -52,10 +52,11 @@ -type pubkey_passphrase_client_options() :: {dsa_pass_phrase, string()} | {rsa_pass_phrase, string()} +%% Not yet implemented: | {ed25519_pass_phrase, string()} +%% Not yet implemented: | {ed448_pass_phrase, string()} | {ecdsa_pass_phrase, string()} . - -define(PERM_700, 8#700). -define(PERM_644, 8#644). @@ -120,6 +121,8 @@ file_base_name('ssh-dss' ) -> "ssh_host_dsa_key"; file_base_name('ecdsa-sha2-nistp256') -> "ssh_host_ecdsa_key"; file_base_name('ecdsa-sha2-nistp384') -> "ssh_host_ecdsa_key"; file_base_name('ecdsa-sha2-nistp521') -> "ssh_host_ecdsa_key"; +file_base_name('ssh-ed25519' ) -> "ssh_host_ed25519_key"; +file_base_name('ssh-ed448' ) -> "ssh_host_ed448_key"; file_base_name(_ ) -> "ssh_host_key". decode(File, Password) -> @@ -257,6 +260,8 @@ identity_key_filename('ssh-rsa' ) -> "id_rsa"; identity_key_filename('rsa-sha2-256' ) -> "id_rsa"; identity_key_filename('rsa-sha2-384' ) -> "id_rsa"; identity_key_filename('rsa-sha2-512' ) -> "id_rsa"; +identity_key_filename('ssh-ed25519' ) -> "id_ed25519"; +identity_key_filename('ssh-ed448' ) -> "id_ed448"; identity_key_filename('ecdsa-sha2-nistp256') -> "id_ecdsa"; identity_key_filename('ecdsa-sha2-nistp384') -> "id_ecdsa"; identity_key_filename('ecdsa-sha2-nistp521') -> "id_ecdsa". @@ -266,9 +271,12 @@ identity_pass_phrase("ssh-rsa" ) -> rsa_pass_phrase; identity_pass_phrase("rsa-sha2-256" ) -> rsa_pass_phrase; identity_pass_phrase("rsa-sha2-384" ) -> rsa_pass_phrase; identity_pass_phrase("rsa-sha2-512" ) -> rsa_pass_phrase; +%% Not yet implemented: identity_pass_phrase("ssh-ed25519" ) -> ed25519_pass_phrase; +%% Not yet implemented: identity_pass_phrase("ssh-ed448" ) -> ed448_pass_phrase; identity_pass_phrase("ecdsa-sha2-"++_) -> ecdsa_pass_phrase; identity_pass_phrase(P) when is_atom(P) -> - identity_pass_phrase(atom_to_list(P)). + identity_pass_phrase(atom_to_list(P)); +identity_pass_phrase(_) -> undefined. lookup_host_key_fd(Fd, KeyToMatch, Host, KeyType) -> case io:get_line(Fd, '') of @@ -318,6 +326,10 @@ key_match({#'ECPoint'{},{namedCurve,Curve}}, Alg) -> _ -> false end; +key_match({ed_pub,ed25519,_}, 'ssh-ed25519') -> + true; +key_match({ed_pub,ed448,_}, 'ssh-ed448') -> + true; key_match(_, _) -> false. diff --git a/lib/ssh/src/ssh_message.erl b/lib/ssh/src/ssh_message.erl index da4027a763..d95e58c1bb 100644 --- a/lib/ssh/src/ssh_message.erl +++ b/lib/ssh/src/ssh_message.erl @@ -611,7 +611,13 @@ encode_signature({_, #'Dss-Parms'{}}, _SigAlg, Signature) -> <<?Ebinary(<<"ssh-dss">>), ?Ebinary(Signature)>>; encode_signature({#'ECPoint'{}, {namedCurve,OID}}, _SigAlg, Signature) -> CurveName = public_key:oid2ssh_curvename(OID), - <<?Ebinary(<<"ecdsa-sha2-",CurveName/binary>>), ?Ebinary(Signature)>>. + <<?Ebinary(<<"ecdsa-sha2-",CurveName/binary>>), ?Ebinary(Signature)>>; +encode_signature({ed_pub, ed25519,_}, _SigAlg, Signature) -> + <<?Ebinary(<<"ssh-ed25519">>), ?Ebinary(Signature)>>; +encode_signature({ed_pub, ed448,_}, _SigAlg, Signature) -> + <<?Ebinary(<<"ssh-ed448">>), ?Ebinary(Signature)>>. + + %%%################################################################ %%%# diff --git a/lib/ssh/src/ssh_options.erl b/lib/ssh/src/ssh_options.erl index bc9f2156bc..1010c9be55 100644 --- a/lib/ssh/src/ssh_options.erl +++ b/lib/ssh/src/ssh_options.erl @@ -434,6 +434,18 @@ default(client) -> class => user_options }, +%%% Not yet implemented {ed25519_pass_phrase, def} => +%%% Not yet implemented #{default => undefined, +%%% Not yet implemented chk => fun check_string/1, +%%% Not yet implemented class => user_options +%%% Not yet implemented }, +%%% Not yet implemented +%%% Not yet implemented {ed448_pass_phrase, def} => +%%% Not yet implemented #{default => undefined, +%%% Not yet implemented chk => fun check_string/1, +%%% Not yet implemented class => user_options +%%% Not yet implemented }, +%%% Not yet implemented {silently_accept_hosts, def} => #{default => false, chk => fun check_silently_accept_hosts/1, @@ -452,12 +464,6 @@ default(client) -> class => user_options }, - {pref_public_key_algs, def} => - #{default => ssh_transport:default_algorithms(public_key), - chk => fun check_pref_public_key_algs/1, - class => user_options - }, - {dh_gex_limits, def} => #{default => {1024, 6144, 8192}, % FIXME: Is this true nowadays? chk => fun({Min,I,Max}) -> @@ -523,6 +529,12 @@ default(common) -> class => user_options }, + {pref_public_key_algs, def} => + #{default => ssh_transport:default_algorithms(public_key), + chk => fun check_pref_public_key_algs/1, + class => user_options + }, + {preferred_algorithms, def} => #{default => ssh:default_algorithms(), chk => fun check_preferred_algorithms/1, diff --git a/lib/ssh/src/ssh_transport.erl b/lib/ssh/src/ssh_transport.erl index c5b0704925..9ff20454cd 100644 --- a/lib/ssh/src/ssh_transport.erl +++ b/lib/ssh/src/ssh_transport.erl @@ -51,7 +51,9 @@ extract_public_key/1, ssh_packet/2, pack/2, valid_key_sha_alg/2, - sha/1, sign/3, verify/5]). + sha/1, sign/3, verify/5, + get_host_key/2, + call_KeyCb/3]). -export([dbg_trace/3]). @@ -147,6 +149,8 @@ supported_algorithms(public_key) -> {'ecdsa-sha2-nistp384', [{public_keys,ecdsa}, {hashs,sha384}, {curves,secp384r1}]}, {'ecdsa-sha2-nistp521', [{public_keys,ecdsa}, {hashs,sha512}, {curves,secp521r1}]}, {'ecdsa-sha2-nistp256', [{public_keys,ecdsa}, {hashs,sha256}, {curves,secp256r1}]}, + {'ssh-ed25519', [{public_keys,eddsa}, {curves,ed25519} ]}, + {'ssh-ed448', [{public_keys,eddsa}, {curves,ed448} ]}, {'ssh-rsa', [{public_keys,rsa}, {hashs,sha} ]}, {'rsa-sha2-256', [{public_keys,rsa}, {hashs,sha256} ]}, {'rsa-sha2-512', [{public_keys,rsa}, {hashs,sha512} ]}, @@ -431,7 +435,8 @@ key_exchange_first_msg(Kex, Ssh0) when Kex == 'ecdh-sha2-nistp256' ; %%% handle_kexdh_init(#ssh_msg_kexdh_init{e = E}, Ssh0 = #ssh{algorithms = #alg{kex=Kex, - hkey=SignAlg} = Algs}) -> + hkey=SignAlg} = Algs, + opts = Opts}) -> %% server {G, P} = dh_group(Kex), if @@ -439,7 +444,7 @@ handle_kexdh_init(#ssh_msg_kexdh_init{e = E}, Sz = dh_bits(Algs), {Public, Private} = generate_key(dh, [P,G,2*Sz]), K = compute_key(dh, E, Private, [P,G]), - MyPrivHostKey = get_host_key(Ssh0, SignAlg), + MyPrivHostKey = get_host_key(SignAlg, Opts), MyPubHostKey = extract_public_key(MyPrivHostKey), H = kex_hash(Ssh0, MyPubHostKey, sha(Kex), {E,Public,K}), H_SIG = sign(H, sha(SignAlg), MyPrivHostKey), @@ -578,14 +583,15 @@ handle_kex_dh_gex_init(#ssh_msg_kex_dh_gex_init{e = E}, #ssh{keyex_key = {{Private, Public}, {G, P}}, keyex_info = {Min, Max, NBits}, algorithms = #alg{kex=Kex, - hkey=SignAlg}} = Ssh0) -> + hkey=SignAlg}, + opts = Opts} = Ssh0) -> %% server if 1=<E, E=<(P-1) -> K = compute_key(dh, E, Private, [P,G]), if 1<K, K<(P-1) -> - MyPrivHostKey = get_host_key(Ssh0, SignAlg), + MyPrivHostKey = get_host_key(SignAlg, Opts), MyPubHostKey = extract_public_key(MyPrivHostKey), H = kex_hash(Ssh0, MyPubHostKey, sha(Kex), {Min,NBits,Max,P,G,E,Public,K}), H_SIG = sign(H, sha(SignAlg), MyPrivHostKey), @@ -653,7 +659,8 @@ handle_kex_dh_gex_reply(#ssh_msg_kex_dh_gex_reply{public_host_key = PeerPubHostK %%% handle_kex_ecdh_init(#ssh_msg_kex_ecdh_init{q_c = PeerPublic}, Ssh0 = #ssh{algorithms = #alg{kex=Kex, - hkey=SignAlg}}) -> + hkey=SignAlg}, + opts = Opts}) -> %% at server Curve = ecdh_curve(Kex), {MyPublic, MyPrivate} = generate_key(ecdh, Curve), @@ -661,7 +668,7 @@ handle_kex_ecdh_init(#ssh_msg_kex_ecdh_init{q_c = PeerPublic}, compute_key(ecdh, PeerPublic, MyPrivate, Curve) of K -> - MyPrivHostKey = get_host_key(Ssh0, SignAlg), + MyPrivHostKey = get_host_key(SignAlg, Opts), MyPubHostKey = extract_public_key(MyPrivHostKey), H = kex_hash(Ssh0, MyPubHostKey, sha(Curve), {PeerPublic, MyPublic, K}), H_SIG = sign(H, sha(SignAlg), MyPrivHostKey), @@ -759,8 +766,7 @@ ext_info_message(#ssh{role=server, send_ext_info=true, opts = Opts} = Ssh0) -> AlgsList = lists:map(fun erlang:atom_to_list/1, - proplists:get_value(public_key, - ?GET_OPT(preferred_algorithms, Opts))), + ?GET_OPT(pref_public_key_algs, Opts)), Msg = #ssh_msg_ext_info{nr_extensions = 1, data = [{"server-sig-algs", string:join(AlgsList,",")}] }, @@ -778,10 +784,8 @@ sid(#ssh{session_id = Id}, _) -> Id. %% %% The host key should be read from storage %% -get_host_key(SSH, SignAlg) -> - #ssh{key_cb = {KeyCb,KeyCbOpts}, opts = Opts} = SSH, - UserOpts = ?GET_OPT(user_options, Opts), - case KeyCb:host_key(SignAlg, [{key_cb_private,KeyCbOpts}|UserOpts]) of +get_host_key(SignAlg, Opts) -> + case call_KeyCb(host_key, [SignAlg], Opts) of {ok, PrivHostKey} -> %% Check the key - the KeyCb may be a buggy plugin case valid_key_sha_alg(PrivHostKey, SignAlg) of @@ -792,6 +796,11 @@ get_host_key(SSH, SignAlg) -> exit({error, {Result, unsupported_key_type}}) end. +call_KeyCb(F, Args, Opts) -> + {KeyCb,KeyCbOpts} = ?GET_OPT(key_cb, Opts), + UserOpts = ?GET_OPT(user_options, Opts), + apply(KeyCb, F, Args ++ [[{key_cb_private,KeyCbOpts}|UserOpts]]). + extract_public_key(#'RSAPrivateKey'{modulus = N, publicExponent = E}) -> #'RSAPublicKey'{modulus = N, publicExponent = E}; extract_public_key(#'DSAPrivateKey'{y = Y, p = P, q = Q, g = G}) -> @@ -799,6 +808,8 @@ extract_public_key(#'DSAPrivateKey'{y = Y, p = P, q = Q, g = G}) -> extract_public_key(#'ECPrivateKey'{parameters = {namedCurve,OID}, publicKey = Q}) -> {#'ECPoint'{point=Q}, {namedCurve,OID}}; +extract_public_key({ed_pri, Alg, Pub, _Priv}) -> + {ed_pub, Alg, Pub}; extract_public_key(#{engine:=_, key_id:=_, algorithm:=Alg} = M) -> case {Alg, crypto:privkey_to_pubkey(Alg, M)} of {rsa, [E,N]} -> @@ -858,29 +869,30 @@ accepted_host(Ssh, PeerName, Public, Opts) -> end. -yes_no(Ssh, Prompt) -> - (Ssh#ssh.io_cb):yes_no(Prompt, Ssh#ssh.opts). +yes_no(#ssh{opts=Opts}, Prompt) -> + IoCb = ?GET_INTERNAL_OPT(io_cb, Opts, ssh_io), + IoCb:yes_no(Prompt, Opts). fmt_hostkey('ssh-rsa') -> "RSA"; fmt_hostkey('ssh-dss') -> "DSA"; +fmt_hostkey('ssh-ed25519') -> "ED25519"; +fmt_hostkey('ssh-ed448') -> "ED448"; fmt_hostkey(A) when is_atom(A) -> fmt_hostkey(atom_to_list(A)); fmt_hostkey("ecdsa"++_) -> "ECDSA"; fmt_hostkey(X) -> X. -known_host_key(#ssh{opts = Opts, key_cb = {KeyCb,KeyCbOpts}, peer = {PeerName,_}} = Ssh, +known_host_key(#ssh{opts = Opts, peer = {PeerName,_}} = Ssh, Public, Alg) -> - UserOpts = ?GET_OPT(user_options, Opts), - case is_host_key(KeyCb, Public, PeerName, Alg, [{key_cb_private,KeyCbOpts}|UserOpts]) of - {_,true} -> + case call_KeyCb(is_host_key, [Public, PeerName, Alg], Opts) of + true -> ok; - {_,false} -> + false -> DoAdd = ?GET_OPT(save_accepted_host, Opts), case accepted_host(Ssh, PeerName, Public, Opts) of true when DoAdd == true -> - {_,R} = add_host_key(KeyCb, PeerName, Public, [{key_cb_private,KeyCbOpts}|UserOpts]), - R; + call_KeyCb(add_host_key, [PeerName, Public], Opts); true when DoAdd == false -> ok; false -> @@ -890,13 +902,6 @@ known_host_key(#ssh{opts = Opts, key_cb = {KeyCb,KeyCbOpts}, peer = {PeerName,_} end end. -is_host_key(KeyCb, Public, PeerName, Alg, Data) -> - {KeyCb, KeyCb:is_host_key(Public, PeerName, Alg, Data)}. - -add_host_key(KeyCb, PeerName, Public, Data) -> - {KeyCb, KeyCb:add_host_key(PeerName, Public, Data)}. - - %% Each of the algorithm strings MUST be a comma-separated list of %% algorithm names (see ''Algorithm Naming'' in [SSH-ARCH]). Each %% supported (allowed) algorithm MUST be listed in order of preference. @@ -1937,6 +1942,11 @@ valid_key_sha_alg(#'RSAPrivateKey'{}, 'ssh-rsa' ) -> true; valid_key_sha_alg({_, #'Dss-Parms'{}}, 'ssh-dss') -> true; valid_key_sha_alg(#'DSAPrivateKey'{}, 'ssh-dss') -> true; +valid_key_sha_alg({ed_pub, ed25519,_}, 'ssh-ed25519') -> true; +valid_key_sha_alg({ed_pri, ed25519,_,_},'ssh-ed25519') -> true; +valid_key_sha_alg({ed_pub, ed448,_}, 'ssh-ed448') -> true; +valid_key_sha_alg({ed_pri, ed448,_,_}, 'ssh-ed448') -> true; + valid_key_sha_alg({#'ECPoint'{},{namedCurve,OID}}, Alg) -> valid_key_sha_alg_ec(OID, Alg); valid_key_sha_alg(#'ECPrivateKey'{parameters = {namedCurve,OID}}, Alg) -> valid_key_sha_alg_ec(OID, Alg); valid_key_sha_alg(_, _) -> false. @@ -1946,12 +1956,17 @@ valid_key_sha_alg_ec(OID, Alg) -> Alg == list_to_atom("ecdsa-sha2-" ++ binary_to_list(Curve)). +-dialyzer({no_match, public_algo/1}). + public_algo(#'RSAPublicKey'{}) -> 'ssh-rsa'; % FIXME: Not right with draft-curdle-rsa-sha2 public_algo({_, #'Dss-Parms'{}}) -> 'ssh-dss'; +public_algo({ed_pub, ed25519,_}) -> 'ssh-ed25519'; +public_algo({ed_pub, ed448,_}) -> 'ssh-ed448'; public_algo({#'ECPoint'{},{namedCurve,OID}}) -> Curve = public_key:oid2ssh_curvename(OID), list_to_atom("ecdsa-sha2-" ++ binary_to_list(Curve)). + sha('ssh-rsa') -> sha; sha('rsa-sha2-256') -> sha256; sha('rsa-sha2-384') -> sha384; @@ -1960,6 +1975,8 @@ sha('ssh-dss') -> sha; sha('ecdsa-sha2-nistp256') -> sha(secp256r1); sha('ecdsa-sha2-nistp384') -> sha(secp384r1); sha('ecdsa-sha2-nistp521') -> sha(secp521r1); +sha('ssh-ed25519') -> undefined; % Included in the spec of ed25519 +sha('ssh-ed448') -> undefined; % Included in the spec of ed448 sha(secp256r1) -> sha256; sha(secp384r1) -> sha384; sha(secp521r1) -> sha512; @@ -2054,7 +2071,6 @@ ecdh_curve('curve448-sha512' ) -> x448; ecdh_curve('curve25519-sha256' ) -> x25519; ecdh_curve('[email protected]' ) -> x25519. - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% %% Utils for default_algorithms/1 and supported_algorithms/1 diff --git a/lib/ssh/test/ssh_algorithms_SUITE.erl b/lib/ssh/test/ssh_algorithms_SUITE.erl index 5e589e585f..02e5f40c38 100644 --- a/lib/ssh/test/ssh_algorithms_SUITE.erl +++ b/lib/ssh/test/ssh_algorithms_SUITE.erl @@ -184,12 +184,15 @@ init_per_testcase(TC, {public_key,Alg}, Config) -> | ExtraOpts], [{extra_daemon,true}|Config]); {{ok,_}, {error,Err}} -> + ct:log("Alg = ~p~nOpts = ~p",[Alg,Opts]), {skip, io_lib:format("No host key: ~p",[Err])}; {{error,Err}, {ok,_}} -> + ct:log("Alg = ~p~nOpts = ~p",[Alg,Opts]), {skip, io_lib:format("No user key: ~p",[Err])}; _ -> + ct:log("Alg = ~p~nOpts = ~p",[Alg,Opts]), {skip, "Neither host nor user key"} end; @@ -470,7 +473,9 @@ setup_pubkey(Alg, Config) -> 'rsa-sha2-512' -> ssh_test_lib:setup_rsa(DataDir, UserDir); 'ecdsa-sha2-nistp256' -> ssh_test_lib:setup_ecdsa("256", DataDir, UserDir); 'ecdsa-sha2-nistp384' -> ssh_test_lib:setup_ecdsa("384", DataDir, UserDir); - 'ecdsa-sha2-nistp521' -> ssh_test_lib:setup_ecdsa("521", DataDir, UserDir) + 'ecdsa-sha2-nistp521' -> ssh_test_lib:setup_ecdsa("521", DataDir, UserDir); + 'ssh-ed25519' -> ssh_test_lib:setup_eddsa(ed25519, DataDir, UserDir); + 'ssh-ed448' -> ssh_test_lib:setup_eddsa(ed448, DataDir, UserDir) end, Config. diff --git a/lib/ssh/test/ssh_algorithms_SUITE_data/id_ed25519 b/lib/ssh/test/ssh_algorithms_SUITE_data/id_ed25519 new file mode 100644 index 0000000000..401a3e4a9a --- /dev/null +++ b/lib/ssh/test/ssh_algorithms_SUITE_data/id_ed25519 @@ -0,0 +1,7 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW +QyNTUxOQAAACDm9P8/gC0IOKmwHLSvkmEtS2Xx0RRqUDqC6wY6UgDVnwAAAJg3+6xpN/us +aQAAAAtzc2gtZWQyNTUxOQAAACDm9P8/gC0IOKmwHLSvkmEtS2Xx0RRqUDqC6wY6UgDVnw +AAAEBzC/Z2WGJhZ3l3tIBnUc6DCbp+lXY2yc2RRpWQTdf8sub0/z+ALQg4qbActK+SYS1L +ZfHRFGpQOoLrBjpSANWfAAAAE3VhYmhuaWxAZWx4YWRsajNxMzIBAg== +-----END OPENSSH PRIVATE KEY----- diff --git a/lib/ssh/test/ssh_algorithms_SUITE_data/id_ed25519.pub b/lib/ssh/test/ssh_algorithms_SUITE_data/id_ed25519.pub new file mode 100644 index 0000000000..a5c03b19c1 --- /dev/null +++ b/lib/ssh/test/ssh_algorithms_SUITE_data/id_ed25519.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOb0/z+ALQg4qbActK+SYS1LZfHRFGpQOoLrBjpSANWf uabhnil@elxadlj3q32 diff --git a/lib/ssh/test/ssh_algorithms_SUITE_data/id_ed448 b/lib/ssh/test/ssh_algorithms_SUITE_data/id_ed448 new file mode 100644 index 0000000000..8ecfd710dc --- /dev/null +++ b/lib/ssh/test/ssh_algorithms_SUITE_data/id_ed448 @@ -0,0 +1,10 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAASgAAAAlz +c2gtZWQ0NDgAAAA53OqeePNaG/NJmoMbELhskKrAHNhLZ6AQm1WjbpMoseNl/OFh +1xznExpUPqTLX36fHYsAaWRHABQAAAAA0AAAEREAABERAAAACXNzaC1lZDQ0OAAA +ADnc6p5481ob80magxsQuGyQqsAc2EtnoBCbVaNukyix42X84WHXHOcTGlQ+pMtf +fp8diwBpZEcAFAAAAAByzSPST3FCdOdENDI3uTKQ9RH2Ql+Y5kRZ/yA+iYUIP/32 +BQBVOrwOBc0CGEvbicTM1n4YeVEmfrMo3OqeePNaG/NJmoMbELhskKrAHNhLZ6AQ +m1WjbpMoseNl/OFh1xznExpUPqTLX36fHYsAaWRHABQAAAAAAAECAwQ= +-----END OPENSSH PRIVATE KEY----- + diff --git a/lib/ssh/test/ssh_algorithms_SUITE_data/id_ed448.pub b/lib/ssh/test/ssh_algorithms_SUITE_data/id_ed448.pub new file mode 100644 index 0000000000..cec0765a5d --- /dev/null +++ b/lib/ssh/test/ssh_algorithms_SUITE_data/id_ed448.pub @@ -0,0 +1 @@ +ssh-ed448 AAAACXNzaC1lZDQ0OAAAADnc6p5481ob80magxsQuGyQqsAc2EtnoBCbVaNukyix42X84WHXHOcTGlQ+pMtffp8diwBpZEcAFAA= uabhnil@elxadlj3q32 diff --git a/lib/ssh/test/ssh_algorithms_SUITE_data/ssh_host_ed25519_key b/lib/ssh/test/ssh_algorithms_SUITE_data/ssh_host_ed25519_key new file mode 100644 index 0000000000..13a8fcf491 --- /dev/null +++ b/lib/ssh/test/ssh_algorithms_SUITE_data/ssh_host_ed25519_key @@ -0,0 +1,7 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW +QyNTUxOQAAACBJSOuiYGWaO9lye8Bgafod1kw8P6cV3Xb2qJgCB6yJfQAAAJi+h4O7voeD +uwAAAAtzc2gtZWQyNTUxOQAAACBJSOuiYGWaO9lye8Bgafod1kw8P6cV3Xb2qJgCB6yJfQ +AAAEBaOcJfGPNemKc1wPHTCmM4Kwvh6dZ0CqY14UT361UnN0lI66JgZZo72XJ7wGBp+h3W +TDw/pxXddvaomAIHrIl9AAAAE3VhYmhuaWxAZWx4YWRsajNxMzIBAg== +-----END OPENSSH PRIVATE KEY----- diff --git a/lib/ssh/test/ssh_algorithms_SUITE_data/ssh_host_ed25519_key.pub b/lib/ssh/test/ssh_algorithms_SUITE_data/ssh_host_ed25519_key.pub new file mode 100644 index 0000000000..156ef4045c --- /dev/null +++ b/lib/ssh/test/ssh_algorithms_SUITE_data/ssh_host_ed25519_key.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIElI66JgZZo72XJ7wGBp+h3WTDw/pxXddvaomAIHrIl9 uabhnil@elxadlj3q32 diff --git a/lib/ssh/test/ssh_algorithms_SUITE_data/ssh_host_ed448_key b/lib/ssh/test/ssh_algorithms_SUITE_data/ssh_host_ed448_key new file mode 100644 index 0000000000..31a7e4e8c3 --- /dev/null +++ b/lib/ssh/test/ssh_algorithms_SUITE_data/ssh_host_ed448_key @@ -0,0 +1,10 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAASgAAAAlz +c2gtZWQ0NDgAAAA5X9dEm1m0Yf0s54fsYWrUah2hNCSFpw4fig6nXYDpZ3jt8SR2 +m0bHBhvWeD3x5Q9s0foavq/oJWGAAAAA0AAAEREAABERAAAACXNzaC1lZDQ0OAAA +ADlf10SbWbRh/Sznh+xhatRqHaE0JIWnDh+KDqddgOlneO3xJHabRscGG9Z4PfHl +D2zR+hq+r+glYYAAAABybIKlYsuAjRDWMr6JyFE+v2ySnzTd+oyfY8mWDvbjSKNS +jIo/zC8ETjmj/FuUSS+PAy51SaIAmPlbX9dEm1m0Yf0s54fsYWrUah2hNCSFpw4f +ig6nXYDpZ3jt8SR2m0bHBhvWeD3x5Q9s0foavq/oJWGAAAAAAAECAwQ= +-----END OPENSSH PRIVATE KEY----- + diff --git a/lib/ssh/test/ssh_algorithms_SUITE_data/ssh_host_ed448_key.pub b/lib/ssh/test/ssh_algorithms_SUITE_data/ssh_host_ed448_key.pub new file mode 100644 index 0000000000..8c390dcb58 --- /dev/null +++ b/lib/ssh/test/ssh_algorithms_SUITE_data/ssh_host_ed448_key.pub @@ -0,0 +1 @@ +ssh-ed448 AAAACXNzaC1lZDQ0OAAAADlf10SbWbRh/Sznh+xhatRqHaE0JIWnDh+KDqddgOlneO3xJHabRscGG9Z4PfHlD2zR+hq+r+glYYA= diff --git a/lib/ssh/test/ssh_basic_SUITE.erl b/lib/ssh/test/ssh_basic_SUITE.erl index 778ae1e7b6..da94b5722f 100644 --- a/lib/ssh/test/ssh_basic_SUITE.erl +++ b/lib/ssh/test/ssh_basic_SUITE.erl @@ -56,6 +56,8 @@ groups() -> {group, ecdsa_sha2_nistp256_key}, {group, ecdsa_sha2_nistp384_key}, {group, ecdsa_sha2_nistp521_key}, + {group, ed25519_key}, + {group, ed448_key}, {group, dsa_pass_key}, {group, rsa_pass_key}, {group, ecdsa_sha2_nistp256_pass_key}, @@ -94,6 +96,8 @@ groups() -> {ecdsa_sha2_nistp256_key, [], [{group, basic}]}, {ecdsa_sha2_nistp384_key, [], [{group, basic}]}, {ecdsa_sha2_nistp521_key, [], [{group, basic}]}, + {ed25519_key, [], [{group, basic}]}, + {ed448_key, [], [{group, basic}]}, {rsa_host_key_is_actualy_ecdsa, [], [fail_daemon_start]}, {host_user_key_differs, [parallel], [exec_key_differs1, exec_key_differs2, @@ -222,6 +226,28 @@ init_per_group(ecdsa_sha2_nistp521_key, Config) -> false -> {skip, unsupported_pub_key} end; +init_per_group(ed25519_key, Config) -> + case lists:member('ssh-ed25519', + ssh_transport:default_algorithms(public_key)) of + true -> + DataDir = proplists:get_value(data_dir, Config), + PrivDir = proplists:get_value(priv_dir, Config), + ssh_test_lib:setup_eddsa(ed25519, DataDir, PrivDir), + Config; + false -> + {skip, unsupported_pub_key} + end; +init_per_group(ed448_key, Config) -> + case lists:member('ssh-ed448', + ssh_transport:default_algorithms(public_key)) of + true -> + DataDir = proplists:get_value(data_dir, Config), + PrivDir = proplists:get_value(priv_dir, Config), + ssh_test_lib:setup_eddsa(ed448, DataDir, PrivDir), + Config; + false -> + {skip, unsupported_pub_key} + end; init_per_group(rsa_pass_key, Config) -> case lists:member('ssh-rsa', ssh_transport:default_algorithms(public_key)) of diff --git a/lib/ssh/test/ssh_basic_SUITE_data/id_ed25519 b/lib/ssh/test/ssh_basic_SUITE_data/id_ed25519 new file mode 100644 index 0000000000..401a3e4a9a --- /dev/null +++ b/lib/ssh/test/ssh_basic_SUITE_data/id_ed25519 @@ -0,0 +1,7 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW +QyNTUxOQAAACDm9P8/gC0IOKmwHLSvkmEtS2Xx0RRqUDqC6wY6UgDVnwAAAJg3+6xpN/us +aQAAAAtzc2gtZWQyNTUxOQAAACDm9P8/gC0IOKmwHLSvkmEtS2Xx0RRqUDqC6wY6UgDVnw +AAAEBzC/Z2WGJhZ3l3tIBnUc6DCbp+lXY2yc2RRpWQTdf8sub0/z+ALQg4qbActK+SYS1L +ZfHRFGpQOoLrBjpSANWfAAAAE3VhYmhuaWxAZWx4YWRsajNxMzIBAg== +-----END OPENSSH PRIVATE KEY----- diff --git a/lib/ssh/test/ssh_basic_SUITE_data/id_ed25519.pub b/lib/ssh/test/ssh_basic_SUITE_data/id_ed25519.pub new file mode 100644 index 0000000000..a5c03b19c1 --- /dev/null +++ b/lib/ssh/test/ssh_basic_SUITE_data/id_ed25519.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOb0/z+ALQg4qbActK+SYS1LZfHRFGpQOoLrBjpSANWf uabhnil@elxadlj3q32 diff --git a/lib/ssh/test/ssh_basic_SUITE_data/id_ed448 b/lib/ssh/test/ssh_basic_SUITE_data/id_ed448 new file mode 100644 index 0000000000..8ecfd710dc --- /dev/null +++ b/lib/ssh/test/ssh_basic_SUITE_data/id_ed448 @@ -0,0 +1,10 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAASgAAAAlz +c2gtZWQ0NDgAAAA53OqeePNaG/NJmoMbELhskKrAHNhLZ6AQm1WjbpMoseNl/OFh +1xznExpUPqTLX36fHYsAaWRHABQAAAAA0AAAEREAABERAAAACXNzaC1lZDQ0OAAA +ADnc6p5481ob80magxsQuGyQqsAc2EtnoBCbVaNukyix42X84WHXHOcTGlQ+pMtf +fp8diwBpZEcAFAAAAAByzSPST3FCdOdENDI3uTKQ9RH2Ql+Y5kRZ/yA+iYUIP/32 +BQBVOrwOBc0CGEvbicTM1n4YeVEmfrMo3OqeePNaG/NJmoMbELhskKrAHNhLZ6AQ +m1WjbpMoseNl/OFh1xznExpUPqTLX36fHYsAaWRHABQAAAAAAAECAwQ= +-----END OPENSSH PRIVATE KEY----- + diff --git a/lib/ssh/test/ssh_basic_SUITE_data/id_ed448.pub b/lib/ssh/test/ssh_basic_SUITE_data/id_ed448.pub new file mode 100644 index 0000000000..cec0765a5d --- /dev/null +++ b/lib/ssh/test/ssh_basic_SUITE_data/id_ed448.pub @@ -0,0 +1 @@ +ssh-ed448 AAAACXNzaC1lZDQ0OAAAADnc6p5481ob80magxsQuGyQqsAc2EtnoBCbVaNukyix42X84WHXHOcTGlQ+pMtffp8diwBpZEcAFAA= uabhnil@elxadlj3q32 diff --git a/lib/ssh/test/ssh_basic_SUITE_data/ssh_host_ed25519_key b/lib/ssh/test/ssh_basic_SUITE_data/ssh_host_ed25519_key new file mode 100644 index 0000000000..13a8fcf491 --- /dev/null +++ b/lib/ssh/test/ssh_basic_SUITE_data/ssh_host_ed25519_key @@ -0,0 +1,7 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW +QyNTUxOQAAACBJSOuiYGWaO9lye8Bgafod1kw8P6cV3Xb2qJgCB6yJfQAAAJi+h4O7voeD +uwAAAAtzc2gtZWQyNTUxOQAAACBJSOuiYGWaO9lye8Bgafod1kw8P6cV3Xb2qJgCB6yJfQ +AAAEBaOcJfGPNemKc1wPHTCmM4Kwvh6dZ0CqY14UT361UnN0lI66JgZZo72XJ7wGBp+h3W +TDw/pxXddvaomAIHrIl9AAAAE3VhYmhuaWxAZWx4YWRsajNxMzIBAg== +-----END OPENSSH PRIVATE KEY----- diff --git a/lib/ssh/test/ssh_basic_SUITE_data/ssh_host_ed25519_key.pub b/lib/ssh/test/ssh_basic_SUITE_data/ssh_host_ed25519_key.pub new file mode 100644 index 0000000000..156ef4045c --- /dev/null +++ b/lib/ssh/test/ssh_basic_SUITE_data/ssh_host_ed25519_key.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIElI66JgZZo72XJ7wGBp+h3WTDw/pxXddvaomAIHrIl9 uabhnil@elxadlj3q32 diff --git a/lib/ssh/test/ssh_basic_SUITE_data/ssh_host_ed448_key b/lib/ssh/test/ssh_basic_SUITE_data/ssh_host_ed448_key new file mode 100644 index 0000000000..31a7e4e8c3 --- /dev/null +++ b/lib/ssh/test/ssh_basic_SUITE_data/ssh_host_ed448_key @@ -0,0 +1,10 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAASgAAAAlz +c2gtZWQ0NDgAAAA5X9dEm1m0Yf0s54fsYWrUah2hNCSFpw4fig6nXYDpZ3jt8SR2 +m0bHBhvWeD3x5Q9s0foavq/oJWGAAAAA0AAAEREAABERAAAACXNzaC1lZDQ0OAAA +ADlf10SbWbRh/Sznh+xhatRqHaE0JIWnDh+KDqddgOlneO3xJHabRscGG9Z4PfHl +D2zR+hq+r+glYYAAAABybIKlYsuAjRDWMr6JyFE+v2ySnzTd+oyfY8mWDvbjSKNS +jIo/zC8ETjmj/FuUSS+PAy51SaIAmPlbX9dEm1m0Yf0s54fsYWrUah2hNCSFpw4f +ig6nXYDpZ3jt8SR2m0bHBhvWeD3x5Q9s0foavq/oJWGAAAAAAAECAwQ= +-----END OPENSSH PRIVATE KEY----- + diff --git a/lib/ssh/test/ssh_basic_SUITE_data/ssh_host_ed448_key.pub b/lib/ssh/test/ssh_basic_SUITE_data/ssh_host_ed448_key.pub new file mode 100644 index 0000000000..8c390dcb58 --- /dev/null +++ b/lib/ssh/test/ssh_basic_SUITE_data/ssh_host_ed448_key.pub @@ -0,0 +1 @@ +ssh-ed448 AAAACXNzaC1lZDQ0OAAAADlf10SbWbRh/Sznh+xhatRqHaE0JIWnDh+KDqddgOlneO3xJHabRscGG9Z4PfHlD2zR+hq+r+glYYA= diff --git a/lib/ssh/test/ssh_compat_SUITE.erl b/lib/ssh/test/ssh_compat_SUITE.erl index 1c607bebe8..f4eef2dc77 100644 --- a/lib/ssh/test/ssh_compat_SUITE.erl +++ b/lib/ssh/test/ssh_compat_SUITE.erl @@ -648,6 +648,7 @@ setup_remote_priv_and_local_auth_keys(KeyAlg, IP, Port, UserDir, Config) -> {silently_accept_hosts,true}, {user_interaction,false} ]), + rm_id_in_remote_dir(Ch, ".ssh"), _ = ssh_sftp:make_dir(Ch, ".ssh"), DstFile = filename:join(".ssh", dst_filename(user,KeyAlg)), ok = ssh_sftp:write_file(Ch, DstFile, Priv), @@ -658,6 +659,18 @@ setup_remote_priv_and_local_auth_keys(KeyAlg, IP, Port, UserDir, Config) -> ok = ssh:close(Cc), UserDir. +rm_id_in_remote_dir(Ch, Dir) -> + case ssh_sftp:list_dir(Ch, Dir) of + {error,_Error} -> + ok; + {ok,FileNames} -> + lists:foreach(fun("id_"++_ = F) -> + ok = ssh_sftp:delete(Ch, filename:join(Dir,F)); + (_) -> + leave + end, FileNames) + end. + user_priv_pub_keys(Config, KeyAlg) -> priv_pub_keys("users_keys", user, Config, KeyAlg). host_priv_pub_keys(Config, KeyAlg) -> priv_pub_keys("host_keys", host, Config, KeyAlg). @@ -673,6 +686,8 @@ src_filename(user, 'ssh-rsa' ) -> "id_rsa"; src_filename(user, 'rsa-sha2-256' ) -> "id_rsa"; src_filename(user, 'rsa-sha2-512' ) -> "id_rsa"; src_filename(user, 'ssh-dss' ) -> "id_dsa"; +src_filename(user, 'ssh-ed25519' ) -> "id_ed25519"; +src_filename(user, 'ssh-ed448' ) -> "id_ed448"; src_filename(user, 'ecdsa-sha2-nistp256') -> "id_ecdsa256"; src_filename(user, 'ecdsa-sha2-nistp384') -> "id_ecdsa384"; src_filename(user, 'ecdsa-sha2-nistp521') -> "id_ecdsa521"; @@ -680,6 +695,8 @@ src_filename(host, 'ssh-rsa' ) -> "ssh_host_rsa_key"; src_filename(host, 'rsa-sha2-256' ) -> "ssh_host_rsa_key"; src_filename(host, 'rsa-sha2-512' ) -> "ssh_host_rsa_key"; src_filename(host, 'ssh-dss' ) -> "ssh_host_dsa_key"; +src_filename(host, 'ssh-ed25519' ) -> "ssh_host_ed25519_key"; +src_filename(host, 'ssh-ed448' ) -> "ssh_host_ed448_key"; src_filename(host, 'ecdsa-sha2-nistp256') -> "ssh_host_ecdsa_key256"; src_filename(host, 'ecdsa-sha2-nistp384') -> "ssh_host_ecdsa_key384"; src_filename(host, 'ecdsa-sha2-nistp521') -> "ssh_host_ecdsa_key521". @@ -688,6 +705,8 @@ dst_filename(user, 'ssh-rsa' ) -> "id_rsa"; dst_filename(user, 'rsa-sha2-256' ) -> "id_rsa"; dst_filename(user, 'rsa-sha2-512' ) -> "id_rsa"; dst_filename(user, 'ssh-dss' ) -> "id_dsa"; +dst_filename(user, 'ssh-ed25519' ) -> "id_ed25519"; +dst_filename(user, 'ssh-ed448' ) -> "id_ed448"; dst_filename(user, 'ecdsa-sha2-nistp256') -> "id_ecdsa"; dst_filename(user, 'ecdsa-sha2-nistp384') -> "id_ecdsa"; dst_filename(user, 'ecdsa-sha2-nistp521') -> "id_ecdsa"; @@ -695,6 +714,8 @@ dst_filename(host, 'ssh-rsa' ) -> "ssh_host_rsa_key"; dst_filename(host, 'rsa-sha2-256' ) -> "ssh_host_rsa_key"; dst_filename(host, 'rsa-sha2-512' ) -> "ssh_host_rsa_key"; dst_filename(host, 'ssh-dss' ) -> "ssh_host_dsa_key"; +dst_filename(host, 'ssh-ed25519' ) -> "ssh_host_ed25519_key"; +dst_filename(host, 'ssh-ed448' ) -> "ssh_host_ed448_key"; dst_filename(host, 'ecdsa-sha2-nistp256') -> "ssh_host_ecdsa_key"; dst_filename(host, 'ecdsa-sha2-nistp384') -> "ssh_host_ecdsa_key"; dst_filename(host, 'ecdsa-sha2-nistp521') -> "ssh_host_ecdsa_key". diff --git a/lib/ssh/test/ssh_compat_SUITE_data/build_scripts/create_all b/lib/ssh/test/ssh_compat_SUITE_data/build_scripts/create_all index 0dcf8cb570..c2e77fcc79 100755 --- a/lib/ssh/test/ssh_compat_SUITE_data/build_scripts/create_all +++ b/lib/ssh/test/ssh_compat_SUITE_data/build_scripts/create_all @@ -18,6 +18,12 @@ SSH_SSL_VERSIONS=(\ openssh 7.6p1 openssl 1.0.2n \ \ openssh 7.6p1 libressl 2.6.4 \ + \ + openssh 7.7p1 openssl 1.0.2p \ + openssh 7.8p1 openssl 1.0.2p \ + openssh 7.9p1 openssl 1.0.2p \ + \ + openssh 7.9p1 libressl 2.6.4 \ ) if [ "x$1" == "x-b" ] diff --git a/lib/ssh/test/ssh_compat_SUITE_data/host_keys/ssh_host_ed25519_key b/lib/ssh/test/ssh_compat_SUITE_data/host_keys/ssh_host_ed25519_key new file mode 100644 index 0000000000..13a8fcf491 --- /dev/null +++ b/lib/ssh/test/ssh_compat_SUITE_data/host_keys/ssh_host_ed25519_key @@ -0,0 +1,7 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW +QyNTUxOQAAACBJSOuiYGWaO9lye8Bgafod1kw8P6cV3Xb2qJgCB6yJfQAAAJi+h4O7voeD +uwAAAAtzc2gtZWQyNTUxOQAAACBJSOuiYGWaO9lye8Bgafod1kw8P6cV3Xb2qJgCB6yJfQ +AAAEBaOcJfGPNemKc1wPHTCmM4Kwvh6dZ0CqY14UT361UnN0lI66JgZZo72XJ7wGBp+h3W +TDw/pxXddvaomAIHrIl9AAAAE3VhYmhuaWxAZWx4YWRsajNxMzIBAg== +-----END OPENSSH PRIVATE KEY----- diff --git a/lib/ssh/test/ssh_compat_SUITE_data/host_keys/ssh_host_ed25519_key.pub b/lib/ssh/test/ssh_compat_SUITE_data/host_keys/ssh_host_ed25519_key.pub new file mode 100644 index 0000000000..156ef4045c --- /dev/null +++ b/lib/ssh/test/ssh_compat_SUITE_data/host_keys/ssh_host_ed25519_key.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIElI66JgZZo72XJ7wGBp+h3WTDw/pxXddvaomAIHrIl9 uabhnil@elxadlj3q32 diff --git a/lib/ssh/test/ssh_compat_SUITE_data/host_keys/ssh_host_ed448_key b/lib/ssh/test/ssh_compat_SUITE_data/host_keys/ssh_host_ed448_key new file mode 100644 index 0000000000..31a7e4e8c3 --- /dev/null +++ b/lib/ssh/test/ssh_compat_SUITE_data/host_keys/ssh_host_ed448_key @@ -0,0 +1,10 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAASgAAAAlz +c2gtZWQ0NDgAAAA5X9dEm1m0Yf0s54fsYWrUah2hNCSFpw4fig6nXYDpZ3jt8SR2 +m0bHBhvWeD3x5Q9s0foavq/oJWGAAAAA0AAAEREAABERAAAACXNzaC1lZDQ0OAAA +ADlf10SbWbRh/Sznh+xhatRqHaE0JIWnDh+KDqddgOlneO3xJHabRscGG9Z4PfHl +D2zR+hq+r+glYYAAAABybIKlYsuAjRDWMr6JyFE+v2ySnzTd+oyfY8mWDvbjSKNS +jIo/zC8ETjmj/FuUSS+PAy51SaIAmPlbX9dEm1m0Yf0s54fsYWrUah2hNCSFpw4f +ig6nXYDpZ3jt8SR2m0bHBhvWeD3x5Q9s0foavq/oJWGAAAAAAAECAwQ= +-----END OPENSSH PRIVATE KEY----- + diff --git a/lib/ssh/test/ssh_compat_SUITE_data/host_keys/ssh_host_ed448_key.pub b/lib/ssh/test/ssh_compat_SUITE_data/host_keys/ssh_host_ed448_key.pub new file mode 100644 index 0000000000..8c390dcb58 --- /dev/null +++ b/lib/ssh/test/ssh_compat_SUITE_data/host_keys/ssh_host_ed448_key.pub @@ -0,0 +1 @@ +ssh-ed448 AAAACXNzaC1lZDQ0OAAAADlf10SbWbRh/Sznh+xhatRqHaE0JIWnDh+KDqddgOlneO3xJHabRscGG9Z4PfHlD2zR+hq+r+glYYA= diff --git a/lib/ssh/test/ssh_compat_SUITE_data/users_keys/id_ed25519 b/lib/ssh/test/ssh_compat_SUITE_data/users_keys/id_ed25519 new file mode 100644 index 0000000000..401a3e4a9a --- /dev/null +++ b/lib/ssh/test/ssh_compat_SUITE_data/users_keys/id_ed25519 @@ -0,0 +1,7 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW +QyNTUxOQAAACDm9P8/gC0IOKmwHLSvkmEtS2Xx0RRqUDqC6wY6UgDVnwAAAJg3+6xpN/us +aQAAAAtzc2gtZWQyNTUxOQAAACDm9P8/gC0IOKmwHLSvkmEtS2Xx0RRqUDqC6wY6UgDVnw +AAAEBzC/Z2WGJhZ3l3tIBnUc6DCbp+lXY2yc2RRpWQTdf8sub0/z+ALQg4qbActK+SYS1L +ZfHRFGpQOoLrBjpSANWfAAAAE3VhYmhuaWxAZWx4YWRsajNxMzIBAg== +-----END OPENSSH PRIVATE KEY----- diff --git a/lib/ssh/test/ssh_compat_SUITE_data/users_keys/id_ed25519.pub b/lib/ssh/test/ssh_compat_SUITE_data/users_keys/id_ed25519.pub new file mode 100644 index 0000000000..a5c03b19c1 --- /dev/null +++ b/lib/ssh/test/ssh_compat_SUITE_data/users_keys/id_ed25519.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOb0/z+ALQg4qbActK+SYS1LZfHRFGpQOoLrBjpSANWf uabhnil@elxadlj3q32 diff --git a/lib/ssh/test/ssh_compat_SUITE_data/users_keys/id_ed448 b/lib/ssh/test/ssh_compat_SUITE_data/users_keys/id_ed448 new file mode 100644 index 0000000000..8ecfd710dc --- /dev/null +++ b/lib/ssh/test/ssh_compat_SUITE_data/users_keys/id_ed448 @@ -0,0 +1,10 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAASgAAAAlz +c2gtZWQ0NDgAAAA53OqeePNaG/NJmoMbELhskKrAHNhLZ6AQm1WjbpMoseNl/OFh +1xznExpUPqTLX36fHYsAaWRHABQAAAAA0AAAEREAABERAAAACXNzaC1lZDQ0OAAA +ADnc6p5481ob80magxsQuGyQqsAc2EtnoBCbVaNukyix42X84WHXHOcTGlQ+pMtf +fp8diwBpZEcAFAAAAAByzSPST3FCdOdENDI3uTKQ9RH2Ql+Y5kRZ/yA+iYUIP/32 +BQBVOrwOBc0CGEvbicTM1n4YeVEmfrMo3OqeePNaG/NJmoMbELhskKrAHNhLZ6AQ +m1WjbpMoseNl/OFh1xznExpUPqTLX36fHYsAaWRHABQAAAAAAAECAwQ= +-----END OPENSSH PRIVATE KEY----- + diff --git a/lib/ssh/test/ssh_compat_SUITE_data/users_keys/id_ed448.pub b/lib/ssh/test/ssh_compat_SUITE_data/users_keys/id_ed448.pub new file mode 100644 index 0000000000..cec0765a5d --- /dev/null +++ b/lib/ssh/test/ssh_compat_SUITE_data/users_keys/id_ed448.pub @@ -0,0 +1 @@ +ssh-ed448 AAAACXNzaC1lZDQ0OAAAADnc6p5481ob80magxsQuGyQqsAc2EtnoBCbVaNukyix42X84WHXHOcTGlQ+pMtffp8diwBpZEcAFAA= uabhnil@elxadlj3q32 diff --git a/lib/ssh/test/ssh_options_SUITE.erl b/lib/ssh/test/ssh_options_SUITE.erl index daf62483cd..60d0da2a39 100644 --- a/lib/ssh/test/ssh_options_SUITE.erl +++ b/lib/ssh/test/ssh_options_SUITE.erl @@ -49,7 +49,7 @@ server_userpassword_option/1, server_pwdfun_option/1, server_pwdfun_4_option/1, - server_pwdfun_4_option_repeat/1, + server_keyboard_interactive/1, ssh_connect_arg4_timeout/1, ssh_connect_negtimeout_parallel/1, ssh_connect_negtimeout_sequential/1, @@ -99,7 +99,7 @@ all() -> server_userpassword_option, server_pwdfun_option, server_pwdfun_4_option, - server_pwdfun_4_option_repeat, + server_keyboard_interactive, {group, dir_options}, ssh_connect_timeout, ssh_connect_arg4_timeout, @@ -381,7 +381,7 @@ server_pwdfun_4_option(Config) -> %%-------------------------------------------------------------------- -server_pwdfun_4_option_repeat(Config) -> +server_keyboard_interactive(Config) -> UserDir = proplists:get_value(user_dir, Config), SysDir = proplists:get_value(data_dir, Config), %% Test that the state works @@ -396,19 +396,28 @@ server_pwdfun_4_option_repeat(Config) -> {pwdfun,PWDFUN}]), %% Try with passwords "incorrect", "Bad again" and finally "bar" - KIFFUN = fun(_,_,_) -> + KIFFUN = fun(_Name, _Instr, _PromptInfos) -> K={k,self()}, - case get(K) of - undefined -> - put(K,1), - ["incorrect"]; - 2 -> - put(K,3), - ["bar"]; - S-> - put(K,S+1), - ["Bad again"] - end + Answer = + case get(K) of + undefined -> + put(K,1), + ["incorrect"]; + 2 -> + put(K,3), + ["bar"]; + S-> + put(K,S+1), + ["Bad again"] + end, + ct:log("keyboard_interact_fun:~n" + " Name = ~p~n" + " Instruction = ~p~n" + " Prompts = ~p~n" + "~nAnswer:~n ~p~n", + [_Name, _Instr, _PromptInfos, Answer]), + + Answer end, ConnectionRef2 = diff --git a/lib/ssh/test/ssh_test_lib.erl b/lib/ssh/test/ssh_test_lib.erl index 416cc301db..a1a7eebcde 100644 --- a/lib/ssh/test/ssh_test_lib.erl +++ b/lib/ssh/test/ssh_test_lib.erl @@ -408,6 +408,21 @@ ct:log("DataDir ~p:~n ~p~n~nSystDir ~p:~n ~p~n~nUserDir ~p:~n ~p",[DataDir, file setup_ecdsa_known_host(Size, System, UserDir), setup_ecdsa_auth_keys(Size, DataDir, UserDir). +setup_eddsa(Alg, DataDir, UserDir) -> + {IdPriv, IdPub, HostPriv, HostPub} = + case Alg of + ed25519 -> {"id_ed25519", "id_ed25519.pub", "ssh_host_ed25519_key", "ssh_host_ed25519_key.pub"}; + ed448 -> {"id_ed448", "id_ed448.pub", "ssh_host_ed448_key", "ssh_host_ed448_key.pub"} + end, + file:copy(filename:join(DataDir, IdPriv), filename:join(UserDir, IdPriv)), + System = filename:join(UserDir, "system"), + file:make_dir(System), + file:copy(filename:join(DataDir, HostPriv), filename:join(System, HostPriv)), + file:copy(filename:join(DataDir, HostPub), filename:join(System, HostPub)), +ct:log("DataDir ~p:~n ~p~n~nSystDir ~p:~n ~p~n~nUserDir ~p:~n ~p",[DataDir, file:list_dir(DataDir), System, file:list_dir(System), UserDir, file:list_dir(UserDir)]), + setup_eddsa_known_host(HostPub, DataDir, UserDir), + setup_eddsa_auth_keys(IdPriv, DataDir, UserDir). + clean_dsa(UserDir) -> del_dirs(filename:join(UserDir, "system")), file:delete(filename:join(UserDir,"id_dsa")), @@ -487,6 +502,11 @@ setup_ecdsa_known_host(_Size, SystemDir, UserDir) -> [{Key, _}] = public_key:ssh_decode(SshBin, public_key), setup_known_hosts(Key, UserDir). +setup_eddsa_known_host(HostPub, SystemDir, UserDir) -> + {ok, SshBin} = file:read_file(filename:join(SystemDir, HostPub)), + [{Key, _}] = public_key:ssh_decode(SshBin, public_key), + setup_known_hosts(Key, UserDir). + setup_known_hosts(Key, UserDir) -> {ok, Hostname} = inet:gethostname(), {ok, {A, B, C, D}} = inet:getaddr(Hostname, inet), @@ -529,6 +549,11 @@ setup_ecdsa_auth_keys(Size, Dir, UserDir) -> PKey = #'ECPoint'{point = Q}, setup_auth_keys([{ {PKey,Param}, [{comment, "Test"}]}], UserDir). +setup_eddsa_auth_keys(IdPriv, Dir, UserDir) -> + {ok, Pem} = file:read_file(filename:join(Dir, IdPriv)), + {ed_pri, Alg, Pub, _} = public_key:pem_entry_decode(hd(public_key:pem_decode(Pem))), + setup_auth_keys([{{ed_pub,Alg,Pub}, [{comment, "Test"}]}], UserDir). + setup_auth_keys(Keys, Dir) -> AuthKeys = public_key:ssh_encode(Keys, auth_keys), AuthKeysFile = filename:join(Dir, "authorized_keys"), diff --git a/lib/ssl/doc/src/notes.xml b/lib/ssl/doc/src/notes.xml index 673431ed0a..46fd8ab180 100644 --- a/lib/ssl/doc/src/notes.xml +++ b/lib/ssl/doc/src/notes.xml @@ -333,6 +333,38 @@ </section> +<section><title>SSL 8.2.6.4</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Add engine support for RSA key exchange</p> + <p> + Own Id: OTP-15420</p> + </item> + </list> + </section> + +</section> + +<section><title>SSL 8.2.6.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Extend check for undelivered data at closing, could under + some circumstances fail to deliverd all data that was + acctualy recivied.</p> + <p> + Own Id: OTP-15412</p> + </item> + </list> + </section> + +</section> + <section><title>SSL 8.2.6.2</title> <section><title>Fixed Bugs and Malfunctions</title> @@ -3163,5 +3195,3 @@ </section> </section> </chapter> - - diff --git a/lib/ssl/src/ssl_cipher.erl b/lib/ssl/src/ssl_cipher.erl index 0e22d63b4b..54c04c13e5 100644 --- a/lib/ssl/src/ssl_cipher.erl +++ b/lib/ssl/src/ssl_cipher.erl @@ -932,7 +932,7 @@ filter_suites_pubkey(ec, Ciphers, _, OtpCert) -> ec_ecdhe_suites(Ciphers)), filter_keyuse_suites(keyAgreement, Uses, CiphersSuites, ec_ecdh_suites(Ciphers)). -filter_suites_signature(rsa, Ciphers, {3, N}) when N >= 3 -> +filter_suites_signature(_, Ciphers, {3, N}) when N >= 3 -> Ciphers; filter_suites_signature(rsa, Ciphers, Version) -> (Ciphers -- ecdsa_signed_suites(Ciphers, Version)) -- dsa_signed_suites(Ciphers, Version); diff --git a/lib/ssl/src/ssl_connection.erl b/lib/ssl/src/ssl_connection.erl index 8ccad74646..acd9f14f7b 100644 --- a/lib/ssl/src/ssl_connection.erl +++ b/lib/ssl/src/ssl_connection.erl @@ -621,6 +621,8 @@ init({call, From}, {start, {Opts, EmOpts}, Timeout}, catch throw:Error -> stop_and_reply(normal, {reply, From, {error, Error}}, State0) end; +init({call, From}, {new_user, _} = Msg, State, Connection) -> + handle_call(Msg, From, ?FUNCTION_NAME, State, Connection); init({call, From}, _Msg, _State, _Connection) -> {keep_state_and_data, [{reply, From, {error, notsup_on_transport_accept_socket}}]}; init(_Type, _Event, _State, _Connection) -> diff --git a/lib/ssl/src/ssl_handshake.erl b/lib/ssl/src/ssl_handshake.erl index dc89fb0029..14df1d2e02 100644 --- a/lib/ssl/src/ssl_handshake.erl +++ b/lib/ssl/src/ssl_handshake.erl @@ -925,6 +925,13 @@ premaster_secret(EncSecret, #'RSAPrivateKey'{} = RSAPrivateKey) -> catch _:_ -> throw(?ALERT_REC(?FATAL, ?DECRYPT_ERROR)) + end; +premaster_secret(EncSecret, #{algorithm := rsa} = Engine) -> + try crypto:private_decrypt(rsa, EncSecret, maps:remove(algorithm, Engine), + [{rsa_pad, rsa_pkcs1_padding}]) + catch + _:_ -> + throw(?ALERT_REC(?FATAL, ?DECRYPT_ERROR)) end. %%==================================================================== %% Extensions handling diff --git a/lib/ssl/src/tls_connection.erl b/lib/ssl/src/tls_connection.erl index adb4f6d9ea..4dfb50967d 100644 --- a/lib/ssl/src/tls_connection.erl +++ b/lib/ssl/src/tls_connection.erl @@ -783,6 +783,7 @@ handle_info({CloseTag, Socket}, StateName, #state{socket = Socket, close_tag = CloseTag, socket_options = #socket_options{active = Active}, protocol_buffers = #protocol_buffers{tls_cipher_texts = CTs}, + user_data_buffer = Buffer, negotiated_version = Version} = State) -> %% Note that as of TLS 1.1, @@ -790,7 +791,7 @@ handle_info({CloseTag, Socket}, StateName, %% session not be resumed. This is a change from TLS 1.0 to conform %% with widespread implementation practice. - case (Active == false) andalso (CTs =/= []) of + case (Active == false) andalso ((CTs =/= []) or (Buffer =/= <<>>)) of false -> case Version of {1, N} when N >= 1 -> diff --git a/lib/ssl/src/tls_sender.erl b/lib/ssl/src/tls_sender.erl index 8d1938cee7..a245ee2465 100644 --- a/lib/ssl/src/tls_sender.erl +++ b/lib/ssl/src/tls_sender.erl @@ -319,7 +319,7 @@ handle_info({'DOWN', Monitor, _, _, _}, _, #data{connection_monitor = Monitor} = StateData) -> {stop, normal, StateData}; handle_info(_,_,_) -> - {keep_state_and_data}. + keep_state_and_data. send_tls_alert(Alert, #data{negotiated_version = Version, socket = Socket, diff --git a/lib/ssl/test/ssl_ECC_SUITE.erl b/lib/ssl/test/ssl_ECC_SUITE.erl index c93f066825..a5309e866b 100644 --- a/lib/ssl/test/ssl_ECC_SUITE.erl +++ b/lib/ssl/test/ssl_ECC_SUITE.erl @@ -395,10 +395,25 @@ client_ecdhe_rsa_server_ecdhe_ecdsa_client_custom(Config) -> end. mix_sign(Config) -> - {COpts0, SOpts0} = ssl_test_lib:make_mix_cert(Config), + mix_sign_rsa_peer(Config), + mix_sign_ecdsa_peer(Config). + +mix_sign_ecdsa_peer(Config) -> + {COpts0, SOpts0} = ssl_test_lib:make_mix_cert([{mix, peer_ecc} |Config]), COpts = ssl_test_lib:ssl_options(COpts0, Config), SOpts = ssl_test_lib:ssl_options(SOpts0, Config), ECDHE_ECDSA = ssl:filter_cipher_suites(ssl:cipher_suites(default, 'tlsv1.2'), [{key_exchange, fun(ecdhe_ecdsa) -> true; (_) -> false end}]), ssl_test_lib:basic_test(COpts, [{ciphers, ECDHE_ECDSA} | SOpts], Config). + + +mix_sign_rsa_peer(Config) -> + {COpts0, SOpts0} = ssl_test_lib:make_mix_cert([{mix, peer_rsa} |Config]), + COpts = ssl_test_lib:ssl_options(COpts0, Config), + SOpts = ssl_test_lib:ssl_options(SOpts0, Config), + ECDHE_RSA = + ssl:filter_cipher_suites(ssl:cipher_suites(default, 'tlsv1.2'), + [{key_exchange, fun(ecdhe_rsa) -> true; (_) -> false end}]), + ssl_test_lib:basic_test(COpts, [{ciphers, ECDHE_RSA} | SOpts], Config). + diff --git a/lib/ssl/test/ssl_basic_SUITE.erl b/lib/ssl/test/ssl_basic_SUITE.erl index f474d7f978..6f668f0c00 100644 --- a/lib/ssl/test/ssl_basic_SUITE.erl +++ b/lib/ssl/test/ssl_basic_SUITE.erl @@ -245,7 +245,8 @@ error_handling_tests()-> recv_error_handling, call_in_error_state, close_in_error_state, - abuse_transport_accept_socket + abuse_transport_accept_socket, + controlling_process_transport_accept_socket ]. error_handling_tests_tls()-> @@ -4057,7 +4058,7 @@ close_in_error_state(Config) when is_list(Config) -> end. %%-------------------------------------------------------------------- abuse_transport_accept_socket() -> - [{doc,"Only ssl:handshake is allowed for transport_accept:sockets"}]. + [{doc,"Only ssl:handshake and ssl:controlling_process is allowed for transport_accept:sockets"}]. abuse_transport_accept_socket(Config) when is_list(Config) -> ServerOpts = ssl_test_lib:ssl_options(server_opts, Config), ClientOpts = ssl_test_lib:ssl_options(client_opts, Config), @@ -4077,6 +4078,28 @@ abuse_transport_accept_socket(Config) when is_list(Config) -> ssl_test_lib:close(Server), ssl_test_lib:close(Client). + +%%-------------------------------------------------------------------- +controlling_process_transport_accept_socket() -> + [{doc,"Only ssl:handshake and ssl:controlling_process is allowed for transport_accept:sockets"}]. +controlling_process_transport_accept_socket(Config) when is_list(Config) -> + ServerOpts = ssl_test_lib:ssl_options(server_opts, Config), + ClientOpts = ssl_test_lib:ssl_options(client_opts, Config), + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server_transport_control([{node, ServerNode}, + {port, 0}, + {from, self()}, + {options, ServerOpts}]), + Port = ssl_test_lib:inet_port(Server), + + _Client = ssl_test_lib:start_client_error([{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {options, ClientOpts}]), + ssl_test_lib:check_result(Server, ok), + ssl_test_lib:close(Server). + %%-------------------------------------------------------------------- run_error_server_close([Pid | Opts]) -> {ok, Listen} = ssl:listen(0, Opts), diff --git a/lib/ssl/test/ssl_bench_SUITE.erl b/lib/ssl/test/ssl_bench_SUITE.erl index 3fe6338d69..13097b08b6 100644 --- a/lib/ssl/test/ssl_bench_SUITE.erl +++ b/lib/ssl/test/ssl_bench_SUITE.erl @@ -44,6 +44,7 @@ init_per_suite(Config) -> nonode@nohost -> {skipped, "Node not distributed"}; _ -> + ssl_test_lib:clean_start(), [{server_node, ssl_bench_test_lib:setup(perf_server)}|Config] end. diff --git a/lib/ssl/test/ssl_bench_test_lib.erl b/lib/ssl/test/ssl_bench_test_lib.erl index e5cbb911bd..47bcd41608 100644 --- a/lib/ssl/test/ssl_bench_test_lib.erl +++ b/lib/ssl/test/ssl_bench_test_lib.erl @@ -58,13 +58,13 @@ setup(Name) -> Path = code:get_path(), true = rpc:call(Node, code, set_path, [Path]), ok = rpc:call(Node, ?MODULE, setup_server, [node()]), - io:format("Client (~p) using ~s~n",[node(), code:which(ssl)]), + io:format("Client (~p) using ~ts~n",[node(), code:which(ssl)]), (Node =:= node()) andalso restrict_schedulers(client), Node. setup_server(ClientNode) -> (ClientNode =:= node()) andalso restrict_schedulers(server), - io:format("Server (~p) using ~s~n",[node(), code:which(ssl)]), + io:format("Server (~p) using ~ts~n",[node(), code:which(ssl)]), ok. restrict_schedulers(Type) -> diff --git a/lib/ssl/test/ssl_engine_SUITE.erl b/lib/ssl/test/ssl_engine_SUITE.erl index 1423c99dc2..e6c82d3eb5 100644 --- a/lib/ssl/test/ssl_engine_SUITE.erl +++ b/lib/ssl/test/ssl_engine_SUITE.erl @@ -90,12 +90,14 @@ end_per_testcase(_TestCase, Config) -> private_key(Config) when is_list(Config) -> ClientFileBase = filename:join([proplists:get_value(priv_dir, Config), "client_engine"]), ServerFileBase = filename:join([proplists:get_value(priv_dir, Config), "server_engine"]), + Ext = x509_test:extensions([{key_usage, [digitalSignature, keyEncipherment]}]), #{server_config := ServerConf, client_config := ClientConf} = GenCertData = public_key:pkix_test_data(#{server_chain => #{root => [{key, ssl_test_lib:hardcode_rsa_key(1)}], intermediates => [[{key, ssl_test_lib:hardcode_rsa_key(2)}]], - peer => [{key, ssl_test_lib:hardcode_rsa_key(3)} + peer => [{extensions, Ext}, + {key, ssl_test_lib:hardcode_rsa_key(3)} ]}, client_chain => #{root => [{key, ssl_test_lib:hardcode_rsa_key(4)}], @@ -131,6 +133,12 @@ private_key(Config) when is_list(Config) -> %% Test with engine test_tls_connection(EngineServerConf, EngineClientConf, Config), + %% Test with engine and rsa keyexchange + RSASuites = all_kex_rsa_suites([{tls_version, 'tlsv1.2'} | Config]), + + test_tls_connection([{ciphers, RSASuites}, {versions, ['tlsv1.2']} | EngineServerConf], + [{ciphers, RSASuites}, {versions, ['tlsv1.2']} | EngineClientConf], Config), + %% Test with engine and present file arugments test_tls_connection(EngineFileServerConf, EngineFileClientConf, Config), @@ -160,3 +168,8 @@ test_tls_connection(ServerConf, ClientConf, Config) -> ssl_test_lib:check_result(Server, ok, Client, ok), ssl_test_lib:close(Server), ssl_test_lib:close(Client). + +all_kex_rsa_suites(Config) -> + Version = proplists:get_value(tls_version, Config), + All = ssl:cipher_suites(all, Version), + ssl:filter_cipher_suites(All,[{key_exchange, fun(rsa) -> true;(_) -> false end}]). diff --git a/lib/ssl/test/ssl_test_lib.erl b/lib/ssl/test/ssl_test_lib.erl index 0ab845c4d1..8a2f0824fb 100644 --- a/lib/ssl/test/ssl_test_lib.erl +++ b/lib/ssl/test/ssl_test_lib.erl @@ -204,6 +204,14 @@ start_server_transport_abuse_socket(Args) -> Result end. +start_server_transport_control(Args) -> + Result = spawn_link(?MODULE, transport_switch_control, [Args]), + receive + {listen, up} -> + Result + end. + + transport_accept_abuse(Opts) -> Node = proplists:get_value(node, Opts), Port = proplists:get_value(port, Opts), @@ -220,6 +228,23 @@ transport_accept_abuse(Opts) -> _ = rpc:call(Node, ssl, handshake, [AcceptSocket, infinity]), Pid ! {self(), ok}. + +transport_switch_control(Opts) -> + Node = proplists:get_value(node, Opts), + Port = proplists:get_value(port, Opts), + Options = proplists:get_value(options, Opts), + Pid = proplists:get_value(from, Opts), + Transport = proplists:get_value(transport, Opts, ssl), + ct:log("~p:~p~nssl:listen(~p, ~p)~n", [?MODULE,?LINE, Port, Options]), + {ok, ListenSocket} = rpc:call(Node, Transport, listen, [Port, Options]), + Pid ! {listen, up}, + send_selected_port(Pid, Port, ListenSocket), + {ok, AcceptSocket} = rpc:call(Node, ssl, transport_accept, + [ListenSocket]), + ok = rpc:call(Node, ssl, controlling_process, [AcceptSocket, self()]), + Pid ! {self(), ok}. + + remove_close_msg(0) -> ok; remove_close_msg(ReconnectTimes) -> @@ -717,20 +742,12 @@ make_mix_cert(Config) -> Ext = x509_test:extensions([{key_usage, [digitalSignature]}]), Digest = {digest, appropriate_sha(crypto:supports())}, CurveOid = hd(tls_v1:ecc_curves(0)), - ClientFileBase = filename:join([proplists:get_value(priv_dir, Config), "mix"]), - ServerFileBase = filename:join([proplists:get_value(priv_dir, Config), "mix"]), - ClientChain = [[Digest, {key, {namedCurve, CurveOid}}], - [Digest, {key, hardcode_rsa_key(1)}], - [Digest, {key, {namedCurve, CurveOid}}, {extensions, Ext}] - ], - ServerChain = [[Digest, {key, {namedCurve, CurveOid}}], - [Digest, {key, hardcode_rsa_key(2)}], - [Digest, {key, {namedCurve, CurveOid}},{extensions, Ext}] - ], + Mix = proplists:get_value(mix, Config, peer_ecc), ClientChainType =ServerChainType = mix, + {ClientChain, ServerChain} = mix(Mix, Digest, CurveOid, Ext), CertChainConf = gen_conf(ClientChainType, ServerChainType, ClientChain, ServerChain), - ClientFileBase = filename:join([proplists:get_value(priv_dir, Config), atom_to_list(ClientChainType)]), - ServerFileBase = filename:join([proplists:get_value(priv_dir, Config), atom_to_list(ServerChainType)]), + ClientFileBase = filename:join([proplists:get_value(priv_dir, Config), "mix" ++ atom_to_list(Mix)]), + ServerFileBase = filename:join([proplists:get_value(priv_dir, Config), "mix" ++ atom_to_list(Mix)]), GenCertData = public_key:pkix_test_data(CertChainConf), [{server_config, ServerConf}, {client_config, ClientConf}] = @@ -739,6 +756,28 @@ make_mix_cert(Config) -> [{reuseaddr, true}, {verify, verify_peer} | ServerConf] }. +mix(peer_ecc, Digest, CurveOid, Ext) -> + ClientChain = [[Digest, {key, {namedCurve, CurveOid}}], + [Digest, {key, hardcode_rsa_key(1)}], + [Digest, {key, {namedCurve, CurveOid}}, {extensions, Ext}] + ], + ServerChain = [[Digest, {key, {namedCurve, CurveOid}}], + [Digest, {key, hardcode_rsa_key(2)}], + [Digest, {key, {namedCurve, CurveOid}},{extensions, Ext}] + ], + {ClientChain, ServerChain}; + +mix(peer_rsa, Digest, CurveOid, Ext) -> + ClientChain = [[Digest, {key, {namedCurve, CurveOid}}], + [Digest, {key, {namedCurve, CurveOid}}], + [Digest, {key, hardcode_rsa_key(1)}, {extensions, Ext}] + ], + ServerChain = [[Digest, {key, {namedCurve, CurveOid}}], + [Digest, {key, {namedCurve, CurveOid}}], + [Digest, {key, hardcode_rsa_key(2)},{extensions, Ext}] + ], + {ClientChain, ServerChain}. + make_ecdsa_cert(Config) -> CryptoSupport = crypto:supports(), case proplists:get_bool(ecdsa, proplists:get_value(public_keys, CryptoSupport)) of diff --git a/lib/stdlib/doc/src/notes.xml b/lib/stdlib/doc/src/notes.xml index d800885b16..039f087708 100644 --- a/lib/stdlib/doc/src/notes.xml +++ b/lib/stdlib/doc/src/notes.xml @@ -504,6 +504,21 @@ </section> +<section><title>STDLIB 3.4.5.1</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p>List subtraction (The <c>--</c> operator) will now + yield properly on large inputs.</p> + <p> + Own Id: OTP-15371</p> + </item> + </list> + </section> + +</section> + <section><title>STDLIB 3.4.5</title> <section><title>Fixed Bugs and Malfunctions</title> @@ -1658,6 +1673,21 @@ </section> +<section><title>STDLIB 2.8.0.1</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p>List subtraction (The <c>--</c> operator) will now + yield properly on large inputs.</p> + <p> + Own Id: OTP-15371</p> + </item> + </list> + </section> + +</section> + <section><title>STDLIB 2.8</title> <section><title>Fixed Bugs and Malfunctions</title> @@ -7827,4 +7857,3 @@ </section> </section> </chapter> - diff --git a/lib/stdlib/src/erl_lint.erl b/lib/stdlib/src/erl_lint.erl index e9ac2fcdff..e0cd68617b 100644 --- a/lib/stdlib/src/erl_lint.erl +++ b/lib/stdlib/src/erl_lint.erl @@ -2262,8 +2262,7 @@ expr({'fun',Line,Body}, Vt, St) -> {[],St}; {function,M,F,A} -> %% New in R15. - {Bvt, St1} = expr_list([M,F,A], Vt, St), - {vtupdate(Bvt, Vt),St1} + expr_list([M,F,A], Vt, St) end; expr({named_fun,_,'_',Cs}, Vt, St) -> fun_clauses(Cs, Vt, St); diff --git a/lib/stdlib/test/erl_lint_SUITE.erl b/lib/stdlib/test/erl_lint_SUITE.erl index f9ab83a120..d18c7c255c 100644 --- a/lib/stdlib/test/erl_lint_SUITE.erl +++ b/lib/stdlib/test/erl_lint_SUITE.erl @@ -67,7 +67,8 @@ record_errors/1, otp_11879_cont/1, non_latin1_module/1, otp_14323/1, stacktrace_syntax/1, - otp_14285/1, otp_14378/1]). + otp_14285/1, otp_14378/1, + external_funs/1]). suite() -> [{ct_hooks,[ts_install_cth]}, @@ -88,7 +89,7 @@ all() -> maps, maps_type, maps_parallel_match, otp_11851, otp_11879, otp_13230, record_errors, otp_11879_cont, non_latin1_module, otp_14323, - stacktrace_syntax, otp_14285, otp_14378]. + stacktrace_syntax, otp_14285, otp_14378, external_funs]. groups() -> [{unused_vars_warn, [], @@ -4134,6 +4135,21 @@ otp_14285(Config) -> run(Config, Ts), ok. +external_funs(Config) when is_list(Config) -> + Ts = [{external_funs_1, + %% ERL-762: Unused variable warning not being emitted. + <<"f() -> + BugVar = process_info(self()), + if true -> fun m:f/1 end. + f(M, F) -> + BugVar = process_info(self()), + if true -> fun M:F/1 end.">>, + [], + {warnings,[{2,erl_lint,{unused_var,'BugVar'}}, + {5,erl_lint,{unused_var,'BugVar'}}]}}], + run(Config, Ts), + ok. + format_error(E) -> lists:flatten(erl_lint:format_error(E)). diff --git a/lib/stdlib/test/lists_SUITE.erl b/lib/stdlib/test/lists_SUITE.erl index 9a94bcc012..984b51e7ae 100644 --- a/lib/stdlib/test/lists_SUITE.erl +++ b/lib/stdlib/test/lists_SUITE.erl @@ -2604,6 +2604,13 @@ subtract(Config) when is_list(Config) -> %% certain thresholds, and we need proper coverage for all corner cases. [sub_thresholds(N) || N <- lists:seq(0, 32)], + %% Trapping, both crashing and otherwise. + [sub_trapping(N) || N <- lists:seq(0, 18)], + + %% The current implementation chooses which algorithm to use based on + %% certain thresholds, and we need proper coverage for all corner cases. + [sub_thresholds(N) || N <- lists:seq(0, 32)], + ok. sub_non_matching(A, B) -> diff --git a/otp_versions.table b/otp_versions.table index 523b3d94ab..7bea0fbb2d 100644 --- a/otp_versions.table +++ b/otp_versions.table @@ -1,3 +1,5 @@ +OTP-21.1.3 : erts-10.1.3 # asn1-5.0.7 common_test-1.16.1 compiler-7.2.7 crypto-4.3.3 debugger-4.2.6 dialyzer-3.3.1 diameter-2.1.6 edoc-0.9.4 eldap-1.2.6 erl_docgen-0.8.1 erl_interface-3.10.4 et-1.6.3 eunit-2.3.7 ftp-1.0.1 hipe-3.18.1 inets-7.0.2 jinterface-1.9.1 kernel-6.1 megaco-3.18.4 mnesia-4.15.5 observer-2.8.1 odbc-2.12.2 os_mon-2.4.6 otp_mibs-1.2.1 parsetools-2.1.8 public_key-1.6.3 reltool-0.7.7 runtime_tools-1.13.1 sasl-3.2.1 snmp-5.2.12 ssh-4.7.1 ssl-9.0.3 stdlib-3.6 syntax_tools-2.1.6 tftp-1.0.1 tools-3.0.1 wx-1.8.5 xmerl-1.3.18 : +OTP-21.1.2 : compiler-7.2.7 erts-10.1.2 public_key-1.6.3 # asn1-5.0.7 common_test-1.16.1 crypto-4.3.3 debugger-4.2.6 dialyzer-3.3.1 diameter-2.1.6 edoc-0.9.4 eldap-1.2.6 erl_docgen-0.8.1 erl_interface-3.10.4 et-1.6.3 eunit-2.3.7 ftp-1.0.1 hipe-3.18.1 inets-7.0.2 jinterface-1.9.1 kernel-6.1 megaco-3.18.4 mnesia-4.15.5 observer-2.8.1 odbc-2.12.2 os_mon-2.4.6 otp_mibs-1.2.1 parsetools-2.1.8 reltool-0.7.7 runtime_tools-1.13.1 sasl-3.2.1 snmp-5.2.12 ssh-4.7.1 ssl-9.0.3 stdlib-3.6 syntax_tools-2.1.6 tftp-1.0.1 tools-3.0.1 wx-1.8.5 xmerl-1.3.18 : OTP-21.1.1 : compiler-7.2.6 eldap-1.2.6 erts-10.1.1 ssl-9.0.3 # asn1-5.0.7 common_test-1.16.1 crypto-4.3.3 debugger-4.2.6 dialyzer-3.3.1 diameter-2.1.6 edoc-0.9.4 erl_docgen-0.8.1 erl_interface-3.10.4 et-1.6.3 eunit-2.3.7 ftp-1.0.1 hipe-3.18.1 inets-7.0.2 jinterface-1.9.1 kernel-6.1 megaco-3.18.4 mnesia-4.15.5 observer-2.8.1 odbc-2.12.2 os_mon-2.4.6 otp_mibs-1.2.1 parsetools-2.1.8 public_key-1.6.2 reltool-0.7.7 runtime_tools-1.13.1 sasl-3.2.1 snmp-5.2.12 ssh-4.7.1 stdlib-3.6 syntax_tools-2.1.6 tftp-1.0.1 tools-3.0.1 wx-1.8.5 xmerl-1.3.18 : OTP-21.1 : asn1-5.0.7 common_test-1.16.1 compiler-7.2.5 crypto-4.3.3 debugger-4.2.6 dialyzer-3.3.1 diameter-2.1.6 edoc-0.9.4 eldap-1.2.5 erl_docgen-0.8.1 erl_interface-3.10.4 erts-10.1 et-1.6.3 eunit-2.3.7 ftp-1.0.1 hipe-3.18.1 inets-7.0.2 jinterface-1.9.1 kernel-6.1 megaco-3.18.4 mnesia-4.15.5 observer-2.8.1 odbc-2.12.2 os_mon-2.4.6 otp_mibs-1.2.1 parsetools-2.1.8 public_key-1.6.2 reltool-0.7.7 runtime_tools-1.13.1 sasl-3.2.1 snmp-5.2.12 ssh-4.7.1 ssl-9.0.2 stdlib-3.6 syntax_tools-2.1.6 tftp-1.0.1 tools-3.0.1 wx-1.8.5 xmerl-1.3.18 # : OTP-21.0.9 : compiler-7.2.4 erts-10.0.8 # asn1-5.0.6 common_test-1.16 crypto-4.3.2 debugger-4.2.5 dialyzer-3.3 diameter-2.1.5 edoc-0.9.3 eldap-1.2.4 erl_docgen-0.8 erl_interface-3.10.3 et-1.6.2 eunit-2.3.6 ftp-1.0 hipe-3.18 inets-7.0.1 jinterface-1.9 kernel-6.0.1 megaco-3.18.3 mnesia-4.15.4 observer-2.8 odbc-2.12.1 os_mon-2.4.5 otp_mibs-1.2 parsetools-2.1.7 public_key-1.6.1 reltool-0.7.6 runtime_tools-1.13 sasl-3.2 snmp-5.2.11 ssh-4.7 ssl-9.0.1 stdlib-3.5.1 syntax_tools-2.1.5 tftp-1.0 tools-3.0 wx-1.8.4 xmerl-1.3.17 : @@ -10,6 +12,9 @@ OTP-21.0.3 : erts-10.0.3 # asn1-5.0.6 common_test-1.16 compiler-7.2.2 crypto-4.3 OTP-21.0.2 : compiler-7.2.2 erts-10.0.2 public_key-1.6.1 stdlib-3.5.1 # asn1-5.0.6 common_test-1.16 crypto-4.3 debugger-4.2.5 dialyzer-3.3 diameter-2.1.5 edoc-0.9.3 eldap-1.2.4 erl_docgen-0.8 erl_interface-3.10.3 et-1.6.2 eunit-2.3.6 ftp-1.0 hipe-3.18 inets-7.0 jinterface-1.9 kernel-6.0 megaco-3.18.3 mnesia-4.15.4 observer-2.8 odbc-2.12.1 os_mon-2.4.5 otp_mibs-1.2 parsetools-2.1.7 reltool-0.7.6 runtime_tools-1.13 sasl-3.2 snmp-5.2.11 ssh-4.7 ssl-9.0 syntax_tools-2.1.5 tftp-1.0 tools-3.0 wx-1.8.4 xmerl-1.3.17 : OTP-21.0.1 : compiler-7.2.1 erts-10.0.1 # asn1-5.0.6 common_test-1.16 crypto-4.3 debugger-4.2.5 dialyzer-3.3 diameter-2.1.5 edoc-0.9.3 eldap-1.2.4 erl_docgen-0.8 erl_interface-3.10.3 et-1.6.2 eunit-2.3.6 ftp-1.0 hipe-3.18 inets-7.0 jinterface-1.9 kernel-6.0 megaco-3.18.3 mnesia-4.15.4 observer-2.8 odbc-2.12.1 os_mon-2.4.5 otp_mibs-1.2 parsetools-2.1.7 public_key-1.6 reltool-0.7.6 runtime_tools-1.13 sasl-3.2 snmp-5.2.11 ssh-4.7 ssl-9.0 stdlib-3.5 syntax_tools-2.1.5 tftp-1.0 tools-3.0 wx-1.8.4 xmerl-1.3.17 : OTP-21.0 : asn1-5.0.6 common_test-1.16 compiler-7.2 crypto-4.3 debugger-4.2.5 dialyzer-3.3 diameter-2.1.5 edoc-0.9.3 eldap-1.2.4 erl_docgen-0.8 erl_interface-3.10.3 erts-10.0 et-1.6.2 eunit-2.3.6 ftp-1.0 hipe-3.18 inets-7.0 jinterface-1.9 kernel-6.0 mnesia-4.15.4 observer-2.8 os_mon-2.4.5 otp_mibs-1.2 parsetools-2.1.7 public_key-1.6 reltool-0.7.6 runtime_tools-1.13 sasl-3.2 ssh-4.7 ssl-9.0 stdlib-3.5 syntax_tools-2.1.5 tftp-1.0 tools-3.0 wx-1.8.4 xmerl-1.3.17 # megaco-3.18.3 odbc-2.12.1 snmp-5.2.11 : +OTP-20.3.8.14 : ssh-4.6.9.2 # asn1-5.0.5.1 common_test-1.15.4 compiler-7.1.5.2 cosEvent-2.2.2 cosEventDomain-1.2.2 cosFileTransfer-1.2.2 cosNotification-1.2.3 cosProperty-1.2.3 cosTime-1.2.3 cosTransactions-1.3.3 crypto-4.2.2.2 debugger-4.2.4 dialyzer-3.2.4 diameter-2.1.4 edoc-0.9.2 eldap-1.2.3.1 erl_docgen-0.7.3 erl_interface-3.10.2.1 erts-9.3.3.6 et-1.6.1 eunit-2.3.5 hipe-3.17.1 ic-4.4.4.2 inets-6.5.2.4 jinterface-1.8.1 kernel-5.4.3.2 megaco-3.18.3 mnesia-4.15.3.2 observer-2.7 odbc-2.12.1 orber-3.8.4 os_mon-2.4.4 otp_mibs-1.1.2 parsetools-2.1.6 public_key-1.5.2 reltool-0.7.5 runtime_tools-1.12.5 sasl-3.1.2 snmp-5.2.11 ssl-8.2.6.4 stdlib-3.4.5.1 syntax_tools-2.1.4.1 tools-2.11.2 wx-1.8.3 xmerl-1.3.16 : +OTP-20.3.8.13 : ssl-8.2.6.4 # asn1-5.0.5.1 common_test-1.15.4 compiler-7.1.5.2 cosEvent-2.2.2 cosEventDomain-1.2.2 cosFileTransfer-1.2.2 cosNotification-1.2.3 cosProperty-1.2.3 cosTime-1.2.3 cosTransactions-1.3.3 crypto-4.2.2.2 debugger-4.2.4 dialyzer-3.2.4 diameter-2.1.4 edoc-0.9.2 eldap-1.2.3.1 erl_docgen-0.7.3 erl_interface-3.10.2.1 erts-9.3.3.6 et-1.6.1 eunit-2.3.5 hipe-3.17.1 ic-4.4.4.2 inets-6.5.2.4 jinterface-1.8.1 kernel-5.4.3.2 megaco-3.18.3 mnesia-4.15.3.2 observer-2.7 odbc-2.12.1 orber-3.8.4 os_mon-2.4.4 otp_mibs-1.1.2 parsetools-2.1.6 public_key-1.5.2 reltool-0.7.5 runtime_tools-1.12.5 sasl-3.1.2 snmp-5.2.11 ssh-4.6.9.1 stdlib-3.4.5.1 syntax_tools-2.1.4.1 tools-2.11.2 wx-1.8.3 xmerl-1.3.16 : +OTP-20.3.8.12 : erts-9.3.3.6 ssl-8.2.6.3 stdlib-3.4.5.1 # asn1-5.0.5.1 common_test-1.15.4 compiler-7.1.5.2 cosEvent-2.2.2 cosEventDomain-1.2.2 cosFileTransfer-1.2.2 cosNotification-1.2.3 cosProperty-1.2.3 cosTime-1.2.3 cosTransactions-1.3.3 crypto-4.2.2.2 debugger-4.2.4 dialyzer-3.2.4 diameter-2.1.4 edoc-0.9.2 eldap-1.2.3.1 erl_docgen-0.7.3 erl_interface-3.10.2.1 et-1.6.1 eunit-2.3.5 hipe-3.17.1 ic-4.4.4.2 inets-6.5.2.4 jinterface-1.8.1 kernel-5.4.3.2 megaco-3.18.3 mnesia-4.15.3.2 observer-2.7 odbc-2.12.1 orber-3.8.4 os_mon-2.4.4 otp_mibs-1.1.2 parsetools-2.1.6 public_key-1.5.2 reltool-0.7.5 runtime_tools-1.12.5 sasl-3.1.2 snmp-5.2.11 ssh-4.6.9.1 syntax_tools-2.1.4.1 tools-2.11.2 wx-1.8.3 xmerl-1.3.16 : OTP-20.3.8.11 : erts-9.3.3.5 # asn1-5.0.5.1 common_test-1.15.4 compiler-7.1.5.2 cosEvent-2.2.2 cosEventDomain-1.2.2 cosFileTransfer-1.2.2 cosNotification-1.2.3 cosProperty-1.2.3 cosTime-1.2.3 cosTransactions-1.3.3 crypto-4.2.2.2 debugger-4.2.4 dialyzer-3.2.4 diameter-2.1.4 edoc-0.9.2 eldap-1.2.3.1 erl_docgen-0.7.3 erl_interface-3.10.2.1 et-1.6.1 eunit-2.3.5 hipe-3.17.1 ic-4.4.4.2 inets-6.5.2.4 jinterface-1.8.1 kernel-5.4.3.2 megaco-3.18.3 mnesia-4.15.3.2 observer-2.7 odbc-2.12.1 orber-3.8.4 os_mon-2.4.4 otp_mibs-1.1.2 parsetools-2.1.6 public_key-1.5.2 reltool-0.7.5 runtime_tools-1.12.5 sasl-3.1.2 snmp-5.2.11 ssh-4.6.9.1 ssl-8.2.6.2 stdlib-3.4.5 syntax_tools-2.1.4.1 tools-2.11.2 wx-1.8.3 xmerl-1.3.16 : OTP-20.3.8.10 : eldap-1.2.3.1 erts-9.3.3.4 # asn1-5.0.5.1 common_test-1.15.4 compiler-7.1.5.2 cosEvent-2.2.2 cosEventDomain-1.2.2 cosFileTransfer-1.2.2 cosNotification-1.2.3 cosProperty-1.2.3 cosTime-1.2.3 cosTransactions-1.3.3 crypto-4.2.2.2 debugger-4.2.4 dialyzer-3.2.4 diameter-2.1.4 edoc-0.9.2 erl_docgen-0.7.3 erl_interface-3.10.2.1 et-1.6.1 eunit-2.3.5 hipe-3.17.1 ic-4.4.4.2 inets-6.5.2.4 jinterface-1.8.1 kernel-5.4.3.2 megaco-3.18.3 mnesia-4.15.3.2 observer-2.7 odbc-2.12.1 orber-3.8.4 os_mon-2.4.4 otp_mibs-1.1.2 parsetools-2.1.6 public_key-1.5.2 reltool-0.7.5 runtime_tools-1.12.5 sasl-3.1.2 snmp-5.2.11 ssh-4.6.9.1 ssl-8.2.6.2 stdlib-3.4.5 syntax_tools-2.1.4.1 tools-2.11.2 wx-1.8.3 xmerl-1.3.16 : OTP-20.3.8.9 : compiler-7.1.5.2 # asn1-5.0.5.1 common_test-1.15.4 cosEvent-2.2.2 cosEventDomain-1.2.2 cosFileTransfer-1.2.2 cosNotification-1.2.3 cosProperty-1.2.3 cosTime-1.2.3 cosTransactions-1.3.3 crypto-4.2.2.2 debugger-4.2.4 dialyzer-3.2.4 diameter-2.1.4 edoc-0.9.2 eldap-1.2.3 erl_docgen-0.7.3 erl_interface-3.10.2.1 erts-9.3.3.3 et-1.6.1 eunit-2.3.5 hipe-3.17.1 ic-4.4.4.2 inets-6.5.2.4 jinterface-1.8.1 kernel-5.4.3.2 megaco-3.18.3 mnesia-4.15.3.2 observer-2.7 odbc-2.12.1 orber-3.8.4 os_mon-2.4.4 otp_mibs-1.1.2 parsetools-2.1.6 public_key-1.5.2 reltool-0.7.5 runtime_tools-1.12.5 sasl-3.1.2 snmp-5.2.11 ssh-4.6.9.1 ssl-8.2.6.2 stdlib-3.4.5 syntax_tools-2.1.4.1 tools-2.11.2 wx-1.8.3 xmerl-1.3.16 : @@ -90,6 +95,7 @@ OTP-19.0.3 : inets-6.3.2 kernel-5.0.1 ssl-8.0.1 # asn1-4.0.3 common_test-1.12.2 OTP-19.0.2 : compiler-7.0.1 erts-8.0.2 stdlib-3.0.1 # asn1-4.0.3 common_test-1.12.2 cosEvent-2.2.1 cosEventDomain-1.2.1 cosFileTransfer-1.2.1 cosNotification-1.2.2 cosProperty-1.2.1 cosTime-1.2.2 cosTransactions-1.3.2 crypto-3.7 debugger-4.2 dialyzer-3.0.1 diameter-1.12 edoc-0.7.19 eldap-1.2.2 erl_docgen-0.5 erl_interface-3.9 et-1.6 eunit-2.3 gs-1.6.1 hipe-3.15.1 ic-4.4.1 inets-6.3.1 jinterface-1.7 kernel-5.0 megaco-3.18.1 mnesia-4.14 observer-2.2.1 odbc-2.11.2 orber-3.8.2 os_mon-2.4.1 otp_mibs-1.1.1 parsetools-2.1.2 percept-0.9 public_key-1.2 reltool-0.7.1 runtime_tools-1.10 sasl-3.0 snmp-5.2.3 ssh-4.3.1 ssl-8.0 syntax_tools-2.0 tools-2.8.5 typer-0.9.11 wx-1.7 xmerl-1.3.11 : OTP-19.0.1 : dialyzer-3.0.1 erts-8.0.1 inets-6.3.1 observer-2.2.1 ssh-4.3.1 tools-2.8.5 # asn1-4.0.3 common_test-1.12.2 compiler-7.0 cosEvent-2.2.1 cosEventDomain-1.2.1 cosFileTransfer-1.2.1 cosNotification-1.2.2 cosProperty-1.2.1 cosTime-1.2.2 cosTransactions-1.3.2 crypto-3.7 debugger-4.2 diameter-1.12 edoc-0.7.19 eldap-1.2.2 erl_docgen-0.5 erl_interface-3.9 et-1.6 eunit-2.3 gs-1.6.1 hipe-3.15.1 ic-4.4.1 jinterface-1.7 kernel-5.0 megaco-3.18.1 mnesia-4.14 odbc-2.11.2 orber-3.8.2 os_mon-2.4.1 otp_mibs-1.1.1 parsetools-2.1.2 percept-0.9 public_key-1.2 reltool-0.7.1 runtime_tools-1.10 sasl-3.0 snmp-5.2.3 ssl-8.0 stdlib-3.0 syntax_tools-2.0 typer-0.9.11 wx-1.7 xmerl-1.3.11 : OTP-19.0 : asn1-4.0.3 common_test-1.12.2 compiler-7.0 cosEvent-2.2.1 cosEventDomain-1.2.1 cosFileTransfer-1.2.1 cosNotification-1.2.2 cosProperty-1.2.1 cosTime-1.2.2 cosTransactions-1.3.2 crypto-3.7 debugger-4.2 dialyzer-3.0 diameter-1.12 edoc-0.7.19 eldap-1.2.2 erl_docgen-0.5 erl_interface-3.9 erts-8.0 et-1.6 eunit-2.3 gs-1.6.1 hipe-3.15.1 ic-4.4.1 inets-6.3 jinterface-1.7 kernel-5.0 megaco-3.18.1 mnesia-4.14 observer-2.2 odbc-2.11.2 orber-3.8.2 os_mon-2.4.1 otp_mibs-1.1.1 parsetools-2.1.2 percept-0.9 public_key-1.2 reltool-0.7.1 runtime_tools-1.10 sasl-3.0 snmp-5.2.3 ssh-4.3 ssl-8.0 stdlib-3.0 syntax_tools-2.0 tools-2.8.4 typer-0.9.11 wx-1.7 xmerl-1.3.11 # : +OTP-18.3.4.11 : erts-7.3.1.6 stdlib-2.8.0.1 # asn1-4.0.2 common_test-1.12.1.1 compiler-6.0.3.1 cosEvent-2.2 cosEventDomain-1.2 cosFileTransfer-1.2 cosNotification-1.2.1 cosProperty-1.2 cosTime-1.2.1 cosTransactions-1.3.1 crypto-3.6.3.1 debugger-4.1.2 dialyzer-2.9 diameter-1.11.2 edoc-0.7.18 eldap-1.2.1.1 erl_docgen-0.4.2 erl_interface-3.8.2 et-1.5.1 eunit-2.2.13 gs-1.6 hipe-3.15 ic-4.4 inets-6.2.4.1 jinterface-1.6.1 kernel-4.2 megaco-3.18 mnesia-4.13.4 observer-2.1.2 odbc-2.11.1 orber-3.8.1 os_mon-2.4 ose-1.1 otp_mibs-1.1 parsetools-2.1.1 percept-0.8.11 public_key-1.1.1 reltool-0.7 runtime_tools-1.9.3 sasl-2.7 snmp-5.2.2 ssh-4.2.2.6 ssl-7.3.3.2 syntax_tools-1.7 test_server-3.10 tools-2.8.3 typer-0.9.10 webtool-0.9.1 wx-1.6.1 xmerl-1.3.10 : OTP-18.3.4.10 : erts-7.3.1.5 # asn1-4.0.2 common_test-1.12.1.1 compiler-6.0.3.1 cosEvent-2.2 cosEventDomain-1.2 cosFileTransfer-1.2 cosNotification-1.2.1 cosProperty-1.2 cosTime-1.2.1 cosTransactions-1.3.1 crypto-3.6.3.1 debugger-4.1.2 dialyzer-2.9 diameter-1.11.2 edoc-0.7.18 eldap-1.2.1.1 erl_docgen-0.4.2 erl_interface-3.8.2 et-1.5.1 eunit-2.2.13 gs-1.6 hipe-3.15 ic-4.4 inets-6.2.4.1 jinterface-1.6.1 kernel-4.2 megaco-3.18 mnesia-4.13.4 observer-2.1.2 odbc-2.11.1 orber-3.8.1 os_mon-2.4 ose-1.1 otp_mibs-1.1 parsetools-2.1.1 percept-0.8.11 public_key-1.1.1 reltool-0.7 runtime_tools-1.9.3 sasl-2.7 snmp-5.2.2 ssh-4.2.2.6 ssl-7.3.3.2 stdlib-2.8 syntax_tools-1.7 test_server-3.10 tools-2.8.3 typer-0.9.10 webtool-0.9.1 wx-1.6.1 xmerl-1.3.10 : OTP-18.3.4.9 : ssh-4.2.2.6 # asn1-4.0.2 common_test-1.12.1.1 compiler-6.0.3.1 cosEvent-2.2 cosEventDomain-1.2 cosFileTransfer-1.2 cosNotification-1.2.1 cosProperty-1.2 cosTime-1.2.1 cosTransactions-1.3.1 crypto-3.6.3.1 debugger-4.1.2 dialyzer-2.9 diameter-1.11.2 edoc-0.7.18 eldap-1.2.1.1 erl_docgen-0.4.2 erl_interface-3.8.2 erts-7.3.1.4 et-1.5.1 eunit-2.2.13 gs-1.6 hipe-3.15 ic-4.4 inets-6.2.4.1 jinterface-1.6.1 kernel-4.2 megaco-3.18 mnesia-4.13.4 observer-2.1.2 odbc-2.11.1 orber-3.8.1 os_mon-2.4 ose-1.1 otp_mibs-1.1 parsetools-2.1.1 percept-0.8.11 public_key-1.1.1 reltool-0.7 runtime_tools-1.9.3 sasl-2.7 snmp-5.2.2 ssl-7.3.3.2 stdlib-2.8 syntax_tools-1.7 test_server-3.10 tools-2.8.3 typer-0.9.10 webtool-0.9.1 wx-1.6.1 xmerl-1.3.10 : OTP-18.3.4.8 : ssh-4.2.2.5 # asn1-4.0.2 common_test-1.12.1.1 compiler-6.0.3.1 cosEvent-2.2 cosEventDomain-1.2 cosFileTransfer-1.2 cosNotification-1.2.1 cosProperty-1.2 cosTime-1.2.1 cosTransactions-1.3.1 crypto-3.6.3.1 debugger-4.1.2 dialyzer-2.9 diameter-1.11.2 edoc-0.7.18 eldap-1.2.1.1 erl_docgen-0.4.2 erl_interface-3.8.2 erts-7.3.1.4 et-1.5.1 eunit-2.2.13 gs-1.6 hipe-3.15 ic-4.4 inets-6.2.4.1 jinterface-1.6.1 kernel-4.2 megaco-3.18 mnesia-4.13.4 observer-2.1.2 odbc-2.11.1 orber-3.8.1 os_mon-2.4 ose-1.1 otp_mibs-1.1 parsetools-2.1.1 percept-0.8.11 public_key-1.1.1 reltool-0.7 runtime_tools-1.9.3 sasl-2.7 snmp-5.2.2 ssl-7.3.3.2 stdlib-2.8 syntax_tools-1.7 test_server-3.10 tools-2.8.3 typer-0.9.10 webtool-0.9.1 wx-1.6.1 xmerl-1.3.10 : |