diff options
222 files changed, 9058 insertions, 2223 deletions
diff --git a/.gitignore b/.gitignore index 9497169cde..bd0e9615f7 100644 --- a/.gitignore +++ b/.gitignore @@ -230,12 +230,6 @@ JAVADOC-GENERATED /erts/test/install_SUITE_data/install_bin /erts/test/autoimport_SUITE_data/erlang.xml -# asn1 - -/lib/asn1/test/asn1_SUITE.erl -/lib/asn1/test/asn1_bin_SUITE.erl -/lib/asn1/test/asn1_bin_v2_SUITE.erl - # common_test /lib/common_test/priv/install.sh @@ -323,6 +317,7 @@ JAVADOC-GENERATED # kernel /lib/kernel/src/inet_dns_record_adts.hrl +/lib/kernel/test/gen_tcp_dist.erl # kernel diff --git a/Makefile.in b/Makefile.in index 49a0d9f8af..6cbd92a55b 100644 --- a/Makefile.in +++ b/Makefile.in @@ -492,31 +492,14 @@ else $(MAKE) opt BUILD_ALL=true $(V_at)test -f $(ERL_TOP)/make/otp_built || echo "OTP built" > $(ERL_TOP)/make/otp_built endif -kernel: - $(make_verbose)cd lib/kernel && \ - ERL_TOP=$(ERL_TOP) PATH=$(BOOT_PREFIX)"$${PATH}" \ - $(MAKE) opt BUILD_ALL=true -stdlib: - $(make_verbose)cd lib/stdlib && \ - ERL_TOP=$(ERL_TOP) PATH=$(BOOT_PREFIX)"$${PATH}" \ - $(MAKE) opt BUILD_ALL=true +APPS=$(patsubst $(ERL_TOP)/lib/%/doc,%,$(wildcard $(ERL_TOP)/lib/*/doc)) -compiler: - $(make_verbose)cd lib/compiler && \ +$(APPS): + $(make_verbose)cd lib/$@ && \ ERL_TOP=$(ERL_TOP) PATH=$(BOOT_PREFIX)"$${PATH}" \ $(MAKE) opt BUILD_ALL=true -hipe: - $(make_verbose)cd lib/hipe && \ - ERL_TOP=$(ERL_TOP) PATH=$(BOOT_PREFIX)"$${PATH}" \ - $(MAKE) opt BUILD_ALL=true - -syntax_tools: - $(make_verbose)cd lib/syntax_tools && \ - ERL_TOP=$(ERL_TOP) PATH=$(BOOT_PREFIX)"$${PATH}" \ - $(MAKE) opt BUILD_ALL=true - preloaded: $(make_verbose)cd erts/preloaded/src && \ ERL_TOP=$(ERL_TOP) PATH=$(BOOT_PREFIX)"$${PATH}" \ diff --git a/OTP_VERSION b/OTP_VERSION index da0a41680b..a5a30a3cfe 100644 --- a/OTP_VERSION +++ b/OTP_VERSION @@ -1 +1 @@ -22.0-rc1 +22.0-rc2 diff --git a/bootstrap/bin/no_dot_erlang.boot b/bootstrap/bin/no_dot_erlang.boot Binary files differindex b48d22fe41..5428e202e7 100644 --- a/bootstrap/bin/no_dot_erlang.boot +++ b/bootstrap/bin/no_dot_erlang.boot diff --git a/bootstrap/bin/start.boot b/bootstrap/bin/start.boot Binary files differindex b48d22fe41..5428e202e7 100644 --- a/bootstrap/bin/start.boot +++ b/bootstrap/bin/start.boot diff --git a/bootstrap/bin/start_clean.boot b/bootstrap/bin/start_clean.boot Binary files differindex b48d22fe41..5428e202e7 100644 --- a/bootstrap/bin/start_clean.boot +++ b/bootstrap/bin/start_clean.boot diff --git a/bootstrap/lib/compiler/ebin/beam_asm.beam b/bootstrap/lib/compiler/ebin/beam_asm.beam Binary files differindex 5c91f09b30..5dafb89f68 100644 --- a/bootstrap/lib/compiler/ebin/beam_asm.beam +++ b/bootstrap/lib/compiler/ebin/beam_asm.beam diff --git a/bootstrap/lib/compiler/ebin/beam_ssa_opt.beam b/bootstrap/lib/compiler/ebin/beam_ssa_opt.beam Binary files differindex 7b34cc5906..b1bd549a8a 100644 --- a/bootstrap/lib/compiler/ebin/beam_ssa_opt.beam +++ b/bootstrap/lib/compiler/ebin/beam_ssa_opt.beam diff --git a/bootstrap/lib/compiler/ebin/beam_validator.beam b/bootstrap/lib/compiler/ebin/beam_validator.beam Binary files differindex a5db96bdaf..71186a12cd 100644 --- a/bootstrap/lib/compiler/ebin/beam_validator.beam +++ b/bootstrap/lib/compiler/ebin/beam_validator.beam diff --git a/bootstrap/lib/compiler/ebin/compile.beam b/bootstrap/lib/compiler/ebin/compile.beam Binary files differindex 5829ab4867..57cfc6b932 100644 --- a/bootstrap/lib/compiler/ebin/compile.beam +++ b/bootstrap/lib/compiler/ebin/compile.beam diff --git a/bootstrap/lib/compiler/ebin/compiler.app b/bootstrap/lib/compiler/ebin/compiler.app index 0a5e6de039..c0483e7801 100644 --- a/bootstrap/lib/compiler/ebin/compiler.app +++ b/bootstrap/lib/compiler/ebin/compiler.app @@ -19,7 +19,7 @@ {application, compiler, [{description, "ERTS CXC 138 10"}, - {vsn, "7.3.1"}, + {vsn, "7.3.2"}, {modules, [ beam_a, beam_asm, diff --git a/bootstrap/lib/compiler/ebin/compiler.appup b/bootstrap/lib/compiler/ebin/compiler.appup index 8370664167..3d4fe9c021 100644 --- a/bootstrap/lib/compiler/ebin/compiler.appup +++ b/bootstrap/lib/compiler/ebin/compiler.appup @@ -16,7 +16,7 @@ %% limitations under the License. %% %% %CopyrightEnd% -{"7.3.1", +{"7.3.2", [{<<".*">>,[{restart_application, compiler}]}], [{<<".*">>,[{restart_application, compiler}]}] }. diff --git a/bootstrap/lib/kernel/ebin/application.beam b/bootstrap/lib/kernel/ebin/application.beam Binary files differindex 9db02f55fd..84c5b7636e 100644 --- a/bootstrap/lib/kernel/ebin/application.beam +++ b/bootstrap/lib/kernel/ebin/application.beam diff --git a/bootstrap/lib/kernel/ebin/application_controller.beam b/bootstrap/lib/kernel/ebin/application_controller.beam Binary files differindex 32612f1bf6..c33895a44e 100644 --- a/bootstrap/lib/kernel/ebin/application_controller.beam +++ b/bootstrap/lib/kernel/ebin/application_controller.beam diff --git a/bootstrap/lib/kernel/ebin/code_server.beam b/bootstrap/lib/kernel/ebin/code_server.beam Binary files differindex 6add687cf2..6cbda0c9bb 100644 --- a/bootstrap/lib/kernel/ebin/code_server.beam +++ b/bootstrap/lib/kernel/ebin/code_server.beam diff --git a/bootstrap/lib/kernel/ebin/global.beam b/bootstrap/lib/kernel/ebin/global.beam Binary files differindex f57cf03afd..f4ca5b8620 100644 --- a/bootstrap/lib/kernel/ebin/global.beam +++ b/bootstrap/lib/kernel/ebin/global.beam diff --git a/bootstrap/lib/kernel/ebin/hipe_unified_loader.beam b/bootstrap/lib/kernel/ebin/hipe_unified_loader.beam Binary files differindex fc80f2d1bd..9270358896 100644 --- a/bootstrap/lib/kernel/ebin/hipe_unified_loader.beam +++ b/bootstrap/lib/kernel/ebin/hipe_unified_loader.beam diff --git a/bootstrap/lib/kernel/ebin/inet_db.beam b/bootstrap/lib/kernel/ebin/inet_db.beam Binary files differindex 76d7ac2adb..e53e183312 100644 --- a/bootstrap/lib/kernel/ebin/inet_db.beam +++ b/bootstrap/lib/kernel/ebin/inet_db.beam diff --git a/bootstrap/lib/kernel/ebin/kernel.app b/bootstrap/lib/kernel/ebin/kernel.app index ab106113c2..c3f4e2b892 100644 --- a/bootstrap/lib/kernel/ebin/kernel.app +++ b/bootstrap/lib/kernel/ebin/kernel.app @@ -22,7 +22,7 @@ {application, kernel, [ {description, "ERTS CXC 138 10"}, - {vsn, "6.2"}, + {vsn, "6.3"}, {modules, [application, application_controller, application_master, @@ -147,6 +147,6 @@ {logger_sasl_compatible, false} ]}, {mod, {kernel, []}}, - {runtime_dependencies, ["erts-10.1", "stdlib-3.5", "sasl-3.0"]} + {runtime_dependencies, ["erts-10.2.5", "stdlib-3.5", "sasl-3.0"]} ] }. diff --git a/bootstrap/lib/kernel/ebin/kernel.appup b/bootstrap/lib/kernel/ebin/kernel.appup index 59e716b2d0..e126f18b35 100644 --- a/bootstrap/lib/kernel/ebin/kernel.appup +++ b/bootstrap/lib/kernel/ebin/kernel.appup @@ -26,7 +26,7 @@ %% versions that have branched off from the above %% stated previous versions. %% -{"6.2", +{"6.3", [{<<"^5\\.3$">>,[restart_new_emulator]}, {<<"^5\\.3\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, {<<"^5\\.3\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, @@ -40,7 +40,10 @@ {<<"^6\\.0\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, {<<"^6\\.1$">>,[restart_new_emulator]}, {<<"^6\\.1\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, - {<<"^6\\.1\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}], + {<<"^6\\.1\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, + {<<"^6\\.2$">>,[restart_new_emulator]}, + {<<"^6\\.2\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, + {<<"^6\\.2\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}], [{<<"^5\\.3$">>,[restart_new_emulator]}, {<<"^5\\.3\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, {<<"^5\\.3\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, @@ -54,4 +57,7 @@ {<<"^6\\.0\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, {<<"^6\\.1$">>,[restart_new_emulator]}, {<<"^6\\.1\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, - {<<"^6\\.1\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}]}. + {<<"^6\\.1\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, + {<<"^6\\.2$">>,[restart_new_emulator]}, + {<<"^6\\.2\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, + {<<"^6\\.2\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}]}. diff --git a/bootstrap/lib/kernel/ebin/kernel.beam b/bootstrap/lib/kernel/ebin/kernel.beam Binary files differindex 8f75029c0f..11be83a49e 100644 --- a/bootstrap/lib/kernel/ebin/kernel.beam +++ b/bootstrap/lib/kernel/ebin/kernel.beam diff --git a/bootstrap/lib/kernel/ebin/logger.beam b/bootstrap/lib/kernel/ebin/logger.beam Binary files differindex bd17885e22..1dc1c4fef4 100644 --- a/bootstrap/lib/kernel/ebin/logger.beam +++ b/bootstrap/lib/kernel/ebin/logger.beam diff --git a/bootstrap/lib/kernel/ebin/logger_formatter.beam b/bootstrap/lib/kernel/ebin/logger_formatter.beam Binary files differindex 83df11e66f..e3e40460f0 100644 --- a/bootstrap/lib/kernel/ebin/logger_formatter.beam +++ b/bootstrap/lib/kernel/ebin/logger_formatter.beam diff --git a/bootstrap/lib/kernel/ebin/logger_h_common.beam b/bootstrap/lib/kernel/ebin/logger_h_common.beam Binary files differindex 5e14216b5d..9e8c25b6b9 100644 --- a/bootstrap/lib/kernel/ebin/logger_h_common.beam +++ b/bootstrap/lib/kernel/ebin/logger_h_common.beam diff --git a/bootstrap/lib/kernel/ebin/logger_simple_h.beam b/bootstrap/lib/kernel/ebin/logger_simple_h.beam Binary files differindex 954bbdc071..583c77b290 100644 --- a/bootstrap/lib/kernel/ebin/logger_simple_h.beam +++ b/bootstrap/lib/kernel/ebin/logger_simple_h.beam diff --git a/bootstrap/lib/kernel/ebin/logger_std_h.beam b/bootstrap/lib/kernel/ebin/logger_std_h.beam Binary files differindex 3b5d3cd9f8..b0f56d9a62 100644 --- a/bootstrap/lib/kernel/ebin/logger_std_h.beam +++ b/bootstrap/lib/kernel/ebin/logger_std_h.beam diff --git a/bootstrap/lib/stdlib/ebin/array.beam b/bootstrap/lib/stdlib/ebin/array.beam Binary files differindex ac3193515c..9e783452f5 100644 --- a/bootstrap/lib/stdlib/ebin/array.beam +++ b/bootstrap/lib/stdlib/ebin/array.beam diff --git a/bootstrap/lib/stdlib/ebin/beam_lib.beam b/bootstrap/lib/stdlib/ebin/beam_lib.beam Binary files differindex c13a82a074..44d783a6d0 100644 --- a/bootstrap/lib/stdlib/ebin/beam_lib.beam +++ b/bootstrap/lib/stdlib/ebin/beam_lib.beam diff --git a/bootstrap/lib/stdlib/ebin/calendar.beam b/bootstrap/lib/stdlib/ebin/calendar.beam Binary files differindex d0abc17e48..89102adda8 100644 --- a/bootstrap/lib/stdlib/ebin/calendar.beam +++ b/bootstrap/lib/stdlib/ebin/calendar.beam diff --git a/bootstrap/lib/stdlib/ebin/io_lib_pretty.beam b/bootstrap/lib/stdlib/ebin/io_lib_pretty.beam Binary files differindex 7ffe569c60..ba644430da 100644 --- a/bootstrap/lib/stdlib/ebin/io_lib_pretty.beam +++ b/bootstrap/lib/stdlib/ebin/io_lib_pretty.beam diff --git a/bootstrap/lib/stdlib/ebin/stdlib.app b/bootstrap/lib/stdlib/ebin/stdlib.app index 2bb0b06f12..9f6ff2942b 100644 --- a/bootstrap/lib/stdlib/ebin/stdlib.app +++ b/bootstrap/lib/stdlib/ebin/stdlib.app @@ -20,7 +20,7 @@ %% {application, stdlib, [{description, "ERTS CXC 138 10"}, - {vsn, "3.7.1"}, + {vsn, "3.8"}, {modules, [array, base64, beam_lib, diff --git a/bootstrap/lib/stdlib/ebin/stdlib.appup b/bootstrap/lib/stdlib/ebin/stdlib.appup index d22e917574..76372566d5 100644 --- a/bootstrap/lib/stdlib/ebin/stdlib.appup +++ b/bootstrap/lib/stdlib/ebin/stdlib.appup @@ -26,7 +26,7 @@ %% versions that have branched off from the above %% stated previous versions. %% -{"3.7.1", +{"3.8", [{<<"^3\\.4$">>,[restart_new_emulator]}, {<<"^3\\.4\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, {<<"^3\\.4\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, @@ -40,7 +40,8 @@ {<<"^3\\.6$">>,[restart_new_emulator]}, {<<"^3\\.6\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, {<<"^3\\.7$">>,[restart_new_emulator]}, - {<<"^3\\.7\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}], + {<<"^3\\.7\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, + {<<"^3\\.7\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}], [{<<"^3\\.4$">>,[restart_new_emulator]}, {<<"^3\\.4\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, {<<"^3\\.4\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, @@ -54,4 +55,5 @@ {<<"^3\\.6$">>,[restart_new_emulator]}, {<<"^3\\.6\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, {<<"^3\\.7$">>,[restart_new_emulator]}, - {<<"^3\\.7\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}]}. + {<<"^3\\.7\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, + {<<"^3\\.7\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}]}. diff --git a/bootstrap/lib/stdlib/ebin/string.beam b/bootstrap/lib/stdlib/ebin/string.beam Binary files differindex da647da3ca..437f63357c 100644 --- a/bootstrap/lib/stdlib/ebin/string.beam +++ b/bootstrap/lib/stdlib/ebin/string.beam diff --git a/bootstrap/lib/stdlib/ebin/unicode_util.beam b/bootstrap/lib/stdlib/ebin/unicode_util.beam Binary files differindex 8ddf73ca26..c9db0483ca 100644 --- a/bootstrap/lib/stdlib/ebin/unicode_util.beam +++ b/bootstrap/lib/stdlib/ebin/unicode_util.beam diff --git a/erts/Makefile b/erts/Makefile index 60c70b6a2c..e62c896170 100644 --- a/erts/Makefile +++ b/erts/Makefile @@ -129,9 +129,13 @@ makefiles: .PHONY: release release: +ifeq ($(TYPE),) for t in $(TYPES); do \ ( cd emulator && $(MAKE) release TYPE=$$t ) || exit $$?; \ done +else + ( cd emulator && $(MAKE) release TYPE=$(TYPE) ) || exit $$?; +endif $(V_at)for d in $(ERTSDIRS) $(XINSTDIRS); do \ if test -d $$d; then \ ( cd $$d && $(MAKE) $@ ) || exit $$? ; \ diff --git a/erts/configure.in b/erts/configure.in index b070ad0649..5f969a0a8b 100644 --- a/erts/configure.in +++ b/erts/configure.in @@ -2783,6 +2783,32 @@ AC_SUBST(HIPE_ENABLED) NATIVE_LIBS_ENABLED= if test X${enable_native_libs} = Xyes -a X${HIPE_ENABLED} = Xyes; then NATIVE_LIBS_ENABLED=yes + cat >> $ERL_TOP/erts/CONF_INFO <<EOF + + WARNING: In OTP 22, HiPE (the native code compiler) is + not fully functional. The reasons for this are: + + 1. There are new BEAM instructions for binary + matching that the HiPE native code compiler does not + support. + + 2. The new optimizations in the Erlang compiler create + new combination of instructions that HiPE currently + does not handle correctly. + + If erlc is invoked like so: + + erlc +native some_file.erl + + or like so: + + erlc +native some_file.beam + + and if any of the new binary matching instructions + are used, the compiler will issue a warning and + produce a BEAM file without native code. + +EOF fi AC_SUBST(NATIVE_LIBS_ENABLED) diff --git a/erts/doc/src/erlang.xml b/erts/doc/src/erlang.xml index e78ded4ae1..0d94f83493 100644 --- a/erts/doc/src/erlang.xml +++ b/erts/doc/src/erlang.xml @@ -1269,6 +1269,42 @@ end</code> data is available by calling <seealso marker="erlang#dist_ctrl_get_data_notification/1"><c>erlang:dist_ctrl_get_data_notification(DHandle)</c></seealso>. </p> + <p>The returned value when there are data available depends + on the value of the <c>get_size</c> option configured on the + distribution channel identified by <c><anno>DHandle</anno></c>. + For more information see the documentation of the <c>get_size</c> + option for the + <seealso marker="#dist_ctrl_set_opt/3"><c>erlang:dist_ctrl_set_opt/3</c></seealso> + function.</p> + <note><p> + Only the process registered as distribution + controller for the distribution channel identified by + <c><anno>DHandle</anno></c> is allowed to call this + function. + </p></note> + <p> + This function is used when implementing an alternative + distribution carrier using processes as distribution + controllers. <c><anno>DHandle</anno></c> is retrived + via the callback + <seealso marker="erts:alt_dist#hs_data_f_handshake_complete"><c>f_handshake_complete</c></seealso>. + More information can be found in the documentation of + <seealso marker="erts:alt_dist#distribution_module">ERTS + User's Guide ➜ How to implement an Alternative Carrier + for the Erlang Distribution ➜ Distribution Module</seealso>. + </p> + </desc> + </func> + + <func> + <name name="dist_ctrl_get_opt" arity="2" clause_i="1" since="OTP @OTP-15617@"/> + <fsummary>Get value of the get_size option on a distribution channel</fsummary> + <desc> + <p>Returns the value of the <c>get_size</c> option on the distribution channel + identified by <c><anno>DHandle</anno></c>. For more information see the + documentation of the <c>get_size</c> option for the + <seealso marker="#dist_ctrl_set_opt/3"><c>erlang:dist_ctrl_set_opt/3</c></seealso> + function.</p> <note><p> Only the process registered as distribution controller for the distribution channel identified by @@ -1392,6 +1428,55 @@ end</code> </func> <func> + <name name="dist_ctrl_set_opt" arity="3" clause_i="1" since="OTP @OTP-15617@"/> + <fsummary>Set value of the get_size option on a distribution channel</fsummary> + <desc> + <p>Sets the value of the <c>get_size</c> option on the distribution channel + identified by <c><anno>DHandle</anno></c>. This option controls the return + value of calls to + <seealso marker="#dist_ctrl_get_data/1">erlang:dist_ctrl_get_data(<anno>DHandle</anno>)</seealso> + where <c><anno>DHandle</anno></c> equals <c><anno>DHandle</anno></c> used + when setting this option. + When the <c>get_size</c> option is:</p> + <taglist> + <tag><c>false</c></tag> + <item> + and there are distribution data available, a call to + <c>erlang:dist_ctrl_get_data(<anno>DHandle</anno>)</c> + will just return <c>Data</c> to pass over the channel. + This is the default value of the <c>get_size</c> option. + </item> + <tag><c>true</c></tag> + <item> + and there are distribution data available, a call to + <c>erlang:dist_ctrl_get_data(<anno>DHandle</anno>)</c> + will return <c>Data</c> to pass over the channel as well as + the <c>Size</c> of <c>Data</c> in bytes. This is returned as + a tuple on the form <c>{Size, Data}</c>. + </item> + </taglist> + <p>All options are set to default when a channel is closed.</p> + <note><p> + Only the process registered as distribution + controller for the distribution channel identified by + <c><anno>DHandle</anno></c> is allowed to call this + function. + </p></note> + <p> + This function is used when implementing an alternative + distribution carrier using processes as distribution + controllers. <c><anno>DHandle</anno></c> is retrived + via the callback + <seealso marker="erts:alt_dist#hs_data_f_handshake_complete"><c>f_handshake_complete</c></seealso>. + More information can be found in the documentation of + <seealso marker="erts:alt_dist#distribution_module">ERTS + User's Guide ➜ How to implement an Alternative Carrier + for the Erlang Distribution ➜ Distribution Module</seealso>. + </p> + </desc> + </func> + + <func> <name name="element" arity="2" since=""/> <fsummary>Return the Nth element of a tuple.</fsummary> <type_desc variable="N">1..tuple_size(<anno>Tuple</anno>)</type_desc> diff --git a/erts/doc/src/notes.xml b/erts/doc/src/notes.xml index bdae994d06..7c5a8aefad 100644 --- a/erts/doc/src/notes.xml +++ b/erts/doc/src/notes.xml @@ -31,6 +31,60 @@ </header> <p>This document describes the changes made to the ERTS application.</p> +<section><title>Erts 10.3.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p>Fixed a bug in <c>seq_trace:reset_trace/0</c> that + could crash the emulator.</p> + <p> + Own Id: OTP-15704</p> + </item> + <item> + <p> + Fixed bug in <c>process_info(reductions)</c> causing it + to sometimes return invalid results.</p> + <p> + Own Id: OTP-15709 Aux Id: ERIERL-337 </p> + </item> + </list> + </section> + +</section> + +<section><title>Erts 10.3.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p>If a suspend/resume signal pair was sent to a process + while it was executing dirty, the receiving process could + later end up in a suspended state indefinitely. This bug + was introduced in ERTS version 10.0 (OTP 21.0).</p> + <p>Suspend/resume signals are sent from <seealso + marker="erts:erlang#suspend_process/1"><c>erlang:suspend_process()</c></seealso>/<seealso + marker="erts:erlang#resume_process/1"><c>erlang:resume_process()</c></seealso>. + The <seealso + marker="runtime_tools:dbg"><c>dbg</c></seealso> trace + tool utilize this functionality and could thus trigger + this bug.</p> + <p> + Own Id: OTP-15688</p> + </item> + <item> + <p> + Fix a possible deadlock when terminating the ERTS caused + by a dirty scheduler not releasing it's run-queue lock + when terminating.</p> + <p> + Own Id: OTP-15690 Aux Id: PR-2172 </p> + </item> + </list> + </section> + +</section> + <section><title>Erts 10.3</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/erts/doc/src/tty.xml b/erts/doc/src/tty.xml index 51db1ba8e2..c33e082f4f 100644 --- a/erts/doc/src/tty.xml +++ b/erts/doc/src/tty.xml @@ -165,6 +165,10 @@ erl</pre> <cell align="left" valign="middle">C-y</cell> <cell align="left" valign="middle">Insert previously killed text</cell> </row> + <row> + <cell align="left" valign="middle">C-]</cell> + <cell align="left" valign="middle">Insert matching closing bracket</cell> + </row> <tcaption>tty Text Editing</tcaption> </table> </section> diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index f81082a698..412d689246 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -290,6 +290,7 @@ atom Ge='>=' atom generational atom get_all_trap atom get_seq_token +atom get_size atom get_tcw atom gather_gc_info_result atom gather_io_bytes diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index f71efd708f..762c5da9be 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -332,6 +332,13 @@ erts_debug_disassemble_1(BIF_ALIST_1) "unknown " HEXF "\n", instr); code_ptr++; } + if (i == op_call_nif) { + /* + * The rest of the code will not be executed. Don't disassemble any + * more code in this function. + */ + code_ptr = 0; + } bin = new_binary(p, (byte *) dsbufp->str, dsbufp->str_len); erts_destroy_tmp_dsbuf(dsbufp); hsz = 4+4; @@ -569,6 +576,7 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr) } break; case op_i_bs_match_string_xfWW: + case op_i_bs_match_string_yfWW: if (ap - first_arg < 3) { erts_print(to, to_arg, "%d", *ap); } else { @@ -772,8 +780,11 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr) case op_put_tuple2_xI: case op_put_tuple2_yI: case op_new_map_dtI: - case op_update_map_assoc_sdtI: - case op_update_map_exact_jsdtI: + case op_update_map_assoc_xdtI: + case op_update_map_assoc_ydtI: + case op_update_map_assoc_cdtI: + case op_update_map_exact_xjdtI: + case op_update_map_exact_yjdtI: { int n = unpacked[-1]; diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 04a2a83123..ea01ce597d 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -1479,9 +1479,24 @@ next_catch(Process* c_p, Eterm *reg) { BeamInstr i_return_time_trace = beam_return_time_trace[0]; ptr = prev = c_p->stop; - ASSERT(is_CP(*ptr)); ASSERT(ptr <= STACK_START(c_p)); - if (ptr == STACK_START(c_p)) return NULL; + + /* This function is only called if we have active catch tags or have + * previously called a function that was exception-traced. As the exception + * trace flag isn't cleared after the traced function returns (and the + * catch tag inserted by it is gone), it's possible to land here with an + * empty stack, and the process should simply die when that happens. */ + if (ptr == STACK_START(c_p)) { + ASSERT(!active_catches && IS_TRACED_FL(c_p, F_EXCEPTION_TRACE)); + return NULL; + } + + /* + * Better safe than sorry here. In debug builds, produce a core + * dump if the top of the stack doesn't point to a continuation + * pointer. In other builds, ignore a non-CP at the top of stack. + */ + ASSERT(is_CP(*ptr)); if ((is_not_CP(*ptr) || (*cp_val(*ptr) != i_return_trace && *cp_val(*ptr) != i_return_to_trace && *cp_val(*ptr) != i_return_time_trace )) @@ -2310,12 +2325,16 @@ fixed_apply(Process* p, Eterm* reg, Uint arity, function = reg[arity+1]; if (is_not_atom(function)) { + Eterm bad_args; error: - p->freason = BADARG; - reg[0] = module; - reg[1] = function; - reg[2] = NIL; - return 0; + bad_args = make_arglist(p, reg, arity); + + p->freason = BADARG; + reg[0] = module; + reg[1] = function; + reg[2] = bad_args; + + return 0; } if (is_not_atom(module)) goto error; @@ -3248,20 +3267,23 @@ erts_is_builtin(Eterm Mod, Eterm Name, int arity) /* - * Return the current number of reductions for the given process. + * Return the current number of reductions consumed by the given process. * To get the total number of reductions, p->reds must be added. */ Uint -erts_current_reductions(Process *current, Process *p) +erts_current_reductions(Process *c_p, Process *p) { - if (current != p) { + Sint reds_left; + if (c_p != p || !(erts_atomic32_read_nob(&c_p->state) + & ERTS_PSFLG_RUNNING)) { return 0; - } else if (current->fcalls < 0 && ERTS_PROC_GET_SAVED_CALLS_BUF(current)) { - return current->fcalls + CONTEXT_REDS; + } else if (c_p->fcalls < 0 && ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) { + reds_left = c_p->fcalls + CONTEXT_REDS; } else { - return REDS_IN(current) - current->fcalls; + reds_left = c_p->fcalls; } + return REDS_IN(c_p) - reds_left; } int diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 0ad5329b2f..941c3ebbbe 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -492,6 +492,11 @@ typedef struct LoaderState { (Genop)->arity * sizeof(GenOpArg)); \ } while (0) +#define GENOP_NAME_ARITY(Genop, Name, Arity) \ + do { \ + (Genop)->op = genop_##Name##_##Arity; \ + (Genop)->arity = Arity; \ + } while (0) static void free_loader_state(Binary* magic); static ErlHeapFragment* new_literal_fragment(Uint size); @@ -3140,6 +3145,42 @@ is_killed(LoaderState* stp, GenOpArg Reg, GenOpArg Live) Live.val <= Reg.val; } +static int +is_killed_by_call_fun(LoaderState* stp, GenOpArg Reg, GenOpArg Live) +{ + return Reg.type == TAG_x && Live.type == TAG_u && + Live.val+1 <= Reg.val; +} + +/* + * Test whether register Reg is killed by make_fun instruction that + * creates the fun given by index idx. + */ + +static int +is_killed_by_make_fun(LoaderState* stp, GenOpArg Reg, GenOpArg idx) +{ + Uint num_free; + + if (idx.val >= stp->num_lambdas) { + /* Invalid index. Ignore the error for now. */ + return 0; + } else { + num_free = stp->lambdas[idx.val].num_free; + return Reg.type == TAG_x && num_free <= Reg.val; + } +} + +/* + * Test whether register Reg is killed by the send instruction that follows. + */ + +static int +is_killed_by_send(LoaderState* stp, GenOpArg Reg) +{ + return Reg.type == TAG_x && 2 <= Reg.val; +} + /* * Generate an instruction for element/2. */ @@ -3151,20 +3192,19 @@ gen_element(LoaderState* stp, GenOpArg Fail, GenOpArg Index, GenOp* op; NEW_GENOP(stp, op); - op->arity = 4; op->next = NULL; if (Index.type == TAG_i && Index.val > 0 && Index.val <= ERTS_MAX_TUPLE_SIZE && (Tuple.type == TAG_x || Tuple.type == TAG_y)) { - op->op = genop_i_fast_element_4; + GENOP_NAME_ARITY(op, i_fast_element, 4); op->a[0] = Tuple; op->a[1] = Fail; op->a[2].type = TAG_u; op->a[2].val = Index.val; op->a[3] = Dst; } else { - op->op = genop_i_element_4; + GENOP_NAME_ARITY(op, i_element, 4); op->a[0] = Tuple; op->a[1] = Fail; op->a[2] = Index; @@ -3180,8 +3220,7 @@ gen_bs_save(LoaderState* stp, GenOpArg Reg, GenOpArg Index) GenOp* op; NEW_GENOP(stp, op); - op->op = genop_i_bs_save2_2; - op->arity = 2; + GENOP_NAME_ARITY(op, i_bs_save2, 2); op->a[0] = Reg; op->a[1] = Index; if (Index.type == TAG_u) { @@ -3200,8 +3239,7 @@ gen_bs_restore(LoaderState* stp, GenOpArg Reg, GenOpArg Index) GenOp* op; NEW_GENOP(stp, op); - op->op = genop_i_bs_restore2_2; - op->arity = 2; + GENOP_NAME_ARITY(op, i_bs_restore2, 2); op->a[0] = Reg; op->a[1] = Index; if (Index.type == TAG_u) { @@ -3235,21 +3273,18 @@ gen_get_integer2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live, } else if ((Flags.val & BSF_SIGNED) != 0) { goto generic; } else if (bits == 8) { - op->op = genop_i_bs_get_integer_8_3; - op->arity = 3; - op->a[0] = Ms; + GENOP_NAME_ARITY(op, i_bs_get_integer_8, 3); + op->a[0] = Ms; op->a[1] = Fail; op->a[2] = Dst; } else if (bits == 16 && (Flags.val & BSF_LITTLE) == 0) { - op->op = genop_i_bs_get_integer_16_3; - op->arity = 3; + GENOP_NAME_ARITY(op, i_bs_get_integer_16, 3); op->a[0] = Ms; op->a[1] = Fail; op->a[2] = Dst; #ifdef ARCH_64 } else if (bits == 32 && (Flags.val & BSF_LITTLE) == 0) { - op->op = genop_i_bs_get_integer_32_3; - op->arity = 3; + GENOP_NAME_ARITY(op, i_bs_get_integer_32, 3); op->a[0] = Ms; op->a[1] = Fail; op->a[2] = Dst; @@ -3257,8 +3292,7 @@ gen_get_integer2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live, } else { generic: if (bits < SMALL_BITS) { - op->op = genop_i_bs_get_integer_small_imm_5; - op->arity = 5; + GENOP_NAME_ARITY(op, i_bs_get_integer_small_imm, 5); op->a[0] = Ms; op->a[1].type = TAG_u; op->a[1].val = bits; @@ -3266,8 +3300,7 @@ gen_get_integer2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live, op->a[3] = Flags; op->a[4] = Dst; } else { - op->op = genop_i_bs_get_integer_imm_6; - op->arity = 6; + GENOP_NAME_ARITY(op, i_bs_get_integer_imm, 6); op->a[0] = Ms; op->a[1].type = TAG_u; op->a[1].val = bits; @@ -3283,8 +3316,7 @@ gen_get_integer2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live, if (!term_to_Uint(big, &bigval)) { error: - op->op = genop_jump_1; - op->arity = 1; + GENOP_NAME_ARITY(op, jump, 1); op->a[0] = Fail; } else { if (!safe_mul(bigval, Unit.val, &bits)) { @@ -3293,8 +3325,7 @@ gen_get_integer2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live, goto generic; } } else { - op->op = genop_i_bs_get_integer_6; - op->arity = 6; + GENOP_NAME_ARITY(op, i_bs_get_integer, 6); op->a[0] = Ms; op->a[1] = Fail; op->a[2] = Live; @@ -3323,24 +3354,14 @@ gen_get_binary2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live, NATIVE_ENDIAN(Flags); if (Size.type == TAG_a && Size.val == am_all) { - if (Ms.type == Dst.type && Ms.val == Dst.val) { - op->op = genop_i_bs_get_binary_all_reuse_3; - op->arity = 3; - op->a[0] = Ms; - op->a[1] = Fail; - op->a[2] = Unit; - } else { - op->op = genop_i_bs_get_binary_all2_5; - op->arity = 5; - op->a[0] = Ms; - op->a[1] = Fail; - op->a[2] = Live; - op->a[3] = Unit; - op->a[4] = Dst; - } + GENOP_NAME_ARITY(op, i_bs_get_binary_all2, 5); + op->a[0] = Ms; + op->a[1] = Fail; + op->a[2] = Live; + op->a[3] = Unit; + op->a[4] = Dst; } else if (Size.type == TAG_i) { - op->op = genop_i_bs_get_binary_imm2_6; - op->arity = 6; + GENOP_NAME_ARITY(op, i_bs_get_binary_imm2, 6); op->a[0] = Ms; op->a[1] = Fail; op->a[2] = Live; @@ -3356,12 +3377,10 @@ gen_get_binary2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live, if (!term_to_Uint(big, &bigval)) { error: - op->op = genop_jump_1; - op->arity = 1; + GENOP_NAME_ARITY(op, jump, 1); op->a[0] = Fail; } else { - op->op = genop_i_bs_get_binary_imm2_6; - op->arity = 6; + GENOP_NAME_ARITY(op, i_bs_get_binary_imm2, 6); op->a[0] = Ms; op->a[1] = Fail; op->a[2] = Live; @@ -3373,8 +3392,7 @@ gen_get_binary2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live, op->a[5] = Dst; } } else { - op->op = genop_i_bs_get_binary2_6; - op->arity = 6; + GENOP_NAME_ARITY(op, i_bs_get_binary2, 6); op->a[0] = Ms; op->a[1] = Fail; op->a[2] = Live; @@ -3407,22 +3425,19 @@ gen_put_binary(LoaderState* stp, GenOpArg Fail,GenOpArg Size, NATIVE_ENDIAN(Flags); if (Size.type == TAG_a && Size.val == am_all) { - op->op = genop_i_new_bs_put_binary_all_3; - op->arity = 3; + GENOP_NAME_ARITY(op, i_new_bs_put_binary_all, 3); op->a[0] = Src; op->a[1] = Fail; op->a[2] = Unit; } else if (Size.type == TAG_i) { - op->op = genop_i_new_bs_put_binary_imm_3; - op->arity = 3; + GENOP_NAME_ARITY(op, i_new_bs_put_binary_imm, 3); op->a[0] = Fail; op->a[1].type = TAG_u; if (safe_mul(Size.val, Unit.val, &op->a[1].val)) { op->a[2] = Src; } else { error: - op->op = genop_badarg_1; - op->arity = 1; + GENOP_NAME_ARITY(op, badarg, 1); op->a[0] = Fail; } } else if (Size.type == TAG_q) { @@ -3440,16 +3455,14 @@ gen_put_binary(LoaderState* stp, GenOpArg Fail,GenOpArg Size, !safe_mul(bigval, Unit.val, &size)) { goto error; } - op->op = genop_i_new_bs_put_binary_imm_3; - op->arity = 3; + GENOP_NAME_ARITY(op, i_new_bs_put_binary_imm, 3); op->a[0] = Fail; op->a[1].type = TAG_u; op->a[1].val = size; op->a[2] = Src; #endif } else { - op->op = genop_i_new_bs_put_binary_4; - op->arity = 4; + GENOP_NAME_ARITY(op, i_new_bs_put_binary, 4); op->a[0] = Fail; op->a[1] = Size; op->a[2].type = TAG_u; @@ -3474,14 +3487,12 @@ gen_put_integer(LoaderState* stp, GenOpArg Fail, GenOpArg Size, Uint size; if (!safe_mul(Size.val, Unit.val, &size)) { error: - op->op = genop_badarg_1; - op->arity = 1; + GENOP_NAME_ARITY(op, badarg, 1); op->a[0] = Fail; op->next = NULL; return op; } - op->op = genop_i_new_bs_put_integer_imm_4; - op->arity = 4; + GENOP_NAME_ARITY(op, i_new_bs_put_integer_imm, 4); op->a[0] = Src; op->a[1] = Fail; op->a[2].type = TAG_u; @@ -3497,8 +3508,7 @@ gen_put_integer(LoaderState* stp, GenOpArg Fail, GenOpArg Size, !safe_mul(bigval, Unit.val, &size)) { goto error; } - op->op = genop_i_new_bs_put_integer_imm_4; - op->arity = 4; + GENOP_NAME_ARITY(op, i_new_bs_put_integer_imm, 4); op->a[0] = Src; op->a[1] = Fail; op->a[2].type = TAG_u; @@ -3506,8 +3516,7 @@ gen_put_integer(LoaderState* stp, GenOpArg Fail, GenOpArg Size, op->a[3].type = Flags.type; op->a[3].val = (Flags.val & 7); } else { - op->op = genop_i_new_bs_put_integer_4; - op->arity = 4; + GENOP_NAME_ARITY(op, i_new_bs_put_integer, 4); op->a[0] = Fail; op->a[1] = Size; op->a[2].type = TAG_u; @@ -3527,21 +3536,18 @@ gen_put_float(LoaderState* stp, GenOpArg Fail, GenOpArg Size, NATIVE_ENDIAN(Flags); if (Size.type == TAG_i) { - op->op = genop_i_new_bs_put_float_imm_4; - op->arity = 4; + GENOP_NAME_ARITY(op, i_new_bs_put_float_imm, 4); op->a[0] = Fail; op->a[1].type = TAG_u; if (!safe_mul(Size.val, Unit.val, &op->a[1].val)) { - op->op = genop_badarg_1; - op->arity = 1; + GENOP_NAME_ARITY(op, badarg, 1); op->a[0] = Fail; } else { op->a[2] = Flags; op->a[3] = Src; } } else { - op->op = genop_i_new_bs_put_float_4; - op->arity = 4; + GENOP_NAME_ARITY(op, i_new_bs_put_float, 4); op->a[0] = Fail; op->a[1] = Size; op->a[2].type = TAG_u; @@ -3564,8 +3570,7 @@ gen_get_float2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live, NEW_GENOP(stp, op); NATIVE_ENDIAN(Flags); - op->op = genop_i_bs_get_float2_6; - op->arity = 6; + GENOP_NAME_ARITY(op, i_bs_get_float2, 6); op->a[0] = Ms; op->a[1] = Fail; op->a[2] = Live; @@ -3582,7 +3587,7 @@ gen_get_float2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live, */ static GenOp* -gen_skip_bits2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, +gen_skip_bits2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Size, GenOpArg Unit, GenOpArg Flags) { GenOp* op; @@ -3602,16 +3607,14 @@ gen_skip_bits2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, * a bs_restore2 instruction which will overwrite the position * by one of the stored positions. */ - op->op = genop_bs_test_unit_3; - op->arity = 3; + GENOP_NAME_ARITY(op, bs_test_unit, 3); op->a[0] = Fail; op->a[1] = Ms; op->a[2] = Unit; } else if (Size.type == TAG_i) { - op->op = genop_i_bs_skip_bits_imm2_3; - op->arity = 3; + GENOP_NAME_ARITY(op, i_bs_skip_bits_imm2, 3); op->a[0] = Fail; - op->a[1] = Ms; + op->a[1] = Ms; op->a[2].type = TAG_u; if (!safe_mul(Size.val, Unit.val, &op->a[2].val)) { goto error; @@ -3622,22 +3625,19 @@ gen_skip_bits2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, if (!term_to_Uint(big, &bigval)) { error: - op->op = genop_jump_1; - op->arity = 1; + GENOP_NAME_ARITY(op, jump, 1); op->a[0] = Fail; } else { - op->op = genop_i_bs_skip_bits_imm2_3; - op->arity = 3; + GENOP_NAME_ARITY(op, i_bs_skip_bits_imm2, 3); op->a[0] = Fail; - op->a[1] = Ms; + op->a[1] = Ms; op->a[2].type = TAG_u; if (!safe_mul(bigval, Unit.val, &op->a[2].val)) { goto error; } } } else { - op->op = genop_i_bs_skip_bits2_4; - op->arity = 4; + GENOP_NAME_ARITY(op, i_bs_skip_bits2, 4); op->a[0] = Ms; op->a[1] = Size; op->a[2] = Fail; @@ -3654,8 +3654,7 @@ gen_increment(LoaderState* stp, GenOpArg Reg, GenOp* op; NEW_GENOP(stp, op); - op->op = genop_i_increment_3; - op->arity = 3; + GENOP_NAME_ARITY(op, i_increment, 3); op->next = NULL; op->a[0] = Reg; op->a[1].type = TAG_u; @@ -3671,8 +3670,7 @@ gen_increment_from_minus(LoaderState* stp, GenOpArg Reg, GenOp* op; NEW_GENOP(stp, op); - op->op = genop_i_increment_3; - op->arity = 3; + GENOP_NAME_ARITY(op, i_increment, 3); op->next = NULL; op->a[0] = Reg; op->a[1].type = TAG_u; @@ -3681,6 +3679,24 @@ gen_increment_from_minus(LoaderState* stp, GenOpArg Reg, return op; } +static GenOp* +gen_plus_from_minus(LoaderState* stp, GenOpArg Fail, GenOpArg Live, + GenOpArg Src, GenOpArg Integer, GenOpArg Dst) +{ + GenOp* op; + + NEW_GENOP(stp, op); + GENOP_NAME_ARITY(op, gen_plus, 5); + op->next = NULL; + op->a[0] = Fail; + op->a[1] = Live; + op->a[2] = Src; + op->a[3].type = TAG_i; + op->a[3].val = -Integer.val; + op->a[4] = Dst; + return op; +} + /* * Test whether the negation of the given number is small. */ @@ -3727,12 +3743,11 @@ gen_literal_timeout(LoaderState* stp, GenOpArg Fail, GenOpArg Time) Sint timeout; NEW_GENOP(stp, op); - op->op = genop_wait_timeout_unlocked_int_2; + GENOP_NAME_ARITY(op, wait_timeout_unlocked_int, 2); op->next = NULL; - op->arity = 2; op->a[0].type = TAG_u; op->a[1] = Fail; - + if (Time.type == TAG_i && (timeout = Time.val) >= 0 && #if defined(ARCH_64) (timeout >> 32) == 0 @@ -3761,8 +3776,7 @@ gen_literal_timeout(LoaderState* stp, GenOpArg Fail, GenOpArg Time) #if !defined(ARCH_64) error: #endif - op->op = genop_i_wait_error_0; - op->arity = 0; + GENOP_NAME_ARITY(op, i_wait_error, 0); } return op; } @@ -3774,9 +3788,8 @@ gen_literal_timeout_locked(LoaderState* stp, GenOpArg Fail, GenOpArg Time) Sint timeout; NEW_GENOP(stp, op); - op->op = genop_wait_timeout_locked_int_2; + GENOP_NAME_ARITY(op, wait_timeout_locked_int, 2); op->next = NULL; - op->arity = 2; op->a[0].type = TAG_u; op->a[1] = Fail; @@ -3808,8 +3821,7 @@ gen_literal_timeout_locked(LoaderState* stp, GenOpArg Fail, GenOpArg Time) #if !defined(ARCH_64) error: #endif - op->op = genop_i_wait_error_locked_0; - op->arity = 0; + GENOP_NAME_ARITY(op, i_wait_error_locked, 0); } return op; } @@ -3846,9 +3858,9 @@ gen_select_tuple_arity(LoaderState* stp, GenOpArg S, GenOpArg Fail, */ if (size == 2) { NEW_GENOP(stp, op); - op->next = NULL; - op->op = genop_i_select_tuple_arity2_4; + GENOP_NAME_ARITY(op, i_select_tuple_arity2, 4); GENOP_ARITY(op, arity - 1); + op->next = NULL; op->a[0] = S; op->a[1] = Fail; op->a[2].type = TAG_u; @@ -3874,9 +3886,9 @@ gen_select_tuple_arity(LoaderState* stp, GenOpArg S, GenOpArg Fail, size += align; NEW_GENOP(stp, op); - op->next = NULL; - op->op = genop_i_select_tuple_arity_3; + GENOP_NAME_ARITY(op, i_select_tuple_arity, 3); GENOP_ARITY(op, arity); + op->next = NULL; op->a[0] = S; op->a[1] = Fail; op->a[2].type = TAG_u; @@ -3932,21 +3944,18 @@ gen_split_values(LoaderState* stp, GenOpArg S, GenOpArg TypeFail, ASSERT(Size.val >= 2 && Size.val % 2 == 0); NEW_GENOP(stp, is_integer); - is_integer->op = genop_is_integer_2; - is_integer->arity = 2; + GENOP_NAME_ARITY(is_integer, is_integer, 2); is_integer->a[0] = TypeFail; is_integer->a[1] = S; NEW_GENOP(stp, label); - label->op = genop_label_1; - label->arity = 1; + GENOP_NAME_ARITY(label, label, 1); label->a[0].type = TAG_u; label->a[0].val = new_label(stp); NEW_GENOP(stp, op1); - op1->op = genop_select_val_3; + GENOP_NAME_ARITY(op1, select_val, 3); GENOP_ARITY(op1, 3 + Size.val); - op1->arity = 3; op1->a[0] = S; op1->a[1].type = TAG_f; op1->a[1].val = label->a[0].val; @@ -3954,9 +3963,8 @@ gen_split_values(LoaderState* stp, GenOpArg S, GenOpArg TypeFail, op1->a[2].val = 0; NEW_GENOP(stp, op2); - op2->op = genop_select_val_3; + GENOP_NAME_ARITY(op2, select_val, 3); GENOP_ARITY(op2, 3 + Size.val); - op2->arity = 3; op2->a[0] = S; op2->a[1] = Fail; op2->a[2].type = TAG_u; @@ -4034,19 +4042,17 @@ gen_jump_tab(LoaderState* stp, GenOpArg S, GenOpArg Fail, GenOpArg Size, GenOpAr GenOp* jump; NEW_GENOP(stp, op); - op->arity = 3; - op->op = genop_is_ne_exact_3; + GENOP_NAME_ARITY(op, is_ne_exact, 3); op->a[0] = Rest[1]; op->a[1] = S; op->a[2] = Rest[0]; NEW_GENOP(stp, jump); - jump->next = NULL; - jump->arity = 1; - jump->op = genop_jump_1; + GENOP_NAME_ARITY(jump, jump, 1); jump->a[0] = Fail; op->next = jump; + jump->next = NULL; return op; } @@ -4073,12 +4079,11 @@ gen_jump_tab(LoaderState* stp, GenOpArg S, GenOpArg Fail, GenOpArg Size, GenOpAr NEW_GENOP(stp, op); op->next = NULL; if (min == 0) { - op->op = genop_i_jump_on_val_zero_3; - fixed_args = 3; + GENOP_NAME_ARITY(op, i_jump_on_val_zero, 3); } else { - op->op = genop_i_jump_on_val_4; - fixed_args = 4; + GENOP_NAME_ARITY(op, i_jump_on_val, 4); } + fixed_args = op->arity; arity = fixed_args + size; GENOP_ARITY(op, arity); op->a[0] = S; @@ -4143,7 +4148,7 @@ gen_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail, NEW_GENOP(stp, op); op->next = NULL; - op->op = genop_i_select_val2_4; + GENOP_NAME_ARITY(op, i_select_val2, 4); GENOP_ARITY(op, arity - 1); op->a[0] = S; op->a[1] = Fail; @@ -4165,7 +4170,11 @@ gen_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail, NEW_GENOP(stp, op); op->next = NULL; - op->op = (align == 0) ? genop_i_select_val_bins_3 : genop_i_select_val_lins_3; + if (align == 0) { + GENOP_NAME_ARITY(op, i_select_val_bins, 3); + } else { + GENOP_NAME_ARITY(op, i_select_val_lins, 3); + } GENOP_ARITY(op, arity); op->a[0] = S; op->a[1] = Fail; @@ -4229,8 +4238,7 @@ gen_select_literals(LoaderState* stp, GenOpArg S, GenOpArg Fail, ASSERT(Rest[i].type == TAG_q); NEW_GENOP(stp, op); - op->op = genop_is_ne_exact_3; - op->arity = 3; + GENOP_NAME_ARITY(op, is_ne_exact, 3); op->a[0] = Rest[i+1]; op->a[1] = S; op->a[2] = Rest[i]; @@ -4239,9 +4247,8 @@ gen_select_literals(LoaderState* stp, GenOpArg S, GenOpArg Fail, } NEW_GENOP(stp, jump); + GENOP_NAME_ARITY(jump, jump, 1); jump->next = NULL; - jump->op = genop_jump_1; - jump->arity = 1; jump->a[0] = Fail; *prev_next = jump; return op; @@ -4263,9 +4270,8 @@ const_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail, ASSERT(Size.type == TAG_u); NEW_GENOP(stp, op); + GENOP_NAME_ARITY(op, jump, 1); op->next = NULL; - op->op = genop_jump_1; - op->arity = 1; /* * Search for a literal matching the controlling expression. @@ -4346,15 +4352,17 @@ gen_make_fun2(LoaderState* stp, GenOpArg idx) funp->creator = erts_init_process_id; funp->arity = arity; - op->op = genop_move_2; - op->arity = 2; + /* + * Use a move_fun/2 instruction to load the fun to enable + * further optimizations. + */ + GENOP_NAME_ARITY(op, move_fun, 2); op->a[0].type = TAG_q; op->a[0].val = lit; op->a[1].type = TAG_x; op->a[1].val = 0; } else { - op->op = genop_i_make_fun_2; - op->arity = 2; + GENOP_NAME_ARITY(op, i_make_fun, 2); op->a[0].type = TAG_u; op->a[0].val = (BeamInstr) fe; op->a[1].type = TAG_u; @@ -4382,13 +4390,11 @@ gen_is_function2(LoaderState* stp, GenOpArg Fail, GenOpArg Fun, GenOpArg Arity) */ if (Arity.val > MAX_ARG) { /* Arity is negative or too big. */ - op->op = genop_jump_1; - op->arity = 1; + GENOP_NAME_ARITY(op, jump, 1); op->a[0] = Fail; return op; } else { - op->op = genop_hot_is_function2_3; - op->arity = 3; + GENOP_NAME_ARITY(op, hot_is_function2, 3); op->a[0] = Fail; op->a[1] = Fun; op->a[2].type = TAG_u; @@ -4408,20 +4414,17 @@ gen_is_function2(LoaderState* stp, GenOpArg Fail, GenOpArg Fun, GenOpArg Arity) move_fun->next = move_arity; move_arity->next = op; - move_fun->arity = 2; - move_fun->op = genop_move_2; + GENOP_NAME_ARITY(move_fun, move, 2); move_fun->a[0] = Fun; move_fun->a[1].type = TAG_x; move_fun->a[1].val = 1022; - move_arity->arity = 2; - move_arity->op = genop_move_2; + GENOP_NAME_ARITY(move_arity, move, 2); move_arity->a[0] = Arity; move_arity->a[1].type = TAG_x; move_arity->a[1].val = 1023; - op->op = genop_cold_is_function2_3; - op->arity = 3; + GENOP_NAME_ARITY(op, cold_is_function2, 3); op->a[0] = Fail; op->a[1].type = TAG_x; op->a[1].val = 1022; @@ -4442,8 +4445,8 @@ tuple_append_put5(LoaderState* stp, GenOpArg Arity, GenOpArg Dst, NEW_GENOP(stp, op); op->next = NULL; + GENOP_NAME_ARITY(op, i_put_tuple, 2); GENOP_ARITY(op, arity+2+5); - op->op = genop_i_put_tuple_2; op->a[0] = Dst; op->a[1].type = TAG_u; op->a[1].val = arity + 5; @@ -4468,8 +4471,8 @@ tuple_append_put(LoaderState* stp, GenOpArg Arity, GenOpArg Dst, NEW_GENOP(stp, op); op->next = NULL; + GENOP_NAME_ARITY(op, i_put_tuple, 2); GENOP_ARITY(op, arity+2+1); - op->op = genop_i_put_tuple_2; op->a[0] = Dst; op->a[1].type = TAG_u; op->a[1].val = arity + 1; @@ -4537,9 +4540,9 @@ gen_new_small_map_lit(LoaderState* stp, GenOpArg Dst, GenOpArg Live, Eterm keys; NEW_GENOP(stp, op); + GENOP_NAME_ARITY(op, i_new_small_map_lit, 3); GENOP_ARITY(op, 3 + size/2); op->next = NULL; - op->op = genop_i_new_small_map_lit_3; tmp = thp = erts_alloc(ERTS_ALC_T_LOADER_TMP, (1 + size/2) * sizeof(*tmp)); keys = make_tuple(thp); @@ -4720,14 +4723,12 @@ gen_get_map_element(LoaderState* stp, GenOpArg Fail, GenOpArg Src, Key = Rest[0]; if (hash_genop_arg(stp, Key, &hx)) { - op->arity = 5; - op->op = genop_i_get_map_element_hash_5; + GENOP_NAME_ARITY(op, i_get_map_element_hash, 5); op->a[3].type = TAG_u; op->a[3].val = (BeamInstr) hx; op->a[4] = Rest[1]; } else { - op->arity = 4; - op->op = genop_i_get_map_element_4; + GENOP_NAME_ARITY(op, i_get_map_element, 4); op->a[3] = Rest[1]; } return op; @@ -4767,15 +4768,13 @@ gen_get(LoaderState* stp, GenOpArg Src, GenOpArg Dst) NEW_GENOP(stp, op); op->next = NULL; if (hash_internal_genop_arg(stp, Src, &hx)) { - op->arity = 3; - op->op = genop_i_get_hash_3; + GENOP_NAME_ARITY(op, i_get_hash, 3); op->a[0] = Src; op->a[1].type = TAG_u; op->a[1].val = (BeamInstr) hx; op->a[2] = Dst; } else { - op->arity = 2; - op->op = genop_i_get_2; + GENOP_NAME_ARITY(op, i_get, 2); op->a[0] = Src; op->a[1] = Dst; } @@ -4799,7 +4798,7 @@ gen_get_map_elements(LoaderState* stp, GenOpArg Fail, GenOpArg Src, ASSERT(Size.type == TAG_u); NEW_GENOP(stp, op); - op->op = genop_i_get_map_elements_3; + GENOP_NAME_ARITY(op, i_get_map_elements, 3); GENOP_ARITY(op, 3 + 3*(Size.val/2)); op->next = NULL; op->a[0] = Fail; @@ -4837,9 +4836,9 @@ gen_has_map_fields(LoaderState* stp, GenOpArg Fail, GenOpArg Src, n = Size.val; NEW_GENOP(stp, op); + GENOP_NAME_ARITY(op, get_map_elements, 3); GENOP_ARITY(op, 3 + 2*n); op->next = NULL; - op->op = genop_get_map_elements_3; op->a[0] = Fail; op->a[1] = Src; diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index acb7d472fd..d0e2d9afc2 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -2463,6 +2463,7 @@ BIF_RETTYPE iolist_size_1(BIF_ALIST_1) ErtsIOListSizeContext* context = NULL; Eterm state_mref; int is_trap_at_L_iter_list; + ERTS_UNDEF(state_mref, THE_NON_VALUE); ESTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK); #ifdef DEBUG iterations_until_trap = iterations_until_trap / 10; diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 11941db8cd..34a0be4f2d 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -160,6 +160,8 @@ bif erlang:dist_ctrl_input_handler/2 bif erlang:dist_ctrl_put_data/2 bif erlang:dist_ctrl_get_data/1 bif erlang:dist_ctrl_get_data_notification/1 +bif erlang:dist_ctrl_get_opt/2 +bif erlang:dist_ctrl_set_opt/3 # Static native functions in erts_internal bif erts_internal:port_info/1 diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 27bf2187c2..06d8c7cda8 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -39,6 +39,7 @@ #include "erl_hl_timer.h" #include "erl_thr_progress.h" #include "erl_proc_sig_queue.h" +#include "dist.h" /* Forward declarations -- should really appear somewhere else */ static void process_killer(void); @@ -74,6 +75,7 @@ port_info(fmtfn_t to, void *to_arg) void process_info(fmtfn_t to, void *to_arg) { + ErtsSchedulerData *esdp = erts_get_scheduler_data(); int i, max = erts_ptab_max(&erts_proc); for (i = 0; i < max; i++) { Process *p = erts_pix2proc(i); @@ -81,11 +83,35 @@ process_info(fmtfn_t to, void *to_arg) /* Do not include processes with no heap, * they are most likely just created and has invalid data */ - if (!ERTS_PROC_IS_EXITING(p) && p->heap != NULL) - print_process_info(to, to_arg, p, 0); + if (p->heap != NULL) { + ErtsProcLocks locks = (p == esdp->current_process || + p == esdp->free_process) ? ERTS_PROC_LOCK_MAIN : 0; + print_process_info(to, to_arg, p, locks); + } } } + /* Look for FREE processes in the run-queues and dist entries. + These have been removed from the ptab but we still want them + in the crash dump for debugging. */ + + /* First loop through all run-queues */ + for (i = 0; i < erts_no_schedulers + ERTS_NUM_DIRTY_RUNQS; i++) { + ErtsRunQueue *rq = ERTS_RUNQ_IX(i); + int j; + for (j = 0; j < ERTS_NO_PROC_PRIO_QUEUES; j++) { + Process *p = rq->procs.prio[j].first; + while (p) { + if (ERTS_PSFLG_FREE & erts_atomic32_read_acqb(&p->state)) + print_process_info(to, to_arg, p, 0); + p = p->next; + } + } + } + + /* Then check all dist entries */ + erts_dist_print_procs_suspended_on_de(to, to_arg); + port_info(to, to_arg); } @@ -199,13 +225,14 @@ static int doit_print_monitor(ErtsMonitor *mon, void *vpcontext, Sint reds) } return 1; } - + /* Display info about an individual Erlang process */ void print_process_info(fmtfn_t to, void *to_arg, Process *p, ErtsProcLocks orig_locks) { int garbing = 0; int running = 0; + int exiting = 0; Sint len; struct saved_calls *scb; erts_aint32_t state; @@ -226,9 +253,12 @@ print_process_info(fmtfn_t to, void *to_arg, Process *p, ErtsProcLocks orig_lock | ERTS_PSFLG_DIRTY_RUNNING)) running = 1; + if (state & ERTS_PSFLG_EXITING) + exiting = 1; + if (!(locks & ERTS_PROC_LOCK_MAIN)) { locks |= ERTS_PROC_LOCK_MAIN; - if (ERTS_IS_CRASH_DUMPING && running) { + if (ERTS_IS_CRASH_DUMPING) { if (erts_proc_trylock(p, locks)) { /* crash dumping and main lock taken, this probably means that the process is doing a GC on a dirty-scheduler... so we cannot @@ -246,7 +276,7 @@ print_process_info(fmtfn_t to, void *to_arg, Process *p, ErtsProcLocks orig_lock * If the process is registered as a global process, display the * registered name */ - if (p->common.u.alive.reg) + if (!ERTS_PROC_IS_EXITING(p) && p->common.u.alive.reg) erts_print(to, to_arg, "Name: %T\n", p->common.u.alive.reg->name); /* @@ -332,7 +362,7 @@ print_process_info(fmtfn_t to, void *to_arg, Process *p, ErtsProcLocks orig_lock } /* display the links only if there are any*/ - if (ERTS_P_LINKS(p) || ERTS_P_MONITORS(p) || ERTS_P_LT_MONITORS(p)) { + if (!exiting && (ERTS_P_LINKS(p) || ERTS_P_MONITORS(p) || ERTS_P_LT_MONITORS(p))) { PrintMonitorContext context = {1, to, to_arg}; erts_print(to, to_arg,"Link list: ["); erts_link_tree_foreach(ERTS_P_LINKS(p), doit_print_link, &context); diff --git a/erts/emulator/beam/bs_instrs.tab b/erts/emulator/beam/bs_instrs.tab index 652460a66d..9cad2b03c5 100644 --- a/erts/emulator/beam/bs_instrs.tab +++ b/erts/emulator/beam/bs_instrs.tab @@ -1136,7 +1136,6 @@ i_bs_get_utf16.execute(Fail, Flags, Dst) { } bs_context_to_binary := ctx_to_bin.fetch.execute; -i_bs_get_binary_all_reuse := ctx_to_bin.fetch_bin.execute; ctx_to_bin.head() { Eterm context; @@ -1159,16 +1158,6 @@ ctx_to_bin.fetch(Src) { } } -ctx_to_bin.fetch_bin(Src, Fail, Unit) { - context = $Src; - mb = ms_matchbuffer(context); - size = mb->size - mb->offset; - if (size % $Unit != 0) { - $FAIL($Fail); - } - offs = mb->offset; -} - ctx_to_bin.execute() { Uint hole_size; Uint orig = mb->orig; diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index 45ada4e38a..8bbe6450eb 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -772,77 +772,25 @@ void init_dist(void) #define ErtsDistOutputBuf2Binary(OB) OB->bin -#ifdef DEBUG - -struct obuf_list; -struct obuf_list { - erts_refc_t refc; - struct obuf_list *next; - struct obuf_list *prev; -}; -#define obuf_list_size sizeof(struct obuf_list) -static struct obuf_list *erts_obuf_list = NULL; -static erts_mtx_t erts_obuf_list_mtx; - -static void -insert_obuf(struct obuf_list *obuf, erts_aint_t initial) { - erts_mtx_lock(&erts_obuf_list_mtx); - obuf->next = erts_obuf_list; - obuf->prev = NULL; - erts_refc_init(&obuf->refc, initial); - if (erts_obuf_list) - erts_obuf_list->prev = obuf; - erts_obuf_list = obuf; - erts_mtx_unlock(&erts_obuf_list_mtx); -} - -static void -remove_obuf(struct obuf_list *obuf) { - if (erts_refc_dectest(&obuf->refc, 0) == 0) { - erts_mtx_lock(&erts_obuf_list_mtx); - if (obuf->prev) { - obuf->prev->next = obuf->next; - } else { - erts_obuf_list = obuf->next; - } - if (obuf->next) obuf->next->prev = obuf->prev; - erts_mtx_unlock(&erts_obuf_list_mtx); - } -} - -void check_obuf(void); -void check_obuf(void) { - erts_mtx_lock(&erts_obuf_list_mtx); - ERTS_ASSERT(erts_obuf_list == NULL); - erts_mtx_unlock(&erts_obuf_list_mtx); -} -#else -#define insert_obuf(...) -#define remove_obuf(...) -#define obuf_list_size 0 -#endif - static ERTS_INLINE ErtsDistOutputBuf * alloc_dist_obuf(Uint size, Uint headers) { int i; ErtsDistOutputBuf *obuf; Uint obuf_size = sizeof(ErtsDistOutputBuf)*(headers) + - sizeof(byte)*size + obuf_list_size; + sizeof(byte)*size; Binary *bin = erts_bin_drv_alloc(obuf_size); - size += obuf_list_size; obuf = (ErtsDistOutputBuf *) &bin->orig_bytes[size]; erts_refc_add(&bin->intern.refc, headers - 1, 1); for (i = 0; i < headers; i++) { obuf[i].bin = bin; - obuf[i].extp = (byte *)&bin->orig_bytes[0] + obuf_list_size; + obuf[i].extp = (byte *)&bin->orig_bytes[0]; #ifdef DEBUG obuf[i].dbg_pattern = ERTS_DIST_OUTPUT_BUF_DBG_PATTERN; obuf[i].alloc_endp = obuf->extp + size; ASSERT(bin == ErtsDistOutputBuf2Binary(obuf)); #endif } - insert_obuf((struct obuf_list*)&bin->orig_bytes[0], headers); return obuf; } @@ -851,10 +799,7 @@ free_dist_obuf(ErtsDistOutputBuf *obuf) { Binary *bin = ErtsDistOutputBuf2Binary(obuf); ASSERT(obuf->dbg_pattern == ERTS_DIST_OUTPUT_BUF_DBG_PATTERN); - remove_obuf((struct obuf_list*)&bin->orig_bytes[0]); - if (erts_refc_dectest(&bin->intern.refc, 0) == 0) { - erts_bin_free(bin); - } + erts_bin_release(bin); } static ERTS_INLINE Sint @@ -1968,6 +1913,7 @@ int erts_net_message(Port *prt, goto invalid_message; } reason = tuple[5]; + edep = NULL; } if (is_not_ref(ref)) @@ -2014,12 +1960,14 @@ int erts_net_message(Port *prt, } token = NIL; reason = tuple[4]; + edep = NULL; } else if (type == DOP_EXIT_TT){ if (tuple_arity != 5) { goto invalid_message; } token = tuple[4]; reason = tuple[5]; + edep = NULL; } else if (type == DOP_PAYLOAD_EXIT) { if (tuple_arity != 3) { goto invalid_message; @@ -2067,12 +2015,14 @@ int erts_net_message(Port *prt, } reason = tuple[4]; token = NIL; + edep = NULL; } else if (type == DOP_EXIT2_TT) { if (tuple_arity != 5) { goto invalid_message; } token = tuple[4]; reason = tuple[5]; + edep = NULL; } else if (type == DOP_PAYLOAD_EXIT2) { if (tuple_arity != 3) { goto invalid_message; @@ -2430,8 +2380,8 @@ erts_dsig_send(ErtsDSigSendContext *ctx) case ERTS_DSIG_SEND_PHASE_FIN: { ASSERT(ctx->obuf->extp < ctx->obuf->ext_endp); - ASSERT(((byte*)&ctx->obuf->bin->orig_bytes[0]+obuf_list_size) <= ctx->obuf->extp - ctx->max_finalize_prepend); - ASSERT(ctx->obuf->ext_endp <= ((byte*)ctx->obuf->bin->orig_bytes+obuf_list_size) + ctx->data_size + ctx->dhdr_ext_size); + ASSERT(((byte*)&ctx->obuf->bin->orig_bytes[0]) <= ctx->obuf->extp - ctx->max_finalize_prepend); + ASSERT(ctx->obuf->ext_endp <= ((byte*)ctx->obuf->bin->orig_bytes) + ctx->data_size + ctx->dhdr_ext_size); ctx->data_size = ctx->obuf->ext_endp - ctx->obuf->extp; @@ -2935,7 +2885,9 @@ erts_dist_command(Port *prt, int initial_reds) ob = oq.first; ASSERT(ob); do { + obufsize += size_obuf(ob); reds = erts_encode_ext_dist_header_finalize(ob, dep, flags, reds); + obufsize -= size_obuf(ob); if (reds < 0) break; last_finalized = ob; @@ -2974,7 +2926,9 @@ erts_dist_command(Port *prt, int initial_reds) while (oq.first && !preempt) { ErtsDistOutputBuf *fob; Uint size; + obufsize += size_obuf(oq.first); reds = erts_encode_ext_dist_header_finalize(oq.first, dep, flags, reds); + obufsize -= size_obuf(oq.first); if (reds < 0) { preempt = 1; break; @@ -3253,6 +3207,86 @@ dist_ctrl_put_data_2(BIF_ALIST_2) } BIF_RETTYPE +dist_ctrl_set_opt_3(BIF_ALIST_3) +{ + DistEntry *dep = ERTS_PROC_GET_DIST_ENTRY(BIF_P); + Uint32 conn_id; + BIF_RETTYPE ret; + + if (!dep) + BIF_ERROR(BIF_P, EXC_NOTSUP); + + if (erts_dhandle_to_dist_entry(BIF_ARG_1, &conn_id) != dep) + BIF_ERROR(BIF_P, BADARG); + + erts_de_rlock(dep); + + if (dep->connection_id != conn_id) + ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG); + else { + + switch (BIF_ARG_2) { + case am_get_size: + ERTS_BIF_PREP_RET(ret, (dep->opts & ERTS_DIST_CTRL_OPT_GET_SIZE + ? am_true + : am_false)); + if (BIF_ARG_3 == am_true) + dep->opts |= ERTS_DIST_CTRL_OPT_GET_SIZE; + else if (BIF_ARG_3 == am_false) + dep->opts &= ~ERTS_DIST_CTRL_OPT_GET_SIZE; + else + ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG); + break; + default: + ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG); + break; + } + + } + + erts_de_runlock(dep); + + return ret; +} + +BIF_RETTYPE +dist_ctrl_get_opt_2(BIF_ALIST_2) +{ + DistEntry *dep = ERTS_PROC_GET_DIST_ENTRY(BIF_P); + Uint32 conn_id; + BIF_RETTYPE ret; + + if (!dep) + BIF_ERROR(BIF_P, EXC_NOTSUP); + + if (erts_dhandle_to_dist_entry(BIF_ARG_1, &conn_id) != dep) + BIF_ERROR(BIF_P, BADARG); + + erts_de_rlock(dep); + + if (dep->connection_id != conn_id) + ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG); + else { + + switch (BIF_ARG_2) { + case am_get_size: + ERTS_BIF_PREP_RET(ret, (dep->opts & ERTS_DIST_CTRL_OPT_GET_SIZE + ? am_true + : am_false)); + break; + default: + ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG); + break; + } + + } + + erts_de_runlock(dep); + + return ret; +} + +BIF_RETTYPE dist_get_stat_1(BIF_ALIST_1) { Sint64 read, write, pend; @@ -3327,12 +3361,13 @@ dist_ctrl_get_data_1(BIF_ALIST_1) { DistEntry *dep = ERTS_PROC_GET_DIST_ENTRY(BIF_P); const Sint initial_reds = ERTS_BIF_REDS_LEFT(BIF_P); - Sint reds = initial_reds; + Sint reds = initial_reds, obufsize = 0; ErtsDistOutputBuf *obuf; - Eterm *hp; + Eterm *hp, res; ProcBin *pb; erts_aint_t qsize; - Uint32 conn_id; + Uint32 conn_id, get_size; + Uint hsz = 0, bin_sz; if (!dep) BIF_ERROR(BIF_P, EXC_NOTSUP); @@ -3384,7 +3419,9 @@ dist_ctrl_get_data_1(BIF_ALIST_1) } obuf = dep->tmp_out_queue.first; + obufsize += size_obuf(obuf); reds = erts_encode_ext_dist_header_finalize(obuf, dep, dep->flags, reds); + obufsize -= size_obuf(obuf); if (reds < 0) { erts_de_runlock(dep); ERTS_BIF_YIELD1(bif_export[BIF_dist_ctrl_get_data_1], @@ -3400,17 +3437,59 @@ dist_ctrl_get_data_1(BIF_ALIST_1) erts_de_runlock(dep); - hp = HAlloc(BIF_P, PROC_BIN_SIZE); - pb = (ProcBin *) (char *) hp; - pb->thing_word = HEADER_PROC_BIN; - pb->size = obuf->ext_endp - obuf->extp; - pb->next = MSO(BIF_P).first; - MSO(BIF_P).first = (struct erl_off_heap_header*) pb; - pb->val = ErtsDistOutputBuf2Binary(obuf); - pb->bytes = (byte*) obuf->extp; - pb->flags = 0; - - qsize = erts_atomic_add_read_nob(&dep->qsize, -size_obuf(obuf)); + bin_sz = obuf->ext_endp - obuf->extp + obuf->hdr_endp - obuf->hdrp; + + get_size = dep->opts & ERTS_DIST_CTRL_OPT_GET_SIZE; + if (get_size) { + hsz += 3; /* 2 tuple */ + if (!IS_USMALL(0, bin_sz)) + hsz += BIG_UINT_HEAP_SIZE; + } + + if (!obuf->hdrp) { + hp = HAlloc(BIF_P, PROC_BIN_SIZE + hsz); + pb = (ProcBin *) (char *) hp; + pb->thing_word = HEADER_PROC_BIN; + pb->size = obuf->ext_endp - obuf->extp; + pb->next = MSO(BIF_P).first; + MSO(BIF_P).first = (struct erl_off_heap_header*) pb; + pb->val = ErtsDistOutputBuf2Binary(obuf); + pb->bytes = (byte*) obuf->extp; + pb->flags = 0; + res = make_binary(pb); + } else { + hp = HAlloc(BIF_P, PROC_BIN_SIZE * 2 + 4 + hsz); + pb = (ProcBin *) (char *) hp; + pb->thing_word = HEADER_PROC_BIN; + pb->size = obuf->ext_endp - obuf->extp; + pb->next = MSO(BIF_P).first; + MSO(BIF_P).first = (struct erl_off_heap_header*) pb; + pb->val = ErtsDistOutputBuf2Binary(obuf); + pb->bytes = (byte*) obuf->extp; + pb->flags = 0; + hp += PROC_BIN_SIZE; + + res = CONS(hp, make_binary(pb), NIL); + hp += 2; + + pb = (ProcBin *) (char *) hp; + pb->thing_word = HEADER_PROC_BIN; + pb->size = obuf->hdr_endp - obuf->hdrp; + pb->next = MSO(BIF_P).first; + MSO(BIF_P).first = (struct erl_off_heap_header*) pb; + pb->val = ErtsDistOutputBuf2Binary(obuf); + erts_refc_inc(&pb->val->intern.refc, 1); + pb->bytes = (byte*) obuf->hdrp; + pb->flags = 0; + hp += PROC_BIN_SIZE; + res = CONS(hp, make_binary(pb), res); + hp += 2; + } + + obufsize += size_obuf(obuf); + + qsize = erts_atomic_add_read_nob(&dep->qsize, (erts_aint_t) -obufsize); + ASSERT(qsize >= 0); if (qsize < erts_dist_buf_busy_limit/2 @@ -3425,7 +3504,18 @@ dist_ctrl_get_data_1(BIF_ALIST_1) } } - BIF_RET2(make_binary(pb), (initial_reds - reds)); + if (get_size) { + Eterm sz_term; + if (IS_USMALL(0, bin_sz)) + sz_term = make_small(bin_sz); + else { + sz_term = uint_to_big(bin_sz, hp); + hp += BIG_UINT_HEAP_SIZE; + } + res = TUPLE2(hp, sz_term, res); + } + + BIF_RET2(res, (initial_reds - reds)); } void @@ -4605,10 +4695,6 @@ init_nodes_monitors(void) { erts_mtx_init(&nodes_monitors_mtx, "nodes_monitors", NIL, ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION); -#ifdef DEBUG - erts_mtx_init(&erts_obuf_list_mtx, "sad", NIL, - ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION); -#endif nodes_monitors = NULL; no_nodes_monitors = 0; } @@ -4999,3 +5085,22 @@ erts_processes_monitoring_nodes(Process *c_p) return ctxt.res; } + +static void +print_suspended_on_de(fmtfn_t to, void *to_arg, DistEntry *dep) +{ + for (; dep; dep = dep->next) { + ErtsProcList *curr = erts_proclist_peek_first(dep->suspended); + while (curr) { + if (!is_internal_pid(curr->u.pid)) + print_process_info(to, to_arg, curr->u.p, 0); + curr = erts_proclist_peek_next(dep->suspended, curr); + } + } +} + +void +erts_dist_print_procs_suspended_on_de(fmtfn_t to, void *to_arg) { + print_suspended_on_de(to, to_arg, erts_hidden_dist_entries); + print_suspended_on_de(to, to_arg, erts_visible_dist_entries); +} diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h index 38ac972f51..f953a2ab8c 100644 --- a/erts/emulator/beam/dist.h +++ b/erts/emulator/beam/dist.h @@ -172,6 +172,9 @@ extern int erts_is_alive; /* Pending connection; signals can be enqueued */ #define ERTS_DSIG_PREP_PENDING 4 +/* dist_ctrl_{g,s}et_option/2 */ +#define ERTS_DIST_CTRL_OPT_GET_SIZE ((Uint32) (1 << 0)) + #ifdef DEBUG #define ERTS_DBG_CHK_NO_DIST_LNK(D, R, L) \ erts_dbg_chk_no_dist_proc_link((D), (R), (L)) @@ -316,5 +319,6 @@ extern int erts_dsig_prepare(ErtsDSigSendContext *, int, int); +void erts_dist_print_procs_suspended_on_de(fmtfn_t to, void *to_arg); int erts_auto_connect(DistEntry* dep, Process *proc, ErtsProcLocks proc_locks); #endif diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 137f87a75c..a7424bbcb8 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -4442,6 +4442,7 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) BIF_P->fcalls = reds; else BIF_P->fcalls = reds - CONTEXT_REDS; + BIF_P->scheduler_data->virtual_reds = 0; } BIF_RET(am_true); } @@ -4606,18 +4607,17 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) } } else if (ERTS_IS_ATOM_STR("wait", BIF_ARG_1)) { - if (ERTS_IS_ATOM_STR("deallocations", BIF_ARG_2)) { - int flag = ERTS_DEBUG_WAIT_COMPLETED_DEALLOCATIONS; - if (erts_debug_wait_completed(BIF_P, flag)) { - ERTS_BIF_YIELD_RETURN(BIF_P, am_ok); - } - } - if (ERTS_IS_ATOM_STR("timer_cancellations", BIF_ARG_2)) { - int flag = ERTS_DEBUG_WAIT_COMPLETED_TIMER_CANCELLATIONS; - if (erts_debug_wait_completed(BIF_P, flag)) { - ERTS_BIF_YIELD_RETURN(BIF_P, am_ok); - } - } + int flag = 0; + if (ERTS_IS_ATOM_STR("deallocations", BIF_ARG_2)) + flag = ERTS_DEBUG_WAIT_COMPLETED_DEALLOCATIONS; + else if (ERTS_IS_ATOM_STR("timer_cancellations", BIF_ARG_2)) + flag = ERTS_DEBUG_WAIT_COMPLETED_TIMER_CANCELLATIONS; + else if (ERTS_IS_ATOM_STR("aux_work", BIF_ARG_2)) + flag = ERTS_DEBUG_WAIT_COMPLETED_AUX_WORK; + + if (flag && erts_debug_wait_completed(BIF_P, flag)) { + ERTS_BIF_YIELD_RETURN(BIF_P, am_ok); + } } else if (ERTS_IS_ATOM_STR("broken_halt", BIF_ARG_1)) { erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h index 4bf77988f7..c9c047255a 100644 --- a/erts/emulator/beam/erl_binary.h +++ b/erts/emulator/beam/erl_binary.h @@ -314,6 +314,7 @@ ERTS_GLB_INLINE Binary *erts_bin_drv_alloc(Uint size); ERTS_GLB_INLINE Binary *erts_bin_nrml_alloc(Uint size); ERTS_GLB_INLINE Binary *erts_bin_realloc_fnf(Binary *bp, Uint size); ERTS_GLB_INLINE Binary *erts_bin_realloc(Binary *bp, Uint size); +ERTS_GLB_INLINE void erts_magic_binary_free(Binary *bp); ERTS_GLB_INLINE void erts_bin_free(Binary *bp); ERTS_GLB_INLINE void erts_bin_release(Binary *bp); ERTS_GLB_INLINE Binary *erts_create_magic_binary_x(Uint size, @@ -446,6 +447,13 @@ erts_bin_realloc(Binary *bp, Uint size) } ERTS_GLB_INLINE void +erts_magic_binary_free(Binary *bp) +{ + erts_magic_ref_remove_bin(ERTS_MAGIC_BIN_REFN(bp)); + erts_free(ERTS_MAGIC_BIN_ATYPE(bp), (void *) bp); +} + +ERTS_GLB_INLINE void erts_bin_free(Binary *bp) { if (bp->intern.flags & BIN_FLAG_MAGIC) { @@ -453,8 +461,7 @@ erts_bin_free(Binary *bp) /* Destructor took control of the deallocation */ return; } - erts_magic_ref_remove_bin(ERTS_MAGIC_BIN_REFN(bp)); - erts_free(ERTS_MAGIC_BIN_ATYPE(bp), (void *) bp); + erts_magic_binary_free(bp); } else if (bp->intern.flags & BIN_FLAG_DRV) erts_free(ERTS_ALC_T_DRV_BINARY, (void *) bp); diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 9317850d96..67a73e4d57 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -577,7 +577,7 @@ force_reschedule: } static ERTS_FORCE_INLINE Uint -young_gen_usage(Process *p) +young_gen_usage(Process *p, Uint *ext_msg_usage) { Uint hsz; Eterm *aheap; @@ -604,7 +604,10 @@ young_gen_usage(Process *p) if (ERTS_SIG_IS_MSG(mp) && mp->data.attached && mp->data.attached != ERTS_MSG_COMBINED_HFRAG) { - hsz += erts_msg_attached_data_size(mp); + Uint sz = erts_msg_attached_data_size(mp); + if (ERTS_SIG_IS_EXTERNAL_MSG(mp)) + *ext_msg_usage += sz; + hsz += sz; } }); } @@ -676,6 +679,7 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end, { Uint reclaimed_now = 0; Uint ygen_usage; + Uint ext_msg_usage = 0; Eterm gc_trace_end_tag; int reds; ErtsMonotonicTime start_time; @@ -698,7 +702,7 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end, return delay_garbage_collection(p, live_hf_end, need, fcalls); } - ygen_usage = max_young_gen_usage ? max_young_gen_usage : young_gen_usage(p); + ygen_usage = max_young_gen_usage ? max_young_gen_usage : young_gen_usage(p, &ext_msg_usage); if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { check_for_possibly_long_gc(p, ygen_usage); @@ -739,7 +743,7 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end, trace_gc(p, am_gc_minor_start, need, THE_NON_VALUE); } DTRACE2(gc_minor_start, pidbuf, need); - reds = minor_collection(p, live_hf_end, need, objv, nobj, + reds = minor_collection(p, live_hf_end, need + ext_msg_usage, objv, nobj, ygen_usage, &reclaimed_now); DTRACE2(gc_minor_end, pidbuf, reclaimed_now); if (reds == -1) { @@ -764,7 +768,7 @@ do_major_collection: trace_gc(p, am_gc_major_start, need, THE_NON_VALUE); } DTRACE2(gc_major_start, pidbuf, need); - reds = major_collection(p, live_hf_end, need, objv, nobj, + reds = major_collection(p, live_hf_end, need + ext_msg_usage, objv, nobj, ygen_usage, &reclaimed_now); if (ERTS_SCHEDULER_IS_DIRTY(esdp)) p->flags &= ~(F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC); @@ -1101,6 +1105,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, Eterm* old_htop; Uint n; Uint ygen_usage = 0; + Uint ext_msg_usage = 0; struct erl_off_heap_header** prev = NULL; Sint64 reds; int hibernated = !!(p->flags & F_HIBERNATED); @@ -1118,7 +1123,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, p->flags &= ~F_DIRTY_CLA; else { Uint size = byte_lit_size/sizeof(Uint); - ygen_usage = young_gen_usage(p); + ygen_usage = young_gen_usage(p, &ext_msg_usage); if (hibernated) size = size*2 + 3*ygen_usage; else @@ -1130,7 +1135,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, } } - reds = (Sint64) garbage_collect(p, ERTS_INVALID_HFRAG_PTR, 0, + reds = (Sint64) garbage_collect(p, ERTS_INVALID_HFRAG_PTR, ext_msg_usage, p->arg_reg, p->arity, fcalls, ygen_usage); if (ERTS_PROC_IS_EXITING(p)) { @@ -1348,6 +1353,9 @@ minor_collection(Process* p, ErlHeapFragment *live_hf_end, Eterm *mature = p->abandoned_heap ? p->abandoned_heap : p->heap; Uint mature_size = p->high_water - mature; Uint size_before = ygen_usage; +#ifdef DEBUG + Uint debug_tmp = 0; +#endif /* * Check if we have gone past the max heap size limit @@ -1484,7 +1492,7 @@ minor_collection(Process* p, ErlHeapFragment *live_hf_end, process from there */ ASSERT(!MAX_HEAP_SIZE_GET(p) || !(MAX_HEAP_SIZE_FLAGS_GET(p) & MAX_HEAP_SIZE_KILL) || - MAX_HEAP_SIZE_GET(p) > (young_gen_usage(p) + + MAX_HEAP_SIZE_GET(p) > (young_gen_usage(p, &debug_tmp) + (OLD_HEND(p) - OLD_HEAP(p)) + (HEAP_END(p) - HEAP_TOP(p)))); diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 82d5140d1c..12750b9aa6 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -2417,17 +2417,12 @@ erts_exit_vv(int n, int flush_async, char *fmt, va_list args1, va_list args2) erts_exit_epilogue(); } -void check_obuf(void); __decl_noreturn void __noreturn erts_exit_epilogue(void) { int n = erts_exit_code; sys_tty_reset(n); -#ifdef DEBUG - check_obuf(); -#endif - if (n == ERTS_INTR_EXIT) exit(0); else if (n == ERTS_DUMP_EXIT) diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 39eabb6710..3aab4828cc 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -164,8 +164,7 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "os_monotonic_time", NULL }, { "erts_alloc_hard_debug", NULL }, { "hard_dbg_mseg", NULL }, - { "erts_mmap", NULL }, - { "sad", NULL} + { "erts_mmap", NULL } }; #define ERTS_LOCK_ORDER_SIZE \ diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index e350a20339..2a0fb9e2aa 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -1137,10 +1137,28 @@ change_to_off_heap: return res; } -void erts_factory_proc_init(ErtsHeapFactory* factory, - Process* p) +void erts_factory_proc_init(ErtsHeapFactory* factory, Process* p) { - erts_factory_proc_prealloc_init(factory, p, HEAP_LIMIT(p) - HEAP_TOP(p)); + /* This function does not use HAlloc to allocate on the heap + as we do not want to use INIT_HEAP_MEM on the allocated + heap as that completely destroys the DEBUG emulators + performance. */ + ErlHeapFragment *bp = p->mbuf; + factory->mode = FACTORY_HALLOC; + factory->p = p; + factory->hp_start = HEAP_TOP(p); + factory->hp = factory->hp_start; + factory->hp_end = HEAP_LIMIT(p); + factory->off_heap = &p->off_heap; + factory->message = NULL; + factory->off_heap_saved.first = p->off_heap.first; + factory->off_heap_saved.overhead = p->off_heap.overhead; + factory->heap_frags_saved = bp; + factory->heap_frags_saved_used = bp ? bp->used_size : 0; + factory->heap_frags = NULL; /* not used */ + factory->alloc_type = 0; /* not used */ + + HEAP_TOP(p) = HEAP_LIMIT(p); } void erts_factory_proc_prealloc_init(ErtsHeapFactory* factory, diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index e5f623a370..5bd25737a7 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -22,6 +22,7 @@ #define __ERL_MESSAGE_H__ #include "sys.h" +#include "erl_vm.h" #define ERTS_PROC_SIG_QUEUE_TYPE_ONLY #include "erl_proc_sig_queue.h" #undef ERTS_PROC_SIG_QUEUE_TYPE_ONLY @@ -117,6 +118,7 @@ erts_produce_heap(ErtsHeapFactory* factory, Uint need, Uint xtra) } res = factory->hp; factory->hp += need; + INIT_HEAP_MEM(res, need); return res; } diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index af1acbfc90..deaf35c2a1 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -1024,7 +1024,7 @@ static Eterm call_whereis(ErlNifEnv *env, Eterm name) int scheduler; execution_state(env, &c_p, &scheduler); - ASSERT((c_p && scheduler) || (!c_p && !scheduler)); + ASSERT(scheduler || !c_p); if (scheduler < 0) { /* dirty scheduler */ @@ -2442,10 +2442,26 @@ int erts_dbg_is_resource_dying(ErtsResource* resource) } #endif -# define NIF_RESOURCE_DTOR &nif_resource_dtor +#define NIF_RESOURCE_DTOR &nif_resource_dtor_prologue -static int nif_resource_dtor(Binary* bin) +static void run_resource_dtor(void* vbin); + +static int nif_resource_dtor_prologue(Binary* bin) { + /* + * Schedule user resource destructor as aux work to get a context + * where we know what locks we have for example. + */ + Uint sched_id = erts_get_scheduler_id(); + if (!sched_id) + sched_id = 1; + erts_schedule_misc_aux_work(sched_id, run_resource_dtor, bin); + return 0; /* don't free */ +} + +static void run_resource_dtor(void* vbin) +{ + Binary* bin = (Binary*) vbin; ErtsResource* resource = (ErtsResource*) ERTS_MAGIC_BIN_UNALIGNED_DATA(bin); ErlNifResourceType* type = resource->type; ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == NIF_RESOURCE_DTOR); @@ -2477,11 +2493,11 @@ static int nif_resource_dtor(Binary* bin) * If resource->monitors->refc != 0 there are * outstanding references to the resource from * monitors that has not been removed yet. - * nif_resource_dtor() will be called again this + * nif_resource_dtor_prologue() will be called again when this * reference count reach zero. */ if (refc != 0) - return 0; /* we'll be back... */ + return; /* we'll be back... */ erts_mtx_destroy(&rm->lock); } @@ -2498,7 +2514,7 @@ static int nif_resource_dtor(Binary* bin) steal_resource_type(type); erts_free(ERTS_ALC_T_NIF, type); } - return 1; + erts_magic_binary_free((Binary*)vbin); } void erts_resource_stop(ErtsResource* resource, ErlNifEvent e, diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index a599511c78..cc389a093f 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -59,12 +59,17 @@ */ #define ERL_NIF_MAJOR_VERSION 2 #define ERL_NIF_MINOR_VERSION 15 + /* - * WHEN CHANGING INTERFACE VERSION, also replace erts version below - * with ticket syntax like "erts-@OTP-12345@", or a temporary placeholder - * between two @ like "erts-@MyName@", if you don't know what a ticket is. + * WHEN CHANGING INTERFACE VERSION, also replace erts version below with + * a ticket number e.g. "erts-@OTP-12345@". The syntax is the same as for + * runtime dependencies so multiple tickets should be separated with ":", e.g. + * "erts-@OTP-12345:OTP-54321@". + * + * If you're not on the OTP team, you should use a placeholder like + * erts-@MyName@ instead. */ -#define ERL_NIF_MIN_ERTS_VERSION "erts-@OTP-15095 OTP-15640@ (OTP-22)" +#define ERL_NIF_MIN_ERTS_VERSION "erts-@OTP-15095:OTP-15640@" /* * The emulator will refuse to load a nif-lib with a major version diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index 09edcbbb7f..49dea8919b 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -177,6 +177,7 @@ dist_table_alloc(void *dep_tmpl) dep->connection_id = 0; dep->state = ERTS_DE_STATE_IDLE; dep->flags = 0; + dep->opts = 0; dep->version = 0; dep->mld = NULL; @@ -659,6 +660,7 @@ erts_set_dist_entry_not_connected(DistEntry *dep) dep->state = ERTS_DE_STATE_IDLE; dep->flags = 0; + dep->opts = 0; dep->prev = NULL; dep->cid = NIL; @@ -2293,7 +2295,7 @@ reference_table_term(Uint **hpp, ErlOffHeap *ohp, Uint *szp) tup = MK_2TUP(AM_ets, drp->id); } else { - ASSERT(!drp->ctrl_ref && drp->node_ref && !drp->signal_ref); + ASSERT(!drp->ctrl_ref && (drp->node_ref || drp->sequence_ref) && !drp->signal_ref); ASSERT(is_atom(drp->id)); tup = MK_2TUP(drp->id, MK_UINT(drp->creation)); tup = MK_2TUP(AM_node, tup); diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h index eb0e13ccbe..c434926142 100644 --- a/erts/emulator/beam/erl_node_tables.h +++ b/erts/emulator/beam/erl_node_tables.h @@ -148,6 +148,7 @@ struct dist_entry_ { enum dist_entry_state state; Uint32 flags; /* Distribution flags, like hidden, atom cache etc. */ + Uint32 opts; unsigned long version; /* Protocol version */ ErtsMonLnkDist *mld; /* Monitors and links */ diff --git a/erts/emulator/beam/erl_posix_str.c b/erts/emulator/beam/erl_posix_str.c index 7b3e640d3f..5b515d6e78 100644 --- a/erts/emulator/beam/erl_posix_str.c +++ b/erts/emulator/beam/erl_posix_str.c @@ -171,6 +171,9 @@ erl_errno_id(error) #if defined(EIDRM) && (!defined(EINPROGRESS) || (EIDRM != EINPROGRESS)) case EIDRM: return "eidrm"; #endif +#ifdef EILSEQ + case EILSEQ: return "eilseq"; +#endif #ifdef EINIT case EINIT: return "einit"; #endif diff --git a/erts/emulator/beam/erl_proc_sig_queue.c b/erts/emulator/beam/erl_proc_sig_queue.c index 9c74a2c355..4e9f177e51 100644 --- a/erts/emulator/beam/erl_proc_sig_queue.c +++ b/erts/emulator/beam/erl_proc_sig_queue.c @@ -611,7 +611,8 @@ proc_queue_signal(Process *c_p, Eterm pid, ErtsSignal *sig, int op) #endif return 1; } - ASSERT(esdp->pending_signal.dbg_from == esdp->current_process); + ASSERT(esdp->pending_signal.dbg_from == esdp->current_process || + esdp->pending_signal.dbg_from == esdp->free_process); if (pend_sig != sig) { /* Switch them and send previously pending signal instead */ Eterm pend_to = esdp->pending_signal.to; @@ -3429,9 +3430,15 @@ erts_proc_sig_handle_incoming(Process *c_p, erts_aint32_t *statep, erts_nif_demonitored((ErtsResource *) tmon->other.ptr); cnt++; break; - case ERTS_MON_TYPE_SUSPEND: - erts_resume(c_p, ERTS_PROC_LOCK_MAIN); + case ERTS_MON_TYPE_SUSPEND: { + ErtsMonitorSuspend *msp; + erts_aint_t mstate; + msp = (ErtsMonitorSuspend *) erts_monitor_to_data(tmon); + mstate = erts_atomic_read_acqb(&msp->state); + if (mstate & ERTS_MSUSPEND_STATE_FLG_ACTIVE) + erts_resume(c_p, ERTS_PROC_LOCK_MAIN); break; + } default: break; } @@ -3967,6 +3974,9 @@ clear_seq_trace_token(ErtsMessage *sig) case ERTS_MON_TYPE_PROC: case ERTS_MON_TYPE_DIST_PROC: case ERTS_MON_TYPE_NODE: + case ERTS_MON_TYPE_NODES: + case ERTS_MON_TYPE_SUSPEND: + case ERTS_MON_TYPE_TIME_OFFSET: break; default: ERTS_INTERNAL_ERROR("Unexpected sig type"); @@ -3983,6 +3993,11 @@ clear_seq_trace_token(ErtsMessage *sig) case ERTS_SIG_Q_OP_LINK: case ERTS_SIG_Q_OP_UNLINK: case ERTS_SIG_Q_OP_TRACE_CHANGE_STATE: + case ERTS_SIG_Q_OP_GROUP_LEADER: + case ERTS_SIG_Q_OP_IS_ALIVE: + case ERTS_SIG_Q_OP_PROCESS_INFO: + case ERTS_SIG_Q_OP_SYNC_SUSPEND: + case ERTS_SIG_Q_OP_RPC: break; default: diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 0a099e69bb..9e662632b4 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -2375,9 +2375,12 @@ struct debug_lop { static void later_thr_debug_wait_completed(void *vlop) { struct debug_lop *lop = vlop; - erts_aint32_t count = (erts_aint32_t) erts_no_schedulers; - count += 1; /* aux thread */ - if (erts_atomic32_dec_read_mb(&debug_wait_completed_count) == count) { + + if (erts_atomic32_dec_read_mb(&debug_wait_completed_count) == 1) { + erts_aint32_t count = (erts_aint32_t) erts_no_schedulers; + count += 1; /* aux thread */ + erts_atomic32_set_nob(&debug_wait_completed_count, count); + /* scheduler threads */ erts_schedule_multi_misc_aux_work(0, erts_no_schedulers, @@ -2395,19 +2398,28 @@ static void later_thr_debug_wait_completed(void *vlop) static void init_thr_debug_wait_completed(void *vproc) { - struct debug_lop* lop = erts_alloc(ERTS_ALC_T_DEBUG, - sizeof(struct debug_lop)); - lop->proc = vproc; - erts_schedule_thr_prgr_later_op(later_thr_debug_wait_completed, lop, &lop->lop); + if (debug_wait_completed_flags == ERTS_DEBUG_WAIT_COMPLETED_AUX_WORK) { + if (erts_atomic32_dec_read_mb(&debug_wait_completed_count) == 1) { + erts_atomic32_set_nob(&debug_wait_completed_count, 0); + erts_resume((Process *) vproc, (ErtsProcLocks) 0); + erts_proc_dec_refc((Process *) vproc); + } + } + else { + struct debug_lop* lop = erts_alloc(ERTS_ALC_T_DEBUG, + sizeof(struct debug_lop)); + lop->proc = vproc; + erts_schedule_thr_prgr_later_op(later_thr_debug_wait_completed, lop, &lop->lop); + } } int erts_debug_wait_completed(Process *c_p, int flags) { - /* Only one process at a time can do this */ - erts_aint32_t count = (erts_aint32_t) (2*erts_no_schedulers); - count += 1; /* aux thread */ + /* Only one process at a time can do this, +1 to mark as busy */ + erts_aint32_t count = (erts_aint32_t) (erts_no_schedulers + 1); + if (0 == erts_atomic32_cmpxchg_mb(&debug_wait_completed_count, count, 0)) { @@ -9460,6 +9472,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) if (!is_normal_sched & !!(flags & ERTS_RUNQ_FLG_HALTING)) { /* Wait for emulator to terminate... */ + erts_runq_unlock(rq); while (1) erts_milli_sleep(1000*1000); } @@ -12077,9 +12090,11 @@ erts_proc_exit_handle_dist_monitor(ErtsMonitor *mon, void *vctxt, Sint reds) ErtsDSigSendContext ctx; ErtsMonLnkDist *dist; DistEntry *dep; - Eterm watcher; + Eterm watcher, ref, *hp; ErtsMonitorData *mdp = NULL; Eterm watched; + Uint watcher_sz, ref_sz; + ErtsHeapFactory factory; ASSERT(erts_monitor_is_target(mon) && mon->type == ERTS_MON_TYPE_DIST_PROC); @@ -12111,10 +12126,18 @@ erts_proc_exit_handle_dist_monitor(ErtsMonitor *mon, void *vctxt, Sint reds) case ERTS_DSIG_PREP_CONNECTED: if (dist->connection_id != ctx.connection_id) break; + erts_factory_proc_init(&factory, c_p); + watcher_sz = size_object(watcher); + hp = erts_produce_heap(&factory, watcher_sz, 0); + watcher = copy_struct(watcher, watcher_sz, &hp, factory.off_heap); + ref_sz = size_object(mdp->ref); + hp = erts_produce_heap(&factory, ref_sz, 0); + ref = copy_struct(mdp->ref, ref_sz, &hp, factory.off_heap); + erts_factory_close(&factory); code = erts_dsig_send_m_exit(&ctx, watcher, watched, - mdp->ref, + ref, reason); switch (code) { case ERTS_DSIG_SEND_CONTINUE: @@ -12319,13 +12342,15 @@ erts_proc_exit_handle_dist_link(ErtsLink *lnk, void *vctxt, Sint reds) { ErtsProcExitContext *ctxt = (ErtsProcExitContext *) vctxt; Process *c_p = ctxt->c_p; - Eterm reason = ctxt->reason; + Eterm reason = ctxt->reason, item, *hp; + Uint item_sz; int code; ErtsDSigSendContext ctx; ErtsMonLnkDist *dist; DistEntry *dep; ErtsLink *dlnk; ErtsLinkData *ldp = NULL; + ErtsHeapFactory factory; ASSERT(lnk->type == ERTS_LNK_TYPE_DIST_PROC); dlnk = erts_link_to_other(lnk, &ldp); @@ -12352,9 +12377,14 @@ erts_proc_exit_handle_dist_link(ErtsLink *lnk, void *vctxt, Sint reds) case ERTS_DSIG_PREP_CONNECTED: if (dist->connection_id != ctx.connection_id) break; + erts_factory_proc_init(&factory, c_p); + item_sz = size_object(lnk->other.item); + hp = erts_produce_heap(&factory, item_sz, 0); + item = copy_struct(lnk->other.item, item_sz, &hp, factory.off_heap); + erts_factory_close(&factory); code = erts_dsig_send_exit_tt(&ctx, c_p->common.id, - lnk->other.item, + item, reason, SEQ_TRACE_TOKEN(c_p)); switch (code) { @@ -12571,6 +12601,8 @@ erts_continue_exit_process(Process *p) if (p->u.terminate) { trap_state = p->u.terminate; + /* Re-set the reason as it may have been gc:ed */ + trap_state->reason = p->fvalue; } else { trap_state->phase = ERTS_CONTINUE_EXIT_TIMERS; trap_state->reason = p->fvalue; @@ -12648,11 +12680,11 @@ restart: p->flags &= ~F_USING_DB; } - erts_set_gc_state(p, 1); - trap_state->phase = ERTS_CONTINUE_EXIT_CLEAN_SYS_TASKS; case ERTS_CONTINUE_EXIT_CLEAN_SYS_TASKS: - + + /* We enable GC again as it can produce more sys-tasks */ + erts_set_gc_state(p, 1); state = erts_atomic32_read_acqb(&p->state); if ((state & ERTS_PSFLG_SYS_TASKS) || p->dirty_sys_tasks) { reds -= cleanup_sys_tasks(p, state, reds); @@ -12775,6 +12807,9 @@ restart: erts_deref_dist_entry(trap_state->dep); } + /* Disable GC so that reason does not get moved */ + erts_set_gc_state(p, 0); + trap_state->pectxt.c_p = p; trap_state->pectxt.reason = trap_state->reason; trap_state->pectxt.dist_links = NULL; @@ -12859,13 +12894,13 @@ restart: switch (result) { case ERTS_DSIG_SEND_OK: case ERTS_DSIG_SEND_TOO_LRG: /*SEND_SYSTEM_LIMIT*/ - case ERTS_DSIG_SEND_YIELD: /*SEND_YIELD_RETURN*/ break; + case ERTS_DSIG_SEND_YIELD: /*SEND_YIELD_RETURN*/ case ERTS_DSIG_SEND_CONTINUE: { /*SEND_YIELD_CONTINUE*/ goto yield; } } - erts_set_gc_state(p, 1); + trap_state->pectxt.dist_state = NIL; if (reds <= 0) goto yield; @@ -12904,11 +12939,12 @@ restart: * From this point on we are no longer allowed to yield * this process. */ - #ifdef DEBUG yield_allowed = 0; #endif + erts_set_gc_state(p, 1); + /* Set state to not active as we don't want this process to be scheduled in again after this. */ state = erts_atomic32_read_band_relb(&p->state, @@ -12947,7 +12983,7 @@ restart: erts_free(ERTS_ALC_T_CONT_EXIT_TRAP, trap_state); p->u.terminate = NULL; } - + ERTS_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); if (IS_TRACED_FL(p, F_TRACE_SCHED_EXIT)) @@ -13403,10 +13439,10 @@ void erts_halt(int code) if (-1 == erts_atomic32_cmpxchg_acqb(&erts_halt_progress, erts_no_schedulers, -1)) { + notify_reap_ports_relb(); ERTS_RUNQ_FLGS_SET(ERTS_DIRTY_CPU_RUNQ, ERTS_RUNQ_FLG_HALTING); ERTS_RUNQ_FLGS_SET(ERTS_DIRTY_IO_RUNQ, ERTS_RUNQ_FLG_HALTING); erts_halt_code = code; - notify_reap_ports_relb(); } } diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 3b593bce02..711b73417d 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -1322,9 +1322,6 @@ ERTS_GLB_INLINE void erts_heap_frag_shrink(Process* p, Eterm* hp) #endif /* inline */ Eterm* erts_heap_alloc(Process* p, Uint need, Uint xtra); -#ifdef CHECK_FOR_HOLES -Eterm* erts_set_hole_marker(Eterm* ptr, Uint sz); -#endif extern erts_rwmtx_t erts_cpu_bind_rwmtx; /* If any of the erts_system_monitor_* variables are set (enabled), @@ -1856,6 +1853,7 @@ Uint erts_debug_nbalance(void); #define ERTS_DEBUG_WAIT_COMPLETED_DEALLOCATIONS (1 << 0) #define ERTS_DEBUG_WAIT_COMPLETED_TIMER_CANCELLATIONS (1 << 1) +#define ERTS_DEBUG_WAIT_COMPLETED_AUX_WORK (1 << 2) int erts_debug_wait_completed(Process *c_p, int flags); diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c index a164ed543e..71262061dd 100644 --- a/erts/emulator/beam/erl_process_dump.c +++ b/erts/emulator/beam/erl_process_dump.c @@ -121,12 +121,14 @@ Uint erts_process_memory(Process *p, int include_sigs_in_transit) size += sizeof(Process); - erts_link_tree_foreach(ERTS_P_LINKS(p), - link_size, (void *) &size); - erts_monitor_tree_foreach(ERTS_P_MONITORS(p), - monitor_size, (void *) &size); - erts_monitor_list_foreach(ERTS_P_LT_MONITORS(p), - monitor_size, (void *) &size); + if ((erts_atomic32_read_nob(&p->state) & ERTS_PSFLG_EXITING) == 0) { + erts_link_tree_foreach(ERTS_P_LINKS(p), + link_size, (void *) &size); + erts_monitor_tree_foreach(ERTS_P_MONITORS(p), + monitor_size, (void *) &size); + erts_monitor_list_foreach(ERTS_P_LT_MONITORS(p), + monitor_size, (void *) &size); + } size += (p->heap_sz + p->mbuf_sz) * sizeof(Eterm); if (p->abandoned_heap) size += (p->hend - p->heap) * sizeof(Eterm); diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h index 880febba8b..430ac305c5 100644 --- a/erts/emulator/beam/erl_utils.h +++ b/erts/emulator/beam/erl_utils.h @@ -69,7 +69,6 @@ int erts_fit_in_bits_int32(Sint32); int erts_fit_in_bits_uint(Uint); Sint erts_list_length(Eterm); int erts_is_builtin(Eterm, Eterm, int); -Uint32 block_hash(byte *, unsigned, Uint32); Uint32 make_hash2(Eterm); Uint32 make_hash(Eterm); Uint32 make_internal_hash(Eterm, Uint32 salt); diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h index 35eae18394..e623148587 100644 --- a/erts/emulator/beam/erl_vm.h +++ b/erts/emulator/beam/erl_vm.h @@ -67,9 +67,10 @@ (unsigned long)HEAP_TOP(p),(sz),__FILE__,__LINE__)), \ */ # ifdef CHECK_FOR_HOLES -# define INIT_HEAP_MEM(p,sz) erts_set_hole_marker(HEAP_TOP(p), (sz)) +Eterm* erts_set_hole_marker(Eterm* ptr, Uint sz); +# define INIT_HEAP_MEM(p,sz) erts_set_hole_marker(p, (sz)) # else -# define INIT_HEAP_MEM(p,sz) sys_memset(HEAP_TOP(p),0x01,(sz)*sizeof(Eterm*)) +# define INIT_HEAP_MEM(p,sz) sys_memset(p,0x01,(sz)*sizeof(Eterm*)) # endif #else # define INIT_HEAP_MEM(p,sz) ((void)0) @@ -91,7 +92,7 @@ ErtsHAllocLockCheck(p), \ (IS_FORCE_HEAP_FRAGS || (((HEAP_LIMIT(p) - HEAP_TOP(p)) < (sz))) \ ? erts_heap_alloc((p),(sz),(xtra)) \ - : (INIT_HEAP_MEM(p,sz), \ + : (INIT_HEAP_MEM(HEAP_TOP(p),sz), \ HEAP_TOP(p) = HEAP_TOP(p) + (sz), HEAP_TOP(p) - (sz)))) #define HAlloc(P, SZ) HAllocX(P,SZ,0) diff --git a/erts/emulator/beam/instrs.tab b/erts/emulator/beam/instrs.tab index 1eb83b61f2..692408e212 100644 --- a/erts/emulator/beam/instrs.tab +++ b/erts/emulator/beam/instrs.tab @@ -477,6 +477,13 @@ i_make_fun(FunP, NumFree) { HEAVY_SWAPIN; } +move_trim(Src, Dst, Words) { + Uint cp = E[0]; + $Dst = $Src; + E += $Words; + E[0] = cp; +} + i_trim(Words) { Uint cp = E[0]; E += $Words; @@ -493,10 +500,6 @@ move3(S1, D1, S2, D2, S3, D3) { $D3 = $S3; } -move_dup(Src, D1, D2) { - $D1 = $D2 = $Src; -} - move2_par(S1, D1, S2, D2) { Eterm V1, V2; V1 = $S1; @@ -512,6 +515,39 @@ move_shift(Src, SD, D) { $SD = V; } +move_src_window2(Src, D1, D2) { + Eterm* src = &$Src; + Eterm s1, s2; + s1 = src[0]; + s2 = src[1]; + $D1 = s1; + $D2 = s2; +} + +move_src_window3(Src, D1, D2, D3) { + Eterm* src = &$Src; + Eterm s1, s2, s3; + s1 = src[0]; + s2 = src[1]; + s3 = src[2]; + $D1 = s1; + $D2 = s2; + $D3 = s3; +} + +move_src_window4(Src, D1, D2, D3, D4) { + Eterm* src = &$Src; + Eterm s1, s2, s3, s4; + s1 = src[0]; + s2 = src[1]; + s3 = src[2]; + s4 = src[3]; + $D1 = s1; + $D2 = s2; + $D3 = s3; + $D4 = s4; +} + move_window2(S1, S2, D) { Eterm xt0, xt1; Eterm* y = &$D; @@ -703,9 +739,11 @@ jump(Fail) { $JUMP($Fail); } -move_jump(Fail, Src) { - x(0) = $Src; - $jump($Fail); +move_jump(Lbl, Src, Dst) { + Eterm lbl = $Lbl; + Eterm src = $Src; + $Dst = src; + $JUMP(lbl); } // @@ -817,6 +855,16 @@ is_tagged_tuple(Fail, Src, Arityval, Tag) { } } +is_tagged_tuple_ff(NotTupleFail, WrongRecordFail, Src, Arityval, Tag) { + Eterm term = $Src; + if (is_not_tuple(term)) { + $FAIL($NotTupleFail); + } else if (ERTS_UNLIKELY((tuple_val(term))[0] != $Arityval || + (tuple_val(term))[1] != $Tag)) { + $FAIL($WrongRecordFail); + } +} + is_tuple(Fail, Src) { if (is_not_tuple($Src)) { $FAIL($Fail); diff --git a/erts/emulator/beam/map_instrs.tab b/erts/emulator/beam/map_instrs.tab index c594a87298..5620d5a2d1 100644 --- a/erts/emulator/beam/map_instrs.tab +++ b/erts/emulator/beam/map_instrs.tab @@ -127,11 +127,21 @@ i_get_map_elements(Fail, Src, N) { } } -update_map_assoc(Src, Dst, Live, N) { +update_map_assoc := update_map_assoc.fetch.execute; + +update_map_assoc.head() { + Eterm map; +} + +update_map_assoc.fetch(Src) { + map = $Src; +} + +update_map_assoc.execute(Dst, Live, N) { Eterm res; Uint live = $Live; - reg[live] = $Src; + reg[live] = map; HEAVY_SWAPOUT; res = erts_gc_update_map_assoc(c_p, reg, live, $N, $NEXT_INSTRUCTION); HEAVY_SWAPIN; @@ -141,11 +151,21 @@ update_map_assoc(Src, Dst, Live, N) { $NEXT($NEXT_INSTRUCTION+$N); } -update_map_exact(Fail, Src, Dst, Live, N) { +update_map_exact := update_map_exact.fetch.execute; + +update_map_exact.head() { + Eterm map; +} + +update_map_exact.fetch(Src) { + map = $Src; +} + +update_map_exact.execute(Fail, Dst, Live, N) { Eterm res; Uint live = $Live; - reg[live] = $Src; + reg[live] = map; HEAVY_SWAPOUT; res = erts_gc_update_map_exact(c_p, reg, live, $N, $NEXT_INSTRUCTION); HEAVY_SWAPIN; diff --git a/erts/emulator/beam/msg_instrs.tab b/erts/emulator/beam/msg_instrs.tab index 6f8d1469ef..b08466c830 100644 --- a/erts/emulator/beam/msg_instrs.tab +++ b/erts/emulator/beam/msg_instrs.tab @@ -105,6 +105,7 @@ i_loop_rec(Dest) { $SET_CP_I_ABS(I); c_p->arity = 0; c_p->current = NULL; + c_p->fcalls = FCALLS; FCALLS -= erts_proc_sig_receive_helper(c_p, FCALLS, neg_o_reds, &msgp, &get_out); SWAPIN; diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab index da5364183c..7a125b0f67 100644 --- a/erts/emulator/beam/ops.tab +++ b/erts/emulator/beam/ops.tab @@ -106,7 +106,10 @@ init y allocate_zero t t? allocate_heap_zero t I t? +move Src=y Dst=x | trim N Remaining => move_trim Src Dst N trim N Remaining => i_trim N + +move_trim y x t i_trim t test_heap I t? @@ -261,12 +264,14 @@ system_limit j # Move instructions. # -move C=cxy x==0 | jump Lbl => move_jump Lbl C +move Src=cxy Dst=xy | jump Lbl => move_jump Lbl Src Dst + +move_jump f cxy xy +move_jump f c r -move_jump f ncxy -# Movement to and from the stack is common -# Try to pack as much as we can into one instruction +# Movement to and from the stack is common. +# Try to pack as much as we can into one instruction. # Window move move_window/5 @@ -294,11 +299,49 @@ move_window3 x x x y move_window4 x x x x y move_window5 x x x x x y +# y -> x + +move_src_window/4 +move_src_window/5 + +move Y1=y X1=x | move Y2=y X2=x | succ(Y1, Y2) => \ + move_src_window Y1 Y2 X1 X2 + +move_src_window Y1 Y2 X1 X2 | move Y3=y X3=x | succ(Y2, Y3) => \ + move_src_window Y1 Y3 X1 X2 X3 +move_src_window Y1 Y2 X1 X2 | move Y3=y X3=x | move Y4=y X4=x | succ(Y3, Y4) => \ + move_src_window2 Y1 X1 X2 | move_src_window Y3 Y4 X3 X4 +move_src_window Y1 Y2 X1 X2 | move Y3=y X3=x => \ + move3 Y1 X1 Y2 X2 Y3 X3 + +move_src_window Y1 Y3 X1 X2 X3 | move Y4=y X4=x | succ(Y3, Y4) => \ + move_src_window4 Y1 X1 X2 X3 X4 + +move_src_window Y1 y X1 X2 => move_src_window2 Y1 X1 X2 +move_src_window Y1 y X1 X2 X3 => move_src_window3 Y1 X1 X2 X3 + +move_src_window2 y x x +move_src_window3 y x x x +move_src_window4 y x x x x + # Swap registers. -move R1=x Tmp=x | move R2=x R1 | move Tmp R2 => swap_temp R1 R2 Tmp +move R1=xy Tmp=x | move R2=xy R1 | move Tmp R2 => swap_temp R1 R2 Tmp + +# The compiler uses x(1022) when swapping registers. It will definitely +# not be used again. +swap_temp R1 R2 Tmp=x==1022 => swap R1 R2 + +swap_temp R1 R2 Tmp | move Src Tmp => swap R1 R2 | move Src Tmp swap_temp R1 R2 Tmp | line Loc | apply Live | is_killed_apply(Tmp, Live) => \ swap R1 R2 | line Loc | apply Live +swap_temp R1 R2 Tmp | line Loc | apply_last Live D | is_killed_apply(Tmp, Live) => \ + swap R1 R2 | line Loc | apply_last Live D + +swap_temp R1 R2 Tmp | line Loc | call_fun Live | is_killed_by_call_fun(Tmp, Live) => \ + swap R1 R2 | line Loc | call_fun Live +swap_temp R1 R2 Tmp | make_fun2 OldIndex=u | is_killed_by_make_fun(Tmp, OldIndex) => \ + swap R1 R2 | make_fun2 OldIndex swap_temp R1 R2 Tmp | line Loc | call Live Addr | is_killed(Tmp, Live) => \ swap R1 R2 | line Loc | call Live Addr @@ -314,27 +357,55 @@ swap_temp R1 R2 Tmp | line Loc | call_ext_only Live Addr | \ swap_temp R1 R2 Tmp | line Loc | call_ext_last Live Addr D | \ is_killed(Tmp, Live) => swap R1 R2 | line Loc | call_ext_last Live Addr D -swap_temp x x x - -swap x x +swap_temp R1 R2 Tmp | call_ext Live Addr | is_killed(Tmp, Live) => \ + swap R1 R2 | call_ext Live Addr +swap_temp R1 R2 Tmp | call_ext_only Live Addr | is_killed(Tmp, Live) => \ + swap R1 R2 | call_ext_only Live Addr +swap_temp R1 R2 Tmp | call_ext_last Live Addr D | is_killed(Tmp, Live) => \ + swap R1 R2 | call_ext_last Live Addr D + +swap_temp R1 R2 Tmp | move Src Any | line Loc | call Live Addr | \ + is_killed(Tmp, Live) | distinct(Tmp, Src) => \ + swap R1 R2 | move Src Any | line Loc | call Live Addr +swap_temp R1 R2 Tmp | move Src Any | line Loc | call_ext Live Addr | \ + is_killed(Tmp, Live) | distinct(Tmp, Src) => \ + swap R1 R2 | move Src Any | line Loc | call_ext Live Addr +swap_temp R1 R2 Tmp | move Src Any | call_only Live Addr | \ + is_killed(Tmp, Live) | distinct(Tmp, Src) => \ + swap R1 R2 | move Src Any | call_only Live Addr +swap_temp R1 R2 Tmp | move Src Any | line Loc | call_ext_only Live Addr | \ + is_killed(Tmp, Live) | distinct(Tmp, Src) => \ + swap R1 R2 | move Src Any | line Loc | call_ext_only Live Addr +swap_temp R1 R2 Tmp | move Src Any | line Loc | call_fun Live | \ + is_killed(Tmp, Live) | distinct(Tmp, Src) => \ + swap R1 R2 | move Src Any | line Loc | call_fun Live + +swap_temp R1 R2 Tmp | line Loc | send | is_killed_by_send(Tmp) => \ + swap R1 R2 | line Loc | send + +# swap_temp/3 with Y register operands are rare. +swap_temp R1 R2=y Tmp => swap R1 R2 | move R2 Tmp +swap_temp R1=y R2 Tmp => swap R1 R2 | move R2 Tmp + +swap R1=x R2=y => swap R2 R1 -# move_dup - -move Src=x D1=x | move Src=x D2=x => move_dup Src D1 D2 -move Src=x SD=x | move SD=x D=x => move_dup Src SD D +swap_temp x x x -move_dup x x x +swap xy x +swap y y # move_shift -move SD=x D=x | move Src=xy SD=x | distinct(D, Src) => move_shift Src SD D -move SD=y D=x | move Src=x SD=y | distinct(D, Src) => move_shift Src SD D -move SD=x D=y | move Src=x SD=x | distinct(D, Src) => move_shift Src SD D +move SD=x D=x | move Src=cxy SD=x | distinct(D, Src) => move_shift Src SD D +move SD=y D=x | move Src=x SD=y | distinct(D, Src) => move_shift Src SD D +move SD=y D=x | init SD | => move_shift n SD D +move SD=x D=y | move Src=x SD=x | distinct(D, Src) => move_shift Src SD D +move SD=x==0 D=y | move Src=y SD=x==0 | distinct(D, Src) => move_shift Src SD D -move_shift x x x -move_shift y x x -move_shift x y x +move_shift cxy x x +move_shift nx y x move_shift x x y +move_shift y r y # move2_par x x x x @@ -530,32 +601,26 @@ put_list Src Dst=x Dst => update_list Src Dst update_list xyc x -put_list x n x -put_list y n x -put_list x x x -put_list y x x +# put_list SrcReg1 SrcReg2 => Dst -put_list y y x -put_list x y x +put_list xy xy x -# put_list SrcReg Constant Dst +# put_list SrcReg [] => Dst -put_list x c x -put_list x c y +put_list xy n xy -put_list y c x +# put_list SrcReg Constant => x -# put_list Constant SrcReg Dst +put_list xy c x -put_list c x x -put_list c y x +# put_list Constant SrcReg => Dst + +put_list c xy x # The following put_list instructions using x(0) are frequently used. -put_list r n r -put_list r n x -put_list r x x -put_list r x r +put_list r n rx +put_list r x rx put_list x x r %cold @@ -629,6 +694,11 @@ test_arity f? xy A test_arity_get_tuple_element f? x A P x +is_tuple NotTupleFail Tuple=x | is_tagged_tuple WrongRecordFail Tuple Arity Atom => \ + is_tagged_tuple_ff NotTupleFail WrongRecordFail Tuple Arity Atom + +is_tagged_tuple_ff f? f? rx A a + get_tuple_element Reg=x P1 D1=x | get_tuple_element Reg=x P2 D2=x | \ get_tuple_element Reg=x P3 D3=x | \ succ(P1, P2) | succ(P2, P3) | \ @@ -1103,8 +1173,25 @@ call_fun Arity => i_call_fun Arity i_call_fun t i_call_fun_last t Q + +# +# A fun with an empty environment can be converted to a literal. +# As a further optimization, the we try to move the fun to its +# final destination directly. + make_fun2 OldIndex=u => gen_make_fun2(OldIndex) +move_fun/2 +move_fun Fun X0 | move X0 Dst | move Src X0 => move Fun Dst | move Src X0 +move_fun Fun X0 | move A B | move X0 Dst | move Src X0 | \ + independent_moves(Fun, X0, A, B) | distinct(Dst, A) => \ + move Fun Dst | move A B | move Src X0 +move_fun Fun X0 | move X0 Dst | make_fun2 OldIndex | \ + is_killed_by_make_fun(X0, OldIndex)=> \ + move Fun Dst | make_fun2 OldIndex + +move_fun Fun Dst => move Fun Dst + %cold i_make_fun W t %hot @@ -1175,7 +1262,6 @@ bs_get_binary2 Fail=f Ms=xy Live=u Sz=sq Unit=u Flags=u Dst=d => \ i_bs_get_binary_imm2 xy f? t W t d i_bs_get_binary2 xy f t? s t d i_bs_get_binary_all2 xy f? t t d -i_bs_get_binary_all_reuse xy f? t # Fetching float from binaries. bs_get_float2 Fail=f Ms=xy Live=u Sz=s Unit=u Flags=u Dst=d => \ @@ -1484,23 +1570,22 @@ put_map_exact F Map Dst Live Size Rest=* | map_key_sort(Size, Rest) => \ sorted_put_map_assoc Map Dst Live Size Rest=* | is_empty_map(Map) => \ new_map Dst Live Size Rest -sorted_put_map_assoc Src=s Dst Live Size Rest=* => \ - update_map_assoc Src Dst Live Size Rest -sorted_put_map_assoc Src Dst Live Size Rest=* => \ - move Src x | update_map_assoc x Dst Live Size Rest +sorted_put_map_assoc Src=xyc Dst Live Size Rest=* => \ + update_map_assoc Src Dst Live Size Rest -sorted_put_map_exact F Src=s Dst Live Size Rest=* => \ - update_map_exact F Src Dst Live Size Rest -sorted_put_map_exact F Src Dst Live Size Rest=* => \ - move Src x | update_map_exact F x Dst Live Size Rest +sorted_put_map_exact Fail Src=xy Dst Live Size Rest=* => \ + update_map_exact Src Fail Dst Live Size Rest +# Literal map arguments for an exact update operation are extremely rare. +sorted_put_map_exact Fail Src Dst Live Size Rest=* => \ + move Src x | update_map_exact x Fail Dst Live Size Rest new_map Dst Live Size Rest=* | is_small_map_literal_keys(Size, Rest) => \ gen_new_small_map_lit(Dst, Live, Size, Rest) new_map d t I i_new_small_map_lit d t q -update_map_assoc s d t I -update_map_exact j? s d t I +update_map_assoc xyc d t I +update_map_exact xy j? d t I is_map Fail Lit=q | literal_is_map(Lit) => is_map Fail cq => jump Fail @@ -1562,6 +1647,11 @@ gen_minus p Live Reg=d Int=i Dst | negation_is_small(Int) => \ # Arithmetic instructions. # +# It is OK to swap arguments for '+' in a guard. It is also +# OK to turn minus into plus in a guard. +gen_plus Fail=f Live S1=c S2 Dst => i_plus S2 S1 Fail Dst +gen_minus Fail=f Live S1 S2=i Dst => gen_plus_from_minus(Fail, Live, S1, S2, Dst) + gen_plus Fail Live S1 S2 Dst => i_plus S1 S2 Fail Dst gen_minus Fail Live S1 S2 Dst => i_minus S1 S2 Fail Dst @@ -1595,10 +1685,13 @@ gc_bif1 Fail Live u$bif:erlang:bnot/1 Src Dst=d => i_int_bnot Fail Src Dst i_increment rxy W d -i_plus x xy j? d -i_plus s s j? d +# Handle unoptimized code. +i_plus S1=c S2=c Fail Dst => move S1 x | i_plus x S2 Fail Dst + +i_plus xy xyc j? d i_minus x x j? d +i_minus c x j? d i_minus s s j? d i_times j? s s d @@ -1639,8 +1732,9 @@ bif1 Fail u$bif:erlang:trunc/1 s d => too_old_compiler gc_bif1 Fail=j Live u$bif:erlang:length/1 Src Dst => \ i_length_setup Live Src | i_length Fail Live Dst -i_length_setup t xyc +i_length_setup Live Src=c => move Src x | i_length_setup Live x +i_length_setup t xy i_length j? t d # diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 36cfe0548e..0bbae65e28 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -1069,11 +1069,11 @@ do { \ #define HCONST 0x9e3779b9UL /* the golden ratio; an arbitrary value */ -Uint32 -block_hash(byte *k, unsigned length, Uint32 initval) +static Uint32 +block_hash(byte *k, Uint length, Uint32 initval) { Uint32 a,b,c; - unsigned len; + Uint len; /* Set up the internal state */ len = length; @@ -1749,7 +1749,7 @@ make_internal_hash(Eterm term, Uint32 salt) case SUB_BINARY_SUBTAG: { byte* bptr; - unsigned sz = binary_size(term); + Uint sz = binary_size(term); Uint32 con = HCONST_13 + hash; Uint bitoffs; Uint bitsize; diff --git a/erts/emulator/nifs/common/socket_dbg.c b/erts/emulator/nifs/common/socket_dbg.c index fe9135e5a0..96f75a328f 100644 --- a/erts/emulator/nifs/common/socket_dbg.c +++ b/erts/emulator/nifs/common/socket_dbg.c @@ -38,8 +38,10 @@ static FILE* dbgout = NULL; +#if defined(CLOCK_REALTIME) static int realtime(struct timespec* tsP); static int timespec2str(char *buf, unsigned int len, struct timespec *ts); +#endif extern @@ -71,41 +73,48 @@ void esock_dbg_printf( const char* prefix, const char* format, ... ) { va_list args; char f[512 + sizeof(format)]; // This has to suffice... +#if defined(CLOCK_REALTIME) char stamp[30]; struct timespec ts; +#endif int res; /* - * We should really include self in the printout, so we can se which process - * are executing the code. But then I must change the API.... - * ....something for later. + * We should really include self in the printout, + * so we can se which process are executing the code. + * But then I must change the API....something for later. */ - if (!realtime(&ts)) { - if (timespec2str(stamp, sizeof(stamp), &ts) != 0) { - res = enif_snprintf(f, sizeof(f), "%s [%s] %s", prefix, TSNAME(), format); - // res = enif_snprintf(f, sizeof(f), "%s [%s]", prefix, format); - } else { - res = enif_snprintf(f, sizeof(f), "%s [%s] [%s] %s", prefix, stamp, TSNAME(), format); - // res = enif_snprintf(f, sizeof(f), "%s [%s] %s", prefix, stamp, format); - } - - if (res > 0) { +#if defined(CLOCK_REALTIME) + if (!realtime(&ts) && + (timespec2str(stamp, sizeof(stamp), &ts) == 0)) { + res = enif_snprintf(f, sizeof(f), "%s [%s] [%s] %s", + prefix, stamp, TSNAME(), format); + } else { + res = enif_snprintf(f, sizeof(f), "%s [%s] %s", + prefix, TSNAME(), format); + } +#else + res = enif_snprintf(f, sizeof(f), "%s [%s] %s", + prefix, TSNAME(), format); +#endif + + if (res > 0) { va_start (args, format); enif_vfprintf (dbgout, f, args); va_end (args); fflush(stdout); - } } return; } +#if defined(CLOCK_REALTIME) static int realtime(struct timespec* tsP) { - return clock_gettime(CLOCK_REALTIME, tsP); + return clock_gettime(CLOCK_REALTIME, tsP); } @@ -136,3 +145,4 @@ int timespec2str(char *buf, unsigned int len, struct timespec *ts) return 0; } +#endif diff --git a/erts/emulator/nifs/common/socket_nif.c b/erts/emulator/nifs/common/socket_nif.c index 052c585032..870ab63bdf 100644 --- a/erts/emulator/nifs/common/socket_nif.c +++ b/erts/emulator/nifs/common/socket_nif.c @@ -2410,7 +2410,8 @@ static void socket_down_reader(ErlNifEnv* env, const ErlNifPid* pid); static char* esock_send_close_msg(ErlNifEnv* env, - SocketDescriptor* descP); + SocketDescriptor* descP, + ERL_NIF_TERM sockRef); static char* esock_send_abort_msg(ErlNifEnv* env, ERL_NIF_TERM sockRef, ERL_NIF_TERM recvRef, @@ -16896,15 +16897,18 @@ char* send_msg_error(ErlNifEnv* env, */ static char* esock_send_close_msg(ErlNifEnv* env, - SocketDescriptor* descP) -{ - ERL_NIF_TERM sockRef = enif_make_resource(descP->closeEnv, descP); - char* res = esock_send_socket_msg(env, - sockRef, - esock_atom_close, - descP->closeRef, - &descP->closerPid, - descP->closeEnv); + SocketDescriptor* descP, + ERL_NIF_TERM sockRef) +{ + ERL_NIF_TERM sr = ((descP->closeEnv != NULL) ? + enif_make_copy(descP->closeEnv, sockRef) : + sockRef); + char* res = esock_send_socket_msg(env, + sr, + esock_atom_close, + descP->closeRef, + &descP->closerPid, + descP->closeEnv); descP->closeEnv = NULL; @@ -17754,11 +17758,12 @@ void socket_stop(ErlNifEnv* env, void* obj, int fd, int is_direct_call) if (descP->sock != INVALID_SOCKET) { if (descP->closeLocal) { + if (!is_direct_call) { /* +++ send close message to the waiting process +++ */ - esock_send_close_msg(env, descP); + esock_send_close_msg(env, descP, sockRef); DEMONP("socket_stop -> closer", env, descP, &descP->closerMon); @@ -17768,7 +17773,11 @@ void socket_stop(ErlNifEnv* env, void* obj, int fd, int is_direct_call) * since the message send takes care of it if scheduled. */ - if (descP->closeEnv != NULL) enif_free_env(descP->closeEnv); + if (descP->closeEnv != NULL) { + enif_clear_env(descP->closeEnv); + enif_free_env(descP->closeEnv); + descP->closeEnv = NULL; + } } } diff --git a/erts/emulator/nifs/common/socket_util.c b/erts/emulator/nifs/common/socket_util.c index b817ae7636..5e18355308 100644 --- a/erts/emulator/nifs/common/socket_util.c +++ b/erts/emulator/nifs/common/socket_util.c @@ -51,8 +51,12 @@ extern char* erl_errno_id(int error); /* THIS IS JUST TEMPORARY??? */ +#if defined(CLOCK_REALTIME) static int realtime(struct timespec* tsP); -static int timespec2str(char *buf, unsigned int len, struct timespec *ts); +static int timespec2str(char *buf, + unsigned int len, + struct timespec *ts); +#endif static char* make_sockaddr_in4(ErlNifEnv* env, ERL_NIF_TERM port, @@ -1506,39 +1510,46 @@ void esock_warning_msg( const char* format, ... ) { va_list args; char f[512 + sizeof(format)]; // This has to suffice... +#if defined(CLOCK_REALTIME) char stamp[64]; // Just in case... struct timespec ts; +#endif int res; /* - * We should really include self in the printout, so we can se which process - * are executing the code. But then I must change the API.... - * ....something for later. + * We should really include self in the printout, + * so we can se which process are executing the code. + * But then I must change the API....something for later. */ // 2018-06-29 12:13:21.232089 // 29-Jun-2018::13:47:25.097097 - - if (!realtime(&ts)) { - if (timespec2str(stamp, sizeof(stamp), &ts) != 0) { - res = enif_snprintf(f, sizeof(f), "=WARNING MSG==== %s", format); - } else { - res = enif_snprintf(f, sizeof(f), - "=WARNING MSG==== %s ===\r\n%s" , stamp, format); - } - if (res > 0) { +#if defined(CLOCK_REALTIME) + if (!realtime(&ts) && + (timespec2str(stamp, sizeof(stamp), &ts) == 0)) { + res = enif_snprintf(f, sizeof(f), + "=WARNING MSG==== %s ===\r\n%s", + stamp, format); + } else { + res = enif_snprintf(f, sizeof(f), "=WARNING MSG==== %s", format); + } +#else + res = enif_snprintf(f, sizeof(f), "=WARNING MSG==== %s", format); +#endif + + if (res > 0) { va_start (args, format); enif_vfprintf (stdout, f, args); va_end (args); fflush(stdout); - } } return; } +#if defined(CLOCK_REALTIME) static int realtime(struct timespec* tsP) { @@ -1574,6 +1585,7 @@ int timespec2str(char *buf, unsigned int len, struct timespec *ts) return 0; } +#endif /* =================================================================== * diff --git a/erts/emulator/nifs/win32/win_prim_file.c b/erts/emulator/nifs/win32/win_prim_file.c index d0aa70542f..e7d3924240 100644 --- a/erts/emulator/nifs/win32/win_prim_file.c +++ b/erts/emulator/nifs/win32/win_prim_file.c @@ -142,12 +142,15 @@ static posix_errno_t get_full_path(ErlNifEnv *env, WCHAR *input, efile_path_t *r maybe_unc_path = !sys_memcmp(result->data, L"\\\\", sizeof(WCHAR) * 2); if(maybe_unc_path && !is_long_path) { - /* \\localhost\c$\gurka -> \\?\UNC\localhost\c$\gurka */ + /* \\localhost\c$\gurka -> \\?\UNC\localhost\c$\gurka + * + * Note that the length is reduced by 2 as the "\\" is replaced by + * the UNC prefix */ sys_memmove(result->data + LP_UNC_PREFIX_SIZE, &((WCHAR*)result->data)[2], - (actual_length - 1) * sizeof(WCHAR)); + (actual_length + 1 - 2) * sizeof(WCHAR)); sys_memcpy(result->data, LP_UNC_PREFIX, LP_UNC_PREFIX_SIZE); - actual_length += LP_UNC_PREFIX_LENGTH; + actual_length += LP_UNC_PREFIX_LENGTH - 2; } else if(!is_long_path) { /* C:\gurka -> \\?\C:\gurka */ sys_memmove(result->data + LP_PREFIX_SIZE, result->data, diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c index c71d23f58c..9662996039 100644 --- a/erts/emulator/sys/common/erl_poll.c +++ b/erts/emulator/sys/common/erl_poll.c @@ -2354,6 +2354,7 @@ uint32_t epoll_events(int kp_fd, int fd) fprintf(stderr,"failed to parse file %s on line %d, errno = %d\n", fname, line, errno); + fclose(f); return 0; } if (fd == ev_fd) { @@ -2408,6 +2409,7 @@ ERTS_POLL_EXPORT(erts_poll_get_selected_events)(ErtsPollSet *ps, if (fscanf(f,"pos:\t%x\nflags:\t%x", &pos, &flags) != 2) { fprintf(stderr,"failed to parse file %s, errno = %d\n", fname, errno); ASSERT(0); + fclose(f); return; } if (fscanf(f,"\nmnt_id:\t%x\n", &mnt_id)); @@ -2422,6 +2424,7 @@ ERTS_POLL_EXPORT(erts_poll_get_selected_events)(ErtsPollSet *ps, fprintf(stderr,"failed to parse file %s on line %d, errno = %d\n", fname, line, errno); ASSERT(0); + fclose(f); return; } if (fd == ps->wake_fds[0] || fd == ps->wake_fds[1]) @@ -2437,6 +2440,7 @@ ERTS_POLL_EXPORT(erts_poll_get_selected_events)(ErtsPollSet *ps, ev[fd] = (ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT) & ERTS_POLL_EV_N2E(events); line++; } + fclose(f); #else for (fd = 0; fd < len; fd++) ev[fd] = ERTS_POLL_EV_NONE; diff --git a/erts/emulator/test/bif_SUITE.erl b/erts/emulator/test/bif_SUITE.erl index 3eedf2f6a6..43975d1800 100644 --- a/erts/emulator/test/bif_SUITE.erl +++ b/erts/emulator/test/bif_SUITE.erl @@ -38,7 +38,8 @@ is_process_alive/1, process_info_blast/1, os_env_case_sensitivity/1, - test_length/1]). + test_length/1, + fixed_apply_badarg/1]). suite() -> [{ct_hooks,[ts_install_cth]}, @@ -54,7 +55,7 @@ all() -> error_stacktrace, error_stacktrace_during_call_trace, group_leader_prio, group_leader_prio_dirty, is_process_alive, process_info_blast, os_env_case_sensitivity, - test_length]. + test_length,fixed_apply_badarg]. %% Uses erlang:display to test that erts_printf does not do deep recursion display(Config) when is_list(Config) -> @@ -1230,6 +1231,23 @@ test_length(I, N, Inc, Good, Bad) when I < N -> lists:reverse(IncSeq, Bad)); test_length(_, _, _, _, _) -> ok. +%% apply/3 with a fixed number of arguments didn't include all arguments on +%% badarg exceptions. +fixed_apply_badarg(Config) when is_list(Config) -> + Bad = id({}), + + {'EXIT',{badarg, [{erlang,apply,[{},baz,[a,b]],[]} | _]}} = + (catch Bad:baz(a,b)), + {'EXIT',{badarg, [{erlang,apply,[baz,{},[c,d]],[]} | _]}} = + (catch baz:Bad(c,d)), + + {'EXIT',{badarg, [{erlang,apply,[{},baz,[e,f]],[]} | _]}} = + (catch apply(Bad,baz,[e,f])), + {'EXIT',{badarg, [{erlang,apply,[baz,{},[g,h]],[]} | _]}} = + (catch apply(baz,Bad,[g,h])), + + ok. + %% helpers id(I) -> I. diff --git a/erts/emulator/test/binary_SUITE.erl b/erts/emulator/test/binary_SUITE.erl index 1406ddc9dc..563d60cc3f 100644 --- a/erts/emulator/test/binary_SUITE.erl +++ b/erts/emulator/test/binary_SUITE.erl @@ -1703,13 +1703,16 @@ error_after_yield_bad_ext_term() -> BadAtomExt]). %% Invalid atom at the end cmp_old_impl(Config) when is_list(Config) -> - %% Compare results from new yielding implementations with - %% old non yielding implementations + %% This test was originally a comparison with the non yielding + %% implementation in R16B. Since OTP 22 we can't talk distribution with such + %% old nodes (< 19). The test case it kept but compares with previous major + %% version for semantic regression test. Cookie = atom_to_list(erlang:get_cookie()), - Rel = "r16b_latest", + Rel = (integer_to_list(list_to_integer(erlang:system_info(otp_release)) - 1) + ++ "_latest"), case test_server:is_release_available(Rel) of false -> - {skipped, "No "++Rel++" available"}; + {skipped, "No OTP "++Rel++" available"}; true -> {ok, Node} = test_server:start_node(list_to_atom(atom_to_list(?MODULE)++"_"++Rel), peer, diff --git a/erts/emulator/test/bs_match_misc_SUITE.erl b/erts/emulator/test/bs_match_misc_SUITE.erl index 17759d78f3..cae4eb54d2 100644 --- a/erts/emulator/test/bs_match_misc_SUITE.erl +++ b/erts/emulator/test/bs_match_misc_SUITE.erl @@ -24,7 +24,7 @@ kenneth/1,encode_binary/1,native/1,happi/1, size_var/1,wiger/1,x0_context/1,huge_float_field/1, writable_binary_matched/1,otp_7198/1,unordered_bindings/1, - float_middle_endian/1]). + float_middle_endian/1,unsafe_get_binary_reuse/1]). -include_lib("common_test/include/ct.hrl"). @@ -36,7 +36,8 @@ all() -> [bound_var, bound_tail, t_float, little_float, sean, kenneth, encode_binary, native, happi, size_var, wiger, x0_context, huge_float_field, writable_binary_matched, - otp_7198, unordered_bindings, float_middle_endian]. + otp_7198, unordered_bindings, float_middle_endian, + unsafe_get_binary_reuse]. %% Test matching of bound variables. @@ -556,5 +557,21 @@ unordered_bindings(CompressedLength, HashSize, PadLength, T) -> Padding:PadLength/binary,PadLength>> = T, {Content,Mac,Padding}. +%% ERL-901: A load-time optimization assumed that match contexts had no further +%% uses when a bs_get_binary2 overwrote the match context's register, and +%% figured it would be safe to reuse the match context's memory for the +%% resulting binary. +%% +%% This is no longer safe as of OTP 22, as a match context may be reused after +%% being passed to another function. +unsafe_get_binary_reuse(Config) when is_list(Config) -> + <<_First, Rest/binary>> = <<"hello">>, + ubgr_1(Rest), + <<Second,_/bits>> = Rest, + $e = Second, + ok. + +ubgr_1(<<_CP/utf8, Rest/binary>>) -> id(Rest); +ubgr_1(_) -> false. id(I) -> I. diff --git a/erts/emulator/test/dirty_bif_SUITE.erl b/erts/emulator/test/dirty_bif_SUITE.erl index 46eb0cba58..4f5ad0295a 100644 --- a/erts/emulator/test/dirty_bif_SUITE.erl +++ b/erts/emulator/test/dirty_bif_SUITE.erl @@ -38,7 +38,8 @@ dirty_process_info/1, dirty_process_register/1, dirty_process_trace/1, - code_purge/1]). + code_purge/1, + otp_15688/1]). suite() -> [{ct_hooks,[ts_install_cth]}]. @@ -64,7 +65,8 @@ all() -> dirty_process_info, dirty_process_register, dirty_process_trace, - code_purge]. + code_purge, + otp_15688]. init_per_suite(Config) -> case erlang:system_info(dirty_cpu_schedulers) of @@ -498,10 +500,58 @@ code_purge(Config) when is_list(Config) -> true = Time =< 1000, ok. +otp_15688(Config) when is_list(Config) -> + ImBack = make_ref(), + {See, SeeMon} = spawn_monitor(fun () -> + erts_debug:dirty_io(wait, 2000), + exit(ImBack) + end), + wait_until(fun () -> + [{current_function, {erts_debug, dirty_io, 2}}, + {status, running}] + == process_info(See, + [current_function, status]) + end), + {Ser1, Ser1Mon} = spawn_monitor(fun () -> + erlang:suspend_process(See, + [asynchronous]) + end), + erlang:suspend_process(See, [asynchronous]), + receive {'DOWN', Ser1Mon, process, Ser1, normal} -> ok end, + + %% Verify that we sent the suspend request while it was executing dirty... + [{current_function, {erts_debug, dirty_io, 2}}, + {status, running}] = process_info(See, [current_function, status]), + + wait_until(fun () -> + {status, suspended} == process_info(See, status) + end), + erlang:resume_process(See), + + receive + {'DOWN', SeeMon, process, See, Reason} -> + ImBack = Reason + after 4000 -> + %% Resume bug seems to have hit us... + PI = process_info(See), + exit(See, kill), + ct:fail({suspendee_stuck, PI}) + end. + + %% %% Internal... %% +wait_until(Fun) -> + case Fun() of + true -> + ok; + _ -> + receive after 100 -> ok end, + wait_until(Fun) + end. + access_dirty_process(Config, Start, Test, Finish) -> {ok, Node} = start_node(Config, ""), [ok] = mcall(Node, diff --git a/erts/emulator/test/distribution_SUITE.erl b/erts/emulator/test/distribution_SUITE.erl index 4f70b51aa0..449821e5ad 100644 --- a/erts/emulator/test/distribution_SUITE.erl +++ b/erts/emulator/test/distribution_SUITE.erl @@ -744,6 +744,7 @@ link_to_dead_new_node(Config) when is_list(Config) -> {'EXIT', Pid, noproc} -> ok; Other -> + stop_node(Node), ct:fail({unexpected_message, Other}) after 5000 -> ct:fail(nothing_received) @@ -1421,7 +1422,7 @@ message_latency_large_exit(Nodename, ReasonFun) -> FlushTrace = fun F() -> receive - {trace, Pid, _, _} = M -> + {trace, Pid, _, _} -> F() after 0 -> ok @@ -1455,8 +1456,14 @@ measure_latency_large_message(Nodename, DataFun) -> Echo = spawn(N, fun F() -> receive {From, Msg} -> From ! Msg, F() end end), - %% Test 32 MB and 320 MB and test the latency difference of sent messages - Payloads = [{I, <<0:(I * 32 * 1024 * 1024 * 8)>>} || I <- [1,10]], + case erlang:system_info(build_type) of + debug -> + %% Test 3.2 MB and 32 MB and test the latency difference of sent messages + Payloads = [{I, <<0:(I * 32 * 1024 * 8)>>} || I <- [1,10]]; + _ -> + %% Test 32 MB and 320 MB and test the latency difference of sent messages + Payloads = [{I, <<0:(I * 32 * 1024 * 1024 * 8)>>} || I <- [1,10]] + end, IndexTimes = [{I, measure_latency(DataFun, Dropper, Echo, P)} || {I, P} <- Payloads], @@ -1465,6 +1472,8 @@ measure_latency_large_message(Nodename, DataFun) -> ct:pal("~p",[IndexTimes]), + stop_node(N), + case {lists:max(Times), lists:min(Times)} of {Max, Min} when Max * 0.25 > Min -> ct:fail({incorrect_latency, IndexTimes}); @@ -1487,7 +1496,7 @@ measure_latency(DataFun, Dropper, Echo, Payload) -> end) || _ <- lists:seq(1,2)], [receive - {monitor, _Sender, busy_dist_port, _Info} = M -> + {monitor, _Sender, busy_dist_port, _Info} -> ok end || _ <- lists:seq(1,10)], @@ -1733,7 +1742,7 @@ bad_dist_fragments(Config) when is_list(Config) -> start_monitor(Offender,P), Exit2Victim = spawn(Victim, fun() -> receive ok -> ok end end), - send_bad_fragments(Offender, Victim, P,{?DOP_PAYLOAD_EXIT2,P,ExitVictim},2, + send_bad_fragments(Offender, Victim, P,{?DOP_PAYLOAD_EXIT2,P,Exit2Victim},2, [{hdr, 1, [132]}]), start_monitor(Offender,P), @@ -2428,17 +2437,22 @@ stop_node(Node) -> verify_nc(Node) -> P = self(), Ref = make_ref(), - spawn(Node, - fun() -> - R = erts_test_utils:check_node_dist(fun(E) -> E end), - P ! {Ref, R} - end), + Pid = spawn(Node, + fun() -> + R = erts_test_utils:check_node_dist(fun(E) -> E end), + P ! {Ref, R} + end), + MonRef = monitor(process, Pid), receive {Ref, ok} -> + demonitor(MonRef,[flush]), ok; {Ref, Error} -> - ct:log("~s",[Error]), - ct:fail(failed_nc_refc_check) + ct:log("~p",[Error]), + ct:fail(failed_nc_refc_check); + {'DOWN', MonRef, _, _, _} = Down -> + ct:log("~p",[Down]), + ct:fail(crashed_nc_refc_check) end. freeze_node(Node, MS) -> diff --git a/erts/emulator/test/dump_SUITE.erl b/erts/emulator/test/dump_SUITE.erl index d0237b78cc..3b860ebdf6 100644 --- a/erts/emulator/test/dump_SUITE.erl +++ b/erts/emulator/test/dump_SUITE.erl @@ -24,7 +24,7 @@ -export([all/0, suite/0, init_per_testcase/2, end_per_testcase/2]). --export([signal_abort/1]). +-export([signal_abort/1, exiting_dump/1, free_dump/1]). -export([load/0]). @@ -35,7 +35,7 @@ suite() -> {timetrap, {minutes, 2}}]. all() -> - [signal_abort]. + [signal_abort, exiting_dump, free_dump]. init_per_testcase(signal_abort, Config) -> SO = erlang:system_info(schedulers_online), @@ -48,7 +48,10 @@ init_per_testcase(signal_abort, Config) -> {skip, "the platform does not support scheduler dump"}; Dump -> Config - end. + end; +init_per_testcase(_, Config) -> + Config. + end_per_testcase(_, Config) -> Config. @@ -79,8 +82,6 @@ signal_abort(Config) -> {ok, Bin} = get_dump_when_done(Dump), - ct:log("~s",[Bin]), - {match, Matches} = re:run(Bin,"Current Process: <",[global]), ct:log("Found ~p",[Matches]), @@ -91,6 +92,85 @@ signal_abort(Config) -> ok. +load() -> + lists:seq(1,10000), + load(). + + +%% Test that crash dumping when a process is in the state EXITING works +exiting_dump(Config) when is_list(Config) -> + Dump = filename:join(proplists:get_value(priv_dir, Config),"signal_abort.dump"), + + {ok, Node} = start_node(Config), + + Self = self(), + + Pid = spawn_link(Node, + fun() -> + [begin + T = ets:new(hej,[]), + [ets:insert(T,{I,I}) || I <- lists:seq(1,1000)] + end || _ <- lists:seq(1,1000)], + Self ! ready, + receive ok -> ok end + end), + + true = rpc:call(Node, os, putenv, ["ERL_CRASH_DUMP",Dump]), + + receive ready -> unlink(Pid), Pid ! ok end, + + rpc:call(Node, erlang, halt, ["dump"]), + + {ok, Bin} = get_dump_when_done(Dump), + + {match, Matches} = re:run(Bin,"^State: Exiting", [global, multiline]), + + ct:log("Found ~p",[Matches]), + + true = length(Matches) == 1, + + file:delete(Dump), + + ok. + +%% Test that crash dumping when a process is in the state FREE works +free_dump(Config) when is_list(Config) -> + Dump = filename:join(proplists:get_value(priv_dir, Config),"signal_abort.dump"), + + {ok, Node} = start_node(Config), + + Self = self(), + + Pid = spawn_link(Node, + fun() -> + Self ! ready, + receive + ok -> + unlink(Self), + exit(lists:duplicate(1000,1000)) + end + end), + + true = rpc:call(Node, os, putenv, ["ERL_CRASH_DUMP",Dump]), + + [erlang:monitor(process, Pid) || _ <- lists:seq(1,10000)], + receive ready -> unlink(Pid), Pid ! ok end, + + rpc:call(Node, erlang, halt, ["dump"]), + + {ok, Bin} = get_dump_when_done(Dump), + + {match, Matches} = re:run(Bin,"^State: Non Existing", [global, multiline]), + + ct:log("Found ~p",[Matches]), + + true = length(Matches) == 1, + + file:delete(Dump), + + ok. + + get_dump_when_done(Dump) -> case file:read_file_info(Dump) of {ok, #file_info{ size = Sz }} -> @@ -104,15 +184,13 @@ get_dump_when_done(Dump, Sz) -> timer:sleep(1000), case file:read_file_info(Dump) of {ok, #file_info{ size = Sz }} -> - file:read_file(Dump); + {ok, Bin} = file:read_file(Dump), + ct:log("~s",[Bin]), + {ok, Bin}; {ok, #file_info{ size = NewSz }} -> get_dump_when_done(Dump, NewSz) end. -load() -> - lists:seq(1,10000), - load(). - start_node(Config) when is_list(Config) -> Pa = filename:dirname(code:which(?MODULE)), Name = list_to_atom(atom_to_list(?MODULE) diff --git a/erts/emulator/test/efile_SUITE.erl b/erts/emulator/test/efile_SUITE.erl index 7dcf302742..045b351e02 100644 --- a/erts/emulator/test/efile_SUITE.erl +++ b/erts/emulator/test/efile_SUITE.erl @@ -45,7 +45,12 @@ iter_max_files(Config) when is_list(Config) -> iter_max_files_1(Config) -> DataDir = proplists:get_value(data_dir,Config), TestFile = filename:join(DataDir, "existing_file"), - N = 10, + case erlang:system_info(debug_compiled) of + true -> + N = 5; + false -> + N = 10 + end, %% Run on a different node in order to make the test more stable. Dir = filename:dirname(code:which(?MODULE)), {ok,Node} = test_server:start_node(test_iter_max_files,slave, @@ -100,34 +105,27 @@ open_files(Name) -> %% a /proc directory), let's read some zero sized files 500 times each, while %% ensuring that response isn't empty << >> proc_zero_sized_files(Config) when is_list(Config) -> - {Type, Flavor} = os:type(), - %% Some files which exist on Linux but might be missing on other systems - Inputs = ["/proc/cpuinfo", - "/proc/meminfo", - "/proc/partitions", - "/proc/swaps", - "/proc/version", - "/proc/uptime", - %% curproc is present on freebsd - "/proc/curproc/cmdline"], - case filelib:is_dir("/proc") of - false -> {skip, "/proc not found"}; % skip the test if no /proc - _ when Type =:= unix andalso Flavor =:= sunos -> - %% SunOS has a /proc, but no zero sized special files - {skip, "sunos does not have any zero sized special files"}; - true -> - %% Take away files which do not exist in proc - Inputs1 = lists:filter(fun filelib:is_file/1, Inputs), - - %% Fail if none of mentioned files exist in /proc, did we just get - %% a normal /proc directory without any special files? - ?assertNotEqual([], Inputs1), - + TestFiles0 = [%% Some files which exist on Linux but might be missing on + %% other systems + "/proc/cpuinfo", + "/proc/meminfo", + "/proc/partitions", + "/proc/swaps", + "/proc/version", + "/proc/uptime", + %% curproc is present on FreeBSD + "/proc/curproc/cmdline"], + + TestFiles = [F || F <- TestFiles0, filelib:is_file(F)], + + case TestFiles of + [_|_] -> %% For 6 inputs and 500 attempts each this do run anywhere %% between 500 and 3000 function calls. - lists:foreach( - fun(Filename) -> do_proc_zero_sized(Filename, 500) end, - Inputs1) + [do_proc_zero_sized(F, 500) || F <- TestFiles], + ok; + [] -> + {skip, "Failed to find any known zero-sized files"} end. %% @doc Test one file N times to also trigger possible leaking fds and memory diff --git a/erts/emulator/test/erts_test_utils.erl b/erts/emulator/test/erts_test_utils.erl index 0c3ef3e0fc..e4e00a0a16 100644 --- a/erts/emulator/test/erts_test_utils.erl +++ b/erts/emulator/test/erts_test_utils.erl @@ -19,6 +19,7 @@ %% -module(erts_test_utils). +-compile(r16). %% %% THIS MODULE IS ALSO USED BY *OTHER* APPLICATIONS TEST CODE diff --git a/erts/emulator/test/fun_SUITE.erl b/erts/emulator/test/fun_SUITE.erl index 4042b58ff2..7f6caa08f1 100644 --- a/erts/emulator/test/fun_SUITE.erl +++ b/erts/emulator/test/fun_SUITE.erl @@ -592,7 +592,8 @@ refc_dist(Config) when is_list(Config) -> 3 = fun_refc(F2), true = erlang:garbage_collect(), 2 = fun_refc(F), - refc_dist_send(Node, F). + refc_dist_send(Node, F), + test_server:stop_node(Node). refc_dist_send(Node, F) -> Pid = spawn_link(Node, fun() -> receive @@ -682,6 +683,7 @@ t_arity(Config) when is_list(Config) -> 43 = spawn_call(Node, fun(X, Y) -> A+X+Y end), 1 = spawn_call(Node, fun(X, Y) -> X+Y end), 45 = spawn_call(Node, fun(X, Y, Z) -> A+X+Y+Z end), + test_server:stop_node(Node), ok. t_is_function2(Config) when is_list(Config) -> diff --git a/erts/emulator/test/map_SUITE.erl b/erts/emulator/test/map_SUITE.erl index d0a6763fe5..9ea59e1084 100644 --- a/erts/emulator/test/map_SUITE.erl +++ b/erts/emulator/test/map_SUITE.erl @@ -17,7 +17,7 @@ %% %CopyrightEnd% %% -module(map_SUITE). --export([all/0, suite/0]). +-export([all/0, suite/0, init_per_suite/1, end_per_suite/1]). -export([t_build_and_match_literals/1, t_build_and_match_literals_large/1, t_update_literals/1, t_update_literals_large/1, @@ -84,7 +84,10 @@ %% instruction-level tests t_has_map_fields/1, y_regs/1, - badmap_17/1]). + badmap_17/1, + + %%Bugs + t_large_unequal_bins_same_hash_bug/1]). -include_lib("stdlib/include/ms_transform.hrl"). @@ -149,7 +152,26 @@ all() -> [t_build_and_match_literals, t_build_and_match_literals_large, %% instruction-level tests t_has_map_fields, y_regs, - badmap_17]. + badmap_17, + + %% Bugs + t_large_unequal_bins_same_hash_bug]. + +init_per_suite(Config) -> + A0 = case application:start(sasl) of + ok -> [sasl]; + _ -> [] + end, + A = case application:start(os_mon) of + ok -> [os_mon|A0]; + _ -> A0 + end, + [{started_apps, A}|Config]. + +end_per_suite(Config) -> + As = proplists:get_value(started_apps, Config), + lists:foreach(fun (A) -> application:stop(A) end, As), + Config. %% tests @@ -3374,3 +3396,67 @@ fannerl() -> 104,2,97,9,97,16,70,63,184,100,97,32,0,0,0,104,2,97,10,97,16,70,63,169,174, 254,64,0,0,0,104,2,97,11,97,16,70,191,119,121,234,0,0,0,0,104,2,97,12,97, 16,70,63,149,12,170,128,0,0,0,104,2,97,13,97,16,70,191,144,193,191,0,0,0,0>>. + +%% This test case checks that the bug with ticket number OTP-15707 is +%% fixed. The bug could cause a crash or memory usage to grow until +%% the machine ran out of memory. +t_large_unequal_bins_same_hash_bug(Config) when is_list(Config) -> + run_when_enough_resources( + fun() -> + K1 = get_4GB_bin(1), + K2 = get_4GB_bin(2), + Map = make_map(500), + Map2 = maps:put(K1, 42, Map), + %% The map needed to contain at least 32 key-value pairs + %% at this point to get the crash or out of memory + %% problem on the next line + Map3 = maps:put(K2, 43, Map2), + %% The following line should avoid that the compiler + %% optimizes away the above + io:format("~p ~p~n", [erlang:phash2(Map3), maps:size(Map3)]) + end). + +make_map(0) -> + #{}; +make_map(Size) -> + maps:put(Size, Size, make_map(Size-1)). + +get_4GB_bin(Value) -> + List = lists:duplicate(65536, Value), + Bin = erlang:iolist_to_binary(List), + IOList4GB = duplicate_iolist(Bin, 16), + Bin4GB = erlang:iolist_to_binary(IOList4GB), + 4294967296 = size(Bin4GB), + Bin4GB. + +duplicate_iolist(IOList, 0) -> + IOList; +duplicate_iolist(IOList, NrOfTimes) -> + duplicate_iolist([IOList, IOList], NrOfTimes - 1). + +run_when_enough_resources(Fun) -> + case {total_memory(), erlang:system_info(wordsize)} of + {Mem, 8} when is_integer(Mem) andalso Mem >= 31 -> + Fun(); + {Mem, WordSize} -> + {skipped, + io_lib:format("Not enough resources (System Memory >= ~p, Word Size = ~p)", + [Mem, WordSize])} + end. + +total_memory() -> + %% Total memory in GB. + try + MemoryData = memsup:get_system_memory_data(), + case lists:keysearch(total_memory, 1, MemoryData) of + {value, {total_memory, TM}} -> + TM div (1024*1024*1024); + false -> + {value, {system_total_memory, STM}} = + lists:keysearch(system_total_memory, 1, MemoryData), + STM div (1024*1024*1024) + end + catch + _ : _ -> + undefined + end. diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl index 2309f844b9..b824daea67 100644 --- a/erts/emulator/test/nif_SUITE.erl +++ b/erts/emulator/test/nif_SUITE.erl @@ -28,6 +28,7 @@ -include_lib("stdlib/include/assert.hrl"). -export([all/0, suite/0, groups/0, + init_per_suite/1, end_per_suite/1, init_per_group/2, end_per_group/2, init_per_testcase/2, end_per_testcase/2, basic/1, reload_error/1, upgrade/1, heap_frag/1, @@ -109,6 +110,14 @@ all() -> pid, nif_term_type]. +init_per_suite(Config) -> + erts_debug:set_internal_state(available_internal_state, true), + Config. + +end_per_suite(_Config) -> + catch erts_debug:set_internal_state(available_internal_state, false), + ok. + groups() -> [{G, [], api_repeaters()} || G <- api_groups()] ++ @@ -118,7 +127,6 @@ groups() -> monitor_process_d, demonitor_process]}]. - api_groups() -> [api_latest, api_2_4, api_2_0]. api_repeaters() -> [upgrade, resource_takeover, t_on_load]. @@ -1223,7 +1231,7 @@ maps(Config) when is_list(Config) -> repeat_while(fun({35,_}) -> false; ({K,Map}) -> Map = maps_from_list_nif(maps:to_list(Map)), - Map = maps:filter(fun(K,V) -> V =:= K*100 end, Map), + Map = maps:filter(fun(K2,V) -> V =:= K2*100 end, Map), {K+1, maps:put(K,K*100,Map)} end, {1,#{}}), @@ -1294,24 +1302,29 @@ resource_hugo_do(Type) -> release_resource(HugoPtr), erlang:garbage_collect(), {HugoPtr,HugoBin} = get_resource(Type,Hugo), - Pid = spawn_link(fun() -> - receive {Pid, Type, Resource, Ptr, Bin} -> - Pid ! {self(), got_it}, - receive {Pid, check_it} -> - {Ptr,Bin} = get_resource(Type,Resource), - Pid ! {self(), ok} - end - end - end), + {Pid,_} = + spawn_monitor(fun() -> + receive {Pid, Type, Resource, Ptr, Bin} -> + Pid ! {self(), got_it}, + receive {Pid, check_it} -> + {Ptr,Bin} = get_resource(Type,Resource) + end + end, + gc_and_exit(ok) + end), Pid ! {self(), Type, Hugo, HugoPtr, HugoBin}, {Pid, got_it} = receive_any(), erlang:garbage_collect(), % just to make our ProcBin move in memory Pid ! {self(), check_it}, - {Pid, ok} = receive_any(), + {'DOWN', _, process, Pid, ok} = receive_any(), [] = last_resource_dtor_call(), {HugoPtr,HugoBin} = get_resource(Type,Hugo), {HugoPtr, HugoBin, 1}. +gc_and_exit(Reason) -> + erlang:garbage_collect(), + exit(Reason). + resource_otto(Type) -> {OttoPtr, OttoBin} = resource_otto_do(Type), erlang:garbage_collect(), @@ -1388,14 +1401,14 @@ resource_binary_do() -> ResInfo = {Ptr,_} = get_resource(binary_resource_type,ResBin1), Papa = self(), - Forwarder = spawn_link(fun() -> forwarder(Papa) end), + {Forwarder,_} = spawn_monitor(fun() -> forwarder(Papa) end), io:format("sending to forwarder pid=~p\n",[Forwarder]), Forwarder ! ResBin1, ResBin2 = receive_any(), ResBin2 = ResBin1, ResInfo = get_resource(binary_resource_type,ResBin2), Forwarder ! terminate, - {Forwarder, 1} = receive_any(), + {'DOWN', _, process, Forwarder, 1} = receive_any(), erlang:garbage_collect(), ResInfo = get_resource(binary_resource_type,ResBin1), ResInfo = get_resource(binary_resource_type,ResBin2), @@ -1755,6 +1768,7 @@ read_resource(Type, {Holder,Id}) -> forget_resource({Holder,Id}) -> Holder ! {self(), forget, Id}, {Holder, forget_ok, Id} = receive_any(), + erts_debug:set_internal_state(wait, aux_work), ok. @@ -1915,11 +1929,11 @@ send2_do1(SendBlobF) -> send2_do2(SendBlobF, self()), Papa = self(), - Forwarder = spawn_link(fun() -> forwarder(Papa) end), + {Forwarder,_} = spawn_monitor(fun() -> forwarder(Papa) end), io:format("sending to forwarder pid=~p\n",[Forwarder]), send2_do2(SendBlobF, Forwarder), Forwarder ! terminate, - {Forwarder, 4} = receive_any(), + {'DOWN', _, process, Forwarder, 4} = receive_any(), ok. send2_do2(SendBlobF, To) -> @@ -1975,7 +1989,7 @@ forwarder(To) -> forwarder(To, N) -> case receive_any() of terminate -> - To ! {self(), N}; + gc_and_exit(N); Msg -> To ! Msg, forwarder(To, N+1) @@ -3100,22 +3114,31 @@ nif_whereis_threaded(Config) when is_list(Config) -> RegName = nif_whereis_test_threaded, undefined = erlang:whereis(RegName), - Ref = make_ref(), - {Pid, Mon} = spawn_monitor(?MODULE, nif_whereis_proxy, [Ref]), - true = register(RegName, Pid), + Self = self(), + true = register(RegName, Self), - {ok, ProcThr} = whereis_thd_lookup(pid, RegName), - {ok, Pid} = whereis_thd_result(ProcThr), + {ok, ProcThr} = whereis_thd_lookup(pid, RegName, "dtor to proc"), + {ok, Self} = whereis_thd_result(ProcThr), - Pid ! {Ref, quit}, - ok = receive {'DOWN', Mon, process, Pid, normal} -> ok end, + nif_whereis_threaded_2(RegName). + +nif_whereis_threaded_2(RegName) -> + erlang:garbage_collect(), + "dtor to proc" = receive_any(1000), + true = unregister(RegName), Port = open_port({spawn, echo_drv}, [eof]), true = register(RegName, Port), - {ok, PortThr} = whereis_thd_lookup(port, RegName), + {ok, PortThr} = whereis_thd_lookup(port, RegName, "dtor to port"), {ok, Port} = whereis_thd_result(PortThr), + nif_whereis_threaded_3(Port). + +nif_whereis_threaded_3(Port) -> + erlang:garbage_collect(), + {Port, {data, "dtor to port"}} = receive_any(1000), + port_close(Port), ok. @@ -3430,6 +3453,10 @@ nif_term_type(Config) -> ok. +last_resource_dtor_call() -> + erts_debug:set_internal_state(wait, aux_work), + last_resource_dtor_call_nif(). + id(I) -> I. %% The NIFs: @@ -3457,7 +3484,7 @@ make_resource(_) -> ?nif_stub. get_resource(_,_) -> ?nif_stub. release_resource(_) -> ?nif_stub. release_resource_from_thread(_) -> ?nif_stub. -last_resource_dtor_call() -> ?nif_stub. +last_resource_dtor_call_nif() -> ?nif_stub. make_new_resource(_,_) -> ?nif_stub. check_is(_,_,_,_,_,_,_,_,_,_,_) -> ?nif_stub. check_is_exception() -> ?nif_stub. @@ -3517,7 +3544,7 @@ ioq_nif(_,_,_,_) -> ?nif_stub. %% whereis whereis_send(_Type,_Name,_Msg) -> ?nif_stub. whereis_term(_Type,_Name) -> ?nif_stub. -whereis_thd_lookup(_Type,_Name) -> ?nif_stub. +whereis_thd_lookup(_Type,_Name, _Msg) -> ?nif_stub. whereis_thd_result(_Thd) -> ?nif_stub. %% maps diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c index 1906384af4..ff47cfe500 100644 --- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c +++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c @@ -27,6 +27,7 @@ #ifndef __WIN32__ #include <unistd.h> #include <fcntl.h> +#include <sys/uio.h> #endif #include "nif_mod.h" @@ -707,28 +708,23 @@ static ERL_NIF_TERM tuple_2_list(ErlNifEnv* env, int argc, const ERL_NIF_TERM ar static ERL_NIF_TERM is_identical(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - if (argc != 2) { - return enif_make_badarg(env); - } + assert(argc == 2); return enif_make_atom(env, (enif_is_identical(argv[0],argv[1]) ? "true" : "false")); } static ERL_NIF_TERM compare(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - if (argc != 2) { - return enif_make_badarg(env); - } + assert(argc == 2); return enif_make_int(env, enif_compare(argv[0],argv[1])); } static ERL_NIF_TERM hash_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - if (argc != 3) { - return enif_make_badarg(env); - } - ErlNifHash type; + ErlNifUInt64 salt; + + assert(argc == 3); if (enif_is_identical(argv[0], enif_make_atom(env, "internal"))) { type = ERL_NIF_INTERNAL_HASH; } @@ -739,7 +735,6 @@ static ERL_NIF_TERM hash_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[] return enif_make_badarg(env); } - ErlNifUInt64 salt; if (! enif_get_uint64(env, argv[2], &salt)) { return enif_make_badarg(env); } @@ -866,7 +861,7 @@ static ERL_NIF_TERM iolist_2_bin(ErlNifEnv* env, int argc, const ERL_NIF_TERM ar return enif_make_binary(env,&obin); } -static ERL_NIF_TERM last_resource_dtor_call(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +static ERL_NIF_TERM last_resource_dtor_call_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { ERL_NIF_TERM ret; if (resource_dtor_last != NULL) { @@ -1010,6 +1005,7 @@ static ERL_NIF_TERM release_resource(ErlNifEnv* env, int argc, const ERL_NIF_TER static void* threaded_release_resource(void* resource) { enif_release_resource(resource); + return NULL; } static ERL_NIF_TERM release_resource_from_thread(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) @@ -1201,7 +1197,6 @@ static void fill(void* dst, unsigned bytes, int seed) enum { /* results */ WHEREIS_SUCCESS, - WHEREIS_ERROR_TYPE, WHEREIS_ERROR_LOOKUP, WHEREIS_ERROR_SEND, /* types */ @@ -1221,6 +1216,8 @@ typedef struct { whereis_term_data_t res; ErlNifTid tid; int type; + int rc; + ERL_NIF_TERM dtor_msg; } whereis_thread_resource_t; static whereis_thread_resource_t* whereis_thread_resource_create(void) @@ -1233,21 +1230,35 @@ static whereis_thread_resource_t* whereis_thread_resource_create(void) return rp; } +static int whereis_lookup_internal(ErlNifEnv*, int type, ERL_NIF_TERM name, + whereis_term_data_t* out); +static int whereis_send_internal(ErlNifEnv*, int type, whereis_term_data_t* to, + ERL_NIF_TERM msg); + + static void whereis_thread_resource_dtor(ErlNifEnv* env, void* obj) { whereis_thread_resource_t* rp = (whereis_thread_resource_t*) obj; + whereis_term_data_t to; + + if (whereis_lookup_internal(env, rp->type, rp->name, &to) + == WHEREIS_SUCCESS) { + whereis_send_internal(env, rp->type, &to, rp->dtor_msg); + } enif_free_env(rp->env); } -static int whereis_type(ERL_NIF_TERM type) +static int whereis_type(ERL_NIF_TERM type_term, int* type_p) { - if (enif_is_identical(type, atom_pid)) - return WHEREIS_LOOKUP_PID; - - if (enif_is_identical(type, atom_port)) - return WHEREIS_LOOKUP_PORT; - - return WHEREIS_ERROR_TYPE; + if (enif_is_identical(type_term, atom_pid)) { + *type_p = WHEREIS_LOOKUP_PID; + return 1; + } + if (enif_is_identical(type_term, atom_port)) { + *type_p = WHEREIS_LOOKUP_PORT; + return 1; + } + return 0; } static int whereis_lookup_internal( @@ -1261,7 +1272,7 @@ static int whereis_lookup_internal( return enif_whereis_port(env, name, & out->port) ? WHEREIS_SUCCESS : WHEREIS_ERROR_LOOKUP; - return WHEREIS_ERROR_TYPE; + abort(); } static int whereis_send_internal( @@ -1275,23 +1286,20 @@ static int whereis_send_internal( return enif_port_command(env, & to->port, NULL, msg) ? WHEREIS_SUCCESS : WHEREIS_ERROR_SEND; - return WHEREIS_ERROR_TYPE; + abort(); } -static int whereis_resolved_term( - ErlNifEnv* env, int type, whereis_term_data_t* res, ERL_NIF_TERM* out) +static ERL_NIF_TERM whereis_resolved_term( + ErlNifEnv* env, int type, whereis_term_data_t* res) { switch (type) { case WHEREIS_LOOKUP_PID: - *out = enif_make_pid(env, & res->pid); - break; + return enif_make_pid(env, &res->pid); case WHEREIS_LOOKUP_PORT: - *out = enif_make_port(env, & res->port); - break; + return enif_make_port(env, &res->port); default: - return WHEREIS_ERROR_TYPE; + abort(); } - return WHEREIS_SUCCESS; } static ERL_NIF_TERM whereis_result_term(ErlNifEnv* env, int result) @@ -1307,9 +1315,6 @@ static ERL_NIF_TERM whereis_result_term(ErlNifEnv* env, int result) case WHEREIS_ERROR_SEND: err = atom_send; break; - case WHEREIS_ERROR_TYPE: - err = atom_badarg; - break; default: err = enif_make_int(env, -result); break; @@ -1320,14 +1325,10 @@ static ERL_NIF_TERM whereis_result_term(ErlNifEnv* env, int result) static void* whereis_lookup_thread(void* arg) { whereis_thread_resource_t* rp = (whereis_thread_resource_t*) arg; - int rc; - /* enif_whereis_xxx should work with allocated or null env */ - rc = whereis_lookup_internal( - ((rp->type == WHEREIS_LOOKUP_PID) ? NULL : rp->env), - rp->type, rp->name, & rp->res); + rp->rc = whereis_lookup_internal(NULL, rp->type, rp->name, &rp->res); - return (((char*) NULL) + rc); + return NULL; } /* whereis_term(Type, Name) -> pid() | port() | false */ @@ -1335,20 +1336,16 @@ static ERL_NIF_TERM whereis_term(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { whereis_term_data_t res; - ERL_NIF_TERM ret; int type, rc; - if (argc != 2) /* allow non-atom name for testing */ - return enif_make_badarg(env); - - if ((type = whereis_type(argv[0])) == WHEREIS_ERROR_TYPE) + assert(argc == 2); + if (!whereis_type(argv[0], &type)) return enif_make_badarg(env); rc = whereis_lookup_internal(env, type, argv[1], & res); - if (rc == WHEREIS_SUCCESS) { - rc = whereis_resolved_term(env, type, & res, & ret); - } - return (rc == WHEREIS_SUCCESS) ? ret : atom_false; + return (rc == WHEREIS_SUCCESS ? + whereis_resolved_term(env, type, &res) : + atom_false); } /* whereis_send(Type, Name, Message) -> ok | {error, Reason} */ @@ -1358,10 +1355,11 @@ whereis_send(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) whereis_term_data_t to; int type, rc; - if (argc != 3 || !enif_is_atom(env, argv[1])) + assert(argc == 3); + if (!enif_is_atom(env, argv[1])) return enif_make_badarg(env); - if ((type = whereis_type(argv[0])) == WHEREIS_ERROR_TYPE) + if (!whereis_type(argv[0], &type)) return enif_make_badarg(env); rc = whereis_lookup_internal(env, type, argv[1], & to); @@ -1371,33 +1369,35 @@ whereis_send(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) return whereis_result_term(env, rc); } -/* whereis_thd_lookup(Type, Name) -> {ok, Resource} | {error, SysErrno} */ +/* whereis_thd_lookup(Type, Name, DtorMsg) -> {ok, Resource} | {error, SysErrno} */ static ERL_NIF_TERM whereis_thd_lookup(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { whereis_thread_resource_t* rp; int type, rc; + ERL_NIF_TERM ret; - if (argc != 2 || !enif_is_atom(env, argv[1])) + assert(argc == 3); + if (!enif_is_atom(env, argv[1])) return enif_make_badarg(env); - if ((type = whereis_type(argv[0])) == WHEREIS_ERROR_TYPE) + if (!whereis_type(argv[0], &type)) return enif_make_badarg(env); rp = whereis_thread_resource_create(); rp->type = type; rp->name = enif_make_copy(rp->env, argv[1]); + rp->dtor_msg = enif_make_copy(rp->env, argv[2]); rc = enif_thread_create( "nif_SUITE:whereis_thd", & rp->tid, whereis_lookup_thread, rp, NULL); - if (rc == 0) { - return enif_make_tuple2(env, atom_ok, enif_make_resource(env, rp)); - } - else { - enif_release_resource(rp); - return enif_make_tuple2(env, atom_error, enif_make_int(env, rc)); - } + if (rc == 0) + ret = enif_make_tuple2(env, atom_ok, enif_make_resource(env, rp)); + else + ret = enif_make_tuple2(env, atom_error, enif_make_int(env, rc)); + enif_release_resource(rp); + return ret; } /* whereis_thd_result(Resource) -> {ok, pid() | port()} | {error, ErrNum} */ @@ -1406,24 +1406,22 @@ whereis_thd_result(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { whereis_thread_resource_t* rp; ERL_NIF_TERM ret; - char* thdret; /* so we can keep compilers happy converting to int */ - int rc; + int join_rc; - if (argc != 1 - || !enif_get_resource(env, argv[0], whereis_resource_type, (void**) & rp)) + assert(argc == 1); + if (!enif_get_resource(env, argv[0], whereis_resource_type, (void**) & rp)) return enif_make_badarg(env); - if ((rc = enif_thread_join(rp->tid, (void**) & thdret)) != 0) - return enif_make_tuple2(env, atom_error, enif_make_int(env, rc)); + if ((join_rc = enif_thread_join(rp->tid, NULL)) != 0) + return enif_make_tuple2(env, atom_error, enif_make_int(env, join_rc)); - rc = (int)(thdret - ((char*) NULL)); - if (rc == WHEREIS_SUCCESS) { - rc = whereis_resolved_term(env, rp->type, & rp->res, & ret); + if (rp->rc == WHEREIS_SUCCESS) { + ret = enif_make_tuple2(env, atom_ok, + whereis_resolved_term(env, rp->type, &rp->res)); } - ret = (rc == WHEREIS_SUCCESS) - ? enif_make_tuple2(env, atom_ok, ret) : whereis_result_term(env, rc); + else + ret = whereis_result_term(env, rp->rc); - enif_release_resource(rp); return ret; } @@ -2007,8 +2005,7 @@ static ERL_NIF_TERM nif_sched1(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv static ERL_NIF_TERM call_nif_schedule(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { ERL_NIF_TERM result; - if (argc != 2) - return enif_make_atom(env, "false"); + assert(argc == 2); result = enif_schedule_nif(env, "nif_sched1", 0, nif_sched1, argc, argv); assert(!enif_is_exception(env, result)); return result; @@ -2155,7 +2152,8 @@ static ERL_NIF_TERM maps_from_list_nif(ErlNifEnv* env, int argc, const ERL_NIF_T ERL_NIF_TERM result, cell; unsigned count; - if (argc != 1 || !enif_get_list_length(env, argv[0], &count)) { + assert(argc == 1); + if (!enif_get_list_length(env, argv[0], &count)) { return enif_make_badarg(env); } @@ -2202,7 +2200,8 @@ static ERL_NIF_TERM sorted_list_from_maps_nif(ErlNifEnv* env, int argc, const ER ErlNifMapIterator iter_b; int cnt, next_ret, prev_ret; - if (argc != 1 && !enif_is_map(env, map)) + assert(argc == 1); + if (!enif_is_map(env, map)) return enif_make_int(env, __LINE__); if(!enif_map_iterator_create(env, map, &iter_f, ERL_NIF_MAP_ITERATOR_FIRST)) @@ -2263,9 +2262,7 @@ static ERL_NIF_TERM monotonic_time(ErlNifEnv* env, int argc, const ERL_NIF_TERM { ErlNifTimeUnit time_unit; - if (argc != 1) - return atom_false; - + assert(argc == 1); if (enif_compare(argv[0], atom_second) == 0) time_unit = ERL_NIF_SEC; else if (enif_compare(argv[0], atom_millisecond) == 0) @@ -2284,9 +2281,7 @@ static ERL_NIF_TERM time_offset(ErlNifEnv* env, int argc, const ERL_NIF_TERM arg { ErlNifTimeUnit time_unit; - if (argc != 1) - return atom_false; - + assert(argc == 1); if (enif_compare(argv[0], atom_second) == 0) time_unit = ERL_NIF_SEC; else if (enif_compare(argv[0], atom_millisecond) == 0) @@ -2306,9 +2301,7 @@ static ERL_NIF_TERM convert_time_unit(ErlNifEnv* env, int argc, const ERL_NIF_TE ErlNifTime val; ErlNifTimeUnit from, to; - if (argc != 3) - return atom_false; - + assert(argc == 3); if (!enif_get_int64(env, argv[0], &i64)) return enif_make_badarg(env); @@ -3319,7 +3312,7 @@ static int writeiovec(ErlNifEnv *env, ERL_NIF_TERM term, ERL_NIF_TERM *tail, Erl } /* Attempt to write the data */ - n = writev(fd, sysiovec, iovcnt); + n = writev(fd, (struct iovec*)sysiovec, iovcnt); saved_errno = errno; if (enif_ioq_size(q) == 0) { @@ -3393,10 +3386,11 @@ static ERL_NIF_TERM ioq(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) if (enif_is_identical(argv[0], enif_make_atom(env, "example"))) { #ifndef __WIN32__ - int fd[2], res = 0, cnt = 0, queue_cnt; + int fd[2], res = 0, cnt = 0; ERL_NIF_TERM tail; char buff[255]; - pipe(fd); + res = pipe(fd); + assert(res == 0); fcntl(fd[0], F_SETFL, fcntl(fd[0], F_GETFL) | O_NONBLOCK); fcntl(fd[1], F_SETFL, fcntl(fd[1], F_GETFL) | O_NONBLOCK); @@ -3639,7 +3633,7 @@ static ErlNifFunc nif_funcs[] = {"get_resource", 2, get_resource}, {"release_resource", 1, release_resource}, {"release_resource_from_thread", 1, release_resource_from_thread}, - {"last_resource_dtor_call", 0, last_resource_dtor_call}, + {"last_resource_dtor_call_nif", 0, last_resource_dtor_call_nif}, {"make_new_resource", 2, make_new_resource}, {"check_is", 11, check_is}, {"check_is_exception", 0, check_is_exception}, @@ -3710,7 +3704,7 @@ static ErlNifFunc nif_funcs[] = {"monitor_frenzy_nif", 4, monitor_frenzy_nif}, {"whereis_send", 3, whereis_send}, {"whereis_term", 2, whereis_term}, - {"whereis_thd_lookup", 2, whereis_thd_lookup}, + {"whereis_thd_lookup", 3, whereis_thd_lookup}, {"whereis_thd_result", 1, whereis_thd_result}, {"ioq_nif", 1, ioq}, {"ioq_nif", 2, ioq}, diff --git a/erts/emulator/test/process_SUITE.erl b/erts/emulator/test/process_SUITE.erl index edf08ce0bd..b530ced566 100644 --- a/erts/emulator/test/process_SUITE.erl +++ b/erts/emulator/test/process_SUITE.erl @@ -44,6 +44,7 @@ process_info_garbage_collection/1, process_info_smoke_all/1, process_info_status_handled_signal/1, + process_info_reductions/1, bump_reductions/1, low_prio/1, binary_owner/1, yield/1, yield2/1, otp_4725/1, bad_register/1, garbage_collect/1, otp_6237/1, process_info_messages/1, process_flag_badarg/1, process_flag_heap_size/1, @@ -84,6 +85,7 @@ all() -> process_info_garbage_collection, process_info_smoke_all, process_info_status_handled_signal, + process_info_reductions, bump_reductions, low_prio, yield, yield2, otp_4725, bad_register, garbage_collect, process_info_messages, process_flag_badarg, process_flag_heap_size, @@ -133,6 +135,15 @@ init_per_group(_GroupName, Config) -> end_per_group(_GroupName, Config) -> Config. +init_per_testcase(Func, Config) + when Func =:= processes_default_tab; + Func =:= processes_this_tab -> + case erlang:system_info(debug_compiled) of + true -> + {skip, "Don't run in debug"}; + false -> + [{testcase, Func} | Config] + end; init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> [{testcase, Func}|Config]. @@ -1084,6 +1095,46 @@ process_info_status_handled_signal(Config) when is_list(Config) -> false = erlang:is_process_alive(P), ok. +%% OTP-15709 +%% Provoke a bug where process_info(reductions) returned wrong result +%% because REDS_IN (def_arg_reg[5]) is read when the process in not running. +process_info_reductions(Config) when is_list(Config) -> + pi_reductions_tester(spawn_link(fun() -> pi_reductions_spinnloop() end)), + pi_reductions_tester(spawn_link(fun() -> pi_reductions_recvloop() end)), + ok. + +pi_reductions_tester(Pid) -> + {_, DiffList} = + lists:foldl(fun(_, {Prev, Acc}) -> + %% Add another item that force sending the request + %% as a signal, like 'current_function'. + PI = process_info(Pid, [reductions, current_function]), + [{reductions,Reds}, {current_function,_}] = PI, + Diff = Reds - Prev, + {Diff, true} = {Diff, (Diff >= 0)}, + {Diff, true} = {Diff, (Diff =< 1000*1000)}, + {Reds, [Diff | Acc]} + end, + {0, []}, + lists:seq(1,10)), + unlink(Pid), + exit(Pid,kill), + io:format("Reduction diffs: ~p\n", [DiffList]), + ok. + +pi_reductions_spinnloop() -> + %% 6 args to make use of def_arg_reg[5] which is also used as REDS_IN + pi_reductions_spinnloop(1, atom, "hej", self(), make_ref(), 3.14). + +pi_reductions_spinnloop(A,B,C,D,E,F) -> + pi_reductions_spinnloop(B,C,D,E,F,A). + +pi_reductions_recvloop() -> + receive + "a free lunch" -> false + end. + + %% Tests erlang:bump_reductions/1. bump_reductions(Config) when is_list(Config) -> erlang:garbage_collect(), diff --git a/erts/emulator/test/socket_SUITE.erl b/erts/emulator/test/socket_SUITE.erl index 8a32efcd85..2e3f40a350 100644 --- a/erts/emulator/test/socket_SUITE.erl +++ b/erts/emulator/test/socket_SUITE.erl @@ -3404,7 +3404,9 @@ api_to_connect_tcp4(suite) -> api_to_connect_tcp4(doc) -> []; api_to_connect_tcp4(_Config) when is_list(_Config) -> + Cond = fun() -> api_to_connect_cond() end, tc_try(api_to_connect_tcp4, + Cond, fun() -> ?TT(?SECS(10)), InitState = #{domain => inet, @@ -3414,6 +3416,41 @@ api_to_connect_tcp4(_Config) when is_list(_Config) -> ok = api_to_connect_tcp(InitState) end). +api_to_connect_cond() -> + api_to_connect_cond(os:type(), os:version()). + +%% I don't know exactly at which version this starts to work. +%% I know it does not work for 4.4.*, but is does for 4.15. +%% So, just to simplify, we require atleast 4.15 +api_to_connect_cond({unix, linux}, {Maj, Min, _Rev}) -> + if + ((Maj >= 4) andalso (Min >= 15)) -> + ok; + true -> + skip("TC does not work") + end; +%% Only test on one machine, which has version 6.3, and there it does +%% not work, so disable for all. +api_to_connect_cond({unix, openbsd}, _) -> + skip("TC does not work"); +api_to_connect_cond({unix, freebsd}, {Maj, Min, _Rev}) -> + if + ((Maj >= 10) andalso (Min >= 4)) -> + ok; + true -> + skip("TC may not work") + end; +api_to_connect_cond({unix, sunos}, {Maj, Min, _Rev}) -> + if + ((Maj >= 5) andalso (Min >= 10)) -> + ok; + true -> + skip("TC may not work") + end; +api_to_connect_cond(_, _) -> + skip("TC may not work"). + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -3425,8 +3462,8 @@ api_to_connect_tcp6(doc) -> []; api_to_connect_tcp6(_Config) when is_list(_Config) -> tc_try(api_to_connect_tcp6, + fun() -> has_support_ipv6(), api_to_connect_cond() end, fun() -> - not_yet_implemented(), ?TT(?SECS(10)), InitState = #{domain => inet6, backlog => 1, @@ -3937,8 +3974,8 @@ api_to_accept_tcp6(doc) -> []; api_to_accept_tcp6(_Config) when is_list(_Config) -> tc_try(api_to_accept_tcp4, + fun() -> has_support_ipv6() end, fun() -> - not_yet_implemented(), ?TT(?SECS(10)), InitState = #{domain => inet6, timeout => 5000}, ok = api_to_accept_tcp(InitState) @@ -4053,8 +4090,8 @@ api_to_maccept_tcp6(doc) -> api_to_maccept_tcp6(_Config) when is_list(_Config) -> ?TT(?SECS(20)), tc_try(api_to_maccept_tcp4, + fun() -> has_support_ipv6() end, fun() -> - not_yet_implemented(), InitState = #{domain => inet6, timeout => 5000}, ok = api_to_maccept_tcp(InitState) end). @@ -4556,8 +4593,8 @@ api_to_recv_tcp6(doc) -> []; api_to_recv_tcp6(_Config) when is_list(_Config) -> tc_try(api_to_recv_tcp6, + fun() -> has_support_ipv6() end, fun() -> - not_yet_implemented(), case socket:supports(ipv6) of true -> ?TT(?SECS(10)), @@ -4915,8 +4952,8 @@ api_to_recvfrom_udp6(doc) -> []; api_to_recvfrom_udp6(_Config) when is_list(_Config) -> tc_try(api_to_recvfrom_udp6, + fun() -> has_support_ipv6() end, fun() -> - not_yet_implemented(), ?TT(?SECS(10)), Recv = fun(Sock, To) -> socket:recvfrom(Sock, 0, To) end, InitState = #{domain => inet6, @@ -5031,8 +5068,8 @@ api_to_recvmsg_udp6(doc) -> []; api_to_recvmsg_udp6(_Config) when is_list(_Config) -> tc_try(api_to_recvmsg_udp6, + fun() -> has_support_ipv6() end, fun() -> - not_yet_implemented(), ?TT(?SECS(10)), Recv = fun(Sock, To) -> socket:recvmsg(Sock, To) end, InitState = #{domain => inet6, @@ -5072,8 +5109,8 @@ api_to_recvmsg_tcp6(doc) -> []; api_to_recvmsg_tcp6(_Config) when is_list(_Config) -> tc_try(api_to_recvmsg_tcp6, + fun() -> has_support_ipv6() end, fun() -> - not_yet_implemented(), ?TT(?SECS(10)), Recv = fun(Sock, To) -> socket:recvmsg(Sock, To) end, InitState = #{domain => inet6, @@ -5104,7 +5141,6 @@ sc_cpe_socket_cleanup_tcp4(doc) -> sc_cpe_socket_cleanup_tcp4(_Config) when is_list(_Config) -> tc_try(sc_cpe_socket_cleanup_tcp4, fun() -> - %% not_yet_implemented(), ?TT(?SECS(5)), InitState = #{domain => inet, type => stream, @@ -5124,8 +5160,8 @@ sc_cpe_socket_cleanup_tcp6(doc) -> []; sc_cpe_socket_cleanup_tcp6(_Config) when is_list(_Config) -> tc_try(sc_cpe_socket_cleanup_tcp6, + fun() -> has_support_ipv6() end, fun() -> - not_yet_implemented(), ?TT(?SECS(5)), InitState = #{domain => inet6, type => stream, @@ -5166,8 +5202,8 @@ sc_cpe_socket_cleanup_udp6(doc) -> []; sc_cpe_socket_cleanup_udp6(_Config) when is_list(_Config) -> tc_try(sc_cpe_socket_cleanup_udp6, + fun() -> has_support_ipv6() end, fun() -> - not_yet_implemented(), ?TT(?SECS(5)), InitState = #{domain => inet6, type => dgram, @@ -5342,8 +5378,8 @@ sc_lc_recv_response_tcp6(doc) -> []; sc_lc_recv_response_tcp6(_Config) when is_list(_Config) -> tc_try(sc_lc_recv_response_tcp6, + fun() -> has_support_ipv6() end, fun() -> - not_yet_implemented(), ?TT(?SECS(10)), Recv = fun(Sock) -> socket:recv(Sock) end, InitState = #{domain => inet6, @@ -5958,8 +5994,8 @@ sc_lc_recvfrom_response_udp6(doc) -> []; sc_lc_recvfrom_response_udp6(_Config) when is_list(_Config) -> tc_try(sc_lc_recvfrom_response_udp6, + fun() -> has_support_ipv6() end, fun() -> - not_yet_implemented(), ?TT(?SECS(30)), Recv = fun(Sock, To) -> socket:recvfrom(Sock, [], To) end, InitState = #{domain => inet6, @@ -6378,8 +6414,8 @@ sc_lc_recvmsg_response_tcp6(doc) -> []; sc_lc_recvmsg_response_tcp6(_Config) when is_list(_Config) -> tc_try(sc_recvmsg_response_tcp6, + fun() -> has_support_ipv6() end, fun() -> - not_yet_implemented(), ?TT(?SECS(10)), Recv = fun(Sock) -> socket:recvmsg(Sock) end, InitState = #{domain => inet6, @@ -6421,8 +6457,8 @@ sc_lc_recvmsg_response_udp6(doc) -> []; sc_lc_recvmsg_response_udp6(_Config) when is_list(_Config) -> tc_try(sc_recvmsg_response_udp6, + fun() -> has_support_ipv6() end, fun() -> - not_yet_implemented(), ?TT(?SECS(10)), Recv = fun(Sock, To) -> socket:recvmsg(Sock, To) end, InitState = #{domain => inet6, @@ -6467,8 +6503,8 @@ sc_lc_acceptor_response_tcp6(doc) -> []; sc_lc_acceptor_response_tcp6(_Config) when is_list(_Config) -> tc_try(sc_lc_acceptor_response_tcp6, + fun() -> has_support_ipv6() end, fun() -> - not_yet_implemented(), ?TT(?SECS(10)), InitState = #{domain => inet, type => stream, @@ -6902,8 +6938,8 @@ sc_rc_recv_response_tcp6(doc) -> []; sc_rc_recv_response_tcp6(_Config) when is_list(_Config) -> tc_try(sc_rc_recv_response_tcp6, + fun() -> has_support_ipv6() end, fun() -> - not_yet_implemented(), ?TT(?SECS(10)), Recv = fun(Sock) -> socket:recv(Sock) end, InitState = #{domain => inet6, @@ -7783,8 +7819,8 @@ sc_rc_recvmsg_response_tcp6(doc) -> []; sc_rc_recvmsg_response_tcp6(_Config) when is_list(_Config) -> tc_try(sc_rc_recvmsg_response_tcp6, + fun() -> has_support_ipv6() end, fun() -> - not_yet_implemented(), ?TT(?SECS(10)), Recv = fun(Sock) -> socket:recvmsg(Sock) end, InitState = #{domain => inet6, @@ -7843,8 +7879,8 @@ sc_rs_recv_send_shutdown_receive_tcp6(doc) -> []; sc_rs_recv_send_shutdown_receive_tcp6(_Config) when is_list(_Config) -> tc_try(sc_rs_recv_send_shutdown_receive_tcp6, + fun() -> has_support_ipv6() end, fun() -> - not_yet_implemented(), ?TT(?SECS(10)), MsgData = ?DATA, Recv = fun(Sock) -> @@ -8667,8 +8703,8 @@ sc_rs_recvmsg_send_shutdown_receive_tcp6(doc) -> []; sc_rs_recvmsg_send_shutdown_receive_tcp6(_Config) when is_list(_Config) -> tc_try(sc_rs_recvmsg_send_shutdown_receive_tcp6, + fun() -> has_support_ipv6() end, fun() -> - not_yet_implemented(), ?TT(?SECS(10)), MsgData = ?DATA, Recv = fun(Sock) -> @@ -8728,8 +8764,8 @@ traffic_send_and_recv_chunks_tcp6(doc) -> []; traffic_send_and_recv_chunks_tcp6(_Config) when is_list(_Config) -> tc_try(traffic_send_and_recv_chunks_tcp6, + fun() -> has_support_ipv6() end, fun() -> - not_yet_implemented(), ?TT(?SECS(30)), InitState = #{domain => inet6}, ok = traffic_send_and_recv_chunks_tcp(InitState) @@ -9730,8 +9766,8 @@ traffic_ping_pong_small_send_and_recv_tcp6(_Config) when is_list(_Config) -> Msg = l2b(?TPP_SMALL), Num = ?TPP_SMALL_NUM, tc_try(traffic_ping_pong_small_send_and_recv_tcp6, + fun() -> has_support_ipv6() end, fun() -> - not_yet_implemented(), ?TT(?SECS(15)), InitState = #{domain => inet6, msg => Msg, @@ -9783,8 +9819,8 @@ traffic_ping_pong_medium_send_and_recv_tcp6(_Config) when is_list(_Config) -> Msg = l2b(?TPP_MEDIUM), Num = ?TPP_MEDIUM_NUM, tc_try(traffic_ping_pong_medium_send_and_recv_tcp6, + fun() -> has_support_ipv6() end, fun() -> - not_yet_implemented(), ?TT(?SECS(30)), InitState = #{domain => inet6, msg => Msg, @@ -9837,8 +9873,8 @@ traffic_ping_pong_large_send_and_recv_tcp6(_Config) when is_list(_Config) -> Msg = l2b(?TPP_LARGE), Num = ?TPP_LARGE_NUM, tc_try(traffic_ping_pong_large_send_and_recv_tcp6, + fun() -> has_support_ipv6() end, fun() -> - not_yet_implemented(), ?TT(?SECS(45)), InitState = #{domain => inet6, msg => Msg, @@ -9944,8 +9980,8 @@ traffic_ping_pong_medium_sendto_and_recvfrom_udp6(_Config) when is_list(_Config) Msg = l2b(?TPP_MEDIUM), Num = ?TPP_MEDIUM_NUM, tc_try(traffic_ping_pong_medium_sendto_and_recvfrom_udp6, + fun() -> has_support_ipv6() end, fun() -> - not_yet_implemented(), ?TT(?SECS(45)), InitState = #{domain => inet6, msg => Msg, @@ -9998,8 +10034,8 @@ traffic_ping_pong_small_sendmsg_and_recvmsg_tcp6(_Config) when is_list(_Config) Msg = l2b(?TPP_SMALL), Num = ?TPP_SMALL_NUM, tc_try(traffic_ping_pong_small_sendmsg_and_recvmsg_tcp6, + fun() -> has_support_ipv6() end, fun() -> - not_yet_implemented(), ?TT(?SECS(20)), InitState = #{domain => inet6, msg => Msg, @@ -10051,8 +10087,8 @@ traffic_ping_pong_medium_sendmsg_and_recvmsg_tcp6(_Config) when is_list(_Config) Msg = l2b(?TPP_MEDIUM), Num = ?TPP_MEDIUM_NUM, tc_try(traffic_ping_pong_medium_sendmsg_and_recvmsg_tcp6, + fun() -> has_support_ipv6() end, fun() -> - not_yet_implemented(), ?TT(?SECS(20)), InitState = #{domain => ine6, msg => Msg, @@ -10104,8 +10140,8 @@ traffic_ping_pong_large_sendmsg_and_recvmsg_tcp6(_Config) when is_list(_Config) Msg = l2b(?TPP_LARGE), Num = ?TPP_LARGE_NUM, tc_try(traffic_ping_pong_large_sendmsg_and_recvmsg_tcp6, + fun() -> has_support_ipv6() end, fun() -> - not_yet_implemented(), ?TT(?SECS(30)), InitState = #{domain => inet6, msg => Msg, @@ -10158,8 +10194,8 @@ traffic_ping_pong_small_sendmsg_and_recvmsg_udp6(_Config) when is_list(_Config) Msg = l2b(?TPP_SMALL), Num = ?TPP_SMALL_NUM, tc_try(traffic_ping_pong_small_sendmsg_and_recvmsg_udp6, + fun() -> has_support_ipv6() end, fun() -> - not_yet_implemented(), ?TT(?SECS(20)), InitState = #{domain => inet, msg => Msg, @@ -10211,8 +10247,8 @@ traffic_ping_pong_medium_sendmsg_and_recvmsg_udp6(_Config) when is_list(_Config) Msg = l2b(?TPP_MEDIUM), Num = ?TPP_MEDIUM_NUM, tc_try(traffic_ping_pong_medium_sendmsg_and_recvmsg_udp6, + fun() -> has_support_ipv6() end, fun() -> - not_yet_implemented(), ?TT(?SECS(20)), InitState = #{domain => ine6, msg => Msg, @@ -17172,17 +17208,17 @@ convert_time(TStrRev, Convert) -> ?TTEST_RUNTIME end. -ttest_tcp(TC, - Domain, - ServerMod, ServerActive, - ClientMod, ClientActive, - MsgID, MaxOutstanding) -> - ttest_tcp(TC, - ?TTEST_RUNTIME, - Domain, - ServerMod, ServerActive, - ClientMod, ClientActive, - MsgID, MaxOutstanding). +%% ttest_tcp(TC, +%% Domain, +%% ServerMod, ServerActive, +%% ClientMod, ClientActive, +%% MsgID, MaxOutstanding) -> +%% ttest_tcp(TC, +%% ?TTEST_RUNTIME, +%% Domain, +%% ServerMod, ServerActive, +%% ClientMod, ClientActive, +%% MsgID, MaxOutstanding). ttest_tcp(TC, Runtime, Domain, @@ -17191,7 +17227,12 @@ ttest_tcp(TC, MsgID, MaxOutstanding) -> tc_try(TC, fun() -> - if (Domain =/= inet) -> not_yet_implemented(); true -> ok end, + if + (Domain =/= inet) -> has_support_ipv6(); + true -> ok + end + end, + fun() -> %% This may be overkill, depending on the runtime, %% but better safe then sorry... ?TT(Runtime + ?SECS(60)), @@ -17833,6 +17874,18 @@ which_addr2(Domain, [_|IFO]) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Here are all the *general* test vase condition functions. + +%% The idea is that this function shall test if the test host has +%% support for IPv6. If not there is no point in running IPv6 tests. +%% Currently we just skip. +has_support_ipv6() -> + not_yet_implemented(). + + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + not_yet_implemented() -> skip("not yet implemented"). @@ -17886,15 +17939,45 @@ tc_end(Result) when is_list(Result) -> "", "----------------------------------------------------~n~n"), ok. - -tc_try(Case, Fun) when is_atom(Case) andalso is_function(Fun, 0) -> +%% *** tc_try/2,3 *** +%% Case: Basically the test case name +%% TCCondFun: A fun that is evaluated before the actual test case +%% The point of this is that it can performs checks to +%% see if we shall run the test case at all. +%% For instance, the test case may only work in specific +%% conditions. +%% FCFun: The test case fun +tc_try(Case, TCFun) -> + TCCondFun = fun() -> ok end, + tc_try(Case, TCCondFun, TCFun). + +tc_try(Case, TCCondFun, TCFun) + when is_atom(Case) andalso + is_function(TCCondFun, 0) andalso + is_function(TCFun, 0) -> tc_begin(Case), - try - begin - Fun(), - ?SLEEP(?SECS(1)), - tc_end("ok") - end + try TCCondFun() of + ok -> + try + begin + TCFun(), + ?SLEEP(?SECS(1)), + tc_end("ok") + end + catch + throw:{skip, _} = SKIP -> + tc_end("skipping"), + SKIP; + Class:Error:Stack -> + tc_end("failed"), + erlang:raise(Class, Error, Stack) + end; + {skip, _} = SKIP -> + tc_end("skipping"), + SKIP; + {error, Reason} -> + tc_end("failed"), + exit({tc_cond_failed, Reason}) catch throw:{skip, _} = SKIP -> tc_end("skipping"), diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops index 1625b2cc65..605a402f2a 100755 --- a/erts/emulator/utils/beam_makeops +++ b/erts/emulator/utils/beam_makeops @@ -2737,7 +2737,7 @@ sub tr_maybe_keep { return; } } elsif ($op eq 'store_var_next_arg') { - return unless shift(@last_instr) eq $args[0]; + return unless @last_instr and shift(@last_instr) eq $args[0]; } elsif (defined $pos) { return; } diff --git a/erts/etc/unix/etp-commands.in b/erts/etc/unix/etp-commands.in index e8dc59156f..dc28107ef5 100644 --- a/erts/etc/unix/etp-commands.in +++ b/erts/etc/unix/etp-commands.in @@ -1453,8 +1453,10 @@ define etp-stack-preamble set $etp_stack_p = ($arg0)->stop set $etp_stack_end = ($arg0)->hend printf "%% Stacktrace (%u)\n", $etp_stack_end-$etp_stack_p - etp-1 ((Eterm)($arg0)->i) 0 - printf " (I)\n" + if ($arg0)->i != 0 + etp-1 ((Eterm)($arg0)->i) 0 + printf " (I)\n" + end if ($arg0)->cp != 0 etp-1 ((Eterm)($arg0)->cp) 0 printf " (cp)\n" diff --git a/erts/preloaded/ebin/erlang.beam b/erts/preloaded/ebin/erlang.beam Binary files differindex 1b0cb5b50c..62dc8702e7 100644 --- a/erts/preloaded/ebin/erlang.beam +++ b/erts/preloaded/ebin/erlang.beam diff --git a/erts/preloaded/src/erlang.erl b/erts/preloaded/src/erlang.erl index a5b60cc845..ac73946dc0 100644 --- a/erts/preloaded/src/erlang.erl +++ b/erts/preloaded/src/erlang.erl @@ -48,6 +48,8 @@ dist_ctrl_put_data/2, dist_ctrl_get_data/1, dist_ctrl_get_data_notification/1, + dist_ctrl_get_opt/2, + dist_ctrl_set_opt/3, dist_get_stat/1]). -deprecated([get_stacktrace/0,now/0]). @@ -3326,7 +3328,8 @@ dist_ctrl_input_handler(_DHandle, _InputHandler) -> dist_ctrl_put_data(_DHandle, _Data) -> erlang:nif_error(undefined). --spec erlang:dist_ctrl_get_data(DHandle) -> Data | 'none' when +-spec erlang:dist_ctrl_get_data(DHandle) -> {Size, Data} | Data | 'none' when + Size :: non_neg_integer(), DHandle :: dist_handle(), Data :: iodata(). @@ -3339,6 +3342,21 @@ dist_ctrl_get_data(_DHandle) -> dist_ctrl_get_data_notification(_DHandle) -> erlang:nif_error(undefined). +-spec erlang:dist_ctrl_set_opt(DHandle, 'get_size', Value) -> OldValue when + DHandle :: dist_handle(), + Value :: boolean(), + OldValue :: boolean(). + +dist_ctrl_set_opt(_DHandle, _Opt, _Val) -> + erlang:nif_error(undefined). + +-spec erlang:dist_ctrl_get_opt(DHandle, 'get_size') -> Value when + DHandle :: dist_handle(), + Value :: boolean(). + +dist_ctrl_get_opt(_DHandle, _Opt) -> + erlang:nif_error(undefined). + -spec erlang:dist_get_stat(DHandle) -> Res when DHandle :: dist_handle(), InputPackets :: non_neg_integer(), diff --git a/erts/vsn.mk b/erts/vsn.mk index fac608ed4e..bce5ce4167 100644 --- a/erts/vsn.mk +++ b/erts/vsn.mk @@ -18,7 +18,7 @@ # %CopyrightEnd% # -VSN = 10.3 +VSN = 10.3.2 # Port number 4365 in 4.2 # Port number 4366 in 4.3 diff --git a/lib/asn1/src/asn1ct_check.erl b/lib/asn1/src/asn1ct_check.erl index 9ec0d93e93..9eec05abd1 100644 --- a/lib/asn1/src/asn1ct_check.erl +++ b/lib/asn1/src/asn1ct_check.erl @@ -1710,7 +1710,7 @@ check_value(S,#valuedef{pos=Pos,name=Name,type=Type, {valueset, check_type(S,#typedef{pos=Pos,name=Name,typespec=NewType},NewType)}; check_value(S, #valuedef{}=V) -> - ?dbg("check_value, V: ~p~n",[V0]), + ?dbg("check_value, V: ~p~n",[V]), case V of #valuedef{checked=true} -> V; @@ -1721,7 +1721,8 @@ check_value(S, #valuedef{}=V) -> check_valuedef(#state{recordtopname=TopName}=S0, V0) -> #valuedef{name=Name,type=Vtype0,value=Value,module=ModName} = V0, V = V0#valuedef{checked=true}, - Vtype = check_type(S0, #typedef{name=Name,typespec=Vtype0},Vtype0), + Vtype1 = expand_valuedef_type(Vtype0), + Vtype = check_type(S0, #typedef{name=Name,typespec=Vtype1},Vtype1), Def = Vtype#type.def, S1 = S0#state{tname=Def}, SVal = update_state(S1, ModName), @@ -1767,6 +1768,27 @@ check_valuedef(#state{recordtopname=TopName}=S0, V0) -> V#valuedef{value=normalize_value(SVal, Vtype, Value, TopName)} end. +expand_valuedef_type(#type{def=Seq}=Type) + when is_record(Seq,'SEQUENCE') -> + NewComponents = case Seq#'SEQUENCE'.components of + {R1,_Ext,R2} -> R1 ++ R2; + {Root,_Ext} -> Root; + Root -> take_only_rootset(Root) + end, + NewSeq = Seq#'SEQUENCE'{components = NewComponents}, + Type#type{def=NewSeq}; +expand_valuedef_type(#type{def=Set}=Type) + when is_record(Set,'SET') -> + NewComponents = case Set#'SET'.components of + {R1,_Ext,R2} -> R1 ++ R2; + {Root,_Ext} -> Root; + Root -> take_only_rootset(Root) + end, + NewSet = Set#'SET'{components = NewComponents}, + Type#type{def=NewSet}; +expand_valuedef_type(Type) -> + Type. + is_contextswitchtype(#typedef{name='EXTERNAL'})-> true; is_contextswitchtype(#typedef{name='EMBEDDED PDV'}) -> @@ -1998,7 +2020,8 @@ normalize_value(S, Type, {'DEFAULT',Value}, NameList) -> {'ENUMERATED',CType,_} -> normalize_enumerated(S,Value,CType); {'CHOICE',CType,NewNameList} -> - normalize_choice(S,Value,CType,NewNameList); + ChoiceComponents = get_choice_components(S, {'CHOICE',CType}), + normalize_choice(S,Value,ChoiceComponents,NewNameList); {'SEQUENCE',CType,NewNameList} -> normalize_sequence(S,Value,CType,NewNameList); {'SEQUENCE OF',CType,NewNameList} -> @@ -2140,6 +2163,9 @@ normalize_octetstring(S, Value) -> _ -> asn1_error(S, illegal_octet_string_value) end; + Val when is_binary(Val) -> + %% constant default value + Val; _ -> asn1_error(S, illegal_octet_string_value) end. @@ -2751,8 +2777,9 @@ check_type(S=#state{recordtopname=TopName},Type,Ts) when is_record(Ts,type) -> TempNewDef#newt{type={'SEQUENCE OF',check_sequenceof(S,Type,Components)}, tag= merge_tags(Tag,?TAG_CONSTRUCTED(?N_SEQUENCE))}; - {'CHOICE',Components} -> + {'CHOICE',_} = Choice-> Ct = maybe_illicit_implicit_tag(S, choice, Tag), + Components = get_choice_components(S, Choice), TempNewDef#newt{type={'CHOICE',check_choice(S,Type,Components)},tag=Ct}; Set when is_record(Set,'SET') -> RecordName= diff --git a/lib/asn1/test/Makefile b/lib/asn1/test/Makefile index c38d1c6ebd..6ff4aa8d0f 100644 --- a/lib/asn1/test/Makefile +++ b/lib/asn1/test/Makefile @@ -60,6 +60,7 @@ MODULES= \ testSeqOf \ testSeqOfIndefinite \ testSeqOfCho \ + testSeqOfChoExt \ testSeqOfExternal \ testSeqOfTag \ testSetDefault \ @@ -72,6 +73,7 @@ MODULES= \ testSetTypeRefPrim \ testSetTypeRefSeq \ testSetTypeRefSet \ + testDefaultOctetString \ testChoiceIndefinite \ testSetOf \ testSetOfCho \ diff --git a/lib/asn1/test/asn1_SUITE.erl b/lib/asn1/test/asn1_SUITE.erl index ab78678110..a88e464996 100644 --- a/lib/asn1/test/asn1_SUITE.erl +++ b/lib/asn1/test/asn1_SUITE.erl @@ -99,6 +99,7 @@ groups() -> testChoTypeRefPrim, testChoTypeRefSeq, testChoTypeRefSet, + testDefaultOctetString, testMultipleLevels, testOpt, testSeqDefault, @@ -118,6 +119,7 @@ groups() -> {group, [], [testSeqOf, testSeqOfIndefinite]}, % Uses 'Mvrasn*' testSeqOfCho, + testSeqOfChoExt, testSetDefault, testExtensionAdditionGroup, testSetOptional, @@ -430,6 +432,11 @@ testChoTypeRefSet(Config, Rule, Opts) -> asn1_test_lib:compile("ChoTypeRefSet", Config, [Rule|Opts]), testChoTypeRefSet:set(Rule). +testDefaultOctetString(Config) -> test(Config, fun testDefaultOctetString/3). +testDefaultOctetString(Config, Rule, Opts) -> + asn1_test_lib:compile("DefaultOctetString", Config, [Rule|Opts]), + testDefaultOctetString:dos(Rule). + testMultipleLevels(Config) -> test(Config, fun testMultipleLevels/3). testMultipleLevels(Config, Rule, Opts) -> asn1_test_lib:compile("MultipleLevels", Config, [Rule|Opts]), @@ -535,6 +542,11 @@ testSeqOfCho(Config, Rule, Opts) -> asn1_test_lib:compile("SeqOfCho", Config, [Rule|Opts]), testSeqOfCho:main(Rule). +testSeqOfChoExt(Config) -> test(Config, fun testSeqOfChoExt/3). +testSeqOfChoExt(Config, Rule, Opts) -> + asn1_test_lib:compile("SeqOfChoExt", Config, [Rule|Opts]), + testSeqOfChoExt:main(Rule). + testSeqOfIndefinite(Config) -> test(Config, fun testSeqOfIndefinite/3, [ber]). testSeqOfIndefinite(Config, Rule, Opts) -> diff --git a/lib/asn1/test/asn1_SUITE_data/ChoExtension.asn1 b/lib/asn1/test/asn1_SUITE_data/ChoExtension.asn1 index 18473bae30..c488704196 100644 --- a/lib/asn1/test/asn1_SUITE_data/ChoExtension.asn1 +++ b/lib/asn1/test/asn1_SUITE_data/ChoExtension.asn1 @@ -41,4 +41,6 @@ ChoExt4 ::= CHOICE str OCTET STRING } +choExt1 ChoExt1 ::= int : 1 + END diff --git a/lib/asn1/test/asn1_SUITE_data/DefaultOctetString.asn b/lib/asn1/test/asn1_SUITE_data/DefaultOctetString.asn new file mode 100644 index 0000000000..076e965d58 --- /dev/null +++ b/lib/asn1/test/asn1_SUITE_data/DefaultOctetString.asn @@ -0,0 +1,15 @@ +DefaultOctetString +DEFINITIONS +AUTOMATIC TAGS + ::= +BEGIN +Dos ::= SEQUENCE { + opt [2] OCTET STRING (SIZE(2..4)) OPTIONAL, + def [10] OCTET STRING (SIZE (1)) DEFAULT '05'H +} + +dos Dos ::= { + opt '1234'H +} + +END diff --git a/lib/asn1/test/asn1_SUITE_data/SeqOfChoExt.asn1 b/lib/asn1/test/asn1_SUITE_data/SeqOfChoExt.asn1 new file mode 100644 index 0000000000..51077754fd --- /dev/null +++ b/lib/asn1/test/asn1_SUITE_data/SeqOfChoExt.asn1 @@ -0,0 +1,27 @@ +SeqOfChoExt DEFINITIONS AUTOMATIC TAGS EXTENSIBILITY IMPLIED ::=
+BEGIN
+
+Seq2 ::= SEQUENCE {
+ octstr [PRIVATE 6] OCTET STRING OPTIONAL
+}
+
+SeqOfCho ::= SEQUENCE OF CHOICE {
+ nullable NULL,
+ seq2 Seq2
+}
+
+Seq1 ::= SEQUENCE {
+ int INTEGER,
+ soc SeqOfCho
+}
+
+seq1 Seq1 ::= {
+ int 10,
+ soc {
+ seq2 : {
+ octstr '01020A'H
+ }
+ }
+}
+
+END
diff --git a/lib/asn1/test/testChoExtension.erl b/lib/asn1/test/testChoExtension.erl index 4c632aab81..cfb28be5c7 100644 --- a/lib/asn1/test/testChoExtension.erl +++ b/lib/asn1/test/testChoExtension.erl @@ -28,6 +28,7 @@ extension(_Rules) -> roundtrip('ChoExt1', {bool,true}), roundtrip('ChoExt1', {int,33}), + {int, 1} = 'ChoExtension':choExt1(), %% A trick to encode with another compatible CHOICE type to test reception %% extension alternative diff --git a/lib/asn1/test/testDefaultOctetString.erl b/lib/asn1/test/testDefaultOctetString.erl new file mode 100644 index 0000000000..82cd5810e5 --- /dev/null +++ b/lib/asn1/test/testDefaultOctetString.erl @@ -0,0 +1,34 @@ +-module(testDefaultOctetString). + +-export([dos/1]). + +-include_lib("common_test/include/ct.hrl"). + +-record('Dos', { + opt = asn1_NOVALUE, + def = asn1_DEFAULT +}). + +-define(def_DEFAULT, <<5>>). + +dos(Rules) -> + %% test roundtrip default + E1 = roundtrip(#'Dos'{}, #'Dos'{def = ?def_DEFAULT}), + %% test the value dos defined in the .asn file + E2 = roundtrip('DefaultOctetString':dos()), + %% sanity test a fully specified SEQUENCE + E3 = roundtrip(#'Dos'{opt = <<1,2,3>>, def = <<6>>}), + %% test def is/isn't encoded according to the value + if Rules == ber -> + <<48, 0>> = E1, + <<48, 4, 16#82, 2, 16#12, 16#34>> = E2, + <<48, 8, 16#82, 3, 1, 2, 3, 16#8A, 1, 6>> = E3; + true -> + ignore + end, + ok. + +roundtrip(Value) -> + roundtrip(Value, Value). +roundtrip(Value, Exp) -> + asn1_test_lib:roundtrip('DefaultOctetString', 'Dos', Value, Exp). diff --git a/lib/asn1/test/testSeqOfChoExt.erl b/lib/asn1/test/testSeqOfChoExt.erl new file mode 100644 index 0000000000..1e72c60841 --- /dev/null +++ b/lib/asn1/test/testSeqOfChoExt.erl @@ -0,0 +1,15 @@ +-module(testSeqOfChoExt). + +-export([main/1]). + +%-record('Seq2', { octstr = asn1_NOVALUE }). +%-record('Seq1', { int, soc }). + +main(_Rules) -> + roundtrip('SeqOfChoExt':seq1()). + +roundtrip(Value) -> + roundtrip(Value, Value). +roundtrip(Value, Exp) -> + Type = element(1,Value), + asn1_test_lib:roundtrip('SeqOfChoExt', Type, Value, Exp). diff --git a/lib/common_test/test_server/configure.in b/lib/common_test/test_server/configure.in index e07bd4c2aa..a32d050081 100644 --- a/lib/common_test/test_server/configure.in +++ b/lib/common_test/test_server/configure.in @@ -171,7 +171,7 @@ case $system in fi SHLIB_EXTRACT_ALL="" ;; - *-openbsd*) + *-openbsd*|*-netbsd*|*-freebsd*|*-dragonfly*) # Not available on all versions: check for include file. AC_CHECK_HEADER(dlfcn.h, [ SHLIB_CFLAGS="-fpic" @@ -194,29 +194,6 @@ case $system in ]) SHLIB_EXTRACT_ALL="" ;; - *-netbsd*|*-freebsd*|*-dragonfly*) - # Not available on all versions: check for include file. - AC_CHECK_HEADER(dlfcn.h, [ - SHLIB_CFLAGS="-fpic" - SHLIB_LD="ld" - SHLIB_LDFLAGS="$LDFLAGS -Bshareable -x" - SHLIB_SUFFIX=".so" - if test X${enable_m64_build} = Xyes; then - AC_MSG_ERROR(don't know how to link 64-bit dynamic drivers) - fi - if test X${enable_m32_build} = Xyes; then - AC_MSG_ERROR(don't know how to link 32-bit dynamic drivers) - fi - ], [ - # No dynamic loading. - SHLIB_CFLAGS="" - SHLIB_LD="ld" - SHLIB_LDFLAGS="" - SHLIB_SUFFIX="" - AC_MSG_ERROR(don't know how to compile and link dynamic drivers) - ]) - SHLIB_EXTRACT_ALL="" - ;; *-solaris2*|*-sysv4*) SHLIB_CFLAGS="-KPIC" SHLIB_LD="/usr/ccs/bin/ld" diff --git a/lib/compiler/scripts/smoke-mix.exs b/lib/compiler/scripts/smoke-mix.exs index 82ae3370fe..ba0815e465 100644 --- a/lib/compiler/scripts/smoke-mix.exs +++ b/lib/compiler/scripts/smoke-mix.exs @@ -25,6 +25,14 @@ defmodule Smoke.MixProject do [ {:bear, "~> 0.8.7"}, {:cloudi_core, "~> 1.7"}, + {:cloudi_service_monitoring, "~> 1.7"}, + {:cloudi_service_tcp, "~> 1.7"}, + {:cloudi_service_queue, "~> 1.7"}, + {:cloudi_service_udp, "~> 1.7"}, + {:cloudi_service_map_reduce, "~> 1.7"}, + {:cloudi_service_api_requests, "~> 1.7"}, + {:cloudi_service_router, "~> 1.7"}, + {:cloudi_service_request_rate, "~> 1.7"}, {:concuerror, "~> 0.20.0"}, {:cowboy, "~> 2.6.1"}, {:ecto, "~> 3.0.6"}, diff --git a/lib/compiler/src/Makefile b/lib/compiler/src/Makefile index c971e8844d..9f8d63baa1 100644 --- a/lib/compiler/src/Makefile +++ b/lib/compiler/src/Makefile @@ -129,9 +129,10 @@ APPUP_TARGET= $(EBIN)/$(APPUP_FILE) ifeq ($(NATIVE_LIBS_ENABLED),yes) ERL_COMPILE_FLAGS += +native +else +ERL_COMPILE_FLAGS += -Werror endif ERL_COMPILE_FLAGS += +inline +warn_unused_import \ - -Werror \ -I../../stdlib/include -I$(EGEN) -W +warn_missing_spec # ---------------------------------------------------- diff --git a/lib/compiler/src/beam_ssa_type.erl b/lib/compiler/src/beam_ssa_type.erl index c01ea4af91..06b42f1928 100644 --- a/lib/compiler/src/beam_ssa_type.erl +++ b/lib/compiler/src/beam_ssa_type.erl @@ -267,10 +267,29 @@ opt_is([#b_set{op=call,args=Args0,dst=Dst}=I0|Is], I1 = beam_ssa:normalize(I0#b_set{args=Args}), {Ts1,Ds,Fdb,I2} = opt_call(I1, D, Ts0, Ds0, Fdb0), case {map_get(Dst, Ts1),Is} of - {_,[#b_set{op=succeeded}]} -> + {Type,[#b_set{op=succeeded}]} when Type =/= none -> %% This call instruction is inside a try/catch - %% block. Don't attempt to optimize it. + %% block. Don't attempt to simplify it. opt_is(Is, Ts1, Ds, Fdb, D, Sub0, [I2|Acc]); + {none,[#b_set{op=succeeded}]} -> + %% This call instruction is inside a try/catch + %% block, but we know it will never return and + %% later optimizations may try to exploit that. + %% + %% For example, if we have an expression that + %% either returns this call or a tuple, we know + %% that the expression always returns a tuple + %% and can turn a later element/3 into + %% get_tuple_element. + %% + %% This is sound but difficult to validate in a + %% meaningful way as try/catch currently forces + %% us to maintain the illusion that the success + %% block is reachable even when its not, so we + %% disable the optimization to keep things + %% simple. + Ts = Ts1#{ Dst := any }, + opt_is(Is, Ts, Ds, Fdb, D, Sub0, [I2|Acc]); {none,_} -> %% This call never returns. The rest of the %% instructions will not be executed. diff --git a/lib/compiler/src/beam_validator.erl b/lib/compiler/src/beam_validator.erl index 4fba3fa1c6..efd2be94cb 100644 --- a/lib/compiler/src/beam_validator.erl +++ b/lib/compiler/src/beam_validator.erl @@ -2899,8 +2899,6 @@ lists_mod_return_type(filter, 2, _Vst) -> list; lists_mod_return_type(flatten, 1, _Vst) -> list; -lists_mod_return_type(flatten, 2, _Vst) -> - list; lists_mod_return_type(map, 2, Vst) -> same_length_type({x,1}, Vst); lists_mod_return_type(MF, 3, Vst) when MF =:= mapfoldl; MF =:= mapfoldr -> @@ -2912,8 +2910,6 @@ lists_mod_return_type(reverse, 1, Vst) -> same_length_type({x,0}, Vst); lists_mod_return_type(seq, 2, _Vst) -> list; -lists_mod_return_type(seq, 3, _Vst) -> - list; lists_mod_return_type(sort, 1, Vst) -> same_length_type({x,0}, Vst); lists_mod_return_type(sort, 2, Vst) -> @@ -2927,16 +2923,10 @@ lists_mod_return_type(unzip, 1, Vst) -> two_tuple(ListType, ListType); lists_mod_return_type(usort, 1, Vst) -> same_length_type({x,0}, Vst); -lists_mod_return_type(usort, 2, Vst) -> - same_length_type({x,1}, Vst); lists_mod_return_type(zip, 2, _Vst) -> list; -lists_mod_return_type(zip3, 3, _Vst) -> - list; lists_mod_return_type(zipwith, 3, _Vst) -> list; -lists_mod_return_type(zipwith3, 4, _Vst) -> - list; lists_mod_return_type(_, _, _) -> term. diff --git a/lib/compiler/src/compile.erl b/lib/compiler/src/compile.erl index 11dea9524b..28db8986ff 100644 --- a/lib/compiler/src/compile.erl +++ b/lib/compiler/src/compile.erl @@ -290,6 +290,10 @@ format_error(bad_crypto_key) -> "invalid crypto key."; format_error(no_crypto_key) -> "no crypto key supplied."; +format_error({unimplemented_instruction,Instruction}) -> + io_lib:fwrite("native-code compilation failed because of an " + "unimplemented instruction (~s).", + [Instruction]); format_error({native, E}) -> io_lib:fwrite("native-code compilation failed with reason: ~tP.", [E, 25]); @@ -1651,18 +1655,22 @@ native_compile_1(Code, St) -> case IgnoreErrors of true -> Ws = [{St#compile.ifile,[{none,?MODULE,{native,R}}]}], - {ok,St#compile{warnings=St#compile.warnings ++ Ws}}; + {ok,Code,St#compile{warnings=St#compile.warnings ++ Ws}}; false -> Es = [{St#compile.ifile,[{none,?MODULE,{native,R}}]}], {error,St#compile{errors=St#compile.errors ++ Es}} end catch + exit:{unimplemented_instruction,_}=Unimplemented -> + Ws = [{St#compile.ifile, + [{none,?MODULE,Unimplemented}]}], + {ok,Code,St#compile{warnings=St#compile.warnings ++ Ws}}; Class:R:Stack -> case IgnoreErrors of true -> Ws = [{St#compile.ifile, [{none,?MODULE,{native_crash,R,Stack}}]}], - {ok,St#compile{warnings=St#compile.warnings ++ Ws}}; + {ok,Code,St#compile{warnings=St#compile.warnings ++ Ws}}; false -> erlang:raise(Class, R, Stack) end diff --git a/lib/compiler/src/sys_core_fold.erl b/lib/compiler/src/sys_core_fold.erl index 7e219da0af..4939a94a92 100644 --- a/lib/compiler/src/sys_core_fold.erl +++ b/lib/compiler/src/sys_core_fold.erl @@ -1997,53 +1997,22 @@ case_opt_compiler_generated(Core) -> %% case_expand_var(Expr0, Sub) -> Expr -%% If Expr0 is a variable that has been previously matched and -%% is known to be a tuple, return the tuple instead. Otherwise +%% If Expr0 is a variable that is known to be bound to a +%% constructed tuple, return the tuple instead. Otherwise %% return Expr0 unchanged. -%% + case_expand_var(E, #sub{t=Tdb}) -> Key = cerl:var_name(E), case Tdb of - #{Key:=T0} -> - case cerl:is_c_tuple(T0) of - false -> - E; - true -> - %% The pattern was a tuple. Now we must make sure - %% that the elements of the tuple are suitable. In - %% particular, we don't want binary or map - %% construction here, since that means that the - %% binary or map will be constructed in the 'case' - %% argument. That is wasteful for binaries. Even - %% worse is that any map pattern that use the ':=' - %% operator will fail when used in map - %% construction (only the '=>' operator is allowed - %% when constructing a map from scratch). - try - cerl_trees:map(fun coerce_to_data/1, T0) - catch - throw:impossible -> - %% Something unsuitable was found (map or - %% or binary). Keep the variable. - E - end + #{Key:=T} -> + case cerl:is_c_tuple(T) of + false -> E; + true -> T end; _ -> E end. -%% coerce_to_data(Core) -> Core' -%% Coerce an element originally from a pattern to an data item or or -%% variable. Throw an 'impossible' exception if non-data Core Erlang -%% terms such as binary construction or map construction are -%% encountered. - -coerce_to_data(C) -> - case cerl:is_data(C) orelse cerl:is_c_var(C) of - true -> C; - false -> throw(impossible) - end. - %% case_opt_nomatch(E, Clauses, LitExpr) -> Clauses' %% Remove all clauses that cannot possibly match. diff --git a/lib/compiler/test/beam_type_SUITE.erl b/lib/compiler/test/beam_type_SUITE.erl index a7ffc3f60a..882e281a44 100644 --- a/lib/compiler/test/beam_type_SUITE.erl +++ b/lib/compiler/test/beam_type_SUITE.erl @@ -24,7 +24,7 @@ integers/1,numbers/1,coverage/1,booleans/1,setelement/1, cons/1,tuple/1,record_float/1,binary_float/1,float_compare/1, arity_checks/1,elixir_binaries/1,find_best/1, - test_size/1]). + test_size/1,cover_lists_functions/1]). suite() -> [{ct_hooks,[ts_install_cth]}]. @@ -46,7 +46,8 @@ groups() -> arity_checks, elixir_binaries, find_best, - test_size + test_size, + cover_lists_functions ]}]. init_per_suite(Config) -> @@ -473,5 +474,18 @@ do_test_size(Term) when is_tuple(Term) -> do_test_size(Term) when is_binary(Term) -> size(Term). +cover_lists_functions(Config) -> + case lists:suffix([no|Config], Config) of + true -> + ct:fail(should_be_false); + false -> + ok + end, + Zipped = lists:zipwith(fun(A, B) -> {A,B} end, + lists:duplicate(length(Config), zip), + Config), + true = is_list(Zipped), + ok. + id(I) -> I. diff --git a/lib/compiler/test/bs_construct_SUITE.erl b/lib/compiler/test/bs_construct_SUITE.erl index 69017d87e7..bccd70d6cb 100644 --- a/lib/compiler/test/bs_construct_SUITE.erl +++ b/lib/compiler/test/bs_construct_SUITE.erl @@ -27,6 +27,7 @@ -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2, init_per_testcase/2,end_per_testcase/2, + verify_highest_opcode/1, two/1,test1/1,fail/1,float_bin/1,in_guard/1,in_catch/1, nasty_literals/1,coerce_to_float/1,side_effect/1, opt/1,otp_7556/1,float_arith/1,otp_8054/1, @@ -43,7 +44,8 @@ all() -> groups() -> [{p,[parallel], - [two,test1,fail,float_bin,in_guard,in_catch, + [verify_highest_opcode, + two,test1,fail,float_bin,in_guard,in_catch, nasty_literals,side_effect,opt,otp_7556,float_arith, otp_8054,cover]}]. @@ -68,6 +70,20 @@ init_per_testcase(Case, Config) when is_atom(Case), is_list(Config) -> end_per_testcase(Case, Config) when is_atom(Case), is_list(Config) -> ok. +verify_highest_opcode(_Config) -> + case ?MODULE of + bs_construct_r21_SUITE -> + {ok,Beam} = file:read_file(code:which(?MODULE)), + case test_lib:highest_opcode(Beam) of + Highest when Highest =< 163 -> + ok; + TooHigh -> + ct:fail({too_high_opcode_for_21,TooHigh}) + end; + _ -> + ok + end. + two(Config) when is_list(Config) -> <<0,1,2,3,4,6,7,8,9>> = two_1([0], [<<1,2,3,4>>,<<6,7,8,9>>]), ok. diff --git a/lib/compiler/test/bs_match_SUITE.erl b/lib/compiler/test/bs_match_SUITE.erl index 2cfcb841a7..41e4918b1e 100644 --- a/lib/compiler/test/bs_match_SUITE.erl +++ b/lib/compiler/test/bs_match_SUITE.erl @@ -24,6 +24,7 @@ -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2, init_per_testcase/2,end_per_testcase/2, + verify_highest_opcode/1, size_shadow/1,int_float/1,otp_5269/1,null_fields/1,wiger/1, bin_tail/1,save_restore/1, partitioned_bs_match/1,function_clause/1, @@ -60,7 +61,8 @@ all() -> groups() -> [{p,[], - [size_shadow,int_float,otp_5269,null_fields,wiger, + [verify_highest_opcode, + size_shadow,int_float,otp_5269,null_fields,wiger, bin_tail,save_restore, partitioned_bs_match,function_clause,unit, shared_sub_bins,bin_and_float,dec_subidentifiers, @@ -101,6 +103,20 @@ init_per_testcase(Case, Config) when is_atom(Case), is_list(Config) -> end_per_testcase(Case, Config) when is_atom(Case), is_list(Config) -> ok. +verify_highest_opcode(_Config) -> + case ?MODULE of + bs_match_r21_SUITE -> + {ok,Beam} = file:read_file(code:which(?MODULE)), + case test_lib:highest_opcode(Beam) of + Highest when Highest =< 163 -> + ok; + TooHigh -> + ct:fail({too_high_opcode_for_21,TooHigh}) + end; + _ -> + ok + end. + size_shadow(Config) when is_list(Config) -> %% Originally OTP-5270. 7 = size_shadow_1(), diff --git a/lib/compiler/test/compile_SUITE.erl b/lib/compiler/test/compile_SUITE.erl index 408af80dd9..53627b9d81 100644 --- a/lib/compiler/test/compile_SUITE.erl +++ b/lib/compiler/test/compile_SUITE.erl @@ -1431,9 +1431,7 @@ bc_options(Config) -> highest_opcode(DataDir, Mod, Opt) -> Src = filename:join(DataDir, atom_to_list(Mod)++".erl"), {ok,Mod,Beam} = compile:file(Src, [binary|Opt]), - {ok,{Mod,[{"Code",Code}]}} = beam_lib:chunks(Beam, ["Code"]), - <<16:32,0:32,HighestOpcode:32,_/binary>> = Code, - HighestOpcode. + test_lib:highest_opcode(Beam). deterministic_include(Config) when is_list(Config) -> DataDir = proplists:get_value(data_dir, Config), diff --git a/lib/compiler/test/test_lib.erl b/lib/compiler/test/test_lib.erl index 39c26c6142..3348c6e9ea 100644 --- a/lib/compiler/test/test_lib.erl +++ b/lib/compiler/test/test_lib.erl @@ -22,7 +22,8 @@ -include_lib("common_test/include/ct.hrl"). -compile({no_auto_import,[binary_part/2]}). -export([id/1,recompile/1,parallel/0,uniq/0,opt_opts/1,get_data_dir/1, - is_cloned_mod/1,smoke_disasm/1,p_run/2]). + is_cloned_mod/1,smoke_disasm/1,p_run/2, + highest_opcode/1]). %% Used by test case that override BIFs. -export([binary_part/2,binary/1]). @@ -113,6 +114,14 @@ is_cloned_mod_1("_no_module_opt_SUITE") -> true; is_cloned_mod_1([_|T]) -> is_cloned_mod_1(T); is_cloned_mod_1([]) -> false. +%% Return the highest opcode use in the BEAM module. + +highest_opcode(Beam) -> + {ok,{_Mod,[{"Code",Code}]}} = beam_lib:chunks(Beam, ["Code"]), + FormatNumber = 0, + <<16:32,FormatNumber:32,HighestOpcode:32,_/binary>> = Code, + HighestOpcode. + %% p_run(fun(Data) -> ok|error, List) -> ok %% Will fail the test case if there were any errors. diff --git a/lib/compiler/test/trycatch_SUITE.erl b/lib/compiler/test/trycatch_SUITE.erl index 8f9cd9ab1e..539f9d69fa 100644 --- a/lib/compiler/test/trycatch_SUITE.erl +++ b/lib/compiler/test/trycatch_SUITE.erl @@ -27,7 +27,8 @@ nested_horrid/1,last_call_optimization/1,bool/1, plain_catch_coverage/1,andalso_orelse/1,get_in_try/1, hockey/1,handle_info/1,catch_in_catch/1,grab_bag/1, - stacktrace/1,nested_stacktrace/1,raise/1]). + stacktrace/1,nested_stacktrace/1,raise/1, + no_return_in_try_block/1]). -include_lib("common_test/include/ct.hrl"). @@ -43,7 +44,8 @@ groups() -> nested_after,nested_horrid,last_call_optimization, bool,plain_catch_coverage,andalso_orelse,get_in_try, hockey,handle_info,catch_in_catch,grab_bag, - stacktrace,nested_stacktrace,raise]}]. + stacktrace,nested_stacktrace,raise, + no_return_in_try_block]}]. init_per_suite(Config) -> @@ -1287,5 +1289,26 @@ do_test_raise_4(Expr) -> erlang:raise(exit, {exception,C,E,Stk}, Stk) end. +no_return_in_try_block(Config) when is_list(Config) -> + 1.0 = no_return_in_try_block_1(0), + 1.0 = no_return_in_try_block_1(0.0), + + gurka = no_return_in_try_block_1(gurka), + [] = no_return_in_try_block_1([]), + + ok. + +no_return_in_try_block_1(H) -> + try + Float = if + is_number(H) -> float(H); + true -> no_return() + end, + Float + 1 + catch + throw:no_return -> H + end. + +no_return() -> throw(no_return). id(I) -> I. diff --git a/lib/crypto/c_src/aead.c b/lib/crypto/c_src/aead.c index 3ee04f1be9..4ed16615a5 100644 --- a/lib/crypto/c_src/aead.c +++ b/lib/crypto/c_src/aead.c @@ -39,87 +39,79 @@ ERL_NIF_TERM aead_encrypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) ASSERT(argc == 6); if (!enif_is_atom(env, type)) - goto bad_arg; + {ret = EXCP_BADARG(env, "non-atom cipher type"); goto done;} if (!enif_inspect_iolist_as_binary(env, argv[1], &key)) - goto bad_arg; + {ret = EXCP_BADARG(env, "non-binary key"); goto done;} if (!enif_inspect_binary(env, argv[2], &iv)) - goto bad_arg; + {ret = EXCP_BADARG(env, "non-binary iv"); goto done;} if (!enif_inspect_iolist_as_binary(env, argv[3], &aad)) - goto bad_arg; + {ret = EXCP_BADARG(env, "non-binary AAD"); goto done;} if (!enif_inspect_iolist_as_binary(env, argv[4], &in)) - goto bad_arg; + {ret = EXCP_BADARG(env, "non-binary text"); goto done;} if (!enif_get_uint(env, argv[5], &tag_len)) - goto bad_arg; + {ret = EXCP_BADARG(env, ""); goto done;} if (tag_len > INT_MAX || iv.size > INT_MAX || in.size > INT_MAX || aad.size > INT_MAX) - goto bad_arg; + {ret = EXCP_BADARG(env, "binary too long"); goto done;} if ((cipherp = get_cipher_type(type, key.size)) == NULL) - goto bad_arg; + {ret = EXCP_BADARG(env, "Unknown cipher"); goto done;} if (cipherp->flags & NON_EVP_CIPHER) - goto bad_arg; + {ret = EXCP_BADARG(env, "Bad cipher"); goto done;} if (! (cipherp->flags & AEAD_CIPHER) ) - goto bad_arg; + {ret = EXCP_BADARG(env, "Not aead cipher"); goto done;} if ((cipher = cipherp->cipher.p) == NULL) - return enif_raise_exception(env, atom_notsup); + {ret = EXCP_NOTSUP(env, "Cipher not supported in this libcrypto version"); goto done;} ctx_ctrl_set_ivlen = cipherp->extra.aead.ctx_ctrl_set_ivlen; ctx_ctrl_get_tag = cipherp->extra.aead.ctx_ctrl_get_tag; ctx_ctrl_set_tag = cipherp->extra.aead.ctx_ctrl_set_tag; if ((ctx = EVP_CIPHER_CTX_new()) == NULL) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} if (EVP_EncryptInit_ex(ctx, cipher, NULL, NULL, NULL) != 1) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} if (EVP_CIPHER_CTX_ctrl(ctx, ctx_ctrl_set_ivlen, (int)iv.size, NULL) != 1) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} #if defined(HAVE_CCM) if (type == atom_aes_ccm) { if (EVP_CIPHER_CTX_ctrl(ctx, ctx_ctrl_set_tag, (int)tag_len, NULL) != 1) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} if (EVP_EncryptInit_ex(ctx, NULL, NULL, key.data, iv.data) != 1) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} if (EVP_EncryptUpdate(ctx, NULL, &len, NULL, (int)in.size) != 1) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} } else #endif { if (EVP_EncryptInit_ex(ctx, NULL, NULL, key.data, iv.data) != 1) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} } if (EVP_EncryptUpdate(ctx, NULL, &len, aad.data, (int)aad.size) != 1) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} if ((outp = enif_make_new_binary(env, in.size, &out)) == NULL) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} if (EVP_EncryptUpdate(ctx, outp, &len, in.data, (int)in.size) != 1) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} if (EVP_EncryptFinal_ex(ctx, outp/*+len*/, &len) != 1) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} if ((tagp = enif_make_new_binary(env, tag_len, &out_tag)) == NULL) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} if (EVP_CIPHER_CTX_ctrl(ctx, ctx_ctrl_get_tag, (int)tag_len, tagp) != 1) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} CONSUME_REDS(env, in); ret = enif_make_tuple2(env, out, out_tag); - goto done; - - bad_arg: - ret = enif_make_badarg(env); - goto done; - - err: - ret = atom_error; done: if (ctx) @@ -127,7 +119,7 @@ ERL_NIF_TERM aead_encrypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) return ret; #else - return enif_raise_exception(env, atom_notsup); + return EXCP_NOTSUP(env, ""); #endif } @@ -151,72 +143,72 @@ ERL_NIF_TERM aead_decrypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) #endif if (!enif_is_atom(env, type)) - goto bad_arg; + {ret = EXCP_BADARG(env, "non-atom cipher type"); goto done;} if (!enif_inspect_iolist_as_binary(env, argv[1], &key)) - goto bad_arg; + {ret = EXCP_BADARG(env, "non-binary key"); goto done;} if (!enif_inspect_binary(env, argv[2], &iv)) - goto bad_arg; + {ret = EXCP_BADARG(env, "non-binary iv"); goto done;} if (!enif_inspect_iolist_as_binary(env, argv[3], &aad)) - goto bad_arg; + {ret = EXCP_BADARG(env, "non-binary AAD"); goto done;} if (!enif_inspect_iolist_as_binary(env, argv[4], &in)) - goto bad_arg; + {ret = EXCP_BADARG(env, ""); goto done;} if (!enif_inspect_iolist_as_binary(env, argv[5], &tag)) - goto bad_arg; + {ret = EXCP_BADARG(env, "non-binary text"); goto done;} if (tag.size > INT_MAX || key.size > INT_MAX || iv.size > INT_MAX || in.size > INT_MAX || aad.size > INT_MAX) - goto bad_arg; + {ret = EXCP_BADARG(env, "binary too long"); goto done;} if ((cipherp = get_cipher_type(type, key.size)) == NULL) - goto bad_arg; + {ret = EXCP_BADARG(env, "Unknown cipher"); goto done;} if (cipherp->flags & NON_EVP_CIPHER) - goto bad_arg; + {ret = EXCP_BADARG(env, "Bad cipher"); goto done;} if ( !(cipherp->flags & AEAD_CIPHER) ) - goto bad_arg; + {ret = EXCP_BADARG(env, "Not aead cipher"); goto done;} if ((cipher = cipherp->cipher.p) == NULL) - return enif_raise_exception(env, atom_notsup); + {ret = EXCP_NOTSUP(env, "Cipher not supported in this libcrypto version"); goto done;} ctx_ctrl_set_ivlen = cipherp->extra.aead.ctx_ctrl_set_ivlen; ctx_ctrl_set_tag = cipherp->extra.aead.ctx_ctrl_set_tag; if ((outp = enif_make_new_binary(env, in.size, &out)) == NULL) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} if ((ctx = EVP_CIPHER_CTX_new()) == NULL) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} if (EVP_DecryptInit_ex(ctx, cipher, NULL, NULL, NULL) != 1) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} if (EVP_CIPHER_CTX_ctrl(ctx, ctx_ctrl_set_ivlen, (int)iv.size, NULL) != 1) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} #if defined(HAVE_CCM) if (type == atom_aes_ccm) { if (EVP_CIPHER_CTX_ctrl(ctx, ctx_ctrl_set_tag, (int)tag.size, tag.data) != 1) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} if (EVP_DecryptInit_ex(ctx, NULL, NULL, key.data, iv.data) != 1) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} if (EVP_DecryptUpdate(ctx, NULL, &len, NULL, (int)in.size) != 1) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} } else #endif { if (EVP_DecryptInit_ex(ctx, NULL, NULL, key.data, iv.data) != 1) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} } if (EVP_DecryptUpdate(ctx, NULL, &len, aad.data, (int)aad.size) != 1) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} if (EVP_DecryptUpdate(ctx, outp, &len, in.data, (int)in.size) != 1) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} #if defined(HAVE_GCM) if (type == atom_aes_gcm) { if (EVP_CIPHER_CTX_ctrl(ctx, ctx_ctrl_set_tag, (int)tag.size, tag.data) != 1) - goto err; + goto err; if (EVP_DecryptFinal_ex(ctx, outp+len, &len) != 1) goto err; } @@ -225,11 +217,8 @@ ERL_NIF_TERM aead_decrypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) ret = out; goto done; - bad_arg: - ret = enif_make_badarg(env); - goto done; - err: + /* Decrypt failed, that is, wrong tag */ ret = atom_error; done: @@ -238,6 +227,6 @@ ERL_NIF_TERM aead_decrypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) return ret; #else - return enif_raise_exception(env, atom_notsup); + return EXCP_NOTSUP(env, ""); #endif } diff --git a/lib/crypto/c_src/algorithms.c b/lib/crypto/c_src/algorithms.c index 06cd109fc1..1d45ed9df2 100644 --- a/lib/crypto/c_src/algorithms.c +++ b/lib/crypto/c_src/algorithms.c @@ -68,9 +68,15 @@ void init_algorithms_types(ErlNifEnv* env) // Non-validated algorithms follow algo_hash_fips_cnt = algo_hash_cnt; +#ifdef HAVE_MD4 algo_hash[algo_hash_cnt++] = enif_make_atom(env, "md4"); +#endif +#ifdef HAVE_MD5 algo_hash[algo_hash_cnt++] = enif_make_atom(env, "md5"); +#endif +#ifdef HAVE_RIPEMD160 algo_hash[algo_hash_cnt++] = enif_make_atom(env, "ripemd160"); +#endif algo_pubkey_cnt = 0; algo_pubkey[algo_pubkey_cnt++] = enif_make_atom(env, "rsa"); diff --git a/lib/crypto/c_src/api_ng.c b/lib/crypto/c_src/api_ng.c index 6a833a0984..5d063c3ae4 100644 --- a/lib/crypto/c_src/api_ng.c +++ b/lib/crypto/c_src/api_ng.c @@ -29,18 +29,6 @@ ERL_NIF_TERM ng_crypto_update(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); ERL_NIF_TERM ng_crypto_one_shot(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); - - -/* All nif functions return a valid value or throws an exception */ -#define EXCP(Env, Class, Str) enif_raise_exception((Env), \ - enif_make_tuple2((Env), (Class), \ - enif_make_string((Env),(Str),(ERL_NIF_LATIN1)) )) - -#define EXCP_NOTSUP(Env, Str) EXCP((Env), atom_notsup, (Str)) -#define EXCP_BADARG(Env, Str) EXCP((Env), atom_badarg, (Str)) -#define EXCP_ERROR(Env, Str) EXCP((Env), atom_error, (Str)) - - #ifdef HAVE_ECB_IVEC_BUG /* <= 0.9.8l returns faulty ivec length */ # define GET_IV_LEN(Ciph) ((Ciph)->flags & ECB_BUG_0_9_8L) ? 0 : EVP_CIPHER_iv_length((Ciph)->cipher.p) @@ -207,7 +195,7 @@ static int get_init_args(ErlNifEnv* env, goto err; } - +#ifdef HAVE_RC2 if (EVP_CIPHER_type((*cipherp)->cipher.p) == NID_rc2_cbc) { if (key_bin.size > INT_MAX / 8) { *return_term = EXCP_BADARG(env, "To large rc2_cbc key"); @@ -218,6 +206,7 @@ static int get_init_args(ErlNifEnv* env, goto err; } } +#endif if (ivec_arg == atom_undefined || ivec_len == 0) { @@ -346,7 +335,7 @@ ERL_NIF_TERM ng_crypto_init_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM arg ret = enif_make_resource(env, ctx_res); if(ctx_res) enif_release_resource(ctx_res); - } else if (enif_get_resource(env, argv[0], evp_cipher_ctx_rtype, (void**)&ctx_res)) { + } else if (enif_get_resource(env, argv[0], (ErlNifResourceType*)evp_cipher_ctx_rtype, (void**)&ctx_res)) { /* Fetch the flag telling if we are going to encrypt (=true) or decrypt (=false) */ if (argv[3] == atom_true) encflg = 1; @@ -426,7 +415,7 @@ ERL_NIF_TERM ng_crypto_update(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[ struct evp_cipher_ctx *ctx_res; ERL_NIF_TERM ret; - if (!enif_get_resource(env, argv[0], evp_cipher_ctx_rtype, (void**)&ctx_res)) + if (!enif_get_resource(env, argv[0], (ErlNifResourceType*)evp_cipher_ctx_rtype, (void**)&ctx_res)) return EXCP_BADARG(env, "Bad 1:st arg"); if (argc == 3) { diff --git a/lib/crypto/c_src/atoms.c b/lib/crypto/c_src/atoms.c index 114e3c1985..0793ffa6ca 100644 --- a/lib/crypto/c_src/atoms.c +++ b/lib/crypto/c_src/atoms.c @@ -52,6 +52,12 @@ ERL_NIF_TERM atom_ecb_mode; ERL_NIF_TERM atom_cbc_mode; ERL_NIF_TERM atom_cfb_mode; ERL_NIF_TERM atom_ofb_mode; +ERL_NIF_TERM atom_ctr_mode; +ERL_NIF_TERM atom_gcm_mode; +ERL_NIF_TERM atom_ccm_mode; +ERL_NIF_TERM atom_xts_mode; +ERL_NIF_TERM atom_wrap_mode; +ERL_NIF_TERM atom_ocb_mode; ERL_NIF_TERM atom_stream_cipher; #if defined(HAVE_EC) @@ -164,6 +170,12 @@ int init_atoms(ErlNifEnv *env, const ERL_NIF_TERM fips_mode, const ERL_NIF_TERM atom_cbc_mode = enif_make_atom(env,"cbc_mode"); atom_cfb_mode = enif_make_atom(env,"cfb_mode"); atom_ofb_mode = enif_make_atom(env,"ofb_mode"); + atom_ctr_mode = enif_make_atom(env,"ctr_mode"); + atom_gcm_mode = enif_make_atom(env,"gcm_mode"); + atom_ccm_mode = enif_make_atom(env,"ccm_mode"); + atom_xts_mode = enif_make_atom(env,"xts_mode"); + atom_wrap_mode = enif_make_atom(env,"wrap_mode"); + atom_ocb_mode = enif_make_atom(env,"ocb_mode"); atom_stream_cipher = enif_make_atom(env,"stream_cipher"); #if defined(HAVE_EC) diff --git a/lib/crypto/c_src/atoms.h b/lib/crypto/c_src/atoms.h index fc46d838aa..24f6dc26fd 100644 --- a/lib/crypto/c_src/atoms.h +++ b/lib/crypto/c_src/atoms.h @@ -56,6 +56,12 @@ extern ERL_NIF_TERM atom_ecb_mode; extern ERL_NIF_TERM atom_cbc_mode; extern ERL_NIF_TERM atom_cfb_mode; extern ERL_NIF_TERM atom_ofb_mode; +extern ERL_NIF_TERM atom_ctr_mode; +extern ERL_NIF_TERM atom_gcm_mode; +extern ERL_NIF_TERM atom_ccm_mode; +extern ERL_NIF_TERM atom_xts_mode; +extern ERL_NIF_TERM atom_wrap_mode; +extern ERL_NIF_TERM atom_ocb_mode; extern ERL_NIF_TERM atom_stream_cipher; #if defined(HAVE_EC) diff --git a/lib/crypto/c_src/cipher.c b/lib/crypto/c_src/cipher.c index 5c57898c50..2652e1db4e 100644 --- a/lib/crypto/c_src/cipher.c +++ b/lib/crypto/c_src/cipher.c @@ -28,12 +28,12 @@ static struct cipher_type_t cipher_types[] = { -#ifndef OPENSSL_NO_RC2 +#ifdef HAVE_RC2 {{"rc2_cbc"}, {&EVP_rc2_cbc}, 0, NO_FIPS_CIPHER}, #else {{"rc2_cbc"}, {NULL}, 0, NO_FIPS_CIPHER}, #endif -#ifndef OPENSSL_NO_RC4 +#ifdef HAVE_RC4 {{"rc4"}, {&EVP_rc4}, 0, NO_FIPS_CIPHER}, #else {{"rc4"}, {NULL}, 0, NO_FIPS_CIPHER}, @@ -274,6 +274,42 @@ ERL_NIF_TERM cipher_info_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[] ret_mode = atom_ofb_mode; break; +#ifdef EVP_CIPH_CTR_MODE + case EVP_CIPH_CTR_MODE: + ret_mode = atom_ctr_mode; + break; +#endif + +#ifdef EVP_CIPH_GCM_MODE + case EVP_CIPH_GCM_MODE: + ret_mode = atom_gcm_mode; + break; +#endif + +#ifdef EVP_CIPH_CCM_MODE + case EVP_CIPH_CCM_MODE: + ret_mode = atom_ccm_mode; + break; +#endif + +#ifdef EVP_CIPH_XTS_MODE + case EVP_CIPH_XTS_MODE: + ret_mode = atom_xts_mode; + break; +#endif + +#ifdef EVP_CIPH_WRAP_MODE + case EVP_CIPH_WRAP_MODE: + ret_mode = atom_wrap_mode; + break; +#endif + +#ifdef EVP_CIPH_OCB_MODE + case EVP_CIPH_OCB_MODE: + ret_mode = atom_ocb_mode; + break; +#endif + case EVP_CIPH_STREAM_CIPHER: ret_mode = atom_stream_cipher; break; diff --git a/lib/crypto/c_src/common.h b/lib/crypto/c_src/common.h index 2bc8bdd73c..0bf7f09f4f 100644 --- a/lib/crypto/c_src/common.h +++ b/lib/crypto/c_src/common.h @@ -35,4 +35,15 @@ #include "openssl_config.h" #include "atoms.h" + +/* All nif functions return a valid value or throws an exception */ +#define EXCP(Env, Id, Str) enif_raise_exception((Env), \ + enif_make_tuple2((Env), \ + (Id), \ + enif_make_string((Env),(Str),(ERL_NIF_LATIN1)) )) + +#define EXCP_NOTSUP(Env, Str) EXCP((Env), atom_notsup, (Str)) +#define EXCP_BADARG(Env, Str) EXCP((Env), atom_badarg, (Str)) +#define EXCP_ERROR(Env, Str) EXCP((Env), atom_error, (Str)) + #endif /* E_COMMON_H__ */ diff --git a/lib/crypto/c_src/digest.c b/lib/crypto/c_src/digest.c index fec286c000..c987a664d5 100644 --- a/lib/crypto/c_src/digest.c +++ b/lib/crypto/c_src/digest.c @@ -22,10 +22,32 @@ static struct digest_type_t digest_types[] = { - {{"md4"}, {&EVP_md4}}, - {{"md5"}, {&EVP_md5}}, - {{"ripemd160"}, {&EVP_ripemd160}}, + {{"md4"}, +#ifdef HAVE_MD4 + {&EVP_md4} +#else + {NULL} +#endif + }, + + {{"md5"}, +#ifdef HAVE_MD5 + {&EVP_md5} +#else + {NULL} +#endif + }, + + {{"ripemd160"}, +#ifdef HAVE_RIPEMD160 + {&EVP_ripemd160} +#else + {NULL} +#endif + }, + {{"sha"}, {&EVP_sha1}}, + {{"sha224"}, #ifdef HAVE_SHA224 {&EVP_sha224} @@ -33,6 +55,7 @@ static struct digest_type_t digest_types[] = {NULL} #endif }, + {{"sha256"}, #ifdef HAVE_SHA256 {&EVP_sha256} @@ -40,6 +63,7 @@ static struct digest_type_t digest_types[] = {NULL} #endif }, + {{"sha384"}, #ifdef HAVE_SHA384 {&EVP_sha384} @@ -47,6 +71,7 @@ static struct digest_type_t digest_types[] = {NULL} #endif }, + {{"sha512"}, #ifdef HAVE_SHA512 {&EVP_sha512} @@ -54,6 +79,7 @@ static struct digest_type_t digest_types[] = {NULL} #endif }, + {{"sha3_224"}, #ifdef HAVE_SHA3_224 {&EVP_sha3_224} @@ -61,6 +87,7 @@ static struct digest_type_t digest_types[] = {NULL} #endif }, + {{"sha3_256"}, #ifdef HAVE_SHA3_256 {&EVP_sha3_256} @@ -68,6 +95,7 @@ static struct digest_type_t digest_types[] = {NULL} #endif }, + {{"sha3_384"}, #ifdef HAVE_SHA3_384 {&EVP_sha3_384} @@ -75,6 +103,7 @@ static struct digest_type_t digest_types[] = {NULL} #endif }, + {{"sha3_512"}, #ifdef HAVE_SHA3_512 {&EVP_sha3_512} @@ -82,6 +111,7 @@ static struct digest_type_t digest_types[] = {NULL} #endif }, + {{"blake2b"}, #ifdef HAVE_BLAKE2 {&EVP_blake2b512} @@ -89,6 +119,7 @@ static struct digest_type_t digest_types[] = {NULL} #endif }, + {{"blake2s"}, #ifdef HAVE_BLAKE2 {&EVP_blake2s256} diff --git a/lib/crypto/c_src/engine.c b/lib/crypto/c_src/engine.c index 7ffbb9e70d..ea5d9a588f 100644 --- a/lib/crypto/c_src/engine.c +++ b/lib/crypto/c_src/engine.c @@ -106,15 +106,13 @@ int init_engine_ctx(ErlNifEnv *env) { (ErlNifResourceDtor*) engine_ctx_dtor, ERL_NIF_RT_CREATE|ERL_NIF_RT_TAKEOVER, NULL); - if (engine_ctx_rtype == NULL) - goto err; + if (engine_ctx_rtype == NULL) { + PRINTF_ERR0("CRYPTO: Could not open resource type 'ENGINE_CTX'"); + return 0; + } #endif return 1; - - err: - PRINTF_ERR0("CRYPTO: Could not open resource type 'ENGINE_CTX'"); - return 0; } ERL_NIF_TERM engine_by_id_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) diff --git a/lib/crypto/c_src/hash.c b/lib/crypto/c_src/hash.c index 0a9f64acef..9b79258585 100644 --- a/lib/crypto/c_src/hash.c +++ b/lib/crypto/c_src/hash.c @@ -21,9 +21,15 @@ #include "hash.h" #include "digest.h" -#define MD5_CTX_LEN (sizeof(MD5_CTX)) -#define MD4_CTX_LEN (sizeof(MD4_CTX)) -#define RIPEMD160_CTX_LEN (sizeof(RIPEMD160_CTX)) +#ifdef HAVE_MD5 +# define MD5_CTX_LEN (sizeof(MD5_CTX)) +#endif +#ifdef HAVE_MD4 +# define MD4_CTX_LEN (sizeof(MD4_CTX)) +#endif +#ifdef HAVE_RIPEMD160 +# define RIPEMD160_CTX_LEN (sizeof(RIPEMD160_CTX)) +#endif #if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,0,0) struct evp_md_ctx { @@ -261,18 +267,24 @@ ERL_NIF_TERM hash_init_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) switch (EVP_MD_type(digp->md.p)) { +#ifdef HAVE_MD4 case NID_md4: ctx_size = MD4_CTX_LEN; ctx_init = (init_fun)(&MD4_Init); break; +#endif +#ifdef HAVE_MD5 case NID_md5: ctx_size = MD5_CTX_LEN; ctx_init = (init_fun)(&MD5_Init); break; +#endif +#ifdef HAVE_RIPEMD160 case NID_ripemd160: ctx_size = RIPEMD160_CTX_LEN; ctx_init = (init_fun)(&RIPEMD160_Init); break; +#endif case NID_sha1: ctx_size = sizeof(SHA_CTX); ctx_init = (init_fun)(&SHA1_Init); @@ -352,18 +364,24 @@ ERL_NIF_TERM hash_update_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[] switch (EVP_MD_type(digp->md.p)) { +#ifdef HAVE_MD4 case NID_md4: ctx_size = MD4_CTX_LEN; ctx_update = (update_fun)(&MD4_Update); break; +#endif +#ifdef HAVE_MD5 case NID_md5: ctx_size = MD5_CTX_LEN; ctx_update = (update_fun)(&MD5_Update); break; +#endif +#ifdef HAVE_RIPEMD160 case NID_ripemd160: ctx_size = RIPEMD160_CTX_LEN; ctx_update = (update_fun)(&RIPEMD160_Update); break; +#endif case NID_sha1: ctx_size = sizeof(SHA_CTX); ctx_update = (update_fun)(&SHA1_Update); @@ -448,18 +466,24 @@ ERL_NIF_TERM hash_final_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) switch (EVP_MD_type(md)) { +#ifdef HAVE_MD4 case NID_md4: ctx_size = MD4_CTX_LEN; ctx_final = (final_fun)(&MD4_Final); break; +#endif +#ifdef HAVE_MD5 case NID_md5: ctx_size = MD5_CTX_LEN; ctx_final = (final_fun)(&MD5_Final); break; - case NID_ripemd160: +#endif +#ifdef HAVE_RIPEMD160 + case NID_ripemd160: ctx_size = RIPEMD160_CTX_LEN; ctx_final = (final_fun)(&RIPEMD160_Final); break; +#endif case NID_sha1: ctx_size = sizeof(SHA_CTX); ctx_final = (final_fun)(&SHA1_Final); diff --git a/lib/crypto/c_src/hmac.c b/lib/crypto/c_src/hmac.c index c41e50eb35..ff7005d75e 100644 --- a/lib/crypto/c_src/hmac.c +++ b/lib/crypto/c_src/hmac.c @@ -181,7 +181,7 @@ ERL_NIF_TERM hmac_update_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[] ASSERT(argc == 2); - if (!enif_get_resource(env, argv[0], hmac_context_rtype, (void**)&obj)) + if (!enif_get_resource(env, argv[0], (ErlNifResourceType*)hmac_context_rtype, (void**)&obj)) goto bad_arg; if (!enif_inspect_iolist_as_binary(env, argv[1], &data)) goto bad_arg; @@ -224,7 +224,7 @@ ERL_NIF_TERM hmac_final_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) ASSERT(argc == 1 || argc == 2); - if (!enif_get_resource(env, argv[0], hmac_context_rtype, (void**)&obj)) + if (!enif_get_resource(env, argv[0], (ErlNifResourceType*)hmac_context_rtype, (void**)&obj)) goto bad_arg; if (argc == 2) { if (!enif_get_uint(env, argv[1], &req_len)) diff --git a/lib/crypto/c_src/openssl_config.h b/lib/crypto/c_src/openssl_config.h index 46868cb987..f926f8af13 100644 --- a/lib/crypto/c_src/openssl_config.h +++ b/lib/crypto/c_src/openssl_config.h @@ -166,6 +166,28 @@ # define HAVE_BLAKE2 #endif +#ifndef OPENSSL_NO_MD4 +# define HAVE_MD4 +#endif + +#ifndef OPENSSL_NO_MD5 +# define HAVE_MD5 +#endif + +#ifndef OPENSSL_NO_RC2 +# define HAVE_RC2 +#endif + +#ifndef OPENSSL_NO_RC4 +# define HAVE_RC4 +#endif + +#ifndef OPENSSL_NO_RMD160 +/* Note RMD160 vs RIPEMD160 */ +# define HAVE_RIPEMD160 +#endif + + #if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION(0,9,8,'o') \ && !defined(OPENSSL_NO_EC) \ && !defined(OPENSSL_NO_ECDH) \ @@ -192,7 +214,9 @@ # define HAVE_AEAD # define HAVE_GCM # define HAVE_CCM -# define HAVE_CMAC +# ifndef OPENSSL_NO_CMAC +# define HAVE_CMAC +# endif # if defined(RSA_PKCS1_OAEP_PADDING) # define HAVE_RSA_OAEP_PADDING # endif @@ -204,21 +228,27 @@ #if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,1,0) # ifndef HAS_LIBRESSL -# define HAVE_CHACHA20_POLY1305 +# if !defined(OPENSSL_NO_CHACHA) && !defined(OPENSSL_NO_POLY1305) +# define HAVE_CHACHA20_POLY1305 +# endif # define HAVE_RSA_OAEP_MD # endif #endif #if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION(1,1,0,'d') # ifndef HAS_LIBRESSL -# define HAVE_CHACHA20 +# ifndef OPENSSL_NO_CHACHA +# define HAVE_CHACHA20 +# endif # endif #endif // OPENSSL_VERSION_NUMBER >= 1.1.1-pre8 #if OPENSSL_VERSION_NUMBER >= (PACKED_OPENSSL_VERSION_PLAIN(1,1,1)-7) # ifndef HAS_LIBRESSL -# define HAVE_POLY1305 +# if !defined(OPENSSL_NO_POLY1305) +# define HAVE_POLY1305 +# endif # endif #endif diff --git a/lib/crypto/doc/src/crypto.xml b/lib/crypto/doc/src/crypto.xml index 83e10c4c78..8a4fad67de 100644 --- a/lib/crypto/doc/src/crypto.xml +++ b/lib/crypto/doc/src/crypto.xml @@ -192,7 +192,16 @@ <datatypes> <datatype_title>Ciphers</datatype_title> <datatype> + <name name="cipher"/> <name name="stream_cipher"/> + <name name="block_cipher"/> + <desc> + <p>Ciphers known byt the CRYPTO application. Note that this list might be reduced if the + underlying libcrypto does not support all of them.</p> + </desc> + </datatype> + + <datatype> <name name="stream_cipher_iv"/> <name name="stream_cipher_no_iv"/> <desc> @@ -204,7 +213,7 @@ </datatype> <datatype> - <name name="block_cipher_with_iv"/> + <name name="block_cipher_iv"/> <name name="cbc_cipher"/> <name name="cfb_cipher"/> <desc> @@ -228,7 +237,7 @@ </datatype> <datatype> - <name name="block_cipher_without_iv"/> + <name name="block_cipher_no_iv"/> <name name="ecb_cipher"/> <desc> <p>Block ciphers without initialization vector for @@ -248,20 +257,16 @@ </desc> </datatype> - <datatype_title>Digests</datatype_title> + <datatype_title>Digests and hash</datatype_title> <datatype> - <name name="sha1"/> - <name name="sha2"/> - <name name="sha3"/> - <name name="blake2"/> + <name name="hash_algorithm"/> <desc> </desc> </datatype> <datatype> - <name name="compatibility_only_hash"/> + <name name="hmac_hash_algorithm"/> <desc> - <p>The <c>compatibility_only_hash()</c> algorithms are recommended only for compatibility with existing applications.</p> </desc> </datatype> @@ -283,6 +288,17 @@ </desc> </datatype> + <datatype> + <name name="sha1"/> + <name name="sha2"/> + <name name="sha3"/> + <name name="blake2"/> + <name name="compatibility_only_hash"/> + <desc> + <p>The <c>compatibility_only_hash()</c> algorithms are recommended only for compatibility with existing applications.</p> + </desc> + </datatype> + <datatype_title>Elliptic Curves</datatype_title> <datatype> <name name="ec_named_curve"/> @@ -537,6 +553,52 @@ </desc> </datatype> + <datatype_title>Error types</datatype_title> + + <datatype> + <name name="run_time_error"/> + <desc> + <p>The exception <c>error:badarg</c> signifies that one or more arguments are of wrong data type, + or are otherwise badly formed. + </p> + <p>The exception <c>error:notsup</c> signifies that the algorithm is known but is not supported + by current underlying libcrypto or explicitly disabled when building that. + </p> + <p>For a list of supported algorithms, see <seealso marker="#supports-0">supports/0</seealso>. + </p> + </desc> + </datatype> + + <datatype> + <name name="descriptive_error"/> + <desc> + <p>This is a more developed variant of the older + <seealso marker="#type-run_time_error">run_time_error()</seealso>. + </p> + <p>It is like the older type an exception of the <c>error</c> class. In addition they contain + a descriptive text in English. That text is targeted to a developer. Examples are "Bad key size" + or "Cipher id is not an atom". + </p> + <p>The exceptions are:</p> + <taglist> + <tag><c>{badarg, Description::string()}</c></tag> + <item><p>Signifies that one or more arguments are of wrong data type or are otherwise badly formed.</p> + </item> + + <tag><c>{notsup, Description::string()}</c></tag> + <item><p>Signifies that the algorithm is known but is not supported by current underlying libcrypto + or explicitly disabled when building that one.</p> + </item> + + <tag><c>{error, Description::string()}</c></tag> + <item><p>An error condition that should not occur, for example a memory allocation failed or + the underlying cryptolib returned an error code, for example "Can't initialize context, step 1". + Thoose text usually needs searching the C-code to be understood.</p> + </item> + </taglist> + </desc> + </datatype> + </datatypes> <!--================ FUNCTIONS ================--> @@ -568,17 +630,18 @@ </func> <func> - <name since="OTP R16B01">block_encrypt(Type, Key, Ivec, PlainText) -> CipherText</name> - <name since="OTP R16B01">block_encrypt(AeadType, Key, Ivec, {AAD, PlainText}) -> {CipherText, CipherTag}</name> - <name since="OTP R16B01">block_encrypt(aes_gcm | aes_ccm, Key, Ivec, {AAD, PlainText, TagLength}) -> {CipherText, CipherTag}</name> + <name since="OTP R16B01">block_encrypt(Type, Key, Ivec, PlainText) -> CipherText | Error</name> + <name since="OTP R16B01">block_encrypt(AeadType, Key, Ivec, {AAD, PlainText}) -> {CipherText, CipherTag} | Error</name> + <name since="OTP R16B01">block_encrypt(aes_gcm | aes_ccm, Key, Ivec, {AAD, PlainText, TagLength}) -> {CipherText, CipherTag} | Error </name> <fsummary>Encrypt <c>PlainText</c> according to <c>Type</c> block cipher</fsummary> <type> - <v>Type = <seealso marker="#type-block_cipher_with_iv">block_cipher_with_iv()</seealso></v> + <v>Type = <seealso marker="#type-block_cipher_iv">block_cipher_iv()</seealso></v> <v>AeadType = <seealso marker="#type-aead_cipher">aead_cipher()</seealso></v> <v>Key = <seealso marker="#type-key">key()</seealso> | <seealso marker="#type-des3_key">des3_key()</seealso></v> <v>PlainText = iodata()</v> <v>AAD = IVec = CipherText = CipherTag = binary()</v> <v>TagLength = 1..16</v> + <v>Error = <seealso marker="#type-run_time_error">run_time_error()</seealso></v> </type> <desc> <p>Encrypt <c>PlainText</c> according to <c>Type</c> block cipher. @@ -595,15 +658,17 @@ </func> <func> - <name since="OTP R16B01">block_decrypt(Type, Key, Ivec, CipherText) -> PlainText</name> - <name since="OTP R16B01">block_decrypt(AeadType, Key, Ivec, {AAD, CipherText, CipherTag}) -> PlainText | error</name> + <name since="OTP R16B01">block_decrypt(Type, Key, Ivec, CipherText) -> PlainText | Error</name> + <name since="OTP R16B01">block_decrypt(AeadType, Key, Ivec, {AAD, CipherText, CipherTag}) -> PlainText | Error</name> <fsummary>Decrypt <c>CipherText</c> according to <c>Type</c> block cipher</fsummary> <type> - <v>Type = <seealso marker="#type-block_cipher_with_iv">block_cipher_with_iv()</seealso></v> + <v>Type = <seealso marker="#type-block_cipher_iv">block_cipher_iv()</seealso></v> <v>AeadType = <seealso marker="#type-aead_cipher">aead_cipher()</seealso></v> <v>Key = <seealso marker="#type-key">key()</seealso> | <seealso marker="#type-des3_key">des3_key()</seealso></v> <v>PlainText = iodata()</v> <v>AAD = IVec = CipherText = CipherTag = binary()</v> + <v>Error = BadTag | <seealso marker="#type-run_time_error">run_time_error()</seealso></v> + <v>BadTag = error</v> </type> <desc> <p>Decrypt <c>CipherText</c> according to <c>Type</c> block cipher. @@ -844,6 +909,39 @@ </func> <func> + <name name="hash_info" arity="1" since="OTP 22.0"/> + <fsummary>Information about supported hash algorithms.</fsummary> + <desc> + <p>Provides a map with information about block_size, size and possibly other properties of the + hash algorithm in question. + </p> + <p>For a list of supported hash algorithms, see <seealso marker="#supports-0">supports/0</seealso>. + </p> + </desc> + </func> + + <func> + <name name="cipher_info" arity="1" since="OTP 22.0"/> + <fsummary>Information about supported ciphers.</fsummary> + <desc> + <p>Provides a map with information about block_size, key_length, iv_length and possibly other properties of the + cipher algorithm in question. + </p> + <note> + <p>The ciphers <c>aes_cbc</c>, <c>aes_cfb8</c>, <c>aes_cfb128</c>, <c>aes_ctr</c>, + <c>aes_ecb</c>, <c>aes_gcm</c> and <c>aes_ccm</c> + has no keylength in the <c>Type</c> as opposed to for example <c>aes_128_ctr</c>. They adapt to the length of + the key provided in the encrypt and decrypt function. Therefor it is impossible to return a valid keylength + in the map.</p> + <p>Always use a <c>Type</c> with an explicit key length, + </p> + </note> + <p>For a list of supported cipher algorithms, see <seealso marker="#supports-0">supports/0</seealso>. + </p> + </desc> + </func> + + <func> <name name="mod_pow" arity="3" since="OTP R16B01"/> <fsummary>Computes the function: N^P mod M</fsummary> <desc> @@ -1289,8 +1387,8 @@ FloatValue = rand:uniform(). % again <desc> <p> Can be used to determine which crypto algorithms that are supported by the underlying libcrypto library</p> - <p>Note: the <c>rsa_opts</c> entry is in an experimental state and may change or be removed without notice. - No guarantee for the accuarcy of the rsa option's value list should be assumed. + <p>See <seealso marker="#hash_info-1">hash_info/1</seealso> and <seealso marker="#cipher_info-1">cipher_info/1</seealso> + for information about the hash and cipher algorithms. </p> </desc> </func> diff --git a/lib/crypto/src/crypto.erl b/lib/crypto/src/crypto.erl index 5cf34f8069..fd13481951 100644 --- a/lib/crypto/src/crypto.erl +++ b/lib/crypto/src/crypto.erl @@ -277,7 +277,13 @@ -type edwards_curve_ed() :: ed25519 | ed448 . %%% --type block_cipher_with_iv() :: cbc_cipher() +-type cipher() :: block_cipher() + | stream_cipher() + | aead_cipher() . + +-type block_cipher() :: block_cipher_iv() | block_cipher_no_iv() . + +-type block_cipher_iv() :: cbc_cipher() | cfb_cipher() | aes_ige256 | blowfish_ofb64 @@ -310,7 +316,7 @@ | des3_cfb . --type block_cipher_without_iv() :: ecb_cipher() . +-type block_cipher_no_iv() :: ecb_cipher() . -type ecb_cipher() :: des_ecb | blowfish_ecb | aes_ecb . -type key() :: iodata(). @@ -330,6 +336,20 @@ -type crypto_integer() :: binary() | integer(). +%%% +%% Exceptions +%% error:badarg +%% error:notsup +-type run_time_error() :: no_return(). + +%% Exceptions +%% error:{badarg,Reason::term()} +%% error:{notsup,Reason::term()} +%% error:{error,Reason::term()} +-type descriptive_error() :: no_return() . + + +%%-------------------------------------------------------------------- -compile(no_native). -on_load(on_load/0). -define(CRYPTO_NIF_VSN,302). @@ -368,10 +388,7 @@ stop() -> | {curves, Curves} | {rsa_opts, RSAopts}, Hashs :: [sha1() | sha2() | sha3() | blake2() | ripemd160 | compatibility_only_hash()], - Ciphers :: [stream_cipher() - | block_cipher_with_iv() | block_cipher_without_iv() - | aead_cipher() - ], + Ciphers :: [cipher()], PKs :: [rsa | dss | ecdsa | dh | ecdh | ec_gf2m], Macs :: [hmac | cmac | poly1305], Curves :: [ec_named_curve() | edwards_curve_dh() | edwards_curve_ed()], @@ -405,14 +422,18 @@ enable_fips_mode(_) -> ?nif_stub. %%% %%%================================================================ --define(HASH_HASH_ALGORITHM, sha1() | sha2() | sha3() | blake2() | ripemd160 | compatibility_only_hash() ). - --spec hash_info(Type) -> map() when Type :: ?HASH_HASH_ALGORITHM. +-type hash_algorithm() :: sha1() | sha2() | sha3() | blake2() | ripemd160 | compatibility_only_hash() . +-spec hash_info(Type) -> Result | run_time_error() + when Type :: hash_algorithm(), + Result :: #{size := integer(), + block_size := integer(), + type := integer() + } . hash_info(Type) -> notsup_to_error(hash_info_nif(Type)). --spec hash(Type, Data) -> Digest when Type :: ?HASH_HASH_ALGORITHM, +-spec hash(Type, Data) -> Digest when Type :: hash_algorithm(), Data :: iodata(), Digest :: binary(). hash(Type, Data) -> @@ -422,7 +443,7 @@ hash(Type, Data) -> -opaque hash_state() :: reference(). --spec hash_init(Type) -> State when Type :: ?HASH_HASH_ALGORITHM, +-spec hash_init(Type) -> State when Type :: hash_algorithm(), State :: hash_state(). hash_init(Type) -> notsup_to_error(hash_init_nif(Type)). @@ -448,12 +469,12 @@ hash_final(Context) -> %%%---- HMAC --define(HMAC_HASH_ALGORITHM, sha1() | sha2() | sha3() | compatibility_only_hash()). +-type hmac_hash_algorithm() :: sha1() | sha2() | sha3() | compatibility_only_hash(). %%%---- hmac/3,4 -spec hmac(Type, Key, Data) -> - Mac when Type :: ?HMAC_HASH_ALGORITHM, + Mac when Type :: hmac_hash_algorithm(), Key :: iodata(), Data :: iodata(), Mac :: binary() . @@ -462,7 +483,7 @@ hmac(Type, Key, Data) -> hmac(Type, Key, Data1, undefined, erlang:byte_size(Data1), max_bytes()). -spec hmac(Type, Key, Data, MacLength) -> - Mac when Type :: ?HMAC_HASH_ALGORITHM, + Mac when Type :: hmac_hash_algorithm(), Key :: iodata(), Data :: iodata(), MacLength :: integer(), @@ -477,7 +498,7 @@ hmac(Type, Key, Data, MacLength) -> -opaque hmac_state() :: binary(). -spec hmac_init(Type, Key) -> - State when Type :: ?HMAC_HASH_ALGORITHM, + State when Type :: hmac_hash_algorithm(), Key :: iodata(), State :: hmac_state() . hmac_init(Type, Key) -> @@ -541,79 +562,124 @@ poly1305(Key, Data) -> %%%================================================================ -define(COMPAT(CALL), - try CALL + try begin CALL end catch + error:{error,_} -> + error(badarg); error:{E,_Reason} when E==notsup ; E==badarg -> error(E) end). --spec cipher_info(Type) -> map() when Type :: block_cipher_with_iv() - | aead_cipher() - | block_cipher_without_iv(). +%%%---- Cipher info +%%%---------------------------------------------------------------- +-spec cipher_info(Type) -> Result | run_time_error() + when Type :: cipher(), + Result :: #{key_length := integer(), + iv_length := integer(), + block_size := integer(), + mode := CipherModes, + type := undefined | integer() + }, + CipherModes :: undefined + | cbc_mode + | ccm_mode + | cfb_mode + | ctr_mode + | ecb_mode + | gcm_mode + | ige_mode + | ocb_mode + | ofb_mode + | wrap_mode + | xts_mode + . + +%% These ciphers are not available via the EVP interface on older cryptolibs. +cipher_info(aes_ctr) -> + #{block_size => 1,iv_length => 16,key_length => 32,mode => ctr_mode,type => undefined}; +cipher_info(aes_128_ctr) -> + #{block_size => 1,iv_length => 16,key_length => 16,mode => ctr_mode,type => undefined}; +cipher_info(aes_192_ctr) -> + #{block_size => 1,iv_length => 16,key_length => 24,mode => ctr_mode,type => undefined}; +cipher_info(aes_256_ctr) -> + #{block_size => 1,iv_length => 16,key_length => 32,mode => ctr_mode,type => undefined}; +%% This cipher is handled specialy. +cipher_info(aes_ige256) -> + #{block_size => 16,iv_length => 32,key_length => 16,mode => ige_mode,type => undefined}; cipher_info(Type) -> - cipher_info_nif(Type). + cipher_info_nif(alias(Type)). %%%---- Block ciphers %%%---------------------------------------------------------------- --spec block_encrypt(Type::block_cipher_with_iv(), Key::key()|des3_key(), Ivec::binary(), PlainText::iodata()) -> binary(); +-spec block_encrypt(Type::block_cipher_iv(), Key::key()|des3_key(), Ivec::binary(), PlainText::iodata()) -> + binary() | run_time_error(); (Type::aead_cipher(), Key::iodata(), Ivec::binary(), {AAD::binary(), PlainText::iodata()}) -> - {binary(), binary()}; + {binary(), binary()} | run_time_error(); (aes_gcm | aes_ccm, Key::iodata(), Ivec::binary(), {AAD::binary(), PlainText::iodata(), TagLength::1..16}) -> - {binary(), binary()}. + {binary(), binary()} | run_time_error(). -block_encrypt(Type, Key, Ivec, Data) -> - do_block_encrypt(alias(Type), Key, Ivec, Data). - -do_block_encrypt(Type, Key, Ivec, PlainText) when Type =:= aes_ige256 -> +block_encrypt(aes_ige256, Key, Ivec, PlainText) -> notsup_to_error(aes_ige_crypt_nif(Key, Ivec, PlainText, true)); -do_block_encrypt(Type, Key, Ivec, {AAD, PlainText}) when Type =:= chacha20_poly1305 -> - aead_encrypt(Type, Key, Ivec, AAD, PlainText, 16); - -do_block_encrypt(Type, Key, Ivec, Data) when Type =:= aes_gcm; - Type =:= aes_ccm -> - case Data of - {AAD, PlainText} -> - aead_encrypt(Type, Key, Ivec, AAD, PlainText); - {AAD, PlainText, TagLength} -> - aead_encrypt(Type, Key, Ivec, AAD, PlainText, TagLength) - end; - -do_block_encrypt(Type, Key, Ivec, PlainText) -> - ?COMPAT(crypto_one_shot(Type, Key, Ivec, PlainText, true)). - - --spec block_encrypt(Type::block_cipher_without_iv(), Key::key(), PlainText::iodata()) -> binary(). - -block_encrypt(Type, Key, PlainText) -> - ?COMPAT(crypto_one_shot(Type, Key, <<>>, PlainText, true)). +block_encrypt(Type, Key0, Ivec, Data) -> + Key = iolist_to_binary(Key0), + ?COMPAT( + case Data of + {AAD, PlainText} -> + aead_encrypt(alias(Type,Key), Key, Ivec, AAD, PlainText, aead_tag_len(Type)); + {AAD, PlainText, TagLength} -> + aead_encrypt(alias(Type,Key), Key, Ivec, AAD, PlainText, TagLength); + PlainText -> + crypto_one_shot(alias(Type,Key), Key, Ivec, PlainText, true) + end). + +-spec block_encrypt(Type::block_cipher_no_iv(), Key::key(), PlainText::iodata()) -> + binary() | run_time_error(). + +block_encrypt(Type, Key0, PlainText) -> + Key = iolist_to_binary(Key0), + ?COMPAT(crypto_one_shot(alias(Type,Key), Key, <<>>, PlainText, true)). + + +aead_tag_len(chacha20_poly1305) -> 16; +aead_tag_len(aes_ccm) -> 12; +aead_tag_len(aes_128_ccm) -> 12; +aead_tag_len(aes_192_ccm) -> 12; +aead_tag_len(aes_256_ccm) -> 12; +aead_tag_len(aes_gcm) -> 16; +aead_tag_len(aes_128_gcm) -> 16; +aead_tag_len(aes_192_gcm) -> 16; +aead_tag_len(aes_256_gcm) -> 16. %%%---------------------------------------------------------------- %%%---------------------------------------------------------------- --spec block_decrypt(Type::block_cipher_with_iv(), Key::key()|des3_key(), Ivec::binary(), Data::iodata()) -> binary(); +-spec block_decrypt(Type::block_cipher_iv(), Key::key()|des3_key(), Ivec::binary(), Data::iodata()) -> + binary() | run_time_error(); (Type::aead_cipher(), Key::iodata(), Ivec::binary(), - {AAD::binary(), Data::iodata(), Tag::binary()}) -> binary() | error. - -block_decrypt(Type, Key, Ivec, Data) -> - do_block_decrypt(alias(Type), Key, Ivec, Data). + {AAD::binary(), Data::iodata(), Tag::binary()}) -> + binary() | error | run_time_error() . -do_block_decrypt(aes_ige256, Key, Ivec, Data) -> +block_decrypt(aes_ige256, Key, Ivec, Data) -> notsup_to_error(aes_ige_crypt_nif(Key, Ivec, Data, false)); -do_block_decrypt(Type, Key, Ivec, {AAD, Data, Tag}) when Type =:= aes_gcm; - Type =:= aes_ccm; - Type =:= chacha20_poly1305 -> - aead_decrypt(Type, Key, Ivec, AAD, Data, Tag); +block_decrypt(Type, Key0, Ivec, Data) -> + Key = iolist_to_binary(Key0), + ?COMPAT( + case Data of + {AAD, CryptoText, Tag} -> + aead_decrypt(alias(Type,Key), Key, Ivec, AAD, CryptoText, Tag); + CryptoText -> + crypto_one_shot(alias(Type,Key), Key, Ivec, CryptoText, false) + end). -do_block_decrypt(Type, Key, Ivec, Data) -> - ?COMPAT(crypto_one_shot(Type, Key, Ivec, Data, false)). +-spec block_decrypt(Type::block_cipher_no_iv(), Key::key(), Data::iodata()) -> + binary() | run_time_error(). --spec block_decrypt(Type::block_cipher_without_iv(), Key::key(), Data::iodata()) -> binary(). - -block_decrypt(Type, Key, Data) -> - ?COMPAT(crypto_one_shot(Type, Key, <<>>, Data, false)). +block_decrypt(Type, Key0, CryptoText) -> + Key = iolist_to_binary(Key0), + ?COMPAT(crypto_one_shot(alias(Type,Key), Key, <<>>, CryptoText, false)). %%%-------- Stream ciphers API @@ -630,32 +696,34 @@ block_decrypt(Type, Key, Data) -> | chacha20 . %%%---- stream_init --spec stream_init(Type, Key, IVec) -> State | no_return() +-spec stream_init(Type, Key, IVec) -> State | run_time_error() when Type :: stream_cipher_iv(), Key :: iodata(), IVec ::binary(), State :: stream_state() . -stream_init(Type, Key, IVec) when is_binary(IVec) -> - Ref = ?COMPAT(ng_crypto_init_nif(alias(Type), - iolist_to_binary(Key), iolist_to_binary(IVec), +stream_init(Type, Key0, IVec) when is_binary(IVec) -> + Key = iolist_to_binary(Key0), + Ref = ?COMPAT(ng_crypto_init_nif(alias(Type,Key), + Key, iolist_to_binary(IVec), undefined) ), {Type, {Ref,flg_undefined}}. --spec stream_init(Type, Key) -> State | no_return() +-spec stream_init(Type, Key) -> State | run_time_error() when Type :: stream_cipher_no_iv(), Key :: iodata(), State :: stream_state() . -stream_init(rc4 = Type, Key) -> - Ref = ?COMPAT(ng_crypto_init_nif(alias(Type), - iolist_to_binary(Key), <<>>, +stream_init(rc4 = Type, Key0) -> + Key = iolist_to_binary(Key0), + Ref = ?COMPAT(ng_crypto_init_nif(alias(Type,Key), + Key, <<>>, undefined) ), {Type, {Ref,flg_undefined}}. %%%---- stream_encrypt --spec stream_encrypt(State, PlainText) -> {NewState, CipherText} | no_return() +-spec stream_encrypt(State, PlainText) -> {NewState, CipherText} | run_time_error() when State :: stream_state(), PlainText :: iodata(), NewState :: stream_state(), @@ -664,7 +732,7 @@ stream_encrypt(State, Data) -> crypto_stream_emulate(State, Data, true). %%%---- stream_decrypt --spec stream_decrypt(State, CipherText) -> {NewState, PlainText} | no_return() +-spec stream_decrypt(State, CipherText) -> {NewState, PlainText} | run_time_error() when State :: stream_state(), CipherText :: iodata(), NewState :: stream_state(), @@ -723,8 +791,8 @@ next_iv(Type, Data, _Ivec) -> %%% Create and initialize a new state for encryption or decryption %%% --spec crypto_init(Cipher, Key, EncryptFlag) -> State | ng_crypto_error() - when Cipher :: block_cipher_without_iv() +-spec crypto_init(Cipher, Key, EncryptFlag) -> State | descriptive_error() + when Cipher :: block_cipher_no_iv() | stream_cipher_no_iv(), Key :: iodata(), EncryptFlag :: boolean(), @@ -734,9 +802,9 @@ crypto_init(Cipher, Key, EncryptFlag) -> ng_crypto_init_nif(alias(Cipher), iolist_to_binary(Key), <<>>, EncryptFlag). --spec crypto_init(Cipher, Key, IV, EncryptFlag) -> State | ng_crypto_error() +-spec crypto_init(Cipher, Key, IV, EncryptFlag) -> State | descriptive_error() when Cipher :: stream_cipher_iv() - | block_cipher_with_iv(), + | block_cipher_iv(), Key :: iodata(), IV :: iodata(), EncryptFlag :: boolean(), @@ -747,9 +815,9 @@ crypto_init(Cipher, Key, IV, EncryptFlag) -> %%%---------------------------------------------------------------- --spec crypto_init_dyn_iv(Cipher, Key, EncryptFlag) -> State | ng_crypto_error() +-spec crypto_init_dyn_iv(Cipher, Key, EncryptFlag) -> State | descriptive_error() when Cipher :: stream_cipher_iv() - | block_cipher_with_iv(), + | block_cipher_iv(), Key :: iodata(), EncryptFlag :: boolean(), State :: crypto_state() . @@ -764,7 +832,7 @@ crypto_init_dyn_iv(Cipher, Key, EncryptFlag) -> %%% blocksize. %%% --spec crypto_update(State, Data) -> Result | ng_crypto_error() +-spec crypto_update(State, Data) -> Result | descriptive_error() when State :: crypto_state(), Data :: iodata(), Result :: binary() . @@ -778,7 +846,7 @@ crypto_update(State, Data0) -> %%%---------------------------------------------------------------- --spec crypto_update_dyn_iv(State, Data, IV) -> Result | ng_crypto_error() +-spec crypto_update_dyn_iv(State, Data, IV) -> Result | descriptive_error() when State :: crypto_state(), Data :: iodata(), IV :: iodata(), @@ -798,15 +866,16 @@ crypto_update_dyn_iv(State, Data0, IV) -> %%% The size must be an integer multiple of the crypto's blocksize. %%% --spec crypto_one_shot(Cipher, Key, IV, Data, EncryptFlag) -> Result | ng_crypto_error() - when Cipher :: stream_cipher() - | block_cipher_with_iv() - | block_cipher_without_iv(), - Key :: iodata(), - IV :: iodata() | undefined, - Data :: iodata(), - EncryptFlag :: boolean(), - Result :: binary() . +-spec crypto_one_shot(Cipher, Key, IV, Data, EncryptFlag) -> + Result | descriptive_error() + when Cipher :: stream_cipher() + | block_cipher(), + Key :: iodata(), + IV :: iodata() | undefined, + Data :: iodata(), + EncryptFlag :: boolean(), + Result :: binary() . + crypto_one_shot(Cipher, Key, undefined, Data, EncryptFlag) -> crypto_one_shot(Cipher, Key, <<>>, Data, EncryptFlag); @@ -823,21 +892,25 @@ crypto_one_shot(Cipher, Key, IV, Data0, EncryptFlag) -> %%%---------------------------------------------------------------- %%% NIFs --type ng_crypto_error() :: no_return() . +-spec ng_crypto_init_nif(atom(), binary(), binary()|undefined, boolean()|undefined ) -> + crypto_state() | descriptive_error() + ; (crypto_state(), <<>>, <<>>, boolean()) + -> crypto_state() | descriptive_error(). --spec ng_crypto_init_nif(atom(), binary(), binary()|undefined, boolean()|undefined ) -> crypto_state() | ng_crypto_error() - ; (crypto_state(), <<>>, <<>>, boolean()) -> crypto_state() | ng_crypto_error(). ng_crypto_init_nif(_Cipher, _Key, _IVec, _EncryptFlg) -> ?nif_stub. --spec ng_crypto_update_nif(crypto_state(), binary()) -> binary() | ng_crypto_error() . +-spec ng_crypto_update_nif(crypto_state(), binary()) -> + binary() | descriptive_error() . ng_crypto_update_nif(_State, _Data) -> ?nif_stub. --spec ng_crypto_update_nif(crypto_state(), binary(), binary()) -> binary() | ng_crypto_error() . +-spec ng_crypto_update_nif(crypto_state(), binary(), binary()) -> + binary() | descriptive_error() . ng_crypto_update_nif(_State, _Data, _IV) -> ?nif_stub. --spec ng_crypto_one_shot_nif(atom(), binary(), binary(), binary(), boolean() ) -> binary() | ng_crypto_error(). +-spec ng_crypto_one_shot_nif(atom(), binary(), binary(), binary(), boolean() ) -> + binary() | descriptive_error(). ng_crypto_one_shot_nif(_Cipher, _Key, _IVec, _Data, _EncryptFlg) -> ?nif_stub. %%%---------------------------------------------------------------- @@ -859,6 +932,44 @@ alias(aes_cbc256) -> aes_256_cbc; alias(Alg) -> Alg. + +%%%---- des_ede3_cbc +alias(des3_cbc, _) -> des_ede3_cbc; +alias(des_ede3, _) -> des_ede3_cbc; +%%%---- des_ede3_cfb +alias(des_ede3_cbf,_ ) -> des_ede3_cfb; +alias(des3_cbf, _) -> des_ede3_cfb; +alias(des3_cfb, _) -> des_ede3_cfb; +%%%---- aes_*_cbc +alias(aes_cbc128, _) -> aes_128_cbc; +alias(aes_cbc256, _) -> aes_256_cbc; + +alias(aes_cbc, Key) when size(Key)==128 -> aes_128_cbc; +alias(aes_cbc, Key) when size(Key)==192 -> aes_192_cbc; +alias(aes_cbc, Key) when size(Key)==256 -> aes_256_cbc; + +alias(aes_cfb8, Key) when size(Key)==128 -> aes_128_cfb8; +alias(aes_cfb8, Key) when size(Key)==192 -> aes_192_cfb8; +alias(aes_cfb8, Key) when size(Key)==256 -> aes_256_cfb8; + +alias(aes_cfb128, Key) when size(Key)==128 -> aes_128_cfb128; +alias(aes_cfb128, Key) when size(Key)==192 -> aes_192_cfb128; +alias(aes_cfb128, Key) when size(Key)==256 -> aes_256_cfb128; + +alias(aes_ctr, Key) when size(Key)==128 -> aes_128_ctr; +alias(aes_ctr, Key) when size(Key)==192 -> aes_192_ctr; +alias(aes_ctr, Key) when size(Key)==256 -> aes_256_ctr; + +alias(aes_gcm, Key) when size(Key)==128 -> aes_128_gcm; +alias(aes_gcm, Key) when size(Key)==192 -> aes_192_gcm; +alias(aes_gcm, Key) when size(Key)==256 -> aes_256_gcm; + +alias(aes_ccm, Key) when size(Key)==128 -> aes_128_ccm; +alias(aes_ccm, Key) when size(Key)==192 -> aes_192_ccm; +alias(aes_ccm, Key) when size(Key)==256 -> aes_256_ccm; + +alias(Alg, _) -> Alg. + %%%================================================================ %%% %%% RAND - pseudo random numbers using RN_ and BN_ functions in crypto lib @@ -1949,9 +2060,6 @@ cipher_info_nif(_Type) -> ?nif_stub. %% AES - in Galois/Counter Mode (GCM) %% %% The default tag length is EVP_GCM_TLS_TAG_LEN(16), -aead_encrypt(Type=aes_ccm, Key, Ivec, AAD, In) -> aead_encrypt(Type, Key, Ivec, AAD, In, 12); -aead_encrypt(Type=aes_gcm, Key, Ivec, AAD, In) -> aead_encrypt(Type, Key, Ivec, AAD, In, 16). - aead_encrypt(_Type, _Key, _Ivec, _AAD, _In, _TagLength) -> ?nif_stub. aead_decrypt(_Type, _Key, _Ivec, _AAD, _In, _Tag) -> ?nif_stub. diff --git a/lib/crypto/test/crypto_SUITE.erl b/lib/crypto/test/crypto_SUITE.erl index 7dbbde68e9..ce5097de47 100644 --- a/lib/crypto/test/crypto_SUITE.erl +++ b/lib/crypto/test/crypto_SUITE.erl @@ -323,12 +323,11 @@ end_per_group(_GroupName, Config) -> init_per_testcase(info, Config) -> Config; init_per_testcase(cmac, Config) -> - case crypto:info_lib() of - [{<<"OpenSSL">>,LibVer,_}] when is_integer(LibVer), LibVer > 16#10001000 -> + case is_supported(cmac) of + true -> Config; - _Else -> - % The CMAC functionality was introduced in OpenSSL 1.0.1 - {skip, "OpenSSL is too old"} + false -> + {skip, "CMAC is not supported"} end; init_per_testcase(generate, Config) -> case proplists:get_value(type, Config) of @@ -848,7 +847,8 @@ cipher_info(Config) when is_list(Config) -> #{type := _,key_length := _,iv_length := _, block_size := _,mode := _} = crypto:cipher_info(aes_128_cbc), {'EXIT',_} = (catch crypto:cipher_info(not_a_cipher)), - ok. + lists:foreach(fun(C) -> crypto:cipher_info(C) end, + proplists:get_value(ciphers, crypto:supports())). %%-------------------------------------------------------------------- hash_info() -> @@ -856,7 +856,8 @@ hash_info() -> hash_info(Config) when is_list(Config) -> #{type := _,size := _,block_size := _} = crypto:hash_info(sha256), {'EXIT',_} = (catch crypto:hash_info(not_a_hash)), - ok. + lists:foreach(fun(H) -> crypto:hash_info(H) end, + proplists:get_value(hashs, crypto:supports())). %%-------------------------------------------------------------------- %% Internal functions ------------------------------------------------ diff --git a/lib/dialyzer/src/Makefile b/lib/dialyzer/src/Makefile index fc08e7ca2f..bddd761705 100644 --- a/lib/dialyzer/src/Makefile +++ b/lib/dialyzer/src/Makefile @@ -90,8 +90,10 @@ APPUP_TARGET= $(EBIN)/$(APPUP_FILE) ifeq ($(NATIVE_LIBS_ENABLED),yes) ERL_COMPILE_FLAGS += +native +else +ERL_COMPILE_FLAGS += -Werror endif -ERL_COMPILE_FLAGS += +warn_export_vars +warn_unused_import +warn_untyped_record +warn_missing_spec +warnings_as_errors +ERL_COMPILE_FLAGS += +warn_export_vars +warn_unused_import +warn_untyped_record +warn_missing_spec # ---------------------------------------------------- # Targets diff --git a/lib/hipe/cerl/Makefile b/lib/hipe/cerl/Makefile index f653dce36f..5c367b5b77 100644 --- a/lib/hipe/cerl/Makefile +++ b/lib/hipe/cerl/Makefile @@ -66,7 +66,10 @@ DOC_FILES= $(MODULES:%=$(DOCS)/%.html) include ../native.mk -ERL_COMPILE_FLAGS += +inline -Werror +warn_export_vars +warn_unused_import +warn_missing_spec #+warn_untyped_record +ERL_COMPILE_FLAGS += +inline +warn_export_vars +warn_unused_import +warn_missing_spec #+warn_untyped_record +ifneq ($(NATIVE_LIBS_ENABLED),yes) +ERL_COMPILE_FLAGS += -Werror +endif # ---------------------------------------------------- # Targets diff --git a/lib/hipe/icode/hipe_beam_to_icode.erl b/lib/hipe/icode/hipe_beam_to_icode.erl index ffe81ef9b8..8e7e56b6c4 100644 --- a/lib/hipe/icode/hipe_beam_to_icode.erl +++ b/lib/hipe/icode/hipe_beam_to_icode.erl @@ -1189,6 +1189,21 @@ trans_fun([raw_raise|Instructions], Env) -> [hipe_icode:mk_primop(Dst,raw_raise,Vars) | trans_fun(Instructions, Env)]; %%-------------------------------------------------------------------- +%% New binary matching added in OTP 22. +%%-------------------------------------------------------------------- +%%--- bs_get_tail --- +trans_fun([{bs_get_tail=Name,_,_,_}|_Instructions], _Env) -> + nyi(Name); +%%--- bs_start_match3 --- +trans_fun([{bs_start_match3=Name,_,_,_,_}|_Instructions], _Env) -> + nyi(Name); +%%--- bs_get_position --- +trans_fun([{bs_get_position=Name,_,_,_}|_Instructions], _Env) -> + nyi(Name); +%%--- bs_set_position --- +trans_fun([{bs_set_position=Name,_,_}|_Instructions], _Env) -> + nyi(Name); +%%-------------------------------------------------------------------- %%--- ERROR HANDLING --- %%-------------------------------------------------------------------- trans_fun([X|_], _) -> @@ -1196,6 +1211,9 @@ trans_fun([X|_], _) -> trans_fun([], _) -> []. +nyi(Name) -> + throw({unimplemented_instruction,Name}). + %%-------------------------------------------------------------------- %% trans_call and trans_enter generate correct Icode calls/tail-calls, %% recognizing explicit fails. diff --git a/lib/hipe/llvm/Makefile b/lib/hipe/llvm/Makefile index 817ff67dcd..9f7a2def6d 100644 --- a/lib/hipe/llvm/Makefile +++ b/lib/hipe/llvm/Makefile @@ -70,7 +70,10 @@ TARGET_FILES= $(MODULES:%=$(EBIN)/%.$(EMULATOR)) include ../native.mk -ERL_COMPILE_FLAGS += -Werror +inline +warn_export_vars #+warn_missing_spec +ERL_COMPILE_FLAGS += +inline +warn_export_vars #+warn_missing_spec +ifneq ($(NATIVE_LIBS_ENABLED),yes) +ERL_COMPILE_FLAGS += -Werror +endif # if in 32 bit backend define BIT32 symbol ifneq ($(BITS64),yes) diff --git a/lib/hipe/main/hipe.erl b/lib/hipe/main/hipe.erl index 2348e9b1f6..094b7bc508 100644 --- a/lib/hipe/main/hipe.erl +++ b/lib/hipe/main/hipe.erl @@ -583,9 +583,8 @@ fix_beam_exports([], Exports) -> Exports. get_beam_icode(Mod, {BeamCode, Exports}, File, Options) -> - {ok, Icode} = - ?option_time((catch {ok, hipe_beam_to_icode:module(BeamCode, Options)}), - "BEAM-to-Icode", Options), + Icode = ?option_time(hipe_beam_to_icode:module(BeamCode, Options), + "BEAM-to-Icode", Options), BeamBin = get_beam_code(File), {{Mod, Exports, Icode}, BeamBin}. @@ -662,9 +661,12 @@ run_compiler_1(Name, DisasmFun, IcodeFun, Options) -> {Icode, WholeModule} = IcodeFun(Code, Opts), CompRes = compile_finish(Icode, WholeModule, Opts), compiler_return(CompRes, Parent) - catch error:Error:StackTrace -> + catch + error:Error:StackTrace -> print_crash_message(Name, Error, StackTrace), - exit(Error) + exit(Error); + throw:{unimplemented_instruction,_Instruction}=Error -> + exit(Error) end end), Timeout = case proplists:get_value(timeout, Options) of diff --git a/lib/hipe/rtl/Makefile b/lib/hipe/rtl/Makefile index becdd0b7d8..0c0f6e24f5 100644 --- a/lib/hipe/rtl/Makefile +++ b/lib/hipe/rtl/Makefile @@ -75,7 +75,10 @@ TARGET_FILES= $(MODULES:%=$(EBIN)/%.$(EMULATOR)) include ../native.mk -ERL_COMPILE_FLAGS += -Werror +inline +warn_unused_import +warn_export_vars +ERL_COMPILE_FLAGS += +inline +warn_unused_import +warn_export_vars +ifneq ($(NATIVE_LIBS_ENABLED),yes) +ERL_COMPILE_FLAGS += -Werror +endif # ---------------------------------------------------- # Targets diff --git a/lib/inets/test/httpc_SUITE.erl b/lib/inets/test/httpc_SUITE.erl index 8357e02014..d4b33ae2c6 100644 --- a/lib/inets/test/httpc_SUITE.erl +++ b/lib/inets/test/httpc_SUITE.erl @@ -339,14 +339,6 @@ end_per_testcase(Case, Config) end_per_testcase(_Case, _Config) -> ok. -is_ipv6_supported() -> - case gen_udp:open(0, [inet6]) of - {ok, Socket} -> - gen_udp:close(Socket), - true; - _ -> - false - end. %%-------------------------------------------------------------------- @@ -1612,7 +1604,8 @@ post_with_content_type(Config) when is_list(Config) -> %%-------------------------------------------------------------------- request_options() -> - [{doc, "Test http get request with socket options against local server (IPv6)"}]. + [{require, ipv6_hosts}, + {doc, "Test http get request with socket options against local server (IPv6)"}]. request_options(Config) when is_list(Config) -> Request = {url(group_name(Config), "/dummy.html", Config), []}, {ok, {{_,200,_}, [_ | _], _ = [_ | _]}} = httpc:request(get, Request, [], @@ -2945,3 +2938,12 @@ receive_stream_n(Ref, N) -> ct:pal("Data: ~p", [Data]), receive_stream_n(Ref, N-1) end. + +is_ipv6_supported() -> + {ok, Hostname0} = inet:gethostname(), + try + lists:member(list_to_atom(Hostname0), ct:get_config(ipv6_hosts)) + catch + _: _ -> + false + end. diff --git a/lib/kernel/doc/src/logger_disk_log_h.xml b/lib/kernel/doc/src/logger_disk_log_h.xml index 5b2374690e..aa577f3c62 100644 --- a/lib/kernel/doc/src/logger_disk_log_h.xml +++ b/lib/kernel/doc/src/logger_disk_log_h.xml @@ -66,7 +66,7 @@ corresponds to the <c>name</c> property in the <seealso marker="disk_log#open-1"><c>dlog_option()</c></seealso> datatype.</p> - <p>The value is set when the handler is added, and it can not + <p>The value is set when the handler is added, and it cannot be changed in runtime.</p> <p>Defaults to the same name as the handler identity, in the current directory.</p> @@ -77,7 +77,7 @@ corresponds to the <c>type</c> property in the <seealso marker="disk_log#open-1"><c>dlog_option()</c></seealso> datatype.</p> - <p>The value is set when the handler is added, and it can not + <p>The value is set when the handler is added, and it cannot be changed in runtime.</p> <p>Defaults to <c>wrap</c>.</p> </item> @@ -88,7 +88,7 @@ corresponds to the <c>MaxNoFiles</c> element in the <c>size</c> property in the <seealso marker="disk_log#open-1"><c>dlog_option()</c></seealso> datatype.</p> - <p>The value is set when the handler is added, and it can not + <p>The value is set when the handler is added, and it cannot be changed in runtime.</p> <p>Defaults to <c>10</c>.</p> <p>The setting has no effect on a halt log.</p> @@ -101,7 +101,7 @@ corresponds to the <c>MaxNoBytes</c> element in the <c>size</c> property in the <seealso marker="disk_log#open-1"><c>dlog_option()</c></seealso> datatype.</p> - <p>The value is set when the handler is added, and it can not + <p>The value is set when the handler is added, and it cannot be changed in runtime.</p> <p>Defaults to <c>1048576</c> bytes for a wrap log, and <c>infinity</c> for a halt log.</p> diff --git a/lib/kernel/doc/src/logger_std_h.xml b/lib/kernel/doc/src/logger_std_h.xml index 5ed1a2f210..5ac4f58d12 100644 --- a/lib/kernel/doc/src/logger_std_h.xml +++ b/lib/kernel/doc/src/logger_std_h.xml @@ -58,7 +58,7 @@ <tag><marker id="type"/><c>type = standard_io | standard_error | file</c></tag> <item> <p>Specifies the log destination.</p> - <p>The value is set when the handler is added, and it can not + <p>The value is set when the handler is added, and it cannot be changed in runtime.</p> <p>Defaults to <c>standard_io</c>, unless parameter <seealso marker="#file"><c>file</c></seealso> is @@ -68,7 +68,7 @@ <item> <p>This specifies the name of the log file when the handler is of type <c>file</c>.</p> - <p>The value is set when the handler is added, and it can not + <p>The value is set when the handler is added, and it cannot be changed in runtime.</p> <p>Defaults to the same name as the handler identity, in the current directory.</p> @@ -93,9 +93,9 @@ or <c>{delayed_write,Size,Delay}</c> is found in the list, <c>delayed_write</c> is added.</item> </list> - <p>Log files are always UTF-8 encoded. The encoding can not be + <p>Log files are always UTF-8 encoded. The encoding cannot be changed by setting the mode <c>{encoding,Encoding}</c>.</p> - <p>The value is set when the handler is added, and it can not + <p>The value is set when the handler is added, and it cannot be changed in runtime.</p> <p>Defaults to <c>[raw,append,delayed_write]</c>.</p> </item> diff --git a/lib/kernel/doc/src/notes.xml b/lib/kernel/doc/src/notes.xml index 0c187eb19f..61bd598145 100644 --- a/lib/kernel/doc/src/notes.xml +++ b/lib/kernel/doc/src/notes.xml @@ -31,6 +31,21 @@ </header> <p>This document describes the changes made to the Kernel application.</p> +<section><title>Kernel 6.3.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p>Fixed a performance regression when reading files + opened with the <c>compressed</c> flag.</p> + <p> + Own Id: OTP-15706 Aux Id: ERIERL-336 </p> + </item> + </list> + </section> + +</section> + <section><title>Kernel 6.3</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/kernel/examples/gen_tcp_dist/Makefile b/lib/kernel/examples/gen_tcp_dist/Makefile index 65513a1729..0c916835ea 100644 --- a/lib/kernel/examples/gen_tcp_dist/Makefile +++ b/lib/kernel/examples/gen_tcp_dist/Makefile @@ -2,9 +2,7 @@ RM=rm -f CP=cp EBIN=ebin ERLC=erlc -# Works if building in open source source tree -KERNEL_INCLUDE=$(ERL_TOP)/lib/kernel/include -ERLCFLAGS+= -W -I$(KERNEL_INCLUDE) +ERLCFLAGS+= -W MODULES=gen_tcp_dist diff --git a/lib/kernel/examples/gen_tcp_dist/src/gen_tcp_dist.erl b/lib/kernel/examples/gen_tcp_dist/src/gen_tcp_dist.erl index 98554ed805..cda4c470f9 100644 --- a/lib/kernel/examples/gen_tcp_dist/src/gen_tcp_dist.erl +++ b/lib/kernel/examples/gen_tcp_dist/src/gen_tcp_dist.erl @@ -53,10 +53,10 @@ -import(error_logger,[error_msg/2]). --include("net_address.hrl"). +-include_lib("kernel/include/net_address.hrl"). --include("dist.hrl"). --include("dist_util.hrl"). +-include_lib("kernel/include/dist.hrl"). +-include_lib("kernel/include/dist_util.hrl"). %% ------------------------------------------------------------ %% Select this protocol based on node name @@ -679,7 +679,14 @@ dist_cntrlr_setup_loop(Socket, TickHandler, Sup) -> %% From now on we execute on normal priority process_flag(priority, normal), erlang:dist_ctrl_get_data_notification(DHandle), - dist_cntrlr_output_loop(DHandle, Socket) + case init:get_argument(gen_tcp_dist_output_loop) of + error -> + dist_cntrlr_output_loop(DHandle, Socket); + {ok, [[ModStr, FuncStr]]} -> % For testing... + apply(list_to_atom(ModStr), + list_to_atom(FuncStr), + [DHandle, Socket]) + end end. %% We use active 10 for good throughput while still diff --git a/lib/kernel/src/Makefile b/lib/kernel/src/Makefile index 43b776f37e..fcb599859b 100644 --- a/lib/kernel/src/Makefile +++ b/lib/kernel/src/Makefile @@ -175,8 +175,10 @@ APPUP_TARGET= $(EBIN)/$(APPUP_FILE) ifeq ($(NATIVE_LIBS_ENABLED),yes) ERL_COMPILE_FLAGS += +native +else +ERL_COMPILE_FLAGS += -Werror endif -ERL_COMPILE_FLAGS += -I../include -Werror +ERL_COMPILE_FLAGS += -I../include # ---------------------------------------------------- # Targets diff --git a/lib/kernel/src/application_controller.erl b/lib/kernel/src/application_controller.erl index 9a8091fb2e..7715dca7c6 100644 --- a/lib/kernel/src/application_controller.erl +++ b/lib/kernel/src/application_controller.erl @@ -537,14 +537,12 @@ check_conf_data(ConfData) when is_list(ConfData) -> {AppName, List} when is_atom(AppName), is_list(List) -> case lists:keymember(AppName, 1, ConfDataRem) of true -> - ?LOG_WARNING("duplicate application config: " ++ atom_to_list(AppName)); + {error, "duplicate application config: " ++ atom_to_list(AppName)}; false -> - ok - end, - - case check_para(List, AppName) of - ok -> check_conf_data(ConfDataRem); - Error -> Error + case check_para(List, AppName) of + ok -> check_conf_data(ConfDataRem); + Error -> Error + end end; {AppName, List} when is_list(List) -> ErrMsg = "application: " @@ -570,15 +568,14 @@ check_para([], _AppName) -> check_para([{Para, Val} | ParaList], AppName) when is_atom(Para) -> case lists:keymember(Para, 1, ParaList) of true -> - ?LOG_WARNING("application: " ++ atom_to_list(AppName) ++ - "; duplicate parameter: " ++ atom_to_list(Para)); + ErrMsg = "application: " ++ atom_to_list(AppName) + ++ "; duplicate parameter: " ++ atom_to_list(Para), + {error, ErrMsg}; false -> - ok - end, - - case check_para_value(Para, Val, AppName) of - ok -> check_para(ParaList, AppName); - {error, _} = Error -> Error + case check_para_value(Para, Val, AppName) of + ok -> check_para(ParaList, AppName); + {error, _} = Error -> Error + end end; check_para([{Para, _Val} | _ParaList], AppName) -> {error, "application: " ++ atom_to_list(AppName) ++ "; invalid parameter name: " ++ diff --git a/lib/kernel/src/global.erl b/lib/kernel/src/global.erl index a38522eb5c..3875074d74 100644 --- a/lib/kernel/src/global.erl +++ b/lib/kernel/src/global.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2017. All Rights Reserved. +%% Copyright Ericsson AB 1996-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -50,10 +50,9 @@ %% This is for backward compatibility only; the functionality is broken. -define(WARN_DUPLICATED_NAME, global_multi_name_action). -%% Undocumented Kernel variable. Set this to 0 (zero) to get the old -%% behaviour. +%% Undocumented Kernel variable. -define(N_CONNECT_RETRIES, global_connect_retries). --define(DEFAULT_N_CONNECT_RETRIES, 5). +-define(DEFAULT_N_CONNECT_RETRIES, 0). %%% In certain places in the server, calling io:format hangs everything, %%% so we'd better use erlang:display/1. @@ -125,16 +124,12 @@ %%% There are also ETS tables used for bookkeeping of locks and names %%% (the first position is the key): %%% -%%% global_locks (set): {ResourceId, LockRequesterId, [{Pid,RPid,ref()]} +%%% global_locks (set): {ResourceId, LockRequesterId, [{Pid,ref()]} %%% Pid is locking ResourceId, ref() is the monitor ref. -%%% RPid =/= Pid if there is an extra process calling erlang:monitor(). -%%% global_names (set): {Name, Pid, Method, RPid, ref()} +%%% global_names (set): {Name, Pid, Method, ref()} %%% Registered names. ref() is the monitor ref. -%%% RPid =/= Pid if there is an extra process calling erlang:monitor(). %%% global_names_ext (set): {Name, Pid, RegNode} %%% External registered names (C-nodes). -%%% (The RPid:s can be removed when/if erlang:monitor() returns before -%%% trying to connect to the other node.) %%% %%% Helper tables: %%% global_pid_names (bag): {Pid, Name} | {ref(), Name} @@ -310,7 +305,7 @@ re_register_name(Name, Pid, Method0) when is_pid(Pid) -> -spec registered_names() -> [Name] when Name :: term(). registered_names() -> - MS = ets:fun2ms(fun({Name,_Pid,_M,_RP,_R}) -> Name end), + MS = ets:fun2ms(fun({Name,_Pid,_M,_R}) -> Name end), ets:select(global_names, MS). %%----------------------------------------------------------------- @@ -1235,7 +1230,7 @@ ins_name_ext(Name, Pid, Method, RegNode, FromPidOrNode, ExtraInfo, S0) -> where(Name) -> case ets:lookup(global_names, Name) of - [{_Name, Pid, _Method, _RPid, _Ref}] -> + [{_Name, Pid, _Method, _Ref}] -> if node(Pid) == node() -> case is_process_alive(Pid) of true -> Pid; @@ -1272,10 +1267,10 @@ can_set_lock({ResourceId, LockRequesterId}) -> end. insert_lock({ResourceId, LockRequesterId}=Id, Pid, PidRefs, S) -> - {RPid, Ref} = do_monitor(Pid), + Ref = erlang:monitor(process, Pid), true = ets:insert(global_pid_ids, {Pid, ResourceId}), true = ets:insert(global_pid_ids, {Ref, ResourceId}), - Lock = {ResourceId, LockRequesterId, [{Pid,RPid,Ref} | PidRefs]}, + Lock = {ResourceId, LockRequesterId, [{Pid,Ref} | PidRefs]}, true = ets:insert(global_locks, Lock), trace_message(S, {ins_lock, node(Pid)}, [Id, Pid]). @@ -1293,10 +1288,9 @@ handle_del_lock({ResourceId, LockReqId}, Pid, S0) -> _ -> S0 end. -remove_lock(ResourceId, LockRequesterId, Pid, [{Pid,RPid,Ref}], Down, S0) -> +remove_lock(ResourceId, LockRequesterId, Pid, [{Pid,Ref}], Down, S0) -> ?trace({remove_lock_1, {id,ResourceId},{pid,Pid}}), true = erlang:demonitor(Ref, [flush]), - kill_monitor_proc(RPid, Pid), true = ets:delete(global_locks, ResourceId), true = ets:delete_object(global_pid_ids, {Pid, ResourceId}), true = ets:delete_object(global_pid_ids, {Ref, ResourceId}), @@ -1309,9 +1303,8 @@ remove_lock(ResourceId, LockRequesterId, Pid, [{Pid,RPid,Ref}], Down, S0) -> remove_lock(ResourceId, LockRequesterId, Pid, PidRefs0, _Down, S) -> ?trace({remove_lock_2, {id,ResourceId},{pid,Pid}}), PidRefs = case lists:keyfind(Pid, 1, PidRefs0) of - {Pid, RPid, Ref} -> + {Pid, Ref} -> true = erlang:demonitor(Ref, [flush]), - kill_monitor_proc(RPid, Pid), true = ets:delete_object(global_pid_ids, {Ref, ResourceId}), lists:keydelete(Pid, 1, PidRefs0); @@ -1324,11 +1317,6 @@ remove_lock(ResourceId, LockRequesterId, Pid, PidRefs0, _Down, S) -> trace_message(S, {rem_lock, node(Pid)}, [{ResourceId, LockRequesterId}, Pid]). -kill_monitor_proc(Pid, Pid) -> - ok; -kill_monitor_proc(RPid, _Pid) -> - exit(RPid, kill). - do_ops(Ops, ConnNode, Names_ext, ExtraInfo, S0) -> ?trace({do_ops, {ops,Ops}}), @@ -1394,8 +1382,8 @@ sync_other(Node, N) -> % exit(normal). insert_global_name(Name, Pid, Method, FromPidOrNode, ExtraInfo, S) -> - {RPid, Ref} = do_monitor(Pid), - true = ets:insert(global_names, {Name, Pid, Method, RPid, Ref}), + Ref = erlang:monitor(process, Pid), + true = ets:insert(global_names, {Name, Pid, Method, Ref}), true = ets:insert(global_pid_names, {Pid, Name}), true = ets:insert(global_pid_names, {Ref, Name}), case lock_still_set(FromPidOrNode, ExtraInfo, S) of @@ -1437,7 +1425,7 @@ extra_info(Tag, ExtraInfo) -> del_name(Ref, S) -> NameL = [Name || {_, Name} <- ets:lookup(global_pid_names, Ref), - {_, _Pid, _Method, _RPid, Ref1} <- + {_, _Pid, _Method, Ref1} <- ets:lookup(global_names, Name), Ref1 =:= Ref], case NameL of @@ -1450,24 +1438,23 @@ del_name(Ref, S) -> %% Keeps the entry in global_names for whereis_name/1. delete_global_name_keep_pid(Name, S) -> case ets:lookup(global_names, Name) of - [{Name, Pid, _Method, RPid, Ref}] -> - delete_global_name2(Name, Pid, RPid, Ref, S); + [{Name, Pid, _Method, Ref}] -> + delete_global_name2(Name, Pid, Ref, S); [] -> S end. delete_global_name2(Name, S) -> case ets:lookup(global_names, Name) of - [{Name, Pid, _Method, RPid, Ref}] -> + [{Name, Pid, _Method, Ref}] -> true = ets:delete(global_names, Name), - delete_global_name2(Name, Pid, RPid, Ref, S); + delete_global_name2(Name, Pid, Ref, S); [] -> S end. -delete_global_name2(Name, Pid, RPid, Ref, S) -> +delete_global_name2(Name, Pid, Ref, S) -> true = erlang:demonitor(Ref, [flush]), - kill_monitor_proc(RPid, Pid), delete_global_name(Name, Pid), ?trace({delete_global_name,{item,Name},{pid,Pid}}), true = ets:delete_object(global_pid_names, {Pid, Name}), @@ -1929,9 +1916,9 @@ reset_node_state(Node) -> %% from the same partition. exchange_names([{Name, Pid, Method} | Tail], Node, Ops, Res) -> case ets:lookup(global_names, Name) of - [{Name, Pid, _Method, _RPid2, _Ref2}] -> + [{Name, Pid, _Method, _Ref2}] -> exchange_names(Tail, Node, Ops, Res); - [{Name, Pid2, Method2, _RPid2, _Ref2}] when node() < Node -> + [{Name, Pid2, Method2, _Ref2}] when node() < Node -> %% Name clash! Add the result of resolving to Res(olved). %% We know that node(Pid) =/= node(), so we don't %% need to link/unlink to Pid. @@ -1960,7 +1947,7 @@ exchange_names([{Name, Pid, Method} | Tail], Node, Ops, Res) -> Op = {delete, Name}, exchange_names(Tail, Node, [Op | Ops], [Op | Res]) end; - [{Name, _Pid2, _Method, _RPid, _Ref}] -> + [{Name, _Pid2, _Method, _Ref}] -> %% The other node will solve the conflict. exchange_names(Tail, Node, Ops, Res); _ -> @@ -2036,7 +2023,7 @@ pid_is_locking(Pid, PidRefs) -> delete_lock(Ref, S0) -> Locks = pid_locks(Ref), F = fun({ResourceId, LockRequesterId, PidRefs}, S) -> - {Pid, _RPid, Ref} = lists:keyfind(Ref, 3, PidRefs), + {Pid, Ref} = lists:keyfind(Ref, 2, PidRefs), remove_lock(ResourceId, LockRequesterId, Pid, PidRefs, true, S) end, lists:foldl(F, S0, Locks). @@ -2046,10 +2033,10 @@ pid_locks(Ref) -> ets:lookup(global_locks, ResourceId) end, ets:lookup(global_pid_ids, Ref)), [Lock || Lock = {_Id, _Req, PidRefs} <- L, - rpid_is_locking(Ref, PidRefs)]. + ref_is_locking(Ref, PidRefs)]. -rpid_is_locking(Ref, PidRefs) -> - lists:keyfind(Ref, 3, PidRefs) =/= false. +ref_is_locking(Ref, PidRefs) -> + lists:keyfind(Ref, 2, PidRefs) =/= false. handle_nodedown(Node, S) -> %% DOWN signals from monitors have removed locks and registered names. @@ -2062,7 +2049,7 @@ handle_nodedown(Node, S) -> get_names() -> ets:select(global_names, - ets:fun2ms(fun({Name, Pid, Method, _RPid, _Ref}) -> + ets:fun2ms(fun({Name, Pid, Method, _Ref}) -> {Name, Pid, Method} end)). @@ -2205,24 +2192,6 @@ unexpected_message(Message, What) -> %%% Utilities -%% When/if erlang:monitor() returns before trying to connect to the -%% other node this function can be removed. -do_monitor(Pid) -> - case (node(Pid) =:= node()) orelse lists:member(node(Pid), nodes()) of - true -> - %% Assume the node is still up - {Pid, erlang:monitor(process, Pid)}; - false -> - F = fun() -> - Ref = erlang:monitor(process, Pid), - receive - {'DOWN', Ref, process, Pid, _Info} -> - exit(normal) - end - end, - erlang:spawn_monitor(F) - end. - intersection(_, []) -> []; intersection(L1, L2) -> diff --git a/lib/kernel/src/kernel.appup.src b/lib/kernel/src/kernel.appup.src index 8fa3f5c588..aca3247c8f 100644 --- a/lib/kernel/src/kernel.appup.src +++ b/lib/kernel/src/kernel.appup.src @@ -43,7 +43,9 @@ {<<"^6\\.1\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, {<<"^6\\.2$">>,[restart_new_emulator]}, {<<"^6\\.2\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, - {<<"^6\\.2\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}], + {<<"^6\\.2\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, + {<<"^6\\.3$">>,[restart_new_emulator]}, + {<<"^6\\.3\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}], [{<<"^5\\.3$">>,[restart_new_emulator]}, {<<"^5\\.3\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, {<<"^5\\.3\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, @@ -60,4 +62,6 @@ {<<"^6\\.1\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, {<<"^6\\.2$">>,[restart_new_emulator]}, {<<"^6\\.2\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, - {<<"^6\\.2\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}]}. + {<<"^6\\.2\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, + {<<"^6\\.3$">>,[restart_new_emulator]}, + {<<"^6\\.3\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}]}. diff --git a/lib/kernel/src/raw_file_io_inflate.erl b/lib/kernel/src/raw_file_io_inflate.erl index 7e9780310c..d3ed02dd03 100644 --- a/lib/kernel/src/raw_file_io_inflate.erl +++ b/lib/kernel/src/raw_file_io_inflate.erl @@ -26,7 +26,7 @@ -include("file_int.hrl"). --define(INFLATE_CHUNK_SIZE, (1 bsl 10)). +-define(INFLATE_CHUNK_SIZE, (8 bsl 10)). -define(GZIP_WBITS, (16 + 15)). callback_mode() -> state_functions. diff --git a/lib/kernel/test/Makefile b/lib/kernel/test/Makefile index d203597fc2..6763a04d9f 100644 --- a/lib/kernel/test/Makefile +++ b/lib/kernel/test/Makefile @@ -57,6 +57,7 @@ MODULES= \ prim_file_SUITE \ ram_file_SUITE \ gen_tcp_api_SUITE \ + gen_tcp_dist \ gen_tcp_echo_SUITE \ gen_tcp_misc_SUITE \ gen_udp_SUITE \ @@ -137,7 +138,10 @@ TARGETS = $(MODULES:%=$(EBIN)/%.$(EMULATOR)) # Targets # ---------------------------------------------------- -make_emakefile: +gen_tcp_dist.erl: ../examples/gen_tcp_dist/src/gen_tcp_dist.erl + cp $< $@ + +make_emakefile: $(ERL_FILES) $(ERL_TOP)/make/make_emakefile $(ERL_COMPILE_FLAGS) -o$(EBIN) '*_SUITE_make' \ > $(EMAKEFILE) $(ERL_TOP)/make/make_emakefile $(ERL_COMPILE_FLAGS) -o$(EBIN) $(MODULES) \ diff --git a/lib/kernel/test/application_SUITE.erl b/lib/kernel/test/application_SUITE.erl index 94d7c17712..1ab554db7c 100644 --- a/lib/kernel/test/application_SUITE.erl +++ b/lib/kernel/test/application_SUITE.erl @@ -2020,18 +2020,11 @@ set_env_errors(Conf) when is_list(Conf) -> "application: kernel; erroneous parameter: distributed" = badarg_msg(fun() -> application:set_env([{kernel, [{distributed, config}]}]) end), - %% This will raise in the future - ct:capture_start(), - _ = application:set_env([{foo, []}, {foo, []}]), - timer:sleep(100), - ct:capture_stop(), - [_ | _] = string:find(ct:capture_get(), "duplicate application config: foo"), - - ct:capture_start(), - _ = application:set_env([{foo, [{bar, baz}, {bar, bat}]}]), - timer:sleep(100), - ct:capture_stop(), - [_ | _] = string:find(ct:capture_get(), "application: foo; duplicate parameter: bar"), + "duplicate application config: foo" = + badarg_msg(fun() -> application:set_env([{foo, []}, {foo, []}]) end), + + "application: foo; duplicate parameter: bar" = + badarg_msg(fun() -> application:set_env([{foo, [{bar, baz}, {bar, bat}]}]) end), ok. diff --git a/lib/kernel/test/code_SUITE.erl b/lib/kernel/test/code_SUITE.erl index 64e0b9d8dd..4f0847084f 100644 --- a/lib/kernel/test/code_SUITE.erl +++ b/lib/kernel/test/code_SUITE.erl @@ -25,8 +25,8 @@ -export([all/0, suite/0,groups/0,init_per_group/2,end_per_group/2]). -export([set_path/1, get_path/1, add_path/1, add_paths/1, del_path/1, replace_path/1, load_file/1, load_abs/1, ensure_loaded/1, - delete/1, purge/1, purge_many_exits/1, soft_purge/1, is_loaded/1, - all_loaded/1, + delete/1, purge/1, purge_many_exits/0, purge_many_exits/1, + soft_purge/1, is_loaded/1, all_loaded/1, load_binary/1, dir_req/1, object_code/1, set_path_file/1, upgrade/1, sticky_dir/1, pa_pz_option/1, add_del_path/1, @@ -55,7 +55,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}, - {timetrap,{minutes,5}}]. + {timetrap,{seconds,30}}]. all() -> [set_path, get_path, add_path, add_paths, del_path, @@ -140,6 +140,11 @@ end_per_testcase(on_load_embedded, Config) -> LinkName = proplists:get_value(link_name, Config), _ = del_link(LinkName), end_per_testcase(Config); +end_per_testcase(upgrade, Config) -> + %% Make sure tracing is turned off even if the test times out. + erlang:trace_pattern({error_handler,undefined_function,3}, false, [global]), + erlang:trace(self(), false, [call]), + end_per_testcase(Config); end_per_testcase(_Func, Config) -> end_per_testcase(Config). @@ -391,6 +396,9 @@ purge(Config) when is_list(Config) -> process_flag(trap_exit, OldFlag), ok. +purge_many_exits() -> + [{timetrap, {minutes, 2}}]. + purge_many_exits(Config) when is_list(Config) -> OldFlag = process_flag(trap_exit, true), @@ -1556,6 +1564,11 @@ on_load_update_code_1(3, Mod) -> %% Test -on_load while trace feature 'on_load' is enabled (OTP-14612) on_load_trace_on_load(Config) -> + %% 'on_load' enables tracing for all newly loaded modules, so we make a dry + %% run to ensure that ancillary modules like 'merl' won't be loaded during + %% the actual test. + on_load_update(Config), + Papa = self(), Tracer = spawn_link(fun F() -> receive M -> Papa ! M end, F() end), {tracer,[]} = erlang:trace_info(self(),tracer), diff --git a/lib/kernel/test/erl_distribution_SUITE.erl b/lib/kernel/test/erl_distribution_SUITE.erl index 5a8bbd56c4..8dd4ef1987 100644 --- a/lib/kernel/test/erl_distribution_SUITE.erl +++ b/lib/kernel/test/erl_distribution_SUITE.erl @@ -40,7 +40,8 @@ monitor_nodes_errors/1, monitor_nodes_combinations/1, monitor_nodes_cleanup/1, - monitor_nodes_many/1]). + monitor_nodes_many/1, + dist_ctrl_proc_smoke/1]). %% Performs the test at another node. -export([get_socket_priorities/0, @@ -52,7 +53,7 @@ -export([init_per_testcase/2, end_per_testcase/2]). --export([start_node/2]). +-export([dist_cntrlr_output_test/2]). -export([pinger/1]). @@ -67,10 +68,11 @@ suite() -> [{ct_hooks,[ts_install_cth]}, - {timetrap,{minutes,4}}]. + {timetrap,{minutes,12}}]. all() -> - [tick, tick_change, nodenames, hostnames, illegal_nodenames, + [dist_ctrl_proc_smoke, + tick, tick_change, nodenames, hostnames, illegal_nodenames, connect_node, hidden_node, setopts, table_waste, net_setuptime, inet_dist_options_options, @@ -116,10 +118,12 @@ connect_node(Config) when is_list(Config) -> ok. tick(Config) when is_list(Config) -> - PaDir = filename:dirname(code:which(erl_distribution_SUITE)), + run_dist_configs(fun tick/2, Config). +tick(DCfg, _Config) -> %% First check that the normal case is OK! - {ok, Node} = start_node(dist_test, "-pa " ++ PaDir), + [Name1, Name2] = get_nodenames(2, dist_test), + {ok, Node} = start_node(DCfg, Name1), rpc:call(Node, erl_distribution_SUITE, tick_cli_test, [node()]), erlang:monitor_node(Node, true), @@ -143,14 +147,12 @@ tick(Config) when is_list(Config) -> %% Set the ticktime on the server node to 100 secs so the server %% node doesn't tick the client node within the interval ... - {ok, ServNode} = start_node(dist_test_server, - "-kernel net_ticktime 100 " - "-pa " ++ PaDir), + {ok, ServNode} = start_node(DCfg, Name2, + "-kernel net_ticktime 100"), rpc:call(ServNode, erl_distribution_SUITE, tick_serv_test, [Node, node()]), - {ok, _} = start_node(dist_test, - "-kernel net_ticktime 12 " - "-pa " ++ PaDir), + {ok, Node} = start_node(DCfg, Name1, + "-kernel net_ticktime 12"), rpc:call(Node, erl_distribution_SUITE, tick_cli_test, [ServNode]), spawn_link(erl_distribution_SUITE, keep_conn, [Node]), @@ -180,6 +182,9 @@ tick(Config) when is_list(Config) -> %% Checks that pinging nonexistyent nodes does not waste space in distribution table. table_waste(Config) when is_list(Config) -> + run_dist_configs(fun table_waste/2, Config). + +table_waste(DCfg, _Config) -> {ok, HName} = inet:gethostname(), F = fun(0,_F) -> []; (N,F) -> @@ -189,7 +194,7 @@ table_waste(Config) when is_list(Config) -> F(N-1,F) end, F(256,F), - {ok, N} = start_node(erl_distribution_300,""), + {ok, N} = start_node(DCfg, erl_distribution_300), stop_node(N), ok. @@ -295,13 +300,16 @@ gethostname() -> %% Test that pinging an illegal nodename does not kill the node. illegal_nodenames(Config) when is_list(Config) -> - PaDir = filename:dirname(code:which(erl_distribution_SUITE)), - {ok, Node}=start_node(illegal_nodenames, "-pa " ++ PaDir), + run_dist_configs(fun illegal_nodenames/2, Config). + +illegal_nodenames(DCfg, _Config) -> + {ok, Node}=start_node(DCfg, illegal_nodenames), monitor_node(Node, true), RPid=rpc:call(Node, erlang, spawn, [?MODULE, pinger, [self()]]), receive {RPid, pinged} -> + monitor_node(Node, false), ok; {nodedown, Node} -> ct:fail("Remote node died.") @@ -318,22 +326,25 @@ pinger(Starter) -> %% Test that you can set the net_setuptime properly. net_setuptime(Config) when is_list(Config) -> + run_dist_configs(fun net_setuptime/2, Config). + +net_setuptime(DCfg, _Config) -> + %% In this test case, we reluctantly accept shorter times than the given %% setup time, because the connection attempt can end in a %% "Host unreachable" error before the timeout fires. - Res0 = do_test_setuptime("2"), + Res0 = do_test_setuptime(DCfg, "2"), io:format("Res0 = ~p", [Res0]), true = (Res0 =< 4000), - Res1 = do_test_setuptime("0.3"), + Res1 = do_test_setuptime(DCfg, "0.3"), io:format("Res1 = ~p", [Res1]), true = (Res1 =< 500), ok. -do_test_setuptime(Setuptime) when is_list(Setuptime) -> - PaDir = filename:dirname(code:which(?MODULE)), - {ok, Node} = start_node(dist_setuptime_test, "-pa " ++ PaDir ++ - " -kernel net_setuptime " ++ Setuptime), +do_test_setuptime(DCfg, Setuptime) when is_list(Setuptime) -> + {ok, Node} = start_node(DCfg, dist_setuptime_test, + "-kernel net_setuptime " ++ Setuptime), Res = rpc:call(Node,?MODULE,time_ping,[?DUMMY_NODE]), stop_node(Node), Res. @@ -399,32 +410,36 @@ tick_cli_test1(Node) -> end. setopts(Config) when is_list(Config) -> + run_dist_configs(fun setopts/2, Config). + +setopts(DCfg, _Config) -> register(setopts_regname, self()), [N1,N2,N3,N4] = get_nodenames(4, setopts), - {_N1F,Port1} = start_node_unconnected(N1, ?MODULE, run_remote_test, + {_N1F,Port1} = start_node_unconnected(DCfg, N1, ?MODULE, run_remote_test, ["setopts_do", atom_to_list(node()), "1", "ping"]), 0 = wait_for_port_exit(Port1), - {_N2F,Port2} = start_node_unconnected(N2, ?MODULE, run_remote_test, + {_N2F,Port2} = start_node_unconnected(DCfg, N2, ?MODULE, run_remote_test, ["setopts_do", atom_to_list(node()), "2", "ping"]), 0 = wait_for_port_exit(Port2), {ok, LSock} = gen_tcp:listen(0, [{packet,2}, {active,false}]), {ok, LTcpPort} = inet:port(LSock), - {N3F,Port3} = start_node_unconnected(N3, ?MODULE, run_remote_test, + {N3F,Port3} = start_node_unconnected(DCfg, N3, ?MODULE, run_remote_test, ["setopts_do", atom_to_list(node()), "1", integer_to_list(LTcpPort)]), wait_and_connect(LSock, N3F, Port3), 0 = wait_for_port_exit(Port3), - {N4F,Port4} = start_node_unconnected(N4, ?MODULE, run_remote_test, + {N4F,Port4} = start_node_unconnected(DCfg, N4, ?MODULE, run_remote_test, ["setopts_do", atom_to_list(node()), "2", integer_to_list(LTcpPort)]), wait_and_connect(LSock, N4F, Port4), 0 = wait_for_port_exit(Port4), + unregister(setopts_regname), ok. wait_and_connect(LSock, NodeName, NodePort) -> @@ -518,9 +533,9 @@ opt_from_nr("2") -> {nodelay, false}. change_val(true) -> false; change_val(false) -> true. -start_node_unconnected(Name, Mod, Func, Args) -> +start_node_unconnected(DCfg, Name, Mod, Func, Args) -> FullName = full_node_name(Name), - CmdLine = mk_node_cmdline(Name,Mod,Func,Args), + CmdLine = mk_node_cmdline(DCfg, Name,Mod,Func,Args), io:format("Starting node ~p: ~s~n", [FullName, CmdLine]), case open_port({spawn, CmdLine}, [exit_status]) of Port when is_port(Port) -> @@ -534,7 +549,7 @@ full_node_name(PreName) -> atom_to_list(node())), list_to_atom(atom_to_list(PreName) ++ HostSuffix). -mk_node_cmdline(Name,Mod,Func,Args) -> +mk_node_cmdline(DCfg, Name,Mod,Func,Args) -> Static = "-noinput", Pa = filename:dirname(code:which(?MODULE)), Prog = case catch init:get_argument(progname) of @@ -551,6 +566,7 @@ mk_node_cmdline(Name,Mod,Func,Args) -> Prog ++ " " ++ Static ++ " " ++ NameSw ++ " " ++ NameStr + ++ " " ++ DCfg ++ " -pa " ++ Pa ++ " -env ERL_CRASH_DUMP " ++ Pwd ++ "/erl_crash_dump." ++ NameStr ++ " -setcookie " ++ atom_to_list(erlang:get_cookie()) @@ -560,7 +576,9 @@ mk_node_cmdline(Name,Mod,Func,Args) -> %% OTP-4255. tick_change(Config) when is_list(Config) -> - PaDir = filename:dirname(code:which(?MODULE)), + run_dist_configs(fun tick_change/2, Config). + +tick_change(DCfg, _Config) -> [BN, CN] = get_nodenames(2, tick_change), DefaultTT = net_kernel:get_net_ticktime(), unchanged = net_kernel:set_net_ticktime(DefaultTT, 60), @@ -577,14 +595,13 @@ tick_change(Config) when is_list(Config) -> end, wait_until(fun () -> 10 == net_kernel:get_net_ticktime() end), - {ok, B} = start_node(BN, "-kernel net_ticktime 10 -pa " ++ PaDir), - {ok, C} = start_node(CN, "-kernel net_ticktime 10 -hidden -pa " - ++ PaDir), + {ok, B} = start_node(DCfg, BN, "-kernel net_ticktime 10"), + {ok, C} = start_node(DCfg, CN, "-kernel net_ticktime 10 -hidden"), OTE = process_flag(trap_exit, true), case catch begin - run_tick_change_test(B, C, 10, 1, PaDir), - run_tick_change_test(B, C, 1, 10, PaDir) + run_tick_change_test(DCfg, B, C, 10, 1), + run_tick_change_test(DCfg, B, C, 1, 10) end of {'EXIT', Reason} -> stop_node(B), @@ -626,7 +643,7 @@ wait_for_nodedowns(Tester, Ref) -> end, wait_for_nodedowns(Tester, Ref). -run_tick_change_test(B, C, PrevTT, TT, PaDir) -> +run_tick_change_test(DCfg, B, C, PrevTT, TT) -> [DN, EN] = get_nodenames(2, tick_change), Tester = self(), @@ -640,8 +657,8 @@ run_tick_change_test(B, C, PrevTT, TT, PaDir) -> wait_for_nodedowns(Tester, Ref) end, - {ok, D} = start_node(DN, "-kernel net_ticktime " - ++ integer_to_list(PrevTT) ++ " -pa " ++ PaDir), + {ok, D} = start_node(DCfg, DN, "-kernel net_ticktime " + ++ integer_to_list(PrevTT)), NMA = spawn_link(fun () -> MonitorNodes([B, C, D]) end), NMB = spawn_link(B, fun () -> MonitorNodes([node(), C, D]) end), @@ -674,8 +691,8 @@ run_tick_change_test(B, C, PrevTT, TT, PaDir) -> sleep(7), change_initiated = rpc:call(C,net_kernel,set_net_ticktime,[TT,10]), - {ok, E} = start_node(EN, "-kernel net_ticktime " - ++ integer_to_list(TT) ++ " -pa " ++ PaDir), + {ok, E} = start_node(DCfg, EN, "-kernel net_ticktime " + ++ integer_to_list(TT)), NME = spawn_link(E, fun () -> MonitorNodes([node(), B, C, D]) end), NMA2 = spawn_link(fun () -> MonitorNodes([E]) end), NMB2 = spawn_link(B, fun () -> MonitorNodes([E]) end), @@ -735,12 +752,13 @@ run_tick_change_test(B, C, PrevTT, TT, PaDir) -> %% %% Basic test of hidden node. hidden_node(Config) when is_list(Config) -> - PaDir = filename:dirname(code:which(?MODULE)), - VArgs = "-pa " ++ PaDir, - HArgs = "-hidden -pa " ++ PaDir, - {ok, V} = start_node(visible_node, VArgs), + run_dist_configs(fun hidden_node/2, Config). + +hidden_node(DCfg, _Config) -> + HArgs = "-hidden", + {ok, V} = start_node(DCfg, visible_node), VMN = start_monitor_nodes_proc(V), - {ok, H} = start_node(hidden_node, HArgs), + {ok, H} = start_node(DCfg, hidden_node, HArgs), %% Connect visible_node -> hidden_node connect_nodes(V, H), test_nodes(V, H), @@ -748,9 +766,9 @@ hidden_node(Config) when is_list(Config) -> sleep(5), check_monitor_nodes_res(VMN, H), stop_node(V), - {ok, H} = start_node(hidden_node, HArgs), + {ok, H} = start_node(DCfg, hidden_node, HArgs), HMN = start_monitor_nodes_proc(H), - {ok, V} = start_node(visible_node, VArgs), + {ok, V} = start_node(DCfg, visible_node), %% Connect hidden_node -> visible_node connect_nodes(H, V), test_nodes(V, H), @@ -850,9 +868,9 @@ do_inet_dist_options_options(Prio) -> "-kernel inet_dist_connect_options "++PriorityString++" " "-kernel inet_dist_listen_options "++PriorityString, {ok,Node1} = - start_node(inet_dist_options_1, InetDistOptions), + start_node("", inet_dist_options_1, InetDistOptions), {ok,Node2} = - start_node(inet_dist_options_2, InetDistOptions), + start_node("", inet_dist_options_2, InetDistOptions), %% pong = rpc:call(Node1, net_adm, ping, [Node2]), @@ -885,6 +903,9 @@ get_socket_priorities() -> %% monitor_nodes_nodedown_reason(Config) when is_list(Config) -> + run_dist_configs(fun monitor_nodes_nodedown_reason/2, Config). + +monitor_nodes_nodedown_reason(DCfg, _Config) -> MonNodeState = monitor_node_state(), ok = net_kernel:monitor_nodes(true), ok = net_kernel:monitor_nodes(true, [nodedown_reason]), @@ -892,10 +913,10 @@ monitor_nodes_nodedown_reason(Config) when is_list(Config) -> Names = get_numbered_nodenames(5, node), [NN1, NN2, NN3, NN4, NN5] = Names, - {ok, N1} = start_node(NN1), - {ok, N2} = start_node(NN2), - {ok, N3} = start_node(NN3), - {ok, N4} = start_node(NN4, "-hidden"), + {ok, N1} = start_node(DCfg, NN1), + {ok, N2} = start_node(DCfg, NN2), + {ok, N3} = start_node(DCfg, NN3), + {ok, N4} = start_node(DCfg, NN4, "-hidden"), receive {nodeup, N1} -> ok end, receive {nodeup, N2} -> ok end, @@ -925,7 +946,7 @@ monitor_nodes_nodedown_reason(Config) when is_list(Config) -> ok = net_kernel:monitor_nodes(false, [nodedown_reason]), - {ok, N5} = start_node(NN5), + {ok, N5} = start_node(DCfg, NN5), stop_node(N5), receive {nodeup, N5} -> ok end, @@ -938,11 +959,14 @@ monitor_nodes_nodedown_reason(Config) when is_list(Config) -> monitor_nodes_complex_nodedown_reason(Config) when is_list(Config) -> + run_dist_configs(fun monitor_nodes_complex_nodedown_reason/2, Config). + +monitor_nodes_complex_nodedown_reason(DCfg, _Config) -> MonNodeState = monitor_node_state(), Me = self(), ok = net_kernel:monitor_nodes(true, [nodedown_reason]), [Name] = get_nodenames(1, monitor_nodes_complex_nodedown_reason), - {ok, Node} = start_node(Name, ""), + {ok, Node} = start_node(DCfg, Name, ""), Pid = spawn(Node, fun() -> Me ! {stuff, @@ -981,16 +1005,19 @@ monitor_nodes_complex_nodedown_reason(Config) when is_list(Config) -> %% monitor_nodes_node_type(Config) when is_list(Config) -> + run_dist_configs(fun monitor_nodes_node_type/2, Config). + +monitor_nodes_node_type(DCfg, _Config) -> MonNodeState = monitor_node_state(), ok = net_kernel:monitor_nodes(true), ok = net_kernel:monitor_nodes(true, [{node_type, all}]), Names = get_numbered_nodenames(9, node), [NN1, NN2, NN3, NN4, NN5, NN6, NN7, NN8, NN9] = Names, - {ok, N1} = start_node(NN1), - {ok, N2} = start_node(NN2), - {ok, N3} = start_node(NN3, "-hidden"), - {ok, N4} = start_node(NN4, "-hidden"), + {ok, N1} = start_node(DCfg, NN1), + {ok, N2} = start_node(DCfg, NN2), + {ok, N3} = start_node(DCfg, NN3, "-hidden"), + {ok, N4} = start_node(DCfg, NN4, "-hidden"), receive {nodeup, N1} -> ok end, receive {nodeup, N2} -> ok end, @@ -1014,15 +1041,15 @@ monitor_nodes_node_type(Config) when is_list(Config) -> receive {nodedown, N4, [{node_type, hidden}]} -> ok end, ok = net_kernel:monitor_nodes(false, [{node_type, all}]), - {ok, N5} = start_node(NN5), + {ok, N5} = start_node(DCfg, NN5), receive {nodeup, N5} -> ok end, stop_node(N5), receive {nodedown, N5} -> ok end, ok = net_kernel:monitor_nodes(true, [{node_type, hidden}]), - {ok, N6} = start_node(NN6), - {ok, N7} = start_node(NN7, "-hidden"), + {ok, N6} = start_node(DCfg, NN6), + {ok, N7} = start_node(DCfg, NN7, "-hidden"), receive {nodeup, N6} -> ok end, @@ -1037,8 +1064,8 @@ monitor_nodes_node_type(Config) when is_list(Config) -> ok = net_kernel:monitor_nodes(false, [{node_type, hidden}]), ok = net_kernel:monitor_nodes(false), - {ok, N8} = start_node(NN8), - {ok, N9} = start_node(NN9, "-hidden"), + {ok, N8} = start_node(DCfg, NN8), + {ok, N9} = start_node(DCfg, NN9, "-hidden"), receive {nodeup, N8, [{node_type, visible}]} -> ok end, stop_node(N8), @@ -1058,6 +1085,9 @@ monitor_nodes_node_type(Config) when is_list(Config) -> %% monitor_nodes_misc(Config) when is_list(Config) -> + run_dist_configs(fun monitor_nodes_misc/2, Config). + +monitor_nodes_misc(DCfg, _Config) -> MonNodeState = monitor_node_state(), ok = net_kernel:monitor_nodes(true), ok = net_kernel:monitor_nodes(true, [{node_type, all}, nodedown_reason]), @@ -1065,8 +1095,8 @@ monitor_nodes_misc(Config) when is_list(Config) -> Names = get_numbered_nodenames(3, node), [NN1, NN2, NN3] = Names, - {ok, N1} = start_node(NN1), - {ok, N2} = start_node(NN2, "-hidden"), + {ok, N1} = start_node(DCfg, NN1), + {ok, N2} = start_node(DCfg, NN2, "-hidden"), receive {nodeup, N1} -> ok end, @@ -1092,7 +1122,7 @@ monitor_nodes_misc(Config) when is_list(Config) -> ok = net_kernel:monitor_nodes(false, [{node_type, all}, nodedown_reason]), - {ok, N3} = start_node(NN3), + {ok, N3} = start_node(DCfg, NN3), receive {nodeup, N3} -> ok end, stop_node(N3), receive {nodedown, N3} -> ok end, @@ -1107,15 +1137,18 @@ monitor_nodes_misc(Config) when is_list(Config) -> %% messages from Node and that {nodedown, Node} messages are %% received after messages from Node. monitor_nodes_otp_6481(Config) when is_list(Config) -> + run_dist_configs(fun monitor_nodes_otp_6481/2, Config). + +monitor_nodes_otp_6481(DCfg, Config) -> io:format("Testing nodedown...~n"), - monitor_nodes_otp_6481_test(Config, nodedown), + monitor_nodes_otp_6481_test(DCfg, Config, nodedown), io:format("ok~n"), io:format("Testing nodeup...~n"), - monitor_nodes_otp_6481_test(Config, nodeup), + monitor_nodes_otp_6481_test(DCfg, Config, nodeup), io:format("ok~n"), ok. -monitor_nodes_otp_6481_test(Config, TestType) when is_list(Config) -> +monitor_nodes_otp_6481_test(DCfg, Config, TestType) when is_list(Config) -> MonNodeState = monitor_node_state(), NodeMsg = make_ref(), Me = self(), @@ -1164,7 +1197,7 @@ monitor_nodes_otp_6481_test(Config, TestType) when is_list(Config) -> end ++ MonNodeState, - {ok, Node} = start_node(Name, "", this), + {ok, Node} = start_node(DCfg, Name, "", this), receive {nodeup, Node} -> ok end, RemotePid = spawn(Node, @@ -1249,17 +1282,20 @@ monitor_nodes_errors(Config) when is_list(Config) -> ok. monitor_nodes_combinations(Config) when is_list(Config) -> + run_dist_configs(fun monitor_nodes_combinations/2, Config). + +monitor_nodes_combinations(DCfg, _Config) -> MonNodeState = monitor_node_state(), monitor_nodes_all_comb(true), [VisibleName, HiddenName] = get_nodenames(2, monitor_nodes_combinations), - {ok, Visible} = start_node(VisibleName, ""), + {ok, Visible} = start_node(DCfg, VisibleName, ""), receive_all_comb_nodeup_msgs(visible, Visible), no_msgs(), stop_node(Visible), receive_all_comb_nodedown_msgs(visible, Visible, connection_closed), no_msgs(), - {ok, Hidden} = start_node(HiddenName, "-hidden"), + {ok, Hidden} = start_node(DCfg, HiddenName, "-hidden"), receive_all_comb_nodeup_msgs(hidden, Hidden), no_msgs(), stop_node(Hidden), @@ -1395,6 +1431,9 @@ monitor_nodes_cleanup(Config) when is_list(Config) -> ok. monitor_nodes_many(Config) when is_list(Config) -> + run_dist_configs(fun monitor_nodes_many/2, Config). + +monitor_nodes_many(DCfg, _Config) -> MonNodeState = monitor_node_state(), [Name] = get_nodenames(1, monitor_nodes_many), %% We want to perform more than 2^16 net_kernel:monitor_nodes @@ -1402,7 +1441,7 @@ monitor_nodes_many(Config) when is_list(Config) -> No = (1 bsl 16) + 17, repeat(fun () -> ok = net_kernel:monitor_nodes(true) end, No), No = length(monitor_node_state()) - length(MonNodeState), - {ok, Node} = start_node(Name), + {ok, Node} = start_node(DCfg, Name), repeat(fun () -> receive {nodeup, Node} -> ok end end, No), stop_node(Node), repeat(fun () -> receive {nodedown, Node} -> ok end end, No), @@ -1411,8 +1450,118 @@ monitor_nodes_many(Config) when is_list(Config) -> MonNodeState = monitor_node_state(), ok. +dist_ctrl_proc_smoke(Config) when is_list(Config) -> + ThisNode = node(), + [Name1, Name2] = get_nodenames(2, dist_ctrl_proc_example_smoke), + GetSizeArg = " -gen_tcp_dist_output_loop " + ++ atom_to_list(?MODULE) ++ " " + ++ "dist_cntrlr_output_test", + {ok, Node1} = start_node("", Name1, "-proto_dist gen_tcp"), + {ok, Node2} = start_node("", Name2, "-proto_dist gen_tcp" ++ GetSizeArg), + pong = rpc:call(Node1, net_adm, ping, [Node2]), + NL1 = lists:sort([ThisNode, Node2]), + NL2 = lists:sort([ThisNode, Node1]), + NL1 = lists:sort(rpc:call(Node1, erlang, nodes, [])), + NL2 = lists:sort(rpc:call(Node2, erlang, nodes, [])), + + %% Verify that we actually are executing the distribution + %% module we expect and also massage message passing over + %% it a bit... + Ps1 = rpc:call(Node1, erlang, processes, []), + try + lists:foreach( + fun (P) -> + case rpc:call(Node1, erlang, process_info, [P, current_stacktrace]) of + undefined -> + ok; + {current_stacktrace, StkTrace} -> + lists:foreach(fun ({gen_tcp_dist, + dist_cntrlr_output_loop, + 2, _}) -> + io:format("~p ~p~n", [P, StkTrace]), + throw(found_it); + (_) -> + ok + end, StkTrace) + end + end, Ps1), + exit({missing, dist_cntrlr_output_loop}) + catch + throw:found_it -> ok + end, + + Ps2 = rpc:call(Node2, erlang, processes, []), + try + lists:foreach( + fun (P) -> + case rpc:call(Node2, erlang, process_info, [P, current_stacktrace]) of + undefined -> + ok; + {current_stacktrace, StkTrace} -> + lists:foreach(fun ({erl_distribution_SUITE, + dist_cntrlr_output_loop, + 2, _}) -> + io:format("~p ~p~n", [P, StkTrace]), + throw(found_it); + (_) -> + ok + end, StkTrace) + end + end, Ps2), + exit({missing, dist_cntrlr_output_loop}) + catch + throw:found_it -> ok + end, + + stop_node(Node1), + stop_node(Node2), + ok. + %% Misc. functions +run_dist_configs(Func, Config) -> + GetSizeArg = " -gen_tcp_dist_output_loop " + ++ atom_to_list(?MODULE) ++ " " + ++ "dist_cntrlr_output_test", + lists:map(fun ({DCfgName, DCfg}) -> + io:format("~n~n=== Running ~s configuration ===~n~n", + [DCfgName]), + Func(DCfg, Config) + end, + [{"default", ""}, + {"gen_tcp_dist", "-proto_dist gen_tcp"}, + {"gen_tcp_dist (get_size)", "-proto_dist gen_tcp" ++ GetSizeArg}]). + +dist_cntrlr_output_test(DHandle, Socket) -> + false = erlang:dist_ctrl_get_opt(DHandle, get_size), + false = erlang:dist_ctrl_set_opt(DHandle, get_size, true), + true = erlang:dist_ctrl_get_opt(DHandle, get_size), + true = erlang:dist_ctrl_set_opt(DHandle, get_size, false), + false = erlang:dist_ctrl_get_opt(DHandle, get_size), + false = erlang:dist_ctrl_set_opt(DHandle, get_size, true), + true = erlang:dist_ctrl_get_opt(DHandle, get_size), + dist_cntrlr_output_loop(DHandle, Socket). + +dist_cntrlr_send_data(DHandle, Socket) -> + case erlang:dist_ctrl_get_data(DHandle) of + none -> + erlang:dist_ctrl_get_data_notification(DHandle); + {Size, Data} -> + Size = erlang:iolist_size(Data), + ok = gen_tcp:send(Socket, Data), + dist_cntrlr_send_data(DHandle, Socket) + end. + +dist_cntrlr_output_loop(DHandle, Socket) -> + receive + dist_data -> + %% Outgoing data from this node... + dist_cntrlr_send_data(DHandle, Socket); + _ -> + ok %% Drop garbage message... + end, + dist_cntrlr_output_loop(DHandle, Socket). + monitor_node_state() -> erts_debug:set_internal_state(available_internal_state, true), MonitoringNodes = erts_debug:get_internal_state(monitoring_nodes), @@ -1438,25 +1587,25 @@ print_my_messages() -> sleep(T) -> receive after T * 1000 -> ok end. -start_node(Name, Param, this) -> +start_node(DCfg, Name, Param, this) -> NewParam = Param ++ " -pa " ++ filename:dirname(code:which(?MODULE)), test_server:start_node(Name, peer, [{args, NewParam}, {erl, [this]}]); -start_node(Name, Param, "this") -> - NewParam = Param ++ " -pa " ++ filename:dirname(code:which(?MODULE)), +start_node(DCfg, Name, Param, "this") -> + NewParam = Param ++ " -pa " ++ filename:dirname(code:which(?MODULE)) ++ " " ++ DCfg, test_server:start_node(Name, peer, [{args, NewParam}, {erl, [this]}]); -start_node(Name, Param, Rel) when is_atom(Rel) -> - NewParam = Param ++ " -pa " ++ filename:dirname(code:which(?MODULE)), +start_node(DCfg, Name, Param, Rel) when is_atom(Rel) -> + NewParam = Param ++ " -pa " ++ filename:dirname(code:which(?MODULE)) ++ " " ++ DCfg, test_server:start_node(Name, peer, [{args, NewParam}, {erl, [{release, atom_to_list(Rel)}]}]); -start_node(Name, Param, Rel) when is_list(Rel) -> - NewParam = Param ++ " -pa " ++ filename:dirname(code:which(?MODULE)), +start_node(DCfg, Name, Param, Rel) when is_list(Rel) -> + NewParam = Param ++ " -pa " ++ filename:dirname(code:which(?MODULE)) ++ " " ++ DCfg, test_server:start_node(Name, peer, [{args, NewParam}, {erl, [{release, Rel}]}]). -start_node(Name, Param) -> - NewParam = Param ++ " -pa " ++ filename:dirname(code:which(?MODULE)), +start_node(DCfg, Name, Param) -> + NewParam = Param ++ " -pa " ++ filename:dirname(code:which(?MODULE)) ++ " " ++ DCfg, test_server:start_node(Name, slave, [{args, NewParam}]). -start_node(Name) -> - start_node(Name, ""). +start_node(DCfg, Name) -> + start_node(DCfg, Name, ""). stop_node(Node) -> test_server:stop_node(Node). diff --git a/lib/kernel/test/file_SUITE.erl b/lib/kernel/test/file_SUITE.erl index a51025cba6..e095e589a3 100644 --- a/lib/kernel/test/file_SUITE.erl +++ b/lib/kernel/test/file_SUITE.erl @@ -2191,6 +2191,9 @@ unc_paths(Config) when is_list(Config) -> {ok, _} = file:read_file_info("C:\\Windows\\explorer.exe"), {ok, _} = file:read_file_info("\\\\localhost\\c$\\Windows\\explorer.exe"), + {ok, Files} = file:list_dir("C:\\Windows\\"), + {ok, Files} = file:list_dir("\\\\localhost\\c$\\Windows\\"), + {ok, Cwd} = file:get_cwd(), try @@ -3741,19 +3744,33 @@ otp_10852(Config) when is_list(Config) -> ok = rpc_call(Node, read_file, [B]), ok = rpc_call(Node, make_link, [B,B]), case rpc_call(Node, make_symlink, [B,B]) of - ok -> ok; - {error, E} when (E =:= enotsup) or (E =:= eperm) -> - {win32,_} = os:type() + {error, eilseq} -> + %% Some versions of OS X refuse to create files with illegal names. + {unix,darwin} = os:type(); + {error, eperm} -> + %% The test user might not have permission to create symlinks. + {win32,_} = os:type(); + ok -> + ok end, ok = rpc_call(Node, delete, [B]), - ok = rpc_call(Node, make_dir, [B]), + case rpc_call(Node, make_dir, [B]) of + {error, eilseq} -> + {unix,darwin} = os:type(); + ok -> + ok + end, ok = rpc_call(Node, del_dir, [B]), - ok = rpc_call(Node, write_file, [B,B]), - {ok, Fd} = rpc_call(Node, open, [B,[read]]), - ok = rpc_call(Node, close, [Fd]), - {ok,0} = rpc_call(Node, copy, [B,B]), - {ok, Fd2, B} = rpc_call(Node, path_open, [["."], B, [read]]), - ok = rpc_call(Node, close, [Fd2]), + case rpc_call(Node, write_file, [B,B]) of + {error, eilseq} -> + {unix,darwin} = os:type(); + ok -> + {ok, Fd} = rpc_call(Node, open, [B,[read]]), + ok = rpc_call(Node, close, [Fd]), + {ok,0} = rpc_call(Node, copy, [B,B]), + {ok, Fd2, B} = rpc_call(Node, path_open, [["."], B, [read]]), + ok = rpc_call(Node, close, [Fd2]) + end, true = test_server:stop_node(Node), ok. diff --git a/lib/kernel/test/file_name_SUITE.erl b/lib/kernel/test/file_name_SUITE.erl index 3afc647081..26cfd187c7 100644 --- a/lib/kernel/test/file_name_SUITE.erl +++ b/lib/kernel/test/file_name_SUITE.erl @@ -632,10 +632,13 @@ make_icky_dir(Mod, IckyDirName) -> hopeless_darwin() -> case {os:type(),os:version()} of - {{unix,darwin},{Major,_,_}} when Major < 9 -> - true; - _ -> - false + {{unix,darwin},{Major,_,_}} -> + %% icky file names worked between 10 and 17, but started returning + %% EILSEQ in 18. The check against 18 is exact in case newer + %% versions of Darwin support them again. + Major < 9 orelse Major =:= 18; + _ -> + false end. make_very_icky_dir(Mod, DirName) -> diff --git a/lib/kernel/test/gen_sctp_SUITE.erl b/lib/kernel/test/gen_sctp_SUITE.erl index a0ae792ba9..e4c489bd10 100644 --- a/lib/kernel/test/gen_sctp_SUITE.erl +++ b/lib/kernel/test/gen_sctp_SUITE.erl @@ -1459,11 +1459,11 @@ do_open_and_connect(ServerAddresses, AddressToConnectTo) -> do_open_and_connect(ServerAddresses, AddressToConnectTo, Fun). %% do_open_and_connect(ServerAddresses, AddressToConnectTo, Fun) -> - ServerFamily = get_family_by_addrs(ServerAddresses), + {ServerFamily, ServerOpts} = get_family_by_addrs(ServerAddresses), io:format("Serving ~p addresses: ~p~n", [ServerFamily, ServerAddresses]), S1 = ok(gen_sctp:open(0, [{ip,Addr} || Addr <- ServerAddresses] ++ - [ServerFamily])), + [ServerFamily|ServerOpts])), ok = gen_sctp:listen(S1, true), P1 = ok(inet:port(S1)), ClientFamily = get_family_by_addr(AddressToConnectTo), @@ -1493,9 +1493,9 @@ do_open_and_connect(ServerAddresses, AddressToConnectTo, Fun) -> %% If at least one of the addresses is an ipv6 address, return inet6, else inet. get_family_by_addrs(Addresses) -> case lists:usort([get_family_by_addr(Addr) || Addr <- Addresses]) of - [inet, inet6] -> inet6; - [inet] -> inet; - [inet6] -> inet6 + [inet, inet6] -> {inet6, [{ipv6_v6only, false}]}; + [inet] -> {inet, []}; + [inet6] -> {inet6, []} end. get_family_by_addr(Addr) when tuple_size(Addr) =:= 4 -> inet; diff --git a/lib/kernel/test/gen_tcp_api_SUITE_data/gen_tcp_api_SUITE.c b/lib/kernel/test/gen_tcp_api_SUITE_data/gen_tcp_api_SUITE.c index b91dca61d4..96938f9071 100644 --- a/lib/kernel/test/gen_tcp_api_SUITE_data/gen_tcp_api_SUITE.c +++ b/lib/kernel/test/gen_tcp_api_SUITE_data/gen_tcp_api_SUITE.c @@ -30,6 +30,7 @@ #define sock_close(s) closesocket(s) #else #include <sys/socket.h> +#include <unistd.h> #define sock_close(s) close(s) #endif diff --git a/lib/kernel/test/gen_tcp_misc_SUITE.erl b/lib/kernel/test/gen_tcp_misc_SUITE.erl index 52edfaee29..edf30448c4 100644 --- a/lib/kernel/test/gen_tcp_misc_SUITE.erl +++ b/lib/kernel/test/gen_tcp_misc_SUITE.erl @@ -2086,8 +2086,39 @@ test_pktoptions(Family, Spec, CheckConnect, OSType, OSVer) -> %%% {ok,<<"hi">>} = gen_tcp:recv(S1, 2, Timeout), %% %% Verify returned remote options - {ok,[{pktoptions,OptsVals1}]} = inet:getopts(S1, [pktoptions]), - {ok,[{pktoptions,OptsVals2}]} = inet:getopts(S2, [pktoptions]), + VerifyRemOpts = + fun(S, Role) -> + case inet:getopts(S, [pktoptions]) of + {ok, [{pktoptions, PktOpts1}]} -> + PktOpts1; + {ok, UnexpOK1} -> + io:format("Unexpected OK (~w): " + "~n ~p" + "~n", [Role, UnexpOK1]), + exit({unexpected_getopts_ok, + Role, + Spec, + TrueRecvOpts, + OptsVals, + OptsValsDefault, + UnexpOK1}); + {error, UnexpERR1} -> + io:format("Unexpected ERROR (~w): " + "~n ~p" + "~n", [Role, UnexpERR1]), + exit({unexpected_getopts_failure, + Role, + Spec, + TrueRecvOpts, + OptsVals, + OptsValsDefault, + UnexpERR1}) + end + end, + OptsVals1 = VerifyRemOpts(S1, dest), + OptsVals2 = VerifyRemOpts(S2, orig), + %% {ok,[{pktoptions,OptsVals1}]} = inet:getopts(S1, [pktoptions]), + %% {ok,[{pktoptions,OptsVals2}]} = inet:getopts(S2, [pktoptions]), (Result1 = sets_eq(OptsVals1, OptsVals)) orelse io:format( "Accept differs: ~p neq ~p~n", [OptsVals1,OptsVals]), @@ -3430,7 +3461,7 @@ wait(Mref) -> %% OTP-15536 %% Test that send error works correctly for delay_send -delay_send_error(Config) -> +delay_send_error(_Config) -> {ok, LS} = gen_tcp:listen(0, [{reuseaddr, true}, {packet, 1}, {active, false}]), {ok,{{0,0,0,0},PortNum}}=inet:sockname(LS), P = spawn_link( diff --git a/lib/kernel/test/logger_disk_log_h_SUITE.erl b/lib/kernel/test/logger_disk_log_h_SUITE.erl index 13b30835a1..36d98eaa25 100644 --- a/lib/kernel/test/logger_disk_log_h_SUITE.erl +++ b/lib/kernel/test/logger_disk_log_h_SUITE.erl @@ -1243,12 +1243,12 @@ restart_after(Config) -> {Log,HConfig,DLHConfig} = start_handler(?MODULE, ?FUNCTION_NAME, Config), NewHConfig1 = HConfig#{config=>DLHConfig#{overload_kill_enable=>true, - overload_kill_qlen=>10, + overload_kill_qlen=>4, overload_kill_restart_after=>infinity}}, ok = logger:update_handler_config(?MODULE, NewHConfig1), MRef1 = erlang:monitor(process, whereis(h_proc_name())), %% kill handler - send_burst({n,100}, {spawn,4,0}, {chars,79}, notice), + send_burst({n,100}, {spawn,5,0}, {chars,79}, notice), receive {'DOWN', MRef1, _, _, _Reason1} -> file_delete(Log), @@ -1265,13 +1265,13 @@ restart_after(Config) -> RestartAfter = ?OVERLOAD_KILL_RESTART_AFTER, NewHConfig2 = HConfig#{config=>DLHConfig#{overload_kill_enable=>true, - overload_kill_qlen=>10, + overload_kill_qlen=>4, overload_kill_restart_after=>RestartAfter}}, ok = logger:update_handler_config(?MODULE, NewHConfig2), Pid0 = whereis(h_proc_name()), MRef2 = erlang:monitor(process, Pid0), %% kill handler - send_burst({n,100}, {spawn,4,0}, {chars,79}, notice), + send_burst({n,100}, {spawn,5,0}, {chars,79}, notice), receive {'DOWN', MRef2, _, _, _Reason2} -> file_delete(Log), diff --git a/lib/kernel/vsn.mk b/lib/kernel/vsn.mk index 7bebe1ba70..b1ae513223 100644 --- a/lib/kernel/vsn.mk +++ b/lib/kernel/vsn.mk @@ -1 +1 @@ -KERNEL_VSN = 6.3 +KERNEL_VSN = 6.3.1 diff --git a/lib/mnesia/src/mnesia.erl b/lib/mnesia/src/mnesia.erl index 77afb8250c..02bc884e36 100644 --- a/lib/mnesia/src/mnesia.erl +++ b/lib/mnesia/src/mnesia.erl @@ -160,7 +160,7 @@ {'sync_transaction', Retries::non_neg_integer()}. -type table() :: atom(). -type storage_type() :: 'ram_copies' | 'disc_copies' | 'disc_only_copies'. --type index_attr() :: atom() | non_neg_integer(). +-type index_attr() :: atom() | non_neg_integer() | {atom()}. -type write_locks() :: 'write' | 'sticky_write'. -type read_locks() :: 'read'. -type lock_kind() :: write_locks() | read_locks(). @@ -1277,6 +1277,14 @@ match_object(Tid, Ts, Tab, Pat, LockKind) match_object(_Tid, _Ts, Tab, Pat, _LockKind) -> abort({bad_type, Tab, Pat}). +add_written_index(Store, Pos, Tab, Key, Objs) when is_integer(Pos) -> + Pat = setelement(Pos, val({Tab, wild_pattern}), Key), + add_written_match(Store, Pat, Tab, Objs); +add_written_index(Store, Pos, Tab, Key, Objs) when is_tuple(Pos) -> + IxF = mnesia_index:index_vals_f(val({Tab, storage_type}), Tab, Pos), + Ops = find_ops(Store, Tab, '_'), + add_ix_match(Ops, Objs, IxF, Key, val({Tab, setorbag})). + add_written_match(S, Pat, Tab, Objs) -> Ops = find_ops(S, Tab, Pat), FixedRes = add_match(Ops, Objs, val({Tab, setorbag})), @@ -1303,6 +1311,46 @@ add_match([{_Oid, Val, write}|R], Objs, bag) -> add_match([{Oid, Val, write}|R], Objs, set) -> add_match(R, [Val | deloid(Oid,Objs)],set). +add_ix_match([], Objs, _IxF, _Key, _Type) -> + Objs; +add_ix_match(Written, Objs, IxF, Key, ordered_set) -> + %% Must use keysort which is stable + add_ordered_match(lists:keysort(1, ix_filter_ops(IxF, Key, Written)), Objs, []); +add_ix_match([{Oid, _, delete}|R], Objs, IxF, Key, Type) -> + add_ix_match(R, deloid(Oid, Objs), IxF, Key, Type); +add_ix_match([{_Oid, Val, delete_object}|R], Objs, IxF, Key, Type) -> + case ix_match(Val, IxF, Key) of + true -> + add_ix_match(R, lists:delete(Val, Objs), IxF, Key, Type); + false -> + add_ix_match(R, Objs, IxF, Key, Type) + end; +add_ix_match([{_Oid, Val, write}|R], Objs, IxF, Key, bag) -> + case ix_match(Val, IxF, Key) of + true -> + add_ix_match(R, [Val | lists:delete(Val, Objs)], IxF, Key, bag); + false -> + add_ix_match(R, Objs, IxF, Key, bag) + end; +add_ix_match([{Oid, Val, write}|R], Objs, IxF, Key, set) -> + case ix_match(Val, IxF, Key) of + true -> + add_ix_match(R, [Val | deloid(Oid,Objs)],IxF,Key,set); + false -> + add_ix_match(R, Objs, IxF, Key, set) + end. + +ix_match(Val, IxF, Key) -> + lists:member(Key, IxF(Val)). + +ix_filter_ops(IxF, Key, Ops) -> + lists:filter( + fun({_Oid, Obj, write}) -> + ix_match(Obj, IxF, Key); + (_) -> + true + end, Ops). + %% For ordered_set only !! add_ordered_match(Written = [{{_, Key}, _, _}|_], [Obj|Objs], Acc) when Key > element(2, Obj) -> @@ -1641,6 +1689,16 @@ index_match_object(Tid, Ts, Tab, Pat, Attr, LockKind) dirty_index_match_object(Tab, Pat, Attr); % Should be optimized? tid -> case mnesia_schema:attr_tab_to_pos(Tab, Attr) of + {_} -> + case LockKind of + read -> + Store = Ts#tidstore.store, + mnesia_locker:rlock_table(Tid, Store, Tab), + Objs = dirty_match_object(Tab, Pat), + add_written_match(Store, Pat, Tab, Objs); + _ -> + abort({bad_type, Tab, LockKind}) + end; Pos when Pos =< tuple_size(Pat) -> case LockKind of read -> @@ -1688,8 +1746,8 @@ index_read(Tid, Ts, Tab, Key, Attr, LockKind) false -> Store = Ts#tidstore.store, Objs = mnesia_index:read(Tid, Store, Tab, Key, Pos), - Pat = setelement(Pos, val({Tab, wild_pattern}), Key), - add_written_match(Store, Pat, Tab, Objs); + add_written_index( + Ts#tidstore.store, Pos, Tab, Key, Objs); true -> abort({bad_type, Tab, Attr, Key}) end; @@ -1825,7 +1883,7 @@ remote_dirty_match_object(Tab, Pat) -> false -> mnesia_lib:db_match_object(Tab, Pat); true -> - PosList = val({Tab, index}), + PosList = regular_indexes(Tab), remote_dirty_match_object(Tab, Pat, PosList) end. @@ -1857,7 +1915,7 @@ remote_dirty_select(Tab, Spec) -> false -> mnesia_lib:db_select(Tab, Spec); true -> - PosList = val({Tab, index}), + PosList = regular_indexes(Tab), remote_dirty_select(Tab, Spec, PosList) end; _ -> @@ -1924,6 +1982,8 @@ dirty_index_match_object(Pat, _Attr) -> dirty_index_match_object(Tab, Pat, Attr) when is_atom(Tab), Tab /= schema, is_tuple(Pat), tuple_size(Pat) > 2 -> case mnesia_schema:attr_tab_to_pos(Tab, Attr) of + {_} -> + dirty_match_object(Tab, Pat); Pos when Pos =< tuple_size(Pat) -> case has_var(element(2, Pat)) of false -> @@ -3254,3 +3314,7 @@ put_activity_id(Activity) -> mnesia_tm:put_activity_id(Activity). put_activity_id(Activity,Fun) -> mnesia_tm:put_activity_id(Activity,Fun). + +regular_indexes(Tab) -> + PosList = val({Tab, index}), + [P || P <- PosList, is_integer(P)]. diff --git a/lib/mnesia/src/mnesia_index.erl b/lib/mnesia/src/mnesia_index.erl index 098265d5fc..6f1c21e3b9 100644 --- a/lib/mnesia/src/mnesia_index.erl +++ b/lib/mnesia/src/mnesia_index.erl @@ -1,8 +1,8 @@ %% %% %CopyrightBegin% -%% +%% %% Copyright Ericsson AB 1996-2018. All Rights Reserved. -%% +%% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. %% You may obtain a copy of the License at @@ -14,7 +14,7 @@ %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and %% limitations under the License. -%% +%% %% %CopyrightEnd% %% @@ -37,7 +37,7 @@ db_match_erase/2, get_index_table/2, get_index_table/3, - + tab2filename/2, init_index/2, init_indecies/3, @@ -45,6 +45,7 @@ del_transient/3, del_index_table/3, + index_vals_f/3, index_info/2, ext_index_instances/1]). @@ -60,9 +61,14 @@ read(Tid, Store, Tab, IxKey, Pos) -> ResList = mnesia_locker:ixrlock(Tid, Store, Tab, IxKey, Pos), %% Remove all tuples which don't include Ixkey, happens when Tab is a bag case val({Tab, setorbag}) of - bag -> + bag when is_integer(Pos) -> mnesia_lib:key_search_all(IxKey, Pos, ResList); - _ -> + bag when is_tuple(Pos) -> + TabStorage = val({Tab, storage_type}), + ValsF = index_vals_f(TabStorage, Tab, Pos), + [Obj || Obj <- ResList, + lists:member(IxKey, ValsF(Obj))]; + _ -> ResList end. @@ -136,7 +142,7 @@ del_object_index2([], _, _Storage, _Tab, _K, _Obj) -> ok; del_object_index2([{{Pos, Type}, Ixt} | Tail], SoB, Storage, Tab, K, Obj) -> ValsF = index_vals_f(Storage, Tab, Pos), case SoB of - bag -> + bag -> del_object_bag(Type, ValsF, Tab, K, Obj, Ixt); _ -> %% If set remove the tuple in index table del_ixes(Type, Ixt, ValsF, Obj, K) @@ -197,7 +203,7 @@ merge([], _, _, Ack) -> realkeys(Tab, Pos, IxKey) -> Index = get_index_table(Tab, Pos), db_get(Index, IxKey). % a list on the form [{IxKey, RealKey1} , .... - + dirty_select(Tab, Spec, Pos) when is_integer(Pos) -> %% Assume that we are on the node where the replica is %% Returns the records without applying the match spec @@ -233,7 +239,7 @@ dirty_read2(Tab, IxKey, Pos) -> end, Acc, mnesia_lib:db_get(Storage, Tab, K)) end, [], Keys)). -pick_index([{{{Pfx,_},IxType}, Ixt}|_], _Tab, {_} = Pfx) -> +pick_index([{{{Pfx,_,_},IxType}, Ixt}|_], _Tab, {_} = Pfx) -> {IxType, Ixt}; pick_index([{{Pos,IxType}, Ixt}|_], _Tab, Pos) -> {IxType, Ixt}; @@ -242,7 +248,7 @@ pick_index([_|T], Tab, Pos) -> pick_index([], Tab, Pos) -> mnesia:abort({no_exist, Tab, {index, Pos}}). - + %%%%%%% Creation, Init and deletion routines for index tables %% We can have several indexes on the same table @@ -387,12 +393,12 @@ init_ext_index(Tab, Storage, Alias, Mod, [{Pos,Type} | Tail]) -> create_fun(Cont, Tab, Pos) -> IxF = index_vals_f(disc_only_copies, Tab, Pos), fun(read) -> - Data = + Data = case Cont of {start, KeysPerChunk} -> mnesia_lib:db_init_chunk( disc_only_copies, Tab, KeysPerChunk); - '$end_of_table' -> + '$end_of_table' -> '$end_of_table'; _Else -> mnesia_lib:db_chunk(disc_only_copies, Cont) @@ -462,7 +468,7 @@ add_index_info(Tab, SetOrBag, IxElem) -> %% Check later if mnesia_tm is sensitive about the order mnesia_lib:set({Tab, index_info}, IndexInfo), mnesia_lib:set({Tab, index}, index_positions(IndexInfo)), - mnesia_lib:set({Tab, commit_work}, + mnesia_lib:set({Tab, commit_work}, mnesia_lib:sort_commit([IndexInfo | Commit])); {value, Old} -> %% We could check for consistency here @@ -470,7 +476,7 @@ add_index_info(Tab, SetOrBag, IxElem) -> mnesia_lib:set({Tab, index_info}, Index), mnesia_lib:set({Tab, index}, index_positions(Index)), NewC = lists:keyreplace(index, 1, Commit, Index), - mnesia_lib:set({Tab, commit_work}, + mnesia_lib:set({Tab, commit_work}, mnesia_lib:sort_commit(NewC)) end. @@ -488,19 +494,19 @@ del_index_info(Tab, Pos) -> element(1,P)=/=Pos end, Old#index.pos_list) of - [] -> + [] -> IndexInfo = index_info(Old#index.setorbag,[]), mnesia_lib:set({Tab, index_info}, IndexInfo), mnesia_lib:set({Tab, index}, index_positions(IndexInfo)), NewC = lists:keydelete(index, 1, Commit), - mnesia_lib:set({Tab, commit_work}, + mnesia_lib:set({Tab, commit_work}, mnesia_lib:sort_commit(NewC)); New -> Index = Old#index{pos_list = New}, mnesia_lib:set({Tab, index_info}, Index), mnesia_lib:set({Tab, index}, index_positions(Index)), NewC = lists:keyreplace(index, 1, Commit, Index), - mnesia_lib:set({Tab, commit_work}, + mnesia_lib:set({Tab, commit_work}, mnesia_lib:sort_commit(NewC)) end end. @@ -537,7 +543,7 @@ db_match_erase({{ext,_,_} = Ext, Ixt}, Pat) -> mnesia_lib:db_match_erase(Ext, Ixt, Pat); db_match_erase({dets, Ixt}, Pat) -> ok = dets:match_delete(Ixt, Pat). - + db_select({ram, Ixt}, Pat) -> ets:select(Ixt, Pat); db_select({{ext,_,_} = Ext, Ixt}, Pat) -> @@ -545,7 +551,7 @@ db_select({{ext,_,_} = Ext, Ixt}, Pat) -> db_select({dets, Ixt}, Pat) -> dets:select(Ixt, Pat). - + get_index_table(Tab, Pos) -> get_index_table(Tab, val({Tab, storage_type}), Pos). diff --git a/lib/mnesia/test/Makefile b/lib/mnesia/test/Makefile index 5b61b1af65..b43bc82801 100644 --- a/lib/mnesia/test/Makefile +++ b/lib/mnesia/test/Makefile @@ -53,7 +53,8 @@ MODULES= \ mnesia_measure_test \ mnesia_cost \ mnesia_dbn_meters \ - ext_test + ext_test \ + mnesia_index_plugin_test DocExamplesDir := ../doc/src/ diff --git a/lib/mnesia/test/mnesia_SUITE.erl b/lib/mnesia/test/mnesia_SUITE.erl index 24c1def6da..b41bf22efa 100644 --- a/lib/mnesia/test/mnesia_SUITE.erl +++ b/lib/mnesia/test/mnesia_SUITE.erl @@ -69,12 +69,13 @@ groups() -> %% covered. [{light, [], [{group, install}, {group, nice}, {group, evil}, - {group, mnesia_frag_test, light}, {group, qlc}, + {group, mnesia_frag_test, light}, {group, qlc}, {group, index_plugins}, {group, registry}, {group, config}, {group, examples}]}, {install, [], [{mnesia_install_test, all}]}, {nice, [], [{mnesia_nice_coverage_test, all}]}, {evil, [], [{mnesia_evil_coverage_test, all}]}, {qlc, [], [{mnesia_qlc_test, all}]}, + {index_plugins, [], [{mnesia_index_plugin_test, all}]}, {registry, [], [{mnesia_registry_test, all}]}, {config, [], [{mnesia_config_test, all}]}, {examples, [], [{mnesia_examples_test, all}]}, diff --git a/lib/mnesia/test/mnesia_index_plugin_test.erl b/lib/mnesia/test/mnesia_index_plugin_test.erl new file mode 100644 index 0000000000..44fe047c50 --- /dev/null +++ b/lib/mnesia/test/mnesia_index_plugin_test.erl @@ -0,0 +1,261 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 1996-2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% +-module(mnesia_index_plugin_test). +-author('[email protected]'). + +-export([init_per_testcase/2, end_per_testcase/2, + init_per_group/2, end_per_group/2, + init_per_suite/1, end_per_suite/1, + all/0, groups/0]). + +-export([ + add_rm_plugin/1, + tab_with_plugin_index/1, + tab_with_multiple_plugin_indexes/1, + ix_match_w_plugin/1, + ix_match_w_plugin_ordered/1, + ix_match_w_plugin_bag/1 + ]). + +-export([ix_prefixes/3, % test plugin + ix_prefixes2/3]). % test plugin 2 + +-include("mnesia_test_lib.hrl"). + +init_per_suite(Conf) -> + Conf. + +end_per_suite(Conf) -> + Conf. + +init_per_testcase(Func, Conf) -> + mnesia_test_lib:init_per_testcase(Func, Conf). + +end_per_testcase(Func, Conf) -> + mnesia_test_lib:end_per_testcase(Func, Conf). + +all() -> + [add_rm_plugin, + tab_with_plugin_index, + tab_with_multiple_plugin_indexes, + ix_match_w_plugin, + ix_match_w_plugin_ordered, + ix_match_w_plugin_bag]. + +groups() -> + []. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + + +add_rm_plugin(suite) -> []; +add_rm_plugin(Config) when is_list(Config) -> + [N1, N2] = Nodes = ?acquire_nodes(2, Config), + ok = add_plugin(), + ok = rpc_check_plugin(N1), + ok = rpc_check_plugin(N2), + ok = add_plugin2(), + ok = del_plugin(), + ok = del_plugin2(), + ok = add_plugin(), + ok = add_plugin2(), + ok = del_plugin(), + ok = del_plugin2(), + ?verify_mnesia(Nodes, []). + +-define(PLUGIN1, {{pfx},?MODULE,ix_prefixes}). +-define(PLUGIN2, {{pfx2},?MODULE,ix_prefixes2}). + +add_plugin() -> + {atomic, ok} = mnesia_schema:add_index_plugin({pfx}, ?MODULE, ix_prefixes), + [?PLUGIN1] = mnesia_schema:index_plugins(), + ok. + +add_plugin2() -> + {atomic, ok} = mnesia_schema:add_index_plugin({pfx2}, ?MODULE, ix_prefixes2), + [?PLUGIN1, ?PLUGIN2] = lists:sort(mnesia_schema:index_plugins()), + ok. + +del_plugin() -> + {atomic, ok} = mnesia_schema:delete_index_plugin({pfx}), + [?PLUGIN2] = mnesia_schema:index_plugins(), + ok. + +del_plugin2() -> + {atomic, ok} = mnesia_schema:delete_index_plugin({pfx2}), + [] = mnesia_schema:index_plugins(), + ok. + +rpc_check_plugin(N) -> + [?PLUGIN1] = + rpc:call(N, mnesia_schema, index_plugins, []), + ok. + +tab_with_plugin_index(suite) -> []; +tab_with_plugin_index(Config) when is_list(Config) -> + [_N1] = Nodes = ?acquire_nodes(1, Config), + ok = add_plugin(), + {atomic, ok} = mnesia:create_table(t, [{attributes, [k,v1,v2]}, + {index, [{{pfx}, ordered}, + {v1, ordered}, + v2]}]), + [ok,ok,ok,ok] = + [mnesia:dirty_write({t, K, V1, V2}) + || {K,V1,V2} <- [{1,a,"123"}, + {2,b,"12345"}, + {3,c,"6789"}, + {4,d,nil}]], + [{t,1,a,"123"},{t,2,b,"12345"}] = + mnesia:dirty_index_read(t,<<"123">>,{pfx}), + [{t,3,c,"6789"}] = + mnesia:dirty_index_read(t,"6789",v2), + [{t,1,a,"123"}] = + mnesia:dirty_match_object({t,'_',a,"123"}), + [{t,1,a,"123"}] = + mnesia:dirty_select(t, [{ {t,'_',a,"123"}, [], ['$_']}]), + mnesia:dirty_delete(t,2), + [{t,1,a,"123"}] = + mnesia:dirty_index_read(t,<<"123">>,{pfx}), + ?verify_mnesia(Nodes, []). + +tab_with_multiple_plugin_indexes(suite) -> []; +tab_with_multiple_plugin_indexes(Config) when is_list(Config) -> + [_N1] = Nodes = ?acquire_nodes(1, Config), + ok = add_plugin(), + ok = add_plugin2(), + {atomic, ok} = + mnesia:create_table(u, [{attributes, [k,v1,v2]}, + {index, [{{pfx}, ordered}, + {{pfx2}, ordered}]}]), + [ok,ok,ok,ok] = + [mnesia:dirty_write({u, K, V1, V2}) + || {K,V1,V2} <- [{1,a,"123"}, + {2,b,"12345"}, + {3,c,"6789"}, + {4,d,nil}]], + [{u,1,a,"123"},{u,2,b,"12345"}] = + mnesia:dirty_index_read(u,<<"123">>,{pfx}), + [{u,1,a,"123"},{u,2,b,"12345"}] = + mnesia:dirty_index_read(u,<<"321">>,{pfx2}), + ?verify_mnesia(Nodes, []). + +ix_match_w_plugin(suite) -> []; +ix_match_w_plugin(Config) when is_list(Config) -> + [_N1] = Nodes = ?acquire_nodes(1, Config), + ok = add_plugin(), + {atomic, ok} = mnesia:create_table(im1, [{attributes, [k, v1, v2]}, + {index, [{{pfx}, ordered}, + {v1, ordered}]}]), + fill_and_test_index_match(im1, set), + ?verify_mnesia(Nodes, []). + + +ix_match_w_plugin_ordered(suite) -> []; +ix_match_w_plugin_ordered(Config) when is_list(Config) -> + [_N1] = Nodes = ?acquire_nodes(1, Config), + ok = add_plugin(), + {atomic, ok} = mnesia:create_table(im2, [{attributes, [k, v1, v2]}, + {type, ordered_set}, + {index, [{{pfx}, ordered}, + {v1, ordered}]}]), + fill_and_test_index_match(im2, ordered_set), + ?verify_mnesia(Nodes, []). + +ix_match_w_plugin_bag(suite) -> []; +ix_match_w_plugin_bag(Config) when is_list(Config) -> + [_N1] = Nodes = ?acquire_nodes(1, Config), + ok = add_plugin(), + {atomic, ok} = mnesia:create_table(im3, [{attributes, [k, v1, v2]}, + {type, bag}, + {index, [{{pfx}, ordered}, + {v1, ordered}]}]), + fill_and_test_index_match(im3, bag), + ?verify_mnesia(Nodes, []). + +fill_and_test_index_match(Tab, Type) -> + [ok,ok,ok,ok,ok,ok,ok,ok,ok] = + [mnesia:dirty_write({Tab, K, V1, V2}) + || {K,V1,V2} <- [{1,a,"123"}, + {2,b,"12345"}, + {3,c,"123"}, + {4,d,nil}, + {5,e,nil}, + {6,f,nil}, + {7,g,nil}, %% overwritten if not bag + {7,g,"234"}, + {8,h,"123"}]], + mnesia:activity( + transaction, + fun() -> + ok = mnesia:write({Tab, 1, aa, "1234"}), %% replaces if not bag + ok = mnesia:delete({Tab, 2}), + ok = mnesia:delete({Tab, 4}), + ok = mnesia:write({Tab, 6, ff, nil}), + ok = mnesia:write({Tab, 7, gg, "123"}), + ok = mnesia:write({Tab, 100, x, nil}), + ok = mnesia:delete_object({Tab,3,c,"123"}), + ok = mnesia:delete_object({Tab,5,e,nil}), + Res = mnesia:index_read(Tab, <<"123">>, {pfx}), + SetRes = [{Tab,1,aa,"1234"}, {Tab,7,gg,"123"}, {Tab,8,h,"123"}], + case Type of + set -> + SetRes = lists:sort(Res); + ordered_set -> + SetRes = Res; + bag -> + [{Tab,1,a,"123"}, {Tab,1,aa,"1234"}, + {Tab,7,gg,"123"}, {Tab,8,h,"123"}] = lists:sort(Res) + end + end). + +%% ============================================================ +%% +ix_prefixes(_Tab, _Pos, Obj) -> + lists:foldl( + fun(V, Acc) when is_list(V) -> + try Pfxs = prefixes(list_to_binary(V)), + Pfxs ++ Acc + catch + error:_ -> + Acc + end; + (V, Acc) when is_binary(V) -> + Pfxs = prefixes(V), + Pfxs ++ Acc; + (_, Acc) -> + Acc + end, [], tl(tuple_to_list(Obj))). + +ix_prefixes2(Tab, Pos, Obj) -> + [rev(P) || P <- ix_prefixes(Tab, Pos, Obj)]. + +rev(B) when is_binary(B) -> + list_to_binary(lists:reverse(binary_to_list(B))). + +prefixes(<<P:3/binary, _/binary>>) -> + [P]; +prefixes(_) -> + []. diff --git a/lib/mnesia/test/mt.erl b/lib/mnesia/test/mt.erl index 5a981bf539..037d6adb38 100644 --- a/lib/mnesia/test/mt.erl +++ b/lib/mnesia/test/mt.erl @@ -67,6 +67,7 @@ alias(recovery) -> mnesia_recovery_test; alias(registry) -> mnesia_registry_test; alias(suite) -> mnesia_SUITE; alias(trans) -> mnesia_trans_access_test; +alias(ixp) -> mnesia_index_plugin_test; alias(Other) -> Other. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% diff --git a/lib/observer/test/observer_SUITE.erl b/lib/observer/test/observer_SUITE.erl index 75336cedcc..7d54bb9b3b 100644 --- a/lib/observer/test/observer_SUITE.erl +++ b/lib/observer/test/observer_SUITE.erl @@ -41,7 +41,8 @@ %% Default timetrap timeout (set in init_per_testcase) -define(default_timeout, ?t:minutes(2)). -suite() -> [{ct_hooks,[ts_install_cth]}]. +suite() -> [{timetrap, {minutes, 5}}, + {ct_hooks,[ts_install_cth]}]. all() -> [app_file, appup_file, {group, gui}]. diff --git a/lib/public_key/asn1/OTP-PKIX.asn1 b/lib/public_key/asn1/OTP-PKIX.asn1 index 9bcd99fba3..ff3250b383 100644 --- a/lib/public_key/asn1/OTP-PKIX.asn1 +++ b/lib/public_key/asn1/OTP-PKIX.asn1 @@ -233,9 +233,13 @@ countryName ATTRIBUTE-TYPE-AND-VALUE-CLASS ::= { -- regarding how to handle and sometimes accept incorrect certificates -- we define and use the type below instead of X520countryName + -- We accept utf8String encoding of the US-ASCII + -- country name code and the mix up with other country code systems + -- that uses three characters instead of two. + OTP-X520countryname ::= CHOICE { - printableString PrintableString (SIZE (2)), - utf8String UTF8String (SIZE (2)) + printableString PrintableString (SIZE (2..3)), + utf8String UTF8String (SIZE (2..3)) } serialNumber ATTRIBUTE-TYPE-AND-VALUE-CLASS ::= { diff --git a/lib/ssl/doc/src/Makefile b/lib/ssl/doc/src/Makefile index 7cf251d8f9..064131944c 100644 --- a/lib/ssl/doc/src/Makefile +++ b/lib/ssl/doc/src/Makefile @@ -47,6 +47,7 @@ XML_CHAPTER_FILES = \ ssl_protocol.xml \ using_ssl.xml \ ssl_distribution.xml \ + standards_compliance.xml \ notes.xml BOOK_FILES = book.xml diff --git a/lib/ssl/doc/src/ssl.xml b/lib/ssl/doc/src/ssl.xml index 37bf9033a1..74a0a0a03e 100644 --- a/lib/ssl/doc/src/ssl.xml +++ b/lib/ssl/doc/src/ssl.xml @@ -145,8 +145,13 @@ </datatype> <datatype> - <name name="legacy_version"/> + <name name="tls_legacy_version"/> </datatype> + + <datatype> + <name name="dtls_legacy_version"/> + </datatype> + <datatype> <name name="prf_random"/> diff --git a/lib/ssl/doc/src/ssl_app.xml b/lib/ssl/doc/src/ssl_app.xml index 893919aeb4..b05caf44ea 100644 --- a/lib/ssl/doc/src/ssl_app.xml +++ b/lib/ssl/doc/src/ssl_app.xml @@ -35,45 +35,10 @@ <description> <p> - The ssl application is an implementation of the SSL/TLS/DTLS protocol in Erlang. + The ssl application is an implementation of the SSL, TLS and DTLS protocols in Erlang. </p> - <list type="bulleted"> - <item>Supported SSL/TLS/DTLS-versions are SSL-3.0, TLS-1.0, - TLS-1.1, TLS-1.2, DTLS-1.0 (based on TLS-1.1), DTLS-1.2 (based on TLS-1.2)</item> - <item>For security reasons SSL-2.0 is not supported. - Interoperability with SSL-2.0 enabled clients dropped. (OTP 21) </item> - <item>For security reasons SSL-3.0 is no longer supported by default, - but can be configured. (OTP 19) </item> - <item>For security reasons RSA key exchange cipher suites are no longer supported by default, - but can be configured. (OTP 21) </item> - <item>For security reasons DES cipher suites are no longer supported by default, - but can be configured. (OTP 20) </item> - <item>For security reasons 3DES cipher suites are no longer supported by default, - but can be configured. (OTP 21) </item> - <item> Renegotiation Indication Extension <url href="http://www.ietf.org/rfc/rfc5746.txt">RFC 5746</url> is supported - </item> - <item>Ephemeral Diffie-Hellman cipher suites are supported, - but not Diffie Hellman Certificates cipher suites.</item> - <item>Elliptic Curve cipher suites are supported if the Crypto - application supports it and named curves are used. - </item> - <item>Export cipher suites are not supported as the - U.S. lifted its export restrictions in early 2000.</item> - <item>IDEA cipher suites are not supported as they have - become deprecated by the latest TLS specification so it is not - motivated to implement them.</item> - <item>Compression is not supported.</item> - <item>CRL validation is supported.</item> - <item>Policy certificate extensions are not supported.</item> - <item>'Server Name Indication' extension - (<url href="http://www.ietf.org/rfc/rfc6066.txt">RFC 6066</url>) is supported.</item> - <item>Application Layer Protocol Negotiation (ALPN) and its successor Next Protocol Negotiation (NPN) - are supported. </item> - <item>It is possible to use Pre-Shared Key (PSK) and Secure Remote Password (SRP) - cipher suites, but they are not enabled by default. - </item> - </list> - </description> + <p>For current statement of standards compliance see the <seealso marker="standards_compliance">User's Guide</seealso>.</p> + </description> <section> <title>DEPENDENCIES</title> diff --git a/lib/ssl/doc/src/standards_compliance.xml b/lib/ssl/doc/src/standards_compliance.xml new file mode 100644 index 0000000000..c20bab4e50 --- /dev/null +++ b/lib/ssl/doc/src/standards_compliance.xml @@ -0,0 +1,2312 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE chapter SYSTEM "chapter.dtd"> + +<chapter> + <header> + <copyright> + <year>2015</year> + <year>2019</year> + <holder>Ericsson AB, All Rights Reserved</holder> + </copyright> + <legalnotice> + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + The Initial Developer of the Original Code is Ericsson AB. + </legalnotice> + + <title>Standards Compliance</title> + <prepared>OTP team</prepared> + <docno></docno> + <date>2019-03-20</date> + <rev>A</rev> + <file>standards_compliance.xml</file> + </header> + + <section> + <title>Purpose</title> + <p>This section describes the current state of standards compliance of the ssl application.</p> + </section> + + <section> + <title>Common (pre TLS 1.3)</title> + <list type="bulleted"> + <item>For security reasons RSA key exchange cipher suites are no longer supported by default, + but can be configured. (OTP 21) + </item> + <item>For security reasons DES cipher suites are no longer supported by default, + but can be configured. (OTP 20) + </item> + <item>For security reasons 3DES cipher suites are no longer supported by default, + but can be configured. (OTP 21) + </item> + <item>Renegotiation Indication Extension <url href="http://www.ietf.org/rfc/rfc5746.txt">RFC 5746</url> is supported + </item> + <item>Ephemeral Diffie-Hellman cipher suites are supported, + but not Diffie Hellman Certificates cipher suites. + </item> + <item>Elliptic Curve cipher suites are supported if the Crypto + application supports it and named curves are used. + </item> + <item>Export cipher suites are not supported as the + U.S. lifted its export restrictions in early 2000. + </item> + <item>IDEA cipher suites are not supported as they have + become deprecated by the TLS 1.2 specification so it is not + motivated to implement them. + </item> + <item>Compression is not supported. + </item> + </list> + </section> + + <section> + <title>Common</title> + <list type="bulleted"> + <item>CRL validation is supported.</item> + <item>Policy certificate extensions are not supported.</item> + <item>'Server Name Indication' extension + (<url href="http://www.ietf.org/rfc/rfc6066.txt">RFC 6066</url>) is supported.</item> + <item>Application Layer Protocol Negotiation (ALPN) and its successor Next Protocol Negotiation (NPN) are supported. </item> + <item>It is possible to use Pre-Shared Key (PSK) and Secure Remote Password (SRP) + cipher suites, but they are not enabled by default. + </item> + </list> + </section> + + + <section> + <title>SSL 2.0</title> + <p>For security reasons SSL-2.0 is not supported. Interoperability with SSL-2.0 enabled clients dropped. (OTP 21)</p> + </section> + + <section> + <title>SSL 3.0</title> + <p>For security reasons SSL-3.0 is no longer supported by default, but can be configured. (OTP 19)</p> + </section> + + <section> + <title>TLS 1.0</title> + <p>For security reasons TLS-1.0 is no longer supported by default, but can be configured. (OTP 22)</p> + </section> + + <section> + <title>TLS 1.1</title> + <p>For security reasons TLS-1.1 is no longer supported by default, but can be configured. (OTP 22)</p> + </section> + + <section> + <title>TLS 1.2</title> + <p>Supported</p> + </section> + + <section> + <title>DTLS 1.0</title> + <p>For security reasons DTLS-1.0 (based on TLS 1.1) is no longer supported by default, but can be configured. (OTP 22)</p> + </section> + + <section> + <title>DTLS 1.2</title> + <p>Supported (based on TLS 1.2)</p> + </section> + + <section> + <title>DTLS 1.3</title> + <p>Not yet supported</p> + </section> + + <section> + <title>TLS 1.3</title> + <p> This section describes the current state of standards compliance for TLS 1.3.</p> + <p>(C = Compliant, NC = Non-Compliant, P = Partially-Compliant, NA = Not Applicable)</p> + <table> + <row> + <cell align="left" valign="middle"><em>Section</em></cell> + <cell align="left" valign="middle"><em>Feature</em></cell> + <cell align="left" valign="middle"><em>State</em></cell> + <cell align="left" valign="middle"><em>Since</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-1.2"> + 1.3. Updates Affecting TLS 1.2 + </url> + </cell> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">Version downgrade protection mechanism</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">RSASSA-PSS signature schemes</cell> + <cell align="left" valign="middle"><em>P</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">supported_versions (ClientHello) extension</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signature_algorithms_cert extension</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-2"> + 2. Protocol Overview + </url> + </cell> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>P</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">(EC)DHE</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">PSK-only</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">PSK with (EC)DHE</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-2.1"> + 2.1. Incorrect DHE share + </url> + </cell> + <cell align="left" valign="middle">HelloRetryRequest</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-2.2"> + 2.2. Resumption and Pre-Shared Key (PSK) + </url> + </cell> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-2.3"> + 2.3. 0-RTT Data + </url> + </cell> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.1.1"> + 4.1.1. Cryptographic Negotiation + </url> + </cell> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>P</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">supported_groups extension</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signature_algorithms extension</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">pre_shared_key extension</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.1.2"> + 4.1.2. Client Hello + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">server_name (RFC6066)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">max_fragment_length (RFC6066)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">status_request (RFC6066)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">supported_groups (RFC7919)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signature_algorithms (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">use_srtp (RFC5764)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">heartbeat (RFC6520)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">application_layer_protocol_negotiation (RFC7301)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signed_certificate_timestamp (RFC6962)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">client_certificate_type (RFC7250)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">server_certificate_type (RFC7250)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">padding (RFC7685)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">key_share (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">pre_shared_key (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">psk_key_exchange_modes (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">early_data (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">cookie (RFC8446) </cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">supported_versions (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">certificate_authorities (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">oid_filters (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">post_handshake_auth (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signature_algorithms_cert (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>PC</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">server_name (RFC6066)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">max_fragment_length (RFC6066)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">status_request (RFC6066)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">supported_groups (RFC7919)</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signature_algorithms (RFC8446)</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">use_srtp (RFC5764)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">heartbeat (RFC6520)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">application_layer_protocol_negotiation (RFC7301)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signed_certificate_timestamp (RFC6962)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">client_certificate_type (RFC7250)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">server_certificate_type (RFC7250)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">padding (RFC7685)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">key_share (RFC8446)</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">pre_shared_key (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">psk_key_exchange_modes (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">early_data (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">cookie (RFC8446) </cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">supported_versions (RFC8446)</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">certificate_authorities (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">oid_filters (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">post_handshake_auth (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signature_algorithms_cert (RFC8446)</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.1.3"> + 4.1.3. Server Hello + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">Version downgrade protection</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">key_share (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">pre_shared_key (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">supported_versions (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>PC</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">Version downgrade protection</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">key_share (RFC8446)</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">pre_shared_key (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">supported_versions (RFC8446)</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.1.4"> + 4.1.4. Hello Retry Request + </url> + </cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>PC</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">key_share (RFC8446)</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">cookie (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">supported_versions (RFC8446)</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.2.1"> + 4.2.1. Supported Versions + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.2.2"> + 4.2.2. Cookie + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.2.3"> + 4.2.3. Signature Algorithms + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pkcs1_sha256</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pkcs1_sha384</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pkcs1_sha512</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ecdsa_secp256r1_sha256</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ecdsa_secp384r1_sha384</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ecdsa_secp521r1_sha512</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pss_rsae_sha256</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pss_rsae_sha384</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pss_rsae_sha512</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ed25519</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ed448</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pss_pss_sha256</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pss_pss_sha384</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pss_pss_sha512</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pkcs1_sha1</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ecdsa_sha1</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>P</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pkcs1_sha256</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pkcs1_sha384</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pkcs1_sha512</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ecdsa_secp256r1_sha256</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ecdsa_secp384r1_sha384</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ecdsa_secp521r1_sha512</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pss_rsae_sha256</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pss_rsae_sha384</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pss_rsae_sha512</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ed25519</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ed448</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pss_pss_sha256</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pss_pss_sha384</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pss_pss_sha512</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pkcs1_sha1</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ecdsa_sha1</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.2.4"> + 4.2.4. Certificate Authorities + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.2.5"> + 4.2.5. OID Filters + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.2.6"> + 4.2.6. Post-Handshake Client Authentication + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.2.7"> + 4.2.7. Supported Groups + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">secp256r1</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">secp384r1</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">secp521r1</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">x25519</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">x448</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ffdhe2048</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ffdhe3072</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ffdhe4096</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ffdhe6144</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ffdhe8192</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">secp256r1</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">secp384r1</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">secp521r1</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">x25519</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">x448</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ffdhe2048</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ffdhe3072</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ffdhe4096</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ffdhe6144</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ffdhe8192</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.2.8"> + 4.2.8. Key Share + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.2.9"> + 4.2.9. Pre-Shared Key Exchange Modes + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.2.10"> + 4.2.10. Early Data Indication + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.2.11"> + 4.2.11. Pre-Shared Key Extension + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.2.11.1"> + 4.2.11.1. Ticket Age + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.2.11.2"> + 4.2.11.2. PSK Binder + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.2.11.3"> + 4.2.11.3. Processing Order + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.3.1"> + 4.3.1. Encrypted Extensions + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">server_name (RFC6066)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">max_fragment_length (RFC6066)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">supported_groups (RFC7919)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">use_srtp (RFC5764)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">heartbeat (RFC6520)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">application_layer_protocol_negotiation (RFC7301)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">client_certificate_type (RFC7250)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">server_certificate_type (RFC7250)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">early_data (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">supported_versions (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>P</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">server_name (RFC6066)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">max_fragment_length (RFC6066)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">supported_groups (RFC7919)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">use_srtp (RFC5764)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">heartbeat (RFC6520)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">application_layer_protocol_negotiation (RFC7301)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">client_certificate_type (RFC7250)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">server_certificate_type (RFC7250)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">early_data (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">supported_versions (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.3.2"> + 4.3.2. Certificate Request + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">status_request (RFC6066)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signature_algorithms (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signed_certificate_timestamp (RFC6962)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">certificate_authorities (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">oid_filters (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signature_algorithms_cert (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>P</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">status_request (RFC6066)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signature_algorithms (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signed_certificate_timestamp (RFC6962)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">certificate_authorities (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">oid_filters (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signature_algorithms_cert (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.4.1"> + 4.4.1. The Transcript Hash + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.4.2"> + 4.4.2. Certificate + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">status_request (RFC6066)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signed_certificate_timestamp (RFC6962)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>P</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">status_request (RFC6066)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signed_certificate_timestamp (RFC6962)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.4.2.1"> + 4.4.2.1. OCSP Status and SCT Extensions + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.4.2.2"> + 4.4.2.2. Server Certificate Selection + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">certificate type MUST be X.509v3</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">certificate's public key is compatible</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">The certificate MUST allow the key to be used for signing</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">server_name and certificate_authorities are used</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>P</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">certificate type MUST be X.509v3</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">certificate's public key is compatible</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">The certificate MUST allow the key to be used for signing</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">server_name and certificate_authorities are used</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.4.2.3"> + 4.4.2.3. Client Certificate Selection + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.4.2.4"> + 4.4.2.4. Receiving a Certificate Message + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.4.3"> + 4.4.3. Certificate Verify + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.4.4"> + 4.4.4. Finished + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.5"> + 4.5. End of Early Data + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.6.1"> + 4.6.1. New Session Ticket Message + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">early_data (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">early_data (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.6.2"> + 4.6.2. Post-Handshake Authentication + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.6.3"> + 4.6.3. Key and Initialization Vector Update + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-5.1"> + 5.1. Record Layer + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">MUST NOT be interleaved with other record types</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">MUST NOT span key changes</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">MUST NOT send zero-length fragments</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">Alert messages MUST NOT be fragmented</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-5.2"> + 5.2. Record Payload Protection + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-5.3"> + 5.3. Per-Record Nonce + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-5.4"> + 5.4. Record Padding + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>P</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">MAY choose to pad</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">MUST NOT send Handshake and Alert records that have a zero-length TLSInnerPlaintext.content</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">The padding sent is automatically verified</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-5.5"> + 5.5. Limits on Key Usage + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-6.1"> + 6.1. Closure Alerts + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">close_notify</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">user_cancelled</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-6.2"> + 6.2. Error Alerts + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>PC</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-7.1"> + 7.1. Key Schedule + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-7.2"> + 7.2. Updating Traffic Secrets + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-7.3"> + 7.3. Traffic Key Calculation + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-7.5"> + 7.5. Exporters + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-8"> + 8. 0-RTT and Anti-Replay + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-8.1"> + 8.1. Single-Use Tickets + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-8.2"> + 8.2. Client Hello Recording + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-8.3"> + 8.3. Freshness Checks + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-9.1"> + 9.1. Mandatory-to-Implement Cipher Suites + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>P</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">MUST implement the TLS_AES_128_GCM_SHA256</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">SHOULD implement the TLS_AES_256_GCM_SHA384</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">SHOULD implement the TLS_CHACHA20_POLY1305_SHA256</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Digital signatures</em></cell> + <cell align="left" valign="middle"><em>P</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">MUST support rsa_pkcs1_sha256 (for certificates)</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">MUST support rsa_pss_rsae_sha256 (for CertificateVerify and certificates)</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">MUST support ecdsa_secp256r1_sha256</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Key Exchange</em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">MUST support key exchange with secp256r1</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">SHOULD support key exchange with X25519</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-9.2"> + 9.2. Mandatory-to-Implement Extensions + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>P</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">Supported Versions</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">Cookie</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">Signature Algorithms</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">Signature Algorithms Certificate</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">Negotiated Groups</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">Key Share</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">Server Name Indication</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>MUST send and use these extensions</em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">"supported_versions" is REQUIRED for ClientHello, ServerHello and HelloRetryRequest</cell> + <cell align="left" valign="middle"><em>PC</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">"signature_algorithms" is REQUIRED for certificate authentication</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">"supported_groups" is REQUIRED for ClientHello messages using (EC)DHE key exchange</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">"key_share" is REQUIRED for (EC)DHE key exchange</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">"pre_shared_key" is REQUIRED for PSK key agreement</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">"psk_key_exchange_modes" is REQUIRED for PSK key agreement</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>TLS 1.3 ClientHello</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">If not containing a "pre_shared_key" extension, it MUST contain both a "signature_algorithms" extension and a "supported_groups" extension.</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">If containing a "supported_groups" extension, it MUST also contain a "key_share" extension, and vice versa. An empty KeyShare.client_shares vector is permitted.</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>TLS 1.3 ServerHello</em></cell> + <cell align="left" valign="middle"><em>P</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">MUST support the use of the "server_name" extension</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-9.3"> + 9.3. Protocol Invariants + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>MUST correctly handle extensible fields</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">A client sending a ClientHello MUST support all parameters advertised in it.</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">A middlebox which terminates a TLS connection MUST behave as a compliant TLS server</cell> + <cell align="left" valign="middle"><em>NA</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">A middlebox which forwards ClientHello parameters it does not understand MUST NOT process any messages beyond that ClientHello.</cell> + <cell align="left" valign="middle"><em>NA</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-B.4"> + B.4. Cipher Suites + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>P</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">TLS_AES_128_GCM_SHA256</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">TLS_AES_256_GCM_SHA384</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">TLS_CHACHA20_POLY1305_SHA256</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">TLS_AES_128_CCM_SHA256</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">TLS_AES_128_CCM_8_SHA256</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-C.1"> + C.1. Random Number Generation and Seeding + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-C.2"> + C.2. Certificates and Authentication + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-C.3"> + C.3. Implementation Pitfalls + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>P</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-C.4"> + C.4. Client Tracking Prevention + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-C.5"> + C.5. Unauthenticated Operation + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-D.1"> + D.1. Negotiating with an Older Server + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-D.2"> + D.2. Negotiating with an Older Client + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-D.3"> + D.3. 0-RTT Backward Compatibility + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-D.4"> + D.4. Middlebox Compatibility Mode + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>P</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-D.5"> + D.5. Security Restrictions Related to Backward Compatibility + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <tcaption>Standards Compliance</tcaption> + </table> + + </section> + +</chapter> diff --git a/lib/ssl/doc/src/usersguide.xml b/lib/ssl/doc/src/usersguide.xml index 23ccf668c3..b22b2456e4 100644 --- a/lib/ssl/doc/src/usersguide.xml +++ b/lib/ssl/doc/src/usersguide.xml @@ -38,6 +38,7 @@ <xi:include href="ssl_protocol.xml"/> <xi:include href="using_ssl.xml"/> <xi:include href="ssl_distribution.xml"/> + <xi:include href="standards_compliance.xml"/> </part> diff --git a/lib/ssl/src/dtls_connection.erl b/lib/ssl/src/dtls_connection.erl index ed47980a69..7993be8a74 100644 --- a/lib/ssl/src/dtls_connection.erl +++ b/lib/ssl/src/dtls_connection.erl @@ -193,7 +193,8 @@ next_event(StateName, no_record, %% TODO maybe buffer later epoch next_event(StateName, no_record, State, Actions); {#alert{} = Alert, State} -> - {next_state, StateName, State, [{next_event, internal, Alert} | Actions]} + Version = State#state.connection_env#connection_env.negotiated_version, + handle_own_alert(Alert, Version, StateName, State) end; next_event(connection = StateName, Record, #state{connection_states = #{current_read := #{epoch := CurrentEpoch}}} = State0, Actions) -> @@ -233,7 +234,8 @@ next_event(StateName, Record, %% TODO maybe buffer later epoch next_event(StateName, no_record, State0, Actions); #alert{} = Alert -> - {next_state, StateName, State0, [{next_event, internal, Alert} | Actions]} + Version = State0#state.connection_env#connection_env.negotiated_version, + handle_own_alert(Alert, Version, StateName, State0) end. %%% DTLS record protocol level application data messages @@ -840,7 +842,7 @@ next_dtls_record(Data, StateName, #state{protocol_buffers = #protocol_buffers{ end. acceptable_record_versions(hello, _) -> - [dtls_record:protocol_version(Vsn) || Vsn <- ?ALL_DATAGRAM_SUPPORTED_VERSIONS]; + [dtls_record:protocol_version(Vsn) || Vsn <- ?ALL_AVAILABLE_DATAGRAM_VERSIONS]; acceptable_record_versions(_, #state{connection_env = #connection_env{negotiated_version = Version}}) -> [Version]. @@ -1075,10 +1077,10 @@ start_retransmision_timer(Timeout, #state{protocol_specific = PS} = State) -> {State#state{protocol_specific = PS#{flight_state => {retransmit, new_timeout(Timeout)}}}, [{state_timeout, Timeout, flight_retransmission_timeout}]}. -new_timeout(N) when N =< 30 -> +new_timeout(N) when N =< 30000 -> N * 2; new_timeout(_) -> - 60. + 60000. send_handshake_flight(#state{static_env = #static_env{socket = Socket, transport_cb = Transport}, diff --git a/lib/ssl/src/dtls_packet_demux.erl b/lib/ssl/src/dtls_packet_demux.erl index e0423b07b4..2e9184b7ac 100644 --- a/lib/ssl/src/dtls_packet_demux.erl +++ b/lib/ssl/src/dtls_packet_demux.erl @@ -203,9 +203,9 @@ dispatch(Client, Msg, #state{dtls_msq_queues = MsgQueues} = State) -> Pid ! Msg, State#state{dtls_msq_queues = kv_update(Client, Queue, MsgQueues)}; - {{value, _}, Queue} -> + {{value, _UDP}, _Queue} -> State#state{dtls_msq_queues = - kv_update(Client, queue:in(Msg, Queue), MsgQueues)}; + kv_update(Client, queue:in(Msg, Queue0), MsgQueues)}; {empty, Queue} -> State#state{dtls_msq_queues = kv_update(Client, queue:in(Msg, Queue), MsgQueues)} diff --git a/lib/ssl/src/inet_tls_dist.erl b/lib/ssl/src/inet_tls_dist.erl index e7fab7ebc5..8d9b92361b 100644 --- a/lib/ssl/src/inet_tls_dist.erl +++ b/lib/ssl/src/inet_tls_dist.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2011-2018. All Rights Reserved. +%% Copyright Ericsson AB 2011-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -132,8 +132,8 @@ f_recv(SslSocket, Length, Timeout) -> f_setopts_pre_nodeup(_SslSocket) -> ok. -f_setopts_post_nodeup(_SslSocket) -> - ok. +f_setopts_post_nodeup(SslSocket) -> + ssl:setopts(SslSocket, [nodelay()]). f_getll(DistCtrl) -> {ok, DistCtrl}. @@ -199,7 +199,7 @@ listen(Name) -> gen_listen(Driver, Name) -> case inet_tcp_dist:gen_listen(Driver, Name) of {ok, {Socket, Address, Creation}} -> - inet:setopts(Socket, [{packet, 4}]), + inet:setopts(Socket, [{packet, 4}, {nodelay, true}]), {ok, {Socket, Address#net_address{protocol=tls}, Creation}}; Other -> Other @@ -532,7 +532,7 @@ do_setup_connect(Driver, Kernel, Node, Address, Ip, TcpPort, Version, Type, MyNo case ssl:connect( Address, TcpPort, [binary, {active, false}, {packet, 4}, - Driver:family(), nodelay()] ++ Opts, + Driver:family(), {nodelay, true}] ++ Opts, net_kernel:connecttime()) of {ok, #sslsocket{pid = [_, DistCtrl| _]} = SslSocket} -> _ = monitor_pid(DistCtrl), diff --git a/lib/ssl/src/ssl.erl b/lib/ssl/src/ssl.erl index bfa349c8d8..8807c575b1 100644 --- a/lib/ssl/src/ssl.erl +++ b/lib/ssl/src/ssl.erl @@ -103,14 +103,19 @@ -type ip_address() :: inet:ip_address(). -type session_id() :: binary(). -type protocol_version() :: tls_version() | dtls_version(). --type tls_version() :: tlsv1 | 'tlsv1.1' | 'tlsv1.2' | 'tlsv1.3' | legacy_version(). --type dtls_version() :: 'dtlsv1' | 'dtlsv1.2'. --type legacy_version() :: sslv3. +-type tls_version() :: 'tlsv1.2' | 'tlsv1.3' | tls_legacy_version(). +-type dtls_version() :: 'dtlsv1.2' | dtls_legacy_version(). +-type tls_legacy_version() :: tlsv1 | 'tlsv1.1' | sslv3. +-type dtls_legacy_version() :: 'dtlsv1'. -type verify_type() :: verify_none | verify_peer. -type cipher() :: aes_128_cbc | aes_256_cbc | aes_128_gcm | aes_256_gcm | + aes_128_ccm | + aes_256_ccm | + aes_128_ccm_8 | + aes_256_ccm_8 | chacha20_poly1305 | legacy_cipher(). -type legacy_cipher() :: rc4_128 | diff --git a/lib/ssl/src/ssl_cipher.erl b/lib/ssl/src/ssl_cipher.erl index fe8736d2df..97878431a6 100644 --- a/lib/ssl/src/ssl_cipher.erl +++ b/lib/ssl/src/ssl_cipher.erl @@ -35,7 +35,7 @@ -include_lib("public_key/include/public_key.hrl"). -export([security_parameters/2, security_parameters/3, security_parameters_1_3/2, - cipher_init/3, nonce_seed/2, decipher/6, cipher/5, aead_encrypt/5, aead_decrypt/6, + cipher_init/3, nonce_seed/2, decipher/6, cipher/5, aead_encrypt/6, aead_decrypt/6, suites/1, all_suites/1, crypto_support_filters/0, chacha_suites/1, anonymous_suites/1, psk_suites/1, psk_suites_anon/1, srp_suites/0, srp_suites_anon/0, @@ -106,9 +106,13 @@ security_parameters_1_3(SecParams, CipherSuite) -> cipher_init(?RC4, IV, Key) -> State = crypto:stream_init(rc4, Key), #cipher_state{iv = IV, key = Key, state = State}; -cipher_init(?AES_GCM, IV, Key) -> +cipher_init(Type, IV, Key) when Type == ?AES_GCM; + Type == ?AES_CCM -> <<Nonce:64>> = random_bytes(8), #cipher_state{iv = IV, key = Key, nonce = Nonce, tag_len = 16}; +cipher_init(?AES_CCM_8, IV, Key) -> + <<Nonce:64>> = random_bytes(8), + #cipher_state{iv = IV, key = Key, nonce = Nonce, tag_len = 8}; cipher_init(?CHACHA20_POLY1305, IV, Key) -> #cipher_state{iv = IV, key = Key, tag_len = 16}; cipher_init(_BCA, IV, Key) -> @@ -148,14 +152,18 @@ cipher(?AES_CBC, CipherState, Mac, Fragment, Version) -> crypto:block_encrypt(aes_cbc256, Key, IV, T) end, block_size(aes_128_cbc), CipherState, Mac, Fragment, Version). -aead_encrypt(Type, Key, Nonce, Fragment, AdditionalData) -> - crypto:block_encrypt(aead_type(Type), Key, Nonce, {AdditionalData, Fragment}). +aead_encrypt(Type, Key, Nonce, Fragment, AdditionalData, TagLen) -> + crypto:block_encrypt(aead_type(Type), Key, Nonce, {AdditionalData, Fragment, TagLen}). aead_decrypt(Type, Key, Nonce, CipherText, CipherTag, AdditionalData) -> crypto:block_decrypt(aead_type(Type), Key, Nonce, {AdditionalData, CipherText, CipherTag}). aead_type(?AES_GCM) -> aes_gcm; +aead_type(?AES_CCM) -> + aes_ccm; +aead_type(?AES_CCM_8) -> + aes_ccm; aead_type(?CHACHA20_POLY1305) -> chacha20_poly1305. @@ -311,8 +319,7 @@ anonymous_suites({254, _} = Version) -> dtls_v1:anonymous_suites(Version); anonymous_suites(4) -> []; %% Raw public key negotiation may be used instead -anonymous_suites(N) - when N >= 3 -> +anonymous_suites( 3 = N) -> psk_suites_anon(N) ++ [?TLS_DH_anon_WITH_AES_128_GCM_SHA256, ?TLS_DH_anon_WITH_AES_256_GCM_SHA384, @@ -347,8 +354,7 @@ psk_suites({3, N}) -> psk_suites(N); psk_suites(4) -> []; %% TODO Add new PSK, PSK_(EC)DHE suites -psk_suites(N) - when N >= 3 -> +psk_suites(3) -> [ ?TLS_RSA_PSK_WITH_AES_256_GCM_SHA384, ?TLS_RSA_PSK_WITH_AES_256_CBC_SHA384, @@ -369,20 +375,32 @@ psk_suites(_) -> %%-------------------------------------------------------------------- psk_suites_anon({3, N}) -> psk_suites_anon(N); -psk_suites_anon(N) - when N >= 3 -> +psk_suites_anon(3) -> [ ?TLS_DHE_PSK_WITH_AES_256_GCM_SHA384, ?TLS_PSK_WITH_AES_256_GCM_SHA384, ?TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA384, ?TLS_DHE_PSK_WITH_AES_256_CBC_SHA384, ?TLS_PSK_WITH_AES_256_CBC_SHA384, + ?TLS_DHE_PSK_WITH_AES_256_CCM, + ?TLS_PSK_DHE_WITH_AES_256_CCM_8, + ?TLS_PSK_WITH_AES_256_CCM, + ?TLS_PSK_WITH_AES_256_CCM_8, ?TLS_ECDHE_PSK_WITH_AES_128_GCM_SHA256, + ?TLS_ECDHE_PSK_WITH_AES_128_CCM_SHA256, + ?TLS_ECDHE_PSK_WITH_AES_128_CCM_8_SHA256, ?TLS_DHE_PSK_WITH_AES_128_GCM_SHA256, ?TLS_PSK_WITH_AES_128_GCM_SHA256, + ?TLS_ECDHE_PSK_WITH_AES_128_GCM_SHA256, + ?TLS_ECDHE_PSK_WITH_AES_128_CCM_8_SHA256, ?TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA256, ?TLS_DHE_PSK_WITH_AES_128_CBC_SHA256, - ?TLS_PSK_WITH_AES_128_CBC_SHA256 + ?TLS_PSK_WITH_AES_128_CBC_SHA256, + ?TLS_DHE_PSK_WITH_AES_128_CCM, + ?TLS_PSK_DHE_WITH_AES_128_CCM_8, + ?TLS_PSK_WITH_AES_128_CCM, + ?TLS_PSK_WITH_AES_128_CCM_8, + ?TLS_ECDHE_PSK_WITH_RC4_128_SHA ] ++ psk_suites_anon(0); psk_suites_anon(_) -> [?TLS_DHE_PSK_WITH_AES_256_CBC_SHA, @@ -589,7 +607,7 @@ is_acceptable_keyexchange(dhe_rsa, Algos) -> proplists:get_bool(dh, Algos) andalso proplists:get_bool(rsa, Algos); is_acceptable_keyexchange(KeyExchange, Algos) when KeyExchange == ecdh_anon; - KeyExchange == ecdhe_psk -> + KeyExchange == ecdhe_psk -> proplists:get_bool(ecdh, Algos); is_acceptable_keyexchange(KeyExchange, Algos) when KeyExchange == ecdh_ecdsa; KeyExchange == ecdhe_ecdsa -> @@ -629,6 +647,12 @@ is_acceptable_cipher(Cipher, Algos) when Cipher == aes_128_gcm; Cipher == aes_256_gcm -> proplists:get_bool(aes_gcm, Algos); +is_acceptable_cipher(Cipher, Algos) + when Cipher == aes_128_ccm; + Cipher == aes_256_ccm; + Cipher == aes_128_ccm_8; + Cipher == aes_256_ccm_8 -> + proplists:get_bool(aes_ccm, Algos); is_acceptable_cipher(Cipher, Algos) -> proplists:get_bool(Cipher, Algos). @@ -721,6 +745,12 @@ bulk_cipher_algorithm(Cipher) when Cipher == aes_128_cbc; bulk_cipher_algorithm(Cipher) when Cipher == aes_128_gcm; Cipher == aes_256_gcm -> ?AES_GCM; +bulk_cipher_algorithm(Cipher) when Cipher == aes_128_ccm; + Cipher == aes_256_ccm -> + ?AES_CCM; +bulk_cipher_algorithm(Cipher) when Cipher == aes_128_ccm_8; + Cipher == aes_256_ccm_8 -> + ?AES_CCM_8; bulk_cipher_algorithm(chacha20_poly1305) -> ?CHACHA20_POLY1305. @@ -735,6 +765,10 @@ type(Cipher) when Cipher == des_cbc; ?BLOCK; type(Cipher) when Cipher == aes_128_gcm; Cipher == aes_256_gcm; + Cipher == aes_128_ccm; + Cipher == aes_256_ccm; + Cipher == aes_128_ccm_8; + Cipher == aes_256_ccm_8; Cipher == chacha20_poly1305 -> ?AEAD. @@ -752,8 +786,16 @@ key_material(aes_256_cbc) -> 32; key_material(aes_128_gcm) -> 16; +key_material(aes_128_ccm) -> + 16; +key_material(aes_128_ccm_8) -> + 16; key_material(aes_256_gcm) -> 32; +key_material(aes_256_ccm_8) -> + 32; +key_material(aes_256_ccm) -> + 32; key_material(chacha20_poly1305) -> 32. @@ -769,6 +811,10 @@ expanded_key_material(Cipher) when Cipher == aes_128_cbc; Cipher == aes_256_cbc; Cipher == aes_128_gcm; Cipher == aes_256_gcm; + Cipher == aes_128_ccm; + Cipher == aes_256_ccm; + Cipher == aes_128_ccm_8; + Cipher == aes_256_ccm_8; Cipher == chacha20_poly1305 -> unknown. @@ -778,12 +824,16 @@ effective_key_bits(des_cbc) -> 56; effective_key_bits(Cipher) when Cipher == rc4_128; Cipher == aes_128_cbc; - Cipher == aes_128_gcm -> + Cipher == aes_128_gcm; + Cipher == aes_128_ccm; + Cipher == aes_128_ccm_8 -> 128; effective_key_bits('3des_ede_cbc') -> 168; effective_key_bits(Cipher) when Cipher == aes_256_cbc; Cipher == aes_256_gcm; + Cipher == aes_256_ccm; + Cipher == aes_256_ccm_8; Cipher == chacha20_poly1305 -> 256. @@ -792,7 +842,11 @@ iv_size(Cipher) when Cipher == null; Cipher == chacha20_poly1305-> 0; iv_size(Cipher) when Cipher == aes_128_gcm; - Cipher == aes_256_gcm -> + Cipher == aes_256_gcm; + Cipher == aes_128_ccm; + Cipher == aes_256_ccm; + Cipher == aes_128_ccm_8; + Cipher == aes_256_ccm_8 -> 4; iv_size(Cipher) -> block_size(Cipher). @@ -804,6 +858,10 @@ block_size(Cipher) when Cipher == aes_128_cbc; Cipher == aes_256_cbc; Cipher == aes_128_gcm; Cipher == aes_256_gcm; + Cipher == aes_128_ccm; + Cipher == aes_256_ccm; + Cipher == aes_128_ccm_8; + Cipher == aes_256_ccm_8; Cipher == chacha20_poly1305 -> 16. diff --git a/lib/ssl/src/ssl_cipher.hrl b/lib/ssl/src/ssl_cipher.hrl index 00822ad9de..5d2f5e2951 100644 --- a/lib/ssl/src/ssl_cipher.hrl +++ b/lib/ssl/src/ssl_cipher.hrl @@ -612,6 +612,58 @@ %% TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256 = {0xcc, 0x15} -define(TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256, <<?BYTE(16#CC), ?BYTE(16#15)>>). + +%% RFC 6655 - TLS-1.2 cipher suites + +%% TLS_RSA_WITH_AES_128_CCM = {0xC0,0x9C} +-define(TLS_RSA_WITH_AES_128_CCM, <<?BYTE(16#C0), ?BYTE(16#9C)>>). + +%% TLS_RSA_WITH_AES_256_CCM = {0xC0,0x9D} +-define(TLS_RSA_WITH_AES_256_CCM, <<?BYTE(16#C0), ?BYTE(16#9D)>>). + +%% TLS_DHE_RSA_WITH_AES_256_CCM = {0xC0,0x9E} +-define(TLS_DHE_RSA_WITH_AES_256_CCM, <<?BYTE(16#C0), ?BYTE(16#9E)>>). + +%% TLS_DHE_RSA_WITH_AES_128_CCM = {0xC0,0x9F} +-define(TLS_DHE_RSA_WITH_AES_128_CCM, <<?BYTE(16#C0), ?BYTE(16#9F)>>). + +%% TLS_RSA_WITH_AES_256_CCM_8 = {0xC0,0x9A0} +-define(TLS_RSA_WITH_AES_256_CCM_8, <<?BYTE(16#C0), ?BYTE(16#A0)>>). + +%% TLS_RSA_WITH_AES_128_CCM_8 = {0xC0,0xA1} +-define(TLS_RSA_WITH_AES_128_CCM_8, <<?BYTE(16#C0), ?BYTE(16#A1)>>). + +%% TLS_DHE_RSA_WITH_AES_128_CCM_8 = {0xC0,0xA2} +-define(TLS_DHE_RSA_WITH_AES_128_CCM_8, <<?BYTE(16#C0), ?BYTE(16#A2)>>). + +%% TLS_DHE_RSA_WITH_AES_256_CCM_8 = {0xC0,0xA3} +-define(TLS_DHE_RSA_WITH_AES_256_CCM_8, <<?BYTE(16#C0), ?BYTE(16#A3)>>). + +%% TLS_PSK_WITH_AES_128_CCM = {0xC0,0xA4} +-define(TLS_PSK_WITH_AES_128_CCM, <<?BYTE(16#C0), ?BYTE(16#A4)>>). + +%% TLS_PSK_WITH_AES_256_CCM = {0xC0,0xA5) +-define(TLS_PSK_WITH_AES_256_CCM, <<?BYTE(16#C0), ?BYTE(16#A5)>>). + +%% TLS_DHE_PSK_WITH_AES_128_CCM = {0xC0,0xA6} +-define(TLS_DHE_PSK_WITH_AES_128_CCM, <<?BYTE(16#C0), ?BYTE(16#A6)>>). + +%% TLS_DHE_PSK_WITH_AES_256_CCM = {0xC0,0xA7} +-define(TLS_DHE_PSK_WITH_AES_256_CCM, <<?BYTE(16#C0), ?BYTE(16#A7)>>). + +%% TLS_PSK_WITH_AES_128_CCM_8 = {0xC0,0xA8} +-define(TLS_PSK_WITH_AES_128_CCM_8, <<?BYTE(16#C0), ?BYTE(16#A8)>>). + +%% TLS_PSK_WITH_AES_256_CCM_8 = {0xC0,0xA9) +-define(TLS_PSK_WITH_AES_256_CCM_8, <<?BYTE(16#C0), ?BYTE(16#A9)>>). + +%% TLS_PSK_DHE_WITH_AES_128_CCM_8 = {0xC0,0xAA} +-define(TLS_PSK_DHE_WITH_AES_128_CCM_8, <<?BYTE(16#C0), ?BYTE(16#AA)>>). + +%% TLS_PSK_DHE_WITH_AES_256_CCM_8 = << ?BYTE(0xC0,0xAB} +-define(TLS_PSK_DHE_WITH_AES_256_CCM_8, <<?BYTE(16#C0),?BYTE(16#AB)>>). + + %%% TLS 1.3 cipher suites RFC8446 %% TLS_AES_128_GCM_SHA256 = {0x13,0x01} diff --git a/lib/ssl/src/ssl_cipher_format.erl b/lib/ssl/src/ssl_cipher_format.erl index b592295d56..8737181922 100644 --- a/lib/ssl/src/ssl_cipher_format.erl +++ b/lib/ssl/src/ssl_cipher_format.erl @@ -467,16 +467,16 @@ suite_definition(?TLS_ECDHE_PSK_WITH_AES_256_GCM_SHA384) -> cipher => aes_256_gcm, mac => null, prf => sha384}; -%% suite_definition(?TLS_ECDHE_PSK_WITH_AES_128_CCM_8_SHA256) -> -%% #{key_exchange => ecdhe_psk, -%% cipher => aes_128_ccm, -%% mac => null, -%% prf =>sha256}; -%% suite_definition(?TLS_ECDHE_PSK_WITH_AES_128_CCM_SHA256) -> -%% #{key_exchange => ecdhe_psk, -%% cipher => aes_256_ccm, -%% mac => null, -%% prf => sha256}; +suite_definition(?TLS_ECDHE_PSK_WITH_AES_128_CCM_SHA256) -> + #{key_exchange => ecdhe_psk, + cipher => aes_128_ccm, + mac => null, + prf =>sha256}; +suite_definition(?TLS_ECDHE_PSK_WITH_AES_128_CCM_8_SHA256) -> + #{key_exchange => ecdhe_psk, + cipher => aes_128_ccm_8, + mac => null, + prf =>sha256}; %%% SRP Cipher Suites RFC 5054 suite_definition(?TLS_SRP_SHA_WITH_3DES_EDE_CBC_SHA) -> #{key_exchange => srp_anon, @@ -792,7 +792,53 @@ suite_definition(?TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384) -> cipher => aes_256_gcm, mac => aead, prf => sha384}; -%% draft-agl-tls-chacha20poly1305-04 Chacha20/Poly1305 Suites +suite_definition(?TLS_PSK_WITH_AES_128_CCM) -> + #{key_exchange => psk, + cipher => aes_128_ccm, + mac => aead, + prf => sha256}; +suite_definition(?TLS_PSK_WITH_AES_256_CCM) -> + #{key_exchange => psk, + cipher => aes_256_ccm, + mac => aead, + prf => sha256}; +suite_definition(?TLS_DHE_PSK_WITH_AES_128_CCM) -> + #{key_exchange => dhe_psk, + cipher => aes_128_ccm, + mac => aead, + prf => sha256}; +suite_definition(?TLS_DHE_PSK_WITH_AES_256_CCM) -> + #{key_exchange => dhe_psk, + cipher => aes_256_ccm, + mac => aead, + prf => sha256}; +suite_definition(?TLS_PSK_WITH_AES_128_CCM_8) -> + #{key_exchange => psk, + cipher => aes_128_ccm_8, + mac => aead, + prf => sha256}; +suite_definition(?TLS_PSK_WITH_AES_256_CCM_8) -> + #{key_exchange => psk, + cipher => aes_256_ccm_8, + mac => aead, + prf => sha256}; +suite_definition(?TLS_PSK_DHE_WITH_AES_128_CCM_8) -> + #{key_exchange => dhe_psk, + cipher => aes_128_ccm_8, + mac => aead, + prf => sha256}; +suite_definition(?TLS_PSK_DHE_WITH_AES_256_CCM_8) -> + #{key_exchange => dhe_psk, + cipher => aes_256_ccm_8, + mac => aead, + prf => sha256}; +suite_definition(#{key_exchange := psk_dhe, + cipher := aes_256_ccm_8, + mac := aead, + prf := sha256}) -> + ?TLS_PSK_DHE_WITH_AES_256_CCM_8; + +% draft-agl-tls-chacha20poly1305-04 Chacha20/Poly1305 Suites suite_definition(?TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256) -> #{key_exchange => ecdhe_rsa, cipher => chacha20_poly1305, @@ -825,16 +871,15 @@ suite_definition(?TLS_CHACHA20_POLY1305_SHA256) -> mac => aead, prf => sha256}. %% suite_definition(?TLS_AES_128_CCM_SHA256) -> -%% #{key_exchange => any, -%% cipher => aes_128_ccm, -%% mac => aead, -%% prf => sha256}; +%% #{key_exchange => any, +%% cipher => aes_128_ccm, +%% mac => aead, +%% prf => sha256}; %% suite_definition(?TLS_AES_128_CCM_8_SHA256) -> -%% #{key_exchange => any, +%% #{key_exchange => any, %% cipher => aes_128_ccm_8, -%% mac => aead, -%% prf => sha256}. - +%% mac => aead, +%% prf => sha256}. %%-------------------------------------------------------------------- -spec erl_suite_definition(cipher_suite() | internal_erl_cipher_suite()) -> old_erl_cipher_suite(). @@ -1154,16 +1199,16 @@ suite(#{key_exchange := ecdhe_psk, mac := null, prf := sha384}) -> ?TLS_ECDHE_PSK_WITH_AES_256_GCM_SHA384; - %% suite(#{key_exchange := ecdhe_psk, - %% cipher := aes_128_ccm, - %% mac := null, - %% prf := sha256}) -> - %% ?TLS_ECDHE_PSK_WITH_AES_128_CCM_8_SHA256; - %% suite(#{key_exchange := ecdhe_psk, - %% cipher := aes_256_ccm, - %% mac := null, - %% prf := sha256}) -> - %% ?TLS_ECDHE_PSK_WITH_AES_128_CCM_SHA256; +suite(#{key_exchange := ecdhe_psk, + cipher := aes_128_ccm_8, + mac := null, + prf := sha256}) -> + ?TLS_ECDHE_PSK_WITH_AES_128_CCM_8_SHA256; +suite(#{key_exchange := ecdhe_psk, + cipher := aes_128_ccm, + mac := null, + prf := sha256}) -> + ?TLS_ECDHE_PSK_WITH_AES_128_CCM_SHA256; %%% SRP Cipher Suites RFC 5054 suite(#{key_exchange := srp_anon, cipher := '3des_ede_cbc', @@ -1460,6 +1505,90 @@ suite(#{key_exchange := dhe_rsa, mac := aead, prf := sha256}) -> ?TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256; + +%% RFC 6655 - TLS-1.2 cipher suites +suite(#{key_exchange := psk, + cipher := aes_128_ccm, + mac := aead, + prf := sha256}) -> + ?TLS_PSK_WITH_AES_128_CCM; +suite(#{key_exchange := psk, + cipher := aes_256_ccm, + mac := aead, + prf := sha256}) -> + ?TLS_PSK_WITH_AES_256_CCM; +suite(#{key_exchange := dhe_psk, + cipher := aes_128_ccm, + mac := aead, + prf := sha256}) -> + ?TLS_DHE_PSK_WITH_AES_128_CCM; +suite(#{key_exchange := dhe_psk, + cipher := aes_256_ccm, + mac := aead, + prf := sha256}) -> + ?TLS_DHE_PSK_WITH_AES_256_CCM; +suite(#{key_exchange := rsa, + cipher := aes_128_ccm, + mac := aead, + prf := sha256}) -> + ?TLS_RSA_WITH_AES_128_CCM; +suite(#{key_exchange := rsa, + cipher := aes_256_ccm, + mac := aead, + prf := sha256}) -> + ?TLS_RSA_WITH_AES_256_CCM; +suite(#{key_exchange := dhe_rsa, + cipher := aes_128_ccm, + mac := aead, + prf := sha256}) -> + ?TLS_DHE_RSA_WITH_AES_128_CCM; +suite(#{key_exchange := dhe_rsa, + cipher := aes_256_ccm, + mac := aead, + prf := sha256}) -> + ?TLS_DHE_RSA_WITH_AES_256_CCM; + +suite(#{key_exchange := psk, + cipher := aes_128_ccm_8, + mac := aead, + prf := sha256}) -> + ?TLS_PSK_WITH_AES_128_CCM_8; +suite(#{key_exchange := psk, + cipher := aes_256_ccm_8, + mac := aead, + prf := sha256}) -> + ?TLS_PSK_WITH_AES_256_CCM_8; +suite(#{key_exchange := dhe_psk, + cipher := aes_128_ccm_8, + mac := aead, + prf := sha256}) -> + ?TLS_PSK_DHE_WITH_AES_128_CCM_8; +suite(#{key_exchange := dhe_psk, + cipher := aes_256_ccm_8, + mac := aead, + prf := sha256}) -> + ?TLS_PSK_DHE_WITH_AES_256_CCM_8; +suite(#{key_exchange := rsa, + cipher := aes_128_ccm_8, + mac := aead, + prf := sha256}) -> + ?TLS_RSA_WITH_AES_128_CCM_8; +suite(#{key_exchange := rsa, + cipher := aes_256_ccm_8, + mac := aead, + prf := sha256}) -> + ?TLS_RSA_WITH_AES_256_CCM_8; +suite(#{key_exchange := dhe_rsa, + cipher := aes_128_ccm_8, + mac := aead, + prf := sha256}) -> + ?TLS_DHE_RSA_WITH_AES_128_CCM_8; +suite(#{key_exchange := dhe_rsa, + cipher := aes_256_ccm_8, + mac := aead, + prf := sha256}) -> + ?TLS_DHE_RSA_WITH_AES_256_CCM_8; + %% TLS 1.3 Cipher Suites RFC8446 suite(#{key_exchange := any, cipher := aes_128_gcm, diff --git a/lib/ssl/src/ssl_internal.hrl b/lib/ssl/src/ssl_internal.hrl index 3d117a655f..4ee0230d88 100644 --- a/lib/ssl/src/ssl_internal.hrl +++ b/lib/ssl/src/ssl_internal.hrl @@ -72,12 +72,13 @@ %% sslv3 is considered insecure due to lack of padding check (Poodle attack) %% Keep as interop with legacy software but do not support as default +%% tlsv1.0 and tlsv1.1 is now also considered legacy %% tlsv1.3 is under development (experimental). -define(ALL_AVAILABLE_VERSIONS, ['tlsv1.3', 'tlsv1.2', 'tlsv1.1', tlsv1, sslv3]). -define(ALL_AVAILABLE_DATAGRAM_VERSIONS, ['dtlsv1.2', dtlsv1]). %% Defines the default versions when not specified by an ssl option. --define(ALL_SUPPORTED_VERSIONS, ['tlsv1.2', 'tlsv1.1', tlsv1]). --define(MIN_SUPPORTED_VERSIONS, ['tlsv1.1', tlsv1]). +-define(ALL_SUPPORTED_VERSIONS, ['tlsv1.2']). +-define(MIN_SUPPORTED_VERSIONS, ['tlsv1.1']). %% Versions allowed in TLSCiphertext.version (TLS 1.2 and prior) and %% TLSCiphertext.legacy_record_version (TLS 1.3). @@ -86,7 +87,7 @@ %% Thus, the allowed range is limited to 0x0300 - 0x0303. -define(ALL_TLS_RECORD_VERSIONS, ['tlsv1.2', 'tlsv1.1', tlsv1, sslv3]). --define(ALL_DATAGRAM_SUPPORTED_VERSIONS, ['dtlsv1.2', dtlsv1]). +-define(ALL_DATAGRAM_SUPPORTED_VERSIONS, ['dtlsv1.2']). -define(MIN_DATAGRAM_SUPPORTED_VERSIONS, [dtlsv1]). %% TLS 1.3 - Section 4.1.3 diff --git a/lib/ssl/src/ssl_record.erl b/lib/ssl/src/ssl_record.erl index 91f1876980..9cc131c3cb 100644 --- a/lib/ssl/src/ssl_record.erl +++ b/lib/ssl/src/ssl_record.erl @@ -471,34 +471,41 @@ initial_security_params(ConnectionEnd) -> -define(end_additional_data(AAD, Len), << (begin(AAD)end)/binary, ?UINT16(begin(Len)end) >>). -do_cipher_aead(?CHACHA20_POLY1305 = Type, Fragment, #cipher_state{key=Key} = CipherState, AAD0) -> +do_cipher_aead(?CHACHA20_POLY1305 = Type, Fragment, #cipher_state{key=Key, tag_len = TagLen} = CipherState, AAD0) -> AAD = ?end_additional_data(AAD0, erlang:iolist_size(Fragment)), Nonce = encrypt_nonce(Type, CipherState), - {Content, CipherTag} = ssl_cipher:aead_encrypt(Type, Key, Nonce, Fragment, AAD), + {Content, CipherTag} = ssl_cipher:aead_encrypt(Type, Key, Nonce, Fragment, AAD, TagLen), {<<Content/binary, CipherTag/binary>>, CipherState}; -do_cipher_aead(Type, Fragment, #cipher_state{key=Key, nonce = ExplicitNonce} = CipherState, AAD0) -> +do_cipher_aead(Type, Fragment, #cipher_state{key=Key, tag_len = TagLen, nonce = ExplicitNonce} = CipherState, AAD0) -> AAD = ?end_additional_data(AAD0, erlang:iolist_size(Fragment)), Nonce = encrypt_nonce(Type, CipherState), - {Content, CipherTag} = ssl_cipher:aead_encrypt(Type, Key, Nonce, Fragment, AAD), + {Content, CipherTag} = ssl_cipher:aead_encrypt(Type, Key, Nonce, Fragment, AAD, TagLen), {<<ExplicitNonce:64/integer, Content/binary, CipherTag/binary>>, CipherState#cipher_state{nonce = ExplicitNonce + 1}}. encrypt_nonce(?CHACHA20_POLY1305, #cipher_state{nonce = Nonce, iv = IV}) -> crypto:exor(<<?UINT32(0), Nonce/binary>>, IV); -encrypt_nonce(?AES_GCM, #cipher_state{iv = IV, nonce = ExplicitNonce}) -> +encrypt_nonce(Type, #cipher_state{iv = IV, nonce = ExplicitNonce}) when Type == ?AES_GCM; + Type == ?AES_CCM; + Type == ?AES_CCM_8 -> <<Salt:4/bytes, _/binary>> = IV, <<Salt/binary, ExplicitNonce:64/integer>>. decrypt_nonce(?CHACHA20_POLY1305, #cipher_state{nonce = Nonce, iv = IV}, _) -> crypto:exor(<<Nonce:96/unsigned-big-integer>>, IV); -decrypt_nonce(?AES_GCM, #cipher_state{iv = <<Salt:4/bytes, _/binary>>}, <<ExplicitNonce:8/bytes, _/binary>>) -> - <<Salt/binary, ExplicitNonce/binary>>. +decrypt_nonce(Type, #cipher_state{iv = <<Salt:4/bytes, _/binary>>}, <<ExplicitNonce:8/bytes, _/binary>>) when + Type == ?AES_GCM; + Type == ?AES_CCM; + Type == ?AES_CCM_8 -> + <<Salt/binary, ExplicitNonce/binary>>. -compile({inline, [aead_ciphertext_split/4]}). aead_ciphertext_split(?CHACHA20_POLY1305, #cipher_state{tag_len = Len}, CipherTextFragment, AAD) -> CipherLen = byte_size(CipherTextFragment) - Len, <<CipherText:CipherLen/bytes, CipherTag:Len/bytes>> = CipherTextFragment, {?end_additional_data(AAD, CipherLen), CipherText, CipherTag}; -aead_ciphertext_split(?AES_GCM, #cipher_state{tag_len = Len}, CipherTextFragment, AAD) -> +aead_ciphertext_split(Type, #cipher_state{tag_len = Len}, CipherTextFragment, AAD) when Type == ?AES_GCM; + Type == ?AES_CCM; + Type == ?AES_CCM_8 -> CipherLen = byte_size(CipherTextFragment) - (Len + 8), %% 8 is length of explicit Nonce << _:8/bytes, CipherText:CipherLen/bytes, CipherTag:Len/bytes>> = CipherTextFragment, {?end_additional_data(AAD, CipherLen), CipherText, CipherTag}. diff --git a/lib/ssl/src/ssl_record.hrl b/lib/ssl/src/ssl_record.hrl index eb718fd20c..6d4d47cedb 100644 --- a/lib/ssl/src/ssl_record.hrl +++ b/lib/ssl/src/ssl_record.hrl @@ -96,6 +96,11 @@ -define(AES_CBC, 7). -define(AES_GCM, 8). -define(CHACHA20_POLY1305, 9). +%% Following two are not defined in any RFC but we want to have the +%% same type of handling internaly, all of these "bulk_cipher_algorithm" +%% enums are only used internaly anyway. +-define(AES_CCM, 10). +-define(AES_CCM_8, 11). %% CipherType -define(STREAM, 0). diff --git a/lib/ssl/src/tls_connection.erl b/lib/ssl/src/tls_connection.erl index fde73cdef1..a05858221a 100644 --- a/lib/ssl/src/tls_connection.erl +++ b/lib/ssl/src/tls_connection.erl @@ -934,7 +934,7 @@ wait_sh(Type, Event, State) -> callback_mode() -> state_functions. -terminate({shutdown, sender_died, Reason}, _StateName, +terminate({shutdown, {sender_died, Reason}}, _StateName, #state{static_env = #static_env{socket = Socket, transport_cb = Transport}} = State) -> @@ -1119,7 +1119,7 @@ handle_info({CloseTag, Socket}, StateName, end; handle_info({'EXIT', Sender, Reason}, _, #state{protocol_specific = #{sender := Sender}} = State) -> - {stop, {shutdown, sender_died, Reason}, State}; + {stop, {shutdown, {sender_died, Reason}}, State}; handle_info(Msg, StateName, State) -> ssl_connection:StateName(info, Msg, State, ?MODULE). diff --git a/lib/ssl/src/tls_handshake_1_3.erl b/lib/ssl/src/tls_handshake_1_3.erl index 1e8b046c1e..0efedf3400 100644 --- a/lib/ssl/src/tls_handshake_1_3.erl +++ b/lib/ssl/src/tls_handshake_1_3.erl @@ -1007,7 +1007,8 @@ update_start_state(#state{connection_states = ConnectionStates0, session = Session#session{session_id = SessionId, ecc = Group, sign_alg = SelectedSignAlg, - dh_public_value = ClientPubKey}, + dh_public_value = ClientPubKey, + cipher_suite = Cipher}, connection_env = CEnv#connection_env{negotiated_version = {3,4}}}. diff --git a/lib/ssl/src/tls_record_1_3.erl b/lib/ssl/src/tls_record_1_3.erl index 97331e1510..74321a1ae2 100644 --- a/lib/ssl/src/tls_record_1_3.erl +++ b/lib/ssl/src/tls_record_1_3.erl @@ -252,7 +252,7 @@ cipher_aead(Fragment, BulkCipherAlgo, Key, Seq, IV, TagLen) -> AAD = additional_data(erlang:iolist_size(Fragment) + TagLen), Nonce = nonce(Seq, IV), {Content, CipherTag} = - ssl_cipher:aead_encrypt(BulkCipherAlgo, Key, Nonce, Fragment, AAD), + ssl_cipher:aead_encrypt(BulkCipherAlgo, Key, Nonce, Fragment, AAD, TagLen), <<Content/binary, CipherTag/binary>>. encode_tls_cipher_text(#tls_cipher_text{opaque_type = Type, diff --git a/lib/ssl/test/Makefile b/lib/ssl/test/Makefile index 57b74115ed..f7fae16088 100644 --- a/lib/ssl/test/Makefile +++ b/lib/ssl/test/Makefile @@ -43,6 +43,7 @@ MODULES = \ ssl_basic_SUITE \ ssl_bench_SUITE \ ssl_cipher_SUITE \ + ssl_cipher_suite_SUITE \ ssl_certificate_verify_SUITE\ ssl_crl_SUITE\ ssl_dist_SUITE \ @@ -63,8 +64,9 @@ MODULES = \ ssl_sni_SUITE \ ssl_eqc_SUITE \ ssl_rfc_5869_SUITE \ - make_certs\ - x509_test + make_certs \ + x509_test \ + inet_crypto_dist ERL_FILES = $(MODULES:%=%.erl) diff --git a/lib/ssl/test/inet_crypto_dist.erl b/lib/ssl/test/inet_crypto_dist.erl new file mode 100644 index 0000000000..5aafaac983 --- /dev/null +++ b/lib/ssl/test/inet_crypto_dist.erl @@ -0,0 +1,1323 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2019. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +%% ------------------------------------------------------------------------- +%% +%% Module for encrypted Erlang protocol - a minimal encrypted +%% distribution protocol based on only a shared secret +%% and the crypto application +%% +-module(inet_crypto_dist). +-define(DIST_NAME, inet_crypto). +-define(DIST_PROTO, crypto). +-define(DRIVER, inet_tcp). +-define(FAMILY, inet). + +-define(PROTOCOL, inet_crypto_dist_v1). +-define(DEFAULT_BLOCK_CRYPTO, aes_128_gcm). +-define(DEFAULT_HASH_ALGORITHM, sha256). +-define(DEFAULT_REKEY_INTERVAL, 32768). + +-export([listen/1, accept/1, accept_connection/5, + setup/5, close/1, select/1, is_node_name/1]). +-export([is_supported/0]). + +%% Generalized dist API, for sibling IPv6 module inet6_crypto_dist +-export([gen_listen/2, gen_accept/2, gen_accept_connection/6, + gen_setup/6, gen_close/2, gen_select/2]). + +-export([nodelay/0]). + +%% Debug +%%%-compile(export_all). +-export([dbg/0, test_server/0, test_client/1]). + +-include_lib("kernel/include/net_address.hrl"). +-include_lib("kernel/include/dist.hrl"). +-include_lib("kernel/include/dist_util.hrl"). + +%% Test if crypto has got enough capabilities for this module to run +%% +is_supported() -> + try {crypto:cipher_info(?DEFAULT_BLOCK_CRYPTO), + crypto:hash_info(?DEFAULT_HASH_ALGORITHM)} + of + {#{block_size := _, iv_length := _, key_length := _}, + #{size := _}} -> + true + catch + error:undef -> + false + end. + +%% ------------------------------------------------------------------------- +%% Erlang distribution plugin structure explained to myself +%% ------- +%% These are the processes involved in the distribution: +%% * net_kernel +%% * The Acceptor +%% * The Controller | Handshaker | Ticker +%% * The DistCtrl process that may be split into: +%% + The Output controller +%% + The Input controller +%% For the regular inet_tcp_dist distribution module, DistCtrl +%% is not one or two processes, but one port - a gen_tcp socket +%% +%% When the VM is started with the argument "-proto_dist inet_crypto" +%% net_kernel registers the module inet_crypto_dist as distribution +%% module. net_kernel calls listen/1 to create a listen socket +%% and then accept/1 with the listen socket as argument to spawn +%% the Acceptor process, which is linked to net_kernel. Apparently +%% the listen socket is owned by net_kernel - I wonder if it could +%% be owned by the Acceptor process instead... +%% +%% The Acceptor process calls blocking accept on the listen socket +%% and when an incoming socket is returned it spawns the DistCtrl +%% process a linked to the Acceptor. The ownership of the accepted +%% socket is transferred to the DistCtrl process. +%% A message is sent to net_kernel to inform it that an incoming +%% connection has appeared and the Acceptor awaits a reply from net_kernel. +%% +%% net_kernel then calls accept_connection/5 to spawn the Controller | +%% Handshaker | Ticker process that is linked to net_kernel. +%% The Controller then awaits a message from the Acceptor process. +%% +%% When net_kernel has spawned the Controller it replies with a message +%% to the Acceptor that then calls DistCtrl to changes its links +%% so DistCtrl ends up linked to the Controller and not to the Acceptor. +%% The Acceptor then sends a message to the Controller. The Controller +%% then changes role into the Handshaker creates a #hs_data{} record +%% and calls dist_util:handshake_other_started/1. After this +%% the Acceptor goes back into a blocking accept on the listen socket. +%% +%% For the regular distribution inet_tcp_dist DistCtrl is a gen_tcp socket +%% and when it is a process it also acts as a socket. The #hs_data{} +%% record used by dist_util presents a set of funs that are used +%% by dist_util to perform the distribution handshake. These funs +%% make sure to transfer the handshake messages through the DistCtrl +%% "socket". +%% +%% When the handshake is finished a fun for this purpose in #hs_data{} +%% is called, which tells DistCtrl that it does not need to be prepared +%% for any more #hs_data{} handshake calls. The DistCtrl process in this +%% module then spawns the Input controller process that gets ownership +%% of the connection's gen_tcp socket and changes into {active, N} mode +%% so now it gets all incoming traffic and delivers that to the VM. +%% The original DistCtrl process changes role into the Output controller +%% process and starts asking the VM for outbound messages and transfers +%% them on the connection socket. +%% +%% The Handshaker now changes into the Ticker role, and uses only two +%% functions in the #hs_data{} record; one to get socket statistics +%% and one to send a tick. None of these may block for any reason +%% in particular not for a congested socket since that would destroy +%% connection supervision. +%% +%% +%% For an connection net_kernel calls setup/5 which spawns the +%% Controller process as linked to net_kernel. This Controller process +%% connects to the other node's listen socket and when that is succesful +%% spawns the DistCtrl process as linked to the controller and transfers +%% socket ownership to it. +%% +%% Then the Controller creates the #hs_data{} record and calls +%% dist_util:handshake_we_started/1 which changes the process role +%% into Handshaker. +%% +%% When the distribution handshake is finished the procedure is just +%% as for an incoming connection above. +%% +%% +%% To sum it up. +%% +%% There is an Acceptor process that is linked to net_kernel and +%% informs it when new connections arrive. +%% +%% net_kernel spawns Controllers for incoming and for outgoing connections. +%% these Controllers use the DistCtrl processes to do distribution +%% handshake and after that becomes Tickers that supervise the connection. +%% +%% The Controller | Handshaker | Ticker is linked to net_kernel, and to +%% DistCtrl, one or both. If any of these connection processes would die +%% all others should be killed by the links. Therefore none of them may +%% terminate with reason 'normal'. +%% ------------------------------------------------------------------------- + +%% ------------------------------------------------------------------------- +%% select/1 is called by net_kernel to ask if this distribution protocol +%% is willing to handle Node +%% + +select(Node) -> + gen_select(Node, ?DRIVER). + +gen_select(Node, Driver) -> + case dist_util:split_node(Node) of + {node, _, Host} -> + case Driver:getaddr(Host) of + {ok, _} -> true; + _ -> false + end; + _ -> + false + end. + +%% ------------------------------------------------------------------------- + +is_node_name(Node) -> + dist_util:is_node_name(Node). + +%% ------------------------------------------------------------------------- +%% Called by net_kernel to create a listen socket for this +%% distribution protocol. This listen socket is used by +%% the Acceptor process. +%% + +listen(Name) -> + gen_listen(Name, ?DRIVER). + +gen_listen(Name, Driver) -> + case inet_tcp_dist:gen_listen(Driver, Name) of + {ok, {Socket, Address, Creation}} -> + inet:setopts(Socket, [binary, {nodelay, true}]), + {ok, + {Socket, Address#net_address{protocol = ?DIST_PROTO}, Creation}}; + Other -> + Other + end. + +%% ------------------------------------------------------------------------- +%% Called by net_kernel to spawn the Acceptor process that awaits +%% new connection in a blocking accept and informs net_kernel +%% when a new connection has appeared, and starts the DistCtrl +%% "socket" process for the connection. +%% + +accept(Listen) -> + gen_accept(Listen, ?DRIVER). + +gen_accept(Listen, Driver) -> + NetKernel = self(), + %% + %% Spawn Acceptor process + %% + Config = config(), + monitor_dist_proc( + spawn_opt( + fun () -> + accept_loop(Listen, Driver, NetKernel, Config) + end, + [link, {priority, max}])). + +accept_loop(Listen, Driver, NetKernel, Config) -> + case Driver:accept(Listen) of + {ok, Socket} -> + wait_for_code_server(), + Timeout = net_kernel:connecttime(), + DistCtrl = start_dist_ctrl(Socket, Config, Timeout), + %% DistCtrl is a "socket" + NetKernel ! + {accept, + self(), DistCtrl, Driver:family(), ?DIST_PROTO}, + receive + {NetKernel, controller, Controller} -> + call_dist_ctrl(DistCtrl, {controller, Controller, self()}), + Controller ! {self(), controller, Socket}; + {NetKernel, unsupported_protocol} -> + exit(unsupported_protocol) + end, + accept_loop(Listen, Driver, NetKernel, Config); + AcceptError -> + exit({accept, AcceptError}) + end. + +wait_for_code_server() -> + %% This is an ugly hack. Starting encryption on a connection + %% requires the crypto module to be loaded. Loading the crypto + %% module triggers its on_load function, which calls + %% code:priv_dir/1 to find the directory where its NIF library is. + %% However, distribution is started earlier than the code server, + %% so the code server is not necessarily started yet, and + %% code:priv_dir/1 might fail because of that, if we receive + %% an incoming connection on the distribution port early enough. + %% + %% If the on_load function of a module fails, the module is + %% unloaded, and the function call that triggered loading it fails + %% with 'undef', which is rather confusing. + %% + %% So let's avoid that by waiting for the code server to start. + %% + case whereis(code_server) of + undefined -> + timer:sleep(10), + wait_for_code_server(); + Pid when is_pid(Pid) -> + ok + end. + +%% ------------------------------------------------------------------------- +%% Called by net_kernel when a new connection has appeared, to spawn +%% a Controller process that performs the handshake with the new node, +%% and then becomes the Ticker connection supervisor. +%% ------------------------------------------------------------------------- + +accept_connection(Acceptor, DistCtrl, MyNode, Allowed, SetupTime) -> + gen_accept_connection( + Acceptor, DistCtrl, MyNode, Allowed, SetupTime, ?DRIVER). + +gen_accept_connection( + Acceptor, DistCtrl, MyNode, Allowed, SetupTime, Driver) -> + NetKernel = self(), + %% + %% Spawn Controller/handshaker/ticker process + %% + monitor_dist_proc( + spawn_opt( + fun() -> + do_accept( + Acceptor, DistCtrl, + MyNode, Allowed, SetupTime, Driver, NetKernel) + end, + [link, {priority, max}])). + +do_accept( + Acceptor, DistCtrl, MyNode, Allowed, SetupTime, Driver, NetKernel) -> + receive + {Acceptor, controller, Socket} -> + Timer = dist_util:start_timer(SetupTime), + HSData = + hs_data_common( + NetKernel, MyNode, DistCtrl, Timer, + Socket, Driver:family()), + HSData_1 = + HSData#hs_data{ + this_node = MyNode, + this_flags = 0, + allowed = Allowed}, + dist_util:handshake_other_started(trace(HSData_1)) + end. + +%% ------------------------------------------------------------------------- +%% Called by net_kernel to spawn a Controller process that sets up +%% a new connection to another Erlang node, performs the handshake +%% with the other it, and then becomes the Ticker process +%% that supervises the connection. +%% ------------------------------------------------------------------------- + +setup(Node, Type, MyNode, LongOrShortNames, SetupTime) -> + gen_setup(Node, Type, MyNode, LongOrShortNames, SetupTime, ?DRIVER). + +gen_setup(Node, Type, MyNode, LongOrShortNames, SetupTime, Driver) -> + NetKernel = self(), + %% + %% Spawn Controller/handshaker/ticker process + %% + monitor_dist_proc( + spawn_opt( + setup_fun( + Node, Type, MyNode, LongOrShortNames, SetupTime, Driver, NetKernel), + [link, {priority, max}])). + +-spec setup_fun(_,_,_,_,_,_,_) -> fun(() -> no_return()). +setup_fun( + Node, Type, MyNode, LongOrShortNames, SetupTime, Driver, NetKernel) -> + fun() -> + do_setup( + Node, Type, MyNode, LongOrShortNames, SetupTime, + Driver, NetKernel) + end. + +-spec do_setup(_,_,_,_,_,_,_) -> no_return(). +do_setup( + Node, Type, MyNode, LongOrShortNames, SetupTime, Driver, NetKernel) -> + {Name, Address} = split_node(Driver, Node, LongOrShortNames), + ErlEpmd = net_kernel:epmd_module(), + {ARMod, ARFun} = get_address_resolver(ErlEpmd, Driver), + Timer = trace(dist_util:start_timer(SetupTime)), + case ARMod:ARFun(Name, Address, Driver:family()) of + {ok, Ip, TcpPort, Version} -> + do_setup_connect( + Node, Type, MyNode, Timer, Driver, NetKernel, + Ip, TcpPort, Version); + {ok, Ip} -> + case ErlEpmd:port_please(Name, Ip) of + {port, TcpPort, Version} -> + do_setup_connect( + Node, Type, MyNode, Timer, Driver, NetKernel, + Ip, TcpPort, Version); + Other -> + ?shutdown2( + Node, + trace( + {port_please_failed, ErlEpmd, Name, Ip, Other})) + end; + Other -> + ?shutdown2( + Node, + trace({getaddr_failed, Driver, Address, Other})) + end. + +-spec do_setup_connect(_,_,_,_,_,_,_,_,_) -> no_return(). + +do_setup_connect( + Node, Type, MyNode, Timer, Driver, NetKernel, + Ip, TcpPort, Version) -> + dist_util:reset_timer(Timer), + ConnectOpts = + trace( + connect_options( + [binary, {active, false}, {packet, 2}, {nodelay, true}])), + case Driver:connect(Ip, TcpPort, ConnectOpts) of + {ok, Socket} -> + Config = config(), + DistCtrl = + start_dist_ctrl(Socket, Config, net_kernel:connecttime()), + %% DistCtrl is a "socket" + HSData = + hs_data_common( + NetKernel, MyNode, DistCtrl, Timer, + Socket, Driver:family()), + HSData_1 = + HSData#hs_data{ + other_node = Node, + this_flags = 0, + other_version = Version, + request_type = Type}, + dist_util:handshake_we_started(trace(HSData_1)); + ConnectError -> + ?shutdown2(Node, + trace({connect_failed, Ip, TcpPort, ConnectError})) + end. + +%% ------------------------------------------------------------------------- +%% close/1 is only called by net_kernel on the socket returned by listen/1. + +close(Socket) -> + gen_close(Socket, ?DRIVER). + +gen_close(Socket, Driver) -> + trace(Driver:close(Socket)). + +%% ------------------------------------------------------------------------- + + +hs_data_common(NetKernel, MyNode, DistCtrl, Timer, Socket, Family) -> + %% Field 'socket' below is set to DistCtrl, which makes + %% the distribution handshake process (ticker) call + %% the funs below with DistCtrl as the S argument. + %% So, S =:= DistCtrl below... + #hs_data{ + kernel_pid = NetKernel, + this_node = MyNode, + socket = DistCtrl, + timer = Timer, + %% + f_send = + fun (S, Packet) when S =:= DistCtrl -> + call_dist_ctrl(S, {send, Packet}) + end, + f_recv = + fun (S, 0, infinity) when S =:= DistCtrl -> + case call_dist_ctrl(S, recv) of + {ok, Bin} when is_binary(Bin) -> + {ok, binary_to_list(Bin)}; + Error -> + Error + end + end, + f_setopts_pre_nodeup = + fun (S) when S =:= DistCtrl -> + ok + end, + f_setopts_post_nodeup = + fun (S) when S =:= DistCtrl -> + ok + end, + f_getll = + fun (S) when S =:= DistCtrl -> + {ok, S} %% DistCtrl is the distribution port + end, + f_address = + fun (S, Node) when S =:= DistCtrl -> + case call_dist_ctrl(S, peername) of + {ok, Address} -> + case dist_util:split_node(Node) of + {node, _, Host} -> + #net_address{ + address = Address, + host = Host, + protocol = ?DIST_PROTO, + family = Family}; + _ -> + {error, no_node} + end + end + end, + f_handshake_complete = + fun (S, _Node, DistHandle) when S =:= DistCtrl -> + call_dist_ctrl(S, {handshake_complete, DistHandle}) + end, + %% + %% mf_tick/1, mf_getstat/1, mf_setopts/2 and mf_getopts/2 + %% are called by the ticker any time after f_handshake_complete/3 + %% so they may not block the caller even for congested socket + mf_tick = + fun (S) when S =:= DistCtrl -> + S ! dist_tick + end, + mf_getstat = + fun (S) when S =:= DistCtrl -> + case + inet:getstat(Socket, [recv_cnt, send_cnt, send_pend]) + of + {ok, Stat} -> + split_stat(Stat, 0, 0, 0); + Error -> + Error + end + end, + mf_setopts = + fun (S, Opts) when S =:= DistCtrl -> + inet:setopts(Socket, setopts_filter(Opts)) + end, + mf_getopts = + fun (S, Opts) when S =:= DistCtrl -> + inet:getopts(Socket, Opts) + end}. + +setopts_filter(Opts) -> + [Opt || + Opt <- Opts, + case Opt of + {K, _} when K =:= active; K =:= deliver; K =:= packet -> false; + K when K =:= list; K =:= binary -> false; + K when K =:= inet; K =:= inet6 -> false; + _ -> true + end]. + +split_stat([{recv_cnt, R}|Stat], _, W, P) -> + split_stat(Stat, R, W, P); +split_stat([{send_cnt, W}|Stat], R, _, P) -> + split_stat(Stat, R, W, P); +split_stat([{send_pend, P}|Stat], R, W, _) -> + split_stat(Stat, R, W, P); +split_stat([], R, W, P) -> + {ok, R, W, P}. + +%% ------------------------------------------------------------ +%% Determine if EPMD module supports address resolving. Default +%% is to use inet_tcp:getaddr/2. +%% ------------------------------------------------------------ +get_address_resolver(EpmdModule, _Driver) -> + case erlang:function_exported(EpmdModule, address_please, 3) of + true -> {EpmdModule, address_please}; + _ -> {erl_epmd, address_please} + end. + + +%% If Node is illegal terminate the connection setup!! +split_node(Driver, Node, LongOrShortNames) -> + case dist_util:split_node(Node) of + {node, Name, Host} -> + check_node(Driver, Node, Name, Host, LongOrShortNames); + {host, _} -> + error_logger:error_msg( + "** Nodename ~p illegal, no '@' character **~n", + [Node]), + ?shutdown2(Node, trace({illegal_node_n@me, Node})); + _ -> + error_logger:error_msg( + "** Nodename ~p illegal **~n", [Node]), + ?shutdown2(Node, trace({illegal_node_name, Node})) + end. + +check_node(Driver, Node, Name, Host, LongOrShortNames) -> + case string:split(Host, ".", all) of + [_] when LongOrShortNames =:= longnames -> + case Driver:parse_address(Host) of + {ok, _} -> + {Name, Host}; + _ -> + error_logger:error_msg( + "** System running to use " + "fully qualified hostnames **~n" + "** Hostname ~s is illegal **~n", + [Host]), + ?shutdown2(Node, trace({not_longnames, Host})) + end; + [_, _|_] when LongOrShortNames =:= shortnames -> + error_logger:error_msg( + "** System NOT running to use " + "fully qualified hostnames **~n" + "** Hostname ~s is illegal **~n", + [Host]), + ?shutdown2(Node, trace({not_shortnames, Host})); + _ -> + {Name, Host} + end. + +%% ------------------------------------------------------------------------- + +connect_options(Opts) -> + case application:get_env(kernel, inet_dist_connect_options) of + {ok, ConnectOpts} -> + Opts ++ setopts_filter(ConnectOpts); + _ -> + Opts + end. + +%% we may not always want the nodelay behaviour +%% for performance reasons +nodelay() -> + case application:get_env(kernel, dist_nodelay) of + undefined -> + {nodelay, true}; + {ok, true} -> + {nodelay, true}; + {ok, false} -> + {nodelay, false}; + _ -> + {nodelay, true} + end. + +config() -> + case init:get_argument(?DIST_NAME) of + error -> + error({missing_argument, ?DIST_NAME}); + {ok, [[String]]} -> + {ok, Tokens, _} = erl_scan:string(String ++ "."), + case erl_parse:parse_term(Tokens) of + {ok, #{secret := Secret} = Config} + when is_binary(Secret); is_list(Secret) -> + Config; + {ok, #{} = Config} -> + error({missing_secret, [{?DIST_NAME,Config}]}); + _ -> + error({bad_argument_value, [{?DIST_NAME,String}]}) + end; + {ok, Value} -> + error({malformed_argument, [{?DIST_NAME,Value}]}) + end. + +%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% +%% The DistCtrl process(es). +%% +%% At net_kernel handshake_complete spawns off the input controller that +%% takes over the socket ownership, and itself becomes the output controller +%% +%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +%%% XXX Missing to "productified": +%%% * Cryptoanalysis by experts +%%% * Proof of usefulness +%%% * Unifying exit reasons using a death_row() function +%%% * Verification (and rejection) of other end's crypto parameters +%%% * OTP:ification (proc_lib?) +%%% * An application to belong to (crypto|kernel?) +%%% * Secret on file (cookie as default?), parameter handling +%%% * Restart and/or code reload policy + +%% Debug client and server + +test_config() -> + #{secret => <<"Secret Cluster Password 123456">>}. + +test_server() -> + {ok, Listen} = gen_tcp:listen(0, [{packet, 2}, {active, false}, binary]), + {ok, Port} = inet:port(Listen), + io:format(?MODULE_STRING":test_client(~w).~n", [Port]), + {ok, Socket} = gen_tcp:accept(Listen), + test(Socket). + +test_client(Port) -> + {ok, Socket} = + gen_tcp:connect( + localhost, Port, [{packet, 2}, {active, false}, binary]), + test(Socket). + +test(Socket) -> + start_dist_ctrl(Socket, test_config(), 10000). + +%% ------------------------------------------------------------------------- + +start_dist_ctrl(Socket, Config, Timeout) -> + Protocol = ?PROTOCOL, + Controller = self(), + Server = + monitor_dist_proc( + spawn_opt( + fun () -> + receive + {?MODULE, From, start} -> + {SendParams, RecvParams} = + init(Socket, Config, Protocol), + reply(From, self()), + handshake(SendParams, 0, RecvParams, 0, Controller) + end + end, + [link, + {priority, max}, + {message_queue_data, off_heap}, + {fullsweep_after, 0}])), + ok = gen_tcp:controlling_process(Socket, Server), + call_dist_ctrl(Server, start, Timeout). + + +call_dist_ctrl(Server, Msg) -> + call_dist_ctrl(Server, Msg, infinity). +%% +call_dist_ctrl(Server, Msg, Timeout) -> + Ref = erlang:monitor(process, Server), + Server ! {?MODULE, {Ref, self()}, Msg}, + receive + {Ref, Res} -> + erlang:demonitor(Ref, [flush]), + Res; + {'DOWN', Ref, process, Server, Reason} -> + exit({?PROTOCOL, Reason}) + after Timeout -> + exit(Server, timeout), + receive + {'DOWN', Ref, process, Server, _} -> + exit({?PROTOCOL, timeout}) + end + end. + +reply({Ref, Pid}, Msg) -> + Pid ! {Ref, Msg}, + ok. + +%% ------------------------------------------------------------------------- + +-record(params, + {protocol, % Encryption protocol tag + socket, + dist_handle, + hash_algorithm, + block_crypto, + rekey_interval, + iv, + key, + tag_len}). + +-define(TCP_ACTIVE, 64). +-define(CHUNK_SIZE, (65536 - 512)). +%% The start chunk starts with zeros, so it seems logical to not have +%% a chunk type with value 0 +-define(HANDSHAKE_CHUNK, 1). +-define(DATA_CHUNK, 2). +-define(TICK_CHUNK, 3). +-define(REKEY_CHUNK, 4). + +%% ------------------------------------------------------------------------- +%% Crypto strategy +%% ------- +%% The crypto strategy is as simple as possible to get an encrypted +%% connection as benchmark reference. It is geared around AEAD +%% ciphers in particular AES-GCM. +%% +%% The init message and the start message must fit in the TCP buffers +%% since both sides start with sending the init message, waits +%% for the other end's init message, sends the start message +%% and waits for the other end's start message. So if the send +%% blocks we have a deadlock. +%% +%% The init message is unencrypted and contains the block cipher and hash +%% algorithms the sender will use, the IV and a key salt. Both sides' +%% key salt is used with the mutual secret as input to the hash algorithm +%% to create different encryption/decryption keys for both directions. +%% +%% The start message is the first encrypted message and contains just +%% encrypted zeros the width of the key, with the header of the init +%% message as AAD data. Successfully decrypting this message +%% verifies that we have an encrypted channel. +%% +%% Subsequent encrypted messages has the sequence number and the length +%% of the message as AAD data. These messages has got a message type +%% differentiating data from ticks. Ticks have a random size in an +%% attempt to make them less obvious to spot. +%% +%% The only reaction to errors is to crash noisily wich will bring +%% down the connection and hopefully produce something useful +%% in the local log, but all the other end sees is a closed connection. +%% ------------------------------------------------------------------------- + +init(Socket, Config, Protocol) -> + Secret = maps:get(secret, Config), + HashAlgorithm = + maps:get(hash_algorithm, Config, ?DEFAULT_HASH_ALGORITHM), + BlockCrypto = + maps:get(block_crypto, Config, ?DEFAULT_BLOCK_CRYPTO), + RekeyInterval = + maps:get(rekey_interval, Config, ?DEFAULT_REKEY_INTERVAL), + %% + SendParams = + init_params( + Socket, Protocol, HashAlgorithm, BlockCrypto, RekeyInterval), + send_init(SendParams, Secret). + +send_init( + #params{ + protocol = Protocol, + socket = Socket, + block_crypto = BlockCrypto, + iv = IVLen, + key = KeyLen, + hash_algorithm = HashAlgorithm} = SendParams, + Secret) -> + %% + ProtocolString = atom_to_binary(Protocol, utf8), + BlockCryptoString = atom_to_binary(BlockCrypto, utf8), + HashAlgorithmString = atom_to_binary(HashAlgorithm, utf8), + SendHeader = + <<ProtocolString/binary, 0, + HashAlgorithmString/binary, 0, + BlockCryptoString/binary, 0>>, + <<IV:IVLen/binary, KeySalt:KeyLen/binary>> = IV_KeySalt = + crypto:strong_rand_bytes(IVLen + KeyLen), + InitPacket = [SendHeader, IV_KeySalt], + ok = gen_tcp:send(Socket, InitPacket), + recv_init(SendParams#params{iv = IV, key = KeySalt}, Secret, SendHeader). + +recv_init( + #params{ + socket = Socket, + hash_algorithm = SendHashAlgorithm, + key = SendKeySalt} = SendParams, Secret, SendHeader) -> + %% + {ok, InitPacket} = gen_tcp:recv(Socket, 0), + [ProtocolString, Rest_1] = binary:split(InitPacket, <<0>>), + Protocol = binary_to_existing_atom(ProtocolString, utf8), + case Protocol of + ?PROTOCOL -> + [HashAlgorithmString, Rest_2] = binary:split(Rest_1, <<0>>), + HashAlgorithm = binary_to_existing_atom(HashAlgorithmString, utf8), + [BlockCryptoString, Rest_3] = binary:split(Rest_2, <<0>>), + BlockCrypto = binary_to_existing_atom(BlockCryptoString, utf8), + #params{ + hash_algorithm = RecvHashAlgorithm, + iv = RecvIVLen, + key = RecvKeyLen} = RecvParams = + init_params( + Socket, Protocol, HashAlgorithm, BlockCrypto, undefined), + <<RecvIV:RecvIVLen/binary, + RecvKeySalt:RecvKeyLen/binary>> = Rest_3, + SendKey = + hash_key(SendHashAlgorithm, SendKeySalt, [RecvKeySalt, Secret]), + RecvKey = + hash_key(RecvHashAlgorithm, RecvKeySalt, [SendKeySalt, Secret]), + SendParams_1 = SendParams#params{key = SendKey}, + RecvParams_1 = RecvParams#params{iv = RecvIV, key = RecvKey}, + RecvHeaderLen = byte_size(InitPacket) - RecvIVLen - RecvKeyLen, + <<RecvHeader:RecvHeaderLen/binary, _/binary>> = InitPacket, + send_start(SendParams_1, SendHeader), + RecvRekeyInterval = recv_start(RecvParams_1, RecvHeader), + {SendParams_1, + RecvParams_1#params{rekey_interval = RecvRekeyInterval}} + end. + +send_start( + #params{ + socket = Socket, + block_crypto = BlockCrypto, + rekey_interval= RekeyInterval, + iv = IV, + key = Key, + tag_len = TagLen}, AAD) -> + %% + KeyLen = byte_size(Key), + Zeros = binary:copy(<<0>>, KeyLen), + {Ciphertext, CipherTag} = + crypto:block_encrypt( + crypto_cipher_name(BlockCrypto), + Key, IV, {AAD, [Zeros, <<RekeyInterval:32>>], TagLen}), + ok = gen_tcp:send(Socket, [Ciphertext, CipherTag]). + +recv_start( + #params{ + socket = Socket, + block_crypto = BlockCrypto, + iv = IV, + key = Key, + tag_len = TagLen}, AAD) -> + {ok, Packet} = gen_tcp:recv(Socket, 0), + KeyLen = byte_size(Key), + PacketLen = KeyLen + 4, + <<Ciphertext:PacketLen/binary, CipherTag:TagLen/binary>> = Packet, + Zeros = binary:copy(<<0>>, KeyLen), + case + crypto:block_decrypt( + crypto_cipher_name(BlockCrypto), + Key, IV, {AAD, Ciphertext, CipherTag}) + of + <<Zeros:KeyLen/binary, RekeyInterval:32>> + when 1 =< RekeyInterval -> + RekeyInterval; + _ -> + error(decrypt_error) + end. + +init_params(Socket, Protocol, HashAlgorithm, BlockCrypto, RekeyInterval) -> + #{block_size := 1, + iv_length := IVLen, + key_length := KeyLen} = crypto:cipher_info(BlockCrypto), + case crypto:hash_info(HashAlgorithm) of + #{size := HashSize} when HashSize >= KeyLen -> + #params{ + socket = Socket, + protocol = Protocol, + hash_algorithm = HashAlgorithm, + block_crypto = BlockCrypto, + rekey_interval = RekeyInterval, + iv = IVLen, + key = KeyLen, + tag_len = 16} + end. + +crypto_cipher_name(BlockCrypto) -> + case BlockCrypto of + aes_128_gcm -> aes_gcm; + aes_192_gcm -> aes_gcm; + aes_256_gcm -> aes_gcm + end. + +hash_key(HashAlgorithm, KeySalt, OtherSalt) -> + KeyLen = byte_size(KeySalt), + <<Key:KeyLen/binary, _/binary>> = + crypto:hash(HashAlgorithm, [KeySalt, OtherSalt]), + Key. + +%% ------------------------------------------------------------------------- +%% net_kernel distribution handshake in progress +%% + +handshake( + SendParams, SendSeq, + #params{socket = Socket} = RecvParams, RecvSeq, Controller) -> + receive + {?MODULE, From, {controller, Controller_1, Parent}} -> + Result = link(Controller_1), + true = unlink(Parent), + reply(From, Result), + handshake(SendParams, SendSeq, RecvParams, RecvSeq, Controller_1); + {?MODULE, From, {handshake_complete, DistHandle}} -> + reply(From, ok), + InputHandler = + monitor_dist_proc( + spawn_opt( + fun () -> + link(Controller), + receive + DistHandle -> + ok = + inet:setopts( + Socket, + [{active, ?TCP_ACTIVE}, + nodelay()]), + input_handler( + RecvParams#params{ + dist_handle = DistHandle}, + RecvSeq, empty_q(), infinity) + end + end, + [link, + {priority, normal}, + {message_queue_data, off_heap}, + {fullsweep_after, 0}])), + _ = monitor(process, InputHandler), % For the benchmark test + ok = gen_tcp:controlling_process(Socket, InputHandler), + ok = erlang:dist_ctrl_input_handler(DistHandle, InputHandler), + InputHandler ! DistHandle, + process_flag(priority, normal), + erlang:dist_ctrl_get_data_notification(DistHandle), + crypto:rand_seed_alg(crypto_cache), + output_handler( + SendParams#params{dist_handle = DistHandle}, SendSeq); + %% + {?MODULE, From, {send, Data}} -> + {SendParams_1, SendSeq_1} = + encrypt_and_send_chunk( + SendParams, SendSeq, [?HANDSHAKE_CHUNK, Data]), + reply(From, ok), + handshake( + SendParams_1, SendSeq_1, RecvParams, RecvSeq, Controller); + {?MODULE, From, recv} -> + {RecvParams_1, RecvSeq_1, Reply} = + recv_and_decrypt_chunk(RecvParams, RecvSeq), + reply(From, Reply), + handshake( + SendParams, SendSeq, RecvParams_1, RecvSeq_1, Controller); + {?MODULE, From, peername} -> + reply(From, inet:peername(Socket)), + handshake(SendParams, SendSeq, RecvParams, RecvSeq, Controller); + %% + _Alien -> + handshake(SendParams, SendSeq, RecvParams, RecvSeq, Controller) + end. + +recv_and_decrypt_chunk(#params{socket = Socket} = RecvParams, RecvSeq) -> + case gen_tcp:recv(Socket, 0) of + {ok, Chunk} -> + case decrypt_chunk(RecvParams, RecvSeq, Chunk) of + <<?HANDSHAKE_CHUNK, Cleartext/binary>> -> + {RecvParams, RecvSeq + 1, {ok, Cleartext}}; + #params{} = RecvParams_1 -> + recv_and_decrypt_chunk(RecvParams_1, 0); + _ -> + error(decrypt_error) + end; + Error -> + {RecvParams, RecvSeq, Error} + end. + +%% ------------------------------------------------------------------------- +%% Output handler process +%% +%% The game here is to flush all dist_data and dist_tick messages, +%% prioritize dist_data over dist_tick, and to not use selective receive + +output_handler(Params, Seq) -> + receive + Msg -> + case Msg of + dist_data -> + output_handler_data(Params, Seq); + dist_tick -> + output_handler_tick(Params, Seq); + _Other -> + %% Ignore + output_handler(Params, Seq) + end + end. + +output_handler_data(Params, Seq) -> + receive + Msg -> + case Msg of + dist_data -> + output_handler_data(Params, Seq); + dist_tick -> + output_handler_data(Params, Seq); + _Other -> + %% Ignore + output_handler_data(Params, Seq) + end + after 0 -> + DistHandle = Params#params.dist_handle, + Q = get_data(DistHandle, empty_q()), + {Params_1, Seq_1} = output_handler_send(Params, Seq, Q, true), + erlang:dist_ctrl_get_data_notification(DistHandle), + output_handler(Params_1, Seq_1) + end. + +output_handler_tick(Params, Seq) -> + receive + Msg -> + case Msg of + dist_data -> + output_handler_data(Params, Seq); + dist_tick -> + output_handler_tick(Params, Seq); + _Other -> + %% Ignore + output_handler_tick(Params, Seq) + end + after 0 -> + TickSize = 8 + rand:uniform(56), + TickData = binary:copy(<<0>>, TickSize), + {Params_1, Seq_1} = + encrypt_and_send_chunk(Params, Seq, [?TICK_CHUNK, TickData]), + output_handler(Params_1, Seq_1) + end. + +output_handler_send( + #params{dist_handle = DistHandle} = Params, Seq, {_, Size, _} = Q, Retry) -> + %% + if + ?CHUNK_SIZE < Size -> + {Cleartext, Q_1} = deq_iovec(?CHUNK_SIZE, Q), + {Params_1, Seq_1} = + encrypt_and_send_chunk(Params, Seq, [?DATA_CHUNK, Cleartext]), + output_handler_send(Params_1, Seq_1, Q_1, Retry); + Retry -> + Q_1 = get_data(DistHandle, Q), + output_handler_send(Params, Seq, Q_1, false); + true -> + {Cleartext, _} = deq_iovec(Size, Q), + encrypt_and_send_chunk(Params, Seq, [?DATA_CHUNK, Cleartext]) + end. + +%% ------------------------------------------------------------------------- +%% Input handler process +%% +%% Here is T 0 or infinity to steer if we should try to receive +%% more data or not; start with infinity, and when we get some +%% data try with 0 to see if more is waiting + +input_handler(#params{socket = Socket} = Params, Seq, Q, T) -> + receive + Msg -> + case Msg of + {tcp_passive, Socket} -> + ok = inet:setopts(Socket, [{active, ?TCP_ACTIVE}]), + Q_1 = + case T of + 0 -> + deliver_data(Params#params.dist_handle, Q); + infinity -> + Q + end, + input_handler(Params, Seq, Q_1, infinity); + {tcp, Socket, Chunk} -> + input_chunk(Params, Seq, Q, Chunk); + {tcp_closed, Socket} -> + error(connection_closed); + _Other -> + %% Ignore... + input_handler(Params, Seq, Q, T) + end + after T -> + Q_1 = deliver_data(Params#params.dist_handle, Q), + input_handler(Params, Seq, Q_1, infinity) + end. + +input_chunk(Params, Seq, Q, Chunk) -> + case decrypt_chunk(Params, Seq, Chunk) of + <<?DATA_CHUNK, Cleartext/binary>> -> + input_handler(Params, Seq + 1, enq_binary(Cleartext, Q), 0); + <<?TICK_CHUNK, _/binary>> -> + input_handler(Params, Seq + 1, Q, 0); + #params{} = Params_1 -> + input_handler(Params_1, 0, Q, 0); + _ -> + error(decrypt_error) + end. + +%% ------------------------------------------------------------------------- +%% erlang:dist_ctrl_* helpers + +%% Get data for sending from the VM and place it in a queue +%% +get_data(DistHandle, {Front, Size, Rear}) -> + get_data(DistHandle, Front, Size, Rear). +%% +get_data(DistHandle, Front, Size, Rear) -> + case erlang:dist_ctrl_get_data(DistHandle) of + none -> + {Front, Size, Rear}; + Bin when is_binary(Bin) -> + Len = byte_size(Bin), + get_data( + DistHandle, Front, Size + 4 + Len, + [Bin, <<Len:32>>|Rear]); + [Bin1, Bin2] -> + Len = byte_size(Bin1) + byte_size(Bin2), + get_data( + DistHandle, Front, Size + 4 + Len, + [Bin2, Bin1, <<Len:32>>|Rear]); + Iovec -> + Len = iolist_size(Iovec), + get_data( + DistHandle, Front, Size + 4 + Len, + lists:reverse(Iovec, [<<Len:32>>|Rear])) + end. + +%% De-packet and deliver received data to the VM from a queue +%% +deliver_data(DistHandle, Q) -> + case Q of + {[], Size, []} -> + Size = 0, % Assert + Q; + {[], Size, Rear} -> + [Bin|Front] = lists:reverse(Rear), + deliver_data(DistHandle, Front, Size, [], Bin); + {[Bin|Front], Size, Rear} -> + deliver_data(DistHandle, Front, Size, Rear, Bin) + end. +%% +deliver_data(DistHandle, Front, Size, Rear, Bin) -> + case Bin of + <<DataSizeA:32, DataA:DataSizeA/binary, + DataSizeB:32, DataB:DataSizeB/binary, Rest/binary>> -> + erlang:dist_ctrl_put_data(DistHandle, DataA), + erlang:dist_ctrl_put_data(DistHandle, DataB), + deliver_data( + DistHandle, + Front, Size - (4 + DataSizeA + 4 + DataSizeB), Rear, + Rest); + <<DataSize:32, Data:DataSize/binary, Rest/binary>> -> + erlang:dist_ctrl_put_data(DistHandle, Data), + deliver_data(DistHandle, Front, Size - (4 + DataSize), Rear, Rest); + <<DataSize:32, FirstData/binary>> -> + TotalSize = 4 + DataSize, + if + TotalSize =< Size -> + BinSize = byte_size(Bin), + {MoreData, Q} = + deq_iovec( + TotalSize - BinSize, + Front, Size - BinSize, Rear), + erlang:dist_ctrl_put_data(DistHandle, [FirstData|MoreData]), + deliver_data(DistHandle, Q); + true -> % Incomplete data + {[Bin|Front], Size, Rear} + end; + <<_/binary>> -> + BinSize = byte_size(Bin), + if + 4 =< Size -> % Fragmented header - extract a header bin + {RestHeader, {Front_1, _Size_1, Rear_1}} = + deq_iovec(4 - BinSize, Front, Size - BinSize, Rear), + Header = iolist_to_binary([Bin|RestHeader]), + deliver_data(DistHandle, Front_1, Size, Rear_1, Header); + true -> % Incomplete header + {[Bin|Front], Size, Rear} + end + end. + +%% ------------------------------------------------------------------------- +%% Encryption and decryption helpers + +encrypt_and_send_chunk( + #params{ + socket = Socket, rekey_interval = Seq, + key = Key, iv = IV, hash_algorithm = HashAlgorithm} = Params, + Seq, Cleartext) -> + %% + KeyLen = byte_size(Key), + IVLen = byte_size(IV), + Chunk = <<IV_1:IVLen/binary, KeySalt:KeyLen/binary>> = + crypto:strong_rand_bytes(IVLen + KeyLen), + ok = gen_tcp:send(Socket, encrypt_chunk(Params, Seq, [?REKEY_CHUNK, Chunk])), + Key_1 = hash_key(HashAlgorithm, Key, KeySalt), + Params_1 = Params#params{key = Key_1, iv = IV_1}, + ok = gen_tcp:send(Socket, encrypt_chunk(Params_1, 0, Cleartext)), + {Params_1, 1}; +encrypt_and_send_chunk(#params{socket = Socket} = Params, Seq, Cleartext) -> + ok = gen_tcp:send(Socket, encrypt_chunk(Params, Seq, Cleartext)), + {Params, Seq + 1}. + +encrypt_chunk( + #params{ + block_crypto = BlockCrypto, + iv = IV, key = Key, tag_len = TagLen}, Seq, Cleartext) -> + %% + ChunkLen = iolist_size(Cleartext) + TagLen, + AAD = <<Seq:32, ChunkLen:32>>, + {Ciphertext, CipherTag} = + crypto:block_encrypt( + crypto_cipher_name(BlockCrypto), Key, IV, {AAD, Cleartext, TagLen}), + Chunk = [Ciphertext,CipherTag], + Chunk. + +decrypt_chunk( + #params{ + block_crypto = BlockCrypto, + iv = IV, key = Key, tag_len = TagLen} = Params, Seq, Chunk) -> + %% + ChunkLen = byte_size(Chunk), + true = TagLen =< ChunkLen, % Assert + AAD = <<Seq:32, ChunkLen:32>>, + CiphertextLen = ChunkLen - TagLen, + <<Ciphertext:CiphertextLen/binary, CipherTag:TagLen/binary>> = Chunk, + block_decrypt( + Params, Seq, crypto_cipher_name(BlockCrypto), + Key, IV, {AAD, Ciphertext, CipherTag}). + +block_decrypt( + #params{rekey_interval = Seq} = Params, Seq, CipherName, Key, IV, Data) -> + %% + KeyLen = byte_size(Key), + IVLen = byte_size(IV), + case crypto:block_decrypt(CipherName, Key, IV, Data) of + <<?REKEY_CHUNK, IV_1:IVLen/binary, KeySalt:KeyLen/binary>> -> + Key_1 = hash_key(Params#params.hash_algorithm, Key, KeySalt), + Params#params{iv = IV_1, key = Key_1}; + _ -> + error(decrypt_error) + end; +block_decrypt(_Params, _Seq, CipherName, Key, IV, Data) -> + crypto:block_decrypt(CipherName, Key, IV, Data). + +%% ------------------------------------------------------------------------- +%% Queue of binaries i.e an iovec queue + +empty_q() -> + {[], 0, []}. + +enq_binary(Bin, {Front, Size, Rear}) -> + {Front, Size + byte_size(Bin), [Bin|Rear]}. + +deq_iovec(GetSize, {Front, Size, Rear}) when GetSize =< Size -> + deq_iovec(GetSize, Front, Size, Rear, []). +%% +deq_iovec(GetSize, Front, Size, Rear) -> + deq_iovec(GetSize, Front, Size, Rear, []). +%% +deq_iovec(GetSize, [], Size, Rear, Acc) -> + deq_iovec(GetSize, lists:reverse(Rear), Size, [], Acc); +deq_iovec(GetSize, [Bin|Front], Size, Rear, Acc) -> + BinSize = byte_size(Bin), + if + BinSize < GetSize -> + deq_iovec( + GetSize - BinSize, Front, Size - BinSize, Rear, [Bin|Acc]); + GetSize < BinSize -> + {Bin1,Bin2} = erlang:split_binary(Bin, GetSize), + {lists:reverse(Acc, [Bin1]), {[Bin2|Front], Size - GetSize, Rear}}; + true -> + {lists:reverse(Acc, [Bin]), {Front, Size - BinSize, Rear}} + end. + +%% ------------------------------------------------------------------------- + +%% Trace point +trace(Term) -> Term. + +%% Keep an eye on this Pid (debug) +monitor_dist_proc(Pid) -> +%%% spawn( +%%% fun () -> +%%% MRef = erlang:monitor(process, Pid), +%%% receive +%%% {'DOWN', MRef, _, _, normal} -> +%%% error_logger:error_report( +%%% [dist_proc_died, +%%% {reason, normal}, +%%% {pid, Pid}]); +%%% {'DOWN', MRef, _, _, Reason} -> +%%% error_logger:info_report( +%%% [dist_proc_died, +%%% {reason, Reason}, +%%% {pid, Pid}]) +%%% end +%%% end), + Pid. + +dbg() -> + dbg:stop(), + dbg:tracer(), + dbg:p(all, c), + dbg:tpl(?MODULE, cx), + dbg:tpl(erlang, dist_ctrl_get_data_notification, cx), + dbg:tpl(erlang, dist_ctrl_get_data, cx), + dbg:tpl(erlang, dist_ctrl_put_data, cx), + ok. diff --git a/lib/ssl/test/ssl.spec b/lib/ssl/test/ssl.spec index 24272327c3..15587abecd 100644 --- a/lib/ssl/test/ssl.spec +++ b/lib/ssl/test/ssl.spec @@ -6,5 +6,7 @@ {skip_groups,dir,ssl_bench_SUITE,payload,"Benchmarks run separately"}. {skip_groups,dir,ssl_bench_SUITE,pem_cache,"Benchmarks run separately"}. {skip_groups,dir,ssl_dist_bench_SUITE,setup,"Benchmarks run separately"}. +{skip_groups,dir,ssl_dist_bench_SUITE,roundtrip,"Benchmarks run separately"}. {skip_groups,dir,ssl_dist_bench_SUITE,throughput,"Benchmarks run separately"}. +{skip_groups,dir,ssl_dist_bench_SUITE,sched_utilization,"Benchmarks run separately"}. diff --git a/lib/ssl/test/ssl_basic_SUITE.erl b/lib/ssl/test/ssl_basic_SUITE.erl index ff5638ff34..03ee97de5d 100644 --- a/lib/ssl/test/ssl_basic_SUITE.erl +++ b/lib/ssl/test/ssl_basic_SUITE.erl @@ -76,11 +76,9 @@ groups() -> {'sslv3', [], all_versions_groups() ++ tls_versions_groups() ++ rizzo_tests() ++ [tls_ciphersuite_vs_version]}, {api,[], api_tests()}, {api_tls,[], api_tests_tls()}, - {tls_ciphers,[], tls_cipher_tests()}, {session, [], session_tests()}, {renegotiate, [], renegotiate_tests()}, {ciphers, [], cipher_tests()}, - {ciphers_ec, [], cipher_tests_ec()}, {error_handling_tests, [], error_handling_tests()}, {error_handling_tests_tls, [], error_handling_tests_tls()} ]. @@ -88,14 +86,12 @@ groups() -> tls_versions_groups ()-> [ {group, api_tls}, - {group, tls_ciphers}, {group, error_handling_tests_tls}]. all_versions_groups ()-> [{group, api}, {group, renegotiate}, {group, ciphers}, - {group, ciphers_ec}, {group, error_handling_tests}]. @@ -211,38 +207,11 @@ renegotiate_tests() -> renegotiate_dos_mitigate_passive, renegotiate_dos_mitigate_absolute]. -tls_cipher_tests() -> - [rc4_rsa_cipher_suites, - rc4_ecdh_rsa_cipher_suites, - rc4_ecdsa_cipher_suites]. - cipher_tests() -> [old_cipher_suites, - cipher_suites_mix, - ciphers_rsa_signed_certs, - ciphers_rsa_signed_certs_openssl_names, - ciphers_dsa_signed_certs, - ciphers_dsa_signed_certs_openssl_names, - chacha_rsa_cipher_suites, - chacha_ecdsa_cipher_suites, - anonymous_cipher_suites, - psk_cipher_suites, - psk_with_hint_cipher_suites, - psk_anon_cipher_suites, - psk_anon_with_hint_cipher_suites, - srp_cipher_suites, - srp_anon_cipher_suites, - srp_dsa_cipher_suites, - des_rsa_cipher_suites, - des_ecdh_rsa_cipher_suites, + cipher_suites_mix, default_reject_anonymous]. -cipher_tests_ec() -> - [ciphers_ecdsa_signed_certs, - ciphers_ecdsa_signed_certs_openssl_names, - ciphers_ecdh_rsa_signed_certs, - ciphers_ecdh_rsa_signed_certs_openssl_names]. - error_handling_tests()-> [close_transport_accept, recv_active, @@ -289,7 +258,8 @@ tls13_test_group() -> tls13_hrr_client_auth_empty_cert_ssl_server_openssl_client, tls13_hrr_client_auth_ssl_server_openssl_client, tls13_unsupported_sign_algo_client_auth_ssl_server_openssl_client, - tls13_unsupported_sign_algo_cert_client_auth_ssl_server_openssl_client]. + tls13_unsupported_sign_algo_cert_client_auth_ssl_server_openssl_client, + tls13_connection_information]. %%-------------------------------------------------------------------- init_per_suite(Config0) -> @@ -409,26 +379,7 @@ init_per_testcase(TestCase, Config) when TestCase == client_renegotiate; ct:timetrap({seconds, ?SEC_RENEGOTIATION_TIMEOUT + 5}), Config; -init_per_testcase(TestCase, Config) when TestCase == psk_cipher_suites; - TestCase == psk_with_hint_cipher_suites; - TestCase == ciphers_rsa_signed_certs; - TestCase == ciphers_rsa_signed_certs_openssl_names; - TestCase == ciphers_ecdh_rsa_signed_certs_openssl_names; - TestCase == ciphers_ecdh_rsa_signed_certs; - TestCase == ciphers_dsa_signed_certs; - TestCase == ciphers_dsa_signed_certs_openssl_names; - TestCase == anonymous_cipher_suites; - TestCase == ciphers_ecdsa_signed_certs; - TestCase == ciphers_ecdsa_signed_certs_openssl_names; - TestCase == anonymous_cipher_suites; - TestCase == psk_anon_cipher_suites; - TestCase == psk_anon_with_hint_cipher_suites; - TestCase == srp_cipher_suites; - TestCase == srp_anon_cipher_suites; - TestCase == srp_dsa_cipher_suites; - TestCase == des_rsa_cipher_suites; - TestCase == des_ecdh_rsa_cipher_suites; - TestCase == versions_option; +init_per_testcase(TestCase, Config) when TestCase == versions_option; TestCase == tls_tcp_connect_big -> ssl_test_lib:ct_log_supported_protocol_versions(Config), ct:timetrap({seconds, 60}), @@ -2708,144 +2659,6 @@ tls_shutdown_error(Config) when is_list(Config) -> ok = ssl:close(Listen), {error, closed} = ssl:shutdown(Listen, read_write). -%%------------------------------------------------------------------- -ciphers_rsa_signed_certs() -> - [{doc,"Test all rsa ssl cipher suites in highest support ssl/tls version"}]. - -ciphers_rsa_signed_certs(Config) when is_list(Config) -> - Ciphers = ssl_test_lib:rsa_suites(crypto), - run_suites(Ciphers, Config, rsa). -%%------------------------------------------------------------------- -ciphers_rsa_signed_certs_openssl_names() -> - [{doc,"Test all rsa ssl cipher suites in highest support ssl/tls version"}]. - -ciphers_rsa_signed_certs_openssl_names(Config) when is_list(Config) -> - Ciphers = ssl_test_lib:openssl_rsa_suites(), - run_suites(Ciphers, Config, rsa). - -%%------------------------------------------------------------------- -ciphers_dsa_signed_certs() -> - [{doc,"Test all dsa ssl cipher suites in highest support ssl/tls version"}]. - -ciphers_dsa_signed_certs(Config) when is_list(Config) -> - NVersion = ssl_test_lib:protocol_version(Config, tuple), - Ciphers = ssl_test_lib:dsa_suites(NVersion), - run_suites(Ciphers, Config, dsa). -%%------------------------------------------------------------------- -ciphers_dsa_signed_certs_openssl_names() -> - [{doc,"Test all dsa ssl cipher suites in highest support ssl/tls version"}]. - -ciphers_dsa_signed_certs_openssl_names(Config) when is_list(Config) -> - Ciphers = ssl_test_lib:openssl_dsa_suites(), - run_suites(Ciphers, Config, dsa). - -%%------------------------------------------------------------------- -chacha_rsa_cipher_suites()-> - [{doc,"Test the cacha with ECDSA signed certs ciphersuites"}]. -chacha_rsa_cipher_suites(Config) when is_list(Config) -> - NVersion = ssl_test_lib:protocol_version(Config, tuple), - Ciphers = [S || {KeyEx,_,_} = S <- ssl_test_lib:chacha_suites(NVersion), - KeyEx == ecdhe_rsa, KeyEx == dhe_rsa], - run_suites(Ciphers, Config, chacha_ecdsa). - -%%------------------------------------------------------------------- -chacha_ecdsa_cipher_suites()-> - [{doc,"Test the cacha with ECDSA signed certs ciphersuites"}]. -chacha_ecdsa_cipher_suites(Config) when is_list(Config) -> - NVersion = ssl_test_lib:protocol_version(Config, tuple), - Ciphers = [S || {ecdhe_ecdsa,_,_} = S <- ssl_test_lib:chacha_suites(NVersion)], - run_suites(Ciphers, Config, chacha_rsa). -%%----------------------------------------------------------------- -anonymous_cipher_suites()-> - [{doc,"Test the anonymous ciphersuites"}]. -anonymous_cipher_suites(Config) when is_list(Config) -> - NVersion = ssl_test_lib:protocol_version(Config, tuple), - Ciphers = ssl_test_lib:ecdh_dh_anonymous_suites(NVersion), - run_suites(Ciphers, Config, anonymous). -%%------------------------------------------------------------------- -psk_cipher_suites() -> - [{doc, "Test the PSK ciphersuites WITHOUT server supplied identity hint"}]. -psk_cipher_suites(Config) when is_list(Config) -> - NVersion = ssl_test_lib:protocol_version(Config, tuple), - Ciphers = ssl_test_lib:psk_suites(NVersion), - run_suites(Ciphers, Config, psk). -%%------------------------------------------------------------------- -psk_with_hint_cipher_suites()-> - [{doc, "Test the PSK ciphersuites WITH server supplied identity hint"}]. -psk_with_hint_cipher_suites(Config) when is_list(Config) -> - NVersion = ssl_test_lib:protocol_version(Config, tuple), - Ciphers = ssl_test_lib:psk_suites(NVersion), - run_suites(Ciphers, Config, psk_with_hint). -%%------------------------------------------------------------------- -psk_anon_cipher_suites() -> - [{doc, "Test the anonymous PSK ciphersuites WITHOUT server supplied identity hint"}]. -psk_anon_cipher_suites(Config) when is_list(Config) -> - NVersion = ssl_test_lib:protocol_version(Config, tuple), - Ciphers = ssl_test_lib:psk_anon_suites(NVersion), - run_suites(Ciphers, Config, psk_anon). -%%------------------------------------------------------------------- -psk_anon_with_hint_cipher_suites()-> - [{doc, "Test the anonymous PSK ciphersuites WITH server supplied identity hint"}]. -psk_anon_with_hint_cipher_suites(Config) when is_list(Config) -> - NVersion = ssl_test_lib:protocol_version(Config, tuple), - Ciphers = ssl_test_lib:psk_anon_suites(NVersion), - run_suites(Ciphers, Config, psk_anon_with_hint). -%%------------------------------------------------------------------- -srp_cipher_suites()-> - [{doc, "Test the SRP ciphersuites"}]. -srp_cipher_suites(Config) when is_list(Config) -> - Ciphers = ssl_test_lib:srp_suites(), - run_suites(Ciphers, Config, srp). -%%------------------------------------------------------------------- -srp_anon_cipher_suites()-> - [{doc, "Test the anonymous SRP ciphersuites"}]. -srp_anon_cipher_suites(Config) when is_list(Config) -> - Ciphers = ssl_test_lib:srp_anon_suites(), - run_suites(Ciphers, Config, srp_anon). -%%------------------------------------------------------------------- -srp_dsa_cipher_suites()-> - [{doc, "Test the SRP DSA ciphersuites"}]. -srp_dsa_cipher_suites(Config) when is_list(Config) -> - Ciphers = ssl_test_lib:srp_dss_suites(), - run_suites(Ciphers, Config, srp_dsa). -%%------------------------------------------------------------------- -rc4_rsa_cipher_suites()-> - [{doc, "Test the RC4 ciphersuites"}]. -rc4_rsa_cipher_suites(Config) when is_list(Config) -> - NVersion = ssl_test_lib:protocol_version(Config, tuple), - Ciphers = [S || {rsa,_,_} = S <- ssl_test_lib:rc4_suites(NVersion)], - run_suites(Ciphers, Config, rc4_rsa). -%------------------------------------------------------------------- -rc4_ecdh_rsa_cipher_suites()-> - [{doc, "Test the RC4 ciphersuites"}]. -rc4_ecdh_rsa_cipher_suites(Config) when is_list(Config) -> - NVersion = ssl_test_lib:protocol_version(Config, tuple), - Ciphers = [S || {ecdh_rsa,_,_} = S <- ssl_test_lib:rc4_suites(NVersion)], - run_suites(Ciphers, Config, rc4_ecdh_rsa). - -%%------------------------------------------------------------------- -rc4_ecdsa_cipher_suites()-> - [{doc, "Test the RC4 ciphersuites"}]. -rc4_ecdsa_cipher_suites(Config) when is_list(Config) -> - NVersion = tls_record:highest_protocol_version([]), - Ciphers = [S || {ecdhe_ecdsa,_,_} = S <- ssl_test_lib:rc4_suites(NVersion)], - run_suites(Ciphers, Config, rc4_ecdsa). - -%%------------------------------------------------------------------- -des_rsa_cipher_suites()-> - [{doc, "Test the des_rsa ciphersuites"}]. -des_rsa_cipher_suites(Config) when is_list(Config) -> - NVersion = tls_record:highest_protocol_version([]), - Ciphers = [S || {rsa,_,_} = S <- ssl_test_lib:des_suites(NVersion)], - run_suites(Ciphers, Config, des_rsa). -%------------------------------------------------------------------- -des_ecdh_rsa_cipher_suites()-> - [{doc, "Test ECDH rsa signed ciphersuites"}]. -des_ecdh_rsa_cipher_suites(Config) when is_list(Config) -> - NVersion = ssl_test_lib:protocol_version(Config, tuple), - Ciphers = [S || {dhe_rsa,_,_} = S <- ssl_test_lib:des_suites(NVersion)], - run_suites(Ciphers, Config, des_dhe_rsa). - %%-------------------------------------------------------------------- default_reject_anonymous()-> [{doc,"Test that by default anonymous cipher suites are rejected "}]. @@ -2872,36 +2685,6 @@ default_reject_anonymous(Config) when is_list(Config) -> ssl_test_lib:check_server_alert(Server, Client, insufficient_security). %%-------------------------------------------------------------------- -ciphers_ecdsa_signed_certs() -> - [{doc, "Test all ecdsa ssl cipher suites in highest support ssl/tls version"}]. - -ciphers_ecdsa_signed_certs(Config) when is_list(Config) -> - NVersion = ssl_test_lib:protocol_version(Config, tuple), - Ciphers = ssl_test_lib:ecdsa_suites(NVersion), - run_suites(Ciphers, Config, ecdsa). -%%-------------------------------------------------------------------- -ciphers_ecdsa_signed_certs_openssl_names() -> - [{doc, "Test all ecdsa ssl cipher suites in highest support ssl/tls version"}]. - -ciphers_ecdsa_signed_certs_openssl_names(Config) when is_list(Config) -> - Ciphers = ssl_test_lib:openssl_ecdsa_suites(), - run_suites(Ciphers, Config, ecdsa). -%%-------------------------------------------------------------------- -ciphers_ecdh_rsa_signed_certs() -> - [{doc, "Test all ecdh_rsa ssl cipher suites in highest support ssl/tls version"}]. - -ciphers_ecdh_rsa_signed_certs(Config) when is_list(Config) -> - NVersion = ssl_test_lib:protocol_version(Config, tuple), - Ciphers = ssl_test_lib:ecdh_rsa_suites(NVersion), - run_suites(Ciphers, Config, ecdh_rsa). -%%-------------------------------------------------------------------- -ciphers_ecdh_rsa_signed_certs_openssl_names() -> - [{doc, "Test all ecdh_rsa ssl cipher suites in highest support ssl/tls version"}]. - -ciphers_ecdh_rsa_signed_certs_openssl_names(Config) when is_list(Config) -> - Ciphers = ssl_test_lib:openssl_ecdh_rsa_suites(), - run_suites(Ciphers, Config, ecdh_rsa). -%%-------------------------------------------------------------------- reuse_session() -> [{doc,"Test reuse of sessions (short handshake)"}]. reuse_session(Config) when is_list(Config) -> @@ -3463,9 +3246,9 @@ defaults(Config) when is_list(Config)-> true = lists:member(sslv3, proplists:get_value(available, Versions)), false = lists:member(sslv3, proplists:get_value(supported, Versions)), true = lists:member('tlsv1', proplists:get_value(available, Versions)), - true = lists:member('tlsv1', proplists:get_value(supported, Versions)), + false = lists:member('tlsv1', proplists:get_value(supported, Versions)), true = lists:member('tlsv1.1', proplists:get_value(available, Versions)), - true = lists:member('tlsv1.1', proplists:get_value(supported, Versions)), + false = lists:member('tlsv1.1', proplists:get_value(supported, Versions)), true = lists:member('tlsv1.2', proplists:get_value(available, Versions)), true = lists:member('tlsv1.2', proplists:get_value(supported, Versions)), false = lists:member({rsa,rc4_128,sha}, ssl:cipher_suites()), @@ -3477,7 +3260,7 @@ defaults(Config) when is_list(Config)-> true = lists:member('dtlsv1.2', proplists:get_value(available_dtls, Versions)), true = lists:member('dtlsv1', proplists:get_value(available_dtls, Versions)), true = lists:member('dtlsv1.2', proplists:get_value(supported_dtls, Versions)), - true = lists:member('dtlsv1', proplists:get_value(supported_dtls, Versions)). + false = lists:member('dtlsv1', proplists:get_value(supported_dtls, Versions)). %%-------------------------------------------------------------------- reuseaddr() -> @@ -5849,6 +5632,29 @@ tls13_unsupported_sign_algo_cert_client_auth_ssl_server_openssl_client(Config) - ssl_test_lib:close_port(Client). +tls13_connection_information() -> + [{doc,"Test the API function ssl:connection_information/1 in a TLS 1.3 connection"}]. + +tls13_connection_information(Config) -> + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_opts, Config), + %% Set versions + ServerOpts = [{versions, ['tlsv1.2','tlsv1.3']}|ServerOpts0], + {_ClientNode, ServerNode, _Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {?MODULE, connection_information_result, []}}, + {options, ServerOpts}]), + Port = ssl_test_lib:inet_port(Server), + + Client = ssl_test_lib:start_basic_client(openssl, 'tlsv1.3', Port, ClientOpts), + + ssl_test_lib:check_result(Server, ok), + ssl_test_lib:close(Server), + ssl_test_lib:close_port(Client). + + %%-------------------------------------------------------------------- %% Internal functions ------------------------------------------------ %%-------------------------------------------------------------------- @@ -6332,147 +6138,6 @@ client_server_opts(#{key_exchange := KeyAlgo}, Config) when KeyAlgo == ecdh_rsa {ssl_test_lib:ssl_options(client_opts, Config), ssl_test_lib:ssl_options(server_ecdh_rsa_opts, Config)}. -run_suites(Ciphers, Config, Type) -> - Version = ssl_test_lib:protocol_version(Config), - ct:log("Running cipher suites ~p~n", [Ciphers]), - {ClientOpts, ServerOpts} = - case Type of - rsa -> - {ssl_test_lib:ssl_options(client_rsa_verify_opts, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_rsa_opts, Config)]}; - dsa -> - {ssl_test_lib:ssl_options(client_dsa_verify_opts, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_dsa_opts, Config)]}; - anonymous -> - %% No certs in opts! - {ssl_test_lib:ssl_options(client_rsa_verify_opts, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options([], Config)]}; - psk -> - {ssl_test_lib:ssl_options(client_psk, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_psk, Config)]}; - psk_with_hint -> - {ssl_test_lib:ssl_options(client_psk, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_psk_hint, Config) - ]}; - psk_anon -> - {ssl_test_lib:ssl_options(client_psk, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_psk_anon, Config)]}; - psk_anon_with_hint -> - {ssl_test_lib:ssl_options(client_psk, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_psk_anon_hint, Config)]}; - srp -> - {ssl_test_lib:ssl_options(client_srp, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_srp, Config)]}; - srp_anon -> - {ssl_test_lib:ssl_options(client_srp, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_srp_anon, Config)]}; - srp_dsa -> - {ssl_test_lib:ssl_options(client_srp_dsa, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_srp_dsa, Config)]}; - ecdsa -> - {ssl_test_lib:ssl_options(client_ecdsa_opts, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_ecdsa_opts, Config)]}; - ecdh_rsa -> - {ssl_test_lib:ssl_options(client_ecdh_rsa_opts, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_ecdh_rsa_opts, Config)]}; - rc4_rsa -> - {ssl_test_lib:ssl_options(client_rsa_verify_opts, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_rsa_verify_opts, Config)]}; - rc4_ecdh_rsa -> - {ssl_test_lib:ssl_options(client_ecdh_rsa_opts, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_ecdh_rsa_opts, Config)]}; - rc4_ecdsa -> - {ssl_test_lib:ssl_options(client_rsa_verify_opts, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_ecdsa_opts, Config)]}; - des_dhe_rsa -> - {ssl_test_lib:ssl_options(client_rsa_verify_opts, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_verification_opts, Config)]}; - des_rsa -> - {ssl_test_lib:ssl_options(client_rsa_verify_opts, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_rsa_verify_opts, Config)]}; - chacha_rsa -> - {ssl_test_lib:ssl_options(client_rsa_verify_opts, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_rsa_verify_opts, Config)]}; - chacha_ecdsa -> - {ssl_test_lib:ssl_options(client_ecdsa_opts, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_ecdsa_opts, Config)]} - end, - Suites = ssl_test_lib:filter_suites(Ciphers, Version), - ct:pal("ssl_test_lib:filter_suites(~p ~p) -> ~p ", [Ciphers, Version, Suites]), - Results0 = lists:map(fun(Cipher) -> - cipher(Cipher, Version, Config, ClientOpts, ServerOpts) end, - ssl_test_lib:filter_suites(Ciphers, Version)), - Results = lists:flatten(Results0), - true = length(Results) == length(Suites), - check_cipher_result(Results). - -check_cipher_result([]) -> - ok; -check_cipher_result([ok | Rest]) -> - check_cipher_result(Rest); -check_cipher_result([_ |_] = Error) -> - ct:fail(Error). - -erlang_cipher_suite(Suite) when is_list(Suite)-> - ssl_cipher_format:suite_definition(ssl_cipher_format:openssl_suite(Suite)); -erlang_cipher_suite(Suite) -> - Suite. - -cipher(CipherSuite, Version, Config, ClientOpts, ServerOpts) -> - %% process_flag(trap_exit, true), - ct:log("Testing CipherSuite ~p~n", [CipherSuite]), - ct:log("Server Opts ~p~n", [ServerOpts]), - ct:log("Client Opts ~p~n", [ClientOpts]), - {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), - - ErlangCipherSuite = erlang_cipher_suite(CipherSuite), - - ConnectionInfo = {ok, {Version, ErlangCipherSuite}}, - - Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, - {from, self()}, - {mfa, {ssl_test_lib, cipher_result, [ConnectionInfo]}}, - {options, ServerOpts}]), - Port = ssl_test_lib:inet_port(Server), - Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, - {host, Hostname}, - {from, self()}, - {mfa, {ssl_test_lib, cipher_result, [ConnectionInfo]}}, - {options, - [{ciphers,[CipherSuite]} | - ClientOpts]}]), - - Result = ssl_test_lib:wait_for_result(Server, ok, Client, ok), - - ssl_test_lib:close(Server), - ssl_test_lib:close(Client), - - case Result of - ok -> - [ok]; - Error -> - [{ErlangCipherSuite, Error}] - end. - connection_information_result(Socket) -> {ok, Info = [_ | _]} = ssl:connection_information(Socket), case length(Info) > 3 of diff --git a/lib/ssl/test/ssl_certificate_verify_SUITE.erl b/lib/ssl/test/ssl_certificate_verify_SUITE.erl index 8690faed54..4f340af4f5 100644 --- a/lib/ssl/test/ssl_certificate_verify_SUITE.erl +++ b/lib/ssl/test/ssl_certificate_verify_SUITE.erl @@ -147,6 +147,7 @@ init_per_testcase(_TestCase, Config) -> ssl:stop(), ssl:start(), ssl_test_lib:ct_log_supported_protocol_versions(Config), + ct:pal(" ~p", [ dtls_record:supported_protocol_versions()]), ct:timetrap({seconds, 10}), Config. diff --git a/lib/ssl/test/ssl_cipher_suite_SUITE.erl b/lib/ssl/test/ssl_cipher_suite_SUITE.erl new file mode 100644 index 0000000000..bf1bc0e752 --- /dev/null +++ b/lib/ssl/test/ssl_cipher_suite_SUITE.erl @@ -0,0 +1,772 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2019-2019. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% + +-module(ssl_cipher_suite_SUITE). + +%% Note: This directive should only be used in test suites. +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +%%-------------------------------------------------------------------- +%% Common Test interface functions ----------------------------------- +%%-------------------------------------------------------------------- +all() -> + [ + {group, 'tlsv1.2'}, + {group, 'tlsv1.1'}, + {group, 'tlsv1'}, + {group, 'sslv3'}, + {group, 'dtlsv1.2'}, + {group, 'dtlsv1'} + ]. + +groups() -> + [ + {'tlsv1.2', [], kex()}, + {'tlsv1.1', [], kex()}, + {'tlsv1', [], kex()}, + {'sslv3', [], kex()}, + {'dtlsv1.2', [], kex()}, + {'dtlsv1', [], kex()}, + {dhe_rsa, [],[dhe_rsa_3des_ede_cbc, + dhe_rsa_aes_128_cbc, + dhe_rsa_aes_256_cbc, + dhe_rsa_chacha20_poly1305 + ]}, + {ecdhe_rsa, [], [ecdhe_rsa_3des_ede_cbc, + ecdhe_rsa_aes_128_cbc, + ecdhe_rsa_aes_128_gcm, + ecdhe_rsa_aes_256_cbc, + ecdhe_rsa_aes_256_gcm, + ecdhe_rsa_chacha20_poly1305 + ]}, + {ecdhe_ecdsa, [],[ecdhe_ecdsa_rc4_128, + ecdhe_ecdsa_3des_ede_cbc, + ecdhe_ecdsa_aes_128_cbc, + ecdhe_ecdsa_aes_128_gcm, + ecdhe_ecdsa_aes_256_cbc, + ecdhe_ecdsa_aes_256_gcm, + ecdhe_ecdsa_chacha20_poly1305 + ]}, + {rsa, [], [rsa_3des_ede_cbc, + rsa_aes_128_cbc, + rsa_aes_256_cbc, + rsa_rc4_128 + ]}, + {dhe_dss, [], [dhe_dss_3des_ede_cbc, + dhe_dss_aes_128_cbc, + dhe_dss_aes_256_cbc]}, + {srp_rsa, [], [srp_rsa_3des_ede_cbc, + srp_rsa_aes_128_cbc, + srp_rsa_aes_256_cbc]}, + {srp_dss, [], [srp_dss_3des_ede_cbc, + srp_dss_aes_128_cbc, + srp_dss_aes_256_cbc]}, + {rsa_psk, [], [rsa_psk_3des_ede_cbc, + rsa_psk_rc4_128, + rsa_psk_aes_128_cbc, + rsa_psk_aes_256_cbc + ]}, + {dh_anon, [], [dh_anon_rc4_128, + dh_anon_3des_ede_cbc, + dh_anon_aes_128_cbc, + dh_anon_aes_128_gcm, + dh_anon_aes_256_cbc, + dh_anon_aes_256_gcm]}, + {ecdh_anon, [], [ecdh_anon_3des_ede_cbc, + ecdh_anon_aes_128_cbc, + ecdh_anon_aes_256_cbc + ]}, + {srp_anon, [], [srp_anon_3des_ede_cbc, + srp_anon_aes_128_cbc, + srp_anon_aes_256_cbc]}, + {psk, [], [psk_3des_ede_cbc, + psk_rc4_128, + psk_aes_128_cbc, + psk_aes_128_ccm, + psk_aes_128_ccm_8, + psk_aes_256_cbc, + psk_aes_256_ccm, + psk_aes_256_ccm_8 + ]}, + {dhe_psk, [], [dhe_psk_3des_ede_cbc, + dhe_psk_rc4_128, + dhe_psk_aes_128_cbc, + dhe_psk_aes_128_ccm, + dhe_psk_aes_128_ccm_8, + dhe_psk_aes_256_cbc, + dhe_psk_aes_256_ccm, + dhe_psk_aes_256_ccm_8 + ]}, + {ecdhe_psk, [], [ecdhe_psk_3des_ede_cbc, + ecdhe_psk_rc4_128, + ecdhe_psk_aes_128_cbc, + ecdhe_psk_aes_128_ccm, + ecdhe_psk_aes_128_ccm_8, + ecdhe_psk_aes_256_cbc + ]} + ]. + + +kex() -> + rsa() ++ ecdsa() ++ dss() ++ anonymous(). + +rsa() -> + [{group, dhe_rsa}, + {group, ecdhe_rsa}, + {group, rsa}, + {group, srp_rsa}, + {group, rsa_psk} + ]. + +ecdsa() -> + [{group, ecdhe_ecdsa}]. + +dss() -> + [{group, dhe_dss}, + {group, srp_dss}]. + +anonymous() -> + [{group, dh_anon}, + {group, ecdh_anon}, + {group, psk}, + {group, dhe_psk}, + {group, ecdhe_psk}, + {group, srp_anon} + ]. + + +init_per_suite(Config) -> + catch crypto:stop(), + try crypto:start() of + ok -> + ssl_test_lib:clean_start(), + Config + catch _:_ -> + {skip, "Crypto did not start"} + end. + +end_per_suite(_Config) -> + ssl:stop(), + application:stop(crypto). + +%%-------------------------------------------------------------------- +init_per_group(GroupName, Config) when GroupName == ecdh_anon; + GroupName == ecdhe_rsa; + GroupName == ecdhe_psk -> + case proplists:get_bool(ecdh, proplists:get_value(public_keys, crypto:supports())) of + true -> + init_certs(GroupName, Config); + false -> + {skip, "Missing EC crypto support"} + end; +init_per_group(ecdhe_ecdsa = GroupName, Config) -> + PKAlg = proplists:get_value(public_keys, crypto:supports()), + case lists:member(ecdh, PKAlg) andalso lists:member(ecdsa, PKAlg) of + true -> + init_certs(GroupName, Config); + false -> + {skip, "Missing EC crypto support"} + end; +init_per_group(dhe_dss = GroupName, Config) -> + PKAlg = proplists:get_value(public_keys, crypto:supports()), + case lists:member(dss, PKAlg) andalso lists:member(dh, PKAlg) of + true -> + init_certs(GroupName, Config); + false -> + {skip, "Missing DSS crypto support"} + end; +init_per_group(srp_dss = GroupName, Config) -> + PKAlg = proplists:get_value(public_keys, crypto:supports()), + case lists:member(dss, PKAlg) andalso lists:member(srp, PKAlg) of + true -> + init_certs(GroupName, Config); + false -> + {skip, "Missing DSS_SRP crypto support"} + end; +init_per_group(GroupName, Config) when GroupName == srp_anon; + GroupName == srp_rsa -> + PKAlg = proplists:get_value(public_keys, crypto:supports()), + case lists:member(srp, PKAlg) of + true -> + init_certs(GroupName, Config); + false -> + {skip, "Missing SRP crypto support"} + end; +init_per_group(dhe_psk = GroupName, Config) -> + PKAlg = proplists:get_value(public_keys, crypto:supports()), + case lists:member(dh, PKAlg) of + true -> + init_certs(GroupName, Config); + false -> + {skip, "Missing SRP crypto support"} + end; +init_per_group(GroupName, Config0) -> + case ssl_test_lib:is_tls_version(GroupName) of + true -> + ssl_test_lib:init_tls_version(GroupName, end_per_group(GroupName, Config0)); + false -> + init_certs(GroupName, Config0) + end. + +end_per_group(GroupName, Config) -> + case ssl_test_lib:is_tls_version(GroupName) of + true -> + ssl_test_lib:clean_tls_version(Config); + false -> + Config + end. +init_per_testcase(TestCase, Config) when TestCase == psk_3des_ede_cbc; + TestCase == srp_anon_3des_ede_cbc; + TestCase == dhe_psk_3des_ede_cbc; + TestCase == ecdhe_psk_3des_ede_cbc; + TestCase == srp_rsa_3des_ede_cbc; + TestCase == srp_dss_3des_ede_cbc; + TestCase == rsa_psk_3des_ede_cbc; + TestCase == rsa_3des_ede_cbc; + TestCase == dhe_rsa_3des_ede_cbc; + TestCase == dhe_dss_3des_ede_cbc; + TestCase == ecdhe_rsa_3des_ede_cbc; + TestCase == srp_anon_dss_3des_ede_cbc; + TestCase == dh_anon_3des_ede_cbc; + TestCase == ecdh_anon_3des_ede_cbc; + TestCase == ecdhe_ecdsa_3des_ede_cbc -> + SupCiphers = proplists:get_value(ciphers, crypto:supports()), + case lists:member(des_ede3, SupCiphers) of + true -> + ct:timetrap({seconds, 5}), + Config; + _ -> + {skip, "Missing 3DES crypto support"} + end; +init_per_testcase(TestCase, Config) when TestCase == psk_rc4_128; + TestCase == ecdhe_psk_rc4_128; + TestCase == dhe_psk_rc4_128; + TestCase == rsa_psk_rc4_128; + TestCase == rsa_rc4_128; + TestCase == ecdhe_rsa_rc4_128; + TestCase == ecdhe_ecdsa_rc4_128; + TestCase == dh_anon_rc4_128 -> + SupCiphers = proplists:get_value(ciphers, crypto:supports()), + case lists:member(rc4, SupCiphers) of + true -> + ct:timetrap({seconds, 5}), + Config; + _ -> + {skip, "Missing RC4 crypto support"} + end; +init_per_testcase(TestCase, Config) when TestCase == psk_aes_128_ccm_8; + TestCase == rsa_psk_aes_128_ccm_8; + TestCase == psk_aes_128_ccm_8; + TestCase == dhe_psk_aes_128_ccm_8; + TestCase == ecdhe_psk_aes_128_ccm_8 -> + SupCiphers = proplists:get_value(ciphers, crypto:supports()), + case lists:member(aes_128_ccm, SupCiphers) of + true -> + ct:timetrap({seconds, 5}), + Config; + _ -> + {skip, "Missing AES_128_CCM crypto support"} + end; +init_per_testcase(TestCase, Config) when TestCase == psk_aes_256_ccm_8; + TestCase == rsa_psk_aes_256_ccm_8; + TestCase == psk_aes_256_ccm_8; + TestCase == dhe_psk_aes_256_ccm_8; + TestCase == ecdhe_psk_aes_256_ccm_8 -> + SupCiphers = proplists:get_value(ciphers, crypto:supports()), + case lists:member(aes_256_ccm, SupCiphers) of + true -> + ct:timetrap({seconds, 5}), + Config; + _ -> + {skip, "Missing AES_256_CCM crypto support"} + end; +init_per_testcase(TestCase, Config) -> + Cipher = test_cipher(TestCase, Config), + %%Reason = io_lib:format("Missing ~p crypto support", [Cipher]), + SupCiphers = proplists:get_value(ciphers, crypto:supports()), + case lists:member(Cipher, SupCiphers) of + true -> + ct:timetrap({seconds, 5}), + Config; + _ -> + {skip, {Cipher, SupCiphers}} + end. + +end_per_testcase(_TestCase, Config) -> + Config. + +init_certs(srp_rsa, Config) -> + DefConf = ssl_test_lib:default_cert_chain_conf(), + CertChainConf = ssl_test_lib:gen_conf(rsa, rsa, DefConf, DefConf), + #{server_config := ServerOpts, + client_config := ClientOpts} + = public_key:pkix_test_data(CertChainConf), + [{tls_config, #{server_config => [{user_lookup_fun, {fun user_lookup/3, undefined}} | ServerOpts], + client_config => [{srp_identity, {"Test-User", "secret"}} | ClientOpts]}} | + proplists:delete(tls_config, Config)]; +init_certs(srp_anon, Config) -> + [{tls_config, #{server_config => [{user_lookup_fun, {fun user_lookup/3, undefined}}], + client_config => [{srp_identity, {"Test-User", "secret"}}]}} | + proplists:delete(tls_config, Config)]; +init_certs(rsa_psk, Config) -> + ClientExt = x509_test:extensions([{key_usage, [digitalSignature, keyEncipherment]}]), + {ClientOpts, ServerOpts} = ssl_test_lib:make_rsa_cert_chains([{server_chain, + [[],[],[{extensions, ClientExt}]]}], + Config, "_peer_keyEncipherment"), + PskSharedSecret = <<1,2,3,4,5,6,7,8,9,10,11,12,13,14,15>>, + [{tls_config, #{server_config => [{user_lookup_fun, {fun user_lookup/3, PskSharedSecret}} | ServerOpts], + client_config => [{psk_identity, "Test-User"}, + {user_lookup_fun, {fun user_lookup/3, PskSharedSecret}} | ClientOpts]}} | + proplists:delete(tls_config, Config)]; +init_certs(rsa, Config) -> + ClientExt = x509_test:extensions([{key_usage, [digitalSignature, keyEncipherment]}]), + {ClientOpts, ServerOpts} = ssl_test_lib:make_rsa_cert_chains([{server_chain, + [[],[],[{extensions, ClientExt}]]}], + Config, "_peer_keyEncipherment"), + [{tls_config, #{server_config => ServerOpts, + client_config => ClientOpts}} | + proplists:delete(tls_config, Config)]; +init_certs(dhe_dss, Config) -> + DefConf = ssl_test_lib:default_cert_chain_conf(), + CertChainConf = ssl_test_lib:gen_conf(dsa, dsa, DefConf, DefConf), + #{server_config := ServerOpts, + client_config := ClientOpts} + = public_key:pkix_test_data(CertChainConf), + [{tls_config, #{server_config => ServerOpts, + client_config => ClientOpts}} | + proplists:delete(tls_config, Config)]; +init_certs(srp_dss, Config) -> + DefConf = ssl_test_lib:default_cert_chain_conf(), + CertChainConf = ssl_test_lib:gen_conf(dsa, dsa, DefConf, DefConf), + #{server_config := ServerOpts, + client_config := ClientOpts} + = public_key:pkix_test_data(CertChainConf), + [{tls_config, #{server_config => [{user_lookup_fun, {fun user_lookup/3, undefined}} | ServerOpts], + client_config => [{srp_identity, {"Test-User", "secret"}} | ClientOpts]}} | + proplists:delete(tls_config, Config)]; +init_certs(GroupName, Config) when GroupName == dhe_rsa; + GroupName == ecdhe_rsa -> + DefConf = ssl_test_lib:default_cert_chain_conf(), + CertChainConf = ssl_test_lib:gen_conf(rsa, rsa, DefConf, DefConf), + #{server_config := ServerOpts, + client_config := ClientOpts} + = public_key:pkix_test_data(CertChainConf), + [{tls_config, #{server_config => ServerOpts, + client_config => ClientOpts}} | + proplists:delete(tls_config, Config)]; +init_certs(GroupName, Config) when GroupName == dhe_ecdsa; + GroupName == ecdhe_ecdsa -> + DefConf = ssl_test_lib:default_cert_chain_conf(), + CertChainConf = ssl_test_lib:gen_conf(ecdsa, ecdsa, DefConf, DefConf), + #{server_config := ServerOpts, + client_config := ClientOpts} + = public_key:pkix_test_data(CertChainConf), + [{tls_config, #{server_config => ServerOpts, + client_config => ClientOpts}} | + proplists:delete(tls_config, Config)]; +init_certs(GroupName, Config) when GroupName == psk; + GroupName == dhe_psk; + GroupName == ecdhe_psk -> + PskSharedSecret = <<1,2,3,4,5,6,7,8,9,10,11,12,13,14,15>>, + [{tls_config, #{server_config => [{user_lookup_fun, {fun user_lookup/3, PskSharedSecret}}], + client_config => [{psk_identity, "Test-User"}, + {user_lookup_fun, {fun user_lookup/3, PskSharedSecret}}]}} | + proplists:delete(tls_config, Config)]; +init_certs(srp, Config) -> + [{tls_config, #{server_config => [{user_lookup_fun, {fun user_lookup/3, undefined}}], + client_config => [{srp_identity, {"Test-User", "secret"}}]}} | + proplists:delete(tls_config, Config)]; +init_certs(_GroupName, Config) -> + %% Anonymous does not need certs + [{tls_config, #{server_config => [], + client_config => []}} | + proplists:delete(tls_config, Config)]. +%%-------------------------------------------------------------------- +%% Test Cases -------------------------------------------------------- +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% SRP -------------------------------------------------------- +%%-------------------------------------------------------------------- +srp_rsa_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(srp_rsa, '3des_ede_cbc', Config). + +srp_rsa_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(srp_rsa, 'aes_128_cbc', Config). + +srp_rsa_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(srp_rsa, 'aes_256_cbc', Config). + +srp_dss_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(srp_dss, '3des_ede_cbc', Config). + +srp_dss_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(srp_dss, 'aes_128_cbc', Config). + +srp_dss_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(srp_dss, 'aes_256_cbc', Config). + +%%-------------------------------------------------------------------- +%% PSK -------------------------------------------------------- +%%-------------------------------------------------------------------- +rsa_psk_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(rsa_psk, '3des_ede_cbc', Config). + +rsa_psk_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(rsa_psk, 'aes_128_cbc', Config). + +rsa_psk_aes_128_ccm(Config) when is_list(Config) -> + run_ciphers_test(rsa_psk, 'aes_128_ccm', Config). + +rsa_psk_aes_128_ccm_8(Config) when is_list(Config) -> + run_ciphers_test(rsa_psk, 'aes_128_ccm_8', Config). + +rsa_psk_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(rsa_psk, 'aes_256_cbc', Config). + +rsa_psk_aes_256_ccm(Config) when is_list(Config) -> + run_ciphers_test(rsa_psk, 'aes_256_ccm', Config). + +rsa_psk_aes_256_ccm_8(Config) when is_list(Config) -> + run_ciphers_test(rsa_psk, 'aes_256_ccm_8', Config). + +rsa_psk_rc4_128(Config) when is_list(Config) -> + run_ciphers_test(rsa_psk, 'rc4_128', Config). + +%%-------------------------------------------------------------------- +%% RSA -------------------------------------------------------- +%%-------------------------------------------------------------------- +rsa_des_cbc(Config) when is_list(Config) -> + run_ciphers_test(rsa, 'des_cbc', Config). + +rsa_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(rsa, '3des_ede_cbc', Config). + +rsa_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(rsa, 'aes_128_cbc', Config). + +rsa_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(rsa, 'aes_256_cbc', Config). + +rsa_aes_128_gcm(Config) when is_list(Config) -> + run_ciphers_test(rsa, 'aes_128_gcm', Config). + +rsa_aes_256_gcm(Config) when is_list(Config) -> + run_ciphers_test(rsa, 'aes_256_gcm', Config). + +rsa_rc4_128(Config) when is_list(Config) -> + run_ciphers_test(rsa, 'rc4_128', Config). +%%-------------------------------------------------------------------- +%% DHE_RSA -------------------------------------------------------- +%%-------------------------------------------------------------------- +dhe_rsa_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_rsa, '3des_ede_cbc', Config). + +dhe_rsa_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_rsa, 'aes_128_cbc', Config). + +dhe_rsa_aes_128_gcm(Config) when is_list(Config) -> + run_ciphers_test(dhe_rsa, 'aes_128_gcm', Config). + +dhe_rsa_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_rsa, 'aes_256_cbc', Config). + +dhe_rsa_aes_256_gcm(Config) when is_list(Config) -> + run_ciphers_test(dhe_rsa, 'aes_256_gcm', Config). + +dhe_rsa_chacha20_poly1305(Config) when is_list(Config) -> + run_ciphers_test(dhe_rsa, 'chacha20_poly1305', Config). +%%-------------------------------------------------------------------- +%% ECDHE_RSA -------------------------------------------------------- +%%-------------------------------------------------------------------- +ecdhe_rsa_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_rsa, '3des_ede_cbc', Config). + +ecdhe_rsa_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_rsa, 'aes_128_cbc', Config). + +ecdhe_rsa_aes_128_gcm(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_rsa, 'aes_128_gcm', Config). + +ecdhe_rsa_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_rsa, 'aes_256_cbc', Config). + +ecdhe_rsa_aes_256_gcm(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_rsa, 'aes_256_gcm', Config). + +ecdhe_rsa_rc4_128(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_rsa, 'rc4_128', Config). + +ecdhe_rsa_chacha20_poly1305(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_rsa, 'chacha20_poly1305', Config). + +%%-------------------------------------------------------------------- +%% ECDHE_ECDSA -------------------------------------------------------- +%%-------------------------------------------------------------------- +ecdhe_ecdsa_rc4_128(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_ecdsa, 'rc4_128', Config). + +ecdhe_ecdsa_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_ecdsa, '3des_ede_cbc', Config). + +ecdhe_ecdsa_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_ecdsa, 'aes_128_cbc', Config). + +ecdhe_ecdsa_aes_128_gcm(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_ecdsa, 'aes_128_gcm', Config). + +ecdhe_ecdsa_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_ecdsa, 'aes_256_cbc', Config). + +ecdhe_ecdsa_aes_256_gcm(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_ecdsa, 'aes_256_gcm', Config). + +ecdhe_ecdsa_chacha20_poly1305(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_ecdsa, 'chacha20_poly1305', Config). +%%-------------------------------------------------------------------- +%% DHE_DSS -------------------------------------------------------- +%%-------------------------------------------------------------------- +dhe_dss_des_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_dss, 'des_cbc', Config). + +dhe_dss_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_dss, '3des_ede_cbc', Config). + +dhe_dss_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_dss, 'aes_128_cbc', Config). + +dhe_dss_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_dss, 'aes_256_cbc', Config). + +dhe_dss_aes_128_gcm(Config) when is_list(Config) -> + run_ciphers_test(dhe_dss, 'aes_128_gcm', Config). + +dhe_dss_aes_256_gcm(Config) when is_list(Config) -> + run_ciphers_test(dhe_dss, 'aes_256_gcm', Config). + +%%-------------------------------------------------------------------- +%% Anonymous -------------------------------------------------------- +%%-------------------------------------------------------------------- +dh_anon_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(dh_anon, '3des_ede_cbc', Config). + +dh_anon_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(dh_anon, 'aes_128_cbc', Config). + +dh_anon_aes_128_gcm(Config) when is_list(Config) -> + run_ciphers_test(dh_anon, 'aes_128_gcm', Config). + +dh_anon_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(dh_anon, 'aes_256_cbc', Config). + +dh_anon_aes_256_gcm(Config) when is_list(Config) -> + run_ciphers_test(dh_anon, 'aes_256_gcm', Config). + +dh_anon_rc4_128(Config) when is_list(Config) -> + run_ciphers_test(dh_anon, 'rc4_128', Config). + +ecdh_anon_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdh_anon, '3des_ede_cbc', Config). + +ecdh_anon_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdh_anon, 'aes_128_cbc', Config). + +ecdh_anon_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdh_anon, 'aes_256_cbc', Config). + +srp_anon_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(srp_anon, '3des_ede_cbc', Config). + +srp_anon_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(srp_anon, 'aes_128_cbc', Config). + +srp_anon_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(srp_anon, 'aes_256_cbc', Config). + +dhe_psk_des_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'des_cbc', Config). + +dhe_psk_rc4_128(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'rc4_128', Config). + +dhe_psk_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, '3des_ede_cbc', Config). + +dhe_psk_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'aes_128_cbc', Config). + +dhe_psk_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'aes_256_cbc', Config). + +dhe_psk_aes_128_gcm(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'aes_128_gcm', Config). + +dhe_psk_aes_256_gcm(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'aes_256_gcm', Config). + +dhe_psk_aes_128_ccm(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'aes_128_ccm', Config). + +dhe_psk_aes_256_ccm(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'aes_256_ccm', Config). + +dhe_psk_aes_128_ccm_8(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'aes_128_ccm_8', Config). + +dhe_psk_aes_256_ccm_8(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'aes_256_ccm_8', Config). + +ecdhe_psk_des_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_psk, 'des_cbc', Config). + +ecdhe_psk_rc4_128(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_psk, 'rc4_128', Config). + +ecdhe_psk_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_psk, '3des_ede_cbc', Config). + +ecdhe_psk_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_psk, 'aes_128_cbc', Config). + +ecdhe_psk_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_psk, 'aes_256_cbc', Config). + +ecdhe_psk_aes_128_gcm(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_psk, 'aes_128_gcm', Config). + +ecdhe_psk_aes_256_gcm(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_psk, 'aes_256_gcm', Config). + +ecdhe_psk_aes_128_ccm(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_psk, 'aes_128_ccm', Config). + +ecdhe_psk_aes_128_ccm_8(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_psk, 'aes_128_ccm_8', Config). + +psk_des_cbc(Config) when is_list(Config) -> + run_ciphers_test(psk, 'des_cbc', Config). + +psk_rc4_128(Config) when is_list(Config) -> + run_ciphers_test(psk, 'rc4_128', Config). + +psk_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(psk, '3des_ede_cbc', Config). + +psk_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(psk, 'aes_128_cbc', Config). + +psk_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(psk, 'aes_256_cbc', Config). + +psk_aes_128_gcm(Config) when is_list(Config) -> + run_ciphers_test(psk, 'aes_128_gcm', Config). + +psk_aes_256_gcm(Config) when is_list(Config) -> + run_ciphers_test(psk, 'aes_256_gcm', Config). + +psk_aes_128_ccm(Config) when is_list(Config) -> + run_ciphers_test(psk, 'aes_128_ccm', Config). + +psk_aes_256_ccm(Config) when is_list(Config) -> + run_ciphers_test(psk, 'aes_256_ccm', Config). + +psk_aes_128_ccm_8(Config) when is_list(Config) -> + run_ciphers_test(psk, 'aes_128_ccm_8', Config). + +psk_aes_256_ccm_8(Config) when is_list(Config) -> + run_ciphers_test(psk, 'aes_256_ccm_8', Config). + +%%-------------------------------------------------------------------- +%% Internal functions ---------------------------------------------- +%%-------------------------------------------------------------------- +test_cipher(TestCase, Config) -> + [{name, Group} |_] = proplists:get_value(tc_group_properties, Config), + list_to_atom(re:replace(atom_to_list(TestCase), atom_to_list(Group) ++ "_", "", [{return, list}])). + +run_ciphers_test(Kex, Cipher, Config) -> + Version = ssl_test_lib:protocol_version(Config), + TestCiphers = test_ciphers(Kex, Cipher, Version), + + case TestCiphers of + [_|_] -> + lists:foreach(fun(TestCipher) -> + cipher_suite_test(TestCipher, Version, Config) + end, TestCiphers); + [] -> + {skip, {not_sup, Kex, Cipher, Version}} + end. + +cipher_suite_test(CipherSuite, Version, Config) -> + #{server_config := SOpts, + client_config := COpts} = proplists:get_value(tls_config, Config), + ServerOpts = ssl_test_lib:ssl_options(SOpts, Config), + ClientOpts = ssl_test_lib:ssl_options(COpts, Config), + ct:log("Testing CipherSuite ~p~n", [CipherSuite]), + ct:log("Server Opts ~p~n", [ServerOpts]), + ct:log("Client Opts ~p~n", [ClientOpts]), + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + + ErlangCipherSuite = erlang_cipher_suite(CipherSuite), + + ConnectionInfo = {ok, {Version, ErlangCipherSuite}}, + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, cipher_result, [ConnectionInfo]}}, + {options, [{versions, [Version]}, {ciphers, [CipherSuite]} | ServerOpts]}]), + Port = ssl_test_lib:inet_port(Server), + Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {mfa, {ssl_test_lib, cipher_result, [ConnectionInfo]}}, + {options, [{versions, [Version]}, {ciphers, [CipherSuite]} | + ClientOpts]}]), + + ssl_test_lib:check_result(Server, ok, Client, ok), + + ssl_test_lib:close(Server), + ssl_test_lib:close(Client). + +erlang_cipher_suite(Suite) when is_list(Suite)-> + ssl_cipher_format:suite_definition(ssl_cipher_format:openssl_suite(Suite)); +erlang_cipher_suite(Suite) -> + Suite. + +user_lookup(psk, _Identity, UserState) -> + {ok, UserState}; +user_lookup(srp, Username, _UserState) -> + Salt = ssl_cipher:random_bytes(16), + UserPassHash = crypto:hash(sha, [Salt, crypto:hash(sha, [Username, <<$:>>, <<"secret">>])]), + {ok, {srp_1024, Salt, UserPassHash}}. + +test_ciphers(Kex, Cipher, Version) -> + ssl:filter_cipher_suites(ssl:cipher_suites(all, Version) ++ ssl:cipher_suites(anonymous, Version), + [{key_exchange, + fun(Kex0) when Kex0 == Kex -> true; + (_) -> false + end}, + {cipher, + fun(Cipher0) when Cipher0 == Cipher -> true; + (_) -> false + end}]). diff --git a/lib/ssl/test/ssl_dist_bench_SUITE.erl b/lib/ssl/test/ssl_dist_bench_SUITE.erl index 7e7de5c9bf..1fea6f6f72 100644 --- a/lib/ssl/test/ssl_dist_bench_SUITE.erl +++ b/lib/ssl/test/ssl_dist_bench_SUITE.erl @@ -49,10 +49,14 @@ suite() -> [{ct_hooks, [{ts_install_cth, [{nodenames, 2}]}]}]. -all() -> [{group, ssl}, {group, plain}]. +all() -> + [{group, ssl}, + {group, crypto}, + {group, plain}]. groups() -> [{ssl, all_groups()}, + {crypto, all_groups()}, {plain, all_groups()}, %% {setup, [{repeat, 1}], [setup]}, @@ -164,6 +168,17 @@ end_per_suite(Config) -> init_per_group(ssl, Config) -> [{ssl_dist, true}, {ssl_dist_prefix, "SSL"}|Config]; +init_per_group(crypto, Config) -> + case inet_crypto_dist:is_supported() of + true -> + [{ssl_dist, false}, {ssl_dist_prefix, "Crypto"}, + {ssl_dist_args, + "-proto_dist inet_crypto " + "-inet_crypto '#{secret => \"123456\"}'"} + |Config]; + false -> + {skip, "Not supported on this OTP version"} + end; init_per_group(plain, Config) -> [{ssl_dist, false}, {ssl_dist_prefix, "Plain"}|Config]; init_per_group(_GroupName, Config) -> @@ -374,29 +389,46 @@ sched_utilization(A, B, Prefix, HA, HB, SSL) -> [A] = ssl_apply(HB, erlang, nodes, []), msacc:print(ClientMsacc), msacc:print(ServerMsacc), - ct:pal("Got ~p msgs",[length(Msgs)]), - report(Prefix++" Sched Utilization Client", - 10000 * msacc:stats(system_runtime,ClientMsacc) / - msacc:stats(system_realtime,ClientMsacc), "util 0.01 %"), - report(Prefix++" Sched Utilization Server", - 10000 * msacc:stats(system_runtime,ServerMsacc) / - msacc:stats(system_realtime,ServerMsacc), "util 0.01 %"), - ok. + ct:pal("Got ~p busy_dist_port msgs",[length(Msgs)]), + ct:log("Stats of B from A: ~p", + [ssl_apply(HA, net_kernel, node_info, [B])]), + ct:log("Stats of A from B: ~p", + [ssl_apply(HB, net_kernel, node_info, [A])]), + SchedUtilClient = + round(10000 * msacc:stats(system_runtime,ClientMsacc) / + msacc:stats(system_realtime,ClientMsacc)), + SchedUtilServer = + round(10000 * msacc:stats(system_runtime,ServerMsacc) / + msacc:stats(system_realtime,ServerMsacc)), + Verdict = + case Msgs of + [] -> + ""; + _ -> + " ???" + end, + {comment, ClientComment} = + report(Prefix ++ " Sched Utilization Client" ++ Verdict, + SchedUtilClient, "/100 %" ++ Verdict), + {comment, ServerComment} = + report(Prefix++" Sched Utilization Server" ++ Verdict, + SchedUtilServer, "/100 %" ++ Verdict), + {comment, "Client " ++ ClientComment ++ ", Server " ++ ServerComment}. %% Runs on node A and spawns a server on node B %% We want to avoid getting busy_dist_port as it hides the true SU usage %% of the receiver and sender. sched_util_runner(A, B, true) -> - sched_util_runner(A, B, 50); + sched_util_runner(A, B, 250); sched_util_runner(A, B, false) -> sched_util_runner(A, B, 250); sched_util_runner(A, B, Senders) -> Payload = payload(5), [A] = rpc:call(B, erlang, nodes, []), - ServerPid = - erlang:spawn( - B, - fun () -> throughput_server() end), + ServerPids = + [erlang:spawn_link( + B, fun () -> throughput_server() end) + || _ <- lists:seq(1, Senders)], ServerMsacc = erlang:spawn( B, @@ -404,24 +436,28 @@ sched_util_runner(A, B, Senders) -> receive {start,Pid} -> msacc:start(10000), - Pid ! {ServerPid,msacc:stats()} + receive + {done,Pid} -> + Pid ! {self(),msacc:stats()} + end end end), - spawn_link( - fun() -> - %% We spawn 250 senders which should mean that we - %% have a load of 250 msgs/msec - [spawn_link( - fun() -> - throughput_client(ServerPid,Payload) - end) || _ <- lists:seq(1, Senders)] - end), - erlang:system_monitor(self(),[busy_dist_port]), + %% We spawn 250 senders which should mean that we + %% have a load of 250 msgs/msec + [spawn_link( + fun() -> + throughput_client(Pid, Payload) + end) || Pid <- ServerPids], + %% + receive after 1000 -> ok end, ServerMsacc ! {start,self()}, msacc:start(10000), ClientMsaccStats = msacc:stats(), - ServerMsaccStats = receive {ServerPid,Stats} -> Stats end, + receive after 1000 -> ok end, + ServerMsacc ! {done,self()}, + ServerMsaccStats = receive {ServerMsacc,Stats} -> Stats end, + %% {ClientMsaccStats,ServerMsaccStats, flush()}. flush() -> @@ -522,15 +558,20 @@ throughput(A, B, Prefix, HA, HB, Packets, Size) -> + byte_size(erlang:term_to_binary([0|<<>>])), % Benchmark overhead Bytes = Packets * (Size + Overhead), io:format("~w bytes, ~.4g s~n", [Bytes,Time/1000000]), + SizeString = integer_to_list(Size), ClientMsaccStats =:= undefined orelse - io:format( - "Sender core usage ratio: ~.4g ns/byte~n", - [msacc:stats(system_runtime, ClientMsaccStats)*1000/Bytes]), + report( + Prefix ++ " Sender_RelativeCoreLoad_" ++ SizeString, + round(msacc:stats(system_runtime, ClientMsaccStats) + * 1000000 / Bytes), + "ps/byte"), ServerMsaccStats =:= undefined orelse begin - io:format( - "Receiver core usage ratio: ~.4g ns/byte~n", - [msacc:stats(system_runtime, ServerMsaccStats)*1000/Bytes]), + report( + Prefix ++ " Receiver_RelativeCoreLoad_" ++ SizeString, + round(msacc:stats(system_runtime, ServerMsaccStats) + * 1000000 / Bytes), + "ps/byte"), msacc:print(ServerMsaccStats) end, io:format("******* ClientProf:~n", []), prof_print(ClientProf), @@ -538,7 +579,7 @@ throughput(A, B, Prefix, HA, HB, Packets, Size) -> io:format("******* Server GC Before:~n~p~n", [Server_GC_Before]), io:format("******* Server GC After:~n~p~n", [Server_GC_After]), Speed = round((Bytes * 1000000) / (1024 * Time)), - report(Prefix++" Throughput_"++integer_to_list(Size), Speed, "kB/s"). + report(Prefix ++ " Throughput_" ++ SizeString, Speed, "kB/s"). %% Runs on node A and spawns a server on node B throughput_runner(A, B, Rounds, Size) -> @@ -546,11 +587,12 @@ throughput_runner(A, B, Rounds, Size) -> [A] = rpc:call(B, erlang, nodes, []), ClientPid = self(), ServerPid = - erlang:spawn( + erlang:spawn_opt( B, - fun () -> throughput_server(ClientPid, Rounds) end), + fun () -> throughput_server(ClientPid, Rounds) end, + [{message_queue_data, off_heap}]), ServerMon = erlang:monitor(process, ServerPid), - msacc:available() andalso + msacc_available() andalso begin msacc:stop(), msacc:reset(), @@ -562,7 +604,7 @@ throughput_runner(A, B, Rounds, Size) -> throughput_client(ServerPid, ServerMon, Payload, Rounds), prof_stop(), MsaccStats = - case msacc:available() of + case msacc_available() of true -> MStats = msacc:stats(), msacc:stop(), @@ -602,7 +644,7 @@ throughput_server(Pid, N) -> GC_Before = get_server_gc_info(), %% dbg:tracer(port, dbg:trace_port(file, "throughput_server_gc.log")), %% dbg:p(TLSDistReceiver, garbage_collection), - msacc:available() andalso + msacc_available() andalso begin msacc:stop(), msacc:reset(), @@ -615,7 +657,7 @@ throughput_server(Pid, N) -> throughput_server_loop(_Pid, GC_Before, 0) -> prof_stop(), MsaccStats = - case msacc:available() of + case msacc_available() of true -> msacc:stop(), MStats = msacc:stats(), @@ -632,8 +674,13 @@ throughput_server_loop(_Pid, GC_Before, 0) -> server_gc_after => get_server_gc_info()}); throughput_server_loop(Pid, GC_Before, N) -> receive - {Pid, N, _} -> - throughput_server_loop(Pid, GC_Before, N-1) + Msg -> + case Msg of + {Pid, N, _} -> + throughput_server_loop(Pid, GC_Before, N - 1); + Other -> + erlang:error({self(),?FUNCTION_NAME,Other}) + end end. get_server_gc_info() -> @@ -773,7 +820,7 @@ get_node_args(Tag, Config) -> true -> proplists:get_value(Tag, Config); false -> - "" + proplists:get_value(ssl_dist_args, Config, "") end. @@ -828,3 +875,6 @@ report(Name, Value, Unit) -> term_to_string(Term) -> unicode:characters_to_list( io_lib:write(Term, [{encoding, unicode}])). + +msacc_available() -> + msacc:available(). diff --git a/lib/ssl/test/ssl_test_lib.erl b/lib/ssl/test/ssl_test_lib.erl index 7f8e81dbd8..22169035f3 100644 --- a/lib/ssl/test/ssl_test_lib.erl +++ b/lib/ssl/test/ssl_test_lib.erl @@ -514,13 +514,6 @@ wait_for_result(Pid, Msg) -> %% Unexpected end. -user_lookup(psk, _Identity, UserState) -> - {ok, UserState}; -user_lookup(srp, Username, _UserState) -> - Salt = ssl_cipher:random_bytes(16), - UserPassHash = crypto:hash(sha, [Salt, crypto:hash(sha, [Username, <<$:>>, <<"secret">>])]), - {ok, {srp_1024, Salt, UserPassHash}}. - cert_options(Config) -> ClientCaCertFile = filename:join([proplists:get_value(priv_dir, Config), "client", "cacerts.pem"]), @@ -1540,10 +1533,13 @@ cipher_result(Socket, Result) -> ct:log("~p:~p~nSuccessfull connect: ~p~n", [?MODULE,?LINE, Result]), %% Importante to send two packets here %% to properly test "cipher state" handling - ssl:send(Socket, "Hello\n"), - "Hello\n" = active_recv(Socket, length( "Hello\n")), - ssl:send(Socket, " world\n"), - " world\n" = active_recv(Socket, length(" world\n")), + Hello = "Hello\n", + World = " world\n", + ssl:send(Socket, Hello), + ct:sleep(500), + ssl:send(Socket, World), + Expected = Hello ++ World, + Expected = active_recv(Socket, length(Expected)), ok. session_info_result(Socket) -> @@ -2378,4 +2374,10 @@ reuse_session(ClientOpts, ServerOpts, Config) -> ssl_test_lib:close(Server1), ssl_test_lib:close(Client3), ssl_test_lib:close(Client4). - + +user_lookup(psk, _Identity, UserState) -> + {ok, UserState}; +user_lookup(srp, Username, _UserState) -> + Salt = ssl_cipher:random_bytes(16), + UserPassHash = crypto:hash(sha, [Salt, crypto:hash(sha, [Username, <<$:>>, <<"secret">>])]), + {ok, {srp_1024, Salt, UserPassHash}}. diff --git a/lib/ssl/test/ssl_to_openssl_SUITE.erl b/lib/ssl/test/ssl_to_openssl_SUITE.erl index df84411b6d..f22eb4ecdf 100644 --- a/lib/ssl/test/ssl_to_openssl_SUITE.erl +++ b/lib/ssl/test/ssl_to_openssl_SUITE.erl @@ -39,16 +39,14 @@ all() -> case ssl_test_lib:openssl_sane_dtls() of true -> - [{group, basic}, - {group, 'tlsv1.2'}, + [{group, 'tlsv1.2'}, {group, 'tlsv1.1'}, {group, 'tlsv1'}, {group, 'sslv3'}, {group, 'dtlsv1.2'}, {group, 'dtlsv1'}]; false -> - [{group, basic}, - {group, 'tlsv1.2'}, + [{group, 'tlsv1.2'}, {group, 'tlsv1.1'}, {group, 'tlsv1'}, {group, 'sslv3'}] @@ -57,8 +55,7 @@ all() -> groups() -> case ssl_test_lib:openssl_sane_dtls() of true -> - [{basic, [], basic_tests()}, - {'tlsv1.2', [], all_versions_tests() ++ alpn_tests() ++ npn_tests() ++ sni_server_tests()}, + [{'tlsv1.2', [], all_versions_tests() ++ alpn_tests() ++ npn_tests() ++ sni_server_tests()}, {'tlsv1.1', [], all_versions_tests() ++ alpn_tests() ++ npn_tests() ++ sni_server_tests()}, {'tlsv1', [], all_versions_tests()++ alpn_tests() ++ npn_tests() ++ sni_server_tests()}, {'sslv3', [], all_versions_tests()}, @@ -66,20 +63,13 @@ groups() -> {'dtlsv1', [], dtls_all_versions_tests()} ]; false -> - [{basic, [], basic_tests()}, - {'tlsv1.2', [], all_versions_tests() ++ alpn_tests() ++ npn_tests() ++ sni_server_tests()}, + [{'tlsv1.2', [], all_versions_tests() ++ alpn_tests() ++ npn_tests() ++ sni_server_tests()}, {'tlsv1.1', [], all_versions_tests() ++ alpn_tests() ++ npn_tests() ++ sni_server_tests()}, {'tlsv1', [], all_versions_tests()++ alpn_tests() ++ npn_tests() ++ sni_server_tests()}, {'sslv3', [], all_versions_tests()} ] end. - -basic_tests() -> - [basic_erlang_client_openssl_server, - basic_erlang_server_openssl_client, - expired_session - ]. - + all_versions_tests() -> [ erlang_client_openssl_server, @@ -357,85 +347,7 @@ end_per_testcase(_, Config) -> %%-------------------------------------------------------------------- %% Test Cases -------------------------------------------------------- %%-------------------------------------------------------------------- -basic_erlang_client_openssl_server() -> - [{doc,"Test erlang client with openssl server"}]. -basic_erlang_client_openssl_server(Config) when is_list(Config) -> - process_flag(trap_exit, true), - ServerOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config), - ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), - - {ClientNode, _, Hostname} = ssl_test_lib:run_where(Config), - - Data = "From openssl to erlang", - - Port = ssl_test_lib:inet_port(node()), - CertFile = proplists:get_value(certfile, ServerOpts), - KeyFile = proplists:get_value(keyfile, ServerOpts), - - Exe = "openssl", - Args = ["s_server", "-accept", integer_to_list(Port), - "-cert", CertFile, "-key", KeyFile], - - OpensslPort = ssl_test_lib:portable_open_port(Exe, Args), - - - ssl_test_lib:wait_for_openssl_server(Port, tls), - - Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, - {host, Hostname}, - {from, self()}, - {mfa, {?MODULE, - erlang_ssl_receive, [Data]}}, - {options, ClientOpts}]), - true = port_command(OpensslPort, Data), - - ssl_test_lib:check_result(Client, ok), - - %% Clean close down! Server needs to be closed first !! - ssl_test_lib:close_port(OpensslPort), - ssl_test_lib:close(Client), - process_flag(trap_exit, false). - -%%-------------------------------------------------------------------- -basic_erlang_server_openssl_client() -> - [{doc,"Test erlang server with openssl client"}]. -basic_erlang_server_openssl_client(Config) when is_list(Config) -> - process_flag(trap_exit, true), - ServerOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config), - - {_, ServerNode, Hostname} = ssl_test_lib:run_where(Config), - - Data = "From openssl to erlang", - Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, - {from, self()}, - {mfa, {?MODULE, erlang_ssl_receive, [Data]}}, - {options,ServerOpts}]), - - Port = ssl_test_lib:inet_port(Server), - - Exe = "openssl", - Args = case no_low_flag("-no_ssl2") of - [] -> - ["s_client", "-connect", hostname_format(Hostname) ++ - ":" ++ integer_to_list(Port), no_low_flag("-no_ssl3") - | workaround_openssl_s_clinent()]; - Flag -> - ["s_client", "-connect", hostname_format(Hostname) ++ - ":" ++ integer_to_list(Port), no_low_flag("-no_ssl3"), Flag - | workaround_openssl_s_clinent()] - end, - - OpenSslPort = ssl_test_lib:portable_open_port(Exe, Args), - true = port_command(OpenSslPort, Data), - - ssl_test_lib:check_result(Server, ok), - - %% Clean close down! Server needs to be closed first !! - ssl_test_lib:close(Server), - ssl_test_lib:close_port(OpenSslPort), - process_flag(trap_exit, false). -%%-------------------------------------------------------------------- erlang_client_openssl_server() -> [{doc,"Test erlang client with openssl server"}]. erlang_client_openssl_server(Config) when is_list(Config) -> diff --git a/lib/ssl/test/ssl_upgrade_SUITE.erl b/lib/ssl/test/ssl_upgrade_SUITE.erl index 875399db76..ead18aeb73 100644 --- a/lib/ssl/test/ssl_upgrade_SUITE.erl +++ b/lib/ssl/test/ssl_upgrade_SUITE.erl @@ -47,10 +47,7 @@ init_per_suite(Config0) -> {skip, Reason} -> {skip, Reason}; Config -> - Result = - {ok, _} = make_certs:all(proplists:get_value(data_dir, Config), - proplists:get_value(priv_dir, Config)), - ssl_test_lib:cert_options(Config) + ssl_test_lib:make_rsa_cert(Config) end catch _:_ -> {skip, "Crypto did not start"} @@ -149,8 +146,8 @@ use_connection(Socket) -> end. soft_start_connection(Config, ResulProxy) -> - ClientOpts = proplists:get_value(client_verification_opts, Config), - ServerOpts = proplists:get_value(server_verification_opts, Config), + ClientOpts = proplists:get_value(client_rsa_verify_opts, Config), + ServerOpts = proplists:get_value(server_rsa_verify_opts, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = start_server([{node, ServerNode}, {port, 0}, {from, ResulProxy}, @@ -166,8 +163,8 @@ soft_start_connection(Config, ResulProxy) -> {Server, Client}. restart_start_connection(Config, ResulProxy) -> - ClientOpts = proplists:get_value(client_verification_opts, Config), - ServerOpts = proplists:get_value(server_verification_opts, Config), + ClientOpts = proplists:get_value(client_rsa_verify_opts, Config), + ServerOpts = proplists:get_value(server_rsa_verify_opts, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = start_server([{node, ServerNode}, {port, 0}, {from, ResulProxy}, diff --git a/lib/stdlib/doc/src/gen_statem.xml b/lib/stdlib/doc/src/gen_statem.xml index d4a4ba268b..6f6849a19d 100644 --- a/lib/stdlib/doc/src/gen_statem.xml +++ b/lib/stdlib/doc/src/gen_statem.xml @@ -757,7 +757,7 @@ handle_event(_, _, State, Data) -> right before entering the initial state even though this actually is not a <em>state change</em>. In this case <c>OldState =:= State</c>, - which can not happen for a subsequent state change, + which cannot happen for a subsequent state change, but will happen when repeating the <em>state enter call</em>. </p> </desc> diff --git a/lib/stdlib/doc/src/notes.xml b/lib/stdlib/doc/src/notes.xml index 23c3f6e981..65650a25c7 100644 --- a/lib/stdlib/doc/src/notes.xml +++ b/lib/stdlib/doc/src/notes.xml @@ -31,6 +31,21 @@ </header> <p>This document describes the changes made to the STDLIB application.</p> +<section><title>STDLIB 3.8.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p>Fixed a performance regression when reading files + opened with the <c>compressed</c> flag.</p> + <p> + Own Id: OTP-15706 Aux Id: ERIERL-336 </p> + </item> + </list> + </section> + +</section> + <section><title>STDLIB 3.8</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/stdlib/src/Makefile b/lib/stdlib/src/Makefile index c95f7637f7..86003c953d 100644 --- a/lib/stdlib/src/Makefile +++ b/lib/stdlib/src/Makefile @@ -155,8 +155,10 @@ APPUP_TARGET= $(EBIN)/$(APPUP_FILE) ifeq ($(NATIVE_LIBS_ENABLED),yes) ERL_COMPILE_FLAGS += +native +else +ERL_COMPILE_FLAGS += -Werror endif -ERL_COMPILE_FLAGS += -I../include -I../../kernel/include -Werror +ERL_COMPILE_FLAGS += -I../include -I../../kernel/include # ---------------------------------------------------- # Targets diff --git a/lib/stdlib/src/array.erl b/lib/stdlib/src/array.erl index 939b1fb488..1504326c61 100644 --- a/lib/stdlib/src/array.erl +++ b/lib/stdlib/src/array.erl @@ -126,11 +126,12 @@ %% per write than base 10, but the speedup is only 21%.) -define(DEFAULT, undefined). --define(LEAFSIZE, 10). % the "base" --define(NODESIZE, ?LEAFSIZE). % (no reason to have a different size) +-define(LEAFSIZE, 10). % the "base" (assumed to be > 1) +-define(NODESIZE, ?LEAFSIZE). % must not be LEAFSIZE-1; keep same as leaf -define(NODEPATTERN(S), {_,_,_,_,_,_,_,_,_,_,S}). % NODESIZE+1 elements! --define(NEW_NODE(S), % beware of argument duplication! - setelement((?NODESIZE+1),erlang:make_tuple((?NODESIZE+1),(S)),(S))). +-define(NEW_NODE(E,S), % general case (currently unused) + setelement((?NODESIZE+1),erlang:make_tuple((?NODESIZE+1),(E)),(S))). +-define(NEW_NODE(S), erlang:make_tuple((?NODESIZE+1),(S))). % when E = S -define(NEW_LEAF(D), erlang:make_tuple(?LEAFSIZE,(D))). -define(NODELEAFS, ?NODESIZE*?LEAFSIZE). @@ -605,7 +606,7 @@ grow(I, E, M) -> grow_1(I, E, M). grow_1(I, E, M) when I >= M -> - grow(I, setelement(1, ?NEW_NODE(M), E), ?extend(M)); + grow_1(I, setelement(1, ?NEW_NODE(M), E), ?extend(M)); grow_1(_I, E, M) -> {E, M}. @@ -1631,12 +1632,11 @@ foldl_test_() -> ?_assert(foldl(Sum, 0, from_list(lists:seq(0,10))) =:= 55), ?_assert(foldl(Reverse, [], from_list(lists:seq(0,1000))) =:= lists:reverse(lists:seq(0,1000))), - ?_assert({999,[N0*100+1+2,N0*2+1+1,0]} =:= - foldl(Vals, {0,[]}, + ?_assertEqual({N0*100+1-2,[N0*100+1+2,N0*2+1+1,0]}, + foldl(Vals, {0,[]}, set(N0*100+1,2, set(N0*2+1,1, set(0,0,new()))))) - ]. -endif. @@ -1786,12 +1786,11 @@ foldr_test_() -> ?_assert(foldr(Sum, 0, from_list(lists:seq(0,10))) =:= 55), ?_assert(foldr(List, [], from_list(lists:seq(0,1000))) =:= lists:seq(0,1000)), - ?_assert({999,[0,N0*2+1+1,N0*100+1+2]} =:= - foldr(Vals, {0,[]}, + ?_assertEqual({N0*100+1-2,[0,N0*2+1+1,N0*100+1+2]}, + foldr(Vals, {0,[]}, set(N0*100+1,2, set(N0*2+1,1, set(0,0,new()))))) - ]. -endif. diff --git a/lib/stdlib/src/erl_tar.erl b/lib/stdlib/src/erl_tar.erl index d8b8f466b1..7064fcacfa 100644 --- a/lib/stdlib/src/erl_tar.erl +++ b/lib/stdlib/src/erl_tar.erl @@ -324,7 +324,7 @@ do_open(Name, Mode) when is_list(Mode) -> open1({binary,Bin}, read, _Raw, Opts) when is_binary(Bin) -> case file:open(Bin, [ram,binary,read]) of {ok,File} -> - _ = [ram_file:uncompress(File) || Opts =:= [compressed]], + _ = [ram_file:uncompress(File) || lists:member(compressed, Opts)], {ok, #reader{handle=File,access=read,func=fun file_op/2}}; Error -> Error @@ -357,7 +357,7 @@ open_mode([read|Rest], false, Raw, Opts) -> open_mode([write|Rest], false, Raw, Opts) -> open_mode(Rest, write, Raw, Opts); open_mode([compressed|Rest], Access, Raw, Opts) -> - open_mode(Rest, Access, Raw, [compressed|Opts]); + open_mode(Rest, Access, Raw, [compressed,read_ahead|Opts]); open_mode([cooked|Rest], Access, _Raw, Opts) -> open_mode(Rest, Access, [], Opts); open_mode([], Access, Raw, Opts) -> diff --git a/lib/stdlib/src/stdlib.appup.src b/lib/stdlib/src/stdlib.appup.src index 37ea97c353..08612ed17f 100644 --- a/lib/stdlib/src/stdlib.appup.src +++ b/lib/stdlib/src/stdlib.appup.src @@ -41,7 +41,9 @@ {<<"^3\\.6\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, {<<"^3\\.7$">>,[restart_new_emulator]}, {<<"^3\\.7\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, - {<<"^3\\.7\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}], + {<<"^3\\.7\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, + {<<"^3\\.8$">>,[restart_new_emulator]}, + {<<"^3\\.8\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}], [{<<"^3\\.4$">>,[restart_new_emulator]}, {<<"^3\\.4\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, {<<"^3\\.4\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, @@ -56,4 +58,6 @@ {<<"^3\\.6\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, {<<"^3\\.7$">>,[restart_new_emulator]}, {<<"^3\\.7\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, - {<<"^3\\.7\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}]}. + {<<"^3\\.7\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, + {<<"^3\\.8$">>,[restart_new_emulator]}, + {<<"^3\\.8\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}]}. diff --git a/lib/stdlib/src/string.erl b/lib/stdlib/src/string.erl index 2939e78d9d..1f8bdc5432 100644 --- a/lib/stdlib/src/string.erl +++ b/lib/stdlib/src/string.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2018. All Rights Reserved. +%% Copyright Ericsson AB 1996-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -1247,18 +1247,20 @@ split_1(Bin, [_C|_]=Needle, Start, Where, Curr0, Acc) -> end end. -lexemes_m([CP|_]=Cs0, {GCs,CPs,_}=Seps, Ts) when is_integer(CP) -> +lexemes_m([CP|_]=Cs0, {GCs,CPs,_}=Seps0, Ts) when is_integer(CP) -> case lists:member(CP, CPs) of true -> [GC|Cs2] = unicode_util:gc(Cs0), case lists:member(GC, GCs) of true -> - lexemes_m(Cs2, Seps, Ts); + lexemes_m(Cs2, Seps0, Ts); false -> + Seps = search_compile(Seps0), {Lexeme,Rest} = lexeme_pick(Cs0, Seps, []), lexemes_m(Rest, Seps, [Lexeme|Ts]) end; false -> + Seps = search_compile(Seps0), {Lexeme,Rest} = lexeme_pick(Cs0, Seps, []), lexemes_m(Rest, Seps, [Lexeme|Ts]) end; diff --git a/lib/stdlib/test/ets_SUITE.erl b/lib/stdlib/test/ets_SUITE.erl index 8561491d50..87ca9bd32c 100644 --- a/lib/stdlib/test/ets_SUITE.erl +++ b/lib/stdlib/test/ets_SUITE.erl @@ -70,7 +70,10 @@ -export([smp_insert/1, smp_fixed_delete/1, smp_unfix_fix/1, smp_select_delete/1, smp_ordered_iteration/1, smp_select_replace/1, otp_8166/1, otp_8732/1, delete_unfix_race/1]). --export([throughput_benchmark/0, test_throughput_benchmark/1]). +-export([throughput_benchmark/0, + throughput_benchmark/1, + test_throughput_benchmark/1, + long_throughput_benchmark/1]). -export([exit_large_table_owner/1, exit_many_large_table_owner/1, exit_many_tables_owner/1, @@ -93,6 +96,7 @@ -include_lib("stdlib/include/ms_transform.hrl"). % ets:fun2ms -include_lib("common_test/include/ct.hrl"). +-include_lib("common_test/include/ct_event.hrl"). -define(m(A,B), assert_eq(A,B)). -define(heap_binary_size, 64). @@ -151,7 +155,8 @@ all() -> take, whereis_table, delete_unfix_race, - test_throughput_benchmark]. + test_throughput_benchmark, + {group, benchmark}]. groups() -> [{new, [], @@ -179,7 +184,9 @@ groups() -> {meta_smp, [], [meta_lookup_unnamed_read, meta_lookup_unnamed_write, meta_lookup_named_read, meta_lookup_named_write, - meta_newdel_unnamed, meta_newdel_named]}]. + meta_newdel_unnamed, meta_newdel_named]}, + {benchmark, [], + [long_throughput_benchmark]}]. init_per_suite(Config) -> erts_debug:set_internal_state(available_internal_state, true), @@ -192,9 +199,61 @@ end_per_suite(_Config) -> catch erts_debug:set_internal_state(available_internal_state, false), ok. +init_per_group(benchmark, Config) -> + P = self(), + %% Spawn owner of ETS table that is alive until end_per_group is run + EtsProcess = + spawn( + fun()-> + Tab = ets:new(ets_benchmark_result_summary_tab, [public]), + P ! {the_table, Tab}, + receive + kill -> ok + end + end), + Tab = receive {the_table, T} -> T end, + CounterNames = [nr_of_benchmarks, + total_throughput, + nr_of_set_benchmarks, + total_throughput_set, + nr_of_ordered_set_benchmarks, + total_throughput_ordered_set], + lists:foreach(fun(CtrName) -> + ets:insert(Tab, {CtrName, 0.0}) + end, + CounterNames), + [{ets_benchmark_result_summary_tab, Tab}, + {ets_benchmark_result_summary_tab_process, EtsProcess} | Config]; init_per_group(_GroupName, Config) -> Config. +end_per_group(benchmark, Config) -> + T = proplists:get_value(ets_benchmark_result_summary_tab, Config), + EtsProcess = proplists:get_value(ets_benchmark_result_summary_tab_process, Config), + Report = + fun(NOfBenchmarksCtr, TotThroughoutCtr, Name) -> + Average = + ets:lookup_element(T, TotThroughoutCtr, 2) / + ets:lookup_element(T, NOfBenchmarksCtr, 2), + io:format("~p ~p~n", [Name, Average]), + ct_event:notify( + #event{name = benchmark_data, + data = [{suite,"ets_bench"}, + {name, Name}, + {value, Average}]}) + end, + Report(nr_of_benchmarks, + total_throughput, + "Average Throughput"), + Report(nr_of_set_benchmarks, + total_throughput_set, + "Average Throughput Set"), + Report(nr_of_ordered_set_benchmarks, + total_throughput_ordered_set, + "Average Throughput Ordered Set"), + ets:delete(T), + EtsProcess ! kill, + Config; end_per_group(_GroupName, Config) -> Config. @@ -6530,8 +6589,8 @@ whereis_table(Config) when is_list(Config) -> ok. -%% The following work functions are used by -%% throughput_benchmark/4. They are declared on the top level beacuse +%% The following help functions are used by +%% throughput_benchmark. They are declared on the top level beacuse %% declaring them as function local funs cause a scalability issue. get_op([{_,O}], _RandNum) -> O; @@ -6566,10 +6625,131 @@ prefill_table_loop(T, RS0, N, ObjFun) -> ets:insert(T, ObjFun(Key)), prefill_table_loop(T, RS1, N-1, ObjFun). -throughput_benchmark() -> - throughput_benchmark(false, not_set, not_set). +-record(ets_throughput_bench_config, + {benchmark_duration_ms = 3000, + recover_time_ms = 1000, + thread_counts = not_set, + key_ranges = [1000000], + scenarios = + [ + [ + {0.5, insert}, + {0.5, delete} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.8, lookup} + ], + [ + {0.01, insert}, + {0.01, delete}, + {0.98, lookup} + ], + [ + {1.0, lookup} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.4, lookup}, + {0.4, nextseq10} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.4, lookup}, + {0.4, nextseq100} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.4, lookup}, + {0.4, nextseq1000} + ], + [ + {1.0, nextseq1000} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.79, lookup}, + {0.01, selectAll} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.7999, lookup}, + {0.0001, selectAll} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.799999, lookup}, + {0.000001, selectAll} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.79, lookup}, + {0.01, partial_select1000} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.7999, lookup}, + {0.0001, partial_select1000} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.799999, lookup}, + {0.000001, partial_select1000} + ] + ], + table_types = + [ + [ordered_set, public], + [ordered_set, public, {write_concurrency, true}], + [ordered_set, public, {read_concurrency, true}], + [ordered_set, public, {write_concurrency, true}, {read_concurrency, true}], + [set, public], + [set, public, {write_concurrency, true}], + [set, public, {read_concurrency, true}], + [set, public, {write_concurrency, true}, {read_concurrency, true}] + ], + etsmem_fun = fun() -> ok end, + verify_etsmem_fun = fun(_) -> true end, + notify_res_fun = fun(_Name, _Throughput) -> ok end, + print_result_paths_fun = + fun(ResultPath, _LatestResultPath) -> + Comment = + io_lib:format("<a href=\"file:///~s\">Result visualization</a>",[ResultPath]), + {comment, Comment} + end + }). + +stdout_notify_res(ResultPath, LatestResultPath) -> + io:format("Result Location: /~s~n", [ResultPath]), + io:format("Latest Result Location: ~s~n", [LatestResultPath]). -throughput_benchmark(TestMode, BenchmarkRunMs, RecoverTimeMs) -> +throughput_benchmark() -> + throughput_benchmark( + #ets_throughput_bench_config{ + print_result_paths_fun = fun stdout_notify_res/2}). + +throughput_benchmark( + #ets_throughput_bench_config{ + benchmark_duration_ms = BenchmarkDurationMs, + recover_time_ms = RecoverTimeMs, + thread_counts = ThreadCountsOpt, + key_ranges = KeyRanges, + scenarios = Scenarios, + table_types = TableTypes, + etsmem_fun = ETSMemFun, + verify_etsmem_fun = VerifyETSMemFun, + notify_res_fun = NotifyResFun, + print_result_paths_fun = PrintResultPathsFun}) -> NrOfSchedulers = erlang:system_info(schedulers), %% Definitions of operations that are supported by the benchmark NextSeqOp = @@ -6634,7 +6814,7 @@ throughput_benchmark(TestMode, BenchmarkRunMs, RecoverTimeMs) -> fun(T,KeyRange) -> NextSeqOp(T,KeyRange,1000) end, selectAll => fun(T,_KeyRange) -> - case -1 =:= ets:select_count(T, ets:fun2ms(fun(X) -> true end)) of + case -1 =:= ets:select_count(T, ets:fun2ms(fun(_X) -> true end)) of true -> io:format("Will never be printed"); false -> ok end @@ -6683,11 +6863,28 @@ throughput_benchmark(TestMode, BenchmarkRunMs, RecoverTimeMs) -> false -> ok end end, + DataHolder = + fun DataHolderFun(Data)-> + receive + {get_data, Pid} -> Pid ! {ets_bench_data, Data}; + D -> DataHolderFun([Data,D]) + end + end, + DataHolderPid = spawn_link(fun()-> DataHolder([]) end), + PrintData = + fun (Str, List) -> + io:format(Str, List), + DataHolderPid ! io_lib:format(Str, List) + end, + GetData = + fun () -> + DataHolderPid ! {get_data, self()}, + receive {ets_bench_data, Data} -> Data end + end, %% Function that runs a benchmark instance and returns the number %% of operations that were performed RunBenchmark = - fun(NrOfProcs, TableConfig, Scenario, - Range, Duration, RecoverTime) -> + fun({NrOfProcs, TableConfig, Scenario, Range, Duration}) -> ProbHelpTab = CalculateOpsProbHelpTab(Scenario, 0), Table = ets:new(t, TableConfig), Nobj = Range div 2, @@ -6695,16 +6892,15 @@ throughput_benchmark(TestMode, BenchmarkRunMs, RecoverTimeMs) -> Nobj = ets:info(Table, size), SafeFixTableIfRequired(Table, Scenario, true), ParentPid = self(), + Worker = + fun() -> + receive start -> ok end, + WorksDone = + do_work(0, Table, ProbHelpTab, Range, Operations), + ParentPid ! WorksDone + end, ChildPids = - lists:map( - fun(_N) -> - spawn(fun() -> - receive start -> ok end, - WorksDone = - do_work(0, Table, ProbHelpTab, Range, Operations), - ParentPid ! WorksDone - end) - end, lists:seq(1, NrOfProcs)), + lists:map(fun(_N) ->spawn_link(Worker)end, lists:seq(1, NrOfProcs)), lists:foreach(fun(Pid) -> Pid ! start end, ChildPids), timer:sleep(Duration), lists:foreach(fun(Pid) -> Pid ! stop end, ChildPids), @@ -6716,185 +6912,194 @@ throughput_benchmark(TestMode, BenchmarkRunMs, RecoverTimeMs) -> end, 0, ChildPids), SafeFixTableIfRequired(Table, Scenario, false), ets:delete(Table), - timer:sleep(RecoverTime), TotalWorksDone end, - %% - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - %%%% Benchmark Configuration %%%%%%%%%%%%%%%%%%%%%%%% - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - %% - %% Change the following variables to configure the benchmark runs - ThreadCounts = - case TestMode of - true -> [1, NrOfSchedulers]; - false -> CalculateThreadCounts([1]) - end, - KeyRanges = % Sizes of the key ranges - case TestMode of - true -> [50000]; - false -> [1000000] + RunBenchmarkInSepProcess = + fun(ParameterTuple) -> + P = self(), + spawn_link(fun()-> P ! {bench_result, RunBenchmark(ParameterTuple)} end), + Result = receive {bench_result, Res} -> Res end, + timer:sleep(RecoverTimeMs), + Result end, - Duration = - case BenchmarkRunMs of % Duration of a benchmark run in milliseconds - not_set -> 30000; - _ -> BenchmarkRunMs + RunBenchmarkAndReport = + fun(ThreadCount, + TableType, + Scenario, + KeyRange, + Duration) -> + Result = RunBenchmarkInSepProcess({ThreadCount, + TableType, + Scenario, + KeyRange, + Duration}), + Throughput = Result/(Duration/1000.0), + PrintData("; ~f",[Throughput]), + Name = io_lib:format("Scenario: ~w, Key Range Size: ~w, " + "# of Processes: ~w, Table Type: ~w", + [Scenario, KeyRange, ThreadCount, TableType]), + NotifyResFun(Name, Throughput) end, - TimeMsToSleepAfterEachBenchmarkRun = - case RecoverTimeMs of - not_set -> 1000; - _ -> RecoverTimeMs + ThreadCounts = + case ThreadCountsOpt of + not_set -> + CalculateThreadCounts([1]); + _ -> ThreadCountsOpt end, - TableTypes = % The table types that will be benchmarked - [ - [ordered_set, public], - [ordered_set, public, {write_concurrency, true}], - [ordered_set, public, {read_concurrency, true}], - [ordered_set, public, {write_concurrency, true}, {read_concurrency, true}], - [set, public], - [set, public, {write_concurrency, true}], - [set, public, {read_concurrency, true}], - [set, public, {write_concurrency, true}, {read_concurrency, true}] - ], - Scenarios = % Benchmark scenarios (the fractions should add up to approximately 1.0) - [ - [ - {0.5, insert}, - {0.5, delete} - ], - [ - {0.1, insert}, - {0.1, delete}, - {0.8, lookup} - ], - [ - {0.01, insert}, - {0.01, delete}, - {0.98, lookup} - ], - [ - {1.0, lookup} - ], - [ - {0.1, insert}, - {0.1, delete}, - {0.4, lookup}, - {0.4, nextseq10} - ], - [ - {0.1, insert}, - {0.1, delete}, - {0.4, lookup}, - {0.4, nextseq100} - ], - [ - {0.1, insert}, - {0.1, delete}, - {0.4, lookup}, - {0.4, nextseq1000} - ], - [ - {1.0, nextseq1000} - ], - [ - {0.1, insert}, - {0.1, delete}, - {0.79, lookup}, - {0.01, selectAll} - ], - [ - {0.1, insert}, - {0.1, delete}, - {0.7999, lookup}, - {0.0001, selectAll} - ], - [ - {0.1, insert}, - {0.1, delete}, - {0.799999, lookup}, - {0.000001, selectAll} - ], - [ - {0.1, insert}, - {0.1, delete}, - {0.79, lookup}, - {0.01, partial_select1000} - ], - [ - {0.1, insert}, - {0.1, delete}, - {0.7999, lookup}, - {0.0001, partial_select1000} - ], - [ - {0.1, insert}, - {0.1, delete}, - {0.799999, lookup}, - {0.000001, partial_select1000} - ] - ], - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - %%%% End of Benchmark Configuration %%%%%%%%%%%%%%%% - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - %% Prepare for memory check - EtsMem = case TestMode of - true -> etsmem(); - false -> ok - end, %% Run the benchmark - io:format("# Each instance of the benchmark runs for ~w seconds:~n", [Duration/1000]), - io:format("# The result of a benchmark instance is presented as a number representing~n"), - io:format("# the number of operations performed per second:~n~n~n"), - io:format("# To plot graphs for the results below:~n"), - io:format("# 1. Open \"$ERL_TOP/lib/stdlib/test/ets_SUITE_data/visualize_throughput.html\" in a web browser~n"), - io:format("# 2. Copy the lines between \"#BENCHMARK STARTED$\" and \"#BENCHMARK ENDED$\" below~n"), - io:format("# 3. Paste the lines copied in step 2 to the text box in the browser window opened in~n"), - io:format("# step 1 and press the Render button~n~n"), - io:format("#BENCHMARK STARTED$~n"), + PrintData("# Each instance of the benchmark runs for ~w seconds:~n", [BenchmarkDurationMs/1000]), + PrintData("# The result of a benchmark instance is presented as a number representing~n",[]), + PrintData("# the number of operations performed per second:~n~n~n",[]), + PrintData("# To plot graphs for the results below:~n",[]), + PrintData("# 1. Open \"$ERL_TOP/lib/stdlib/test/ets_SUITE_data/visualize_throughput.html\" in a web browser~n",[]), + PrintData("# 2. Copy the lines between \"#BENCHMARK STARTED$\" and \"#BENCHMARK ENDED$\" below~n",[]), + PrintData("# 3. Paste the lines copied in step 2 to the text box in the browser window opened in~n",[]), + PrintData("# step 1 and press the Render button~n~n",[]), + PrintData("#BENCHMARK STARTED$~n",[]), + EtsMem = ETSMemFun(), %% The following loop runs all benchmark scenarios and prints the results (i.e, operations/second) lists:foreach( fun(KeyRange) -> lists:foreach( fun(Scenario) -> - io:format("Scenario: ~s | Key Range Size: ~w$~n", - [RenderScenario(Scenario, ""), - KeyRange]), + PrintData("Scenario: ~s | Key Range Size: ~w$~n", + [RenderScenario(Scenario, ""), KeyRange]), lists:foreach( fun(ThreadCount) -> - io:format("; ~w",[ThreadCount]) + PrintData("; ~w",[ThreadCount]) end, ThreadCounts), - io:format("$~n",[]), + PrintData("$~n",[]), lists:foreach( fun(TableType) -> - io:format("~w ",[TableType]), + PrintData("~w ",[TableType]), lists:foreach( fun(ThreadCount) -> - Result = RunBenchmark(ThreadCount, + RunBenchmarkAndReport(ThreadCount, TableType, Scenario, KeyRange, - Duration, - TimeMsToSleepAfterEachBenchmarkRun), - io:format("; ~f",[Result/(Duration/1000.0)]) + BenchmarkDurationMs) end, ThreadCounts), - io:format("$~n",[]) + PrintData("$~n",[]) end, TableTypes) end, Scenarios) end, KeyRanges), - io:format("~n#BENCHMARK ENDED$~n~n"), - case TestMode of - true -> verify_etsmem(EtsMem); - false -> ok - end. + PrintData("~n#BENCHMARK ENDED$~n~n",[]), + VerifyETSMemFun(EtsMem), + DataDir = filename:join(filename:dirname(code:which(?MODULE)), "ets_SUITE_data"), + TemplatePath = filename:join(DataDir, "visualize_throughput.html"), + {ok, Template} = file:read_file(TemplatePath), + OutputData = string:replace(Template, "#bench_data_placeholder", GetData()), + OutputPath1 = filename:join(DataDir, "ets_bench_result.html"), + {{Year, Month, Day}, {Hour, Minute, Second}} = calendar:now_to_datetime(erlang:timestamp()), + StrTime = lists:flatten(io_lib:format("~4..0w-~2..0w-~2..0wT~2..0w:~2..0w:~2..0w",[Year,Month,Day,Hour,Minute,Second])), + OutputPath2 = filename:join(DataDir, io_lib:format("ets_bench_result_~s.html", [StrTime])), + file:write_file(OutputPath1, OutputData), + file:write_file(OutputPath2, OutputData), + PrintResultPathsFun(OutputPath2, OutputPath1). test_throughput_benchmark(Config) when is_list(Config) -> - throughput_benchmark(true, 100, 0). - + throughput_benchmark( + #ets_throughput_bench_config{ + benchmark_duration_ms = 100, + recover_time_ms = 0, + thread_counts = [1, erlang:system_info(schedulers)], + key_ranges = [50000], + etsmem_fun = fun etsmem/0, + verify_etsmem_fun = fun verify_etsmem/1}). + +long_throughput_benchmark(Config) when is_list(Config) -> + N = erlang:system_info(schedulers), + throughput_benchmark( + #ets_throughput_bench_config{ + benchmark_duration_ms = 3000, + recover_time_ms = 1000, + thread_counts = [1, N div 2, N], + key_ranges = [1000000], + scenarios = + [ + [ + {0.5, insert}, + {0.5, delete} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.8, lookup} + ], + [ + {0.01, insert}, + {0.01, delete}, + {0.98, lookup} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.4, lookup}, + {0.4, nextseq100} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.79, lookup}, + {0.01, selectAll} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.79, lookup}, + {0.01, partial_select1000} + ] + ], + table_types = + [ + [ordered_set, public, {write_concurrency, true}, {read_concurrency, true}], + [set, public, {write_concurrency, true}, {read_concurrency, true}] + ], + etsmem_fun = fun etsmem/0, + verify_etsmem_fun = fun verify_etsmem/1, + notify_res_fun = + fun(Name, Throughput) -> + SummaryTable = + proplists:get_value(ets_benchmark_result_summary_tab, Config), + AddToSummaryCounter = + case SummaryTable of + undefined -> + fun(_, _) -> + ok + end; + Tab -> + fun(CounterName, ToAdd) -> + OldVal = ets:lookup_element(Tab, CounterName, 2), + NewVal = OldVal + ToAdd, + ets:insert(Tab, {CounterName, NewVal}) + end + end, + Record = + fun(NoOfBenchsCtr, TotThrputCtr) -> + AddToSummaryCounter(NoOfBenchsCtr, 1), + AddToSummaryCounter(TotThrputCtr, Throughput) + end, + Record(nr_of_benchmarks, total_throughput), + case string:find(Name, "ordered_set") of + nomatch -> + Record(nr_of_set_benchmarks, total_throughput_set); + _ -> + Record(nr_of_ordered_set_benchmarks, + total_throughput_ordered_set) + end, + ct_event:notify( + #event{name = benchmark_data, + data = [{suite,"ets_bench"}, + {name, Name}, + {value,Throughput}]}) + end + }). add_lists(L1,L2) -> add_lists(L1,L2,[]). diff --git a/lib/stdlib/test/ets_SUITE_data/visualize_throughput.html b/lib/stdlib/test/ets_SUITE_data/visualize_throughput.html index a2c61aa938..27d6849c60 100644 --- a/lib/stdlib/test/ets_SUITE_data/visualize_throughput.html +++ b/lib/stdlib/test/ets_SUITE_data/visualize_throughput.html @@ -42,7 +42,7 @@ </p> Paste the generated data in the field below and press the Render button: <br> - <textarea id="dataField" rows="4" cols="50"></textarea> + <textarea id="dataField" rows="4" cols="50">#bench_data_placeholder</textarea> <br> <input type="checkbox" id="barPlot"> Bar Plot <br> @@ -56,13 +56,13 @@ <br> <input type="checkbox" class="showCheck" value="[ordered_set,public,{write_concurrency,true},{read_concurrency,true}]" checked> Show <code>[ordered_set,public,{write_concurrency,true},{read_concurrency,true}]</code> <br> - <input type="checkbox" class="showCheck" value="[set,public]"> Show <code>[set,public]</code> + <input type="checkbox" class="showCheck" value="[set,public]" checked> Show <code>[set,public]</code> <br> - <input type="checkbox" class="showCheck" value="[set,public,{write_concurrency,true}]"> Show <code>[set,public,{write_concurrency,true}]</code> + <input type="checkbox" class="showCheck" value="[set,public,{write_concurrency,true}]" checked> Show <code>[set,public,{write_concurrency,true}]</code> <br> - <input type="checkbox" class="showCheck" value="[set,public,{read_concurrency,true}]"> Show <code>[set,public,{read_concurrency,true}]</code> + <input type="checkbox" class="showCheck" value="[set,public,{read_concurrency,true}]" checked> Show <code>[set,public,{read_concurrency,true}]</code> <br> - <input type="checkbox" class="showCheck" value="[set,public,{write_concurrency,true},{read_concurrency,true}]"> Show <code>[set,public,{write_concurrency,true},{read_concurrency,true}]</code> + <input type="checkbox" class="showCheck" value="[set,public,{write_concurrency,true},{read_concurrency,true}]" checked> Show <code>[set,public,{write_concurrency,true},{read_concurrency,true}]</code> <br> <button id="renderButton" type="button">Render</button> diff --git a/lib/stdlib/test/stdlib.spec b/lib/stdlib/test/stdlib.spec index 4de7c1a0eb..bf64eae2c7 100644 --- a/lib/stdlib/test/stdlib.spec +++ b/lib/stdlib/test/stdlib.spec @@ -2,3 +2,6 @@ {skip_groups,"../stdlib_test",stdlib_bench_SUITE, [binary,base64,gen_server,gen_statem,unicode], "Benchmark only"}. +{skip_groups,"../stdlib_test",ets_SUITE, + [benchmark], + "Benchmark only"}. diff --git a/lib/stdlib/test/stdlib_bench.spec b/lib/stdlib/test/stdlib_bench.spec index 7a0da811a0..6d665f22b6 100644 --- a/lib/stdlib/test/stdlib_bench.spec +++ b/lib/stdlib/test/stdlib_bench.spec @@ -8,3 +8,4 @@ {skip_groups,"../stdlib_test",stdlib_bench_SUITE, [gen_server_comparison,gen_statem_comparison], "Not a benchmark"}. +{groups,"../stdlib_test",ets_SUITE,[benchmark]}. diff --git a/lib/stdlib/test/string_SUITE.erl b/lib/stdlib/test/string_SUITE.erl index 251e09121c..248912c3f2 100644 --- a/lib/stdlib/test/string_SUITE.erl +++ b/lib/stdlib/test/string_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2018. All Rights Reserved. +%% Copyright Ericsson AB 2004-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -754,19 +754,22 @@ do_measure(DataDir) -> io:format("~p~n",[byte_size(Bin)]), Do = fun(Name, Func, Mode) -> {N, Mean, Stddev, _} = time_func(Func, Mode, Bin, 20), - io:format("~15w ~6w ~6.2fms ±~5.2fms #~.2w gc included~n", + io:format("~15w ~15w ~8.2fms ±~6.2fms #~.2w gc included~n", [Name, Mode, Mean/1000, Stddev/1000, N]) end, Do2 = fun(Name, Func, Mode) -> {N, Mean, Stddev, _} = time_func(Func, binary, <<>>, 20), - io:format("~15w ~6w ~6.2fms ±~5.2fms #~.2w gc included~n", + io:format("~15w ~15w ~8.2fms ±~6.2fms #~.2w gc included~n", [Name, Mode, Mean/1000, Stddev/1000, N]) end, + %% lefty_list means a list balanced to the left, like + %% [[[30],31],32]. Only some functions check such lists. + Modes = [list, lefty_list, binary, {many_lists,1}, {many_lists, 4}], io:format("----------------------~n"), Do(old_tokens, fun(Str) -> string:tokens(Str, [$\n,$\r]) end, list), Tokens = {lexemes, fun(Str) -> string:lexemes(Str, [$\n,$\r]) end}, - [Do(Name,Fun,Mode) || {Name,Fun} <- [Tokens], Mode <- [list, binary]], + [Do(Name,Fun,Mode) || {Name,Fun} <- [Tokens], Mode <- Modes], S0 = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxy.....", S0B = <<"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxy.....">>, @@ -824,17 +827,17 @@ do_measure(DataDir) -> io:format("--~n",[]), NthTokens = {nth_lexemes, fun(Str) -> string:nth_lexeme(Str, 18000, [$\n,$\r]) end}, - [Do(Name,Fun,Mode) || {Name,Fun} <- [NthTokens], Mode <- [list, binary]], + [Do(Name,Fun,Mode) || {Name,Fun} <- [NthTokens], Mode <- Modes], Do2(take_t, repeat(fun() -> string:take(S0, [$.,$y], false, trailing) end), list), Do2(take_t, repeat(fun() -> string:take(S0B, [$.,$y], false, trailing) end), binary), Do2(take_tc, repeat(fun() -> string:take(S0, [$x], true, trailing) end), list), Do2(take_tc, repeat(fun() -> string:take(S0B, [$x], true, trailing) end), binary), Length = {length, fun(Str) -> string:length(Str) end}, - [Do(Name,Fun,Mode) || {Name,Fun} <- [Length], Mode <- [list, binary]], + [Do(Name,Fun,Mode) || {Name,Fun} <- [Length], Mode <- Modes], Reverse = {reverse, fun(Str) -> string:reverse(Str) end}, - [Do(Name,Fun,Mode) || {Name,Fun} <- [Reverse], Mode <- [list, binary]], + [Do(Name,Fun,Mode) || {Name,Fun} <- [Reverse], Mode <- Modes], ok. @@ -1064,7 +1067,33 @@ time_func(N,Sum,SumSq, _, _, Res, _) -> {N, Mean, Stdev, Res}. mode(binary, Bin) -> Bin; -mode(list, Bin) -> unicode:characters_to_list(Bin). +mode(list, Bin) -> unicode:characters_to_list(Bin); +mode(lefty_list, Bin) -> + L = unicode:characters_to_list(Bin), + to_left(L); +mode({many_lists, N}, Bin) -> + group(unicode:characters_to_list(Bin), N). + +group([], _N) -> + []; +group(L, N) -> + try lists:split(N, L) of + {L1, L2} -> + [L1 | group(L2, N)] + catch + _:_ -> + [L] + end. + +to_left([]) -> + []; +to_left([H|L]) -> + to_left([H], L). + +to_left(V, []) -> + V; +to_left(V, [H|L]) -> + to_left([V,H], L). %% %% Old string lists Test cases starts here. diff --git a/lib/stdlib/uc_spec/gen_unicode_mod.escript b/lib/stdlib/uc_spec/gen_unicode_mod.escript index 70eec1a6f2..8636c69a0d 100755..100644 --- a/lib/stdlib/uc_spec/gen_unicode_mod.escript +++ b/lib/stdlib/uc_spec/gen_unicode_mod.escript @@ -4,7 +4,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2017. All Rights Reserved. +%% Copyright Ericsson AB 2017-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -460,17 +460,73 @@ gen_cp(Fd) -> " maybe_improper_list() | {error, unicode:chardata()}.\n"), io:put_chars(Fd, "cp([C|_]=L) when is_integer(C) -> L;\n"), io:put_chars(Fd, "cp([List]) -> cp(List);\n"), - io:put_chars(Fd, "cp([List|R]) ->\n"), - io:put_chars(Fd, " case cp(List) of\n"), - io:put_chars(Fd, " [] -> cp(R);\n"), - io:put_chars(Fd, " [CP] -> [CP|R];\n"), - io:put_chars(Fd, " [C|R0] -> [C|[R0|R]];\n"), - io:put_chars(Fd, " {error,Error} -> {error,[Error|R]}\n"), - io:put_chars(Fd, " end;\n"), + io:put_chars(Fd, "cp([List|R]) -> cpl(List, R);\n"), io:put_chars(Fd, "cp([]) -> [];\n"), io:put_chars(Fd, "cp(<<C/utf8, R/binary>>) -> [C|R];\n"), io:put_chars(Fd, "cp(<<>>) -> [];\n"), - io:put_chars(Fd, "cp(<<R/binary>>) -> {error,R}.\n\n"), + io:put_chars(Fd, "cp(<<R/binary>>) -> {error,R}.\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "cpl([C], R) when is_integer(C) -> [C|cpl_1_cont(R)];\n"), + io:put_chars(Fd, "cpl([C|T], R) when is_integer(C) -> [C|cpl_cont(T, R)];\n"), + io:put_chars(Fd, "cpl([List], R) -> cpl(List, R);\n"), + io:put_chars(Fd, "cpl([List|T], R) -> cpl(List, [T|R]);\n"), + io:put_chars(Fd, "cpl([], R) -> cp(R);\n"), + io:put_chars(Fd, "cpl(<<C/utf8, T/binary>>, R) -> [C,T|R];\n"), + io:put_chars(Fd, "cpl(<<>>, R) -> cp(R);\n"), + io:put_chars(Fd, "cpl(<<B/binary>>, R) -> {error,[B|R]}.\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "%%%\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "cpl_cont([C|T], R) when is_integer(C) -> [C|cpl_cont2(T, R)];\n"), + io:put_chars(Fd, "cpl_cont([L], R) -> cpl_cont(L, R);\n"), + io:put_chars(Fd, "cpl_cont([L|T], R) -> cpl_cont(L, [T|R]);\n"), + io:put_chars(Fd, "cpl_cont([], R) -> cpl_1_cont(R);\n"), + io:put_chars(Fd, "cpl_cont(T, R) -> [T|R].\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "cpl_cont2([C|T], R) when is_integer(C) -> [C|cpl_cont3(T, R)];\n"), + io:put_chars(Fd, "cpl_cont2([L], R) -> cpl_cont2(L, R);\n"), + io:put_chars(Fd, "cpl_cont2([L|T], R) -> cpl_cont2(L, [T|R]);\n"), + io:put_chars(Fd, "cpl_cont2([], R) -> cpl_1_cont2(R);\n"), + io:put_chars(Fd, "cpl_cont2(T, R) -> [T|R].\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "cpl_cont3([C], R) when is_integer(C) -> [C|R];\n"), + io:put_chars(Fd, "cpl_cont3([C|T], R) when is_integer(C) -> [C,T|R];\n"), + io:put_chars(Fd, "cpl_cont3([L], R) -> cpl_cont3(L, R);\n"), + io:put_chars(Fd, "cpl_cont3([L|T], R) -> cpl_cont3(L, [T|R]);\n"), + io:put_chars(Fd, "cpl_cont3([], R) -> cpl_1_cont3(R);\n"), + io:put_chars(Fd, "cpl_cont3(T, R) -> [T|R].\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "%%%\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "cpl_1_cont([C|T]) when is_integer(C) -> [C|cpl_1_cont2(T)];\n"), + io:put_chars(Fd, "cpl_1_cont([L]) -> cpl_1_cont(L);\n"), + io:put_chars(Fd, "cpl_1_cont([L|T]) -> cpl_cont(L, T);\n"), + io:put_chars(Fd, "cpl_1_cont(T) -> T.\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "cpl_1_cont2([C|T]) when is_integer(C) -> [C|cpl_1_cont3(T)];\n"), + io:put_chars(Fd, "cpl_1_cont2([L]) -> cpl_1_cont2(L);\n"), + io:put_chars(Fd, "cpl_1_cont2([L|T]) -> cpl_cont2(L, T);\n"), + io:put_chars(Fd, "cpl_1_cont2(T) -> T.\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "cpl_1_cont3([C|_]=T) when is_integer(C) -> T;\n"), + io:put_chars(Fd, "cpl_1_cont3([L]) -> cpl_1_cont3(L);\n"), + io:put_chars(Fd, "cpl_1_cont3([L|T]) -> cpl_cont3(L, T);\n"), + io:put_chars(Fd, "cpl_1_cont3(T) -> T.\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "%%%\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "cp_no_bin([C|_]=L) when is_integer(C) -> L;\n"), + io:put_chars(Fd, "cp_no_bin([List]) -> cp_no_bin(List);\n"), + io:put_chars(Fd, "cp_no_bin([List|R]) -> cp_no_binl(List, R);\n"), + io:put_chars(Fd, "cp_no_bin([]) -> [];\n"), + io:put_chars(Fd, "cp_no_bin(_) -> binary_found.\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "cp_no_binl([C], R) when is_integer(C) -> [C|cpl_1_cont(R)];\n"), + io:put_chars(Fd, "cp_no_binl([C|T], R) when is_integer(C) -> [C|cpl_cont(T, R)];\n"), + io:put_chars(Fd, "cp_no_binl([List], R) -> cp_no_binl(List, R);\n"), + io:put_chars(Fd, "cp_no_binl([List|T], R) -> cp_no_binl(List, [T|R]);\n"), + io:put_chars(Fd, "cp_no_binl([], R) -> cp_no_bin(R);\n"), + io:put_chars(Fd, "cp_no_binl(_, _) -> binary_found.\n\n"), ok. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -481,11 +537,26 @@ gen_gc(Fd, GBP) -> "-spec gc(String::unicode:chardata()) ->" " maybe_improper_list() | {error, unicode:chardata()}.\n"), io:put_chars(Fd, - "gc([CP1, CP2|_]=T)\n" - " when CP1 < 256, CP2 < 256, CP1 =/= $\r -> %% Ascii Fast path\n" - " T;\n" + "gc([]=R) -> R;\n" + "gc([CP]=R) when is_integer(CP) -> R;\n" + "gc([$\\r=CP|R0]) ->\n" + " case cp(R0) of % Don't break CRLF\n" + " [$\\n|R1] -> [[$\\r,$\\n]|R1];\n" + " T -> [CP|T]\n" + " end;\n" + "gc([CP1|T1]=T) when CP1 < 256 ->\n" + " case T1 of\n" + " [CP2|_] when CP2 < 256 -> T; %% Ascii Fast path\n" + " _ -> %% Keep the tail binary.\n" + " case cp_no_bin(T1) of\n" + " [CP2|_]=T3 when CP2 < 256 -> [CP1|T3]; %% Asciii Fast path\n" + " binary_found -> gc_1(T);\n" + " T4 -> gc_1([CP1|T4])\n" + " end\n" + " end;\n" + "gc(<<>>) -> [];\n" "gc(<<CP1/utf8, Rest/binary>>) ->\n" - " if CP1 < 256, CP1 =/= $\r ->\n" + " if CP1 < 256, CP1 =/= $\\r ->\n" " case Rest of\n" " <<CP2/utf8, _/binary>> when CP2 < 256 -> %% Ascii Fast path\n" " [CP1|Rest];\n" @@ -493,13 +564,12 @@ gen_gc(Fd, GBP) -> " end;\n" " true -> gc_1([CP1|Rest])\n" " end;\n" + "gc([CP|_]=T) when is_integer(CP) -> gc_1(T);\n" "gc(Str) ->\n" - " gc_1(cp(Str)).\n\n" - "gc_1([$\\r|R0] = R) ->\n" - " case cp(R0) of % Don't break CRLF\n" - " [$\\n|R1] -> [[$\\r,$\\n]|R1];\n" - " _ -> R\n" - " end;\n" + " case cp(Str) of\n" + " {error,_}=Error -> Error;\n" + " CPs -> gc(CPs)\n" + " end.\n" ), GenExtP = fun(Range) -> io:format(Fd, "gc_1~s gc_ext_pict(R1,[CP]);\n", [gen_clause(Range)]) end, @@ -507,7 +577,12 @@ gen_gc(Fd, GBP) -> %% Pick codepoints below 256 (some data knowledge here) {ExtendedPictographicLow,ExtendedPictographicHigh} = lists:splitwith(fun({Start,undefined}) -> Start < 256 end,ExtendedPictographic0), - + io:put_chars(Fd, + "\ngc_1([$\\r|R0] = R) ->\n" + " case cp(R0) of % Don't break CRLF\n" + " [$\\n|R1] -> [[$\\r,$\\n]|R1];\n" + " _ -> R\n" + " end;\n"), io:put_chars(Fd, "\n%% Handle control\n"), GenControl = fun(Range) -> io:format(Fd, "gc_1~s R0;\n", [gen_clause(Range)]) end, CRs0 = merge_ranges(maps:get(cr, GBP) ++ maps:get(lf, GBP) ++ maps:get(control, GBP), false), @@ -516,7 +591,14 @@ gen_gc(Fd, GBP) -> %%GenControl(R1),GenControl(R2),GenControl(R3), io:put_chars(Fd, "\n%% Optimize Latin-1\n"), [GenExtP(CP) || CP <- merge_ranges(ExtendedPictographicLow)], - io:format(Fd, "gc_1([CP|R]) when CP < 256 -> gc_extend(R,CP);\n\n", []), + + io:format(Fd, + "gc_1([CP|R]=R0) when CP < 256 ->\n" + " case R of\n" + " [CP2|_] when CP2 < 256 -> R0;\n" + " _ -> gc_extend(cp(R), R, CP)\n" + " end;\n", + []), io:put_chars(Fd, "\n%% Continue control\n"), [GenControl(CP) || CP <- Crs], %% One clause per CP @@ -540,7 +622,7 @@ gen_gc(Fd, GBP) -> io:put_chars(Fd, "gc_1([CP|_]=R0) when 44000 < CP, CP < 56000 -> gc_h_lv_lvt(R0, []);\n"), io:put_chars(Fd, "\n%% Handle Regional\n"), - GenRegional = fun(Range) -> io:format(Fd, "gc_1~s gc_regional(R1,[CP]);\n", [gen_clause(Range)]) end, + GenRegional = fun(Range) -> io:format(Fd, "gc_1~s gc_regional(R1,CP);\n", [gen_clause(Range)]) end, [GenRegional(CP) || CP <- merge_ranges(maps:get(regional_indicator,GBP))], %% io:put_chars(Fd, "%% Handle E_Base\n"), %% GenEBase = fun(Range) -> io:format(Fd, "gc_1~s gc_e_cont(R1,[CP]);\n", [gen_clause(Range)]) end, @@ -552,9 +634,7 @@ gen_gc(Fd, GBP) -> io:put_chars(Fd, "%% Handle extended_pictographic\n"), [GenExtP(CP) || CP <- merge_ranges(ExtendedPictographicHigh)], io:put_chars(Fd, "\n%% default clauses\n"), - io:put_chars(Fd, "gc_1([CP|R]) -> gc_extend(R, CP);\n"), - io:put_chars(Fd, "gc_1([]) -> [];\n"), - io:put_chars(Fd, "gc_1({error,_}=Error) -> Error.\n\n"), + io:put_chars(Fd, "gc_1([CP|R]) -> gc_extend(cp(R), R, CP).\n\n"), io:put_chars(Fd, "%% Handle Prepend\n"), io:put_chars(Fd, @@ -581,31 +661,24 @@ gen_gc(Fd, GBP) -> "%% To simplify binary handling in libraries the tail should be kept binary\n" "%% and not a lookahead CP\n" ), - io:put_chars(Fd, "gc_extend(T, Acc) ->\n" - " gc_extend(cp(T), T, Acc).\n\n"), io:put_chars(Fd, - "gc_extend([CP|T], T0, Acc0) ->\n" + "gc_extend([CP|T], T0, CP0) ->\n" " case is_extend(CP) of\n" - " false ->\n" - " case Acc0 of\n" - " [Acc] -> [Acc|T0];\n" - " [_|_]=Acc -> [lists:reverse(Acc)|T0];\n" - " Acc -> [Acc|T0]\n" - " end;\n" - " _TrueOrZWJ ->\n" - " case Acc0 of\n" - " [_|_] -> gc_extend(T, [CP|Acc0]);\n" - " Acc -> gc_extend(T, [CP,Acc])\n" - " end\n" + " false -> [CP0|T0]; % losing work done on T\n" + " _TrueOrZWJ -> gc_extend2(cp(T), T, [CP,CP0])\n" " end;\n" - "gc_extend([], _, Acc0) ->\n" - " case Acc0 of\n" - " [_]=Acc -> Acc;\n" - " [_|_]=Acc -> [lists:reverse(Acc)];\n" - " Acc -> [Acc]\n" + "gc_extend([], _, CP) -> [CP];\n" + "gc_extend({error,R}, _, CP) -> [CP|R].\n\n"), + io:put_chars(Fd, + "gc_extend2([CP|T], T0, Acc) ->\n" + " case is_extend(CP) of\n" + " false -> [lists:reverse(Acc)|T0]; % losing work done on T\n" + " _TrueOrZWJ -> gc_extend2(cp(T), T, [CP|Acc])\n" " end;\n" - "gc_extend({error,R}, T, Acc0) ->\n" - " gc_extend([], T, Acc0) ++ [R].\n\n" + "gc_extend2([], _, Acc) ->\n" + " [lists:reverse(Acc)];\n" + "gc_extend2({error,R}, _, Acc) ->\n" + " [lists:reverse(Acc)] ++ [R].\n\n" ), [ZWJ] = maps:get(zwj, GBP), GenExtend = fun(R) when R =:= ZWJ -> io:format(Fd, "is_extend~s zwj;\n", [gen_single_clause(ZWJ)]); @@ -660,10 +733,10 @@ gen_gc(Fd, GBP) -> %% -------------------- io:put_chars(Fd, "%% Handle Regional\n"), [{RLess,RLarge}] = merge_ranges(maps:get(regional_indicator,GBP)), - io:put_chars(Fd,"gc_regional(R0, Acc) ->\n" + io:put_chars(Fd,"gc_regional(R0, CP0) ->\n" " case cp(R0) of\n"), - io:format(Fd, " [CP|R1] when ~w =< CP,CP =< ~w-> gc_extend(R1,[CP|Acc]);~n",[RLess, RLarge]), - io:put_chars(Fd," R1 -> gc_extend(R1, R0, Acc)\n" + io:format(Fd, " [CP|R1] when ~w =< CP,CP =< ~w-> gc_extend2(cp(R1),R1,[CP,CP0]);~n",[RLess, RLarge]), + io:put_chars(Fd," R1 -> gc_extend(R1, R0, CP0)\n" " end.\n\n"), %% Special hangul @@ -685,16 +758,23 @@ gen_gc(Fd, GBP) -> GenHangulV_2 = fun(Range) -> io:format(Fd, "~8c~s gc_h_T(R1,[CP|Acc]);\n", [$\s,gen_case_clause(Range)]) end, [GenHangulV_2(CP) || CP <- merge_ranges(maps:get(t,GBP))], - io:put_chars(Fd, " R1 -> gc_extend(R1, R0, Acc)\n end.\n\n"), - + io:put_chars(Fd, + " R1 ->\n" + " case Acc of\n" + " [CP] -> gc_extend(R1, R0, CP);\n" + " _ -> gc_extend2(R1, R0, Acc)\n" + " end\n end.\n\n"), io:put_chars(Fd, "%% Handle Hangul T\n"), io:put_chars(Fd, "gc_h_T(R0, Acc) ->\n case cp(R0) of\n"), GenHangulT_1 = fun(Range) -> io:format(Fd, "~8c~s gc_h_T(R1,[CP|Acc]);\n", [$\s,gen_case_clause(Range)]) end, [GenHangulT_1(CP) || CP <- merge_ranges(maps:get(t,GBP))], - io:put_chars(Fd, " R1 -> gc_extend(R1, R0, Acc)\n end.\n\n"), - - io:put_chars(Fd, "gc_h_lv_lvt({error,_}=Error, Acc) -> gc_extend(Error, [], Acc);\n"), + io:put_chars(Fd, + " R1 ->\n" + " case Acc of\n" + " [CP] -> gc_extend(R1, R0, CP);\n" + " _ -> gc_extend2(R1, R0, Acc)\n" + " end\n end.\n\n"), io:put_chars(Fd, "%% Handle Hangul LV\n"), GenHangulLV = fun(Range) -> io:format(Fd, "gc_h_lv_lvt~s gc_h_V(R1,[CP|Acc]);\n", [gen_clause2(Range)]) end, @@ -703,8 +783,10 @@ gen_gc(Fd, GBP) -> GenHangulLVT = fun(Range) -> io:format(Fd, "gc_h_lv_lvt~s gc_h_T(R1,[CP|Acc]);\n", [gen_clause2(Range)]) end, [GenHangulLVT(CP) || CP <- merge_ranges(maps:get(lvt,GBP))], - io:put_chars(Fd, "gc_h_lv_lvt([CP|R], []) -> gc_extend(R, CP);\n"), %% From gc_1/1 - io:put_chars(Fd, "gc_h_lv_lvt(R, Acc) -> gc_extend(R, Acc).\n\n"), + io:put_chars(Fd, "gc_h_lv_lvt([CP|R], []) -> gc_extend(cp(R), R, CP);\n"), %% From gc_1/1 + io:put_chars(Fd, "%% Also handles error tuples\n"), + io:put_chars(Fd, "gc_h_lv_lvt(R, [CP]) -> gc_extend(R, R, CP);\n"), + io:put_chars(Fd, "gc_h_lv_lvt(R, Acc) -> gc_extend2(R, R, Acc).\n\n"), ok. gen_compose_pairs(Fd, ExclData, Data) -> diff --git a/lib/stdlib/vsn.mk b/lib/stdlib/vsn.mk index cbefd6590a..6471dc70e0 100644 --- a/lib/stdlib/vsn.mk +++ b/lib/stdlib/vsn.mk @@ -1 +1 @@ -STDLIB_VSN = 3.8 +STDLIB_VSN = 3.8.1 diff --git a/lib/syntax_tools/test/syntax_tools_SUITE.erl b/lib/syntax_tools/test/syntax_tools_SUITE.erl index 6b42f7a0a1..e1dd1bd73b 100644 --- a/lib/syntax_tools/test/syntax_tools_SUITE.erl +++ b/lib/syntax_tools/test/syntax_tools_SUITE.erl @@ -74,7 +74,7 @@ smoke_test_file(File) -> [print_error_markers(F, File) || F <- Forms], ok; {error,Reason} -> - io:format("~s: ~p\n", [File,Reason]), + io:format("~ts: ~p\n", [File,Reason]), error end. @@ -82,7 +82,7 @@ print_error_markers(F, File) -> case erl_syntax:type(F) of error_marker -> {L,M,Info} = erl_syntax:error_marker_info(F), - io:format("~ts:~p: ~s", [File,L,M:format_error(Info)]); + io:format("~ts:~p: ~ts", [File,L,M:format_error(Info)]); _ -> ok end. @@ -362,7 +362,7 @@ test_comment_scan([File|Files],DataDir) -> end, Fs1 = erl_recomment:recomment_forms(Fs0, Comments), Fs2 = erl_syntax_lib:map(Fun, Fs1), - io:format("File: ~s~n", [Filename]), + io:format("File: ~ts~n", [Filename]), io:put_chars(erl_prettypr:format(Fs2, [{paper, 120}, {ribbon, 110}])), test_comment_scan(Files,DataDir). @@ -377,8 +377,8 @@ test_prettypr([File|Files],DataDir,PrivDir) -> PP = erl_prettypr:format(Fs, [{paper, 120}, {ribbon, 110}]), io:put_chars(PP), OutFile = filename:join(PrivDir, File), - ok = file:write_file(OutFile,iolist_to_binary(PP)), - io:format("Parsing OutFile: ~s~n", [OutFile]), + ok = file:write_file(OutFile,unicode:characters_to_binary(PP)), + io:format("Parsing OutFile: ~ts~n", [OutFile]), {ok, Fs2} = epp:parse_file(OutFile, [], []), case [Error || {error, _} = Error <- Fs2] of [] -> @@ -445,7 +445,7 @@ pretty_print_parse_forms([{Fs0,Type}|FsForms],PrivDir,Filename) -> {Fs2,{CC,CT}} = erl_syntax_lib:mapfold(Comment,{0,0}, Fs1), io:format("Commented on ~w cases and ~w tries~n", [CC,CT]), PP = erl_prettypr:format(Fs2), - ok = file:write_file(OutFile,iolist_to_binary(PP)), + ok = file:write_file(OutFile,unicode:characters_to_binary(PP)), pretty_print_parse_forms(FsForms,PrivDir,Filename). diff --git a/lib/tools/emacs/erlang.el b/lib/tools/emacs/erlang.el index 3cbe9daa60..38c0eba92b 100644 --- a/lib/tools/emacs/erlang.el +++ b/lib/tools/emacs/erlang.el @@ -905,8 +905,10 @@ resulting regexp is surrounded by \\_< and \\_>." "dist_get_stat" "dist_ctrl_get_data" "dist_ctrl_get_data_notification" + "dist_ctrl_get_opt" "dist_ctrl_input_handler" "dist_ctrl_put_data" + "dist_ctrl_set_opt" "dmonitor_node" "dt_append_vm_tag_data" "dt_get_tag" diff --git a/lib/tools/emacs/erldoc.el b/lib/tools/emacs/erldoc.el index 770ab299ee..bc16d7a14d 100644 --- a/lib/tools/emacs/erldoc.el +++ b/lib/tools/emacs/erldoc.el @@ -89,6 +89,13 @@ up the indexing." :type 'file :group 'erldoc) +(defcustom erldoc-no-signature-function #'ignore + "Notification function called if no function signature was found." + :type '(choice (function-item :tag "Ignore" ignore) + (function-item :tag "Warn" warn) + (function-item :tag "Error" error)) + :group 'erldoc) + (defun erldoc-strip-string (s) (let* ((re "[ \t\n\r\f\v\u00a0]+") (from (if (string-match (concat "\\`" re) s) (match-end 0) 0)) @@ -212,11 +219,21 @@ up the indexing." ;; Get the full function signature. (when (and (eq (car-safe d) 'a) (gethash (erldoc-dom-get-attribute d 'name) table)) - (push (append (gethash (erldoc-dom-get-attribute d 'name) table) - (list (funcall span-content - (or (erldoc-dom-get-element d 'span) - (cadr (memq d erldoc-dom-walk-siblings)))))) - entries)) + (let* ((name (erldoc-dom-get-attribute d 'name)) + (mfa-url (gethash name table)) + (mfa (car mfa-url)) + (sig (or (funcall span-content d) + (funcall span-content + (or (erldoc-dom-get-element d 'span) + (cadr + (memq d erldoc-dom-walk-siblings)))) + (progn + (funcall erldoc-no-signature-function + "erldoc-parse-man: no sig for %s" + mfa) + nil)))) + (push (append mfa-url (list sig)) + entries))) ;; Get data types (when (and (eq (car-safe d) 'a) (string-prefix-p "type-" @@ -280,7 +297,7 @@ up the indexing." (unless (file-exists-p of) (erldoc-parse-all erldoc-man-index of)) (unless (string= erldoc-output-file of) - (make-symbolic-link of erldoc-output-file)))) + (make-symbolic-link (expand-file-name of) erldoc-output-file)))) (setq erldoc-lookup-table (with-temp-buffer (insert-file-contents erldoc-output-file) @@ -356,9 +373,12 @@ up the indexing." (sigs)) (maphash (lambda (k v) (when (string-match re k) - (push (cons (string-to-number (match-string 1 k)) - (cdr (erldoc-tokenize-signature (cadr v)))) - sigs))) + (if (cadr v) + (push (cons (string-to-number (match-string 1 k)) + (cdr (erldoc-tokenize-signature (cadr v)))) + sigs) + (funcall erldoc-no-signature-function + "erldoc-format-signature: No sig for %s" k)))) (erldoc-lookup-table)) (when sigs ;; Mostly single return type but there are exceptions such as diff --git a/lib/wx/src/Makefile b/lib/wx/src/Makefile index 21b45af2c4..52f4008e0a 100644 --- a/lib/wx/src/Makefile +++ b/lib/wx/src/Makefile @@ -111,10 +111,10 @@ $(APPUP_TARGET): $(APPUP_SRC) ../vsn.mk Makefile # Rules $(EBIN)/%.beam: $(ESRC)/%.erl $(HEADER_FILES) - $(V_ERLC) -W -bbeam $(ERL_FLAGS) $(ERL_COMPILE_FLAGS) -o$(EBIN) $< + $(V_ERLC) -W -bbeam $(ERL_COMPILE_FLAGS) -o$(EBIN) $< $(EBIN)/%.beam: $(EGEN)/%.erl $(HEADER_FILES) - $(V_ERLC) -W -bbeam $(ERL_FLAGS) $(ERL_COMPILE_FLAGS) -o$(EBIN) $< + $(V_ERLC) -W -bbeam $(ERL_COMPILE_FLAGS) -o$(EBIN) $< # ---------------------------------------------------- # Release Target diff --git a/lib/xmerl/doc/src/notes.xml b/lib/xmerl/doc/src/notes.xml index 7f6874e36b..d6b6dfdfb5 100644 --- a/lib/xmerl/doc/src/notes.xml +++ b/lib/xmerl/doc/src/notes.xml @@ -32,6 +32,28 @@ <p>This document describes the changes made to the Xmerl application.</p> +<section><title>Xmerl 1.3.20</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Handling of character references in attributes are fixed.</p> + <p> + Own Id: OTP-15684 Aux Id: ERL-837 </p> + </item> + <item> + <p> + Normalization of whitespace characters in attributes are + fixed so it works when character references are used.</p> + <p> + Own Id: OTP-15685 Aux Id: ERL-475 </p> + </item> + </list> + </section> + +</section> + <section><title>Xmerl 1.3.19</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/xmerl/src/xmerl_scan.erl b/lib/xmerl/src/xmerl_scan.erl index e543a5a11e..d76ed5c820 100644 --- a/lib/xmerl/src/xmerl_scan.erl +++ b/lib/xmerl/src/xmerl_scan.erl @@ -2410,15 +2410,22 @@ scan_att_chars("&" ++ T, S0, Delim, Acc, TmpAcc,AT,IsNorm) -> % Reference true -> scan_att_chars(T1,S1,Delim,[ExpRef|Acc],[ExpRef|TmpAcc],AT,IsNorm); _ -> - Ch = string_to_char_set(S#xmerl_scanner.encoding, ExpRef), case T of "#" ++ _ -> %% normalization rules (sec 3.3.3) require that for %% character references, the referenced character be %% added directly to the normalized value - scan_att_chars(T1, S1, Delim, Ch ++ Acc,TmpAcc, AT,IsNorm); + {T2,S2,IsNorm2} = + if + ?whitespace(hd(ExpRef)) -> + normalize(T1, S1, IsNorm); + true -> + {T1, S1, IsNorm} + end, + scan_att_chars(T2, S2, Delim, ExpRef ++ Acc, TmpAcc, AT, IsNorm2); _ -> - scan_att_chars(Ch ++ T1, S1, Delim, Acc,TmpAcc, AT,IsNorm) + Ch = string_to_char_set(S#xmerl_scanner.encoding, ExpRef), + scan_att_chars(Ch ++ T1, S1, Delim, Acc, TmpAcc, AT, IsNorm) end end; scan_att_chars("<" ++ _T, S0, _Delim, _Acc,_, _,_) -> % Tags not allowed here @@ -3964,7 +3971,7 @@ normalize(T,S,IsNorm) -> {_,T,S} -> {T,S,IsNorm}; {_,T1,S1} -> - {T1,S1,true} + normalize(T1,S1,true) end. diff --git a/lib/xmerl/vsn.mk b/lib/xmerl/vsn.mk index b6486681c2..31ffa6e749 100644 --- a/lib/xmerl/vsn.mk +++ b/lib/xmerl/vsn.mk @@ -1 +1 @@ -XMERL_VSN = 1.3.19 +XMERL_VSN = 1.3.20 diff --git a/make/otp_patch_solve_forward_merge_version b/make/otp_patch_solve_forward_merge_version index 7f8f011eb7..45a4fb75db 100644 --- a/make/otp_patch_solve_forward_merge_version +++ b/make/otp_patch_solve_forward_merge_version @@ -1 +1 @@ -7 +8 diff --git a/otp_versions.table b/otp_versions.table index a31759a8ef..37b5061519 100644 --- a/otp_versions.table +++ b/otp_versions.table @@ -1,3 +1,5 @@ +OTP-21.3.3 : erts-10.3.2 kernel-6.3.1 stdlib-3.8.1 # asn1-5.0.8 common_test-1.17 compiler-7.3.2 crypto-4.4.1 debugger-4.2.6 dialyzer-3.3.2 diameter-2.2 edoc-0.10 eldap-1.2.6 erl_docgen-0.9 erl_interface-3.11.1 et-1.6.4 eunit-2.3.7 ftp-1.0.2 hipe-3.18.3 inets-7.0.6 jinterface-1.9.1 megaco-3.18.4 mnesia-4.15.6 observer-2.9 odbc-2.12.3 os_mon-2.4.7 otp_mibs-1.2.1 parsetools-2.1.8 public_key-1.6.5 reltool-0.7.8 runtime_tools-1.13.2 sasl-3.3 snmp-5.2.12 ssh-4.7.4 ssl-9.2.1 syntax_tools-2.1.7 tftp-1.0.1 tools-3.1 wx-1.8.7 xmerl-1.3.20 : +OTP-21.3.2 : erts-10.3.1 xmerl-1.3.20 # asn1-5.0.8 common_test-1.17 compiler-7.3.2 crypto-4.4.1 debugger-4.2.6 dialyzer-3.3.2 diameter-2.2 edoc-0.10 eldap-1.2.6 erl_docgen-0.9 erl_interface-3.11.1 et-1.6.4 eunit-2.3.7 ftp-1.0.2 hipe-3.18.3 inets-7.0.6 jinterface-1.9.1 kernel-6.3 megaco-3.18.4 mnesia-4.15.6 observer-2.9 odbc-2.12.3 os_mon-2.4.7 otp_mibs-1.2.1 parsetools-2.1.8 public_key-1.6.5 reltool-0.7.8 runtime_tools-1.13.2 sasl-3.3 snmp-5.2.12 ssh-4.7.4 ssl-9.2.1 stdlib-3.8 syntax_tools-2.1.7 tftp-1.0.1 tools-3.1 wx-1.8.7 : OTP-21.3.1 : erl_interface-3.11.1 ssl-9.2.1 # asn1-5.0.8 common_test-1.17 compiler-7.3.2 crypto-4.4.1 debugger-4.2.6 dialyzer-3.3.2 diameter-2.2 edoc-0.10 eldap-1.2.6 erl_docgen-0.9 erts-10.3 et-1.6.4 eunit-2.3.7 ftp-1.0.2 hipe-3.18.3 inets-7.0.6 jinterface-1.9.1 kernel-6.3 megaco-3.18.4 mnesia-4.15.6 observer-2.9 odbc-2.12.3 os_mon-2.4.7 otp_mibs-1.2.1 parsetools-2.1.8 public_key-1.6.5 reltool-0.7.8 runtime_tools-1.13.2 sasl-3.3 snmp-5.2.12 ssh-4.7.4 stdlib-3.8 syntax_tools-2.1.7 tftp-1.0.1 tools-3.1 wx-1.8.7 xmerl-1.3.19 : OTP-21.3 : common_test-1.17 compiler-7.3.2 crypto-4.4.1 dialyzer-3.3.2 diameter-2.2 edoc-0.10 erl_docgen-0.9 erl_interface-3.11 erts-10.3 ftp-1.0.2 hipe-3.18.3 inets-7.0.6 kernel-6.3 mnesia-4.15.6 observer-2.9 odbc-2.12.3 public_key-1.6.5 runtime_tools-1.13.2 ssh-4.7.4 ssl-9.2 stdlib-3.8 syntax_tools-2.1.7 tools-3.1 wx-1.8.7 # asn1-5.0.8 debugger-4.2.6 eldap-1.2.6 et-1.6.4 eunit-2.3.7 jinterface-1.9.1 megaco-3.18.4 os_mon-2.4.7 otp_mibs-1.2.1 parsetools-2.1.8 reltool-0.7.8 sasl-3.3 snmp-5.2.12 tftp-1.0.1 xmerl-1.3.19 : OTP-21.2.7 : erts-10.2.5 kernel-6.2.1 # asn1-5.0.8 common_test-1.16.1 compiler-7.3.1 crypto-4.4 debugger-4.2.6 dialyzer-3.3.1 diameter-2.1.6 edoc-0.9.4 eldap-1.2.6 erl_docgen-0.8.1 erl_interface-3.10.4 et-1.6.4 eunit-2.3.7 ftp-1.0.1 hipe-3.18.2 inets-7.0.5 jinterface-1.9.1 megaco-3.18.4 mnesia-4.15.5 observer-2.8.2 odbc-2.12.2 os_mon-2.4.7 otp_mibs-1.2.1 parsetools-2.1.8 public_key-1.6.4 reltool-0.7.8 runtime_tools-1.13.1 sasl-3.3 snmp-5.2.12 ssh-4.7.3 ssl-9.1.2 stdlib-3.7.1 syntax_tools-2.1.6 tftp-1.0.1 tools-3.0.2 wx-1.8.6 xmerl-1.3.19 : diff --git a/system/doc/design_principles/statem.xml b/system/doc/design_principles/statem.xml index 45ea972ff2..23e9054547 100644 --- a/system/doc/design_principles/statem.xml +++ b/system/doc/design_principles/statem.xml @@ -759,7 +759,7 @@ StateName(EventType, EventContent, Data) -> <p> Since the <em>state enter call</em> is not an event there are restrictions on the allowed return value and - <seealso marker="#State Transition Actions">State Transition Actions</seealso>. + <seealso marker="#Transition Actions">State Transition Actions</seealso>. You may not change the state, <seealso marker="#Postponing Events">postpone</seealso> this non-event, or diff --git a/system/doc/reference_manual/expressions.xml b/system/doc/reference_manual/expressions.xml index 8c47070890..ea3b2159fc 100644 --- a/system/doc/reference_manual/expressions.xml +++ b/system/doc/reference_manual/expressions.xml @@ -1221,10 +1221,10 @@ Ei = Value | <<1,17,0,42>> 10> <input>H.</input> <<17,0,42>> -11> <input><<G,H/bitstring>> = <<1,17,42:12>>.</input> -<<1,17,1,10:4>> -12> <input>H.</input> -<<17,1,10:4>> +11> <input><<G,J/bitstring>> = <<1,17,42:12>>.</input> +<<1,17,2,10:4>> +12> <input>J.</input> +<<17,2,10:4>> 13> <input><<1024/utf8>>.</input> <<208,128>> </pre> |