diff options
88 files changed, 2384 insertions, 818 deletions
diff --git a/bootstrap/lib/compiler/ebin/beam_disasm.beam b/bootstrap/lib/compiler/ebin/beam_disasm.beam Binary files differindex 3fe71cfe1b..eb2a13d620 100644 --- a/bootstrap/lib/compiler/ebin/beam_disasm.beam +++ b/bootstrap/lib/compiler/ebin/beam_disasm.beam diff --git a/bootstrap/lib/compiler/ebin/beam_ssa_pre_codegen.beam b/bootstrap/lib/compiler/ebin/beam_ssa_pre_codegen.beam Binary files differindex 329f31f546..f7c9c34c9a 100644 --- a/bootstrap/lib/compiler/ebin/beam_ssa_pre_codegen.beam +++ b/bootstrap/lib/compiler/ebin/beam_ssa_pre_codegen.beam diff --git a/bootstrap/lib/compiler/ebin/beam_validator.beam b/bootstrap/lib/compiler/ebin/beam_validator.beam Binary files differindex 07daf4ed81..79d64b800b 100644 --- a/bootstrap/lib/compiler/ebin/beam_validator.beam +++ b/bootstrap/lib/compiler/ebin/beam_validator.beam diff --git a/bootstrap/lib/kernel/ebin/disk_log_1.beam b/bootstrap/lib/kernel/ebin/disk_log_1.beam Binary files differindex 36beba1f57..17f65d6ceb 100644 --- a/bootstrap/lib/kernel/ebin/disk_log_1.beam +++ b/bootstrap/lib/kernel/ebin/disk_log_1.beam diff --git a/bootstrap/lib/kernel/ebin/erl_distribution.beam b/bootstrap/lib/kernel/ebin/erl_distribution.beam Binary files differindex 6a096155d6..4490f9c81f 100644 --- a/bootstrap/lib/kernel/ebin/erl_distribution.beam +++ b/bootstrap/lib/kernel/ebin/erl_distribution.beam diff --git a/bootstrap/lib/kernel/ebin/file.beam b/bootstrap/lib/kernel/ebin/file.beam Binary files differindex 9c2759adc9..a7a82a1f12 100644 --- a/bootstrap/lib/kernel/ebin/file.beam +++ b/bootstrap/lib/kernel/ebin/file.beam diff --git a/bootstrap/lib/kernel/ebin/user_drv.beam b/bootstrap/lib/kernel/ebin/user_drv.beam Binary files differindex ce4d4328de..1b59423b71 100644 --- a/bootstrap/lib/kernel/ebin/user_drv.beam +++ b/bootstrap/lib/kernel/ebin/user_drv.beam diff --git a/bootstrap/lib/stdlib/ebin/beam_lib.beam b/bootstrap/lib/stdlib/ebin/beam_lib.beam Binary files differindex 44d783a6d0..a96ccb6f8c 100644 --- a/bootstrap/lib/stdlib/ebin/beam_lib.beam +++ b/bootstrap/lib/stdlib/ebin/beam_lib.beam diff --git a/bootstrap/lib/stdlib/ebin/dets_utils.beam b/bootstrap/lib/stdlib/ebin/dets_utils.beam Binary files differindex 25bb217db1..e38eb384a1 100644 --- a/bootstrap/lib/stdlib/ebin/dets_utils.beam +++ b/bootstrap/lib/stdlib/ebin/dets_utils.beam diff --git a/bootstrap/lib/stdlib/ebin/dets_v9.beam b/bootstrap/lib/stdlib/ebin/dets_v9.beam Binary files differindex 1532c51f8e..046de3bd5a 100644 --- a/bootstrap/lib/stdlib/ebin/dets_v9.beam +++ b/bootstrap/lib/stdlib/ebin/dets_v9.beam diff --git a/bootstrap/lib/stdlib/ebin/epp.beam b/bootstrap/lib/stdlib/ebin/epp.beam Binary files differindex a507106263..e3d8c21edf 100644 --- a/bootstrap/lib/stdlib/ebin/epp.beam +++ b/bootstrap/lib/stdlib/ebin/epp.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_pp.beam b/bootstrap/lib/stdlib/ebin/erl_pp.beam Binary files differindex be2fe2c4c0..1965b9e656 100644 --- a/bootstrap/lib/stdlib/ebin/erl_pp.beam +++ b/bootstrap/lib/stdlib/ebin/erl_pp.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_tar.beam b/bootstrap/lib/stdlib/ebin/erl_tar.beam Binary files differindex 526d151a12..3c5004871d 100644 --- a/bootstrap/lib/stdlib/ebin/erl_tar.beam +++ b/bootstrap/lib/stdlib/ebin/erl_tar.beam diff --git a/bootstrap/lib/stdlib/ebin/filename.beam b/bootstrap/lib/stdlib/ebin/filename.beam Binary files differindex 2b76d5bce8..d353a11f3e 100644 --- a/bootstrap/lib/stdlib/ebin/filename.beam +++ b/bootstrap/lib/stdlib/ebin/filename.beam diff --git a/bootstrap/lib/stdlib/ebin/re.beam b/bootstrap/lib/stdlib/ebin/re.beam Binary files differindex d6920d54ce..5662594d4c 100644 --- a/bootstrap/lib/stdlib/ebin/re.beam +++ b/bootstrap/lib/stdlib/ebin/re.beam diff --git a/bootstrap/lib/stdlib/ebin/string.beam b/bootstrap/lib/stdlib/ebin/string.beam Binary files differindex 437f63357c..e6d68ea84e 100644 --- a/bootstrap/lib/stdlib/ebin/string.beam +++ b/bootstrap/lib/stdlib/ebin/string.beam diff --git a/bootstrap/lib/stdlib/ebin/unicode_util.beam b/bootstrap/lib/stdlib/ebin/unicode_util.beam Binary files differindex c9db0483ca..c0523f2e52 100644 --- a/bootstrap/lib/stdlib/ebin/unicode_util.beam +++ b/bootstrap/lib/stdlib/ebin/unicode_util.beam diff --git a/bootstrap/lib/stdlib/ebin/uri_string.beam b/bootstrap/lib/stdlib/ebin/uri_string.beam Binary files differindex 001118f0b9..b57c84a302 100644 --- a/bootstrap/lib/stdlib/ebin/uri_string.beam +++ b/bootstrap/lib/stdlib/ebin/uri_string.beam diff --git a/erts/configure.in b/erts/configure.in index 5f969a0a8b..506ce0d0fb 100644 --- a/erts/configure.in +++ b/erts/configure.in @@ -2,7 +2,7 @@ dnl Process this file with autoconf to produce a configure script. -*-m4-*- dnl %CopyrightBegin% dnl -dnl Copyright Ericsson AB 1997-2018. All Rights Reserved. +dnl Copyright Ericsson AB 1997-2019. All Rights Reserved. dnl dnl Licensed under the Apache License, Version 2.0 (the "License"); dnl you may not use this file except in compliance with the License. @@ -590,6 +590,22 @@ AC_SUBST(WERRORFLAGS) ## Check if we can do profile guided optimization of beam_emu LM_CHECK_ENABLE_CFLAG([-fprofile-generate -Werror],[PROFILE_GENERATE]) LM_CHECK_ENABLE_CFLAG([-fprofile-use -Werror],[PROFILE_USE]) +LM_CHECK_ENABLE_CFLAG([-fprofile-use -fprofile-correction -Werror],[PROFILE_CORRECTION]) + +if test "X$PROFILE_CORRECTION" = "Xtrue"; then + saved_CFLAGS=$CFLAGS + saved_LDFLAGS=$LDFLAGS + CFLAGS="-fprofile-generate $saved_CFLAGS" + LDFLAGS="-fprofile-generate $saved_LDFLAGS" + AC_MSG_CHECKING([whether $CC links with -fprofile-generate]) + AC_LINK_IFELSE([AC_LANG_PROGRAM([],[return 0;])], + [AC_MSG_RESULT([yes]) + PROFILE_GENERATE=true], + [AC_MSG_RESULT([no]) + PROFILE_GENERATE=false]) + CFLAGS=$saved_CFLAGS + LDFLAGS=$saved_LDFLAGS +fi ## Check if this is clang LM_CHECK_ENABLE_CFLAG([-fprofile-instr-generate -Werror],[PROFILE_INSTR_GENERATE]) @@ -614,8 +630,8 @@ if test "X$PROFILE_INSTR_GENERATE" = "Xtrue"; then if test "X$LLVM_PROFDATA" != "X"; then CFLAGS="-fprofile-instr-use=default.profdata -Werror $saved_CFLAGS"; $LLVM_PROFDATA merge -output=default.profdata *.profraw; - AC_MSG_CHECKING([whether gcc accepts -fprofile-instr-use=default.profdata -Werror]) - AC_COMPILE_IFELSE([], + AC_MSG_CHECKING([whether $CC accepts -fprofile-instr-use=default.profdata -Werror]) + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([],[return 0;])], [AC_MSG_RESULT([yes]) PROFILE_INSTR_USE=true], [AC_MSG_RESULT([no]) @@ -637,8 +653,6 @@ AS_HELP_STRING([--enable-pgo], esac ],enable_pgo=default) -LM_CHECK_ENABLE_CFLAG([-fprofile-use -fprofile-correction -Werror],[PROFILE_CORRECTION]) - USE_PGO=false AC_MSG_CHECKING([whether to do PGO of erts]) if test $enable_pgo = no; then @@ -1329,6 +1343,28 @@ LIBS=$zlib_save_LIBS fi AC_SUBST(Z_LIB) + +dnl ------------- +dnl esock +dnl ------------- + +AC_ARG_ENABLE(esock, +AS_HELP_STRING([--enable-esock], [enable builtin experimental socket (as a nif) support (default)]) +AS_HELP_STRING([--disable-esock], [disable builtin experimental socket (as a nif) support])) + +dnl Default value +USE_ESOCK=yes + +if test "x$enable_esock" = "xyes"; then + USE_ESOCK=yes +else + if test "x$enable_esock" = "xno"; then + USE_ESOCK=no + fi +fi +AC_SUBST(USE_ESOCK) + + dnl dnl This test kindly borrowed from Tcl dnl diff --git a/erts/doc/src/erl.xml b/erts/doc/src/erl.xml index 471d7caa5a..ed1b0880b4 100644 --- a/erts/doc/src/erl.xml +++ b/erts/doc/src/erl.xml @@ -460,7 +460,10 @@ <tag><c><![CDATA[-remsh Node]]></c></tag> <item> <p>Starts Erlang with a remote shell connected to - <c><![CDATA[Node]]></c>.</p> + <c><![CDATA[Node]]></c>. Requires either <c><![CDATA[-name]]></c> + or <c><![CDATA[-sname]]></c> to be given. If <c><![CDATA[Node]]></c> + does not contain a hostname, one is automatically taken from + <c><![CDATA[-name]]></c> or <c><![CDATA[-sname]]></c></p> </item> <tag><c><![CDATA[-rsh Program]]></c></tag> <item> diff --git a/erts/doc/src/erl_dist_protocol.xml b/erts/doc/src/erl_dist_protocol.xml index 185c75fe84..f924c8a70b 100644 --- a/erts/doc/src/erl_dist_protocol.xml +++ b/erts/doc/src/erl_dist_protocol.xml @@ -850,10 +850,15 @@ DiB == gen_digest(ChA, ICA)? <tag><c>-define(DFLAG_EXIT_PAYLOAD, 16#400000).</c></tag> <item> <p>Use the <c>PAYLOAD_EXIT</c>, <c>PAYLOAD_EXIT_TT</c>, - <c>PAYLOAD_EXIT2</c>, <c>PAYLOAD_EXIT2_TT</c> - and <c>PAYLOAD_MONITOR_P_EXIT</c> - <seealso marker="#control_message">control message</seealso>s - instead of the non-PAYLOAD variants.</p> + <c>PAYLOAD_EXIT2</c>, <c>PAYLOAD_EXIT2_TT</c> + and <c>PAYLOAD_MONITOR_P_EXIT</c> + <seealso marker="#control_message">control message</seealso>s + instead of the non-PAYLOAD variants.</p> + </item> + <tag><c>-define(DFLAG_FRAGMENTS, 16#800000).</c></tag> + <item> + <p>Use <seealso marker="erl_ext_dist#fragments">fragmented</seealso> + distribution messages to send large messages.</p> </item> </taglist> <p> diff --git a/erts/doc/src/erl_ext_dist.xml b/erts/doc/src/erl_ext_dist.xml index a6bc44b8c8..3730f0e8ac 100644 --- a/erts/doc/src/erl_ext_dist.xml +++ b/erts/doc/src/erl_ext_dist.xml @@ -140,162 +140,366 @@ <marker id="distribution_header"/> <title>Distribution Header</title> <p> - The distribution header only contains an atom cache - reference section, but can in the future contain more - information. The distribution header precedes one or more Erlang - terms on the external format. For more information, see the - documentation of the + The distribution header is sent by the erlang distribution to + carry metadata about the coming + <seealso marker="erl_dist_protocol#control_message">control message</seealso> + and potential payload. It is primarily used to handle the atom cache + in the Erlang distribution. Since OTP-22 it is also used to fragment + large distribution messages into multiple smaller fragments. + For more information about how the distribution uses the distribution header, + see the documentation of the <seealso marker="erl_dist_protocol#connected_nodes">protocol between connected nodes</seealso> in the <seealso marker="erl_dist_protocol">distribution protocol</seealso> documentation. </p> <p> - <seealso marker="#ATOM_CACHE_REF">ATOM_CACHE_REF</seealso> + Any <seealso marker="#ATOM_CACHE_REF">ATOM_CACHE_REF</seealso> entries with corresponding <c>AtomCacheReferenceIndex</c> in terms encoded on the external format following a distribution header refer to the atom cache references made in the distribution header. The range is 0 <= <c>AtomCacheReferenceIndex</c> < 255, that is, at most 255 different atom cache references from the following terms can be made. </p> - <p> - The distribution header format is as follows: - </p> - <table align="left"> - <row> - <cell align="center">1</cell> - <cell align="center">1</cell> - <cell align="center">1</cell> - <cell align="center">NumberOfAtomCacheRefs/2+1 | 0</cell> - <cell align="center">N | 0</cell> - </row> - <row> - <cell align="center"><c>131</c></cell> - <cell align="center"><c>68</c></cell> - <cell align="center"><c>NumberOfAtomCacheRefs</c></cell> - <cell align="center"><c>Flags</c></cell> - <cell align="center"><c>AtomCacheRefs</c></cell> - </row> - <tcaption>Distribution Header Format</tcaption></table> - <p> - <c>Flags</c> consist of <c>NumberOfAtomCacheRefs/2+1</c> bytes, - unless <c>NumberOfAtomCacheRefs</c> is <c>0</c>. If - <c>NumberOfAtomCacheRefs</c> is <c>0</c>, <c>Flags</c> and - <c>AtomCacheRefs</c> are omitted. Each atom cache reference has - a half byte flag field. Flags corresponding to a specific - <c>AtomCacheReferenceIndex</c> are located in flag byte number - <c>AtomCacheReferenceIndex/2</c>. Flag byte 0 is the first byte - after the <c>NumberOfAtomCacheRefs</c> byte. Flags for an even - <c>AtomCacheReferenceIndex</c> are located in the least significant - half byte and flags for an odd <c>AtomCacheReferenceIndex</c> are - located in the most significant half byte. - </p> - <p> - The flag field of an atom cache reference has the following - format: - </p> - <table align="left"> - <row> - <cell align="center">1 bit</cell> - <cell align="center">3 bits</cell> - </row> - <row> - <cell align="center"><c>NewCacheEntryFlag</c></cell> - <cell align="center"><c>SegmentIndex</c></cell> - </row> - <tcaption></tcaption></table> - <p> - The most significant bit is the <c>NewCacheEntryFlag</c>. If set, - the corresponding cache reference is new. The three least - significant bits are the <c>SegmentIndex</c> of the corresponding - atom cache entry. An atom cache consists of 8 segments, each of size - 256, that is, an atom cache can contain 2048 entries. - </p> - <p> - After flag fields for atom cache references, another half byte flag - field is located with the following format: - </p> - <table align="left"> - <row> - <cell align="center">3 bits</cell> - <cell align="center">1 bit</cell> - </row> - <row> - <cell align="center"><c>CurrentlyUnused</c></cell> - <cell align="center"><c>LongAtoms</c></cell> - </row> - <tcaption></tcaption></table> - <p> - The least significant bit in that half byte is flag <c>LongAtoms</c>. - If it is set, 2 bytes are used for atom lengths instead of - 1 byte in the distribution header. - </p> - <p> - After the <c>Flags</c> field follow the <c>AtomCacheRefs</c>. The - first <c>AtomCacheRef</c> is the one corresponding to - <c>AtomCacheReferenceIndex</c> 0. Higher indices follow - in sequence up to index <c>NumberOfAtomCacheRefs - 1</c>. - </p> - <p> - If the <c>NewCacheEntryFlag</c> for the next <c>AtomCacheRef</c> has - been set, a <c>NewAtomCacheRef</c> on the following format follows: - </p> - <table align="left"> - <row> - <cell align="center">1</cell> - <cell align="center">1 | 2</cell> - <cell align="center">Length</cell> - </row> - <row> - <cell align="center"><c>InternalSegmentIndex</c></cell> - <cell align="center"><c>Length</c></cell> - <cell align="center"><c>AtomText</c></cell> - </row> - <tcaption></tcaption></table> - <p> - <c>InternalSegmentIndex</c> together with the <c>SegmentIndex</c> - completely identify the location of an atom cache entry in the - atom cache. <c>Length</c> is the number of bytes that <c>AtomText</c> - consists of. Length is a 2 byte big-endian integer - if flag <c>LongAtoms</c> has been set, otherwise a 1 byte - integer. When distribution flag - <seealso marker="erl_dist_protocol#dflags"> - <c>DFLAG_UTF8_ATOMS</c></seealso> - has been exchanged between both nodes in the - <seealso marker="erl_dist_protocol#distribution_handshake"> - distribution handshake</seealso>, - characters in <c>AtomText</c> are encoded in UTF-8, otherwise - in Latin-1. The following <c>CachedAtomRef</c>s with the same - <c>SegmentIndex</c> and <c>InternalSegmentIndex</c> as this - <c>NewAtomCacheRef</c> refer to this atom until a new - <c>NewAtomCacheRef</c> with the same <c>SegmentIndex</c> - and <c>InternalSegmentIndex</c> appear. - </p> - <p> - For more information on encoding of atoms, see the - <seealso marker="#utf8_atoms">note on UTF-8 encoded atoms</seealso> - in the beginning of this section. - </p> - <p> - If the <c>NewCacheEntryFlag</c> for the next <c>AtomCacheRef</c> - has not been set, a <c>CachedAtomRef</c> on the following format - follows: - </p> - <table align="left"> - <row> - <cell align="center">1</cell> - </row> - <row> - <cell align="center"><c>InternalSegmentIndex</c></cell> - </row> - <tcaption></tcaption></table> - <p> - <c>InternalSegmentIndex</c> together with the <c>SegmentIndex</c> - identify the location of the atom cache entry in the atom cache. - The atom corresponding to this <c>CachedAtomRef</c> is the - latest <c>NewAtomCacheRef</c> preceding this <c>CachedAtomRef</c> - in another previously passed distribution header. - </p> + <section> + <title>Normal Distribution Header</title> + <p> + The non-fragmented distribution header format is as follows: + </p> + <table align="left"> + <row> + <cell align="center">1</cell> + <cell align="center">1</cell> + <cell align="center">1</cell> + <cell align="center">NumberOfAtomCacheRefs/2+1 | 0</cell> + <cell align="center">N | 0</cell> + </row> + <row> + <cell align="center"><c>131</c></cell> + <cell align="center"><c>68</c></cell> + <cell align="center"><c>NumberOfAtomCacheRefs</c></cell> + <cell align="center"><c>Flags</c></cell> + <cell align="center"><c>AtomCacheRefs</c></cell> + </row> + <tcaption>Normal Distribution Header Format</tcaption></table> + <p> + <c>Flags</c> consist of <c>NumberOfAtomCacheRefs/2+1</c> bytes, + unless <c>NumberOfAtomCacheRefs</c> is <c>0</c>. If + <c>NumberOfAtomCacheRefs</c> is <c>0</c>, <c>Flags</c> and + <c>AtomCacheRefs</c> are omitted. Each atom cache reference has + a half byte flag field. Flags corresponding to a specific + <c>AtomCacheReferenceIndex</c> are located in flag byte number + <c>AtomCacheReferenceIndex/2</c>. Flag byte 0 is the first byte + after the <c>NumberOfAtomCacheRefs</c> byte. Flags for an even + <c>AtomCacheReferenceIndex</c> are located in the least significant + half byte and flags for an odd <c>AtomCacheReferenceIndex</c> are + located in the most significant half byte. + </p> + <p> + The flag field of an atom cache reference has the following + format: + </p> + <table align="left"> + <row> + <cell align="center">1 bit</cell> + <cell align="center">3 bits</cell> + </row> + <row> + <cell align="center"><c>NewCacheEntryFlag</c></cell> + <cell align="center"><c>SegmentIndex</c></cell> + </row> + <tcaption></tcaption></table> + <p> + The most significant bit is the <c>NewCacheEntryFlag</c>. If set, + the corresponding cache reference is new. The three least + significant bits are the <c>SegmentIndex</c> of the corresponding + atom cache entry. An atom cache consists of 8 segments, each of size + 256, that is, an atom cache can contain 2048 entries. + </p> + <p> + After flag fields for atom cache references, another half byte flag + field is located with the following format: + </p> + <table align="left"> + <row> + <cell align="center">3 bits</cell> + <cell align="center">1 bit</cell> + </row> + <row> + <cell align="center"><c>CurrentlyUnused</c></cell> + <cell align="center"><c>LongAtoms</c></cell> + </row> + <tcaption></tcaption></table> + <p> + The least significant bit in that half byte is flag <c>LongAtoms</c>. + If it is set, 2 bytes are used for atom lengths instead of + 1 byte in the distribution header. + </p> + <p> + After the <c>Flags</c> field follow the <c>AtomCacheRefs</c>. The + first <c>AtomCacheRef</c> is the one corresponding to + <c>AtomCacheReferenceIndex</c> 0. Higher indices follow + in sequence up to index <c>NumberOfAtomCacheRefs - 1</c>. + </p> + <p> + If the <c>NewCacheEntryFlag</c> for the next <c>AtomCacheRef</c> has + been set, a <c>NewAtomCacheRef</c> on the following format follows: + </p> + <table align="left"> + <row> + <cell align="center">1</cell> + <cell align="center">1 | 2</cell> + <cell align="center">Length</cell> + </row> + <row> + <cell align="center"><c>InternalSegmentIndex</c></cell> + <cell align="center"><c>Length</c></cell> + <cell align="center"><c>AtomText</c></cell> + </row> + <tcaption></tcaption></table> + <p> + <c>InternalSegmentIndex</c> together with the <c>SegmentIndex</c> + completely identify the location of an atom cache entry in the + atom cache. <c>Length</c> is the number of bytes that <c>AtomText</c> + consists of. Length is a 2 byte big-endian integer + if flag <c>LongAtoms</c> has been set, otherwise a 1 byte + integer. When distribution flag + <seealso marker="erl_dist_protocol#dflags"> + <c>DFLAG_UTF8_ATOMS</c></seealso> + has been exchanged between both nodes in the + <seealso marker="erl_dist_protocol#distribution_handshake"> + distribution handshake</seealso>, + characters in <c>AtomText</c> are encoded in UTF-8, otherwise + in Latin-1. The following <c>CachedAtomRef</c>s with the same + <c>SegmentIndex</c> and <c>InternalSegmentIndex</c> as this + <c>NewAtomCacheRef</c> refer to this atom until a new + <c>NewAtomCacheRef</c> with the same <c>SegmentIndex</c> + and <c>InternalSegmentIndex</c> appear. + </p> + <p> + For more information on encoding of atoms, see the + <seealso marker="#utf8_atoms">note on UTF-8 encoded atoms</seealso> + in the beginning of this section. + </p> + <p> + If the <c>NewCacheEntryFlag</c> for the next <c>AtomCacheRef</c> + has not been set, a <c>CachedAtomRef</c> on the following format + follows: + </p> + <table align="left"> + <row> + <cell align="center">1</cell> + </row> + <row> + <cell align="center"><c>InternalSegmentIndex</c></cell> + </row> + <tcaption></tcaption></table> + <p> + <c>InternalSegmentIndex</c> together with the <c>SegmentIndex</c> + identify the location of the atom cache entry in the atom cache. + The atom corresponding to this <c>CachedAtomRef</c> is the + latest <c>NewAtomCacheRef</c> preceding this <c>CachedAtomRef</c> + in another previously passed distribution header. + </p> + </section> + <section> + <marker id="fragments"/> + <title>Distribution Header for fragmented messages</title> + <p>Messages sent between Erlang nodes can sometimes be + quite large. Since OTP-22 it is possible to split large messages + into smaller fragments in order to allow smaller messages to be interleaved + between larges messages. It is only the <c>message</c> part of each + <seealso marker="erl_dist_protocol#connected_nodes">distributed message</seealso> + that may be split using fragmentation. Therefore it is recommended to use the + <seealso marker="erl_dist_protocol#new-ctrlmessages-for-erlang-otp-22"> + PAYLOAD control messages</seealso> introduced in OTP-22. + </p> + <p>Fragmented distribution messages are only used if the receiving node + signals that it supports them via the + <seealso marker="erl_dist_protocol#dflags">DFLAG_FRAGMENTS</seealso> distribution + flag.</p> + <p>A process must complete the sending of a fragmented message before it + can start sending any other message on the same distribution channel.</p> + + <p>The start of a sequence of fragmented messages looks like this:</p> + <table align="left"> + <row> + <cell align="center">1</cell> + <cell align="center">1</cell> + <cell align="center">8</cell> + <cell align="center">8</cell> + <cell align="center">1</cell> + <cell align="center">NumberOfAtomCacheRefs/2+1 | 0</cell> + <cell align="center">N | 0</cell> + </row> + <row> + <cell align="center"><c>131</c></cell> + <cell align="center"><c>69</c></cell> + <cell align="center"><c>SequenceId</c></cell> + <cell align="center"><c>FragmentId</c></cell> + <cell align="center"><c>NumberOfAtomCacheRefs</c></cell> + <cell align="center"><c>Flags</c></cell> + <cell align="center"><c>AtomCacheRefs</c></cell> + </row> + <tcaption>Starting Fragmented Distribution Header Format</tcaption> + </table> + + <p>The continuation of a sequence of fragmented messages looks like this:</p> + <table align="left"> + <row> + <cell align="center">1</cell> + <cell align="center">1</cell> + <cell align="center">8</cell> + <cell align="center">8</cell> + </row> + <row> + <cell align="center"><c>131</c></cell> + <cell align="center"><c>70</c></cell> + <cell align="center"><c>SequenceId</c></cell> + <cell align="center"><c>FragmentId</c></cell> + </row> + <tcaption>Continuing Fragmented Distribution Header Format</tcaption> + </table> + + <p> + The starting distribution header is very similar to a non-fragmented distribution + header. The atom cache works the same as for normal distribution header and + is the same for the entire sequence. The additional fields added are the + sequence id and fragment id. + </p> + + <taglist> + <tag>Sequence ID</tag> + <item> + <p> + The sequence id is used to uniquely identify a fragmented message sent + from one process to another on the same distributed connection. This is used + to identify which sequence a fragment is a part of as the same process can + be in the process of receiving multiple sequences at the same time. + </p> + <p> + As one process can only be sending one fragmented message at once, + it can be convenient to use the local PID as the sequence id. + </p> + </item> + <tag>Fragments ID</tag> + <item> + <p> + The Fragment ID is used to number the fragments in a sequence. + The id starts at the total number of fragments and then decrements to 1 + (which is the final fragment). So if a sequence consists of 3 fragments + the fragment id in the starting header will be 3, and then fragments 2 and 1 + are sent. + </p> + <p> + The fragments must be delivered in the correct order, so if an unordered + distribution carrier is used, they must be ordered before delivered to the + Erlang run-time. + </p> + </item> + </taglist> + + <section> + <title>Example:</title> + <p> + As an example, let say that we want to send + <c>{call, <0.245.2>, {set_get_state, <<0:1024>>}}</c> to + registered process <c>reg</c> using a fragment size of 128. To send + this message we need a distribution header, atom cache updates, + the control message (which would be <c>{6, <0.245.2>, [], reg}</c> in this case) + and finally the actual message. This would all be encoded into: + </p> + + <code> +131,69,0,0,2,168,0,0,5,83,0,0,0,0,0,0,0,2, %% Header with seq and frag id +5,4,137,9,10,5,236,3,114,101,103,9,4,99,97,108,108, %% Atom cache updates +238,13,115,101,116,95,103,101,116,95,115,116,97,116,101, +104,4,97,6,103,82,0,0,0,0,85,0,0,0,0,2,82,1,82,2, %% Control message +104,3,82,3,103,82,0,0,0,0,245,0,0,0,2,2, %% Actual message using cached atoms +104,2,82,4,109,0,0,0,128,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 + +131,70,0,0,2,168,0,0,5,83,0,0,0,0,0,0,0,1, %% Cont Header with seq and frag id +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, %% Rest of payload +0,0,0,0</code> + + <p> + Let us break that apart into its components. First we have the + distribution header tags together with the sequence id and + a fragment id of 2. + </p> + <code> +131,69, %% Start fragment header +0,0,2,168,0,0,5,83, %% The sequence ID +0,0,0,0,0,0,0,2, %% The fragment ID +</code> + <p>Then we have the updates to the atom cache:</p> + <code> +5,4,137,9, %% 5 atoms and their flags +10,5, %% The already cached atom ids +236,3,114,101,103, %% The atom 'reg' +9,4,99,97,108,108, %% The atom 'call' +238,13,115,101,116,95,103,101,116,95,115,116,97,116,101, %% The atom 'set_get_state' + </code> + <p> + The first byte says that we have 5 atoms that are part + of the cache. Then follows three bytes that are the + atom cache ref flags. Each of the flags uses 4 bits so + they are a bit hard to read in decimal byte form. In + binary half-byte form they look like this: + </p> + <code>0000, 0100, 1000, 1001, 1001</code> + <p> + As the high bit of the first two atoms in the + cache are not set we know that they are already in the cache, + so they do not have to be sent again (this is the node name of the + receiving and sending node). Then follows the atoms that have to be sent, + together with their segment ids. + </p> + <p> + Then the listing of the atoms comes, starting with 10 and 5 + which are the atom refs of the already cached atoms. Then the + new atoms are sent. + </p> + <p> + When the atom cache is setup correctly the control message is sent. + </p> + <code>104,4,97,6,103,82,0,0,0,0,85,0,0,0,0,2,82,1,82,2,</code> + <p> + Note that up until here it is not allowed to fragments the message. + The entire atom cache and control message has to be part of the + starting fragment. After the control message the payload of the message + is sent using 128 bytes: + </p> + <code> +104,3,82,3,103,82,0,0,0,0,245,0,0,0,2,2, +104,2,82,4,109,0,0,0,128,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 + </code> + <p> + Since the payload is larger than 128-bytes it is split into two + fragments. The second fragment does not have any atom cache update + instructions so it is a lot simpler: + </p> + <code> +131,70,0,0,2,168,0,0,5,83,0,0,0,0,0,0,0,1, %% Continuation dist header 70 with seq and frag id +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, %% remaining payload +0,0,0,0 + </code> + <note> + <p> + The fragment size of 128 is only used as an example. + Any fragments size may be used when sending fragmented messages. + </p> + </note> + </section> + </section> </section> <section> diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index 448f41b523..a9f3bb8e89 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -633,6 +633,15 @@ GENERATE += $(TTF_DIR)/driver_tab.c # This list must be consistent with PRE_LOADED_MODULES in # erts/preloaded/src/Makefile. +ifeq ($(USE_ESOCK), yes) +ESOCK_PRELOAD_BEAM = \ + $(ERL_TOP)/erts/preloaded/ebin/socket.beam \ + $(ERL_TOP)/erts/preloaded/ebin/net.beam +else +ESOCK_PRELOAD_BEAM = \ + $(ERL_TOP)/erts/preloaded/ebin/net.beam +endif + PRELOAD_BEAM = $(ERL_TOP)/erts/preloaded/ebin/erts_code_purger.beam \ $(ERL_TOP)/erts/preloaded/ebin/erl_init.beam \ $(ERL_TOP)/erts/preloaded/ebin/init.beam \ @@ -641,8 +650,7 @@ PRELOAD_BEAM = $(ERL_TOP)/erts/preloaded/ebin/erts_code_purger.beam \ $(ERL_TOP)/erts/preloaded/ebin/prim_inet.beam \ $(ERL_TOP)/erts/preloaded/ebin/prim_file.beam \ $(ERL_TOP)/erts/preloaded/ebin/zlib.beam \ - $(ERL_TOP)/erts/preloaded/ebin/socket.beam \ - $(ERL_TOP)/erts/preloaded/ebin/net.beam \ + $(ESOCK_PRELOAD_BEAM) \ $(ERL_TOP)/erts/preloaded/ebin/prim_zip.beam \ $(ERL_TOP)/erts/preloaded/ebin/erl_prim_loader.beam \ $(ERL_TOP)/erts/preloaded/ebin/erlang.beam \ @@ -835,6 +843,15 @@ EMU_OBJS = \ $(OBJDIR)/beam_catches.o $(OBJDIR)/code_ix.o \ $(OBJDIR)/beam_ranges.o + +ifeq ($(USE_ESOCK), yes) + +# WE ARE USING ESOCK + +ESOCK_NIF_OBJS = \ + $(OBJDIR)/socket_nif.o \ + $(OBJDIR)/net_nif.o + ifneq ($(TARGET), win32) # These are *currently* only needed for non-win32, # since the nif-functions for socket and net are basically @@ -847,6 +864,16 @@ else ESOCK_RUN_OBJS = endif +else + +# WE ARE *NOT* USING ESOCK + +ESOCK_NIF_OBJS = +ESOCK_RUN_OBJS = + +endif + + RUN_OBJS += \ $(OBJDIR)/erl_alloc.o $(OBJDIR)/erl_mtrace.o \ $(OBJDIR)/erl_alloc_util.o $(OBJDIR)/erl_goodfit_alloc.o \ @@ -903,8 +930,7 @@ NIF_OBJS = \ $(OBJDIR)/prim_buffer_nif.o \ $(OBJDIR)/prim_file_nif.o \ $(OBJDIR)/zlib_nif.o \ - $(OBJDIR)/socket_nif.o \ - $(OBJDIR)/net_nif.o + $(ESOCK_NIF_OBJS) ifeq ($(TARGET),win32) DRV_OBJS = \ diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 8e93e53003..bae64afb97 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -3272,7 +3272,7 @@ erts_current_reductions(Process *c_p, Process *p) } else { reds_left = c_p->fcalls; } - return REDS_IN(c_p) - reds_left; + return REDS_IN(c_p) - reds_left - erts_proc_sched_data(p)->virtual_reds; } int diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index 27599f38e0..ff19ef018e 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -55,7 +55,6 @@ */ #if 0 #define ERTS_DIST_MSG_DBG -FILE *dbg_file; #endif #if 0 /* Enable this to print the dist debug messages to a file instead */ @@ -67,6 +66,7 @@ FILE *dbg_file; #endif #if defined(ERTS_DIST_MSG_DBG) || defined(ERTS_RAW_DIST_MSG_DBG) +FILE *dbg_file; static void bw(byte *buf, ErlDrvSizeT sz) { bin_write(ERTS_PRINT_FILE, dbg_file, buf, sz); @@ -743,7 +743,7 @@ void init_dist(void) sprintf(buff, ERTS_DIST_MSG_DBG_FILE, getpid()); dbg_file = fopen(buff,"w+"); } -#elif defined (ERTS_DIST_MSG_DBG) +#elif defined(ERTS_DIST_MSG_DBG) || defined(ERTS_RAW_DIST_MSG_DBG) dbg_file = stderr; #endif @@ -1844,7 +1844,7 @@ int erts_net_message(Port *prt, if (locks) erts_proc_unlock(rp, locks); - } else if (ede_hfrag) { + } else if (ede_hfrag != NULL) { erts_free_dist_ext_copy(erts_get_dist_ext(ede_hfrag)); free_message_buffer(ede_hfrag); } @@ -1886,16 +1886,18 @@ int erts_net_message(Port *prt, goto invalid_message; } rp = erts_proc_lookup(to); + if (rp) { ErtsProcLocks locks = 0; erts_queue_dist_message(rp, locks, edep, ede_hfrag, token, am_Empty); if (locks) erts_proc_unlock(rp, locks); - } else if (ede_hfrag) { + } else if (ede_hfrag != NULL) { erts_free_dist_ext_copy(erts_get_dist_ext(ede_hfrag)); free_message_buffer(ede_hfrag); } + break; } @@ -1936,15 +1938,19 @@ int erts_net_message(Port *prt, goto invalid_message; } - if (!erts_proc_lookup(watcher)) break; /* Process not alive */ - - if (reason == THE_NON_VALUE) { + if (!erts_proc_lookup(watcher)) { + if (ede_hfrag != NULL) { + erts_free_dist_ext_copy(erts_get_dist_ext(ede_hfrag)); + free_message_buffer(ede_hfrag); + } + break; /* Process not alive */ + } #ifdef ERTS_DIST_MSG_DBG + if (reason == THE_NON_VALUE) { dist_msg_dbg(edep, "MSG", buf, orig_len); -#endif - } +#endif erts_proc_sig_send_dist_monitor_down( dep, ref, watched, watcher, edep, ede_hfrag, reason); @@ -1993,13 +1999,19 @@ int erts_net_message(Port *prt, goto invalid_message; } - if (!erts_proc_lookup(to)) break; /* Process not alive */ + if (!erts_proc_lookup(to)) { + if (ede_hfrag != NULL) { + erts_free_dist_ext_copy(erts_get_dist_ext(ede_hfrag)); + free_message_buffer(ede_hfrag); + } + break; /* Process not alive */ + } - if (reason == THE_NON_VALUE) { #ifdef ERTS_DIST_MSG_DBG + if (reason == THE_NON_VALUE) { dist_msg_dbg(edep, "MSG", buf, orig_len); -#endif } +#endif erts_proc_sig_send_dist_link_exit(dep, from, to, edep, ede_hfrag, @@ -2048,13 +2060,19 @@ int erts_net_message(Port *prt, goto invalid_message; } - if (!erts_proc_lookup(to)) break; /* Process not alive */ + if (!erts_proc_lookup(to)) { + if (ede_hfrag != NULL) { + erts_free_dist_ext_copy(erts_get_dist_ext(ede_hfrag)); + free_message_buffer(ede_hfrag); + } + break; /* Process not alive */ + } - if (reason == THE_NON_VALUE) { #ifdef ERTS_DIST_MSG_DBG + if (reason == THE_NON_VALUE) { dist_msg_dbg(edep, "MSG", buf, orig_len); -#endif } +#endif erts_proc_sig_send_dist_exit(dep, from, to, edep, ede_hfrag, reason, token); break; @@ -3455,6 +3473,8 @@ dist_ctrl_get_data_1(BIF_ALIST_1) obufsize -= size_obuf(obuf); if (reds < 0) { erts_de_runlock(dep); + if (obufsize) + erts_atomic_add_nob(&dep->qsize, (erts_aint_t) -obufsize); ERTS_BIF_YIELD1(bif_export[BIF_dist_ctrl_get_data_1], BIF_P, BIF_ARG_1); } diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index b34916271e..b46b311c59 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -336,7 +336,9 @@ void erts_alcu_sched_spec_data_init(struct ErtsSchedulerData_ *esdp); #if MBC_ABLK_OFFSET_BITS /* The shift is reduced by 1 since the highest bit is used for a flag. */ # define MBC_ABLK_OFFSET_SHIFT (sizeof(UWord)*8 - 1 - MBC_ABLK_OFFSET_BITS) -# define MBC_ABLK_OFFSET_MASK (((1ul << MBC_ABLK_OFFSET_BITS) - 1ul) << MBC_ABLK_OFFSET_SHIFT) +# define MBC_ABLK_OFFSET_MASK \ + (((UWORD_CONSTANT(1) << MBC_ABLK_OFFSET_BITS) - UWORD_CONSTANT(1)) \ + << MBC_ABLK_OFFSET_SHIFT) # define MBC_ABLK_SZ_MASK (~MBC_ABLK_OFFSET_MASK & ~BLK_FLG_MASK) #else # define MBC_ABLK_SZ_MASK (~BLK_FLG_MASK) diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 0339589b79..2704b99aa4 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -768,7 +768,7 @@ static ErtsProcessInfoArgs pi_args[] = { {am_memory, 0, ERTS_PI_FLAG_NEED_MSGQ_LEN|ERTS_PI_FLAG_FORCE_SIG_SEND, ERTS_PROC_LOCK_MAIN}, {am_garbage_collection, 3+2 + 3+2 + 3+2 + 3+2 + 3+2 + ERTS_MAX_HEAP_SIZE_MAP_SZ, 0, ERTS_PROC_LOCK_MAIN}, {am_group_leader, 0, 0, ERTS_PROC_LOCK_MAIN}, - {am_reductions, 0, 0, ERTS_PROC_LOCK_MAIN}, + {am_reductions, 0, ERTS_PI_FLAG_FORCE_SIG_SEND, ERTS_PROC_LOCK_MAIN}, {am_priority, 0, 0, 0}, {am_trace, 0, 0, ERTS_PROC_LOCK_MAIN}, {am_binary, 0, ERTS_PI_FLAG_FORCE_SIG_SEND, ERTS_PROC_LOCK_MAIN}, diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 31c941337f..45fef0c0e5 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -5298,44 +5298,31 @@ erts_get_port_names(Eterm id, ErlDrvPort drv_port) pnp->driver_name = NULL; } else { - int do_realloc = 1; - int len = -1; - size_t pnp_len = sizeof(ErtsPortNames); -#ifndef DEBUG - pnp_len += 100; /* In most cases 100 characters will be enough... */ - ASSERT(prt->common.id == id); -#endif - pnp = erts_alloc(ERTS_ALC_T_PORT_NAMES, pnp_len); - do { - int nlen; - char *name, *driver_name; - if (len > 0) { - erts_free(ERTS_ALC_T_PORT_NAMES, pnp); - pnp_len = sizeof(ErtsPortNames) + len; - pnp = erts_alloc(ERTS_ALC_T_PORT_NAMES, pnp_len); - } - name = prt->name; - len = nlen = name ? sys_strlen(name) + 1 : 0; - driver_name = (prt->drv_ptr ? prt->drv_ptr->name : NULL); - len += driver_name ? sys_strlen(driver_name) + 1 : 0; - if (len <= pnp_len - sizeof(ErtsPortNames)) { - if (!name) - pnp->name = NULL; - else { - pnp->name = ((char *) pnp) + sizeof(ErtsPortNames); - sys_strcpy(pnp->name, name); - } - if (!driver_name) - pnp->driver_name = NULL; - else { - pnp->driver_name = (((char *) pnp) - + sizeof(ErtsPortNames) - + nlen); - sys_strcpy(pnp->driver_name, driver_name); - } - do_realloc = 0; - } - } while (do_realloc); + int len; + int nlen; + char *driver_name; + + len = nlen = prt->name ? sys_strlen(prt->name) + 1 : 0; + driver_name = (prt->drv_ptr ? prt->drv_ptr->name : NULL); + len += driver_name ? sys_strlen(driver_name) + 1 : 0; + + pnp = erts_alloc(ERTS_ALC_T_PORT_NAMES, + sizeof(ErtsPortNames) + len); + + if (!prt->name) + pnp->name = NULL; + else { + pnp->name = ((char *) pnp) + sizeof(ErtsPortNames); + sys_strcpy(pnp->name, prt->name); + } + if (!driver_name) + pnp->driver_name = NULL; + else { + pnp->driver_name = (((char *) pnp) + + sizeof(ErtsPortNames) + + nlen); + sys_strcpy(pnp->driver_name, driver_name); + } } return pnp; } diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c index a3069e419a..9eb020d070 100644 --- a/erts/emulator/beam/time.c +++ b/erts/emulator/beam/time.c @@ -316,7 +316,7 @@ struct ErtsTimerWheel_ { #define ERTS_TW_SLOT_AT_ONCE (-1) #define ERTS_TW_BUMP_LATER_WHEEL(TIW) \ - ((tiw)->pos + ERTS_TW_LATER_WHEEL_SLOT_SIZE >= (TIW)->later.pos) + ((TIW)->pos + ERTS_TW_LATER_WHEEL_SLOT_SIZE >= (TIW)->later.pos) static int bump_later_wheel(ErtsTimerWheel *tiw, int *yield_count_p); @@ -701,7 +701,8 @@ remove_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p) if (slot < ERTS_TW_SOON_WHEEL_END_SLOT) { if (empty_slot && tiw->true_next_timeout_time - && p->timeout_pos == tiw->next_timeout_pos) { + && p->timeout_pos == tiw->next_timeout_pos + && tiw->yield_slot == ERTS_TW_SLOT_INACTIVE) { tiw->true_next_timeout_time = 0; } if (--tiw->soon.nto == 0) @@ -714,7 +715,8 @@ remove_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p) ErtsMonotonicTime tpos = tiw->later.min_tpos; tpos &= ERTS_TW_LATER_WHEEL_POS_MASK; tpos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE; - if (tpos == tiw->next_timeout_pos) + if (tpos == tiw->next_timeout_pos + && tiw->yield_slot == ERTS_TW_SLOT_INACTIVE) tiw->true_next_timeout_time = 0; } if (--tiw->later.nto == 0) { @@ -908,7 +910,6 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) { ErtsMonotonicTime tmp_slots = bump_to - tiw->pos; - tmp_slots = (bump_to - tiw->pos); if (tmp_slots < ERTS_TW_SOON_WHEEL_SIZE) slots = (int) tmp_slots; else diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c index 98be50815c..fb18c837ab 100644 --- a/erts/emulator/sys/common/erl_check_io.c +++ b/erts/emulator/sys/common/erl_check_io.c @@ -534,6 +534,7 @@ erts_io_notify_port_task_executed(ErtsPortTaskType type, if (state->active_events & ERTS_POLL_EV_OUT) oready(state->driver.select->outport, state); state->active_events = 0; + active_events = 0; } } diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c index 9662996039..1b125056f5 100644 --- a/erts/emulator/sys/common/erl_poll.c +++ b/erts/emulator/sys/common/erl_poll.c @@ -924,7 +924,7 @@ update_pollset(ErtsPollSet *ps, int fd, ErtsPollOp op, ErtsPollEvents events) ERTS_EV_SET(&evts[len++], fd, EVFILT_WRITE, flags, (void *) ERTS_POLL_EV_OUT); } #else - uint32_t flags = EV_ADD; + uint32_t flags = EV_ADD|EV_ENABLE; if (ps->oneshot) flags |= EV_ONESHOT; @@ -932,9 +932,27 @@ update_pollset(ErtsPollSet *ps, int fd, ErtsPollOp op, ErtsPollEvents events) erts_atomic_dec_nob(&ps->no_of_user_fds); /* We don't do anything when a delete is issued. The fds will be removed when they are triggered, or when they are closed. */ - events = 0; + if (ps->oneshot) + events = 0; + else { + flags = EV_DELETE; + events = ERTS_POLL_EV_IN; + } } else if (op == ERTS_POLL_OP_ADD) { erts_atomic_inc_nob(&ps->no_of_user_fds); + /* Only allow EV_IN in non-oneshot poll-sets */ + ASSERT(ps->oneshot || events == ERTS_POLL_EV_IN); + } else if (!ps->oneshot) { + ASSERT(op == ERTS_POLL_OP_MOD); + /* If we are not oneshot and do a mod we should disable the FD. + We assume that it is only the read side that is active as + currently only read is selected upon in the non-oneshot + poll-sets. */ + if (!events) + flags = EV_DISABLE; + else + flags = EV_ENABLE; + events = ERTS_POLL_EV_IN; } if (events & ERTS_POLL_EV_IN) { @@ -961,16 +979,15 @@ update_pollset(ErtsPollSet *ps, int fd, ErtsPollOp op, ErtsPollEvents events) for (i = 0; i < len; i++) { const char *flags = "UNKNOWN"; if (evts[i].flags == (EV_DELETE)) flags = "EV_DELETE"; - if (evts[i].flags == (EV_ADD|EV_ONESHOT)) flags = "EV_ADD|EV_ONESHOT"; if (evts[i].flags == (EV_ADD)) flags = "EV_ADD"; + if (evts[i].flags == (EV_ADD|EV_ONESHOT)) flags = "EV_ADD|EV_ONESHOT"; + if (evts[i].flags == (EV_ENABLE)) flags = "EV_ENABLE"; + if (evts[i].flags == (EV_DISABLE)) flags = "EV_DISABLE"; + if (evts[i].flags == (EV_ADD|EV_DISABLE)) flags = "EV_ADD|EV_DISABLE"; #ifdef EV_DISPATCH if (evts[i].flags == (EV_ADD|EV_DISPATCH)) flags = "EV_ADD|EV_DISPATCH"; - if (evts[i].flags == (EV_ADD|EV_DISABLE)) flags = "EV_ADD|EV_DISABLE"; if (evts[i].flags == (EV_ENABLE|EV_DISPATCH)) flags = "EV_ENABLE|EV_DISPATCH"; - if (evts[i].flags == (EV_ENABLE)) flags = "EV_ENABLE"; - if (evts[i].flags == (EV_DISABLE)) flags = "EV_DISABLE"; if (evts[i].flags == (EV_DISABLE|EV_DISPATCH)) flags = "EV_DISABLE|EV_DISABLE"; - if (evts[i].flags == (EV_DISABLE)) flags = "EV_DISABLE"; #endif keventbp += sprintf(keventbp, "%s{%lu, %s, %s}",i > 0 ? ", " : "", diff --git a/erts/emulator/test/driver_SUITE.erl b/erts/emulator/test/driver_SUITE.erl index cbed71cedd..f6d7c55017 100644 --- a/erts/emulator/test/driver_SUITE.erl +++ b/erts/emulator/test/driver_SUITE.erl @@ -120,29 +120,6 @@ -define(heap_binary_size, 64). -init_per_testcase(Case, Config) when is_atom(Case), is_list(Config) -> - CIOD = rpc(Config, - fun() -> - case catch erts_debug:get_internal_state(available_internal_state) of - true -> ok; - _ -> erts_debug:set_internal_state(available_internal_state, true) - end, - erts_debug:get_internal_state(check_io_debug) - end), - erlang:display({init_per_testcase, Case}), - 0 = element(1, CIOD), - [{testcase, Case}|Config]. - -end_per_testcase(Case, Config) -> - erlang:display({end_per_testcase, Case}), - CIOD = rpc(Config, - fun() -> - get_stable_check_io_info(), - erts_debug:get_internal_state(check_io_debug) - end), - 0 = element(1, CIOD), - ok. - suite() -> [{ct_hooks,[ts_install_cth]}, {timetrap, {minutes, 1}}]. @@ -219,6 +196,48 @@ end_per_group(_GroupName, Config) -> end, Config. +init_per_testcase(Case, Config) when is_atom(Case), is_list(Config) -> + CIOD = rpc(Config, + fun() -> + case catch erts_debug:get_internal_state(available_internal_state) of + true -> ok; + _ -> erts_debug:set_internal_state(available_internal_state, true) + end, + erts_debug:get_internal_state(check_io_debug) + end), + erlang:display({init_per_testcase, Case}), + 0 = element(1, CIOD), + [{testcase, Case}|Config]. + +end_per_testcase(Case, Config) -> + erlang:display({end_per_testcase, Case}), + try rpc(Config, fun() -> + get_stable_check_io_info(), + erts_debug:get_internal_state(check_io_debug) + end) of + CIOD -> + 0 = element(1, CIOD) + catch _E:_R:_ST -> + %% Logs some info about the system + ct_os_cmd("epmd -names"), + ct_os_cmd("ps aux"), + %% Restart the node + case proplists:get_value(node, Config) of + undefined -> + ok; + Node -> + timer:sleep(1000), %% Give the node time to die + [NodeName, _] = string:lexemes(atom_to_list(Node),"@"), + {ok, Node} = start_node_final( + list_to_atom(NodeName), + proplists:get_value(node_args, Config)) + end + end, + ok. + +ct_os_cmd(Cmd) -> + ct:log("~s: ~s",[Cmd,os:cmd(Cmd)]). + %% Test sending bad types to port with an outputv-capable driver. outputv_errors(Config) when is_list(Config) -> Path = proplists:get_value(data_dir, Config), @@ -2644,7 +2663,6 @@ start_node(Config) when is_list(Config) -> start_node(Name) -> start_node(Name, ""). start_node(NodeName, Args) -> - Pa = filename:dirname(code:which(?MODULE)), Name = list_to_atom(atom_to_list(?MODULE) ++ "-" ++ atom_to_list(NodeName) @@ -2652,7 +2670,17 @@ start_node(NodeName, Args) -> ++ integer_to_list(erlang:system_time(second)) ++ "-" ++ integer_to_list(erlang:unique_integer([positive]))), - test_server:start_node(Name, slave, [{args, Args ++ " -pa "++Pa}]). + start_node_final(Name, Args). +start_node_final(Name, Args) -> + {ok, Pwd} = file:get_cwd(), + FinalArgs = [Args, " -pa ", filename:dirname(code:which(?MODULE))], + {ok, Node} = test_server:start_node(Name, slave, [{args, FinalArgs}]), + LogPath = Pwd ++ "/error_log." ++ atom_to_list(Name), + ct:pal("Logging to: ~s", [LogPath]), + rpc:call(Node, logger, add_handler, [file_handler, logger_std_h, + #{formatter => {logger_formatter,#{ single_line => false }}, + config => #{file => LogPath }}]), + {ok, Node}. stop_node(Node) -> test_server:stop_node(Node). diff --git a/erts/emulator/test/net_SUITE.erl b/erts/emulator/test/net_SUITE.erl index 1a973cacb2..6111fc76a5 100644 --- a/erts/emulator/test/net_SUITE.erl +++ b/erts/emulator/test/net_SUITE.erl @@ -127,12 +127,17 @@ api_basic_cases() -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% init_per_suite(Config) -> - case os:type() of - {win32, _} -> - not_yet_implemented(); - _ -> - %% ?LOGGER:start(), - Config + case lists:member(socket, erlang:loaded()) of + true -> + case os:type() of + {win32, _} -> + not_yet_implemented(); + _ -> + %% ?LOGGER:start(), + Config + end; + false -> + {skip, "esock disabled"} end. end_per_suite(_) -> diff --git a/erts/emulator/test/socket_SUITE.erl b/erts/emulator/test/socket_SUITE.erl index cefbe4c1f8..e3545ccbf9 100644 --- a/erts/emulator/test/socket_SUITE.erl +++ b/erts/emulator/test/socket_SUITE.erl @@ -1385,22 +1385,27 @@ ttest_ssockt_csockt_cases() -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% init_per_suite(Config) -> - case os:type() of - {win32, _} -> - not_yet_implemented(); - _ -> - case quiet_mode(Config) of - default -> - ?LOGGER:start(), - Config; - Quiet -> - ?LOGGER:start(Quiet), - [{esock_test_quiet, Quiet}|Config] - end + case lists:member(socket, erlang:loaded()) of + true -> + case os:type() of + {win32, _} -> + (catch not_yet_implemented()); + _ -> + case quiet_mode(Config) of + default -> + ?LOGGER:start(), + Config; + Quiet -> + ?LOGGER:start(Quiet), + [{esock_test_quiet, Quiet}|Config] + end + end; + false -> + {skip, "esock disabled"} end. end_per_suite(_) -> - ?LOGGER:stop(), + (catch ?LOGGER:stop()), ok. diff --git a/erts/etc/common/erlexec.c b/erts/etc/common/erlexec.c index ec4a4ead23..8203c46a39 100644 --- a/erts/etc/common/erlexec.c +++ b/erts/etc/common/erlexec.c @@ -408,7 +408,6 @@ int main(int argc, char **argv) int process_args = 1; int print_args_exit = 0; int print_qouted_cmd_exit = 0; - erts_cpu_info_t *cpuinfo = NULL; char* emu_name; #ifdef __WIN32__ @@ -467,8 +466,6 @@ int main(int argc, char **argv) /* * Construct the path of the executable. */ - cpuinfo = erts_cpu_info_create(); - #if defined(__WIN32__) && defined(WIN32_ALWAYS_DEBUG) emu_type = "debug"; #endif @@ -526,9 +523,6 @@ int main(int argc, char **argv) i++; } - erts_cpu_info_destroy(cpuinfo); - cpuinfo = NULL; - if (malloc_lib) { if (strcmp(malloc_lib, "libc") != 0) usage("+MYm"); @@ -662,15 +656,6 @@ int main(int argc, char **argv) } break; - case 'i': - if (strcmp(argv[i], "-instr") == 0) { - add_Eargs("-Mim"); - add_Eargs("true"); - } - else - add_arg(argv[i]); - break; - case 'e': if (strcmp(argv[i], "-extra") == 0) { process_args = 0; diff --git a/erts/etc/unix/etp-commands.in b/erts/etc/unix/etp-commands.in index f2b2ab1d9a..66d6d20c4e 100644 --- a/erts/etc/unix/etp-commands.in +++ b/erts/etc/unix/etp-commands.in @@ -4124,14 +4124,6 @@ define etp-block-size-1 set $etp_blk_sz = ($arg0)->bhdr & ~7 else # Allocated block - if !$etp_MBC_ABLK_SZ_MASK - if etp_arch_bits == 64 - set $etp_MBC_ABLK_OFFSET_SHIFT = (64 - 24) - else - set $etp_MBC_ABLK_OFFSET_SHIFT = (32 - 9) - end - set $etp_MBC_ABLK_SZ_MASK = ((UWord)1 << $etp_MBC_ABLK_OFFSET_SHIFT) - 1 - 7 - end set $etp_blk_sz = ($arg0)->bhdr & $etp_MBC_ABLK_SZ_MASK end end @@ -4146,14 +4138,7 @@ define etp-block2mbc-1 set $etp_mbc = ($arg0)->u.carrier else # Allocated block - if !$etp_MBC_ABLK_OFFSET_SHIFT - if etp_arch_bits == 64 - set $etp_MBC_ABLK_OFFSET_SHIFT = (64 - 24) - else - set $etp_MBC_ABLK_OFFSET_SHIFT = (32 - 9) - end - end - set $etp_mbc = (Carrier_t*) ((((UWord)($arg0) >> 18) - (($arg0)->bhdr >> $etp_MBC_ABLK_OFFSET_SHIFT)) << 18) + set $etp_mbc = (Carrier_t*) ((((UWord)($arg0) >> 18) - ((($arg0)->bhdr & $etp_MBC_ABLK_OFFSET_MASK) >> $etp_MBC_ABLK_OFFSET_SHIFT)) << 18) end end @@ -4197,7 +4182,7 @@ document etp-smp-atomic %--------------------------------------------------------------------------- end -define etp-carrier-blocks +define etp-carrier-blocks-1 set $etp_crr = (Carrier_t*) $arg0 etp-smp-atomic $etp_crr->allctr $etp_alc set $etp_alc = (Allctr_t*)($etp_alc & ~7) @@ -4209,12 +4194,6 @@ define etp-carrier-blocks set $etp_fblk_cnt = 0 set $etp_aborted = 0 - if $argc == 2 - set $etp_be_silent = $arg1 - else - set $etp_be_silent = 0 - end - while 1 if !$etp_be_silent etp-block $etp_blk @@ -4237,6 +4216,12 @@ define etp-carrier-blocks printf "ERROR: Missing PREV_FREE_BLK_HDR_FLG (2) in block at %#lx\n", $etp_blk set $etp_error_cnt = $etp_error_cnt + 1 end + else + # Prev is ALLOCATED + if ($etp_blk->bhdr & 2) + printf "ERROR: Invalid PREV_FREE_BLK_HDR_FLG (2) set in block at %#lx\n", $etp_blk + set $etp_error_cnt = $etp_error_cnt + 1 + end end end if $etp_blk->bhdr & 1 @@ -4274,13 +4259,31 @@ define etp-carrier-blocks end end -document etp-carrier-blocks +define etp-carrier-print + set $etp_be_silent = 0 + etp-carrier-blocks-1 $arg0 +end + +document etp-carrier-print +%--------------------------------------------------------------------------- +% Print all memory blocks in carrier +% Args: (Carrier_t*) +%--------------------------------------------------------------------------- +end + +define etp-carrier-check + set $etp_be_silent = 1 + etp-carrier-blocks-1 $arg0 +end + +document etp-carrier-check %--------------------------------------------------------------------------- -% Check and (maybe) print all memory blocks in carrier -% Args: (Carrier_t*) [1=be_silent] +% Check all memory blocks in carrier +% Args: (Carrier_t*) %--------------------------------------------------------------------------- end + define etp-address-to-beam-opcode set $etp_i = 0 set $etp_min_diff = ((UWord)1 << (sizeof(UWord)*8 - 1)) @@ -4412,9 +4415,14 @@ define etp-init set $etp_arch64 = (sizeof(void *) == 8) if $etp_arch64 set $etp_nil = 0xfffffffffffffffb + set $etp_MBC_ABLK_OFFSET_BITS = 23 else set $etp_nil = 0xfffffffb + set $etp_MBC_ABLK_OFFSET_BITS = 8 end + set $etp_MBC_ABLK_OFFSET_SHIFT = (sizeof(UWord)*8 - 1 - $etp_MBC_ABLK_OFFSET_BITS) + set $etp_MBC_ABLK_OFFSET_MASK = ((((UWord)1 << $etp_MBC_ABLK_OFFSET_BITS) - 1) << $etp_MBC_ABLK_OFFSET_SHIFT) + set $etp_MBC_ABLK_SZ_MASK = ((UWord)1 << $etp_MBC_ABLK_OFFSET_SHIFT) - 1 - 7 set $etp_flat = 0 set $etp_chart_id = 0 set $etp_chart = 0 diff --git a/erts/preloaded/ebin/erl_init.beam b/erts/preloaded/ebin/erl_init.beam Binary files differindex 81be5b021a..0313988e3e 100644 --- a/erts/preloaded/ebin/erl_init.beam +++ b/erts/preloaded/ebin/erl_init.beam diff --git a/erts/preloaded/ebin/net.beam b/erts/preloaded/ebin/net.beam Binary files differindex ebb1296b95..f61b2b4a69 100644 --- a/erts/preloaded/ebin/net.beam +++ b/erts/preloaded/ebin/net.beam diff --git a/erts/preloaded/src/Makefile b/erts/preloaded/src/Makefile index efeb92dce9..27d450c873 100644 --- a/erts/preloaded/src/Makefile +++ b/erts/preloaded/src/Makefile @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2008-2018. All Rights Reserved. +# Copyright Ericsson AB 2008-2019. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -33,14 +33,22 @@ STATIC_EBIN=../ebin include $(ERL_TOP)/erts/vsn.mk include $(ERL_TOP)/lib/kernel/vsn.mk +ifeq ($(USE_ESOCK), yes) +PRE_LOADED_ERL_ESOCK_MODULES = \ + socket \ + net +else +PRE_LOADED_ERL_ESOCK_MODULES = \ + net +endif + PRE_LOADED_ERL_MODULES = \ erl_prim_loader \ init \ prim_buffer \ prim_file \ prim_inet \ - socket \ - net \ + $(PRE_LOADED_ERL_ESOCK_MODULES) \ zlib \ prim_zip \ erl_init \ @@ -73,6 +81,11 @@ STATIC_TARGET_FILES = $(PRE_LOADED_MODULES:%=$(STATIC_EBIN)/%.$(EMULATOR)) APP_FILE= erts.app APP_SRC= $(APP_FILE).src APP_TARGET= $(STATIC_EBIN)/$(APP_FILE) +ifeq ($(USE_ESOCK), yes) +APP_ESOCK_MODS= net, socket +else +APP_ESOCK_MODS= net +endif KERNEL_SRC=$(ERL_TOP)/lib/kernel/src @@ -94,7 +107,7 @@ copy: cp *.beam $(STATIC_EBIN) $(APP_TARGET): $(APP_SRC) $(ERL_TOP)/erts/vsn.mk - $(vsn_verbose)sed -e 's;%VSN%;$(VSN);' $< > $@ + $(vsn_verbose)sed -e 's;%VSN%;$(VSN);' -e 's;%ESOCK_MODS%;$(APP_ESOCK_MODS);' $< > $@ include $(ERL_TOP)/make/otp_release_targets.mk diff --git a/erts/preloaded/src/erl_init.erl b/erts/preloaded/src/erl_init.erl index 6edead362c..d209c4033b 100644 --- a/erts/preloaded/src/erl_init.erl +++ b/erts/preloaded/src/erl_init.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2000-2016. All Rights Reserved. +%% Copyright Ericsson AB 2000-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -35,8 +35,7 @@ start(Mod, BootArgs) -> erl_tracer:on_load(), prim_buffer:on_load(), prim_file:on_load(), - socket:on_load(), - net:on_load(), + conditional_load(socket, [socket, net]), % socket:on_load(), net:on_load(), %% Proceed to the specified boot module run(Mod, boot, BootArgs). @@ -48,3 +47,24 @@ run(M, F, A) -> true -> M:F(A) end. + +conditional_load(CondMod, Mods2Load) -> + conditional_load(CondMod, erlang:loaded(), Mods2Load). + +conditional_load(_CondMod, [], _Mods2LOad) -> + ok; +conditional_load(CondMod, [CondMod|_], Mods2Load) -> + on_load(Mods2Load); +conditional_load(CondMod, [_|T], Mods2Load) -> + conditional_load(CondMod, T, Mods2Load). + +on_load([]) -> + ok; +on_load([Mod|Mods]) -> + Mod:on_load(), + on_load(Mods). + + + + + diff --git a/erts/preloaded/src/erts.app.src b/erts/preloaded/src/erts.app.src index c2a8511b6d..132397b478 100644 --- a/erts/preloaded/src/erts.app.src +++ b/erts/preloaded/src/erts.app.src @@ -36,8 +36,7 @@ atomics, counters, zlib, - net, - socket + %ESOCK_MODS% ]}, {registered, []}, {applications, []}, diff --git a/erts/preloaded/src/net.erl b/erts/preloaded/src/net.erl index a24b5c8ce3..13d2e3a117 100644 --- a/erts/preloaded/src/net.erl +++ b/erts/preloaded/src/net.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2018-2018. All Rights Reserved. +%% Copyright Ericsson AB 2018-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -178,12 +178,28 @@ getnameinfo(SockAddr, [] = _Flags) -> getnameinfo(#{family := Fam, addr := _Addr} = SockAddr, Flags) when ((Fam =:= inet) orelse (Fam =:= inet6)) andalso (is_list(Flags) orelse (Flags =:= undefined)) -> - nif_getnameinfo(socket:ensure_sockaddr(SockAddr), Flags); + nif_getnameinfo((catch ensure_sockaddr(SockAddr)), Flags); getnameinfo(#{family := Fam, path := _Path} = SockAddr, Flags) when (Fam =:= local) andalso (is_list(Flags) orelse (Flags =:= undefined)) -> nif_getnameinfo(SockAddr, Flags). +%% This function is intended to "handle" the case when the user +%% has built their (OTP) system with "--disable-esock". +%% That means the socket module does not exist. This is not really +%% a problem since the nif_getnameinfo won't work either (since +%% the nif file is not part of the system). The result of calling +%% getnameinfo will be a undef exception (erlang:nif_error(undef)). +%% +%% The only functions in this module that actually work in this case +%% (--disable-esock) is the depricated stuff (call, cast, ...). +%% +ensure_sockaddr(SockAddr) -> + try socket:ensure_sockaddr(SockAddr) + catch + error:undef:_ -> + undefined + end. %% =========================================================================== %% @@ -334,3 +350,4 @@ nif_if_index2name(_Id) -> nif_if_names() -> erlang:nif_error(undef). + diff --git a/lib/compiler/src/beam_ssa_pre_codegen.erl b/lib/compiler/src/beam_ssa_pre_codegen.erl index bad43a9c4e..bf99e8fc26 100644 --- a/lib/compiler/src/beam_ssa_pre_codegen.erl +++ b/lib/compiler/src/beam_ssa_pre_codegen.erl @@ -342,21 +342,22 @@ make_save_point_dict_1([], Ctx, I, Acc) -> [{Ctx,I}|Acc]. bs_restores([{L,#b_blk{is=Is,last=Last}}|Bs], CtxChain, D0, Rs0) -> - FPos = case D0 of - #{L:=Pos0} -> Pos0; - #{} -> #{} - end, - {SPos,Rs} = bs_restores_is(Is, CtxChain, FPos, Rs0), - D = bs_update_successors(Last, SPos, FPos, D0), + InPos = maps:get(L, D0, #{}), + {SuccPos, FailPos, Rs} = bs_restores_is(Is, CtxChain, InPos, InPos, Rs0), + + D = bs_update_successors(Last, SuccPos, FailPos, D0), bs_restores(Bs, CtxChain, D, Rs); bs_restores([], _, _, Rs) -> Rs. bs_update_successors(#b_br{succ=Succ,fail=Fail}, SPos, FPos, D) -> join_positions([{Succ,SPos},{Fail,FPos}], D); -bs_update_successors(#b_switch{fail=Fail,list=List}, SPos, _FPos, D) -> +bs_update_successors(#b_switch{fail=Fail,list=List}, SPos, FPos, D) -> + SPos = FPos, %Assertion. Update = [{L,SPos} || {_,L} <- List] ++ [{Fail,SPos}], join_positions(Update, D); -bs_update_successors(#b_ret{}, _, _, D) -> D. +bs_update_successors(#b_ret{}, SPos, FPos, D) -> + SPos = FPos, %Assertion. + D. join_positions([{L,MapPos0}|T], D) -> case D of @@ -382,75 +383,91 @@ join_positions_1(MapPos0, MapPos1) -> end, MapPos1), maps:merge(MapPos0, MapPos2). +%% +%% Updates the restore and position maps according to the given instructions. +%% +%% Note that positions may be updated even when a match fails; if a match +%% requires a restore, the position at the fail block will be the position +%% we've *restored to* and not the one we entered the current block with. +%% + bs_restores_is([#b_set{op=bs_start_match,dst=Start}|Is], - CtxChain, PosMap0, Rs) -> - PosMap = PosMap0#{Start=>Start}, - bs_restores_is(Is, CtxChain, PosMap, Rs); + CtxChain, SPos0, FPos, Rs) -> + %% We only allow one match per block. + SPos0 = FPos, %Assertion. + SPos = SPos0#{Start=>Start}, + bs_restores_is(Is, CtxChain, SPos, FPos, Rs); bs_restores_is([#b_set{op=bs_match,dst=NewPos,args=Args}=I|Is], - CtxChain, PosMap0, Rs0) -> + CtxChain, SPos0, FPos0, Rs0) -> + SPos0 = FPos0, %Assertion. Start = bs_subst_ctx(NewPos, CtxChain), [_,FromPos|_] = Args, - case PosMap0 of + case SPos0 of #{Start:=FromPos} -> %% Same position, no restore needed. - PosMap = case bs_match_type(I) of + SPos = case bs_match_type(I) of plain -> %% Update position to new position. - PosMap0#{Start:=NewPos}; + SPos0#{Start:=NewPos}; _ -> %% Position will not change (test_unit %% instruction or no instruction at %% all). - PosMap0#{Start:=FromPos} + SPos0#{Start:=FromPos} end, - bs_restores_is(Is, CtxChain, PosMap, Rs0); + bs_restores_is(Is, CtxChain, SPos, FPos0, Rs0); #{Start:=_} -> %% Different positions, might need a restore instruction. case bs_match_type(I) of none -> - %% The tail test will be optimized away. - %% No need to do a restore. - PosMap = PosMap0#{Start:=FromPos}, - bs_restores_is(Is, CtxChain, PosMap, Rs0); + %% This is a tail test that will be optimized away. + %% There's no need to do a restore, and all + %% positions are unchanged. + bs_restores_is(Is, CtxChain, SPos0, FPos0, Rs0); test_unit -> %% This match instruction will be replaced by %% a test_unit instruction. We will need a %% restore. The new position will be the position %% restored to (NOT NewPos). - PosMap = PosMap0#{Start:=FromPos}, + SPos = SPos0#{Start:=FromPos}, + FPos = FPos0#{Start:=FromPos}, Rs = Rs0#{NewPos=>{Start,FromPos}}, - bs_restores_is(Is, CtxChain, PosMap, Rs); + bs_restores_is(Is, CtxChain, SPos, FPos, Rs); plain -> %% Match or skip. Position will be changed. - PosMap = PosMap0#{Start:=NewPos}, + SPos = SPos0#{Start:=NewPos}, + FPos = FPos0#{Start:=FromPos}, Rs = Rs0#{NewPos=>{Start,FromPos}}, - bs_restores_is(Is, CtxChain, PosMap, Rs) + bs_restores_is(Is, CtxChain, SPos, FPos, Rs) end end; bs_restores_is([#b_set{op=bs_extract,args=[FromPos|_]}|Is], - CtxChain, PosMap, Rs) -> + CtxChain, SPos, FPos, Rs) -> Start = bs_subst_ctx(FromPos, CtxChain), - #{Start:=FromPos} = PosMap, %Assertion. - bs_restores_is(Is, CtxChain, PosMap, Rs); + #{Start:=FromPos} = SPos, %Assertion. + #{Start:=FromPos} = FPos, %Assertion. + bs_restores_is(Is, CtxChain, SPos, FPos, Rs); bs_restores_is([#b_set{op=call,dst=Dst,args=Args}|Is], - CtxChain, PosMap0, Rs0) -> - {Rs,PosMap1} = bs_restore_args(Args, PosMap0, CtxChain, Dst, Rs0), - PosMap = bs_invalidate_pos(Args, PosMap1, CtxChain), - bs_restores_is(Is, CtxChain, PosMap, Rs); -bs_restores_is([#b_set{op=landingpad}|Is], CtxChain, PosMap0, Rs) -> + CtxChain, SPos0, FPos0, Rs0) -> + {Rs, SPos1, FPos1} = bs_restore_args(Args, SPos0, FPos0, CtxChain, Dst, Rs0), + {SPos, FPos} = bs_invalidate_pos(Args, SPos1, FPos1, CtxChain), + bs_restores_is(Is, CtxChain, SPos, FPos, Rs); +bs_restores_is([#b_set{op=landingpad}|Is], CtxChain, SPos0, FPos0, Rs) -> %% We can land here from any point, so all positions are invalid. - PosMap = maps:map(fun(_Start,_Pos) -> unknown end, PosMap0), - bs_restores_is(Is, CtxChain, PosMap, Rs); + Invalidate = fun(_Start,_Pos) -> unknown end, + SPos = maps:map(Invalidate, SPos0), + FPos = maps:map(Invalidate, FPos0), + bs_restores_is(Is, CtxChain, SPos, FPos, Rs); bs_restores_is([#b_set{op=Op,dst=Dst,args=Args}|Is], - CtxChain, PosMap0, Rs0) + CtxChain, SPos0, FPos0, Rs0) when Op =:= bs_test_tail; Op =:= bs_get_tail -> - {Rs,PosMap} = bs_restore_args(Args, PosMap0, CtxChain, Dst, Rs0), - bs_restores_is(Is, CtxChain, PosMap, Rs); -bs_restores_is([_|Is], CtxChain, PosMap, Rs) -> - bs_restores_is(Is, CtxChain, PosMap, Rs); -bs_restores_is([], _CtxChain, PosMap, Rs) -> - {PosMap,Rs}. + {Rs, SPos, FPos} = bs_restore_args(Args, SPos0, FPos0, CtxChain, Dst, Rs0), + bs_restores_is(Is, CtxChain, SPos, FPos, Rs); +bs_restores_is([_|Is], CtxChain, SPos, FPos, Rs) -> + bs_restores_is(Is, CtxChain, SPos, FPos, Rs); +bs_restores_is([], _CtxChain, SPos, FPos, Rs) -> + {SPos, FPos, Rs}. bs_match_type(#b_set{args=[#b_literal{val=skip},_Ctx, #b_literal{val=binary},_Flags, @@ -464,40 +481,42 @@ bs_match_type(_) -> %% Call instructions leave the match position in an undefined state, %% requiring us to invalidate each affected argument. -bs_invalidate_pos([#b_var{}=Arg|Args], PosMap0, CtxChain) -> +bs_invalidate_pos([#b_var{}=Arg|Args], SPos0, FPos0, CtxChain) -> Start = bs_subst_ctx(Arg, CtxChain), - case PosMap0 of + case SPos0 of #{Start:=_} -> - PosMap = PosMap0#{Start:=unknown}, - bs_invalidate_pos(Args, PosMap, CtxChain); + SPos = SPos0#{Start:=unknown}, + FPos = FPos0#{Start:=unknown}, + bs_invalidate_pos(Args, SPos, FPos, CtxChain); #{} -> %% Not a match context. - bs_invalidate_pos(Args, PosMap0, CtxChain) + bs_invalidate_pos(Args, SPos0, FPos0, CtxChain) end; -bs_invalidate_pos([_|Args], PosMap, CtxChain) -> - bs_invalidate_pos(Args, PosMap, CtxChain); -bs_invalidate_pos([], PosMap, _CtxChain) -> - PosMap. +bs_invalidate_pos([_|Args], SPos, FPos, CtxChain) -> + bs_invalidate_pos(Args, SPos, FPos, CtxChain); +bs_invalidate_pos([], SPos, FPos, _CtxChain) -> + {SPos, FPos}. -bs_restore_args([#b_var{}=Arg|Args], PosMap0, CtxChain, Dst, Rs0) -> +bs_restore_args([#b_var{}=Arg|Args], SPos0, FPos0, CtxChain, Dst, Rs0) -> Start = bs_subst_ctx(Arg, CtxChain), - case PosMap0 of + case SPos0 of #{Start:=Arg} -> %% Same position, no restore needed. - bs_restore_args(Args, PosMap0, CtxChain, Dst, Rs0); + bs_restore_args(Args, SPos0, FPos0, CtxChain, Dst, Rs0); #{Start:=_} -> %% Different positions, need a restore instruction. - PosMap = PosMap0#{Start:=Arg}, + SPos = SPos0#{Start:=Arg}, + FPos = FPos0#{Start:=Arg}, Rs = Rs0#{Dst=>{Start,Arg}}, - bs_restore_args(Args, PosMap, CtxChain, Dst, Rs); + bs_restore_args(Args, SPos, FPos, CtxChain, Dst, Rs); #{} -> %% Not a match context. - bs_restore_args(Args, PosMap0, CtxChain, Dst, Rs0) + bs_restore_args(Args, SPos0, FPos0, CtxChain, Dst, Rs0) end; -bs_restore_args([_|Args], PosMap, CtxChain, Dst, Rs) -> - bs_restore_args(Args, PosMap, CtxChain, Dst, Rs); -bs_restore_args([], PosMap, _CtxChain, _Dst, Rs) -> - {Rs,PosMap}. +bs_restore_args([_|Args], SPos, FPos, CtxChain, Dst, Rs) -> + bs_restore_args(Args, SPos, FPos, CtxChain, Dst, Rs); +bs_restore_args([], SPos, FPos, _CtxChain, _Dst, Rs) -> + {Rs,SPos,FPos}. %% Insert all bs_save and bs_restore instructions. diff --git a/lib/compiler/test/bs_match_SUITE.erl b/lib/compiler/test/bs_match_SUITE.erl index 41e4918b1e..d97f49c56e 100644 --- a/lib/compiler/test/bs_match_SUITE.erl +++ b/lib/compiler/test/bs_match_SUITE.erl @@ -1891,15 +1891,37 @@ expression_before_match_1(R) -> %% Make sure that context positions are updated on calls. restore_on_call(Config) when is_list(Config) -> - ok = restore_on_call_1(<<0, 1, 2>>). + ok = restore_on_call_plain(<<0, 1, 2>>), + <<"x">> = restore_on_call_match(<<0, "x">>), + ok. -restore_on_call_1(<<0, Rest/binary>>) -> - <<2>> = restore_on_call_2(Rest), - <<2>> = restore_on_call_2(Rest), %% {badmatch, <<>>} on missing restore. +restore_on_call_plain(<<0, Rest/binary>>) -> + <<2>> = restore_on_call_plain_1(Rest), + %% {badmatch, <<>>} on missing restore. + <<2>> = restore_on_call_plain_1(Rest), ok. -restore_on_call_2(<<1, Rest/binary>>) -> Rest; -restore_on_call_2(Other) -> Other. +restore_on_call_plain_1(<<1, Rest/binary>>) -> Rest; +restore_on_call_plain_1(Other) -> Other. + +%% Calls a function that moves the match context passed to it, and then matches +%% on its result to confuse the reposition algorithm's success/fail logic. +restore_on_call_match(<<0, Bin/binary>>) -> + case skip_until_zero(Bin) of + {skipped, Rest} -> + Rest; + not_found -> + %% The match context did not get repositioned before the + %% bs_get_tail instruction here. + Bin + end. + +skip_until_zero(<<0,Rest/binary>>) -> + {skipped, Rest}; +skip_until_zero(<<_C,Rest/binary>>) -> + skip_until_zero(Rest); +skip_until_zero(_) -> + not_found. %% 'catch' must invalidate positions. restore_after_catch(Config) when is_list(Config) -> @@ -1983,5 +2005,4 @@ do_matching_meets_apply(_Bin, {Handler, State}) -> %% Another case of the above. Handler:abs(State). - id(I) -> I. diff --git a/lib/crypto/doc/src/crypto.xml b/lib/crypto/doc/src/crypto.xml index 641738247e..d1d1252f29 100644 --- a/lib/crypto/doc/src/crypto.xml +++ b/lib/crypto/doc/src/crypto.xml @@ -678,7 +678,8 @@ </p> <p>For encryption, set the <c>EncryptFlag</c> to <c>true</c>. For decryption, set it to <c>false</c>. </p> - <p>See <seealso marker="crypto:new_api#the-new-api">examples in the User's Guide.</seealso> + <p>See <seealso marker="crypto:new_api#examples-of-crypto_init-4-and-crypto_update-2"> + examples in the User's Guide.</seealso> </p> </desc> </func> @@ -696,7 +697,8 @@ or <seealso marker="crypto#crypto_init/4">crypto_init/4</seealso>. </p> - <p>See <seealso marker="crypto:new_api#the-new-api">examples in the User's Guide.</seealso> + <p>See <seealso marker="crypto:new_api#examples-of-crypto_init-4-and-crypto_update-2"> + examples in the User's Guide.</seealso> </p> </desc> </func> @@ -712,8 +714,6 @@ </p> <p>For encryption, set the <c>EncryptFlag</c> to <c>true</c>. For decryption, set it to <c>false</c>. </p> - <p>See <seealso marker="crypto:new_api#the-new-api">examples in the User's Guide.</seealso> - </p> </desc> </func> @@ -726,8 +726,6 @@ The <c>State</c> should be created with <seealso marker="crypto#crypto_dyn_iv_init/3">crypto_dyn_iv_init/3</seealso>. </p> - <p>See <seealso marker="crypto:new_api#the-new-api">examples in the User's Guide.</seealso> - </p> </desc> </func> @@ -744,11 +742,11 @@ <fsummary>Do a complete encrypt or decrypt of the full text</fsummary> <desc> <p>Part of the <seealso marker="crypto:new_api#the-new-api">new API</seealso>. - Do a complete encrypt or decrypt of the full text. + Do a complete encrypt or decrypt of the full text in the argument <c>Data</c>. </p> <p>For encryption, set the <c>EncryptFlag</c> to <c>true</c>. For decryption, set it to <c>false</c>. </p> - <p>See <seealso marker="crypto:new_api#the-new-api">examples in the User's Guide.</seealso> + <p>See <seealso marker="crypto:new_api#example-of-crypto_one_time-5">examples in the User's Guide.</seealso> </p> </desc> </func> @@ -768,10 +766,23 @@ <p>For decryption, set the <c>EncryptFlag</c> to <c>false</c> and put the tag to be checked in the argument <c>TagOrTagLength</c>. </p> - <p>See <seealso marker="crypto:new_api#the-new-api">examples in the User's Guide.</seealso> + <p>See <seealso marker="crypto:new_api#example-of-crypto_one_time_aead-6">examples in the User's Guide.</seealso> </p> </desc> </func> + + <func> + <name name="supports" arity="1" since="OTP 22.0"/> + <fsummary>Provide a list of available crypto algorithms.</fsummary> + <desc> + <p> Can be used to determine which crypto algorithms that are supported + by the underlying libcrypto library</p> + <p>See <seealso marker="#hash_info-1">hash_info/1</seealso> and <seealso marker="#cipher_info-1">cipher_info/1</seealso> + for information about the hash and cipher algorithms. + </p> + </desc> + </func> + </funcs> <section> @@ -1425,18 +1436,6 @@ FloatValue = rand:uniform(). % again </desc> </func> - <func> - <name name="supports" arity="0" since="OTP R16B01"/> - <fsummary>Provide a list of available crypto algorithms.</fsummary> - <desc> - <p> Can be used to determine which crypto algorithms that are supported - by the underlying libcrypto library</p> - <p>See <seealso marker="#hash_info-1">hash_info/1</seealso> and <seealso marker="#cipher_info-1">cipher_info/1</seealso> - for information about the hash and cipher algorithms. - </p> - </desc> - </func> - <func> <name name="ec_curves" arity="0" since="OTP 17.0"/> <fsummary>Provide a list of available named elliptic curves.</fsummary> @@ -1947,6 +1946,21 @@ FloatValue = rand:uniform(). % again </desc> </func> + <func> + <name name="supports" arity="0" since="OTP R16B01"/> + <fsummary>Provide a list of available crypto algorithms.</fsummary> + <desc> + <dont><p>Don't use this function for new programs! Use + <seealso marker="crypto#supports-1">supports/1</seealso> in + <seealso marker="crypto:new_api">the new api</seealso>.</p></dont> + <p> Can be used to determine which crypto algorithms that are supported + by the underlying libcrypto library</p> + <p>See <seealso marker="#hash_info-1">hash_info/1</seealso> and <seealso marker="#cipher_info-1">cipher_info/1</seealso> + for information about the hash and cipher algorithms. + </p> + </desc> + </func> + </funcs> diff --git a/lib/crypto/doc/src/new_api.xml b/lib/crypto/doc/src/new_api.xml index 79096b55e8..bd2334ac9f 100644 --- a/lib/crypto/doc/src/new_api.xml +++ b/lib/crypto/doc/src/new_api.xml @@ -59,8 +59,9 @@ <item><seealso marker="crypto#stream_init-2">stream_init/3</seealso></item> <item><seealso marker="crypto#stream_encrypt-2">stream_encrypt/2</seealso></item> <item><seealso marker="crypto#stream_decrypt-2">stream_decrypt/2</seealso></item> + <item><seealso marker="crypto#supports-0">supports/0</seealso></item> </list> - <p>They are not deprecated for now, but may be in a future. + <p>They are not deprecated for now, but may be in a future release. </p> </section> @@ -75,7 +76,8 @@ <item><seealso marker="crypto#crypto_one_time_aead/7">crypto_one_time_aead/7</seealso></item> </list> <p>In those functions the internal crypto state is first created and initialized - with the cipher type, the key and possibly other data. Then the data is encrypted or decrypted, + with the cipher type, the key and possibly other data. Then the single binary is encrypted + or decrypted, the crypto state is de-allocated and the result of the crypto operation is returned. </p> <p>The <c>crypto_one_time_aead</c> functions are for the ciphers of mode <c>ccm</c> or @@ -94,7 +96,7 @@ <c>crypto_update</c> does the acual encryption or decryption. Note that AEAD ciphers can't be handled this way due to their nature. </p> - <p>Finally, for repeated encryption or decryption of a text divided in parts where the + <p>For repeated encryption or decryption of a text divided in parts where the same cipher and same key is used, but a new initialization vector (nounce) should be applied for each part, the functions are: </p> @@ -103,32 +105,49 @@ <item><seealso marker="crypto#crypto_dyn_iv_update/3">crypto_dyn_iv_update/3</seealso></item> </list> <p>An example of where those functions are needed, is when handling the TLS protocol.</p> + <p>For information about available algorithms, use: + </p> + <list> + <item><seealso marker="crypto#supports-1">supports/1</seealso></item> + <item><seealso marker="crypto#hash_info-1">hash_info/1</seealso></item> + <item><seealso marker="crypto#cipher_info-1">cipher_info/1</seealso></item> + </list> <section> <title>Examples of crypto_init/4 and crypto_update/2</title> - <p>Encrypting two blocks:</p> + <p>The functions <seealso marker="crypto#crypto_init/4">crypto_init/4</seealso> + and <seealso marker="crypto#crypto_update/2">crypto_update/2</seealso> are intended + to be used for encrypting or decrypting a sequence of blocks. First one call of + <c>crypto_init/4</c> initialises the crypto context. One or more calls <c>crypto_update/2</c> + does the actual encryption or decryption for each block. + </p> + <p>This example shows first the encryption of two blocks and then decryptions of the cipher + text, but divided into three blocks just to show that it is possible to divide the plain text and + cipher text differently for some ciphers:</p> <code type="erl"> 1> crypto:start(). ok 2> Key = <<1:128>>. - 2> IV = <<0:128>>. - 2> StateEnc = crypto:crypto_init(aes_128_ctr, Key, IV, true). % encrypt -> true + <<0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1>> + 3> IV = <<0:128>>. + <<0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0>> + 4> StateEnc = crypto:crypto_init(aes_128_ctr, Key, IV, true). % encrypt -> true #Ref<0.3768901617.1128660993.124047> - 3> crypto:crypto_update(StateEnc, <<"First bytes">>). + 5> crypto:crypto_update(StateEnc, <<"First bytes">>). <<67,44,216,166,25,130,203,5,66,6,162>> - 4> crypto:crypto_update(StateEnc, <<"Second bytes">>). + 6> crypto:crypto_update(StateEnc, <<"Second bytes">>). <<16,79,94,115,234,197,94,253,16,144,151,41>> - 5> - 5> StateDec = crypto:crypto_init(aes_128_ctr, Key, IV, false). % decrypt -> false + 7> + 7> StateDec = crypto:crypto_init(aes_128_ctr, Key, IV, false). % decrypt -> false #Ref<0.3768901617.1128660994.124255> - 6> crypto:crypto_update(StateDec, <<67,44,216,166,25,130,203>>). + 8> crypto:crypto_update(StateDec, <<67,44,216,166,25,130,203>>). <<"First b">> - 7> crypto:crypto_update(StateDec, <<5,66,6,162,16,79,94,115,234,197, - 94,253,16,144,151>>). + 9> crypto:crypto_update(StateDec, <<5,66,6,162,16,79,94,115,234,197, + 94,253,16,144,151>>). <<"ytesSecond byte">> - 8> crypto:crypto_update(StateDec, <<41>>). + 10> crypto:crypto_update(StateDec, <<41>>). <<"s">> - 9> + 11> </code> <p>Note that the internal data that the <c>StateEnc</c> and <c>StateDec</c> references are destructivly updated by the calls to <seealso marker="crypto#crypto_update/2">crypto_update/2</seealso>. @@ -155,21 +174,51 @@ <title>Example of crypto_one_time/5</title> <p>The same example as in the <seealso marker="#examples-of-crypto_init-4-and-crypto_update-2">previous section</seealso>, - but now with one call to <c>crypto_one_time/5</c>: + but now with one call to <seealso marker="crypto#crypto_one_time/5">crypto_one_time/5</seealso>: </p> <code> - 2> Key = <<1:128>>. + 1> Key = <<1:128>>. + <<0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1>> 2> IV = <<0:128>>. - 2> Txt = [<<"First bytes">>,<<"Second bytes">>], - 2> crypto:crypto_one_time(aes_128_ctr, Key, IV, Txt, true). + <<0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0>> + 3> Txt = [<<"First bytes">>,<<"Second bytes">>]. + [<<"First bytes">>,<<"Second bytes">>] + 4> crypto:crypto_one_time(aes_128_ctr, Key, IV, Txt, true). <<67,44,216,166,25,130,203,5,66,6,162,16,79,94,115,234, 197,94,253,16,144,151,41>> - 3> + 5> + </code> + <p>The <c>[<<"First bytes">>,<<"Second bytes">>]</c> could of course have been one + single binary: <c><<"First bytesSecond bytes">></c>. + </p> + </section> + + <section> + <title>Example of crypto_one_time_aead/6</title> + <p>The same example as in the + <seealso marker="#example-of-crypto_one_time-5">previous section</seealso>, + but now with one call to <seealso marker="crypto#crypto_one_time_aead/6">crypto_one_time_aead/6</seealso>: + </p> + <code> + 1> Key = <<1:128>>. + <<0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1>> + 2> IV = <<0:128>>. + <<0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0>> + 3> Txt = [<<"First bytes">>,<<"Second bytes">>]. + [<<"First bytes">>,<<"Second bytes">>] + 4> AAD = <<"Some bytes">>. + <<"Some bytes">> + 5> crypto:crypto_one_time_aead(aes_128_gcm, Key, IV, Txt, AAD, true). + {<<240,130,38,96,130,241,189,52,3,190,179,213,132,1,72, + 192,103,176,90,104,15,71,158>>, + <<131,47,45,91,142,85,9,244,21,141,214,71,31,135,2,155>>} + 9> </code> <p>The <c>[<<"First bytes">>,<<"Second bytes">>]</c> could of course have been one single binary: <c><<"First bytesSecond bytes">></c>. </p> </section> + </section> <section> diff --git a/lib/crypto/src/crypto.erl b/lib/crypto/src/crypto.erl index 98378412d4..8ffdde2b90 100644 --- a/lib/crypto/src/crypto.erl +++ b/lib/crypto/src/crypto.erl @@ -509,6 +509,27 @@ supports() -> {rsa_opts, rsa_opts_algorithms()} ]. + +-spec supports(Type) -> Support + when Type :: hashs + | ciphers + | public_keys + | macs + | curves + | rsa_opts, + Support :: Hashs + | Ciphers + | PKs + | Macs + | Curves + | RSAopts, + Hashs :: [sha1() | sha2() | sha3() | blake2() | ripemd160 | compatibility_only_hash()], + Ciphers :: [cipher()], + PKs :: [rsa | dss | ecdsa | dh | ecdh | ec_gf2m], + Macs :: [hmac | cmac | poly1305], + Curves :: [ec_named_curve() | edwards_curve_dh() | edwards_curve_ed()], + RSAopts :: [rsa_sign_verify_opt() | rsa_opt()] . + supports(hashs) -> hash_algorithms(); supports(public_keys) -> pubkey_algorithms(); supports(ciphers) -> cipher_algorithms(); diff --git a/lib/dialyzer/test/small_SUITE_data/results/spec_other_module b/lib/dialyzer/test/small_SUITE_data/results/spec_other_module deleted file mode 100644 index ab2e35cf55..0000000000 --- a/lib/dialyzer/test/small_SUITE_data/results/spec_other_module +++ /dev/null @@ -1,2 +0,0 @@ - -spec_other_module.erl:7: Contract for function that does not exist: lists:flatten/1 diff --git a/lib/dialyzer/test/small_SUITE_data/src/spec_other_module.erl b/lib/dialyzer/test/small_SUITE_data/src/spec_other_module.erl deleted file mode 100644 index b36742b1bd..0000000000 --- a/lib/dialyzer/test/small_SUITE_data/src/spec_other_module.erl +++ /dev/null @@ -1,7 +0,0 @@ --module(spec_other_module). - -%% OTP-15562 and ERL-845. Example provided by Kostis. - --type deep_list(A) :: [A | deep_list(A)]. - --spec lists:flatten(deep_list(A)) -> [A]. diff --git a/lib/erl_interface/doc/src/ei.xml b/lib/erl_interface/doc/src/ei.xml index 254ae27cc8..7808bfd94f 100644 --- a/lib/erl_interface/doc/src/ei.xml +++ b/lib/erl_interface/doc/src/ei.xml @@ -183,28 +183,36 @@ typedef enum { </func> <func> - <name since="OTP @OTP-15712@"><ret>int</ret><nametext>ei_decode_bitstring(const char *buf, int *index, void *p, size_t plen, size_t *bitsp)</nametext></name> + <name since="OTP @OTP-15712@"><ret>int</ret><nametext>ei_decode_bitstring(const char *buf, int *index, const char **pp, unsigned int *bitoffsp, size_t *nbitsp)</nametext></name> <fsummary>Decode a bitstring.</fsummary> <desc> - <p>Decodes a bitstring from the binary format.</p> + <p>Decodes a bit string from the binary format.</p> <taglist> - <tag><c>p</c></tag> - <item><p>Either <c>NULL</c> or points to a buffer where the bytes of the - bitstring will be written.</p> + <tag><c>pp</c></tag> + <item><p>Either <c>NULL</c> or <c>*pp</c> returns a pointer to + the first byte of the bit string. The returned bit string is + readable as long as the buffer pointed to by <c>buf</c> is + readable and not written to.</p> </item> - <tag><c>plen</c></tag> - <item><p>The max size of the bitstring in <em>bytes</em>, that is the - size of the buffer if <c>p != NULL</c>.</p> + <tag><c>bitoffsp</c></tag> + <item><p>Either <c>NULL</c> or <c>*bitoffsp</c> returns the + number of unused bits in the first byte pointed to by + <c>*pp</c>. The value of <c>*bitoffsp</c> is between 0 and 7. + Unused bits in the first byte are the most significant bits.</p> </item> - <tag><c>*bitsp</c></tag> - <item><p>If <c>bitsp</c> is not <c>NULL</c>, set to the actual - number of <em>bits</em> of the bitstring.</p> + <tag><c>nbitsp</c></tag> + <item><p>Either <c>NULL</c> or <c>*nbitsp</c> returns the length + of the bit string in <em>bits</em>.</p> </item> </taglist> - <p>Returns <c>0</c> if it was a bitstring no longer than <c>plen</c> - bytes. The actual length of the bitstring will be - <c>(*bitsp+7)/8</c> bytes. If <c>(*bitsp % 8) > 0</c> only the high - <c>(*bitsp % 8)</c> bits of the last byte are significant.</p> + <p>Returns <c>0</c> if it was a bit string term.</p> + <p>The number of <em>bytes</em> pointed to by <c>*pp</c>, which are + part of the bit string, is <c>(*bitoffsp + *nbitsp + 7)/8</c>. If + <c>(*bitoffsp + *bitsp)%8 > 0</c> then only <c>(*bitoffsp + + *bitsp)%8</c> bits of the last byte are used. Unused bits in + the last byte are the least significant bits.</p> + <p>The values of unused bits in the first and last byte are undefined + and cannot be relied on.</p> <p>Number of bits may be divisible by 8, which means a binary decodable by <c>ei_decode_binary</c> is also decodable by <c>ei_decode_bitstring</c>.</p> @@ -490,14 +498,24 @@ typedef enum { </func> <func> - <name since="OTP @OTP-15712@"><ret>int</ret><nametext>ei_encode_bitstring(char *buf, int *index, const void *p, size_t bits)</nametext></name> - <name since="OTP @OTP-15712@"><ret>int</ret><nametext>ei_x_encode_bitstring(ei_x_buff* x, const void *p, size_t bits)</nametext></name> + <name since="OTP @OTP-15712@"><ret>int</ret> + <nametext>ei_encode_bitstring(char *buf, int *index, const char *p, size_t bitoffs, size_t nbits)</nametext></name> + <name since="OTP @OTP-15712@"><ret>int</ret> + <nametext>ei_x_encode_bitstring(ei_x_buff* x, const char *p, size_t bitoffs, size_t nbits)</nametext></name> <fsummary>Encode a bitstring.</fsummary> <desc> - <p>Encodes a bitstring in the binary format. The data is at - <c>p</c>. The size of the data is <c>bits</c> bits or - <c>(bits+7)/8</c> bytes. If <c>(bits%8) > 0</c> only the high - <c>(bits%8)</c> bits of the last byte are significant.</p> + <p>Encodes a bit string in the binary format.</p> + <p>The data is at <c>p</c>. The length of the bit string is <c>nbits</c> + bits. The first <c>bitoffs</c> bits of the data at <c>p</c> are unused. + The first byte which is part of the bit string is + <c>p[bitoffs/8]</c>. The <c>bitoffs%8</c> most significant bits of + the first byte <c>p[bitoffs/8]</c> are unused.</p> + <p>The number of bytes which is part of the bit string is <c>(bitoffs + + nbits + 7)/8</c>. If <c>(bitoffs + nbits)%8 > 0</c> then only <c>(bitoffs + + nbits)%8</c> bits of the last byte are used. Unused bits in + the last byte are the least significant bits.</p> + <p>The values of unused bits are disregarded and does not need to be + cleared.</p> </desc> </func> diff --git a/lib/erl_interface/include/ei.h b/lib/erl_interface/include/ei.h index a860df3f77..b138118f04 100644 --- a/lib/erl_interface/include/ei.h +++ b/lib/erl_interface/include/ei.h @@ -213,12 +213,12 @@ extern volatile int __erl_errno; * library and when using the library we set a value that we use */ -#define EI_MAXHOSTNAMELEN 64 -#define EI_MAXALIVELEN 63 #define EI_MAX_COOKIE_SIZE 512 #define MAXATOMLEN (255 + 1) #define MAXATOMLEN_UTF8 (255*4 + 1) -#define MAXNODELEN EI_MAXALIVELEN+1+EI_MAXHOSTNAMELEN +#define EI_MAXHOSTNAMELEN (MAXATOMLEN - 2) +#define EI_MAXALIVELEN (MAXATOMLEN - 2) +#define MAXNODELEN MAXATOMLEN typedef enum { ERLANG_ASCII = 1, @@ -526,9 +526,9 @@ int ei_x_encode_atom_len(ei_x_buff* x, const char* s, int len); int ei_x_encode_atom_len_as(ei_x_buff* x, const char* s, int len, erlang_char_encoding from, erlang_char_encoding to); int ei_encode_binary(char *buf, int *index, const void *p, long len); -int ei_encode_bitstring(char *buf, int *index, const void *p, size_t bits); +int ei_encode_bitstring(char *buf, int *index, const char *p, size_t bitoffs, size_t bits); int ei_x_encode_binary(ei_x_buff* x, const void* s, int len); -int ei_x_encode_bitstring(ei_x_buff* x, const void* p, size_t bits); +int ei_x_encode_bitstring(ei_x_buff* x, const char* p, size_t bitoffs, size_t bits); int ei_encode_pid(char *buf, int *index, const erlang_pid *p); int ei_x_encode_pid(ei_x_buff* x, const erlang_pid* pid); int ei_encode_fun(char* buf, int* index, const erlang_fun* p); @@ -578,7 +578,9 @@ int ei_decode_string(const char *buf, int *index, char *p); int ei_decode_atom(const char *buf, int *index, char *p); int ei_decode_atom_as(const char *buf, int *index, char *p, int destlen, erlang_char_encoding want, erlang_char_encoding* was, erlang_char_encoding* result); int ei_decode_binary(const char *buf, int *index, void *p, long *len); -int ei_decode_bitstring(const char *buf, int *index, void *p, size_t plen, size_t *bitsp); +int ei_decode_bitstring(const char *buf, int *index, const char** pp, + unsigned int* bitoffsp, size_t *nbitsp); + int ei_decode_fun(const char* buf, int* index, erlang_fun* p); void free_fun(erlang_fun* f); int ei_decode_pid(const char *buf, int *index, erlang_pid *p); diff --git a/lib/erl_interface/src/connect/ei_connect.c b/lib/erl_interface/src/connect/ei_connect.c index 0cbad235cc..1b1479d2e9 100644 --- a/lib/erl_interface/src/connect/ei_connect.c +++ b/lib/erl_interface/src/connect/ei_connect.c @@ -791,14 +791,17 @@ int ei_connect_init_ussi(ei_cnode* ec, const char* this_node_name, if (strcmp(hp->h_name, "localhost") == 0) { /* We use a short node name */ if ((ct = strchr(thishostname, '.')) != NULL) *ct = '\0'; - sprintf(thisnodename, "%s@%s", this_node_name, thishostname); } else { /* We use a short node name */ if ((ct = strchr(hp->h_name, '.')) != NULL) *ct = '\0'; strcpy(thishostname, hp->h_name); - sprintf(thisnodename, "%s@%s", this_node_name, hp->h_name); } } + if (strlen(this_node_name) + 1 + strlen(thishostname) > MAXNODELEN) { + EI_TRACE_ERR0("ei_connect_init_ussi","this node name is too long"); + return ERL_ERROR; + } + sprintf(thisnodename, "%s@%s", this_node_name, thishostname); res = ei_connect_xinit_ussi(ec, thishostname, thisalivename, thisnodename, (struct in_addr *)*hp->h_addr_list, cookie, creation, cbs, cbs_sz, setup_context); @@ -889,6 +892,11 @@ int ei_connect_tmo(ei_cnode* ec, char *nodename, unsigned ms) int ei_h_errno; #endif /* !win32 */ int res; + + if (strlen(nodename) > MAXNODELEN) { + EI_TRACE_ERR0("ei_connect","Too long nodename"); + return ERL_ERROR; + } /* extract the host and alive parts from nodename */ if (!(hostname = strchr(nodename,'@'))) { diff --git a/lib/erl_interface/src/decode/decode_binary.c b/lib/erl_interface/src/decode/decode_binary.c index 2799438bef..0d28c67230 100644 --- a/lib/erl_interface/src/decode/decode_binary.c +++ b/lib/erl_interface/src/decode/decode_binary.c @@ -40,40 +40,41 @@ int ei_decode_binary(const char *buf, int *index, void *p, long *lenp) return 0; } -int ei_decode_bitstring(const char *buf, int *index, void *p, size_t plen, - size_t *bitsp) +int ei_decode_bitstring(const char *buf, int *index, + const char** pp, + unsigned int* bitoffsp, + size_t *nbitsp) { - const char *s = buf + *index; - const char *s0 = s; - unsigned long len; - unsigned char last_bits; - const unsigned char tag = get8(s); + const char *s = buf + *index; + const char *s0 = s; + unsigned char last_bits; + const unsigned char tag = get8(s); + size_t len = get32be(s); - if (tag == ERL_BINARY_EXT) { - long bytes; - int ret = ei_decode_binary(buf, index, p, &bytes); - if (bitsp) - *bitsp = (size_t)bytes * 8; - return ret; - } + switch(tag) { + case ERL_BINARY_EXT: + if (nbitsp) + *nbitsp = len * 8; + break; + case ERL_BIT_BINARY_EXT: + last_bits = get8(s); + if (((last_bits==0) != (len==0)) || last_bits > 8) + return -1; - if (tag != ERL_BIT_BINARY_EXT) - return -1; - - len = get32be(s); - last_bits = get8(s); - - if (len > plen || ((last_bits==0) != (len==0)) || last_bits > 8) - return -1; - - if (p) - memcpy(p, s, len); - s += len; + if (nbitsp) + *nbitsp = (len == 0) ? 0 : ((len-1) * 8) + last_bits; + break; + default: + return -1; + } - if (bitsp) - *bitsp = (len == 0) ? 0 : ((len-1) * 8) + last_bits; + if (pp) + *pp = s; + if (bitoffsp) + *bitoffsp = 0; - *index += s-s0; - return 0; + s += len; + *index += s-s0; + return 0; } diff --git a/lib/erl_interface/src/decode/decode_skip.c b/lib/erl_interface/src/decode/decode_skip.c index 11d3bc1786..736c00e074 100644 --- a/lib/erl_interface/src/decode/decode_skip.c +++ b/lib/erl_interface/src/decode/decode_skip.c @@ -21,14 +21,6 @@ #include "eiext.h" #include "decode_skip.h" -#ifdef HAVE_STDINT_H -# include <stdint.h> -#endif - -#ifndef SIZE_MAX -# define SIZE_MAX (~((size_t)0)) -#endif - int ei_skip_term(const char* buf, int* index) { int i, n, ty; @@ -88,7 +80,7 @@ int ei_skip_term(const char* buf, int* index) return -1; break; case ERL_BIT_BINARY_EXT: - if (ei_decode_bitstring(buf, index, NULL, SIZE_MAX, NULL) < 0) + if (ei_decode_bitstring(buf, index, NULL, NULL, NULL) < 0) return -1; break; case ERL_SMALL_INTEGER_EXT: diff --git a/lib/erl_interface/src/encode/encode_binary.c b/lib/erl_interface/src/encode/encode_binary.c index 4aa9f6bc16..0562979417 100644 --- a/lib/erl_interface/src/encode/encode_binary.c +++ b/lib/erl_interface/src/encode/encode_binary.c @@ -22,6 +22,10 @@ #include "eiext.h" #include "putget.h" +static void copy_bits(const unsigned char* src, size_t soffs, + unsigned char* dst, size_t n); + + int ei_encode_binary(char *buf, int *index, const void *p, long len) { char *s = buf + *index; @@ -40,23 +44,28 @@ int ei_encode_binary(char *buf, int *index, const void *p, long len) return 0; } -int ei_encode_bitstring(char *buf, int *index, const void *p, size_t bits) +int ei_encode_bitstring(char *buf, int *index, + const char *p, + size_t bitoffs, + size_t bits) { char *s = buf + *index; char *s0 = s; size_t bytes = (bits + 7) / 8; char last_bits = bits % 8; - if (bytes == 0 || last_bits == 0) - return ei_encode_binary(buf, index, p, bytes); - - if (!buf) s += 6; + if (!buf) s += last_bits ? 6 : 5; else { - put8(s, ERL_BIT_BINARY_EXT); + char* tagp = s++; put32be(s, bytes); - put8(s, last_bits); - memcpy(s, p, bytes); - s[bytes-1] &= (0xff << (8-last_bits)); + if (last_bits) { + *tagp = ERL_BIT_BINARY_EXT; + put8(s, last_bits); + } + else + *tagp = ERL_BINARY_EXT; + + copy_bits((const unsigned char*)p, bitoffs, (unsigned char*)s, bits); } s += bytes; @@ -64,3 +73,77 @@ int ei_encode_bitstring(char *buf, int *index, const void *p, size_t bits) return 0; } + + +/* + * MAKE_MASK(n) constructs a mask with n bits. + * Example: MAKE_MASK(3) returns the binary number 00000111. + */ +#define MAKE_MASK(n) ((((unsigned) 1) << (n))-1) + + +static +void copy_bits(const unsigned char* src, /* Base pointer to source. */ + size_t soffs, /* Bit offset for source relative to src. */ + unsigned char* dst, /* Destination. */ + size_t n) /* Number of bits to copy. */ +{ + unsigned rmask; + unsigned count; + unsigned deoffs; + unsigned bits; + unsigned bits1; + unsigned rshift; + + if (n == 0) + return; + + deoffs = n & 7; + rmask = deoffs ? (MAKE_MASK(deoffs) << (8-deoffs)) : 0; + + if (soffs == 0) { + unsigned nbytes = (n + 7) / 8; + memcpy(dst, src, nbytes); + if (rmask) + dst[nbytes-1] &= rmask; + return; + } + + src += soffs / 8; + soffs &= 7; + + if (n < 8) { /* Less than one byte */ + bits = (*src << soffs); + if (soffs+n > 8) { + src++; + bits |= (*src >> (8 - soffs)); + } + *dst = bits & rmask; + return; + } + + count = n >> 3; + + rshift = 8 - soffs; + bits = *src; + if (soffs + n > 8) { + src++; + } + + while (count--) { + bits1 = bits << soffs; + bits = *src; + src++; + *dst = bits1 | (bits >> rshift); + dst++; + } + + if (rmask) { + bits1 = bits << soffs; + if ((rmask << rshift) & 0xff) { + bits = *src; + bits1 |= (bits >> rshift); + } + *dst = bits1 & rmask; + } +} diff --git a/lib/erl_interface/src/misc/ei_printterm.c b/lib/erl_interface/src/misc/ei_printterm.c index a89b990ac1..5c40fb7747 100644 --- a/lib/erl_interface/src/misc/ei_printterm.c +++ b/lib/erl_interface/src/misc/ei_printterm.c @@ -250,12 +250,13 @@ static int print_term(FILE* fp, ei_x_buff* x, ei_free(p); break; case ERL_BIT_BINARY_EXT: { + const char* cp; size_t bits; + unsigned int bitoffs; int trunc = 0; - p = ei_malloc(n); - if (p == NULL) goto err; - if (ei_decode_bitstring(buf, index, p, n, &bits) < 0) { - ei_free(p); + + if (ei_decode_bitstring(buf, index, &cp, &bitoffs, &bits) < 0 + || bitoffs != 0) { goto err; } ch_written += xprintf(fp, x, "#Bits<"); @@ -266,15 +267,14 @@ static int print_term(FILE* fp, ei_x_buff* x, } --m; for (i = 0; i < m; ++i) { - ch_written += xprintf(fp, x, "%d,", p[i]); + ch_written += xprintf(fp, x, "%d,", cp[i]); } - ch_written += xprintf(fp, x, "%d", p[i]); + ch_written += xprintf(fp, x, "%d", cp[i]); if (trunc) ch_written += xprintf(fp, x, ",..."); else if (bits % 8 != 0) ch_written += xprintf(fp, x, ":%u", (unsigned)(bits % 8)); xputc('>', fp, x); ++ch_written; - ei_free(p); break; } case ERL_SMALL_INTEGER_EXT: diff --git a/lib/erl_interface/src/misc/ei_x_encode.c b/lib/erl_interface/src/misc/ei_x_encode.c index 2da271795f..8e77679d2a 100644 --- a/lib/erl_interface/src/misc/ei_x_encode.c +++ b/lib/erl_interface/src/misc/ei_x_encode.c @@ -117,14 +117,14 @@ int ei_x_encode_binary(ei_x_buff* x, const void* p, int len) return ei_encode_binary(x->buff, &x->index, p, len); } -int ei_x_encode_bitstring(ei_x_buff* x, const void* p, size_t bits) +int ei_x_encode_bitstring(ei_x_buff* x, const char* p, size_t bitoffs, size_t bits) { int i = x->index; - if (ei_encode_bitstring(NULL, &i, p, bits) == -1) + if (ei_encode_bitstring(NULL, &i, p, bitoffs, bits) == -1) return -1; if (!x_fix_buff(x, i)) return -1; - return ei_encode_bitstring(x->buff, &x->index, p, bits); + return ei_encode_bitstring(x->buff, &x->index, p, bitoffs, bits); } int ei_x_encode_long(ei_x_buff* x, long n) diff --git a/lib/erl_interface/src/misc/show_msg.c b/lib/erl_interface/src/misc/show_msg.c index 2d49eb6449..805d69e9b3 100644 --- a/lib/erl_interface/src/misc/show_msg.c +++ b/lib/erl_interface/src/misc/show_msg.c @@ -24,13 +24,6 @@ #include <stdlib.h> #include <stdarg.h> #include <string.h> -#ifdef HAVE_STDINT_H -# include <stdint.h> -#endif - -#ifndef SIZE_MAX -# define SIZE_MAX (~((size_t)0)) -#endif #include <sys/types.h> @@ -464,7 +457,7 @@ static void show_term(const char *termbuf, int *index, FILE *stream) case ERL_BIT_BINARY_EXT: { size_t bits; - ei_decode_bitstring(termbuf, index, NULL, SIZE_MAX, &bits); + ei_decode_bitstring(termbuf, index, NULL, NULL, &bits); fprintf(stream, "#Bits<%lu>", (unsigned long)bits); break; } diff --git a/lib/erl_interface/test/ei_connect_SUITE_data/ei_connect_test.c b/lib/erl_interface/test/ei_connect_SUITE_data/ei_connect_test.c index 7c9e79f837..385bcdd422 100644 --- a/lib/erl_interface/test/ei_connect_SUITE_data/ei_connect_test.c +++ b/lib/erl_interface/test/ei_connect_SUITE_data/ei_connect_test.c @@ -209,8 +209,9 @@ static void cmd_ei_send_funs(char* buf, int len) erlang_pid pid; ei_x_buff x; erlang_fun fun1, fun2; - unsigned char bitstring[10]; + char* bitstring; size_t bits; + int bitoffs; if (ei_decode_long(buf, &index, &fd) < 0) fail("expected long"); @@ -224,7 +225,7 @@ static void cmd_ei_send_funs(char* buf, int len) fail("expected Fun1"); if (ei_decode_fun(buf, &index, &fun2) < 0) fail("expected Fun2"); - if (ei_decode_bitstring(buf, &index, bitstring, sizeof(bitstring), &bits) < 0) + if (ei_decode_bitstring(buf, &index, &bitstring, &bitoffs, &bits) < 0) fail("expected bitstring"); if (ei_x_new_with_version(&x) < 0) fail("ei_x_new_with_version"); @@ -234,7 +235,7 @@ static void cmd_ei_send_funs(char* buf, int len) fail("encode fun1"); if (ei_x_encode_fun(&x, &fun2) < 0) fail("encode fun2"); - if (ei_x_encode_bitstring(&x, bitstring, bits) < 0) + if (ei_x_encode_bitstring(&x, bitstring, bitoffs, bits) < 0) fail("encode bitstring"); free_fun(&fun1); free_fun(&fun2); diff --git a/lib/erl_interface/test/ei_decode_SUITE_data/ei_decode_test.c b/lib/erl_interface/test/ei_decode_SUITE_data/ei_decode_test.c index d39970a857..46d6b8f2af 100644 --- a/lib/erl_interface/test/ei_decode_SUITE_data/ei_decode_test.c +++ b/lib/erl_interface/test/ei_decode_SUITE_data/ei_decode_test.c @@ -319,17 +319,18 @@ static void decode_bin(int exp_size, const char* val, int exp_len) static void decode_bits(int exp_size, const char* val, size_t exp_bits) { - char p[1024]; + const char* p; char *buf; size_t bits; + int bitoffs; int size1 = 0; int size2 = 0; int err; message("ei_decode_bitstring should be %d bits", (int)exp_bits); buf = read_packet(NULL); - err = ei_decode_bitstring(buf+1, &size1, NULL, sizeof(p), &bits); - message("err = %d, size = %d, len = %d, expected size = %d, expected bits = %d\n",\ - err,size1, (int)bits, exp_size, (int)exp_bits); + err = ei_decode_bitstring(buf+1, &size1, NULL, &bitoffs, &bits); + message("err = %d, size = %d, bitoffs = %d, bits = %d, expected size = %d, expected bits = %d\n",\ + err,size1, bitoffs, (int)bits, exp_size, (int)exp_bits); if (err != 0) { if (err != -1) { @@ -344,8 +345,12 @@ static void decode_bits(int exp_size, const char* val, size_t exp_bits) fail("number of bits is not correct"); return; } + if (bitoffs != 0) { + fail("non zero bit offset"); + return; + } - err = ei_decode_bitstring(buf+1, &size2, p, sizeof(p), &bits); + err = ei_decode_bitstring(buf+1, &size2, &p, NULL, &bits); message("err = %d, size = %d, len = %d, expected size = %d, expected len = %d\n",\ err,size2, (int)bits, exp_size, (int)exp_bits); if (err != 0) { diff --git a/lib/erl_interface/test/ei_decode_encode_SUITE.erl b/lib/erl_interface/test/ei_decode_encode_SUITE.erl index d8b0bce3ae..3451d9f503 100644 --- a/lib/erl_interface/test/ei_decode_encode_SUITE.erl +++ b/lib/erl_interface/test/ei_decode_encode_SUITE.erl @@ -122,9 +122,29 @@ test_ei_decode_encode(Config) when is_list(Config) -> [send_rec(P, <<16#dec0deb175:B/little>>) || B <- lists:seq(0,48)], + % And last an ugly duckling to test ei_encode_bitstring with bitoffs != 0 + encode_bitstring(P), + runner:recv_eot(P), ok. +encode_bitstring(P) -> + %% Send one bitstring to c-node + Bits = <<16#18f6d4b2907e5c3a1:66>>, + P ! {self(), {command, term_to_binary(Bits, [{minor_version, 2}])}}, + + %% and then receive and verify a number of different sub-bitstrings + receive_sub_bitstring(P, Bits, 0, bit_size(Bits)). + +receive_sub_bitstring(_, _, _, NBits) when NBits < 0 -> + ok; +receive_sub_bitstring(P, Bits, BitOffs, NBits) -> + <<_:BitOffs, Sub:NBits/bits, _/bits>> = Bits, + %%io:format("expecting term_to_binary(~p) = ~p\n", [Sub, term_to_binary(Sub)]), + {_B,Sub} = get_buf_and_term(P), + receive_sub_bitstring(P, Bits, BitOffs+1, NBits - ((NBits div 20)+1)). + + %% ######################################################################## %% diff --git a/lib/erl_interface/test/ei_decode_encode_SUITE_data/ei_decode_encode_test.c b/lib/erl_interface/test/ei_decode_encode_SUITE_data/ei_decode_encode_test.c index f9c05b2739..85ca6c56e9 100644 --- a/lib/erl_interface/test/ei_decode_encode_SUITE_data/ei_decode_encode_test.c +++ b/lib/erl_interface/test/ei_decode_encode_SUITE_data/ei_decode_encode_test.c @@ -42,7 +42,8 @@ typedef struct typedef struct { - char bytes[MAXATOMLEN_UTF8]; + const char* bytes; + unsigned int bitoffs; size_t nbits; }my_bitstring; @@ -128,17 +129,17 @@ struct Type my_atom_type = { int ei_decode_my_bits(const char *buf, int *index, my_bitstring* a) { - return ei_decode_bitstring(buf, index, (a ? a->bytes : NULL), - sizeof(a->bytes), + return ei_decode_bitstring(buf, index, (a ? &a->bytes : NULL), + (a ? &a->bitoffs : NULL), (a ? &a->nbits : NULL)); } int ei_encode_my_bits(char *buf, int *index, my_bitstring* a) { - return ei_encode_bitstring(buf, index, a->bytes, a->nbits); + return ei_encode_bitstring(buf, index, a->bytes, a->bitoffs, a->nbits); } int ei_x_encode_my_bits(ei_x_buff* x, my_bitstring* a) { - return ei_x_encode_bitstring(x, a->bytes, a->nbits); + return ei_x_encode_bitstring(x, a->bytes, a->bitoffs, a->nbits); } struct Type my_bitstring_type = { @@ -264,11 +265,7 @@ void decode_encode(struct Type** tv, int nobj) size1 = 0; err = t->ei_decode_fp(inp, &size1, NULL); if (err != 0) { - if (err != -1) { - fail("decode returned non zero but not -1"); - } else { - fail1("decode '%s' returned non zero", t->name); - } + fail2("decode '%s' returned non zero %d", t->name, err); return; } if (size1 < 1) { @@ -497,6 +494,66 @@ void decode_encode_big(struct Type* t) } +void encode_bitstring(void) +{ + char* packet; + char* inp; + char out_buf[BUFSZ]; + int size; + int err, i; + ei_x_buff arg; + const char* p; + unsigned int bitoffs; + size_t nbits, org_nbits; + + packet = read_packet(NULL); + inp = packet+1; + + size = 0; + err = ei_decode_bitstring(inp, &size, &p, &bitoffs, &nbits); + if (err != 0) { + fail1("ei_decode_bitstring returned non zero %d", err); + return; + } + + /* + * Now send a bunch of different sub-bitstrings back + * encoded both with ei_encode_ and ei_x_encode_. + */ + org_nbits = nbits; + do { + size = 0; + err = ei_encode_bitstring(out_buf, &size, p, bitoffs, nbits); + if (err != 0) { + fail1("ei_encode_bitstring returned non zero %d", err); + return; + } + + ei_x_new(&arg); + err = ei_x_encode_bitstring(&arg, p, bitoffs, nbits); + if (err != 0) { + fail1("ei_x_encode_bitstring returned non zero %d", err); + ei_x_free(&arg); + return; + } + + if (arg.index < 1) { + fail("size is < 1"); + ei_x_free(&arg); + return; + } + + send_buffer(out_buf, size); + send_buffer(arg.buff, arg.index); + ei_x_free(&arg); + + bitoffs++; + nbits -= (nbits / 20) + 1; + } while (nbits < org_nbits); + + free_packet(packet); +} + /* ******************************************************************** */ @@ -568,6 +625,8 @@ TESTCASE(test_ei_decode_encode) decode_encode_one(&my_bitstring_type); } + encode_bitstring(); + report(1); } diff --git a/lib/kernel/src/erl_distribution.erl b/lib/kernel/src/erl_distribution.erl index 0bec78e938..cdb2d2f1f6 100644 --- a/lib/kernel/src/erl_distribution.erl +++ b/lib/kernel/src/erl_distribution.erl @@ -21,6 +21,8 @@ -behaviour(supervisor). +-include_lib("kernel/include/logger.hrl"). + -export([start_link/0,start_link/2,init/1,start/1,stop/0]). -define(DBG,erlang:display([?MODULE,?LINE])). @@ -83,6 +85,10 @@ do_start_link([{Arg,Flag}|T]) -> case init:get_argument(Arg) of {ok,[[Name]]} -> start_link([list_to_atom(Name),Flag|ticktime()], true); + {ok,[[Name]|_Rest]} -> + ?LOG_WARNING("Multiple -~p given to erl, using the first, ~p", + [Arg, Name]), + start_link([list_to_atom(Name),Flag|ticktime()], true); _ -> do_start_link(T) end; diff --git a/lib/kernel/src/user_drv.erl b/lib/kernel/src/user_drv.erl index 08286dd476..69ff8e7971 100644 --- a/lib/kernel/src/user_drv.erl +++ b/lib/kernel/src/user_drv.erl @@ -120,7 +120,7 @@ server1(Iport, Oport, Shell) -> {Curr,Shell1} = case init:get_argument(remsh) of {ok,[[Node]]} -> - ANode = list_to_atom(Node), + ANode = list_to_atom(append_hostname(Node)), RShell = {ANode,shell,start,[]}, RGr = group:start(self(), RShell, rem_sh_opts(ANode)), {RGr,RShell}; @@ -139,6 +139,12 @@ server1(Iport, Oport, Shell) -> %% Enter the server loop. server_loop(Iport, Oport, Curr, User, Gr, {false, queue:new()}). +append_hostname(Node) -> + case string:find(Node, "@") of + nomatch -> Node ++ string:find(atom_to_list(node()), "@"); + _ -> Node + end. + rem_sh_opts(Node) -> [{expand_fun,fun(B)-> rpc:call(Node,edlin_expand,expand,[B]) end}]. diff --git a/lib/kernel/test/erl_distribution_SUITE.erl b/lib/kernel/test/erl_distribution_SUITE.erl index 8dd4ef1987..c3a022df0a 100644 --- a/lib/kernel/test/erl_distribution_SUITE.erl +++ b/lib/kernel/test/erl_distribution_SUITE.erl @@ -205,6 +205,9 @@ nodenames(Config) when is_list(Config) -> legal("a-1@b"), legal("a_1@b"), + %% Test that giving two -sname works as it should + test_node("a_1@b", false, long_or_short() ++ "a_0@b"), + illegal("cdé@a"), illegal("te欢st@a"). @@ -258,8 +261,11 @@ illegal(Name) -> test_node(Name) -> test_node(Name, false). test_node(Name, Illigal) -> + test_node(Name, Illigal, ""). +test_node(Name, Illigal, ExtraArgs) -> ProgName = ct:get_progname(), - Command = ProgName ++ " -noinput " ++ long_or_short() ++ Name ++ + Command = ProgName ++ " -noinput " ++ ExtraArgs ++ + long_or_short() ++ Name ++ " -eval \"net_adm:ping('" ++ atom_to_list(node()) ++ "')\"" ++ case Illigal of true -> diff --git a/lib/sasl/src/Makefile b/lib/sasl/src/Makefile index 7338bdf016..fd62588f5c 100644 --- a/lib/sasl/src/Makefile +++ b/lib/sasl/src/Makefile @@ -61,7 +61,11 @@ TARGET_FILES= $(MODULES:%=$(EBIN)/%.$(EMULATOR)) $(APP_TARGET) $(APPUP_TARGET) # ---------------------------------------------------- # FLAGS # ---------------------------------------------------- + ERL_COMPILE_FLAGS += -I../../stdlib/include -Werror +ifeq ($(USE_ESOCK), yes) +ERL_COMPILE_FLAGS += -DUSE_ESOCK=true +endif # ---------------------------------------------------- diff --git a/lib/sasl/src/systools_make.erl b/lib/sasl/src/systools_make.erl index c2c91fd667..b5a6b44f93 100644 --- a/lib/sasl/src/systools_make.erl +++ b/lib/sasl/src/systools_make.erl @@ -33,6 +33,7 @@ -export([read_application/4]). -export([make_hybrid_boot/4]). +-export([preloaded/0]). % Exported just for testing -import(lists, [filter/2, keysort/2, keysearch/3, map/2, reverse/1, append/1, foldl/3, member/2, foreach/2]). @@ -45,6 +46,13 @@ -compile({inline,[{badarg,2}]}). +-ifdef(USE_ESOCK). +-define(ESOCK_MODS, [socket]). +-else. +-define(ESOCK_MODS, []). +-endif. + + %%----------------------------------------------------------------- %% Create a boot script from a release file. %% Options is a list of {path, Path} | silent | local @@ -1566,7 +1574,7 @@ preloaded() -> erts_code_purger,erts_dirty_process_signal_handler, erts_internal,erts_literal_area_collector, init,net,persistent_term,prim_buffer,prim_eval,prim_file, - prim_inet,prim_zip,socket,zlib]. + prim_inet,prim_zip] ++ ?ESOCK_MODS ++ [zlib]. %%______________________________________________________________________ %% Kernel processes; processes that are specially treated by the init diff --git a/lib/ssh/src/ssh.app.src b/lib/ssh/src/ssh.app.src index 410061cded..7449405d20 100644 --- a/lib/ssh/src/ssh.app.src +++ b/lib/ssh/src/ssh.app.src @@ -44,10 +44,10 @@ {env, []}, {mod, {ssh_app, []}}, {runtime_dependencies, [ - "crypto-4.2", - "erts-6.0", - "kernel-3.0", - "public_key-1.5.2", - "stdlib-3.3" + "crypto-@OTP-15644@", + "erts-9.0", + "kernel-5.3", + "public_key-1.6.1", + "stdlib-3.4.1" ]}]}. diff --git a/lib/ssl/src/tls_connection.erl b/lib/ssl/src/tls_connection.erl index a05858221a..872a557e67 100644 --- a/lib/ssl/src/tls_connection.erl +++ b/lib/ssl/src/tls_connection.erl @@ -171,19 +171,21 @@ next_record(#state{protocol_buffers = connection_states = ConnectionStates, ssl_options = #ssl_options{padding_check = Check}} = State) -> next_record(State, CipherTexts, ConnectionStates, Check); -next_record(#state{protocol_buffers = #protocol_buffers{tls_cipher_texts = []}, - protocol_specific = #{active_n_toggle := true, active_n := N} = ProtocolSpec, +next_record(#state{user_data_buffer = {_,0,_}, + protocol_buffers = #protocol_buffers{tls_cipher_texts = []}, + protocol_specific = #{active_n_toggle := true, + active_n := N} = ProtocolSpec, static_env = #static_env{socket = Socket, close_tag = CloseTag, transport_cb = Transport} - } = State) -> + } = State) -> case tls_socket:setopts(Transport, Socket, [{active, N}]) of ok -> - {no_record, State#state{protocol_specific = ProtocolSpec#{active_n_toggle => false}}}; + {no_record, State#state{protocol_specific = ProtocolSpec#{active_n_toggle => false}}}; _ -> - self() ! {CloseTag, Socket}, - {no_record, State} - end; + self() ! {CloseTag, Socket}, + {no_record, State} + end; next_record(State) -> {no_record, State}. diff --git a/lib/ssl/test/Makefile b/lib/ssl/test/Makefile index f7fae16088..dba90aaff0 100644 --- a/lib/ssl/test/Makefile +++ b/lib/ssl/test/Makefile @@ -44,6 +44,7 @@ MODULES = \ ssl_bench_SUITE \ ssl_cipher_SUITE \ ssl_cipher_suite_SUITE \ + openssl_server_cipher_suite_SUITE\ ssl_certificate_verify_SUITE\ ssl_crl_SUITE\ ssl_dist_SUITE \ diff --git a/lib/ssl/test/openssl_server_cipher_suite_SUITE.erl b/lib/ssl/test/openssl_server_cipher_suite_SUITE.erl new file mode 100644 index 0000000000..907de1abe2 --- /dev/null +++ b/lib/ssl/test/openssl_server_cipher_suite_SUITE.erl @@ -0,0 +1,768 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2019-2019. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% + +-module(openssl_server_cipher_suite_SUITE). + +%% Note: This directive should only be used in test suites. +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +%%-------------------------------------------------------------------- +%% Common Test interface functions ----------------------------------- +%%-------------------------------------------------------------------- +all() -> + [ + {group, 'tlsv1.2'}, + {group, 'tlsv1.1'}, + {group, 'tlsv1'}, + {group, 'sslv3'}, + {group, 'dtlsv1.2'}, + {group, 'dtlsv1'} + ]. + +groups() -> + %% TODO: Enable SRP, PSK suites (needs OpenSSL s_server conf) + %% TODO: Enable all "kex" on DTLS + [ + {'tlsv1.2', [], kex()}, + {'tlsv1.1', [], kex()}, + {'tlsv1', [], kex()}, + {'sslv3', [], kex()}, + {'dtlsv1.2', [], dtls_kex()}, + {'dtlsv1', [], dtls_kex()}, + {dhe_rsa, [],[dhe_rsa_3des_ede_cbc, + dhe_rsa_aes_128_cbc, + dhe_rsa_aes_256_cbc, + dhe_rsa_chacha20_poly1305 + ]}, + {ecdhe_rsa, [], [ecdhe_rsa_3des_ede_cbc, + ecdhe_rsa_aes_128_cbc, + ecdhe_rsa_aes_128_gcm, + ecdhe_rsa_aes_256_cbc, + ecdhe_rsa_aes_256_gcm, + ecdhe_rsa_chacha20_poly1305 + ]}, + {ecdhe_ecdsa, [],[ecdhe_ecdsa_rc4_128, + ecdhe_ecdsa_3des_ede_cbc, + ecdhe_ecdsa_aes_128_cbc, + ecdhe_ecdsa_aes_128_gcm, + ecdhe_ecdsa_aes_256_cbc, + ecdhe_ecdsa_aes_256_gcm, + ecdhe_ecdsa_chacha20_poly1305 + ]}, + {rsa, [], [rsa_3des_ede_cbc, + rsa_aes_128_cbc, + rsa_aes_256_cbc, + rsa_rc4_128 + ]}, + {dhe_dss, [], [dhe_dss_3des_ede_cbc, + dhe_dss_aes_128_cbc, + dhe_dss_aes_256_cbc]}, + %% {srp_rsa, [], [srp_rsa_3des_ede_cbc, + %% srp_rsa_aes_128_cbc, + %% srp_rsa_aes_256_cbc]}, + %% {srp_dss, [], [srp_dss_3des_ede_cbc, + %% srp_dss_aes_128_cbc, + %% srp_dss_aes_256_cbc]}, + %% {rsa_psk, [], [rsa_psk_3des_ede_cbc, + %% rsa_psk_rc4_128, + %% rsa_psk_aes_128_cbc, + %% rsa_psk_aes_256_cbc + %% ]}, + {dh_anon, [], [dh_anon_rc4_128, + dh_anon_3des_ede_cbc, + dh_anon_aes_128_cbc, + dh_anon_aes_128_gcm, + dh_anon_aes_256_cbc, + dh_anon_aes_256_gcm]}, + {ecdh_anon, [], [ecdh_anon_3des_ede_cbc, + ecdh_anon_aes_128_cbc, + ecdh_anon_aes_256_cbc + ]} + %% {srp_anon, [], [srp_anon_3des_ede_cbc, + %% srp_anon_aes_128_cbc, + %% srp_anon_aes_256_cbc]}, + %% {psk, [], [psk_3des_ede_cbc, + %% psk_rc4_128, + %% psk_aes_128_cbc, + %% psk_aes_128_ccm, + %% psk_aes_128_ccm_8, + %% psk_aes_256_cbc, + %% psk_aes_256_ccm, + %% psk_aes_256_ccm_8 + %% ]}, + %% {dhe_psk, [], [dhe_psk_3des_ede_cbc, + %% dhe_psk_rc4_128, + %% dhe_psk_aes_128_cbc, + %% dhe_psk_aes_128_ccm, + %% dhe_psk_aes_128_ccm_8, + %% dhe_psk_aes_256_cbc, + %% dhe_psk_aes_256_ccm, + %% dhe_psk_aes_256_ccm_8 + %% ]}, + %% {ecdhe_psk, [], [ecdhe_psk_3des_ede_cbc, + %% ecdhe_psk_rc4_128, + %% ecdhe_psk_aes_128_cbc, + %% ecdhe_psk_aes_128_ccm, + %% ecdhe_psk_aes_128_ccm_8, + %% ecdhe_psk_aes_256_cbc + %% ]} + ]. + +kex() -> + rsa() ++ ecdsa() ++ dss() ++ anonymous(). + +dtls_kex() -> %% Should be all kex in the future + dtls_rsa() ++ dss() ++ anonymous(). + +rsa() -> + [{group, dhe_rsa}, + {group, ecdhe_rsa}, + {group, rsa} %%, {group, srp_rsa}, + %%{group, rsa_psk} + ]. + +dtls_rsa() -> + [ + {group, rsa} + %%,{group, rsa_psk} + ]. + +ecdsa() -> + [{group, ecdhe_ecdsa}]. + +dss() -> + [{group, dhe_dss} + %%{group, srp_dss} + ]. + +anonymous() -> + [{group, dh_anon}, + {group, ecdh_anon} + %% {group, psk}, + %%{group, dhe_psk}, + %%{group, ecdhe_psk} + %%{group, srp_anon} + ]. + +init_per_suite(Config) -> + catch crypto:stop(), + try crypto:start() of + ok -> + ssl_test_lib:clean_start(), + Config + catch _:_ -> + {skip, "Crypto did not start"} + end. + +end_per_suite(_Config) -> + ssl:stop(), + application:stop(crypto). + +%%-------------------------------------------------------------------- +init_per_group(GroupName, Config) -> + case ssl_test_lib:is_tls_version(GroupName) of + true -> + case ssl_test_lib:supports_ssl_tls_version(GroupName) of + true -> + do_init_per_group(GroupName, Config); + false -> + {skip, {openssl_does_not_support, GroupName}} + end; + false -> + do_init_per_group(GroupName, Config) + end. + +do_init_per_group(GroupName, Config) when GroupName == ecdh_anon; + GroupName == ecdhe_rsa; + GroupName == ecdhe_psk -> + case proplists:get_bool(ecdh, proplists:get_value(public_keys, crypto:supports())) of + true -> + init_certs(GroupName, Config); + false -> + {skip, "Missing EC crypto support"} + end; +do_init_per_group(ecdhe_ecdsa = GroupName, Config) -> + PKAlg = proplists:get_value(public_keys, crypto:supports()), + case lists:member(ecdh, PKAlg) andalso lists:member(ecdsa, PKAlg) of + true -> + init_certs(GroupName, Config); + false -> + {skip, "Missing EC crypto support"} + end; +do_init_per_group(dhe_dss = GroupName, Config) -> + PKAlg = proplists:get_value(public_keys, crypto:supports()), + case lists:member(dss, PKAlg) andalso lists:member(dh, PKAlg) of + true -> + init_certs(GroupName, Config); + false -> + {skip, "Missing DSS crypto support"} + end; +do_init_per_group(srp_dss = GroupName, Config) -> + PKAlg = proplists:get_value(public_keys, crypto:supports()), + case lists:member(dss, PKAlg) andalso lists:member(srp, PKAlg) of + true -> + init_certs(GroupName, Config); + false -> + {skip, "Missing DSS_SRP crypto support"} + end; +do_init_per_group(GroupName, Config) when GroupName == srp_anon; + GroupName == srp_rsa -> + PKAlg = proplists:get_value(public_keys, crypto:supports()), + case lists:member(srp, PKAlg) of + true -> + init_certs(GroupName, Config); + false -> + {skip, "Missing SRP crypto support"} + end; +do_init_per_group(dhe_psk = GroupName, Config) -> + PKAlg = proplists:get_value(public_keys, crypto:supports()), + case lists:member(dh, PKAlg) of + true -> + init_certs(GroupName, Config); + false -> + {skip, "Missing SRP crypto support"} + end; +do_init_per_group(GroupName, Config0) -> + case ssl_test_lib:is_tls_version(GroupName) of + true -> + ssl_test_lib:init_tls_version(GroupName, end_per_group(GroupName, Config0)); + false -> + init_certs(GroupName, Config0) + end. + +end_per_group(GroupName, Config) -> + case ssl_test_lib:is_tls_version(GroupName) of + true -> + ssl_test_lib:clean_tls_version(Config); + false -> + Config + end. + +init_per_testcase(TestCase, Config) when TestCase == psk_3des_ede_cbc; + TestCase == srp_anon_3des_ede_cbc; + TestCase == dhe_psk_3des_ede_cbc; + TestCase == ecdhe_psk_3des_ede_cbc; + TestCase == srp_rsa_3des_ede_cbc; + TestCase == srp_dss_3des_ede_cbc; + TestCase == rsa_psk_3des_ede_cbc; + TestCase == rsa_3des_ede_cbc; + TestCase == dhe_rsa_3des_ede_cbc; + TestCase == dhe_dss_3des_ede_cbc; + TestCase == ecdhe_rsa_3des_ede_cbc; + TestCase == srp_anon_dss_3des_ede_cbc; + TestCase == dh_anon_3des_ede_cbc; + TestCase == ecdh_anon_3des_ede_cbc; + TestCase == ecdhe_ecdsa_3des_ede_cbc -> + SupCiphers = proplists:get_value(ciphers, crypto:supports()), + case lists:member(des_ede3, SupCiphers) of + true -> + ct:timetrap({seconds, 5}), + Config; + _ -> + {skip, "Missing 3DES crypto support"} + end; +init_per_testcase(TestCase, Config) when TestCase == psk_rc4_128; + TestCase == ecdhe_psk_rc4_128; + TestCase == dhe_psk_rc4_128; + TestCase == rsa_psk_rc4_128; + TestCase == rsa_rc4_128; + TestCase == ecdhe_rsa_rc4_128; + TestCase == ecdhe_ecdsa_rc4_128; + TestCase == dh_anon_rc4_128 -> + SupCiphers = proplists:get_value(ciphers, crypto:supports()), + case lists:member(rc4, SupCiphers) of + true -> + ct:timetrap({seconds, 5}), + Config; + _ -> + {skip, "Missing RC4 crypto support"} + end; +init_per_testcase(TestCase, Config) when TestCase == psk_aes_128_ccm_8; + TestCase == rsa_psk_aes_128_ccm_8; + TestCase == psk_aes_128_ccm_8; + TestCase == dhe_psk_aes_128_ccm_8; + TestCase == ecdhe_psk_aes_128_ccm_8 -> + SupCiphers = proplists:get_value(ciphers, crypto:supports()), + case lists:member(aes_128_ccm, SupCiphers) of + true -> + ct:timetrap({seconds, 5}), + Config; + _ -> + {skip, "Missing AES_128_CCM crypto support"} + end; +init_per_testcase(TestCase, Config) when TestCase == psk_aes_256_ccm_8; + TestCase == rsa_psk_aes_256_ccm_8; + TestCase == psk_aes_256_ccm_8; + TestCase == dhe_psk_aes_256_ccm_8; + TestCase == ecdhe_psk_aes_256_ccm_8 -> + SupCiphers = proplists:get_value(ciphers, crypto:supports()), + case lists:member(aes_256_ccm, SupCiphers) of + true -> + ct:timetrap({seconds, 5}), + Config; + _ -> + {skip, "Missing AES_256_CCM crypto support"} + end; +init_per_testcase(TestCase, Config) -> + Cipher = ssl_test_lib:test_cipher(TestCase, Config), + SupCiphers = proplists:get_value(ciphers, crypto:supports()), + case lists:member(Cipher, SupCiphers) of + true -> + ct:timetrap({seconds, 5}), + Config; + _ -> + {skip, {Cipher, SupCiphers}} + end. + +end_per_testcase(_TestCase, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Initializtion ------------------------------------------ +%%-------------------------------------------------------------------- +init_certs(srp_rsa, Config) -> + {ClientOpts, ServerOpts} = ssl_test_lib:make_rsa_cert_chains([{server_chain, ssl_test_lib:default_cert_chain_conf()}, + {client_chain, ssl_test_lib:default_cert_chain_conf()}], + Config, ""), + [{tls_config, #{server_config => [{user_lookup_fun, {fun ssl_test_lib:user_lookup/3, undefined}} | ServerOpts], + client_config => [{srp_identity, {"Test-User", "secret"}} | ClientOpts]}} | + proplists:delete(tls_config, Config)]; +init_certs(srp_anon, Config) -> + [{tls_config, #{server_config => [{user_lookup_fun, {fun ssl_test_lib:user_lookup/3, undefined}}], + client_config => [{srp_identity, {"Test-User", "secret"}}]}} | + proplists:delete(tls_config, Config)]; +init_certs(rsa_psk, Config) -> + Ext = x509_test:extensions([{key_usage, [digitalSignature, keyEncipherment]}]), + {ClientOpts, ServerOpts} = ssl_test_lib:make_rsa_cert_chains([{server_chain, + [[ssl_test_lib:digest()],[ssl_test_lib:digest()], + [ssl_test_lib:digest(), {extensions, Ext}]]}, + {client_chain, ssl_test_lib:default_cert_chain_conf()}], + Config, "_peer_keyEncipherment"), + PskSharedSecret = <<1,2,3,4,5,6,7,8,9,10,11,12,13,14,15>>, + [{tls_config, #{server_config => [{user_lookup_fun, {fun ssl_test_lib:user_lookup/3, PskSharedSecret}} | ServerOpts], + client_config => [{psk_identity, "Test-User"}, + {user_lookup_fun, {fun ssl_test_lib:user_lookup/3, PskSharedSecret}} | ClientOpts]}} | + proplists:delete(tls_config, Config)]; +init_certs(rsa, Config) -> + Ext = x509_test:extensions([{key_usage, [digitalSignature, keyEncipherment]}]), + {ClientOpts, ServerOpts} = ssl_test_lib:make_rsa_cert_chains([{server_chain, + [[ssl_test_lib:digest()],[ssl_test_lib:digest()], + [ssl_test_lib:digest(), {extensions, Ext}]]} + ], + Config, "_peer_keyEncipherment"), + [{tls_config, #{server_config => ServerOpts, + client_config => ClientOpts}} | + proplists:delete(tls_config, Config)]; +init_certs(dhe_dss, Config) -> + {ClientOpts, ServerOpts} = ssl_test_lib:make_dsa_cert_chains([{server_chain, ssl_test_lib:default_cert_chain_conf()}, + {client_chain, ssl_test_lib:default_cert_chain_conf()}], + Config, ""), + [{tls_config, #{server_config => ServerOpts, + client_config => ClientOpts}} | + proplists:delete(tls_config, Config)]; +init_certs(srp_dss, Config) -> + {ClientOpts, ServerOpts} = ssl_test_lib:make_dsa_cert_chains([{server_chain, ssl_test_lib:default_cert_chain_conf()}, + {client_chain, ssl_test_lib:default_cert_chain_conf()}], + Config, ""), + [{tls_config, #{server_config => [{user_lookup_fun, {fun ssl_test_lib:user_lookup/3, undefined}} | ServerOpts], + client_config => [{srp_identity, {"Test-User", "secret"}} | ClientOpts]}} | + proplists:delete(tls_config, Config)]; +init_certs(GroupName, Config) when GroupName == dhe_rsa; + GroupName == ecdhe_rsa -> + {ClientOpts, ServerOpts} = ssl_test_lib:make_rsa_cert_chains([{server_chain, ssl_test_lib:default_cert_chain_conf()}, + {client_chain, ssl_test_lib:default_cert_chain_conf()}], + Config, ""), + [{tls_config, #{server_config => ServerOpts, + client_config => ClientOpts}} | + proplists:delete(tls_config, Config)]; +init_certs(GroupName, Config) when GroupName == dhe_ecdsa; + GroupName == ecdhe_ecdsa -> + {ClientOpts, ServerOpts} = ssl_test_lib:make_ecc_cert_chains([{server_chain, ssl_test_lib:default_cert_chain_conf()}, + {client_chain, ssl_test_lib:default_cert_chain_conf()}], + Config, ""), + [{tls_config, #{server_config => ServerOpts, + client_config => ClientOpts}} | + proplists:delete(tls_config, Config)]; +init_certs(GroupName, Config) when GroupName == psk; + GroupName == dhe_psk; + GroupName == ecdhe_psk -> + PskSharedSecret = <<1,2,3,4,5,6,7,8,9,10,11,12,13,14,15>>, + [{tls_config, #{server_config => [{user_lookup_fun, {fun ssl_test_lib:user_lookup/3, PskSharedSecret}}], + client_config => [{psk_identity, "Test-User"}, + {user_lookup_fun, {fun ssl_test_lib:user_lookup/3, PskSharedSecret}}]}} | + proplists:delete(tls_config, Config)]; +init_certs(srp, Config) -> + [{tls_config, #{server_config => [{user_lookup_fun, {fun ssl_test_lib:user_lookup/3, undefined}}], + client_config => [{srp_identity, {"Test-User", "secret"}}]}} | + proplists:delete(tls_config, Config)]; +init_certs(_GroupName, Config) -> + %% Anonymous does not need certs + [{tls_config, #{server_config => [], + client_config => []}} | + proplists:delete(tls_config, Config)]. +%%-------------------------------------------------------------------- +%% Test Cases -------------------------------------------------------- +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% SRP -------------------------------------------------------- +%%-------------------------------------------------------------------- +srp_rsa_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(srp_rsa, '3des_ede_cbc', Config). + +srp_rsa_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(srp_rsa, 'aes_128_cbc', Config). + +srp_rsa_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(srp_rsa, 'aes_256_cbc', Config). + +srp_dss_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(srp_dss, '3des_ede_cbc', Config). + +srp_dss_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(srp_dss, 'aes_128_cbc', Config). + +srp_dss_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(srp_dss, 'aes_256_cbc', Config). + +%%-------------------------------------------------------------------- +%% PSK -------------------------------------------------------- +%%-------------------------------------------------------------------- +rsa_psk_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(rsa_psk, '3des_ede_cbc', Config). + +rsa_psk_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(rsa_psk, 'aes_128_cbc', Config). + +rsa_psk_aes_128_ccm(Config) when is_list(Config) -> + run_ciphers_test(rsa_psk, 'aes_128_ccm', Config). + +rsa_psk_aes_128_ccm_8(Config) when is_list(Config) -> + run_ciphers_test(rsa_psk, 'aes_128_ccm_8', Config). + +rsa_psk_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(rsa_psk, 'aes_256_cbc', Config). + +rsa_psk_aes_256_ccm(Config) when is_list(Config) -> + run_ciphers_test(rsa_psk, 'aes_256_ccm', Config). + +rsa_psk_aes_256_ccm_8(Config) when is_list(Config) -> + run_ciphers_test(rsa_psk, 'aes_256_ccm_8', Config). + +rsa_psk_rc4_128(Config) when is_list(Config) -> + run_ciphers_test(rsa_psk, 'rc4_128', Config). + +%%-------------------------------------------------------------------- +%% RSA -------------------------------------------------------- +%%-------------------------------------------------------------------- +rsa_des_cbc(Config) when is_list(Config) -> + run_ciphers_test(rsa, 'des_cbc', Config). + +rsa_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(rsa, '3des_ede_cbc', Config). + +rsa_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(rsa, 'aes_128_cbc', Config). + +rsa_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(rsa, 'aes_256_cbc', Config). + +rsa_aes_128_gcm(Config) when is_list(Config) -> + run_ciphers_test(rsa, 'aes_128_gcm', Config). + +rsa_aes_256_gcm(Config) when is_list(Config) -> + run_ciphers_test(rsa, 'aes_256_gcm', Config). + +rsa_rc4_128(Config) when is_list(Config) -> + run_ciphers_test(rsa, 'rc4_128', Config). +%%-------------------------------------------------------------------- +%% DHE_RSA -------------------------------------------------------- +%%-------------------------------------------------------------------- +dhe_rsa_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_rsa, '3des_ede_cbc', Config). + +dhe_rsa_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_rsa, 'aes_128_cbc', Config). + +dhe_rsa_aes_128_gcm(Config) when is_list(Config) -> + run_ciphers_test(dhe_rsa, 'aes_128_gcm', Config). + +dhe_rsa_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_rsa, 'aes_256_cbc', Config). + +dhe_rsa_aes_256_gcm(Config) when is_list(Config) -> + run_ciphers_test(dhe_rsa, 'aes_256_gcm', Config). + +dhe_rsa_chacha20_poly1305(Config) when is_list(Config) -> + run_ciphers_test(dhe_rsa, 'chacha20_poly1305', Config). +%%-------------------------------------------------------------------- +%% ECDHE_RSA -------------------------------------------------------- +%%-------------------------------------------------------------------- +ecdhe_rsa_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_rsa, '3des_ede_cbc', Config). + +ecdhe_rsa_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_rsa, 'aes_128_cbc', Config). + +ecdhe_rsa_aes_128_gcm(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_rsa, 'aes_128_gcm', Config). + +ecdhe_rsa_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_rsa, 'aes_256_cbc', Config). + +ecdhe_rsa_aes_256_gcm(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_rsa, 'aes_256_gcm', Config). + +ecdhe_rsa_rc4_128(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_rsa, 'rc4_128', Config). + +ecdhe_rsa_chacha20_poly1305(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_rsa, 'chacha20_poly1305', Config). + +%%-------------------------------------------------------------------- +%% ECDHE_ECDSA -------------------------------------------------------- +%%-------------------------------------------------------------------- +ecdhe_ecdsa_rc4_128(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_ecdsa, 'rc4_128', Config). + +ecdhe_ecdsa_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_ecdsa, '3des_ede_cbc', Config). + +ecdhe_ecdsa_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_ecdsa, 'aes_128_cbc', Config). + +ecdhe_ecdsa_aes_128_gcm(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_ecdsa, 'aes_128_gcm', Config). + +ecdhe_ecdsa_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_ecdsa, 'aes_256_cbc', Config). + +ecdhe_ecdsa_aes_256_gcm(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_ecdsa, 'aes_256_gcm', Config). + +ecdhe_ecdsa_chacha20_poly1305(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_ecdsa, 'chacha20_poly1305', Config). +%%-------------------------------------------------------------------- +%% DHE_DSS -------------------------------------------------------- +%%-------------------------------------------------------------------- +dhe_dss_des_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_dss, 'des_cbc', Config). + +dhe_dss_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_dss, '3des_ede_cbc', Config). + +dhe_dss_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_dss, 'aes_128_cbc', Config). + +dhe_dss_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_dss, 'aes_256_cbc', Config). + +dhe_dss_aes_128_gcm(Config) when is_list(Config) -> + run_ciphers_test(dhe_dss, 'aes_128_gcm', Config). + +dhe_dss_aes_256_gcm(Config) when is_list(Config) -> + run_ciphers_test(dhe_dss, 'aes_256_gcm', Config). + +%%-------------------------------------------------------------------- +%% Anonymous -------------------------------------------------------- +%%-------------------------------------------------------------------- +dh_anon_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(dh_anon, '3des_ede_cbc', Config). + +dh_anon_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(dh_anon, 'aes_128_cbc', Config). + +dh_anon_aes_128_gcm(Config) when is_list(Config) -> + run_ciphers_test(dh_anon, 'aes_128_gcm', Config). + +dh_anon_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(dh_anon, 'aes_256_cbc', Config). + +dh_anon_aes_256_gcm(Config) when is_list(Config) -> + run_ciphers_test(dh_anon, 'aes_256_gcm', Config). + +dh_anon_rc4_128(Config) when is_list(Config) -> + run_ciphers_test(dh_anon, 'rc4_128', Config). + +ecdh_anon_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdh_anon, '3des_ede_cbc', Config). + +ecdh_anon_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdh_anon, 'aes_128_cbc', Config). + +ecdh_anon_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdh_anon, 'aes_256_cbc', Config). + +srp_anon_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(srp_anon, '3des_ede_cbc', Config). + +srp_anon_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(srp_anon, 'aes_128_cbc', Config). + +srp_anon_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(srp_anon, 'aes_256_cbc', Config). + +dhe_psk_des_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'des_cbc', Config). + +dhe_psk_rc4_128(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'rc4_128', Config). + +dhe_psk_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, '3des_ede_cbc', Config). + +dhe_psk_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'aes_128_cbc', Config). + +dhe_psk_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'aes_256_cbc', Config). + +dhe_psk_aes_128_gcm(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'aes_128_gcm', Config). + +dhe_psk_aes_256_gcm(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'aes_256_gcm', Config). + +dhe_psk_aes_128_ccm(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'aes_128_ccm', Config). + +dhe_psk_aes_256_ccm(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'aes_256_ccm', Config). + +dhe_psk_aes_128_ccm_8(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'aes_128_ccm_8', Config). + +dhe_psk_aes_256_ccm_8(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'aes_256_ccm_8', Config). + +ecdhe_psk_des_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_psk, 'des_cbc', Config). + +ecdhe_psk_rc4_128(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_psk, 'rc4_128', Config). + +ecdhe_psk_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_psk, '3des_ede_cbc', Config). + +ecdhe_psk_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_psk, 'aes_128_cbc', Config). + +ecdhe_psk_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_psk, 'aes_256_cbc', Config). + +ecdhe_psk_aes_128_gcm(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_psk, 'aes_128_gcm', Config). + +ecdhe_psk_aes_256_gcm(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_psk, 'aes_256_gcm', Config). + +ecdhe_psk_aes_128_ccm(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_psk, 'aes_128_ccm', Config). + +ecdhe_psk_aes_128_ccm_8(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_psk, 'aes_128_ccm_8', Config). + +psk_des_cbc(Config) when is_list(Config) -> + run_ciphers_test(psk, 'des_cbc', Config). + +psk_rc4_128(Config) when is_list(Config) -> + run_ciphers_test(psk, 'rc4_128', Config). + +psk_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(psk, '3des_ede_cbc', Config). + +psk_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(psk, 'aes_128_cbc', Config). + +psk_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(psk, 'aes_256_cbc', Config). + +psk_aes_128_gcm(Config) when is_list(Config) -> + run_ciphers_test(psk, 'aes_128_gcm', Config). + +psk_aes_256_gcm(Config) when is_list(Config) -> + run_ciphers_test(psk, 'aes_256_gcm', Config). + +psk_aes_128_ccm(Config) when is_list(Config) -> + run_ciphers_test(psk, 'aes_128_ccm', Config). + +psk_aes_256_ccm(Config) when is_list(Config) -> + run_ciphers_test(psk, 'aes_256_ccm', Config). + +psk_aes_128_ccm_8(Config) when is_list(Config) -> + run_ciphers_test(psk, 'aes_128_ccm_8', Config). + +psk_aes_256_ccm_8(Config) when is_list(Config) -> + run_ciphers_test(psk, 'aes_256_ccm_8', Config). + +%%-------------------------------------------------------------------- +%% Internal functions ---------------------------------------------- +%%-------------------------------------------------------------------- +run_ciphers_test(Kex, Cipher, Config) -> + Version = ssl_test_lib:protocol_version(Config), + TestCiphers = test_ciphers(Kex, Cipher, Version), + + case TestCiphers of + [_|_] -> + lists:foreach(fun(TestCipher) -> + cipher_suite_test(TestCipher, Version, Config) + end, TestCiphers); + [] -> + {skip, {not_sup, Kex, Cipher, Version}} + end. + +cipher_suite_test(CipherSuite, _Version, Config) -> + #{server_config := SOpts, + client_config := COpts} = proplists:get_value(tls_config, Config), + ServerOpts = ssl_test_lib:ssl_options(SOpts, Config), + ClientOpts = ssl_test_lib:ssl_options(COpts, Config), + ct:log("Testing CipherSuite ~p~n", [CipherSuite]), + ct:log("Server Opts ~p~n", [ServerOpts]), + ct:log("Client Opts ~p~n", [ClientOpts]), + ssl_test_lib:basic_test([{ciphers, [CipherSuite]} | COpts], SOpts, [{client_type, erlang}, + {server_type, openssl} | Config]). + + +test_ciphers(Kex, Cipher, Version) -> + Ciphers = ssl:filter_cipher_suites(ssl:cipher_suites(default, Version) ++ ssl:cipher_suites(anonymous, Version), + [{key_exchange, + fun(Kex0) when Kex0 == Kex -> true; + (_) -> false + end}, + {cipher, + fun(Cipher0) when Cipher0 == Cipher -> true; + (_) -> false + end}]), + ct:log("Version ~p Testing ~p~n", [Version, Ciphers]), + OpenSSLCiphers = openssl_ciphers(), + ct:log("OpenSSLCiphers ~p~n", [OpenSSLCiphers]), + lists:filter(fun(C) -> + ct:log("Cipher ~p~n", [C]), + lists:member(ssl_cipher_format:suite_map_to_openssl_str(C), OpenSSLCiphers) + end, Ciphers). + + +openssl_ciphers() -> + Str = os:cmd("openssl ciphers"), + string:split(string:strip(Str, right, $\n), ":", all). diff --git a/lib/ssl/test/ssl_ECC_SUITE.erl b/lib/ssl/test/ssl_ECC_SUITE.erl index c64358960c..d02888793c 100644 --- a/lib/ssl/test/ssl_ECC_SUITE.erl +++ b/lib/ssl/test/ssl_ECC_SUITE.erl @@ -51,35 +51,7 @@ groups() -> ]. test_cases()-> - key_cert_combinations() - ++ misc() - ++ ecc_negotiation(). - -key_cert_combinations() -> - server_ecdh_rsa() ++ - server_ecdhe_rsa() ++ - server_ecdh_ecdsa() ++ - server_ecdhe_ecdsa(). - -server_ecdh_rsa() -> - [client_ecdh_rsa_server_ecdh_rsa, - client_ecdhe_rsa_server_ecdh_rsa, - client_ecdhe_ecdsa_server_ecdh_rsa]. - -server_ecdhe_rsa() -> - [client_ecdh_rsa_server_ecdhe_rsa, - client_ecdhe_rsa_server_ecdhe_rsa, - client_ecdhe_ecdsa_server_ecdhe_rsa]. - -server_ecdh_ecdsa() -> - [client_ecdh_ecdsa_server_ecdh_ecdsa, - client_ecdhe_rsa_server_ecdh_ecdsa, - client_ecdhe_ecdsa_server_ecdh_ecdsa]. - -server_ecdhe_ecdsa() -> - [client_ecdh_rsa_server_ecdhe_ecdsa, - client_ecdh_ecdsa_server_ecdhe_ecdsa, - client_ecdhe_ecdsa_server_ecdhe_ecdsa]. + misc() ++ ecc_negotiation(). misc()-> [client_ecdsa_server_ecdsa_with_raw_key]. @@ -160,35 +132,6 @@ end_per_testcase(_TestCase, Config) -> %% Test diffrent certificate chain types, note that it is the servers %% chain that affect what cipher suit that will be choosen -%% ECDH_RSA -client_ecdh_rsa_server_ecdh_rsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdh_rsa_server_ecdh_rsa(Config). -client_ecdhe_rsa_server_ecdh_rsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdhe_rsa_server_ecdh_rsa(Config). -client_ecdhe_ecdsa_server_ecdh_rsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdhe_ecdsa_server_ecdh_rsa(Config). -%% ECDHE_RSA -client_ecdh_rsa_server_ecdhe_rsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdh_rsa_server_ecdhe_rsa(Config). -client_ecdhe_rsa_server_ecdhe_rsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdhe_rsa_server_ecdhe_rsa(Config). -client_ecdhe_ecdsa_server_ecdhe_rsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdhe_ecdsa_server_ecdhe_rsa(Config). -%% ECDH_ECDSA -client_ecdh_ecdsa_server_ecdh_ecdsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdh_ecdsa_server_ecdh_ecdsa(Config). -client_ecdhe_rsa_server_ecdh_ecdsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdhe_rsa_server_ecdh_ecdsa(Config). -client_ecdhe_ecdsa_server_ecdh_ecdsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdhe_ecdsa_server_ecdh_ecdsa(Config). -%% ECDHE_ECDSA -client_ecdh_rsa_server_ecdhe_ecdsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdh_rsa_server_ecdhe_ecdsa(Config). -client_ecdh_ecdsa_server_ecdhe_ecdsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdh_ecdsa_server_ecdhe_ecdsa(Config). -client_ecdhe_ecdsa_server_ecdhe_ecdsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdhe_ecdsa_server_ecdhe_ecdsa(Config). - client_ecdsa_server_ecdsa_with_raw_key(Config) when is_list(Config) -> Default = ssl_test_lib:default_cert_chain_conf(), {COpts0, SOpts0} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, diff --git a/lib/ssl/test/ssl_ECC_openssl_SUITE.erl b/lib/ssl/test/ssl_ECC_openssl_SUITE.erl index 81a7dfd2da..68d4e910fd 100644 --- a/lib/ssl/test/ssl_ECC_openssl_SUITE.erl +++ b/lib/ssl/test/ssl_ECC_openssl_SUITE.erl @@ -33,77 +33,23 @@ %%-------------------------------------------------------------------- all() -> - case test_cases() of - [_|_] -> - all_groups(); - [] -> - [skip] - end. - -all_groups() -> case ssl_test_lib:openssl_sane_dtls() of true -> [{group, 'tlsv1.2'}, - {group, 'tlsv1.1'}, - {group, 'tlsv1'}, - {group, 'dtlsv1.2'}, - {group, 'dtlsv1'}]; + {group, 'dtlsv1.2'}]; false -> - [{group, 'tlsv1.2'}, - {group, 'tlsv1.1'}, - {group, 'tlsv1'}] + [{group, 'tlsv1.2'}] end. groups() -> case ssl_test_lib:openssl_sane_dtls() of true -> - [{'tlsv1.2', [], [mix_sign | test_cases()]}, - {'tlsv1.1', [], test_cases()}, - {'tlsv1', [], test_cases()}, - {'dtlsv1.2', [], [mix_sign | test_cases()]}, - {'dtlsv1', [], test_cases()}]; + [{'tlsv1.2', [], [mix_sign]}, + {'dtlsv1.2', [], [mix_sign]}]; false -> - [{'tlsv1.2', [], [mix_sign | test_cases()]}, - {'tlsv1.1', [], test_cases()}, - {'tlsv1', [], test_cases()}] + [{'tlsv1.2', [], [mix_sign]}] end. - -test_cases()-> - cert_combinations(). -cert_combinations() -> - lists:append(lists:map(fun({Name, Suites}) -> - case ssl_test_lib:openssl_filter(Name) of - [] -> - []; - [_|_] -> - Suites - end - end, [{"ECDH-ECDSA", server_ecdh_ecdsa()}, - {"ECDH-RSA", server_ecdh_rsa()}, - {"ECDHE-RSA", server_ecdhe_rsa()}, - {"ECDHE-ECDSA", server_ecdhe_ecdsa()} - ])). -server_ecdh_rsa() -> - [client_ecdh_rsa_server_ecdh_rsa, - client_ecdhe_rsa_server_ecdh_rsa, - client_ecdhe_ecdsa_server_ecdh_rsa]. - -server_ecdhe_rsa() -> - [client_ecdh_rsa_server_ecdhe_rsa, - client_ecdhe_rsa_server_ecdhe_rsa, - client_ecdhe_ecdsa_server_ecdhe_rsa]. - -server_ecdh_ecdsa() -> - [client_ecdh_ecdsa_server_ecdh_ecdsa, - client_ecdhe_rsa_server_ecdh_ecdsa, - client_ecdhe_ecdsa_server_ecdh_ecdsa]. - -server_ecdhe_ecdsa() -> - [client_ecdh_rsa_server_ecdhe_ecdsa, - client_ecdh_ecdsa_server_ecdhe_ecdsa, - client_ecdhe_ecdsa_server_ecdhe_ecdsa]. - %%-------------------------------------------------------------------- init_per_suite(Config0) -> end_per_suite(Config0), @@ -171,38 +117,6 @@ end_per_testcase(_TestCase, Config) -> skip(Config) when is_list(Config) -> {skip, openssl_does_not_support_ECC}. -%% Test diffrent certificate chain types, note that it is the servers -%% chain that affect what cipher suit that will be choosen - -%% ECDH_RSA -client_ecdh_rsa_server_ecdh_rsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdh_rsa_server_ecdh_rsa(Config). -client_ecdhe_rsa_server_ecdh_rsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdhe_rsa_server_ecdh_rsa(Config). -client_ecdhe_ecdsa_server_ecdh_rsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdhe_ecdsa_server_ecdh_rsa(Config). -%% ECDHE_RSA -client_ecdh_rsa_server_ecdhe_rsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdh_rsa_server_ecdhe_rsa(Config). -client_ecdhe_rsa_server_ecdhe_rsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdhe_rsa_server_ecdhe_rsa(Config). -client_ecdhe_ecdsa_server_ecdhe_rsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdhe_ecdsa_server_ecdhe_rsa(Config). -%% ECDH_ECDSA -client_ecdh_ecdsa_server_ecdh_ecdsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdh_ecdsa_server_ecdh_ecdsa(Config). -client_ecdhe_rsa_server_ecdh_ecdsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdhe_rsa_server_ecdh_ecdsa(Config). -client_ecdhe_ecdsa_server_ecdh_ecdsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdhe_ecdsa_server_ecdh_ecdsa(Config). -%% ECDHE_ECDSA -client_ecdh_rsa_server_ecdhe_ecdsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdh_rsa_server_ecdhe_ecdsa(Config). -client_ecdh_ecdsa_server_ecdhe_ecdsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdh_ecdsa_server_ecdhe_ecdsa(Config). -client_ecdhe_ecdsa_server_ecdhe_ecdsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdhe_ecdsa_server_ecdhe_ecdsa(Config). - mix_sign(Config) -> {COpts0, SOpts0} = ssl_test_lib:make_mix_cert(Config), COpts = ssl_test_lib:ssl_options(COpts0, Config), diff --git a/lib/ssl/test/ssl_cipher_suite_SUITE.erl b/lib/ssl/test/ssl_cipher_suite_SUITE.erl index 8805df7b52..51788c29e7 100644 --- a/lib/ssl/test/ssl_cipher_suite_SUITE.erl +++ b/lib/ssl/test/ssl_cipher_suite_SUITE.erl @@ -127,7 +127,6 @@ groups() -> ]} ]. - kex() -> rsa() ++ ecdsa() ++ dss() ++ anonymous(). @@ -154,7 +153,6 @@ anonymous() -> {group, ecdhe_psk}, {group, srp_anon} ]. - init_per_suite(Config) -> catch crypto:stop(), @@ -170,7 +168,7 @@ end_per_suite(_Config) -> ssl:stop(), application:stop(crypto). -%%-------------------------------------------------------------------- + init_per_group(GroupName, Config) when GroupName == ecdh_anon; GroupName == ecdhe_rsa; GroupName == ecdhe_psk -> @@ -236,6 +234,7 @@ end_per_group(GroupName, Config) -> false -> Config end. + init_per_testcase(TestCase, Config) when TestCase == psk_3des_ede_cbc; TestCase == srp_anon_3des_ede_cbc; TestCase == dhe_psk_3des_ede_cbc; @@ -302,8 +301,7 @@ init_per_testcase(TestCase, Config) when TestCase == psk_aes_256_ccm_8; {skip, "Missing AES_256_CCM crypto support"} end; init_per_testcase(TestCase, Config) -> - Cipher = test_cipher(TestCase, Config), - %%Reason = io_lib:format("Missing ~p crypto support", [Cipher]), + Cipher = ssl_test_lib:test_cipher(TestCase, Config), SupCiphers = proplists:get_value(ciphers, crypto:supports()), case lists:member(Cipher, SupCiphers) of true -> @@ -316,17 +314,21 @@ init_per_testcase(TestCase, Config) -> end_per_testcase(_TestCase, Config) -> Config. +%%-------------------------------------------------------------------- +%% Initializtion ------------------------------------------ +%%-------------------------------------------------------------------- + init_certs(srp_rsa, Config) -> DefConf = ssl_test_lib:default_cert_chain_conf(), CertChainConf = ssl_test_lib:gen_conf(rsa, rsa, DefConf, DefConf), #{server_config := ServerOpts, client_config := ClientOpts} = public_key:pkix_test_data(CertChainConf), - [{tls_config, #{server_config => [{user_lookup_fun, {fun user_lookup/3, undefined}} | ServerOpts], + [{tls_config, #{server_config => [{user_lookup_fun, {fun ssl_test_lib:user_lookup/3, undefined}} | ServerOpts], client_config => [{srp_identity, {"Test-User", "secret"}} | ClientOpts]}} | proplists:delete(tls_config, Config)]; init_certs(srp_anon, Config) -> - [{tls_config, #{server_config => [{user_lookup_fun, {fun user_lookup/3, undefined}}], + [{tls_config, #{server_config => [{user_lookup_fun, {fun ssl_test_lib:user_lookup/3, undefined}}], client_config => [{srp_identity, {"Test-User", "secret"}}]}} | proplists:delete(tls_config, Config)]; init_certs(rsa_psk, Config) -> @@ -335,9 +337,9 @@ init_certs(rsa_psk, Config) -> [[],[],[{extensions, ClientExt}]]}], Config, "_peer_keyEncipherment"), PskSharedSecret = <<1,2,3,4,5,6,7,8,9,10,11,12,13,14,15>>, - [{tls_config, #{server_config => [{user_lookup_fun, {fun user_lookup/3, PskSharedSecret}} | ServerOpts], + [{tls_config, #{server_config => [{user_lookup_fun, {fun ssl_test_lib:user_lookup/3, PskSharedSecret}} | ServerOpts], client_config => [{psk_identity, "Test-User"}, - {user_lookup_fun, {fun user_lookup/3, PskSharedSecret}} | ClientOpts]}} | + {user_lookup_fun, {fun ssl_test_lib:user_lookup/3, PskSharedSecret}} | ClientOpts]}} | proplists:delete(tls_config, Config)]; init_certs(rsa, Config) -> ClientExt = x509_test:extensions([{key_usage, [digitalSignature, keyEncipherment]}]), @@ -362,7 +364,7 @@ init_certs(srp_dss, Config) -> #{server_config := ServerOpts, client_config := ClientOpts} = public_key:pkix_test_data(CertChainConf), - [{tls_config, #{server_config => [{user_lookup_fun, {fun user_lookup/3, undefined}} | ServerOpts], + [{tls_config, #{server_config => [{user_lookup_fun, {fun ssl_test_lib:user_lookup/3, undefined}} | ServerOpts], client_config => [{srp_identity, {"Test-User", "secret"}} | ClientOpts]}} | proplists:delete(tls_config, Config)]; init_certs(GroupName, Config) when GroupName == dhe_rsa; @@ -389,12 +391,12 @@ init_certs(GroupName, Config) when GroupName == psk; GroupName == dhe_psk; GroupName == ecdhe_psk -> PskSharedSecret = <<1,2,3,4,5,6,7,8,9,10,11,12,13,14,15>>, - [{tls_config, #{server_config => [{user_lookup_fun, {fun user_lookup/3, PskSharedSecret}}], + [{tls_config, #{server_config => [{user_lookup_fun, {fun ssl_test_lib:user_lookup/3, PskSharedSecret}}], client_config => [{psk_identity, "Test-User"}, - {user_lookup_fun, {fun user_lookup/3, PskSharedSecret}}]}} | + {user_lookup_fun, {fun ssl_test_lib:user_lookup/3, PskSharedSecret}}]}} | proplists:delete(tls_config, Config)]; init_certs(srp, Config) -> - [{tls_config, #{server_config => [{user_lookup_fun, {fun user_lookup/3, undefined}}], + [{tls_config, #{server_config => [{user_lookup_fun, {fun ssl_test_lib:user_lookup/3, undefined}}], client_config => [{srp_identity, {"Test-User", "secret"}}]}} | proplists:delete(tls_config, Config)]; init_certs(_GroupName, Config) -> @@ -402,6 +404,7 @@ init_certs(_GroupName, Config) -> [{tls_config, #{server_config => [], client_config => []}} | proplists:delete(tls_config, Config)]. + %%-------------------------------------------------------------------- %% Test Cases -------------------------------------------------------- %%-------------------------------------------------------------------- @@ -700,10 +703,6 @@ psk_aes_256_ccm_8(Config) when is_list(Config) -> %%-------------------------------------------------------------------- %% Internal functions ---------------------------------------------- %%-------------------------------------------------------------------- -test_cipher(TestCase, Config) -> - [{name, Group} |_] = proplists:get_value(tc_group_properties, Config), - list_to_atom(re:replace(atom_to_list(TestCase), atom_to_list(Group) ++ "_", "", [{return, list}])). - run_ciphers_test(Kex, Cipher, Config) -> Version = ssl_test_lib:protocol_version(Config), TestCiphers = test_ciphers(Kex, Cipher, Version), @@ -717,30 +716,28 @@ run_ciphers_test(Kex, Cipher, Config) -> {skip, {not_sup, Kex, Cipher, Version}} end. -cipher_suite_test(CipherSuite, Version, Config) -> +cipher_suite_test(ErlangCipherSuite, Version, Config) -> #{server_config := SOpts, client_config := COpts} = proplists:get_value(tls_config, Config), ServerOpts = ssl_test_lib:ssl_options(SOpts, Config), ClientOpts = ssl_test_lib:ssl_options(COpts, Config), - ct:log("Testing CipherSuite ~p~n", [CipherSuite]), + ct:log("Testing CipherSuite ~p~n", [ErlangCipherSuite]), ct:log("Server Opts ~p~n", [ServerOpts]), ct:log("Client Opts ~p~n", [ClientOpts]), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), - ErlangCipherSuite = erlang_cipher_suite(CipherSuite), - ConnectionInfo = {ok, {Version, ErlangCipherSuite}}, Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, {from, self()}, {mfa, {ssl_test_lib, cipher_result, [ConnectionInfo]}}, - {options, [{versions, [Version]}, {ciphers, [CipherSuite]} | ServerOpts]}]), + {options, [{versions, [Version]}, {ciphers, [ErlangCipherSuite]} | ServerOpts]}]), Port = ssl_test_lib:inet_port(Server), Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, {host, Hostname}, {from, self()}, {mfa, {ssl_test_lib, cipher_result, [ConnectionInfo]}}, - {options, [{versions, [Version]}, {ciphers, [CipherSuite]} | + {options, [{versions, [Version]}, {ciphers, [ErlangCipherSuite]} | ClientOpts]}]), ssl_test_lib:check_result(Server, ok, Client, ok), @@ -748,17 +745,6 @@ cipher_suite_test(CipherSuite, Version, Config) -> ssl_test_lib:close(Server), ssl_test_lib:close(Client). -erlang_cipher_suite(Suite) when is_list(Suite)-> - ssl_cipher_format:suite_definition(ssl_cipher_format:suite_openssl_str_to_map(Suite)); -erlang_cipher_suite(Suite) -> - Suite. - -user_lookup(psk, _Identity, UserState) -> - {ok, UserState}; -user_lookup(srp, Username, _UserState) -> - Salt = ssl_cipher:random_bytes(16), - UserPassHash = crypto:hash(sha, [Salt, crypto:hash(sha, [Username, <<$:>>, <<"secret">>])]), - {ok, {srp_1024, Salt, UserPassHash}}. test_ciphers(Kex, Cipher, Version) -> ssl:filter_cipher_suites(ssl:cipher_suites(all, Version) ++ ssl:cipher_suites(anonymous, Version), @@ -770,3 +756,4 @@ test_ciphers(Kex, Cipher, Version) -> fun(Cipher0) when Cipher0 == Cipher -> true; (_) -> false end}]). + diff --git a/lib/ssl/test/ssl_test_lib.erl b/lib/ssl/test/ssl_test_lib.erl index 65b8998cc3..3b161a0c8a 100644 --- a/lib/ssl/test/ssl_test_lib.erl +++ b/lib/ssl/test/ssl_test_lib.erl @@ -631,6 +631,40 @@ make_rsa_cert_chains(UserConf, Config, Suffix) -> [{reuseaddr, true}, {verify, verify_peer} | ServerConf] }. +make_ecc_cert_chains(UserConf, Config, Suffix) -> + ClientChain = proplists:get_value(client_chain, UserConf, default_cert_chain_conf()), + ServerChain = proplists:get_value(server_chain, UserConf, default_cert_chain_conf()), + CertChainConf = gen_conf(ecdsa, ecdsa, ClientChain, ServerChain), + ClientFileBase = filename:join([proplists:get_value(priv_dir, Config), "ecdsa" ++ Suffix]), + ServerFileBase = filename:join([proplists:get_value(priv_dir, Config), "ecdsa" ++ Suffix]), + GenCertData = public_key:pkix_test_data(CertChainConf), + [{server_config, ServerConf}, + {client_config, ClientConf}] = + x509_test:gen_pem_config_files(GenCertData, ClientFileBase, ServerFileBase), + {[{verify, verify_peer} | ClientConf], + [{reuseaddr, true}, {verify, verify_peer} | ServerConf] + }. + + +make_dsa_cert_chains(UserConf, Config, Suffix) -> + CryptoSupport = crypto:supports(), + case proplists:get_bool(dss, proplists:get_value(public_keys, CryptoSupport)) of + true -> + ClientChain = proplists:get_value(client_chain, UserConf, default_cert_chain_conf()), + ServerChain = proplists:get_value(server_chain, UserConf, default_cert_chain_conf()), + CertChainConf = gen_conf(dsa, dsa, ClientChain, ServerChain), + ClientFileBase = filename:join([proplists:get_value(priv_dir, Config), "dsa" ++ Suffix]), + ServerFileBase = filename:join([proplists:get_value(priv_dir, Config), "dsa" ++ Suffix]), + GenCertData = public_key:pkix_test_data(CertChainConf), + [{server_config, ServerConf}, + {client_config, ClientConf}] = + x509_test:gen_pem_config_files(GenCertData, ClientFileBase, ServerFileBase), + {[{verify, verify_peer} | ClientConf], + [{reuseaddr, true}, {verify, verify_peer} | ServerConf]}; + false -> + Config + end. + make_ec_cert_chains(UserConf, ClientChainType, ServerChainType, Config) -> make_ec_cert_chains(UserConf, ClientChainType, ServerChainType, Config, ?DEFAULT_CURVE). %% @@ -1067,7 +1101,7 @@ accepters(Acc, N) -> basic_test(COpts, SOpts, Config) -> SType = proplists:get_value(server_type, Config), CType = proplists:get_value(client_type, Config), - {Server, Port} = start_server(SType, SOpts, Config), + {Server, Port} = start_server(SType, COpts, SOpts, Config), Client = start_client(CType, Port, COpts, Config), gen_check_result(Server, SType, Client, CType), stop(Server, Client). @@ -1134,7 +1168,7 @@ start_client(erlang, Port, ClientOpts, Config) -> {host, Hostname}, {from, self()}, {mfa, {ssl_test_lib, check_key_exchange_send_active, [KeyEx]}}, - {options, [{verify, verify_peer} | ClientOpts]}]). + {options, ClientOpts}]). %% Workaround for running tests on machines where openssl %% s_client would use an IPv6 address with localhost. As @@ -1169,20 +1203,19 @@ start_client_ecc_error(erlang, Port, ClientOpts, ECCOpts, Config) -> [{verify, verify_peer} | ClientOpts]}]). -start_server(openssl, ServerOpts, Config) -> - Cert = proplists:get_value(certfile, ServerOpts), - Key = proplists:get_value(keyfile, ServerOpts), - CA = proplists:get_value(cacertfile, ServerOpts), +start_server(openssl, ClientOpts, ServerOpts, Config) -> Port = inet_port(node()), Version = protocol_version(Config), Exe = "openssl", - Args = ["s_server", "-accept", integer_to_list(Port), ssl_test_lib:version_flag(Version), - "-verify", "2", "-cert", Cert, "-CAfile", CA, - "-key", Key, "-msg", "-debug"], + CertArgs = openssl_cert_options(ServerOpts), + [Cipher|_] = proplists:get_value(ciphers, ClientOpts, ssl:cipher_suites(default,Version)), + Args = ["s_server", "-accept", integer_to_list(Port), "-cipher", + ssl_cipher_format:suite_map_to_openssl_str(Cipher), + ssl_test_lib:version_flag(Version)] ++ CertArgs ++ ["-msg", "-debug"], OpenSslPort = portable_open_port(Exe, Args), true = port_command(OpenSslPort, "Hello world"), {OpenSslPort, Port}; -start_server(erlang, ServerOpts, Config) -> +start_server(erlang, _, ServerOpts, Config) -> {_, ServerNode, _} = ssl_test_lib:run_where(Config), KeyEx = proplists:get_value(check_keyex, Config, false), Server = start_server([{node, ServerNode}, {port, 0}, @@ -1245,6 +1278,29 @@ stop(Client, Server) -> close(Server), close(Client). + +openssl_cert_options(ServerOpts) -> + Cert = proplists:get_value(certfile, ServerOpts, undefined), + Key = proplists:get_value(keyfile, ServerOpts, undefined), + CA = proplists:get_value(cacertfile, ServerOpts, undefined), + case CA of + undefined -> + case cert_option("-cert", Cert) ++ cert_option("-key", Key) of + [] -> + ["-nocert"]; + Other -> + Other + end; + _ -> + cert_option("-cert", Cert) ++ cert_option("-CAfile", CA) ++ + cert_option("-key", Key) ++ ["-verify", "2"] + end. + +cert_option(_, undefined) -> + []; +cert_option(Opt, Value) -> + [Opt, Value]. + supported_eccs(Opts) -> ToCheck = proplists:get_value(eccs, Opts, []), Supported = ssl:eccs(), @@ -2374,3 +2430,16 @@ user_lookup(srp, Username, _UserState) -> Salt = ssl_cipher:random_bytes(16), UserPassHash = crypto:hash(sha, [Salt, crypto:hash(sha, [Username, <<$:>>, <<"secret">>])]), {ok, {srp_1024, Salt, UserPassHash}}. + +test_cipher(TestCase, Config) -> + [{name, Group} |_] = proplists:get_value(tc_group_properties, Config), + list_to_atom(re:replace(atom_to_list(TestCase), atom_to_list(Group) ++ "_", "", [{return, list}])). + +digest() -> + case application:get_env(ssl, protocol_version, application:get_env(ssl, dtls_protocol_version)) of + Ver when Ver == 'tlsv1.2'; + Ver == 'dtlsv1.2' -> + {digest, sha256}; + _ -> + {digest, sha1} + end. diff --git a/lib/stdlib/doc/src/gb_trees.xml b/lib/stdlib/doc/src/gb_trees.xml index 570c9c7cb6..08aa1865e8 100644 --- a/lib/stdlib/doc/src/gb_trees.xml +++ b/lib/stdlib/doc/src/gb_trees.xml @@ -42,11 +42,8 @@ <section> <title>Data Structure</title> - <code type="none"> -{Size, Tree}</code> - - <p><c>Tree</c> is composed of nodes of the form <c>{Key, Value, Smaller, - Bigger}</c> and the "empty tree" node <c>nil</c>.</p> + <p>Trees and iterators are built using opaque data structures that should + not be pattern-matched from outside this module.</p> <p>There is no attempt to balance trees after deletions. As deletions do not increase the height of a tree, this should be OK.</p> diff --git a/lib/stdlib/doc/src/queue.xml b/lib/stdlib/doc/src/queue.xml index 83a8afea81..89cce6d85b 100644 --- a/lib/stdlib/doc/src/queue.xml +++ b/lib/stdlib/doc/src/queue.xml @@ -168,7 +168,7 @@ <fsummary>Test if a queue is empty.</fsummary> <desc> <p>Tests if <c><anno>Q</anno></c> is empty and returns <c>true</c> if - so, otherwise otherwise.</p> + so, otherwise <c>false</c>.</p> </desc> </func> diff --git a/lib/stdlib/src/erl_lint.erl b/lib/stdlib/src/erl_lint.erl index e0c37ca030..0cd0aef124 100644 --- a/lib/stdlib/src/erl_lint.erl +++ b/lib/stdlib/src/erl_lint.erl @@ -2,7 +2,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2018. All Rights Reserved. +%% Copyright Ericsson AB 1996-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -386,6 +386,8 @@ format_error({redefine_callback, {F, A}}) -> format_error({bad_callback, {M, F, A}}) -> io_lib:format("explicit module not allowed for callback ~tw:~tw/~w", [M, F, A]); +format_error({bad_module, {M, F, A}}) -> + io_lib:format("spec for function ~w:~tw/~w from other module", [M, F, A]); format_error({spec_fun_undefined, {F, A}}) -> io_lib:format("spec for undefined function ~tw/~w", [F, A]); format_error({missing_spec, {F,A}}) -> @@ -3010,7 +3012,13 @@ spec_decl(Line, MFA0, TypeSpecs, St00 = #lint{specs = Specs, module = Mod}) -> St1 = St0#lint{specs = dict:store(MFA, Line, Specs)}, case dict:is_key(MFA, Specs) of true -> add_error(Line, {redefine_spec, MFA0}, St1); - false -> check_specs(TypeSpecs, spec_wrong_arity, Arity, St1) + false -> + case MFA of + {Mod, _, _} -> + check_specs(TypeSpecs, spec_wrong_arity, Arity, St1); + _ -> + add_error(Line, {bad_module, MFA}, St1) + end end. %% callback_decl(Line, Fun, Types, State) -> State. diff --git a/lib/stdlib/src/string.erl b/lib/stdlib/src/string.erl index 1f8bdc5432..a418754caf 100644 --- a/lib/stdlib/src/string.erl +++ b/lib/stdlib/src/string.erl @@ -128,7 +128,8 @@ length(CD) -> to_graphemes(CD0) -> case unicode_util:gc(CD0) of [GC|CD] -> [GC|to_graphemes(CD)]; - [] -> [] + [] -> []; + {error, Err} -> error({badarg, Err}) end. %% Compare two strings return boolean, assumes that the input are @@ -332,7 +333,10 @@ uppercase(<<CP1/utf8, Rest/binary>>=Orig) -> catch unchanged -> Orig end; uppercase(<<>>) -> - <<>>. + <<>>; +uppercase(Bin) -> + error({badarg, Bin}). + %% Lowercase all chars in Str -spec lowercase(String::unicode:chardata()) -> unicode:chardata(). @@ -346,7 +350,10 @@ lowercase(<<CP1/utf8, Rest/binary>>=Orig) -> catch unchanged -> Orig end; lowercase(<<>>) -> - <<>>. + <<>>; +lowercase(Bin) -> + error({badarg, Bin}). + %% Make a titlecase of the first char in Str -spec titlecase(String::unicode:chardata()) -> unicode:chardata(). @@ -375,7 +382,9 @@ casefold(<<CP1/utf8, Rest/binary>>=Orig) -> catch unchanged -> Orig end; casefold(<<>>) -> - <<>>. + <<>>; +casefold(Bin) -> + error({badarg, Bin}). -spec to_integer(String) -> {Int, Rest} | {'error', Reason} when String :: unicode:chardata(), @@ -544,7 +553,8 @@ length_1([CP1|[CP2|_]=Cont], N) when ?ASCII_LIST(CP1,CP2) -> length_1(Str, N) -> case unicode_util:gc(Str) of [] -> N; - [_|Rest] -> length_1(Rest, N+1) + [_|Rest] -> length_1(Rest, N+1); + {error, Err} -> error({badarg, Err}) end. length_b(<<CP2/utf8, Rest/binary>>, CP1, N) @@ -554,7 +564,8 @@ length_b(Bin0, CP1, N) -> [_|Bin1] = unicode_util:gc([CP1|Bin0]), case unicode_util:cp(Bin1) of [] -> N+1; - [CP3|Bin] -> length_b(Bin, CP3, N+1) + [CP3|Bin] -> length_b(Bin, CP3, N+1); + {error, Err} -> error({badarg, Err}) end. equal_1([A|AR], [B|BR]) when is_integer(A), is_integer(B) -> @@ -599,7 +610,8 @@ reverse_1([CP1|[CP2|_]=Cont], Acc) when ?ASCII_LIST(CP1,CP2) -> reverse_1(CD, Acc) -> case unicode_util:gc(CD) of [GC|Rest] -> reverse_1(Rest, [GC|Acc]); - [] -> Acc + [] -> Acc; + {error, Err} -> error({badarg, Err}) end. reverse_b(<<CP2/utf8, Rest/binary>>, CP1, Acc) @@ -609,7 +621,8 @@ reverse_b(Bin0, CP1, Acc) -> [GC|Bin1] = unicode_util:gc([CP1|Bin0]), case unicode_util:cp(Bin1) of [] -> [GC|Acc]; - [CP3|Bin] -> reverse_b(Bin, CP3, [GC|Acc]) + [CP3|Bin] -> reverse_b(Bin, CP3, [GC|Acc]); + {error, Err} -> error({badarg, Err}) end. slice_l0(<<CP1/utf8, Bin/binary>>, N) when N > 0 -> @@ -622,7 +635,8 @@ slice_l([CP1|[CP2|_]=Cont], N) when ?ASCII_LIST(CP1,CP2),N > 0 -> slice_l(CD, N) when N > 0 -> case unicode_util:gc(CD) of [_|Cont] -> slice_l(Cont, N-1); - [] -> [] + [] -> []; + {error, Err} -> error({badarg, Err}) end; slice_l(Cont, 0) -> Cont. @@ -634,7 +648,8 @@ slice_lb(Bin, CP1, N) -> if N > 1 -> case unicode_util:cp(Rest) of [CP2|Cont] -> slice_lb(Cont, CP2, N-1); - [] -> <<>> + [] -> <<>>; + {error, Err} -> error({badarg, Err}) end; N =:= 1 -> Rest @@ -647,7 +662,10 @@ slice_trail(Orig, N) when is_binary(Orig) -> Sz = byte_size(Orig) - Length, <<Keep:Sz/binary, _/binary>> = Orig, Keep; - _ -> <<>> + <<_, _/binary>> when N > 0 -> + error({badarg, Orig}); + _ -> + <<>> end; slice_trail(CD, N) when is_list(CD) -> slice_list(CD, N). @@ -657,7 +675,8 @@ slice_list([CP1|[CP2|_]=Cont], N) when ?ASCII_LIST(CP1,CP2),N > 0 -> slice_list(CD, N) when N > 0 -> case unicode_util:gc(CD) of [GC|Cont] -> append(GC, slice_list(Cont, N-1)); - [] -> [] + [] -> []; + {error, Err} -> error({badarg, Err}) end; slice_list(_, 0) -> []. @@ -668,7 +687,8 @@ slice_bin(CD, CP1, N) when N > 0 -> [_|Bin] = unicode_util:gc([CP1|CD]), case unicode_util:cp(Bin) of [CP2|Cont] -> slice_bin(Cont, CP2, N-1); - [] -> 0 + [] -> 0; + {error, Err} -> error({badarg, Err}) end; slice_bin(CD, CP1, 0) -> byte_size(CD)+byte_size(<<CP1/utf8>>). @@ -703,14 +723,18 @@ uppercase_bin(CP1, Bin, Changed) -> [] when Changed -> [CP1]; [] -> - throw(unchanged) + throw(unchanged); + {error, Err} -> + error({badarg, Err}) end; [Char|CPs] -> case unicode_util:cp(CPs) of [Next|Rest] -> [Char|uppercase_bin(Next, Rest, true)]; [] -> - [Char] + [Char]; + {error, Err} -> + error({badarg, Err}) end end. @@ -744,14 +768,18 @@ lowercase_bin(CP1, Bin, Changed) -> [] when Changed -> [CP1]; [] -> - throw(unchanged) + throw(unchanged); + {error, Err} -> + error({badarg, Err}) end; [Char|CPs] -> case unicode_util:cp(CPs) of [Next|Rest] -> [Char|lowercase_bin(Next, Rest, true)]; [] -> - [Char] + [Char]; + {error, Err} -> + error({badarg, Err}) end end. @@ -785,14 +813,18 @@ casefold_bin(CP1, Bin, Changed) -> [] when Changed -> [CP1]; [] -> - throw(unchanged) + throw(unchanged); + {error, Err} -> + error({badarg, Err}) end; [Char|CPs] -> case unicode_util:cp(CPs) of [Next|Rest] -> [Char|casefold_bin(Next, Rest, true)]; [] -> - [Char] + [Char]; + {error, Err} -> + error({badarg, Err}) end end. @@ -1634,7 +1666,9 @@ bin_search_inv_1(<<CP1/utf8, BinRest/binary>>=Bin0, Cont, Sep) -> bin_search_inv_1(<<>>, Cont, _Sep) -> {nomatch, Cont}; bin_search_inv_1([], Cont, _Sep) -> - {nomatch, Cont}. + {nomatch, Cont}; +bin_search_inv_1(Bin, _, _) -> + error({badarg, Bin}). bin_search_inv_n(<<CP1/utf8, BinRest/binary>>=Bin0, Cont, Seps) -> @@ -1666,7 +1700,9 @@ bin_search_inv_n(<<CP1/utf8, BinRest/binary>>=Bin0, Cont, Seps) -> bin_search_inv_n(<<>>, Cont, _Sep) -> {nomatch, Cont}; bin_search_inv_n([], Cont, _Sep) -> - {nomatch, Cont}. + {nomatch, Cont}; +bin_search_inv_n(Bin, _, _) -> + error({badarg, Bin}). bin_search_str(Bin0, Start, [], SearchCPs) -> Compiled = binary:compile_pattern(unicode:characters_to_binary(SearchCPs)), diff --git a/lib/stdlib/test/erl_lint_SUITE.erl b/lib/stdlib/test/erl_lint_SUITE.erl index fe98a3796d..e7882e0daf 100644 --- a/lib/stdlib/test/erl_lint_SUITE.erl +++ b/lib/stdlib/test/erl_lint_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2018. All Rights Reserved. +%% Copyright Ericsson AB 1999-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -68,7 +68,7 @@ non_latin1_module/1, otp_14323/1, stacktrace_syntax/1, otp_14285/1, otp_14378/1, - external_funs/1,otp_15456/1]). + external_funs/1,otp_15456/1,otp_15563/1]). suite() -> [{ct_hooks,[ts_install_cth]}, @@ -90,7 +90,7 @@ all() -> otp_11851, otp_11879, otp_13230, record_errors, otp_11879_cont, non_latin1_module, otp_14323, stacktrace_syntax, otp_14285, otp_14378, external_funs, - otp_15456]. + otp_15456, otp_15563]. groups() -> [{unused_vars_warn, [], @@ -4010,6 +4010,8 @@ non_latin1_module(Config) -> format_error(non_latin1_module_unsupported), BadCallback = {bad_callback,{'кирилли́ческий атом','кирилли́ческий атом',0}}, + BadModule = + {bad_module,{'кирилли́ческий атом','кирилли́ческий атом',0}}, "explicit module not allowed for callback " "'кирилли́ческий атом':'кирилли́ческий атом'/0" = format_error(BadCallback), @@ -4052,6 +4054,7 @@ non_latin1_module(Config) -> {11,erl_lint,illegal_guard_expr}, {15,erl_lint,non_latin1_module_unsupported}, {17,erl_lint,non_latin1_module_unsupported}, + {17,erl_lint,BadModule}, {20,erl_lint,non_latin1_module_unsupported}, {23,erl_lint,non_latin1_module_unsupported}, {25,erl_lint,non_latin1_module_unsupported}], @@ -4229,6 +4232,21 @@ external_funs(Config) when is_list(Config) -> run(Config, Ts), ok. +otp_15563(Config) when is_list(Config) -> + Ts = [{otp_15563, + <<"-type deep_list(A) :: [A | deep_list(A)]. + -spec lists:flatten(deep_list(A)) -> [A]. + -callback lists:concat([_]) -> string(). + -spec ?MODULE:foo() -> any(). + foo() -> a. + ">>, + [warn_unused_vars], + {errors,[{2,erl_lint,{bad_module,{lists,flatten,1}}}, + {3,erl_lint,{bad_callback,{lists,concat,1}}}], + []}}], + [] = run(Config, Ts), + ok. + format_error(E) -> lists:flatten(erl_lint:format_error(E)). diff --git a/lib/stdlib/test/erl_pp_SUITE.erl b/lib/stdlib/test/erl_pp_SUITE.erl index 3eb1670806..c0cfd26925 100644 --- a/lib/stdlib/test/erl_pp_SUITE.erl +++ b/lib/stdlib/test/erl_pp_SUITE.erl @@ -829,7 +829,7 @@ type_examples() -> "(t24()) -> D when is_subtype(D, atom())," " is_subtype(D, t14())," " is_subtype(D, '\\'t::4'()).">>}, - {ex32,<<"-spec mod:t2() -> any(). ">>}, + {ex32,<<"-spec erl_pp_test:t2() -> any(). ">>}, {ex33,<<"-opaque attributes_data() :: " "[{'column', column()} | {'line', info_line()} |" " {'text', string()}] | {line(),column()}. ">>}, diff --git a/lib/stdlib/test/string_SUITE.erl b/lib/stdlib/test/string_SUITE.erl index 248912c3f2..c9aadd7f10 100644 --- a/lib/stdlib/test/string_SUITE.erl +++ b/lib/stdlib/test/string_SUITE.erl @@ -103,6 +103,15 @@ debug() -> test(?LINE,?FUNCTION_NAME,B,C,D, false), test(?LINE,?FUNCTION_NAME,hd(C),[B|tl(C)],D, false)). +-define(TRY(Exp), + fun() -> + try Exp + catch _E:Reason:_ST -> + %% io:format("~p:~w: ~p: ~.0p ~p~n", + %% [?FUNCTION_NAME, ?LINE,_E,Reason, hd(_ST)]), + {'EXIT', Reason} + end + end()). is_empty(_) -> ?TEST("", [], true), @@ -126,6 +135,10 @@ length(_) -> ?TEST(["abc"|<<"abc">>], [], 6), ?TEST(["abc",["def"]], [], 6), ?TEST([<<97/utf8, 778/utf8, 98/utf8>>, [776,111,776]], [], 3), %% åäö in nfd + + InvalidUTF8 = <<192,192>>, + {'EXIT', {badarg, _}} = ?TRY(string:length(InvalidUTF8)), + {'EXIT', {badarg, _}} = ?TRY(string:length(<<$a, InvalidUTF8/binary, $z>>)), ok. equal(_) -> @@ -226,6 +239,8 @@ to_graphemes(_) -> true = erlang:length(GCs) =:= erlang:length(string:to_graphemes(NFD)), true = erlang:length(GCs) =:= erlang:length(string:to_graphemes(unicode:characters_to_nfc_list(String))), + + {'EXIT', {badarg, _}} = ?TRY(string:to_graphemes(<<$a,192,192,$z>>)), ok. reverse(_) -> @@ -238,6 +253,11 @@ reverse(_) -> ?TEST(Str2, [], lists:reverse(Str2)), ?TEST(Str3, [], lists:reverse(Str3)), true = string:reverse(Str3) =:= lists:reverse(string:to_graphemes(Str3)), + + InvalidUTF8 = <<192,192>>, + {'EXIT', {badarg, _}} = ?TRY(string:reverse(InvalidUTF8)), + {'EXIT', {badarg, _}} = ?TRY(string:reverse(<<$a, InvalidUTF8/binary, $z>>)), + ok. slice(_) -> @@ -258,6 +278,14 @@ slice(_) -> ?TEST([<<"aå"/utf8>>,"äöbcd"], [3,3], "öbc"), ?TEST([<<"aåä"/utf8>>,"öbcd"], [3,10], "öbcd"), + InvalidUTF8 = <<192,192>>, + [$b, $c|InvalidUTF8] = string:slice(["abc", InvalidUTF8], 1), + InvalidUTF8 = string:slice(["abc", InvalidUTF8], 3), + {'EXIT', {badarg, _}} = ?TRY(string:slice(["abc", InvalidUTF8], 1, 5)), + BadUtf8 = <<$a, InvalidUTF8/binary, "teststring">>, + {'EXIT', {badarg, _}} = ?TRY(string:slice(BadUtf8, 2)), + {'EXIT', {badarg, _}} = ?TRY(string:slice(BadUtf8, 1, 5)), + {'EXIT', {badarg, _}} = ?TRY(string:slice(BadUtf8, 0, 5)), ok. pad(_) -> @@ -270,6 +298,10 @@ pad(_) -> ?TEST(Str, [10, trailing, $.], "Hallå....."), ?TEST(Str++["f"], [10, trailing, $.], "Hallåf...."), ?TEST(Str++[" flåwer"], [10, trailing, $.], "Hallå flåwer"), + + InvalidUTF8 = <<192,192>>, + {'EXIT', {badarg, _}} = ?TRY(string:pad(InvalidUTF8, 10, both, $.)), + {'EXIT', {badarg, _}} = ?TRY(string:pad(<<$a, InvalidUTF8/binary, $z>>, 10, both, $.)), ok. trim(_) -> @@ -300,6 +332,11 @@ trim(_) -> ?TEST([[<<"!v">>|<<204,128,$v,204,129>>]],[trailing, [[$v,769]]], [$!,$v,768]), ?TEST([[[<<"v">>|<<204,129,118,204,128,118>>],769,118,769]], [trailing, [[118,769]]], [$v,769,$v,768]), ?TEST([<<"vv">>|<<204,128,118,204,128>>], [trailing, [[118,768]]], "v"), + + InvalidUTF8 = <<192,192>>, + {'EXIT', {badarg, _}} = ?TRY(string:trim(InvalidUTF8, both, "az")), + %% Not checked (using binary search) + %% {'EXIT', {badarg, _}} = ?TRY(string:trim(<<$a, $b, InvalidUTF8/binary, $z>>, both, "az")), ok. chomp(_) -> @@ -400,6 +437,13 @@ take(_) -> ?TEST([<<"e">>,778,"åäöe", <<778/utf8>>, $e, 779], [[[$e,778]], true, trailing], {[$e,778]++"åäöe"++[778], [$e,779]}), + InvalidUTF8 = <<192,192>>, + {'EXIT', {badarg, _}} = ?TRY(string:take(InvalidUTF8, [$.], false, leading)), + %% Not checked (using binary search) + %% {'EXIT', {badarg, _}} = ?TRY(string:take(InvalidUTF8, [$.], true, leading)), + %% {'EXIT', {badarg, _}} = ?TRY(string:take(InvalidUTF8, [$.], false, trailing)), + {'EXIT', {badarg, _}} = ?TRY(string:take(InvalidUTF8, [$.], true, trailing)), + ok. @@ -416,6 +460,11 @@ uppercase(_) -> ?TEST("ljLJ", [], "LJLJ"), ?TEST("LJlj", [], "LJLJ"), ?TEST("ß sharp s", [], "SS SHARP S"), + + InvalidUTF8 = <<192,192>>, + {'EXIT', {badarg, _}} = ?TRY(string:uppercase(InvalidUTF8)), + {'EXIT', {badarg, _}} = ?TRY(string:uppercase(<<$a, InvalidUTF8/binary, $z>>)), + ok. lowercase(_) -> @@ -429,6 +478,10 @@ lowercase(_) -> ?TEST(["Mic",<<"HAŁ"/utf8>>], [], "michał"), ?TEST("ß SHARP S", [], "ß sharp s"), ?TEST("İ I WITH DOT ABOVE", [], "i̇ i with dot above"), + + InvalidUTF8 = <<192,192>>, + {'EXIT', {badarg, _}} = ?TRY(string:lowercase(InvalidUTF8)), + {'EXIT', {badarg, _}} = ?TRY(string:lowercase(<<$a, InvalidUTF8/binary, $z>>)), ok. titlecase(_) -> @@ -442,6 +495,10 @@ titlecase(_) -> ?TEST("ljLJ", [], "LjLJ"), ?TEST("LJlj", [], "Ljlj"), ?TEST("ß sharp s", [], "Ss sharp s"), + + InvalidUTF8 = <<192,192>>, + {'EXIT', {badarg, _}} = ?TRY(string:titlecase(InvalidUTF8)), + <<$A, _/binary>> = ?TRY(string:titlecase(<<$a, InvalidUTF8/binary, $z>>)), ok. casefold(_) -> @@ -456,6 +513,10 @@ casefold(_) -> ?TEST("ß SHARP S", [], "ss sharp s"), ?TEST("ẞ SHARP S", [], "ss sharp s"), ?TEST("İ I WITH DOT ABOVE", [], "i̇ i with dot above"), + + InvalidUTF8 = <<192,192>>, + {'EXIT', {badarg, _}} = ?TRY(string:casefold(InvalidUTF8)), + {'EXIT', {badarg, _}} = ?TRY(string:casefold(<<$a, InvalidUTF8/binary, $z>>)), ok. @@ -740,7 +801,7 @@ meas(Config) -> _ -> % No scaling, run at most 1.5 min Tester = spawn(Exec), receive {test_done, Tester} -> ok - after 90000 -> + after 118000 -> io:format("Timelimit reached stopping~n",[]), exit(Tester, die) end, diff --git a/lib/stdlib/test/unicode_util_SUITE.erl b/lib/stdlib/test/unicode_util_SUITE.erl index 044b4e5834..6f55f204f4 100644 --- a/lib/stdlib/test/unicode_util_SUITE.erl +++ b/lib/stdlib/test/unicode_util_SUITE.erl @@ -428,7 +428,15 @@ mode(deep_l, Bin) -> [unicode:characters_to_list(Bin)]. fetch(Str, F) -> case F(Str) of [] -> []; - [CP|R] -> [CP|fetch(R,F)] + [CP|R] -> + %% If input is a binary R should be binary + if is_binary(Str) == false -> ok; + is_binary(R); R =:= [] -> ok; + true -> + io:format("Char: ~tc Tail:~tP~n", [CP,R,10]), + exit({bug, F}) + end, + [CP|fetch(R,F)] end. %% *Test.txt file helpers diff --git a/lib/stdlib/uc_spec/gen_unicode_mod.escript b/lib/stdlib/uc_spec/gen_unicode_mod.escript index 8636c69a0d..de67b18afc 100644 --- a/lib/stdlib/uc_spec/gen_unicode_mod.escript +++ b/lib/stdlib/uc_spec/gen_unicode_mod.escript @@ -202,7 +202,8 @@ gen_static(Fd) -> io:put_chars(Fd, " {Upper,_} -> [Upper|Str];\n"), io:put_chars(Fd, " {Upper,_,_,_} -> [Upper|Str]\n"), io:put_chars(Fd, " end;\n"), - io:put_chars(Fd, " [] -> []\n"), + io:put_chars(Fd, " [] -> [];\n"), + io:put_chars(Fd, " {error,Err} -> error({badarg, Err})\n"), io:put_chars(Fd, " end.\n\n"), io:put_chars(Fd, "-spec lowercase(unicode:chardata()) -> " "maybe_improper_list(gc(),unicode:chardata()).\n"), @@ -213,7 +214,8 @@ gen_static(Fd) -> io:put_chars(Fd, " {_,Lower} -> [Lower|Str];\n"), io:put_chars(Fd, " {_,Lower,_,_} -> [Lower|Str]\n"), io:put_chars(Fd, " end;\n"), - io:put_chars(Fd, " [] -> []\n"), + io:put_chars(Fd, " [] -> [];\n"), + io:put_chars(Fd, " {error,Err} -> error({badarg, Err})\n"), io:put_chars(Fd, " end.\n\n"), io:put_chars(Fd, "-spec titlecase(unicode:chardata()) -> " "maybe_improper_list(gc(),unicode:chardata()).\n"), @@ -224,7 +226,8 @@ gen_static(Fd) -> io:put_chars(Fd, " {_,_,Title,_} -> [Title|Str];\n"), io:put_chars(Fd, " {Upper,_} -> [Upper|Str]\n"), io:put_chars(Fd, " end;\n"), - io:put_chars(Fd, " [] -> []\n"), + io:put_chars(Fd, " [] -> [];\n"), + io:put_chars(Fd, " {error,Err} -> error({badarg, Err})\n"), io:put_chars(Fd, " end.\n\n"), io:put_chars(Fd, "-spec casefold(unicode:chardata()) -> " "maybe_improper_list(gc(),unicode:chardata()).\n"), @@ -235,7 +238,8 @@ gen_static(Fd) -> io:put_chars(Fd, " {_,_,_,Fold} -> [Fold|Str];\n"), io:put_chars(Fd, " {_,Lower} -> [Lower|Str]\n"), io:put_chars(Fd, " end;\n"), - io:put_chars(Fd, " [] -> []\n"), + io:put_chars(Fd, " [] -> [];\n"), + io:put_chars(Fd, " {error,Err} -> error({badarg, Err})\n"), io:put_chars(Fd, " end.\n\n"), ok. @@ -619,7 +623,7 @@ gen_gc(Fd, GBP) -> GenHangulT = fun(Range) -> io:format(Fd, "gc_1~s gc_h_T(R1,[CP]);\n", [gen_clause(Range)]) end, [GenHangulT(CP) || CP <- merge_ranges(maps:get(t,GBP))], io:put_chars(Fd, "%% Handle Hangul LV and LVT special, since they are large\n"), - io:put_chars(Fd, "gc_1([CP|_]=R0) when 44000 < CP, CP < 56000 -> gc_h_lv_lvt(R0, []);\n"), + io:put_chars(Fd, "gc_1([CP|_]=R0) when 44000 < CP, CP < 56000 -> gc_h_lv_lvt(R0, R0, []);\n"), io:put_chars(Fd, "\n%% Handle Regional\n"), GenRegional = fun(Range) -> io:format(Fd, "gc_1~s gc_regional(R1,CP);\n", [gen_clause(Range)]) end, @@ -748,7 +752,7 @@ gen_gc(Fd, GBP) -> GenHangulL_2 = fun(Range) -> io:format(Fd, "~8c~s gc_h_V(R1,[CP|Acc]);\n", [$\s,gen_case_clause(Range)]) end, [GenHangulL_2(CP) || CP <- merge_ranges(maps:get(v,GBP))], - io:put_chars(Fd, " R1 -> gc_h_lv_lvt(R1, Acc)\n end.\n\n"), + io:put_chars(Fd, " R1 -> gc_h_lv_lvt(R1, R0, Acc)\n end.\n\n"), io:put_chars(Fd, "%% Handle Hangul V\n"), io:put_chars(Fd, "gc_h_V(R0, Acc) ->\n case cp(R0) of\n"), @@ -783,10 +787,10 @@ gen_gc(Fd, GBP) -> GenHangulLVT = fun(Range) -> io:format(Fd, "gc_h_lv_lvt~s gc_h_T(R1,[CP|Acc]);\n", [gen_clause2(Range)]) end, [GenHangulLVT(CP) || CP <- merge_ranges(maps:get(lvt,GBP))], - io:put_chars(Fd, "gc_h_lv_lvt([CP|R], []) -> gc_extend(cp(R), R, CP);\n"), %% From gc_1/1 + io:put_chars(Fd, "gc_h_lv_lvt([CP|R1], _, []) -> gc_extend(cp(R1), R1, CP);\n"), %% From gc_1/1 io:put_chars(Fd, "%% Also handles error tuples\n"), - io:put_chars(Fd, "gc_h_lv_lvt(R, [CP]) -> gc_extend(R, R, CP);\n"), - io:put_chars(Fd, "gc_h_lv_lvt(R, Acc) -> gc_extend2(R, R, Acc).\n\n"), + io:put_chars(Fd, "gc_h_lv_lvt(R1, R0, [CP]) -> gc_extend(R1, R0, CP);\n"), + io:put_chars(Fd, "gc_h_lv_lvt(R1, R0, Acc) -> gc_extend2(R1, R0, Acc).\n\n"), ok. gen_compose_pairs(Fd, ExclData, Data) -> @@ -887,9 +891,9 @@ gen_clause({R0, R1}) -> io_lib:format("([CP|R1]=R0) when ~w =< CP, CP =< ~w ->", [R0,R1]). gen_clause2({R0, undefined}) -> - io_lib:format("([~w=CP|R1], Acc) ->", [R0]); + io_lib:format("([~w=CP|R1], R0, Acc) ->", [R0]); gen_clause2({R0, R1}) -> - io_lib:format("([CP|R1], Acc) when ~w =< CP, CP =< ~w ->", [R0,R1]). + io_lib:format("([CP|R1], R0, Acc) when ~w =< CP, CP =< ~w ->", [R0,R1]). gen_case_clause({R0, undefined}) -> io_lib:format("[~w=CP|R1] ->", [R0]); diff --git a/lib/tools/doc/src/make.xml b/lib/tools/doc/src/make.xml index af2404707f..322d77323f 100644 --- a/lib/tools/doc/src/make.xml +++ b/lib/tools/doc/src/make.xml @@ -145,5 +145,10 @@ Modules. {'*',[debug_info]}. </code> <p></p> </section> + + <section> + <title>See Also</title> + <p><seealso marker="compiler:compile"><c>compile(3)</c></seealso></p> + </section> </erlref> diff --git a/make/configure.in b/make/configure.in index bf3fd0751f..c4b89c4f45 100644 --- a/make/configure.in +++ b/make/configure.in @@ -2,7 +2,7 @@ dnl Process this file with autoconf to produce a configure script. dnl %CopyrightBegin% dnl -dnl Copyright Ericsson AB 1998-2016. All Rights Reserved. +dnl Copyright Ericsson AB 1998-2019. All Rights Reserved. dnl dnl Licensed under the Apache License, Version 2.0 (the "License"); dnl you may not use this file except in compliance with the License. @@ -298,6 +298,10 @@ AC_ARG_ENABLE(builtin-zlib, AS_HELP_STRING([--enable-builtin-zlib], [force use of our own built-in zlib])) +AC_ARG_ENABLE(esock, +AS_HELP_STRING([--enable-esock], [enable builtin experimental socket (as a nif) support (default)]) +AS_HELP_STRING([--disable-esock], [disable builtin experimental socket (as a nif) support])) + AC_ARG_ENABLE(sharing-preserving, AS_HELP_STRING([--enable-sharing-preserving], [enable copying of terms without destroying sharing])) diff --git a/make/otp.mk.in b/make/otp.mk.in index ceff8f7c31..cdddb90734 100644 --- a/make/otp.mk.in +++ b/make/otp.mk.in @@ -4,7 +4,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1997-2016. All Rights Reserved. +# Copyright Ericsson AB 1997-2019. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -51,6 +51,8 @@ TYPES = @TYPES@ USE_PGO = @USE_PGO@ +USE_ESOCK = @USE_ESOCK@ + # Slash separated list of return values from $(origin VAR) # that are untrusted - set default in this file instead. # The list is not space separated since some return values diff --git a/scripts/run-dialyzer b/scripts/run-dialyzer index 621de3fa65..44436594d3 100755 --- a/scripts/run-dialyzer +++ b/scripts/run-dialyzer @@ -1,17 +1,47 @@ #!/bin/bash set -e -set -x -$ERL_TOP/bin/dialyzer --build_plt --apps asn1 compiler crypto dialyzer edoc erts et ftp hipe inets kernel mnesia observer public_key runtime_tools snmp ssh ssl stdlib syntax_tools tftp wx xmerl --statistics -$ERL_TOP/bin/dialyzer -n -Wunknown -Wunmatched_returns --apps compiler erts ftp tftp kernel stdlib asn1 crypto dialyzer hipe parsetools public_key runtime_tools sasl tools --statistics -$ERL_TOP/bin/dialyzer -n --apps common_test debugger edoc ftp inets mnesia observer ssh ssl syntax_tools tftp wx xmerl --statistics +filter () { + FILTER_RESULT="" + for app in $1; do + if echo " $2 " | grep -v " $app " > /dev/null; then + FILTER_RESULT="$FILTER_RESULT $app" + fi + done +} + +ALL_APPLICATIONS=$(ls -d -1 lib/*/ | sed 's:^lib\|/::g') +echo "All applications: $ALL_APPLICATIONS" |tr '\n' ' ' && echo "" + +BASE_PLT="compiler crypto erts hipe kernel stdlib syntax_tools" +APP_PLT="asn1 edoc et ftp inets mnesia observer public_key sasl runtime_tools snmp ssl tftp wx xmerl tools" +NO_UNMATCHED="common_test debugger edoc eunit ftp inets mnesia observer reltool ssh ssl syntax_tools tftp wx xmerl" +WARNINGS="diameter megaco snmp" + +TRAVIS_SKIP="" +if [ "X$TRAVIS" = "Xtrue" ]; then + TRAVIS_SKIP="common_test eldap erl_docgen odbc eunit reltool os_mon diameter megaco snmp" +fi + +filter "$ALL_APPLICATIONS" "$NO_UNMATCHED $WARNINGS $TRAVIS_SKIP" +UNMATCHED=$FILTER_RESULT +filter "$APP_PLT" "$TRAVIS_SKIP" +APP_PLT=$FILTER_RESULT +filter "$NO_UNMATCHED" "$TRAVIS_SKIP" +NO_UNMATCHED=$FILTER_RESULT +filter "$WARNINGS" "$TRAVIS_SKIP" +WARNINGS=$FILTER_RESULT -# In travis we don't dialyze everything as it takes too much time -if [ "X$TRAVIS" != "Xtrue" ]; then - $ERL_TOP/bin/dialyzer -n -Wunknown -Wunmatched_returns --apps eldap erl_docgen et odbc --statistics - $ERL_TOP/bin/dialyzer -n --apps eunit reltool os_mon --statistics +echo "UNMATCHED = $UNMATCHED" +echo "NO_UNMATCHED = $NO_UNMATCHED" +echo "WARNINGS = $WARNINGS" + +set -x - # These application are not run always as the currently have dialyzer warnings - # $ERL_TOP/bin/dialyzer -n --apps diameter megaco snmp --statistics +$ERL_TOP/bin/dialyzer --build_plt -Wunknown --apps $BASE_PLT $APP_PLT --statistics +$ERL_TOP/bin/dialyzer -n -Wunknown -Wunmatched_returns --apps $UNMATCHED --statistics +$ERL_TOP/bin/dialyzer -n -Wunknown --apps $NO_UNMATCHED --statistics +if [ "X$WARNINGS" != "X" ]; then + $ERL_TOP/bin/dialyzer -n --apps $WARNINGS --statistics fi |