diff options
39 files changed, 996 insertions, 385 deletions
diff --git a/erts/configure.in b/erts/configure.in index 5f969a0a8b..506ce0d0fb 100644 --- a/erts/configure.in +++ b/erts/configure.in @@ -2,7 +2,7 @@ dnl Process this file with autoconf to produce a configure script. -*-m4-*- dnl %CopyrightBegin% dnl -dnl Copyright Ericsson AB 1997-2018. All Rights Reserved. +dnl Copyright Ericsson AB 1997-2019. All Rights Reserved. dnl dnl Licensed under the Apache License, Version 2.0 (the "License"); dnl you may not use this file except in compliance with the License. @@ -590,6 +590,22 @@ AC_SUBST(WERRORFLAGS) ## Check if we can do profile guided optimization of beam_emu LM_CHECK_ENABLE_CFLAG([-fprofile-generate -Werror],[PROFILE_GENERATE]) LM_CHECK_ENABLE_CFLAG([-fprofile-use -Werror],[PROFILE_USE]) +LM_CHECK_ENABLE_CFLAG([-fprofile-use -fprofile-correction -Werror],[PROFILE_CORRECTION]) + +if test "X$PROFILE_CORRECTION" = "Xtrue"; then + saved_CFLAGS=$CFLAGS + saved_LDFLAGS=$LDFLAGS + CFLAGS="-fprofile-generate $saved_CFLAGS" + LDFLAGS="-fprofile-generate $saved_LDFLAGS" + AC_MSG_CHECKING([whether $CC links with -fprofile-generate]) + AC_LINK_IFELSE([AC_LANG_PROGRAM([],[return 0;])], + [AC_MSG_RESULT([yes]) + PROFILE_GENERATE=true], + [AC_MSG_RESULT([no]) + PROFILE_GENERATE=false]) + CFLAGS=$saved_CFLAGS + LDFLAGS=$saved_LDFLAGS +fi ## Check if this is clang LM_CHECK_ENABLE_CFLAG([-fprofile-instr-generate -Werror],[PROFILE_INSTR_GENERATE]) @@ -614,8 +630,8 @@ if test "X$PROFILE_INSTR_GENERATE" = "Xtrue"; then if test "X$LLVM_PROFDATA" != "X"; then CFLAGS="-fprofile-instr-use=default.profdata -Werror $saved_CFLAGS"; $LLVM_PROFDATA merge -output=default.profdata *.profraw; - AC_MSG_CHECKING([whether gcc accepts -fprofile-instr-use=default.profdata -Werror]) - AC_COMPILE_IFELSE([], + AC_MSG_CHECKING([whether $CC accepts -fprofile-instr-use=default.profdata -Werror]) + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([],[return 0;])], [AC_MSG_RESULT([yes]) PROFILE_INSTR_USE=true], [AC_MSG_RESULT([no]) @@ -637,8 +653,6 @@ AS_HELP_STRING([--enable-pgo], esac ],enable_pgo=default) -LM_CHECK_ENABLE_CFLAG([-fprofile-use -fprofile-correction -Werror],[PROFILE_CORRECTION]) - USE_PGO=false AC_MSG_CHECKING([whether to do PGO of erts]) if test $enable_pgo = no; then @@ -1329,6 +1343,28 @@ LIBS=$zlib_save_LIBS fi AC_SUBST(Z_LIB) + +dnl ------------- +dnl esock +dnl ------------- + +AC_ARG_ENABLE(esock, +AS_HELP_STRING([--enable-esock], [enable builtin experimental socket (as a nif) support (default)]) +AS_HELP_STRING([--disable-esock], [disable builtin experimental socket (as a nif) support])) + +dnl Default value +USE_ESOCK=yes + +if test "x$enable_esock" = "xyes"; then + USE_ESOCK=yes +else + if test "x$enable_esock" = "xno"; then + USE_ESOCK=no + fi +fi +AC_SUBST(USE_ESOCK) + + dnl dnl This test kindly borrowed from Tcl dnl diff --git a/erts/doc/src/erl_dist_protocol.xml b/erts/doc/src/erl_dist_protocol.xml index 185c75fe84..f924c8a70b 100644 --- a/erts/doc/src/erl_dist_protocol.xml +++ b/erts/doc/src/erl_dist_protocol.xml @@ -850,10 +850,15 @@ DiB == gen_digest(ChA, ICA)? <tag><c>-define(DFLAG_EXIT_PAYLOAD, 16#400000).</c></tag> <item> <p>Use the <c>PAYLOAD_EXIT</c>, <c>PAYLOAD_EXIT_TT</c>, - <c>PAYLOAD_EXIT2</c>, <c>PAYLOAD_EXIT2_TT</c> - and <c>PAYLOAD_MONITOR_P_EXIT</c> - <seealso marker="#control_message">control message</seealso>s - instead of the non-PAYLOAD variants.</p> + <c>PAYLOAD_EXIT2</c>, <c>PAYLOAD_EXIT2_TT</c> + and <c>PAYLOAD_MONITOR_P_EXIT</c> + <seealso marker="#control_message">control message</seealso>s + instead of the non-PAYLOAD variants.</p> + </item> + <tag><c>-define(DFLAG_FRAGMENTS, 16#800000).</c></tag> + <item> + <p>Use <seealso marker="erl_ext_dist#fragments">fragmented</seealso> + distribution messages to send large messages.</p> </item> </taglist> <p> diff --git a/erts/doc/src/erl_ext_dist.xml b/erts/doc/src/erl_ext_dist.xml index a6bc44b8c8..3730f0e8ac 100644 --- a/erts/doc/src/erl_ext_dist.xml +++ b/erts/doc/src/erl_ext_dist.xml @@ -140,162 +140,366 @@ <marker id="distribution_header"/> <title>Distribution Header</title> <p> - The distribution header only contains an atom cache - reference section, but can in the future contain more - information. The distribution header precedes one or more Erlang - terms on the external format. For more information, see the - documentation of the + The distribution header is sent by the erlang distribution to + carry metadata about the coming + <seealso marker="erl_dist_protocol#control_message">control message</seealso> + and potential payload. It is primarily used to handle the atom cache + in the Erlang distribution. Since OTP-22 it is also used to fragment + large distribution messages into multiple smaller fragments. + For more information about how the distribution uses the distribution header, + see the documentation of the <seealso marker="erl_dist_protocol#connected_nodes">protocol between connected nodes</seealso> in the <seealso marker="erl_dist_protocol">distribution protocol</seealso> documentation. </p> <p> - <seealso marker="#ATOM_CACHE_REF">ATOM_CACHE_REF</seealso> + Any <seealso marker="#ATOM_CACHE_REF">ATOM_CACHE_REF</seealso> entries with corresponding <c>AtomCacheReferenceIndex</c> in terms encoded on the external format following a distribution header refer to the atom cache references made in the distribution header. The range is 0 <= <c>AtomCacheReferenceIndex</c> < 255, that is, at most 255 different atom cache references from the following terms can be made. </p> - <p> - The distribution header format is as follows: - </p> - <table align="left"> - <row> - <cell align="center">1</cell> - <cell align="center">1</cell> - <cell align="center">1</cell> - <cell align="center">NumberOfAtomCacheRefs/2+1 | 0</cell> - <cell align="center">N | 0</cell> - </row> - <row> - <cell align="center"><c>131</c></cell> - <cell align="center"><c>68</c></cell> - <cell align="center"><c>NumberOfAtomCacheRefs</c></cell> - <cell align="center"><c>Flags</c></cell> - <cell align="center"><c>AtomCacheRefs</c></cell> - </row> - <tcaption>Distribution Header Format</tcaption></table> - <p> - <c>Flags</c> consist of <c>NumberOfAtomCacheRefs/2+1</c> bytes, - unless <c>NumberOfAtomCacheRefs</c> is <c>0</c>. If - <c>NumberOfAtomCacheRefs</c> is <c>0</c>, <c>Flags</c> and - <c>AtomCacheRefs</c> are omitted. Each atom cache reference has - a half byte flag field. Flags corresponding to a specific - <c>AtomCacheReferenceIndex</c> are located in flag byte number - <c>AtomCacheReferenceIndex/2</c>. Flag byte 0 is the first byte - after the <c>NumberOfAtomCacheRefs</c> byte. Flags for an even - <c>AtomCacheReferenceIndex</c> are located in the least significant - half byte and flags for an odd <c>AtomCacheReferenceIndex</c> are - located in the most significant half byte. - </p> - <p> - The flag field of an atom cache reference has the following - format: - </p> - <table align="left"> - <row> - <cell align="center">1 bit</cell> - <cell align="center">3 bits</cell> - </row> - <row> - <cell align="center"><c>NewCacheEntryFlag</c></cell> - <cell align="center"><c>SegmentIndex</c></cell> - </row> - <tcaption></tcaption></table> - <p> - The most significant bit is the <c>NewCacheEntryFlag</c>. If set, - the corresponding cache reference is new. The three least - significant bits are the <c>SegmentIndex</c> of the corresponding - atom cache entry. An atom cache consists of 8 segments, each of size - 256, that is, an atom cache can contain 2048 entries. - </p> - <p> - After flag fields for atom cache references, another half byte flag - field is located with the following format: - </p> - <table align="left"> - <row> - <cell align="center">3 bits</cell> - <cell align="center">1 bit</cell> - </row> - <row> - <cell align="center"><c>CurrentlyUnused</c></cell> - <cell align="center"><c>LongAtoms</c></cell> - </row> - <tcaption></tcaption></table> - <p> - The least significant bit in that half byte is flag <c>LongAtoms</c>. - If it is set, 2 bytes are used for atom lengths instead of - 1 byte in the distribution header. - </p> - <p> - After the <c>Flags</c> field follow the <c>AtomCacheRefs</c>. The - first <c>AtomCacheRef</c> is the one corresponding to - <c>AtomCacheReferenceIndex</c> 0. Higher indices follow - in sequence up to index <c>NumberOfAtomCacheRefs - 1</c>. - </p> - <p> - If the <c>NewCacheEntryFlag</c> for the next <c>AtomCacheRef</c> has - been set, a <c>NewAtomCacheRef</c> on the following format follows: - </p> - <table align="left"> - <row> - <cell align="center">1</cell> - <cell align="center">1 | 2</cell> - <cell align="center">Length</cell> - </row> - <row> - <cell align="center"><c>InternalSegmentIndex</c></cell> - <cell align="center"><c>Length</c></cell> - <cell align="center"><c>AtomText</c></cell> - </row> - <tcaption></tcaption></table> - <p> - <c>InternalSegmentIndex</c> together with the <c>SegmentIndex</c> - completely identify the location of an atom cache entry in the - atom cache. <c>Length</c> is the number of bytes that <c>AtomText</c> - consists of. Length is a 2 byte big-endian integer - if flag <c>LongAtoms</c> has been set, otherwise a 1 byte - integer. When distribution flag - <seealso marker="erl_dist_protocol#dflags"> - <c>DFLAG_UTF8_ATOMS</c></seealso> - has been exchanged between both nodes in the - <seealso marker="erl_dist_protocol#distribution_handshake"> - distribution handshake</seealso>, - characters in <c>AtomText</c> are encoded in UTF-8, otherwise - in Latin-1. The following <c>CachedAtomRef</c>s with the same - <c>SegmentIndex</c> and <c>InternalSegmentIndex</c> as this - <c>NewAtomCacheRef</c> refer to this atom until a new - <c>NewAtomCacheRef</c> with the same <c>SegmentIndex</c> - and <c>InternalSegmentIndex</c> appear. - </p> - <p> - For more information on encoding of atoms, see the - <seealso marker="#utf8_atoms">note on UTF-8 encoded atoms</seealso> - in the beginning of this section. - </p> - <p> - If the <c>NewCacheEntryFlag</c> for the next <c>AtomCacheRef</c> - has not been set, a <c>CachedAtomRef</c> on the following format - follows: - </p> - <table align="left"> - <row> - <cell align="center">1</cell> - </row> - <row> - <cell align="center"><c>InternalSegmentIndex</c></cell> - </row> - <tcaption></tcaption></table> - <p> - <c>InternalSegmentIndex</c> together with the <c>SegmentIndex</c> - identify the location of the atom cache entry in the atom cache. - The atom corresponding to this <c>CachedAtomRef</c> is the - latest <c>NewAtomCacheRef</c> preceding this <c>CachedAtomRef</c> - in another previously passed distribution header. - </p> + <section> + <title>Normal Distribution Header</title> + <p> + The non-fragmented distribution header format is as follows: + </p> + <table align="left"> + <row> + <cell align="center">1</cell> + <cell align="center">1</cell> + <cell align="center">1</cell> + <cell align="center">NumberOfAtomCacheRefs/2+1 | 0</cell> + <cell align="center">N | 0</cell> + </row> + <row> + <cell align="center"><c>131</c></cell> + <cell align="center"><c>68</c></cell> + <cell align="center"><c>NumberOfAtomCacheRefs</c></cell> + <cell align="center"><c>Flags</c></cell> + <cell align="center"><c>AtomCacheRefs</c></cell> + </row> + <tcaption>Normal Distribution Header Format</tcaption></table> + <p> + <c>Flags</c> consist of <c>NumberOfAtomCacheRefs/2+1</c> bytes, + unless <c>NumberOfAtomCacheRefs</c> is <c>0</c>. If + <c>NumberOfAtomCacheRefs</c> is <c>0</c>, <c>Flags</c> and + <c>AtomCacheRefs</c> are omitted. Each atom cache reference has + a half byte flag field. Flags corresponding to a specific + <c>AtomCacheReferenceIndex</c> are located in flag byte number + <c>AtomCacheReferenceIndex/2</c>. Flag byte 0 is the first byte + after the <c>NumberOfAtomCacheRefs</c> byte. Flags for an even + <c>AtomCacheReferenceIndex</c> are located in the least significant + half byte and flags for an odd <c>AtomCacheReferenceIndex</c> are + located in the most significant half byte. + </p> + <p> + The flag field of an atom cache reference has the following + format: + </p> + <table align="left"> + <row> + <cell align="center">1 bit</cell> + <cell align="center">3 bits</cell> + </row> + <row> + <cell align="center"><c>NewCacheEntryFlag</c></cell> + <cell align="center"><c>SegmentIndex</c></cell> + </row> + <tcaption></tcaption></table> + <p> + The most significant bit is the <c>NewCacheEntryFlag</c>. If set, + the corresponding cache reference is new. The three least + significant bits are the <c>SegmentIndex</c> of the corresponding + atom cache entry. An atom cache consists of 8 segments, each of size + 256, that is, an atom cache can contain 2048 entries. + </p> + <p> + After flag fields for atom cache references, another half byte flag + field is located with the following format: + </p> + <table align="left"> + <row> + <cell align="center">3 bits</cell> + <cell align="center">1 bit</cell> + </row> + <row> + <cell align="center"><c>CurrentlyUnused</c></cell> + <cell align="center"><c>LongAtoms</c></cell> + </row> + <tcaption></tcaption></table> + <p> + The least significant bit in that half byte is flag <c>LongAtoms</c>. + If it is set, 2 bytes are used for atom lengths instead of + 1 byte in the distribution header. + </p> + <p> + After the <c>Flags</c> field follow the <c>AtomCacheRefs</c>. The + first <c>AtomCacheRef</c> is the one corresponding to + <c>AtomCacheReferenceIndex</c> 0. Higher indices follow + in sequence up to index <c>NumberOfAtomCacheRefs - 1</c>. + </p> + <p> + If the <c>NewCacheEntryFlag</c> for the next <c>AtomCacheRef</c> has + been set, a <c>NewAtomCacheRef</c> on the following format follows: + </p> + <table align="left"> + <row> + <cell align="center">1</cell> + <cell align="center">1 | 2</cell> + <cell align="center">Length</cell> + </row> + <row> + <cell align="center"><c>InternalSegmentIndex</c></cell> + <cell align="center"><c>Length</c></cell> + <cell align="center"><c>AtomText</c></cell> + </row> + <tcaption></tcaption></table> + <p> + <c>InternalSegmentIndex</c> together with the <c>SegmentIndex</c> + completely identify the location of an atom cache entry in the + atom cache. <c>Length</c> is the number of bytes that <c>AtomText</c> + consists of. Length is a 2 byte big-endian integer + if flag <c>LongAtoms</c> has been set, otherwise a 1 byte + integer. When distribution flag + <seealso marker="erl_dist_protocol#dflags"> + <c>DFLAG_UTF8_ATOMS</c></seealso> + has been exchanged between both nodes in the + <seealso marker="erl_dist_protocol#distribution_handshake"> + distribution handshake</seealso>, + characters in <c>AtomText</c> are encoded in UTF-8, otherwise + in Latin-1. The following <c>CachedAtomRef</c>s with the same + <c>SegmentIndex</c> and <c>InternalSegmentIndex</c> as this + <c>NewAtomCacheRef</c> refer to this atom until a new + <c>NewAtomCacheRef</c> with the same <c>SegmentIndex</c> + and <c>InternalSegmentIndex</c> appear. + </p> + <p> + For more information on encoding of atoms, see the + <seealso marker="#utf8_atoms">note on UTF-8 encoded atoms</seealso> + in the beginning of this section. + </p> + <p> + If the <c>NewCacheEntryFlag</c> for the next <c>AtomCacheRef</c> + has not been set, a <c>CachedAtomRef</c> on the following format + follows: + </p> + <table align="left"> + <row> + <cell align="center">1</cell> + </row> + <row> + <cell align="center"><c>InternalSegmentIndex</c></cell> + </row> + <tcaption></tcaption></table> + <p> + <c>InternalSegmentIndex</c> together with the <c>SegmentIndex</c> + identify the location of the atom cache entry in the atom cache. + The atom corresponding to this <c>CachedAtomRef</c> is the + latest <c>NewAtomCacheRef</c> preceding this <c>CachedAtomRef</c> + in another previously passed distribution header. + </p> + </section> + <section> + <marker id="fragments"/> + <title>Distribution Header for fragmented messages</title> + <p>Messages sent between Erlang nodes can sometimes be + quite large. Since OTP-22 it is possible to split large messages + into smaller fragments in order to allow smaller messages to be interleaved + between larges messages. It is only the <c>message</c> part of each + <seealso marker="erl_dist_protocol#connected_nodes">distributed message</seealso> + that may be split using fragmentation. Therefore it is recommended to use the + <seealso marker="erl_dist_protocol#new-ctrlmessages-for-erlang-otp-22"> + PAYLOAD control messages</seealso> introduced in OTP-22. + </p> + <p>Fragmented distribution messages are only used if the receiving node + signals that it supports them via the + <seealso marker="erl_dist_protocol#dflags">DFLAG_FRAGMENTS</seealso> distribution + flag.</p> + <p>A process must complete the sending of a fragmented message before it + can start sending any other message on the same distribution channel.</p> + + <p>The start of a sequence of fragmented messages looks like this:</p> + <table align="left"> + <row> + <cell align="center">1</cell> + <cell align="center">1</cell> + <cell align="center">8</cell> + <cell align="center">8</cell> + <cell align="center">1</cell> + <cell align="center">NumberOfAtomCacheRefs/2+1 | 0</cell> + <cell align="center">N | 0</cell> + </row> + <row> + <cell align="center"><c>131</c></cell> + <cell align="center"><c>69</c></cell> + <cell align="center"><c>SequenceId</c></cell> + <cell align="center"><c>FragmentId</c></cell> + <cell align="center"><c>NumberOfAtomCacheRefs</c></cell> + <cell align="center"><c>Flags</c></cell> + <cell align="center"><c>AtomCacheRefs</c></cell> + </row> + <tcaption>Starting Fragmented Distribution Header Format</tcaption> + </table> + + <p>The continuation of a sequence of fragmented messages looks like this:</p> + <table align="left"> + <row> + <cell align="center">1</cell> + <cell align="center">1</cell> + <cell align="center">8</cell> + <cell align="center">8</cell> + </row> + <row> + <cell align="center"><c>131</c></cell> + <cell align="center"><c>70</c></cell> + <cell align="center"><c>SequenceId</c></cell> + <cell align="center"><c>FragmentId</c></cell> + </row> + <tcaption>Continuing Fragmented Distribution Header Format</tcaption> + </table> + + <p> + The starting distribution header is very similar to a non-fragmented distribution + header. The atom cache works the same as for normal distribution header and + is the same for the entire sequence. The additional fields added are the + sequence id and fragment id. + </p> + + <taglist> + <tag>Sequence ID</tag> + <item> + <p> + The sequence id is used to uniquely identify a fragmented message sent + from one process to another on the same distributed connection. This is used + to identify which sequence a fragment is a part of as the same process can + be in the process of receiving multiple sequences at the same time. + </p> + <p> + As one process can only be sending one fragmented message at once, + it can be convenient to use the local PID as the sequence id. + </p> + </item> + <tag>Fragments ID</tag> + <item> + <p> + The Fragment ID is used to number the fragments in a sequence. + The id starts at the total number of fragments and then decrements to 1 + (which is the final fragment). So if a sequence consists of 3 fragments + the fragment id in the starting header will be 3, and then fragments 2 and 1 + are sent. + </p> + <p> + The fragments must be delivered in the correct order, so if an unordered + distribution carrier is used, they must be ordered before delivered to the + Erlang run-time. + </p> + </item> + </taglist> + + <section> + <title>Example:</title> + <p> + As an example, let say that we want to send + <c>{call, <0.245.2>, {set_get_state, <<0:1024>>}}</c> to + registered process <c>reg</c> using a fragment size of 128. To send + this message we need a distribution header, atom cache updates, + the control message (which would be <c>{6, <0.245.2>, [], reg}</c> in this case) + and finally the actual message. This would all be encoded into: + </p> + + <code> +131,69,0,0,2,168,0,0,5,83,0,0,0,0,0,0,0,2, %% Header with seq and frag id +5,4,137,9,10,5,236,3,114,101,103,9,4,99,97,108,108, %% Atom cache updates +238,13,115,101,116,95,103,101,116,95,115,116,97,116,101, +104,4,97,6,103,82,0,0,0,0,85,0,0,0,0,2,82,1,82,2, %% Control message +104,3,82,3,103,82,0,0,0,0,245,0,0,0,2,2, %% Actual message using cached atoms +104,2,82,4,109,0,0,0,128,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 + +131,70,0,0,2,168,0,0,5,83,0,0,0,0,0,0,0,1, %% Cont Header with seq and frag id +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, %% Rest of payload +0,0,0,0</code> + + <p> + Let us break that apart into its components. First we have the + distribution header tags together with the sequence id and + a fragment id of 2. + </p> + <code> +131,69, %% Start fragment header +0,0,2,168,0,0,5,83, %% The sequence ID +0,0,0,0,0,0,0,2, %% The fragment ID +</code> + <p>Then we have the updates to the atom cache:</p> + <code> +5,4,137,9, %% 5 atoms and their flags +10,5, %% The already cached atom ids +236,3,114,101,103, %% The atom 'reg' +9,4,99,97,108,108, %% The atom 'call' +238,13,115,101,116,95,103,101,116,95,115,116,97,116,101, %% The atom 'set_get_state' + </code> + <p> + The first byte says that we have 5 atoms that are part + of the cache. Then follows three bytes that are the + atom cache ref flags. Each of the flags uses 4 bits so + they are a bit hard to read in decimal byte form. In + binary half-byte form they look like this: + </p> + <code>0000, 0100, 1000, 1001, 1001</code> + <p> + As the high bit of the first two atoms in the + cache are not set we know that they are already in the cache, + so they do not have to be sent again (this is the node name of the + receiving and sending node). Then follows the atoms that have to be sent, + together with their segment ids. + </p> + <p> + Then the listing of the atoms comes, starting with 10 and 5 + which are the atom refs of the already cached atoms. Then the + new atoms are sent. + </p> + <p> + When the atom cache is setup correctly the control message is sent. + </p> + <code>104,4,97,6,103,82,0,0,0,0,85,0,0,0,0,2,82,1,82,2,</code> + <p> + Note that up until here it is not allowed to fragments the message. + The entire atom cache and control message has to be part of the + starting fragment. After the control message the payload of the message + is sent using 128 bytes: + </p> + <code> +104,3,82,3,103,82,0,0,0,0,245,0,0,0,2,2, +104,2,82,4,109,0,0,0,128,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 + </code> + <p> + Since the payload is larger than 128-bytes it is split into two + fragments. The second fragment does not have any atom cache update + instructions so it is a lot simpler: + </p> + <code> +131,70,0,0,2,168,0,0,5,83,0,0,0,0,0,0,0,1, %% Continuation dist header 70 with seq and frag id +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, %% remaining payload +0,0,0,0 + </code> + <note> + <p> + The fragment size of 128 is only used as an example. + Any fragments size may be used when sending fragmented messages. + </p> + </note> + </section> + </section> </section> <section> diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index 448f41b523..a9f3bb8e89 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -633,6 +633,15 @@ GENERATE += $(TTF_DIR)/driver_tab.c # This list must be consistent with PRE_LOADED_MODULES in # erts/preloaded/src/Makefile. +ifeq ($(USE_ESOCK), yes) +ESOCK_PRELOAD_BEAM = \ + $(ERL_TOP)/erts/preloaded/ebin/socket.beam \ + $(ERL_TOP)/erts/preloaded/ebin/net.beam +else +ESOCK_PRELOAD_BEAM = \ + $(ERL_TOP)/erts/preloaded/ebin/net.beam +endif + PRELOAD_BEAM = $(ERL_TOP)/erts/preloaded/ebin/erts_code_purger.beam \ $(ERL_TOP)/erts/preloaded/ebin/erl_init.beam \ $(ERL_TOP)/erts/preloaded/ebin/init.beam \ @@ -641,8 +650,7 @@ PRELOAD_BEAM = $(ERL_TOP)/erts/preloaded/ebin/erts_code_purger.beam \ $(ERL_TOP)/erts/preloaded/ebin/prim_inet.beam \ $(ERL_TOP)/erts/preloaded/ebin/prim_file.beam \ $(ERL_TOP)/erts/preloaded/ebin/zlib.beam \ - $(ERL_TOP)/erts/preloaded/ebin/socket.beam \ - $(ERL_TOP)/erts/preloaded/ebin/net.beam \ + $(ESOCK_PRELOAD_BEAM) \ $(ERL_TOP)/erts/preloaded/ebin/prim_zip.beam \ $(ERL_TOP)/erts/preloaded/ebin/erl_prim_loader.beam \ $(ERL_TOP)/erts/preloaded/ebin/erlang.beam \ @@ -835,6 +843,15 @@ EMU_OBJS = \ $(OBJDIR)/beam_catches.o $(OBJDIR)/code_ix.o \ $(OBJDIR)/beam_ranges.o + +ifeq ($(USE_ESOCK), yes) + +# WE ARE USING ESOCK + +ESOCK_NIF_OBJS = \ + $(OBJDIR)/socket_nif.o \ + $(OBJDIR)/net_nif.o + ifneq ($(TARGET), win32) # These are *currently* only needed for non-win32, # since the nif-functions for socket and net are basically @@ -847,6 +864,16 @@ else ESOCK_RUN_OBJS = endif +else + +# WE ARE *NOT* USING ESOCK + +ESOCK_NIF_OBJS = +ESOCK_RUN_OBJS = + +endif + + RUN_OBJS += \ $(OBJDIR)/erl_alloc.o $(OBJDIR)/erl_mtrace.o \ $(OBJDIR)/erl_alloc_util.o $(OBJDIR)/erl_goodfit_alloc.o \ @@ -903,8 +930,7 @@ NIF_OBJS = \ $(OBJDIR)/prim_buffer_nif.o \ $(OBJDIR)/prim_file_nif.o \ $(OBJDIR)/zlib_nif.o \ - $(OBJDIR)/socket_nif.o \ - $(OBJDIR)/net_nif.o + $(ESOCK_NIF_OBJS) ifeq ($(TARGET),win32) DRV_OBJS = \ diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index d15db760a2..ff19ef018e 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -55,7 +55,6 @@ */ #if 0 #define ERTS_DIST_MSG_DBG -FILE *dbg_file; #endif #if 0 /* Enable this to print the dist debug messages to a file instead */ @@ -67,6 +66,7 @@ FILE *dbg_file; #endif #if defined(ERTS_DIST_MSG_DBG) || defined(ERTS_RAW_DIST_MSG_DBG) +FILE *dbg_file; static void bw(byte *buf, ErlDrvSizeT sz) { bin_write(ERTS_PRINT_FILE, dbg_file, buf, sz); @@ -743,7 +743,7 @@ void init_dist(void) sprintf(buff, ERTS_DIST_MSG_DBG_FILE, getpid()); dbg_file = fopen(buff,"w+"); } -#elif defined (ERTS_DIST_MSG_DBG) +#elif defined(ERTS_DIST_MSG_DBG) || defined(ERTS_RAW_DIST_MSG_DBG) dbg_file = stderr; #endif @@ -1844,7 +1844,7 @@ int erts_net_message(Port *prt, if (locks) erts_proc_unlock(rp, locks); - } else if (ede_hfrag) { + } else if (ede_hfrag != NULL) { erts_free_dist_ext_copy(erts_get_dist_ext(ede_hfrag)); free_message_buffer(ede_hfrag); } @@ -1886,16 +1886,18 @@ int erts_net_message(Port *prt, goto invalid_message; } rp = erts_proc_lookup(to); + if (rp) { ErtsProcLocks locks = 0; erts_queue_dist_message(rp, locks, edep, ede_hfrag, token, am_Empty); if (locks) erts_proc_unlock(rp, locks); - } else if (ede_hfrag) { + } else if (ede_hfrag != NULL) { erts_free_dist_ext_copy(erts_get_dist_ext(ede_hfrag)); free_message_buffer(ede_hfrag); } + break; } @@ -1936,15 +1938,19 @@ int erts_net_message(Port *prt, goto invalid_message; } - if (!erts_proc_lookup(watcher)) break; /* Process not alive */ - - if (reason == THE_NON_VALUE) { + if (!erts_proc_lookup(watcher)) { + if (ede_hfrag != NULL) { + erts_free_dist_ext_copy(erts_get_dist_ext(ede_hfrag)); + free_message_buffer(ede_hfrag); + } + break; /* Process not alive */ + } #ifdef ERTS_DIST_MSG_DBG + if (reason == THE_NON_VALUE) { dist_msg_dbg(edep, "MSG", buf, orig_len); -#endif - } +#endif erts_proc_sig_send_dist_monitor_down( dep, ref, watched, watcher, edep, ede_hfrag, reason); @@ -1993,13 +1999,19 @@ int erts_net_message(Port *prt, goto invalid_message; } - if (!erts_proc_lookup(to)) break; /* Process not alive */ + if (!erts_proc_lookup(to)) { + if (ede_hfrag != NULL) { + erts_free_dist_ext_copy(erts_get_dist_ext(ede_hfrag)); + free_message_buffer(ede_hfrag); + } + break; /* Process not alive */ + } - if (reason == THE_NON_VALUE) { #ifdef ERTS_DIST_MSG_DBG + if (reason == THE_NON_VALUE) { dist_msg_dbg(edep, "MSG", buf, orig_len); -#endif } +#endif erts_proc_sig_send_dist_link_exit(dep, from, to, edep, ede_hfrag, @@ -2048,13 +2060,19 @@ int erts_net_message(Port *prt, goto invalid_message; } - if (!erts_proc_lookup(to)) break; /* Process not alive */ + if (!erts_proc_lookup(to)) { + if (ede_hfrag != NULL) { + erts_free_dist_ext_copy(erts_get_dist_ext(ede_hfrag)); + free_message_buffer(ede_hfrag); + } + break; /* Process not alive */ + } - if (reason == THE_NON_VALUE) { #ifdef ERTS_DIST_MSG_DBG + if (reason == THE_NON_VALUE) { dist_msg_dbg(edep, "MSG", buf, orig_len); -#endif } +#endif erts_proc_sig_send_dist_exit(dep, from, to, edep, ede_hfrag, reason, token); break; diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c index a3069e419a..9eb020d070 100644 --- a/erts/emulator/beam/time.c +++ b/erts/emulator/beam/time.c @@ -316,7 +316,7 @@ struct ErtsTimerWheel_ { #define ERTS_TW_SLOT_AT_ONCE (-1) #define ERTS_TW_BUMP_LATER_WHEEL(TIW) \ - ((tiw)->pos + ERTS_TW_LATER_WHEEL_SLOT_SIZE >= (TIW)->later.pos) + ((TIW)->pos + ERTS_TW_LATER_WHEEL_SLOT_SIZE >= (TIW)->later.pos) static int bump_later_wheel(ErtsTimerWheel *tiw, int *yield_count_p); @@ -701,7 +701,8 @@ remove_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p) if (slot < ERTS_TW_SOON_WHEEL_END_SLOT) { if (empty_slot && tiw->true_next_timeout_time - && p->timeout_pos == tiw->next_timeout_pos) { + && p->timeout_pos == tiw->next_timeout_pos + && tiw->yield_slot == ERTS_TW_SLOT_INACTIVE) { tiw->true_next_timeout_time = 0; } if (--tiw->soon.nto == 0) @@ -714,7 +715,8 @@ remove_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p) ErtsMonotonicTime tpos = tiw->later.min_tpos; tpos &= ERTS_TW_LATER_WHEEL_POS_MASK; tpos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE; - if (tpos == tiw->next_timeout_pos) + if (tpos == tiw->next_timeout_pos + && tiw->yield_slot == ERTS_TW_SLOT_INACTIVE) tiw->true_next_timeout_time = 0; } if (--tiw->later.nto == 0) { @@ -908,7 +910,6 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) { ErtsMonotonicTime tmp_slots = bump_to - tiw->pos; - tmp_slots = (bump_to - tiw->pos); if (tmp_slots < ERTS_TW_SOON_WHEEL_SIZE) slots = (int) tmp_slots; else diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c index 98be50815c..fb18c837ab 100644 --- a/erts/emulator/sys/common/erl_check_io.c +++ b/erts/emulator/sys/common/erl_check_io.c @@ -534,6 +534,7 @@ erts_io_notify_port_task_executed(ErtsPortTaskType type, if (state->active_events & ERTS_POLL_EV_OUT) oready(state->driver.select->outport, state); state->active_events = 0; + active_events = 0; } } diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c index 9662996039..1b125056f5 100644 --- a/erts/emulator/sys/common/erl_poll.c +++ b/erts/emulator/sys/common/erl_poll.c @@ -924,7 +924,7 @@ update_pollset(ErtsPollSet *ps, int fd, ErtsPollOp op, ErtsPollEvents events) ERTS_EV_SET(&evts[len++], fd, EVFILT_WRITE, flags, (void *) ERTS_POLL_EV_OUT); } #else - uint32_t flags = EV_ADD; + uint32_t flags = EV_ADD|EV_ENABLE; if (ps->oneshot) flags |= EV_ONESHOT; @@ -932,9 +932,27 @@ update_pollset(ErtsPollSet *ps, int fd, ErtsPollOp op, ErtsPollEvents events) erts_atomic_dec_nob(&ps->no_of_user_fds); /* We don't do anything when a delete is issued. The fds will be removed when they are triggered, or when they are closed. */ - events = 0; + if (ps->oneshot) + events = 0; + else { + flags = EV_DELETE; + events = ERTS_POLL_EV_IN; + } } else if (op == ERTS_POLL_OP_ADD) { erts_atomic_inc_nob(&ps->no_of_user_fds); + /* Only allow EV_IN in non-oneshot poll-sets */ + ASSERT(ps->oneshot || events == ERTS_POLL_EV_IN); + } else if (!ps->oneshot) { + ASSERT(op == ERTS_POLL_OP_MOD); + /* If we are not oneshot and do a mod we should disable the FD. + We assume that it is only the read side that is active as + currently only read is selected upon in the non-oneshot + poll-sets. */ + if (!events) + flags = EV_DISABLE; + else + flags = EV_ENABLE; + events = ERTS_POLL_EV_IN; } if (events & ERTS_POLL_EV_IN) { @@ -961,16 +979,15 @@ update_pollset(ErtsPollSet *ps, int fd, ErtsPollOp op, ErtsPollEvents events) for (i = 0; i < len; i++) { const char *flags = "UNKNOWN"; if (evts[i].flags == (EV_DELETE)) flags = "EV_DELETE"; - if (evts[i].flags == (EV_ADD|EV_ONESHOT)) flags = "EV_ADD|EV_ONESHOT"; if (evts[i].flags == (EV_ADD)) flags = "EV_ADD"; + if (evts[i].flags == (EV_ADD|EV_ONESHOT)) flags = "EV_ADD|EV_ONESHOT"; + if (evts[i].flags == (EV_ENABLE)) flags = "EV_ENABLE"; + if (evts[i].flags == (EV_DISABLE)) flags = "EV_DISABLE"; + if (evts[i].flags == (EV_ADD|EV_DISABLE)) flags = "EV_ADD|EV_DISABLE"; #ifdef EV_DISPATCH if (evts[i].flags == (EV_ADD|EV_DISPATCH)) flags = "EV_ADD|EV_DISPATCH"; - if (evts[i].flags == (EV_ADD|EV_DISABLE)) flags = "EV_ADD|EV_DISABLE"; if (evts[i].flags == (EV_ENABLE|EV_DISPATCH)) flags = "EV_ENABLE|EV_DISPATCH"; - if (evts[i].flags == (EV_ENABLE)) flags = "EV_ENABLE"; - if (evts[i].flags == (EV_DISABLE)) flags = "EV_DISABLE"; if (evts[i].flags == (EV_DISABLE|EV_DISPATCH)) flags = "EV_DISABLE|EV_DISABLE"; - if (evts[i].flags == (EV_DISABLE)) flags = "EV_DISABLE"; #endif keventbp += sprintf(keventbp, "%s{%lu, %s, %s}",i > 0 ? ", " : "", diff --git a/erts/emulator/test/driver_SUITE.erl b/erts/emulator/test/driver_SUITE.erl index cbed71cedd..f6d7c55017 100644 --- a/erts/emulator/test/driver_SUITE.erl +++ b/erts/emulator/test/driver_SUITE.erl @@ -120,29 +120,6 @@ -define(heap_binary_size, 64). -init_per_testcase(Case, Config) when is_atom(Case), is_list(Config) -> - CIOD = rpc(Config, - fun() -> - case catch erts_debug:get_internal_state(available_internal_state) of - true -> ok; - _ -> erts_debug:set_internal_state(available_internal_state, true) - end, - erts_debug:get_internal_state(check_io_debug) - end), - erlang:display({init_per_testcase, Case}), - 0 = element(1, CIOD), - [{testcase, Case}|Config]. - -end_per_testcase(Case, Config) -> - erlang:display({end_per_testcase, Case}), - CIOD = rpc(Config, - fun() -> - get_stable_check_io_info(), - erts_debug:get_internal_state(check_io_debug) - end), - 0 = element(1, CIOD), - ok. - suite() -> [{ct_hooks,[ts_install_cth]}, {timetrap, {minutes, 1}}]. @@ -219,6 +196,48 @@ end_per_group(_GroupName, Config) -> end, Config. +init_per_testcase(Case, Config) when is_atom(Case), is_list(Config) -> + CIOD = rpc(Config, + fun() -> + case catch erts_debug:get_internal_state(available_internal_state) of + true -> ok; + _ -> erts_debug:set_internal_state(available_internal_state, true) + end, + erts_debug:get_internal_state(check_io_debug) + end), + erlang:display({init_per_testcase, Case}), + 0 = element(1, CIOD), + [{testcase, Case}|Config]. + +end_per_testcase(Case, Config) -> + erlang:display({end_per_testcase, Case}), + try rpc(Config, fun() -> + get_stable_check_io_info(), + erts_debug:get_internal_state(check_io_debug) + end) of + CIOD -> + 0 = element(1, CIOD) + catch _E:_R:_ST -> + %% Logs some info about the system + ct_os_cmd("epmd -names"), + ct_os_cmd("ps aux"), + %% Restart the node + case proplists:get_value(node, Config) of + undefined -> + ok; + Node -> + timer:sleep(1000), %% Give the node time to die + [NodeName, _] = string:lexemes(atom_to_list(Node),"@"), + {ok, Node} = start_node_final( + list_to_atom(NodeName), + proplists:get_value(node_args, Config)) + end + end, + ok. + +ct_os_cmd(Cmd) -> + ct:log("~s: ~s",[Cmd,os:cmd(Cmd)]). + %% Test sending bad types to port with an outputv-capable driver. outputv_errors(Config) when is_list(Config) -> Path = proplists:get_value(data_dir, Config), @@ -2644,7 +2663,6 @@ start_node(Config) when is_list(Config) -> start_node(Name) -> start_node(Name, ""). start_node(NodeName, Args) -> - Pa = filename:dirname(code:which(?MODULE)), Name = list_to_atom(atom_to_list(?MODULE) ++ "-" ++ atom_to_list(NodeName) @@ -2652,7 +2670,17 @@ start_node(NodeName, Args) -> ++ integer_to_list(erlang:system_time(second)) ++ "-" ++ integer_to_list(erlang:unique_integer([positive]))), - test_server:start_node(Name, slave, [{args, Args ++ " -pa "++Pa}]). + start_node_final(Name, Args). +start_node_final(Name, Args) -> + {ok, Pwd} = file:get_cwd(), + FinalArgs = [Args, " -pa ", filename:dirname(code:which(?MODULE))], + {ok, Node} = test_server:start_node(Name, slave, [{args, FinalArgs}]), + LogPath = Pwd ++ "/error_log." ++ atom_to_list(Name), + ct:pal("Logging to: ~s", [LogPath]), + rpc:call(Node, logger, add_handler, [file_handler, logger_std_h, + #{formatter => {logger_formatter,#{ single_line => false }}, + config => #{file => LogPath }}]), + {ok, Node}. stop_node(Node) -> test_server:stop_node(Node). diff --git a/erts/emulator/test/net_SUITE.erl b/erts/emulator/test/net_SUITE.erl index 1a973cacb2..6111fc76a5 100644 --- a/erts/emulator/test/net_SUITE.erl +++ b/erts/emulator/test/net_SUITE.erl @@ -127,12 +127,17 @@ api_basic_cases() -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% init_per_suite(Config) -> - case os:type() of - {win32, _} -> - not_yet_implemented(); - _ -> - %% ?LOGGER:start(), - Config + case lists:member(socket, erlang:loaded()) of + true -> + case os:type() of + {win32, _} -> + not_yet_implemented(); + _ -> + %% ?LOGGER:start(), + Config + end; + false -> + {skip, "esock disabled"} end. end_per_suite(_) -> diff --git a/erts/emulator/test/socket_SUITE.erl b/erts/emulator/test/socket_SUITE.erl index cefbe4c1f8..e3545ccbf9 100644 --- a/erts/emulator/test/socket_SUITE.erl +++ b/erts/emulator/test/socket_SUITE.erl @@ -1385,22 +1385,27 @@ ttest_ssockt_csockt_cases() -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% init_per_suite(Config) -> - case os:type() of - {win32, _} -> - not_yet_implemented(); - _ -> - case quiet_mode(Config) of - default -> - ?LOGGER:start(), - Config; - Quiet -> - ?LOGGER:start(Quiet), - [{esock_test_quiet, Quiet}|Config] - end + case lists:member(socket, erlang:loaded()) of + true -> + case os:type() of + {win32, _} -> + (catch not_yet_implemented()); + _ -> + case quiet_mode(Config) of + default -> + ?LOGGER:start(), + Config; + Quiet -> + ?LOGGER:start(Quiet), + [{esock_test_quiet, Quiet}|Config] + end + end; + false -> + {skip, "esock disabled"} end. end_per_suite(_) -> - ?LOGGER:stop(), + (catch ?LOGGER:stop()), ok. diff --git a/erts/etc/common/erlexec.c b/erts/etc/common/erlexec.c index ec4a4ead23..8203c46a39 100644 --- a/erts/etc/common/erlexec.c +++ b/erts/etc/common/erlexec.c @@ -408,7 +408,6 @@ int main(int argc, char **argv) int process_args = 1; int print_args_exit = 0; int print_qouted_cmd_exit = 0; - erts_cpu_info_t *cpuinfo = NULL; char* emu_name; #ifdef __WIN32__ @@ -467,8 +466,6 @@ int main(int argc, char **argv) /* * Construct the path of the executable. */ - cpuinfo = erts_cpu_info_create(); - #if defined(__WIN32__) && defined(WIN32_ALWAYS_DEBUG) emu_type = "debug"; #endif @@ -526,9 +523,6 @@ int main(int argc, char **argv) i++; } - erts_cpu_info_destroy(cpuinfo); - cpuinfo = NULL; - if (malloc_lib) { if (strcmp(malloc_lib, "libc") != 0) usage("+MYm"); @@ -662,15 +656,6 @@ int main(int argc, char **argv) } break; - case 'i': - if (strcmp(argv[i], "-instr") == 0) { - add_Eargs("-Mim"); - add_Eargs("true"); - } - else - add_arg(argv[i]); - break; - case 'e': if (strcmp(argv[i], "-extra") == 0) { process_args = 0; diff --git a/erts/preloaded/ebin/erl_init.beam b/erts/preloaded/ebin/erl_init.beam Binary files differindex 81be5b021a..0313988e3e 100644 --- a/erts/preloaded/ebin/erl_init.beam +++ b/erts/preloaded/ebin/erl_init.beam diff --git a/erts/preloaded/ebin/net.beam b/erts/preloaded/ebin/net.beam Binary files differindex ebb1296b95..f61b2b4a69 100644 --- a/erts/preloaded/ebin/net.beam +++ b/erts/preloaded/ebin/net.beam diff --git a/erts/preloaded/src/Makefile b/erts/preloaded/src/Makefile index efeb92dce9..27d450c873 100644 --- a/erts/preloaded/src/Makefile +++ b/erts/preloaded/src/Makefile @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2008-2018. All Rights Reserved. +# Copyright Ericsson AB 2008-2019. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -33,14 +33,22 @@ STATIC_EBIN=../ebin include $(ERL_TOP)/erts/vsn.mk include $(ERL_TOP)/lib/kernel/vsn.mk +ifeq ($(USE_ESOCK), yes) +PRE_LOADED_ERL_ESOCK_MODULES = \ + socket \ + net +else +PRE_LOADED_ERL_ESOCK_MODULES = \ + net +endif + PRE_LOADED_ERL_MODULES = \ erl_prim_loader \ init \ prim_buffer \ prim_file \ prim_inet \ - socket \ - net \ + $(PRE_LOADED_ERL_ESOCK_MODULES) \ zlib \ prim_zip \ erl_init \ @@ -73,6 +81,11 @@ STATIC_TARGET_FILES = $(PRE_LOADED_MODULES:%=$(STATIC_EBIN)/%.$(EMULATOR)) APP_FILE= erts.app APP_SRC= $(APP_FILE).src APP_TARGET= $(STATIC_EBIN)/$(APP_FILE) +ifeq ($(USE_ESOCK), yes) +APP_ESOCK_MODS= net, socket +else +APP_ESOCK_MODS= net +endif KERNEL_SRC=$(ERL_TOP)/lib/kernel/src @@ -94,7 +107,7 @@ copy: cp *.beam $(STATIC_EBIN) $(APP_TARGET): $(APP_SRC) $(ERL_TOP)/erts/vsn.mk - $(vsn_verbose)sed -e 's;%VSN%;$(VSN);' $< > $@ + $(vsn_verbose)sed -e 's;%VSN%;$(VSN);' -e 's;%ESOCK_MODS%;$(APP_ESOCK_MODS);' $< > $@ include $(ERL_TOP)/make/otp_release_targets.mk diff --git a/erts/preloaded/src/erl_init.erl b/erts/preloaded/src/erl_init.erl index 6edead362c..d209c4033b 100644 --- a/erts/preloaded/src/erl_init.erl +++ b/erts/preloaded/src/erl_init.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2000-2016. All Rights Reserved. +%% Copyright Ericsson AB 2000-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -35,8 +35,7 @@ start(Mod, BootArgs) -> erl_tracer:on_load(), prim_buffer:on_load(), prim_file:on_load(), - socket:on_load(), - net:on_load(), + conditional_load(socket, [socket, net]), % socket:on_load(), net:on_load(), %% Proceed to the specified boot module run(Mod, boot, BootArgs). @@ -48,3 +47,24 @@ run(M, F, A) -> true -> M:F(A) end. + +conditional_load(CondMod, Mods2Load) -> + conditional_load(CondMod, erlang:loaded(), Mods2Load). + +conditional_load(_CondMod, [], _Mods2LOad) -> + ok; +conditional_load(CondMod, [CondMod|_], Mods2Load) -> + on_load(Mods2Load); +conditional_load(CondMod, [_|T], Mods2Load) -> + conditional_load(CondMod, T, Mods2Load). + +on_load([]) -> + ok; +on_load([Mod|Mods]) -> + Mod:on_load(), + on_load(Mods). + + + + + diff --git a/erts/preloaded/src/erts.app.src b/erts/preloaded/src/erts.app.src index c2a8511b6d..132397b478 100644 --- a/erts/preloaded/src/erts.app.src +++ b/erts/preloaded/src/erts.app.src @@ -36,8 +36,7 @@ atomics, counters, zlib, - net, - socket + %ESOCK_MODS% ]}, {registered, []}, {applications, []}, diff --git a/erts/preloaded/src/net.erl b/erts/preloaded/src/net.erl index a24b5c8ce3..13d2e3a117 100644 --- a/erts/preloaded/src/net.erl +++ b/erts/preloaded/src/net.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2018-2018. All Rights Reserved. +%% Copyright Ericsson AB 2018-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -178,12 +178,28 @@ getnameinfo(SockAddr, [] = _Flags) -> getnameinfo(#{family := Fam, addr := _Addr} = SockAddr, Flags) when ((Fam =:= inet) orelse (Fam =:= inet6)) andalso (is_list(Flags) orelse (Flags =:= undefined)) -> - nif_getnameinfo(socket:ensure_sockaddr(SockAddr), Flags); + nif_getnameinfo((catch ensure_sockaddr(SockAddr)), Flags); getnameinfo(#{family := Fam, path := _Path} = SockAddr, Flags) when (Fam =:= local) andalso (is_list(Flags) orelse (Flags =:= undefined)) -> nif_getnameinfo(SockAddr, Flags). +%% This function is intended to "handle" the case when the user +%% has built their (OTP) system with "--disable-esock". +%% That means the socket module does not exist. This is not really +%% a problem since the nif_getnameinfo won't work either (since +%% the nif file is not part of the system). The result of calling +%% getnameinfo will be a undef exception (erlang:nif_error(undef)). +%% +%% The only functions in this module that actually work in this case +%% (--disable-esock) is the depricated stuff (call, cast, ...). +%% +ensure_sockaddr(SockAddr) -> + try socket:ensure_sockaddr(SockAddr) + catch + error:undef:_ -> + undefined + end. %% =========================================================================== %% @@ -334,3 +350,4 @@ nif_if_index2name(_Id) -> nif_if_names() -> erlang:nif_error(undef). + diff --git a/lib/erl_interface/doc/src/ei.xml b/lib/erl_interface/doc/src/ei.xml index 254ae27cc8..7808bfd94f 100644 --- a/lib/erl_interface/doc/src/ei.xml +++ b/lib/erl_interface/doc/src/ei.xml @@ -183,28 +183,36 @@ typedef enum { </func> <func> - <name since="OTP @OTP-15712@"><ret>int</ret><nametext>ei_decode_bitstring(const char *buf, int *index, void *p, size_t plen, size_t *bitsp)</nametext></name> + <name since="OTP @OTP-15712@"><ret>int</ret><nametext>ei_decode_bitstring(const char *buf, int *index, const char **pp, unsigned int *bitoffsp, size_t *nbitsp)</nametext></name> <fsummary>Decode a bitstring.</fsummary> <desc> - <p>Decodes a bitstring from the binary format.</p> + <p>Decodes a bit string from the binary format.</p> <taglist> - <tag><c>p</c></tag> - <item><p>Either <c>NULL</c> or points to a buffer where the bytes of the - bitstring will be written.</p> + <tag><c>pp</c></tag> + <item><p>Either <c>NULL</c> or <c>*pp</c> returns a pointer to + the first byte of the bit string. The returned bit string is + readable as long as the buffer pointed to by <c>buf</c> is + readable and not written to.</p> </item> - <tag><c>plen</c></tag> - <item><p>The max size of the bitstring in <em>bytes</em>, that is the - size of the buffer if <c>p != NULL</c>.</p> + <tag><c>bitoffsp</c></tag> + <item><p>Either <c>NULL</c> or <c>*bitoffsp</c> returns the + number of unused bits in the first byte pointed to by + <c>*pp</c>. The value of <c>*bitoffsp</c> is between 0 and 7. + Unused bits in the first byte are the most significant bits.</p> </item> - <tag><c>*bitsp</c></tag> - <item><p>If <c>bitsp</c> is not <c>NULL</c>, set to the actual - number of <em>bits</em> of the bitstring.</p> + <tag><c>nbitsp</c></tag> + <item><p>Either <c>NULL</c> or <c>*nbitsp</c> returns the length + of the bit string in <em>bits</em>.</p> </item> </taglist> - <p>Returns <c>0</c> if it was a bitstring no longer than <c>plen</c> - bytes. The actual length of the bitstring will be - <c>(*bitsp+7)/8</c> bytes. If <c>(*bitsp % 8) > 0</c> only the high - <c>(*bitsp % 8)</c> bits of the last byte are significant.</p> + <p>Returns <c>0</c> if it was a bit string term.</p> + <p>The number of <em>bytes</em> pointed to by <c>*pp</c>, which are + part of the bit string, is <c>(*bitoffsp + *nbitsp + 7)/8</c>. If + <c>(*bitoffsp + *bitsp)%8 > 0</c> then only <c>(*bitoffsp + + *bitsp)%8</c> bits of the last byte are used. Unused bits in + the last byte are the least significant bits.</p> + <p>The values of unused bits in the first and last byte are undefined + and cannot be relied on.</p> <p>Number of bits may be divisible by 8, which means a binary decodable by <c>ei_decode_binary</c> is also decodable by <c>ei_decode_bitstring</c>.</p> @@ -490,14 +498,24 @@ typedef enum { </func> <func> - <name since="OTP @OTP-15712@"><ret>int</ret><nametext>ei_encode_bitstring(char *buf, int *index, const void *p, size_t bits)</nametext></name> - <name since="OTP @OTP-15712@"><ret>int</ret><nametext>ei_x_encode_bitstring(ei_x_buff* x, const void *p, size_t bits)</nametext></name> + <name since="OTP @OTP-15712@"><ret>int</ret> + <nametext>ei_encode_bitstring(char *buf, int *index, const char *p, size_t bitoffs, size_t nbits)</nametext></name> + <name since="OTP @OTP-15712@"><ret>int</ret> + <nametext>ei_x_encode_bitstring(ei_x_buff* x, const char *p, size_t bitoffs, size_t nbits)</nametext></name> <fsummary>Encode a bitstring.</fsummary> <desc> - <p>Encodes a bitstring in the binary format. The data is at - <c>p</c>. The size of the data is <c>bits</c> bits or - <c>(bits+7)/8</c> bytes. If <c>(bits%8) > 0</c> only the high - <c>(bits%8)</c> bits of the last byte are significant.</p> + <p>Encodes a bit string in the binary format.</p> + <p>The data is at <c>p</c>. The length of the bit string is <c>nbits</c> + bits. The first <c>bitoffs</c> bits of the data at <c>p</c> are unused. + The first byte which is part of the bit string is + <c>p[bitoffs/8]</c>. The <c>bitoffs%8</c> most significant bits of + the first byte <c>p[bitoffs/8]</c> are unused.</p> + <p>The number of bytes which is part of the bit string is <c>(bitoffs + + nbits + 7)/8</c>. If <c>(bitoffs + nbits)%8 > 0</c> then only <c>(bitoffs + + nbits)%8</c> bits of the last byte are used. Unused bits in + the last byte are the least significant bits.</p> + <p>The values of unused bits are disregarded and does not need to be + cleared.</p> </desc> </func> diff --git a/lib/erl_interface/include/ei.h b/lib/erl_interface/include/ei.h index a860df3f77..ed0420300d 100644 --- a/lib/erl_interface/include/ei.h +++ b/lib/erl_interface/include/ei.h @@ -526,9 +526,9 @@ int ei_x_encode_atom_len(ei_x_buff* x, const char* s, int len); int ei_x_encode_atom_len_as(ei_x_buff* x, const char* s, int len, erlang_char_encoding from, erlang_char_encoding to); int ei_encode_binary(char *buf, int *index, const void *p, long len); -int ei_encode_bitstring(char *buf, int *index, const void *p, size_t bits); +int ei_encode_bitstring(char *buf, int *index, const char *p, size_t bitoffs, size_t bits); int ei_x_encode_binary(ei_x_buff* x, const void* s, int len); -int ei_x_encode_bitstring(ei_x_buff* x, const void* p, size_t bits); +int ei_x_encode_bitstring(ei_x_buff* x, const char* p, size_t bitoffs, size_t bits); int ei_encode_pid(char *buf, int *index, const erlang_pid *p); int ei_x_encode_pid(ei_x_buff* x, const erlang_pid* pid); int ei_encode_fun(char* buf, int* index, const erlang_fun* p); @@ -578,7 +578,9 @@ int ei_decode_string(const char *buf, int *index, char *p); int ei_decode_atom(const char *buf, int *index, char *p); int ei_decode_atom_as(const char *buf, int *index, char *p, int destlen, erlang_char_encoding want, erlang_char_encoding* was, erlang_char_encoding* result); int ei_decode_binary(const char *buf, int *index, void *p, long *len); -int ei_decode_bitstring(const char *buf, int *index, void *p, size_t plen, size_t *bitsp); +int ei_decode_bitstring(const char *buf, int *index, const char** pp, + unsigned int* bitoffsp, size_t *nbitsp); + int ei_decode_fun(const char* buf, int* index, erlang_fun* p); void free_fun(erlang_fun* f); int ei_decode_pid(const char *buf, int *index, erlang_pid *p); diff --git a/lib/erl_interface/src/decode/decode_binary.c b/lib/erl_interface/src/decode/decode_binary.c index 2799438bef..0d28c67230 100644 --- a/lib/erl_interface/src/decode/decode_binary.c +++ b/lib/erl_interface/src/decode/decode_binary.c @@ -40,40 +40,41 @@ int ei_decode_binary(const char *buf, int *index, void *p, long *lenp) return 0; } -int ei_decode_bitstring(const char *buf, int *index, void *p, size_t plen, - size_t *bitsp) +int ei_decode_bitstring(const char *buf, int *index, + const char** pp, + unsigned int* bitoffsp, + size_t *nbitsp) { - const char *s = buf + *index; - const char *s0 = s; - unsigned long len; - unsigned char last_bits; - const unsigned char tag = get8(s); + const char *s = buf + *index; + const char *s0 = s; + unsigned char last_bits; + const unsigned char tag = get8(s); + size_t len = get32be(s); - if (tag == ERL_BINARY_EXT) { - long bytes; - int ret = ei_decode_binary(buf, index, p, &bytes); - if (bitsp) - *bitsp = (size_t)bytes * 8; - return ret; - } + switch(tag) { + case ERL_BINARY_EXT: + if (nbitsp) + *nbitsp = len * 8; + break; + case ERL_BIT_BINARY_EXT: + last_bits = get8(s); + if (((last_bits==0) != (len==0)) || last_bits > 8) + return -1; - if (tag != ERL_BIT_BINARY_EXT) - return -1; - - len = get32be(s); - last_bits = get8(s); - - if (len > plen || ((last_bits==0) != (len==0)) || last_bits > 8) - return -1; - - if (p) - memcpy(p, s, len); - s += len; + if (nbitsp) + *nbitsp = (len == 0) ? 0 : ((len-1) * 8) + last_bits; + break; + default: + return -1; + } - if (bitsp) - *bitsp = (len == 0) ? 0 : ((len-1) * 8) + last_bits; + if (pp) + *pp = s; + if (bitoffsp) + *bitoffsp = 0; - *index += s-s0; - return 0; + s += len; + *index += s-s0; + return 0; } diff --git a/lib/erl_interface/src/decode/decode_skip.c b/lib/erl_interface/src/decode/decode_skip.c index 11d3bc1786..736c00e074 100644 --- a/lib/erl_interface/src/decode/decode_skip.c +++ b/lib/erl_interface/src/decode/decode_skip.c @@ -21,14 +21,6 @@ #include "eiext.h" #include "decode_skip.h" -#ifdef HAVE_STDINT_H -# include <stdint.h> -#endif - -#ifndef SIZE_MAX -# define SIZE_MAX (~((size_t)0)) -#endif - int ei_skip_term(const char* buf, int* index) { int i, n, ty; @@ -88,7 +80,7 @@ int ei_skip_term(const char* buf, int* index) return -1; break; case ERL_BIT_BINARY_EXT: - if (ei_decode_bitstring(buf, index, NULL, SIZE_MAX, NULL) < 0) + if (ei_decode_bitstring(buf, index, NULL, NULL, NULL) < 0) return -1; break; case ERL_SMALL_INTEGER_EXT: diff --git a/lib/erl_interface/src/encode/encode_binary.c b/lib/erl_interface/src/encode/encode_binary.c index 4aa9f6bc16..0562979417 100644 --- a/lib/erl_interface/src/encode/encode_binary.c +++ b/lib/erl_interface/src/encode/encode_binary.c @@ -22,6 +22,10 @@ #include "eiext.h" #include "putget.h" +static void copy_bits(const unsigned char* src, size_t soffs, + unsigned char* dst, size_t n); + + int ei_encode_binary(char *buf, int *index, const void *p, long len) { char *s = buf + *index; @@ -40,23 +44,28 @@ int ei_encode_binary(char *buf, int *index, const void *p, long len) return 0; } -int ei_encode_bitstring(char *buf, int *index, const void *p, size_t bits) +int ei_encode_bitstring(char *buf, int *index, + const char *p, + size_t bitoffs, + size_t bits) { char *s = buf + *index; char *s0 = s; size_t bytes = (bits + 7) / 8; char last_bits = bits % 8; - if (bytes == 0 || last_bits == 0) - return ei_encode_binary(buf, index, p, bytes); - - if (!buf) s += 6; + if (!buf) s += last_bits ? 6 : 5; else { - put8(s, ERL_BIT_BINARY_EXT); + char* tagp = s++; put32be(s, bytes); - put8(s, last_bits); - memcpy(s, p, bytes); - s[bytes-1] &= (0xff << (8-last_bits)); + if (last_bits) { + *tagp = ERL_BIT_BINARY_EXT; + put8(s, last_bits); + } + else + *tagp = ERL_BINARY_EXT; + + copy_bits((const unsigned char*)p, bitoffs, (unsigned char*)s, bits); } s += bytes; @@ -64,3 +73,77 @@ int ei_encode_bitstring(char *buf, int *index, const void *p, size_t bits) return 0; } + + +/* + * MAKE_MASK(n) constructs a mask with n bits. + * Example: MAKE_MASK(3) returns the binary number 00000111. + */ +#define MAKE_MASK(n) ((((unsigned) 1) << (n))-1) + + +static +void copy_bits(const unsigned char* src, /* Base pointer to source. */ + size_t soffs, /* Bit offset for source relative to src. */ + unsigned char* dst, /* Destination. */ + size_t n) /* Number of bits to copy. */ +{ + unsigned rmask; + unsigned count; + unsigned deoffs; + unsigned bits; + unsigned bits1; + unsigned rshift; + + if (n == 0) + return; + + deoffs = n & 7; + rmask = deoffs ? (MAKE_MASK(deoffs) << (8-deoffs)) : 0; + + if (soffs == 0) { + unsigned nbytes = (n + 7) / 8; + memcpy(dst, src, nbytes); + if (rmask) + dst[nbytes-1] &= rmask; + return; + } + + src += soffs / 8; + soffs &= 7; + + if (n < 8) { /* Less than one byte */ + bits = (*src << soffs); + if (soffs+n > 8) { + src++; + bits |= (*src >> (8 - soffs)); + } + *dst = bits & rmask; + return; + } + + count = n >> 3; + + rshift = 8 - soffs; + bits = *src; + if (soffs + n > 8) { + src++; + } + + while (count--) { + bits1 = bits << soffs; + bits = *src; + src++; + *dst = bits1 | (bits >> rshift); + dst++; + } + + if (rmask) { + bits1 = bits << soffs; + if ((rmask << rshift) & 0xff) { + bits = *src; + bits1 |= (bits >> rshift); + } + *dst = bits1 & rmask; + } +} diff --git a/lib/erl_interface/src/misc/ei_printterm.c b/lib/erl_interface/src/misc/ei_printterm.c index a89b990ac1..5c40fb7747 100644 --- a/lib/erl_interface/src/misc/ei_printterm.c +++ b/lib/erl_interface/src/misc/ei_printterm.c @@ -250,12 +250,13 @@ static int print_term(FILE* fp, ei_x_buff* x, ei_free(p); break; case ERL_BIT_BINARY_EXT: { + const char* cp; size_t bits; + unsigned int bitoffs; int trunc = 0; - p = ei_malloc(n); - if (p == NULL) goto err; - if (ei_decode_bitstring(buf, index, p, n, &bits) < 0) { - ei_free(p); + + if (ei_decode_bitstring(buf, index, &cp, &bitoffs, &bits) < 0 + || bitoffs != 0) { goto err; } ch_written += xprintf(fp, x, "#Bits<"); @@ -266,15 +267,14 @@ static int print_term(FILE* fp, ei_x_buff* x, } --m; for (i = 0; i < m; ++i) { - ch_written += xprintf(fp, x, "%d,", p[i]); + ch_written += xprintf(fp, x, "%d,", cp[i]); } - ch_written += xprintf(fp, x, "%d", p[i]); + ch_written += xprintf(fp, x, "%d", cp[i]); if (trunc) ch_written += xprintf(fp, x, ",..."); else if (bits % 8 != 0) ch_written += xprintf(fp, x, ":%u", (unsigned)(bits % 8)); xputc('>', fp, x); ++ch_written; - ei_free(p); break; } case ERL_SMALL_INTEGER_EXT: diff --git a/lib/erl_interface/src/misc/ei_x_encode.c b/lib/erl_interface/src/misc/ei_x_encode.c index 2da271795f..8e77679d2a 100644 --- a/lib/erl_interface/src/misc/ei_x_encode.c +++ b/lib/erl_interface/src/misc/ei_x_encode.c @@ -117,14 +117,14 @@ int ei_x_encode_binary(ei_x_buff* x, const void* p, int len) return ei_encode_binary(x->buff, &x->index, p, len); } -int ei_x_encode_bitstring(ei_x_buff* x, const void* p, size_t bits) +int ei_x_encode_bitstring(ei_x_buff* x, const char* p, size_t bitoffs, size_t bits) { int i = x->index; - if (ei_encode_bitstring(NULL, &i, p, bits) == -1) + if (ei_encode_bitstring(NULL, &i, p, bitoffs, bits) == -1) return -1; if (!x_fix_buff(x, i)) return -1; - return ei_encode_bitstring(x->buff, &x->index, p, bits); + return ei_encode_bitstring(x->buff, &x->index, p, bitoffs, bits); } int ei_x_encode_long(ei_x_buff* x, long n) diff --git a/lib/erl_interface/src/misc/show_msg.c b/lib/erl_interface/src/misc/show_msg.c index 2d49eb6449..805d69e9b3 100644 --- a/lib/erl_interface/src/misc/show_msg.c +++ b/lib/erl_interface/src/misc/show_msg.c @@ -24,13 +24,6 @@ #include <stdlib.h> #include <stdarg.h> #include <string.h> -#ifdef HAVE_STDINT_H -# include <stdint.h> -#endif - -#ifndef SIZE_MAX -# define SIZE_MAX (~((size_t)0)) -#endif #include <sys/types.h> @@ -464,7 +457,7 @@ static void show_term(const char *termbuf, int *index, FILE *stream) case ERL_BIT_BINARY_EXT: { size_t bits; - ei_decode_bitstring(termbuf, index, NULL, SIZE_MAX, &bits); + ei_decode_bitstring(termbuf, index, NULL, NULL, &bits); fprintf(stream, "#Bits<%lu>", (unsigned long)bits); break; } diff --git a/lib/erl_interface/test/ei_connect_SUITE_data/ei_connect_test.c b/lib/erl_interface/test/ei_connect_SUITE_data/ei_connect_test.c index 7c9e79f837..385bcdd422 100644 --- a/lib/erl_interface/test/ei_connect_SUITE_data/ei_connect_test.c +++ b/lib/erl_interface/test/ei_connect_SUITE_data/ei_connect_test.c @@ -209,8 +209,9 @@ static void cmd_ei_send_funs(char* buf, int len) erlang_pid pid; ei_x_buff x; erlang_fun fun1, fun2; - unsigned char bitstring[10]; + char* bitstring; size_t bits; + int bitoffs; if (ei_decode_long(buf, &index, &fd) < 0) fail("expected long"); @@ -224,7 +225,7 @@ static void cmd_ei_send_funs(char* buf, int len) fail("expected Fun1"); if (ei_decode_fun(buf, &index, &fun2) < 0) fail("expected Fun2"); - if (ei_decode_bitstring(buf, &index, bitstring, sizeof(bitstring), &bits) < 0) + if (ei_decode_bitstring(buf, &index, &bitstring, &bitoffs, &bits) < 0) fail("expected bitstring"); if (ei_x_new_with_version(&x) < 0) fail("ei_x_new_with_version"); @@ -234,7 +235,7 @@ static void cmd_ei_send_funs(char* buf, int len) fail("encode fun1"); if (ei_x_encode_fun(&x, &fun2) < 0) fail("encode fun2"); - if (ei_x_encode_bitstring(&x, bitstring, bits) < 0) + if (ei_x_encode_bitstring(&x, bitstring, bitoffs, bits) < 0) fail("encode bitstring"); free_fun(&fun1); free_fun(&fun2); diff --git a/lib/erl_interface/test/ei_decode_SUITE_data/ei_decode_test.c b/lib/erl_interface/test/ei_decode_SUITE_data/ei_decode_test.c index d39970a857..46d6b8f2af 100644 --- a/lib/erl_interface/test/ei_decode_SUITE_data/ei_decode_test.c +++ b/lib/erl_interface/test/ei_decode_SUITE_data/ei_decode_test.c @@ -319,17 +319,18 @@ static void decode_bin(int exp_size, const char* val, int exp_len) static void decode_bits(int exp_size, const char* val, size_t exp_bits) { - char p[1024]; + const char* p; char *buf; size_t bits; + int bitoffs; int size1 = 0; int size2 = 0; int err; message("ei_decode_bitstring should be %d bits", (int)exp_bits); buf = read_packet(NULL); - err = ei_decode_bitstring(buf+1, &size1, NULL, sizeof(p), &bits); - message("err = %d, size = %d, len = %d, expected size = %d, expected bits = %d\n",\ - err,size1, (int)bits, exp_size, (int)exp_bits); + err = ei_decode_bitstring(buf+1, &size1, NULL, &bitoffs, &bits); + message("err = %d, size = %d, bitoffs = %d, bits = %d, expected size = %d, expected bits = %d\n",\ + err,size1, bitoffs, (int)bits, exp_size, (int)exp_bits); if (err != 0) { if (err != -1) { @@ -344,8 +345,12 @@ static void decode_bits(int exp_size, const char* val, size_t exp_bits) fail("number of bits is not correct"); return; } + if (bitoffs != 0) { + fail("non zero bit offset"); + return; + } - err = ei_decode_bitstring(buf+1, &size2, p, sizeof(p), &bits); + err = ei_decode_bitstring(buf+1, &size2, &p, NULL, &bits); message("err = %d, size = %d, len = %d, expected size = %d, expected len = %d\n",\ err,size2, (int)bits, exp_size, (int)exp_bits); if (err != 0) { diff --git a/lib/erl_interface/test/ei_decode_encode_SUITE.erl b/lib/erl_interface/test/ei_decode_encode_SUITE.erl index d8b0bce3ae..3451d9f503 100644 --- a/lib/erl_interface/test/ei_decode_encode_SUITE.erl +++ b/lib/erl_interface/test/ei_decode_encode_SUITE.erl @@ -122,9 +122,29 @@ test_ei_decode_encode(Config) when is_list(Config) -> [send_rec(P, <<16#dec0deb175:B/little>>) || B <- lists:seq(0,48)], + % And last an ugly duckling to test ei_encode_bitstring with bitoffs != 0 + encode_bitstring(P), + runner:recv_eot(P), ok. +encode_bitstring(P) -> + %% Send one bitstring to c-node + Bits = <<16#18f6d4b2907e5c3a1:66>>, + P ! {self(), {command, term_to_binary(Bits, [{minor_version, 2}])}}, + + %% and then receive and verify a number of different sub-bitstrings + receive_sub_bitstring(P, Bits, 0, bit_size(Bits)). + +receive_sub_bitstring(_, _, _, NBits) when NBits < 0 -> + ok; +receive_sub_bitstring(P, Bits, BitOffs, NBits) -> + <<_:BitOffs, Sub:NBits/bits, _/bits>> = Bits, + %%io:format("expecting term_to_binary(~p) = ~p\n", [Sub, term_to_binary(Sub)]), + {_B,Sub} = get_buf_and_term(P), + receive_sub_bitstring(P, Bits, BitOffs+1, NBits - ((NBits div 20)+1)). + + %% ######################################################################## %% diff --git a/lib/erl_interface/test/ei_decode_encode_SUITE_data/ei_decode_encode_test.c b/lib/erl_interface/test/ei_decode_encode_SUITE_data/ei_decode_encode_test.c index f9c05b2739..85ca6c56e9 100644 --- a/lib/erl_interface/test/ei_decode_encode_SUITE_data/ei_decode_encode_test.c +++ b/lib/erl_interface/test/ei_decode_encode_SUITE_data/ei_decode_encode_test.c @@ -42,7 +42,8 @@ typedef struct typedef struct { - char bytes[MAXATOMLEN_UTF8]; + const char* bytes; + unsigned int bitoffs; size_t nbits; }my_bitstring; @@ -128,17 +129,17 @@ struct Type my_atom_type = { int ei_decode_my_bits(const char *buf, int *index, my_bitstring* a) { - return ei_decode_bitstring(buf, index, (a ? a->bytes : NULL), - sizeof(a->bytes), + return ei_decode_bitstring(buf, index, (a ? &a->bytes : NULL), + (a ? &a->bitoffs : NULL), (a ? &a->nbits : NULL)); } int ei_encode_my_bits(char *buf, int *index, my_bitstring* a) { - return ei_encode_bitstring(buf, index, a->bytes, a->nbits); + return ei_encode_bitstring(buf, index, a->bytes, a->bitoffs, a->nbits); } int ei_x_encode_my_bits(ei_x_buff* x, my_bitstring* a) { - return ei_x_encode_bitstring(x, a->bytes, a->nbits); + return ei_x_encode_bitstring(x, a->bytes, a->bitoffs, a->nbits); } struct Type my_bitstring_type = { @@ -264,11 +265,7 @@ void decode_encode(struct Type** tv, int nobj) size1 = 0; err = t->ei_decode_fp(inp, &size1, NULL); if (err != 0) { - if (err != -1) { - fail("decode returned non zero but not -1"); - } else { - fail1("decode '%s' returned non zero", t->name); - } + fail2("decode '%s' returned non zero %d", t->name, err); return; } if (size1 < 1) { @@ -497,6 +494,66 @@ void decode_encode_big(struct Type* t) } +void encode_bitstring(void) +{ + char* packet; + char* inp; + char out_buf[BUFSZ]; + int size; + int err, i; + ei_x_buff arg; + const char* p; + unsigned int bitoffs; + size_t nbits, org_nbits; + + packet = read_packet(NULL); + inp = packet+1; + + size = 0; + err = ei_decode_bitstring(inp, &size, &p, &bitoffs, &nbits); + if (err != 0) { + fail1("ei_decode_bitstring returned non zero %d", err); + return; + } + + /* + * Now send a bunch of different sub-bitstrings back + * encoded both with ei_encode_ and ei_x_encode_. + */ + org_nbits = nbits; + do { + size = 0; + err = ei_encode_bitstring(out_buf, &size, p, bitoffs, nbits); + if (err != 0) { + fail1("ei_encode_bitstring returned non zero %d", err); + return; + } + + ei_x_new(&arg); + err = ei_x_encode_bitstring(&arg, p, bitoffs, nbits); + if (err != 0) { + fail1("ei_x_encode_bitstring returned non zero %d", err); + ei_x_free(&arg); + return; + } + + if (arg.index < 1) { + fail("size is < 1"); + ei_x_free(&arg); + return; + } + + send_buffer(out_buf, size); + send_buffer(arg.buff, arg.index); + ei_x_free(&arg); + + bitoffs++; + nbits -= (nbits / 20) + 1; + } while (nbits < org_nbits); + + free_packet(packet); +} + /* ******************************************************************** */ @@ -568,6 +625,8 @@ TESTCASE(test_ei_decode_encode) decode_encode_one(&my_bitstring_type); } + encode_bitstring(); + report(1); } diff --git a/lib/kernel/src/erl_distribution.erl b/lib/kernel/src/erl_distribution.erl index 0bec78e938..cdb2d2f1f6 100644 --- a/lib/kernel/src/erl_distribution.erl +++ b/lib/kernel/src/erl_distribution.erl @@ -21,6 +21,8 @@ -behaviour(supervisor). +-include_lib("kernel/include/logger.hrl"). + -export([start_link/0,start_link/2,init/1,start/1,stop/0]). -define(DBG,erlang:display([?MODULE,?LINE])). @@ -83,6 +85,10 @@ do_start_link([{Arg,Flag}|T]) -> case init:get_argument(Arg) of {ok,[[Name]]} -> start_link([list_to_atom(Name),Flag|ticktime()], true); + {ok,[[Name]|_Rest]} -> + ?LOG_WARNING("Multiple -~p given to erl, using the first, ~p", + [Arg, Name]), + start_link([list_to_atom(Name),Flag|ticktime()], true); _ -> do_start_link(T) end; diff --git a/lib/kernel/test/erl_distribution_SUITE.erl b/lib/kernel/test/erl_distribution_SUITE.erl index 8dd4ef1987..c3a022df0a 100644 --- a/lib/kernel/test/erl_distribution_SUITE.erl +++ b/lib/kernel/test/erl_distribution_SUITE.erl @@ -205,6 +205,9 @@ nodenames(Config) when is_list(Config) -> legal("a-1@b"), legal("a_1@b"), + %% Test that giving two -sname works as it should + test_node("a_1@b", false, long_or_short() ++ "a_0@b"), + illegal("cdé@a"), illegal("te欢st@a"). @@ -258,8 +261,11 @@ illegal(Name) -> test_node(Name) -> test_node(Name, false). test_node(Name, Illigal) -> + test_node(Name, Illigal, ""). +test_node(Name, Illigal, ExtraArgs) -> ProgName = ct:get_progname(), - Command = ProgName ++ " -noinput " ++ long_or_short() ++ Name ++ + Command = ProgName ++ " -noinput " ++ ExtraArgs ++ + long_or_short() ++ Name ++ " -eval \"net_adm:ping('" ++ atom_to_list(node()) ++ "')\"" ++ case Illigal of true -> diff --git a/lib/sasl/src/Makefile b/lib/sasl/src/Makefile index 7338bdf016..fd62588f5c 100644 --- a/lib/sasl/src/Makefile +++ b/lib/sasl/src/Makefile @@ -61,7 +61,11 @@ TARGET_FILES= $(MODULES:%=$(EBIN)/%.$(EMULATOR)) $(APP_TARGET) $(APPUP_TARGET) # ---------------------------------------------------- # FLAGS # ---------------------------------------------------- + ERL_COMPILE_FLAGS += -I../../stdlib/include -Werror +ifeq ($(USE_ESOCK), yes) +ERL_COMPILE_FLAGS += -DUSE_ESOCK=true +endif # ---------------------------------------------------- diff --git a/lib/sasl/src/systools_make.erl b/lib/sasl/src/systools_make.erl index c2c91fd667..b5a6b44f93 100644 --- a/lib/sasl/src/systools_make.erl +++ b/lib/sasl/src/systools_make.erl @@ -33,6 +33,7 @@ -export([read_application/4]). -export([make_hybrid_boot/4]). +-export([preloaded/0]). % Exported just for testing -import(lists, [filter/2, keysort/2, keysearch/3, map/2, reverse/1, append/1, foldl/3, member/2, foreach/2]). @@ -45,6 +46,13 @@ -compile({inline,[{badarg,2}]}). +-ifdef(USE_ESOCK). +-define(ESOCK_MODS, [socket]). +-else. +-define(ESOCK_MODS, []). +-endif. + + %%----------------------------------------------------------------- %% Create a boot script from a release file. %% Options is a list of {path, Path} | silent | local @@ -1566,7 +1574,7 @@ preloaded() -> erts_code_purger,erts_dirty_process_signal_handler, erts_internal,erts_literal_area_collector, init,net,persistent_term,prim_buffer,prim_eval,prim_file, - prim_inet,prim_zip,socket,zlib]. + prim_inet,prim_zip] ++ ?ESOCK_MODS ++ [zlib]. %%______________________________________________________________________ %% Kernel processes; processes that are specially treated by the init diff --git a/lib/ssh/src/ssh.app.src b/lib/ssh/src/ssh.app.src index 410061cded..7449405d20 100644 --- a/lib/ssh/src/ssh.app.src +++ b/lib/ssh/src/ssh.app.src @@ -44,10 +44,10 @@ {env, []}, {mod, {ssh_app, []}}, {runtime_dependencies, [ - "crypto-4.2", - "erts-6.0", - "kernel-3.0", - "public_key-1.5.2", - "stdlib-3.3" + "crypto-@OTP-15644@", + "erts-9.0", + "kernel-5.3", + "public_key-1.6.1", + "stdlib-3.4.1" ]}]}. diff --git a/lib/stdlib/doc/src/gb_trees.xml b/lib/stdlib/doc/src/gb_trees.xml index 570c9c7cb6..08aa1865e8 100644 --- a/lib/stdlib/doc/src/gb_trees.xml +++ b/lib/stdlib/doc/src/gb_trees.xml @@ -42,11 +42,8 @@ <section> <title>Data Structure</title> - <code type="none"> -{Size, Tree}</code> - - <p><c>Tree</c> is composed of nodes of the form <c>{Key, Value, Smaller, - Bigger}</c> and the "empty tree" node <c>nil</c>.</p> + <p>Trees and iterators are built using opaque data structures that should + not be pattern-matched from outside this module.</p> <p>There is no attempt to balance trees after deletions. As deletions do not increase the height of a tree, this should be OK.</p> diff --git a/make/configure.in b/make/configure.in index bf3fd0751f..c4b89c4f45 100644 --- a/make/configure.in +++ b/make/configure.in @@ -2,7 +2,7 @@ dnl Process this file with autoconf to produce a configure script. dnl %CopyrightBegin% dnl -dnl Copyright Ericsson AB 1998-2016. All Rights Reserved. +dnl Copyright Ericsson AB 1998-2019. All Rights Reserved. dnl dnl Licensed under the Apache License, Version 2.0 (the "License"); dnl you may not use this file except in compliance with the License. @@ -298,6 +298,10 @@ AC_ARG_ENABLE(builtin-zlib, AS_HELP_STRING([--enable-builtin-zlib], [force use of our own built-in zlib])) +AC_ARG_ENABLE(esock, +AS_HELP_STRING([--enable-esock], [enable builtin experimental socket (as a nif) support (default)]) +AS_HELP_STRING([--disable-esock], [disable builtin experimental socket (as a nif) support])) + AC_ARG_ENABLE(sharing-preserving, AS_HELP_STRING([--enable-sharing-preserving], [enable copying of terms without destroying sharing])) diff --git a/make/otp.mk.in b/make/otp.mk.in index ceff8f7c31..cdddb90734 100644 --- a/make/otp.mk.in +++ b/make/otp.mk.in @@ -4,7 +4,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1997-2016. All Rights Reserved. +# Copyright Ericsson AB 1997-2019. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -51,6 +51,8 @@ TYPES = @TYPES@ USE_PGO = @USE_PGO@ +USE_ESOCK = @USE_ESOCK@ + # Slash separated list of return values from $(origin VAR) # that are untrusted - set default in this file instead. # The list is not space separated since some return values diff --git a/scripts/run-dialyzer b/scripts/run-dialyzer index 621de3fa65..44436594d3 100755 --- a/scripts/run-dialyzer +++ b/scripts/run-dialyzer @@ -1,17 +1,47 @@ #!/bin/bash set -e -set -x -$ERL_TOP/bin/dialyzer --build_plt --apps asn1 compiler crypto dialyzer edoc erts et ftp hipe inets kernel mnesia observer public_key runtime_tools snmp ssh ssl stdlib syntax_tools tftp wx xmerl --statistics -$ERL_TOP/bin/dialyzer -n -Wunknown -Wunmatched_returns --apps compiler erts ftp tftp kernel stdlib asn1 crypto dialyzer hipe parsetools public_key runtime_tools sasl tools --statistics -$ERL_TOP/bin/dialyzer -n --apps common_test debugger edoc ftp inets mnesia observer ssh ssl syntax_tools tftp wx xmerl --statistics +filter () { + FILTER_RESULT="" + for app in $1; do + if echo " $2 " | grep -v " $app " > /dev/null; then + FILTER_RESULT="$FILTER_RESULT $app" + fi + done +} + +ALL_APPLICATIONS=$(ls -d -1 lib/*/ | sed 's:^lib\|/::g') +echo "All applications: $ALL_APPLICATIONS" |tr '\n' ' ' && echo "" + +BASE_PLT="compiler crypto erts hipe kernel stdlib syntax_tools" +APP_PLT="asn1 edoc et ftp inets mnesia observer public_key sasl runtime_tools snmp ssl tftp wx xmerl tools" +NO_UNMATCHED="common_test debugger edoc eunit ftp inets mnesia observer reltool ssh ssl syntax_tools tftp wx xmerl" +WARNINGS="diameter megaco snmp" + +TRAVIS_SKIP="" +if [ "X$TRAVIS" = "Xtrue" ]; then + TRAVIS_SKIP="common_test eldap erl_docgen odbc eunit reltool os_mon diameter megaco snmp" +fi + +filter "$ALL_APPLICATIONS" "$NO_UNMATCHED $WARNINGS $TRAVIS_SKIP" +UNMATCHED=$FILTER_RESULT +filter "$APP_PLT" "$TRAVIS_SKIP" +APP_PLT=$FILTER_RESULT +filter "$NO_UNMATCHED" "$TRAVIS_SKIP" +NO_UNMATCHED=$FILTER_RESULT +filter "$WARNINGS" "$TRAVIS_SKIP" +WARNINGS=$FILTER_RESULT -# In travis we don't dialyze everything as it takes too much time -if [ "X$TRAVIS" != "Xtrue" ]; then - $ERL_TOP/bin/dialyzer -n -Wunknown -Wunmatched_returns --apps eldap erl_docgen et odbc --statistics - $ERL_TOP/bin/dialyzer -n --apps eunit reltool os_mon --statistics +echo "UNMATCHED = $UNMATCHED" +echo "NO_UNMATCHED = $NO_UNMATCHED" +echo "WARNINGS = $WARNINGS" + +set -x - # These application are not run always as the currently have dialyzer warnings - # $ERL_TOP/bin/dialyzer -n --apps diameter megaco snmp --statistics +$ERL_TOP/bin/dialyzer --build_plt -Wunknown --apps $BASE_PLT $APP_PLT --statistics +$ERL_TOP/bin/dialyzer -n -Wunknown -Wunmatched_returns --apps $UNMATCHED --statistics +$ERL_TOP/bin/dialyzer -n -Wunknown --apps $NO_UNMATCHED --statistics +if [ "X$WARNINGS" != "X" ]; then + $ERL_TOP/bin/dialyzer -n --apps $WARNINGS --statistics fi |