Introduce sender in distributed signals and dflag configuration

9 files changed, 383 insertions, 78 deletions

diff --git a/erts/doc/src/alt_dist.xml b/erts/doc/src/alt_dist.xml
index 300f75dc13..d3731a5391 100644
--- a/erts/doc/src/alt_dist.xml
+++ b/erts/doc/src/alt_dist.xml
@@ -760,11 +760,80 @@
 	  </p>
 	  <p>Only used during handshake phase.</p>
 	</item>
+	
+	<tag><marker id="hs_data_add_flags"/><c>add_flags</c></tag>
+	<item>
+	  <p>
+	    <seealso marker="erl_dist_protocol#dflags">Distribution flags</seealso>
+	    to add to the connection. Currently all (non obsolete) flags will
+	    automatically be enabled.
+	  </p>
+	  <p>
+	    This flag field is optional.
+	  </p>
+	</item>
+
+	<tag><marker id="hs_data_reject_flags"/><c>reject_flags</c></tag>
+	<item>
+	  <p>
+	    <seealso marker="erl_dist_protocol#dflags">Distribution flags</seealso>
+	    to reject. Currently the following distribution flags can be rejected:
+	  </p>
+	  <taglist>
+	    <tag><c>DFLAG_DIST_HDR_ATOM_CACHE</c></tag>
+	    <item>Do not use atom cache over this connection.</item>
+	    <tag><c>DFLAGS_STRICT_ORDER_DELIVERY</c></tag>
+	    <item>Do not use any features that require strict
+	    order delivery.</item>
+	  </taglist>
+	  <p>
+	    This flag field is optional.
+	  </p>
+	</item>
+
+	<tag><marker id="hs_data_require_flags"/><c>require_flags</c></tag>
+	<item>
+	  <p>
+	    Require these <seealso marker="erl_dist_protocol#dflags">distribution
+	    flags</seealso> to be used. The connection will be aborted during the
+	    handshake if the other end does not use them.
+	  </p>
+	  <p>
+	    This flag field is optional.
+	  </p>
+	</item>
 
       </taglist>
     </section>
 
     <section>
+      <marker id="distribution_data_delivery"/>
+      <title>Distribution Data Delivery</title>
+      <p>
+	When using the default configuration, the data to pass
+	over a connection needs to be delivered as is
+	to the node on the receiving end in the <em>exact same
+	order</em>, with no loss of data what so ever, as sent
+	from the sending node.
+      </p>
+      <p>
+	The data delivery order can be relaxed by disabling
+	features that require strict ordering. This is done by
+	passing the <c>?DFLAGS_STRICT_ORDER_DELIVERY</c>
+	<seealso marker="erl_dist_protocol#dflags">distribution
+	flags</seealso> in the
+	<seealso marker="alt_dist#hs_data_reject_flags"><c>reject_flags</c></seealso>
+	field of the <seealso marker="#hs_data_record"><c>#hs_data{}</c></seealso>
+	record used when setting up the connection. When relaxed
+	ordering is used, only the order of signals with the same
+	sender/receiver pair has to be preserved.
+	However, note that disabling the features that require
+	strict ordering may have a negative impact on performance,
+	throughput, and/or latency. 
+      </p>
+    </section>
+
+    <section>
       <marker id="enable_your_distribution_module"/>
       <title>Enable Your Distribution Module</title>
 
diff --git a/erts/doc/src/erl_dist_protocol.xml b/erts/doc/src/erl_dist_protocol.xml
index 610351db6c..a78b13aaa4 100644
--- a/erts/doc/src/erl_dist_protocol.xml
+++ b/erts/doc/src/erl_dist_protocol.xml
@@ -829,7 +829,31 @@ DiB == gen_digest(ChA, ICA)?
         <item>
           <p>The node understand UTF-8 encoded atoms.</p>
         </item>
+        <tag><c>-define(DFLAG_MAP_TAG, 16#20000).</c></tag>
+        <item>
+          <p>The node understand the map tag.</p>
+        </item>
+        <tag><c>-define(DFLAG_BIG_CREATION, 16#40000).</c></tag>
+        <item>
+          <p>The node understand big node creation.</p>
+        </item>
+        <tag><c>-define(DFLAG_SEND_SENDER, 16#80000).</c></tag>
+        <item>
+          <p>
+	    Use the <c>SEND_SENDER</c>
+	    <seealso marker="#control_message">control message</seealso>
+	    instead of the <c>SEND</c> control message and use the
+	    <c>SEND_SENDER_TT</c> control message instead
+	    of the <c>SEND_TT</c> control message.
+	  </p>
+        </item>
       </taglist>
+      <p>
+	There are also a collection of <c>DFLAG</c>s bitwise or:ed
+	together in the <c>DFLAGS_STRICT_ORDER_DELIVERY</c> macro.
+	These flags corresponds to features that require strict
+	ordering of data over distribution channels.
+      </p>
     </section>
   </section>
 
@@ -922,6 +946,7 @@ DiB == gen_digest(ChA, ICA)?
       </item>
     </taglist>
 
+    <marker id="control_message"/>
     <p>The <c>ControlMessage</c> is a tuple, where the first element indicates
       which distributed operation it encodes:</p>
 
@@ -1028,4 +1053,49 @@ DiB == gen_digest(ChA, ICA)?
       </item>
     </taglist>
   </section>
+
+  <section>
+    <title>New Ctrlmessages for Erlang/OTP 21</title>
+    <taglist>
+      <tag><c>SEND_SENDER</c></tag>
+      <item>
+        <p><c>{22, FromPid, ToPid}</c></p>
+        <p>Followed by <c>Message</c>.</p>
+	<p>
+	  This control messages replace the <c>SEND</c> control
+	  message and will be sent when the distribution flag
+	  <seealso marker="erl_dist_protocol#dflags"><c>DFLAG_SEND_SENDER</c></seealso>
+	  has been negotiated in the connection setup handshake.
+	</p>
+	<note><p>
+	  Messages encoded before the connection has
+	  been set up may still use the <c>SEND</c> control
+	  message. However, once a <c>SEND_SENDER</c> or <c>SEND_SENDER_TT</c>
+	  control message has been sent, no more <c>SEND</c>
+	  control messages will be sent in the same direction
+	  on the connection.
+	</p></note>
+      </item>
+      <tag><c>SEND_SENDER_TT</c></tag>
+      <item>
+        <p><c>{23, FromPid, ToPid, TraceToken}</c></p>
+        <p>Followed by <c>Message</c>.</p>
+	<p>
+	  This control messages replace the <c>SEND_TT</c> control
+	  message and will be sent when the distribution flag
+	  <seealso marker="erl_dist_protocol#dflags"><c>DFLAG_SEND_SENDER</c></seealso>
+	  has been negotiated in the connection setup handshake.
+	</p>
+	<note><p>
+	  Messages encoded before the connection has
+	  been set up may still use the <c>SEND_TT</c> control
+	  message. However, once a <c>SEND_SENDER</c> or <c>SEND_SENDER_TT</c>
+	  control message has been sent, no more <c>SEND_TT</c>
+	  control messages will be sent in the same direction
+	  on the connection.
+	</p></note>
+      </item>
+    </taglist>
+  </section>
+
 </chapter>
diff --git a/erts/doc/src/erlang.xml b/erts/doc/src/erlang.xml
index 8b0f97f30f..f1fcbd7532 100644
--- a/erts/doc/src/erlang.xml
+++ b/erts/doc/src/erlang.xml
@@ -1236,12 +1236,6 @@ end</code>
 	  <seealso marker="erlang#dist_ctrl_get_data_notification/1"><c>erlang:dist_ctrl_get_data_notification(DHandle)</c></seealso>.
 	</p>
 	<note><p>
-	  The data retreived from this function needs to be delivered
-	  as is to the node on the other end in the exact same order,
-	  with no loss of data what so ever, as retrived from this
-	  function.
-	</p></note>
-	<note><p>
 	  Only the process registered as distribution
 	  controller for the distribution channel identified by
 	  <c><anno>DHandle</anno></c> is allowed to call this
@@ -1253,7 +1247,7 @@ end</code>
 	  controllers. <c><anno>DHandle</anno></c> is retrived
 	  via the callback
 	  <seealso marker="erts:alt_dist#hs_data_f_handshake_complete"><c>f_handshake_complete</c></seealso>.
-	  More information about this can be found in the documentation of
+	  More information can be found in the documentation of
 	  <seealso marker="erts:alt_dist#distribution_module">ERTS
 	  User's Guide ➜ How to implement an Alternative Carrier
 	  for the Erlang Distribution ➜ Distribution Module</seealso>.
@@ -1289,7 +1283,7 @@ end</code>
 	  controllers. <c><anno>DHandle</anno></c> is retrived
 	  via the callback
 	  <seealso marker="erts:alt_dist#hs_data_f_handshake_complete"><c>f_handshake_complete</c></seealso>.
-	  More information about this can be found in the documentation of
+	  More information can be found in the documentation of
 	  <seealso marker="erts:alt_dist#distribution_module">ERTS
 	  User's Guide ➜ How to implement an Alternative Carrier
 	  for the Erlang Distribution ➜ Distribution Module</seealso>.
@@ -1322,7 +1316,7 @@ end</code>
 	  controllers. <c><anno>DHandle</anno></c> is retrived
 	  via the callback
 	  <seealso marker="erts:alt_dist#hs_data_f_handshake_complete"><c>f_handshake_complete</c></seealso>.
-	  More information about this can be found in the documentation of
+	  More information can be found in the documentation of
 	  <seealso marker="erts:alt_dist#distribution_module">ERTS
 	  User's Guide ➜ How to implement an Alternative Carrier
 	  for the Erlang Distribution ➜ Distribution Module</seealso>.
@@ -1339,11 +1333,6 @@ end</code>
 	  local node.
 	</p>
 	<note><p>
-	  The data passed to the VM using this function needs to be
-	  passed in the exact same order, and with no loss of data
-	  what so ever, as sent from the node on the other end.
-	</p></note>
-	<note><p>
 	  Only the process registered as distribution
 	  controller for the distribution channel identified by
 	  <c><anno>DHandle</anno></c> is allowed to call this
@@ -1360,7 +1349,7 @@ end</code>
 	  controllers. <c><anno>DHandle</anno></c> is retrived
 	  via the callback
 	  <seealso marker="erts:alt_dist#hs_data_f_handshake_complete"><c>f_handshake_complete</c></seealso>.
-	  More information about this can be found in the documentation of
+	  More information can be found in the documentation of
 	  <seealso marker="erts:alt_dist#distribution_module">ERTS
 	  User's Guide ➜ How to implement an Alternative Carrier
 	  for the Erlang Distribution ➜ Distribution Module</seealso>.
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index b1f915b133..53b48ba6e1 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -2035,6 +2035,7 @@ static Sint remote_send(Process *p, DistEntry *dep,
 
     ASSERT(is_atom(to) || is_external_pid(to));
 
+    ctx->dep = dep;
     code = erts_dsig_prepare(&ctx->dsd, dep, p, ERTS_DSP_NO_LOCK, !ctx->suspend);
     switch (code) {
     case ERTS_DSIG_PREP_NOT_ALIVE:
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index d5c39485d0..a50072fe98 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -953,11 +953,30 @@ erts_dsig_send_msg(Eterm remote, Eterm message, ErtsSendContext* ctx)
     }
 #endif
 
-    if (token != NIL)
-	ctl = TUPLE4(&ctx->ctl_heap[0],
-		     make_small(DOP_SEND_TT), am_Empty, remote, token);
-    else
-	ctl = TUPLE3(&ctx->ctl_heap[0], make_small(DOP_SEND), am_Empty, remote);
+    if (token != NIL) {
+        Eterm el1, el2;
+        if (ctx->dep->flags & DFLAG_SEND_SENDER) {
+            el1 = make_small(DOP_SEND_SENDER_TT);
+            el2 = sender->common.id;
+        }
+        else {
+            el1 = make_small(DOP_SEND_TT);
+            el2 = am_Empty;
+        }
+	ctl = TUPLE4(&ctx->ctl_heap[0], el1, el2, remote, token);
+    }
+    else {
+        Eterm el1, el2;
+        if (ctx->dep->flags & DFLAG_SEND_SENDER) {
+            el1 = make_small(DOP_SEND_SENDER);
+            el2 = sender->common.id;
+        }
+        else {
+            el1 = make_small(DOP_SEND);
+            el2 = am_Empty;
+        }
+	ctl = TUPLE3(&ctx->ctl_heap[0], el1, el2, remote);
+    }
     DTRACE6(message_send, sender_name, receiver_name,
             msize, tok_label, tok_lastcnt, tok_serial);
     DTRACE7(message_send_remote, sender_name, node_name, receiver_name,
@@ -1292,6 +1311,7 @@ int erts_net_message(Port *prt,
     }
 
     token_size = 0;
+    token = NIL;
 
     switch (type = unsigned_val(tuple[1])) {
     case DOP_LINK:
@@ -1521,38 +1541,52 @@ int erts_net_message(Port *prt,
 	}
 	break;
 
+    case DOP_SEND_SENDER_TT: {
+        Uint xsize;
     case DOP_SEND_TT:
+
 	if (tuple_arity != 4) {
 	    goto invalid_message;
 	}
-	
-	token_size = size_object(tuple[4]);
-	/* Fall through ... */
+
+	token = tuple[4];
+	token_size = size_object(token);
+        xsize = ERTS_HEAP_FRAG_SIZE(token_size);
+        goto send_common;
+
+    case DOP_SEND_SENDER:
     case DOP_SEND:
+
+        token = NIL;
+        xsize = 0;
+	if (tuple_arity != 3)
+	    goto invalid_message;
+
+    send_common:
+
 	/*
-	 * There is intentionally no testing of the cookie (it is always '')
-	 * from R9B and onwards.
+         * If DOP_SEND_SENDER or DOP_SEND_SENDER_TT element 2 contains
+         * the sender pid (i.e. DFLAG_SEND_SENDER is set); otherwise,
+         * the atom '' (empty cookie).
 	 */
+        ASSERT((type == DOP_SEND_SENDER || type == DOP_SEND_SENDER_TT)
+               ? (is_pid(tuple[2]) && (dep->flags & DFLAG_SEND_SENDER))
+               : tuple[2] == am_Empty);
+
 #ifdef ERTS_DIST_MSG_DBG
 	dist_msg_dbg(&ede, "MSG", buf, orig_len);
 #endif
-	if (type != DOP_SEND_TT && tuple_arity != 3) {
-	    goto invalid_message;
-	}
 	to = tuple[3];
 	if (is_not_pid(to)) {
 	    goto invalid_message;
 	}
 	rp = erts_proc_lookup(to);
 	if (rp) {
-	    Uint xsize = type == DOP_SEND ? 0 : ERTS_HEAP_FRAG_SIZE(token_size);
 	    ErtsProcLocks locks = 0;
 	    ErtsDistExternal *ede_copy;
 
 	    ede_copy = erts_make_dist_ext_copy(&ede, xsize);
-	    if (type == DOP_SEND) {
-		token = NIL;
-	    } else {
+	    if (is_not_nil(token)) {
 		ErlHeapFragment *heap_frag;
 		ErlOffHeap *ohp;
 		ASSERT(xsize);
@@ -1560,15 +1594,15 @@ int erts_net_message(Port *prt,
 		ERTS_INIT_HEAP_FRAG(heap_frag, token_size, token_size);
 		hp = heap_frag->mem;
 		ohp = &heap_frag->off_heap;
-		token = tuple[4];
 		token = copy_struct(token, token_size, &hp, ohp);
 	    }
 
-	    erts_queue_dist_message(rp, locks, ede_copy, token, tuple[2]);
+	    erts_queue_dist_message(rp, locks, ede_copy, token, am_Empty);
 	    if (locks)
 		erts_smp_proc_unlock(rp, locks);
 	}
 	break;
+    }
 
     case DOP_MONITOR_P_EXIT: {
 	/* We are monitoring a process on the remote node which dies, we get
diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index dcd5846ca5..06c13a9f2b 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -44,6 +44,7 @@
 #define DFLAG_UTF8_ATOMS          0x10000
 #define DFLAG_MAP_TAG             0x20000
 #define DFLAG_BIG_CREATION        0x40000
+#define DFLAG_SEND_SENDER         0x80000
 
 /* All flags that should be enabled when term_to_binary/1 is used. */
 #define TERM_TO_BINARY_DFLAGS (DFLAG_EXTENDED_REFERENCES	\
@@ -74,6 +75,9 @@
 #define DOP_DEMONITOR_P		20
 #define DOP_MONITOR_P_EXIT	21
 
+#define DOP_SEND_SENDER         22
+#define DOP_SEND_SENDER_TT      23
+
 /* distribution trap functions */
 extern Export* dsend2_trap;
 extern Export* dsend3_trap;
@@ -346,6 +350,7 @@ typedef struct {
     Eterm ctl_heap[6];
     ErtsDSigData dsd;
     DistEntry* dep_to_deref;
+    DistEntry *dep;
     struct erts_dsig_send_context dss;
 
     Eterm return_term;
diff --git a/lib/kernel/include/dist.hrl b/lib/kernel/include/dist.hrl
index d6bccdf474..db4a5eaebc 100644
--- a/lib/kernel/include/dist.hrl
+++ b/lib/kernel/include/dist.hrl
@@ -40,3 +40,33 @@
 -define(DFLAG_UTF8_ATOMS, 16#10000).
 -define(DFLAG_MAP_TAG, 16#20000).
 -define(DFLAG_BIG_CREATION, 16#40000).
+-define(DFLAG_SEND_SENDER, 16#80000).
+
+%% DFLAGs that require strict ordering or:ed together...
+-define(DFLAGS_STRICT_ORDER_DELIVERY,
+        ?DFLAG_DIST_HDR_ATOM_CACHE).
+
+
+%% Also update dflag2str() in ../src/dist_util.erl
+%% when adding flags...
+
+-define(DFLAGS_ALL,
+        (?DFLAG_PUBLISHED
+             bor ?DFLAG_ATOM_CACHE
+             bor ?DFLAG_EXTENDED_REFERENCES
+             bor ?DFLAG_DIST_MONITOR
+             bor ?DFLAG_FUN_TAGS
+             bor ?DFLAG_DIST_MONITOR_NAME
+             bor ?DFLAG_HIDDEN_ATOM_CACHE
+             bor ?DFLAG_NEW_FUN_TAGS
+             bor ?DFLAG_EXTENDED_PIDS_PORTS
+             bor ?DFLAG_EXPORT_PTR_TAG
+             bor ?DFLAG_BIT_BINARIES
+             bor ?DFLAG_NEW_FLOATS
+             bor ?DFLAG_UNICODE_IO
+             bor ?DFLAG_DIST_HDR_ATOM_CACHE
+             bor ?DFLAG_SMALL_ATOM_TAGS
+             bor ?DFLAG_UTF8_ATOMS
+             bor ?DFLAG_MAP_TAG
+             bor ?DFLAG_BIG_CREATION
+             bor ?DFLAG_SEND_SENDER)).
diff --git a/lib/kernel/include/dist_util.hrl b/lib/kernel/include/dist_util.hrl
index e668b12a82..eeb0f8dd43 100644
--- a/lib/kernel/include/dist_util.hrl
+++ b/lib/kernel/include/dist_util.hrl
@@ -82,6 +82,9 @@
 
           %% New in kernel-6.0 (OTP 21.0)
           f_handshake_complete, %% Notify handshake complete
+          add_flags,         %% dflags to add
+          reject_flags,      %% dflags not to use (not all can be rejected)
+          require_flags     %% dflags that are required
 }).
 	  
 
diff --git a/lib/kernel/src/dist_util.erl b/lib/kernel/src/dist_util.erl
index 81dbf442d1..08bd5946cd 100644
--- a/lib/kernel/src/dist_util.erl
+++ b/lib/kernel/src/dist_util.erl
@@ -74,6 +74,48 @@
 	       ticked = 0
 	       }).
 
+dflag2str(?DFLAG_PUBLISHED) ->
+    "PUBLISHED";
+dflag2str(?DFLAG_ATOM_CACHE) ->
+    "ATOM_CACHE";
+dflag2str(?DFLAG_EXTENDED_REFERENCES) ->
+    "EXTENDED_REFERENCES";
+dflag2str(?DFLAG_DIST_MONITOR) ->
+    "DIST_MONITOR";
+dflag2str(?DFLAG_FUN_TAGS) ->
+    "FUN_TAGS";
+dflag2str(?DFLAG_DIST_MONITOR_NAME) ->
+    "DIST_MONITOR_NAME";
+dflag2str(?DFLAG_HIDDEN_ATOM_CACHE) ->
+    "HIDDEN_ATOM_CACHE";
+dflag2str(?DFLAG_NEW_FUN_TAGS) ->
+    "NEW_FUN_TAGS";
+dflag2str(?DFLAG_EXTENDED_PIDS_PORTS) ->
+    "EXTENDED_PIDS_PORTS";
+dflag2str(?DFLAG_EXPORT_PTR_TAG) ->
+    "EXPORT_PTR_TAG";
+dflag2str(?DFLAG_BIT_BINARIES) ->
+    "BIT_BINARIES";
+dflag2str(?DFLAG_NEW_FLOATS) ->
+    "NEW_FLOATS";
+dflag2str(?DFLAG_UNICODE_IO) ->
+    "UNICODE_IO";
+dflag2str(?DFLAG_DIST_HDR_ATOM_CACHE) ->
+    "DIST_HDR_ATOM_CACHE";
+dflag2str(?DFLAG_SMALL_ATOM_TAGS) ->
+    "SMALL_ATOM_TAGS";
+dflag2str(?DFLAG_UTF8_ATOMS) ->
+    "UTF8_ATOMS";
+dflag2str(?DFLAG_MAP_TAG) ->
+    "MAP_TAG";
+dflag2str(?DFLAG_BIG_CREATION) ->
+    "BIG_CREATION";
+dflag2str(?DFLAG_SEND_SENDER) ->
+    "SEND_SENDER";
+dflag2str(_) ->
+    "UNKNOWN".
+
+
 remove_flag(Flag, Flags) ->
     case Flags band Flag of
 	0 ->
@@ -82,13 +124,13 @@ remove_flag(Flag, Flags) ->
 	    Flags - Flag
     end.
 
-adjust_flags(ThisFlags, OtherFlags) ->
+adjust_flags(ThisFlags, OtherFlags, RejectFlags) ->
     case (?DFLAG_PUBLISHED band ThisFlags) band OtherFlags of
 	0 ->
 	    {remove_flag(?DFLAG_PUBLISHED, ThisFlags),
 	     remove_flag(?DFLAG_PUBLISHED, OtherFlags)};
 	_ ->
-	    {ThisFlags, OtherFlags}
+	    {ThisFlags, OtherFlags band (bnot RejectFlags)}
     end.
 
 publish_flag(hidden, _) ->
@@ -101,36 +143,71 @@ publish_flag(_, OtherNode) ->
 	    0
     end.
 
-make_this_flags(RequestType, OtherNode) ->
-    publish_flag(RequestType, OtherNode) bor
-	%% The parenthesis below makes the compiler generate better code.
-	(?DFLAG_EXPORT_PTR_TAG bor
-	 ?DFLAG_EXTENDED_PIDS_PORTS bor
-	 ?DFLAG_EXTENDED_REFERENCES bor
-	 ?DFLAG_DIST_MONITOR bor
-	 ?DFLAG_FUN_TAGS bor
-	 ?DFLAG_DIST_MONITOR_NAME bor
-	 ?DFLAG_HIDDEN_ATOM_CACHE bor
-	 ?DFLAG_NEW_FUN_TAGS bor
-	 ?DFLAG_BIT_BINARIES bor
-	 ?DFLAG_NEW_FLOATS bor
-	 ?DFLAG_UNICODE_IO bor
-	 ?DFLAG_DIST_HDR_ATOM_CACHE bor
-	 ?DFLAG_SMALL_ATOM_TAGS bor
-	 ?DFLAG_UTF8_ATOMS bor
-	 ?DFLAG_MAP_TAG bor
-	 ?DFLAG_BIG_CREATION).
-
-handshake_other_started(#hs_data{request_type=ReqType}=HSData0) ->
+-define(DFLAGS_REMOVABLE,
+        (?DFLAG_DIST_HDR_ATOM_CACHE
+             bor ?DFLAG_HIDDEN_ATOM_CACHE
+             bor ?DFLAG_ATOM_CACHE)).
+
+-define(DFLAGS_ADDABLE,
+        (?DFLAGS_ALL
+             band (bnot (?DFLAG_PUBLISHED
+                             bor ?DFLAG_HIDDEN_ATOM_CACHE
+                             bor ?DFLAG_ATOM_CACHE)))).
+
+-define(DFLAGS_THIS_DEFAULT,
+        (?DFLAG_EXPORT_PTR_TAG
+             bor ?DFLAG_EXTENDED_PIDS_PORTS
+             bor ?DFLAG_EXTENDED_REFERENCES
+             bor ?DFLAG_DIST_MONITOR
+             bor ?DFLAG_FUN_TAGS
+             bor ?DFLAG_DIST_MONITOR_NAME
+             bor ?DFLAG_NEW_FUN_TAGS
+             bor ?DFLAG_BIT_BINARIES
+             bor ?DFLAG_NEW_FLOATS
+             bor ?DFLAG_UNICODE_IO
+             bor ?DFLAG_DIST_HDR_ATOM_CACHE
+             bor ?DFLAG_SMALL_ATOM_TAGS
+             bor ?DFLAG_UTF8_ATOMS
+             bor ?DFLAG_MAP_TAG
+             bor ?DFLAG_BIG_CREATION
+             bor ?DFLAG_SEND_SENDER)).
+
+make_this_flags(RequestType, AddFlags, RemoveFlags, OtherNode) ->
+    case RemoveFlags band (bnot ?DFLAGS_REMOVABLE) of
+        0 -> ok;
+        Rerror -> exit({"Rejecting non rejectable flags", Rerror})
+    end,
+    case AddFlags band (bnot ?DFLAGS_ADDABLE) of
+        0 -> ok;
+        Aerror -> exit({"Adding non addable flags", Aerror})
+    end,
+    Flgs0 = ?DFLAGS_THIS_DEFAULT,
+    Flgs1 = Flgs0 bor publish_flag(RequestType, OtherNode),
+    Flgs2 = Flgs1 bor AddFlags,
+    Flgs3 = Flgs2 band (bnot (?DFLAG_HIDDEN_ATOM_CACHE
+                                  bor ?DFLAG_ATOM_CACHE)),
+    Flgs3 band (bnot RemoveFlags).
+
+handshake_other_started(#hs_data{request_type=ReqType,
+                                 add_flags=AddFlgs0,
+                                 reject_flags=RejFlgs0,
+                                 require_flags=ReqFlgs0}=HSData0) ->
+    AddFlgs = convert_flags(AddFlgs0),
+    RejFlgs = convert_flags(RejFlgs0),
+    ReqFlgs = convert_flags(ReqFlgs0),
     {PreOtherFlags,Node,Version} = recv_name(HSData0),
-    PreThisFlags = make_this_flags(ReqType, Node),
+    PreThisFlags = make_this_flags(ReqType, AddFlgs, RejFlgs, Node),
     {ThisFlags, OtherFlags} = adjust_flags(PreThisFlags,
-					   PreOtherFlags),
+					   PreOtherFlags,
+                                           RejFlgs),
     HSData = HSData0#hs_data{this_flags=ThisFlags,
 			     other_flags=OtherFlags,
 			     other_version=Version,
 			     other_node=Node,
-			     other_started=true},
+			     other_started=true,
+                             add_flags=AddFlgs,
+                             reject_flags=RejFlgs,
+                             require_flags=ReqFlgs},
     check_dflags(HSData),
     is_allowed(HSData),
     ?debug({"MD5 connection from ~p (V~p)~n",
@@ -165,23 +242,18 @@ is_allowed(#hs_data{other_node = Node,
     end.
 
 %%
-%% Check that both nodes can handle the same types of extended
-%% node containers. If they can not, abort the connection.
+%% Check mandatory flags...
 %%
 check_dflags(#hs_data{other_node = Node,
                       other_flags = OtherFlags,
-                      other_started = OtherStarted} = HSData) ->
-
-    Mandatory = [{?DFLAG_EXTENDED_REFERENCES, "EXTENDED_REFERENCES"},
-                 {?DFLAG_EXTENDED_PIDS_PORTS, "EXTENDED_PIDS_PORTS"},
-                 {?DFLAG_UTF8_ATOMS, "UTF8_ATOMS"}],
-    Missing = lists:filtermap(fun({Bit, Str}) ->
-                                      case Bit band OtherFlags of
-                                          Bit -> false;
-                                          0 -> {true, Str}
-                                      end
-                              end,
-                              Mandatory),
+                      other_started = OtherStarted,
+                      require_flags = RequiredFlags} = HSData) ->
+    Mandatory = ((?DFLAG_EXTENDED_REFERENCES
+                      bor ?DFLAG_EXTENDED_PIDS_PORTS
+                      bor ?DFLAG_UTF8_ATOMS)
+                     bor RequiredFlags),
+    Missing = check_mandatory(0, ?DFLAGS_ALL, Mandatory,
+                              OtherFlags, []),
     case Missing of
         [] ->
             ok;
@@ -201,6 +273,22 @@ check_dflags(#hs_data{other_node = Node,
 	    ?shutdown2(Node, {check_dflags_failed, Missing})
     end.
 
+check_mandatory(_Bit, 0, _Mandatory, _OtherFlags, Missing) ->
+    Missing;
+check_mandatory(Bit, Left, Mandatory, OtherFlags, Missing) ->
+    DFlag = (1 bsl Bit),
+    NewLeft = Left band (bnot DFlag),
+    NewMissing = case {DFlag band Mandatory,
+                       DFlag band OtherFlags} of
+                     {DFlag, 0} ->
+                         %% Mandatory and missing...
+                         [dflag2str(DFlag) | Missing];
+                     _ ->
+                         %% Not mandatory or present...
+                         Missing
+                 end,
+    check_mandatory(Bit+1, NewLeft, Mandatory, OtherFlags, NewMissing).
+                    
 
 %% No nodedown will be sent if we fail before this process has
 %% succeeded to mark the node as pending.
@@ -314,13 +402,24 @@ flush_down() ->
     end.
 
 handshake_we_started(#hs_data{request_type=ReqType,
-			      other_node=Node}=PreHSData) ->
-    PreThisFlags = make_this_flags(ReqType, Node),
-    HSData = PreHSData#hs_data{this_flags=PreThisFlags},
+			      other_node=Node,
+                              add_flags=AddFlgs0,
+                              reject_flags=RejFlgs0,
+                              require_flags=ReqFlgs0}=PreHSData) ->
+    AddFlgs = convert_flags(AddFlgs0),
+    RejFlgs = convert_flags(RejFlgs0),
+    ReqFlgs = convert_flags(ReqFlgs0),
+    PreThisFlags = make_this_flags(ReqType, AddFlgs, RejFlgs, Node),
+    HSData = PreHSData#hs_data{this_flags = PreThisFlags,
+                               add_flags = AddFlgs,
+                               reject_flags = RejFlgs,
+                               require_flags = ReqFlgs},
     send_name(HSData),
     recv_status(HSData),
     {PreOtherFlags,ChallengeA} = recv_challenge(HSData),
-    {ThisFlags,OtherFlags} = adjust_flags(PreThisFlags, PreOtherFlags),
+    {ThisFlags,OtherFlags} = adjust_flags(PreThisFlags,
+                                          PreOtherFlags,
+                                          RejFlgs),
     NewHSData = HSData#hs_data{this_flags = ThisFlags,
 			       other_flags = OtherFlags, 
 			       other_started = false}, 
@@ -342,6 +441,11 @@ convert_old_hsdata(OldHsData) ->
     true = NoMissing > 0,
     list_to_tuple(OHSDL ++ lists:duplicate(NoMissing, undefined)).
 
+convert_flags(Flags) when is_integer(Flags) ->
+    Flags;
+convert_flags(_Undefined) ->
+    0.
+
 %% --------------------------------------------------------------
 %% The connection has been established.
 %% --------------------------------------------------------------