Merge branch 'sverker/net_kernel-setopts/OTP-13564' into maint

10 files changed, 417 insertions, 35 deletions

diff --git a/lib/kernel/doc/src/inet.xml b/lib/kernel/doc/src/inet.xml
index c0dce2f50c..8cc21bf3e2 100644
--- a/lib/kernel/doc/src/inet.xml
+++ b/lib/kernel/doc/src/inet.xml
@@ -151,6 +151,12 @@ fe80::204:acff:fe17:bf38
       <name name="socket_address"/>
     </datatype>
     <datatype>
+      <name name="socket_getopt"/>
+    </datatype>
+    <datatype>
+      <name name="socket_setopt"/>
+    </datatype>
+    <datatype>
       <name name="returned_non_ip_address"/>
       <desc>
 	<p>
@@ -327,8 +333,6 @@ fe80::204:acff:fe17:bf38
     <func>
       <name name="getopts" arity="2"/>
       <fsummary>Get one or more options for a socket.</fsummary>
-      <type name="socket_getopt"/>
-      <type name="socket_setopt"/>
       <desc>
         <p>Gets one or more options for a socket. For a list of available
           options, see
@@ -580,7 +584,6 @@ get_tcpi_sacked(Sock) ->
     <func>
       <name name="setopts" arity="2"/>
       <fsummary>Set one or more options for a socket.</fsummary>
-      <type name="socket_setopt"/>
       <desc>
         <p>Sets one or more options for a socket.</p>
         <p>The following options are available:</p>
diff --git a/lib/kernel/doc/src/kernel_app.xml b/lib/kernel/doc/src/kernel_app.xml
index 9e6fb60bb7..dc0291babe 100644
--- a/lib/kernel/doc/src/kernel_app.xml
+++ b/lib/kernel/doc/src/kernel_app.xml
@@ -215,12 +215,14 @@
         <p>Defines the <c>First..Last</c> port range for the listener
           socket of a distributed Erlang node.</p>
       </item>
+      <marker id="inet_dist_listen_options"></marker>
       <tag><c>{inet_dist_listen_options, Opts}</c></tag>
       <item>
         <p>Defines a list of extra socket options to be used when opening the
 	  listening socket for a distributed Erlang node.
 	  See <seealso marker="gen_tcp#listen/2"><c>gen_tcp:listen/2</c></seealso>.</p>
       </item>
+      <marker id="inet_dist_connect_options"></marker>
       <tag><c>{inet_dist_connect_options, Opts}</c></tag>
       <item>
         <p>Defines a list of extra socket options to be used when connecting to
diff --git a/lib/kernel/doc/src/net_kernel.xml b/lib/kernel/doc/src/net_kernel.xml
index f48a534d4f..3c1b3d5190 100644
--- a/lib/kernel/doc/src/net_kernel.xml
+++ b/lib/kernel/doc/src/net_kernel.xml
@@ -116,6 +116,21 @@ $ <input>erl -sname foobar</input></pre>
     </func>
 
     <func>
+      <name name="getopts" arity="2"/>
+      <fsummary>Get distribution socket options.</fsummary>
+      <desc>
+        <p>Get one or more options for the distribution socket
+	connected to <c><anno>Node</anno></c>.</p>
+        <p>If <c><anno>Node</anno></c> is a connected node
+	the return value is the same as from
+	<seealso marker="inet#getopts/2"><c>inet:getopts(Sock, Options)</c></seealso>
+	where <c>Sock</c> is the distribution socket for <c><anno>Node</anno></c>.</p>
+	<p>Returns <c>ignored</c> if the local node is not alive or
+	<c>{error, noconnection}</c> if <c><anno>Node</anno></c> is not connected.</p>
+      </desc>
+    </func>
+
+    <func>
       <name name="monitor_nodes" arity="1"/>
       <name name="monitor_nodes" arity="2"/>
       <fsummary>Subscribe to node status change messages.</fsummary>
@@ -289,6 +304,27 @@ $ <input>erl -sname foobar</input></pre>
     </func>
 
     <func>
+      <name name="setopts" arity="2"/>
+      <fsummary>Set distribution socket options.</fsummary>
+      <desc>
+        <p>Set one or more options for distribution sockets.
+	Argument <c><anno>Node</anno></c> can be either one node name
+	or the atom <c>new</c> to affect the distribution sockets of all
+	future connected nodes.</p>
+        <p>The return value is the same as from
+	<seealso marker="inet#setopts/2"><c>inet:setopts/2</c></seealso>
+	or <c>{error, noconnection}</c> if <c><anno>Node</anno></c> is not
+	a connected node or <c>new</c>.</p>
+	<p>If <c><anno>Node</anno></c> is <c>new</c> the <c><anno>Options</anno></c>
+	will then also be added to kernel configration parameters
+	<seealso marker="kernel:kernel_app#inet_dist_listen_options">inet_dist_listen_options</seealso>
+	and
+	<seealso marker="kernel:kernel_app#inet_dist_connect_options">inet_dist_connect_options</seealso>.</p>
+	<p>Returns <c>ignored</c> if the local node is not alive.</p>
+      </desc>
+    </func>
+
+    <func>
       <name>start([Name]) -> {ok, pid()} | {error, Reason}</name>
       <name>start([Name, NameType]) -> {ok, pid()} | {error, Reason}</name>
       <name>start([Name, NameType, Ticktime]) -> {ok, pid()} | {error, Reason}</name>
diff --git a/lib/kernel/include/dist_util.hrl b/lib/kernel/include/dist_util.hrl
index 43e50d4325..320e916c04 100644
--- a/lib/kernel/include/dist_util.hrl
+++ b/lib/kernel/include/dist_util.hrl
@@ -63,7 +63,7 @@
 	  f_getll,               %% Get low level port or pid.
 	  f_address,         %% The address of the "socket", 
 	                     %% generated from Socket,Node
-	  %% These two are used in the tick loop,
+	  %% These three are used in the tick loop,
 	  %% so they are not fun's to avoid holding old code.
 	  mf_tick,           %% Takes the socket as parameters and
 	                     %% sends a tick, this is no fun, it
@@ -74,7 +74,11 @@
 			     %% {ok, RecvCnt, SendCnt, SendPend} for
 	                     %% a given socket. This is a {M,F}, 
 	                     %% returning {error, Reason on failure}
-	  request_type = normal
+	  request_type = normal,
+
+	  %% New in kernel-5.1 (OTP 19.1):
+	  mf_setopts,        %% netkernel:setopts on active connection
+	  mf_getopts         %% netkernel:getopts on active connection
 }).
 	  
 
diff --git a/lib/kernel/src/dist_util.erl b/lib/kernel/src/dist_util.erl
index 47d0c1b861..8d2fc4d4b7 100644
--- a/lib/kernel/src/dist_util.erl
+++ b/lib/kernel/src/dist_util.erl
@@ -143,7 +143,11 @@ handshake_other_started(#hs_data{request_type=ReqType}=HSData0) ->
     ChallengeB = recv_challenge_reply(HSData, ChallengeA, MyCookie),
     send_challenge_ack(HSData, gen_digest(ChallengeB, HisCookie)),
     ?debug({dist_util, self(), accept_connection, Node}),
-    connection(HSData).
+    connection(HSData);
+
+handshake_other_started(OldHsData) when element(1,OldHsData) =:= hs_data ->
+    handshake_other_started(convert_old_hsdata(OldHsData)).
+
 
 %%
 %% check if connecting node is allowed to connect
@@ -330,7 +334,20 @@ handshake_we_started(#hs_data{request_type=ReqType,
 			 gen_digest(ChallengeA,HisCookie)),
     reset_timer(NewHSData#hs_data.timer),
     recv_challenge_ack(NewHSData, MyChallenge, MyCookie),
-    connection(NewHSData).
+    connection(NewHSData);
+
+handshake_we_started(OldHsData) when element(1,OldHsData) =:= hs_data ->
+    handshake_we_started(convert_old_hsdata(OldHsData)).
+
+convert_old_hsdata({hs_data, KP, ON, TN, S, T, TF, A, OV, OF, OS, FS, FR,
+		    FS_PRE, FS_POST, FG, FA, MFT, MFG, RT}) ->
+    #hs_data{
+       kernel_pid = KP, other_node = ON, this_node = TN, socket = S, timer = T,
+       this_flags = TF, allowed = A, other_version = OV, other_flags = OF,
+       other_started = OS, f_send = FS, f_recv = FR, f_setopts_pre_nodeup = FS_PRE,
+       f_setopts_post_nodeup = FS_POST, f_getll = FG, f_address = FA,
+       mf_tick = MFT, mf_getstat = MFG, request_type = RT}.
+
 
 %% --------------------------------------------------------------
 %% The connection has been established.
@@ -350,15 +367,15 @@ connection(#hs_data{other_node = Node,
 	    mark_nodeup(HSData,Address),
 	    case FPostNodeup(Socket) of
 		ok ->
-		    con_loop(HSData#hs_data.kernel_pid, 
-			     Node, 
-			     Socket, 
-			     Address,
-			     HSData#hs_data.this_node, 
-			     PType,
-			     #tick{},
-			     HSData#hs_data.mf_tick,
-			     HSData#hs_data.mf_getstat);
+		    con_loop({HSData#hs_data.kernel_pid,
+			      Node,
+			      Socket,
+			      PType,
+			      HSData#hs_data.mf_tick,
+			      HSData#hs_data.mf_getstat,
+			      HSData#hs_data.mf_setopts,
+			      HSData#hs_data.mf_getopts},
+			     #tick{});
 		_ ->
 		    ?shutdown2(Node, connection_setup_failed)
 	    end;
@@ -454,8 +471,8 @@ mark_nodeup(#hs_data{kernel_pid = Kernel,
 	    ?shutdown(Node)
     end.
 
-con_loop(Kernel, Node, Socket, TcpAddress,
-	 MyNode, Type, Tick, MFTick, MFGetstat) ->
+con_loop({Kernel, Node, Socket, Type, MFTick, MFGetstat, MFSetOpts, MFGetOpts}=ConData,
+	 Tick) ->
     receive
 	{tcp_closed, Socket} ->
 	    ?shutdown2(Node, connection_closed);
@@ -468,15 +485,12 @@ con_loop(Kernel, Node, Socket, TcpAddress,
 		_ ->
 		    ignore_it
 	    end,
-	    con_loop(Kernel, Node, Socket, TcpAddress, MyNode, Type,
-		     Tick, MFTick, MFGetstat);
+	    con_loop(ConData, Tick);
 	{Kernel, tick} ->
 	    case send_tick(Socket, Tick, Type, 
 			   MFTick, MFGetstat) of
 		{ok, NewTick} ->
-		    con_loop(Kernel, Node, Socket, TcpAddress,
-			     MyNode, Type, NewTick, MFTick,  
-			     MFGetstat);
+		    con_loop(ConData, NewTick);
 		{error, not_responding} ->
  		    error_msg("** Node ~p not responding **~n"
  			      "** Removing (timedout) connection **~n",
@@ -489,13 +503,24 @@ con_loop(Kernel, Node, Socket, TcpAddress,
 	    case MFGetstat(Socket) of
 		{ok, Read, Write, _} ->
 		    From ! {self(), get_status, {ok, Read, Write}},
-		    con_loop(Kernel, Node, Socket, TcpAddress, 
-			     MyNode, 
-			     Type, Tick, 
-			     MFTick, MFGetstat);
+		    con_loop(ConData, Tick);
 		_ ->
 		    ?shutdown2(Node, get_status_failed)
-	    end
+	    end;
+	{From, Ref, {setopts, Opts}} ->
+	    Ret = case MFSetOpts of
+		      undefined -> {error, enotsup};
+		      _ -> MFSetOpts(Socket, Opts)
+		  end,
+	    From ! {Ref, Ret},
+	    con_loop(ConData, Tick);
+	{From, Ref, {getopts, Opts}} ->
+	    Ret = case MFGetOpts of
+		      undefined -> {error, enotsup};
+		      _ -> MFGetOpts(Socket, Opts)
+		  end,
+	    From ! {Ref, Ret},
+	    con_loop(ConData, Tick)
     end.
 
 
diff --git a/lib/kernel/src/inet.erl b/lib/kernel/src/inet.erl
index a91a6ed517..75dd800c6b 100644
--- a/lib/kernel/src/inet.erl
+++ b/lib/kernel/src/inet.erl
@@ -75,6 +75,7 @@
 -export_type([address_family/0, hostent/0, hostname/0, ip4_address/0,
               ip6_address/0, ip_address/0, port_number/0,
 	      local_address/0, socket_address/0, returned_non_ip_address/0,
+	      socket_setopt/0, socket_getopt/0,
 	      posix/0, socket/0, stat_option/0]).
 %% imports
 -import(lists, [append/1, duplicate/2, filter/2, foldl/3]).
diff --git a/lib/kernel/src/inet6_tcp_dist.erl b/lib/kernel/src/inet6_tcp_dist.erl
index 3aa61973af..9b6c2745d5 100644
--- a/lib/kernel/src/inet6_tcp_dist.erl
+++ b/lib/kernel/src/inet6_tcp_dist.erl
@@ -24,6 +24,7 @@
 -export([listen/1, accept/1, accept_connection/5,
          setup/5, close/1, select/1, is_node_name/1]).
 
+-export([setopts/2, getopts/2]).
 
 %% ------------------------------------------------------------
 %%  Select this protocol based on node name
@@ -72,3 +73,9 @@ close(Socket) ->
     
 is_node_name(Node) when is_atom(Node) ->
     inet_tcp_dist:is_node_name(Node).
+
+setopts(S, Opts) ->
+    inet_tcp_dist:setopts(S, Opts).
+
+getopts(S, Opts) ->
+    inet_tcp_dist:getopts(S, Opts).
diff --git a/lib/kernel/src/inet_tcp_dist.erl b/lib/kernel/src/inet_tcp_dist.erl
index 94bde55133..3084bd599a 100644
--- a/lib/kernel/src/inet_tcp_dist.erl
+++ b/lib/kernel/src/inet_tcp_dist.erl
@@ -24,13 +24,16 @@
 -export([listen/1, accept/1, accept_connection/5,
 	 setup/5, close/1, select/1, is_node_name/1]).
 
+%% Optional
+-export([setopts/2, getopts/2]).
+
 %% Generalized dist API
 -export([gen_listen/2, gen_accept/2, gen_accept_connection/6,
 	 gen_setup/6, gen_select/2]).
 
 %% internal exports
 
--export([accept_loop/3,do_accept/7,do_setup/7,getstat/1]).
+-export([accept_loop/3,do_accept/7,do_setup/7,getstat/1,tick/2]).
 
 -import(error_logger,[error_msg/2]).
 
@@ -215,8 +218,10 @@ do_accept(Driver, Kernel, AcceptPid, Socket, MyNode, Allowed, SetupTime) ->
 					inet:getll(S)
 				end,
 		      f_address = fun(S, Node) -> get_remote_id(Driver, S, Node) end,
-		      mf_tick = fun(S) -> tick(Driver, S) end,
-		      mf_getstat = fun ?MODULE:getstat/1
+		      mf_tick = fun(S) -> ?MODULE:tick(Driver, S) end,
+		      mf_getstat = fun ?MODULE:getstat/1,
+		      mf_setopts = fun ?MODULE:setopts/2,
+		      mf_getopts = fun ?MODULE:getopts/2
 		     },
 		    dist_util:handshake_other_started(HSData);
 		{false,IP} ->
@@ -320,6 +325,7 @@ do_setup(Driver, Kernel, Node, Type, MyNode, LongOrShortNames, SetupTime) ->
 					  {packet, 4},
 					  nodelay()])
 			      end,
+
 			      f_getll = fun inet:getll/1,
 			      f_address = 
 			      fun(_,_) ->
@@ -329,9 +335,11 @@ do_setup(Driver, Kernel, Node, Type, MyNode, LongOrShortNames, SetupTime) ->
 				   protocol = tcp,
 				   family = AddressFamily}
 			      end,
-			      mf_tick = fun(S) -> tick(Driver, S) end,
+			      mf_tick = fun(S) -> ?MODULE:tick(Driver, S) end,
 			      mf_getstat = fun ?MODULE:getstat/1,
-			      request_type = Type
+			      request_type = Type,
+			      mf_setopts = fun ?MODULE:setopts/2,
+			      mf_getopts = fun ?MODULE:getopts/2
 			     },
 			    dist_util:handshake_we_started(HSData);
 			_ ->
@@ -492,3 +500,12 @@ split_stat([], R, W, P) ->
     {ok, R, W, P}.
 
 
+setopts(S, Opts) ->
+    case [Opt || {K,_}=Opt <- Opts,
+		 K =:= active orelse K =:= deliver orelse K =:= packet] of
+	[] -> inet:setopts(S,Opts);
+	Opts1 -> {error, {badopts,Opts1}}
+    end.
+
+getopts(S, Opts) ->
+    inet:getopts(S, Opts).
diff --git a/lib/kernel/src/net_kernel.erl b/lib/kernel/src/net_kernel.erl
index ac19f4935b..0c679e7349 100644
--- a/lib/kernel/src/net_kernel.erl
+++ b/lib/kernel/src/net_kernel.erl
@@ -59,6 +59,8 @@
 	 connect_node/1,
 	 monitor_nodes/1,
 	 monitor_nodes/2,
+	 setopts/2,
+	 getopts/2,
 	 start/1,
 	 stop/0]).
 
@@ -111,7 +113,7 @@
 	 }).
 
 -record(listen, {
-		 listen,     %% listen pid
+		 listen,     %% listen socket
 		 accept,     %% accepting pid
 		 address,    %% #net_address
 		 module      %% proto module
@@ -384,7 +386,7 @@ init({Name, LongOrShortNames, TickT, CleanHalt}) ->
 			connections =
 			ets:new(sys_dist,[named_table,
 					  protected,
-					  {keypos, 2}]),
+					  {keypos, #connection.node}]),
 			listen = Listeners,
 			allowed = [],
 			verbose = 0
@@ -554,6 +556,38 @@ handle_call({new_ticktime,_T,_TP},
 	    #state{tick = #tick_change{time = T}} = State) ->
     async_reply({reply, {ongoing_change_to, T}, State}, From);
 
+handle_call({setopts, new, Opts}, From, State) ->
+    Ret = setopts_new(Opts, State),
+    async_reply({reply, Ret, State}, From);
+
+handle_call({setopts, Node, Opts}, From, State) ->
+    Return =
+	case ets:lookup(sys_dist, Node) of
+	    [Conn] when Conn#connection.state =:= up ->
+		case call_owner(Conn#connection.owner, {setopts, Opts}) of
+		    {ok, Ret} -> Ret;
+		    _ -> {error, noconnection}
+		end;
+
+	    _ ->
+		{error, noconnection}
+    end,
+    async_reply({reply, Return, State}, From);
+
+handle_call({getopts, Node, Opts}, From, State) ->
+    Return =
+	case ets:lookup(sys_dist, Node) of
+	    [Conn] when Conn#connection.state =:= up ->
+		case call_owner(Conn#connection.owner, {getopts, Opts}) of
+		    {ok, Ret} -> Ret;
+		    _ -> {error, noconnection}
+		end;
+
+	    _ ->
+		{error, noconnection}
+    end,
+    async_reply({reply, Return, State}, From);
+
 handle_call(_Msg, _From, State) ->
     {noreply, State}.
 
@@ -1608,3 +1642,93 @@ async_gen_server_reply(From, Msg) ->
         {'EXIT', _} ->
             ok
     end.
+
+call_owner(Owner, Msg) ->
+    Mref = monitor(process, Owner),
+    Owner ! {self(), Mref, Msg},
+    receive
+	{Mref, Reply} ->
+	    erlang:demonitor(Mref, [flush]),
+	    {ok, Reply};
+	{'DOWN', Mref, _, _, _} ->
+	    error
+    end.
+
+
+-spec setopts(Node, Options) -> ok | {error, Reason} | ignored when
+      Node :: node() | new,
+      Options :: [inet:socket_setopt()],
+      Reason :: inet:posix() | noconnection.
+
+setopts(Node, Opts) when is_atom(Node), is_list(Opts) ->
+    request({setopts, Node, Opts}).
+
+setopts_new(Opts, State) ->
+    %% First try setopts on listening socket(s)
+    %% Bail out on failure.
+    %% If successful, we are pretty sure Opts are ok
+    %% and we continue with config params and pending connections.
+    case setopts_on_listen(Opts, State#state.listen) of
+	ok ->
+	    setopts_new_1(Opts);
+	Fail -> Fail
+    end.
+
+setopts_on_listen(_, []) -> ok;
+setopts_on_listen(Opts, [#listen {listen = LSocket, module = Mod} | T]) ->
+    try Mod:setopts(LSocket, Opts) of
+	ok ->
+	    setopts_on_listen(Opts, T);
+	Fail -> Fail
+    catch
+	error:undef -> {error, enotsup}
+    end.
+
+setopts_new_1(Opts) ->
+    ConnectOpts = case application:get_env(kernel, inet_dist_connect_options) of
+		      {ok, CO} -> CO;
+		      _ -> []
+		  end,
+    application:set_env(kernel, inet_dist_connect_options,
+			merge_opts(Opts,ConnectOpts)),
+    ListenOpts = case application:get_env(kernel, inet_dist_listen_options) of
+		     {ok, LO} -> LO;
+		     _ -> []
+		 end,
+    application:set_env(kernel, inet_dist_listen_options,
+			merge_opts(Opts, ListenOpts)),
+    case lists:keyfind(nodelay, 1, Opts) of
+	{nodelay, ND} when is_boolean(ND) ->
+	    application:set_env(kernel, dist_nodelay, ND);
+	_ -> ignore
+    end,
+
+    %% Update any pending connections
+    PendingConns = ets:select(sys_dist, [{'_',
+					  [{'=/=',{element,#connection.state,'$_'},up}],
+					  ['$_']}]),
+    lists:foreach(fun(#connection{state = pending, owner = Owner}) ->
+			  call_owner(Owner, {setopts, Opts});
+		     (#connection{state = up_pending, pending_owner = Owner}) ->
+			  call_owner(Owner, {setopts, Opts});
+		     (_) -> ignore
+		  end, PendingConns),
+    ok.
+
+merge_opts([], B) ->
+    B;
+merge_opts([H|T], B0) ->
+    {Key, _} = H,
+    B1 = lists:filter(fun({K,_}) -> K =/= Key end, B0),
+    merge_opts(T, [H | B1]).
+
+-spec getopts(Node, Options) ->
+	{'ok', OptionValues} | {'error', Reason} | ignored when
+      Node :: node(),
+      Options :: [inet:socket_getopt()],
+      OptionValues :: [inet:socket_setopt()],
+      Reason :: inet:posix() | noconnection.
+
+getopts(Node, Opts) when is_atom(Node), is_list(Opts) ->
+    request({getopts, Node, Opts}).
+
diff --git a/lib/kernel/test/erl_distribution_SUITE.erl b/lib/kernel/test/erl_distribution_SUITE.erl
index eb58e92224..e43be77428 100644
--- a/lib/kernel/test/erl_distribution_SUITE.erl
+++ b/lib/kernel/test/erl_distribution_SUITE.erl
@@ -25,6 +25,7 @@
 	 init_per_group/2,end_per_group/2]).
 
 -export([tick/1, tick_change/1, illegal_nodenames/1, hidden_node/1,
+	 setopts/1,
 	 table_waste/1, net_setuptime/1,
 	 inet_dist_options_options/1,
 
@@ -42,6 +43,8 @@
 -export([get_socket_priorities/0,
 	 tick_cli_test/1, tick_cli_test1/1,
 	 tick_serv_test/2, tick_serv_test1/1,
+	 run_remote_test/1,
+	 setopts_do/2,
 	 keep_conn/1, time_ping/1]).
 
 -export([init_per_testcase/2, end_per_testcase/2]).
@@ -66,6 +69,7 @@ suite() ->
 
 all() -> 
     [tick, tick_change, illegal_nodenames, hidden_node,
+     setopts,
      table_waste, net_setuptime, inet_dist_options_options,
      {group, monitor_nodes}].
 
@@ -282,6 +286,165 @@ tick_cli_test1(Node) ->
 	    end
     end.
 
+setopts(Config) when is_list(Config) ->
+    register(setopts_regname, self()),
+    [N1,N2,N3,N4] = get_nodenames(4, setopts),
+
+    {_N1F,Port1} = start_node_unconnected(N1, ?MODULE, run_remote_test,
+					["setopts_do", atom_to_list(node()), "1", "ping"]),
+    0 = wait_for_port_exit(Port1),
+
+    {_N2F,Port2} = start_node_unconnected(N2, ?MODULE, run_remote_test,
+				 ["setopts_do", atom_to_list(node()), "2", "ping"]),
+    0 = wait_for_port_exit(Port2),
+
+    {ok, LSock} = gen_tcp:listen(0, [{packet,2}, {active,false}]),
+    {ok, LTcpPort} = inet:port(LSock),
+
+    {N3F,Port3} = start_node_unconnected(N3, ?MODULE, run_remote_test,
+					["setopts_do", atom_to_list(node()),
+					 "1", integer_to_list(LTcpPort)]),
+    wait_and_connect(LSock, N3F, Port3),
+    0 = wait_for_port_exit(Port3),
+
+    {N4F,Port4} = start_node_unconnected(N4, ?MODULE, run_remote_test,
+					["setopts_do", atom_to_list(node()),
+					 "2", integer_to_list(LTcpPort)]),
+    wait_and_connect(LSock, N4F, Port4),
+    0 = wait_for_port_exit(Port4),
+
+    ok.
+
+wait_and_connect(LSock, NodeName, NodePort) ->
+    {ok, Sock} = gen_tcp:accept(LSock),
+    {ok, "Connect please"} = gen_tcp:recv(Sock, 0),
+    flush_from_port(NodePort),
+    pong = net_adm:ping(NodeName),
+    gen_tcp:send(Sock, "Connect done"),
+    gen_tcp:close(Sock).
+
+
+flush_from_port(Port) ->
+    flush_from_port(Port, 10).
+
+flush_from_port(Port, Timeout) ->
+    receive
+	{Port,{data,String}} ->
+	    io:format("~p: ~s\n", [Port, String]),
+	    flush_from_port(Port, Timeout)
+    after Timeout ->
+	    timeout
+    end.
+
+wait_for_port_exit(Port) ->
+    case (receive M -> M end) of
+	{Port,{exit_status,Status}} ->
+	    Status;
+	{Port,{data,String}} ->
+	    io:format("~p: ~s\n", [Port, String]),
+	    wait_for_port_exit(Port)
+    end.
+
+run_remote_test([FuncStr, TestNodeStr | Args]) ->
+    Status = try
+	io:format("Node ~p started~n", [node()]),
+	TestNode = list_to_atom(TestNodeStr),
+	io:format("Node ~p spawning function ~p~n", [node(), FuncStr]),
+	{Pid,Ref} = spawn_monitor(?MODULE, list_to_atom(FuncStr), [TestNode, Args]),
+	io:format("Node ~p waiting for function ~p~n", [node(), FuncStr]),
+	receive
+	    {'DOWN', Ref, process, Pid, normal} ->
+		0;
+	    Other ->
+		io:format("Node ~p got unexpected msg: ~p\n",[node(), Other]),
+		1
+	end
+    catch
+	C:E ->
+	    io:format("Node ~p got EXCEPTION ~p:~p\nat ~p\n",
+		      [node(), C, E, erlang:get_stacktrace()]),
+	    2
+    end,
+    io:format("Node ~p doing halt(~p).\n",[node(), Status]),
+    erlang:halt(Status).
+
+% Do the actual test on the remote node
+setopts_do(TestNode, [OptNr, ConnectData]) ->
+    [] = nodes(),
+    {Opt, Val} = opt_from_nr(OptNr),
+    ok = net_kernel:setopts(new, [{Opt, Val}]),
+
+    [] = nodes(),
+    {error, noconnection} = net_kernel:getopts(TestNode, [Opt]),
+
+    case ConnectData of
+	"ping" ->  % We connect
+	    net_adm:ping(TestNode);
+	TcpPort -> % Other connect
+	    {ok, Sock} = gen_tcp:connect("localhost", list_to_integer(TcpPort),
+					 [{active,false},{packet,2}]),
+	    ok = gen_tcp:send(Sock, "Connect please"),
+	    {ok, "Connect done"} = gen_tcp:recv(Sock, 0),
+	    gen_tcp:close(Sock)
+    end,
+    [TestNode] = nodes(),
+    {ok, [{Opt,Val}]} = net_kernel:getopts(TestNode, [Opt]),
+    {error, noconnection} = net_kernel:getopts('pixie@fairyland', [Opt]),
+
+    NewVal = change_val(Val),
+    ok = net_kernel:setopts(TestNode, [{Opt, NewVal}]),
+    {ok, [{Opt,NewVal}]} = net_kernel:getopts(TestNode, [Opt]),
+
+    ok = net_kernel:setopts(TestNode, [{Opt, Val}]),
+    {ok, [{Opt,Val}]} = net_kernel:getopts(TestNode, [Opt]),
+
+    ok.
+
+opt_from_nr("1") -> {nodelay, true};
+opt_from_nr("2") -> {nodelay, false}.
+
+change_val(true)  -> false;
+change_val(false) -> true.
+
+start_node_unconnected(Name, Mod, Func, Args) ->
+    FullName = full_node_name(Name),
+    CmdLine = mk_node_cmdline(Name,Mod,Func,Args),
+    io:format("Starting node ~p: ~s~n", [FullName, CmdLine]),
+    case open_port({spawn, CmdLine}, [exit_status]) of
+	Port when is_port(Port) ->
+	    {FullName, Port};
+	Error ->
+	    exit({failed_to_start_node, FullName, Error})
+    end.
+
+full_node_name(PreName) ->
+    HostSuffix = lists:dropwhile(fun ($@) -> false; (_) -> true end,
+				 atom_to_list(node())),
+    list_to_atom(atom_to_list(PreName) ++ HostSuffix).
+
+mk_node_cmdline(Name,Mod,Func,Args) ->
+    Static = "-noinput",
+    Pa = filename:dirname(code:which(?MODULE)),
+    Prog = case catch init:get_argument(progname) of
+	       {ok,[[P]]} -> P;
+	       _ -> exit(no_progname_argument_found)
+	   end,
+    NameSw = case net_kernel:longnames() of
+		 false -> "-sname ";
+		 true -> "-name ";
+		 _ -> exit(not_distributed_node)
+	     end,
+    {ok, Pwd} = file:get_cwd(),
+    NameStr = atom_to_list(Name),
+    Prog ++ " "
+	++ Static ++ " "
+	++ NameSw ++ " " ++ NameStr
+	++ " -pa " ++ Pa
+	++ " -env ERL_CRASH_DUMP " ++ Pwd ++ "/erl_crash_dump." ++ NameStr
+	++ " -setcookie " ++ atom_to_list(erlang:get_cookie())
+	++ " -run " ++ atom_to_list(Mod) ++ " " ++ atom_to_list(Func)
+	++ " " ++ string:join(Args, " ").
+
 
 %% OTP-4255.
 tick_change(Config) when is_list(Config) ->