Merge branch 'lukas/erts/forker'

* lukas/erts/forker: (28 commits)
  erts: Never abort in the forked child
  erts: Mend ASSERT makro for erl_child_setup
  erts: Allow enomem failures in port_SUITE
  erts: iter_port sleep longer on freebsd
  erts: Allow one dangling fd if there is a gethost port
  erts: Only use forker StackAck on freebsd
  erts: It is not possible to exit the forker driver
  erts: Add forker StartAck for port start flowcontrol
  erts: Fix large open_port arg segfault for win32
  erts: Fix memory leak at async open port
  kernel: Remove cmd server for unix os:cmd
  erts: Add testcase for huge port environment
  erts: Move os_pid to port hash to child setup
  erts: Handle all EINTR and EAGAIN cases in child setup
  erts: Make child_setup work with large environments
  erts: Fix forker driver ifdefs for win32
  erts: Fix uds socket handling for os x
  erts: Fix dereferencing of unaligned integer for sparc
  erts: Flatten too long io vectors in uds write
  erts: Add fd count test for spawn_driver
  ...

Conflicts:
	erts/emulator/beam/erl_node_tables.c
	erts/preloaded/src/erts_internal.erl

4 files changed, 188 insertions, 37 deletions

diff --git a/erts/emulator/test/driver_SUITE.erl b/erts/emulator/test/driver_SUITE.erl
index 4fd7b36e6a..ce55fe3c52 100644
--- a/erts/emulator/test/driver_SUITE.erl
+++ b/erts/emulator/test/driver_SUITE.erl
@@ -2395,13 +2395,35 @@ z_test(Config) when is_list(Config) ->
 
 check_io_debug() ->
     get_stable_check_io_info(),
-    {NoErrorFds, NoUsedFds, NoDrvSelStructs, NoDrvEvStructs}
+    {NoErrorFds, NoUsedFds, NoDrvSelStructs, NoDrvEvStructs} = CheckIoDebug
 	= erts_debug:get_internal_state(check_io_debug),
+    HasGetHost = has_gethost(),
+    ct:log("check_io_debug: ~p~n"
+           "HasGetHost: ~p",[CheckIoDebug, HasGetHost]),
     0 = NoErrorFds,
-    NoUsedFds = NoDrvSelStructs,
+    if
+        NoUsedFds == NoDrvSelStructs ->
+            ok;
+        HasGetHost andalso (NoUsedFds == (NoDrvSelStructs - 1)) ->
+            %% If the inet_gethost port is alive, we may have
+            %% one extra used fd that is not selected on
+            ok
+    end,
     0 = NoDrvEvStructs,
     ok.
 
+has_gethost() ->
+    has_gethost(erlang:ports()).
+has_gethost([P|T]) ->
+    case erlang:port_info(P, name) of
+        {name,"inet_gethost"++_} ->
+            true;
+        _ ->
+            has_gethost(T)
+    end;
+has_gethost([]) ->
+    false.
+
 %flush_msgs() ->
 %    receive
 %	M ->
diff --git a/erts/emulator/test/port_SUITE.erl b/erts/emulator/test/port_SUITE.erl
index b42e02a1e1..ff75ee86d6 100644
--- a/erts/emulator/test/port_SUITE.erl
+++ b/erts/emulator/test/port_SUITE.erl
@@ -82,10 +82,11 @@
 	 mul_basic/1, mul_slow_writes/1,
 	 dying_port/1, port_program_with_path/1,
 	 open_input_file_port/1, open_output_file_port/1,
+         count_fds/1,
 	 iter_max_ports/1, eof/1, input_only/1, output_only/1,
 	 name1/1,
 	 t_binary/1, parallell/1, t_exit/1,
-	 env/1, bad_env/1, cd/1, exit_status/1,
+	 env/1, huge_env/1, bad_env/1, cd/1, exit_status/1,
 	 tps_16_bytes/1, tps_1K/1, line/1, stderr_to_stdout/1,
 	 otp_3906/1, otp_4389/1, win_massive/1, win_massive_client/1,
 	 mix_up_ports/1, otp_5112/1, otp_5119/1, otp_6224/1,
@@ -111,8 +112,8 @@ all() ->
      bad_packet, bad_port_messages, {group, options},
      {group, multiple_packets}, parallell, dying_port,
      port_program_with_path, open_input_file_port,
-     open_output_file_port, name1, env, bad_env, cd,
-     exit_status, iter_max_ports, t_exit, {group, tps}, line,
+     open_output_file_port, name1, env, huge_env, bad_env, cd,
+     exit_status, iter_max_ports, count_fds, t_exit, {group, tps}, line,
      stderr_to_stdout, otp_3906, otp_4389, win_massive,
      mix_up_ports, otp_5112, otp_5119,
      exit_status_multi_scheduling_block, ports, spawn_driver,
@@ -385,27 +386,33 @@ input_only(Config) when is_list(Config) ->
 output_only(Config) when is_list(Config) ->
     Dog = test_server:timetrap(test_server:seconds(100)),
     Dir = ?config(priv_dir, Config),
+
+    %% First we test that the port program gets the data
     Filename = filename:join(Dir, "output_only_stream"),
-    output_and_verify(Config, Filename, "-h0",
-          	    random_packet(35777, "echo")),
+    Data = random_packet(35777, "echo"),
+    output_and_verify(Config, ["-h0 -o", Filename], Data),
+    Wait_time = 500,
+    test_server:sleep(Wait_time),
+    {ok, Written} = file:read_file(Filename),
+    Data = binary_to_list(Written),
+
+    %% Then we test that any writes to stdout from
+    %% the port program is not sent to erlang
+    output_and_verify(Config, ["-h0"], Data),
+
     test_server:timetrap_cancel(Dog),
     ok.
 
-output_and_verify(Config, Filename, Options, Data) ->
+output_and_verify(Config, Options, Data) ->
     PortTest = port_test(Config),
-    Command = lists:concat([PortTest, " ",
-          		  Options, " -o", Filename]),
+    Command = lists:concat([PortTest, " " | Options]),
     Port = open_port({spawn, Command}, [out]),
     Port ! {self(), {command, Data}},
     Port ! {self(), close},
     receive
-        {Port, closed} -> ok
-    end,
-    Wait_time = 500,
-    test_server:sleep(Wait_time),
-    {ok, Written} = file:read_file(Filename),
-    Data = binary_to_list(Written),
-    ok.
+        {Port, closed} -> ok;
+        Msg -> ct:fail({received_unexpected_message, Msg})
+    end.
 
 %% Test that receiving several packages written in the same
 %% write operation works.
@@ -610,6 +617,38 @@ open_output_file_port(Config) when is_list(Config) ->
     test_server:timetrap_cancel(Dog),
     ok.
 
+%% Tests that all appropriate fd's have been closed in the port program
+count_fds(suite) -> [];
+count_fds(Config) when is_list(Config) ->
+    case os:type() of
+        {unix, _} ->
+            PrivDir = proplists:get_value(priv_dir, Config),
+            Filename = filename:join(PrivDir, "my_fd_counter"),
+
+            RunTest = fun(PortOpts) ->
+                              PortTest = port_test(Config),
+                              Command = lists:concat([PortTest, " -n -f -o", Filename]),
+                              Port = open_port({spawn, Command}, PortOpts),
+                              Port ! {self(), close},
+                              receive
+                                  {Port, closed} -> ok
+                              end,
+                              test_server:sleep(500),
+                              {ok, Written} = file:read_file(Filename),
+                              Written
+                      end,
+                    <<4:32/native>> = RunTest([out, nouse_stdio]),
+                    <<4:32/native>> = RunTest([in, nouse_stdio]),
+                    <<5:32/native>> = RunTest([in, out, nouse_stdio]),
+            <<3:32/native>> = RunTest([out, use_stdio]),
+            <<3:32/native>> = RunTest([in, use_stdio]),
+            <<3:32/native>> = RunTest([in, out, use_stdio]),
+            <<3:32/native>> = RunTest([in, out, use_stdio, stderr_to_stdout]),
+            <<3:32/native>> = RunTest([out, use_stdio, stderr_to_stdout]);
+        _ ->
+            {skip, "Skipped on windows"}
+    end.
+
 %%
 %% Open as many ports as possible. Do this several times and check
 %% that we get the same number of ports every time.
@@ -680,7 +719,16 @@ close_ports([]) ->
     ok.
 
 open_ports(Name, Settings) ->
-    test_server:sleep(5),
+    case os:type() of
+        {unix, freebsd} ->
+            %% FreeBsd has issues with sendmsg/recvmsg in fork
+            %% implementation and we therefor have to spawn
+            %% slower to make sure that we always hit the same
+            %% make roof.
+            test_server:sleep(10);
+        _ ->
+            test_server:sleep(5)
+    end,
     case catch open_port(Name, Settings) of
 	P when is_port(P) ->
 	    [P| open_ports(Name, Settings)];
@@ -923,6 +971,40 @@ try_bad_env(Env) ->
 	error:badarg -> ok
     end.
 
+%% Test that we can handle a very very large environment gracefully.
+huge_env(Config) when is_list(Config) ->
+    Vars = case os:type() of
+                {win32,_} -> 500;
+                _ ->
+                   %% We create a huge environment,
+                   %% 20000 variables is about 25MB
+                   %% which seems to be the limit on Linux.
+                   20000
+            end,
+    Env = [{[$a + I div (25*25*25*25) rem 25,
+              $a + I div (25*25*25) rem 25,
+              $a + I div (25*25) rem 25,
+              $a+I div 25 rem 25, $a+I rem 25],
+            lists:duplicate(100,$a+I rem 25)}
+           || I <- lists:seq(1,Vars)],
+    try erlang:open_port({spawn,"ls"},[exit_status, {env, Env}]) of
+        P ->
+            receive
+                {P, {exit_status,N}} = M ->
+                    %% We test that the exit status is an integer, this means
+                    %% that the child program has started. If we get an atom
+                    %% something went wrong in the driver which is not ok.
+                    ct:log("Got ~p",[M]),
+                    true = is_integer(N)
+            end
+    catch E:R ->
+            %% Have to catch the error here, as printing the stackdump
+            %% in the ct log is way to heavy for some test machines.
+            ct:fail("Open port failed ~p:~p",[E,R])
+    end.
+
+
+
 %% 'cd' option
 %% (Can perhaps be made smaller by calling the other utility functions
 %% in this module.)
@@ -1228,13 +1310,15 @@ otp_4389(Config)  when is_list(Config) ->
 							    {P,{exit_status,_}} ->
 								TCR ! {self(),ok};
 							    {'EXIT',_,{R2,_}} when R2 == emfile;
-										   R2 == eagain ->
+										   R2 == eagain;
+                                                                                   R2 == enomem ->
 								TCR ! {self(),ok};
 							    Err2 ->
 								TCR ! {self(),{msg,Err2}}
 							end;
 						    {'EXIT',{R1,_}} when R1 == emfile;
-									 R1 == eagain ->
+									 R1 == eagain;
+                                                                         R1 == enomem ->
 							TCR ! {self(),ok};
 						    Err1 ->
 							TCR ! {self(), {open_port,Err1}}
@@ -1840,10 +1924,12 @@ exit_status_msb_test(Config, SleepSecs) when is_list(Config) ->
 					      {Prt,
 					       erlang:system_info(scheduler_id)};
 					  {'EXIT', {Err, _}} when Err == eagain;
-								  Err == emfile ->
+								  Err == emfile;
+                                                                  Err == enomem ->
 					      noop;
 					  {'EXIT', Err} when Err == eagain;
-							     Err == emfile ->
+							     Err == emfile;
+                                                             Err == enomem ->
 					      noop;
 					  Error ->
 					      ?t:fail(Error)
diff --git a/erts/emulator/test/port_SUITE_data/port_test.c b/erts/emulator/test/port_SUITE_data/port_test.c
index 7abefab2e3..cc3ebdf0f8 100644
--- a/erts/emulator/test/port_SUITE_data/port_test.c
+++ b/erts/emulator/test/port_SUITE_data/port_test.c
@@ -13,6 +13,7 @@
 
 #ifndef __WIN32__
 #include <unistd.h>
+#include <limits.h>
 
 #include <sys/time.h>
 
@@ -48,6 +49,7 @@ typedef struct {
 				 * after reading the header for a packet
 				 * before reading the rest.
 				 */
+    int fd_count;               /* Count the number of open fds */
     int break_mode;		/* If set, this program will close standard
 				 * input, which should case broken pipe
 				 * error in the writer.
@@ -107,7 +109,7 @@ MAIN(argc, argv)
 int argc;
 char *argv[];
 {
-  int ret;
+  int ret, fd_count;
   if((port_data = (PORT_TEST_DATA *) malloc(sizeof(PORT_TEST_DATA))) == NULL) {
     fprintf(stderr, "Couldn't malloc for port_data");
     exit(1);
@@ -115,6 +117,7 @@ char *argv[];
   port_data->header_size = 0;
   port_data->io_buf_size = 0;	
   port_data->delay_mode = 0;	
+  port_data->fd_count = 0;
   port_data->break_mode = 0;	
   port_data->quit_mode = 0;	
   port_data->slow_writes = 0;	
@@ -144,6 +147,9 @@ char *argv[];
     case 'e': 
       port_data->fd_to_erl = 2;
       break;
+    case 'f':
+      port_data->fd_count = 1;
+      break;
     case 'h':			/* Header size for packets. */
       switch (argv[1][2]) {
       case '0': port_data->header_size = 0; break;
@@ -189,18 +195,31 @@ char *argv[];
     /* XXX Add error printout here */
   }
 
+  if (port_data->fd_count) {
+#ifdef __WIN32__
+      DWORD handles;
+      GetProcessHandleCount(GetCurrentProcess(), &handles);
+      fd_count = handles;
+#else
+      int i;
+      for (i = 0, fd_count = 0; i < 1024; i++)
+          if (fcntl(i, F_GETFD) >= 0) {
+              fd_count++;
+          }
+#endif
+  }
+
+  if (port_data->output_file)
+      replace_stdout(port_data->output_file);
+
+  if (port_data->fd_count)
+      reply(&fd_count, sizeof(fd_count));
+
   if (port_data->no_packet_loop){
       free(port_data);
       exit(0);
   }
 
-  /*
-   * If an output file was given, let it replace standard output.
-   */
-
-  if (port_data->output_file)
-    replace_stdout(port_data->output_file);
-
   ret = packet_loop();
   if(port_data->io_buf_size > 0)
       free(port_data->io_buf);
diff --git a/erts/emulator/test/z_SUITE.erl b/erts/emulator/test/z_SUITE.erl
index f4d9030255..abc353fb01 100644
--- a/erts/emulator/test/z_SUITE.erl
+++ b/erts/emulator/test/z_SUITE.erl
@@ -249,6 +249,7 @@ pollset_size(Config) when is_list(Config) ->
     ?line io:format("Initial: ~p~nFinal: ~p~n", [InitChkIo, FinChkIo]),
     ?line InitPollsetSize = lists:keysearch(total_poll_set_size, 1, InitChkIo),
     ?line FinPollsetSize = lists:keysearch(total_poll_set_size, 1, FinChkIo),
+    HasGethost = case has_gethost() of true -> 1; _ -> 0 end,
     ?line case InitPollsetSize =:= FinPollsetSize of
 	      true ->
 		  case InitPollsetSize of
@@ -269,7 +270,7 @@ pollset_size(Config) when is_list(Config) ->
 		      = InitPollsetSize,
 		  ?line {value, {total_poll_set_size, FinSize}}
 		      = FinPollsetSize,
-		  ?line true = FinSize < InitSize,
+		  ?line true = FinSize < (InitSize + HasGethost),
 		  ?line true = 2 =< FinSize,
 		  ?line {comment,
 			 "Start pollset size: "
@@ -289,16 +290,39 @@ check_io_debug(Config) when is_list(Config) ->
 	  end.
 
 check_io_debug_test() ->
-    ?line erlang:display(get_check_io_info()),
-    ?line erts_debug:set_internal_state(available_internal_state, true),
-    ?line {NoErrorFds, NoUsedFds, NoDrvSelStructs, NoDrvEvStructs}
+    erlang:display(get_check_io_info()),
+    erts_debug:set_internal_state(available_internal_state, true),
+    {NoErrorFds, NoUsedFds, NoDrvSelStructs, NoDrvEvStructs} = CheckIoDebug
 	= erts_debug:get_internal_state(check_io_debug),
-    ?line erts_debug:set_internal_state(available_internal_state, false),
-    ?line 0 = NoErrorFds,
-    ?line NoUsedFds = NoDrvSelStructs,
+    erts_debug:set_internal_state(available_internal_state, false),
+    HasGetHost = has_gethost(),
+    ct:log("check_io_debug: ~p~n"
+           "HasGetHost: ~p",[CheckIoDebug, HasGetHost]),
+    0 = NoErrorFds,
+    if
+        NoUsedFds == NoDrvSelStructs ->
+            ok;
+        HasGetHost andalso (NoUsedFds == (NoDrvSelStructs - 1)) ->
+            %% If the inet_gethost port is alive, we may have
+            %% one extra used fd that is not selected on.
+            %% This happens when the initial setup of the
+            %% port returns an EAGAIN
+            ok
+    end,
     ?line 0 = NoDrvEvStructs,
     ?line ok.
 
+has_gethost() ->
+    has_gethost(erlang:ports()).
+has_gethost([P|T]) ->
+    case erlang:port_info(P, name) of
+        {name,"inet_gethost"++_} ->
+            true;
+        _ ->
+            has_gethost(T)
+    end;
+has_gethost([]) ->
+    false.
 
 
 %%