erts/kernel: sendfile no longer uses async threads

This has been done because a slow client attack is possible if the
async thread pool is used. The scenario is:

Client does a request for a file and then slowly receives the file one
byte at a time. This will eventually fill the async thread pool with blocking
sendfile operations and thus starving the vm of all file operations.

If you still want to use the async threads pool for sendfile an option to
enable it has been introduced.

3 files changed, 69 insertions, 32 deletions

diff --git a/lib/kernel/doc/src/file.xml b/lib/kernel/doc/src/file.xml
index 0a4dd3ba47..e2c4aab27a 100644
--- a/lib/kernel/doc/src/file.xml
+++ b/lib/kernel/doc/src/file.xml
@@ -1746,16 +1746,16 @@
           <item>The chunk size used by the erlang fallback to send
 	  data. If using the fallback, this should be set to a value
 	  which comfortably fits in the systems memory. Default is 20 MB.</item>
+	  <tag><c>use_threads</c></tag>
+          <item>Instruct the emulator to use the async thread pool for the
+	  sendfile system call. This could be usefull if the OS you are running
+	  on does not properly support non-blocking sendfile calls. Do note that
+	  using async threads potentially makes your system volnerable to slow
+	  client attacks. If set to true and no async threads are available,
+	  the sendfile call will return <c>{error,einval}</c>.
+	  Introduced in 17.0. Default is false.</item>
 	</taglist>
 	</p>
-	<p>On operating systems with thread support, it is recommended to use
-	async threads. See the command line flag
-	<c>+A</c> in <seealso marker="erts:erl">erl(1)</seealso>. If it is not
-	possible to use async threads for sendfile, it is recommended to use
-	a relatively small value for the send buffer on the socket. Otherwise
-	the Erlang VM might loose some of its soft realtime guarantees.
-	Which size to use depends on the OS/hardware and the requirements
-	of the application.</p>
       </desc>
     </func>
     <func>
diff --git a/lib/kernel/src/file.erl b/lib/kernel/src/file.erl
index b5152018e0..23cf74f80f 100644
--- a/lib/kernel/src/file.erl
+++ b/lib/kernel/src/file.erl
@@ -111,7 +111,8 @@
 -type date_time() :: calendar:datetime().
 -type posix_file_advise() :: 'normal' | 'sequential' | 'random'
                            | 'no_reuse' | 'will_need' | 'dont_need'.
--type sendfile_option() :: {chunk_size, non_neg_integer()}.
+-type sendfile_option() :: {chunk_size, non_neg_integer()}
+			 | {use_threads, boolean()}.
 -type file_info_option() :: {'time', 'local'} | {'time', 'universal'} 
 			  | {'time', 'posix'}.
 %%% BIFs
@@ -1229,8 +1230,7 @@ change_time(Name, {{AY, AM, AD}, {AH, AMin, ASec}}=Atime,
 sendfile(File, _Sock, _Offet, _Bytes, _Opts) when is_pid(File) ->
     {error, badarg};
 sendfile(File, Sock, Offset, Bytes, []) ->
-    sendfile(File, Sock, Offset, Bytes, ?MAX_CHUNK_SIZE, [], [],
-	     false, false, false);
+    sendfile(File, Sock, Offset, Bytes, ?MAX_CHUNK_SIZE, [], [], []);
 sendfile(File, Sock, Offset, Bytes, Opts) ->
     ChunkSize0 = proplists:get_value(chunk_size, Opts, ?MAX_CHUNK_SIZE),
     ChunkSize = if ChunkSize0 > ?MAX_CHUNK_SIZE ->
@@ -1240,8 +1240,7 @@ sendfile(File, Sock, Offset, Bytes, Opts) ->
     %% Support for headers, trailers and options has been removed because the
     %% Darwin and BSD API for using it does not play nice with
     %% non-blocking sockets. See unix_efile.c for more info.
-    sendfile(File, Sock, Offset, Bytes, ChunkSize, [], [],
-	     false,false,false).
+    sendfile(File, Sock, Offset, Bytes, ChunkSize, [], [], Opts).
 
 %% sendfile/2
 -spec sendfile(Filename, Socket) ->
@@ -1261,17 +1260,17 @@ sendfile(Filename, Sock)  ->
 
 %% Internal sendfile functions
 sendfile(#file_descriptor{ module = Mod } = Fd, Sock, Offset, Bytes,
-	 ChunkSize, Headers, Trailers, Nodiskio, MNowait, Sync)
+	 ChunkSize, Headers, Trailers, Opts)
   when is_port(Sock) ->
     case Mod:sendfile(Fd, Sock, Offset, Bytes, ChunkSize, Headers, Trailers,
-		      Nodiskio, MNowait, Sync) of
+		      Opts) of
 	{error, enotsup} ->
 	    sendfile_fallback(Fd, Sock, Offset, Bytes, ChunkSize,
 			      Headers, Trailers);
 	Else ->
 	    Else
     end;
-sendfile(_,_,_,_,_,_,_,_,_,_) ->
+sendfile(_,_,_,_,_,_,_,_) ->
     {error, badarg}.
 
 %%%
diff --git a/lib/kernel/test/sendfile_SUITE.erl b/lib/kernel/test/sendfile_SUITE.erl
index 4cf4c6489d..1d26702c64 100644
--- a/lib/kernel/test/sendfile_SUITE.erl
+++ b/lib/kernel/test/sendfile_SUITE.erl
@@ -24,7 +24,14 @@
 
 -compile(export_all).
 
-all() ->
+all() -> [{group,async_threads},
+	  {group,no_async_threads}].
+
+groups() ->
+    [{async_threads,[],tcs()},
+     {no_async_threads,[],tcs()}].
+
+tcs() ->
     [t_sendfile_small
      ,t_sendfile_big_all
      ,t_sendfile_big_size
@@ -63,6 +70,14 @@ init_per_suite(Config) ->
 end_per_suite(Config) ->
     file:delete(proplists:get_value(big_file, Config)).
 
+init_per_group(async_threads,Config) ->
+    [{sendfile_opts,[{use_threads,true}]}|Config];
+init_per_group(no_async_threads,Config) ->
+    [{sendfile_opts,[{use_threads,false}]}|Config].
+
+end_per_group(_,_Config) ->
+    ok.
+
 init_per_testcase(TC,Config) when TC == t_sendfile_recvduring;
 				  TC == t_sendfile_sendduring ->
     Filename = proplists:get_value(small_file, Config),
@@ -71,7 +86,7 @@ init_per_testcase(TC,Config) when TC == t_sendfile_recvduring;
 		   {_Size, Data} = sendfile_file_info(Filename),
 		   {ok,D} = file:open(Filename, [raw,binary,read]),
 		   prim_file:sendfile(D, Sock, 0, 0, 0,
-				      [],[],false,false,false),
+				      [],[],[]),
 		   Data
 	   end,
 
@@ -92,6 +107,7 @@ t_sendfile_small(Config) when is_list(Config) ->
 
     Send = fun(Sock) ->
 		   {Size, Data} = sendfile_file_info(Filename),
+		   %% Here we make sure to test the sendfile/2 api
 		   {ok, Size} = file:sendfile(Filename, Sock),
 		   Data
 	   end,
@@ -101,6 +117,7 @@ t_sendfile_small(Config) when is_list(Config) ->
 t_sendfile_many_small(Config) when is_list(Config) ->
     Filename = proplists:get_value(small_file, Config),
     FileOpts = proplists:get_value(file_opts, Config, []),
+    SendfileOpts = proplists:get_value(sendfile_opts, Config),
 
     error_logger:add_report_handler(?MODULE,[self()]),
 
@@ -109,7 +126,7 @@ t_sendfile_many_small(Config) when is_list(Config) ->
 		   N = 10000,
 		   {ok,D} = file:open(Filename,[read|FileOpts]),
 		   [begin
-			{ok,Size} = file:sendfile(D,Sock,0,0,[])
+			{ok,Size} = file:sendfile(D,Sock,0,0,SendfileOpts)
 		    end || _I <- lists:seq(1,N)],
 		   file:close(D),
 		   Size*N
@@ -127,11 +144,12 @@ t_sendfile_many_small(Config) when is_list(Config) ->
 
 t_sendfile_big_all(Config) when is_list(Config) ->
     Filename = proplists:get_value(big_file, Config),
+    SendfileOpts = proplists:get_value(sendfile_opts, Config),
 
     Send = fun(Sock) ->
 		   {ok, #file_info{size = Size}} =
 		       file:read_file_info(Filename),
-		   {ok, Size} = file:sendfile(Filename, Sock),
+		   {ok, Size} = sendfile(Filename, Sock, SendfileOpts),
 		   Size
 	   end,
 
@@ -140,12 +158,13 @@ t_sendfile_big_all(Config) when is_list(Config) ->
 t_sendfile_big_size(Config) ->
     Filename = proplists:get_value(big_file, Config),
     FileOpts = proplists:get_value(file_opts, Config, []),
+    SendfileOpts = proplists:get_value(sendfile_opts, Config),
 
     SendAll = fun(Sock) ->
 		      {ok, #file_info{size = Size}} =
 			  file:read_file_info(Filename),
 		      {ok,D} = file:open(Filename,[read|FileOpts]),
-		      {ok, Size} = file:sendfile(D, Sock,0,Size,[]),
+		      {ok, Size} = file:sendfile(D, Sock,0,Size,SendfileOpts),
 		      Size
 	      end,
 
@@ -154,12 +173,13 @@ t_sendfile_big_size(Config) ->
 t_sendfile_partial(Config) ->
     Filename = proplists:get_value(small_file, Config),
     FileOpts = proplists:get_value(file_opts, Config, []),
+    SendfileOpts = proplists:get_value(sendfile_opts, Config),
 
     SendSingle = fun(Sock) ->
 			 {_Size, <<Data:5/binary,_/binary>>} =
 			     sendfile_file_info(Filename),
 			 {ok,D} = file:open(Filename,[read|FileOpts]),
-			 {ok,5} = file:sendfile(D,Sock,0,5,[]),
+			 {ok,5} = file:sendfile(D,Sock,0,5,SendfileOpts),
 			 file:close(D),
 			 Data
 		 end,
@@ -170,14 +190,14 @@ t_sendfile_partial(Config) ->
     {ok,D} = file:open(Filename,[read|FileOpts]),
     {ok, <<FData/binary>>} = file:read(D,5),
     FSend = fun(Sock) ->
-		    {ok,5} = file:sendfile(D,Sock,0,5,[]),
+		    {ok,5} = file:sendfile(D,Sock,0,5,SendfileOpts),
 		    FData
 	    end,
 
     ok = sendfile_send(FSend),
 
     SSend = fun(Sock) ->
-		    {ok,3} = file:sendfile(D,Sock,5,3,[]),
+		    {ok,3} = file:sendfile(D,Sock,5,3,SendfileOpts),
 		    SData
 	    end,
 
@@ -190,12 +210,13 @@ t_sendfile_partial(Config) ->
 t_sendfile_offset(Config) ->
     Filename = proplists:get_value(small_file, Config),
     FileOpts = proplists:get_value(file_opts, Config, []),
+    SendfileOpts = proplists:get_value(sendfile_opts, Config),
 
     Send = fun(Sock) ->
 		   {_Size, <<_:5/binary,Data:3/binary,_/binary>> = AllData} =
 		       sendfile_file_info(Filename),
 		   {ok,D} = file:open(Filename,[read|FileOpts]),
-		   {ok,3} = file:sendfile(D,Sock,5,3,[]),
+		   {ok,3} = file:sendfile(D,Sock,5,3,SendfileOpts),
 		   {ok, AllData} = file:read(D,100),
 		   file:close(D),
 		   Data
@@ -205,10 +226,11 @@ t_sendfile_offset(Config) ->
 
 t_sendfile_sendafter(Config) ->
     Filename = proplists:get_value(small_file, Config),
+    SendfileOpts = proplists:get_value(sendfile_opts, Config),
 
     Send = fun(Sock) ->
 		   {Size, Data} = sendfile_file_info(Filename),
-		   {ok, Size} = file:sendfile(Filename, Sock),
+		   {ok, Size} = sendfile(Filename, Sock, SendfileOpts),
 		   ok = gen_tcp:send(Sock, <<2>>),
 		   <<Data/binary,2>>
 	   end,
@@ -217,10 +239,11 @@ t_sendfile_sendafter(Config) ->
 
 t_sendfile_recvafter(Config) ->
     Filename = proplists:get_value(small_file, Config),
+    SendfileOpts = proplists:get_value(sendfile_opts, Config),
 
     Send = fun(Sock) ->
 		   {Size, Data} = sendfile_file_info(Filename),
-		   {ok, Size} = file:sendfile(Filename, Sock),
+		   {ok, Size} = sendfile(Filename, Sock, SendfileOpts),
 		   ok = gen_tcp:send(Sock, <<1>>),
 		   {ok,<<1>>} = gen_tcp:recv(Sock, 1),
 		   <<Data/binary,1>>
@@ -230,6 +253,7 @@ t_sendfile_recvafter(Config) ->
 
 t_sendfile_sendduring(Config) ->
     Filename = proplists:get_value(big_file, Config),
+    SendfileOpts = proplists:get_value(sendfile_opts, Config),
 
     Send = fun(Sock) ->
 		   {ok, #file_info{size = Size}} =
@@ -238,7 +262,7 @@ t_sendfile_sendduring(Config) ->
 				      timer:sleep(50),
 				      ok = gen_tcp:send(Sock, <<2>>)
 			      end),
-		   {ok, Size} = file:sendfile(Filename, Sock),
+		   {ok, Size} = sendfile(Filename, Sock, SendfileOpts),
 		   Size+1
 	   end,
 
@@ -246,6 +270,7 @@ t_sendfile_sendduring(Config) ->
 
 t_sendfile_recvduring(Config) ->
     Filename = proplists:get_value(big_file, Config),
+    SendfileOpts = proplists:get_value(sendfile_opts, Config),
 
     Send = fun(Sock) ->
 		   {ok, #file_info{size = Size}} =
@@ -255,7 +280,7 @@ t_sendfile_recvduring(Config) ->
 				      ok = gen_tcp:send(Sock, <<1>>),
 				      {ok,<<1>>} = gen_tcp:recv(Sock, 1)
 			      end),
-		   {ok, Size} = file:sendfile(Filename, Sock),
+		   {ok, Size} = sendfile(Filename, Sock, SendfileOpts),
 		   timer:sleep(1000),
 		   Size+1
 	   end,
@@ -264,6 +289,7 @@ t_sendfile_recvduring(Config) ->
 
 t_sendfile_closeduring(Config) ->
     Filename = proplists:get_value(big_file, Config),
+    SendfileOpts = proplists:get_value(sendfile_opts, Config),
 
     Send = fun(Sock,SFServPid) ->
 		   spawn_link(fun() ->
@@ -272,13 +298,14 @@ t_sendfile_closeduring(Config) ->
 			      end),
 		   case erlang:system_info(thread_pool_size) of
 		       0 ->
-			   {error, closed} = file:sendfile(Filename, Sock);
+			   {error, closed} = sendfile(Filename, Sock,
+						      SendfileOpts);
 		       _Else ->
 			   %% This can return how much has been sent or
 			   %% {error,closed} depending on OS.
 			   %% How much is sent impossible to know as
 			   %%  the socket was closed mid sendfile
-			   case file:sendfile(Filename, Sock) of
+			   case sendfile(Filename, Sock, SendfileOpts) of
 			       {error, closed} ->
 				   ok;
 			       {ok, Size}  when is_integer(Size) ->
@@ -292,6 +319,7 @@ t_sendfile_closeduring(Config) ->
 
 t_sendfile_crashduring(Config) ->
     Filename = proplists:get_value(big_file, Config),
+    SendfileOpts = proplists:get_value(sendfile_opts, Config),
 
     error_logger:add_report_handler(?MODULE,[self()]),
 
@@ -300,7 +328,7 @@ t_sendfile_crashduring(Config) ->
 				      timer:sleep(50),
 				      exit(die)
 			      end),
-		   {error, closed} = file:sendfile(Filename, Sock),
+		   {error, closed} = sendfile(Filename, Sock, SendfileOpts),
 		   -1
 	   end,
     process_flag(trap_exit,true),
@@ -395,6 +423,16 @@ sendfile_file_info(File) ->
     {ok, Data} = file:read_file(File),
     {Size, Data}.
 
+sendfile(Filename,Sock,Opts) ->
+    case file:open(Filename, [read, raw, binary]) of
+	{error, Reason} ->
+	    {error, Reason};
+	{ok, Fd} ->
+	    Res = file:sendfile(Fd, Sock, 0, 0, Opts),
+	    _ = file:close(Fd),
+	    Res
+    end.
+
 
 %% Error handler