Merge branch 'lukas/erts/sendfile_no_async_pool/OTP-11639'

* lukas/erts/sendfile_no_async_pool/OTP-11639:
  erts/kernel: sendfile no longer uses async threads

6 files changed, 105 insertions, 46 deletions

diff --git a/erts/emulator/drivers/common/efile_drv.c b/erts/emulator/drivers/common/efile_drv.c
index dca979c13a..e040864d24 100644
--- a/erts/emulator/drivers/common/efile_drv.c
+++ b/erts/emulator/drivers/common/efile_drv.c
@@ -167,7 +167,7 @@ dt_private *get_dt_private(int);
 
 
 #ifdef USE_THREADS
-#define IF_THRDS if (sys_info.async_threads > 0)
+#define THRDS_AVAILABLE (sys_info.async_threads > 0)
 #ifdef HARDDEBUG /* HARDDEBUG in io.c is expected too */
 #define TRACE_DRIVER fprintf(stderr, "Efile: ")
 #else
@@ -177,24 +177,26 @@ dt_private *get_dt_private(int);
 #define MUTEX_LOCK(m)    do { IF_THRDS { TRACE_DRIVER; driver_pdl_lock(m);   } } while (0)
 #define MUTEX_UNLOCK(m)  do { IF_THRDS { TRACE_DRIVER; driver_pdl_unlock(m); } } while (0)
 #else
-#define IF_THRDS if (0)
+#define THRDS_AVAILABLE (0)
 #define MUTEX_INIT(m, p)
 #define MUTEX_LOCK(m)
 #define MUTEX_UNLOCK(m)
 #endif
+#define IF_THRDS if (THRDS_AVAILABLE)
 
 
+#define SENDFILE_FLGS_USE_THREADS (1 << 0)
 /**
  * On DARWIN sendfile can deadlock with close if called in
  * different threads. So until Apple fixes so that sendfile
  * is not buggy we disable usage of the async pool for
  * DARWIN. The testcase t_sendfile_crashduring reproduces
- * this error when using +A 10.
+ * this error when using +A 10 and enabling SENDFILE_FLGS_USE_THREADS.
  */
 #if defined(__APPLE__) && defined(__MACH__)
-#define USE_THRDS_FOR_SENDFILE 0
+#define USE_THRDS_FOR_SENDFILE(DATA) 0
 #else
-#define USE_THRDS_FOR_SENDFILE (sys_info.async_threads > 0)
+#define USE_THRDS_FOR_SENDFILE(DATA) (DATA->flags & SENDFILE_FLGS_USE_THREADS)
 #endif /* defined(__APPLE__) && defined(__MACH__) */
 
 
@@ -300,7 +302,7 @@ static void file_stop_select(ErlDrvEvent event, void* _);
 enum e_timer {timer_idle, timer_again, timer_write};
 #ifdef HAVE_SENDFILE
 enum e_sendfile {sending, not_sending};
-static void free_sendfile(void *data);
+#define SENDFILE_USE_THREADS (1 << 0)
 #endif /* HAVE_SENDFILE */
 
 struct t_data;
@@ -1932,7 +1934,7 @@ static void invoke_sendfile(void *data)
 
     d->c.sendfile.written += nbytes;
 
- if (result == 1 || (result == 0 && USE_THRDS_FOR_SENDFILE)) {
+    if (result == 1 || (result == 0 && USE_THRDS_FOR_SENDFILE(d))) {
       d->result_ok = 0;
     } else if (result == 0 && (d->errInfo.posix_errno == EAGAIN
 				 || d->errInfo.posix_errno == EINTR)) {
@@ -1949,7 +1951,7 @@ static void invoke_sendfile(void *data)
 
 static void free_sendfile(void *data) {
     struct t_data *d = (struct t_data *)data;
-    if (USE_THRDS_FOR_SENDFILE) {
+    if (USE_THRDS_FOR_SENDFILE(d)) {
 	SET_NONBLOCKING(d->c.sendfile.out_fd);
     } else {
 	MUTEX_LOCK(d->c.sendfile.q_mtx);
@@ -4122,8 +4124,16 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) {
 	    goto done;
 	}
 
-	if (hd_len != 0 || tl_len != 0 || flags != 0) {
-	    /* We do not allow header, trailers and/or flags right now */
+	if (hd_len != 0 || tl_len != 0) {
+	    /* We do not allow header, trailers */
+	    reply_posix_error(desc, EINVAL);
+	    goto done;
+	}
+
+	
+	if (flags & SENDFILE_FLGS_USE_THREADS && !THRDS_AVAILABLE) {
+	    /* We do not allow use_threads flag on a system where
+	       no threads are available. */
 	    reply_posix_error(desc, EINVAL);
 	    goto done;
 	}
@@ -4133,6 +4143,7 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) {
 	d->command = command;
 	d->invoke = invoke_sendfile;
 	d->free = free_sendfile;
+	d->flags = flags;
 	d->level = 2;
 
 	d->c.sendfile.out_fd = (int) out_fd;
@@ -4152,7 +4163,7 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) {
 
 	d->c.sendfile.nbytes = nbytes;
 
-	if (USE_THRDS_FOR_SENDFILE) {
+	if (USE_THRDS_FOR_SENDFILE(d)) {
 	    SET_BLOCKING(d->c.sendfile.out_fd);
 	} else {
 	    /**
diff --git a/erts/preloaded/ebin/prim_file.beam b/erts/preloaded/ebin/prim_file.beam
index d5a1ec766d..e40e33b332 100644
--- a/erts/preloaded/ebin/prim_file.beam
+++ b/erts/preloaded/ebin/prim_file.beam
diff --git a/erts/preloaded/src/prim_file.erl b/erts/preloaded/src/prim_file.erl
index 5999e98340..34679404a2 100644
--- a/erts/preloaded/src/prim_file.erl
+++ b/erts/preloaded/src/prim_file.erl
@@ -27,7 +27,7 @@
 %% Generic file contents operations
 -export([open/2, close/1, datasync/1, sync/1, advise/4, position/2, truncate/1,
 	 write/2, pwrite/2, pwrite/3, read/2, read_line/1, pread/2, pread/3,
-	 copy/3, sendfile/10, allocate/3]).
+	 copy/3, sendfile/8, allocate/3]).
 
 %% Specialized file operations
 -export([open/1, open/3]).
@@ -149,6 +149,9 @@
 -define(POSIX_FADV_DONTNEED,   4).
 -define(POSIX_FADV_NOREUSE,    5).
 
+%% Sendfile flags
+-define(EFILE_SENDFILE_USE_THREADS, 1).
+
 
 %%% BIFs
 
@@ -582,13 +585,14 @@ write_file(_, _) ->
 %    {error, enotsup};
 sendfile(#file_descriptor{module = ?MODULE, data = {Port, _}},
 	 Dest, Offset, Bytes, _ChunkSize, Headers, Trailers,
-	 _Nodiskio, _MNowait, _Sync) ->
+	 Flags) ->
     case erlang:port_get_data(Dest) of
 	Data when Data == inet_tcp; Data == inet6_tcp ->
 	    ok = inet:lock_socket(Dest,true),
 	    {ok, DestFD} = prim_inet:getfd(Dest),
+	    IntFlags = translate_sendfile_flags(Flags),
 	    try drv_command(Port, [<<?FILE_SENDFILE, DestFD:32,
-				     0:8,
+				     IntFlags:8,
 				     Offset:64/unsigned,
 				     Bytes:64/unsigned,
 				     (iolist_size(Headers)):32/unsigned,
@@ -601,6 +605,13 @@ sendfile(#file_descriptor{module = ?MODULE, data = {Port, _}},
 	    {error,badarg}
     end.
 
+translate_sendfile_flags([{use_threads,true}|T]) ->
+    ?EFILE_SENDFILE_USE_THREADS bor translate_sendfile_flags(T);
+translate_sendfile_flags([_|T]) ->
+    translate_sendfile_flags(T);
+translate_sendfile_flags([]) ->
+    0.
+
 
 %%%-----------------------------------------------------------------
 %%% Functions operating on files without handle to the file. ?DRV.
diff --git a/lib/kernel/doc/src/file.xml b/lib/kernel/doc/src/file.xml
index 0a4dd3ba47..e2c4aab27a 100644
--- a/lib/kernel/doc/src/file.xml
+++ b/lib/kernel/doc/src/file.xml
@@ -1746,16 +1746,16 @@
           <item>The chunk size used by the erlang fallback to send
 	  data. If using the fallback, this should be set to a value
 	  which comfortably fits in the systems memory. Default is 20 MB.</item>
+	  <tag><c>use_threads</c></tag>
+          <item>Instruct the emulator to use the async thread pool for the
+	  sendfile system call. This could be usefull if the OS you are running
+	  on does not properly support non-blocking sendfile calls. Do note that
+	  using async threads potentially makes your system volnerable to slow
+	  client attacks. If set to true and no async threads are available,
+	  the sendfile call will return <c>{error,einval}</c>.
+	  Introduced in 17.0. Default is false.</item>
 	</taglist>
 	</p>
-	<p>On operating systems with thread support, it is recommended to use
-	async threads. See the command line flag
-	<c>+A</c> in <seealso marker="erts:erl">erl(1)</seealso>. If it is not
-	possible to use async threads for sendfile, it is recommended to use
-	a relatively small value for the send buffer on the socket. Otherwise
-	the Erlang VM might loose some of its soft realtime guarantees.
-	Which size to use depends on the OS/hardware and the requirements
-	of the application.</p>
       </desc>
     </func>
     <func>
diff --git a/lib/kernel/src/file.erl b/lib/kernel/src/file.erl
index b5152018e0..23cf74f80f 100644
--- a/lib/kernel/src/file.erl
+++ b/lib/kernel/src/file.erl
@@ -111,7 +111,8 @@
 -type date_time() :: calendar:datetime().
 -type posix_file_advise() :: 'normal' | 'sequential' | 'random'
                            | 'no_reuse' | 'will_need' | 'dont_need'.
--type sendfile_option() :: {chunk_size, non_neg_integer()}.
+-type sendfile_option() :: {chunk_size, non_neg_integer()}
+			 | {use_threads, boolean()}.
 -type file_info_option() :: {'time', 'local'} | {'time', 'universal'} 
 			  | {'time', 'posix'}.
 %%% BIFs
@@ -1229,8 +1230,7 @@ change_time(Name, {{AY, AM, AD}, {AH, AMin, ASec}}=Atime,
 sendfile(File, _Sock, _Offet, _Bytes, _Opts) when is_pid(File) ->
     {error, badarg};
 sendfile(File, Sock, Offset, Bytes, []) ->
-    sendfile(File, Sock, Offset, Bytes, ?MAX_CHUNK_SIZE, [], [],
-	     false, false, false);
+    sendfile(File, Sock, Offset, Bytes, ?MAX_CHUNK_SIZE, [], [], []);
 sendfile(File, Sock, Offset, Bytes, Opts) ->
     ChunkSize0 = proplists:get_value(chunk_size, Opts, ?MAX_CHUNK_SIZE),
     ChunkSize = if ChunkSize0 > ?MAX_CHUNK_SIZE ->
@@ -1240,8 +1240,7 @@ sendfile(File, Sock, Offset, Bytes, Opts) ->
     %% Support for headers, trailers and options has been removed because the
     %% Darwin and BSD API for using it does not play nice with
     %% non-blocking sockets. See unix_efile.c for more info.
-    sendfile(File, Sock, Offset, Bytes, ChunkSize, [], [],
-	     false,false,false).
+    sendfile(File, Sock, Offset, Bytes, ChunkSize, [], [], Opts).
 
 %% sendfile/2
 -spec sendfile(Filename, Socket) ->
@@ -1261,17 +1260,17 @@ sendfile(Filename, Sock)  ->
 
 %% Internal sendfile functions
 sendfile(#file_descriptor{ module = Mod } = Fd, Sock, Offset, Bytes,
-	 ChunkSize, Headers, Trailers, Nodiskio, MNowait, Sync)
+	 ChunkSize, Headers, Trailers, Opts)
   when is_port(Sock) ->
     case Mod:sendfile(Fd, Sock, Offset, Bytes, ChunkSize, Headers, Trailers,
-		      Nodiskio, MNowait, Sync) of
+		      Opts) of
 	{error, enotsup} ->
 	    sendfile_fallback(Fd, Sock, Offset, Bytes, ChunkSize,
 			      Headers, Trailers);
 	Else ->
 	    Else
     end;
-sendfile(_,_,_,_,_,_,_,_,_,_) ->
+sendfile(_,_,_,_,_,_,_,_) ->
     {error, badarg}.
 
 %%%
diff --git a/lib/kernel/test/sendfile_SUITE.erl b/lib/kernel/test/sendfile_SUITE.erl
index 24884bada5..2c741232c4 100644
--- a/lib/kernel/test/sendfile_SUITE.erl
+++ b/lib/kernel/test/sendfile_SUITE.erl
@@ -24,7 +24,14 @@
 
 -compile(export_all).
 
-all() ->
+all() -> [{group,async_threads},
+	  {group,no_async_threads}].
+
+groups() ->
+    [{async_threads,[],tcs()},
+     {no_async_threads,[],tcs()}].
+
+tcs() ->
     [t_sendfile_small
      ,t_sendfile_big_all
      ,t_sendfile_big_size
@@ -64,6 +71,14 @@ init_per_suite(Config) ->
 end_per_suite(Config) ->
     file:delete(proplists:get_value(big_file, Config)).
 
+init_per_group(async_threads,Config) ->
+    [{sendfile_opts,[{use_threads,true}]}|Config];
+init_per_group(no_async_threads,Config) ->
+    [{sendfile_opts,[{use_threads,false}]}|Config].
+
+end_per_group(_,_Config) ->
+    ok.
+
 init_per_testcase(TC,Config) when TC == t_sendfile_recvduring;
 				  TC == t_sendfile_sendduring ->
     Filename = proplists:get_value(small_file, Config),
@@ -72,7 +87,7 @@ init_per_testcase(TC,Config) when TC == t_sendfile_recvduring;
 		   {_Size, Data} = sendfile_file_info(Filename),
 		   {ok,D} = file:open(Filename, [raw,binary,read]),
 		   prim_file:sendfile(D, Sock, 0, 0, 0,
-				      [],[],false,false,false),
+				      [],[],[]),
 		   Data
 	   end,
 
@@ -93,6 +108,7 @@ t_sendfile_small(Config) when is_list(Config) ->
 
     Send = fun(Sock) ->
 		   {Size, Data} = sendfile_file_info(Filename),
+		   %% Here we make sure to test the sendfile/2 api
 		   {ok, Size} = file:sendfile(Filename, Sock),
 		   Data
 	   end,
@@ -102,6 +118,7 @@ t_sendfile_small(Config) when is_list(Config) ->
 t_sendfile_many_small(Config) when is_list(Config) ->
     Filename = proplists:get_value(small_file, Config),
     FileOpts = proplists:get_value(file_opts, Config, []),
+    SendfileOpts = proplists:get_value(sendfile_opts, Config),
 
     error_logger:add_report_handler(?MODULE,[self()]),
 
@@ -110,7 +127,7 @@ t_sendfile_many_small(Config) when is_list(Config) ->
 		   N = 10000,
 		   {ok,D} = file:open(Filename,[read|FileOpts]),
 		   [begin
-			{ok,Size} = file:sendfile(D,Sock,0,0,[])
+			{ok,Size} = file:sendfile(D,Sock,0,0,SendfileOpts)
 		    end || _I <- lists:seq(1,N)],
 		   file:close(D),
 		   Size*N
@@ -128,11 +145,12 @@ t_sendfile_many_small(Config) when is_list(Config) ->
 
 t_sendfile_big_all(Config) when is_list(Config) ->
     Filename = proplists:get_value(big_file, Config),
+    SendfileOpts = proplists:get_value(sendfile_opts, Config),
 
     Send = fun(Sock) ->
 		   {ok, #file_info{size = Size}} =
 		       file:read_file_info(Filename),
-		   {ok, Size} = file:sendfile(Filename, Sock),
+		   {ok, Size} = sendfile(Filename, Sock, SendfileOpts),
 		   Size
 	   end,
 
@@ -141,12 +159,13 @@ t_sendfile_big_all(Config) when is_list(Config) ->
 t_sendfile_big_size(Config) ->
     Filename = proplists:get_value(big_file, Config),
     FileOpts = proplists:get_value(file_opts, Config, []),
+    SendfileOpts = proplists:get_value(sendfile_opts, Config),
 
     SendAll = fun(Sock) ->
 		      {ok, #file_info{size = Size}} =
 			  file:read_file_info(Filename),
 		      {ok,D} = file:open(Filename,[read|FileOpts]),
-		      {ok, Size} = file:sendfile(D, Sock,0,Size,[]),
+		      {ok, Size} = file:sendfile(D, Sock,0,Size,SendfileOpts),
 		      Size
 	      end,
 
@@ -155,12 +174,13 @@ t_sendfile_big_size(Config) ->
 t_sendfile_partial(Config) ->
     Filename = proplists:get_value(small_file, Config),
     FileOpts = proplists:get_value(file_opts, Config, []),
+    SendfileOpts = proplists:get_value(sendfile_opts, Config),
 
     SendSingle = fun(Sock) ->
 			 {_Size, <<Data:5/binary,_/binary>>} =
 			     sendfile_file_info(Filename),
 			 {ok,D} = file:open(Filename,[read|FileOpts]),
-			 {ok,5} = file:sendfile(D,Sock,0,5,[]),
+			 {ok,5} = file:sendfile(D,Sock,0,5,SendfileOpts),
 			 file:close(D),
 			 Data
 		 end,
@@ -171,14 +191,14 @@ t_sendfile_partial(Config) ->
     {ok,D} = file:open(Filename,[read|FileOpts]),
     {ok, <<FData/binary>>} = file:read(D,5),
     FSend = fun(Sock) ->
-		    {ok,5} = file:sendfile(D,Sock,0,5,[]),
+		    {ok,5} = file:sendfile(D,Sock,0,5,SendfileOpts),
 		    FData
 	    end,
 
     ok = sendfile_send(FSend),
 
     SSend = fun(Sock) ->
-		    {ok,3} = file:sendfile(D,Sock,5,3,[]),
+		    {ok,3} = file:sendfile(D,Sock,5,3,SendfileOpts),
 		    SData
 	    end,
 
@@ -191,12 +211,13 @@ t_sendfile_partial(Config) ->
 t_sendfile_offset(Config) ->
     Filename = proplists:get_value(small_file, Config),
     FileOpts = proplists:get_value(file_opts, Config, []),
+    SendfileOpts = proplists:get_value(sendfile_opts, Config),
 
     Send = fun(Sock) ->
 		   {_Size, <<_:5/binary,Data:3/binary,_/binary>> = AllData} =
 		       sendfile_file_info(Filename),
 		   {ok,D} = file:open(Filename,[read|FileOpts]),
-		   {ok,3} = file:sendfile(D,Sock,5,3,[]),
+		   {ok,3} = file:sendfile(D,Sock,5,3,SendfileOpts),
 		   {ok, AllData} = file:read(D,100),
 		   file:close(D),
 		   Data
@@ -206,10 +227,11 @@ t_sendfile_offset(Config) ->
 
 t_sendfile_sendafter(Config) ->
     Filename = proplists:get_value(small_file, Config),
+    SendfileOpts = proplists:get_value(sendfile_opts, Config),
 
     Send = fun(Sock) ->
 		   {Size, Data} = sendfile_file_info(Filename),
-		   {ok, Size} = file:sendfile(Filename, Sock),
+		   {ok, Size} = sendfile(Filename, Sock, SendfileOpts),
 		   ok = gen_tcp:send(Sock, <<2>>),
 		   <<Data/binary,2>>
 	   end,
@@ -218,10 +240,11 @@ t_sendfile_sendafter(Config) ->
 
 t_sendfile_recvafter(Config) ->
     Filename = proplists:get_value(small_file, Config),
+    SendfileOpts = proplists:get_value(sendfile_opts, Config),
 
     Send = fun(Sock) ->
 		   {Size, Data} = sendfile_file_info(Filename),
-		   {ok, Size} = file:sendfile(Filename, Sock),
+		   {ok, Size} = sendfile(Filename, Sock, SendfileOpts),
 		   ok = gen_tcp:send(Sock, <<1>>),
 		   {ok,<<1>>} = gen_tcp:recv(Sock, 1),
 		   <<Data/binary,1>>
@@ -250,6 +273,7 @@ t_sendfile_recvafter_remoteclose(Config) ->
 
 t_sendfile_sendduring(Config) ->
     Filename = proplists:get_value(big_file, Config),
+    SendfileOpts = proplists:get_value(sendfile_opts, Config),
 
     Send = fun(Sock) ->
 		   {ok, #file_info{size = Size}} =
@@ -258,7 +282,7 @@ t_sendfile_sendduring(Config) ->
 				      timer:sleep(50),
 				      ok = gen_tcp:send(Sock, <<2>>)
 			      end),
-		   {ok, Size} = file:sendfile(Filename, Sock),
+		   {ok, Size} = sendfile(Filename, Sock, SendfileOpts),
 		   Size+1
 	   end,
 
@@ -266,6 +290,7 @@ t_sendfile_sendduring(Config) ->
 
 t_sendfile_recvduring(Config) ->
     Filename = proplists:get_value(big_file, Config),
+    SendfileOpts = proplists:get_value(sendfile_opts, Config),
 
     Send = fun(Sock) ->
 		   {ok, #file_info{size = Size}} =
@@ -275,7 +300,7 @@ t_sendfile_recvduring(Config) ->
 				      ok = gen_tcp:send(Sock, <<1>>),
 				      {ok,<<1>>} = gen_tcp:recv(Sock, 1)
 			      end),
-		   {ok, Size} = file:sendfile(Filename, Sock),
+		   {ok, Size} = sendfile(Filename, Sock, SendfileOpts),
 		   timer:sleep(1000),
 		   Size+1
 	   end,
@@ -284,6 +309,7 @@ t_sendfile_recvduring(Config) ->
 
 t_sendfile_closeduring(Config) ->
     Filename = proplists:get_value(big_file, Config),
+    SendfileOpts = proplists:get_value(sendfile_opts, Config),
 
     Send = fun(Sock,SFServPid) ->
 		   spawn_link(fun() ->
@@ -292,13 +318,14 @@ t_sendfile_closeduring(Config) ->
 			      end),
 		   case erlang:system_info(thread_pool_size) of
 		       0 ->
-			   {error, closed} = file:sendfile(Filename, Sock);
+			   {error, closed} = sendfile(Filename, Sock,
+						      SendfileOpts);
 		       _Else ->
 			   %% This can return how much has been sent or
 			   %% {error,closed} depending on OS.
 			   %% How much is sent impossible to know as
 			   %%  the socket was closed mid sendfile
-			   case file:sendfile(Filename, Sock) of
+			   case sendfile(Filename, Sock, SendfileOpts) of
 			       {error, closed} ->
 				   ok;
 			       {ok, Size}  when is_integer(Size) ->
@@ -312,6 +339,7 @@ t_sendfile_closeduring(Config) ->
 
 t_sendfile_crashduring(Config) ->
     Filename = proplists:get_value(big_file, Config),
+    SendfileOpts = proplists:get_value(sendfile_opts, Config),
 
     error_logger:add_report_handler(?MODULE,[self()]),
 
@@ -320,7 +348,7 @@ t_sendfile_crashduring(Config) ->
 				      timer:sleep(50),
 				      exit(die)
 			      end),
-		   {error, closed} = file:sendfile(Filename, Sock),
+		   {error, closed} = sendfile(Filename, Sock, SendfileOpts),
 		   -1
 	   end,
     process_flag(trap_exit,true),
@@ -415,6 +443,16 @@ sendfile_file_info(File) ->
     {ok, Data} = file:read_file(File),
     {Size, Data}.
 
+sendfile(Filename,Sock,Opts) ->
+    case file:open(Filename, [read, raw, binary]) of
+	{error, Reason} ->
+	    {error, Reason};
+	{ok, Fd} ->
+	    Res = file:sendfile(Fd, Sock, 0, 0, Opts),
+	    _ = file:close(Fd),
+	    Res
+    end.
+
 
 %% Error handler