Fix fd leak when using async thread pool

When using the async thread pool, if an erlang process
asks to open a file and it gets shutdown/killed while
the file:open/2 call hasn't returned, it's possible to
leak a file descriptor against the target file.

This happens because when the file driver is stopped
(file_stop() function is called), an async thread is
executing, about to execute, or executed already the
invoke_open() function. After file_stop() is called,
the file_async_ready() function will not run, and this
function is responsible for setting desc->fd with the
file descriptor that invoke_open() got. The file_stop()
call closes desc->fd if it refers to a valid file
descriptor, which is not the case here, because this
function was called before file_async_ready() could
run.

This leak is easily reproducile in a GNU/Linux system
using the following test code:

-module(t).
-export([t/1]).

t(N) ->
    Pid = spawn_link(fun() ->
       process_flag(trap_exit, true),
       loop(N)
    end),
    Ref = erlang:monitor(process, Pid),
    receive {'DOWN', Ref, _, _, _} ->
         ok
    end.

loop(0) ->
    ok;
loop(N) ->
    Name = integer_to_list(N),
    Server = self(),
    Pid = spawn(fun() ->
        Server ! continue,
        {ok, FdW} = file:open(Name, [raw, write]),
        {ok, FdR} = file:open(Name, [raw, read]),
        % Optional close calls, with or without them
        % it makes no difference.
        %ok = file:close(FdW),
        %ok = file:close(FdR),
        ok
    end),
    receive continue -> ok end,
    exit(Pid, shutdown),
    loop(N - 1).

Running this code with a few iterations is enough to very often
notice, with the lsof command, that the beam.smp process is holding
forever file descriptors open. This issue doesn't happen if the
async thread pool is not used.

Example:

$ erl +A 4
Erlang R15B03 (erts-5.9.3) [source] [64-bit] [smp:4:4] [async-threads:4] [hipe] [kernel-poll:false]

Eshell V5.9.3  (abort with ^G)
1> c(t).
{ok,t}
2> os:getpid().
"31975"
3> t:t(20).
ok

In a separate shell:

$ lsof -p 31975
COMMAND    PID     USER   FD   TYPE DEVICE     SIZE     NODE NAME
beam.smp 31975 fdmanana  cwd    DIR   8,18 22736896 32563204 /home/fdmanana/git/hub/otp/tmp
beam.smp 31975 fdmanana  rtd    DIR    8,1     4096        2 /
beam.smp 31975 fdmanana  txt    REG    8,1  7600263  1835126 /opt/r15b03/lib/erlang/erts-5.9.3/bin/beam.smp
beam.smp 31975 fdmanana  mem    REG    8,1  7220736  2497283 /usr/lib/locale/locale-archive
beam.smp 31975 fdmanana  mem    REG    8,1    10280  2505021 /usr/lib/libsctp.so.1.0.11
beam.smp 31975 fdmanana  mem    REG    8,1  1811128   917795 /lib/x86_64-linux-gnu/libc-2.15.so
beam.smp 31975 fdmanana  mem    REG    8,1    31752   917803 /lib/x86_64-linux-gnu/librt-2.15.so
beam.smp 31975 fdmanana  mem    REG    8,1   135366   917799 /lib/x86_64-linux-gnu/libpthread-2.15.so
beam.smp 31975 fdmanana  mem    REG    8,1   159200   921249 /lib/x86_64-linux-gnu/libtinfo.so.5.9
beam.smp 31975 fdmanana  mem    REG    8,1  1030512   917962 /lib/x86_64-linux-gnu/libm-2.15.so
beam.smp 31975 fdmanana  mem    REG    8,1    14768   917702 /lib/x86_64-linux-gnu/libdl-2.15.so
beam.smp 31975 fdmanana  mem    REG    8,1   149280   917974 /lib/x86_64-linux-gnu/ld-2.15.so
beam.smp 31975 fdmanana    0u   CHR  136,1                 4 /dev/pts/1
beam.smp 31975 fdmanana    1u   CHR  136,1                 4 /dev/pts/1
beam.smp 31975 fdmanana    2u   CHR  136,1                 4 /dev/pts/1
beam.smp 31975 fdmanana    3r  FIFO    0,8           1298297 pipe
beam.smp 31975 fdmanana    4w  FIFO    0,8           1298297 pipe
beam.smp 31975 fdmanana    5r  FIFO    0,8           1298298 pipe
beam.smp 31975 fdmanana    6w  FIFO    0,8           1298298 pipe
beam.smp 31975 fdmanana    7w   REG   8,18        0 32564173 /home/fdmanana/git/hub/otp/tmp/20
beam.smp 31975 fdmanana    8w   REG   8,18        0 32564176 /home/fdmanana/git/hub/otp/tmp/16
beam.smp 31975 fdmanana    9w   REG   8,18        0 32564177 /home/fdmanana/git/hub/otp/tmp/15
beam.smp 31975 fdmanana   10w   REG   8,18        0 32564179 /home/fdmanana/git/hub/otp/tmp/12
beam.smp 31975 fdmanana   11w   REG   8,18        0 32564180 /home/fdmanana/git/hub/otp/tmp/11
beam.smp 31975 fdmanana   12w   REG   8,18        0 32564205 /home/fdmanana/git/hub/otp/tmp/10
beam.smp 31975 fdmanana   13w   REG   8,18        0 32564182 /home/fdmanana/git/hub/otp/tmp/8
beam.smp 31975 fdmanana   14w   REG   8,18        0 32564183 /home/fdmanana/git/hub/otp/tmp/7
beam.smp 31975 fdmanana   15w   REG   8,18        0 32564186 /home/fdmanana/git/hub/otp/tmp/3

1 files changed, 18 insertions, 5 deletions

diff --git a/erts/emulator/drivers/common/efile_drv.c b/erts/emulator/drivers/common/efile_drv.c
index 912f5d3d8b..2ac7f169af 100644
--- a/erts/emulator/drivers/common/efile_drv.c
+++ b/erts/emulator/drivers/common/efile_drv.c
@@ -439,6 +439,7 @@ struct t_data
     Efile_error    errInfo;
     int            flags;
     SWord          fd;
+    int            is_fd_unused;
     /**/
     Efile_info        info;
     EFILE_DIR_HANDLE  dir_handle; /* Handle to open directory. */
@@ -781,11 +782,6 @@ file_start(ErlDrvPort port, char* command)
     return (ErlDrvData) desc;
 }
 
-static void free_data(void *data)
-{
-    EF_FREE(data);
-}
-
 static void do_close(int flags, SWord fd) {
     if (flags & EFILE_COMPRESSED) {
 	erts_gzclose((gzFile)(fd));
@@ -803,6 +799,17 @@ static void invoke_close(void *data)
     DTRACE_INVOKE_RETURN(FILE_CLOSE);
 }
 
+static void free_data(void *data)
+{
+    struct t_data *d = (struct t_data *) data;
+
+    if (d->command == FILE_OPEN && d->is_fd_unused && d->fd != FILE_FD_INVALID) {
+        do_close(d->flags, d->fd);
+    }
+
+    EF_FREE(data);
+}
+
 /*********************************************************************
  * Driver entry point -> stop
  */
@@ -1862,6 +1869,9 @@ static void invoke_open(void *data)
     }
 
     d->result_ok = status;
+    if (!status) {
+        d->fd = FILE_FD_INVALID;
+    }
     DTRACE_INVOKE_RETURN(FILE_OPEN);
 }
 
@@ -2373,8 +2383,10 @@ file_async_ready(ErlDrvData e, ErlDrvThreadData data)
 	if (!d->result_ok) {
 	    reply_error(desc, &d->errInfo);
 	} else {
+	    ASSERT(d->is_fd_unused);
 	    desc->fd = d->fd;
 	    desc->flags = d->flags;
+	    d->is_fd_unused = 0;
 	    reply_Uint(desc, d->fd);
 	}
 	free_data(data);
@@ -2745,6 +2757,7 @@ file_output(ErlDrvData e, char* buf, ErlDrvSizeT count)
 	    d->invoke = invoke_open;
 	    d->free = free_data;
 	    d->level = 2;
+	    d->is_fd_unused = 1;
 	    goto done;
 	}