From 250a556b1e3eb8486ec294f94d3b918c9ac91542 Mon Sep 17 00:00:00 2001 From: Lukas Larsson Date: Fri, 2 Dec 2011 17:13:49 +0100 Subject: Make solaris use sendfilev sendfilev is a richer API which allows us to do non blocking TCP on solaris. The normal sendfile API seems to have some issue with non blocking sockets and the return value of sendfile. --- erts/configure.in | 4 +++- erts/emulator/drivers/common/erl_efile.h | 5 +++++ erts/emulator/drivers/unix/unix_efile.c | 24 +++++++++++++++++++++++- 3 files changed, 31 insertions(+), 2 deletions(-) (limited to 'erts') diff --git a/erts/configure.in b/erts/configure.in index 548e4cc9d5..e937184f62 100644 --- a/erts/configure.in +++ b/erts/configure.in @@ -1697,7 +1697,9 @@ case $host_os in AC_CHECK_FUNCS([sendfile]) ;; solaris*) - AC_SEARCH_LIBS(sendfile, sendfile, AC_DEFINE(HAVE_SENDFILE, 1)) + AC_SEARCH_LIBS(sendfilev, sendfile, + AC_DEFINE([HAVE_SENDFILEV],[1], + [Define to 1 if you have the `sendfilev' function.])) ;; win32) LIBS="$LIBS -lmswsock" diff --git a/erts/emulator/drivers/common/erl_efile.h b/erts/emulator/drivers/common/erl_efile.h index 349ab0e17b..ed8910a8b8 100644 --- a/erts/emulator/drivers/common/erl_efile.h +++ b/erts/emulator/drivers/common/erl_efile.h @@ -67,6 +67,11 @@ #define FILENAMES_16BIT 1 #endif +// We use sendfilev if it exist on solaris +#if !defined(HAVE_SENDFILE) && defined(HAVE_SENDFILEV) +#define HAVE_SENDFILE +#endif + /* * An handle to an open directory. To be cast to the correct type * in the system-dependent directory functions. diff --git a/erts/emulator/drivers/unix/unix_efile.c b/erts/emulator/drivers/unix/unix_efile.c index 72911641d3..2b68334ad7 100644 --- a/erts/emulator/drivers/unix/unix_efile.c +++ b/erts/emulator/drivers/unix/unix_efile.c @@ -1488,7 +1488,7 @@ efile_sendfile(Efile_error* errInfo, int in_fd, int out_fd, off_t *offset, Uint64 *nbytes, struct t_sendfile_hdtl* hdtl) { Uint64 written = 0; -#if defined(__linux__) || (defined(__sun) && defined(__SVR4)) +#if defined(__linux__) ssize_t retval; do { // check if *nbytes is 0 or greater than the largest size_t @@ -1503,6 +1503,28 @@ efile_sendfile(Efile_error* errInfo, int in_fd, int out_fd, } while (retval != -1 && retval == SENDFILE_CHUNK_SIZE); *nbytes = written; return check_error(retval == -1 ? -1 : 0, errInfo); +#elif defined(__sun) && defined(__SVR4) && defined(HAVE_SENDFILEV) + ssize_t retval; + size_t len; + sendfilevec_t fdrec; + fdrec.sfv_fd = in_fd; + fdrec.sfv_flag = SFV_NOWAIT; + do { + fdrec.sfv_off = *offset; + len = 0; + if (*nbytes == 0 || *nbytes > SENDFILE_CHUNK_SIZE) + fdrec.sfv_len = SENDFILE_CHUNK_SIZE; + else + fdrec.sfv_len = *nbytes; + retval = sendfilev(out_fd, &fdrec, 1, &len); + if (retval != -1 || errno == EAGAIN || errno == EINTR) { + *offset += len; + *nbytes -= len; + written += len; + } + } while (len == SENDFILE_CHUNK_SIZE); + *nbytes = written; + return check_error(retval == -1 ? -1 : 0, errInfo); #elif defined(DARWIN) int retval; off_t len; -- cgit v1.2.3 From ec885e883c99825e7d737752b1b0aebbbafbd670 Mon Sep 17 00:00:00 2001 From: Lukas Larsson Date: Mon, 5 Dec 2011 18:45:44 +0100 Subject: Clarify some code comments Thanks Tuncer Ayaz --- erts/emulator/drivers/common/erl_efile.h | 2 +- erts/emulator/drivers/common/inet_drv.c | 6 +++--- erts/emulator/drivers/unix/unix_efile.c | 16 +++++++++++++--- 3 files changed, 17 insertions(+), 7 deletions(-) (limited to 'erts') diff --git a/erts/emulator/drivers/common/erl_efile.h b/erts/emulator/drivers/common/erl_efile.h index ed8910a8b8..0ed4a60efb 100644 --- a/erts/emulator/drivers/common/erl_efile.h +++ b/erts/emulator/drivers/common/erl_efile.h @@ -126,7 +126,7 @@ typedef struct _Efile_info { #ifdef HAVE_SENDFILE /* - * Described the structure of header/trailers for sendfile + * Describes the structure of headers/trailers for sendfile */ struct t_sendfile_hdtl { SysIOVec *headers; diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c index 56799555a9..abbd835605 100644 --- a/erts/emulator/drivers/common/inet_drv.c +++ b/erts/emulator/drivers/common/inet_drv.c @@ -872,9 +872,9 @@ typedef struct { double send_avg; /* average packet size sent */ subs_list empty_out_q_subs; /* Empty out queue subscribers */ - int is_ignored; /* if a fd is ignored by from the inet_drv, - this should be set to true when the fd is used - outside of inet_drv. */ + int is_ignored; /* if a fd is ignored by the inet_drv. + This flag should be set to true when + the fd is used outside of inet_drv. */ } inet_descriptor; diff --git a/erts/emulator/drivers/unix/unix_efile.c b/erts/emulator/drivers/unix/unix_efile.c index 2b68334ad7..92314fce36 100644 --- a/erts/emulator/drivers/unix/unix_efile.c +++ b/erts/emulator/drivers/unix/unix_efile.c @@ -1469,6 +1469,9 @@ efile_fadvise(Efile_error* errInfo, int fd, Sint64 offset, } #ifdef HAVE_SENDFILE + +// For some reason the maximum size_t cannot be used as the max size +// 3GB seems to work on all platforms #define SENDFILE_CHUNK_SIZE ((1 << 30) -1) /* @@ -1477,7 +1480,13 @@ efile_fadvise(Efile_error* errInfo, int fd, Sint64 offset, * we have to emulate some things in linux and play with variables on * bsd/darwin. * - * It could be possible to implement header/trailer in sendfile, though + * All of the calls will split a command which tries to send more than + * SENDFILE_CHUNK_SIZE of data at once. + * + * On platforms where *nbytes of 0 does not mean the entire file, this is + * simulated. + * + * It could be possible to implement header/trailer in sendfile. Though * you would have to emulate it in linux and on BSD/Darwin some complex * calculations have to be made when using a non blocking socket to figure * out how much of the header/file/trailer was sent in each command. @@ -1491,7 +1500,7 @@ efile_sendfile(Efile_error* errInfo, int in_fd, int out_fd, #if defined(__linux__) ssize_t retval; do { - // check if *nbytes is 0 or greater than the largest size_t + // check if *nbytes is 0 or greater than chunk size if (*nbytes == 0 || *nbytes > SENDFILE_CHUNK_SIZE) retval = sendfile(out_fd, in_fd, offset, SENDFILE_CHUNK_SIZE); else @@ -1512,6 +1521,7 @@ efile_sendfile(Efile_error* errInfo, int in_fd, int out_fd, do { fdrec.sfv_off = *offset; len = 0; + // check if *nbytes is 0 or greater than chunk size if (*nbytes == 0 || *nbytes > SENDFILE_CHUNK_SIZE) fdrec.sfv_len = SENDFILE_CHUNK_SIZE; else @@ -1529,7 +1539,7 @@ efile_sendfile(Efile_error* errInfo, int in_fd, int out_fd, int retval; off_t len; do { - // check if *nbytes is 0 or greater than the largest off_t + // check if *nbytes is 0 or greater than chunk size if(*nbytes > SENDFILE_CHUNK_SIZE) len = SENDFILE_CHUNK_SIZE; else -- cgit v1.2.3 From f34a89e0c0cc92226d5fe3fb257c62251068392b Mon Sep 17 00:00:00 2001 From: Lukas Larsson Date: Wed, 7 Dec 2011 10:46:22 +0100 Subject: Do not use SFV_NOWAIT as it does not exist on all solaris --- erts/emulator/drivers/unix/unix_efile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'erts') diff --git a/erts/emulator/drivers/unix/unix_efile.c b/erts/emulator/drivers/unix/unix_efile.c index 92314fce36..630c570f7c 100644 --- a/erts/emulator/drivers/unix/unix_efile.c +++ b/erts/emulator/drivers/unix/unix_efile.c @@ -1517,7 +1517,7 @@ efile_sendfile(Efile_error* errInfo, int in_fd, int out_fd, size_t len; sendfilevec_t fdrec; fdrec.sfv_fd = in_fd; - fdrec.sfv_flag = SFV_NOWAIT; + fdrec.sfv_flag = 0; do { fdrec.sfv_off = *offset; len = 0; -- cgit v1.2.3 From 1137c0a08b25ab5e38286260f0b7c51ba015afbb Mon Sep 17 00:00:00 2001 From: Lukas Larsson Date: Wed, 7 Dec 2011 14:46:26 +0100 Subject: Return {error,closed} from sendfile if closed If the socket is closed by the remote end, sendfile now returns {error,closed}. --- erts/emulator/drivers/common/efile_drv.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) (limited to 'erts') diff --git a/erts/emulator/drivers/common/efile_drv.c b/erts/emulator/drivers/common/efile_drv.c index 5c52b99348..32914d3b22 100644 --- a/erts/emulator/drivers/common/efile_drv.c +++ b/erts/emulator/drivers/common/efile_drv.c @@ -799,7 +799,16 @@ static void reply_Uint_posix_error(file_descriptor *desc, Uint num, driver_output2(desc->port, response, t-response, NULL, 0); } +static void reply_string_error(file_descriptor *desc, char* str) { + char response[256]; /* Response buffer. */ + char* s; + char* t; + response[0] = FILE_RESP_ERROR; + for (s = str, t = response+1; *s; s++, t++) + *t = tolower(*s); + driver_output2(desc->port, response, t-response, NULL, 0); +} static int reply_error(file_descriptor *desc, Efile_error *errInfo) /* The error codes. */ @@ -2208,7 +2217,12 @@ file_async_ready(ErlDrvData e, ErlDrvThreadData data) case FILE_SENDFILE: if (d->result_ok == -1) { desc->sendfile_state = not_sending; - reply_error(desc, &d->errInfo); + if (d->errInfo.posix_errno == ECONNRESET || + d->errInfo.posix_errno == ENOTCONN || + d->errInfo.posix_errno == EPIPE) + reply_string_error(desc,"closed"); + else + reply_error(desc, &d->errInfo); if (sys_info.async_threads != 0) { SET_NONBLOCKING(d->c.sendfile.out_fd); free_sendfile(data); -- cgit v1.2.3 From 4181dc36580a0ec7476aeed389948976a89c755d Mon Sep 17 00:00:00 2001 From: Lukas Larsson Date: Wed, 7 Dec 2011 14:47:40 +0100 Subject: Fix cleanup when sendfile process crashes When sendfile crashes during a send there should be no error_logger printouts about stolen fds. --- erts/emulator/drivers/common/efile_drv.c | 9 +++++++++ erts/emulator/drivers/common/inet_drv.c | 12 +++++++++--- 2 files changed, 18 insertions(+), 3 deletions(-) (limited to 'erts') diff --git a/erts/emulator/drivers/common/efile_drv.c b/erts/emulator/drivers/common/efile_drv.c index 32914d3b22..5943bca27d 100644 --- a/erts/emulator/drivers/common/efile_drv.c +++ b/erts/emulator/drivers/common/efile_drv.c @@ -734,6 +734,15 @@ file_stop(ErlDrvData e) TRACE_C('p'); +#ifdef HAVE_SENDFILE + if (desc->sendfile_state == sending && !USE_THRDS_FOR_SENDFILE) { + driver_select(desc->port,(ErlDrvEvent)(long)desc->d->c.sendfile.out_fd, + ERL_DRV_WRITE|ERL_DRV_USE,0); + } else if (desc->sendfile_state == sending) { + SET_NONBLOCKING(desc->d->c.sendfile.out_fd); + } +#endif /* HAVE_SENDFILE */ + if (desc->fd != FILE_FD_INVALID) { do_close(desc->flags, desc->fd); desc->fd = FILE_FD_INVALID; diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c index abbd835605..634fc8e8e0 100644 --- a/erts/emulator/drivers/common/inet_drv.c +++ b/erts/emulator/drivers/common/inet_drv.c @@ -445,7 +445,7 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n) driver_select(port, e, mode | (on?ERL_DRV_USE:0), on) #define sock_select(d, flags, onoff) do { \ - ASSERT(!onoff || !(d)->is_ignored); \ + ASSERT(!(d)->is_ignored); \ (d)->event_mask = (onoff) ? \ ((d)->event_mask | (flags)) : \ ((d)->event_mask & ~(flags)); \ @@ -3738,7 +3738,13 @@ static void desc_close(inet_descriptor* desc) desc->forced_events = 0; desc->send_would_block = 0; #endif - driver_select(desc->port, (ErlDrvEvent)(long)desc->event, ERL_DRV_USE, 0); + // We should close the fd here, but the other driver might still + // be selecting on it. + if (!desc->is_ignored) + driver_select(desc->port,(ErlDrvEvent)(long)desc->event, + ERL_DRV_USE, 0); + else + inet_stop_select((ErlDrvEvent)(long)desc->event,NULL); desc->event = INVALID_EVENT; /* closed by stop_select callback */ desc->s = INVALID_SOCKET; desc->event_mask = 0; @@ -7612,8 +7618,8 @@ static int inet_ctl(inet_descriptor* desc, int cmd, char* buf, int len, return ctl_error(EINVAL, rbuf, rsize); if (*buf == 1 && !desc->is_ignored) { - desc->is_ignored = INET_IGNORE_READ; sock_select(desc, (FD_READ|FD_WRITE|FD_CLOSE|ERL_DRV_USE_NO_CALLBACK), 0); + desc->is_ignored = INET_IGNORE_READ; } else if (*buf == 0 && desc->is_ignored) { int flags = (FD_READ|FD_CLOSE|((desc->is_ignored & INET_IGNORE_WRITE)?FD_WRITE:0)); desc->is_ignored = INET_IGNORE_NONE; -- cgit v1.2.3 From e406d3f5471913a971254a8420b7a8a429de3bf5 Mon Sep 17 00:00:00 2001 From: Lukas Larsson Date: Wed, 7 Dec 2011 17:55:56 +0100 Subject: Do not use async threads on DARWIN --- erts/emulator/drivers/common/efile_drv.c | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) (limited to 'erts') diff --git a/erts/emulator/drivers/common/efile_drv.c b/erts/emulator/drivers/common/efile_drv.c index 5943bca27d..5f5165bbda 100644 --- a/erts/emulator/drivers/common/efile_drv.c +++ b/erts/emulator/drivers/common/efile_drv.c @@ -104,6 +104,7 @@ #ifndef WANT_NONBLOCKING #define WANT_NONBLOCKING #endif + #include "sys.h" #include "erl_driver.h" @@ -147,6 +148,22 @@ static ErlDrvSysInfo sys_info; #define MUTEX_UNLOCK(m) #endif + +/** + * On DARWIN sendfile can deadlock with close if called in + * different threads. So until Apple fixes so that sendfile + * is not buggy we disable usage of the async pool for + * DARWIN. The testcase t_sendfile_crashduring reproduces + * this error when using +A 10. + */ +#if !defined(DARWIN) +#define USE_THRDS_FOR_SENDFILE (sys_info.async_threads > 0) +#else +#define USE_THRDS_FOR_SENDFILE 0 +#endif /* !DARWIN */ + + + #if 0 /* Experimental, for forcing all file operations to use the same thread. */ static unsigned file_fixed_key = 1; @@ -1762,7 +1779,7 @@ static void invoke_sendfile(void *data) d->c.sendfile.written += nbytes; if (result == 1) { - if (sys_info.async_threads != 0) { + if (USE_THRDS_FOR_SENDFILE) { d->result_ok = 0; } else if (d->c.sendfile.nbytes == 0 && nbytes != 0) { d->result_ok = 1; @@ -2232,7 +2249,7 @@ file_async_ready(ErlDrvData e, ErlDrvThreadData data) reply_string_error(desc,"closed"); else reply_error(desc, &d->errInfo); - if (sys_info.async_threads != 0) { + if (USE_THRDS_FOR_SENDFILE) { SET_NONBLOCKING(d->c.sendfile.out_fd); free_sendfile(data); } else { @@ -2243,7 +2260,7 @@ file_async_ready(ErlDrvData e, ErlDrvThreadData data) } else if (d->result_ok == 0) { desc->sendfile_state = not_sending; reply_Sint64(desc, d->c.sendfile.written); - if (sys_info.async_threads != 0) { + if (USE_THRDS_FOR_SENDFILE) { SET_NONBLOCKING(d->c.sendfile.out_fd); free_sendfile(data); } else { @@ -3448,7 +3465,7 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) { d->c.sendfile.nbytes = nbytes; - if (sys_info.async_threads != 0) { + if (USE_THRDS_FOR_SENDFILE) { SET_BLOCKING(d->c.sendfile.out_fd); } -- cgit v1.2.3