diff options
Diffstat (limited to 'erts')
34 files changed, 1407 insertions, 236 deletions
diff --git a/erts/configure.in b/erts/configure.in index c8b96c50f0..766e35fb2b 100644 --- a/erts/configure.in +++ b/erts/configure.in @@ -3988,7 +3988,7 @@ dnl If set to --with-ssl=PATH we use that path as the prefix, i.e. we dnl use "PATH/include" and "PATH/lib". AC_SUBST(SSL_INCLUDE) -AC_SUBST(SSL_ROOT) +AC_SUBST(SSL_INCDIR) AC_SUBST(SSL_LIBDIR) AC_SUBST(SSL_CRYPTO_LIBNAME) AC_SUBST(SSL_SSL_LIBNAME) @@ -4082,6 +4082,15 @@ AS_HELP_STRING([--with-ssl=PATH], [specify location of OpenSSL include and lib]) AS_HELP_STRING([--with-ssl], [use SSL (default)]) AS_HELP_STRING([--without-ssl], [don't use SSL])) +AC_ARG_WITH(ssl-incl, +AS_HELP_STRING([--with-ssl-incl=PATH], [location of OpenSSL include dir, if different than specified by --with-ssl=PATH]), +[ +case X$with_ssl in + X | Xyes | Xno) AC_MSG_ERROR([--with-ssl-incl=PATH set without --with-ssl=PATH]);; +esac +], +[with_ssl_incl=$with_ssl]) #default + AC_ARG_ENABLE(dynamic-ssl-lib, AS_HELP_STRING([--disable-dynamic-ssl-lib], [disable using dynamic openssl libraries]), @@ -4196,7 +4205,7 @@ case "$erl_xcomp_without_sysroot-$with_ssl" in dir="$erl_xcomp_sysroot$rdir" if test -f "$erl_xcomp_isysroot$rdir/include/openssl/opensslv.h"; then is_real_ssl=yes - SSL_ROOT="$dir" + SSL_INCDIR="$dir" if test "x$MIXED_CYGWIN" = "xyes" -o "x$MIXED_MSYS" = "xyes"; then if test -f "$dir/lib/VC/libeay32.lib"; then SSL_RUNTIME_LIBDIR="$rdir/lib/VC" @@ -4326,8 +4335,8 @@ case "$erl_xcomp_without_sysroot-$with_ssl" in # Trust OpenBSD to have everything the in the correct locations. ssl_found=yes ssl_linkable=yes - SSL_ROOT="$erl_xcomp_sysroot/usr" - AC_MSG_RESULT([$SSL_ROOT]) + SSL_INCDIR="$erl_xcomp_sysroot/usr" + AC_MSG_RESULT([$SSL_INCDIR]) SSL_RUNTIME_LIB="/usr/lib" SSL_LIB="$erl_xcomp_sysroot/usr/lib" SSL_BINDIR="/usr/sbin" @@ -4394,7 +4403,10 @@ dnl so it is - be adoptable if test ! -d "$with_ssl" ; then AC_MSG_ERROR(Invalid path to option --with-ssl=PATH) fi - SSL_ROOT="$with_ssl" + if test ! -d "$with_ssl_incl" ; then + AC_MSG_ERROR(Invalid path to option --with-ssl-incl=PATH) + fi + SSL_INCDIR="$with_ssl_incl" SSL_CRYPTO_LIBNAME=crypto SSL_SSL_LIBNAME=ssl if test "x$MIXED_CYGWIN" = "xyes" -o "x$MIXED_MSYS" = "xyes" && test -d "$with_ssl/lib/VC"; then @@ -4444,12 +4456,12 @@ dnl so it is - be adoptable elif test '!' -f ${SSL_LIBDIR}/lib${SSL_CRYPTO_LIBNAME}.so -a '!' -f "$SSL_LIBDIR/lib${SSL_CRYPTO_LIBNAME}.dylib"; then SSL_STATIC_ONLY=yes fi - SSL_INCLUDE="-I$with_ssl/include" + SSL_INCLUDE="-I$with_ssl_incl/include" SSL_APP=ssl CRYPTO_APP=crypto SSH_APP=ssh if test "$cross_compiling" = "yes"; then - SSL_RUNTIME_LIBDIR=`echo "$SSL_LIBDIR" | sed -n "s|^$erl_xcomp_sysroot\(.*\)\$|\1|p"` + SSL_RUNTIME_LIBDIR=`echo "$SSL_LIBDIR" | sed -n "s|^$erl_xcomp_sysroot\(/*\)\(.*\)\$|/\2|p"` else SSL_RUNTIME_LIBDIR="$SSL_LIBDIR" fi @@ -4507,8 +4519,8 @@ if test "x$SSL_APP" != "x" ; then SSL_KRB5_INCLUDE= if test "x$ssl_krb5_enabled" = "xyes" ; then AC_MSG_CHECKING(for krb5.h in standard locations) - for dir in $extra_dir "$SSL_ROOT/include" "$SSL_ROOT/include/openssl" \ - "$SSL_ROOT/include/kerberos" \ + for dir in $extra_dir "$SSL_INCDIR/include" "$SSL_INCDIR/include/openssl" \ + "$SSL_INCDIR/include/kerberos" \ "$erl_xcomp_isysroot/cygdrive/c/kerberos/include" \ "$erl_xcomp_isysroot/usr/local/kerberos/include" \ "$erl_xcomp_isysroot/usr/kerberos/include" \ diff --git a/erts/doc/src/crash_dump.xml b/erts/doc/src/crash_dump.xml index d3de29b876..2b5fc877c3 100644 --- a/erts/doc/src/crash_dump.xml +++ b/erts/doc/src/crash_dump.xml @@ -115,8 +115,9 @@ sockets/pipes can be used simultaneously by Erlang (due to limitations in the Unix <c><![CDATA[select]]></c> call). The number of open regular files is not affected by this.</item> - <item>"Received SIGUSR1" - The SIGUSR1 signal was sent to the - Erlang machine (Unix only).</item> + <item>"Received SIGUSR1" - Sending the SIGUSR1 signal to a + Erlang machine (Unix only) forces a crash dump. This slogan reflects + that the Erlang machine crash-dumped due to receiving that signal.</item> <item>"Kernel pid terminated (<em>Who</em>) (<em>Exit-reason</em>)" - The kernel supervisor has detected a failure, usually that the <c><![CDATA[application_controller]]></c> diff --git a/erts/doc/src/erl.xml b/erts/doc/src/erl.xml index f856b9ab86..16000191dc 100644 --- a/erts/doc/src/erl.xml +++ b/erts/doc/src/erl.xml @@ -525,7 +525,8 @@ core dump and no crash dump if an internal error is detected.</p> <p>Calling <c>erlang:halt/1</c> with a string argument will still - produce a crash dump.</p> + produce a crash dump. On Unix systems, sending an emulator process + a SIGUSR1 signal will also force a crash dump.</p> </item> <tag><marker id="+e"><c><![CDATA[+e Number]]></c></marker></tag> <item> @@ -1141,6 +1142,23 @@ <p>For more information, see <seealso marker="erlang#system_info_cpu_topology">erlang:system_info(cpu_topology)</seealso>.</p> </item> + <tag><marker id="+secio"><c>+secio true|false</c></marker></tag> + <item> + <p>Enable or disable eager check I/O scheduling. The default + is currently <c>true</c>. The default was changed from <c>false</c> + to <c>true</c> as of erts version 7.0. The behaviour before this + flag was introduced corresponds to <c>+secio false</c>.</p> + <p>The flag effects when schedulers will check for I/O + operations possible to execute, and when such I/O operations + will execute. As the name of the parameter implies, + schedulers will be more eager to check for I/O when + <c>true</c> is passed. This however also implies that + execution of outstanding I/O operation will not be + prioritized to the same extent as when <c>false</c> is + passed.</p> + <p><seealso marker="erlang#system_info_eager_check_io"><c>erlang:system_info(eager_check_io)</c></seealso> + returns the value of this parameter used when starting the VM.</p> + </item> <tag><marker id="+sfwi"><c>+sfwi Interval</c></marker></tag> <item> <p>Set scheduler forced wakeup interval. All run queues will diff --git a/erts/doc/src/erl_driver.xml b/erts/doc/src/erl_driver.xml index 4a1aab75c7..77fc906aca 100644 --- a/erts/doc/src/erl_driver.xml +++ b/erts/doc/src/erl_driver.xml @@ -2033,7 +2033,8 @@ ERL_DRV_MAP int sz entry function is called. If <c>ready_async</c> is null in the driver entry, the <c>async_free</c> function is called instead.</p> - <p>The return value is a handle to the asynchronous task.</p> + <p>The return value is -1 if the <c>driver_async</c> call + fails.</p> <note> <p>As of erts version 5.5.4.3 the default stack size for threads in the async-thread pool is 16 kilowords, diff --git a/erts/doc/src/erlang.xml b/erts/doc/src/erlang.xml index 97fe6d2915..226f2c0150 100644 --- a/erts/doc/src/erlang.xml +++ b/erts/doc/src/erlang.xml @@ -5789,6 +5789,7 @@ ok <name name="system_info" arity="1" clause_i="52"/> <name name="system_info" arity="1" clause_i="53"/> <name name="system_info" arity="1" clause_i="54"/> + <name name="system_info" arity="1" clause_i="55"/> <fsummary>Information about the system</fsummary> <desc> <p>Returns various information about the current system @@ -5984,6 +5985,16 @@ ok The return value will always be <c>false</c> since the elib_malloc allocator has been removed.</p> </item> + <tag><marker id="system_info_eager_check_io"><c>eager_check_io</c></marker></tag> + <item> + <p> + Returns the value of the <c>erl</c> + <seealso marker="erl#+secio">+secio</seealso> command line + flag which is either <c>true</c> or <c>false</c>. See the + documentation of the command line flag for information about + the different values. + </p> + </item> <tag><c>ets_limit</c></tag> <item> <p>Returns the maximum number of ETS tables allowed. This limit diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 37354b7f8d..61def65235 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -267,7 +267,6 @@ type CODE_IX_LOCK_Q SHORT_LIVED SYSTEM code_ix_lock_q type PROC_INTERVAL LONG_LIVED SYSTEM process_interval type BUSY_CALLER_TAB SHORT_LIVED SYSTEM busy_caller_table type BUSY_CALLER SHORT_LIVED SYSTEM busy_caller -type PORT_DATA_HEAP STANDARD SYSTEM port_data_heap type PROC_SYS_TSK SHORT_LIVED PROCESSES proc_sys_task type PROC_SYS_TSK_QS SHORT_LIVED PROCESSES proc_sys_task_queues @@ -364,6 +363,7 @@ type NLINK_SH STANDARD_LOW PROCESSES nlink_sh type AINFO_REQ STANDARD_LOW SYSTEM alloc_info_request type SCHED_WTIME_REQ STANDARD_LOW SYSTEM sched_wall_time_request type GC_INFO_REQ STANDARD_LOW SYSTEM gc_info_request +type PORT_DATA_HEAP STANDARD_LOW SYSTEM port_data_heap +else # "fullword" @@ -383,6 +383,7 @@ type NLINK_SH FIXED_SIZE PROCESSES nlink_sh type AINFO_REQ SHORT_LIVED SYSTEM alloc_info_request type SCHED_WTIME_REQ SHORT_LIVED SYSTEM sched_wall_time_request type GC_INFO_REQ SHORT_LIVED SYSTEM gc_info_request +type PORT_DATA_HEAP STANDARD SYSTEM port_data_heap +endif @@ -397,6 +398,7 @@ type DRV_EV_STATE LONG_LIVED SYSTEM driver_event_state type DRV_EV_D_STATE FIXED_SIZE SYSTEM driver_event_data_state type DRV_SEL_D_STATE FIXED_SIZE SYSTEM driver_select_data_state type FD_LIST SHORT_LIVED SYSTEM fd_list +type ACTIVE_FD_ARR SHORT_LIVED SYSTEM active_fd_array type POLLSET LONG_LIVED SYSTEM pollset type POLLSET_UPDREQ SHORT_LIVED SYSTEM pollset_update_req type POLL_FDS LONG_LIVED SYSTEM poll_fds @@ -413,6 +415,8 @@ type CS_PROG_PATH LONG_LIVED SYSTEM cs_prog_path type ENVIRONMENT TEMPORARY SYSTEM environment type PUTENV_STR SYSTEM SYSTEM putenv_string type PRT_REP_EXIT STANDARD SYSTEM port_report_exit +type SYS_BLOCKING STANDARD SYSTEM sys_blocking +type SYS_WRITE_BUF TEMPORARY SYSTEM sys_write_buf +endif diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 6efe9d9550..61e4469600 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -2696,6 +2696,9 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) ? am_disabled : am_enabled); } + else if (ERTS_IS_ATOM_STR("eager_check_io",BIF_ARG_1)) { + BIF_RET(erts_eager_check_io ? am_true : am_false); + } BIF_ERROR(BIF_P, BADARG); } @@ -3304,17 +3307,38 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) BIF_RET(make_small((Uint) words)); } else if (ERTS_IS_ATOM_STR("check_io_debug", BIF_ARG_1)) { - /* Used by (emulator) */ - int res; + /* Used by driver_SUITE (emulator) */ + Uint sz, *szp; + Eterm res, *hp, **hpp; + int no_errors; + ErtsCheckIoDebugInfo ciodi = {0}; #ifdef HAVE_ERTS_CHECK_IO_DEBUG erts_smp_proc_unlock(BIF_P,ERTS_PROC_LOCK_MAIN); - res = erts_check_io_debug(); + no_errors = erts_check_io_debug(&ciodi); erts_smp_proc_lock(BIF_P,ERTS_PROC_LOCK_MAIN); #else - res = 0; + no_errors = 0; #endif - ASSERT(res >= 0); - BIF_RET(erts_make_integer((Uint) res, BIF_P)); + sz = 0; + szp = &sz; + hpp = NULL; + while (1) { + res = erts_bld_tuple(hpp, szp, 4, + erts_bld_uint(hpp, szp, + (Uint) no_errors), + erts_bld_uint(hpp, szp, + (Uint) ciodi.no_used_fds), + erts_bld_uint(hpp, szp, + (Uint) ciodi.no_driver_select_structs), + erts_bld_uint(hpp, szp, + (Uint) ciodi.no_driver_event_structs)); + if (hpp) + break; + hp = HAlloc(BIF_P, sz); + szp = NULL; + hpp = &hp; + } + BIF_RET(res); } else if (ERTS_IS_ATOM_STR("process_info_args", BIF_ARG_1)) { /* Used by process_SUITE (emulator) */ diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c index afb33c1cdb..64bd598ba6 100644 --- a/erts/emulator/beam/erl_bif_port.c +++ b/erts/emulator/beam/erl_bif_port.c @@ -493,8 +493,8 @@ void erts_cleanup_port_data(Port *prt) { ASSERT(erts_atomic32_read_nob(&prt->state) & ERTS_PORT_SFLGS_INVALID_LOOKUP); - cleanup_old_port_data(erts_smp_atomic_read_nob(&prt->data)); - erts_smp_atomic_set_nob(&prt->data, (erts_aint_t) THE_NON_VALUE); + cleanup_old_port_data(erts_smp_atomic_xchg_nob(&prt->data, + (erts_aint_t) NULL)); } Uint @@ -554,6 +554,7 @@ BIF_RETTYPE port_set_data_2(BIF_ALIST_2) hp = &pdhp->heap[0]; pdhp->off_heap.first = NULL; pdhp->off_heap.overhead = 0; + pdhp->hsize = hsize; pdhp->data = copy_struct(BIF_ARG_2, hsize, &hp, &pdhp->off_heap); data = (erts_aint_t) pdhp; ASSERT((data & 0x3) == 0); @@ -561,8 +562,14 @@ BIF_RETTYPE port_set_data_2(BIF_ALIST_2) data = erts_smp_atomic_xchg_wb(&prt->data, data); + if (data == (erts_aint_t)NULL) { + /* Port terminated by racing thread */ + data = erts_smp_atomic_xchg_wb(&prt->data, data); + ASSERT(data != (erts_aint_t)NULL); + cleanup_old_port_data(data); + BIF_ERROR(BIF_P, BADARG); + } cleanup_old_port_data(data); - BIF_RET(am_true); } @@ -581,6 +588,8 @@ BIF_RETTYPE port_get_data_1(BIF_ALIST_1) BIF_ERROR(BIF_P, BADARG); data = erts_smp_atomic_read_ddrb(&prt->data); + if (data == (erts_aint_t)NULL) + BIF_ERROR(BIF_P, BADARG); /* Port terminated by racing thread */ if ((data & 0x3) != 0) { res = (Eterm) (UWord) data; diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 88c4006934..61f8385efc 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -548,6 +548,8 @@ void erts_usage(void) erts_fprintf(stderr, " see the erl(1) documentation for more info.\n"); erts_fprintf(stderr, "-sct cput set cpu topology,\n"); erts_fprintf(stderr, " see the erl(1) documentation for more info.\n"); + erts_fprintf(stderr, "-secio bool enable/disable eager check I/O scheduling,\n"); + erts_fprintf(stderr, " see the erl(1) documentation for more info.\n"); #if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT erts_fprintf(stderr, "-sub bool enable/disable scheduler utilization balancing,\n"); #else @@ -1674,6 +1676,22 @@ erl_start(int argc, char **argv) erts_usage(); } } + else if (has_prefix("ecio", sub_param)) { + arg = get_arg(sub_param+4, argv[i+1], &i); +#ifndef __OSE__ + if (sys_strcmp("true", arg) == 0) + erts_eager_check_io = 1; + else +#endif + if (sys_strcmp("false", arg) == 0) + erts_eager_check_io = 0; + else { + erts_fprintf(stderr, + "bad schedule eager check I/O value '%s'\n", + arg); + erts_usage(); + } + } else if (has_prefix("pp", sub_param)) { arg = get_arg(sub_param+2, argv[i+1], &i); if (sys_strcmp(arg, "true") == 0) diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c index 682f6f8f4b..2aa0a27197 100644 --- a/erts/emulator/beam/erl_port_task.c +++ b/erts/emulator/beam/erl_port_task.c @@ -32,6 +32,7 @@ #include "global.h" #include "erl_port_task.h" #include "dist.h" +#include "erl_check_io.h" #include "dtrace-wrapper.h" #include <stdarg.h> @@ -550,6 +551,16 @@ reset_handle(ErtsPortTask *ptp) } static ERTS_INLINE void +reset_executed_io_task_handle(ErtsPortTask *ptp) +{ + if (ptp->u.alive.handle) { + ASSERT(ptp == handle2task(ptp->u.alive.handle)); + erts_io_notify_port_task_executed(ptp->u.alive.handle); + reset_port_task_handle(ptp->u.alive.handle); + } +} + +static ERTS_INLINE void set_handle(ErtsPortTask *ptp, ErtsPortTaskHandle *pthp) { ptp->u.alive.handle = pthp; @@ -1396,10 +1407,7 @@ erts_port_task_schedule(Eterm id, erts_aint32_t act, add_flags; unsigned int prof_runnable_ports; - if (pthp && erts_port_task_is_scheduled(pthp)) { - ASSERT(0); - erts_port_task_abort(pthp); - } + ERTS_LC_ASSERT(!pthp || !erts_port_task_is_scheduled(pthp)); ASSERT(is_internal_port(id)); @@ -1699,8 +1707,6 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) goto aborted_port_task; } - reset_handle(ptp); - if (erts_system_monitor_long_schedule != 0) { start_time = erts_timestamp_millis(); } @@ -1711,6 +1717,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) switch (ptp->type) { case ERTS_PORT_TASK_TIMEOUT: + reset_handle(ptp); reds = ERTS_PORT_REDS_TIMEOUT; if (!(state & ERTS_PORT_SFLGS_DEAD)) { DTRACE_DRIVER(driver_timeout, pp); @@ -1725,6 +1732,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) for input and output */ (*pp->drv_ptr->ready_input)((ErlDrvData) pp->drv_data, ptp->u.alive.td.io.event); + reset_executed_io_task_handle(ptp); io_tasks_executed++; break; case ERTS_PORT_TASK_OUTPUT: @@ -1733,6 +1741,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) DTRACE_DRIVER(driver_ready_output, pp); (*pp->drv_ptr->ready_output)((ErlDrvData) pp->drv_data, ptp->u.alive.td.io.event); + reset_executed_io_task_handle(ptp); io_tasks_executed++; break; case ERTS_PORT_TASK_EVENT: @@ -1742,10 +1751,12 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) (*pp->drv_ptr->event)((ErlDrvData) pp->drv_data, ptp->u.alive.td.io.event, ptp->u.alive.td.io.event_data); + reset_executed_io_task_handle(ptp); io_tasks_executed++; break; case ERTS_PORT_TASK_PROC_SIG: { ErtsProc2PortSigData *sigdp = &ptp->u.alive.td.psig.data; + reset_handle(ptp); ASSERT((state & ERTS_PORT_SFLGS_DEAD) == 0); if (!pp->sched.taskq.bpq) reds = ptp->u.alive.td.psig.callback(pp, @@ -1763,6 +1774,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) break; } case ERTS_PORT_TASK_DIST_CMD: + reset_handle(ptp); reds = erts_dist_command(pp, CONTEXT_REDS - pp->reds); break; default: diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h index 9ef0cfcedc..406cd3c492 100644 --- a/erts/emulator/beam/erl_port_task.h +++ b/erts/emulator/beam/erl_port_task.h @@ -156,7 +156,7 @@ erts_port_task_handle_init(ErtsPortTaskHandle *pthp) ERTS_GLB_INLINE int erts_port_task_is_scheduled(ErtsPortTaskHandle *pthp) { - return ((void *) erts_smp_atomic_read_nob(pthp)) != NULL; + return ((void *) erts_smp_atomic_read_acqb(pthp)) != NULL; } ERTS_GLB_INLINE void erts_port_task_pre_init_sched(ErtsPortTaskSched *ptsp, diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 20a88ec581..7b272885a7 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -148,6 +148,12 @@ extern BeamInstr beam_apply[]; extern BeamInstr beam_exit[]; extern BeamInstr beam_continue_exit[]; +#ifdef __OSE__ +/* Eager check I/O not supported on OSE yet. */ +int erts_eager_check_io = 0; +#else +int erts_eager_check_io = 1; +#endif int erts_sched_compact_load; int erts_sched_balance_util = 0; Uint erts_no_schedulers; @@ -2381,29 +2387,47 @@ try_set_sys_scheduling(void) #endif static ERTS_INLINE int -prepare_for_sys_schedule(ErtsSchedulerData *esdp) +prepare_for_sys_schedule(ErtsSchedulerData *esdp, int non_blocking) { + if (non_blocking && erts_eager_check_io) { #ifdef ERTS_SMP - while (!erts_port_task_have_outstanding_io_tasks() - && try_set_sys_scheduling()) { #ifdef ERTS_SCHED_ONLY_POLL_SCHED_1 - if (esdp->no != 1) { - /* If we are not scheduler 1 and ERTS_SCHED_ONLY_POLL_SCHED_1 is used - then we make sure to wake scheduler 1 */ - ErtsRunQueue *rq = ERTS_RUNQ_IX(0); - clear_sys_scheduling(); - wake_scheduler(rq); - return 0; - } + if (esdp->no != 1) { + /* If we are not scheduler 1 and ERTS_SCHED_ONLY_POLL_SCHED_1 is used + then we make sure to wake scheduler 1 */ + ErtsRunQueue *rq = ERTS_RUNQ_IX(0); + wake_scheduler(rq); + return 0; + } #endif - if (!erts_port_task_have_outstanding_io_tasks()) + return try_set_sys_scheduling(); +#else return 1; - clear_sys_scheduling(); +#endif } - return 0; + else { +#ifdef ERTS_SMP + while (!erts_port_task_have_outstanding_io_tasks() + && try_set_sys_scheduling()) { +#ifdef ERTS_SCHED_ONLY_POLL_SCHED_1 + if (esdp->no != 1) { + /* If we are not scheduler 1 and ERTS_SCHED_ONLY_POLL_SCHED_1 is used + then we make sure to wake scheduler 1 */ + ErtsRunQueue *rq = ERTS_RUNQ_IX(0); + clear_sys_scheduling(); + wake_scheduler(rq); + return 0; + } +#endif + if (!erts_port_task_have_outstanding_io_tasks()) + return 1; + clear_sys_scheduling(); + } + return 0; #else - return !erts_port_task_have_outstanding_io_tasks(); + return !erts_port_task_have_outstanding_io_tasks(); #endif + } } #ifdef ERTS_SMP @@ -2780,7 +2804,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) * be waiting in erl_sys_schedule() */ - if (ERTS_SCHEDULER_IS_DIRTY(esdp) || !prepare_for_sys_schedule(esdp)) { + if (ERTS_SCHEDULER_IS_DIRTY(esdp) || !prepare_for_sys_schedule(esdp, 0)) { sched_waiting(esdp->no, rq); @@ -2944,7 +2968,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) * Got to check that we still got I/O tasks; otherwise * we have to continue checking for I/O... */ - if (!prepare_for_sys_schedule(esdp)) { + if (!prepare_for_sys_schedule(esdp, 0)) { spincount *= ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT; goto tse_wait; } @@ -2966,7 +2990,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) * Got to check that we still got I/O tasks; otherwise * we have to wait in erl_sys_schedule() after all... */ - if (!prepare_for_sys_schedule(esdp)) { + if (!prepare_for_sys_schedule(esdp, 0)) { /* * Not allowed to wait in erl_sys_schedule; * do tse wait instead... @@ -9200,7 +9224,7 @@ Process *schedule(Process *p, int calls) } else if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && (fcalls > input_reductions && - prepare_for_sys_schedule(esdp))) { + prepare_for_sys_schedule(esdp, !0))) { /* * Schedule system-level activities. */ @@ -9208,8 +9232,6 @@ Process *schedule(Process *p, int calls) erts_smp_atomic32_set_relb(&function_calls, 0); fcalls = 0; - ASSERT(!erts_port_task_have_outstanding_io_tasks()); - #if 0 /* Not needed since we wont wait in sys schedule */ erts_sys_schedule_interrupt(0); #endif @@ -9241,7 +9263,9 @@ Process *schedule(Process *p, int calls) if (RUNQ_READ_LEN(&rq->ports.info.len)) { int have_outstanding_io; have_outstanding_io = erts_port_task_execute(rq, &esdp->current_port); - if ((have_outstanding_io && fcalls > 2*input_reductions) + if ((!erts_eager_check_io + && have_outstanding_io + && fcalls > 2*input_reductions) || rq->halt_in_progress) { /* * If we have performed more than 2*INPUT_REDUCTIONS since diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 3b0798207e..27a3a3553b 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -104,6 +104,7 @@ struct saved_calls { }; extern Export exp_send, exp_receive, exp_timeout; +extern int erts_eager_check_io; extern int erts_sched_compact_load; extern int erts_sched_balance_util; extern Uint erts_no_schedulers; diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index 3d8dd9c6d0..1bdb1579f5 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -66,8 +66,12 @@ */ #ifndef ERTS_SYS_FD_TYPE +#define ERTS_SYS_FD_INVALID ((ErtsSysFdType) -1) typedef int ErtsSysFdType; #else +#ifndef ERTS_SYS_FD_INVALID +# error missing ERTS_SYS_FD_INVALID +#endif typedef ERTS_SYS_FD_TYPE ErtsSysFdType; #endif @@ -744,6 +748,14 @@ void init_getenv_state(GETENV_STATE *); char * getenv_string(GETENV_STATE *); void fini_getenv_state(GETENV_STATE *); +#define HAVE_ERTS_CHECK_IO_DEBUG +typedef struct { + int no_used_fds; + int no_driver_select_structs; + int no_driver_event_structs; +} ErtsCheckIoDebugInfo; +int erts_check_io_debug(ErtsCheckIoDebugInfo *ip); + /* xxxP */ #define SYS_DEFAULT_FLOAT_DECIMALS 20 void init_sys_float(void); diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 55f9e68e78..f20e6e5665 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -48,6 +48,7 @@ #include "erl_sched_spec_pre_alloc.h" #include "beam_bp.h" #include "erl_ptab.h" +#include "erl_check_io.h" #undef M_TRIM_THRESHOLD #undef M_TOP_PAD diff --git a/erts/emulator/drivers/unix/ttsl_drv.c b/erts/emulator/drivers/unix/ttsl_drv.c index 491e0a090e..be2fee1f25 100644 --- a/erts/emulator/drivers/unix/ttsl_drv.c +++ b/erts/emulator/drivers/unix/ttsl_drv.c @@ -32,6 +32,10 @@ static ErlDrvData ttysl_start(ErlDrvPort, char*); #ifdef HAVE_TERMCAP /* else make an empty driver that can not be opened */ +#ifndef WANT_NONBLOCKING +#define WANT_NONBLOCKING +#endif + #include "sys.h" #include <ctype.h> #include <stdlib.h> @@ -39,6 +43,7 @@ static ErlDrvData ttysl_start(ErlDrvPort, char*); #include <string.h> #include <signal.h> #include <fcntl.h> +#include <limits.h> #include <locale.h> #include <unistd.h> #include <termios.h> @@ -57,6 +62,14 @@ static ErlDrvData ttysl_start(ErlDrvPort, char*); #include <langinfo.h> #endif +#if defined IOV_MAX +#define MAXIOV IOV_MAX +#elif defined UIO_MAXIOV +#define MAXIOV UIO_MAXIOV +#else +#define MAXIOV 16 +#endif + #define TRUE 1 #define FALSE 0 @@ -80,12 +93,15 @@ static volatile int cols_needs_update = FALSE; #define OP_INSC 2 #define OP_DELC 3 #define OP_BEEP 4 +#define OP_PUTC_SYNC 5 /* Control op */ #define CTRL_OP_GET_WINSIZE 100 #define CTRL_OP_GET_UNICODE_STATE 101 #define CTRL_OP_SET_UNICODE_STATE 102 - +/* We use 1024 as the buf size as that was the default buf size of FILE streams + on all platforms that I checked. */ +#define TTY_BUFFSIZE 1024 static int lbuf_size = BUFSIZ; static Uint32 *lbuf; /* The current line buffer */ @@ -113,13 +129,19 @@ static int lpos; /* The current "cursor position" in the line buf /* Main interface functions. */ static void ttysl_stop(ErlDrvData); static void ttysl_from_erlang(ErlDrvData, char*, ErlDrvSizeT); +static void ttysl_to_tty(ErlDrvData, ErlDrvEvent); +static void ttysl_flush_tty(ErlDrvData); static void ttysl_from_tty(ErlDrvData, ErlDrvEvent); static void ttysl_stop_select(ErlDrvEvent, void*); static Sint16 get_sint16(char*); static ErlDrvPort ttysl_port; static int ttysl_fd; -static FILE *ttysl_out; +static int ttysl_terminate = 0; +static int ttysl_send_ok = 0; +static ErlDrvBinary *putcbuf; +static int putcpos; +static int putclen; /* Functions that work on the line buffer. */ static int start_lbuf(void); @@ -201,22 +223,22 @@ struct erl_drv_entry ttsl_driver_entry = { IF_IMPL(ttysl_stop), IF_IMPL(ttysl_from_erlang), IF_IMPL(ttysl_from_tty), - NULL, - "tty_sl", - NULL, - NULL, + IF_IMPL(ttysl_to_tty), + "tty_sl", /* driver_name */ + NULL, /* finish */ + NULL, /* handle */ IF_IMPL(ttysl_control), NULL, /* timeout */ NULL, /* outputv */ NULL, /* ready_async */ - NULL, /* flush */ + IF_IMPL(ttysl_flush_tty), NULL, /* call */ NULL, /* event */ ERL_DRV_EXTENDED_MARKER, ERL_DRV_EXTENDED_MAJOR_VERSION, ERL_DRV_EXTENDED_MINOR_VERSION, 0, /* ERL_DRV_FLAGs */ - NULL, + NULL, /* handle2 */ NULL, /* process_exit */ IF_IMPL(ttysl_stop_select) }; @@ -296,8 +318,7 @@ static ErlDrvData ttysl_start(ErlDrvPort port, char* buf) return ERL_DRV_ERROR_GENERAL; } - /* Open the terminal and set the terminal */ - ttysl_out = fdopen(ttysl_fd, "w"); + SET_NONBLOCKING(ttysl_fd); #ifdef PRIMITIVE_UTF8_CHECK setlocale(LC_CTYPE, ""); /* Set international environment, @@ -400,12 +421,14 @@ static void ttysl_stop(ErlDrvData ttysl_data) stop_lbuf(); stop_termcap(); tty_reset(ttysl_fd); - driver_select(ttysl_port, (ErlDrvEvent)(UWord)ttysl_fd, ERL_DRV_READ|ERL_DRV_USE, 0); + driver_select(ttysl_port, (ErlDrvEvent)(UWord)ttysl_fd, + ERL_DRV_WRITE|ERL_DRV_READ|ERL_DRV_USE, 0); sys_sigset(SIGCONT, SIG_DFL); sys_sigset(SIGWINCH, SIG_DFL); } ttysl_port = (ErlDrvPort)-1; ttysl_fd = -1; + ttysl_terminate = 0; /* return TRUE; */ } @@ -650,10 +673,26 @@ static int check_buf_size(byte *s, int n) static void ttysl_from_erlang(ErlDrvData ttysl_data, char* buf, ErlDrvSizeT count) { + ErlDrvSizeT sz; + + sz = driver_sizeq(ttysl_port); + + putclen = count > TTY_BUFFSIZE ? TTY_BUFFSIZE : count; + putcbuf = driver_alloc_binary(putclen); + putcpos = 0; + if (lpos > MAXSIZE) put_chars((byte*)"\n", 1); switch (buf[0]) { + case OP_PUTC_SYNC: + /* Using sync means that we have to send an ok to the + controlling process for each command call. We delay + sending ok if the driver queue exceeds a certain size. + We do not set ourselves as a busy port, as this + could be very bad for user_drv, if it gets blocked on + the port_command. */ + /* fall through */ case OP_PUTC: DEBUGLOG(("OP: Putc(%lu)",(unsigned long) count-1)); if (check_buf_size((byte*)buf+1, count-1) == 0) @@ -678,10 +717,104 @@ static void ttysl_from_erlang(ErlDrvData ttysl_data, char* buf, ErlDrvSizeT coun /* Unknown op, just ignore. */ break; } - fflush(ttysl_out); + + driver_enq_bin(ttysl_port,putcbuf,0,putcpos); + + if (sz == 0) { + for (;;) { + int written, qlen; + SysIOVec *iov; + + iov = driver_peekq(ttysl_port,&qlen); + if (iov) + written = writev(ttysl_fd, iov, qlen > MAXIOV ? MAXIOV : qlen); + else + written = 0; + if (written < 0) { + if (errno == EAGAIN) { + driver_select(ttysl_port,(ErlDrvEvent)(long)ttysl_fd, + ERL_DRV_USE|ERL_DRV_WRITE,1); + break; + } else { + /* we ignore all other errors */ + break; + } + } else { + if (driver_deq(ttysl_port, written) == 0) + break; + } + } + } + + if (buf[0] == OP_PUTC_SYNC) { + if (driver_sizeq(ttysl_port) > TTY_BUFFSIZE && !ttysl_terminate) { + /* We delay sending the ack until the buffer has been consumed */ + ttysl_send_ok = 1; + } else { + ErlDrvTermData spec[] = { + ERL_DRV_PORT, driver_mk_port(ttysl_port), + ERL_DRV_ATOM, driver_mk_atom("ok"), + ERL_DRV_TUPLE, 2 + }; + ASSERT(ttysl_send_ok == 0); + erl_drv_output_term(driver_mk_port(ttysl_port), spec, + sizeof(spec) / sizeof(spec[0])); + } + } + return; /* TRUE; */ } +static void ttysl_to_tty(ErlDrvData ttysl_data, ErlDrvEvent fd) { + for (;;) { + int written, qlen; + SysIOVec *iov; + ErlDrvSizeT sz; + + iov = driver_peekq(ttysl_port,&qlen); + if (iov) + written = writev(ttysl_fd, iov, qlen > MAXIOV ? MAXIOV : qlen); + else + written = 0; + if (written < 0) { + if (errno == EAGAIN) { + break; + } else { + /* we ignore all other errors */ + } + } else { + sz = driver_deq(ttysl_port, written); + if (sz < TTY_BUFFSIZE && ttysl_send_ok) { + ErlDrvTermData spec[] = { + ERL_DRV_PORT, driver_mk_port(ttysl_port), + ERL_DRV_ATOM, driver_mk_atom("ok"), + ERL_DRV_TUPLE, 2 + }; + ttysl_send_ok = 0; + erl_drv_output_term(driver_mk_port(ttysl_port), spec, + sizeof(spec) / sizeof(spec[0])); + } + if (sz == 0) { + driver_select(ttysl_port,(ErlDrvEvent)(long)ttysl_fd, + ERL_DRV_WRITE,0); + if (ttysl_terminate) + /* flush has been called, which means we should terminate + when queue is empty. This will not send any exit + message */ + driver_failure_atom(ttysl_port, "normal"); + break; + } + } + } + + return; +} + +static void ttysl_flush_tty(ErlDrvData ttysl_data) { + ttysl_terminate = 1; + return; +} + static void ttysl_from_tty(ErlDrvData ttysl_data, ErlDrvEvent fd) { byte b[1024]; @@ -1070,7 +1203,14 @@ static int write_buf(Uint32 *s, int n) /* The basic procedure for outputting one character. */ static int outc(int c) { - return (int)putc(c, ttysl_out); + putcbuf->orig_bytes[putcpos++] = c; + if (putcpos == putclen) { + driver_enq_bin(ttysl_port,putcbuf,0,putclen); + putcpos = 0; + putclen = TTY_BUFFSIZE; + putcbuf = driver_alloc_binary(BUFSIZ); + } + return 1; } static int move_cursor(int from, int to) diff --git a/erts/emulator/drivers/win32/ttsl_drv.c b/erts/emulator/drivers/win32/ttsl_drv.c index 502cb58dfa..851c336a11 100644 --- a/erts/emulator/drivers/win32/ttsl_drv.c +++ b/erts/emulator/drivers/win32/ttsl_drv.c @@ -46,6 +46,7 @@ static int rows; /* Number of rows available. */ #define OP_INSC 2 #define OP_DELC 3 #define OP_BEEP 4 +#define OP_PUTC_SYNC 5 /* Control op */ #define CTRL_OP_GET_WINSIZE 100 @@ -458,6 +459,7 @@ static void ttysl_from_erlang(ErlDrvData ttysl_data, char* buf, ErlDrvSizeT coun switch (buf[0]) { case OP_PUTC: + case OP_PUTC_SYNC: DEBUGLOG(("OP: Putc(%I64u)",(unsigned long long)count-1)); if (check_buf_size((byte*)buf+1, count-1) == 0) return; @@ -481,6 +483,20 @@ static void ttysl_from_erlang(ErlDrvData ttysl_data, char* buf, ErlDrvSizeT coun /* Unknown op, just ignore. */ break; } + + if (buf[0] == OP_PUTC_SYNC) { + /* On windows we do a blocking write to the tty so we just + send the ack immidiately. If at some point in the future + someone has a problem with tty output being blocking + this has to be changed. */ + ErlDrvTermData spec[] = { + ERL_DRV_PORT, driver_mk_port(ttysl_port), + ERL_DRV_ATOM, driver_mk_atom("ok"), + ERL_DRV_TUPLE, 2 + }; + erl_drv_output_term(driver_mk_port(ttysl_port), spec, + sizeof(spec) / sizeof(spec[0])); + } return; } diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c index 1db673e7f3..81cb5dc4bb 100644 --- a/erts/emulator/sys/common/erl_check_io.c +++ b/erts/emulator/sys/common/erl_check_io.c @@ -52,8 +52,17 @@ typedef char EventStateType; #define ERTS_EV_TYPE_STOP_USE ((EventStateType) 3) /* pending stop_select */ typedef char EventStateFlags; -#define ERTS_EV_FLAG_USED ((EventStateFlags) 1) /* ERL_DRV_USE has been turned on */ +#define ERTS_EV_FLAG_USED ((EventStateFlags) 1) /* ERL_DRV_USE has been turned on */ +#define ERTS_EV_FLAG_DEFER_IN_EV ((EventStateFlags) 2) +#define ERTS_EV_FLAG_DEFER_OUT_EV ((EventStateFlags) 4) +#ifdef DEBUG +# define ERTS_ACTIVE_FD_INC 2 +#else +# define ERTS_ACTIVE_FD_INC 128 +#endif + +#define ERTS_CHECK_IO_POLL_RES_LEN 512 #if defined(ERTS_KERNEL_POLL_VERSION) # define ERTS_CIO_EXPORT(FUNC) FUNC ## _kp @@ -67,6 +76,7 @@ typedef char EventStateFlags; (ERTS_POLL_USE_POLL && !ERTS_POLL_USE_KERNEL_POLL) #define ERTS_CIO_POLL_CTL ERTS_POLL_EXPORT(erts_poll_control) +#define ERTS_CIO_POLL_CTLV ERTS_POLL_EXPORT(erts_poll_controlv) #define ERTS_CIO_POLL_WAIT ERTS_POLL_EXPORT(erts_poll_wait) #ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT #define ERTS_CIO_POLL_AS_INTR ERTS_POLL_EXPORT(erts_poll_async_sig_interrupt) @@ -85,6 +95,13 @@ static struct pollset_info { ErtsPollSet ps; erts_smp_atomic_t in_poll_wait; /* set while doing poll */ + struct { + int six; /* start index */ + int eix; /* end index */ + erts_smp_atomic32_t no; + int size; + ErtsSysFdType *array; + } active_fd; #ifdef ERTS_SMP struct removed_fd* removed_list; /* list of deselected fd's*/ erts_smp_spinlock_t removed_list_lock; @@ -97,9 +114,11 @@ typedef struct { SafeHashBucket hb; #endif ErtsSysFdType fd; - union { - ErtsDrvEventDataState *event; /* ERTS_EV_TYPE_DRV_EV */ + struct { ErtsDrvSelectDataState *select; /* ERTS_EV_TYPE_DRV_SEL */ +#if ERTS_CIO_HAVE_DRV_EVENT + ErtsDrvEventDataState *event; /* ERTS_EV_TYPE_DRV_EV */ +#endif erts_driver_t* drv_ptr; /* ERTS_EV_TYPE_STOP_USE */ } driver; ErtsPollEvents events; @@ -169,6 +188,10 @@ static ERTS_INLINE ErtsDrvEventState* hash_new_drv_ev_state(ErtsSysFdType fd) ErtsDrvEventState tmpl; tmpl.fd = fd; tmpl.driver.select = NULL; +#if ERTS_CIO_HAVE_DRV_EVENT + tmpl.driver.event = NULL; +#endif + tmpl.driver.drv_ptr = NULL; tmpl.events = 0; tmpl.remove_cnt = 0; tmpl.type = ERTS_EV_TYPE_NONE; @@ -209,6 +232,65 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(removed_fd, struct removed_fd, 64, ERTS_ALC_T_F #endif static ERTS_INLINE void +init_iotask(ErtsIoTask *io_task) +{ + erts_port_task_handle_init(&io_task->task); + erts_smp_atomic_init_nob(&io_task->executed_time, ~((erts_aint_t) 0)); +} + +static ERTS_INLINE int +is_iotask_active(ErtsIoTask *io_task, erts_aint_t current_cio_time) +{ + if (erts_port_task_is_scheduled(&io_task->task)) + return 1; + if (erts_smp_atomic_read_nob(&io_task->executed_time) == current_cio_time) + return 1; + return 0; +} + +static ERTS_INLINE ErtsDrvSelectDataState * +alloc_drv_select_data(void) +{ + ErtsDrvSelectDataState *dsp = erts_alloc(ERTS_ALC_T_DRV_SEL_D_STATE, + sizeof(ErtsDrvSelectDataState)); + dsp->inport = NIL; + dsp->outport = NIL; + init_iotask(&dsp->iniotask); + init_iotask(&dsp->outiotask); + return dsp; +} + +static ERTS_INLINE void +free_drv_select_data(ErtsDrvSelectDataState *dsp) +{ + ASSERT(!erts_port_task_is_scheduled(&dsp->iniotask.task)); + ASSERT(!erts_port_task_is_scheduled(&dsp->outiotask.task)); + erts_free(ERTS_ALC_T_DRV_SEL_D_STATE, dsp); +} + +static ERTS_INLINE ErtsDrvEventDataState * +alloc_drv_event_data(void) +{ + ErtsDrvEventDataState *dep = erts_alloc(ERTS_ALC_T_DRV_EV_D_STATE, + sizeof(ErtsDrvEventDataState)); + dep->port = NIL; + dep->data = NULL; + dep->removed_events = 0; +#if ERTS_CIO_DEFER_ACTIVE_EVENTS + dep->deferred_events = 0; +#endif + init_iotask(&dep->iotask); + return dep; +} + +static ERTS_INLINE void +free_drv_event_data(ErtsDrvEventDataState *dep) +{ + ASSERT(!erts_port_task_is_scheduled(&dep->iotask.task)); + erts_free(ERTS_ALC_T_DRV_EV_D_STATE, dep); +} + +static ERTS_INLINE void remember_removed(ErtsDrvEventState *state, struct pollset_info* psi) { #ifdef ERTS_SMP @@ -288,7 +370,7 @@ forget_removed(struct pollset_info* psi) drv_ptr = state->driver.drv_ptr; ASSERT(drv_ptr); state->type = ERTS_EV_TYPE_NONE; - state->flags = 0; + state->flags &= ~ERTS_EV_FLAG_USED; state->driver.drv_ptr = NULL; /* Fall through */ case ERTS_EV_TYPE_NONE: @@ -345,6 +427,10 @@ grow_drv_ev_state(int min_ix) for (i = erts_smp_atomic_read_nob(&drv_ev_state_len); i < new_len; i++) { drv_ev_state[i].fd = (ErtsSysFdType) i; drv_ev_state[i].driver.select = NULL; +#if ERTS_CIO_HAVE_DRV_EVENT + drv_ev_state[i].driver.event = NULL; +#endif + drv_ev_state[i].driver.drv_ptr = NULL; drv_ev_state[i].events = 0; drv_ev_state[i].remove_cnt = 0; drv_ev_state[i].type = ERTS_EV_TYPE_NONE; @@ -365,11 +451,7 @@ grow_drv_ev_state(int min_ix) static ERTS_INLINE void abort_task(Eterm id, ErtsPortTaskHandle *pthp, EventStateType type) { - if (is_nil(id)) { - ASSERT(type == ERTS_EV_TYPE_NONE - || !erts_port_task_is_scheduled(pthp)); - } - else if (erts_port_task_is_scheduled(pthp)) { + if (is_not_nil(id) && erts_port_task_is_scheduled(pthp)) { erts_port_task_abort(pthp); ASSERT(erts_is_port_alive(id)); } @@ -384,7 +466,7 @@ abort_tasks(ErtsDrvEventState *state, int mode) #if ERTS_CIO_HAVE_DRV_EVENT case ERTS_EV_TYPE_DRV_EV: abort_task(state->driver.event->port, - &state->driver.event->task, + &state->driver.event->iotask.task, ERTS_EV_TYPE_DRV_EV); return; #endif @@ -398,14 +480,14 @@ abort_tasks(ErtsDrvEventState *state, int mode) case ERL_DRV_WRITE: ASSERT(state->type == ERTS_EV_TYPE_DRV_SEL); abort_task(state->driver.select->outport, - &state->driver.select->outtask, + &state->driver.select->outiotask.task, state->type); if (mode == ERL_DRV_WRITE) break; case ERL_DRV_READ: ASSERT(state->type == ERTS_EV_TYPE_DRV_SEL); abort_task(state->driver.select->inport, - &state->driver.select->intask, + &state->driver.select->iniotask.task, state->type); break; default: @@ -443,16 +525,14 @@ deselect(ErtsDrvEventState *state, int mode) if (!(state->events)) { switch (state->type) { case ERTS_EV_TYPE_DRV_SEL: - ASSERT(!erts_port_task_is_scheduled(&state->driver.select->intask)); - ASSERT(!erts_port_task_is_scheduled(&state->driver.select->outtask)); - erts_free(ERTS_ALC_T_DRV_SEL_D_STATE, - state->driver.select); + state->driver.select->inport = NIL; + state->driver.select->outport = NIL; break; #if ERTS_CIO_HAVE_DRV_EVENT case ERTS_EV_TYPE_DRV_EV: - ASSERT(!erts_port_task_is_scheduled(&state->driver.event->task)); - erts_free(ERTS_ALC_T_DRV_EV_D_STATE, - state->driver.event); + state->driver.event->port = NIL; + state->driver.event->data = NULL; + state->driver.event->removed_events = (ErtsPollEvents) 0; break; #endif case ERTS_EV_TYPE_NONE: @@ -462,20 +542,297 @@ deselect(ErtsDrvEventState *state, int mode) break; } - state->driver.select = NULL; state->type = ERTS_EV_TYPE_NONE; - state->flags = 0; + state->flags &= ~ERTS_EV_FLAG_USED; remember_removed(state, &pollset); } } - #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS # define IS_FD_UNKNOWN(state) ((state)->type == ERTS_EV_TYPE_NONE && (state)->remove_cnt == 0) #else # define IS_FD_UNKNOWN(state) ((state) == NULL) #endif +static ERTS_INLINE void +check_fd_cleanup(ErtsDrvEventState *state, +#if ERTS_CIO_HAVE_DRV_EVENT + ErtsDrvEventDataState **free_event, +#endif + ErtsDrvSelectDataState **free_select) +{ + erts_aint_t current_cio_time; + + ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(fd_mtx(state->fd))); + + current_cio_time = erts_smp_atomic_read_acqb(&erts_check_io_time); + *free_select = NULL; + if (state->driver.select + && (state->type != ERTS_EV_TYPE_DRV_SEL) + && !is_iotask_active(&state->driver.select->iniotask, current_cio_time) + && !is_iotask_active(&state->driver.select->outiotask, current_cio_time)) { + + *free_select = state->driver.select; + state->driver.select = NULL; + } + +#if ERTS_CIO_HAVE_DRV_EVENT + *free_event = NULL; + if (state->driver.event + && (state->type != ERTS_EV_TYPE_DRV_EV) + && !is_iotask_active(&state->driver.event->iotask, current_cio_time)) { + + *free_event = state->driver.event; + state->driver.event = NULL; + } +#endif + +#ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS + if (((state->type != ERTS_EV_TYPE_NONE) + | state->remove_cnt +#if ERTS_CIO_HAVE_DRV_EVENT + | (state->driver.event != NULL) +#endif + | (state->driver.select != NULL)) == 0) { + + hash_erase_drv_ev_state(state); + + } +#endif +} + +static ERTS_INLINE int +check_cleanup_active_fd(ErtsSysFdType fd, +#if ERTS_CIO_DEFER_ACTIVE_EVENTS + ErtsPollControlEntry *pce, + int *pce_ix, +#endif + erts_aint_t current_cio_time) +{ + ErtsDrvEventState *state; + int active = 0; + erts_smp_mtx_t *mtx = fd_mtx(fd); + void *free_select = NULL; +#if ERTS_CIO_HAVE_DRV_EVENT + void *free_event = NULL; +#endif +#if ERTS_CIO_DEFER_ACTIVE_EVENTS + ErtsPollEvents evon = 0, evoff = 0; +#endif + + erts_smp_mtx_lock(mtx); + +#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS + state = &drv_ev_state[(int) fd]; +#else + state = hash_get_drv_ev_state(fd); /* may be NULL! */ + if (state) +#endif + { + if (state->driver.select) { +#if ERTS_CIO_DEFER_ACTIVE_EVENTS + if (is_iotask_active(&state->driver.select->iniotask, current_cio_time)) { + active = 1; + if ((state->events & ERTS_POLL_EV_IN) + && !(state->flags & ERTS_EV_FLAG_DEFER_IN_EV)) { + evoff |= ERTS_POLL_EV_IN; + state->flags |= ERTS_EV_FLAG_DEFER_IN_EV; + } + } + else if (state->flags & ERTS_EV_FLAG_DEFER_IN_EV) { + if (state->events & ERTS_POLL_EV_IN) + evon |= ERTS_POLL_EV_IN; + state->flags &= ~ERTS_EV_FLAG_DEFER_IN_EV; + } + if (is_iotask_active(&state->driver.select->outiotask, current_cio_time)) { + active = 1; + if ((state->events & ERTS_POLL_EV_OUT) + && !(state->flags & ERTS_EV_FLAG_DEFER_OUT_EV)) { + evoff |= ERTS_POLL_EV_OUT; + state->flags |= ERTS_EV_FLAG_DEFER_OUT_EV; + } + } + else if (state->flags & ERTS_EV_FLAG_DEFER_OUT_EV) { + if (state->events & ERTS_POLL_EV_OUT) + evon |= ERTS_POLL_EV_OUT; + state->flags &= ~ERTS_EV_FLAG_DEFER_OUT_EV; + } + if (active) + (void) 0; + else +#else + if (is_iotask_active(&state->driver.select->iniotask, current_cio_time) + || is_iotask_active(&state->driver.select->outiotask, current_cio_time)) + active = 1; + else +#endif + if (state->type != ERTS_EV_TYPE_DRV_SEL) { + free_select = state->driver.select; + state->driver.select = NULL; + } + } + +#if ERTS_CIO_HAVE_DRV_EVENT + if (state->driver.event) { + if (is_iotask_active(&state->driver.event->iotask, current_cio_time)) { +#if ERTS_CIO_DEFER_ACTIVE_EVENTS + ErtsPollEvents evs = state->events & ~state->driver.event->deferred_events; + if (evs) { + evoff |= evs; + state->driver.event->deferred_events |= evs; + } +#endif + active = 1; + } + else if (state->type != ERTS_EV_TYPE_DRV_EV) { + free_event = state->driver.event; + state->driver.event = NULL; + } +#if ERTS_CIO_DEFER_ACTIVE_EVENTS + else { + ErtsPollEvents evs = state->events & state->driver.event->deferred_events; + if (evs) { + evon |= evs; + state->driver.event->deferred_events = 0; + } + } +#endif + + } +#endif + +#ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS + if (((state->type != ERTS_EV_TYPE_NONE) | state->remove_cnt | active) == 0) + hash_erase_drv_ev_state(state); +#endif + + } + + erts_smp_mtx_unlock(mtx); + + if (free_select) + free_drv_select_data(free_select); +#if ERTS_CIO_HAVE_DRV_EVENT + if (free_event) + free_drv_event_data(free_event); +#endif + +#if ERTS_CIO_DEFER_ACTIVE_EVENTS + if (evoff) { + ErtsPollControlEntry *pcep = &pce[(*pce_ix)++]; + pcep->fd = fd; + pcep->events = evoff; + pcep->on = 0; + } + if (evon) { + ErtsPollControlEntry *pcep = &pce[(*pce_ix)++]; + pcep->fd = fd; + pcep->events = evon; + pcep->on = 1; + } +#endif + + return active; +} + +static void +check_cleanup_active_fds(erts_aint_t current_cio_time) +{ + int six = pollset.active_fd.six; + int eix = pollset.active_fd.eix; + erts_aint32_t no = erts_smp_atomic32_read_dirty(&pollset.active_fd.no); + int size = pollset.active_fd.size; + int ix = six; +#if ERTS_CIO_DEFER_ACTIVE_EVENTS + /* every fd might add two entries */ + Uint pce_sz = 2*sizeof(ErtsPollControlEntry)*no; + ErtsPollControlEntry *pctrl_entries = (pce_sz + ? erts_alloc(ERTS_ALC_T_TMP, pce_sz) + : NULL); + int pctrl_ix = 0; +#endif + + while (ix != eix) { + ErtsSysFdType fd = pollset.active_fd.array[ix]; + int nix = ix + 1; + if (nix >= size) + nix = 0; + ASSERT(fd != ERTS_SYS_FD_INVALID); + if (!check_cleanup_active_fd(fd, +#if ERTS_CIO_DEFER_ACTIVE_EVENTS + pctrl_entries, + &pctrl_ix, +#endif + current_cio_time)) { + no--; + if (ix == six) { +#ifdef DEBUG + pollset.active_fd.array[ix] = ERTS_SYS_FD_INVALID; +#endif + six = nix; + } + else { + pollset.active_fd.array[ix] = pollset.active_fd.array[six]; +#ifdef DEBUG + pollset.active_fd.array[six] = ERTS_SYS_FD_INVALID; +#endif + six++; + if (six >= size) + six = 0; + } + } + ix = nix; + } + +#if ERTS_CIO_DEFER_ACTIVE_EVENTS + ASSERT(pctrl_ix <= pce_sz/sizeof(ErtsPollControlEntry)); + if (pctrl_ix) + ERTS_CIO_POLL_CTLV(pollset.ps, pctrl_entries, pctrl_ix); + if (pctrl_entries) + erts_free(ERTS_ALC_T_TMP, pctrl_entries); +#endif + + pollset.active_fd.six = six; + pollset.active_fd.eix = eix; + erts_smp_atomic32_set_relb(&pollset.active_fd.no, no); +} + +static ERTS_INLINE void +add_active_fd(ErtsSysFdType fd) +{ + int eix = pollset.active_fd.eix; + int size = pollset.active_fd.size; + + + pollset.active_fd.array[eix] = fd; + + erts_smp_atomic32_set_relb(&pollset.active_fd.no, + (erts_smp_atomic32_read_dirty(&pollset.active_fd.no) + + 1)); + + eix++; + if (eix >= size) + eix = 0; + if (pollset.active_fd.six == eix) { + pollset.active_fd.six = 0; + eix = size; + size += ERTS_ACTIVE_FD_INC; + pollset.active_fd.array = erts_realloc(ERTS_ALC_T_ACTIVE_FD_ARR, + pollset.active_fd.array, + sizeof(ErtsSysFdType)*size); + pollset.active_fd.size = size; +#ifdef DEBUG + { + int i; + for (i = eix + 1; i < size; i++) + pollset.active_fd.array[i] = ERTS_SYS_FD_INVALID; + } +#endif + + } + + pollset.active_fd.eix = eix; +} int ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix, @@ -492,6 +849,10 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix, ErtsDrvEventState *state; int wake_poller; int ret; +#if ERTS_CIO_HAVE_DRV_EVENT + ErtsDrvEventDataState *free_event = NULL; +#endif + ErtsDrvSelectDataState *free_select = NULL; #ifdef USE_VM_PROBES DTRACE_CHARBUF(name, 64); #endif @@ -593,9 +954,9 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix, if (new_events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL)) { if (state->type == ERTS_EV_TYPE_DRV_SEL && !state->events) { state->type = ERTS_EV_TYPE_NONE; - state->flags = 0; - erts_free(ERTS_ALC_T_DRV_SEL_D_STATE, state->driver.select); - state->driver.select = NULL; + state->flags &= ~ERTS_EV_FLAG_USED; + state->driver.select->inport = NIL; + state->driver.select->outport = NIL; } ret = -1; goto done; @@ -613,18 +974,10 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix, state->events = new_events; if (ctl_events) { if (on) { - if (state->type == ERTS_EV_TYPE_NONE) { - ErtsDrvSelectDataState *dsdsp - = erts_alloc(ERTS_ALC_T_DRV_SEL_D_STATE, - sizeof(ErtsDrvSelectDataState)); - dsdsp->inport = NIL; - dsdsp->outport = NIL; - erts_port_task_handle_init(&dsdsp->intask); - erts_port_task_handle_init(&dsdsp->outtask); - ASSERT(state->driver.select == NULL); - state->driver.select = dsdsp; + if (!state->driver.select) + state->driver.select = alloc_drv_select_data(); + if (state->type == ERTS_EV_TYPE_NONE) state->type = ERTS_EV_TYPE_DRV_SEL; - } ASSERT(state->type == ERTS_EV_TYPE_DRV_SEL); if (ctl_events & ERTS_POLL_EV_IN) state->driver.select->inport = id; @@ -645,17 +998,12 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix, state->driver.select->outport = NIL; } if (new_events == 0) { - ASSERT(!erts_port_task_is_scheduled(&state->driver.select->intask)); - ASSERT(!erts_port_task_is_scheduled(&state->driver.select->outtask)); if (old_events != 0) { remember_removed(state, &pollset); } if ((mode & ERL_DRV_USE) || !(state->flags & ERTS_EV_FLAG_USED)) { state->type = ERTS_EV_TYPE_NONE; - state->flags = 0; - erts_free(ERTS_ALC_T_DRV_SEL_D_STATE, - state->driver.select); - state->driver.select = NULL; + state->flags &= ~ERTS_EV_FLAG_USED; } /*else keep it, as fd will probably be selected upon again */ } @@ -686,13 +1034,15 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix, ret = 0; -done:; -#ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS - if (state->type == ERTS_EV_TYPE_NONE && state->remove_cnt == 0) { - hash_erase_drv_ev_state(state); - } +done: + + check_fd_cleanup(state, +#if ERTS_CIO_HAVE_DRV_EVENT + &free_event, #endif -done_unknown: + &free_select); + +done_unknown: erts_smp_mtx_unlock(fd_mtx(fd)); if (stop_select_fn) { int was_unmasked = erts_block_fpe(); @@ -700,6 +1050,12 @@ done_unknown: (*stop_select_fn)(e, NULL); erts_unblock_fpe(was_unmasked); } + if (free_select) + free_drv_select_data(free_select); +#if ERTS_CIO_HAVE_DRV_EVENT + if (free_event) + free_drv_event_data(free_event); +#endif return ret; } @@ -719,6 +1075,10 @@ ERTS_CIO_EXPORT(driver_event)(ErlDrvPort ix, ErtsDrvEventState *state; int do_wake = 0; int ret; +#if ERTS_CIO_HAVE_DRV_EVENT + ErtsDrvEventDataState *free_event; +#endif + ErtsDrvSelectDataState *free_select; Port *prt = erts_drvport2port(ix); if (prt == ERTS_INVALID_ERL_DRV_PORT) @@ -799,10 +1159,8 @@ ERTS_CIO_EXPORT(driver_event)(ErlDrvPort ix, state->driver.event->removed_events |= remove_events; } else { - state->driver.event - = erts_alloc(ERTS_ALC_T_DRV_EV_D_STATE, - sizeof(ErtsDrvEventDataState)); - erts_port_task_handle_init(&state->driver.event->task); + if (!state->driver.event) + state->driver.event = alloc_drv_event_data(); state->driver.event->port = id; state->driver.event->removed_events = (ErtsPollEvents) 0; state->type = ERTS_EV_TYPE_DRV_EV; @@ -812,10 +1170,10 @@ ERTS_CIO_EXPORT(driver_event)(ErlDrvPort ix, else { if (state->type == ERTS_EV_TYPE_DRV_EV) { abort_tasks(state, 0); - erts_free(ERTS_ALC_T_DRV_EV_D_STATE, - state->driver.event); + state->driver.event->port = NIL; + state->driver.event->data = NULL; + state->driver.event->removed_events = (ErtsPollEvents) 0; } - state->driver.select = NULL; state->type = ERTS_EV_TYPE_NONE; remember_removed(state, &pollset); } @@ -825,12 +1183,22 @@ ERTS_CIO_EXPORT(driver_event)(ErlDrvPort ix, ret = 0; done: -#ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS - if (state->type == ERTS_EV_TYPE_NONE && state->remove_cnt == 0) { - hash_erase_drv_ev_state(state); - } + + check_fd_cleanup(state, +#if ERTS_CIO_HAVE_DRV_EVENT + &free_event, #endif + &free_select); + erts_smp_mtx_unlock(fd_mtx(fd)); + + if (free_select) + free_drv_select_data(free_select); +#if ERTS_CIO_HAVE_DRV_EVENT + if (free_event) + free_drv_event_data(free_event); +#endif + return ret; #endif } @@ -1027,7 +1395,7 @@ steal_pending_stop_select(erts_dsprintf_buf_t *dsbufp, ErlDrvPort ix, * In either case stop_select should not be called. */ state->type = ERTS_EV_TYPE_NONE; - state->flags = 0; + state->flags &= ~ERTS_EV_FLAG_USED; if (state->driver.drv_ptr->handle) { erts_ddll_dereference_driver(state->driver.drv_ptr->handle); } @@ -1099,38 +1467,103 @@ event_large_fd_error(ErlDrvPort ix, ErtsSysFdType fd, ErlDrvEventData event_data #endif #endif +static ERTS_INLINE int +io_task_schedule_allowed(ErtsDrvEventState *state, + ErtsPortTaskType type, + erts_aint_t current_cio_time) +{ + ErtsIoTask *io_task; + + switch (type) { + case ERTS_PORT_TASK_INPUT: + if (!state->driver.select) + return 0; +#if ERTS_CIO_HAVE_DRV_EVENT + if (state->driver.event) + return 0; +#endif + io_task = &state->driver.select->iniotask; + break; + case ERTS_PORT_TASK_OUTPUT: + if (!state->driver.select) + return 0; +#if ERTS_CIO_HAVE_DRV_EVENT + if (state->driver.event) + return 0; +#endif + io_task = &state->driver.select->outiotask; + break; +#if ERTS_CIO_HAVE_DRV_EVENT + case ERTS_PORT_TASK_EVENT: + if (!state->driver.event) + return 0; + if (state->driver.select) + return 0; + io_task = &state->driver.event->iotask; + break; +#endif + default: + ERTS_INTERNAL_ERROR("Invalid I/O-task type"); + return 0; + } + + return !is_iotask_active(io_task, current_cio_time); +} + static ERTS_INLINE void -iready(Eterm id, ErtsDrvEventState *state) +iready(Eterm id, ErtsDrvEventState *state, erts_aint_t current_cio_time) { - if (erts_port_task_schedule(id, - &state->driver.select->intask, - ERTS_PORT_TASK_INPUT, - (ErlDrvEvent) state->fd) != 0) { - stale_drv_select(id, state, ERL_DRV_READ); + if (io_task_schedule_allowed(state, + ERTS_PORT_TASK_INPUT, + current_cio_time)) { + ErtsIoTask *iotask = &state->driver.select->iniotask; + erts_smp_atomic_set_nob(&iotask->executed_time, current_cio_time); + if (erts_port_task_schedule(id, + &iotask->task, + ERTS_PORT_TASK_INPUT, + (ErlDrvEvent) state->fd) != 0) { + stale_drv_select(id, state, ERL_DRV_READ); + } + add_active_fd(state->fd); } } static ERTS_INLINE void -oready(Eterm id, ErtsDrvEventState *state) +oready(Eterm id, ErtsDrvEventState *state, erts_aint_t current_cio_time) { - if (erts_port_task_schedule(id, - &state->driver.select->outtask, - ERTS_PORT_TASK_OUTPUT, - (ErlDrvEvent) state->fd) != 0) { - stale_drv_select(id, state, ERL_DRV_WRITE); + if (io_task_schedule_allowed(state, + ERTS_PORT_TASK_OUTPUT, + current_cio_time)) { + ErtsIoTask *iotask = &state->driver.select->outiotask; + erts_smp_atomic_set_nob(&iotask->executed_time, current_cio_time); + if (erts_port_task_schedule(id, + &iotask->task, + ERTS_PORT_TASK_OUTPUT, + (ErlDrvEvent) state->fd) != 0) { + stale_drv_select(id, state, ERL_DRV_WRITE); + } + add_active_fd(state->fd); } } #if ERTS_CIO_HAVE_DRV_EVENT static ERTS_INLINE void -eready(Eterm id, ErtsDrvEventState *state, ErlDrvEventData event_data) +eready(Eterm id, ErtsDrvEventState *state, ErlDrvEventData event_data, + erts_aint_t current_cio_time) { - if (erts_port_task_schedule(id, - &state->driver.event->task, - ERTS_PORT_TASK_EVENT, - (ErlDrvEvent) state->fd, - event_data) != 0) { - stale_drv_select(id, state, 0); + if (io_task_schedule_allowed(state, + ERTS_PORT_TASK_EVENT, + current_cio_time)) { + ErtsIoTask *iotask = &state->driver.event->iotask; + erts_smp_atomic_set_nob(&iotask->executed_time, current_cio_time); + if (erts_port_task_schedule(id, + &iotask->task, + ERTS_PORT_TASK_EVENT, + (ErlDrvEvent) state->fd, + event_data) != 0) { + stale_drv_select(id, state, 0); + } + add_active_fd(state->fd); } } #endif @@ -1161,10 +1594,11 @@ ERTS_CIO_EXPORT(erts_check_io_interrupt_timed)(int set, void ERTS_CIO_EXPORT(erts_check_io)(int do_wait) { - ErtsPollResFd pollres[256]; + ErtsPollResFd *pollres; int pollres_len; SysTimeval wait_time; int poll_ret, i; + erts_aint_t current_cio_time; restart: @@ -1181,10 +1615,24 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait) wait_time.tv_usec = 0; } + /* + * No need for an atomic inc op when incrementing + * erts_check_io_time, since only one thread can + * check io at a time. + */ + current_cio_time = erts_smp_atomic_read_dirty(&erts_check_io_time); + current_cio_time++; + erts_smp_atomic_set_relb(&erts_check_io_time, current_cio_time); + + check_cleanup_active_fds(current_cio_time); + #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_check_exact(NULL, 0); /* No locks should be locked */ #endif - pollres_len = sizeof(pollres)/sizeof(ErtsPollResFd); + + pollres_len = erts_smp_atomic32_read_dirty(&pollset.active_fd.no) + ERTS_CHECK_IO_POLL_RES_LEN; + + pollres = erts_alloc(ERTS_ALC_T_TMP, sizeof(ErtsPollResFd)*pollres_len); erts_smp_atomic_set_nob(&pollset.in_poll_wait, 1); @@ -1204,6 +1652,7 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait) if (poll_ret != 0) { erts_smp_atomic_set_nob(&pollset.in_poll_wait, 0); forget_removed(&pollset); + erts_free(ERTS_ALC_T_TMP, pollres); if (poll_ret == EAGAIN) { goto restart; } @@ -1263,15 +1712,15 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait) if ((revents & ERTS_POLL_EV_IN) || (!(revents & ERTS_POLL_EV_OUT) && state->events & ERTS_POLL_EV_IN)) { - iready(state->driver.select->inport, state); + iready(state->driver.select->inport, state, current_cio_time); } else if (state->events & ERTS_POLL_EV_OUT) { - oready(state->driver.select->outport, state); + oready(state->driver.select->outport, state, current_cio_time); } } else if (revents & (ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT)) { if (revents & ERTS_POLL_EV_OUT) { - oready(state->driver.select->outport, state); + oready(state->driver.select->outport, state, current_cio_time); } /* Someone might have deselected input since revents was read (true also on the non-smp emulator since @@ -1279,7 +1728,7 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait) revents... */ revents &= ~(~state->events & ERTS_POLL_EV_IN); if (revents & ERTS_POLL_EV_IN) { - iready(state->driver.select->inport, state); + iready(state->driver.select->inport, state, current_cio_time); } } else if (revents & ERTS_POLL_EV_NVAL) { @@ -1287,6 +1736,7 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait) state->driver.select->inport, state->driver.select->outport, state->events); + add_active_fd(state->fd); } break; } @@ -1304,8 +1754,7 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait) if (revents) { event_data->events = state->events; event_data->revents = revents; - - eready(state->driver.event->port, state, event_data); + eready(state->driver.event->port, state, event_data, current_cio_time); } break; } @@ -1323,6 +1772,7 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait) (int) state->type); ASSERT(0); deselect(state, 0); + add_active_fd(state->fd); break; } } @@ -1334,6 +1784,7 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait) } erts_smp_atomic_set_nob(&pollset.in_poll_wait, 0); + erts_free(ERTS_ALC_T_TMP, pollres); forget_removed(&pollset); } @@ -1469,10 +1920,27 @@ static void drv_ev_state_free(void *des) void ERTS_CIO_EXPORT(erts_init_check_io)(void) { + erts_smp_atomic_init_nob(&erts_check_io_time, 0); erts_smp_atomic_init_nob(&pollset.in_poll_wait, 0); + ERTS_CIO_POLL_INIT(); pollset.ps = ERTS_CIO_NEW_POLLSET(); + pollset.active_fd.six = 0; + pollset.active_fd.eix = 0; + erts_smp_atomic32_init_nob(&pollset.active_fd.no, 0); + pollset.active_fd.size = ERTS_ACTIVE_FD_INC; + pollset.active_fd.array = erts_alloc(ERTS_ALC_T_ACTIVE_FD_ARR, + sizeof(ErtsSysFdType)*ERTS_ACTIVE_FD_INC); +#ifdef DEBUG + { + int i; + for (i = 0; i < ERTS_ACTIVE_FD_INC; i++) + pollset.active_fd.array[i] = ERTS_SYS_FD_INVALID; + } +#endif + + #ifdef ERTS_SMP init_removed_fd_alloc(); pollset.removed_list = NULL; @@ -1548,12 +2016,27 @@ Eterm ERTS_CIO_EXPORT(erts_check_io_info)(void *proc) { Process *p = (Process *) proc; - Eterm tags[15], values[15], res; + Eterm tags[16], values[16], res; Uint sz, *szp, *hp, **hpp, memory_size; Sint i; ErtsPollInfo pi; - - ERTS_CIO_POLL_INFO(pollset.ps, &pi); + erts_aint_t cio_time = erts_smp_atomic_read_acqb(&erts_check_io_time); + int active_fds = (int) erts_smp_atomic32_read_acqb(&pollset.active_fd.no); + + while (1) { + erts_aint_t post_cio_time; + int post_active_fds; + + ERTS_CIO_POLL_INFO(pollset.ps, &pi); + + post_cio_time = erts_smp_atomic_read_mb(&erts_check_io_time); + post_active_fds = (int) erts_smp_atomic32_read_acqb(&pollset.active_fd.no); + if (cio_time == post_cio_time && active_fds == post_active_fds) + break; + cio_time = post_cio_time; + active_fds = post_active_fds; + } + memory_size = pi.memory_size; #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS memory_size += sizeof(ErtsDrvEventState) * erts_smp_atomic_read_nob(&drv_ev_state_len); @@ -1617,6 +2100,9 @@ ERTS_CIO_EXPORT(erts_check_io_info)(void *proc) tags[i] = erts_bld_atom(hpp, szp, "max_fds"); values[i++] = erts_bld_uint(hpp, szp, (Uint) pi.max_fds); + tags[i] = erts_bld_atom(hpp, szp, "active_fds"); + values[i++] = erts_bld_uint(hpp, szp, (Uint) active_fds); + #ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS tags[i] = erts_bld_atom(hpp, szp, "no_avoided_wakeups"); values[i++] = erts_bld_uint(hpp, szp, (Uint) pi.no_avoided_wakeups); @@ -1671,6 +2157,8 @@ print_events(ErtsPollEvents ev) typedef struct { int used_fds; int num_errors; + int no_driver_select_structs; + int no_driver_event_structs; #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS int internal_fds; ErtsPollEvents *epep; @@ -1693,6 +2181,13 @@ static void doit_erts_check_io_debug(void *vstate, void *vcounters) struct stat stat_buf; #endif + if (state->driver.select) + counters->no_driver_select_structs++; +#if ERTS_CIO_HAVE_DRV_EVENT + if (state->driver.event) + counters->no_driver_event_structs++; +#endif + #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS if (state->events || ep_events) { if (ep_events & ERTS_POLL_EV_NVAL) { @@ -1831,6 +2326,7 @@ static void doit_erts_check_io_debug(void *vstate, void *vcounters) } } } +#if ERTS_CIO_HAVE_DRV_EVENT else if (state->type == ERTS_EV_TYPE_DRV_EV) { Eterm id; erts_printf("driver_event "); @@ -1866,6 +2362,7 @@ static void doit_erts_check_io_debug(void *vstate, void *vcounters) erts_free_port_names(pnp); } } +#endif #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS else if (internal) { erts_printf("internal "); @@ -1905,7 +2402,7 @@ static void doit_erts_check_io_debug(void *vstate, void *vcounters) } int -ERTS_CIO_EXPORT(erts_check_io_debug)(void) +ERTS_CIO_EXPORT(erts_check_io_debug)(ErtsCheckIoDebugInfo *ciodip) { #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS int fd, len; @@ -1915,6 +2412,10 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(void) ErtsDrvEventState null_des; null_des.driver.select = NULL; +#if ERTS_CIO_HAVE_DRV_EVENT + null_des.driver.event = NULL; +#endif + null_des.driver.drv_ptr = NULL; null_des.events = 0; null_des.remove_cnt = 0; null_des.type = ERTS_EV_TYPE_NONE; @@ -1935,6 +2436,8 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(void) #endif counters.used_fds = 0; counters.num_errors = 0; + counters.no_driver_select_structs = 0; + counters.no_driver_event_structs = 0; #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS len = erts_smp_atomic_read_nob(&drv_ev_state_len); @@ -1951,8 +2454,16 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(void) erts_smp_thr_progress_unblock(); + ciodip->no_used_fds = counters.used_fds; + ciodip->no_driver_select_structs = counters.no_driver_select_structs; + ciodip->no_driver_event_structs = counters.no_driver_event_structs; + erts_printf("\n"); erts_printf("used fds=%d\n", counters.used_fds); + erts_printf("Number of driver_select() structures=%d\n", counters.no_driver_select_structs); +#if ERTS_CIO_HAVE_DRV_EVENT + erts_printf("Number of driver_event() structures=%d\n", counters.no_driver_event_structs); +#endif #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS erts_printf("internal fds=%d\n", counters.internal_fds); #endif @@ -1961,6 +2472,7 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(void) #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS erts_free(ERTS_ALC_T_TMP, (void *) counters.epep); #endif + return counters.num_errors; } diff --git a/erts/emulator/sys/common/erl_check_io.h b/erts/emulator/sys/common/erl_check_io.h index edab7947ba..d01297d55c 100644 --- a/erts/emulator/sys/common/erl_check_io.h +++ b/erts/emulator/sys/common/erl_check_io.h @@ -26,6 +26,7 @@ #ifndef ERL_CHECK_IO_H__ #define ERL_CHECK_IO_H__ +#include "sys.h" #include "erl_sys_driver.h" #ifdef ERTS_ENABLE_KERNEL_POLL @@ -52,8 +53,8 @@ void erts_check_io_kp(int); void erts_check_io_nkp(int); void erts_init_check_io_kp(void); void erts_init_check_io_nkp(void); -int erts_check_io_debug_kp(void); -int erts_check_io_debug_nkp(void); +int erts_check_io_debug_kp(ErtsCheckIoDebugInfo *); +int erts_check_io_debug_nkp(ErtsCheckIoDebugInfo *); #else /* !ERTS_ENABLE_KERNEL_POLL */ @@ -70,6 +71,27 @@ void erts_init_check_io(void); #endif +extern erts_smp_atomic_t erts_check_io_time; + +typedef struct { + ErtsPortTaskHandle task; + erts_smp_atomic_t executed_time; +} ErtsIoTask; + +ERTS_GLB_INLINE void erts_io_notify_port_task_executed(ErtsPortTaskHandle *pthp); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE void +erts_io_notify_port_task_executed(ErtsPortTaskHandle *pthp) +{ + ErtsIoTask *itp = (ErtsIoTask *) (((char *) pthp) - offsetof(ErtsIoTask, task)); + erts_aint_t ci_time = erts_smp_atomic_read_acqb(&erts_check_io_time); + erts_smp_atomic_set_relb(&itp->executed_time, ci_time); +} + +#endif + #endif /* ERL_CHECK_IO_H__ */ #if !defined(ERL_CHECK_IO_C__) && !defined(ERTS_ALLOC_C__) @@ -81,6 +103,16 @@ void erts_init_check_io(void); #include "erl_poll.h" #include "erl_port_task.h" +#ifdef __WIN32__ +/* + * Current erts_poll implementation for Windows cannot handle + * active events in the set of events polled. + */ +# define ERTS_CIO_DEFER_ACTIVE_EVENTS 1 +#else +# define ERTS_CIO_DEFER_ACTIVE_EVENTS 0 +#endif + /* * ErtsDrvEventDataState is used by driver_event() which is almost never * used. We allocate ErtsDrvEventDataState separate since we dont wan't @@ -91,13 +123,16 @@ typedef struct { Eterm port; ErlDrvEventData data; ErtsPollEvents removed_events; - ErtsPortTaskHandle task; +#if ERTS_CIO_DEFER_ACTIVE_EVENTS + ErtsPollEvents deferred_events; +#endif + ErtsIoTask iotask; } ErtsDrvEventDataState; typedef struct { Eterm inport; Eterm outport; - ErtsPortTaskHandle intask; - ErtsPortTaskHandle outtask; + ErtsIoTask iniotask; + ErtsIoTask outiotask; } ErtsDrvSelectDataState; #endif /* #ifndef ERL_CHECK_IO_INTERNAL__ */ diff --git a/erts/emulator/sys/common/erl_sys_common_misc.c b/erts/emulator/sys/common/erl_sys_common_misc.c index e3ba741058..e63f0bda54 100644 --- a/erts/emulator/sys/common/erl_sys_common_misc.c +++ b/erts/emulator/sys/common/erl_sys_common_misc.c @@ -44,6 +44,14 @@ #endif #endif +/* + * erts_check_io_time is used by the erl_check_io implementation. The + * global erts_check_io_time variable is declared here since there + * (often) exist two versions of erl_check_io (kernel-poll and + * non-kernel-poll), and we dont want two versions of this variable. + */ +erts_smp_atomic_t erts_check_io_time; + /* Written once and only once */ static int filename_encoding = ERL_FILENAME_UNKNOWN; diff --git a/erts/emulator/sys/unix/erl_unix_sys.h b/erts/emulator/sys/unix/erl_unix_sys.h index 176fc049a7..b2fe34c1ac 100644 --- a/erts/emulator/sys/unix/erl_unix_sys.h +++ b/erts/emulator/sys/unix/erl_unix_sys.h @@ -135,9 +135,6 @@ /* File descriptors are numbers anc consecutively allocated on Unix */ #define ERTS_SYS_CONTINOUS_FD_NUMBERS -#define HAVE_ERTS_CHECK_IO_DEBUG -int erts_check_io_debug(void); - #ifndef ERTS_SMP # undef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT # define ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c index c3d7440409..5de0c281c4 100644 --- a/erts/emulator/sys/unix/sys.c +++ b/erts/emulator/sys/unix/sys.c @@ -34,6 +34,7 @@ #include <termios.h> #include <ctype.h> #include <sys/utsname.h> +#include <sys/select.h> #ifdef ISC32 #include <sys/bsdtypes.h> @@ -91,8 +92,10 @@ static erts_smp_rwmtx_t environ_rwmtx; # else # define CHLDWTHR 0 # endif +# define FDBLOCK 1 #else # define CHLDWTHR 0 +# define FDBLOCK 0 #endif /* * [OTP-3906] @@ -121,6 +124,15 @@ struct ErtsSysReportExit_ { #endif }; +/* Used by the fd driver iff the fd could not be set to non-blocking */ +typedef struct ErtsSysBlocking_ { + ErlDrvPDL pdl; + int res; + int err; + unsigned int pkey; +} ErtsSysBlocking; + + /* This data is shared by these drivers - initialized by spawn_init() */ static struct driver_data { ErlDrvPort port_num; @@ -129,6 +141,8 @@ static struct driver_data { int pid; int alive; int status; + int terminating; + ErtsSysBlocking *blocking; } *driver_data; /* indexed by fd */ static ErtsSysReportExit *report_exit_list; @@ -284,7 +298,7 @@ struct { void (*check_io)(int); Uint (*size)(void); Eterm (*info)(void *); - int (*check_io_debug)(void); + int (*check_io_debug)(ErtsCheckIoDebugInfo *); } io_func = {0}; @@ -306,9 +320,9 @@ Eterm erts_check_io_info(void *p) } int -erts_check_io_debug(void) +erts_check_io_debug(ErtsCheckIoDebugInfo *ip) { - return (*io_func.check_io_debug)(); + return (*io_func.check_io_debug)(ip); } @@ -1108,11 +1122,16 @@ void fini_getenv_state(GETENV_STATE *state) /* Driver interfaces */ static ErlDrvData spawn_start(ErlDrvPort, char*, SysDriverOpts*); static ErlDrvData fd_start(ErlDrvPort, char*, SysDriverOpts*); +#if FDBLOCK +static void fd_async(void *); +static void fd_ready_async(ErlDrvData drv_data, ErlDrvThreadData thread_data); +#endif static ErlDrvSSizeT fd_control(ErlDrvData, unsigned int, char *, ErlDrvSizeT, char **, ErlDrvSizeT); static ErlDrvData vanilla_start(ErlDrvPort, char*, SysDriverOpts*); static int spawn_init(void); static void fd_stop(ErlDrvData); +static void fd_flush(ErlDrvData); static void stop(ErlDrvData); static void ready_input(ErlDrvData, ErlDrvEvent); static void ready_output(ErlDrvData, ErlDrvEvent); @@ -1157,8 +1176,12 @@ struct erl_drv_entry fd_driver_entry = { fd_control, NULL, outputv, - NULL, /* ready_async */ - NULL, /* flush */ +#if FDBLOCK + fd_ready_async, /* ready_async */ +#else + NULL, +#endif + fd_flush, /* flush */ NULL, /* call */ NULL, /* event */ ERL_DRV_EXTENDED_MARKER, @@ -1212,13 +1235,28 @@ static RETSIGTYPE onchld(int signum) #endif } +static int set_blocking_data(struct driver_data *dd) { + + dd->blocking = erts_alloc(ERTS_ALC_T_SYS_BLOCKING, sizeof(ErtsSysBlocking)); + + erts_smp_atomic_add_nob(&sys_misc_mem_sz, sizeof(ErtsSysBlocking)); + + dd->blocking->pdl = driver_pdl_create(dd->port_num); + dd->blocking->res = 0; + dd->blocking->err = 0; + dd->blocking->pkey = driver_async_port_key(dd->port_num); + + return 1; +} + static int set_driver_data(ErlDrvPort port_num, int ifd, int ofd, int packet_bytes, int read_write, int exit_status, - int pid) + int pid, + int is_blocking) { Port *prt; ErtsSysReportExit *report_exit; @@ -1250,8 +1288,13 @@ static int set_driver_data(ErlDrvPort port_num, driver_data[ifd].pid = pid; driver_data[ifd].alive = 1; driver_data[ifd].status = 0; + driver_data[ifd].terminating = 0; + driver_data[ifd].blocking = NULL; if (read_write & DO_WRITE) { driver_data[ifd].ofd = ofd; + if (is_blocking && FDBLOCK) + if (!set_blocking_data(driver_data+ifd)) + return -1; if (ifd != ofd) driver_data[ofd] = driver_data[ifd]; /* structure copy */ } else { /* DO_READ only */ @@ -1267,6 +1310,11 @@ static int set_driver_data(ErlDrvPort port_num, driver_data[ofd].pid = pid; driver_data[ofd].alive = 1; driver_data[ofd].status = 0; + driver_data[ofd].terminating = 0; + driver_data[ofd].blocking = NULL; + if (is_blocking && FDBLOCK) + if (!set_blocking_data(driver_data+ofd)) + return -1; return(ofd); } } @@ -1276,6 +1324,7 @@ static int spawn_init() int i; #if CHLDWTHR erts_thr_opts_t thr_opts = ERTS_THR_OPTS_DEFAULT_INITER; + thr_opts.detached = 0; thr_opts.suggested_stack_size = 0; /* Smallest possible */ #endif @@ -1755,7 +1804,7 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name, SysDriverOpts* op } res = set_driver_data(port_num, ifd[0], ofd[1], opts->packet_bytes, - opts->read_write, opts->exit_status, pid); + opts->read_write, opts->exit_status, pid, 0); /* Don't unblock SIGCHLD until now, since the call above must first complete putting away the info about our new subprocess. */ unblock_signals(); @@ -1840,6 +1889,7 @@ static ErlDrvData fd_start(ErlDrvPort port_num, char* name, SysDriverOpts* opts) { ErlDrvData res; + int non_blocking = 0; if (((opts->read_write & DO_READ) && opts->ifd >= max_files) || ((opts->read_write & DO_WRITE) && opts->ofd >= max_files)) @@ -1912,6 +1962,20 @@ static ErlDrvData fd_start(ErlDrvPort port_num, char* name, * case - it can be called with any old pre-existing file descriptors, * the relations between which (if they're even two) we can only guess * at - still, we try our best... + * + * Added note OTP 18: Some systems seem to use stdout/stderr to log data + * using unix pipes, so we cannot allow the system to block on a write. + * Therefore we use an async thread to write the data to fd's that could + * not be set to non-blocking. When no async threads are available we + * fall back on the old behaviour. + * + * Also the guarantee about what is delivered to the OS has changed. + * Pre 18 the fd driver did no flushing of data before terminating. + * Now it does. This is because we want to be able to guarantee that things + * such as escripts and friends really have outputted all data before + * terminating. This could potentially block the termination of the system + * for a very long time, but if the user wants to terminate fast she should + * use erlang:halt with flush=false. */ if (opts->read_write & DO_READ) { @@ -1934,6 +1998,7 @@ static ErlDrvData fd_start(ErlDrvPort port_num, char* name, imagine a scenario where setting non-blocking mode here would cause problems - go ahead and do it. */ + non_blocking = 1; SET_NONBLOCKING(opts->ofd); } else { /* output fd is a tty, input fd isn't */ @@ -1976,6 +2041,7 @@ static ErlDrvData fd_start(ErlDrvPort port_num, char* name, (nfd = open(tty, O_WRONLY)) != -1) { dup2(nfd, opts->ofd); close(nfd); + non_blocking = 1; SET_NONBLOCKING(opts->ofd); } } @@ -1984,8 +2050,9 @@ static ErlDrvData fd_start(ErlDrvPort port_num, char* name, } CHLD_STAT_LOCK; res = (ErlDrvData)(long)set_driver_data(port_num, opts->ifd, opts->ofd, - opts->packet_bytes, - opts->read_write, 0, -1); + opts->packet_bytes, + opts->read_write, 0, -1, + !non_blocking); CHLD_STAT_UNLOCK; return res; } @@ -2011,14 +2078,30 @@ static void nbio_stop_fd(ErlDrvPort prt, int fd) SET_BLOCKING(fd); } -static void fd_stop(ErlDrvData fd) /* Does not close the fds */ +static void fd_stop(ErlDrvData ev) /* Does not close the fds */ { int ofd; + int fd = (int)(long)ev; + ErlDrvPort prt = driver_data[fd].port_num; - nbio_stop_fd(driver_data[(int)(long)fd].port_num, (int)(long)fd); - ofd = driver_data[(int)(long)fd].ofd; - if (ofd != (int)(long)fd && ofd != -1) - nbio_stop_fd(driver_data[(int)(long)fd].port_num, (int)(long)ofd); +#if FDBLOCK + if (driver_data[fd].blocking) { + erts_free(ERTS_ALC_T_SYS_BLOCKING,driver_data[fd].blocking); + driver_data[fd].blocking = NULL; + erts_smp_atomic_add_nob(&sys_misc_mem_sz, -1*sizeof(ErtsSysBlocking)); + } +#endif + + nbio_stop_fd(prt, fd); + ofd = driver_data[fd].ofd; + if (ofd != fd && ofd != -1) + nbio_stop_fd(prt, ofd); +} + +static void fd_flush(ErlDrvData fd) +{ + if (!driver_data[(int)(long)fd].terminating) + driver_data[(int)(long)fd].terminating = 1; } static ErlDrvData vanilla_start(ErlDrvPort port_num, char* name, @@ -2041,8 +2124,8 @@ static ErlDrvData vanilla_start(ErlDrvPort port_num, char* name, CHLD_STAT_LOCK; res = (ErlDrvData)(long)set_driver_data(port_num, fd, fd, - opts->packet_bytes, - opts->read_write, 0, -1); + opts->packet_bytes, + opts->read_write, 0, -1, 0); CHLD_STAT_UNLOCK; return res; } @@ -2079,6 +2162,7 @@ static void stop(ErlDrvData fd) } } +/* used by fd_driver */ static void outputv(ErlDrvData e, ErlIOVec* ev) { int fd = (int)(long)e; @@ -2104,12 +2188,21 @@ static void outputv(ErlDrvData e, ErlIOVec* ev) ev->iov[0].iov_base = lbp; ev->iov[0].iov_len = pb; ev->size += pb; + + if (driver_data[fd].blocking && FDBLOCK) + driver_pdl_lock(driver_data[fd].blocking->pdl); + if ((sz = driver_sizeq(ix)) > 0) { driver_enqv(ix, ev, 0); + + if (driver_data[fd].blocking && FDBLOCK) + driver_pdl_unlock(driver_data[fd].blocking->pdl); + if (sz + ev->size >= (1 << 13)) set_busy_port(ix, 1); } - else { + else if (!driver_data[fd].blocking || !FDBLOCK) { + /* We try to write directly if the fd in non-blocking */ int vsize = ev->vsize > MAX_VSIZE ? MAX_VSIZE : ev->vsize; n = writev(ofd, (const void *) (ev->iov), vsize); @@ -2125,10 +2218,22 @@ static void outputv(ErlDrvData e, ErlIOVec* ev) driver_enqv(ix, ev, n); /* n is the skip value */ driver_select(ix, ofd, ERL_DRV_WRITE|ERL_DRV_USE, 1); } +#if FDBLOCK + else { + if (ev->size != 0) { + driver_enqv(ix, ev, 0); + driver_pdl_unlock(driver_data[fd].blocking->pdl); + driver_async(ix, &driver_data[fd].blocking->pkey, + fd_async, driver_data+fd, NULL); + } else { + driver_pdl_unlock(driver_data[fd].blocking->pdl); + } + } +#endif /* return 0;*/ } - +/* Used by spawn_driver and vanilla driver */ static void output(ErlDrvData e, char* buf, ErlDrvSizeT len) { int fd = (int)(long)e; @@ -2191,6 +2296,23 @@ static int port_inp_failure(ErlDrvPort port_num, int ready_fd, int res) ASSERT(res <= 0); (void) driver_select(port_num, ready_fd, ERL_DRV_READ|ERL_DRV_WRITE, 0); clear_fd_data(ready_fd); + + if (driver_data[ready_fd].blocking && FDBLOCK) { + driver_pdl_lock(driver_data[ready_fd].blocking->pdl); + if (driver_sizeq(driver_data[ready_fd].port_num) > 0) { + driver_pdl_unlock(driver_data[ready_fd].blocking->pdl); + /* We have stuff in the output queue, so we just + set the state to terminating and wait for fd_async_ready + to terminate the port */ + if (res == 0) + driver_data[ready_fd].terminating = 2; + else + driver_data[ready_fd].terminating = -err; + return 0; + } + driver_pdl_unlock(driver_data[ready_fd].blocking->pdl); + } + if (res == 0) { if (driver_data[ready_fd].report_exit) { CHLD_STAT_LOCK; @@ -2241,6 +2363,7 @@ static void ready_input(ErlDrvData e, ErlDrvEvent ready_fd) port_num = driver_data[fd].port_num; packet_bytes = driver_data[fd].packet_bytes; + if (packet_bytes == 0) { byte *read_buf = (byte *) erts_alloc(ERTS_ALC_T_SYS_READ_BUF, ERTS_SYS_READ_BUF_SZ); @@ -2364,6 +2487,8 @@ static void ready_output(ErlDrvData e, ErlDrvEvent ready_fd) if ((iv = (struct iovec*) driver_peekq(ix, &vsize)) == NULL) { driver_select(ix, ready_fd, ERL_DRV_WRITE, 0); + if (driver_data[fd].terminating) + driver_failure_atom(driver_data[fd].port_num,"normal"); return; /* 0; */ } vsize = vsize > MAX_VSIZE ? MAX_VSIZE : vsize; @@ -2389,6 +2514,82 @@ static void stop_select(ErlDrvEvent fd, void* _) close((int)fd); } +#if FDBLOCK + +static void +fd_async(void *async_data) +{ + int res; + struct driver_data *dd = (struct driver_data*)async_data; + SysIOVec *iov0; + SysIOVec *iov; + int iovlen; + int iovcnt; + int p; + /* much of this code is stolen from efile_drv:invoke_writev */ + driver_pdl_lock(dd->blocking->pdl); + iov0 = driver_peekq(dd->port_num, &iovlen); + /* Calculate iovcnt */ + for (p = 0, iovcnt = 0; iovcnt < iovlen; + p += iov0[iovcnt++].iov_len) + ; + iov = erts_alloc_fnf(ERTS_ALC_T_SYS_WRITE_BUF, + sizeof(SysIOVec)*iovcnt); + if (!iov) { + res = -1; + errno = ENOMEM; + erts_free(ERTS_ALC_T_SYS_WRITE_BUF, iov); + driver_pdl_unlock(dd->blocking->pdl); + } else { + memcpy(iov,iov0,iovcnt*sizeof(SysIOVec)); + driver_pdl_unlock(dd->blocking->pdl); + + res = writev(dd->ofd, iov, iovlen); + + erts_free(ERTS_ALC_T_SYS_WRITE_BUF, iov); + } + dd->blocking->res = res; + dd->blocking->err = errno; +} + +void fd_ready_async(ErlDrvData drv_data, + ErlDrvThreadData thread_data) { + struct driver_data *dd = (struct driver_data *)thread_data; + ErlDrvPort port_num = dd->port_num; + + ASSERT(dd->blocking); + ASSERT(dd == (driver_data + (int)(long)drv_data)); + + if (dd->blocking->res > 0) { + driver_pdl_lock(dd->blocking->pdl); + if (driver_deq(port_num, dd->blocking->res) == 0) { + driver_pdl_unlock(dd->blocking->pdl); + set_busy_port(port_num, 0); + if (dd->terminating) { + /* The port is has been ordered to terminate + from either fd_flush or port_inp_failure */ + if (dd->terminating == 1) + driver_failure_atom(port_num, "normal"); + else if (dd->terminating == 2) + driver_failure_eof(port_num); + else if (dd->terminating < 0) + driver_failure_posix(port_num, -dd->terminating); + return; /* -1; */ + } + } else { + driver_pdl_unlock(dd->blocking->pdl); + /* still data left to write in queue */ + driver_async(port_num, &dd->blocking->pkey, fd_async, dd, NULL); + return /* 0; */; + } + } else if (dd->blocking->res < 0) { + driver_failure_posix(port_num, dd->blocking->err); + return; /* -1; */ + } + return; /* 0; */ +} + +#endif void erts_do_break_handling(void) { @@ -2658,18 +2859,30 @@ void sys_preload_end(Preload* p) /* Nothing */ } -/* Read a key from console (?) */ - +/* Read a key from console, used by break.c + Here we assume that all schedulers are stopped so that erl_poll + does not interfere with the select below. +*/ int sys_get_key(fd) int fd; { - int c; + int c, ret; unsigned char rbuf[64]; + fd_set fds; fflush(stdout); /* Flush query ??? */ - if ((c = read(fd,rbuf,64)) <= 0) { - return c; + FD_ZERO(&fds); + FD_SET(fd,&fds); + + ret = select(fd+1, &fds, NULL, NULL, NULL); + + if (ret == 1) { + do { + c = read(fd,rbuf,64); + } while (c < 0 && errno == EAGAIN); + if (c <= 0) + return c; } return rbuf[0]; diff --git a/erts/emulator/sys/win32/erl_poll.c b/erts/emulator/sys/win32/erl_poll.c index 7a1d129cd5..972170d465 100644 --- a/erts/emulator/sys/win32/erl_poll.c +++ b/erts/emulator/sys/win32/erl_poll.c @@ -1085,7 +1085,7 @@ void erts_poll_controlv(ErtsPollSet ps, pcev[i].events, pcev[i].on); } - ERTS_POLLSET_LOCK(ps); + ERTS_POLLSET_UNLOCK(ps); HARDTRACEF(("Out erts_poll_controlv")); } diff --git a/erts/emulator/sys/win32/erl_win_sys.h b/erts/emulator/sys/win32/erl_win_sys.h index a78dbf64af..838f0c61eb 100644 --- a/erts/emulator/sys/win32/erl_win_sys.h +++ b/erts/emulator/sys/win32/erl_win_sys.h @@ -113,12 +113,10 @@ /* * Our own type of "FD's" */ +#define ERTS_SYS_FD_INVALID INVALID_HANDLE_VALUE #define ERTS_SYS_FD_TYPE HANDLE #define NO_FSTAT_ON_SYS_FD_TYPE 1 /* They are events, not files */ -#define HAVE_ERTS_CHECK_IO_DEBUG -int erts_check_io_debug(void); - /* * For erl_time_sup */ diff --git a/erts/emulator/test/a_SUITE.erl b/erts/emulator/test/a_SUITE.erl index 195c9c0a5f..17579be416 100644 --- a/erts/emulator/test/a_SUITE.erl +++ b/erts/emulator/test/a_SUITE.erl @@ -97,23 +97,13 @@ display_check_io(ChkIo) -> catch erlang:display('--- CHECK IO INFO ---'), catch erlang:display(ChkIo), catch erts_debug:set_internal_state(available_internal_state, true), - NoOfErrorFds = (catch erts_debug:get_internal_state(check_io_debug)), + NoOfErrorFds = (catch element(1, erts_debug:get_internal_state(check_io_debug))), catch erlang:display({'NoOfErrorFds', NoOfErrorFds}), catch erts_debug:set_internal_state(available_internal_state, false), catch erlang:display('--- CHECK IO INFO ---'), ok. get_check_io_info() -> - ChkIo = erlang:system_info(check_io), - case lists:keysearch(pending_updates, 1, ChkIo) of - {value, {pending_updates, 0}} -> - display_check_io(ChkIo), - ChkIo; - false -> - ChkIo; - _ -> - receive after 10 -> ok end, - get_check_io_info() - end. + z_SUITE:get_check_io_info(). diff --git a/erts/emulator/test/driver_SUITE.erl b/erts/emulator/test/driver_SUITE.erl index 336b6188f6..8d2c620be0 100644 --- a/erts/emulator/test/driver_SUITE.erl +++ b/erts/emulator/test/driver_SUITE.erl @@ -31,8 +31,9 @@ end_per_suite/1, init_per_group/2,end_per_group/2, init_per_testcase/2, end_per_testcase/2, + + a_test/1, outputv_echo/1, - timer_measure/1, timer_cancel/1, timer_change/1, @@ -79,7 +80,8 @@ thr_free_drv/1, async_blast/1, thr_msg_blast/1, - consume_timeslice/1]). + consume_timeslice/1, + z_test/1]). -export([bin_prefix/2]). @@ -122,19 +124,19 @@ init_per_testcase(Case, Config) when is_atom(Case), is_list(Config) -> _ -> erts_debug:set_internal_state(available_internal_state, true) end, erlang:display({init_per_testcase, Case}), - ?line 0 = erts_debug:get_internal_state(check_io_debug), + ?line 0 = element(1, erts_debug:get_internal_state(check_io_debug)), [{watchdog, Dog},{testcase, Case}|Config]. end_per_testcase(Case, Config) -> Dog = ?config(watchdog, Config), erlang:display({end_per_testcase, Case}), - ?line 0 = erts_debug:get_internal_state(check_io_debug), + ?line 0 = element(1, erts_debug:get_internal_state(check_io_debug)), ?t:timetrap_cancel(Dog). suite() -> [{ct_hooks,[ts_install_cth]}]. -all() -> - [outputv_errors, outputv_echo, queue_echo, {group, timer}, +all() -> %% Keep a_test first and z_test last... + [a_test, outputv_errors, outputv_echo, queue_echo, {group, timer}, driver_unloaded, io_ready_exit, use_fallback_pollset, bad_fd_in_pollset, driver_event, fd_change, steal_control, otp_6602, driver_system_info_base_ver, @@ -151,7 +153,8 @@ all() -> thr_free_drv, async_blast, thr_msg_blast, - consume_timeslice]. + consume_timeslice, + z_test]. groups() -> [{timer, [], @@ -917,8 +920,7 @@ steal_control_test(Hndl = {erts_poll_info, Before}) -> end. chkio_test_init(Config) when is_list(Config) -> - ?line wait_until_no_pending_updates(), - ?line ChkIo = erlang:system_info(check_io), + ?line ChkIo = get_stable_check_io_info(), ?line case catch lists:keysearch(name, 1, ChkIo) of {value, {name, erts_poll}} -> ?line ?t:format("Before test: ~p~n", [ChkIo]), @@ -937,8 +939,7 @@ chkio_test_fini({skipped, _} = Res) -> chkio_test_fini({chkio_test_result, Res, Before}) -> ?line ok = erl_ddll:unload_driver('chkio_drv'), ?line ok = erl_ddll:stop(), - ?line wait_until_no_pending_updates(), - ?line After = erlang:system_info(check_io), + ?line After = get_stable_check_io_info(), ?line ?t:format("After test: ~p~n", [After]), ?line verify_chkio_state(Before, After), ?line Res. @@ -985,7 +986,7 @@ chkio_test({erts_poll_info, Before}, ?line Fun(), ?line During = erlang:system_info(check_io), ?line erlang:display(During), - ?line 0 = erts_debug:get_internal_state(check_io_debug), + ?line 0 = element(1, erts_debug:get_internal_state(check_io_debug)), ?line ?t:format("During test: ~p~n", [During]), ?line chk_chkio_port(Port), ?line case erlang:port_control(Port, ?CHKIO_STOP, "") of @@ -1034,18 +1035,22 @@ verify_chkio_state(Before, After) -> After) end, ?line ok. - - -wait_until_no_pending_updates() -> - case lists:keysearch(pending_updates, 1, erlang:system_info(check_io)) of - {value, {pending_updates, 0}} -> - ok; - false -> - ok; +get_stable_check_io_info() -> + ChkIo = erlang:system_info(check_io), + PendUpdNo = case lists:keysearch(pending_updates, 1, ChkIo) of + {value, {pending_updates, PendNo}} -> + PendNo; + false -> + 0 + end, + {value, {active_fds, ActFds}} = lists:keysearch(active_fds, 1, ChkIo), + case {PendUpdNo, ActFds} of + {0, 0} -> + ChkIo; _ -> receive after 10 -> ok end, - wait_until_no_pending_updates() + get_stable_check_io_info() end. otp_6602(doc) -> ["Missed port lock when stealing control of fd from a " @@ -2387,10 +2392,25 @@ count_proc_sched(Ps, PNs) -> PNs end. +a_test(Config) when is_list(Config) -> + check_io_debug(). + +z_test(Config) when is_list(Config) -> + check_io_debug(). + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Utilities %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +check_io_debug() -> + get_stable_check_io_info(), + {NoErrorFds, NoUsedFds, NoDrvSelStructs, NoDrvEvStructs} + = erts_debug:get_internal_state(check_io_debug), + 0 = NoErrorFds, + NoUsedFds = NoDrvSelStructs, + 0 = NoDrvEvStructs, + ok. + %flush_msgs() -> % receive % M -> diff --git a/erts/emulator/test/port_SUITE.erl b/erts/emulator/test/port_SUITE.erl index 738d60b8a4..1bb4cb3637 100644 --- a/erts/emulator/test/port_SUITE.erl +++ b/erts/emulator/test/port_SUITE.erl @@ -90,6 +90,7 @@ mix_up_ports/1, otp_5112/1, otp_5119/1, otp_6224/1, exit_status_multi_scheduling_block/1, ports/1, spawn_driver/1, spawn_executable/1, close_deaf_port/1, + port_setget_data/1, unregister_name/1, parallelism_option/1]). -export([do_iter_max_ports/2]). @@ -115,6 +116,7 @@ all() -> mix_up_ports, otp_5112, otp_5119, exit_status_multi_scheduling_block, ports, spawn_driver, spawn_executable, close_deaf_port, unregister_name, + port_setget_data, parallelism_option]. groups() -> @@ -2339,6 +2341,55 @@ close_deaf_port_1(N, Cmd) -> {comment, "Could not spawn more than " ++ integer_to_list(N) ++ " OS processes."} end. +%% Test undocumented port_set_data/2 and port_get_data/1 +%% Hammer from multiple processes a while +%% and then abrubtly close the port (OTP-12208). +port_setget_data(Config) when is_list(Config) -> + ok = load_driver(?config(data_dir, Config), "echo_drv"), + Port = erlang:open_port({spawn_driver, "echo_drv"}, []), + + NSched = erlang:system_info(schedulers_online), + PRs = lists:map(fun(I) -> + spawn_opt(fun() -> port_setget_data_hammer(Port,1) end, + [monitor, {scheduler, I rem NSched}]) + end, + lists:seq(1,10)), + receive after 100 -> ok end, + Papa = self(), + lists:foreach(fun({Pid,_}) -> Pid ! {Papa,prepare_for_close} end, PRs), + lists:foreach(fun({Pid,_}) -> + receive {Pid,prepare_for_close} -> ok end + end, + PRs), + port_close(Port), + lists:foreach(fun({Pid,Ref}) -> + receive {'DOWN', Ref, process, Pid, normal} -> ok end + end, + PRs), + ok. + +port_setget_data_hammer(Port, N) -> + Rand = random:uniform(3), + try case Rand of + 1 -> true = erlang:port_set_data(Port, atom); + 2 -> true = erlang:port_set_data(Port, {1,2,3}); + 3 -> erlang:port_get_data(Port) + end + catch + error:badarg -> + true = get(prepare_for_close), + io:format("~p did ~p rounds before port closed\n", [self(), N]), + exit(normal) + end, + receive {Papa, prepare_for_close} -> + put(prepare_for_close, true), + Papa ! {self(),prepare_for_close} + after 0 -> + ok + end, + port_setget_data_hammer(Port, N+1). + + wait_until(Fun) -> case catch Fun() of true -> diff --git a/erts/emulator/test/z_SUITE.erl b/erts/emulator/test/z_SUITE.erl index 4b3075a164..b0c6224dfe 100644 --- a/erts/emulator/test/z_SUITE.erl +++ b/erts/emulator/test/z_SUITE.erl @@ -38,7 +38,7 @@ -export([schedulers_alive/1, node_container_refc_check/1, long_timers/1, pollset_size/1, - check_io_debug/1]). + check_io_debug/1, get_check_io_info/0]). -define(DEFAULT_TIMEOUT, ?t:minutes(5)). @@ -288,11 +288,14 @@ check_io_debug(Config) when is_list(Config) -> end. check_io_debug_test() -> + ?line erlang:display(get_check_io_info()), ?line erts_debug:set_internal_state(available_internal_state, true), - ?line erlang:display(erlang:system_info(check_io)), - ?line NoOfErrorFds = erts_debug:get_internal_state(check_io_debug), + ?line {NoErrorFds, NoUsedFds, NoDrvSelStructs, NoDrvEvStructs} + = erts_debug:get_internal_state(check_io_debug), ?line erts_debug:set_internal_state(available_internal_state, false), - ?line 0 = NoOfErrorFds, + ?line 0 = NoErrorFds, + ?line NoUsedFds = NoDrvSelStructs, + ?line 0 = NoDrvEvStructs, ?line ok. @@ -305,7 +308,7 @@ display_check_io(ChkIo) -> catch erlang:display('--- CHECK IO INFO ---'), catch erlang:display(ChkIo), catch erts_debug:set_internal_state(available_internal_state, true), - NoOfErrorFds = (catch erts_debug:get_internal_state(check_io_debug)), + NoOfErrorFds = (catch element(1, erts_debug:get_internal_state(check_io_debug))), catch erlang:display({'NoOfErrorFds', NoOfErrorFds}), catch erts_debug:set_internal_state(available_internal_state, false), catch erlang:display('--- CHECK IO INFO ---'), @@ -313,14 +316,19 @@ display_check_io(ChkIo) -> get_check_io_info() -> ChkIo = erlang:system_info(check_io), - case lists:keysearch(pending_updates, 1, ChkIo) of - {value, {pending_updates, 0}} -> + PendUpdNo = case lists:keysearch(pending_updates, 1, ChkIo) of + {value, {pending_updates, PendNo}} -> + PendNo; + false -> + 0 + end, + {value, {active_fds, ActFds}} = lists:keysearch(active_fds, 1, ChkIo), + case {PendUpdNo, ActFds} of + {0, 0} -> display_check_io(ChkIo), ChkIo; - false -> - ChkIo; _ -> - receive after 10 -> ok end, + receive after 100 -> ok end, get_check_io_info() end. diff --git a/erts/etc/common/erlexec.c b/erts/etc/common/erlexec.c index 709c6f02d1..5ebde8ca3c 100644 --- a/erts/etc/common/erlexec.c +++ b/erts/etc/common/erlexec.c @@ -128,6 +128,7 @@ static char *pluss_val_switches[] = { "bwt", "cl", "ct", + "ecio", "fwi", "tbt", "wct", diff --git a/erts/etc/unix/etp-commands.in b/erts/etc/unix/etp-commands.in index ee27770e51..1a723ad936 100644 --- a/erts/etc/unix/etp-commands.in +++ b/erts/etc/unix/etp-commands.in @@ -3561,6 +3561,39 @@ document etp-carrier-blocks %--------------------------------------------------------------------------- end +define etp-address-to-beam-opcode + set $etp_i = 0 + set $etp_min_diff = ((UWord)1 << (sizeof(UWord)*8 - 1)) + set $etp_min_opcode = -1 + set $etp_addr = (UWord) ($arg0) + + while $etp_i < num_instructions && $etp_min_diff > 0 + if ($etp_addr - (UWord)beam_ops[$etp_i]) < $etp_min_diff + set $etp_min_diff = $etp_addr - (UWord)beam_ops[$etp_i] + set $etp_min_opcode = $etp_i + end + set $etp_i = $etp_i + 1 + end + if $etp_min_diff == 0 + printf "Address %p is start of '%s'\n", $etp_addr, opc[$etp_min_opcode].name + else + if $etp_min_opcode >= 0 + printf "Address is %ld bytes into opcode '%s' at %p\n", $etp_min_diff, opc[$etp_min_opcode].name, beam_ops[$etp_min_opcode] + else + printf "Invalid opcode address\n" + end + end +end + +document etp-address-to-beam-opcode +%--------------------------------------------------------------------------- +% Get beam opcode from a native instruction address (within process_main()) +% Arg: Instructon pointer value +% +% Does not work with NO_JUMP_TABLE +%--------------------------------------------------------------------------- +end + ############################################################################ # Toolbox parameter handling diff --git a/erts/etc/win32/Install.c b/erts/etc/win32/Install.c index 500fd166f8..9d85d642ab 100644 --- a/erts/etc/win32/Install.c +++ b/erts/etc/win32/Install.c @@ -80,7 +80,7 @@ int wmain(int argc, wchar_t **argv) } } if (root == NULL) { - if (module = NULL) { + if (module == NULL) { fprintf(stderr, "Cannot GetModuleHandle()\n"); exit(1); } diff --git a/erts/etc/win32/erl.c b/erts/etc/win32/erl.c index 1d116bf36e..772b668586 100644 --- a/erts/etc/win32/erl.c +++ b/erts/etc/win32/erl.c @@ -264,7 +264,7 @@ static void get_parameters(void) int len; - if (module = NULL) { + if (module == NULL) { error("Cannot GetModuleHandle()"); } diff --git a/erts/preloaded/ebin/erlang.beam b/erts/preloaded/ebin/erlang.beam Binary files differindex 32ff6a3874..d0f9907709 100644 --- a/erts/preloaded/ebin/erlang.beam +++ b/erts/preloaded/ebin/erlang.beam diff --git a/erts/preloaded/src/erlang.erl b/erts/preloaded/src/erlang.erl index b96a601792..83a38da26b 100644 --- a/erts/preloaded/src/erlang.erl +++ b/erts/preloaded/src/erlang.erl @@ -2239,6 +2239,7 @@ tuple_to_list(_Tuple) -> (dynamic_trace) -> none | dtrace | systemtap; (dynamic_trace_probes) -> boolean(); (elib_malloc) -> false; + (eager_check_io) -> boolean(); (ets_limit) -> pos_integer(); (fullsweep_after) -> {fullsweep_after, non_neg_integer()}; (garbage_collection) -> [{atom(), integer()}]; |