Merge branch 'rickard/eager-check-io/OTP-12117' into rickard/maint-17/eager-check-io/OTP-12117

* rickard/eager-check-io/OTP-12117:
  Introduce support for eager check I/O scheduling

Conflicts:
	erts/emulator/beam/erl_bif_info.c
	erts/emulator/beam/erl_init.c
	erts/emulator/beam/erl_process.c
	erts/preloaded/ebin/erlang.beam

9 files changed, 120 insertions, 35 deletions

diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 37354b7f8d..029ac09b26 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -397,6 +397,7 @@ type	DRV_EV_STATE	LONG_LIVED	SYSTEM		driver_event_state
 type	DRV_EV_D_STATE	FIXED_SIZE	SYSTEM		driver_event_data_state
 type	DRV_SEL_D_STATE	FIXED_SIZE	SYSTEM		driver_select_data_state
 type	FD_LIST		SHORT_LIVED	SYSTEM		fd_list
+type	ACTIVE_FD_ARR	SHORT_LIVED	SYSTEM		active_fd_array
 type	POLLSET		LONG_LIVED	SYSTEM		pollset
 type	POLLSET_UPDREQ	SHORT_LIVED	SYSTEM		pollset_update_req
 type	POLL_FDS	LONG_LIVED	SYSTEM		poll_fds
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 6efe9d9550..61e4469600 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -2696,6 +2696,9 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 		? am_disabled
 		: am_enabled);
     }
+    else if (ERTS_IS_ATOM_STR("eager_check_io",BIF_ARG_1)) {
+	BIF_RET(erts_eager_check_io ? am_true : am_false);
+    }
 
     BIF_ERROR(BIF_P, BADARG);
 }
@@ -3304,17 +3307,38 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
 	    BIF_RET(make_small((Uint) words));
 	}
 	else if (ERTS_IS_ATOM_STR("check_io_debug", BIF_ARG_1)) {
-	    /* Used by (emulator) */
-	    int res;
+	    /* Used by driver_SUITE (emulator) */
+	    Uint sz, *szp;
+	    Eterm res, *hp, **hpp;
+	    int no_errors;
+	    ErtsCheckIoDebugInfo ciodi = {0};
 #ifdef HAVE_ERTS_CHECK_IO_DEBUG
 	    erts_smp_proc_unlock(BIF_P,ERTS_PROC_LOCK_MAIN);
-	    res = erts_check_io_debug();
+	    no_errors = erts_check_io_debug(&ciodi);
 	    erts_smp_proc_lock(BIF_P,ERTS_PROC_LOCK_MAIN);
 #else
-	    res = 0;
+	    no_errors = 0;
 #endif
-	    ASSERT(res >= 0);
-	    BIF_RET(erts_make_integer((Uint) res, BIF_P));
+	    sz = 0;
+	    szp = &sz;
+	    hpp = NULL;
+	    while (1) {
+		res = erts_bld_tuple(hpp, szp, 4,
+				     erts_bld_uint(hpp, szp,
+						   (Uint) no_errors),
+				     erts_bld_uint(hpp, szp,
+						   (Uint) ciodi.no_used_fds),
+				     erts_bld_uint(hpp, szp,
+						   (Uint) ciodi.no_driver_select_structs),
+				     erts_bld_uint(hpp, szp,
+						   (Uint) ciodi.no_driver_event_structs));
+		if (hpp)
+		    break;
+		hp = HAlloc(BIF_P, sz);
+		szp = NULL;
+		hpp = &hp;
+	    }
+	    BIF_RET(res);
 	}
 	else if (ERTS_IS_ATOM_STR("process_info_args", BIF_ARG_1)) {
 	    /* Used by process_SUITE (emulator) */
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 88c4006934..ef77e1d7fc 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -548,6 +548,8 @@ void erts_usage(void)
     erts_fprintf(stderr, "               see the erl(1) documentation for more info.\n");
     erts_fprintf(stderr, "-sct cput      set cpu topology,\n");
     erts_fprintf(stderr, "               see the erl(1) documentation for more info.\n");
+    erts_fprintf(stderr, "-secio bool    enable/disable eager check I/O scheduling,\n");
+    erts_fprintf(stderr, "               see the erl(1) documentation for more info.\n");
 #if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT
     erts_fprintf(stderr, "-sub bool      enable/disable scheduler utilization balancing,\n");
 #else
@@ -1674,6 +1676,19 @@ erl_start(int argc, char **argv)
 		    erts_usage();
 		}
 	    }
+	    else if (has_prefix("ecio", sub_param)) {
+		arg = get_arg(sub_param+4, argv[i+1], &i);
+		if (sys_strcmp("true", arg) == 0)
+		    erts_eager_check_io = 1;
+		else if (sys_strcmp("false", arg) == 0)
+		    erts_eager_check_io = 0;
+		else {
+		    erts_fprintf(stderr,
+				 "bad schedule eager check I/O value '%s'\n",
+				 arg);
+		    erts_usage();
+		}
+	    }
 	    else if (has_prefix("pp", sub_param)) {
 		arg = get_arg(sub_param+2, argv[i+1], &i);
 		if (sys_strcmp(arg, "true") == 0)
diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index 682f6f8f4b..2aa0a27197 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -32,6 +32,7 @@
 #include "global.h"
 #include "erl_port_task.h"
 #include "dist.h"
+#include "erl_check_io.h"
 #include "dtrace-wrapper.h"
 #include <stdarg.h>
 
@@ -550,6 +551,16 @@ reset_handle(ErtsPortTask *ptp)
 }
 
 static ERTS_INLINE void
+reset_executed_io_task_handle(ErtsPortTask *ptp)
+{
+    if (ptp->u.alive.handle) {
+	ASSERT(ptp == handle2task(ptp->u.alive.handle));
+	erts_io_notify_port_task_executed(ptp->u.alive.handle);
+	reset_port_task_handle(ptp->u.alive.handle);
+    }
+}
+
+static ERTS_INLINE void
 set_handle(ErtsPortTask *ptp, ErtsPortTaskHandle *pthp)
 {
     ptp->u.alive.handle = pthp;
@@ -1396,10 +1407,7 @@ erts_port_task_schedule(Eterm id,
     erts_aint32_t act, add_flags;
     unsigned int prof_runnable_ports;
 
-    if (pthp && erts_port_task_is_scheduled(pthp)) {
-	ASSERT(0);
-	erts_port_task_abort(pthp);
-    }
+    ERTS_LC_ASSERT(!pthp || !erts_port_task_is_scheduled(pthp));
 
     ASSERT(is_internal_port(id));
 
@@ -1699,8 +1707,6 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 	    goto aborted_port_task;
 	}
 
-	reset_handle(ptp);
-
 	if (erts_system_monitor_long_schedule != 0) {
 	    start_time = erts_timestamp_millis();
 	}
@@ -1711,6 +1717,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 
 	switch (ptp->type) {
 	case ERTS_PORT_TASK_TIMEOUT:
+	    reset_handle(ptp);
 	    reds = ERTS_PORT_REDS_TIMEOUT;
 	    if (!(state & ERTS_PORT_SFLGS_DEAD)) {
                 DTRACE_DRIVER(driver_timeout, pp);
@@ -1725,6 +1732,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 	       for input and output */
 	    (*pp->drv_ptr->ready_input)((ErlDrvData) pp->drv_data,
 					ptp->u.alive.td.io.event);
+	    reset_executed_io_task_handle(ptp);
 	    io_tasks_executed++;
 	    break;
 	case ERTS_PORT_TASK_OUTPUT:
@@ -1733,6 +1741,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
             DTRACE_DRIVER(driver_ready_output, pp);
 	    (*pp->drv_ptr->ready_output)((ErlDrvData) pp->drv_data,
 					 ptp->u.alive.td.io.event);
+	    reset_executed_io_task_handle(ptp);
 	    io_tasks_executed++;
 	    break;
 	case ERTS_PORT_TASK_EVENT:
@@ -1742,10 +1751,12 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 	    (*pp->drv_ptr->event)((ErlDrvData) pp->drv_data,
 				  ptp->u.alive.td.io.event,
 				  ptp->u.alive.td.io.event_data);
+	    reset_executed_io_task_handle(ptp);
 	    io_tasks_executed++;
 	    break;
 	case ERTS_PORT_TASK_PROC_SIG: {
 	    ErtsProc2PortSigData *sigdp = &ptp->u.alive.td.psig.data;
+	    reset_handle(ptp);
 	    ASSERT((state & ERTS_PORT_SFLGS_DEAD) == 0);
 	    if (!pp->sched.taskq.bpq)
 		reds = ptp->u.alive.td.psig.callback(pp,
@@ -1763,6 +1774,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 	    break;
 	}
 	case ERTS_PORT_TASK_DIST_CMD:
+	    reset_handle(ptp);
 	    reds = erts_dist_command(pp, CONTEXT_REDS - pp->reds);
 	    break;
 	default:
diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h
index 9ef0cfcedc..406cd3c492 100644
--- a/erts/emulator/beam/erl_port_task.h
+++ b/erts/emulator/beam/erl_port_task.h
@@ -156,7 +156,7 @@ erts_port_task_handle_init(ErtsPortTaskHandle *pthp)
 ERTS_GLB_INLINE int
 erts_port_task_is_scheduled(ErtsPortTaskHandle *pthp)
 {
-    return ((void *) erts_smp_atomic_read_nob(pthp)) != NULL;
+    return ((void *) erts_smp_atomic_read_acqb(pthp)) != NULL;
 }
 
 ERTS_GLB_INLINE void erts_port_task_pre_init_sched(ErtsPortTaskSched *ptsp,
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 20a88ec581..65abd87c08 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -148,6 +148,7 @@ extern BeamInstr beam_apply[];
 extern BeamInstr beam_exit[];
 extern BeamInstr beam_continue_exit[];
 
+int erts_eager_check_io = 0;
 int erts_sched_compact_load;
 int erts_sched_balance_util = 0;
 Uint erts_no_schedulers;
@@ -2381,29 +2382,47 @@ try_set_sys_scheduling(void)
 #endif
 
 static ERTS_INLINE int
-prepare_for_sys_schedule(ErtsSchedulerData *esdp)
+prepare_for_sys_schedule(ErtsSchedulerData *esdp, int non_blocking)
 {
+    if (non_blocking && erts_eager_check_io) {
 #ifdef ERTS_SMP
-    while (!erts_port_task_have_outstanding_io_tasks()
-	   && try_set_sys_scheduling()) {
 #ifdef ERTS_SCHED_ONLY_POLL_SCHED_1
-      if (esdp->no != 1) {
-	/* If we are not scheduler 1 and ERTS_SCHED_ONLY_POLL_SCHED_1 is used
-	   then we make sure to wake scheduler 1 */
-	ErtsRunQueue *rq = ERTS_RUNQ_IX(0);
-	clear_sys_scheduling();
-	wake_scheduler(rq);
-	return 0;
-      }
+	if (esdp->no != 1) {
+	    /* If we are not scheduler 1 and ERTS_SCHED_ONLY_POLL_SCHED_1 is used
+	       then we make sure to wake scheduler 1 */
+	    ErtsRunQueue *rq = ERTS_RUNQ_IX(0);
+	    wake_scheduler(rq);
+	    return 0;
+	}
 #endif
-      if (!erts_port_task_have_outstanding_io_tasks())
+	return try_set_sys_scheduling();
+#else
 	return 1;
-      clear_sys_scheduling();
+#endif
     }
-    return 0;
+    else {
+#ifdef ERTS_SMP
+	while (!erts_port_task_have_outstanding_io_tasks()
+	       && try_set_sys_scheduling()) {
+#ifdef ERTS_SCHED_ONLY_POLL_SCHED_1
+	    if (esdp->no != 1) {
+		/* If we are not scheduler 1 and ERTS_SCHED_ONLY_POLL_SCHED_1 is used
+		   then we make sure to wake scheduler 1 */
+		ErtsRunQueue *rq = ERTS_RUNQ_IX(0);
+		clear_sys_scheduling();
+		wake_scheduler(rq);
+		return 0;
+	    }
+#endif
+	    if (!erts_port_task_have_outstanding_io_tasks())
+		return 1;
+	    clear_sys_scheduling();
+	}
+	return 0;
 #else
-    return !erts_port_task_have_outstanding_io_tasks();
+	return !erts_port_task_have_outstanding_io_tasks();
 #endif
+    }
 }
 
 #ifdef ERTS_SMP
@@ -2780,7 +2799,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
      * be waiting in erl_sys_schedule()
      */
 
-    if (ERTS_SCHEDULER_IS_DIRTY(esdp) || !prepare_for_sys_schedule(esdp)) {
+    if (ERTS_SCHEDULER_IS_DIRTY(esdp) || !prepare_for_sys_schedule(esdp, 0)) {
 
 	sched_waiting(esdp->no, rq);
 
@@ -2944,7 +2963,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 		 * Got to check that we still got I/O tasks; otherwise
 		 * we have to continue checking for I/O...
 		 */
-		if (!prepare_for_sys_schedule(esdp)) {
+		if (!prepare_for_sys_schedule(esdp, 0)) {
 		    spincount *= ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT;
 		    goto tse_wait;
 		}
@@ -2966,7 +2985,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	     * Got to check that we still got I/O tasks; otherwise
 	     * we have to wait in erl_sys_schedule() after all...
 	     */
-	    if (!prepare_for_sys_schedule(esdp)) {
+	    if (!prepare_for_sys_schedule(esdp, 0)) {
 		/*
 		 * Not allowed to wait in erl_sys_schedule;
 		 * do tse wait instead...
@@ -9200,7 +9219,7 @@ Process *schedule(Process *p, int calls)
 	}
 	else if (!ERTS_SCHEDULER_IS_DIRTY(esdp) &&
 		 (fcalls > input_reductions &&
-		  prepare_for_sys_schedule(esdp))) {
+		  prepare_for_sys_schedule(esdp, !0))) {
 	    /*
 	     * Schedule system-level activities.
 	     */
@@ -9208,8 +9227,6 @@ Process *schedule(Process *p, int calls)
 	    erts_smp_atomic32_set_relb(&function_calls, 0);
 	    fcalls = 0;
 
-	    ASSERT(!erts_port_task_have_outstanding_io_tasks());
-
 #if 0 /* Not needed since we wont wait in sys schedule */
 	    erts_sys_schedule_interrupt(0);
 #endif
@@ -9241,7 +9258,9 @@ Process *schedule(Process *p, int calls)
 	if (RUNQ_READ_LEN(&rq->ports.info.len)) {
 	    int have_outstanding_io;
 	    have_outstanding_io = erts_port_task_execute(rq, &esdp->current_port);
-	    if ((have_outstanding_io && fcalls > 2*input_reductions)
+	    if ((!erts_eager_check_io
+		 && have_outstanding_io
+		 && fcalls > 2*input_reductions)
 		|| rq->halt_in_progress) {
 		/*
 		 * If we have performed more than 2*INPUT_REDUCTIONS since
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 3b0798207e..27a3a3553b 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -104,6 +104,7 @@ struct saved_calls {
 };
 
 extern Export exp_send, exp_receive, exp_timeout;
+extern int erts_eager_check_io;
 extern int erts_sched_compact_load;
 extern int erts_sched_balance_util;
 extern Uint erts_no_schedulers;
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 3d8dd9c6d0..1bdb1579f5 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -66,8 +66,12 @@
  */
 
 #ifndef ERTS_SYS_FD_TYPE
+#define ERTS_SYS_FD_INVALID ((ErtsSysFdType) -1)
 typedef int ErtsSysFdType;
 #else
+#ifndef ERTS_SYS_FD_INVALID
+# error missing ERTS_SYS_FD_INVALID
+#endif
 typedef ERTS_SYS_FD_TYPE ErtsSysFdType;
 #endif
 
@@ -744,6 +748,14 @@ void init_getenv_state(GETENV_STATE *);
 char * getenv_string(GETENV_STATE *);
 void fini_getenv_state(GETENV_STATE *);
 
+#define HAVE_ERTS_CHECK_IO_DEBUG
+typedef struct {
+    int no_used_fds;
+    int no_driver_select_structs;
+    int no_driver_event_structs;
+} ErtsCheckIoDebugInfo;
+int erts_check_io_debug(ErtsCheckIoDebugInfo *ip);
+
 /* xxxP */
 #define SYS_DEFAULT_FLOAT_DECIMALS 20
 void init_sys_float(void);
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 55f9e68e78..f20e6e5665 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -48,6 +48,7 @@
 #include "erl_sched_spec_pre_alloc.h"
 #include "beam_bp.h"
 #include "erl_ptab.h"
+#include "erl_check_io.h"
 
 #undef M_TRIM_THRESHOLD
 #undef M_TOP_PAD