Merge branch 'lukas/erts/scheduler-pollset-fixes/OTP-15538' into maint-21

* lukas/erts/scheduler-pollset-fixes/OTP-15538:
  erts: Fix getting of poll events on linux >= 4.15.0
  erts: Use reduction based polling for starved poll-set
  erts: Fix pollset test cases

7 files changed, 71 insertions, 79 deletions

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 19093ebfdd..dca502c939 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -340,6 +340,7 @@ erts_sched_stat_t erts_sched_stat;
 static erts_tsd_key_t ERTS_WRITE_UNLIKELY(sched_data_key);
 
 #if ERTS_POLL_USE_SCHEDULER_POLLING
+static erts_atomic32_t function_calls;
 static erts_atomic32_t doing_sys_schedule;
 #endif
 static erts_atomic32_t no_empty_run_queues;
@@ -3247,6 +3248,7 @@ poll_thread(void *arg)
 static ERTS_INLINE void
 clear_sys_scheduling(void)
 {
+    erts_atomic32_set_relb(&function_calls, 0);
     erts_atomic32_set_mb(&doing_sys_schedule, 0);
 }
 
@@ -3269,28 +3271,6 @@ prepare_for_sys_schedule(void)
     return 0;
 }
 
-static void
-check_io_timer(void *null)
-{
-    ErtsSchedulerData *esdp = erts_get_scheduler_data();
-    if (prepare_for_sys_schedule()) {
-        erts_check_io(esdp->ssi->psi, ERTS_POLL_NO_TIMEOUT);
-        clear_sys_scheduling();
-    }
-
-    /* The timer is cleared if this schedulers run-queue became empty
-       or if the CHECKIO flag was cleared. The CHECKIO flags is cleared
-       when a check_balance assigns another scheduler to be the poller in
-       the overload scenario. */
-    if ((ERTS_RUNQ_FLGS_GET_NOB(esdp->run_queue) & (ERTS_RUNQ_FLG_OUT_OF_WORK|ERTS_RUNQ_FLG_CHECKIO))
-        == ERTS_RUNQ_FLG_CHECKIO) {
-        erts_start_timer_callback(ERTS_POLL_SCHEDULER_POLLING_TIMEOUT,
-                                  check_io_timer, NULL);
-    } else {
-        ERTS_RUNQ_FLGS_UNSET(esdp->run_queue, ERTS_RUNQ_FLG_CHECKIO);
-    }
-}
-
 #else
 #define clear_sys_scheduling()
 #define prepare_for_sys_schedule() 0
@@ -3448,6 +3428,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
                         current_time = erts_get_monotonic_time(esdp);
                     }
                 }
+                *fcalls = 0;
                 clear_sys_scheduling();
             } else {
                 if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
@@ -4711,15 +4692,6 @@ check_balance(ErtsRunQueue *c_rq)
     if (blnc_no_rqs == 1) {
 	c_rq->check_balance_reds = INT_MAX;
 	erts_atomic32_set_nob(&balance_info.checking_balance, 0);
-#if ERTS_POLL_USE_SCHEDULER_POLLING
-	c_rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS;
-        if ((ERTS_RUNQ_FLGS_GET_NOB(c_rq) & (ERTS_RUNQ_FLG_OUT_OF_WORK|ERTS_RUNQ_FLG_CHECKIO))
-             == 0) {
-            ERTS_RUNQ_FLGS_SET(c_rq, ERTS_RUNQ_FLG_CHECKIO);
-            erts_start_timer_callback(ERTS_POLL_SCHEDULER_POLLING_TIMEOUT, check_io_timer, NULL);
-        }
-        ERTS_RUNQ_FLGS_UNSET(c_rq, ERTS_RUNQ_FLGS_MIGRATION_INFO);
-#endif
 	return;
     }
 
@@ -5239,19 +5211,6 @@ erts_fprintf(stderr, "--------------------------------\n");
     /* Publish new migration paths... */
     erts_atomic_set_wb(&erts_migration_paths, (erts_aint_t) new_mpaths);
 
-#if ERTS_POLL_USE_SCHEDULER_POLLING
-    if (full_scheds == current_active) {
-        ERTS_ASSERT(full_scheds <= current_active);
-        /* All active schedulers ran for full, we need to do active polling,
-           so we setup a timer that does active polling */
-        if (!(ERTS_RUNQ_FLGS_GET_NOB(c_rq) & ERTS_RUNQ_FLG_CHECKIO)) {
-            /* Active polling is not running, start it */
-            erts_start_timer_callback(ERTS_POLL_SCHEDULER_POLLING_TIMEOUT, check_io_timer, NULL);
-        }
-        run_queue_info[c_rq->ix].flags |= ERTS_RUNQ_FLG_CHECKIO;
-    }
-#endif
-
     /* Reset balance statistics in all online queues */
     for (qix = 0; qix < blnc_no_rqs; qix++) {
 	Uint32 flags = run_queue_info[qix].flags;
@@ -5261,8 +5220,6 @@ erts_fprintf(stderr, "--------------------------------\n");
 	ASSERT(!(flags & ERTS_RUNQ_FLG_OUT_OF_WORK));
 	if (rq->waiting)
 	    flags |= ERTS_RUNQ_FLG_OUT_OF_WORK;
-        if (rq != c_rq)
-            flags &= ~ERTS_RUNQ_FLG_CHECKIO;
 
 	rq->full_reds_history_sum
 	    = run_queue_info[qix].full_reds_history_sum;
@@ -5272,7 +5229,7 @@ erts_fprintf(stderr, "--------------------------------\n");
 	ERTS_DBG_CHK_FULL_REDS_HISTORY(rq);
 
 	rq->out_of_work_count = 0;
-	(void) ERTS_RUNQ_FLGS_READ_BSET(rq, ERTS_RUNQ_FLGS_MIGRATION_INFO|ERTS_RUNQ_FLG_CHECKIO, flags);
+	(void) ERTS_RUNQ_FLGS_READ_BSET(rq, ERTS_RUNQ_FLGS_MIGRATION_INFO, flags);
 	rq->max_len = erts_atomic32_read_dirty(&rq->len);
 	for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) {
 	    ErtsRunQueueInfo *rqi;
@@ -5923,6 +5880,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online, int no_poll_th
 	erts_alloc_permanent_cache_aligned(ERTS_ALC_T_RUNQS, size_runqs);
 #if ERTS_POLL_USE_SCHEDULER_POLLING
     erts_atomic32_init_nob(&doing_sys_schedule, 0);
+    erts_atomic32_init_nob(&function_calls, 0);
 #endif
     erts_atomic32_init_nob(&no_empty_run_queues, 0);
 
@@ -9277,7 +9235,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
     Process *proxy_p = NULL;
     ErtsRunQueue *rq;
     int context_reds;
-    int fcalls;
+    int fcalls = 0;
     int actual_reds;
     int reds;
     Uint32 flags;
@@ -9351,6 +9309,10 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	    reds = ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST;
 	esdp->virtual_reds = 0;
 
+#if ERTS_POLL_USE_SCHEDULER_POLLING
+        fcalls = (int) erts_atomic32_add_read_acqb(&function_calls, reds);
+#endif
+
 	ASSERT(esdp && esdp == erts_get_scheduler_data());
 
 	rq = erts_get_runq_current(esdp);
@@ -9567,7 +9529,33 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	    non_empty_runq(rq);
 
 	    goto check_activities_to_run;
-	}
+	} else if (is_normal_sched &&
+                   fcalls > (2 * context_reds) &&
+                   prepare_for_sys_schedule()) {
+            ErtsMonotonicTime current_time;
+	    /*
+	     * Schedule system-level activities.
+	     */
+
+	    ERTS_MSACC_PUSH_STATE_CACHED_M();
+
+	    erts_runq_unlock(rq);
+
+            ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_CHECK_IO);
+            LTTNG2(scheduler_poll, esdp->no, 1);
+
+	    erts_check_io(esdp->ssi->psi, ERTS_POLL_NO_TIMEOUT);
+	    ERTS_MSACC_POP_STATE_M();
+
+	    current_time = erts_get_monotonic_time(esdp);
+	    if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref))
+		erts_bump_timers(esdp->timer_wheel, current_time);
+
+	    erts_runq_lock(rq);
+	    fcalls = 0;
+	    clear_sys_scheduling();
+	    goto continue_check_activities_to_run;
+        }
 
         if (flags & ERTS_RUNQ_FLG_MISC_OP)
 	    exec_misc_ops(rq);
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 0aa19e7bde..43937f216c 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -173,8 +173,6 @@ extern int erts_dio_sched_thread_suggested_stack_size;
   (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 9))
 #define ERTS_RUNQ_FLG_HALTING \
   (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 10))
-#define ERTS_RUNQ_FLG_CHECKIO \
-  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 11))
 
 #define ERTS_RUNQ_FLG_MAX (ERTS_RUNQ_FLG_BASE2 + 12)
 
diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index c681fa481f..ac9a070bce 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -2262,14 +2262,14 @@ erts_check_io_info(void *proc)
 
 #if ERTS_POLL_USE_FALLBACK
     erts_poll_info_flbk(get_fallback_pollset(), &piv[0]);
-    piv[0].poll_threads = 1;
+    piv[0].poll_threads = 0;
     piv[0].active_fds = 0;
     piv++;
 #endif
 
 #if ERTS_POLL_USE_SCHEDULER_POLLING
     erts_poll_info(get_scheduler_pollset(0), &piv[0]);
-    piv[0].poll_threads = 1;
+    piv[0].poll_threads = 0;
     piv[0].active_fds = 0;
     piv++;
 #endif
diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c
index 51d50933ff..27ffba58bd 100644
--- a/erts/emulator/sys/common/erl_poll.c
+++ b/erts/emulator/sys/common/erl_poll.c
@@ -2326,6 +2326,7 @@ uint32_t epoll_events(int kp_fd, int fd)
 {
     /* For epoll we read the information about what is selected upon from the proc fs.*/
     char fname[30];
+    char s[256];
     FILE *f;
     unsigned int pos, flags, mnt_id;
     int line = 0;
@@ -2343,12 +2344,12 @@ uint32_t epoll_events(int kp_fd, int fd)
     }
     if (fscanf(f,"\nmnt_id:\t%x\n", &mnt_id));
     line += 3;
-    while (!feof(f)) {
+    while (fgets(s, sizeof(s) / sizeof(*s), f)) {
         /* tfd:       10 events: 40000019 data:       180000000a */
         int ev_fd;
         uint32_t events;
         uint64_t data;
-        if (fscanf(f,"tfd:%d events:%x data:%llx\n", &ev_fd, &events,
+        if (sscanf(s,"tfd:%d events:%x data:%llx", &ev_fd, &events,
                    (unsigned long long*)&data) != 3) {
             fprintf(stderr,"failed to parse file %s on line %d, errno = %d\n", fname,
                     line,
@@ -2392,6 +2393,7 @@ ERTS_POLL_EXPORT(erts_poll_get_selected_events)(ErtsPollSet *ps,
 
     /* For epoll we read the information about what is selected upon from the proc fs.*/
     char fname[30];
+    char s[256];
     FILE *f;
     unsigned int pos, flags, mnt_id;
     int line = 0;
@@ -2410,12 +2412,12 @@ ERTS_POLL_EXPORT(erts_poll_get_selected_events)(ErtsPollSet *ps,
     }
     if (fscanf(f,"\nmnt_id:\t%x\n", &mnt_id));
     line += 3;
-    while (!feof(f)) {
+    while (fgets(s, sizeof(s) / sizeof(*s), f)) {
         /* tfd:       10 events: 40000019 data:       180000000a */
         int fd;
         uint32_t events;
         uint64_t data;
-        if (fscanf(f,"tfd:%d events:%x data:%llx\n", &fd, &events,
+        if (sscanf(s,"tfd:%d events:%x data:%llx", &fd, &events,
                    (unsigned long long*)&data) != 3) {
             fprintf(stderr,"failed to parse file %s on line %d, errno = %d\n",
                     fname, line, errno);
diff --git a/erts/emulator/test/driver_SUITE.erl b/erts/emulator/test/driver_SUITE.erl
index bd62708aa7..1d2ae4fb51 100644
--- a/erts/emulator/test/driver_SUITE.erl
+++ b/erts/emulator/test/driver_SUITE.erl
@@ -1069,14 +1069,19 @@ get_stable_check_io_info(N) ->
 get_check_io_total(ChkIo) ->
     ct:log("ChkIo = ~p~n",[ChkIo]),
     {Fallback, Rest} = get_fallback(ChkIo),
+    OnlyPollThreads = [PS || PS <- Rest, not is_scheduler_pollset(PS)],
     add_fallback_infos(Fallback,
-      lists:foldl(fun(Pollset, Acc) ->
-                          lists:zipwith(fun(A, B) ->
-                                                add_pollset_infos(A,B)
-                                        end,
-                                        Pollset, Acc)
-                  end,
-                  hd(Rest), tl(Rest))).
+      lists:foldl(
+        fun(Pollset, Acc) ->
+                lists:zipwith(fun(A, B) ->
+                                      add_pollset_infos(A,B)
+                              end,
+                              Pollset, Acc)
+        end,
+        hd(OnlyPollThreads), tl(OnlyPollThreads))).
+
+is_scheduler_pollset(Pollset) ->
+    proplists:get_value(poll_threads, Pollset) == 0.
 
 add_pollset_infos({Tag, A}=TA , {Tag, B}=TB) ->
     case tag_type(Tag) of
diff --git a/erts/emulator/test/scheduler_SUITE.erl b/erts/emulator/test/scheduler_SUITE.erl
index 2e0dfa42f3..f61949c75b 100644
--- a/erts/emulator/test/scheduler_SUITE.erl
+++ b/erts/emulator/test/scheduler_SUITE.erl
@@ -1450,28 +1450,26 @@ poll_threads(Config) when is_list(Config) ->
     {Conc, PollType, KP} = get_ioconfig(Config),
     {Sched, SchedOnln, _} = get_sstate(Config, ""),
 
+    [1, 1] = get_ionum(Config,"+IOt 2 +IOp 2"),
+    [1, 1, 1, 1, 1] = get_ionum(Config,"+IOt 5 +IOp 5"),
+    [1, 1] = get_ionum(Config, "+S 2 +IOPt 100 +IOPp 100"),
+
     if
         Conc ->
-            [1, 1, 1] = get_ionum(Config,"+IOt 2 +IOp 2"),
-            [1, 1, 1, 1, 1, 1] = get_ionum(Config,"+IOt 5 +IOp 5"),
-            [1, 1, 1] = get_ionum(Config, "+S 2 +IOPt 100 +IOPp 100"),
 
-            [5, 1] = get_ionum(Config,"+IOt 5 +IOp 1"),
-            [3, 2, 1] = get_ionum(Config,"+IOt 5 +IOp 2"),
-            [2, 2, 2, 2, 2, 1] = get_ionum(Config,"+IOt 10 +IOPp 50"),
+            [5] = get_ionum(Config,"+IOt 5 +IOp 1"),
+            [3, 2] = get_ionum(Config,"+IOt 5 +IOp 2"),
+            [2, 2, 2, 2, 2] = get_ionum(Config,"+IOt 10 +IOPp 50"),
 
-            [2, 1] = get_ionum(Config, "+S 2 +IOPt 100"),
-            [4, 1] = get_ionum(Config, "+S 4 +IOPt 100"),
-            [4, 1] = get_ionum(Config, "+S 4:2 +IOPt 100"),
-            [4, 4, 1] = get_ionum(Config, "+S 8 +IOPt 100 +IOPp 25"),
+            [2] = get_ionum(Config, "+S 2 +IOPt 100"),
+            [4] = get_ionum(Config, "+S 4 +IOPt 100"),
+            [4] = get_ionum(Config, "+S 4:2 +IOPt 100"),
+            [4, 4] = get_ionum(Config, "+S 8 +IOPt 100 +IOPp 25"),
 
             fail = get_ionum(Config, "+IOt 1 +IOp 2"),
 
             ok;
         not Conc ->
-            [1, 1] = get_ionum(Config,"+IOt 2 +IOp 2"),
-            [1, 1, 1, 1, 1] = get_ionum(Config,"+IOt 5 +IOp 5"),
-            [1, 1] = get_ionum(Config, "+S 2 +IOPt 100 +IOPp 100"),
 
             [1, 1, 1, 1, 1] = get_ionum(Config,"+IOt 5 +IOp 1"),
             [1, 1, 1, 1, 1] = get_ionum(Config,"+IOt 5 +IOp 2"),
@@ -1515,7 +1513,8 @@ get_iostate(Config, Cmd)->
                                              erlang:system_info(check_io)
                                      end]),
             IO = [IOState || IOState <- IOStates,
-                             proplists:get_value(fallback, IOState) == false],
+                             proplists:get_value(fallback, IOState) == false,
+                             proplists:get_value(poll_threads, IOState) /= 0],
             stop_node(Node),
             IO;
         {error,timeout} ->
diff --git a/erts/emulator/test/z_SUITE.erl b/erts/emulator/test/z_SUITE.erl
index 1c52e1a934..6549108126 100644
--- a/erts/emulator/test/z_SUITE.erl
+++ b/erts/emulator/test/z_SUITE.erl
@@ -251,7 +251,7 @@ pollset_size(Config) when is_list(Config) ->
 	  end.
 
 check_io_debug(Config) when is_list(Config) ->
-    case lists:keysearch(name, 1, erlang:system_info(check_io)) of
+    case lists:keysearch(name, 1, hd(erlang:system_info(check_io))) of
 	      {value, {name, erts_poll}} -> check_io_debug_test();
 	      _ -> {skipped, "Not implemented in this emulator"}
 	  end.