1 files changed, 124 insertions, 86 deletions

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 0118302eb3..650ec0958c 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -36,7 +36,6 @@
 #include "erl_db.h"
 #include "dist.h"
 #include "beam_catches.h"
-#include "erl_instrument.h"
 #include "erl_threads.h"
 #include "erl_binary.h"
 #include "beam_bp.h"
@@ -172,11 +171,19 @@ int erts_dio_sched_thread_suggested_stack_size = -1;
 ErtsLcPSDLocks erts_psd_required_locks[ERTS_PSD_SIZE];
 #endif
 
-static struct ErtsSchedBusyWait_ {
+typedef struct {
     int aux_work;
     int tse;
     int sys_schedule;
-} sched_busy_wait;
+} ErtsBusyWaitParams;
+
+static ErtsBusyWaitParams sched_busy_wait_params[ERTS_SCHED_TYPE_LAST + 1];
+
+static ERTS_INLINE ErtsBusyWaitParams *
+sched_get_busy_wait_params(ErtsSchedulerData *esdp)
+{
+    return &sched_busy_wait_params[esdp->type];
+}
 
 int erts_disable_proc_not_running_opt;
 
@@ -2500,6 +2507,8 @@ handle_yield(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting)
 
     yield |= erts_handle_yielded_ets_all_request(awdp->esdp,
                                                  &awdp->yield.ets_all);
+    yield |= erts_handle_yielded_alcu_blockscan(awdp->esdp,
+                                                &awdp->yield.alcu_blockscan);
 
     /*
      * Other yielding operations...
@@ -3262,7 +3271,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 
     erts_runq_unlock(rq);
 
-    spincount = sched_busy_wait.tse;
+    spincount = sched_get_busy_wait_params(esdp)->tse;
 
     if (ERTS_SCHEDULER_IS_DIRTY(esdp))
         dirty_sched_wall_time_change(esdp, working = 0);
@@ -3364,7 +3373,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
         }
 
         flgs = sched_prep_cont_spin_wait(ssi);
-        spincount = sched_busy_wait.aux_work;
+        spincount = sched_get_busy_wait_params(esdp)->aux_work;
 
         if (!(flgs & ERTS_SSI_FLG_WAITING)) {
             ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING));
@@ -5250,24 +5259,35 @@ typedef enum {
 #define ERTS_WAKEUP_OTHER_FIXED_INC_LEGACY (CONTEXT_REDS/10)
 
 
-static struct {
+typedef struct {
     ErtsSchedWakeupOtherThreshold threshold;
     ErtsSchedWakeupOtherType type;
     int limit;
     int dec_shift;
     int dec_mask;
     void (*check)(ErtsRunQueue *rq, Uint32 flags);
-} wakeup_other;
+} ErtsWakeupOtherParams;
+
+static ErtsWakeupOtherParams sched_wakeup_other_params[ERTS_SCHED_TYPE_LAST + 1];
+
+static ERTS_INLINE ErtsWakeupOtherParams *
+runq_get_wakeup_other_params(ErtsRunQueue *rq)
+{
+    ErtsSchedulerData *esdp = rq->scheduler;
+    return &sched_wakeup_other_params[esdp->type];
+}
 
 static void
 wakeup_other_check(ErtsRunQueue *rq, Uint32 flags)
 {
+    ErtsWakeupOtherParams *wo_params = runq_get_wakeup_other_params(rq);
     int wo_reds = rq->wakeup_other_reds;
+
     if (wo_reds) {
 	int left_len = erts_atomic32_read_dirty(&rq->len) - 1;
 	if (left_len < 1) {
-	    int wo_reduce = wo_reds << wakeup_other.dec_shift;
-	    wo_reduce &= wakeup_other.dec_mask;
+	    int wo_reduce = wo_reds << wo_params->dec_shift;
+	    wo_reduce &= wo_params->dec_mask;
 	    rq->wakeup_other -= wo_reduce;
 	    if (rq->wakeup_other < 0)
 		rq->wakeup_other = 0;
@@ -5275,7 +5295,7 @@ wakeup_other_check(ErtsRunQueue *rq, Uint32 flags)
 	else {
 	    rq->wakeup_other += (left_len*wo_reds
 				 + ERTS_WAKEUP_OTHER_FIXED_INC);
-	    if (rq->wakeup_other > wakeup_other.limit) {
+	    if (rq->wakeup_other > wo_params->limit) {
 		if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) {
 		    if (rq->waiting) {
 			wake_dirty_scheduler(rq);
@@ -5297,42 +5317,44 @@ wakeup_other_check(ErtsRunQueue *rq, Uint32 flags)
 }
 
 static void
-wakeup_other_set_limit(void)
+wakeup_other_set_limit(ErtsWakeupOtherParams *params)
 {
-    switch (wakeup_other.threshold) {
+    switch (params->threshold) {
     case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_HIGH:
-	wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_HIGH;
-	wakeup_other.dec_shift = ERTS_WAKEUP_OTHER_DEC_SHIFT_VERY_HIGH;
+	params->limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_HIGH;
+	params->dec_shift = ERTS_WAKEUP_OTHER_DEC_SHIFT_VERY_HIGH;
 	break;
     case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_HIGH:
-	wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_HIGH;
-	wakeup_other.dec_shift = ERTS_WAKEUP_OTHER_DEC_SHIFT_HIGH;
+	params->limit = ERTS_WAKEUP_OTHER_LIMIT_HIGH;
+	params->dec_shift = ERTS_WAKEUP_OTHER_DEC_SHIFT_HIGH;
 	break;
     case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_MEDIUM:
-	wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_MEDIUM;
-	wakeup_other.dec_shift = ERTS_WAKEUP_OTHER_DEC_SHIFT_MEDIUM;
+	params->limit = ERTS_WAKEUP_OTHER_LIMIT_MEDIUM;
+	params->dec_shift = ERTS_WAKEUP_OTHER_DEC_SHIFT_MEDIUM;
 	break;
     case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_LOW:
-	wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_LOW;
-	wakeup_other.dec_shift = ERTS_WAKEUP_OTHER_DEC_SHIFT_LOW;
+	params->limit = ERTS_WAKEUP_OTHER_LIMIT_LOW;
+	params->dec_shift = ERTS_WAKEUP_OTHER_DEC_SHIFT_LOW;
 	break;
     case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_LOW:
-	wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_LOW;
-	wakeup_other.dec_shift = ERTS_WAKEUP_OTHER_DEC_SHIFT_VERY_LOW;
+	params->limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_LOW;
+	params->dec_shift = ERTS_WAKEUP_OTHER_DEC_SHIFT_VERY_LOW;
 	break;
     }
-    if (wakeup_other.dec_shift < 0)
-	wakeup_other.dec_mask = (1 << (sizeof(wakeup_other.dec_mask)*8
-				       + wakeup_other.dec_shift)) - 1;
+
+    if (params->dec_shift < 0)
+	params->dec_mask = (1 << (sizeof(params->dec_mask)*8
+				  + params->dec_shift)) - 1;
     else {
-	wakeup_other.dec_mask = 0;
-	wakeup_other.dec_mask = ~wakeup_other.dec_mask;
+	params->dec_mask = 0;
+	params->dec_mask = ~params->dec_mask;
     }
 }
 
 static void
 wakeup_other_check_legacy(ErtsRunQueue *rq, Uint32 flags)
 {
+    ErtsWakeupOtherParams *wo_params = runq_get_wakeup_other_params(rq);
     int wo_reds = rq->wakeup_other_reds;
     if (wo_reds) {
 	erts_aint32_t len = erts_atomic32_read_dirty(&rq->len);
@@ -5341,7 +5363,7 @@ wakeup_other_check_legacy(ErtsRunQueue *rq, Uint32 flags)
 	    if (rq->wakeup_other < 0)
 		rq->wakeup_other = 0;
 	}
-	else if (rq->wakeup_other < wakeup_other.limit)
+	else if (rq->wakeup_other < wo_params->limit)
 	    rq->wakeup_other += len*wo_reds + ERTS_WAKEUP_OTHER_FIXED_INC_LEGACY;
 	else {
 	    if (flags & ERTS_RUNQ_FLG_PROTECTED)
@@ -5357,23 +5379,23 @@ wakeup_other_check_legacy(ErtsRunQueue *rq, Uint32 flags)
 }
 
 static void
-wakeup_other_set_limit_legacy(void)
+wakeup_other_set_limit_legacy(ErtsWakeupOtherParams *params)
 {
-    switch (wakeup_other.threshold) {
+    switch (params->threshold) {
     case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_HIGH:
-	wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_HIGH_LEGACY;
+	params->limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_HIGH_LEGACY;
 	break;
     case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_HIGH:
-	wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_HIGH_LEGACY;
+	params->limit = ERTS_WAKEUP_OTHER_LIMIT_HIGH_LEGACY;
 	break;
     case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_MEDIUM:
-	wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_MEDIUM_LEGACY;
+	params->limit = ERTS_WAKEUP_OTHER_LIMIT_MEDIUM_LEGACY;
 	break;
     case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_LOW:
-	wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_LOW_LEGACY;
+	params->limit = ERTS_WAKEUP_OTHER_LIMIT_LOW_LEGACY;
 	break;
     case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_LOW:
-	wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_LOW_LEGACY;
+	params->limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_LOW_LEGACY;
 	break;
     }
 }
@@ -5381,15 +5403,21 @@ wakeup_other_set_limit_legacy(void)
 static void
 set_wakeup_other_data(void)
 {
-    switch (wakeup_other.type) {
-    case ERTS_SCHED_WAKEUP_OTHER_TYPE_DEFAULT:
-	wakeup_other.check = wakeup_other_check;
-	wakeup_other_set_limit();
-	break;
-    case ERTS_SCHED_WAKEUP_OTHER_TYPE_LEGACY:
-	wakeup_other.check = wakeup_other_check_legacy;
-	wakeup_other_set_limit_legacy();
-	break;
+    ErtsSchedType type;
+
+    for (type = ERTS_SCHED_TYPE_FIRST; type <= ERTS_SCHED_TYPE_LAST; type++) {
+        ErtsWakeupOtherParams *params = &sched_wakeup_other_params[type];
+
+        switch (params->type) {
+        case ERTS_SCHED_WAKEUP_OTHER_TYPE_DEFAULT:
+            params->check = wakeup_other_check;
+            wakeup_other_set_limit(params);
+            break;
+        case ERTS_SCHED_WAKEUP_OTHER_TYPE_LEGACY:
+            params->check = wakeup_other_check_legacy;
+            wakeup_other_set_limit_legacy(params);
+            break;
+        }
     }
 }
 
@@ -5444,56 +5472,64 @@ runq_supervisor(void *unused)
 void
 erts_early_init_scheduling(int no_schedulers)
 {
+    ErtsSchedType type;
+
     aux_work_timeout_early_init(no_schedulers);
-    wakeup_other.threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_MEDIUM;
-    wakeup_other.type = ERTS_SCHED_WAKEUP_OTHER_TYPE_DEFAULT;
-    sched_busy_wait.sys_schedule = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM;
-    sched_busy_wait.tse = (ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM
-			   * ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT);
-    sched_busy_wait.aux_work = (ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM
-				* ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_MEDIUM);
+
+    for (type = ERTS_SCHED_TYPE_FIRST; type <= ERTS_SCHED_TYPE_LAST; type++) {
+        erts_sched_set_wakeup_other_threshold(type, "medium");
+        erts_sched_set_wakeup_other_type(type, "default");
+
+        erts_sched_set_busy_wait_threshold(type, "medium");
+    }
+
+    erts_sched_set_busy_wait_threshold(ERTS_SCHED_DIRTY_CPU, "short");
+    erts_sched_set_busy_wait_threshold(ERTS_SCHED_DIRTY_IO, "short");
 }
 
 int
-erts_sched_set_wakeup_other_thresold(char *str)
-{
-    ErtsSchedWakeupOtherThreshold threshold;
-    if (sys_strcmp(str, "very_high") == 0)
-	threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_HIGH;
-    else if (sys_strcmp(str, "high") == 0)
-	threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_HIGH;
-    else if (sys_strcmp(str, "medium") == 0)
-	threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_MEDIUM;
-    else if (sys_strcmp(str, "low") == 0)
-	threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_LOW;
-    else if (sys_strcmp(str, "very_low") == 0)
-	threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_LOW;
-    else
-	return EINVAL;
-    wakeup_other.threshold = threshold;
-    set_wakeup_other_data();
+erts_sched_set_wakeup_other_threshold(ErtsSchedType sched_type, char *str)
+{
+    ErtsWakeupOtherParams *params = &sched_wakeup_other_params[sched_type];
+
+    if (sys_strcmp(str, "very_high") == 0) {
+        params->threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_HIGH;
+    } else if (sys_strcmp(str, "high") == 0) {
+        params->threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_HIGH;
+    } else if (sys_strcmp(str, "medium") == 0) {
+        params->threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_MEDIUM;
+    } else if (sys_strcmp(str, "low") == 0) {
+        params->threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_LOW;
+    } else if (sys_strcmp(str, "very_low") == 0) {
+        params->threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_LOW;
+    } else {
+        return EINVAL;
+    }
+
     return 0;
 }
 
 int
-erts_sched_set_wakeup_other_type(char *str)
+erts_sched_set_wakeup_other_type(ErtsSchedType sched_type, char *str)
 {
-    ErtsSchedWakeupOtherType type;
-    if (sys_strcmp(str, "default") == 0)
-	type = ERTS_SCHED_WAKEUP_OTHER_TYPE_DEFAULT;
-    else if (sys_strcmp(str, "legacy") == 0)
-	type = ERTS_SCHED_WAKEUP_OTHER_TYPE_LEGACY;
-    else
-	return EINVAL;
-    wakeup_other.type = type;
+    ErtsWakeupOtherParams *params = &sched_wakeup_other_params[sched_type];
+
+    if (sys_strcmp(str, "default") == 0) {
+        params->type = ERTS_SCHED_WAKEUP_OTHER_TYPE_DEFAULT;
+    } else if (sys_strcmp(str, "legacy") == 0) {
+        params->type = ERTS_SCHED_WAKEUP_OTHER_TYPE_LEGACY;
+    } else {
+        return EINVAL;
+    }
+
     return 0;
 }
 
 int
-erts_sched_set_busy_wait_threshold(char *str)
+erts_sched_set_busy_wait_threshold(ErtsSchedType sched_type, char *str)
 {
-    int sys_sched;
-    int aux_work_fact;
+    ErtsBusyWaitParams *params = &sched_busy_wait_params[sched_type];
+    int aux_work_fact, sys_sched;
 
     if (sys_strcmp(str, "very_long") == 0) {
 	sys_sched = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_VERY_LONG;
@@ -5523,9 +5559,9 @@ erts_sched_set_busy_wait_threshold(char *str)
 	return EINVAL;
     }
 
-    sched_busy_wait.sys_schedule = sys_sched;
-    sched_busy_wait.tse = sys_sched*ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT;
-    sched_busy_wait.aux_work = sys_sched*aux_work_fact;
+    params->sys_schedule = sys_sched;
+    params->tse = sys_sched * ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT;
+    params->aux_work = sys_sched * aux_work_fact;
 
     return 0;
 }
@@ -5663,6 +5699,7 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num,
     esdp->ssi = ssi;
     esdp->current_process = NULL;
     esdp->current_port = NULL;
+    esdp->current_nif = NULL;
 
     esdp->virtual_reds = 0;
     esdp->cpu_id = -1;
@@ -6058,7 +6095,7 @@ make_proxy_proc(Process *prev_proxy, Process *proc, erts_aint32_t prio)
 	proxy = prev_proxy;
 	ASSERT(erts_atomic32_read_nob(&proxy->state) & ERTS_PSFLG_PROXY);
 	erts_atomic32_set_nob(&proxy->state, state);
-        (void) erts_set_runq_proc(proc, rq, &bound);
+        (void) erts_set_runq_proc(proxy, rq, &bound);
     }
     else {
 	proxy = erts_alloc(ERTS_ALC_T_PROC, sizeof(Process));
@@ -6071,7 +6108,7 @@ make_proxy_proc(Process *prev_proxy, Process *proc, erts_aint32_t prio)
 	}
 #endif
 	erts_atomic32_init_nob(&proxy->state, state);
-        erts_init_runq_proc(proc, rq, bound);
+        erts_init_runq_proc(proxy, rq, bound);
     }
 
     proxy->common.id = proc->common.id;
@@ -8298,6 +8335,7 @@ sched_thread_func(void *vesdp)
     ERTS_VERIFY_UNUSED_TEMP_ALLOC(NULL);
 #endif
 
+    erts_alcu_sched_spec_data_init(esdp);
     erts_ets_sched_spec_data_init(esdp);
 
     process_main(esdp->x_reg_array, esdp->f_reg_array);
@@ -9894,7 +9932,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
         if (flags & ERTS_RUNQ_FLG_MISC_OP)
 	    exec_misc_ops(rq);
 
-	wakeup_other.check(rq, flags);
+        runq_get_wakeup_other_params(rq)->check(rq, flags);
 
 	/*
 	 * Find a new port to run.