Merge branch 'rickard/mv-dirty-reds-count/OTP-13123'

* rickard/mv-dirty-reds-count/OTP-13123:
  Move dirty reduction count to erts_dirty_process_main()

6 files changed, 115 insertions, 235 deletions

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 25cc41323b..0a59f8785c 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1323,11 +1323,7 @@ void process_main(void)
 
     if (start_time != 0) {
         Sint64 diff = erts_timestamp_millis() - start_time;
-	if (diff > 0 && (Uint) diff >  erts_system_monitor_long_schedule
-#if defined(ERTS_SMP) && defined(ERTS_DIRTY_SCHEDULERS)
-	    && !ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(c_p))
-#endif
-	    ) {
+	if (diff > 0 && (Uint) diff >  erts_system_monitor_long_schedule) {
 	    BeamInstr *inptr = find_function_from_pc(start_time_i);
 	    BeamInstr *outptr = find_function_from_pc(c_p->i);
 	    monitor_long_schedule_proc(c_p,inptr,outptr,(Uint) diff);
@@ -1337,7 +1333,7 @@ void process_main(void)
     PROCESS_MAIN_CHK_LOCKS(c_p);
     ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
     ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
-    c_p = schedule(c_p, reds_used);
+    c_p = erts_schedule(NULL, c_p, reds_used);
     ASSERT(!(c_p->flags & F_HIPE_MODE));
     ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
     start_time = 0;
@@ -3559,7 +3555,9 @@ do {						\
 		typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]);
 		NifF* fp = vbf = (NifF*) I[1];
 		struct enif_environment_t env;
+#ifdef ERTS_SMP
 		ASSERT(c_p->scheduler_data);
+#endif
 		live_hf_end = c_p->mbuf;
 		erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2], NULL);
 		nif_bif_result = (*fp)(&env, bif_nif_arity, reg);
@@ -5156,18 +5154,18 @@ do {						\
 }
 
 /*
- * dirty_process_main() is what dirty schedulers execute. Since they handle
+ * erts_dirty_process_main() is what dirty schedulers execute. Since they handle
  * only NIF calls they do not need to be able to execute all BEAM
  * instructions.
  */
-void dirty_process_main(void)
+void erts_dirty_process_main(ErtsSchedulerData *esdp)
 {
+#ifdef ERTS_DIRTY_SCHEDULERS
     Process* c_p = NULL;
-    int reds_used;
+    ErtsMonotonicTime start_time;
 #ifdef DEBUG
     ERTS_DECLARE_DUMMY(Eterm pid);
 #endif
-    BeamInstr *do_call_nif = OpCode(call_nif);
 
     /* Pointer to X registers: x(1)..x(N); reg[0] is used when doing GC,
      * in all other cases x0 is used.
@@ -5190,36 +5188,30 @@ void dirty_process_main(void)
      */
     register BeamInstr *I REG_I = NULL;
 
-    /* Number of reductions left.  This function
-     * returns to the scheduler when FCALLS reaches zero.
-     */
-    register Sint FCALLS REG_fcalls = 0;
-
-    /*
-     * X registers and floating point registers are located in
-     * scheduler specific data.
-     */
-    register FloatDef *freg = NULL;
+    ERTS_MSACC_DECLARE_CACHE_X() /* a cached value of the tsd pointer for msacc */
 
     /*
-     * For keeping the negative old value of 'reds' when call saving is active.
+     * start_time always positive for dirty CPU schedulers,
+     * and negative for dirty I/O schedulers.
      */
-    int neg_o_reds = 0;
 
-    ERTS_MSACC_DECLARE_CACHE_X() /* a cached value of the tsd pointer for msacc */
-
-    ERL_BITS_DECLARE_STATEP; /* Has to be last declaration */
-
-    c_p = NULL;
-    reds_used = 0;
+    if (ERTS_SCHEDULER_IS_DIRTY_CPU(esdp)) {
+	start_time = erts_get_monotonic_time(NULL);
+	ASSERT(start_time >= 0);
+    }
+    else {
+	start_time = ERTS_SINT64_MIN;
+	ASSERT(start_time < 0);
+    }
 
-    goto do_dirty_schedule1;
+    goto do_dirty_schedule;
 
  context_switch:
     c_p->arity = I[-1];
     c_p->current = I-3;		/* Pointer to Mod, Func, Arity */
 
     {
+	int reds_used;
 	Eterm* argp;
 	int i;
 
@@ -5246,18 +5238,6 @@ void dirty_process_main(void)
 	}
 
 	/*
-	 * Since REDS_IN(c_p) is stored in the save area (c_p->arg_reg) we must read it
-	 * now before saving registers.
-	 */
-
-	ASSERT(c_p->debug_reds_in == REDS_IN(c_p));
-	if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p))
-	    reds_used = REDS_IN(c_p) - FCALLS;
-	else
-	    reds_used = REDS_IN(c_p) - (CONTEXT_REDS + FCALLS);
-	ASSERT(reds_used >= 0);
-
-	/*
 	 * Save the argument registers and everything else.
 	 */
 
@@ -5267,23 +5247,46 @@ void dirty_process_main(void)
 	}
 	SWAPOUT;
 	c_p->i = I;
-	goto do_dirty_schedule1;
-    }
 
- do_dirty_schedule:
-    ASSERT(c_p->debug_reds_in == REDS_IN(c_p));
-    if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p))
-	reds_used = REDS_IN(c_p) - FCALLS;
-    else
-	reds_used = REDS_IN(c_p) - (CONTEXT_REDS + FCALLS);
-    ASSERT(reds_used >= 0);
- do_dirty_schedule1:
+    do_dirty_schedule:
+
+	if (start_time < 0) {
+	    /*
+	     * Dirty I/O scheduler:
+	     *   One reduction consumed regardless of
+	     *   time spent in the dirty NIF.
+	     */
+	    reds_used = esdp->virtual_reds + 1;
+	}
+	else {
+	    /*
+	     * Dirty CPU scheduler:
+	     *   Currently two reductions consumed per
+	     *   micro second spent in the dirty NIF.
+	     */
+	    ErtsMonotonicTime time;
+	    time = erts_get_monotonic_time(esdp);
+	    time -= start_time;
+	    time = ERTS_MONOTONIC_TO_USEC(time);
+	    time *= (CONTEXT_REDS-1)/1000 + 1;
+	    ASSERT(time >= 0);
+	    if (time == 0)
+		time = 1; /* At least one reduction */
+	    time += esdp->virtual_reds;
+	    reds_used = time > INT_MAX ? INT_MAX : (int) time;
+	}
+
+	PROCESS_MAIN_CHK_LOCKS(c_p);
+	ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
+	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+	c_p = erts_schedule(esdp, c_p, reds_used);
+
+	if (start_time >= 0) {
+	    start_time = erts_get_monotonic_time(esdp);
+	    ASSERT(start_time >= 0);
+	}
+    }
 
-    PROCESS_MAIN_CHK_LOCKS(c_p);
-    ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
-    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
-    c_p = schedule(c_p, reds_used);
-    ASSERT(!(c_p->flags & F_HIPE_MODE));
     ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 #ifdef DEBUG
     pid = c_p->common.id; /* Save for debugging purposes */
@@ -5291,13 +5294,11 @@ void dirty_process_main(void)
     ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
     PROCESS_MAIN_CHK_LOCKS(c_p);
 
+    ASSERT(!(c_p->flags & F_HIPE_MODE));
     ERTS_MSACC_UPDATE_CACHE_X();
 
-    reg = erts_proc_sched_data(c_p)->x_reg_array;
-    freg = erts_proc_sched_data(c_p)->f_reg_array;
-    ERL_BITS_RELOAD_STATEP(c_p);
+    reg = esdp->x_reg_array;
     {
-	int reds;
 	Eterm* argp;
 	int i;
 
@@ -5315,21 +5316,13 @@ void dirty_process_main(void)
 
 	I = c_p->i;
 
-	REDS_IN(c_p) = reds = c_p->fcalls;
-#ifdef DEBUG
-	c_p->debug_reds_in = reds;
-#endif
+	ASSERT(BeamOp(op_call_nif) == (BeamInstr *) *I);
 
-	if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) {
-	    neg_o_reds = -CONTEXT_REDS;
-	    FCALLS = neg_o_reds + reds;
-	} else {
-	    neg_o_reds = 0;
-	    FCALLS = reds;
+	if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)
+	    && (ERTS_TRACE_FLAGS(c_p) & F_SENSITIVE) == 0) {
+	    c_p->fcalls = REDS_IN(c_p) = 0;
 	}
 
-	ERTS_DBG_CHK_REDS(c_p, FCALLS);
-
 	SWAPIN;
 
 #ifdef USE_VM_PROBES
@@ -5361,7 +5354,6 @@ void dirty_process_main(void)
 	Eterm nif_bif_result;
 	Eterm bif_nif_arity;
 
-    OpCase(call_nif):
 	{
 	    /*
 	     * call_nif is always first instruction in function:
@@ -5376,18 +5368,11 @@ void dirty_process_main(void)
 	     */
 	    BifFunction vbf;
 
-            if (!((FCALLS - 1) > 0 || (FCALLS - 1) > neg_o_reds)) {
-                /* If we have run out of reductions, we do a context
-                   switch before calling the nif */
-                goto context_switch;
-            }
-
 	    ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF);
 
 	    DTRACE_NIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]);
 	    c_p->current = I-3; /* current and vbf set to please handle_error */
 	    SWAPOUT;
-	    c_p->fcalls = FCALLS - 1;
 	    PROCESS_MAIN_CHK_LOCKS(c_p);
 	    bif_nif_arity = I[-1];
 	    ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
@@ -5398,7 +5383,7 @@ void dirty_process_main(void)
 		NifF* fp = vbf = (NifF*) I[1];
 		struct enif_environment_t env;
 		ASSERT(!c_p->scheduler_data);
-		erts_pre_dirty_nif(&env, c_p, (struct erl_module_nif*)I[2], NULL);
+		erts_pre_dirty_nif(esdp, &env, c_p, (struct erl_module_nif*)I[2], NULL);
 		nif_bif_result = (*fp)(&env, bif_nif_arity, reg);
 		if (env.exception_thrown)
 		    nif_bif_result = THE_NON_VALUE;
@@ -5421,8 +5406,6 @@ void dirty_process_main(void)
 							reg, bif_nif_arity);
 	    }
 	    SWAPIN;  /* There might have been a garbage collection. */
-	    FCALLS = c_p->fcalls;
-	    ERTS_DBG_CHK_REDS(c_p, FCALLS);
 	    if (is_value(nif_bif_result)) {
 		r(0) = nif_bif_result;
 		CHECK_TERM(r(0));
@@ -5431,11 +5414,8 @@ void dirty_process_main(void)
 		Goto(*I);
 	    } else if (c_p->freason == TRAP) {
 		I = c_p->i;
-		if (c_p->flags & F_HIBERNATE_SCHED) {
-		    c_p->flags &= ~F_HIBERNATE_SCHED;
-		    goto do_dirty_schedule;
-		}
-		DispatchMacro();
+		ASSERT(!(c_p->flags & F_HIBERNATE_SCHED));
+		goto context_switch;
 	    }
 	    I = handle_error(c_p, c_p->cp, reg, vbf);
 	}
@@ -5445,8 +5425,9 @@ void dirty_process_main(void)
     } else {
 	ASSERT(!is_value(r(0)));
 	SWAPIN;
-	Goto(do_call_nif);
+	goto context_switch;
     }
+#endif /* ERTS_DIRTY_SCHEDULERS */
 }
 
 static BifFunction
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 4437adb57b..f1c35cd5a5 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -178,11 +178,6 @@ static ERTS_INLINE void ensure_heap(ErlNifEnv* env, size_t may_need)
 void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif,
                   Process* tracee)
 {
-#ifdef DEBUG
-#ifdef ERTS_DIRTY_SCHEDULERS
-    ErtsSchedulerData *esdp;
-#endif
-#endif
     env->mod_nif = mod_nif;
     env->proc = p;
     env->hp = HEAP_TOP(p);
@@ -195,46 +190,41 @@ void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif,
 
     ASSERT(p->common.id != ERTS_INVALID_PID);
 
-#ifdef DEBUG
-#ifdef ERTS_DIRTY_SCHEDULERS
-    esdp = erts_get_scheduler_data();
-    ASSERT(esdp);
+#if defined(DEBUG) && defined(ERTS_DIRTY_SCHEDULERS)
+    {
+	ErtsSchedulerData *esdp = erts_get_scheduler_data();
+	ASSERT(esdp);
 
-    if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
-	erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state);
+	if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+	    erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state);
 
-	ASSERT(p->scheduler_data == esdp);
-	ASSERT((state & (ERTS_PSFLG_RUNNING
-			 | ERTS_PSFLG_RUNNING_SYS))
-	       && !(state & (ERTS_PSFLG_DIRTY_RUNNING
-			     | ERTS_PSFLG_DIRTY_RUNNING_SYS)));
+	    ASSERT(p->scheduler_data == esdp);
+	    ASSERT((state & (ERTS_PSFLG_RUNNING
+			     | ERTS_PSFLG_RUNNING_SYS))
+		   && !(state & (ERTS_PSFLG_DIRTY_RUNNING
+				 | ERTS_PSFLG_DIRTY_RUNNING_SYS)));
+	}
     }
 #endif
-#endif
 }
 
-void erts_pre_dirty_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif,
+void erts_pre_dirty_nif(ErtsSchedulerData *esdp,
+			ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif,
 			Process* tracee)
 {
 #ifdef ERTS_DIRTY_SCHEDULERS
-#ifdef DEBUG
-    erts_aint32_t state;
-#endif
     Process *sproc;
-    ErtsSchedulerData *esdp;
-    esdp = erts_get_scheduler_data();
-    ASSERT(esdp);
-
-    erts_pre_nif(env, p, mod_nif, tracee);
-
 #ifdef DEBUG
-    state = erts_smp_atomic32_read_nob(&p->state);
+    erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state);
 
     ASSERT(!p->scheduler_data);
     ASSERT((state & ERTS_PSFLG_DIRTY_RUNNING)
 	   && !(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)));
+    ASSERT(esdp);
 #endif
 
+    erts_pre_nif(env, p, mod_nif, tracee);
+
     sproc = esdp->dirty_shadow_process;
     ASSERT(sproc);
     ASSERT(sproc->static_flags & ERTS_STC_FLG_SHADOW_PROC);
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index e245c9e6bb..c0b1d7246c 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -8197,7 +8197,7 @@ sched_dirty_cpu_thread_func(void *vesdp)
 #endif
     erts_thread_init_float();
 
-    dirty_process_main();
+    erts_dirty_process_main(esdp);
     /* No schedulers should *ever* terminate */
     erts_exit(ERTS_ABORT_EXIT,
 	     "Dirty CPU scheduler thread number %beu terminated\n",
@@ -8242,7 +8242,7 @@ sched_dirty_io_thread_func(void *vesdp)
 #endif
     erts_thread_init_float();
 
-    dirty_process_main();
+    erts_dirty_process_main(esdp);
     /* No schedulers should *ever* terminate */
     erts_exit(ERTS_ABORT_EXIT,
 	     "Dirty I/O scheduler thread number %beu terminated\n",
@@ -9377,77 +9377,6 @@ scheduler_gc_proc(Process *c_p, int reds_left)
     return reds;
 }
 
-static ERTS_INLINE void
-clean_dirty_start(Process *p)
-{
-#if defined(ERTS_DIRTY_SCHEDULERS) && !defined(ARCH_64)
-    void *ptr = ERTS_PROC_SET_DIRTY_CPU_START(p, NULL);
-    if (ptr)
-	erts_free(ERTS_ALC_T_DIRTY_START, ptr);
-#endif
-}
-
-static ERTS_INLINE void
-save_dirty_start(ErtsSchedulerData *esdp, Process *c_p)
-{
-#ifdef ERTS_DIRTY_SCHEDULERS
-    if (ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(esdp->run_queue)) {
-	ErtsMonotonicTime time = erts_get_monotonic_time(esdp);
-#ifdef ARCH_64
-	ERTS_PROC_SET_DIRTY_CPU_START(c_p, (void *) time);
-#else
-	ErtsMonotonicTime *stimep;
-
-	stimep = (ErtsMonotonicTime *) ERTS_PROC_GET_DIRTY_CPU_START(c_p);
-	if (!stimep) {
-	    stimep = erts_alloc(ERTS_ALC_T_DIRTY_START,
-				sizeof(ErtsMonotonicTime));
-	    ERTS_PROC_SET_DIRTY_CPU_START(c_p, (void *) stimep);
-	}
-	*stimep = time;
-#endif
-    }
-#endif
-}
-
-static ERTS_INLINE int
-get_dirty_reds(ErtsSchedulerData *esdp, Process *c_p)
-{
-
-#ifndef ERTS_DIRTY_SCHEDULERS
-    return -1;
-#else
-    ErtsMonotonicTime stime, time;
-
-    if (!ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(esdp->run_queue))
-	return 1;
-
-#ifdef ARCH_64
-    stime = (ErtsMonotonicTime) ERTS_PROC_GET_DIRTY_CPU_START(c_p);
-#else
-    {
-	ErtsMonotonicTime *stimep;
-	stimep = (ErtsMonotonicTime *) ERTS_PROC_GET_DIRTY_CPU_START(c_p);
-	ASSERT(stimep);
-	stime = *stimep;
-    }
-#endif
-
-    time = erts_get_monotonic_time(esdp);
-
-    ASSERT(stime && stime < time);
-
-    time -= stime;
-    time = ERTS_MONOTONIC_TO_USEC(time);
-    time *= 2;
-
-    if (time > INT_MAX)
-	return INT_MAX;
-    return (int) time;
-#endif
-
-}
-
 /*
  * schedule() is called from BEAM (process_main()) or HiPE
  * (hipe_mode_switch()) when the current process is to be
@@ -9466,11 +9395,10 @@ get_dirty_reds(ErtsSchedulerData *esdp, Process *c_p)
  * so that normal processes get to run more frequently. 
  */
 
-Process *schedule(Process *p, int calls)
+Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 {
     Process *proxy_p = NULL;
     ErtsRunQueue *rq;
-    ErtsSchedulerData *esdp;
     int context_reds;
     int fcalls;
     int input_reductions;
@@ -9507,8 +9435,19 @@ Process *schedule(Process *p, int calls)
      * Clean up after the process being scheduled out.
      */
     if (!p) {	/* NULL in the very first schedule() call */
+#ifdef ERTS_DIRTY_SCHEDULERS
+	is_normal_sched = !esdp;
+	if (is_normal_sched) {
+	    esdp = erts_get_scheduler_data();
+	    ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp));
+	}
+	else {
+	    ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp));
+	}
+#else
 	esdp = erts_get_scheduler_data();
-	is_normal_sched = !ERTS_SCHEDULER_IS_DIRTY(esdp);
+	is_normal_sched = 1;
+#endif
 	rq = erts_get_runq_current(esdp);
 	ASSERT(esdp);
 	fcalls = (int) erts_smp_atomic32_read_acqb(&function_calls);
@@ -9517,12 +9456,12 @@ Process *schedule(Process *p, int calls)
     } else {
 #ifdef ERTS_SMP
 #ifdef ERTS_DIRTY_SCHEDULERS
-	esdp = p->scheduler_data;
-	is_normal_sched = esdp != NULL;
-	if (is_normal_sched)
+	is_normal_sched = !esdp;
+	if (is_normal_sched) {
+	    esdp = p->scheduler_data;
 	    ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp));
+	}
 	else {
-	    esdp = erts_get_scheduler_data();
 	    ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp));
 	}
 #else
@@ -9541,10 +9480,7 @@ Process *schedule(Process *p, int calls)
 
 	ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
 
-	if (is_normal_sched)
-	    reds = actual_reds = calls - esdp->virtual_reds;
-	else
-	    reds = actual_reds = get_dirty_reds(esdp, p);
+	reds = actual_reds = calls - esdp->virtual_reds;
 
 	ASSERT(actual_reds >= 0);
 	if (reds < ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST)
@@ -9994,17 +9930,10 @@ Process *schedule(Process *p, int calls)
 	    calls = 0;
 	    reds = context_reds;
 
-#ifdef ERTS_SMP
-
 	    erts_smp_runq_unlock(rq);
 
-#endif /* ERTS_SMP */
-
 	}
 
-	if (!is_normal_sched)
-	    save_dirty_start(esdp, p);
-
 #ifdef ERTS_SMP
 
 	if (flags & ERTS_RUNQ_FLG_PROTECTED)
@@ -11745,8 +11674,6 @@ delete_process(Process* p)
     if (nif_export)
 	erts_destroy_nif_export(nif_export);
 
-    clean_dirty_start(p);
-
     /* Cleanup psd */
 
     psd = (ErtsPSD *) erts_smp_atomic_read_nob(&p->psd);
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index b44ac442aa..7569fd81bd 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -810,25 +810,13 @@ erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi)
 #define ERTS_PSD_DELAYED_GC_TASK_QS		4
 #define ERTS_PSD_NIF_TRAP_EXPORT		5
 #define ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF	6
-#define ERTS_PSD_DIRTY_CPU_START		7
 
-#define ERTS_PSD_SIZE				8
+#define ERTS_PSD_SIZE				7
 
-#if !defined(HIPE) && !defined(ERTS_DIRTY_SCHEDULERS)
+#if !defined(HIPE)
 #  undef ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF
-#  undef ERTS_PSD_DIRTY_CPU_START
 #  undef ERTS_PSD_SIZE
 #  define ERTS_PSD_SIZE 6
-#elif !defined(HIPE)
-#  undef ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF
-#  undef ERTS_PSD_DIRTY_CPU_START
-#  undef ERTS_PSD_SIZE
-#  define ERTS_PSD_DIRTY_CPU_START		6
-#  define ERTS_PSD_SIZE 7
-#elif !defined(ERTS_DIRTY_SCHEDULERS)
-#  undef ERTS_PSD_DIRTY_CPU_START
-#  undef ERTS_PSD_SIZE
-#  define ERTS_PSD_SIZE 7
 #endif
 
 typedef struct {
@@ -1831,7 +1819,7 @@ Eterm erts_get_schedulers_binds(Process *c_p);
 Eterm erts_set_cpu_topology(Process *c_p, Eterm term);
 Eterm erts_bind_schedulers(Process *c_p, Eterm how);
 ErtsRunQueue *erts_schedid2runq(Uint);
-Process *schedule(Process*, int);
+Process *erts_schedule(ErtsSchedulerData *, Process*, int);
 void erts_schedule_misc_op(void (*)(void *), void *);
 Eterm erl_create_process(Process*, Eterm, Eterm, Eterm, ErlSpawnOpts*);
 void erts_do_exit_process(Process*, Eterm);
@@ -2061,13 +2049,6 @@ erts_psd_set(Process *p, int ix, void *data)
   ((struct saved_calls *) erts_psd_set((P), ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF, (void *) (SCB)))
 #endif
 
-#ifdef ERTS_DIRTY_SCHEDULERS
-#define ERTS_PROC_GET_DIRTY_CPU_START(P) \
-  ((void *) erts_psd_get((P), ERTS_PSD_DIRTY_CPU_START))
-#define ERTS_PROC_SET_DIRTY_CPU_START(P, DCS) \
-  ((void *) erts_psd_set((P), ERTS_PSD_DIRTY_CPU_START, (void *) (DCS)))
-#endif
-
 ERTS_GLB_INLINE Eterm erts_proc_get_error_handler(Process *p);
 ERTS_GLB_INLINE Eterm erts_proc_set_error_handler(Process *p, Eterm handler);
 
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 15253bb53e..f3d4ac56cd 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -62,7 +62,8 @@ struct enif_environment_t /* ErlNifEnv */
 extern void erts_pre_nif(struct enif_environment_t*, Process*,
 			 struct erl_module_nif*, Process* tracee);
 extern void erts_post_nif(struct enif_environment_t* env);
-extern void erts_pre_dirty_nif(struct enif_environment_t*, Process*,
+extern void erts_pre_dirty_nif(ErtsSchedulerData *,
+			       struct enif_environment_t*, Process*,
 			       struct erl_module_nif*, Process* tracee);
 extern Eterm erts_nif_taints(Process* p);
 extern void erts_print_nif_taints(int to, void* to_arg);
@@ -1154,7 +1155,7 @@ void print_pass_through(int, byte*, int);
 int catchlevel(Process*);
 void init_emulator(void);
 void process_main(void);
-void dirty_process_main(void);
+void erts_dirty_process_main(ErtsSchedulerData *);
 Eterm build_stacktrace(Process* c_p, Eterm exc);
 Eterm expand_error_value(Process* c_p, Uint freason, Eterm Value);
 void erts_save_stacktrace(Process* p, struct StackTrace* s, int depth);
diff --git a/erts/emulator/hipe/hipe_mode_switch.c b/erts/emulator/hipe/hipe_mode_switch.c
index 884331e969..ed95045292 100644
--- a/erts/emulator/hipe/hipe_mode_switch.c
+++ b/erts/emulator/hipe/hipe_mode_switch.c
@@ -547,7 +547,7 @@ Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[])
 	      p->flags &= ~F_HIPE_MODE;
 
 	      ERTS_SMP_UNREQ_PROC_MAIN_LOCK(p);
-	      p = schedule(p, reds_in - p->fcalls);
+	      p = erts_schedule(NULL, p, reds_in - p->fcalls);
 	      ERTS_SMP_REQ_PROC_MAIN_LOCK(p);
 	      ASSERT(!(p->flags & F_HIPE_MODE));
 #ifdef ERTS_SMP