diff options
Diffstat (limited to 'erts/emulator/beam/erl_process.c')
| -rw-r--r-- | erts/emulator/beam/erl_process.c | 996 |
1 files changed, 655 insertions, 341 deletions
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index da4b468d66..084ecb14ec 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -107,9 +107,31 @@ #define LOW_BIT (1 << PRIORITY_LOW) #define PORT_BIT (1 << ERTS_PORT_PRIO_LEVEL) -#define ERTS_EMPTY_RUNQ(RQ) \ - ((ERTS_RUNQ_FLGS_GET_NOB((RQ)) & ERTS_RUNQ_FLGS_QMASK) == 0 \ - && (RQ)->misc.start == NULL) +#define ERTS_IS_RUNQ_EMPTY_FLGS(FLGS) \ + (!((FLGS) & (ERTS_RUNQ_FLGS_QMASK|ERTS_RUNQ_FLG_MISC_OP))) + +#define ERTS_IS_RUNQ_EMPTY_PORTS_FLGS(FLGS) \ + (!((FLGS) & (PORT_BIT|ERTS_RUNQ_FLG_MISC_OP))) + +#define ERTS_EMPTY_RUNQ(RQ) \ + ERTS_IS_RUNQ_EMPTY_FLGS(ERTS_RUNQ_FLGS_GET_NOB((RQ))) + +#define ERTS_EMPTY_RUNQ_PORTS(RQ) \ + ERTS_IS_RUNQ_EMPTY_FLGS(ERTS_RUNQ_FLGS_GET_NOB((RQ))) + +static ERTS_INLINE int +runq_got_work_to_execute_flags(Uint32 flags) +{ + if (flags & ERTS_RUNQ_FLG_HALTING) + return !ERTS_IS_RUNQ_EMPTY_PORTS_FLGS(flags); + return !ERTS_IS_RUNQ_EMPTY_FLGS(flags); +} + +static ERTS_INLINE int +runq_got_work_to_execute(ErtsRunQueue *rq) +{ + return runq_got_work_to_execute_flags(ERTS_RUNQ_FLGS_GET_NOB(rq)); +} #undef RUNQ_READ_RQ #undef RUNQ_SET_RQ @@ -141,9 +163,6 @@ do { \ # define ERTS_DBG_VERIFY_VALID_RUNQP(RQP) #endif -#define ERTS_EMPTY_RUNQ_PORTS(RQ) \ - (RUNQ_READ_LEN(&(RQ)->ports.info.len) == 0 && (RQ)->misc.start == NULL) - const Process erts_invalid_process = {{ERTS_INVALID_PID}}; extern BeamInstr beam_apply[]; @@ -200,158 +219,139 @@ typedef struct { ErtsProcList *chngq; } ErtsMultiSchedulingBlock; +typedef struct { + Uint32 normal; +#ifdef ERTS_DIRTY_SCHEDULERS + Uint32 dirty_cpu; + Uint32 dirty_io; +#endif +} ErtsSchedTypeCounters; + static struct { erts_smp_mtx_t mtx; - Uint32 online; - Uint32 curr_online; - Uint32 active; + ErtsSchedTypeCounters online; + ErtsSchedTypeCounters curr_online; + ErtsSchedTypeCounters active; erts_smp_atomic32_t changing; ErtsProcList *chngq; Eterm changer; ErtsMultiSchedulingBlock nmsb; /* Normal multi Scheduling Block */ ErtsMultiSchedulingBlock msb; /* Multi Scheduling Block */ +#ifdef ERTS_DIRTY_SCHEDULERS + ErtsSchedType last_msb_dirty_type; +#endif } schdlr_sspnd; -#define ERTS_SCHDLR_SSPND_S_BITS 10 -#define ERTS_SCHDLR_SSPND_DCS_BITS 11 -#define ERTS_SCHDLR_SSPND_DIS_BITS 11 - -#define ERTS_SCHDLR_SSPND_S_MASK ((1 << ERTS_SCHDLR_SSPND_S_BITS)-1) -#define ERTS_SCHDLR_SSPND_DCS_MASK ((1 << ERTS_SCHDLR_SSPND_DCS_BITS)-1) -#define ERTS_SCHDLR_SSPND_DIS_MASK ((1 << ERTS_SCHDLR_SSPND_DIS_BITS)-1) - -#define ERTS_SCHDLR_SSPND_S_SHIFT 0 -#define ERTS_SCHDLR_SSPND_DCS_SHIFT (ERTS_SCHDLR_SSPND_S_SHIFT \ - + ERTS_SCHDLR_SSPND_S_BITS) -#define ERTS_SCHDLR_SSPND_DIS_SHIFT (ERTS_SCHDLR_SSPND_DCS_SHIFT \ - + ERTS_SCHDLR_SSPND_DCS_BITS) - -#if (ERTS_SCHDLR_SSPND_S_BITS \ - + ERTS_SCHDLR_SSPND_DCS_BITS \ - + ERTS_SCHDLR_SSPND_DIS_BITS) > 32 -# error Wont fit in Uint32 -#endif +static void init_scheduler_suspend(void); -#if (ERTS_MAX_NO_OF_SCHEDULERS-1) > ERTS_SCHDLR_SSPND_S_MASK -# error Max no schedulers wont fit in its bit-field -#endif -#if ERTS_MAX_NO_OF_DIRTY_CPU_SCHEDULERS > ERTS_SCHDLR_SSPND_DCS_MASK -# error Max no dirty cpu schedulers wont fit in its bit-field -#endif -#if ERTS_MAX_NO_OF_DIRTY_IO_SCHEDULERS > ERTS_SCHDLR_SSPND_DIS_MASK -# error Max no dirty io schedulers wont fit in its bit-field +static ERTS_INLINE Uint32 +schdlr_sspnd_eq_nscheds(ErtsSchedTypeCounters *val1p, ErtsSchedTypeCounters *val2p) +{ + int res = val1p->normal == val2p->normal; +#ifdef ERTS_DIRTY_SCHEDULERS + res &= val1p->dirty_cpu == val2p->dirty_cpu; + res &= val1p->dirty_io == val2p->dirty_io; #endif - -#define ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(S, DCS, DIS) \ - ((((Uint32) (((S) & ERTS_SCHDLR_SSPND_S_MASK))-1) \ - << ERTS_SCHDLR_SSPND_S_SHIFT) \ - | ((((Uint32) ((DCS) & ERTS_SCHDLR_SSPND_DCS_MASK)) \ - << ERTS_SCHDLR_SSPND_DCS_SHIFT)) \ - | ((((Uint32) ((DIS) & ERTS_SCHDLR_SSPND_DIS_MASK)) \ - << ERTS_SCHDLR_SSPND_DIS_SHIFT))) - -static void init_scheduler_suspend(void); + return res; +} static ERTS_INLINE Uint32 -schdlr_sspnd_get_nscheds(Uint32 *valp, ErtsSchedType type) +schdlr_sspnd_get_nscheds(ErtsSchedTypeCounters *valp, + ErtsSchedType type) { - Uint32 res = (Uint32) (*valp); switch (type) { case ERTS_SCHED_NORMAL: - res >>= ERTS_SCHDLR_SSPND_S_SHIFT; - res &= (Uint32) ERTS_SCHDLR_SSPND_S_MASK; - res++; - break; + return valp->normal; +#ifdef ERTS_DIRTY_SCHEDULERS case ERTS_SCHED_DIRTY_CPU: - res >>= ERTS_SCHDLR_SSPND_DCS_SHIFT; - res &= (Uint32) ERTS_SCHDLR_SSPND_DCS_MASK; - break; + return valp->dirty_cpu; case ERTS_SCHED_DIRTY_IO: - res >>= ERTS_SCHDLR_SSPND_DIS_SHIFT; - res &= (Uint32) ERTS_SCHDLR_SSPND_DIS_MASK; - break; + return valp->dirty_io; +#else + case ERTS_SCHED_DIRTY_CPU: + case ERTS_SCHED_DIRTY_IO: + return 0; +#endif default: ERTS_INTERNAL_ERROR("Invalid scheduler type"); return 0; } +} +static ERTS_INLINE Uint32 +schdlr_sspnd_get_nscheds_tot(ErtsSchedTypeCounters *valp) +{ + Uint32 res = valp->normal; +#ifdef ERTS_DIRTY_SCHEDULERS + res += valp->dirty_cpu; + res += valp->dirty_io; +#endif return res; } static ERTS_INLINE void -schdlr_sspnd_dec_nscheds(Uint32 *valp, ErtsSchedType type) +schdlr_sspnd_dec_nscheds(ErtsSchedTypeCounters *valp, + ErtsSchedType type) { ASSERT(schdlr_sspnd_get_nscheds(valp, type) > 0); switch (type) { case ERTS_SCHED_NORMAL: - *valp -= ((Uint32) 1) << ERTS_SCHDLR_SSPND_S_SHIFT; + valp->normal--; break; +#ifdef ERTS_DIRTY_SCHEDULERS case ERTS_SCHED_DIRTY_CPU: - *valp -= ((Uint32) 1) << ERTS_SCHDLR_SSPND_DCS_SHIFT; + valp->dirty_cpu--; break; case ERTS_SCHED_DIRTY_IO: - *valp -= ((Uint32) 1) << ERTS_SCHDLR_SSPND_DIS_SHIFT; + valp->dirty_io--; break; +#endif default: ERTS_INTERNAL_ERROR("Invalid scheduler type"); } } static ERTS_INLINE void -schdlr_sspnd_inc_nscheds(Uint32 *valp, ErtsSchedType type) +schdlr_sspnd_inc_nscheds(ErtsSchedTypeCounters *valp, + ErtsSchedType type) { switch (type) { case ERTS_SCHED_NORMAL: - ASSERT(schdlr_sspnd_get_nscheds(valp, type) - < ERTS_MAX_NO_OF_SCHEDULERS-1); - *valp += ((Uint32) 1) << ERTS_SCHDLR_SSPND_S_SHIFT; + valp->normal++; break; +#ifdef ERTS_DIRTY_SCHEDULERS case ERTS_SCHED_DIRTY_CPU: - ASSERT(schdlr_sspnd_get_nscheds(valp, type) - < ERTS_MAX_NO_OF_DIRTY_CPU_SCHEDULERS); - *valp += ((Uint32) 1) << ERTS_SCHDLR_SSPND_DCS_SHIFT; + valp->dirty_cpu++; break; case ERTS_SCHED_DIRTY_IO: - ASSERT(schdlr_sspnd_get_nscheds(valp, type) - < ERTS_MAX_NO_OF_DIRTY_IO_SCHEDULERS); - *valp += ((Uint32) 1) << ERTS_SCHDLR_SSPND_DIS_SHIFT; + valp->dirty_io++; break; +#endif default: ERTS_INTERNAL_ERROR("Invalid scheduler type"); } } static ERTS_INLINE void -schdlr_sspnd_set_nscheds(Uint32 *valp, ErtsSchedType type, Uint32 no) +schdlr_sspnd_set_nscheds(ErtsSchedTypeCounters *valp, + ErtsSchedType type, Uint32 no) { - Uint32 val = *valp; - switch (type) { case ERTS_SCHED_NORMAL: - ASSERT(no > 0); - val &= ~(((Uint32) ERTS_SCHDLR_SSPND_S_MASK) - << ERTS_SCHDLR_SSPND_S_SHIFT); - val |= (((no-1) & ((Uint32) ERTS_SCHDLR_SSPND_S_MASK)) - << ERTS_SCHDLR_SSPND_S_SHIFT); + valp->normal = no; break; +#ifdef ERTS_DIRTY_SCHEDULERS case ERTS_SCHED_DIRTY_CPU: - val &= ~(((Uint32) ERTS_SCHDLR_SSPND_DCS_MASK) - << ERTS_SCHDLR_SSPND_DCS_SHIFT); - val |= ((no & ((Uint32) ERTS_SCHDLR_SSPND_DCS_MASK)) - << ERTS_SCHDLR_SSPND_DCS_SHIFT); + valp->dirty_cpu = no; break; case ERTS_SCHED_DIRTY_IO: - val &= ~(((Uint32) ERTS_SCHDLR_SSPND_DIS_MASK) - << ERTS_SCHDLR_SSPND_DIS_SHIFT); - val |= ((no & ((Uint32) ERTS_SCHDLR_SSPND_DIS_MASK)) - << ERTS_SCHDLR_SSPND_DIS_SHIFT); + valp->dirty_io = no; break; +#endif default: ERTS_INTERNAL_ERROR("Invalid scheduler type"); } - - *valp = val; } static struct { @@ -2293,7 +2293,8 @@ static ERTS_INLINE erts_aint32_t handle_reap_ports(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) { unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_REAP_PORTS); - awdp->esdp->run_queue->halt_in_progress = 1; + ERTS_RUNQ_FLGS_SET(awdp->esdp->run_queue, ERTS_RUNQ_FLG_HALTING); + if (erts_smp_atomic32_dec_read_acqb(&erts_halt_progress) == 0) { int i, max = erts_ptab_max(&erts_port); erts_smp_atomic32_set_nob(&erts_halt_progress, 1); @@ -2833,11 +2834,12 @@ static erts_aint32_t sched_prep_spin_wait(ErtsSchedulerSleepInfo *ssi) { erts_aint32_t oflgs; - erts_aint32_t nflgs = (ERTS_SSI_FLG_SLEEPING - | ERTS_SSI_FLG_WAITING); + erts_aint32_t nflgs; erts_aint32_t xflgs = 0; do { + nflgs = (xflgs & ERTS_SSI_FLG_MSB_EXEC); + nflgs |= ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING; oflgs = erts_smp_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs); if (oflgs == xflgs) return nflgs; @@ -2859,7 +2861,7 @@ sched_prep_cont_spin_wait(ErtsSchedulerSleepInfo *ssi) if (oflgs == xflgs) return nflgs; xflgs = oflgs; - nflgs |= oflgs & ERTS_SSI_FLG_SUSPENDED; + nflgs |= oflgs & (ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC); } while (oflgs & ERTS_SSI_FLG_WAITING); return oflgs; } @@ -2909,7 +2911,7 @@ sched_set_sleeptype(ErtsSchedulerSleepInfo *ssi, erts_aint32_t sleep_type) return oflgs; } xflgs = oflgs; - nflgs |= oflgs & ERTS_SSI_FLG_SUSPENDED; + nflgs |= oflgs & (ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC); } } @@ -3044,6 +3046,8 @@ aux_thread(void *unused) return NULL; } +static void suspend_scheduler(ErtsSchedulerData *esdp); + #endif /* ERTS_SMP */ static void @@ -3209,8 +3213,10 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) } - if (flgs & ~ERTS_SSI_FLG_SUSPENDED) - erts_smp_atomic32_read_band_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); + if (flgs & ~(ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC)) + erts_smp_atomic32_read_band_nob(&ssi->flags, + (ERTS_SSI_FLG_SUSPENDED + | ERTS_SSI_FLG_MSB_EXEC)); if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && !thr_prgr_active) { erts_thr_progress_active(esdp, thr_prgr_active = 1); @@ -3409,8 +3415,10 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) erts_smp_runq_lock(rq); } clear_sys_scheduling(); - if (flgs & ~ERTS_SSI_FLG_SUSPENDED) - erts_smp_atomic32_read_band_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); + if (flgs & ~(ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC)) + erts_smp_atomic32_read_band_nob(&ssi->flags, + (ERTS_SSI_FLG_SUSPENDED + | ERTS_SSI_FLG_MSB_EXEC)); #endif if (!working) sched_wall_time_change(esdp, working = 1); @@ -3433,17 +3441,54 @@ ssi_flags_set_wake(ErtsSchedulerSleepInfo *ssi) oflgs = erts_smp_atomic32_cmpxchg_relb(&ssi->flags, nflgs, xflgs); if (oflgs == xflgs) return oflgs; - nflgs = oflgs & ERTS_SSI_FLG_SUSPENDED; + nflgs = oflgs & (ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC); xflgs = oflgs; } } +static ERTS_INLINE void +ssi_wake(ErtsSchedulerSleepInfo *ssi) +{ + erts_sched_finish_poke(ssi, ssi_flags_set_wake(ssi)); +} + +#ifdef ERTS_DIRTY_SCHEDULERS + static void -wake_scheduler(ErtsRunQueue *rq) +dcpu_sched_ix_suspend_wake(Uint ix) { - ErtsSchedulerSleepInfo *ssi; - erts_aint32_t flgs; + ErtsSchedulerSleepInfo* ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); + erts_smp_atomic32_read_bor_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); + ssi_wake(ssi); +} + +static void +dio_sched_ix_suspend_wake(Uint ix) +{ + ErtsSchedulerSleepInfo* ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix); + erts_smp_atomic32_read_bor_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); + ssi_wake(ssi); +} + +static void +dcpu_sched_ix_wake(Uint ix) +{ + ssi_wake(ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix)); +} + +#if 0 +static void +dio_sched_ix_wake(Uint ix) +{ + ssi_wake(ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix)); +} +#endif +#endif + +static void +wake_scheduler(ErtsRunQueue *rq) +{ /* * The unlocked run queue is not strictly necessary * from a thread safety or deadlock prevention @@ -3455,10 +3500,7 @@ wake_scheduler(ErtsRunQueue *rq) ERTS_SMP_LC_ASSERT(!erts_smp_lc_runq_is_locked(rq) || ERTS_RUNQ_IX_IS_DIRTY(rq->ix)); - ssi = rq->scheduler->ssi; - - flgs = ssi_flags_set_wake(ssi); - erts_sched_finish_poke(ssi, flgs); + ssi_wake(rq->scheduler->ssi); } #ifdef ERTS_DIRTY_SCHEDULERS @@ -3505,6 +3547,13 @@ wake_dirty_schedulers(ErtsRunQueue *rq, int one) } while (ssi); } } + +static void +wake_dirty_scheduler(ErtsRunQueue *rq) +{ + wake_dirty_schedulers(rq, 1); +} + #endif #define ERTS_NO_USED_RUNQS_SHIFT 16 @@ -3645,7 +3694,7 @@ smp_notify_inc_runq(ErtsRunQueue *runq) if (runq) { #ifdef ERTS_DIRTY_SCHEDULERS if (ERTS_RUNQ_IX_IS_DIRTY(runq->ix)) - wake_dirty_schedulers(runq, 1); + wake_dirty_scheduler(runq); else #endif wake_scheduler(runq); @@ -3949,8 +3998,7 @@ suspend_run_queue(ErtsRunQueue *rq) wake_scheduler(rq); } -static void scheduler_ix_resume_wake(Uint ix); -static void scheduler_ssi_resume_wake(ErtsSchedulerSleepInfo *ssi); +static void nrml_sched_ix_resume_wake(Uint ix); static ERTS_INLINE void resume_run_queue(ErtsRunQueue *rq) @@ -3958,16 +4006,19 @@ resume_run_queue(ErtsRunQueue *rq) int pix; Uint32 oflgs; + ASSERT(!ERTS_RUNQ_IX_IS_DIRTY(rq->ix)); + erts_smp_runq_lock(rq); oflgs = ERTS_RUNQ_FLGS_READ_BSET(rq, (ERTS_RUNQ_FLG_OUT_OF_WORK | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK - | ERTS_RUNQ_FLG_SUSPENDED), + | ERTS_RUNQ_FLG_SUSPENDED + | ERTS_RUNQ_FLG_MSB_EXEC), (ERTS_RUNQ_FLG_OUT_OF_WORK | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK)); - if (oflgs & ERTS_RUNQ_FLG_SUSPENDED) { + if (oflgs & (ERTS_RUNQ_FLG_SUSPENDED|ERTS_RUNQ_FLG_MSB_EXEC)) { erts_aint32_t len; rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS; @@ -3986,10 +4037,7 @@ resume_run_queue(ErtsRunQueue *rq) erts_smp_runq_unlock(rq); -#ifdef ERTS_DIRTY_SCHEDULERS - if (!ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) -#endif - scheduler_ix_resume_wake(rq->ix); + nrml_sched_ix_resume_wake(rq->ix); } typedef struct { @@ -4047,28 +4095,22 @@ evacuate_run_queue(ErtsRunQueue *rq, int prio_q; ErtsRunQueue *to_rq; ErtsMigrationPaths *mps; - ErtsMigrationPath *mp = NULL; + ErtsMigrationPath *mp; ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); (void) ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_PROTECTED); -#ifdef ERTS_DIRTY_SCHEDULERS - if (!ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) -#endif - { - mps = erts_get_migration_paths_managed(); - mp = &mps->mpath[rq->ix]; - } + ASSERT(!ERTS_RUNQ_IX_IS_DIRTY(rq->ix)); + + mps = erts_get_migration_paths_managed(); + mp = &mps->mpath[rq->ix]; /* Evacuate scheduled misc ops */ if (rq->misc.start) { ErtsMiscOpList *start, *end; -#ifdef ERTS_DIRTY_SCHEDULERS - ASSERT(!ERTS_RUNQ_IX_IS_DIRTY(rq->ix)); -#endif to_rq = mp->misc_evac_runq; if (!to_rq) return; @@ -4077,6 +4119,7 @@ evacuate_run_queue(ErtsRunQueue *rq, end = rq->misc.end; rq->misc.start = NULL; rq->misc.end = NULL; + ERTS_RUNQ_FLGS_UNSET_NOB(rq, ERTS_RUNQ_FLG_MISC_OP); erts_smp_runq_unlock(rq); erts_smp_runq_lock(to_rq); @@ -4097,9 +4140,6 @@ evacuate_run_queue(ErtsRunQueue *rq, if (rq->ports.start) { Port *prt; -#ifdef ERTS_DIRTY_SCHEDULERS - ASSERT(!ERTS_RUNQ_IX_IS_DIRTY(rq->ix)); -#endif to_rq = mp->prio[ERTS_PORT_PRIO_LEVEL].runq; if (!to_rq) return; @@ -4137,15 +4177,10 @@ evacuate_run_queue(ErtsRunQueue *rq, int notify = 0; to_rq = NULL; -#ifdef ERTS_DIRTY_SCHEDULERS - if (!ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) -#endif - { - if (!mp->prio[prio_q].runq) - return; - if (prio_q == PRIORITY_NORMAL && !mp->prio[PRIORITY_LOW].runq) - return; - } + if (!mp->prio[prio_q].runq) + return; + if (prio_q == PRIORITY_NORMAL && !mp->prio[PRIORITY_LOW].runq) + return; proc = dequeue_process(rq, prio_q, &state); while (proc) { @@ -4201,11 +4236,6 @@ evacuate_run_queue(ErtsRunQueue *rq, goto handle_next_proc; } -#ifdef ERTS_DIRTY_SCHEDULERS - if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) - clear_proc_dirty_queue_bit(real_proc, rq, qbit); -#endif - if (ERTS_PSFLG_BOUND & real_state) { /* Bound processes get stuck here... */ proc->next = NULL; @@ -4219,16 +4249,7 @@ evacuate_run_queue(ErtsRunQueue *rq, int prio = (int) ERTS_PSFLGS_GET_PRQ_PRIO(state); erts_smp_runq_unlock(rq); -#ifdef ERTS_DIRTY_SCHEDULERS - if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) - /* - * dirty run queues evacuate only to run - * queue 0 during multi-scheduling blocking - */ - to_rq = ERTS_RUNQ_IX(0); - else -#endif - to_rq = mp->prio[prio].runq; + to_rq = mp->prio[prio].runq; RUNQ_SET_RQ(&proc->run_queue, to_rq); erts_smp_runq_lock(to_rq); @@ -4263,7 +4284,7 @@ try_steal_task_from_victim(ErtsRunQueue *rq, int *rq_lockedp, ErtsRunQueue *vrq, erts_smp_runq_lock(vrq); - if (rq->halt_in_progress) + if (ERTS_RUNQ_FLGS_GET_NOB(rq) & ERTS_RUNQ_FLG_HALTING) goto no_procs; /* @@ -4362,8 +4383,7 @@ check_possible_steal_victim(ErtsRunQueue *rq, int *rq_lockedp, int vix) { ErtsRunQueue *vrq = ERTS_RUNQ_IX(vix); Uint32 flags = ERTS_RUNQ_FLGS_GET(vrq); - if ((flags & (ERTS_RUNQ_FLG_NONEMPTY - | ERTS_RUNQ_FLG_PROTECTED)) == ERTS_RUNQ_FLG_NONEMPTY) + if (runq_got_work_to_execute_flags(flags) & (!(flags & ERTS_RUNQ_FLG_PROTECTED))) return try_steal_task_from_victim(rq, rq_lockedp, vrq, flags); else return 0; @@ -4431,11 +4451,9 @@ try_steal_task(ErtsRunQueue *rq) if (!rq_locked) erts_smp_runq_lock(rq); - if (!res) - res = rq->halt_in_progress ? - !ERTS_EMPTY_RUNQ_PORTS(rq) : !ERTS_EMPTY_RUNQ(rq); - - return res; + if (res) + return res; + return runq_got_work_to_execute(rq); } /* Run queue balancing */ @@ -5347,7 +5365,7 @@ wakeup_other_check(ErtsRunQueue *rq, Uint32 flags) #ifdef ERTS_DIRTY_SCHEDULERS if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) { if (rq->waiting) { - wake_dirty_schedulers(rq, 1); + wake_dirty_scheduler(rq); } } else #endif @@ -5710,13 +5728,29 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num, erts_alloc_permanent_cache_aligned(ERTS_ALC_T_BEAM_REGISTER, MAX_REG * sizeof(FloatDef)); #ifdef ERTS_DIRTY_SCHEDULERS + esdp->run_queue = runq; if (ERTS_RUNQ_IX_IS_DIRTY(runq->ix)) { esdp->no = 0; + if (runq == ERTS_DIRTY_CPU_RUNQ) + esdp->type = ERTS_SCHED_DIRTY_CPU; + else { + ASSERT(runq == ERTS_DIRTY_IO_RUNQ); + esdp->type = ERTS_SCHED_DIRTY_IO; + } ERTS_DIRTY_SCHEDULER_NO(esdp) = (Uint) num; + if (num == 1) { + /* + * Multi-scheduling block functionality depends + * on finding dirty scheduler number 1 here... + */ + runq->scheduler = esdp; + } } else { + esdp->type = ERTS_SCHED_NORMAL; esdp->no = (Uint) num; ERTS_DIRTY_SCHEDULER_NO(esdp) = 0; + runq->scheduler = esdp; } esdp->dirty_shadow_process = shadow_proc; if (shadow_proc) { @@ -5728,6 +5762,8 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num, shadow_proc->static_flags = ERTS_STC_FLG_SHADOW_PROC; } #else + runq->scheduler = esdp; + esdp->run_queue = runq; esdp->no = (Uint) num; #endif @@ -5740,9 +5776,6 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num, erts_init_atom_cache_map(&esdp->atom_cache_map); - esdp->run_queue = runq; - esdp->run_queue->scheduler = esdp; - esdp->last_monotonic_time = 0; esdp->check_time_reds = 0; @@ -5858,7 +5891,6 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online erts_smp_atomic32_set_nob(&rq->len, 0); rq->wakeup_other = 0; rq->wakeup_other_reds = 0; - rq->halt_in_progress = 0; rq->procs.pending_exiters = NULL; rq->procs.context_switches = 0; @@ -6974,15 +7006,8 @@ resume_process(Process *p, ErtsProcLocks locks) #ifdef ERTS_SMP -static void -scheduler_ix_resume_wake(Uint ix) -{ - ErtsSchedulerSleepInfo *ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); - scheduler_ssi_resume_wake(ssi); -} - -static void -scheduler_ssi_resume_wake(ErtsSchedulerSleepInfo *ssi) +static ERTS_INLINE void +sched_resume_wake__(ErtsSchedulerSleepInfo *ssi) { erts_aint32_t xflgs = (ERTS_SSI_FLG_SLEEPING | ERTS_SSI_FLG_TSE_SLEEPING @@ -6996,9 +7021,31 @@ scheduler_ssi_resume_wake(ErtsSchedulerSleepInfo *ssi) break; } xflgs = oflgs; - } while (oflgs & ERTS_SSI_FLG_SUSPENDED); + } while (oflgs & (ERTS_SSI_FLG_MSB_EXEC|ERTS_SSI_FLG_SUSPENDED)); +} + +static void +nrml_sched_ix_resume_wake(Uint ix) +{ + sched_resume_wake__(ERTS_SCHED_SLEEP_INFO_IX(ix)); } +#ifdef ERTS_DIRTY_SCHEDULERS + +static void +dcpu_sched_ix_resume_wake(Uint ix) +{ + sched_resume_wake__(ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix)); +} + +static void +dio_sched_ix_resume_wake(Uint ix) +{ + sched_resume_wake__(ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix)); +} + +#endif + static erts_aint32_t sched_prep_spin_suspended(ErtsSchedulerSleepInfo *ssi, erts_aint32_t xpct) { @@ -7078,9 +7125,18 @@ static void init_scheduler_suspend(void) { erts_smp_mtx_init(&schdlr_sspnd.mtx, "schdlr_sspnd"); - schdlr_sspnd.online = ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0); - schdlr_sspnd.curr_online = ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0); - schdlr_sspnd.active = ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0); + schdlr_sspnd.online.normal = 1; + schdlr_sspnd.curr_online.normal = 1; + schdlr_sspnd.active.normal = 1; +#ifdef ERTS_DIRTY_SCHEDULERS + schdlr_sspnd.online.dirty_cpu = 0; + schdlr_sspnd.curr_online.dirty_cpu = 0; + schdlr_sspnd.active.dirty_cpu = 0; + schdlr_sspnd.online.dirty_io = 0; + schdlr_sspnd.curr_online.dirty_io = 0; + schdlr_sspnd.active.dirty_io = 0; + schdlr_sspnd.last_msb_dirty_type = ERTS_SCHED_DIRTY_IO; +#endif erts_smp_atomic32_init_nob(&schdlr_sspnd.changing, 0); schdlr_sspnd.chngq = NULL; schdlr_sspnd.changer = am_false; @@ -7141,6 +7197,252 @@ schdlr_sspnd_resume_procs(ErtsSchedType sched_type, } } +#ifdef ERTS_DIRTY_SCHEDULERS + +static ERTS_INLINE int +have_dirty_work(void) +{ + return !(ERTS_EMPTY_RUNQ(ERTS_DIRTY_CPU_RUNQ) + | ERTS_EMPTY_RUNQ(ERTS_DIRTY_IO_RUNQ)); +} + +#define ERTS_MSB_NONE_PRIO_BIT PORT_BIT + +static ERTS_INLINE Uint32 +msb_runq_prio_bit(Uint32 flgs) +{ + int pbit; + + pbit = (int) (flgs & ERTS_RUNQ_FLGS_PROCS_QMASK); + if (flgs & PORT_BIT) { + /* rate ports as proc prio high */ + pbit |= HIGH_BIT; + } + if (flgs & ERTS_RUNQ_FLG_MISC_OP) { + /* rate misc ops as proc prio normal */ + pbit |= NORMAL_BIT; + } + if (flgs & LOW_BIT) { + /* rate low prio as normal (avoid starvation) */ + pbit |= NORMAL_BIT; + } + if (!pbit) + pbit = (int) ERTS_MSB_NONE_PRIO_BIT; + else + pbit &= -pbit; /* least significant bit set... */ + ASSERT(pbit); + + /* High prio low value; low prio high value... */ + return (Uint32) pbit; +} + +static ERTS_INLINE void +msb_runq_prio_bits(Uint32 *nrmlp, Uint32 *dcpup, Uint32 *diop) +{ + Uint32 flgs = ERTS_RUNQ_FLGS_GET(ERTS_RUNQ_IX(0)); + if (flgs & ERTS_RUNQ_FLG_HALTING) { + /* + * Emulator is halting; only execute port jobs + * on normal scheduler. Ensure that we switch + * to the normal scheduler. + */ + *nrmlp = HIGH_BIT; + *dcpup = ERTS_MSB_NONE_PRIO_BIT; + *diop = ERTS_MSB_NONE_PRIO_BIT; + } + else { + *nrmlp = msb_runq_prio_bit(flgs); + + flgs = ERTS_RUNQ_FLGS_GET(ERTS_DIRTY_CPU_RUNQ); + *dcpup = msb_runq_prio_bit(flgs); + + flgs = ERTS_RUNQ_FLGS_GET(ERTS_DIRTY_IO_RUNQ); + *diop = msb_runq_prio_bit(flgs); + } +} + +static int +msb_scheduler_type_switch(ErtsSchedType sched_type, + ErtsSchedulerData *esdp, + long no) +{ + Uint32 nrml_prio, dcpu_prio, dio_prio; + ErtsSchedType exec_type; + ErtsRunQueue *exec_rq; +#ifdef DEBUG + erts_aint32_t dbg_val; +#endif + + ASSERT(schdlr_sspnd.msb.ongoing); + + /* + * This function determines how to switch + * between scheduler types when multi-scheduling + * is blocked. + * + * If no dirty work exist, we always select + * execution of normal scheduler. If nothing + * executes, normal scheduler 1 should be waiting + * in sys_schedule(), otherwise we cannot react + * on I/O events. + * + * We unconditionally switch back to normal + * scheduler after executing dirty in order to + * make sure we check for I/O... + */ + + msb_runq_prio_bits(&nrml_prio, &dcpu_prio, &dio_prio); + + exec_type = ERTS_SCHED_NORMAL; + if (sched_type == ERTS_SCHED_NORMAL) { + + /* + * Check priorities of work in the + * different run-queues and determine + * run-queue with highest prio job... + */ + + if ((dcpu_prio == ERTS_MSB_NONE_PRIO_BIT) + & (dio_prio == ERTS_MSB_NONE_PRIO_BIT)) { + /* + * No dirty work exist; continue on normal + * scheduler... + */ + return 0; + } + + if (dcpu_prio < nrml_prio) { + exec_type = ERTS_SCHED_DIRTY_CPU; + if (dio_prio < dcpu_prio) + exec_type = ERTS_SCHED_DIRTY_IO; + } + else { + if (dio_prio < nrml_prio) + exec_type = ERTS_SCHED_DIRTY_IO; + } + + /* + * Make sure to alternate between dirty types + * inbetween normal execution if highest + * priorities are equal. + */ + + if (exec_type == ERTS_SCHED_NORMAL) { + if (dcpu_prio == nrml_prio) + exec_type = ERTS_SCHED_DIRTY_CPU; + else if (dio_prio == nrml_prio) + exec_type = ERTS_SCHED_DIRTY_IO; + else { + /* + * Normal work has higher prio than + * dirty work; continue on normal + * scheduler... + */ + return 0; + } + } + + ASSERT(exec_type != ERTS_SCHED_NORMAL); + if (dio_prio == dcpu_prio) { + /* Alter between dirty types... */ + if (schdlr_sspnd.last_msb_dirty_type == ERTS_SCHED_DIRTY_IO) + exec_type = ERTS_SCHED_DIRTY_CPU; + else + exec_type = ERTS_SCHED_DIRTY_IO; + } + } + + ASSERT(sched_type != exec_type); + + if (exec_type != ERTS_SCHED_NORMAL) + schdlr_sspnd.last_msb_dirty_type = exec_type; + else { + erts_aint32_t calls; + /* + * Going back to normal scheduler after + * dirty execution; make sure it will check + * for I/O... + */ + if (ERTS_USE_MODIFIED_TIMING()) + calls = ERTS_MODIFIED_TIMING_INPUT_REDS + 1; + else + calls = INPUT_REDUCTIONS + 1; + erts_smp_atomic32_set_nob(&function_calls, calls); + + if ((nrml_prio == ERTS_MSB_NONE_PRIO_BIT) + & ((dcpu_prio != ERTS_MSB_NONE_PRIO_BIT) + | (dio_prio != ERTS_MSB_NONE_PRIO_BIT))) { + /* + * We have dirty work, but an empty + * normal run-queue. + * + * Since the normal run-queue is + * empty, the normal scheduler will + * go to sleep when selected for + * execution. We have dirty work to + * do, so we only want it to check + * I/O, and then come back here and + * switch to dirty execution. + * + * To prevent the scheduler from going + * to sleep we trick it into believing + * it has work to do... + */ + ERTS_RUNQ_FLGS_SET_NOB(ERTS_RUNQ_IX(0), + ERTS_RUNQ_FLG_MISC_OP); + } + } + + /* + * Suspend this scheduler and wake up scheduler + * number one of another type... + */ +#ifdef DEBUG + dbg_val = +#else + (void) +#endif + erts_smp_atomic32_read_bset_mb(&esdp->ssi->flags, + (ERTS_SSI_FLG_SUSPENDED + | ERTS_SSI_FLG_MSB_EXEC), + ERTS_SSI_FLG_SUSPENDED); + ASSERT(dbg_val & ERTS_SSI_FLG_MSB_EXEC); + + switch (exec_type) { + case ERTS_SCHED_NORMAL: + exec_rq = ERTS_RUNQ_IX(0); + break; + case ERTS_SCHED_DIRTY_CPU: + exec_rq = ERTS_DIRTY_CPU_RUNQ; + break; + case ERTS_SCHED_DIRTY_IO: + exec_rq = ERTS_DIRTY_IO_RUNQ; + break; + default: + ERTS_INTERNAL_ERROR("Invalid scheduler type"); + exec_rq = NULL; + break; + } + +#ifdef DEBUG + dbg_val = +#else + (void) +#endif + erts_smp_atomic32_read_bset_mb(&exec_rq->scheduler->ssi->flags, + (ERTS_SSI_FLG_SUSPENDED + | ERTS_SSI_FLG_MSB_EXEC), + ERTS_SSI_FLG_MSB_EXEC); + ASSERT(dbg_val & ERTS_SSI_FLG_SUSPENDED); + + wake_scheduler(exec_rq); + + return 1; /* suspend this scheduler... */ + +} + +#endif + static void suspend_scheduler(ErtsSchedulerData *esdp) { @@ -7166,32 +7468,49 @@ suspend_scheduler(ErtsSchedulerData *esdp) * Regardless of why a scheduler is suspended, it ends up here. */ -#ifdef ERTS_DIRTY_SCHEDULERS - if (ERTS_SCHEDULER_IS_DIRTY(esdp)) { + +#if !defined(ERTS_DIRTY_SCHEDULERS) + + sched_type = ERTS_SCHED_NORMAL; + online_flag = ERTS_SCHDLR_SSPND_CHNG_ONLN; + no = esdp->no; + ASSERT(no != 1); + +#else + + sched_type = esdp->type; + switch (sched_type) { + case ERTS_SCHED_NORMAL: + online_flag = ERTS_SCHDLR_SSPND_CHNG_ONLN; + no = esdp->no; + break; + case ERTS_SCHED_DIRTY_CPU: + online_flag = ERTS_SCHDLR_SSPND_CHNG_DCPU_ONLN; + no = ERTS_DIRTY_SCHEDULER_NO(esdp); + break; + case ERTS_SCHED_DIRTY_IO: + online_flag = 0; no = ERTS_DIRTY_SCHEDULER_NO(esdp); - if (ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(esdp->run_queue)) { - online_flag = ERTS_SCHDLR_SSPND_CHNG_DCPU_ONLN; - sched_type = ERTS_SCHED_DIRTY_CPU; - } - else { - online_flag = 0; - sched_type = ERTS_SCHED_DIRTY_IO; - } + break; + default: + ERTS_INTERNAL_ERROR("Invalid scheduler type"); + return; } - else -#endif - { - online_flag = ERTS_SCHDLR_SSPND_CHNG_ONLN; - no = esdp->no; - sched_type = ERTS_SCHED_NORMAL; + + if (erts_smp_atomic32_read_nob(&ssi->flags) & ERTS_SSI_FLG_MSB_EXEC) { + ASSERT(no == 1); + if (!msb_scheduler_type_switch(sched_type, esdp, no)) + return; + /* Suspend and let scheduler 1 of another type execute... */ } - ASSERT(sched_type != ERTS_SCHED_NORMAL || no != 1); +#endif if (sched_type != ERTS_SCHED_NORMAL) erts_smp_runq_unlock(esdp->run_queue); else { - evacuate_run_queue(esdp->run_queue, &sbp); + if (no != 1) + evacuate_run_queue(esdp->run_queue, &sbp); erts_smp_runq_unlock(esdp->run_queue); @@ -7200,8 +7519,6 @@ suspend_scheduler(ErtsSchedulerData *esdp) if (erts_system_profile_flags.scheduler) profile_scheduler(make_small(esdp->no), am_inactive); - sched_wall_time_change(esdp, 0); - } erts_smp_mtx_lock(&schdlr_sspnd.mtx); @@ -7210,9 +7527,6 @@ suspend_scheduler(ErtsSchedulerData *esdp) schdlr_sspnd_dec_nscheds(&schdlr_sspnd.active, sched_type); - ASSERT(schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, - ERTS_SCHED_NORMAL) >= 1); - changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); while (1) { @@ -7234,9 +7548,13 @@ suspend_scheduler(ErtsSchedulerData *esdp) ERTS_SCHED_NORMAL) == 1) { clr_flg = ERTS_SCHDLR_SSPND_CHNG_NMSB; } - else if (schdlr_sspnd.active - == ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0)) { - clr_flg = ERTS_SCHDLR_SSPND_CHNG_MSB; + else if (schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, + ERTS_SCHED_NORMAL) == 1 + && schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, + ERTS_SCHED_DIRTY_CPU) == 0 + && schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, + ERTS_SCHED_DIRTY_IO) == 0) { + clr_flg = ERTS_SCHDLR_SSPND_CHNG_MSB; } if (clr_flg) { @@ -7308,10 +7626,7 @@ suspend_scheduler(ErtsSchedulerData *esdp) } } - if (curr_online - && (sched_type == ERTS_SCHED_NORMAL - ? !(schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing) - : !schdlr_sspnd.msb.ongoing)) { + if (curr_online) { flgs = erts_smp_atomic32_read_acqb(&ssi->flags); if (!(flgs & ERTS_SSI_FLG_SUSPENDED)) break; @@ -7327,13 +7642,11 @@ suspend_scheduler(ErtsSchedulerData *esdp) if (sched_type != ERTS_SCHED_NORMAL) aux_work = 0; else { - erts_aint32_t qmask; - qmask = (ERTS_RUNQ_FLGS_GET(esdp->run_queue) - & ERTS_RUNQ_FLGS_QMASK); + int evacuate = no == 1 ? 0 : !ERTS_EMPTY_RUNQ(esdp->run_queue); aux_work = erts_atomic32_read_acqb(&ssi->aux_work); - if (aux_work|qmask) { + if (aux_work|evacuate) { if (!thr_prgr_active) { erts_thr_progress_active(esdp, thr_prgr_active = 1); sched_wall_time_change(esdp, 1); @@ -7345,7 +7658,7 @@ suspend_scheduler(ErtsSchedulerData *esdp) if (aux_work && erts_thr_progress_update(esdp)) erts_thr_progress_leader_update(esdp); - if (qmask) { + if (evacuate) { erts_smp_runq_lock(esdp->run_queue); evacuate_run_queue(esdp->run_queue, &sbp); erts_smp_runq_unlock(esdp->run_queue); @@ -7460,7 +7773,8 @@ suspend_scheduler(ErtsSchedulerData *esdp) if (changing) { if ((changing & ERTS_SCHDLR_SSPND_CHNG_MSB) && !schdlr_sspnd.msb.ongoing - && schdlr_sspnd.online == schdlr_sspnd.active) { + && schdlr_sspnd_eq_nscheds(&schdlr_sspnd.online, + &schdlr_sspnd.active)) { erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, ~ERTS_SCHDLR_SSPND_CHNG_MSB); } @@ -7475,19 +7789,16 @@ suspend_scheduler(ErtsSchedulerData *esdp) } } ASSERT(no <= schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, sched_type)); - ASSERT((sched_type == ERTS_SCHED_NORMAL - ? !(schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing) - : !schdlr_sspnd.msb.ongoing)); } erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - ASSERT(!resume.msb.chngrs); schdlr_sspnd_resume_procs(sched_type, &resume); ASSERT(curr_online); - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + if (sched_type == ERTS_SCHED_NORMAL) { + (void) erts_get_monotonic_time(esdp); if (erts_system_profile_flags.scheduler) profile_scheduler(make_small(esdp->no), am_active); @@ -7497,8 +7808,6 @@ suspend_scheduler(ErtsSchedulerData *esdp) } } - if (sched_type == ERTS_SCHED_NORMAL) - (void) erts_get_monotonic_time(esdp); erts_smp_runq_lock(esdp->run_queue); non_empty_runq(esdp->run_queue); @@ -7768,10 +8077,8 @@ erts_set_schedulers_online(Process *p, erts_sched_poke(ssi); } } else { - for (ix = dirty_online; ix < dirty_no; ix++) { - ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); - scheduler_ssi_resume_wake(ssi); - } + for (ix = dirty_online; ix < dirty_no; ix++) + dcpu_sched_ix_resume_wake(ix); } } if (!dirty_only) @@ -7799,19 +8106,19 @@ erts_set_schedulers_online(Process *p, else /* if decrease */ { #ifdef ERTS_DIRTY_SCHEDULERS if (change_dirty) { - ErtsSchedulerSleepInfo* ssi; if (schdlr_sspnd.msb.ongoing) { - for (ix = dirty_no; ix < dirty_online; ix++) { - ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); - erts_sched_poke(ssi); - } - } else { - for (ix = dirty_no; ix < dirty_online; ix++) { - ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); - erts_smp_atomic32_read_bor_nob(&ssi->flags, - ERTS_SSI_FLG_SUSPENDED); - } - wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0); + for (ix = dirty_no; ix < dirty_online; ix++) + erts_sched_poke(ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix)); + } + else { + for (ix = dirty_no; ix < dirty_online; ix++) + dcpu_sched_ix_suspend_wake(ix); + /* + * Newly suspended scheduler may have just been + * about to handle a task. Make sure someone takes + * care of such a task... + */ + dcpu_sched_ix_wake(0); } } if (!dirty_only) @@ -7874,9 +8181,6 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal { int resume_proc, ix, res, have_unlocked_plocks = 0; ErtsProcList *plp; -#ifdef ERTS_DIRTY_SCHEDULERS - ErtsSchedulerSleepInfo* ssi; -#endif ErtsMultiSchedulingBlock *msbp; erts_aint32_t chng_flg; int have_blckd_flg; @@ -7923,8 +8227,9 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal erts_proclist_store_last(&msbp->blckrs, plp); p->flags |= have_blckd_flg; ASSERT(normal - ? 1 == schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, ERTS_SCHED_NORMAL) - : schdlr_sspnd.active == ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0)); + ? 1 == schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, + ERTS_SCHED_NORMAL) + : schdlr_sspnd_get_nscheds_tot(&schdlr_sspnd.active) == 1); ASSERT(erts_proc_sched_data(p)->no == 1); if (schdlr_sspnd.msb.ongoing) res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; @@ -7942,59 +8247,41 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal } ASSERT(!msbp->ongoing); msbp->ongoing = 1; - if (schdlr_sspnd.active == ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0) - || (normal && schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, - ERTS_SCHED_NORMAL) == 1)) { - ASSERT(erts_proc_sched_data(p)->no == 1); - plp = proclist_create(p); - erts_proclist_store_last(&msbp->blckrs, plp); - if (schdlr_sspnd.msb.ongoing) - res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; - else - res = ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED; - } - else { - erts_smp_atomic32_read_bor_nob(&schdlr_sspnd.changing, - chng_flg); - change_no_used_runqs(1); - for (ix = 1; ix < erts_no_run_queues; ix++) - suspend_run_queue(ERTS_RUNQ_IX(ix)); - for (ix = 1; ix < online; ix++) { - ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); - wake_scheduler(rq); - } + erts_smp_atomic32_read_bor_nob(&schdlr_sspnd.changing, + chng_flg); + change_no_used_runqs(1); + for (ix = 1; ix < erts_no_run_queues; ix++) + suspend_run_queue(ERTS_RUNQ_IX(ix)); -#ifdef ERTS_DIRTY_SCHEDULERS - if (!normal) { - for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++) { - ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); - erts_smp_atomic32_read_bor_nob(&ssi->flags, - ERTS_SSI_FLG_SUSPENDED); - } - wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0); + for (ix = 1; ix < online; ix++) { + ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); + wake_scheduler(rq); + } - for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) { - ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix); - erts_smp_atomic32_read_bor_nob(&ssi->flags, - ERTS_SSI_FLG_SUSPENDED); - } - wake_dirty_schedulers(ERTS_DIRTY_IO_RUNQ, 0); - } +#ifdef ERTS_DIRTY_SCHEDULERS + if (!normal) { + ERTS_RUNQ_FLGS_SET_NOB(ERTS_RUNQ_IX(0), ERTS_RUNQ_FLG_MSB_EXEC); + erts_smp_atomic32_read_bor_nob(&ERTS_RUNQ_IX(0)->scheduler->ssi->flags, + ERTS_SSI_FLG_MSB_EXEC); + for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++) + dcpu_sched_ix_suspend_wake(ix); + for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) + dio_sched_ix_suspend_wake(ix); + } #endif - wait_until_msb: + wait_until_msb: - ASSERT(chng_flg & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)); + ASSERT(chng_flg & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)); - plp = proclist_create(p); - erts_proclist_store_last(&msbp->chngq, plp); - resume_proc = 0; - if (schdlr_sspnd.msb.ongoing) - res = ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED; - else - res = ERTS_SCHDLR_SSPND_YIELD_DONE_NMSCHED_BLOCKED; - } + plp = proclist_create(p); + erts_proclist_store_last(&msbp->chngq, plp); + resume_proc = 0; + if (schdlr_sspnd.msb.ongoing) + res = ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED; + else + res = ERTS_SCHDLR_SSPND_YIELD_DONE_NMSCHED_BLOCKED; ASSERT(erts_proc_sched_data(p)); } } @@ -8026,27 +8313,19 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal } } if (!msbp->blckrs && !msbp->chngq) { - int online = (int) schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, - ERTS_SCHED_NORMAL); + int online; erts_smp_atomic32_read_bor_nob(&schdlr_sspnd.changing, chng_flg); p->flags &= ~have_blckd_flg; msbp->ongoing = 0; - if (online == 1) { - /* No normal schedulers to resume */ - ASSERT(schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, - ERTS_SCHED_NORMAL) == 1); -#ifndef ERTS_DIRTY_SCHEDULERS - erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, - ~chng_flg); -#endif - } - else if (!(schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing)) { - if (plocks) { + if (!(schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing)) { + if (plocks) { have_unlocked_plocks = 1; erts_smp_proc_unlock(p, plocks); } + online = (int) schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, + ERTS_SCHED_NORMAL); change_no_used_runqs(online); /* Resume all online run queues */ @@ -8057,19 +8336,15 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal suspend_run_queue(ERTS_RUNQ_IX(ix)); } #ifdef ERTS_DIRTY_SCHEDULERS - if (!normal) { - ASSERT(!schdlr_sspnd.msb.ongoing); + if (!schdlr_sspnd.msb.ongoing) { + /* Get rid of msb-exec flag in run-queue of scheduler 1 */ + resume_run_queue(ERTS_RUNQ_IX(0)); online = (int) schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, ERTS_SCHED_DIRTY_CPU); - for (ix = 0; ix < online; ix++) { - ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); - scheduler_ssi_resume_wake(ssi); - } - - for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) { - ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix); - scheduler_ssi_resume_wake(ssi); - } + for (ix = 0; ix < online; ix++) + dcpu_sched_ix_resume_wake(ix); + for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) + dio_sched_ix_resume_wake(ix); } #endif } @@ -9734,7 +10009,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) if (!is_normal_sched) { if (erts_smp_atomic32_read_acqb(&esdp->ssi->flags) - & ERTS_SSI_FLG_SUSPENDED) { + & (ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC)) { suspend_scheduler(esdp); } } @@ -9744,8 +10019,10 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) ASSERT(is_normal_sched); - if (flags & (ERTS_RUNQ_FLG_CHK_CPU_BIND|ERTS_RUNQ_FLG_SUSPENDED)) { - if (flags & ERTS_RUNQ_FLG_SUSPENDED) { + if (flags & (ERTS_RUNQ_FLG_CHK_CPU_BIND + | ERTS_RUNQ_FLG_SUSPENDED + | ERTS_RUNQ_FLG_MSB_EXEC)) { + if (flags & (ERTS_RUNQ_FLG_SUSPENDED|ERTS_RUNQ_FLG_MSB_EXEC)) { (void) ERTS_RUNQ_FLGS_UNSET_NOB(rq, ERTS_RUNQ_FLG_EXEC); suspend_scheduler(esdp); flags = ERTS_RUNQ_FLGS_SET_NOB(rq, ERTS_RUNQ_FLG_EXEC); @@ -9784,13 +10061,12 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) flags = ERTS_RUNQ_FLGS_GET_NOB(rq); - if (!is_normal_sched && rq->halt_in_progress) { + if (!is_normal_sched & !!(flags & ERTS_RUNQ_FLG_HALTING)) { /* Wait for emulator to terminate... */ while (1) erts_milli_sleep(1000*1000); } - else if ((!(flags & ERTS_RUNQ_FLGS_QMASK) && !rq->misc.start) - || (rq->halt_in_progress && ERTS_EMPTY_RUNQ_PORTS(rq))) { + else if (!runq_got_work_to_execute_flags(flags)) { /* Prepare for scheduler wait */ #ifdef ERTS_SMP ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); @@ -9804,21 +10080,44 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) if (flags & ERTS_RUNQ_FLG_INACTIVE) empty_runq(rq); else { - if (is_normal_sched && try_steal_task(rq)) - goto continue_check_activities_to_run; - - empty_runq(rq); - - /* - * Check for ERTS_RUNQ_FLG_SUSPENDED has to be done - * after trying to steal a task. - */ - flags = ERTS_RUNQ_FLGS_GET_NOB(rq); - if (flags & ERTS_RUNQ_FLG_SUSPENDED) { - non_empty_runq(rq); - flags |= ERTS_RUNQ_FLG_NONEMPTY; - goto continue_check_activities_to_run_known_flags; + ASSERT(!runq_got_work_to_execute(rq)); + if (!is_normal_sched) { + /* Dirty scheduler */ + if (erts_smp_atomic32_read_acqb(&esdp->ssi->flags) + & (ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC)) { + /* Go suspend... */ + goto continue_check_activities_to_run_known_flags; + } + } + else { + /* Normal scheduler */ + if (try_steal_task(rq)) + goto continue_check_activities_to_run; + /* + * Check for suspend has to be done after trying + * to steal a task... + */ + flags = ERTS_RUNQ_FLGS_GET_NOB(rq); + if ((flags & ERTS_RUNQ_FLG_SUSPENDED) +#ifdef ERTS_DIRTY_SCHEDULERS + /* If multi scheduling block and we have + * dirty work, suspend and let dirty + * scheduler handle work... */ + || ((((flags & (ERTS_RUNQ_FLG_HALTING + | ERTS_RUNQ_FLG_MSB_EXEC)) + == ERTS_RUNQ_FLG_MSB_EXEC)) + && have_dirty_work()) +#endif + ) { + non_empty_runq(rq); + flags |= ERTS_RUNQ_FLG_NONEMPTY; + /* + * Go suspend... + */ + goto continue_check_activities_to_run_known_flags; + } } + empty_runq(rq); } #endif @@ -9870,7 +10169,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) #endif } - if (rq->misc.start) + if (flags & ERTS_RUNQ_FLG_MISC_OP) exec_misc_ops(rq); #ifdef ERTS_SMP @@ -9881,13 +10180,15 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) * Find a new port to run. */ - if (RUNQ_READ_LEN(&rq->ports.info.len)) { + flags = ERTS_RUNQ_FLGS_GET_NOB(rq); + + if (flags & PORT_BIT) { int have_outstanding_io; have_outstanding_io = erts_port_task_execute(rq, &esdp->current_port); if ((!erts_eager_check_io && have_outstanding_io && fcalls > 2*input_reductions) - || rq->halt_in_progress) { + || (flags & ERTS_RUNQ_FLG_HALTING)) { /* * If we have performed more than 2*INPUT_REDUCTIONS since * last call to erl_sys_schedule() and we still haven't @@ -11419,6 +11720,8 @@ erts_schedule_misc_op(void (*func)(void *), void *arg) non_empty_runq(rq); #endif + ERTS_RUNQ_FLGS_SET_NOB(rq, ERTS_RUNQ_FLG_MISC_OP); + erts_smp_runq_unlock(rq); smp_notify_inc_runq(rq); @@ -11449,6 +11752,9 @@ exec_misc_ops(ErtsRunQueue *rq) rq->misc.end = NULL; } + if (!rq->misc.start) + ERTS_RUNQ_FLGS_UNSET_NOB(rq, ERTS_RUNQ_FLG_MISC_OP); + erts_smp_runq_unlock(rq); while (molp) { @@ -13576,6 +13882,8 @@ erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) { erts_print(to, to_arg, "WAITING"); break; case ERTS_SSI_FLG_SUSPENDED: erts_print(to, to_arg, "SUSPENDED"); break; + case ERTS_SSI_FLG_MSB_EXEC: + erts_print(to, to_arg, "MSB_EXEC"); break; default: erts_print(to, to_arg, "UNKNOWN(%d)", flg); break; } @@ -13685,6 +13993,12 @@ erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) { erts_print(to, to_arg, "NONEMPTY"); break; case ERTS_RUNQ_FLG_PROTECTED: erts_print(to, to_arg, "PROTECTED"); break; + case ERTS_RUNQ_FLG_EXEC: + erts_print(to, to_arg, "EXEC"); break; + case ERTS_RUNQ_FLG_MSB_EXEC: + erts_print(to, to_arg, "MSB_EXEC"); break; + case ERTS_RUNQ_FLG_MISC_OP: + erts_print(to, to_arg, "MISC_OP"); break; default: erts_print(to, to_arg, "UNKNOWN(%d)", flg); break; } @@ -13734,11 +14048,11 @@ erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) { * A nice system halt closing all open port goes as follows: * 1) This function schedules the aux work ERTS_SSI_AUX_WORK_REAP_PORTS * on all schedulers, then schedules itself out. - * 2) All shedulers detect this and set the flag halt_in_progress + * 2) All shedulers detect this and set the flag ERTS_RUNQ_FLG_HALTING * on their run queue. The last scheduler sets all non-closed ports * ERTS_PORT_SFLG_HALT. Global atomic erts_halt_progress is used * as refcount to determine which is last. - * 3) While the run ques has flag halt_in_progress no processes + * 3) While the run queues has flag ERTS_RUNQ_FLG_HALTING no processes * will be scheduled, only ports. * 4) When the last port closes that scheduler calls erlang:halt/1. * The same global atomic is used as refcount. @@ -13753,8 +14067,8 @@ void erts_halt(int code) erts_no_schedulers, -1)) { #ifdef ERTS_DIRTY_SCHEDULERS - ERTS_DIRTY_CPU_RUNQ->halt_in_progress = 1; - ERTS_DIRTY_IO_RUNQ->halt_in_progress = 1; + ERTS_RUNQ_FLGS_SET(ERTS_DIRTY_CPU_RUNQ, ERTS_RUNQ_FLG_HALTING); + ERTS_RUNQ_FLGS_SET(ERTS_DIRTY_IO_RUNQ, ERTS_RUNQ_FLG_HALTING); #endif erts_halt_code = code; notify_reap_ports_relb(); |