diff options
Diffstat (limited to 'erts/emulator/beam/erl_process.c')
| -rw-r--r-- | erts/emulator/beam/erl_process.c | 117 |
1 files changed, 64 insertions, 53 deletions
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 4d6e982325..2efff4bf10 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -622,14 +622,10 @@ init_misc_aux_work(void) init_misc_aux_work_alloc(); - misc_aux_work_queues = erts_alloc(ERTS_ALC_T_MISC_AUX_WORK_Q, - (sizeof(erts_algnd_misc_aux_work_q_t) - *(erts_no_schedulers+1))); - if ((((UWord) misc_aux_work_queues) & ERTS_CACHE_LINE_MASK) != 0) - misc_aux_work_queues = ((erts_algnd_misc_aux_work_q_t *) - ((((UWord) misc_aux_work_queues) - & ~ERTS_CACHE_LINE_MASK) - + ERTS_CACHE_LINE_SIZE)); + misc_aux_work_queues = + erts_alloc_permanent_cache_aligned(ERTS_ALC_T_MISC_AUX_WORK_Q, + erts_no_schedulers * + sizeof(erts_algnd_misc_aux_work_q_t)); for (ix = 0; ix < erts_no_schedulers; ix++) { erts_smp_mtx_init_x(&misc_aux_work_queues[ix].data.mtx, @@ -815,15 +811,31 @@ erts_active_schedulers(void) return as; } +#ifdef ERTS_SMP + +static ERTS_INLINE void +clear_sys_scheduling(void) +{ + erts_smp_atomic32_set_relb(&doing_sys_schedule, 0); +} + +static ERTS_INLINE int +try_set_sys_scheduling(void) +{ + return 0 == erts_smp_atomic32_cmpxchg_acqb(&doing_sys_schedule, 1, 0); +} + +#endif + static ERTS_INLINE int prepare_for_sys_schedule(void) { #ifdef ERTS_SMP while (!erts_port_task_have_outstanding_io_tasks() - && !erts_smp_atomic32_xchg(&doing_sys_schedule, 1)) { + && try_set_sys_scheduling()) { if (!erts_port_task_have_outstanding_io_tasks()) return 1; - erts_smp_atomic32_set(&doing_sys_schedule, 0); + clear_sys_scheduling(); } return 0; #else @@ -1153,7 +1165,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) * call erl_sys_schedule() until it is handled. */ if (erts_port_task_have_outstanding_io_tasks()) { - erts_smp_atomic32_set(&doing_sys_schedule, 0); + clear_sys_scheduling(); /* * Got to check that we still got I/O tasks; otherwise * we have to continue checking for I/O... @@ -1172,7 +1184,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) * sleep in erl_sys_schedule(). */ if (erts_port_task_have_outstanding_io_tasks()) { - erts_smp_atomic32_set(&doing_sys_schedule, 0); + clear_sys_scheduling(); /* * Got to check that we still got I/O tasks; otherwise @@ -1226,7 +1238,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) sys_woken: erts_smp_runq_lock(rq); sys_locked_woken: - erts_smp_atomic32_set(&doing_sys_schedule, 0); + clear_sys_scheduling(); if (flgs & ~ERTS_SSI_FLG_SUSPENDED) erts_smp_atomic32_band(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); sched_active_sys(esdp->no, rq); @@ -1255,7 +1267,6 @@ ssi_flags_set_wake(ErtsSchedulerSleepInfo *ssi) static void wake_scheduler(ErtsRunQueue *rq, int incq, int one) { - int res; ErtsSchedulerSleepInfo *ssi; ErtsSchedulerSleepList *sl; @@ -1286,9 +1297,9 @@ wake_scheduler(ErtsRunQueue *rq, int incq, int one) if (ssi->next) ssi->next->prev = ssi->prev; - res = sl->list != NULL; erts_smp_spin_unlock(&sl->lock); + ERTS_THR_MEMORY_BARRIER; flgs = ssi_flags_set_wake(ssi); erts_sched_finish_poke(ssi, flgs); @@ -1298,6 +1309,8 @@ wake_scheduler(ErtsRunQueue *rq, int incq, int one) else { sl->list = NULL; erts_smp_spin_unlock(&sl->lock); + + ERTS_THR_MEMORY_BARRIER; do { ErtsSchedulerSleepInfo *wake_ssi = ssi; ssi = ssi->next; @@ -2515,16 +2528,9 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) n = (int) (mrq ? no_schedulers : 1); - erts_aligned_run_queues = erts_alloc(ERTS_ALC_T_RUNQS, - (sizeof(ErtsAlignedRunQueue)*(n+1))); - if ((((UWord) erts_aligned_run_queues) & ERTS_CACHE_LINE_MASK) != 0) - erts_aligned_run_queues = ((ErtsAlignedRunQueue *) - ((((UWord) erts_aligned_run_queues) - & ~ERTS_CACHE_LINE_MASK) - + ERTS_CACHE_LINE_SIZE)); - - ASSERT((((UWord) erts_aligned_run_queues) & ERTS_CACHE_LINE_MASK) == 0); - + erts_aligned_run_queues = + erts_alloc_permanent_cache_aligned(ERTS_ALC_T_RUNQS, + sizeof(ErtsAlignedRunQueue) * n); #ifdef ERTS_SMP erts_smp_atomic32_init(&no_empty_run_queues, 0); #endif @@ -2619,14 +2625,10 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) #ifdef ERTS_SMP /* Create and initialize scheduler sleep info */ - aligned_sched_sleep_info = erts_alloc(ERTS_ALC_T_SCHDLR_SLP_INFO, - (sizeof(ErtsAlignedSchedulerSleepInfo) - *(n+1))); - if ((((UWord) aligned_sched_sleep_info) & ERTS_CACHE_LINE_MASK) == 0) - aligned_sched_sleep_info = ((ErtsAlignedSchedulerSleepInfo *) - ((((UWord) aligned_sched_sleep_info) - & ~ERTS_CACHE_LINE_MASK) - + ERTS_CACHE_LINE_SIZE)); + aligned_sched_sleep_info = + erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_SLP_INFO, + n * sizeof(ErtsAlignedSchedulerSleepInfo)); + for (ix = 0; ix < n; ix++) { ErtsSchedulerSleepInfo *ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); #if 0 /* no need to initialize these... */ @@ -2641,16 +2643,9 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) /* Create and initialize scheduler specific data */ - erts_aligned_scheduler_data = erts_alloc(ERTS_ALC_T_SCHDLR_DATA, - (sizeof(ErtsAlignedSchedulerData) - *(n+1))); - if ((((UWord) erts_aligned_scheduler_data) & ERTS_CACHE_LINE_MASK) != 0) - erts_aligned_scheduler_data = ((ErtsAlignedSchedulerData *) - ((((UWord) erts_aligned_scheduler_data) - & ~ERTS_CACHE_LINE_MASK) - + ERTS_CACHE_LINE_SIZE)); - - ASSERT((((UWord) erts_aligned_scheduler_data) & ERTS_CACHE_LINE_MASK) == 0); + erts_aligned_scheduler_data = + erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA, + n*sizeof(ErtsAlignedSchedulerData)); for (ix = 0; ix < n; ix++) { ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(ix); @@ -2860,7 +2855,6 @@ resume_process(Process *p) return; switch(p->rstatus) { case P_RUNABLE: - *statusp = P_WAITING; /* make erts_add_to_runq work */ erts_add_to_runq(p); break; case P_WAITING: @@ -4656,7 +4650,7 @@ internal_add_to_runq(ErtsRunQueue *runq, Process *p) if (p->status_flags & ERTS_PROC_SFLG_INRUNQ) return NULL; else if (p->runq_flags & ERTS_PROC_RUNQ_FLG_RUNNING) { - ASSERT(p->status != P_SUSPENDED); + ASSERT(p->rcount == 0); ERTS_DBG_CHK_PROCS_RUNQ_NOPROC(runq, p); p->status_flags |= ERTS_PROC_SFLG_PENDADD2SCHEDQ; return NULL; @@ -4667,9 +4661,8 @@ internal_add_to_runq(ErtsRunQueue *runq, Process *p) ERTS_DBG_CHK_PROCS_RUNQ_NOPROC(runq, p); #ifndef ERTS_SMP /* Never schedule a suspended process (ok in smp case) */ - ASSERT(p->status != P_SUSPENDED); + ASSERT(p->rcount == 0); add_runq = runq; - #else ASSERT(!p->bound_runq || p->bound_runq == p->run_queue); if (p->bound_runq) { @@ -5169,7 +5162,7 @@ Process *schedule(Process *p, int calls) handle_pending_suspend(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); ASSERT(!(p->status_flags & ERTS_PROC_SFLG_PENDADD2SCHEDQ) - || p->status != P_SUSPENDED); + || p->rcount == 0); } #endif erts_smp_runq_lock(rq); @@ -5371,7 +5364,7 @@ Process *schedule(Process *p, int calls) if (dt) erts_bump_timer(dt); #ifdef ERTS_SMP erts_smp_runq_lock(rq); - erts_smp_atomic32_set(&doing_sys_schedule, 0); + clear_sys_scheduling(); goto continue_check_activities_to_run; #else if (!runnable) @@ -7614,10 +7607,28 @@ timeout_proc(Process* p) p->flags |= F_TIMO; p->flags &= ~F_INSLPQUEUE; - if (p->status == P_WAITING) - erts_add_to_runq(p); - if (p->status == P_SUSPENDED) + switch (p->status) { + case P_GARBING: + switch (p->gcstatus) { + case P_SUSPENDED: + goto suspended; + case P_WAITING: + goto waiting; + default: + break; + } + break; + case P_WAITING: + waiting: + erts_add_to_runq(p); + break; + case P_SUSPENDED: + suspended: p->rstatus = P_RUNABLE; /* MUST set resume status to runnable */ + break; + default: + break; + } } @@ -7703,7 +7714,7 @@ erts_program_counter_info(int to, void *to_arg, Process *p) * only cause problems. */ for (i = 0; i < p->arity; i++) - erts_print(to, to_arg, " %T\n", p->arg_reg[i]); + erts_print(to, to_arg, " %.*T\n", INT_MAX, p->arg_reg[i]); } } } |