diff options
Diffstat (limited to 'erts/emulator')
| -rw-r--r-- | erts/emulator/beam/erl_port_task.h | 1 | ||||
| -rw-r--r-- | erts/emulator/beam/erl_process.c | 31 | ||||
| -rw-r--r-- | erts/emulator/beam/erl_process.h | 4 | ||||
| -rw-r--r-- | erts/emulator/sys/common/erl_poll.c | 1 | ||||
| -rw-r--r-- | erts/emulator/sys/win32/erl_poll.c | 8 |
5 files changed, 36 insertions, 9 deletions
diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h index 714b4ea7dd..49a0b4c63a 100644 --- a/erts/emulator/beam/erl_port_task.h +++ b/erts/emulator/beam/erl_port_task.h @@ -102,6 +102,7 @@ erts_port_task_init_sched(ErtsPortTaskSched *ptsp) ERTS_GLB_INLINE int erts_port_task_have_outstanding_io_tasks(void) { + ERTS_THR_MEMORY_BARRIER; return erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) != 0; } diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 4d6e982325..fd2bd4469c 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -815,15 +815,31 @@ erts_active_schedulers(void) return as; } +#ifdef ERTS_SMP + +static ERTS_INLINE void +clear_sys_scheduling(void) +{ + erts_smp_atomic32_set_relb(&doing_sys_schedule, 0); +} + +static ERTS_INLINE int +try_set_sys_scheduling(void) +{ + return 0 == erts_smp_atomic32_cmpxchg_acqb(&doing_sys_schedule, 1, 0); +} + +#endif + static ERTS_INLINE int prepare_for_sys_schedule(void) { #ifdef ERTS_SMP while (!erts_port_task_have_outstanding_io_tasks() - && !erts_smp_atomic32_xchg(&doing_sys_schedule, 1)) { + && try_set_sys_scheduling()) { if (!erts_port_task_have_outstanding_io_tasks()) return 1; - erts_smp_atomic32_set(&doing_sys_schedule, 0); + clear_sys_scheduling(); } return 0; #else @@ -1153,7 +1169,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) * call erl_sys_schedule() until it is handled. */ if (erts_port_task_have_outstanding_io_tasks()) { - erts_smp_atomic32_set(&doing_sys_schedule, 0); + clear_sys_scheduling(); /* * Got to check that we still got I/O tasks; otherwise * we have to continue checking for I/O... @@ -1172,7 +1188,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) * sleep in erl_sys_schedule(). */ if (erts_port_task_have_outstanding_io_tasks()) { - erts_smp_atomic32_set(&doing_sys_schedule, 0); + clear_sys_scheduling(); /* * Got to check that we still got I/O tasks; otherwise @@ -1226,7 +1242,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) sys_woken: erts_smp_runq_lock(rq); sys_locked_woken: - erts_smp_atomic32_set(&doing_sys_schedule, 0); + clear_sys_scheduling(); if (flgs & ~ERTS_SSI_FLG_SUSPENDED) erts_smp_atomic32_band(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); sched_active_sys(esdp->no, rq); @@ -1289,6 +1305,7 @@ wake_scheduler(ErtsRunQueue *rq, int incq, int one) res = sl->list != NULL; erts_smp_spin_unlock(&sl->lock); + ERTS_THR_MEMORY_BARRIER; flgs = ssi_flags_set_wake(ssi); erts_sched_finish_poke(ssi, flgs); @@ -1298,6 +1315,8 @@ wake_scheduler(ErtsRunQueue *rq, int incq, int one) else { sl->list = NULL; erts_smp_spin_unlock(&sl->lock); + + ERTS_THR_MEMORY_BARRIER; do { ErtsSchedulerSleepInfo *wake_ssi = ssi; ssi = ssi->next; @@ -5371,7 +5390,7 @@ Process *schedule(Process *p, int calls) if (dt) erts_bump_timer(dt); #ifdef ERTS_SMP erts_smp_runq_lock(rq); - erts_smp_atomic32_set(&doing_sys_schedule, 0); + clear_sys_scheduling(); goto continue_check_activities_to_run; #else if (!runnable) diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index e871a9834a..8f78a7d76e 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -1592,7 +1592,9 @@ ERTS_GLB_INLINE void erts_sched_poke(ErtsSchedulerSleepInfo *ssi); ERTS_GLB_INLINE void erts_sched_poke(ErtsSchedulerSleepInfo *ssi) { - erts_aint32_t flags = erts_smp_atomic32_read(&ssi->flags); + erts_aint32_t flags; + ERTS_THR_MEMORY_BARRIER; + flags = erts_smp_atomic32_read(&ssi->flags); ASSERT(!(flags & ERTS_SSI_FLG_SLEEPING) || (flags & ERTS_SSI_FLG_WAITING)); if (flags & ERTS_SSI_FLG_SLEEPING) { diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c index 77ac2de5f6..3ae5b8d747 100644 --- a/erts/emulator/sys/common/erl_poll.c +++ b/erts/emulator/sys/common/erl_poll.c @@ -347,6 +347,7 @@ reset_wakeup_state(ErtsPollSet ps) { #ifdef ERTS_SMP erts_atomic32_set(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN); + ERTS_THR_MEMORY_BARRIER; #elif ERTS_POLL_ASYNC_INTERRUPT_SUPPORT ps->wakeup_state = 0; #endif diff --git a/erts/emulator/sys/win32/erl_poll.c b/erts/emulator/sys/win32/erl_poll.c index 1f2877b682..7662f190ef 100644 --- a/erts/emulator/sys/win32/erl_poll.c +++ b/erts/emulator/sys/win32/erl_poll.c @@ -452,12 +452,15 @@ poll_wait_timeout(ErtsPollSet ps, SysTimeval *tvp) static ERTS_INLINE void wake_poller(ErtsPollSet ps, int io_ready) { - erts_aint32_t wakeup_state = erts_atomic32_read(&ps->wakeup_state); + erts_aint32_t wakeup_state; if (io_ready) { /* We may set the event multiple times. This is, however, harmless. */ - erts_atomic32_set(&ps->wakeup_state, ERTS_POLL_WOKEN_IO_READY); + wakeup_state = erts_atomic32_read(&ps->wakeup_state); + erts_atomic32_set_relb(&ps->wakeup_state, ERTS_POLL_WOKEN_IO_READY); } else { + ERTS_THR_MEMORY_BARRIER; + wakeup_state = erts_atomic32_read(&ps->wakeup_state); while (wakeup_state != ERTS_POLL_WOKEN_IO_READY && wakeup_state != ERTS_POLL_WOKEN_INTR) { erts_aint32_t act = erts_atomic32_cmpxchg(&ps->wakeup_state, @@ -518,6 +521,7 @@ reset_interrupt(ErtsPollSet ps) break; wakeup_state = act; } + ERTS_THR_MEMORY_BARRIER; } static ERTS_INLINE void |