diff options
Diffstat (limited to 'erts/emulator/sys')
20 files changed, 382 insertions, 848 deletions
diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c index ad580e7d52..834b77eb58 100644 --- a/erts/emulator/sys/common/erl_check_io.c +++ b/erts/emulator/sys/common/erl_check_io.c @@ -84,9 +84,6 @@ typedef char EventStateFlags; #define ERTS_CIO_POLL_CTL ERTS_POLL_EXPORT(erts_poll_control) #define ERTS_CIO_POLL_CTLV ERTS_POLL_EXPORT(erts_poll_controlv) #define ERTS_CIO_POLL_WAIT ERTS_POLL_EXPORT(erts_poll_wait) -#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT -#define ERTS_CIO_POLL_AS_INTR ERTS_POLL_EXPORT(erts_poll_async_sig_interrupt) -#endif #define ERTS_CIO_POLL_INTR ERTS_POLL_EXPORT(erts_poll_interrupt) #define ERTS_CIO_POLL_INTR_TMD ERTS_POLL_EXPORT(erts_poll_interrupt_timed) #define ERTS_CIO_NEW_POLLSET ERTS_POLL_EXPORT(erts_poll_create_pollset) @@ -100,18 +97,16 @@ typedef char EventStateFlags; static struct pollset_info { ErtsPollSet ps; - erts_smp_atomic_t in_poll_wait; /* set while doing poll */ + erts_atomic_t in_poll_wait; /* set while doing poll */ struct { int six; /* start index */ int eix; /* end index */ - erts_smp_atomic32_t no; + erts_atomic32_t no; int size; ErtsSysFdType *array; } active_fd; -#ifdef ERTS_SMP struct removed_fd* removed_list; /* list of deselected fd's*/ - erts_smp_spinlock_t removed_list_lock; -#endif + erts_spinlock_t removed_list_lock; }pollset; #define NUM_OF_POLLSETS 1 @@ -137,7 +132,6 @@ typedef struct { EventStateFlags flags; } ErtsDrvEventState; -#ifdef ERTS_SMP struct removed_fd { struct removed_fd *next; #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS @@ -150,19 +144,17 @@ struct removed_fd { #endif }; -#endif #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS static int max_fds = -1; #endif #define DRV_EV_STATE_LOCK_CNT 16 static union { - erts_smp_mtx_t lck; + erts_mtx_t lck; byte _cache_line_alignment[64]; }drv_ev_state_locks[DRV_EV_STATE_LOCK_CNT]; -#ifdef ERTS_SMP -static ERTS_INLINE erts_smp_mtx_t* fd_mtx(ErtsSysFdType fd) +static ERTS_INLINE erts_mtx_t* fd_mtx(ErtsSysFdType fd) { int hash = (int)fd; # ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS @@ -170,21 +162,18 @@ static ERTS_INLINE erts_smp_mtx_t* fd_mtx(ErtsSysFdType fd) # endif return &drv_ev_state_locks[hash % DRV_EV_STATE_LOCK_CNT].lck; } -#else -# define fd_mtx(fd) NULL -#endif #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS -static erts_smp_atomic_t drv_ev_state_len; +static erts_atomic_t drv_ev_state_len; static ErtsDrvEventState *drv_ev_state; -static erts_smp_mtx_t drv_ev_state_grow_lock; /* prevent lock-hogging of racing growers */ +static erts_mtx_t drv_ev_state_grow_lock; /* prevent lock-hogging of racing growers */ #else static SafeHash drv_ev_state_tab; static int num_state_prealloc; static ErtsDrvEventState *state_prealloc_first; -erts_smp_spinlock_t state_prealloc_lock; +erts_spinlock_t state_prealloc_lock; static ERTS_INLINE ErtsDrvEventState *hash_get_drv_ev_state(ErtsSysFdType fd) { @@ -249,15 +238,13 @@ static void steal_pending_stop_nif(erts_dsprintf_buf_t *dsbufp, ErtsResource*, ErtsDrvEventState *state, int mode, int on); -#ifdef ERTS_SMP ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(removed_fd, struct removed_fd, 64, ERTS_ALC_T_FD_LIST) -#endif static ERTS_INLINE void init_iotask(ErtsIoTask *io_task) { erts_port_task_handle_init(&io_task->task); - erts_smp_atomic_init_nob(&io_task->executed_time, ~((erts_aint_t) 0)); + erts_atomic_init_nob(&io_task->executed_time, ~((erts_aint_t) 0)); } static ERTS_INLINE int @@ -265,7 +252,7 @@ is_iotask_active(ErtsIoTask *io_task, erts_aint_t current_cio_time) { if (erts_port_task_is_scheduled(&io_task->task)) return 1; - if (erts_smp_atomic_read_nob(&io_task->executed_time) == current_cio_time) + if (erts_atomic_read_nob(&io_task->executed_time) == current_cio_time) return 1; return 0; } @@ -337,10 +324,9 @@ free_drv_event_data(ErtsDrvEventDataState *dep) static ERTS_INLINE void remember_removed(ErtsDrvEventState *state, struct pollset_info* psi) { -#ifdef ERTS_SMP struct removed_fd *fdlp; - ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(fd_mtx(state->fd))); - if (erts_smp_atomic_read_nob(&psi->in_poll_wait)) { + ERTS_LC_ASSERT(erts_lc_mtx_is_locked(fd_mtx(state->fd))); + if (erts_atomic_read_nob(&psi->in_poll_wait)) { state->remove_cnt++; ASSERT(state->remove_cnt > 0); fdlp = removed_fd_alloc(); @@ -350,62 +336,56 @@ remember_removed(ErtsDrvEventState *state, struct pollset_info* psi) #ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS fdlp->state = state; #endif - erts_smp_spin_lock(&psi->removed_list_lock); + erts_spin_lock(&psi->removed_list_lock); fdlp->next = psi->removed_list; psi->removed_list = fdlp; - erts_smp_spin_unlock(&psi->removed_list_lock); + erts_spin_unlock(&psi->removed_list_lock); } -#endif } static ERTS_INLINE int is_removed(ErtsDrvEventState *state) { -#ifdef ERTS_SMP /* Note that there is a possible race here, where an fd is removed (increasing remove_cnt) and then added again just before erts_poll_wait is called by erts_check_io. Any polled event on the re-added fd will then be falsely ignored. But that does not matter, as the event will trigger again next time erl_check_io is called. */ return state->remove_cnt > 0; -#else - return 0; -#endif } static void forget_removed(struct pollset_info* psi) { -#ifdef ERTS_SMP struct removed_fd* fdlp; struct removed_fd* tofree; /* Fast track: if (atomic_ptr(removed_list)==NULL) return; */ - erts_smp_spin_lock(&psi->removed_list_lock); + erts_spin_lock(&psi->removed_list_lock); fdlp = psi->removed_list; psi->removed_list = NULL; - erts_smp_spin_unlock(&psi->removed_list_lock); + erts_spin_unlock(&psi->removed_list_lock); while (fdlp) { ErtsResource* resource = NULL; erts_driver_t* drv_ptr = NULL; - erts_smp_mtx_t* mtx; + erts_mtx_t* mtx; ErtsSysFdType fd; ErtsDrvEventState *state; #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS fd = fdlp->fd; mtx = fd_mtx(fd); - erts_smp_mtx_lock(mtx); + erts_mtx_lock(mtx); state = &drv_ev_state[(int) fd]; #else state = fdlp->state; fd = state->fd; ASSERT(fd == fdlp->fd); mtx = fd_mtx(fd); - erts_smp_mtx_lock(mtx); + erts_mtx_lock(mtx); #endif ASSERT(state->remove_cnt > 0); if (--state->remove_cnt == 0) { @@ -440,7 +420,7 @@ forget_removed(struct pollset_info* psi) ASSERT(0); } } - erts_smp_mtx_unlock(mtx); + erts_mtx_unlock(mtx); if (drv_ptr) { int was_unmasked = erts_block_fpe(); DTRACE1(driver_stop_select, drv_ptr->name); @@ -460,7 +440,6 @@ forget_removed(struct pollset_info* psi) fdlp = fdlp->next; removed_fd_free(tofree); } -#endif /* ERTS_SMP */ } #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS @@ -471,15 +450,15 @@ grow_drv_ev_state(int min_ix) int old_len; int new_len; - erts_smp_mtx_lock(&drv_ev_state_grow_lock); - old_len = erts_smp_atomic_read_nob(&drv_ev_state_len); + erts_mtx_lock(&drv_ev_state_grow_lock); + old_len = erts_atomic_read_nob(&drv_ev_state_len); if (min_ix >= old_len) { new_len = erts_poll_new_table_len(old_len, min_ix + 1); if (new_len > max_fds) new_len = max_fds; for (i=0; i<DRV_EV_STATE_LOCK_CNT; i++) { /* lock all fd's */ - erts_smp_mtx_lock(&drv_ev_state_locks[i].lck); + erts_mtx_lock(&drv_ev_state_locks[i].lck); } drv_ev_state = (drv_ev_state ? erts_realloc(ERTS_ALC_T_DRV_EV_STATE, @@ -500,14 +479,14 @@ grow_drv_ev_state(int min_ix) drv_ev_state[i].type = ERTS_EV_TYPE_NONE; drv_ev_state[i].flags = 0; } - erts_smp_atomic_set_nob(&drv_ev_state_len, new_len); + erts_atomic_set_nob(&drv_ev_state_len, new_len); for (i=0; i<DRV_EV_STATE_LOCK_CNT; i++) { - erts_smp_mtx_unlock(&drv_ev_state_locks[i].lck); + erts_mtx_unlock(&drv_ev_state_locks[i].lck); } } /*else already grown by racing thread */ - erts_smp_mtx_unlock(&drv_ev_state_grow_lock); + erts_mtx_unlock(&drv_ev_state_grow_lock); } #endif /* ERTS_SYS_CONTINOUS_FD_NUMBERS */ @@ -565,7 +544,7 @@ deselect(ErtsDrvEventState *state, int mode) { int do_wake = 0; ErtsPollEvents rm_events; - ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(fd_mtx(state->fd))); + ERTS_LC_ASSERT(erts_lc_mtx_is_locked(fd_mtx(state->fd))); ASSERT(state->events); abort_tasks(state, mode); @@ -637,9 +616,9 @@ check_fd_cleanup(ErtsDrvEventState *state, { erts_aint_t current_cio_time; - ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(fd_mtx(state->fd))); + ERTS_LC_ASSERT(erts_lc_mtx_is_locked(fd_mtx(state->fd))); - current_cio_time = erts_smp_atomic_read_acqb(&erts_check_io_time); + current_cio_time = erts_atomic_read_acqb(&erts_check_io_time); *free_select = NULL; if (state->driver.select && (state->type != ERTS_EV_TYPE_DRV_SEL) @@ -692,7 +671,7 @@ check_cleanup_active_fd(ErtsSysFdType fd, { ErtsDrvEventState *state; int active = 0; - erts_smp_mtx_t *mtx = fd_mtx(fd); + erts_mtx_t *mtx = fd_mtx(fd); void *free_select = NULL; void *free_nif = NULL; #if ERTS_CIO_HAVE_DRV_EVENT @@ -702,7 +681,7 @@ check_cleanup_active_fd(ErtsSysFdType fd, ErtsPollEvents evon = 0, evoff = 0; #endif - erts_smp_mtx_lock(mtx); + erts_mtx_lock(mtx); #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS state = &drv_ev_state[(int) fd]; @@ -823,7 +802,7 @@ check_cleanup_active_fd(ErtsSysFdType fd, } - erts_smp_mtx_unlock(mtx); + erts_mtx_unlock(mtx); if (free_select) free_drv_select_data(free_select); @@ -857,7 +836,7 @@ check_cleanup_active_fds(erts_aint_t current_cio_time, int may_sleep) { int six = pollset.active_fd.six; int eix = pollset.active_fd.eix; - erts_aint32_t no = erts_smp_atomic32_read_dirty(&pollset.active_fd.no); + erts_aint32_t no = erts_atomic32_read_dirty(&pollset.active_fd.no); int size = pollset.active_fd.size; int ix = six; #if ERTS_CIO_DEFER_ACTIVE_EVENTS @@ -912,7 +891,7 @@ check_cleanup_active_fds(erts_aint_t current_cio_time, int may_sleep) pollset.active_fd.six = six; pollset.active_fd.eix = eix; - erts_smp_atomic32_set_relb(&pollset.active_fd.no, no); + erts_atomic32_set_relb(&pollset.active_fd.no, no); } static void grow_active_fds(void) @@ -941,8 +920,8 @@ add_active_fd(ErtsSysFdType fd) pollset.active_fd.array[eix] = fd; - erts_smp_atomic32_set_relb(&pollset.active_fd.no, - (erts_smp_atomic32_read_dirty(&pollset.active_fd.no) + erts_atomic32_set_relb(&pollset.active_fd.no, + (erts_atomic32_read_dirty(&pollset.active_fd.no) + 1)); eix++; @@ -982,10 +961,10 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix, if (prt == ERTS_INVALID_ERL_DRV_PORT) return -1; - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); + ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - if ((unsigned)fd >= (unsigned)erts_smp_atomic_read_nob(&drv_ev_state_len)) { + if ((unsigned)fd >= (unsigned)erts_atomic_read_nob(&drv_ev_state_len)) { if (fd < 0) { return -1; } @@ -997,7 +976,7 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix, } #endif - erts_smp_mtx_lock(fd_mtx(fd)); + erts_mtx_lock(fd_mtx(fd)); #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS state = &drv_ev_state[(int) fd]; @@ -1178,7 +1157,7 @@ done: &free_nif); done_unknown: - erts_smp_mtx_unlock(fd_mtx(fd)); + erts_mtx_unlock(fd_mtx(fd)); if (stop_select_fn) { int was_unmasked = erts_block_fpe(); DTRACE1(driver_stop_select, name); @@ -1227,7 +1206,7 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env, ASSERT(!(resource->monitors && resource->monitors->is_dying)); #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - if ((unsigned)fd >= (unsigned)erts_smp_atomic_read_nob(&drv_ev_state_len)) { + if ((unsigned)fd >= (unsigned)erts_atomic_read_nob(&drv_ev_state_len)) { if (fd < 0) { return INT_MIN | ERL_NIF_SELECT_INVALID_EVENT; } @@ -1239,7 +1218,7 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env, } #endif - erts_smp_mtx_lock(fd_mtx(fd)); + erts_mtx_lock(fd_mtx(fd)); #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS state = &drv_ev_state[(int) fd]; @@ -1433,7 +1412,7 @@ done: &free_nif); done_unknown: - erts_smp_mtx_unlock(fd_mtx(fd)); + erts_mtx_unlock(fd_mtx(fd)); if (call_stop) { erts_resource_stop(resource, (ErlNifEvent)fd, 1); if (call_stop == CALL_STOP_AND_RELEASE) { @@ -1479,10 +1458,10 @@ ERTS_CIO_EXPORT(driver_event)(ErlDrvPort ix, if (prt == ERTS_INVALID_ERL_DRV_PORT) return -1; - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); + ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - if ((unsigned)fd >= (unsigned)erts_smp_atomic_read_nob(&drv_ev_state_len)) { + if ((unsigned)fd >= (unsigned)erts_atomic_read_nob(&drv_ev_state_len)) { if (fd < 0) return -1; if (fd >= max_fds) { @@ -1493,7 +1472,7 @@ ERTS_CIO_EXPORT(driver_event)(ErlDrvPort ix, } #endif - erts_smp_mtx_lock(fd_mtx(fd)); + erts_mtx_lock(fd_mtx(fd)); #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS state = &drv_ev_state[(int) fd]; @@ -1586,7 +1565,7 @@ done: &free_select, &free_nif); - erts_smp_mtx_unlock(fd_mtx(fd)); + erts_mtx_unlock(fd_mtx(fd)); if (free_select) free_drv_select_data(free_select); @@ -2016,7 +1995,7 @@ iready(Eterm id, ErtsDrvEventState *state, erts_aint_t current_cio_time) ERTS_PORT_TASK_INPUT, current_cio_time)) { ErtsIoTask *iotask = &state->driver.select->iniotask; - erts_smp_atomic_set_nob(&iotask->executed_time, current_cio_time); + erts_atomic_set_nob(&iotask->executed_time, current_cio_time); if (erts_port_task_schedule(id, &iotask->task, ERTS_PORT_TASK_INPUT, @@ -2034,7 +2013,7 @@ oready(Eterm id, ErtsDrvEventState *state, erts_aint_t current_cio_time) ERTS_PORT_TASK_OUTPUT, current_cio_time)) { ErtsIoTask *iotask = &state->driver.select->outiotask; - erts_smp_atomic_set_nob(&iotask->executed_time, current_cio_time); + erts_atomic_set_nob(&iotask->executed_time, current_cio_time); if (erts_port_task_schedule(id, &iotask->task, ERTS_PORT_TASK_OUTPUT, @@ -2090,7 +2069,7 @@ send_event_tuple(struct erts_nif_select_event* e, ErtsResource* resource, erts_queue_message(rp, rp_locks, mp, tuple, am_system); if (rp_locks) - erts_smp_proc_unlock(rp, rp_locks); + erts_proc_unlock(rp, rp_locks); } @@ -2103,7 +2082,7 @@ eready(Eterm id, ErtsDrvEventState *state, ErlDrvEventData event_data, ERTS_PORT_TASK_EVENT, current_cio_time)) { ErtsIoTask *iotask = &state->driver.event->iotask; - erts_smp_atomic_set_nob(&iotask->executed_time, current_cio_time); + erts_atomic_set_nob(&iotask->executed_time, current_cio_time); if (erts_port_task_schedule(id, &iotask->task, ERTS_PORT_TASK_EVENT, @@ -2118,13 +2097,6 @@ eready(Eterm id, ErtsDrvEventState *state, ErlDrvEventData event_data, static void bad_fd_in_pollset(ErtsDrvEventState *, Eterm inport, Eterm outport); -#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT -void -ERTS_CIO_EXPORT(erts_check_io_async_sig_interrupt)(void) -{ - ERTS_CIO_POLL_AS_INTR(pollset.ps); -} -#endif void ERTS_CIO_EXPORT(erts_check_io_interrupt)(int set) @@ -2185,9 +2157,9 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait) * erts_check_io_time, since only one thread can * check io at a time. */ - current_cio_time = erts_smp_atomic_read_dirty(&erts_check_io_time); + current_cio_time = erts_atomic_read_dirty(&erts_check_io_time); current_cio_time++; - erts_smp_atomic_set_relb(&erts_check_io_time, current_cio_time); + erts_atomic_set_relb(&erts_check_io_time, current_cio_time); check_cleanup_active_fds(current_cio_time, timeout_time != ERTS_POLL_NO_TIMEOUT); @@ -2196,11 +2168,11 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait) erts_lc_check_exact(NULL, 0); /* No locks should be locked */ #endif - pollres_len = erts_smp_atomic32_read_dirty(&pollset.active_fd.no) + ERTS_CHECK_IO_POLL_RES_LEN; + pollres_len = erts_atomic32_read_dirty(&pollset.active_fd.no) + ERTS_CHECK_IO_POLL_RES_LEN; pollres = erts_alloc(ERTS_ALC_T_TMP, sizeof(ErtsPollResFd)*pollres_len); - erts_smp_atomic_set_nob(&pollset.in_poll_wait, 1); + erts_atomic_set_nob(&pollset.in_poll_wait, 1); poll_ret = ERTS_CIO_POLL_WAIT(pollset.ps, pollres, &pollres_len, timeout_time); @@ -2222,7 +2194,7 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait) if (poll_ret != 0) { - erts_smp_atomic_set_nob(&pollset.in_poll_wait, 0); + erts_atomic_set_nob(&pollset.in_poll_wait, 0); forget_removed(&pollset); erts_free(ERTS_ALC_T_TMP, pollres); if (poll_ret == EAGAIN) { @@ -2248,7 +2220,7 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait) ErtsSysFdType fd = (ErtsSysFdType) pollres[i].fd; ErtsDrvEventState *state; - erts_smp_mtx_lock(fd_mtx(fd)); + erts_mtx_lock(fd_mtx(fd)); #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS state = &drv_ev_state[ (int) fd]; @@ -2355,9 +2327,7 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait) add_active_fd(state->fd); } -#ifdef ERTS_SMP - erts_smp_mtx_unlock(fd_mtx(fd)); -#endif + erts_mtx_unlock(fd_mtx(fd)); if (is_not_nil(in.pid)) { send_event_tuple(&in, resource, am_ready_input); } @@ -2404,13 +2374,11 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait) } next_pollres:; -#ifdef ERTS_SMP - erts_smp_mtx_unlock(fd_mtx(fd)); -#endif + erts_mtx_unlock(fd_mtx(fd)); next_pollres_unlocked:; } - erts_smp_atomic_set_nob(&pollset.in_poll_wait, 0); + erts_atomic_set_nob(&pollset.in_poll_wait, 0); erts_free(ERTS_ALC_T_TMP, pollres); forget_removed(&pollset); } @@ -2506,16 +2474,16 @@ static int drv_ev_state_cmp(void *des1, void *des2) static void *drv_ev_state_alloc(void *des_tmpl) { ErtsDrvEventState *evstate; - erts_smp_spin_lock(&state_prealloc_lock); + erts_spin_lock(&state_prealloc_lock); if (state_prealloc_first == NULL) { - erts_smp_spin_unlock(&state_prealloc_lock); + erts_spin_unlock(&state_prealloc_lock); evstate = (ErtsDrvEventState *) erts_alloc(ERTS_ALC_T_DRV_EV_STATE, sizeof(ErtsDrvEventState)); } else { evstate = state_prealloc_first; state_prealloc_first = (ErtsDrvEventState *) evstate->hb.next; --num_state_prealloc; - erts_smp_spin_unlock(&state_prealloc_lock); + erts_spin_unlock(&state_prealloc_lock); } /* XXX: Already valid data if prealloced, could ignore template! */ *evstate = *((ErtsDrvEventState *) des_tmpl); @@ -2525,11 +2493,11 @@ static void *drv_ev_state_alloc(void *des_tmpl) static void drv_ev_state_free(void *des) { - erts_smp_spin_lock(&state_prealloc_lock); + erts_spin_lock(&state_prealloc_lock); ((ErtsDrvEventState *) des)->hb.next = &state_prealloc_first->hb; state_prealloc_first = (ErtsDrvEventState *) des; ++num_state_prealloc; - erts_smp_spin_unlock(&state_prealloc_lock); + erts_spin_unlock(&state_prealloc_lock); } #endif @@ -2540,15 +2508,15 @@ ERTS_CIO_EXPORT(erts_init_check_io)(void) ERL_NIF_SELECT_STOP_SCHEDULED | ERL_NIF_SELECT_INVALID_EVENT | ERL_NIF_SELECT_FAILED)) == 0); - erts_smp_atomic_init_nob(&erts_check_io_time, 0); - erts_smp_atomic_init_nob(&pollset.in_poll_wait, 0); + erts_atomic_init_nob(&erts_check_io_time, 0); + erts_atomic_init_nob(&pollset.in_poll_wait, 0); ERTS_CIO_POLL_INIT(); pollset.ps = ERTS_CIO_NEW_POLLSET(); pollset.active_fd.six = 0; pollset.active_fd.eix = 0; - erts_smp_atomic32_init_nob(&pollset.active_fd.no, 0); + erts_atomic32_init_nob(&pollset.active_fd.no, 0); pollset.active_fd.size = ERTS_ACTIVE_FD_INC; pollset.active_fd.array = erts_alloc(ERTS_ALC_T_ACTIVE_FD_ARR, sizeof(ErtsSysFdType)*ERTS_ACTIVE_FD_INC); @@ -2561,27 +2529,23 @@ ERTS_CIO_EXPORT(erts_init_check_io)(void) #endif -#ifdef ERTS_SMP init_removed_fd_alloc(); pollset.removed_list = NULL; - erts_smp_spinlock_init(&pollset.removed_list_lock, - "pollset_rm_list"); + erts_spinlock_init(&pollset.removed_list_lock, "pollset_rm_list", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO); { - int i; - for (i=0; i<DRV_EV_STATE_LOCK_CNT; i++) { -#ifdef ERTS_ENABLE_LOCK_COUNT - erts_smp_mtx_init_x(&drv_ev_state_locks[i].lck, "drv_ev_state", make_small(i)); -#else - erts_smp_mtx_init(&drv_ev_state_locks[i].lck, "drv_ev_state"); -#endif - } + int i; + for (i=0; i<DRV_EV_STATE_LOCK_CNT; i++) { + erts_mtx_init(&drv_ev_state_locks[i].lck, "drv_ev_state", make_small(i), + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO); + } } -#endif #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS max_fds = ERTS_CIO_POLL_MAX_FDS(); - erts_smp_atomic_init_nob(&drv_ev_state_len, 0); + erts_atomic_init_nob(&drv_ev_state_len, 0); drv_ev_state = NULL; - erts_smp_mtx_init(&drv_ev_state_grow_lock, "drv_ev_state_grow"); + erts_mtx_init(&drv_ev_state_grow_lock, "drv_ev_state_grow", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO); #else { SafeHashFunctions hf; @@ -2591,10 +2555,11 @@ ERTS_CIO_EXPORT(erts_init_check_io)(void) hf.free = &drv_ev_state_free; num_state_prealloc = 0; state_prealloc_first = NULL; - erts_smp_spinlock_init(&state_prealloc_lock,"state_prealloc"); + erts_spinlock_init(&state_prealloc_lock,"state_prealloc", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO); - safe_hash_init(ERTS_ALC_T_DRV_EV_STATE, &drv_ev_state_tab, "drv_ev_state_tab", - DRV_EV_STATE_HTAB_SIZE, hf); + safe_hash_init(ERTS_ALC_T_DRV_EV_STATE, &drv_ev_state_tab, "drv_ev_state_tab", + ERTS_LOCK_FLAGS_CATEGORY_IO, DRV_EV_STATE_HTAB_SIZE, hf); } #endif } @@ -2617,7 +2582,7 @@ ERTS_CIO_EXPORT(erts_check_io_size)(void) ERTS_CIO_POLL_INFO(pollset.ps, &pi); res = pi.memory_size; #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - res += sizeof(ErtsDrvEventState) * erts_smp_atomic_read_nob(&drv_ev_state_len); + res += sizeof(ErtsDrvEventState) * erts_atomic_read_nob(&drv_ev_state_len); #else res += safe_hash_table_sz(&drv_ev_state_tab); { @@ -2625,9 +2590,9 @@ ERTS_CIO_EXPORT(erts_check_io_size)(void) safe_hash_get_info(&hi, &drv_ev_state_tab); res += hi.objs * sizeof(ErtsDrvEventState); } - erts_smp_spin_lock(&state_prealloc_lock); + erts_spin_lock(&state_prealloc_lock); res += num_state_prealloc * sizeof(ErtsDrvEventState); - erts_smp_spin_unlock(&state_prealloc_lock); + erts_spin_unlock(&state_prealloc_lock); #endif return res; } @@ -2640,8 +2605,8 @@ ERTS_CIO_EXPORT(erts_check_io_info)(void *proc) Uint sz, *szp, *hp, **hpp, memory_size; Sint i; ErtsPollInfo pi; - erts_aint_t cio_time = erts_smp_atomic_read_acqb(&erts_check_io_time); - int active_fds = (int) erts_smp_atomic32_read_acqb(&pollset.active_fd.no); + erts_aint_t cio_time = erts_atomic_read_acqb(&erts_check_io_time); + int active_fds = (int) erts_atomic32_read_acqb(&pollset.active_fd.no); while (1) { erts_aint_t post_cio_time; @@ -2649,8 +2614,8 @@ ERTS_CIO_EXPORT(erts_check_io_info)(void *proc) ERTS_CIO_POLL_INFO(pollset.ps, &pi); - post_cio_time = erts_smp_atomic_read_mb(&erts_check_io_time); - post_active_fds = (int) erts_smp_atomic32_read_acqb(&pollset.active_fd.no); + post_cio_time = erts_atomic_read_mb(&erts_check_io_time); + post_active_fds = (int) erts_atomic32_read_acqb(&pollset.active_fd.no); if (cio_time == post_cio_time && active_fds == post_active_fds) break; cio_time = post_cio_time; @@ -2659,7 +2624,7 @@ ERTS_CIO_EXPORT(erts_check_io_info)(void *proc) memory_size = pi.memory_size; #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - memory_size += sizeof(ErtsDrvEventState) * erts_smp_atomic_read_nob(&drv_ev_state_len); + memory_size += sizeof(ErtsDrvEventState) * erts_atomic_read_nob(&drv_ev_state_len); #else memory_size += safe_hash_table_sz(&drv_ev_state_tab); { @@ -2667,9 +2632,9 @@ ERTS_CIO_EXPORT(erts_check_io_info)(void *proc) safe_hash_get_info(&hi, &drv_ev_state_tab); memory_size += hi.objs * sizeof(ErtsDrvEventState); } - erts_smp_spin_lock(&state_prealloc_lock); + erts_spin_lock(&state_prealloc_lock); memory_size += num_state_prealloc * sizeof(ErtsDrvEventState); - erts_smp_spin_unlock(&state_prealloc_lock); + erts_spin_unlock(&state_prealloc_lock); #endif hpp = NULL; @@ -3077,11 +3042,11 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(ErtsCheckIoDebugInfo *ciodip) erts_printf("--- fds in pollset --------------------------------------\n"); -#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) +#if defined(ERTS_ENABLE_LOCK_CHECK) erts_lc_check_exact(NULL, 0); /* No locks should be locked */ #endif - erts_smp_thr_progress_block(); /* stop the world to avoid messy locking */ + erts_thr_progress_block(); /* stop the world to avoid messy locking */ #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS counters.epep = erts_alloc(ERTS_ALC_T_TMP, sizeof(ErtsPollEvents)*max_fds); @@ -3094,7 +3059,7 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(ErtsCheckIoDebugInfo *ciodip) counters.no_driver_event_structs = 0; #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - len = erts_smp_atomic_read_nob(&drv_ev_state_len); + len = erts_atomic_read_nob(&drv_ev_state_len); for (fd = 0; fd < len; fd++) { doit_erts_check_io_debug((void *) &drv_ev_state[fd], (void *) &counters); } @@ -3106,7 +3071,7 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(ErtsCheckIoDebugInfo *ciodip) safe_hash_for_each(&drv_ev_state_tab, &doit_erts_check_io_debug, (void *) &counters); #endif - erts_smp_thr_progress_unblock(); + erts_thr_progress_unblock(); ciodip->no_used_fds = counters.used_fds; ciodip->no_driver_select_structs = counters.no_driver_select_structs; @@ -3130,3 +3095,12 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(ErtsCheckIoDebugInfo *ciodip) return counters.num_errors; } +#ifdef ERTS_ENABLE_LOCK_COUNT +void ERTS_CIO_EXPORT(erts_lcnt_update_cio_locks)(int enable) { +#ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS + erts_lcnt_enable_hash_lock_count(&drv_ev_state_tab, ERTS_LOCK_FLAGS_CATEGORY_IO, enable); +#else + (void)enable; +#endif +} +#endif /* ERTS_ENABLE_LOCK_COUNT */ diff --git a/erts/emulator/sys/common/erl_check_io.h b/erts/emulator/sys/common/erl_check_io.h index ee4abeece9..777942a473 100644 --- a/erts/emulator/sys/common/erl_check_io.h +++ b/erts/emulator/sys/common/erl_check_io.h @@ -44,10 +44,6 @@ Eterm erts_check_io_info_kp(void *); Eterm erts_check_io_info_nkp(void *); int erts_check_io_max_files_kp(void); int erts_check_io_max_files_nkp(void); -#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT -void erts_check_io_async_sig_interrupt_kp(void); -void erts_check_io_async_sig_interrupt_nkp(void); -#endif void erts_check_io_interrupt_kp(int); void erts_check_io_interrupt_nkp(int); void erts_check_io_interrupt_timed_kp(int, ErtsMonotonicTime); @@ -59,26 +55,32 @@ void erts_init_check_io_nkp(void); int erts_check_io_debug_kp(ErtsCheckIoDebugInfo *); int erts_check_io_debug_nkp(ErtsCheckIoDebugInfo *); +#ifdef ERTS_ENABLE_LOCK_COUNT +void erts_lcnt_update_cio_locks_kp(int enable); +void erts_lcnt_update_cio_locks_nkp(int enable); +#endif + #else /* !ERTS_ENABLE_KERNEL_POLL */ Uint erts_check_io_size(void); Eterm erts_check_io_info(void *); int erts_check_io_max_files(void); -#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT -void erts_check_io_async_sig_interrupt(void); -#endif void erts_check_io_interrupt(int); void erts_check_io_interrupt_timed(int, ErtsMonotonicTime); void erts_check_io(int); void erts_init_check_io(void); +#ifdef ERTS_ENABLE_LOCK_COUNT +void erts_lcnt_update_cio_locks(int enable); +#endif + #endif -extern erts_smp_atomic_t erts_check_io_time; +extern erts_atomic_t erts_check_io_time; typedef struct { ErtsPortTaskHandle task; - erts_smp_atomic_t executed_time; + erts_atomic_t executed_time; } ErtsIoTask; ERTS_GLB_INLINE void erts_io_notify_port_task_executed(ErtsPortTaskHandle *pthp); @@ -89,8 +91,8 @@ ERTS_GLB_INLINE void erts_io_notify_port_task_executed(ErtsPortTaskHandle *pthp) { ErtsIoTask *itp = (ErtsIoTask *) (((char *) pthp) - offsetof(ErtsIoTask, task)); - erts_aint_t ci_time = erts_smp_atomic_read_acqb(&erts_check_io_time); - erts_smp_atomic_set_relb(&itp->executed_time, ci_time); + erts_aint_t ci_time = erts_atomic_read_acqb(&erts_check_io_time); + erts_atomic_set_relb(&itp->executed_time, ci_time); } #endif diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c index bb930ff03b..a9c6e72c5f 100644 --- a/erts/emulator/sys/common/erl_mmap.c +++ b/erts/emulator/sys/common/erl_mmap.c @@ -24,7 +24,6 @@ #define ERTS_WANT_MEM_MAPPERS #include "sys.h" #include "erl_process.h" -#include "erl_smp.h" #include "atom.h" #include "erl_mmap.h" #include <stddef.h> @@ -62,11 +61,11 @@ (((UWord) (PTR)) - ((UWord) mm->sa.bot) \ < ((UWord) mm->sua.top) - ((UWord) mm->sa.bot)) #define ERTS_MMAP_IN_SUPERALIGNED_AREA(PTR) \ - (ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&mm->mtx)), \ + (ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&mm->mtx)), \ (((UWord) (PTR)) - ((UWord) mm->sa.bot) \ < ((UWord) mm->sa.top) - ((UWord) mm->sa.bot))) #define ERTS_MMAP_IN_SUPERUNALIGNED_AREA(PTR) \ - (ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&mm->mtx)), \ + (ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&mm->mtx)), \ (((UWord) (PTR)) - ((UWord) mm->sua.bot) \ < ((UWord) mm->sua.top) - ((UWord) mm->sua.bot))) @@ -199,10 +198,10 @@ static ErtsMMapOp mmap_ops[ERTS_MMAP_OP_RINGBUF_SZ]; #define ERTS_MMAP_OP_LCK(RES, IN_SZ, OUT_SZ) \ do { \ - erts_smp_mtx_lock(&mm->mtx); \ + erts_mtx_lock(&mm->mtx); \ ERTS_MMAP_OP_START((IN_SZ)); \ ERTS_MMAP_OP_END((RES), (OUT_SZ)); \ - erts_smp_mtx_unlock(&mm->mtx); \ + erts_mtx_unlock(&mm->mtx); \ } while (0) #define ERTS_MUNMAP_OP(PTR, SZ) \ @@ -221,9 +220,9 @@ static ErtsMMapOp mmap_ops[ERTS_MMAP_OP_RINGBUF_SZ]; #define ERTS_MUNMAP_OP_LCK(PTR, SZ) \ do { \ - erts_smp_mtx_lock(&mm->mtx); \ + erts_mtx_lock(&mm->mtx); \ ERTS_MUNMAP_OP((PTR), (SZ)); \ - erts_smp_mtx_unlock(&mm->mtx); \ + erts_mtx_unlock(&mm->mtx); \ } while (0) #define ERTS_MREMAP_OP_START(OLD_PTR, OLD_SZ, IN_SZ) \ @@ -249,10 +248,10 @@ static ErtsMMapOp mmap_ops[ERTS_MMAP_OP_RINGBUF_SZ]; #define ERTS_MREMAP_OP_LCK(RES, OLD_PTR, OLD_SZ, IN_SZ, OUT_SZ) \ do { \ - erts_smp_mtx_lock(&mm->mtx); \ + erts_mtx_lock(&mm->mtx); \ ERTS_MREMAP_OP_START((OLD_PTR), (OLD_SZ), (IN_SZ)); \ ERTS_MREMAP_OP_END((RES), (OUT_SZ)); \ - erts_smp_mtx_unlock(&mm->mtx); \ + erts_mtx_unlock(&mm->mtx); \ } while (0) #define ERTS_MMAP_OP_ABORT() \ @@ -321,7 +320,7 @@ struct ErtsMemMapper_ { #if HAVE_MMAP && (!defined(MAP_ANON) && !defined(MAP_ANONYMOUS)) int mmap_fd; #endif - erts_smp_mtx_t mtx; + erts_mtx_t mtx; struct { char *free_list; char *unused_start; @@ -1536,7 +1535,7 @@ erts_mmap(ErtsMemMapper* mm, Uint32 flags, UWord *sizep) ErtsFreeSegDesc *desc; Uint32 superaligned = (ERTS_MMAPFLG_SUPERALIGNED & flags); - erts_smp_mtx_lock(&mm->mtx); + erts_mtx_lock(&mm->mtx); ERTS_MMAP_OP_START(*sizep); @@ -1660,7 +1659,7 @@ erts_mmap(ErtsMemMapper* mm, Uint32 flags, UWord *sizep) } ERTS_MMAP_OP_ABORT(); - erts_smp_mtx_unlock(&mm->mtx); + erts_mtx_unlock(&mm->mtx); } #if ERTS_HAVE_OS_MMAP @@ -1724,13 +1723,13 @@ supercarrier_success: #endif ERTS_MMAP_OP_END(seg, asize); - erts_smp_mtx_unlock(&mm->mtx); + erts_mtx_unlock(&mm->mtx); *sizep = asize; return (void *) seg; supercarrier_reserve_failure: - erts_smp_mtx_unlock(&mm->mtx); + erts_mtx_unlock(&mm->mtx); *sizep = 0; return NULL; } @@ -1760,7 +1759,7 @@ erts_munmap(ErtsMemMapper* mm, Uint32 flags, void *ptr, UWord size) start = (char *) ptr; end = start + size; - erts_smp_mtx_lock(&mm->mtx); + erts_mtx_lock(&mm->mtx); ERTS_MUNMAP_OP(ptr, size); @@ -1829,7 +1828,7 @@ erts_munmap(ErtsMemMapper* mm, Uint32 flags, void *ptr, UWord size) if (unres_sz) mm->unreserve_physical(((char *) ptr) + ad_sz, unres_sz); - erts_smp_mtx_unlock(&mm->mtx); + erts_mtx_unlock(&mm->mtx); } } } @@ -1948,12 +1947,12 @@ erts_mremap(ErtsMemMapper* mm, ? ERTS_SUPERALIGNED_CEILING(*sizep) : ERTS_PAGEALIGNED_CEILING(*sizep)); - erts_smp_mtx_lock(&mm->mtx); + erts_mtx_lock(&mm->mtx); if (ERTS_MMAP_IN_SUPERALIGNED_AREA(ptr) ? (!superaligned && lookup_free_seg(&mm->sua.map, asize)) : (superaligned && lookup_free_seg(&mm->sa.map, asize))) { - erts_smp_mtx_unlock(&mm->mtx); + erts_mtx_unlock(&mm->mtx); /* * Segment currently in wrong area (due to a previous memory * shortage), move it to the right area. @@ -2068,7 +2067,7 @@ erts_mremap(ErtsMemMapper* mm, } ERTS_MMAP_OP_ABORT(); - erts_smp_mtx_unlock(&mm->mtx); + erts_mtx_unlock(&mm->mtx); /* Failed to resize... */ } @@ -2090,14 +2089,14 @@ supercarrier_resize_success: #endif ERTS_MREMAP_OP_END(new_ptr, asize); - erts_smp_mtx_unlock(&mm->mtx); + erts_mtx_unlock(&mm->mtx); *sizep = asize; return new_ptr; supercarrier_reserve_failure: ERTS_MREMAP_OP_END(NULL, old_size); - erts_smp_mtx_unlock(&mm->mtx); + erts_mtx_unlock(&mm->mtx); *sizep = old_size; return NULL; @@ -2212,9 +2211,11 @@ erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init, int executable) erts_exit(1, "erts_mmap: Failed to open /dev/zero\n"); #endif - erts_smp_mtx_init(&mm->mtx, "erts_mmap"); + erts_mtx_init(&mm->mtx, "erts_mmap", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC); if (is_first_call) { - erts_mtx_init(&am.init_mutex, "mmap_init_atoms"); + erts_mtx_init(&am.init_mutex, "mmap_init_atoms", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC); } #ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION @@ -2405,7 +2406,7 @@ Eterm erts_mmap_info(ErtsMemMapper* mm, Eterm res = THE_NON_VALUE; if (!hpp) { - erts_smp_mtx_lock(&mm->mtx); + erts_mtx_lock(&mm->mtx); emis->sizes[0] = mm->size.supercarrier.total; emis->sizes[1] = mm->sa.top - mm->sa.bot; emis->sizes[2] = mm->sua.top - mm->sua.bot; @@ -2421,7 +2422,7 @@ Eterm erts_mmap_info(ErtsMemMapper* mm, emis->segs[5] = mm->sua.map.nseg; emis->os_used = mm->size.os.used; - erts_smp_mtx_unlock(&mm->mtx); + erts_mtx_unlock(&mm->mtx); } list[lix] = erts_mmap_info_options(mm, "option ", print_to_p, print_to_arg, @@ -2541,14 +2542,14 @@ Eterm erts_mmap_debug_info(Process* p) Eterm *hp, *hp_end; Uint may_need; - erts_smp_mtx_lock(&mm->mtx); + erts_mtx_lock(&mm->mtx); values[0] = (UWord)mm->sa.bot; values[1] = (UWord)mm->sa.top; values[2] = (UWord)mm->sua.bot; values[3] = (UWord)mm->sua.top; sa_list = build_free_seg_list(p, &mm->sa.map); sua_list = build_free_seg_list(p, &mm->sua.map); - erts_smp_mtx_unlock(&mm->mtx); + erts_mtx_unlock(&mm->mtx); may_need = 4*(2+3+2) + 2*(2+3); hp = HAlloc(p, may_need); @@ -2816,7 +2817,8 @@ static void hard_dbg_mseg_init(void) { ErtsFreeSegDesc_fake* p; - erts_mtx_init(&hard_dbg_mseg_mtx, "hard_dbg_mseg"); + erts_mtx_init(&hard_dbg_mseg_mtx, "hard_dbg_mseg", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DEBUG); hard_dbg_mseg_tree.root = NULL; hard_dbg_mseg_tree.order = ADDR_ORDER; diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c index b8f0bb7150..bf6de9b13a 100644 --- a/erts/emulator/sys/common/erl_mseg.c +++ b/erts/emulator/sys/common/erl_mseg.c @@ -188,7 +188,6 @@ typedef union { static int no_mseg_allocators; static ErtsAlgndMsegAllctr_t *aligned_mseg_allctr; -#ifdef ERTS_SMP #define ERTS_MSEG_ALLCTR_IX(IX) \ (&aligned_mseg_allctr[(IX)].mseg_alloc) @@ -199,18 +198,6 @@ static ErtsAlgndMsegAllctr_t *aligned_mseg_allctr; #define ERTS_MSEG_ALLCTR_OPT(OPT) \ ((OPT)->sched_spec ? ERTS_MSEG_ALLCTR_SS() : ERTS_MSEG_ALLCTR_IX(0)) -#else - -#define ERTS_MSEG_ALLCTR_IX(IX) \ - (&aligned_mseg_allctr[0].mseg_alloc) - -#define ERTS_MSEG_ALLCTR_SS() \ - (&aligned_mseg_allctr[0].mseg_alloc) - -#define ERTS_MSEG_ALLCTR_OPT(OPT) \ - (&aligned_mseg_allctr[0].mseg_alloc) - -#endif #define ERTS_MSEG_LOCK(MA) \ do { \ @@ -352,11 +339,11 @@ mseg_recreate(ErtsMsegAllctr_t *ma, Uint flags, void *old_seg, UWord old_size, U do { \ if ((MA)->is_thread_safe) \ ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&(MA)->mtx) \ - || erts_smp_thr_progress_is_blocking() \ + || erts_thr_progress_is_blocking() \ || ERTS_IS_CRASH_DUMPING); \ else \ ERTS_LC_ASSERT((MA)->ix == (int) erts_get_scheduler_id() \ - || erts_smp_thr_progress_is_blocking() \ + || erts_thr_progress_is_blocking() \ || ERTS_IS_CRASH_DUMPING); \ } while (0) #else @@ -1404,11 +1391,7 @@ erts_mseg_init(ErtsMsegInit_t *init) int i; UWord x; -#ifdef ERTS_SMP no_mseg_allocators = init->nos + 1; -#else - no_mseg_allocators = 1; -#endif x = (UWord) malloc(sizeof(ErtsAlgndMsegAllctr_t) *no_mseg_allocators @@ -1420,7 +1403,8 @@ erts_mseg_init(ErtsMsegInit_t *init) atoms_initialized = 0; - erts_mtx_init(&init_atoms_mutex, "mseg_init_atoms"); + erts_mtx_init(&init_atoms_mutex, "mseg_init_atoms", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC); #ifdef ERTS_HAVE_EXEC_MMAPPER /* Initialize erts_exec_mapper *FIRST*, to increase probability @@ -1449,7 +1433,8 @@ erts_mseg_init(ErtsMsegInit_t *init) ma->is_thread_safe = 0; else { ma->is_thread_safe = 1; - erts_mtx_init(&ma->mtx, "mseg"); + erts_mtx_init(&ma->mtx, "mseg", make_small(i), + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_ALLOCATOR); } ma->is_cache_check_scheduled = 0; diff --git a/erts/emulator/sys/common/erl_os_monotonic_time_extender.c b/erts/emulator/sys/common/erl_os_monotonic_time_extender.c index d53190fdd5..341845cc2a 100644 --- a/erts/emulator/sys/common/erl_os_monotonic_time_extender.c +++ b/erts/emulator/sys/common/erl_os_monotonic_time_extender.c @@ -23,7 +23,6 @@ #endif #include "erl_os_monotonic_time_extender.h" -#ifdef USE_THREADS static void *os_monotonic_time_extender(void *vstatep) { @@ -49,30 +48,22 @@ static void *os_monotonic_time_extender(void *vstatep) } static erts_tid_t os_monotonic_extender_tid; -#endif void erts_init_os_monotonic_time_extender(ErtsOsMonotonicTimeExtendState *statep, Uint32 (*raw_os_monotonic_time)(void), int check_seconds) { -#ifdef USE_THREADS statep->raw_os_monotonic_time = raw_os_monotonic_time; erts_atomic32_init_nob(&statep->extend[0], (erts_aint32_t) 0); erts_atomic32_init_nob(&statep->extend[1], (erts_aint32_t) 0); statep->check_interval = check_seconds; -#else - statep->extend[0] = (Uint32) 0; - statep->extend[1] = (Uint32) 0; - statep->last_msb = (ErtsMonotonicTime) 0; -#endif } void erts_late_init_os_monotonic_time_extender(ErtsOsMonotonicTimeExtendState *statep) { -#ifdef USE_THREADS erts_thr_opts_t thr_opts = ERTS_THR_OPTS_DEFAULT_INITER; thr_opts.detached = 1; thr_opts.suggested_stack_size = 4; @@ -85,5 +76,4 @@ erts_late_init_os_monotonic_time_extender(ErtsOsMonotonicTimeExtendState *statep os_monotonic_time_extender, (void*) statep, &thr_opts); -#endif } diff --git a/erts/emulator/sys/common/erl_os_monotonic_time_extender.h b/erts/emulator/sys/common/erl_os_monotonic_time_extender.h index 8089c9aed9..53c32579d5 100644 --- a/erts/emulator/sys/common/erl_os_monotonic_time_extender.h +++ b/erts/emulator/sys/common/erl_os_monotonic_time_extender.h @@ -25,36 +25,16 @@ #include "erl_threads.h" typedef struct { -#ifdef USE_THREADS Uint32 (*raw_os_monotonic_time)(void); erts_atomic32_t extend[2]; int check_interval; -#else - Uint32 extend[2]; - ErtsMonotonicTime last_msb; -#endif } ErtsOsMonotonicTimeExtendState; -#ifdef USE_THREADS -# define ERTS_CHK_EXTEND_OS_MONOTONIC_TIME(S, RT) ((void) 1) # define ERTS_EXTEND_OS_MONOTONIC_TIME(S, RT) \ ((((ErtsMonotonicTime) \ erts_atomic32_read_nob(&((S)->extend[((int) ((RT) >> 31)) & 1]))) \ << 32) \ + (RT)) -#else -# define ERTS_CHK_EXTEND_OS_MONOTONIC_TIME(S, RT) \ - do { \ - Uint32 msb__ = (RT) & (((Uint32) 1) << 31); \ - if (msb__ != (S)->last_msb) { \ - int ix__ = ((int) ((S)->last_msb >> 31)) & 1; \ - (S)->extend[ix__]++; \ - (S)->last_msb = msb; \ - } \ - } while (0) -# define ERTS_EXTEND_OS_MONOTONIC_TIME(S, RT) \ - ((((ErtsMonotonicTime) (S)->extend[((int) ((RT) >> 31)) & 1]) << 32) + (RT)) -#endif void erts_init_os_monotonic_time_extender(ErtsOsMonotonicTimeExtendState *statep, diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c index 5e7ae8953a..7d26839b0f 100644 --- a/erts/emulator/sys/common/erl_poll.c +++ b/erts/emulator/sys/common/erl_poll.c @@ -95,9 +95,6 @@ #define ERTS_POLL_DEBUG_PRINT #endif -#if defined(DEBUG) && 0 -#define HARD_DEBUG -#endif #ifdef _DARWIN_UNLIMITED_SELECT typedef struct { @@ -147,29 +144,15 @@ int ERTS_SELECT(int nfds, ERTS_fd_set *readfds, ERTS_fd_set *writefds, #define ERTS_POLL_USE_BATCH_UPDATE_POLLSET (ERTS_POLL_USE_DEVPOLL \ || ERTS_POLL_USE_KQUEUE) -#define ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE \ - (defined(ERTS_SMP) || ERTS_POLL_USE_KERNEL_POLL || ERTS_POLL_USE_POLL) -#define ERTS_POLL_USE_CONCURRENT_UPDATE \ - (defined(ERTS_SMP) && ERTS_POLL_USE_EPOLL) +#define ERTS_POLL_USE_CONCURRENT_UPDATE ERTS_POLL_USE_EPOLL #define ERTS_POLL_COALESCE_KP_RES (ERTS_POLL_USE_KQUEUE || ERTS_POLL_USE_EPOLL) -#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT -# define ERTS_POLL_ASYNC_INTERRUPT_SUPPORT 1 -#else -# define ERTS_POLL_ASYNC_INTERRUPT_SUPPORT 0 -#endif - -#define ERTS_POLL_USE_WAKEUP_PIPE \ - (ERTS_POLL_ASYNC_INTERRUPT_SUPPORT || defined(USE_THREADS)) - -#ifdef ERTS_SMP - #define ERTS_POLLSET_LOCK(PS) \ - erts_smp_mtx_lock(&(PS)->mtx) + erts_mtx_lock(&(PS)->mtx) #define ERTS_POLLSET_UNLOCK(PS) \ - erts_smp_mtx_unlock(&(PS)->mtx) + erts_mtx_unlock(&(PS)->mtx) #define ERTS_POLLSET_SET_POLLED_CHK(PS) \ ((int) erts_atomic32_xchg_nob(&(PS)->polled, (erts_aint32_t) 1)) @@ -178,28 +161,13 @@ int ERTS_SELECT(int nfds, ERTS_fd_set *readfds, ERTS_fd_set *writefds, #define ERTS_POLLSET_IS_POLLED(PS) \ ((int) erts_atomic32_read_nob(&(PS)->polled)) -#else -#define ERTS_POLLSET_LOCK(PS) -#define ERTS_POLLSET_UNLOCK(PS) -#define ERTS_POLLSET_SET_POLLED_CHK(PS) 0 -#define ERTS_POLLSET_UNSET_POLLED(PS) -#define ERTS_POLLSET_IS_POLLED(PS) 0 - -#endif - -#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE #define ERTS_POLLSET_SET_HAVE_UPDATE_REQUESTS(PS) \ - erts_smp_atomic32_set_nob(&(PS)->have_update_requests, (erts_aint32_t) 1) + erts_atomic32_set_nob(&(PS)->have_update_requests, (erts_aint32_t) 1) #define ERTS_POLLSET_UNSET_HAVE_UPDATE_REQUESTS(PS) \ - erts_smp_atomic32_set_nob(&(PS)->have_update_requests, (erts_aint32_t) 0) + erts_atomic32_set_nob(&(PS)->have_update_requests, (erts_aint32_t) 0) #define ERTS_POLLSET_HAVE_UPDATE_REQUESTS(PS) \ - ((int) erts_smp_atomic32_read_nob(&(PS)->have_update_requests)) -#else -#define ERTS_POLLSET_SET_HAVE_UPDATE_REQUESTS(PS) -#define ERTS_POLLSET_UNSET_HAVE_UPDATE_REQUESTS(PS) -#define ERTS_POLLSET_HAVE_UPDATE_REQUESTS(PS) 0 -#endif + ((int) erts_atomic32_read_nob(&(PS)->have_update_requests)) #if ERTS_POLL_USE_FALLBACK # if ERTS_POLL_USE_POLL @@ -212,7 +180,6 @@ int ERTS_SELECT(int nfds, ERTS_fd_set *readfds, ERTS_fd_set *writefds, * --- Data types ------------------------------------------------------------ */ -#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE #define ERTS_POLLSET_UPDATE_REQ_BLOCK_SIZE 128 typedef struct ErtsPollSetUpdateRequestsBlock_ ErtsPollSetUpdateRequestsBlock; @@ -222,19 +189,14 @@ struct ErtsPollSetUpdateRequestsBlock_ { int fds[ERTS_POLLSET_UPDATE_REQ_BLOCK_SIZE]; }; -#endif -#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE # define ERTS_POLL_FD_FLG_INURQ (((unsigned short) 1) << 0) -#endif #if ERTS_POLL_USE_FALLBACK # define ERTS_POLL_FD_FLG_INFLBCK (((unsigned short) 1) << 1) # define ERTS_POLL_FD_FLG_USEFLBCK (((unsigned short) 1) << 2) #endif -#if ERTS_POLL_USE_KERNEL_POLL || defined(ERTS_SMP) # define ERTS_POLL_FD_FLG_RST (((unsigned short) 1) << 3) -#endif typedef struct { #if ERTS_POLL_USE_POLL int pix; @@ -244,9 +206,7 @@ typedef struct { #if ERTS_POLL_COALESCE_KP_RES unsigned short res_ev_ix; #endif -#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE || ERTS_POLL_USE_FALLBACK unsigned short flags; -#endif } ErtsFdStatus; @@ -272,7 +232,7 @@ struct ErtsPollSet_ { ErtsPollSet next; int internal_fd_limit; ErtsFdStatus *fds_status; - erts_smp_atomic_t no_of_user_fds; + erts_atomic_t no_of_user_fds; int fds_status_len; #if ERTS_POLL_USE_KERNEL_POLL int kp_fd; @@ -301,32 +261,24 @@ struct ErtsPollSet_ { ERTS_fd_set output_fds; ERTS_fd_set res_output_fds; #endif -#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE ErtsPollSetUpdateRequestsBlock update_requests; ErtsPollSetUpdateRequestsBlock *curr_upd_req_block; - erts_smp_atomic32_t have_update_requests; -#endif -#ifdef ERTS_SMP + erts_atomic32_t have_update_requests; erts_atomic32_t polled; - erts_smp_mtx_t mtx; -#endif -#if ERTS_POLL_USE_WAKEUP_PIPE + erts_mtx_t mtx; int wake_fds[2]; -#endif #if ERTS_POLL_USE_TIMERFD int timer_fd; #endif #if ERTS_POLL_USE_FALLBACK int fallback_used; #endif -#if defined(USE_THREADS) || ERTS_POLL_ASYNC_INTERRUPT_SUPPORT erts_atomic32_t wakeup_state; -#endif erts_atomic64_t timeout_time; #ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS - erts_smp_atomic_t no_avoided_wakeups; - erts_smp_atomic_t no_avoided_interrupts; - erts_smp_atomic_t no_interrupt_timed; + erts_atomic_t no_avoided_wakeups; + erts_atomic_t no_avoided_interrupts; + erts_atomic_t no_interrupt_timed; #endif }; @@ -336,7 +288,7 @@ static void fatal_error_async_signal_safe(char *error_str); static int max_fds = -1; static ErtsPollSet pollsets; -static erts_smp_spinlock_t pollsets_lock; +static erts_mtx_t pollsets_lock; #if ERTS_POLL_USE_POLL @@ -413,50 +365,37 @@ get_timeout_time(ErtsPollSet ps) static ERTS_INLINE void reset_wakeup_state(ErtsPollSet ps) { -#if defined(USE_THREADS) || ERTS_POLL_ASYNC_INTERRUPT_SUPPORT erts_atomic32_set_mb(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN); -#endif } static ERTS_INLINE int is_woken(ErtsPollSet ps) { -#if defined(USE_THREADS) || ERTS_POLL_ASYNC_INTERRUPT_SUPPORT return erts_atomic32_read_acqb(&ps->wakeup_state) != ERTS_POLL_NOT_WOKEN; -#else - return 0; -#endif } static ERTS_INLINE int is_interrupted_reset(ErtsPollSet ps) { -#if defined(USE_THREADS) || ERTS_POLL_ASYNC_INTERRUPT_SUPPORT return (erts_atomic32_xchg_acqb(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN) == ERTS_POLL_WOKEN_INTR); -#else - return 0; -#endif } static ERTS_INLINE void woke_up(ErtsPollSet ps) { -#if defined(USE_THREADS) || ERTS_POLL_ASYNC_INTERRUPT_SUPPORT erts_aint32_t wakeup_state = erts_atomic32_read_acqb(&ps->wakeup_state); if (wakeup_state == ERTS_POLL_NOT_WOKEN) (void) erts_atomic32_cmpxchg_nob(&ps->wakeup_state, ERTS_POLL_WOKEN, ERTS_POLL_NOT_WOKEN); ASSERT(erts_atomic32_read_nob(&ps->wakeup_state) != ERTS_POLL_NOT_WOKEN); -#endif } /* * --- Wakeup pipe ----------------------------------------------------------- */ -#if ERTS_POLL_USE_WAKEUP_PIPE static ERTS_INLINE void wake_poller(ErtsPollSet ps, int interrupted, int async_signal_safe) @@ -507,18 +446,11 @@ wake_poller(ErtsPollSet ps, int interrupted, int async_signal_safe) static ERTS_INLINE void cleanup_wakeup_pipe(ErtsPollSet ps) { -#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT - int intr = 0; -#endif int fd = ps->wake_fds[0]; int res; do { char buf[32]; res = read(fd, buf, sizeof(buf)); -#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT - if (res > 0) - intr = 1; -#endif } while (res > 0 || (res < 0 && errno == EINTR)); if (res < 0 && errno != ERRNO_BLOCK) { fatal_error("%s:%d:cleanup_wakeup_pipe(): " @@ -528,10 +460,6 @@ cleanup_wakeup_pipe(ErtsPollSet ps) fd, erl_errno_id(errno), errno); } -#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT - if (intr) - erts_atomic32_set_nob(&ps->wakeup_state, ERTS_POLL_WOKEN_INTR); -#endif } static void @@ -574,7 +502,6 @@ create_wakeup_pipe(ErtsPollSet ps) ps->wake_fds[1] = wake_fds[1]; } -#endif /* ERTS_POLL_USE_WAKEUP_PIPE */ /* * --- timer fd ----------------------------------------------------------- @@ -648,7 +575,6 @@ timerfd_clear(ErtsPollSet ps, int res, int max_res) { /* * --- Poll set update requests ---------------------------------------------- */ -#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE static ERTS_INLINE void enqueue_update_request(ErtsPollSet ps, int fd) @@ -691,7 +617,6 @@ free_update_requests_block(ErtsPollSet ps, } } -#endif /* ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE */ /* * --- Growing poll set structures ------------------------------------------- @@ -819,9 +744,7 @@ grow_fds_status(ErtsPollSet ps, int min_fd) #if ERTS_POLL_COALESCE_KP_RES ps->fds_status[i].res_ev_ix = (unsigned short) ERTS_POLL_MAX_RES; #endif -#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE || ERTS_POLL_USE_FALLBACK ps->fds_status[i].flags = (unsigned short) 0; -#endif } ps->fds_status_len = new_len; } @@ -849,7 +772,7 @@ need_update(ErtsPollSet ps, int fd) ps->fds_status[fd].flags &= ~ERTS_POLL_FD_FLG_RST; reset = 0; } -#elif defined(ERTS_SMP) +#else ps->fds_status[fd].flags &= ~ERTS_POLL_FD_FLG_RST; #endif @@ -1032,7 +955,7 @@ write_batch_buf(ErtsPollSet ps, ErtsPollBatchBuf *bbp) ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_USEFLBCK; ASSERT(ps->fds_status[fd].used_events); ps->fds_status[fd].used_events = 0; - erts_smp_atomic_dec_nob(&ps->no_of_user_fds); + erts_atomic_dec_nob(&ps->no_of_user_fds); update_fallback_pollset(ps, fd); ASSERT(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK); break; @@ -1082,11 +1005,11 @@ batch_update_pollset(ErtsPollSet ps, int fd, ErtsPollBatchBuf *bbp) events = ERTS_POLL_EV_E2N(ps->fds_status[fd].events); if (!events) { buf[buf_len].events = POLLREMOVE; - erts_smp_atomic_dec_nob(&ps->no_of_user_fds); + erts_atomic_dec_nob(&ps->no_of_user_fds); } else if (!ps->fds_status[fd].used_events) { buf[buf_len].events = events; - erts_smp_atomic_inc_nob(&ps->no_of_user_fds); + erts_atomic_inc_nob(&ps->no_of_user_fds); } else { if ((ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_RST) @@ -1176,12 +1099,12 @@ batch_update_pollset(ErtsPollSet ps, int fd, ErtsPollBatchBuf *bbp) } if (used_events) { if (!events) { - erts_smp_atomic_dec_nob(&ps->no_of_user_fds); + erts_atomic_dec_nob(&ps->no_of_user_fds); } } else { if (events) - erts_smp_atomic_inc_nob(&ps->no_of_user_fds); + erts_atomic_inc_nob(&ps->no_of_user_fds); } ASSERT((events & ~(ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT)) == 0); ASSERT((used_events & ~(ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT)) == 0); @@ -1255,7 +1178,7 @@ update_pollset(ErtsPollSet ps, int fd) epe.data.fd = epe_templ.data.fd; res = epoll_ctl(ps->kp_fd, EPOLL_CTL_DEL, fd, &epe); } while (res != 0 && errno == EINTR); - erts_smp_atomic_dec_nob(&ps->no_of_user_fds); + erts_atomic_dec_nob(&ps->no_of_user_fds); ps->fds_status[fd].used_events = 0; } @@ -1263,11 +1186,11 @@ update_pollset(ErtsPollSet ps, int fd) /* A note on EPOLL_CTL_DEL: linux kernel versions before 2.6.9 need a non-NULL event pointer even though it is ignored... */ op = EPOLL_CTL_DEL; - erts_smp_atomic_dec_nob(&ps->no_of_user_fds); + erts_atomic_dec_nob(&ps->no_of_user_fds); } else if (!ps->fds_status[fd].used_events) { op = EPOLL_CTL_ADD; - erts_smp_atomic_inc_nob(&ps->no_of_user_fds); + erts_atomic_inc_nob(&ps->no_of_user_fds); } else { op = EPOLL_CTL_MOD; @@ -1317,7 +1240,7 @@ update_pollset(ErtsPollSet ps, int fd) /* Fall through ... */ case EPOLL_CTL_ADD: { ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_USEFLBCK; - erts_smp_atomic_dec_nob(&ps->no_of_user_fds); + erts_atomic_dec_nob(&ps->no_of_user_fds); #if ERTS_POLL_USE_CONCURRENT_UPDATE if (!*update_fallback) { *update_fallback = 1; @@ -1405,7 +1328,7 @@ static int update_pollset(ErtsPollSet ps, int fd) #if ERTS_POLL_USE_FALLBACK ASSERT(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK); #endif - erts_smp_atomic_dec_nob(&ps->no_of_user_fds); + erts_atomic_dec_nob(&ps->no_of_user_fds); last_pix = --ps->no_poll_fds; if (pix != last_pix) { /* Move last pix to this pix */ @@ -1432,7 +1355,7 @@ static int update_pollset(ErtsPollSet ps, int fd) ASSERT(!(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK) || fd == ps->kp_fd); #endif - erts_smp_atomic_inc_nob(&ps->no_of_user_fds); + erts_atomic_inc_nob(&ps->no_of_user_fds); ps->fds_status[fd].pix = pix = ps->no_poll_fds++; if (pix >= ps->poll_fds_len) grow_poll_fds(ps, pix); @@ -1484,7 +1407,7 @@ static int update_pollset(ErtsPollSet ps, int fd) if (!ps->fds_status[fd].used_events) { ASSERT(events); - erts_smp_atomic_inc_nob(&ps->no_of_user_fds); + erts_atomic_inc_nob(&ps->no_of_user_fds); #if ERTS_POLL_USE_FALLBACK ps->no_select_fds++; ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_INFLBCK; @@ -1492,7 +1415,7 @@ static int update_pollset(ErtsPollSet ps, int fd) } else if (!events) { ASSERT(ps->fds_status[fd].used_events); - erts_smp_atomic_dec_nob(&ps->no_of_user_fds); + erts_atomic_dec_nob(&ps->no_of_user_fds); ps->fds_status[fd].events = events; #if ERTS_POLL_USE_FALLBACK ps->no_select_fds--; @@ -1518,7 +1441,6 @@ static int update_pollset(ErtsPollSet ps, int fd) #endif /* ERTS_POLL_USE_POLL || ERTS_POLL_USE_SELECT || ERTS_POLL_USE_FALLBACK */ -#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE static void handle_update_requests(ErtsPollSet ps) @@ -1565,7 +1487,6 @@ handle_update_requests(ErtsPollSet ps) ERTS_POLLSET_UNSET_HAVE_UPDATE_REQUESTS(ps); } -#endif /* ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE */ static ERTS_INLINE ErtsPollEvents poll_control(ErtsPollSet ps, int fd, ErtsPollEvents events, int on, int *do_wake) @@ -1583,12 +1504,10 @@ poll_control(ErtsPollSet ps, int fd, ErtsPollEvents events, int on, int *do_wake goto done; } #endif -#if ERTS_POLL_USE_WAKEUP_PIPE if (fd == ps->wake_fds[0] || fd == ps->wake_fds[1]) { new_events = ERTS_POLL_EV_NVAL; goto done; } -#endif #if ERTS_POLL_USE_TIMERFD if (fd == ps->timer_fd) { new_events = ERTS_POLL_EV_NVAL; @@ -1615,9 +1534,7 @@ poll_control(ErtsPollSet ps, int fd, ErtsPollEvents events, int on, int *do_wake new_events &= ~events; if (new_events == (ErtsPollEvents) 0) { -#if ERTS_POLL_USE_KERNEL_POLL || defined(ERTS_SMP) ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_RST; -#endif #if ERTS_POLL_USE_FALLBACK ps->fds_status[fd].flags &= ~ERTS_POLL_FD_FLG_USEFLBCK; #endif @@ -1626,18 +1543,12 @@ poll_control(ErtsPollSet ps, int fd, ErtsPollEvents events, int on, int *do_wake ps->fds_status[fd].events = new_events; if (new_events == ps->fds_status[fd].used_events -#if ERTS_POLL_USE_KERNEL_POLL || defined(ERTS_SMP) && !(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_RST) -#endif ) { *do_wake = 0; goto done; } -#if !ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE - if (update_pollset(ps, fd) != 0) - new_events = ERTS_POLL_EV_ERR; -#else /* ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE */ #if ERTS_POLL_USE_CONCURRENT_UPDATE if (ERTS_POLLSET_IS_POLLED(ps)) { @@ -1652,7 +1563,6 @@ poll_control(ErtsPollSet ps, int fd, ErtsPollEvents events, int on, int *do_wake enqueue_update_request(ps, fd); -#ifdef ERTS_SMP /* * If new events have been added, we need to wake up the * polling thread, but if events have been removed we don't. @@ -1660,9 +1570,7 @@ poll_control(ErtsPollSet ps, int fd, ErtsPollEvents events, int on, int *do_wake if ((new_events && (ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_RST)) || (~ps->fds_status[fd].used_events & new_events)) *do_wake = 1; -#endif /* ERTS_SMP */ -#endif /* ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE */ done: #ifdef ERTS_POLL_DEBUG_PRINT @@ -1695,10 +1603,8 @@ ERTS_POLL_EXPORT(erts_poll_controlv)(ErtsPollSet ps, ERTS_POLLSET_UNLOCK(ps); -#ifdef ERTS_SMP if (final_do_wake) wake_poller(ps, 0, 0); -#endif /* ERTS_SMP */ } @@ -1718,11 +1624,9 @@ ERTS_POLL_EXPORT(erts_poll_control)(ErtsPollSet ps, ERTS_POLLSET_UNLOCK(ps); -#ifdef ERTS_SMP if (*do_wake) { wake_poller(ps, 0, 0); } -#endif /* ERTS_SMP */ return res; } @@ -1739,9 +1643,7 @@ save_kp_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res, int chk_fds_res) int res = 0; int i; int n = chk_fds_res < max_res ? chk_fds_res : max_res; -#if ERTS_POLL_USE_WAKEUP_PIPE int wake_fd = ps->wake_fds[0]; -#endif #if ERTS_POLL_USE_TIMERFD int timer_fd = ps->timer_fd; #endif @@ -1754,12 +1656,10 @@ save_kp_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res, int chk_fds_res) int fd = ps->res_events[i].data.fd; int ix; ErtsPollEvents revents; -#if ERTS_POLL_USE_WAKEUP_PIPE if (fd == wake_fd) { cleanup_wakeup_pipe(ps); continue; } -#endif #if ERTS_POLL_USE_TIMERFD if (fd == timer_fd) { continue; @@ -1805,12 +1705,10 @@ save_kp_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res, int chk_fds_res) } if (ev->filter == EVFILT_READ) { -#if ERTS_POLL_USE_WAKEUP_PIPE if (fd == wake_fd) { cleanup_wakeup_pipe(ps); continue; } -#endif pr[ix].events |= ERTS_POLL_EV_IN; } else if (ev->filter == EVFILT_WRITE) @@ -1833,12 +1731,10 @@ save_kp_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res, int chk_fds_res) if (ps->res_events[i].revents) { int fd = ps->res_events[i].fd; ErtsPollEvents revents; -#if ERTS_POLL_USE_WAKEUP_PIPE if (fd == wake_fd) { cleanup_wakeup_pipe(ps); continue; } -#endif #if ERTS_POLL_USE_TIMERFD if (fd == timer_fd) { continue; @@ -1913,7 +1809,7 @@ save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res, #if ERTS_POLL_USE_POLL /* --- poll -------------------------------- */ int res = 0; -#if ERTS_POLL_USE_WAKEUP_PIPE && !ERTS_POLL_USE_FALLBACK +#if !ERTS_POLL_USE_FALLBACK int wake_fd = ps->wake_fds[0]; #endif int i, first_ix, end_ix; @@ -1938,7 +1834,7 @@ save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res, i++; continue; } -#elif ERTS_POLL_USE_WAKEUP_PIPE +#else if (fd == wake_fd) { cleanup_wakeup_pipe(ps); i++; @@ -1964,7 +1860,7 @@ save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res, #elif ERTS_POLL_USE_SELECT /* --- select ------------------------------ */ int res = 0; -#if ERTS_POLL_USE_WAKEUP_PIPE && !ERTS_POLL_USE_FALLBACK +#if !ERTS_POLL_USE_FALLBACK int wake_fd = ps->wake_fds[0]; #endif int fd, first_fd, end_fd; @@ -1988,7 +1884,7 @@ save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res, fd++; continue; } -#elif ERTS_POLL_USE_WAKEUP_PIPE +#else if (fd == wake_fd) { cleanup_wakeup_pipe(ps); fd++; @@ -2051,7 +1947,7 @@ save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res, fd++; continue; } -#elif ERTS_POLL_USE_WAKEUP_PIPE +#else if (fd == wake_fd) { cleanup_wakeup_pipe(ps); fd++; @@ -2073,7 +1969,7 @@ save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res, fd++; continue; } -#elif ERTS_POLL_USE_WAKEUP_PIPE +#else if (fd == wake_fd) { cleanup_wakeup_pipe(ps); fd++; @@ -2243,7 +2139,7 @@ check_fd_events(ErtsPollSet ps, ErtsMonotonicTime timeout_time, int max_res) { int res; ERTS_MSACC_PUSH_STATE_M(); - if (erts_smp_atomic_read_nob(&ps->no_of_user_fds) == 0 + if (erts_atomic_read_nob(&ps->no_of_user_fds) == 0 && timeout_time == ERTS_POLL_NO_TIMEOUT) { /* Nothing to poll and zero timeout; done... */ return 0; @@ -2261,9 +2157,7 @@ check_fd_events(ErtsPollSet ps, ErtsMonotonicTime timeout_time, int max_res) struct itimerspec its; timeout = get_timeout_itimerspec(ps, &its, timeout_time); if (timeout) { -#ifdef ERTS_SMP erts_thr_progress_prepare_wait(NULL); -#endif ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP); timerfd_set(ps, &its); res = epoll_wait(ps->kp_fd, ps->res_events, max_res, -1); @@ -2275,9 +2169,7 @@ check_fd_events(ErtsPollSet ps, ErtsMonotonicTime timeout_time, int max_res) #else /* !ERTS_POLL_USE_TIMERFD */ timeout = (int) get_timeout(ps, 1000, timeout_time); if (timeout) { -#ifdef ERTS_SMP erts_thr_progress_prepare_wait(NULL); -#endif ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP); } res = epoll_wait(ps->kp_fd, ps->res_events, max_res, timeout); @@ -2288,9 +2180,7 @@ check_fd_events(ErtsPollSet ps, ErtsMonotonicTime timeout_time, int max_res) grow_res_events(ps, max_res); timeout = get_timeout_timespec(ps, &ts, timeout_time); if (timeout) { -#ifdef ERTS_SMP erts_thr_progress_prepare_wait(NULL); -#endif ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP); } res = kevent(ps->kp_fd, NULL, 0, ps->res_events, max_res, &ts); @@ -2307,19 +2197,15 @@ check_fd_events(ErtsPollSet ps, ErtsMonotonicTime timeout_time, int max_res) * the maximum number of file descriptors in the poll set. */ struct dvpoll poll_res; - int nfds = (int) erts_smp_atomic_read_nob(&ps->no_of_user_fds); -#if ERTS_POLL_USE_WAKEUP_PIPE + int nfds = (int) erts_atomic_read_nob(&ps->no_of_user_fds); nfds++; /* Wakeup pipe */ -#endif timeout = (int) get_timeout(ps, 1000, timeout_time); poll_res.dp_nfds = nfds < max_res ? nfds : max_res; if (poll_res.dp_nfds > ps->res_events_len) grow_res_events(ps, poll_res.dp_nfds); poll_res.dp_fds = ps->res_events; if (timeout) { -#ifdef ERTS_SMP erts_thr_progress_prepare_wait(NULL); -#endif ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP); } poll_res.dp_timeout = timeout; @@ -2328,9 +2214,7 @@ check_fd_events(ErtsPollSet ps, ErtsMonotonicTime timeout_time, int max_res) struct timespec ts; timeout = get_timeout_timespec(ps, &ts, timeout_time); if (timeout) { -#ifdef ERTS_SMP erts_thr_progress_prepare_wait(NULL); -#endif ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP); } res = ppoll(ps->poll_fds, ps->no_poll_fds, &ts, NULL); @@ -2338,9 +2222,7 @@ check_fd_events(ErtsPollSet ps, ErtsMonotonicTime timeout_time, int max_res) timeout = (int) get_timeout(ps, 1000, timeout_time); if (timeout) { -#ifdef ERTS_SMP erts_thr_progress_prepare_wait(NULL); -#endif ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP); } res = poll(ps->poll_fds, ps->no_poll_fds, timeout); @@ -2352,9 +2234,7 @@ check_fd_events(ErtsPollSet ps, ErtsMonotonicTime timeout_time, int max_res) ERTS_FD_COPY(&ps->output_fds, &ps->res_output_fds); if (timeout) { -#ifdef ERTS_SMP erts_thr_progress_prepare_wait(NULL); -#endif ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP); } res = ERTS_SELECT(ps->max_fd + 1, @@ -2362,7 +2242,6 @@ check_fd_events(ErtsPollSet ps, ErtsMonotonicTime timeout_time, int max_res) &ps->res_output_fds, NULL, &to); -#ifdef ERTS_SMP if (timeout) { erts_thr_progress_finalize_wait(NULL); ERTS_MSACC_POP_STATE_M(); @@ -2397,14 +2276,11 @@ check_fd_events(ErtsPollSet ps, ErtsMonotonicTime timeout_time, int max_res) res = -1; } } -#endif /* ERTS_SMP */ return res; #endif /* ----------------------------------------- */ } if (timeout) { -#ifdef ERTS_SMP erts_thr_progress_finalize_wait(NULL); -#endif ERTS_MSACC_POP_STATE_M(); } return res; @@ -2420,9 +2296,7 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps, ErtsMonotonicTime to; int res, no_fds; int ebadf = 0; -#ifdef ERTS_SMP int ps_locked = 0; -#endif no_fds = *len; #ifdef ERTS_POLL_MAX_RES @@ -2447,13 +2321,11 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps, ? ERTS_POLL_NO_TIMEOUT /* Use zero timeout */ : timeout_time); -#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE if (ERTS_POLLSET_HAVE_UPDATE_REQUESTS(ps)) { ERTS_POLLSET_LOCK(ps); handle_update_requests(ps); ERTS_POLLSET_UNLOCK(ps); } -#endif while (1) { res = check_fd_events(ps, to, no_fds); @@ -2484,10 +2356,8 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps, save_results: #endif -#ifdef ERTS_SMP ps_locked = 1; ERTS_POLLSET_LOCK(ps); -#endif no_fds = save_poll_result(ps, pr, no_fds, res, ebadf); @@ -2499,11 +2369,9 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps, *len = no_fds; } -#ifdef ERTS_SMP if (ps_locked) ERTS_POLLSET_UNLOCK(ps); ERTS_POLLSET_UNSET_POLLED(ps); -#endif done: set_timeout_time(ps, ERTS_MONOTONIC_TIME_MAX); @@ -2522,25 +2390,12 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps, void ERTS_POLL_EXPORT(erts_poll_interrupt)(ErtsPollSet ps, int set) { -#if defined(USE_THREADS) || ERTS_POLL_ASYNC_INTERRUPT_SUPPORT if (!set) reset_wakeup_state(ps); else wake_poller(ps, 1, 0); -#endif } -#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT -void -ERTS_POLL_EXPORT(erts_poll_async_sig_interrupt)(ErtsPollSet ps) -{ - /* - * NOTE: This function is called from signal handlers, it, - * therefore, it has to be async-signal safe. - */ - wake_poller(ps, 1, 1); -} -#endif /* * erts_poll_interrupt_timed(): @@ -2552,7 +2407,6 @@ ERTS_POLL_EXPORT(erts_poll_interrupt_timed)(ErtsPollSet ps, int set, ErtsMonotonicTime timeout_time) { -#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT || defined(ERTS_SMP) if (!set) reset_wakeup_state(ps); else { @@ -2562,13 +2416,12 @@ ERTS_POLL_EXPORT(erts_poll_interrupt_timed)(ErtsPollSet ps, #ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS else { if (ERTS_POLLSET_IS_POLLED(ps)) - erts_smp_atomic_inc_nob(&ps->no_avoided_wakeups); - erts_smp_atomic_inc_nob(&ps->no_avoided_interrupts); + erts_atomic_inc_nob(&ps->no_avoided_wakeups); + erts_atomic_inc_nob(&ps->no_avoided_interrupts); } - erts_smp_atomic_inc_nob(&ps->no_interrupt_timed); + erts_atomic_inc_nob(&ps->no_interrupt_timed); #endif } -#endif } int @@ -2583,7 +2436,8 @@ ERTS_POLL_EXPORT(erts_poll_max_fds)(void) void ERTS_POLL_EXPORT(erts_poll_init)(void) { - erts_smp_spinlock_init(&pollsets_lock, "pollsets_lock"); + erts_mtx_init(&pollsets_lock, "pollsets_lock", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO); pollsets = NULL; errno = 0; @@ -2622,7 +2476,7 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void) ps->internal_fd_limit = 0; ps->fds_status = NULL; ps->fds_status_len = 0; - erts_smp_atomic_init_nob(&ps->no_of_user_fds, 0); + erts_atomic_init_nob(&ps->no_of_user_fds, 0); #if ERTS_POLL_USE_KERNEL_POLL ps->kp_fd = -1; #if ERTS_POLL_USE_EPOLL @@ -2681,22 +2535,14 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void) ERTS_FD_ZERO(&ps->res_output_fds); #endif #endif -#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE ps->update_requests.next = NULL; ps->update_requests.len = 0; ps->curr_upd_req_block = &ps->update_requests; - erts_smp_atomic32_init_nob(&ps->have_update_requests, 0); -#endif -#ifdef ERTS_SMP + erts_atomic32_init_nob(&ps->have_update_requests, 0); erts_atomic32_init_nob(&ps->polled, 0); - erts_smp_mtx_init(&ps->mtx, "pollset"); -#endif -#if defined(USE_THREADS) || ERTS_POLL_ASYNC_INTERRUPT_SUPPORT + erts_mtx_init(&ps->mtx, "pollset", NIL, ERTS_LOCK_FLAGS_CATEGORY_IO); erts_atomic32_init_nob(&ps->wakeup_state, (erts_aint32_t) 0); -#endif -#if ERTS_POLL_USE_WAKEUP_PIPE create_wakeup_pipe(ps); -#endif #if ERTS_POLL_USE_TIMERFD create_timerfd(ps); #endif @@ -2719,22 +2565,20 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void) #endif init_timeout_time(ps); #ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS - erts_smp_atomic_init_nob(&ps->no_avoided_wakeups, 0); - erts_smp_atomic_init_nob(&ps->no_avoided_interrupts, 0); - erts_smp_atomic_init_nob(&ps->no_interrupt_timed, 0); + erts_atomic_init_nob(&ps->no_avoided_wakeups, 0); + erts_atomic_init_nob(&ps->no_avoided_interrupts, 0); + erts_atomic_init_nob(&ps->no_interrupt_timed, 0); #endif -#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE handle_update_requests(ps); -#endif #if ERTS_POLL_USE_FALLBACK ps->fallback_used = 0; #endif - erts_smp_atomic_set_nob(&ps->no_of_user_fds, 0); /* Don't count wakeup pipe and fallback fd */ + erts_atomic_set_nob(&ps->no_of_user_fds, 0); /* Don't count wakeup pipe and fallback fd */ - erts_smp_spin_lock(&pollsets_lock); + erts_mtx_lock(&pollsets_lock); ps->next = pollsets; pollsets = ps; - erts_smp_spin_unlock(&pollsets_lock); + erts_mtx_unlock(&pollsets_lock); return ps; } @@ -2771,7 +2615,6 @@ ERTS_POLL_EXPORT(erts_poll_destroy_pollset)(ErtsPollSet ps) erts_free(ERTS_ALC_T_SELECT_FDS, (void *) ps->res_output_fds.ptr); #endif #endif -#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE { ErtsPollSetUpdateRequestsBlock *urqbp = ps->update_requests.next; while (urqbp) { @@ -2780,22 +2623,17 @@ ERTS_POLL_EXPORT(erts_poll_destroy_pollset)(ErtsPollSet ps) free_update_requests_block(ps, free_urqbp); } } -#endif -#ifdef ERTS_SMP - erts_smp_mtx_destroy(&ps->mtx); -#endif -#if ERTS_POLL_USE_WAKEUP_PIPE + erts_mtx_destroy(&ps->mtx); if (ps->wake_fds[0] >= 0) close(ps->wake_fds[0]); if (ps->wake_fds[1] >= 0) close(ps->wake_fds[1]); -#endif #if ERTS_POLL_USE_TIMERFD if (ps->timer_fd >= 0) close(ps->timer_fd); #endif - erts_smp_spin_lock(&pollsets_lock); + erts_mtx_lock(&pollsets_lock); if (ps == pollsets) pollsets = pollsets->next; else { @@ -2805,7 +2643,7 @@ ERTS_POLL_EXPORT(erts_poll_destroy_pollset)(ErtsPollSet ps) ASSERT(ps == prev_ps->next); prev_ps->next = ps->next; } - erts_smp_spin_unlock(&pollsets_lock); + erts_mtx_unlock(&pollsets_lock); erts_free(ERTS_ALC_T_POLLSET, (void *) ps); } @@ -2817,9 +2655,7 @@ ERTS_POLL_EXPORT(erts_poll_destroy_pollset)(ErtsPollSet ps) void ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip) { -#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE int pending_updates; -#endif Uint size = 0; ERTS_POLLSET_LOCK(ps); @@ -2844,7 +2680,6 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip) #endif #endif -#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE { ErtsPollSetUpdateRequestsBlock *urqbp = ps->update_requests.next; pending_updates = ps->update_requests.len; @@ -2854,7 +2689,6 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip) urqbp = urqbp->next; } } -#endif pip->primary = #if ERTS_POLL_USE_KQUEUE @@ -2894,10 +2728,8 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip) pip->memory_size = size; - pip->poll_set_size = (int) erts_smp_atomic_read_nob(&ps->no_of_user_fds); -#if ERTS_POLL_USE_WAKEUP_PIPE + pip->poll_set_size = (int) erts_atomic_read_nob(&ps->no_of_user_fds); pip->poll_set_size++; /* Wakeup pipe */ -#endif #if ERTS_POLL_USE_TIMERFD pip->poll_set_size++; /* timerfd */ #endif @@ -2921,19 +2753,11 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip) #endif pip->lazy_updates = -#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE 1 -#else - 0 -#endif ; pip->pending_updates = -#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE pending_updates -#else - 0 -#endif ; pip->batch_updates = @@ -2955,9 +2779,9 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip) pip->max_fds = max_fds; #ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS - pip->no_avoided_wakeups = erts_smp_atomic_read_nob(&ps->no_avoided_wakeups); - pip->no_avoided_interrupts = erts_smp_atomic_read_nob(&ps->no_avoided_interrupts); - pip->no_interrupt_timed = erts_smp_atomic_read_nob(&ps->no_interrupt_timed); + pip->no_avoided_wakeups = erts_atomic_read_nob(&ps->no_avoided_wakeups); + pip->no_avoided_interrupts = erts_atomic_read_nob(&ps->no_avoided_interrupts); + pip->no_interrupt_timed = erts_atomic_read_nob(&ps->no_interrupt_timed); #endif ERTS_POLLSET_UNLOCK(ps); @@ -3030,9 +2854,7 @@ ERTS_POLL_EXPORT(erts_poll_get_selected_events)(ErtsPollSet ps, else { ev[fd] = ps->fds_status[fd].events; if ( -#if ERTS_POLL_USE_WAKEUP_PIPE fd == ps->wake_fds[0] || fd == ps->wake_fds[1] || -#endif #if ERTS_POLL_USE_TIMERFD fd == ps->timer_fd || #endif @@ -3120,11 +2942,7 @@ print_misc_debug_info(void) "select" #endif , -#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE "true" -#else - "false" -#endif , #if ERTS_POLL_USE_BATCH_UPDATE_POLLSET "true" @@ -3148,3 +2966,26 @@ print_misc_debug_info(void) } #endif + +#ifdef ERTS_ENABLE_LOCK_COUNT +static void erts_lcnt_enable_pollset_lock_count(ErtsPollSet pollset, int enable) { + if(enable) { + erts_lcnt_install_new_lock_info(&pollset->mtx.lcnt, "pollset_rm", NIL, + ERTS_LOCK_TYPE_MUTEX | ERTS_LOCK_FLAGS_CATEGORY_IO); + } else { + erts_lcnt_uninstall(&pollset->mtx.lcnt); + } +} + +void ERTS_POLL_EXPORT(erts_lcnt_update_pollset_locks)(int enable) { + ErtsPollSet iterator; + + erts_mtx_lock(&pollsets_lock); + + for(iterator = pollsets; iterator != NULL; iterator = iterator->next) { + erts_lcnt_enable_pollset_lock_count(iterator, enable); + } + + erts_mtx_unlock(&pollsets_lock); +} +#endif diff --git a/erts/emulator/sys/common/erl_poll.h b/erts/emulator/sys/common/erl_poll.h index c16122610d..a57dc51e5b 100644 --- a/erts/emulator/sys/common/erl_poll.h +++ b/erts/emulator/sys/common/erl_poll.h @@ -227,9 +227,6 @@ typedef struct { #endif } ErtsPollInfo; -#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT -void ERTS_POLL_EXPORT(erts_poll_async_sig_interrupt)(ErtsPollSet); -#endif void ERTS_POLL_EXPORT(erts_poll_interrupt)(ErtsPollSet, int); void ERTS_POLL_EXPORT(erts_poll_interrupt_timed)(ErtsPollSet, @@ -260,4 +257,8 @@ void ERTS_POLL_EXPORT(erts_poll_get_selected_events)(ErtsPollSet, int erts_poll_new_table_len(int old_len, int need_len); +#ifdef ERTS_ENABLE_LOCK_COUNT +void ERTS_POLL_EXPORT(erts_lcnt_update_pollset_locks)(int enable); +#endif + #endif /* #ifndef ERL_POLL_H__ */ diff --git a/erts/emulator/sys/common/erl_sys_common_misc.c b/erts/emulator/sys/common/erl_sys_common_misc.c index 79f87eb3a9..09237c81ce 100644 --- a/erts/emulator/sys/common/erl_sys_common_misc.c +++ b/erts/emulator/sys/common/erl_sys_common_misc.c @@ -51,7 +51,7 @@ * (often) exist two versions of erl_check_io (kernel-poll and * non-kernel-poll), and we dont want two versions of this variable. */ -erts_smp_atomic_t erts_check_io_time; +erts_atomic_t erts_check_io_time; /* Written once and only once */ diff --git a/erts/emulator/sys/unix/erl_unix_sys.h b/erts/emulator/sys/unix/erl_unix_sys.h index 22059d21d5..cbbd0e6f40 100644 --- a/erts/emulator/sys/unix/erl_unix_sys.h +++ b/erts/emulator/sys/unix/erl_unix_sys.h @@ -128,10 +128,6 @@ /* File descriptors are numbers anc consecutively allocated on Unix */ #define ERTS_SYS_CONTINOUS_FD_NUMBERS -#ifndef ERTS_SMP -# undef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT -# define ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT -#endif typedef void *GETENV_STATE; @@ -354,9 +350,7 @@ extern void erts_sys_unix_later_init(void); #ifdef NO_FPE_SIGNALS #define erts_get_current_fp_exception() NULL -#ifdef ERTS_SMP #define erts_thread_init_fp_exception() do{}while(0) -#endif # define __ERTS_FP_CHECK_INIT(fpexnp) do {} while (0) # define __ERTS_FP_ERROR(fpexnp, f, Action) if (!isfinite(f)) { Action; } else {} # define __ERTS_FP_ERROR_THOROUGH(fpexnp, f, Action) __ERTS_FP_ERROR(fpexnp, f, Action) @@ -369,9 +363,7 @@ extern void erts_sys_unix_later_init(void); #else /* !NO_FPE_SIGNALS */ extern volatile unsigned long *erts_get_current_fp_exception(void); -#ifdef ERTS_SMP extern void erts_thread_init_fp_exception(void); -#endif # if (defined(__i386__) || defined(__x86_64__)) && defined(__GNUC__) # define erts_fwait(fpexnp,f) \ __asm__ __volatile__("fwait" : "=m"(*(fpexnp)) : "m"(f)) @@ -438,10 +430,8 @@ void erts_sys_unblock_fpe(int); /* Threads */ -#ifdef USE_THREADS extern int init_async(int); extern int exit_async(void); -#endif #define ERTS_EXIT_AFTER_DUMP _exit diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c index 5cf0a49972..237614b0fb 100644 --- a/erts/emulator/sys/unix/sys.c +++ b/erts/emulator/sys/unix/sys.c @@ -58,14 +58,12 @@ #define __DARWIN__ 1 #endif -#ifdef USE_THREADS #include "erl_threads.h" -#endif #include "erl_mseg.h" extern char **environ; -erts_smp_rwmtx_t environ_rwmtx; +erts_rwmtx_t environ_rwmtx; #define MAX_VSIZE 16 /* Max number of entries allowed in an I/O * vector sock_sendv(). @@ -94,19 +92,12 @@ extern void erts_sys_init_float(void); static int debug_log = 0; #endif -#ifdef ERTS_SMP -static erts_smp_atomic32_t have_prepared_crash_dump; -#define ERTS_PREPARED_CRASH_DUMP \ - ((int) erts_smp_atomic32_xchg_nob(&have_prepared_crash_dump, 1)) -#else -static volatile int have_prepared_crash_dump; +static erts_atomic32_t have_prepared_crash_dump; #define ERTS_PREPARED_CRASH_DUMP \ - (have_prepared_crash_dump++) -#endif + ((int) erts_atomic32_xchg_nob(&have_prepared_crash_dump, 1)) -erts_smp_atomic_t sys_misc_mem_sz; +erts_atomic_t sys_misc_mem_sz; -#if defined(ERTS_SMP) static void smp_sig_notify(int signum); static int sig_notify_fds[2] = {-1, -1}; @@ -114,7 +105,6 @@ static int sig_notify_fds[2] = {-1, -1}; static int sig_suspend_fds[2] = {-1, -1}; #endif -#endif jmp_buf erts_sys_sigsegv_jmp; @@ -128,38 +118,12 @@ static int max_files = -1; /* * a few variables used by the break handler */ -#ifdef ERTS_SMP -erts_smp_atomic32_t erts_break_requested; +erts_atomic32_t erts_break_requested; #define ERTS_SET_BREAK_REQUESTED \ - erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 1) + erts_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 1) #define ERTS_UNSET_BREAK_REQUESTED \ - erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 0) -#else -volatile int erts_break_requested = 0; -#define ERTS_SET_BREAK_REQUESTED (erts_break_requested = 1) -#define ERTS_UNSET_BREAK_REQUESTED (erts_break_requested = 0) -#endif + erts_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 0) -#ifndef ERTS_SMP -static Eterm signalstate_sigterm[] = { - am_sigint, /* 0 */ - am_sighup, /* 1 */ - am_sigquit, /* 2 */ - am_sigabrt, /* 3 */ - am_sigalrm, /* 4 */ - am_sigterm, /* 5 */ - am_sigusr1, /* 6 */ - am_sigusr2, /* 7 */ - am_sigchld, /* 8 */ - am_sigstop, /* 9 */ - am_sigtstp /* 10 */ -}; - -volatile Uint erts_signal_state = 0; -#define ERTS_SET_SIGNAL_STATE(S) (erts_signal_state |= signum_to_signalstate(S)) -#define ERTS_CLEAR_SIGNAL_STATE (erts_signal_state = 0) -static ERTS_INLINE Uint signum_to_signalstate(int signum); -#endif /* set early so the break handler has access to initial mode */ static struct termios initial_tty_mode; @@ -223,9 +187,6 @@ init_check_io(void) io_func.select = driver_select_kp; io_func.enif_select = enif_select_kp; io_func.event = driver_event_kp; -#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT - io_func.check_io_as_interrupt = erts_check_io_async_sig_interrupt_kp; -#endif io_func.check_io_interrupt = erts_check_io_interrupt_kp; io_func.check_io_interrupt_tmd = erts_check_io_interrupt_timed_kp; io_func.check_io = erts_check_io_kp; @@ -239,9 +200,6 @@ init_check_io(void) io_func.select = driver_select_nkp; io_func.enif_select = enif_select_nkp; io_func.event = driver_event_nkp; -#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT - io_func.check_io_as_interrupt = erts_check_io_async_sig_interrupt_nkp; -#endif io_func.check_io_interrupt = erts_check_io_interrupt_nkp; io_func.check_io_interrupt_tmd = erts_check_io_interrupt_timed_nkp; io_func.check_io = erts_check_io_nkp; @@ -253,11 +211,7 @@ init_check_io(void) } } -#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT -#define ERTS_CHK_IO_AS_INTR() (*io_func.check_io_as_interrupt)() -#else #define ERTS_CHK_IO_AS_INTR() (*io_func.check_io_interrupt)(1) -#endif #define ERTS_CHK_IO_INTR (*io_func.check_io_interrupt) #define ERTS_CHK_IO_INTR_TMD (*io_func.check_io_interrupt_tmd) #define ERTS_CHK_IO (*io_func.check_io) @@ -272,11 +226,7 @@ init_check_io(void) max_files = erts_check_io_max_files(); } -#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT -#define ERTS_CHK_IO_AS_INTR() erts_check_io_async_sig_interrupt() -#else #define ERTS_CHK_IO_AS_INTR() erts_check_io_interrupt(1) -#endif #define ERTS_CHK_IO_INTR erts_check_io_interrupt #define ERTS_CHK_IO_INTR_TMD erts_check_io_interrupt_timed #define ERTS_CHK_IO erts_check_io @@ -290,13 +240,11 @@ erts_sys_schedule_interrupt(int set) ERTS_CHK_IO_INTR(set); } -#ifdef ERTS_SMP void erts_sys_schedule_interrupt_timed(int set, ErtsMonotonicTime timeout_time) { ERTS_CHK_IO_INTR_TMD(set, timeout_time); } -#endif UWord erts_sys_get_page_size(void) @@ -314,7 +262,7 @@ Uint erts_sys_misc_mem_sz(void) { Uint res = ERTS_CHK_IO_SZ(); - res += erts_smp_atomic_read_mb(&sys_misc_mem_sz); + res += erts_atomic_read_mb(&sys_misc_mem_sz); return res; } @@ -339,7 +287,6 @@ MALLOC_USE_HASH(1); #endif #endif -#ifdef USE_THREADS #ifdef ERTS_THR_HAVE_SIG_FUNCS @@ -418,19 +365,15 @@ thr_create_prepare_child(void *vtcdp) erts_sched_bind_atthrcreate_child(tcdp->sched_bind_data); } -#endif /* #ifdef USE_THREADS */ void erts_sys_pre_init(void) { -#ifdef USE_THREADS erts_thr_init_data_t eid = ERTS_THR_INIT_DATA_DEF_INITER; -#endif erts_printf_add_cr_to_stdout = 1; erts_printf_add_cr_to_stderr = 1; -#ifdef USE_THREADS eid.thread_create_child_func = thr_create_prepare_child; /* Before creation in parent */ @@ -438,33 +381,29 @@ erts_sys_pre_init(void) /* After creation in parent */ eid.thread_create_parent_func = thr_create_cleanup, +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_pre_thr_init(); +#endif + erts_thr_init(&eid); -#ifdef ERTS_ENABLE_LOCK_CHECK - erts_lc_init(); +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_post_thr_init(); #endif -#ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_init(); +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_lc_init(); #endif -#endif /* USE_THREADS */ erts_init_sys_time_sup(); -#ifdef USE_THREADS -#ifdef ERTS_SMP - erts_smp_atomic32_init_nob(&erts_break_requested, 0); - erts_smp_atomic32_init_nob(&have_prepared_crash_dump, 0); -#else - erts_break_requested = 0; - have_prepared_crash_dump = 0; -#endif + erts_atomic32_init_nob(&erts_break_requested, 0); + erts_atomic32_init_nob(&have_prepared_crash_dump, 0); -#endif /* USE_THREADS */ - erts_smp_atomic_init_nob(&sys_misc_mem_sz, 0); + erts_atomic_init_nob(&sys_misc_mem_sz, 0); { /* @@ -527,10 +466,8 @@ SIGFUNC sys_signal(int sig, SIGFUNC func) return(oact.sa_handler); } -#ifdef USE_THREADS #undef sigprocmask #define sigprocmask erts_thr_sigmask -#endif void sys_sigblock(int sig) { @@ -668,7 +605,7 @@ static void signal_notify_requested(Eterm type) { erts_queue_message(p, locks, msgp, msg, am_system); if (locks) - erts_smp_proc_unlock(p, locks); + erts_proc_unlock(p, locks); erts_proc_dec_refc(p); } } @@ -693,11 +630,7 @@ break_requested(void) static RETSIGTYPE request_break(int signum) { -#ifdef ERTS_SMP smp_sig_notify(signum); -#else - break_requested(); -#endif } #ifdef ETHR_UNUSABLE_SIGUSRX @@ -806,35 +739,10 @@ signum_to_signalterm(int signum) } } -#ifndef ERTS_SMP -static ERTS_INLINE Uint -signum_to_signalstate(int signum) -{ - switch (signum) { - case SIGINT: return (1 << 0); - case SIGHUP: return (1 << 1); - case SIGQUIT: return (1 << 2); - case SIGABRT: return (1 << 3); - case SIGALRM: return (1 << 4); - case SIGTERM: return (1 << 5); - case SIGUSR1: return (1 << 6); - case SIGUSR2: return (1 << 7); - case SIGCHLD: return (1 << 8); - case SIGSTOP: return (1 << 9); - case SIGTSTP: return (1 << 10); - default: return 0; - } -} -#endif static RETSIGTYPE generic_signal_handler(int signum) { -#ifdef ERTS_SMP smp_sig_notify(signum); -#else - ERTS_SET_SIGNAL_STATE(signum); - ERTS_CHK_IO_AS_INTR(); /* Make sure we don't sleep in poll */ -#endif } int erts_set_signal(Eterm signal, Eterm type) { @@ -961,7 +869,7 @@ void os_version(int *pMajor, int *pMinor, int *pBuild) { void init_getenv_state(GETENV_STATE *state) { - erts_smp_rwmtx_rlock(&environ_rwmtx); + erts_rwmtx_rlock(&environ_rwmtx); *state = NULL; } @@ -970,7 +878,7 @@ char *getenv_string(GETENV_STATE *state0) char **state = (char **) *state0; char *cp; - ERTS_SMP_LC_ASSERT(erts_smp_lc_rwmtx_is_rlocked(&environ_rwmtx)); + ERTS_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&environ_rwmtx)); if (state == NULL) state = environ; @@ -984,7 +892,7 @@ char *getenv_string(GETENV_STATE *state0) void fini_getenv_state(GETENV_STATE *state) { *state = NULL; - erts_smp_rwmtx_runlock(&environ_rwmtx); + erts_rwmtx_runlock(&environ_rwmtx); } void erts_do_break_handling(void) @@ -997,7 +905,7 @@ void erts_do_break_handling(void) * therefore, make sure that all threads but this one are blocked before * proceeding! */ - erts_smp_thr_progress_block(); + erts_thr_progress_block(); /* during break we revert to initial settings */ /* this is done differently for oldshell */ @@ -1025,25 +933,9 @@ void erts_do_break_handling(void) tcsetattr(0,TCSANOW,&temp_mode); } - erts_smp_thr_progress_unblock(); + erts_thr_progress_unblock(); } -#ifdef ERTS_SIGNAL_STATE -void erts_handle_signal_state(void) { - Uint signal_state = ERTS_SIGNAL_STATE; - Uint i = 0; - - ERTS_CLEAR_SIGNAL_STATE; - - while (signal_state) { - if (signal_state & 0x1) { - signal_notify_requested(signalstate_sigterm[i]); - } - i++; - signal_state = signal_state >> 1; - } -} -#endif /* Fills in the systems representation of the jam/beam process identifier. ** The Pid is put in STRING representation in the supplied buffer, @@ -1071,14 +963,14 @@ erts_sys_putenv(char *key, char *value) env = erts_alloc(ERTS_ALC_T_TMP, need); #else env = erts_alloc(ERTS_ALC_T_PUTENV_STR, need); - erts_smp_atomic_add_nob(&sys_misc_mem_sz, need); + erts_atomic_add_nob(&sys_misc_mem_sz, need); #endif strcpy(env,key); strcat(env,"="); strcat(env,value); - erts_smp_rwmtx_rwlock(&environ_rwmtx); + erts_rwmtx_rwlock(&environ_rwmtx); res = putenv(env); - erts_smp_rwmtx_rwunlock(&environ_rwmtx); + erts_rwmtx_rwunlock(&environ_rwmtx); #ifdef HAVE_COPYING_PUTENV erts_free(ERTS_ALC_T_TMP, env); #endif @@ -1125,9 +1017,9 @@ int erts_sys_getenv(char *key, char *value, size_t *size) { int res; - erts_smp_rwmtx_rlock(&environ_rwmtx); + erts_rwmtx_rlock(&environ_rwmtx); res = erts_sys_getenv__(key, value, size); - erts_smp_rwmtx_runlock(&environ_rwmtx); + erts_rwmtx_runlock(&environ_rwmtx); return res; } @@ -1135,9 +1027,9 @@ int erts_sys_unsetenv(char *key) { int res; - erts_smp_rwmtx_rwlock(&environ_rwmtx); + erts_rwmtx_rwlock(&environ_rwmtx); res = unsetenv(key); - erts_smp_rwmtx_rwunlock(&environ_rwmtx); + erts_rwmtx_rwunlock(&environ_rwmtx); return res; } @@ -1278,16 +1170,6 @@ erl_assert_error(const char* expr, const char* func, const char* file, int line) fprintf(stderr, "%s:%d:%s() Assertion failed: %s\n", file, line, func, expr); fflush(stderr); -#if !defined(ERTS_SMP) && 0 - /* Writing a crashdump from a failed assertion when smp support - * is enabled almost a guaranteed deadlocking, don't even bother. - * - * It could maybe be useful (but I'm not convinced) to write the - * crashdump if smp support is disabled... - */ - if (erts_initialized) - erl_crash_dump(file, line, "Assertion failed: %s\n", expr); -#endif abort(); } @@ -1318,13 +1200,12 @@ void erl_sys_schedule(int runnable) { ERTS_CHK_IO(!runnable); - ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); + ERTS_LC_ASSERT(!erts_thr_progress_is_blocking()); } -#ifdef ERTS_SMP -static erts_smp_tid_t sig_dispatcher_tid; +static erts_tid_t sig_dispatcher_tid; static void smp_sig_notify(int signum) @@ -1398,7 +1279,7 @@ signal_dispatcher_thread_func(void *unused) } signal_notify_requested(signal); } - ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); + ERTS_LC_ASSERT(!erts_thr_progress_is_blocking()); } return NULL; } @@ -1406,7 +1287,7 @@ signal_dispatcher_thread_func(void *unused) static void init_smp_sig_notify(void) { - erts_smp_thr_opts_t thr_opts = ERTS_SMP_THR_OPTS_DEFAULT_INITER; + erts_thr_opts_t thr_opts = ERTS_THR_OPTS_DEFAULT_INITER; thr_opts.detached = 1; thr_opts.name = "sys_sig_dispatcher"; @@ -1418,7 +1299,7 @@ init_smp_sig_notify(void) } /* Start signal handler thread */ - erts_smp_thr_create(&sig_dispatcher_tid, + erts_thr_create(&sig_dispatcher_tid, signal_dispatcher_thread_func, NULL, &thr_opts); @@ -1511,7 +1392,6 @@ erts_sys_main_thread(void) } } -#endif /* ERTS_SMP */ #ifdef ERTS_ENABLE_KERNEL_POLL /* get_value() is currently only used when kernel-poll is enabled */ @@ -1545,7 +1425,8 @@ erl_sys_args(int* argc, char** argv) { int i, j; - erts_smp_rwmtx_init(&environ_rwmtx, "environ"); + erts_rwmtx_init(&environ_rwmtx, "environ", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC); i = 1; @@ -1597,10 +1478,8 @@ erl_sys_args(int* argc, char** argv) init_check_io(); -#ifdef ERTS_SMP init_smp_sig_notify(); init_smp_sig_suspend(); -#endif /* Handled arguments have been marked with NULL. Slide arguments not handled towards the beginning of argv. */ diff --git a/erts/emulator/sys/unix/sys_drivers.c b/erts/emulator/sys/unix/sys_drivers.c index 834706d86f..7c9a532fed 100644 --- a/erts/emulator/sys/unix/sys_drivers.c +++ b/erts/emulator/sys/unix/sys_drivers.c @@ -53,14 +53,12 @@ #define WANT_NONBLOCKING /* must define this to pull in defs from sys.h */ #include "sys.h" -#ifdef USE_THREADS #include "erl_threads.h" -#endif extern char **environ; -extern erts_smp_rwmtx_t environ_rwmtx; +extern erts_rwmtx_t environ_rwmtx; -extern erts_smp_atomic_t sys_misc_mem_sz; +extern erts_atomic_t sys_misc_mem_sz; static Eterm forker_port; @@ -86,12 +84,6 @@ static Eterm forker_port; #define MAXIOV 16 #endif -#ifdef USE_THREADS -# define FDBLOCK 1 -#else -# define FDBLOCK 0 -#endif - /* Used by the fd driver iff the fd could not be set to non-blocking */ typedef struct ErtsSysBlocking_ { ErlDrvPDL pdl; @@ -178,9 +170,7 @@ void erl_sys_late_init(void) { SysDriverOpts opts; -#ifdef ERTS_SMP Port *port; -#endif sys_signal(SIGPIPE, SIG_IGN); /* Ignore - we'll handle the write failure */ @@ -197,13 +187,9 @@ erl_sys_late_init(void) opts.argv = NULL; opts.parallelism = erts_port_parallelism; -#ifdef ERTS_SMP port = -#endif erts_open_driver(&forker_driver, make_internal_pid(0), "forker", &opts, NULL, NULL); -#ifdef ERTS_SMP erts_mtx_unlock(port->lock); -#endif erts_sys_unix_later_init(); /* Need to be called after forker has been started */ } @@ -220,10 +206,8 @@ static ErlDrvData vanilla_start(ErlDrvPort, char*, SysDriverOpts*); /* II.III FD prototypes */ static ErlDrvData fd_start(ErlDrvPort, char*, SysDriverOpts*); -#if FDBLOCK static void fd_async(void *); static void fd_ready_async(ErlDrvData drv_data, ErlDrvThreadData thread_data); -#endif static ErlDrvSSizeT fd_control(ErlDrvData, unsigned int, char *, ErlDrvSizeT, char **, ErlDrvSizeT); static void fd_stop(ErlDrvData); @@ -287,11 +271,7 @@ struct erl_drv_entry fd_driver_entry = { fd_control, NULL, outputv, -#if FDBLOCK fd_ready_async, /* ready_async */ -#else - NULL, -#endif fd_flush, /* flush */ NULL, /* call */ NULL, /* event */ @@ -363,7 +343,7 @@ static int set_blocking_data(ErtsSysDriverData *dd) { dd->blocking = erts_alloc(ERTS_ALC_T_SYS_BLOCKING, sizeof(ErtsSysBlocking)); - erts_smp_atomic_add_nob(&sys_misc_mem_sz, sizeof(ErtsSysBlocking)); + erts_atomic_add_nob(&sys_misc_mem_sz, sizeof(ErtsSysBlocking)); dd->blocking->pdl = driver_pdl_create(dd->port_num); dd->blocking->res = 0; @@ -406,7 +386,7 @@ create_driver_data(ErlDrvPort port_num, size += sizeof(ErtsSysFdData); data = erts_alloc(ERTS_ALC_T_DRV_TAB,size); - erts_smp_atomic_add_nob(&sys_misc_mem_sz, size); + erts_atomic_add_nob(&sys_misc_mem_sz, size); driver_data = (ErtsSysDriverData*)data; data += sizeof(*driver_data); @@ -441,7 +421,7 @@ create_driver_data(ErlDrvPort port_num, data += sizeof(*driver_data->ofd); init_fd_data(driver_data->ofd, ofd); } - if (is_blocking && FDBLOCK) + if (is_blocking) if (!set_blocking_data(driver_data)) { erts_free(ERTS_ALC_T_DRV_TAB, driver_data); return NULL; @@ -472,7 +452,7 @@ static char **build_unix_environment(char *block) char **cpp; char** old_env; - ERTS_SMP_LC_ASSERT(erts_smp_lc_rwmtx_is_rlocked(&environ_rwmtx)); + ERTS_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&environ_rwmtx)); cp = block; len = 0; @@ -620,12 +600,12 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name, len = CMD_LINE_PREFIX_STR_SZ + len + 1; } - erts_smp_rwmtx_rlock(&environ_rwmtx); + erts_rwmtx_rlock(&environ_rwmtx); if (opts->envir == NULL) { new_environ = environ; } else if ((new_environ = build_unix_environment(opts->envir)) == NULL) { - erts_smp_rwmtx_runlock(&environ_rwmtx); + erts_rwmtx_runlock(&environ_rwmtx); close_pipes(ifd, ofd); erts_free(ERTS_ALC_T_TMP, (void *) cmd_line); errno = ENOMEM; @@ -641,7 +621,7 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name, erts_free(ERTS_ALC_T_TMP, (void *) cmd_line); if (new_environ != environ) erts_free(ERTS_ALC_T_ENVIRONMENT, (void *) new_environ); - erts_smp_rwmtx_runlock(&environ_rwmtx); + erts_rwmtx_runlock(&environ_rwmtx); errno = err; return ERL_DRV_ERROR_ERRNO; } @@ -681,7 +661,7 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name, if (!io_vector) { close_pipes(ifd, ofd); - erts_smp_rwmtx_runlock(&environ_rwmtx); + erts_rwmtx_runlock(&environ_rwmtx); erts_free(ERTS_ALC_T_TMP, (void *) cmd_line); if (new_environ != environ) erts_free(ERTS_ALC_T_ENVIRONMENT, (void *) new_environ); @@ -766,7 +746,7 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name, erts_free(ERTS_ALC_T_TMP, io_vector); if (new_environ != environ) erts_free(ERTS_ALC_T_ENVIRONMENT, (void *) new_environ); - erts_smp_rwmtx_runlock(&environ_rwmtx); + erts_rwmtx_runlock(&environ_rwmtx); erts_free(ERTS_ALC_T_TMP, (void *) cmd_line); errno = err; return ERL_DRV_ERROR_ERRNO; @@ -795,7 +775,7 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name, if (new_environ != environ) erts_free(ERTS_ALC_T_ENVIRONMENT, (void *) new_environ); - erts_smp_rwmtx_runlock(&environ_rwmtx); + erts_rwmtx_runlock(&environ_rwmtx); dd = create_driver_data(port_num, ifd[0], ofd[1], opts->packet_bytes, DO_WRITE | DO_READ, opts->exit_status, @@ -1068,8 +1048,8 @@ static void clear_fd_data(ErtsSysFdData *fdd) { if (fdd->sz > 0) { erts_free(ERTS_ALC_T_FD_ENTRY_BUF, (void *) fdd->buf); - ASSERT(erts_smp_atomic_read_nob(&sys_misc_mem_sz) >= fdd->sz); - erts_smp_atomic_add_nob(&sys_misc_mem_sz, -1*fdd->sz); + ASSERT(erts_atomic_read_nob(&sys_misc_mem_sz) >= fdd->sz); + erts_atomic_add_nob(&sys_misc_mem_sz, -1*fdd->sz); } fdd->buf = NULL; fdd->sz = 0; @@ -1092,13 +1072,11 @@ static void fd_stop(ErlDrvData ev) /* Does not close the fds */ ErlDrvPort prt = dd->port_num; int sz = sizeof(ErtsSysDriverData); -#if FDBLOCK if (dd->blocking) { erts_free(ERTS_ALC_T_SYS_BLOCKING, dd->blocking); dd->blocking = NULL; sz += sizeof(ErtsSysBlocking); } -#endif if (dd->ifd) { sz += sizeof(ErtsSysFdData); @@ -1110,7 +1088,7 @@ static void fd_stop(ErlDrvData ev) /* Does not close the fds */ } erts_free(ERTS_ALC_T_DRV_TAB, dd); - erts_smp_atomic_add_nob(&sys_misc_mem_sz, -sz); + erts_atomic_add_nob(&sys_misc_mem_sz, -sz); } static void fd_flush(ErlDrvData ev) @@ -1191,19 +1169,19 @@ static void outputv(ErlDrvData e, ErlIOVec* ev) ev->iov[0].iov_len = pb; ev->size += pb; - if (dd->blocking && FDBLOCK) + if (dd->blocking) driver_pdl_lock(dd->blocking->pdl); if ((sz = driver_sizeq(ix)) > 0) { driver_enqv(ix, ev, 0); - if (dd->blocking && FDBLOCK) + if (dd->blocking) driver_pdl_unlock(dd->blocking->pdl); if (sz + ev->size >= (1 << 13)) set_busy_port(ix, 1); } - else if (!dd->blocking || !FDBLOCK) { + else if (!dd->blocking) { /* We try to write directly if the fd in non-blocking */ int vsize = ev->vsize > MAX_VSIZE ? MAX_VSIZE : ev->vsize; @@ -1220,7 +1198,6 @@ static void outputv(ErlDrvData e, ErlIOVec* ev) driver_enqv(ix, ev, n); /* n is the skip value */ driver_select(ix, ofd, ERL_DRV_WRITE|ERL_DRV_USE, 1); } -#if FDBLOCK else { if (ev->size != 0) { driver_enqv(ix, ev, 0); @@ -1231,7 +1208,6 @@ static void outputv(ErlDrvData e, ErlIOVec* ev) driver_pdl_unlock(dd->blocking->pdl); } } -#endif /* return 0;*/ } @@ -1303,7 +1279,7 @@ static int port_inp_failure(ErtsSysDriverData *dd, int res) clear_fd_data(dd->ifd); } - if (dd->blocking && FDBLOCK) { + if (dd->blocking) { driver_pdl_lock(dd->blocking->pdl); if (driver_sizeq(dd->port_num) > 0) { driver_pdl_unlock(dd->blocking->pdl); @@ -1408,7 +1384,7 @@ static void ready_input(ErlDrvData e, ErlDrvEvent ready_fd) if (dd->ifd->fd < 0) { driver_select(port_num, abs(dd->ifd->fd), ERL_DRV_READ|ERL_DRV_USE, 0); - erts_smp_atomic_add_nob(&sys_misc_mem_sz, -sizeof(ErtsSysFdData)); + erts_atomic_add_nob(&sys_misc_mem_sz, -sizeof(ErtsSysFdData)); dd->ifd = NULL; } @@ -1514,7 +1490,7 @@ static void ready_input(ErlDrvData e, ErlDrvEvent ready_fd) port_inp_failure(dd, -1); } else { - erts_smp_atomic_add_nob(&sys_misc_mem_sz, h); + erts_atomic_add_nob(&sys_misc_mem_sz, h); sys_memcpy(buf, cpos, bytes_left); dd->ifd->buf = buf; dd->ifd->sz = h; @@ -1549,7 +1525,7 @@ static void ready_output(ErlDrvData e, ErlDrvEvent ready_fd) should close the output fd as soon as the command has been sent. */ driver_select(ix, ready_fd, ERL_DRV_WRITE|ERL_DRV_USE, 0); - erts_smp_atomic_add_nob(&sys_misc_mem_sz, -sizeof(ErtsSysFdData)); + erts_atomic_add_nob(&sys_misc_mem_sz, -sizeof(ErtsSysFdData)); dd->ofd = NULL; } if (dd->terminating) @@ -1579,7 +1555,6 @@ static void stop_select(ErlDrvEvent fd, void* _) close((int)fd); } -#if FDBLOCK static void fd_async(void *async_data) @@ -1658,7 +1633,6 @@ void fd_ready_async(ErlDrvData drv_data, return; /* 0; */ } -#endif /* Forker driver */ diff --git a/erts/emulator/sys/unix/sys_float.c b/erts/emulator/sys/unix/sys_float.c index 6435da086f..a82c15bd32 100644 --- a/erts/emulator/sys/unix/sys_float.c +++ b/erts/emulator/sys/unix/sys_float.c @@ -39,7 +39,6 @@ erts_sys_init_float(void) #else /* !NO_FPE_SIGNALS */ -#ifdef ERTS_SMP static erts_tsd_key_t fpe_key; /* once-only initialisation early in the main thread (via erts_sys_init_float()) */ @@ -61,11 +60,6 @@ static ERTS_INLINE volatile unsigned long *erts_thread_get_fp_exception(void) { return (volatile unsigned long*)erts_tsd_get(fpe_key); } -#else /* !SMP */ -#define erts_init_fp_exception() /*empty*/ -static volatile unsigned long fp_exception; -#define erts_thread_get_fp_exception() (&fp_exception) -#endif /* SMP */ volatile unsigned long *erts_get_current_fp_exception(void) { @@ -659,11 +653,9 @@ void erts_sys_init_float(void) void erts_thread_init_float(void) { -#ifdef ERTS_SMP /* This allows Erlang schedulers to leave Erlang-process context and still have working FP exceptions. XXX: is this needed? */ erts_thread_init_fp_exception(); -#endif #ifndef NO_FPE_SIGNALS /* NOTE: diff --git a/erts/emulator/sys/unix/sys_time.c b/erts/emulator/sys/unix/sys_time.c index 4f26639703..ef05380d17 100644 --- a/erts/emulator/sys/unix/sys_time.c +++ b/erts/emulator/sys/unix/sys_time.c @@ -160,7 +160,7 @@ struct sys_time_internal_state_read_mostly__ { #ifdef ERTS_SYS_TIME_INTERNAL_STATE_WRITE_FREQ__ struct sys_time_internal_state_write_freq__ { - erts_smp_mtx_t mtx; + erts_mtx_t mtx; #if defined(__linux__) && defined(OS_MONOTONIC_TIME_USING_CLOCK_GETTIME) ErtsMonotonicTime last_delivered; #endif @@ -304,8 +304,8 @@ sys_init_time(ErtsSysInitTimeResult *init_resp) erts_sys_time_data__.r.o.os_times = clock_gettime_times_verified; #endif - erts_smp_mtx_init(&internal_state.w.f.mtx, - "os_monotonic_time"); + erts_mtx_init(&internal_state.w.f.mtx, "os_monotonic_time", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO); internal_state.w.f.last_delivered = clock_gettime_monotonic(); init_resp->os_monotonic_time_info.locked_use = 1; @@ -525,12 +525,12 @@ static ErtsMonotonicTime clock_gettime_monotonic_verified(void) mtime = (ErtsMonotonicTime) posix_clock_gettime(MONOTONIC_CLOCK_ID, MONOTONIC_CLOCK_ID_STR); - erts_smp_mtx_lock(&internal_state.w.f.mtx); + erts_mtx_lock(&internal_state.w.f.mtx); if (mtime < internal_state.w.f.last_delivered) mtime = internal_state.w.f.last_delivered; else internal_state.w.f.last_delivered = mtime; - erts_smp_mtx_unlock(&internal_state.w.f.mtx); + erts_mtx_unlock(&internal_state.w.f.mtx); return mtime; } @@ -547,12 +547,12 @@ static void clock_gettime_times_verified(ErtsMonotonicTime *mtimep, WALL_CLOCK_ID_STR, stimep); - erts_smp_mtx_lock(&internal_state.w.f.mtx); + erts_mtx_lock(&internal_state.w.f.mtx); if (*mtimep < internal_state.w.f.last_delivered) *mtimep = internal_state.w.f.last_delivered; else internal_state.w.f.last_delivered = *mtimep; - erts_smp_mtx_unlock(&internal_state.w.f.mtx); + erts_mtx_unlock(&internal_state.w.f.mtx); } #endif /* defined(OS_SYSTEM_TIME_USING_CLOCK_GETTIME) */ @@ -878,8 +878,6 @@ ErtsMonotonicTime erts_os_monotonic_time(void) { Uint32 ticks = get_tick_count(); - ERTS_CHK_EXTEND_OS_MONOTONIC_TIME(&internal_state.wr.m.os_mtime_xtnd, - ticks); return ERTS_EXTEND_OS_MONOTONIC_TIME(&internal_state.wr.m.os_mtime_xtnd, ticks) << internal_state.r.o.times_shift; } diff --git a/erts/emulator/sys/win32/erl_poll.c b/erts/emulator/sys/win32/erl_poll.c index b10fc1e430..0bd43bb4fb 100644 --- a/erts/emulator/sys/win32/erl_poll.c +++ b/erts/emulator/sys/win32/erl_poll.c @@ -142,7 +142,8 @@ static erts_mtx_t save_ops_mtx; static void poll_debug_init(void) { - erts_mtx_init(&save_ops_mtx, "save_ops_lock"); + erts_mtx_init(&save_ops_mtx, "save_ops_lock", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DEBUG); } void poll_debug_set_active_fd(ErtsSysFdType fd) @@ -285,41 +286,26 @@ struct ErtsPollSet_ { CRITICAL_SECTION standby_crit; /* CS to guard the counter */ HANDLE standby_wait_event; /* Event signalled when counte == 0 */ erts_atomic32_t wakeup_state; -#ifdef ERTS_SMP - erts_smp_mtx_t mtx; -#endif + erts_mtx_t mtx; erts_atomic64_t timeout_time; }; -#ifdef ERTS_SMP #define ERTS_POLLSET_LOCK(PS) \ - erts_smp_mtx_lock(&(PS)->mtx) + erts_mtx_lock(&(PS)->mtx) #define ERTS_POLLSET_UNLOCK(PS) \ - erts_smp_mtx_unlock(&(PS)->mtx) - -#else + erts_mtx_unlock(&(PS)->mtx) -#define ERTS_POLLSET_LOCK(PS) -#define ERTS_POLLSET_UNLOCK(PS) - -#endif /* * Communication with sys_interrupt */ -#ifdef ERTS_SMP -extern erts_smp_atomic32_t erts_break_requested; +extern erts_atomic32_t erts_break_requested; #define ERTS_SET_BREAK_REQUESTED \ - erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 1) + erts_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 1) #define ERTS_UNSET_BREAK_REQUESTED \ - erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 0) -#else -extern volatile int erts_break_requested; -#define ERTS_SET_BREAK_REQUESTED (erts_break_requested = 1) -#define ERTS_UNSET_BREAK_REQUESTED (erts_break_requested = 0) -#endif + erts_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 0) static erts_mtx_t break_waiter_lock; static HANDLE break_happened_event; @@ -677,7 +663,7 @@ static void new_waiter(ErtsPollSet ps) w->active_events = 1; w->highwater = 1; w->total_events = 1; - erts_mtx_init(&w->mtx, "pollwaiter"); + erts_mtx_init(&w->mtx, "pollwaiter", NIL, ERTS_LOCK_FLAGS_CATEGORY_IO); /* @@ -1192,14 +1178,10 @@ int erts_poll_wait(ErtsPollSet ps, HARDDEBUGF(("Start waiting %d [%d]",num_h, (int) timeout)); ERTS_POLLSET_UNLOCK(ps); -#ifdef ERTS_SMP erts_thr_progress_prepare_wait(NULL); -#endif ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP); WaitForMultipleObjects(num_h, harr, FALSE, timeout); -#ifdef ERTS_SMP erts_thr_progress_finalize_wait(NULL); -#endif ERTS_MSACC_POP_STATE_M(); ERTS_POLLSET_LOCK(ps); HARDDEBUGF(("Stop waiting %d [%d]",num_h, (int) timeout)); @@ -1358,9 +1340,7 @@ ErtsPollSet erts_poll_create_pollset(void) ps->restore_events = 0; erts_atomic32_init_nob(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN); -#ifdef ERTS_SMP - erts_smp_mtx_init(&ps->mtx, "pollset"); -#endif + erts_mtx_init(&ps->mtx, "pollset", NIL, ERTS_LOCK_FLAGS_CATEGORY_IO); init_timeout_time(ps); HARDTRACEF(("Out erts_poll_create_pollset")); @@ -1390,9 +1370,7 @@ void erts_poll_destroy_pollset(ErtsPollSet ps) CloseHandle(ps->event_io_ready); CloseHandle(ps->standby_wait_event); ERTS_POLLSET_UNLOCK(ps); -#ifdef ERTS_SMP - erts_smp_mtx_destroy(&ps->mtx); -#endif + erts_mtx_destroy(&ps->mtx); SEL_FREE(ERTS_ALC_T_POLLSET, (void *) ps); HARDTRACEF(("Out erts_poll_destroy_pollset")); } @@ -1411,7 +1389,8 @@ void erts_poll_init(void) HARDTRACEF(("In erts_poll_init")); erts_sys_break_event = CreateManualEvent(FALSE); - erts_mtx_init(&break_waiter_lock,"break_waiter_lock"); + erts_mtx_init(&break_waiter_lock, "break_waiter_lock", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO); break_happened_event = CreateManualEvent(FALSE); erts_atomic32_init_nob(&break_waiter_state, 0); diff --git a/erts/emulator/sys/win32/erl_win_sys.h b/erts/emulator/sys/win32/erl_win_sys.h index 78005aada9..1f53452d17 100644 --- a/erts/emulator/sys/win32/erl_win_sys.h +++ b/erts/emulator/sys/win32/erl_win_sys.h @@ -311,10 +311,8 @@ typedef long ssize_t; #endif /* Threads */ -#ifdef USE_THREADS int init_async(int); int exit_async(void); -#endif #define ERTS_HAVE_TRY_CATCH 1 diff --git a/erts/emulator/sys/win32/sys.c b/erts/emulator/sys/win32/sys.c index 28019e306c..b23dbecbac 100644 --- a/erts/emulator/sys/win32/sys.c +++ b/erts/emulator/sys/win32/sys.c @@ -80,11 +80,9 @@ static int application_type(const wchar_t* originalName, wchar_t fullPath[MAX_PA HANDLE erts_service_event; -#ifdef ERTS_SMP -static erts_smp_tsd_key_t win32_errstr_key; -#endif +static erts_tsd_key_t win32_errstr_key; -static erts_smp_atomic_t pipe_creation_counter; +static erts_atomic_t pipe_creation_counter; /* Results from application_type(_w) is one of */ #define APPL_NONE 0 @@ -94,10 +92,8 @@ static erts_smp_atomic_t pipe_creation_counter; static int driver_write(long, HANDLE, byte*, int); static int create_file_thread(struct async_io* aio, int mode); -#ifdef ERTS_SMP static void close_active_handle(DriverData *, HANDLE handle); static DWORD WINAPI threaded_handle_closer(LPVOID param); -#endif static DWORD WINAPI threaded_reader(LPVOID param); static DWORD WINAPI threaded_writer(LPVOID param); static DWORD WINAPI threaded_exiter(LPVOID param); @@ -136,7 +132,7 @@ static OSVERSIONINFO int_os_version; /* Version information for Win32. */ Disabled the use of CancelIoEx as its been seen to cause problem with some drivers. Not sure what to blame; faulty drivers or some form of invalid use. */ -#if defined(ERTS_SMP) && defined(USE_CANCELIOEX) +#if defined(USE_CANCELIOEX) static BOOL (WINAPI *fpCancelIoEx)(HANDLE,LPOVERLAPPED); #endif @@ -145,7 +141,7 @@ static BOOL (WINAPI *fpCancelIoEx)(HANDLE,LPOVERLAPPED); - call erl_start() to parse arguments and do other init */ -static erts_smp_atomic_t sys_misc_mem_sz; +static erts_atomic_t sys_misc_mem_sz; HMODULE beam_module = NULL; @@ -196,7 +192,7 @@ Uint erts_sys_misc_mem_sz(void) { Uint res = (Uint) erts_check_io_size(); - res += (Uint) erts_smp_atomic_read_mb(&sys_misc_mem_sz); + res += (Uint) erts_atomic_read_mb(&sys_misc_mem_sz); return res; } @@ -450,9 +446,7 @@ typedef struct async_io { * the console for Windows NT). */ HANDLE fd; /* Handle for file or pipe. */ -#ifdef ERTS_SMP int async_io_active; /* if true, a close of the file will signal the event in ov */ -#endif OVERLAPPED ov; /* Control structure for overlapped reading. * When overlapped reading is simulated with * a thread, the fields are used as follows: @@ -665,7 +659,7 @@ new_driver_data(ErlDrvPort port_num, int packet_bytes, int wait_objs_required, i dp->inbuf = DRV_BUF_ALLOC(dp->inBufSize); if (dp->inbuf == NULL) goto buf_alloc_error; - erts_smp_atomic_add_nob(&sys_misc_mem_sz, dp->inBufSize); + erts_atomic_add_nob(&sys_misc_mem_sz, dp->inBufSize); dp->outBufSize = 0; dp->outbuf = NULL; dp->port_num = port_num; @@ -691,7 +685,6 @@ buf_alloc_error: static void release_driver_data(DriverData* dp) { -#ifdef ERTS_SMP #ifdef USE_CANCELIOEX if (fpCancelIoEx != NULL) { if (dp->in.thread == (HANDLE) -1 && dp->in.fd != INVALID_HANDLE_VALUE) { @@ -734,18 +727,10 @@ release_driver_data(DriverData* dp) DEBUGF(("...done\n")); } } -#else - if (dp->in.thread == (HANDLE) -1 && dp->in.fd != INVALID_HANDLE_VALUE) { - CancelIo(dp->in.fd); - } - if (dp->out.thread == (HANDLE) -1 && dp->out.fd != INVALID_HANDLE_VALUE) { - CancelIo(dp->out.fd); - } -#endif if (dp->inbuf != NULL) { - ASSERT(erts_smp_atomic_read_nob(&sys_misc_mem_sz) >= dp->inBufSize); - erts_smp_atomic_add_nob(&sys_misc_mem_sz, -1*dp->inBufSize); + ASSERT(erts_atomic_read_nob(&sys_misc_mem_sz) >= dp->inBufSize); + erts_atomic_add_nob(&sys_misc_mem_sz, -1*dp->inBufSize); DRV_BUF_FREE(dp->inbuf); dp->inBufSize = 0; dp->inbuf = NULL; @@ -753,8 +738,8 @@ release_driver_data(DriverData* dp) ASSERT(dp->inBufSize == 0); if (dp->outbuf != NULL) { - ASSERT(erts_smp_atomic_read_nob(&sys_misc_mem_sz) >= dp->outBufSize); - erts_smp_atomic_add_nob(&sys_misc_mem_sz, -1*dp->outBufSize); + ASSERT(erts_atomic_read_nob(&sys_misc_mem_sz) >= dp->outBufSize); + erts_atomic_add_nob(&sys_misc_mem_sz, -1*dp->outBufSize); DRV_BUF_FREE(dp->outbuf); dp->outBufSize = 0; dp->outbuf = NULL; @@ -777,7 +762,6 @@ release_driver_data(DriverData* dp) unrefer_driver_data(dp); } -#ifdef ERTS_SMP struct handles_to_be_closed { HANDLE handles[MAXIMUM_WAIT_OBJECTS]; @@ -870,7 +854,6 @@ threaded_handle_closer(LPVOID param) DEBUGF(("threaded_handle_closer %p terminating\r\n", htbc)); return 0; } -#endif /* ERTS_SMP */ /* * Stores input and output file descriptors in the DriverData structure, @@ -946,9 +929,7 @@ init_async_io(DriverData *dp, AsyncIo* aio, int use_threads) aio->flushReplyEvent = NULL; aio->pendingError = 0; aio->bytesTransferred = 0; -#ifdef ERTS_SMP aio->async_io_active = 0; -#endif aio->ov.hEvent = CreateManualEvent(FALSE); if (aio->ov.hEvent == NULL) return -1; @@ -1029,9 +1010,7 @@ async_read_file(AsyncIo* aio, LPVOID buf, DWORD numToRead) ResetEvent(aio->ov.hEvent); SetEvent(aio->ioAllowed); } else { -#ifdef ERTS_SMP aio->async_io_active = 1; /* Will get 0 when the event actually happened */ -#endif if (ReadFile(aio->fd, buf, numToRead, &aio->bytesTransferred, &aio->ov)) { DEBUGF(("async_read_file: ReadFile() suceeded: %d bytes\n", @@ -1079,16 +1058,12 @@ async_write_file(AsyncIo* aio, /* Pointer to async control block. */ ResetEvent(aio->ov.hEvent); SetEvent(aio->ioAllowed); } else { -#ifdef ERTS_SMP aio->async_io_active = 1; /* Will get 0 when the event actually happened */ -#endif if (WriteFile(aio->fd, buf, numToWrite, &aio->bytesTransferred, &aio->ov)) { DEBUGF(("async_write_file: WriteFile() suceeded: %d bytes\n", aio->bytesTransferred)); -#ifdef ERTS_SMP aio->async_io_active = 0; /* The event will not be signalled */ -#endif ResetEvent(aio->ov.hEvent); return TRUE; } else { @@ -1190,7 +1165,7 @@ static int spawn_init(void) { int i; -#if defined(ERTS_SMP) && defined(USE_CANCELIOEX) +#if defined(USE_CANCELIOEX) HMODULE module = GetModuleHandle("kernel32"); fpCancelIoEx = (BOOL (WINAPI *)(HANDLE,LPOVERLAPPED)) ((module != NULL) ? GetProcAddress(module,"CancelIoEx") : NULL); @@ -1762,7 +1737,7 @@ static int create_pipe(HANDLE *phRead, HANDLE *phWrite, BOOL inheritRead, BOOL o * Otherwise, create named pipes. */ - calls = (UWord) erts_smp_atomic_inc_read_nob(&pipe_creation_counter); + calls = (UWord) erts_atomic_inc_read_nob(&pipe_creation_counter); erts_snprintf(pipe_name, sizeof(pipe_name), "\\\\.\\pipe\\erlang44_%d_%bpu", getpid(), calls); @@ -2447,7 +2422,7 @@ output(ErlDrvData drv_data, char* buf, ErlDrvSizeT len) } dp->outBufSize = pb+len; - erts_smp_atomic_add_nob(&sys_misc_mem_sz, dp->outBufSize); + erts_atomic_add_nob(&sys_misc_mem_sz, dp->outBufSize); /* * Store header bytes (if any). @@ -2476,8 +2451,8 @@ output(ErlDrvData drv_data, char* buf, ErlDrvSizeT len) } else { dp->out.ov.Offset += pb+len; /* For vanilla driver. */ /* XXX OffsetHigh should be changed too. */ - ASSERT(erts_smp_atomic_read_nob(&sys_misc_mem_sz) >= dp->outBufSize); - erts_smp_atomic_add_nob(&sys_misc_mem_sz, -1*dp->outBufSize); + ASSERT(erts_atomic_read_nob(&sys_misc_mem_sz) >= dp->outBufSize); + erts_atomic_add_nob(&sys_misc_mem_sz, -1*dp->outBufSize); DRV_BUF_FREE(dp->outbuf); dp->outBufSize = 0; dp->outbuf = NULL; @@ -2511,11 +2486,9 @@ ready_input(ErlDrvData drv_data, ErlDrvEvent ready_event) int pb; pb = dp->packet_bytes; -#ifdef ERTS_SMP if(dp->in.thread == (HANDLE) -1) { dp->in.async_io_active = 0; } -#endif DEBUGF(("ready_input: dp %p, event 0x%x\n", dp, ready_event)); /* @@ -2590,8 +2563,8 @@ ready_input(ErlDrvData drv_data, ErlDrvEvent ready_event) error = ERROR_NOT_ENOUGH_MEMORY; break; /* Break out of loop into error handler. */ } - ASSERT(erts_smp_atomic_read_nob(&sys_misc_mem_sz) >= dp->inBufSize); - erts_smp_atomic_add_nob(&sys_misc_mem_sz, + ASSERT(erts_atomic_read_nob(&sys_misc_mem_sz) >= dp->inBufSize); + erts_atomic_add_nob(&sys_misc_mem_sz, dp->totalNeeded - dp->inBufSize); dp->inBufSize = dp->totalNeeded; dp->inbuf = new_buf; @@ -2680,11 +2653,9 @@ ready_output(ErlDrvData drv_data, ErlDrvEvent ready_event) DriverData *dp = (DriverData *) drv_data; int error; -#ifdef ERTS_SMP if(dp->out.thread == (HANDLE) -1) { dp->out.async_io_active = 0; } -#endif DEBUGF(("ready_output(%p, 0x%x)\n", drv_data, ready_event)); set_busy_port(dp->port_num, 0); if (!(dp->outbuf)) { @@ -2692,8 +2663,8 @@ ready_output(ErlDrvData drv_data, ErlDrvEvent ready_event) write... */ return; } - ASSERT(erts_smp_atomic_read_nob(&sys_misc_mem_sz) >= dp->outBufSize); - erts_smp_atomic_add_nob(&sys_misc_mem_sz, -1*dp->outBufSize); + ASSERT(erts_atomic_read_nob(&sys_misc_mem_sz) >= dp->outBufSize); + erts_atomic_add_nob(&sys_misc_mem_sz, -1*dp->outBufSize); DRV_BUF_FREE(dp->outbuf); dp->outBufSize = 0; dp->outbuf = NULL; @@ -2743,7 +2714,6 @@ sys_init_io(void) max_files = 2*erts_ptab_max(&erts_port); } -#ifdef ERTS_SMP void erts_sys_main_thread(void) { @@ -2756,7 +2726,6 @@ erts_sys_main_thread(void) WaitForSingleObject(dummy, INFINITE); } } -#endif void erts_sys_alloc_init(void) { @@ -2843,7 +2812,7 @@ Preload* sys_preloaded(void) (num_preloaded+1)*sizeof(Preload)); res_name = erts_alloc(ERTS_ALC_T_PRELOADED, (num_preloaded+1)*sizeof(unsigned)); - erts_smp_atomic_add_nob(&sys_misc_mem_sz, + erts_atomic_add_nob(&sys_misc_mem_sz, (num_preloaded+1)*sizeof(Preload) + (num_preloaded+1)*sizeof(unsigned)); for (i = 0; i < num_preloaded; i++) { @@ -2856,7 +2825,7 @@ Preload* sys_preloaded(void) n = GETWORD(data); data += 2; preloaded[i].name = erts_alloc(ERTS_ALC_T_PRELOADED, n+1); - erts_smp_atomic_add_nob(&sys_misc_mem_sz, n+1); + erts_atomic_add_nob(&sys_misc_mem_sz, n+1); sys_memcpy(preloaded[i].name, data, n); preloaded[i].name[n] = '\0'; data += n; @@ -2938,11 +2907,7 @@ sys_get_key(int fd) char* win32_errorstr(int error) { -#ifdef SMP - LPTSTR lpBufPtr = erts_smp_tsd_get(win32_errstr_key); -#else - static LPTSTR lpBufPtr = NULL; -#endif + LPTSTR lpBufPtr = erts_tsd_get(win32_errstr_key); if (lpBufPtr) { LocalFree(lpBufPtr); } @@ -2956,9 +2921,7 @@ char* win32_errorstr(int error) 0, NULL); SetLastError(error); -#ifdef ERTS_SMP - erts_smp_tsd_set(win32_errstr_key,lpBufPtr); -#endif + erts_tsd_set(win32_errstr_key,lpBufPtr); return lpBufPtr; } @@ -3131,7 +3094,6 @@ check_supported_os_version(void) #endif } -#ifdef USE_THREADS typedef struct { int sched_bind_data; @@ -3176,37 +3138,39 @@ thr_create_prepare_child(void *vtcdp) erts_sched_bind_atthrcreate_child(tcdp->sched_bind_data); } -#endif /* USE_THREADS */ void erts_sys_pre_init(void) { -#ifdef USE_THREADS erts_thr_init_data_t eid = ERTS_THR_INIT_DATA_DEF_INITER; -#endif int_os_version.dwOSVersionInfoSize = sizeof(OSVERSIONINFO); GetVersionEx(&int_os_version); check_supported_os_version(); -#ifdef USE_THREADS eid.thread_create_child_func = thr_create_prepare_child; /* Before creation in parent */ eid.thread_create_prepare_func = thr_create_prepare; /* After creation in parent */ eid.thread_create_parent_func = thr_create_cleanup; - erts_thr_init(&eid); -#ifdef ERTS_ENABLE_LOCK_CHECK - erts_lc_init(); +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_pre_thr_init(); #endif + + erts_thr_init(&eid); + #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_init(); + erts_lcnt_post_thr_init(); #endif + +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_lc_init(); #endif + erts_init_sys_time_sup(); - erts_smp_atomic_init_nob(&sys_misc_mem_sz, 0); + erts_atomic_init_nob(&sys_misc_mem_sz, 0); } void noinherit_std_handle(DWORD type) @@ -3226,11 +3190,9 @@ void erl_sys_init(void) noinherit_std_handle(STD_INPUT_HANDLE); noinherit_std_handle(STD_ERROR_HANDLE); -#ifdef ERTS_SMP - erts_smp_tsd_key_create(&win32_errstr_key,"win32_errstr_key"); + erts_tsd_key_create(&win32_errstr_key,"win32_errstr_key"); InitializeCriticalSection(&htbc_lock); -#endif - erts_smp_atomic_init_nob(&pipe_creation_counter,0); + erts_atomic_init_nob(&pipe_creation_counter,0); /* * Test if we have named pipes or not. */ @@ -3292,13 +3254,11 @@ erts_sys_schedule_interrupt(int set) erts_check_io_interrupt(set); } -#ifdef ERTS_SMP void erts_sys_schedule_interrupt_timed(int set, ErtsMonotonicTime timeout_time) { erts_check_io_interrupt_timed(set, timeout_time); } -#endif /* * Called from schedule() when it runs out of runnable processes, @@ -3309,6 +3269,6 @@ void erl_sys_schedule(int runnable) { erts_check_io(!runnable); - ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); + ERTS_LC_ASSERT(!erts_thr_progress_is_blocking()); } diff --git a/erts/emulator/sys/win32/sys_env.c b/erts/emulator/sys/win32/sys_env.c index 21ef71ad9a..5792816267 100644 --- a/erts/emulator/sys/win32/sys_env.c +++ b/erts/emulator/sys/win32/sys_env.c @@ -32,22 +32,23 @@ static WCHAR **env_to_arg(WCHAR *env); static WCHAR **find_arg(WCHAR **arg, WCHAR *str);
static int compare(const void *a, const void *b);
-static erts_smp_rwmtx_t environ_rwmtx;
+static erts_rwmtx_t environ_rwmtx; void
erts_sys_env_init(void)
{
- erts_smp_rwmtx_init(&environ_rwmtx, "environ");
+ erts_rwmtx_init(&environ_rwmtx, "environ", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
}
int
erts_sys_putenv_raw(char *key, char *value)
{
int res;
- erts_smp_rwmtx_rwlock(&environ_rwmtx);
+ erts_rwmtx_rwlock(&environ_rwmtx); res = (SetEnvironmentVariable((LPCTSTR) key,
(LPCTSTR) value) ? 0 : 1);
- erts_smp_rwmtx_rwunlock(&environ_rwmtx);
+ erts_rwmtx_rwunlock(&environ_rwmtx); return res;
}
@@ -57,10 +58,10 @@ erts_sys_putenv(char *key, char *value) int res;
WCHAR *wkey = (WCHAR *) key;
WCHAR *wvalue = (WCHAR *) value;
- erts_smp_rwmtx_rwlock(&environ_rwmtx);
+ erts_rwmtx_rwlock(&environ_rwmtx); res = (SetEnvironmentVariableW(wkey,
wvalue) ? 0 : 1);
- erts_smp_rwmtx_rwunlock(&environ_rwmtx);
+ erts_rwmtx_rwunlock(&environ_rwmtx); return res;
}
@@ -75,12 +76,12 @@ erts_sys_getenv(char *key, char *value, size_t *size) DWORD wsize = *size / (sizeof(WCHAR) / sizeof(char));
SetLastError(0);
- erts_smp_rwmtx_rlock(&environ_rwmtx);
+ erts_rwmtx_rlock(&environ_rwmtx); new_size = GetEnvironmentVariableW(wkey,
wvalue,
(DWORD) wsize);
res = !new_size && GetLastError() == ERROR_ENVVAR_NOT_FOUND ? -1 : 0;
- erts_smp_rwmtx_runlock(&environ_rwmtx);
+ erts_rwmtx_runlock(&environ_rwmtx); if (res < 0)
return res;
res = new_size > wsize ? 1 : 0;
@@ -110,22 +111,22 @@ int erts_sys_getenv_raw(char *key, char *value, size_t *size)
{
int res;
- erts_smp_rwmtx_rlock(&environ_rwmtx);
+ erts_rwmtx_rlock(&environ_rwmtx); res = erts_sys_getenv__(key, value, size);
- erts_smp_rwmtx_runlock(&environ_rwmtx);
+ erts_rwmtx_runlock(&environ_rwmtx); return res;
}
void init_getenv_state(GETENV_STATE *state)
{
- erts_smp_rwmtx_rlock(&environ_rwmtx);
+ erts_rwmtx_rlock(&environ_rwmtx); state->environment_strings = GetEnvironmentStringsW();
state->next_string = state->environment_strings;
}
char *getenv_string(GETENV_STATE *state)
{
- ERTS_SMP_LC_ASSERT(erts_smp_lc_rwmtx_is_rlocked(&environ_rwmtx));
+ ERTS_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&environ_rwmtx)); if (state->next_string[0] == L'\0') {
return NULL;
} else {
@@ -139,7 +140,7 @@ void fini_getenv_state(GETENV_STATE *state) {
FreeEnvironmentStringsW(state->environment_strings);
state->environment_strings = state->next_string = NULL;
- erts_smp_rwmtx_runlock(&environ_rwmtx);
+ erts_rwmtx_runlock(&environ_rwmtx); }
int erts_sys_unsetenv(char *key)
@@ -148,7 +149,7 @@ int erts_sys_unsetenv(char *key) WCHAR *wkey = (WCHAR *) key;
SetLastError(0);
- erts_smp_rwmtx_rlock(&environ_rwmtx);
+ erts_rwmtx_rlock(&environ_rwmtx); GetEnvironmentVariableW(wkey,
NULL,
0);
@@ -156,7 +157,7 @@ int erts_sys_unsetenv(char *key) res = (SetEnvironmentVariableW(wkey,
NULL) ? 0 : 1);
}
- erts_smp_rwmtx_runlock(&environ_rwmtx);
+ erts_rwmtx_runlock(&environ_rwmtx); return res;
}
@@ -170,12 +171,12 @@ win_build_environment(char* new_env) tmp_new = (WCHAR *) new_env;
- erts_smp_rwmtx_rlock(&environ_rwmtx);
+ erts_rwmtx_rlock(&environ_rwmtx); tmp = GetEnvironmentStringsW();
merged = merge_environment(tmp, tmp_new);
FreeEnvironmentStringsW(tmp);
- erts_smp_rwmtx_runlock(&environ_rwmtx);
+ erts_rwmtx_runlock(&environ_rwmtx); return (char *) merged;
}
}
diff --git a/erts/emulator/sys/win32/sys_interrupt.c b/erts/emulator/sys/win32/sys_interrupt.c index df838960eb..02aa50500f 100644 --- a/erts/emulator/sys/win32/sys_interrupt.c +++ b/erts/emulator/sys/win32/sys_interrupt.c @@ -35,17 +35,11 @@ # define WIN_SYS_INLINE __forceinline #endif -#ifdef ERTS_SMP -erts_smp_atomic32_t erts_break_requested; +erts_atomic32_t erts_break_requested; #define ERTS_SET_BREAK_REQUESTED \ - erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 1) + erts_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 1) #define ERTS_UNSET_BREAK_REQUESTED \ - erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 0) -#else -volatile int erts_break_requested = 0; -#define ERTS_SET_BREAK_REQUESTED (erts_break_requested = 1) -#define ERTS_UNSET_BREAK_REQUESTED (erts_break_requested = 0) -#endif + erts_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 0) extern int nohup; HANDLE erts_sys_break_event = NULL; @@ -57,14 +51,14 @@ void erts_do_break_handling(void) * therefore, make sure that all threads but this one are blocked before * proceeding! */ - erts_smp_thr_progress_block(); + erts_thr_progress_block(); /* call the break handling function, reset the flag */ do_break(); ResetEvent(erts_sys_break_event); ERTS_UNSET_BREAK_REQUESTED; - erts_smp_thr_progress_unblock(); + erts_thr_progress_unblock(); } diff --git a/erts/emulator/sys/win32/sys_time.c b/erts/emulator/sys/win32/sys_time.c index e8c67b3928..25c2ad385c 100644 --- a/erts/emulator/sys/win32/sys_time.c +++ b/erts/emulator/sys/win32/sys_time.c @@ -95,7 +95,7 @@ struct sys_time_internal_state_read_mostly__ { }; struct sys_time_internal_state_write_freq__ { - erts_smp_mtx_t mtime_mtx; + erts_mtx_t mtime_mtx; ULONGLONG wrap; ULONGLONG last_tick_count; }; @@ -187,8 +187,6 @@ os_monotonic_time_gtc32(void) { ErtsMonotonicTime mtime; Uint32 ticks = (Uint32) GetTickCount(); - ERTS_CHK_EXTEND_OS_MONOTONIC_TIME(&internal_state.wr.m.os_mtime_xtnd, - ticks); mtime = ERTS_EXTEND_OS_MONOTONIC_TIME(&internal_state.wr.m.os_mtime_xtnd, ticks); mtime <<= ERTS_GET_TICK_COUNT_TIME_UNIT_SHIFT; @@ -205,8 +203,6 @@ os_times_gtc32(ErtsMonotonicTime *mtimep, ErtsSystemTime *stimep) ticks = (Uint32) GetTickCount(); GetSystemTime(&st); - ERTS_CHK_EXTEND_OS_MONOTONIC_TIME(&internal_state.wr.m.os_mtime_xtnd, - ticks); mtime = ERTS_EXTEND_OS_MONOTONIC_TIME(&internal_state.wr.m.os_mtime_xtnd, ticks); mtime <<= ERTS_GET_TICK_COUNT_TIME_UNIT_SHIFT; @@ -265,8 +261,6 @@ sys_hrtime_gtc32(void) { ErtsSysHrTime time; Uint32 ticks = (Uint32) GetTickCount(); - ERTS_CHK_EXTEND_OS_MONOTONIC_TIME(&internal_state.wr.m.os_mtime_xtnd, - tick_count); time = (ErtsSysHrTime) ERTS_EXTEND_OS_MONOTONIC_TIME(&internal_state.wr.m.os_mtime_xtnd, ticks); time *= (ErtsSysHrTime) (1000 * 1000); @@ -300,8 +294,8 @@ sys_init_time(ErtsSysInitTimeResult *init_resp) module = GetModuleHandle(kernel_dll_name); if (!module) { get_tick_count: - erts_smp_mtx_init(&internal_state.w.f.mtime_mtx, - "os_monotonic_time"); + erts_mtx_init(&internal_state.w.f.mtime_mtx, "os_monotonic_time", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC); internal_state.w.f.wrap = 0; internal_state.w.f.last_tick_count = 0; |