diff options
Diffstat (limited to 'erts/emulator/sys/common')
| -rw-r--r-- | erts/emulator/sys/common/erl_check_io.c | 1017 | ||||
| -rw-r--r-- | erts/emulator/sys/common/erl_check_io.h | 29 | ||||
| -rw-r--r-- | erts/emulator/sys/common/erl_mmap.c | 19 | ||||
| -rw-r--r-- | erts/emulator/sys/common/erl_mmap.h | 15 | ||||
| -rw-r--r-- | erts/emulator/sys/common/erl_mseg.c | 17 | ||||
| -rw-r--r-- | erts/emulator/sys/common/erl_poll.c | 39 | ||||
| -rw-r--r-- | erts/emulator/sys/common/erl_poll.h | 4 | ||||
| -rw-r--r-- | erts/emulator/sys/common/erl_sys_common_misc.c | 104 |
8 files changed, 994 insertions, 250 deletions
diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c index 8f2f1f9521..799f67fc45 100644 --- a/erts/emulator/sys/common/erl_check_io.c +++ b/erts/emulator/sys/common/erl_check_io.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2006-2016. All Rights Reserved. + * Copyright Ericsson AB 2006-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -38,6 +38,7 @@ #include "erl_port.h" #include "erl_check_io.h" #include "erl_thr_progress.h" +#include "erl_bif_unique.h" #include "dtrace-wrapper.h" #include "lttng-wrapper.h" #define ERTS_WANT_TIMER_WHEEL_API @@ -53,6 +54,8 @@ typedef char EventStateType; #define ERTS_EV_TYPE_DRV_SEL ((EventStateType) 1) /* driver_select */ #define ERTS_EV_TYPE_DRV_EV ((EventStateType) 2) /* driver_event */ #define ERTS_EV_TYPE_STOP_USE ((EventStateType) 3) /* pending stop_select */ +#define ERTS_EV_TYPE_NIF ((EventStateType) 4) /* enif_select */ +#define ERTS_EV_TYPE_STOP_NIF ((EventStateType) 5) /* pending nif stop */ typedef char EventStateFlags; #define ERTS_EV_FLAG_USED ((EventStateFlags) 1) /* ERL_DRV_USE has been turned on */ @@ -122,7 +125,11 @@ typedef struct { #if ERTS_CIO_HAVE_DRV_EVENT ErtsDrvEventDataState *event; /* ERTS_EV_TYPE_DRV_EV */ #endif - erts_driver_t* drv_ptr; /* ERTS_EV_TYPE_STOP_USE */ + ErtsNifSelectDataState *nif; /* ERTS_EV_TYPE_NIF */ + union { + erts_driver_t* drv_ptr; /* ERTS_EV_TYPE_STOP_USE */ + ErtsResource* resource; /* ERTS_EV_TYPE_STOP_NIF */ + }stop; } driver; ErtsPollEvents events; unsigned short remove_cnt; /* number of removed_fd's referring to this fd */ @@ -194,7 +201,8 @@ static ERTS_INLINE ErtsDrvEventState* hash_new_drv_ev_state(ErtsSysFdType fd) #if ERTS_CIO_HAVE_DRV_EVENT tmpl.driver.event = NULL; #endif - tmpl.driver.drv_ptr = NULL; + tmpl.driver.nif = NULL; + tmpl.driver.stop.drv_ptr = NULL; tmpl.events = 0; tmpl.remove_cnt = 0; tmpl.type = ERTS_EV_TYPE_NONE; @@ -211,24 +219,35 @@ static ERTS_INLINE void hash_erase_drv_ev_state(ErtsDrvEventState *state) #endif /* !ERTS_SYS_CONTINOUS_FD_NUMBERS */ static void stale_drv_select(Eterm id, ErtsDrvEventState *state, int mode); -static void select_steal(ErlDrvPort ix, ErtsDrvEventState *state, - int mode, int on); -static void print_select_op(erts_dsprintf_buf_t *dsbufp, - ErlDrvPort ix, ErtsSysFdType fd, int mode, int on); +static void drv_select_steal(ErlDrvPort ix, ErtsDrvEventState *state, + int mode, int on); +static void nif_select_steal(ErtsDrvEventState *state, int mode, + ErtsResource* resource, Eterm ref); + +static void print_drv_select_op(erts_dsprintf_buf_t *dsbufp, + ErlDrvPort ix, ErtsSysFdType fd, int mode, int on); +static void print_nif_select_op(erts_dsprintf_buf_t*, ErtsSysFdType, + int mode, ErtsResource*, Eterm ref); + #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS -static void select_large_fd_error(ErlDrvPort, ErtsSysFdType, int, int); +static void drv_select_large_fd_error(ErlDrvPort, ErtsSysFdType, int, int); +static void nif_select_large_fd_error(ErtsSysFdType, int, ErtsResource*,Eterm ref); #endif #if ERTS_CIO_HAVE_DRV_EVENT -static void event_steal(ErlDrvPort ix, ErtsDrvEventState *state, +static void drv_event_steal(ErlDrvPort ix, ErtsDrvEventState *state, ErlDrvEventData event_data); -static void print_event_op(erts_dsprintf_buf_t *dsbufp, - ErlDrvPort, ErtsSysFdType, ErlDrvEventData); +static void print_drv_event_op(erts_dsprintf_buf_t *dsbufp, + ErlDrvPort, ErtsSysFdType, ErlDrvEventData); #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS static void event_large_fd_error(ErlDrvPort, ErtsSysFdType, ErlDrvEventData); #endif #endif -static void steal_pending_stop_select(erts_dsprintf_buf_t*, ErlDrvPort, - ErtsDrvEventState*, int mode, int on); +static void +steal_pending_stop_use(erts_dsprintf_buf_t*, ErlDrvPort, ErtsDrvEventState*, + int mode, int on); +static void +steal_pending_stop_nif(erts_dsprintf_buf_t *dsbufp, ErtsResource*, + ErtsDrvEventState *state, int mode, int on); #ifdef ERTS_SMP ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(removed_fd, struct removed_fd, 64, ERTS_ALC_T_FD_LIST) @@ -263,6 +282,18 @@ alloc_drv_select_data(void) return dsp; } +static ERTS_INLINE ErtsNifSelectDataState * +alloc_nif_select_data(void) +{ + ErtsNifSelectDataState *dsp = erts_alloc(ERTS_ALC_T_NIF_SEL_D_STATE, + sizeof(ErtsNifSelectDataState)); + dsp->in.pid = NIL; + dsp->out.pid = NIL; + dsp->in.ddeselect_cnt = 0; + dsp->out.ddeselect_cnt = 0; + return dsp; +} + static ERTS_INLINE void free_drv_select_data(ErtsDrvSelectDataState *dsp) { @@ -271,6 +302,12 @@ free_drv_select_data(ErtsDrvSelectDataState *dsp) erts_free(ERTS_ALC_T_DRV_SEL_D_STATE, dsp); } +static ERTS_INLINE void +free_nif_select_data(ErtsNifSelectDataState *dsp) +{ + erts_free(ERTS_ALC_T_NIF_SEL_D_STATE, dsp); +} + #if ERTS_CIO_HAVE_DRV_EVENT static ERTS_INLINE ErtsDrvEventDataState * @@ -352,6 +389,7 @@ forget_removed(struct pollset_info* psi) erts_smp_spin_unlock(&psi->removed_list_lock); while (fdlp) { + ErtsResource* resource = NULL; erts_driver_t* drv_ptr = NULL; erts_smp_mtx_t* mtx; ErtsSysFdType fd; @@ -372,15 +410,25 @@ forget_removed(struct pollset_info* psi) ASSERT(state->remove_cnt > 0); if (--state->remove_cnt == 0) { switch (state->type) { + case ERTS_EV_TYPE_STOP_NIF: + /* Now we can call stop */ + resource = state->driver.stop.resource; + state->driver.stop.resource = NULL; + ASSERT(resource); + state->type = ERTS_EV_TYPE_NONE; + state->flags &= ~ERTS_EV_FLAG_USED; + goto case_ERTS_EV_TYPE_NONE; + case ERTS_EV_TYPE_STOP_USE: /* Now we can call stop_select */ - drv_ptr = state->driver.drv_ptr; + drv_ptr = state->driver.stop.drv_ptr; ASSERT(drv_ptr); state->type = ERTS_EV_TYPE_NONE; state->flags &= ~ERTS_EV_FLAG_USED; - state->driver.drv_ptr = NULL; + state->driver.stop.drv_ptr = NULL; /* Fall through */ - case ERTS_EV_TYPE_NONE: + case ERTS_EV_TYPE_NONE: + case_ERTS_EV_TYPE_NONE: #ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS hash_erase_drv_ev_state(state); #endif @@ -403,6 +451,11 @@ forget_removed(struct pollset_info* psi) erts_ddll_dereference_driver(drv_ptr->handle); } } + if (resource) { + erts_resource_stop(resource, (ErlNifEvent)fd, 0); + enif_release_resource(resource->data); + } + tofree = fdlp; fdlp = fdlp->next; removed_fd_free(tofree); @@ -440,7 +493,8 @@ grow_drv_ev_state(int min_ix) #if ERTS_CIO_HAVE_DRV_EVENT drv_ev_state[i].driver.event = NULL; #endif - drv_ev_state[i].driver.drv_ptr = NULL; + drv_ev_state[i].driver.stop.drv_ptr = NULL; + drv_ev_state[i].driver.nif = NULL; drv_ev_state[i].events = 0; drv_ev_state[i].remove_cnt = 0; drv_ev_state[i].type = ERTS_EV_TYPE_NONE; @@ -480,6 +534,7 @@ abort_tasks(ErtsDrvEventState *state, int mode) ERTS_EV_TYPE_DRV_EV); return; #endif + case ERTS_EV_TYPE_NIF: case ERTS_EV_TYPE_NONE: return; default: @@ -534,6 +589,14 @@ deselect(ErtsDrvEventState *state, int mode) if (!(state->events)) { switch (state->type) { + case ERTS_EV_TYPE_NIF: + state->driver.nif->in.pid = NIL; + state->driver.nif->out.pid = NIL; + state->driver.nif->in.ddeselect_cnt = 0; + state->driver.nif->out.ddeselect_cnt = 0; + enif_release_resource(state->driver.stop.resource); + state->driver.stop.resource = NULL; + break; case ERTS_EV_TYPE_DRV_SEL: state->driver.select->inport = NIL; state->driver.select->outport = NIL; @@ -569,7 +632,8 @@ check_fd_cleanup(ErtsDrvEventState *state, #if ERTS_CIO_HAVE_DRV_EVENT ErtsDrvEventDataState **free_event, #endif - ErtsDrvSelectDataState **free_select) + ErtsDrvSelectDataState **free_select, + ErtsNifSelectDataState **free_nif) { erts_aint_t current_cio_time; @@ -586,6 +650,12 @@ check_fd_cleanup(ErtsDrvEventState *state, state->driver.select = NULL; } + *free_nif = NULL; + if (state->driver.nif && (state->type != ERTS_EV_TYPE_NIF)) { + *free_nif = state->driver.nif; + state->driver.nif = NULL; + } + #if ERTS_CIO_HAVE_DRV_EVENT *free_event = NULL; if (state->driver.event @@ -617,12 +687,14 @@ check_cleanup_active_fd(ErtsSysFdType fd, ErtsPollControlEntry *pce, int *pce_ix, #endif - erts_aint_t current_cio_time) + erts_aint_t current_cio_time, + int may_sleep) { ErtsDrvEventState *state; int active = 0; erts_smp_mtx_t *mtx = fd_mtx(fd); void *free_select = NULL; + void *free_nif = NULL; #if ERTS_CIO_HAVE_DRV_EVENT void *free_event = NULL; #endif @@ -682,6 +754,39 @@ check_cleanup_active_fd(ErtsSysFdType fd, } } + if (state->driver.nif) { + ErtsPollEvents rm_events = 0; + if (state->driver.nif->in.ddeselect_cnt) { + ASSERT(state->type == ERTS_EV_TYPE_NIF); + ASSERT(state->events & ERTS_POLL_EV_IN); + ASSERT(is_nil(state->driver.nif->in.pid)); + if (may_sleep || state->driver.nif->in.ddeselect_cnt == 1) { + rm_events = ERTS_POLL_EV_IN; + state->driver.nif->in.ddeselect_cnt = 0; + } + } + if (state->driver.nif->out.ddeselect_cnt) { + ASSERT(state->type == ERTS_EV_TYPE_NIF); + ASSERT(state->events & ERTS_POLL_EV_OUT); + ASSERT(is_nil(state->driver.nif->out.pid)); + if (may_sleep || state->driver.nif->out.ddeselect_cnt == 1) { + rm_events |= ERTS_POLL_EV_OUT; + state->driver.nif->out.ddeselect_cnt = 0; + } + } + if (rm_events) { + int do_wake = 0; + state->events = ERTS_CIO_POLL_CTL(pollset.ps, state->fd, + rm_events, 0, &do_wake); + } + if (state->events) + active = 1; + else if (state->type != ERTS_EV_TYPE_NIF) { + free_nif = state->driver.nif; + state->driver.nif = NULL; + } + } + #if ERTS_CIO_HAVE_DRV_EVENT if (state->driver.event) { if (is_iotask_active(&state->driver.event->iotask, current_cio_time)) { @@ -722,6 +827,8 @@ check_cleanup_active_fd(ErtsSysFdType fd, if (free_select) free_drv_select_data(free_select); + if (free_nif) + free_nif_select_data(free_nif); #if ERTS_CIO_HAVE_DRV_EVENT if (free_event) free_drv_event_data(free_event); @@ -746,7 +853,7 @@ check_cleanup_active_fd(ErtsSysFdType fd, } static void -check_cleanup_active_fds(erts_aint_t current_cio_time) +check_cleanup_active_fds(erts_aint_t current_cio_time, int may_sleep) { int six = pollset.active_fd.six; int eix = pollset.active_fd.eix; @@ -773,7 +880,8 @@ check_cleanup_active_fds(erts_aint_t current_cio_time) pctrl_entries, &pctrl_ix, #endif - current_cio_time)) { + current_cio_time, + may_sleep)) { no--; if (ix == six) { #ifdef DEBUG @@ -807,13 +915,30 @@ check_cleanup_active_fds(erts_aint_t current_cio_time) erts_smp_atomic32_set_relb(&pollset.active_fd.no, no); } +static void grow_active_fds(void) +{ + ASSERT(pollset.active_fd.six == pollset.active_fd.eix); + pollset.active_fd.six = 0; + pollset.active_fd.eix = pollset.active_fd.size; + pollset.active_fd.size += ERTS_ACTIVE_FD_INC; + pollset.active_fd.array = erts_realloc(ERTS_ALC_T_ACTIVE_FD_ARR, + pollset.active_fd.array, + pollset.active_fd.size*sizeof(ErtsSysFdType)); +#ifdef DEBUG + { + int i; + for (i = pollset.active_fd.eix + 1; i < pollset.active_fd.size; i++) + pollset.active_fd.array[i] = ERTS_SYS_FD_INVALID; + } +#endif +} + static ERTS_INLINE void add_active_fd(ErtsSysFdType fd) { int eix = pollset.active_fd.eix; int size = pollset.active_fd.size; - pollset.active_fd.array[eix] = fd; erts_smp_atomic32_set_relb(&pollset.active_fd.no, @@ -823,25 +948,11 @@ add_active_fd(ErtsSysFdType fd) eix++; if (eix >= size) eix = 0; - if (pollset.active_fd.six == eix) { - pollset.active_fd.six = 0; - eix = size; - size += ERTS_ACTIVE_FD_INC; - pollset.active_fd.array = erts_realloc(ERTS_ALC_T_ACTIVE_FD_ARR, - pollset.active_fd.array, - sizeof(ErtsSysFdType)*size); - pollset.active_fd.size = size; -#ifdef DEBUG - { - int i; - for (i = eix + 1; i < size; i++) - pollset.active_fd.array[i] = ERTS_SYS_FD_INVALID; - } -#endif + pollset.active_fd.eix = eix; + if (pollset.active_fd.six == eix) { + grow_active_fds(); } - - pollset.active_fd.eix = eix; } int @@ -863,6 +974,7 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix, ErtsDrvEventDataState *free_event = NULL; #endif ErtsDrvSelectDataState *free_select = NULL; + ErtsNifSelectDataState *free_nif = NULL; #ifdef USE_VM_PROBES DTRACE_CHARBUF(name, 64); #endif @@ -878,7 +990,7 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix, return -1; } if (fd >= max_fds) { - select_large_fd_error(ix, fd, mode, on); + drv_select_large_fd_error(ix, fd, mode, on); return -1; } grow_drv_ev_state(fd); @@ -916,26 +1028,38 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix, } #endif + switch (state->type) { #if ERTS_CIO_HAVE_DRV_EVENT - if (state->type == ERTS_EV_TYPE_DRV_EV) - select_steal(ix, state, mode, on); + case ERTS_EV_TYPE_DRV_EV: #endif - if (state->type == ERTS_EV_TYPE_STOP_USE) { - erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - print_select_op(dsbufp, ix, state->fd, mode, on); - steal_pending_stop_select(dsbufp, ix, state, mode, on); - if (state->type == ERTS_EV_TYPE_STOP_USE) { - ret = 0; - goto done; /* stop_select still pending */ - } - ASSERT(state->type == ERTS_EV_TYPE_NONE); - } + case ERTS_EV_TYPE_NIF: + drv_select_steal(ix, state, mode, on); + break; + case ERTS_EV_TYPE_STOP_USE: { + erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); + print_drv_select_op(dsbufp, ix, state->fd, mode, on); + steal_pending_stop_use(dsbufp, ix, state, mode, on); + if (state->type == ERTS_EV_TYPE_STOP_USE) { + ret = 0; + goto done; /* stop_select still pending */ + } + ASSERT(state->type == ERTS_EV_TYPE_NONE); + break; + } + case ERTS_EV_TYPE_STOP_NIF: { + erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); + print_drv_select_op(dsbufp, ix, state->fd, mode, on); + steal_pending_stop_nif(dsbufp, NULL, state, mode, on); + ASSERT(state->type == ERTS_EV_TYPE_NONE); + break; + + }} if (mode & ERL_DRV_READ) { if (state->type == ERTS_EV_TYPE_DRV_SEL) { Eterm owner = state->driver.select->inport; if (owner != id && is_not_nil(owner)) - select_steal(ix, state, mode, on); + drv_select_steal(ix, state, mode, on); } ctl_events |= ERTS_POLL_EV_IN; } @@ -943,7 +1067,7 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix, if (state->type == ERTS_EV_TYPE_DRV_SEL) { Eterm owner = state->driver.select->outport; if (owner != id && is_not_nil(owner)) - select_steal(ix, state, mode, on); + drv_select_steal(ix, state, mode, on); } ctl_events |= ERTS_POLL_EV_OUT; } @@ -1033,7 +1157,7 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix, else { /* Not safe to close fd, postpone stop_select callback. */ state->type = ERTS_EV_TYPE_STOP_USE; - state->driver.drv_ptr = drv_ptr; + state->driver.stop.drv_ptr = drv_ptr; if (drv_ptr->handle) { erts_ddll_reference_referenced_driver(drv_ptr->handle); } @@ -1050,7 +1174,8 @@ done: #if ERTS_CIO_HAVE_DRV_EVENT &free_event, #endif - &free_select); + &free_select, + &free_nif); done_unknown: erts_smp_mtx_unlock(fd_mtx(fd)); @@ -1063,6 +1188,263 @@ done_unknown: } if (free_select) free_drv_select_data(free_select); + if (free_nif) + free_nif_select_data(free_nif); + +#if ERTS_CIO_HAVE_DRV_EVENT + if (free_event) + free_drv_event_data(free_event); +#endif + return ret; +} + +int +ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env, + ErlNifEvent e, + enum ErlNifSelectFlags mode, + void* obj, + const ErlNifPid* pid, + Eterm ref) +{ + int on; + ErtsResource* resource = DATA_TO_RESOURCE(obj); + ErtsSysFdType fd = (ErtsSysFdType) e; + ErtsPollEvents ctl_events = (ErtsPollEvents) 0; + ErtsPollEvents new_events, old_events; + ErtsDrvEventState *state; + int wake_poller; + int ret; + enum { NO_STOP=0, CALL_STOP, CALL_STOP_AND_RELEASE } call_stop = NO_STOP; +#if ERTS_CIO_HAVE_DRV_EVENT + ErtsDrvEventDataState *free_event = NULL; +#endif + ErtsDrvSelectDataState *free_select = NULL; + ErtsNifSelectDataState *free_nif = NULL; +#ifdef USE_VM_PROBES + DTRACE_CHARBUF(name, 64); +#endif + + ASSERT(!(resource->monitors && resource->monitors->is_dying)); + +#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS + if ((unsigned)fd >= (unsigned)erts_smp_atomic_read_nob(&drv_ev_state_len)) { + if (fd < 0) { + return INT_MIN | ERL_NIF_SELECT_INVALID_EVENT; + } + if (fd >= max_fds) { + nif_select_large_fd_error(fd, mode, resource, ref); + return INT_MIN | ERL_NIF_SELECT_INVALID_EVENT; + } + grow_drv_ev_state(fd); + } +#endif + + erts_smp_mtx_lock(fd_mtx(fd)); + +#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS + state = &drv_ev_state[(int) fd]; +#else + state = hash_get_drv_ev_state(fd); /* may be NULL! */ +#endif + + if (mode & ERL_NIF_SELECT_STOP) { + ASSERT(resource->type->stop); + if (IS_FD_UNKNOWN(state)) { + /* fast track to stop callback */ + call_stop = CALL_STOP; + ret = ERL_NIF_SELECT_STOP_CALLED; + goto done_unknown; + } + on = 0; + mode = ERL_DRV_READ | ERL_DRV_WRITE | ERL_DRV_USE; + wake_poller = 1; /* to eject fd from pollset (if needed) */ + ctl_events = ERTS_POLL_EV_IN | ERTS_POLL_EV_OUT; + } + else { + on = 1; + ASSERT(mode); + wake_poller = 0; + if (mode & ERL_DRV_READ) { + ctl_events |= ERTS_POLL_EV_IN; + } + if (mode & ERL_DRV_WRITE) { + ctl_events |= ERTS_POLL_EV_OUT; + } + } + +#ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS + if (state == NULL) { + state = hash_new_drv_ev_state(fd); + } +#endif + + switch (state->type) { + case ERTS_EV_TYPE_NIF: + /* + * Changing resource is considered stealing. + * Changing process and/or ref is ok (I think?). + */ + if (state->driver.stop.resource != resource) + nif_select_steal(state, ERL_DRV_READ | ERL_DRV_WRITE, resource, ref); + break; +#if ERTS_CIO_HAVE_DRV_EVENT + case ERTS_EV_TYPE_DRV_EV: +#endif + case ERTS_EV_TYPE_DRV_SEL: + nif_select_steal(state, mode, resource, ref); + break; + case ERTS_EV_TYPE_STOP_USE: { + erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); + print_nif_select_op(dsbufp, fd, mode, resource, ref); + steal_pending_stop_use(dsbufp, ERTS_INVALID_ERL_DRV_PORT, state, mode, on); + ASSERT(state->type == ERTS_EV_TYPE_NONE); + break; + } + case ERTS_EV_TYPE_STOP_NIF: { + erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); + print_nif_select_op(dsbufp, fd, mode, resource, ref); + steal_pending_stop_nif(dsbufp, resource, state, mode, on); + if (state->type == ERTS_EV_TYPE_STOP_NIF) { + ret = ERL_NIF_SELECT_STOP_SCHEDULED; /* ?? */ + goto done; + } + ASSERT(state->type == ERTS_EV_TYPE_NONE); + break; + }} + + ASSERT((state->type == ERTS_EV_TYPE_NIF) || + (state->type == ERTS_EV_TYPE_NONE && !state->events)); + + new_events = ERTS_CIO_POLL_CTL(pollset.ps, state->fd, ctl_events, on, &wake_poller); + + if (new_events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL)) { + if (state->type == ERTS_EV_TYPE_NIF && !state->events) { + state->type = ERTS_EV_TYPE_NONE; + state->flags &= ~ERTS_EV_FLAG_USED; + state->driver.nif->in.pid = NIL; + state->driver.nif->out.pid = NIL; + state->driver.nif->in.ddeselect_cnt = 0; + state->driver.nif->out.ddeselect_cnt = 0; + state->driver.stop.resource = NULL; + } + ret = INT_MIN | ERL_NIF_SELECT_FAILED; + goto done; + } + + old_events = state->events; + + ASSERT(on + ? (new_events == (state->events | ctl_events)) + : (new_events == (state->events & ~ctl_events))); + + ASSERT(state->type == ERTS_EV_TYPE_NIF + || state->type == ERTS_EV_TYPE_NONE); + + state->events = new_events; + if (on) { + const Eterm recipient = pid ? pid->pid : env->proc->common.id; + Uint32* refn; + if (!state->driver.nif) + state->driver.nif = alloc_nif_select_data(); + if (state->type == ERTS_EV_TYPE_NONE) { + state->type = ERTS_EV_TYPE_NIF; + state->driver.stop.resource = resource; + enif_keep_resource(resource->data); + } + ASSERT(state->type == ERTS_EV_TYPE_NIF); + ASSERT(state->driver.stop.resource == resource); + if (ctl_events & ERTS_POLL_EV_IN) { + state->driver.nif->in.pid = recipient; + if (is_immed(ref)) { + state->driver.nif->in.immed = ref; + } else { + ASSERT(is_internal_ref(ref)); + refn = internal_ref_numbers(ref); + state->driver.nif->in.immed = THE_NON_VALUE; + state->driver.nif->in.refn[0] = refn[0]; + state->driver.nif->in.refn[1] = refn[1]; + state->driver.nif->in.refn[2] = refn[2]; + } + state->driver.nif->in.ddeselect_cnt = 0; + } + if (ctl_events & ERTS_POLL_EV_OUT) { + state->driver.nif->out.pid = recipient; + if (is_immed(ref)) { + state->driver.nif->out.immed = ref; + } else { + ASSERT(is_internal_ref(ref)); + refn = internal_ref_numbers(ref); + state->driver.nif->out.immed = THE_NON_VALUE; + state->driver.nif->out.refn[0] = refn[0]; + state->driver.nif->out.refn[1] = refn[1]; + state->driver.nif->out.refn[2] = refn[2]; + } + state->driver.nif->out.ddeselect_cnt = 0; + } + ret = 0; + } + else { /* off */ + if (state->type == ERTS_EV_TYPE_NIF) { + state->driver.nif->in.pid = NIL; + state->driver.nif->out.pid = NIL; + state->driver.nif->in.ddeselect_cnt = 0; + state->driver.nif->out.ddeselect_cnt = 0; + if (old_events != 0) { + remember_removed(state, &pollset); + } + } + ASSERT(new_events==0); + if (state->remove_cnt == 0 || !wake_poller) { + /* + * Safe to close fd now as it is not in pollset + * or there was no need to eject fd (kernel poll) + */ + if (state->type == ERTS_EV_TYPE_NIF) { + ASSERT(state->driver.stop.resource == resource); + call_stop = CALL_STOP_AND_RELEASE; + state->driver.stop.resource = NULL; + } + else { + ASSERT(!state->driver.stop.resource); + call_stop = CALL_STOP; + } + state->type = ERTS_EV_TYPE_NONE; + ret = ERL_NIF_SELECT_STOP_CALLED; + } + else { + /* Not safe to close fd, postpone stop_select callback. */ + if (state->type == ERTS_EV_TYPE_NONE) { + ASSERT(!state->driver.stop.resource); + state->driver.stop.resource = resource; + enif_keep_resource(resource); + } + state->type = ERTS_EV_TYPE_STOP_NIF; + ret = ERL_NIF_SELECT_STOP_SCHEDULED; + } + } + +done: + + check_fd_cleanup(state, +#if ERTS_CIO_HAVE_DRV_EVENT + &free_event, +#endif + &free_select, + &free_nif); + +done_unknown: + erts_smp_mtx_unlock(fd_mtx(fd)); + if (call_stop) { + erts_resource_stop(resource, (ErlNifEvent)fd, 1); + if (call_stop == CALL_STOP_AND_RELEASE) { + enif_release_resource(resource->data); + } + } + if (free_select) + free_drv_select_data(free_select); + if (free_nif) + free_nif_select_data(free_nif); + #if ERTS_CIO_HAVE_DRV_EVENT if (free_event) free_drv_event_data(free_event); @@ -1070,6 +1452,7 @@ done_unknown: return ret; } + int ERTS_CIO_EXPORT(driver_event)(ErlDrvPort ix, ErlDrvEvent e, @@ -1090,6 +1473,7 @@ ERTS_CIO_EXPORT(driver_event)(ErlDrvPort ix, ErtsDrvEventDataState *free_event; #endif ErtsDrvSelectDataState *free_select; + ErtsNifSelectDataState *free_nif; Port *prt = erts_drvport2port(ix); if (prt == ERTS_INVALID_ERL_DRV_PORT) @@ -1126,12 +1510,12 @@ ERTS_CIO_EXPORT(driver_event)(ErlDrvPort ix, if (state->driver.event->port == id) break; /*fall through*/ case ERTS_EV_TYPE_DRV_SEL: - event_steal(ix, state, event_data); + drv_event_steal(ix, state, event_data); break; case ERTS_EV_TYPE_STOP_USE: { erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - print_event_op(dsbufp, ix, fd, event_data); - steal_pending_stop_select(dsbufp, ix, state, 0, 1); + print_drv_event_op(dsbufp, ix, fd, event_data); + steal_pending_stop_use(dsbufp, ix, state, 0, 1); break; } } @@ -1199,12 +1583,15 @@ done: #if ERTS_CIO_HAVE_DRV_EVENT &free_event, #endif - &free_select); + &free_select, + &free_nif); erts_smp_mtx_unlock(fd_mtx(fd)); if (free_select) free_drv_select_data(free_select); + if (free_nif) + free_nif_select_data(free_nif); #if ERTS_CIO_HAVE_DRV_EVENT if (free_event) free_drv_event_data(free_event); @@ -1240,13 +1627,19 @@ need2steal(ErtsDrvEventState *state, int mode) state, ERL_DRV_WRITE); break; + case ERTS_EV_TYPE_NIF: + ASSERT(state->driver.stop.resource); + do_steal = 1; + break; + #if ERTS_CIO_HAVE_DRV_EVENT case ERTS_EV_TYPE_DRV_EV: do_steal |= chk_stale(state->driver.event->port, state, 0); break; #endif case ERTS_EV_TYPE_STOP_USE: - ASSERT(0); + case ERTS_EV_TYPE_STOP_NIF: + ASSERT(0); break; default: break; @@ -1307,6 +1700,25 @@ steal(erts_dsprintf_buf_t *dsbufp, ErtsDrvEventState *state, int mode) erts_dsprintf(dsbufp, "\n"); break; } + case ERTS_EV_TYPE_NIF: { + Eterm iid = state->driver.nif->in.pid; + Eterm oid = state->driver.nif->out.pid; + const char* with = "with"; + ErlNifResourceType* rt = state->driver.stop.resource->type; + + erts_dsprintf(dsbufp, "resource %T:%T", rt->module, rt->name); + + if (is_not_nil(iid)) { + erts_dsprintf(dsbufp, " %s in-pid %T", with, iid); + with = "and"; + } + if (is_not_nil(oid)) { + erts_dsprintf(dsbufp, " %s out-pid %T", with, oid); + } + deselect(state, 0); + erts_dsprintf(dsbufp, "\n"); + break; + } #if ERTS_CIO_HAVE_DRV_EVENT case ERTS_EV_TYPE_DRV_EV: { Eterm eid = state->driver.event->port; @@ -1324,7 +1736,8 @@ steal(erts_dsprintf_buf_t *dsbufp, ErtsDrvEventState *state, int mode) break; } #endif - case ERTS_EV_TYPE_STOP_USE: { + case ERTS_EV_TYPE_STOP_USE: + case ERTS_EV_TYPE_STOP_NIF: { ASSERT(0); break; } @@ -1335,8 +1748,8 @@ steal(erts_dsprintf_buf_t *dsbufp, ErtsDrvEventState *state, int mode) } static void -print_select_op(erts_dsprintf_buf_t *dsbufp, - ErlDrvPort ix, ErtsSysFdType fd, int mode, int on) +print_drv_select_op(erts_dsprintf_buf_t *dsbufp, + ErlDrvPort ix, ErtsSysFdType fd, int mode, int on) { Port *pp = erts_drvport2port(ix); erts_dsprintf(dsbufp, @@ -1354,11 +1767,40 @@ print_select_op(erts_dsprintf_buf_t *dsbufp, } static void -select_steal(ErlDrvPort ix, ErtsDrvEventState *state, int mode, int on) +print_nif_select_op(erts_dsprintf_buf_t *dsbufp, + ErtsSysFdType fd, int mode, + ErtsResource* resource, Eterm ref) +{ + erts_dsprintf(dsbufp, + "enif_select(_, %d,%s%s%s, %T:%T, %T) ", + (int) GET_FD(fd), + mode & ERL_NIF_SELECT_READ ? " READ" : "", + mode & ERL_NIF_SELECT_WRITE ? " WRITE" : "", + mode & ERL_NIF_SELECT_STOP ? " STOP" : "", + resource->type->module, + resource->type->name, + ref); +} + + +static void +drv_select_steal(ErlDrvPort ix, ErtsDrvEventState *state, int mode, int on) { if (need2steal(state, mode)) { erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - print_select_op(dsbufp, ix, state->fd, mode, on); + print_drv_select_op(dsbufp, ix, state->fd, mode, on); + steal(dsbufp, state, mode); + erts_send_error_to_logger_nogl(dsbufp); + } +} + +static void +nif_select_steal(ErtsDrvEventState *state, int mode, + ErtsResource* resource, Eterm ref) +{ + if (need2steal(state, mode)) { + erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); + print_nif_select_op(dsbufp, state->fd, mode, resource, ref); steal(dsbufp, state, mode); erts_send_error_to_logger_nogl(dsbufp); } @@ -1374,10 +1816,20 @@ large_fd_error_common(erts_dsprintf_buf_t *dsbufp, ErtsSysFdType fd) } static void -select_large_fd_error(ErlDrvPort ix, ErtsSysFdType fd, int mode, int on) +drv_select_large_fd_error(ErlDrvPort ix, ErtsSysFdType fd, int mode, int on) +{ + erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); + print_drv_select_op(dsbufp, ix, fd, mode, on); + erts_dsprintf(dsbufp, "failed: "); + large_fd_error_common(dsbufp, fd); + erts_send_error_to_logger_nogl(dsbufp); +} +static void +nif_select_large_fd_error(ErtsSysFdType fd, int mode, + ErtsResource* resource, Eterm ref) { erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - print_select_op(dsbufp, ix, fd, mode, on); + print_nif_select_op(dsbufp, fd, mode, resource, ref); erts_dsprintf(dsbufp, "failed: "); large_fd_error_common(dsbufp, fd); erts_send_error_to_logger_nogl(dsbufp); @@ -1387,41 +1839,81 @@ select_large_fd_error(ErlDrvPort ix, ErtsSysFdType fd, int mode, int on) static void -steal_pending_stop_select(erts_dsprintf_buf_t *dsbufp, ErlDrvPort ix, - ErtsDrvEventState *state, int mode, int on) +steal_pending_stop_use(erts_dsprintf_buf_t *dsbufp, ErlDrvPort ix, + ErtsDrvEventState *state, int mode, int on) { + int cancel = 0; ASSERT(state->type == ERTS_EV_TYPE_STOP_USE); - erts_dsprintf(dsbufp, "failed: fd=%d (re)selected before stop_select " - "was called for driver %s\n", - (int) GET_FD(state->fd), state->driver.drv_ptr->name); - erts_send_error_to_logger_nogl(dsbufp); if (on) { /* Either fd-owner changed its mind about closing * or closed fd before stop_select callback and fd is now reused. * In either case stop_select should not be called. - */ - state->type = ERTS_EV_TYPE_NONE; - state->flags &= ~ERTS_EV_FLAG_USED; - if (state->driver.drv_ptr->handle) { - erts_ddll_dereference_driver(state->driver.drv_ptr->handle); - } - state->driver.drv_ptr = NULL; + */ + cancel = 1; } else if ((mode & ERL_DRV_USE_NO_CALLBACK) == ERL_DRV_USE) { Port *prt = erts_drvport2port(ix); - erts_driver_t* drv_ptr = prt != ERTS_INVALID_ERL_DRV_PORT ? prt->drv_ptr : NULL; - if (drv_ptr && drv_ptr != state->driver.drv_ptr) { - /* Some other driver wants the stop_select callback */ - if (state->driver.drv_ptr->handle) { - erts_ddll_dereference_driver(state->driver.drv_ptr->handle); - } - if (drv_ptr->handle) { - erts_ddll_reference_referenced_driver(drv_ptr->handle); - } - state->driver.drv_ptr = drv_ptr; - } + if (prt == ERTS_INVALID_ERL_DRV_PORT + || prt->drv_ptr != state->driver.stop.drv_ptr) { + /* Some other driver or nif wants the stop_select callback */ + cancel = 1; + } + } + + if (cancel) { + erts_dsprintf(dsbufp, "called before stop_select was called for driver '%s'\n", + state->driver.stop.drv_ptr->name); + if (state->driver.stop.drv_ptr->handle) { + erts_ddll_dereference_driver(state->driver.stop.drv_ptr->handle); + } + state->type = ERTS_EV_TYPE_NONE; + state->flags &= ~ERTS_EV_FLAG_USED; + state->driver.stop.drv_ptr = NULL; + } + else { + erts_dsprintf(dsbufp, "ignored repeated call\n"); } + erts_send_error_to_logger_nogl(dsbufp); +} + +static void +steal_pending_stop_nif(erts_dsprintf_buf_t *dsbufp, ErtsResource* resource, + ErtsDrvEventState *state, int mode, int on) +{ + int cancel = 0; + + ASSERT(state->type == ERTS_EV_TYPE_STOP_NIF); + ASSERT(state->driver.stop.resource); + + if (on) { + ASSERT(mode & (ERL_NIF_SELECT_READ | ERL_NIF_SELECT_WRITE)); + /* Either fd-owner changed its mind about closing + * or closed fd before stop callback and fd is now reused. + * In either case, stop should not be called. + */ + cancel = 1; + } + else if ((mode & ERL_DRV_USE_NO_CALLBACK) == ERL_DRV_USE + && resource != state->driver.stop.resource) { + /* Some driver or other resource wants the stop callback */ + cancel = 1; + } + + if (cancel) { + ErlNifResourceType* rt = state->driver.stop.resource->type; + erts_dsprintf(dsbufp, "called before stop was called for NIF resource %T:%T\n", + rt->module, rt->name); + + enif_release_resource(state->driver.stop.resource); + state->type = ERTS_EV_TYPE_NONE; + state->flags &= ~ERTS_EV_FLAG_USED; + state->driver.stop.resource = NULL; + } + else { + erts_dsprintf(dsbufp, "ignored repeated call\n"); + } + erts_send_error_to_logger_nogl(dsbufp); } @@ -1429,8 +1921,8 @@ steal_pending_stop_select(erts_dsprintf_buf_t *dsbufp, ErlDrvPort ix, #if ERTS_CIO_HAVE_DRV_EVENT static void -print_event_op(erts_dsprintf_buf_t *dsbufp, - ErlDrvPort ix, ErtsSysFdType fd, ErlDrvEventData event_data) +print_drv_event_op(erts_dsprintf_buf_t *dsbufp, + ErlDrvPort ix, ErtsSysFdType fd, ErlDrvEventData event_data) { Port *pp = erts_drvport2port(ix); erts_dsprintf(dsbufp, "driver_event(%p, %d, ", ix, (int) fd); @@ -1447,11 +1939,11 @@ print_event_op(erts_dsprintf_buf_t *dsbufp, } static void -event_steal(ErlDrvPort ix, ErtsDrvEventState *state, ErlDrvEventData event_data) +drv_event_steal(ErlDrvPort ix, ErtsDrvEventState *state, ErlDrvEventData event_data) { if (need2steal(state, ERL_DRV_READ|ERL_DRV_WRITE)) { erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - print_event_op(dsbufp, ix, state->fd, event_data); + print_drv_event_op(dsbufp, ix, state->fd, event_data); steal(dsbufp, state, ERL_DRV_READ|ERL_DRV_WRITE); erts_send_error_to_logger_nogl(dsbufp); } @@ -1466,7 +1958,7 @@ static void event_large_fd_error(ErlDrvPort ix, ErtsSysFdType fd, ErlDrvEventData event_data) { erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - print_event_op(dsbufp, ix, fd, event_data); + print_drv_event_op(dsbufp, ix, fd, event_data); erts_dsprintf(dsbufp, "failed: "); large_fd_error_common(dsbufp, fd); erts_send_error_to_logger_nogl(dsbufp); @@ -1553,6 +2045,55 @@ oready(Eterm id, ErtsDrvEventState *state, erts_aint_t current_cio_time) } } +static ERTS_INLINE void +send_event_tuple(struct erts_nif_select_event* e, ErtsResource* resource, + Eterm event_atom) +{ + Process* rp = erts_proc_lookup(e->pid); + ErtsProcLocks rp_locks = 0; + ErtsMessage* mp; + ErlOffHeap* ohp; + ErtsBinary* bin; + Eterm* hp; + Uint hsz; + Eterm resource_term, ref_term, tuple; + + if (!rp) { + return; + } + + bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource); + + /* {select, Resource, Ref, EventAtom} */ + if (is_value(e->immed)) { + hsz = 5 + ERTS_MAGIC_REF_THING_SIZE; + } + else { + hsz = 5 + ERTS_MAGIC_REF_THING_SIZE + ERTS_REF_THING_SIZE; + } + + mp = erts_alloc_message_heap(rp, &rp_locks, hsz, &hp, &ohp); + + resource_term = erts_mk_magic_ref(&hp, ohp, &bin->binary); + if (is_value(e->immed)) { + ASSERT(is_immed(e->immed)); + ref_term = e->immed; + } + else { + write_ref_thing(hp, e->refn[0], e->refn[1], e->refn[2]); + ref_term = make_internal_ref(hp); + hp += ERTS_REF_THING_SIZE; + } + tuple = TUPLE4(hp, am_select, resource_term, ref_term, event_atom); + + ERL_MESSAGE_TOKEN(mp) = am_undefined; + erts_queue_message(rp, rp_locks, mp, tuple, am_system); + + if (rp_locks) + erts_smp_proc_unlock(rp, rp_locks); +} + + #if ERTS_CIO_HAVE_DRV_EVENT static ERTS_INLINE void eready(Eterm id, ErtsDrvEventState *state, ErlDrvEventData event_data, @@ -1575,7 +2116,7 @@ eready(Eterm id, ErtsDrvEventState *state, ErlDrvEventData event_data, } #endif -static void bad_fd_in_pollset( ErtsDrvEventState *, Eterm, Eterm, ErtsPollEvents); +static void bad_fd_in_pollset(ErtsDrvEventState *, Eterm inport, Eterm outport); #ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT void @@ -1598,6 +2139,17 @@ ERTS_CIO_EXPORT(erts_check_io_interrupt_timed)(int set, ERTS_CIO_POLL_INTR_TMD(pollset.ps, set, timeout_time); } +#if !ERTS_CIO_DEFER_ACTIVE_EVENTS +/* + * Number of ignored events, for a lingering fd added by enif_select(), + * until we deselect fd-event from pollset. + */ +# define ERTS_NIF_DELAYED_DESELECT 20 +#else +/* Disable delayed deselect as pollset cannot handle active events */ +# define ERTS_NIF_DELAYED_DESELECT 1 +#endif + void ERTS_CIO_EXPORT(erts_check_io)(int do_wait) { @@ -1617,9 +2169,10 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait) erts_do_break_handling(); #endif -#ifdef ERTS_SIGNAL_SIGTERM - if (ERTS_SIGNAL_SIGTERM) - erts_handle_signal_sigterm(); +#ifdef ERTS_SIGNAL_STATE /* ifndef ERTS_SMP */ + if (ERTS_SIGNAL_STATE) { + erts_handle_signal_state(); + } #endif /* Figure out timeout value */ @@ -1636,7 +2189,8 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait) current_cio_time++; erts_smp_atomic_set_relb(&erts_check_io_time, current_cio_time); - check_cleanup_active_fds(current_cio_time); + check_cleanup_active_fds(current_cio_time, + timeout_time != ERTS_POLL_NO_TIMEOUT); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_check_exact(NULL, 0); /* No locks should be locked */ @@ -1659,11 +2213,14 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait) erts_do_break_handling(); #endif -#ifdef ERTS_SIGNAL_SIGTERM - if (ERTS_SIGNAL_SIGTERM) - erts_handle_signal_sigterm(); + +#ifdef ERTS_SIGNAL_STATE /* ifndef ERTS_SMP */ + if (ERTS_SIGNAL_STATE) { + erts_handle_signal_state(); + } #endif + if (poll_ret != 0) { erts_smp_atomic_set_nob(&pollset.in_poll_wait, 0); forget_removed(&pollset); @@ -1709,31 +2266,22 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait) switch (state->type) { case ERTS_EV_TYPE_DRV_SEL: { /* Requested via driver_select()... */ - ErtsPollEvents revents; - ErtsPollEvents revent_mask; - - revent_mask = ~(ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT); - revent_mask |= state->events; - revents = pollres[i].events & revent_mask; - - if (revents & ERTS_POLL_EV_ERR) { - /* - * Let the driver handle the error condition. Only input, - * only output, or nothing might have been selected. - * We *do not* want to call a callback that corresponds - * to an event not selected. revents might give us a clue - * on which one to call. - */ - if ((revents & ERTS_POLL_EV_IN) - || (!(revents & ERTS_POLL_EV_OUT) - && state->events & ERTS_POLL_EV_IN)) { - iready(state->driver.select->inport, state, current_cio_time); - } - else if (state->events & ERTS_POLL_EV_OUT) { - oready(state->driver.select->outport, state, current_cio_time); - } - } - else if (revents & (ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT)) { + ErtsPollEvents revents = pollres[i].events; + + if (revents & ERTS_POLL_EV_ERR) { + /* + * Handle error events by triggering all in/out events + * that the driver has selected. + * We *do not* want to call a callback that corresponds + * to an event not selected. + */ + revents = state->events; + } + else { + revents &= (state->events | ERTS_POLL_EV_NVAL); + } + + if (revents & (ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT)) { if (revents & ERTS_POLL_EV_OUT) { oready(state->driver.select->outport, state, current_cio_time); } @@ -1741,21 +2289,84 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait) was read (true also on the non-smp emulator since oready() may have been called); therefore, update revents... */ - revents &= ~(~state->events & ERTS_POLL_EV_IN); + revents &= state->events; if (revents & ERTS_POLL_EV_IN) { iready(state->driver.select->inport, state, current_cio_time); } } else if (revents & ERTS_POLL_EV_NVAL) { bad_fd_in_pollset(state, - state->driver.select->inport, - state->driver.select->outport, - state->events); + state->driver.select->inport, + state->driver.select->outport); add_active_fd(state->fd); } break; } + case ERTS_EV_TYPE_NIF: { /* Requested via enif_select()... */ + struct erts_nif_select_event in = {NIL}; + struct erts_nif_select_event out = {NIL}; + ErtsResource* resource = NULL; + ErtsPollEvents revents = pollres[i].events; + + if (revents & ERTS_POLL_EV_ERR) { + /* + * Handle error events by triggering all in/out events + * that the NIF has selected. + * We *do not* want to send a message that corresponds + * to an event not selected. + */ + revents = state->events; + } + else { + revents &= (state->events | ERTS_POLL_EV_NVAL); + } + + if (revents & (ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT)) { + if (revents & ERTS_POLL_EV_OUT) { + if (is_not_nil(state->driver.nif->out.pid)) { + out = state->driver.nif->out; + resource = state->driver.stop.resource; + state->driver.nif->out.ddeselect_cnt = ERTS_NIF_DELAYED_DESELECT; + state->driver.nif->out.pid = NIL; + add_active_fd(state->fd); + } + else { + ASSERT(state->driver.nif->out.ddeselect_cnt >= 2); + state->driver.nif->out.ddeselect_cnt--; + } + } + if (revents & ERTS_POLL_EV_IN) { + if (is_not_nil(state->driver.nif->in.pid)) { + in = state->driver.nif->in; + resource = state->driver.stop.resource; + state->driver.nif->in.ddeselect_cnt = ERTS_NIF_DELAYED_DESELECT; + state->driver.nif->in.pid = NIL; + add_active_fd(state->fd); + } + else { + ASSERT(state->driver.nif->in.ddeselect_cnt >= 2); + state->driver.nif->in.ddeselect_cnt--; + } + } + } + else if (revents & ERTS_POLL_EV_NVAL) { + bad_fd_in_pollset(state, NIL, NIL); + add_active_fd(state->fd); + } + +#ifdef ERTS_SMP + erts_smp_mtx_unlock(fd_mtx(fd)); +#endif + if (is_not_nil(in.pid)) { + send_event_tuple(&in, resource, am_ready_input); + } + if (is_not_nil(out.pid)) { + send_event_tuple(&out, resource, am_ready_output); + } + goto next_pollres_unlocked; + } + #if ERTS_CIO_HAVE_DRV_EVENT case ERTS_EV_TYPE_DRV_EV: { /* Requested via driver_event()... */ ErlDrvEventData event_data; @@ -1796,6 +2407,7 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait) #ifdef ERTS_SMP erts_smp_mtx_unlock(fd_mtx(fd)); #endif + next_pollres_unlocked:; } erts_smp_atomic_set_nob(&pollset.in_poll_wait, 0); @@ -1804,9 +2416,9 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait) } static void -bad_fd_in_pollset(ErtsDrvEventState *state, Eterm inport, - Eterm outport, ErtsPollEvents events) +bad_fd_in_pollset(ErtsDrvEventState *state, Eterm inport, Eterm outport) { + ErtsPollEvents events = state->events; erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); if (events & (ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT)) { @@ -1830,27 +2442,36 @@ bad_fd_in_pollset(ErtsDrvEventState *state, Eterm inport, erts_dsprintf(dsbufp, "Bad %s fd in erts_poll()! fd=%d, ", io_str, (int) state->fd); - if (is_nil(port)) { - ErtsPortNames *ipnp = erts_get_port_names(inport, ERTS_INVALID_ERL_DRV_PORT); - ErtsPortNames *opnp = erts_get_port_names(outport, ERTS_INVALID_ERL_DRV_PORT); - erts_dsprintf(dsbufp, "ports=%T/%T, drivers=%s/%s, names=%s/%s\n", - is_nil(inport) ? am_undefined : inport, - is_nil(outport) ? am_undefined : outport, - ipnp->driver_name ? ipnp->driver_name : "<unknown>", - opnp->driver_name ? opnp->driver_name : "<unknown>", - ipnp->name ? ipnp->name : "<unknown>", - opnp->name ? opnp->name : "<unknown>"); - erts_free_port_names(ipnp); - erts_free_port_names(opnp); - } - else { - ErtsPortNames *pnp = erts_get_port_names(port, ERTS_INVALID_ERL_DRV_PORT); - erts_dsprintf(dsbufp, "port=%T, driver=%s, name=%s\n", - is_nil(port) ? am_undefined : port, - pnp->driver_name ? pnp->driver_name : "<unknown>", - pnp->name ? pnp->name : "<unknown>"); - erts_free_port_names(pnp); - } + if (state->type == ERTS_EV_TYPE_DRV_SEL) { + if (is_nil(port)) { + ErtsPortNames *ipnp = erts_get_port_names(inport, ERTS_INVALID_ERL_DRV_PORT); + ErtsPortNames *opnp = erts_get_port_names(outport, ERTS_INVALID_ERL_DRV_PORT); + erts_dsprintf(dsbufp, "ports=%T/%T, drivers=%s/%s, names=%s/%s\n", + is_nil(inport) ? am_undefined : inport, + is_nil(outport) ? am_undefined : outport, + ipnp->driver_name ? ipnp->driver_name : "<unknown>", + opnp->driver_name ? opnp->driver_name : "<unknown>", + ipnp->name ? ipnp->name : "<unknown>", + opnp->name ? opnp->name : "<unknown>"); + erts_free_port_names(ipnp); + erts_free_port_names(opnp); + } + else { + ErtsPortNames *pnp = erts_get_port_names(port, ERTS_INVALID_ERL_DRV_PORT); + erts_dsprintf(dsbufp, "port=%T, driver=%s, name=%s\n", + is_nil(port) ? am_undefined : port, + pnp->driver_name ? pnp->driver_name : "<unknown>", + pnp->name ? pnp->name : "<unknown>"); + erts_free_port_names(pnp); + } + } + else { + ErlNifResourceType* rt; + ASSERT(state->type == ERTS_EV_TYPE_NIF); + ASSERT(state->driver.stop.resource); + rt = state->driver.stop.resource->type; + erts_dsprintf(dsbufp, "resource={%T,%T}\n", rt->module, rt->name); + } } else { erts_dsprintf(dsbufp, "Bad fd in erts_poll()! fd=%d\n", (int) state->fd); @@ -1915,6 +2536,10 @@ static void drv_ev_state_free(void *des) void ERTS_CIO_EXPORT(erts_init_check_io)(void) { + ERTS_CT_ASSERT((INT_MIN & (ERL_NIF_SELECT_STOP_CALLED | + ERL_NIF_SELECT_STOP_SCHEDULED | + ERL_NIF_SELECT_INVALID_EVENT | + ERL_NIF_SELECT_FAILED)) == 0); erts_smp_atomic_init_nob(&erts_check_io_time, 0); erts_smp_atomic_init_nob(&pollset.in_poll_wait, 0); @@ -1939,24 +2564,22 @@ ERTS_CIO_EXPORT(erts_init_check_io)(void) #ifdef ERTS_SMP init_removed_fd_alloc(); pollset.removed_list = NULL; - erts_smp_spinlock_init(&pollset.removed_list_lock, - "pollset_rm_list"); + erts_smp_spinlock_init(&pollset.removed_list_lock, "pollset_rm_list", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO); { - int i; - for (i=0; i<DRV_EV_STATE_LOCK_CNT; i++) { -#ifdef ERTS_ENABLE_LOCK_COUNT - erts_smp_mtx_init_x(&drv_ev_state_locks[i].lck, "drv_ev_state", make_small(i)); -#else - erts_smp_mtx_init(&drv_ev_state_locks[i].lck, "drv_ev_state"); -#endif - } + int i; + for (i=0; i<DRV_EV_STATE_LOCK_CNT; i++) { + erts_smp_mtx_init(&drv_ev_state_locks[i].lck, "drv_ev_state", make_small(i), + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO); + } } #endif #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS max_fds = ERTS_CIO_POLL_MAX_FDS(); erts_smp_atomic_init_nob(&drv_ev_state_len, 0); drv_ev_state = NULL; - erts_smp_mtx_init(&drv_ev_state_grow_lock, "drv_ev_state_grow"); + erts_smp_mtx_init(&drv_ev_state_grow_lock, "drv_ev_state_grow", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO); #else { SafeHashFunctions hf; @@ -1966,10 +2589,11 @@ ERTS_CIO_EXPORT(erts_init_check_io)(void) hf.free = &drv_ev_state_free; num_state_prealloc = 0; state_prealloc_first = NULL; - erts_smp_spinlock_init(&state_prealloc_lock,"state_prealloc"); + erts_smp_spinlock_init(&state_prealloc_lock,"state_prealloc", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO); - safe_hash_init(ERTS_ALC_T_DRV_EV_STATE, &drv_ev_state_tab, "drv_ev_state_tab", - DRV_EV_STATE_HTAB_SIZE, hf); + safe_hash_init(ERTS_ALC_T_DRV_EV_STATE, &drv_ev_state_tab, "drv_ev_state_tab", + ERTS_LOCK_FLAGS_CATEGORY_IO, DRV_EV_STATE_HTAB_SIZE, hf); } #endif } @@ -2321,6 +2945,39 @@ static void doit_erts_check_io_debug(void *vstate, void *vcounters) } } } + else if (state->type == ERTS_EV_TYPE_NIF) { + ErtsResource* r; + erts_printf("enif_select "); + +#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS + if (internal) { + erts_printf("internal "); + err = 1; + } + + if (cio_events == ep_events) { + erts_printf("ev="); + if (print_events(cio_events) != 0) + err = 1; + } + else { + err = 1; + erts_printf("cio_ev="); + print_events(cio_events); + erts_printf(" ep_ev="); + print_events(ep_events); + } +#else + if (print_events(cio_events) != 0) + err = 1; +#endif + erts_printf(" inpid=%T dd_cnt=%b32d", state->driver.nif->in.pid, + state->driver.nif->in.ddeselect_cnt); + erts_printf(" outpid=%T dd_cnt=%b32d", state->driver.nif->out.pid, + state->driver.nif->out.ddeselect_cnt); + r = state->driver.stop.resource; + erts_printf(" resource=%p(%T:%T)", r, r->type->module, r->type->name); + } #if ERTS_CIO_HAVE_DRV_EVENT else if (state->type == ERTS_EV_TYPE_DRV_EV) { Eterm id; @@ -2377,11 +3034,12 @@ static void doit_erts_check_io_debug(void *vstate, void *vcounters) erts_printf("control_type=%d ", (int)state->type); #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS if (cio_events == ep_events) { - erts_printf("ev=0x%b32x", (Uint32) cio_events); + erts_printf("ev="); + print_events(cio_events); } else { - erts_printf("cio_ev=0x%b32x", (Uint32) cio_events); - erts_printf(" ep_ev=0x%b32x", (Uint32) ep_events); + erts_printf("cio_ev="); print_events(cio_events); + erts_printf(" ep_ev="); print_events(ep_events); } #else erts_printf("ev=0x%b32x", (Uint32) cio_events); @@ -2410,7 +3068,7 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(ErtsCheckIoDebugInfo *ciodip) #if ERTS_CIO_HAVE_DRV_EVENT null_des.driver.event = NULL; #endif - null_des.driver.drv_ptr = NULL; + null_des.driver.stop.drv_ptr = NULL; null_des.events = 0; null_des.remove_cnt = 0; null_des.type = ERTS_EV_TYPE_NONE; @@ -2471,3 +3129,12 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(ErtsCheckIoDebugInfo *ciodip) return counters.num_errors; } +#ifdef ERTS_ENABLE_LOCK_COUNT +void ERTS_CIO_EXPORT(erts_lcnt_update_cio_locks)(int enable) { +#ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS + erts_lcnt_enable_hash_lock_count(&drv_ev_state_tab, ERTS_LOCK_FLAGS_CATEGORY_IO, enable); +#else + (void)enable; +#endif +} +#endif /* ERTS_ENABLE_LOCK_COUNT */ diff --git a/erts/emulator/sys/common/erl_check_io.h b/erts/emulator/sys/common/erl_check_io.h index 14f1ea3f43..2d3bb98afa 100644 --- a/erts/emulator/sys/common/erl_check_io.h +++ b/erts/emulator/sys/common/erl_check_io.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2006-2016. All Rights Reserved. + * Copyright Ericsson AB 2006-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -34,6 +34,8 @@ int driver_select_kp(ErlDrvPort, ErlDrvEvent, int, int); int driver_select_nkp(ErlDrvPort, ErlDrvEvent, int, int); +int enif_select_kp(ErlNifEnv*, ErlNifEvent, enum ErlNifSelectFlags, void*, const ErlNifPid*, Eterm); +int enif_select_nkp(ErlNifEnv*, ErlNifEvent, enum ErlNifSelectFlags, void*, const ErlNifPid*, Eterm); int driver_event_kp(ErlDrvPort, ErlDrvEvent, ErlDrvEventData); int driver_event_nkp(ErlDrvPort, ErlDrvEvent, ErlDrvEventData); Uint erts_check_io_size_kp(void); @@ -57,6 +59,11 @@ void erts_init_check_io_nkp(void); int erts_check_io_debug_kp(ErtsCheckIoDebugInfo *); int erts_check_io_debug_nkp(ErtsCheckIoDebugInfo *); +#ifdef ERTS_ENABLE_LOCK_COUNT +void erts_lcnt_update_cio_locks_kp(int enable); +void erts_lcnt_update_cio_locks_nkp(int enable); +#endif + #else /* !ERTS_ENABLE_KERNEL_POLL */ Uint erts_check_io_size(void); @@ -70,6 +77,10 @@ void erts_check_io_interrupt_timed(int, ErtsMonotonicTime); void erts_check_io(int); void erts_init_check_io(void); +#ifdef ERTS_ENABLE_LOCK_COUNT +void erts_lcnt_update_cio_locks(int enable); +#endif + #endif extern erts_smp_atomic_t erts_check_io_time; @@ -136,4 +147,20 @@ typedef struct { ErtsIoTask iniotask; ErtsIoTask outiotask; } ErtsDrvSelectDataState; + +struct erts_nif_select_event { + Eterm pid; + Eterm immed; + Uint32 refn[ERTS_REF_NUMBERS]; + Sint32 ddeselect_cnt; /* 0: No delayed deselect in progress + * 1: Do deselect before next poll + * >1: Countdown of ignored events + */ +}; + +typedef struct { + struct erts_nif_select_event in; + struct erts_nif_select_event out; +} ErtsNifSelectDataState; + #endif /* #ifndef ERL_CHECK_IO_INTERNAL__ */ diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c index a5714f8325..214ed01c82 100644 --- a/erts/emulator/sys/common/erl_mmap.c +++ b/erts/emulator/sys/common/erl_mmap.c @@ -21,6 +21,7 @@ # include "config.h" #endif +#define ERTS_WANT_MEM_MAPPERS #include "sys.h" #include "erl_process.h" #include "erl_smp.h" @@ -358,12 +359,11 @@ char* erts_literals_start; UWord erts_literals_size; #endif -#ifdef ERTS_ALC_A_EXEC +#ifdef ERTS_HAVE_EXEC_MMAPPER ErtsMemMapper erts_exec_mmapper; #endif - #define ERTS_MMAP_SIZE_SC_SA_INC(SZ) \ do { \ mm->size.supercarrier.used.total += (SZ); \ @@ -1878,7 +1878,7 @@ erts_mremap(ErtsMemMapper* mm, return NULL; } -#if ERTS_HAVE_OS_MREMAP || ERTS_HAVE_GENUINE_OS_MMAP +#if defined(ERTS_HAVE_OS_MREMAP) || defined(ERTS_HAVE_GENUINE_OS_MMAP) superaligned = (ERTS_MMAPFLG_SUPERALIGNED & flags); if (superaligned) { @@ -1898,7 +1898,7 @@ erts_mremap(ErtsMemMapper* mm, } } -#if ERTS_HAVE_GENUINE_OS_MMAP +#ifdef ERTS_HAVE_GENUINE_OS_MMAP if (asize < old_size && (!superaligned || ERTS_IS_SUPERALIGNED(ptr))) { @@ -1913,7 +1913,7 @@ erts_mremap(ErtsMemMapper* mm, return ptr; } #endif -#if ERTS_HAVE_OS_MREMAP +#ifdef ERTS_HAVE_OS_MREMAP if (superaligned) return remap_move(mm, flags, new_ptr, old_size, sizep); else { @@ -2212,9 +2212,11 @@ erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init, int executable) erts_exit(1, "erts_mmap: Failed to open /dev/zero\n"); #endif - erts_smp_mtx_init(&mm->mtx, "erts_mmap"); + erts_smp_mtx_init(&mm->mtx, "erts_mmap", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC); if (is_first_call) { - erts_mtx_init(&am.init_mutex, "mmap_init_atoms"); + erts_mtx_init(&am.init_mutex, "mmap_init_atoms", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC); } #ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION @@ -2816,7 +2818,8 @@ static void hard_dbg_mseg_init(void) { ErtsFreeSegDesc_fake* p; - erts_mtx_init(&hard_dbg_mseg_mtx, "hard_dbg_mseg"); + erts_mtx_init(&hard_dbg_mseg_mtx, "hard_dbg_mseg", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DEBUG); hard_dbg_mseg_tree.root = NULL; hard_dbg_mseg_tree.order = ADDR_ORDER; diff --git a/erts/emulator/sys/common/erl_mmap.h b/erts/emulator/sys/common/erl_mmap.h index 623b5188ac..2a07d93c8c 100644 --- a/erts/emulator/sys/common/erl_mmap.h +++ b/erts/emulator/sys/common/erl_mmap.h @@ -158,12 +158,23 @@ Eterm erts_mmap_info_options(ErtsMemMapper*, # include "erl_alloc_types.h" extern ErtsMemMapper erts_dflt_mmapper; -# if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) + +# if defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) + +# if defined(ARCH_64) extern ErtsMemMapper erts_literal_mmapper; # endif -# ifdef ERTS_ALC_A_EXEC + +# if defined(ERTS_ALC_A_EXEC) && defined(__x86_64__) + /* + * On x86_64, exec_alloc employs its own super carrier 'erts_exec_mmaper' + * to ensure low memory for HiPE AMD64 small code model. + */ +# define ERTS_HAVE_EXEC_MMAPPER extern ErtsMemMapper erts_exec_mmapper; # endif + +# endif /* ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION */ #endif /* ERTS_WANT_MEM_MAPPERS */ /*#define HARD_DEBUG_MSEG*/ diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c index 968f71211c..d69a79dc2a 100644 --- a/erts/emulator/sys/common/erl_mseg.c +++ b/erts/emulator/sys/common/erl_mseg.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2016. All Rights Reserved. + * Copyright Ericsson AB 2002-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -83,12 +83,13 @@ static const int debruijn[32] = { 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9 }; -#define LOG2(X) (debruijn[((Uint32)(((X) & -(X)) * 0x077CB531U)) >> 27]) +#define LSB(X) (debruijn[((Uint32)(((X) & -(X)) * 0x077CB531U)) >> 27]) #define CACHE_AREAS (32 - MSEG_ALIGN_BITS) /* FIXME: segment sizes > 2 GB result in bogus negative indices */ -#define SIZE_TO_CACHE_AREA_IDX(S) (LOG2((S)) - MSEG_ALIGN_BITS) +/* NOTE: using LSB instead of proper log2 only works if S is a power of 2 */ +#define SIZE_TO_CACHE_AREA_IDX(S) (LSB((S)) - MSEG_ALIGN_BITS) #define MAX_CACHE_SIZE (30) #define MSEG_FLG_IS_2POW(X) ((X) & ERTS_MSEG_FLG_2POW) @@ -379,7 +380,7 @@ static ERTS_INLINE int cache_bless_segment(ErtsMsegAllctr_t *ma, void *seg, UWor ASSERT(!MSEG_FLG_IS_2POW(flags) || (MSEG_FLG_IS_2POW(flags) && MAP_IS_ALIGNED(seg) && IS_2POW(size))); - /* The idea is that sbc caching is prefered over mbc caching. + /* The idea is that sbc caching is preferred over mbc caching. * Blocks are normally allocated in mb carriers and thus cached there. * Large blocks has no such cache and it is up to mseg to cache them to speed things up. */ @@ -1419,9 +1420,10 @@ erts_mseg_init(ErtsMsegInit_t *init) atoms_initialized = 0; - erts_mtx_init(&init_atoms_mutex, "mseg_init_atoms"); + erts_mtx_init(&init_atoms_mutex, "mseg_init_atoms", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC); -#ifdef ERTS_ALC_A_EXEC +#ifdef ERTS_HAVE_EXEC_MMAPPER /* Initialize erts_exec_mapper *FIRST*, to increase probability * of getting low memory for HiPE AMD64's small code model. */ @@ -1448,7 +1450,8 @@ erts_mseg_init(ErtsMsegInit_t *init) ma->is_thread_safe = 0; else { ma->is_thread_safe = 1; - erts_mtx_init(&ma->mtx, "mseg"); + erts_mtx_init(&ma->mtx, "mseg", make_small(i), + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_ALLOCATOR); } ma->is_cache_check_scheduled = 0; diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c index b8a28bcc18..52a8b6a53f 100644 --- a/erts/emulator/sys/common/erl_poll.c +++ b/erts/emulator/sys/common/erl_poll.c @@ -51,7 +51,6 @@ #ifndef WANT_NONBLOCKING # define WANT_NONBLOCKING #endif -#define ERTS_WANT_GOT_SIGUSR1 #include "erl_poll.h" #if ERTS_POLL_USE_KQUEUE @@ -337,7 +336,7 @@ static void fatal_error_async_signal_safe(char *error_str); static int max_fds = -1; static ErtsPollSet pollsets; -static erts_smp_spinlock_t pollsets_lock; +static erts_smp_mtx_t pollsets_lock; #if ERTS_POLL_USE_POLL @@ -2584,7 +2583,8 @@ ERTS_POLL_EXPORT(erts_poll_max_fds)(void) void ERTS_POLL_EXPORT(erts_poll_init)(void) { - erts_smp_spinlock_init(&pollsets_lock, "pollsets_lock"); + erts_smp_mtx_init(&pollsets_lock, "pollsets_lock", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO); pollsets = NULL; errno = 0; @@ -2690,7 +2690,7 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void) #endif #ifdef ERTS_SMP erts_atomic32_init_nob(&ps->polled, 0); - erts_smp_mtx_init(&ps->mtx, "pollset"); + erts_smp_mtx_init(&ps->mtx, "pollset", NIL, ERTS_LOCK_FLAGS_CATEGORY_IO); #endif #if defined(USE_THREADS) || ERTS_POLL_ASYNC_INTERRUPT_SUPPORT erts_atomic32_init_nob(&ps->wakeup_state, (erts_aint32_t) 0); @@ -2732,10 +2732,10 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void) #endif erts_smp_atomic_set_nob(&ps->no_of_user_fds, 0); /* Don't count wakeup pipe and fallback fd */ - erts_smp_spin_lock(&pollsets_lock); + erts_smp_mtx_lock(&pollsets_lock); ps->next = pollsets; pollsets = ps; - erts_smp_spin_unlock(&pollsets_lock); + erts_smp_mtx_unlock(&pollsets_lock); return ps; } @@ -2796,7 +2796,7 @@ ERTS_POLL_EXPORT(erts_poll_destroy_pollset)(ErtsPollSet ps) close(ps->timer_fd); #endif - erts_smp_spin_lock(&pollsets_lock); + erts_smp_mtx_lock(&pollsets_lock); if (ps == pollsets) pollsets = pollsets->next; else { @@ -2806,7 +2806,7 @@ ERTS_POLL_EXPORT(erts_poll_destroy_pollset)(ErtsPollSet ps) ASSERT(ps == prev_ps->next); prev_ps->next = ps->next; } - erts_smp_spin_unlock(&pollsets_lock); + erts_smp_mtx_unlock(&pollsets_lock); erts_free(ERTS_ALC_T_POLLSET, (void *) ps); } @@ -3149,3 +3149,26 @@ print_misc_debug_info(void) } #endif + +#ifdef ERTS_ENABLE_LOCK_COUNT +static void erts_lcnt_enable_pollset_lock_count(ErtsPollSet pollset, int enable) { + if(enable) { + erts_lcnt_install_new_lock_info(&pollset->mtx.lcnt, "pollset_rm", NIL, + ERTS_LOCK_TYPE_MUTEX | ERTS_LOCK_FLAGS_CATEGORY_IO); + } else { + erts_lcnt_uninstall(&pollset->mtx.lcnt); + } +} + +void ERTS_POLL_EXPORT(erts_lcnt_update_pollset_locks)(int enable) { + ErtsPollSet iterator; + + erts_smp_mtx_lock(&pollsets_lock); + + for(iterator = pollsets; iterator != NULL; iterator = iterator->next) { + erts_lcnt_enable_pollset_lock_count(iterator, enable); + } + + erts_smp_mtx_unlock(&pollsets_lock); +} +#endif diff --git a/erts/emulator/sys/common/erl_poll.h b/erts/emulator/sys/common/erl_poll.h index c16122610d..b3b4d79984 100644 --- a/erts/emulator/sys/common/erl_poll.h +++ b/erts/emulator/sys/common/erl_poll.h @@ -260,4 +260,8 @@ void ERTS_POLL_EXPORT(erts_poll_get_selected_events)(ErtsPollSet, int erts_poll_new_table_len(int old_len, int need_len); +#ifdef ERTS_ENABLE_LOCK_COUNT +void ERTS_POLL_EXPORT(erts_lcnt_update_pollset_locks)(int enable); +#endif + #endif /* #ifndef ERL_POLL_H__ */ diff --git a/erts/emulator/sys/common/erl_sys_common_misc.c b/erts/emulator/sys/common/erl_sys_common_misc.c index 79f87eb3a9..ba03a11405 100644 --- a/erts/emulator/sys/common/erl_sys_common_misc.c +++ b/erts/emulator/sys/common/erl_sys_common_misc.c @@ -158,59 +158,50 @@ int sys_double_to_chars_fast(double f, char *buffer, int buffer_size, int decimals, int compact) { - /* Note that some C compilers don't support "static const" propagation - * so we use a defines */ - #define SYS_DOUBLE_RND_CONST 0.55555555555555555 + #define SYS_DOUBLE_RND_CONST 0.5 #define FRAC_SIZE 52 #define EXP_SIZE 11 - #define EXP_MASK ((1ll << EXP_SIZE) - 1) + #define EXP_MASK (((Uint64)1 << EXP_SIZE) - 1) #define MAX_DECIMALS (sizeof(cs_sys_double_pow10) \ / sizeof(cs_sys_double_pow10[0])) - #define FRAC_MASK ((1ll << FRAC_SIZE) - 1) - #define FRAC_MASK2 ((1ll << (FRAC_SIZE + 1)) - 1) - #define MAX_FLOAT (1ll << (FRAC_SIZE+1)) + #define FRAC_MASK (((Uint64)1 << FRAC_SIZE) - 1) + #define FRAC_MASK2 (((Uint64)1 << (FRAC_SIZE + 1)) - 1) + #define MAX_FLOAT ((Uint64)1 << (FRAC_SIZE+1)) static const double cs_sys_double_pow10[] = { - SYS_DOUBLE_RND_CONST / 1ll, - SYS_DOUBLE_RND_CONST / 10ll, - SYS_DOUBLE_RND_CONST / 100ll, - SYS_DOUBLE_RND_CONST / 1000ll, - SYS_DOUBLE_RND_CONST / 10000ll, - SYS_DOUBLE_RND_CONST / 100000ll, - SYS_DOUBLE_RND_CONST / 1000000ll, - SYS_DOUBLE_RND_CONST / 10000000ll, - SYS_DOUBLE_RND_CONST / 100000000ll, - SYS_DOUBLE_RND_CONST / 1000000000ll, - SYS_DOUBLE_RND_CONST / 10000000000ll, - SYS_DOUBLE_RND_CONST / 100000000000ll, - SYS_DOUBLE_RND_CONST / 1000000000000ll, - SYS_DOUBLE_RND_CONST / 10000000000000ll, - SYS_DOUBLE_RND_CONST / 100000000000000ll, - SYS_DOUBLE_RND_CONST / 1000000000000000ll, - SYS_DOUBLE_RND_CONST / 10000000000000000ll, - SYS_DOUBLE_RND_CONST / 100000000000000000ll, - SYS_DOUBLE_RND_CONST / 1000000000000000000ll + SYS_DOUBLE_RND_CONST / 1e0, + SYS_DOUBLE_RND_CONST / 1e1, + SYS_DOUBLE_RND_CONST / 1e2, + SYS_DOUBLE_RND_CONST / 1e3, + SYS_DOUBLE_RND_CONST / 1e4, + SYS_DOUBLE_RND_CONST / 1e5, + SYS_DOUBLE_RND_CONST / 1e6, + SYS_DOUBLE_RND_CONST / 1e7, + SYS_DOUBLE_RND_CONST / 1e8, + SYS_DOUBLE_RND_CONST / 1e9, + SYS_DOUBLE_RND_CONST / 1e10, + SYS_DOUBLE_RND_CONST / 1e11, + SYS_DOUBLE_RND_CONST / 1e12, + SYS_DOUBLE_RND_CONST / 1e13, + SYS_DOUBLE_RND_CONST / 1e14, + SYS_DOUBLE_RND_CONST / 1e15, + SYS_DOUBLE_RND_CONST / 1e16, + SYS_DOUBLE_RND_CONST / 1e17, + SYS_DOUBLE_RND_CONST / 1e18 }; - long long mantissa, int_part = 0, frac_part = 0; - short exp; + Uint64 mantissa, int_part, frac_part; + int exp; + int fbits; int max; int neg; double fr; - union { long long L; double F; } x; + union { Uint64 L; double F; } x; char *p = buffer; if (decimals < 0) return -1; - /* Round the number to given decimal places. The number of 5's in the - * SYS_DOUBLE_RND_CONST constant is chosen such that adding any more 5's doesn't - * change the double precision of the number, i.e.: - * 1> term_to_binary(0.55555555555555555, [{minor_version, 1}]). - * <<131,70,63,225,199,28,113,199,28,114>> - * 2> term_to_binary(0.5555555555555555555, [{minor_version, 1}]). - * <<131,70,63,225,199,28,113,199,28,114>> - */ if (f >= 0) { neg = 0; fr = decimals < MAX_DECIMALS ? (f + cs_sys_double_pow10[decimals]) : f; @@ -241,7 +232,7 @@ sys_double_to_chars_fast(double f, char *buffer, int buffer_size, int decimals, } exp -= EXP_MASK >> 1; - mantissa |= (1ll << FRAC_SIZE); + mantissa |= ((Uint64)1 << FRAC_SIZE); /* Don't bother with optimizing too large numbers or too large precision */ if (x.F > MAX_FLOAT || decimals >= MAX_DECIMALS) { @@ -256,11 +247,16 @@ sys_double_to_chars_fast(double f, char *buffer, int buffer_size, int decimals, return p - buffer; } else if (exp >= FRAC_SIZE) { int_part = mantissa << (exp - FRAC_SIZE); + frac_part = 0; + fbits = FRAC_SIZE; /* not important as frac_part==0 */ } else if (exp >= 0) { - int_part = mantissa >> (FRAC_SIZE - exp); - frac_part = (mantissa << (exp + 1)) & FRAC_MASK2; + fbits = FRAC_SIZE - exp; + int_part = mantissa >> fbits; + frac_part = mantissa & (((Uint64)1 << fbits) -1); } else /* if (exp < 0) */ { - frac_part = (mantissa & FRAC_MASK2) >> -(exp + 1); + int_part = 0; + frac_part = mantissa; + fbits = FRAC_SIZE - exp; } if (!int_part) { @@ -270,9 +266,8 @@ sys_double_to_chars_fast(double f, char *buffer, int buffer_size, int decimals, } else { int ret, i, n; while (int_part != 0) { - long long j = int_part / 10; - *p++ = (char)(int_part - ((j << 3) + (j << 1)) + '0'); - int_part = j; + *p++ = (char)((int_part % 10) + '0'); + int_part /= 10; } if (neg) *p++ = '-'; @@ -298,11 +293,22 @@ sys_double_to_chars_fast(double f, char *buffer, int buffer_size, int decimals, max = decimals; for (i = 0; i < max; i++) { - /* frac_part *= 10; */ - frac_part = (frac_part << 3) + (frac_part << 1); - - *p++ = (char)((frac_part >> (FRAC_SIZE + 1)) + '0'); - frac_part &= FRAC_MASK2; + if (frac_part > (ERTS_UINT64_MAX/5)) { + frac_part >>= 3; + fbits -= 3; + } + + /* Multiply by 10 (5*2) to extract decimal digit as integer part */ + frac_part *= 5; + fbits--; + + if (fbits >= 64) { + *p++ = '0'; + } + else { + *p++ = (char)((frac_part >> fbits) + '0'); + frac_part &= ((Uint64)1 << fbits) - 1; + } } /* Delete trailing zeroes */ |