diff options
Diffstat (limited to 'erts/emulator/sys')
| -rw-r--r-- | erts/emulator/sys/common/erl_check_io.c | 283 | ||||
| -rw-r--r-- | erts/emulator/sys/common/erl_check_io.h | 3 | ||||
| -rw-r--r-- | erts/emulator/sys/common/erl_mmap.h | 57 | ||||
| -rw-r--r-- | erts/emulator/sys/win32/sys.c | 4 |
4 files changed, 231 insertions, 116 deletions
diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c index c39cd01e1c..e367087455 100644 --- a/erts/emulator/sys/common/erl_check_io.c +++ b/erts/emulator/sys/common/erl_check_io.c @@ -591,6 +591,96 @@ abort_tasks(ErtsDrvEventState *state, int mode) } } +static void prepare_select_msg(struct erts_nif_select_event* e, + enum ErlNifSelectFlags mode, + Eterm recipient, + ErtsResource* resource, + Eterm msg, + ErlNifEnv* msg_env, + Eterm event_atom) +{ + ErtsMessage* mp; + Eterm* hp; + Uint hsz; + + if (is_not_nil(e->pid)) { + ASSERT(e->mp); + erts_cleanup_messages(e->mp); + } + + if (mode & ERL_NIF_SELECT_CUSTOM_MSG) { + if (msg_env) { + mp = erts_create_message_from_nif_env(msg_env); + ERL_MESSAGE_TERM(mp) = msg; + } + else { + hsz = size_object(msg); + mp = erts_alloc_message(hsz, &hp); + ERL_MESSAGE_TERM(mp) = copy_struct(msg, hsz, &hp, &mp->hfrag.off_heap); + } + } + else { + ErtsBinary* bin; + Eterm resource_term, ref_term, tuple; + Eterm* hp_start; + + /* {select, Resource, Ref, EventAtom} */ + hsz = 5 + ERTS_MAGIC_REF_THING_SIZE; + if (is_internal_ref(msg)) + hsz += ERTS_REF_THING_SIZE; + else + ASSERT(is_immed(msg)); + + mp = erts_alloc_message(hsz, &hp); + hp_start = hp; + + bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource); + resource_term = erts_mk_magic_ref(&hp, &mp->hfrag.off_heap, &bin->binary); + if (is_internal_ref(msg)) { + Uint32* refn = internal_ref_numbers(msg); + write_ref_thing(hp, refn[0], refn[1], refn[2]); + ref_term = make_internal_ref(hp); + hp += ERTS_REF_THING_SIZE; + } + else { + ASSERT(is_immed(msg)); + ref_term = msg; + } + tuple = TUPLE4(hp, am_select, resource_term, ref_term, event_atom); + hp += 5; + ERL_MESSAGE_TERM(mp) = tuple; + ASSERT(hp == hp_start + hsz); (void)hp_start; + } + + ASSERT(is_not_nil(recipient)); + e->pid = recipient; + e->mp = mp; +} + +static ERTS_INLINE void send_select_msg(struct erts_nif_select_event* e) +{ + Process* rp = erts_proc_lookup(e->pid); + + ASSERT(is_internal_pid(e->pid)); + if (!rp) { + erts_cleanup_messages(e->mp); + return; + } + + erts_queue_message(rp, 0, e->mp, ERL_MESSAGE_TERM(e->mp), am_system); +} + +static void clear_select_event(struct erts_nif_select_event* e) +{ + if (is_not_nil(e->pid)) { + /* Discard unsent message */ + ASSERT(e->mp); + erts_cleanup_messages(e->mp); + e->mp = NULL; + e->pid = NIL; + } +} + static void deselect(ErtsDrvEventState *state, int mode) { @@ -621,8 +711,8 @@ deselect(ErtsDrvEventState *state, int mode) erts_io_control(state, ERTS_POLL_OP_DEL, 0); switch (state->type) { case ERTS_EV_TYPE_NIF: - state->driver.nif->in.pid = NIL; - state->driver.nif->out.pid = NIL; + clear_select_event(&state->driver.nif->in); + clear_select_event(&state->driver.nif->out); enif_release_resource(state->driver.stop.resource->data); state->driver.stop.resource = NULL; break; @@ -943,12 +1033,21 @@ done_unknown: } int -enif_select(ErlNifEnv* env, - ErlNifEvent e, - enum ErlNifSelectFlags mode, - void* obj, - const ErlNifPid* pid, - Eterm ref) +enif_select(ErlNifEnv* env, ErlNifEvent e, enum ErlNifSelectFlags mode, + void* obj, const ErlNifPid* pid, Eterm msg) +{ + return enif_select_x(env, e, mode, obj, pid, msg, NULL); +} + + +int +enif_select_x(ErlNifEnv* env, + ErlNifEvent e, + enum ErlNifSelectFlags mode, + void* obj, + const ErlNifPid* pid, + Eterm msg, + ErlNifEnv* msg_env) { int on; ErtsResource* resource = DATA_TO_RESOURCE(obj); @@ -966,7 +1065,7 @@ enif_select(ErlNifEnv* env, #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS if (!grow_drv_ev_state(fd)) { - if (fd > 0) nif_select_large_fd_error(fd, mode, resource, ref); + if (fd > 0) nif_select_large_fd_error(fd, mode, resource, msg); return INT_MIN | ERL_NIF_SELECT_INVALID_EVENT; } #endif @@ -993,7 +1092,7 @@ enif_select(ErlNifEnv* env, ctl_op = ERTS_POLL_OP_DEL; } else { - on = 1; + on = !(mode & ERL_NIF_SELECT_CANCEL); ASSERT(mode); if (mode & ERL_DRV_READ) { ctl_events |= ERTS_POLL_EV_IN; @@ -1012,21 +1111,21 @@ enif_select(ErlNifEnv* env, * Changing process and/or ref is ok (I think?). */ if (state->driver.stop.resource != resource) - nif_select_steal(state, ERL_DRV_READ | ERL_DRV_WRITE, resource, ref); + nif_select_steal(state, ERL_DRV_READ | ERL_DRV_WRITE, resource, msg); break; case ERTS_EV_TYPE_DRV_SEL: - nif_select_steal(state, mode, resource, ref); + nif_select_steal(state, mode, resource, msg); break; case ERTS_EV_TYPE_STOP_USE: { erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - print_nif_select_op(dsbufp, fd, mode, resource, ref); + print_nif_select_op(dsbufp, fd, mode, resource, msg); steal_pending_stop_use(dsbufp, ERTS_INVALID_ERL_DRV_PORT, state, mode, on); ASSERT(state->type == ERTS_EV_TYPE_NONE); break; } case ERTS_EV_TYPE_STOP_NIF: { erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - print_nif_select_op(dsbufp, fd, mode, resource, ref); + print_nif_select_op(dsbufp, fd, mode, resource, msg); steal_pending_stop_nif(dsbufp, resource, state, mode, on); if (state->type == ERTS_EV_TYPE_STOP_NIF) { ret = ERL_NIF_SELECT_STOP_SCHEDULED; /* ?? */ @@ -1082,7 +1181,6 @@ enif_select(ErlNifEnv* env, if (on) { const Eterm recipient = pid ? pid->pid : env->proc->common.id; - Uint32* refn; if (!state->driver.nif) state->driver.nif = alloc_nif_select_data(); if (state->type == ERTS_EV_TYPE_NONE) { @@ -1093,64 +1191,62 @@ enif_select(ErlNifEnv* env, ASSERT(state->type == ERTS_EV_TYPE_NIF); ASSERT(state->driver.stop.resource == resource); if (mode & ERL_DRV_READ) { - state->driver.nif->in.pid = recipient; - if (is_immed(ref)) { - state->driver.nif->in.immed = ref; - } else { - ASSERT(is_internal_ref(ref)); - refn = internal_ref_numbers(ref); - state->driver.nif->in.immed = THE_NON_VALUE; - sys_memcpy(state->driver.nif->in.refn, refn, - sizeof(state->driver.nif->in.refn)); - } + prepare_select_msg(&state->driver.nif->in, mode, recipient, + resource, msg, msg_env, am_ready_input); + msg_env = NULL; } if (mode & ERL_DRV_WRITE) { - state->driver.nif->out.pid = recipient; - if (is_immed(ref)) { - state->driver.nif->out.immed = ref; - } else { - ASSERT(is_internal_ref(ref)); - refn = internal_ref_numbers(ref); - state->driver.nif->out.immed = THE_NON_VALUE; - sys_memcpy(state->driver.nif->out.refn, refn, - sizeof(state->driver.nif->out.refn)); - } + prepare_select_msg(&state->driver.nif->out, mode, recipient, + resource, msg, msg_env, am_ready_output); } ret = 0; } else { /* off */ + ret = 0; if (state->type == ERTS_EV_TYPE_NIF) { - state->driver.nif->in.pid = NIL; - state->driver.nif->out.pid = NIL; - } - ASSERT(state->events==0); - if (!wake_poller) { - /* - * Safe to close fd now as it is not in pollset - * or there was no need to eject fd (kernel poll) - */ - if (state->type == ERTS_EV_TYPE_NIF) { - ASSERT(state->driver.stop.resource == resource); - call_stop = CALL_STOP_AND_RELEASE; - state->driver.stop.resource = NULL; + if (mode & ERL_NIF_SELECT_READ + && is_not_nil(state->driver.nif->in.pid)) { + clear_select_event(&state->driver.nif->in); + ret |= ERL_NIF_SELECT_READ_CANCELLED; } - else { - ASSERT(!state->driver.stop.resource); - call_stop = CALL_STOP; + if (mode & ERL_NIF_SELECT_WRITE + && is_not_nil(state->driver.nif->out.pid)) { + clear_select_event(&state->driver.nif->out); + ret |= ERL_NIF_SELECT_WRITE_CANCELLED; } - state->type = ERTS_EV_TYPE_NONE; - ret = ERL_NIF_SELECT_STOP_CALLED; } - else { - /* Not safe to close fd, postpone stop_select callback. */ - if (state->type == ERTS_EV_TYPE_NONE) { - ASSERT(!state->driver.stop.resource); - state->driver.stop.resource = resource; - enif_keep_resource(resource); + if (mode & ERL_NIF_SELECT_STOP) { + ASSERT(state->events==0); + if (!wake_poller) { + /* + * Safe to close fd now as it is not in pollset + * or there was no need to eject fd (kernel poll) + */ + if (state->type == ERTS_EV_TYPE_NIF) { + ASSERT(state->driver.stop.resource == resource); + call_stop = CALL_STOP_AND_RELEASE; + state->driver.stop.resource = NULL; + } + else { + ASSERT(!state->driver.stop.resource); + call_stop = CALL_STOP; + } + state->type = ERTS_EV_TYPE_NONE; + ret |= ERL_NIF_SELECT_STOP_CALLED; + } + else { + /* Not safe to close fd, postpone stop_select callback. */ + if (state->type == ERTS_EV_TYPE_NONE) { + ASSERT(!state->driver.stop.resource); + state->driver.stop.resource = resource; + enif_keep_resource(resource); + } + state->type = ERTS_EV_TYPE_STOP_NIF; + ret |= ERL_NIF_SELECT_STOP_SCHEDULED; } - state->type = ERTS_EV_TYPE_STOP_NIF; - ret = ERL_NIF_SELECT_STOP_SCHEDULED; } + else + ASSERT(mode & ERL_NIF_SELECT_CANCEL); } done: @@ -1328,7 +1424,8 @@ print_nif_select_op(erts_dsprintf_buf_t *dsbufp, (int) fd, mode & ERL_NIF_SELECT_READ ? " READ" : "", mode & ERL_NIF_SELECT_WRITE ? " WRITE" : "", - mode & ERL_NIF_SELECT_STOP ? " STOP" : "", + (mode & ERL_NIF_SELECT_STOP ? " STOP" + : (mode & ERL_NIF_SELECT_CANCEL ? " CANCEL" : "")), resource->type->module, resource->type->name, ref); @@ -1531,53 +1628,6 @@ oready(Eterm id, ErtsDrvEventState *state) } } -static ERTS_INLINE void -send_event_tuple(struct erts_nif_select_event* e, ErtsResource* resource, - Eterm event_atom) -{ - Process* rp = erts_proc_lookup(e->pid); - ErtsProcLocks rp_locks = 0; - ErtsMessage* mp; - ErlOffHeap* ohp; - ErtsBinary* bin; - Eterm* hp; - Uint hsz; - Eterm resource_term, ref_term, tuple; - - if (!rp) { - return; - } - - bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource); - - /* {select, Resource, Ref, EventAtom} */ - if (is_value(e->immed)) { - hsz = 5 + ERTS_MAGIC_REF_THING_SIZE; - } - else { - hsz = 5 + ERTS_MAGIC_REF_THING_SIZE + ERTS_REF_THING_SIZE; - } - - mp = erts_alloc_message_heap(rp, &rp_locks, hsz, &hp, &ohp); - - resource_term = erts_mk_magic_ref(&hp, ohp, &bin->binary); - if (is_value(e->immed)) { - ASSERT(is_immed(e->immed)); - ref_term = e->immed; - } - else { - write_ref_thing(hp, e->refn[0], e->refn[1], e->refn[2]); - ref_term = make_internal_ref(hp); - hp += ERTS_REF_THING_SIZE; - } - tuple = TUPLE4(hp, am_select, resource_term, ref_term, event_atom); - - erts_queue_message(rp, rp_locks, mp, tuple, am_system); - - if (rp_locks) - erts_proc_unlock(rp, rp_locks); -} - static void bad_fd_in_pollset(ErtsDrvEventState *, Eterm inport, Eterm outport); void @@ -1751,7 +1801,6 @@ erts_check_io(ErtsPollThread *psi, ErtsMonotonicTime timeout_time) case ERTS_EV_TYPE_NIF: { /* Requested via enif_select()... */ struct erts_nif_select_event in = {NIL}; struct erts_nif_select_event out = {NIL}; - ErtsResource* resource = NULL; if (revents & (ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT)) { if (revents & ERTS_POLL_EV_OUT) { @@ -1759,6 +1808,7 @@ erts_check_io(ErtsPollThread *psi, ErtsMonotonicTime timeout_time) out = state->driver.nif->out; resource = state->driver.stop.resource; state->driver.nif->out.pid = NIL; + state->driver.nif->out.mp = NULL; } } if (revents & ERTS_POLL_EV_IN) { @@ -1766,6 +1816,7 @@ erts_check_io(ErtsPollThread *psi, ErtsMonotonicTime timeout_time) in = state->driver.nif->in; resource = state->driver.stop.resource; state->driver.nif->in.pid = NIL; + state->driver.nif->in.mp = NULL; } } state->events &= ~revents; @@ -1778,10 +1829,10 @@ erts_check_io(ErtsPollThread *psi, ErtsMonotonicTime timeout_time) erts_mtx_unlock(fd_mtx(fd)); if (is_not_nil(in.pid)) { - send_event_tuple(&in, resource, am_ready_input); + send_select_msg(&in); } if (is_not_nil(out.pid)) { - send_event_tuple(&out, resource, am_ready_output); + send_select_msg(&out); } continue; } @@ -2448,10 +2499,16 @@ drvmode2str(int mode) { static ERTS_INLINE char * nifmode2str(enum ErlNifSelectFlags mode) { + if (mode & ERL_NIF_SELECT_STOP) + return "STOP"; switch (mode) { case ERL_NIF_SELECT_READ: return "READ"; case ERL_NIF_SELECT_WRITE: return "WRITE"; - case ERL_NIF_SELECT_STOP: return "STOP"; + case ERL_NIF_SELECT_READ|ERL_NIF_SELECT_WRITE: return "READ|WRITE"; + case ERL_NIF_SELECT_CANCEL|ERL_NIF_SELECT_READ: return "CANCEL|READ"; + case ERL_NIF_SELECT_CANCEL|ERL_NIF_SELECT_WRITE: return "CANCEL|WRITE"; + case ERL_NIF_SELECT_CANCEL|ERL_NIF_SELECT_READ|ERL_NIF_SELECT_WRITE: + return "CANCEL|READ|WRITE"; default: return "UNKNOWN"; } } diff --git a/erts/emulator/sys/common/erl_check_io.h b/erts/emulator/sys/common/erl_check_io.h index 31182be5ec..0f3fc4f7a2 100644 --- a/erts/emulator/sys/common/erl_check_io.h +++ b/erts/emulator/sys/common/erl_check_io.h @@ -138,8 +138,7 @@ typedef struct { struct erts_nif_select_event { Eterm pid; - Eterm immed; - Uint32 refn[ERTS_REF_NUMBERS]; + ErtsMessage *mp; }; typedef struct { diff --git a/erts/emulator/sys/common/erl_mmap.h b/erts/emulator/sys/common/erl_mmap.h index 539daea419..e1ff0fe80a 100644 --- a/erts/emulator/sys/common/erl_mmap.h +++ b/erts/emulator/sys/common/erl_mmap.h @@ -176,4 +176,61 @@ void hard_dbg_remove_mseg(void* seg, UWord sz); #endif /* HAVE_ERTS_MMAP */ +/* Marks the given memory region as unused without freeing it, letting the OS + * reclaim its physical memory with the promise that we'll get it back (without + * its contents) the next time it's accessed. */ +ERTS_GLB_INLINE void erts_mem_discard(void *p, UWord size); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +#ifdef VALGRIND + #include <valgrind/memcheck.h> + + ERTS_GLB_INLINE void erts_mem_discard(void *ptr, UWord size) { + VALGRIND_MAKE_MEM_UNDEFINED(ptr, size); + } +#elif defined(DEBUG) + /* Try to provoke crashes by filling the discard region with garbage. It's + * extremely hard to find bugs where we've discarded too much, as the + * region often retains its old contents if it's accessed before the OS + * reclaims it. */ + ERTS_GLB_INLINE void erts_mem_discard(void *ptr, UWord size) { + static const char pattern[] = "DISCARDED"; + char *data; + int i; + + for(i = 0, data = ptr; i < size; i++) { + data[i] = pattern[i % sizeof(pattern)]; + } + } +#elif defined(HAVE_SYS_MMAN_H) + #include <sys/mman.h> + + ERTS_GLB_INLINE void erts_mem_discard(void *ptr, UWord size) { + #ifdef MADV_FREE + /* This is preferred as it doesn't necessarily free the pages right + * away, which is a bit faster than MADV_DONTNEED. */ + madvise(ptr, size, MADV_FREE); + #else + madvise(ptr, size, MADV_DONTNEED); + #endif + } +#elif defined(_WIN32) + #include <winbase.h> + + /* MEM_RESET is defined on all supported versions of Windows, and has the + * same semantics as MADV_FREE. */ + ERTS_GLB_INLINE void erts_mem_discard(void *ptr, UWord size) { + VirtualAlloc(ptr, size, MEM_RESET, PAGE_READWRITE); + } +#else + /* Dummy implementation. */ + ERTS_GLB_INLINE void erts_mem_discard(void *ptr, UWord size) { + (void)ptr; + (void)size; + } +#endif + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + #endif /* ERL_MMAP_H__ */ diff --git a/erts/emulator/sys/win32/sys.c b/erts/emulator/sys/win32/sys.c index a1c630d68a..b95aadc9b2 100644 --- a/erts/emulator/sys/win32/sys.c +++ b/erts/emulator/sys/win32/sys.c @@ -186,7 +186,9 @@ void sys_primitive_init(HMODULE beam) UWord erts_sys_get_page_size(void) { - return (UWord) 4*1024; /* Guess 4 KB */ + SYSTEM_INFO info; + GetSystemInfo(&info); + return (UWord)info.dwPageSize; } Uint |