aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator/sys
diff options
context:
space:
mode:
Diffstat (limited to 'erts/emulator/sys')
-rw-r--r--erts/emulator/sys/common/erl_check_io.c283
-rw-r--r--erts/emulator/sys/common/erl_check_io.h3
-rw-r--r--erts/emulator/sys/common/erl_mmap.h57
-rw-r--r--erts/emulator/sys/win32/sys.c4
4 files changed, 231 insertions, 116 deletions
diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index c39cd01e1c..e367087455 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -591,6 +591,96 @@ abort_tasks(ErtsDrvEventState *state, int mode)
}
}
+static void prepare_select_msg(struct erts_nif_select_event* e,
+ enum ErlNifSelectFlags mode,
+ Eterm recipient,
+ ErtsResource* resource,
+ Eterm msg,
+ ErlNifEnv* msg_env,
+ Eterm event_atom)
+{
+ ErtsMessage* mp;
+ Eterm* hp;
+ Uint hsz;
+
+ if (is_not_nil(e->pid)) {
+ ASSERT(e->mp);
+ erts_cleanup_messages(e->mp);
+ }
+
+ if (mode & ERL_NIF_SELECT_CUSTOM_MSG) {
+ if (msg_env) {
+ mp = erts_create_message_from_nif_env(msg_env);
+ ERL_MESSAGE_TERM(mp) = msg;
+ }
+ else {
+ hsz = size_object(msg);
+ mp = erts_alloc_message(hsz, &hp);
+ ERL_MESSAGE_TERM(mp) = copy_struct(msg, hsz, &hp, &mp->hfrag.off_heap);
+ }
+ }
+ else {
+ ErtsBinary* bin;
+ Eterm resource_term, ref_term, tuple;
+ Eterm* hp_start;
+
+ /* {select, Resource, Ref, EventAtom} */
+ hsz = 5 + ERTS_MAGIC_REF_THING_SIZE;
+ if (is_internal_ref(msg))
+ hsz += ERTS_REF_THING_SIZE;
+ else
+ ASSERT(is_immed(msg));
+
+ mp = erts_alloc_message(hsz, &hp);
+ hp_start = hp;
+
+ bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource);
+ resource_term = erts_mk_magic_ref(&hp, &mp->hfrag.off_heap, &bin->binary);
+ if (is_internal_ref(msg)) {
+ Uint32* refn = internal_ref_numbers(msg);
+ write_ref_thing(hp, refn[0], refn[1], refn[2]);
+ ref_term = make_internal_ref(hp);
+ hp += ERTS_REF_THING_SIZE;
+ }
+ else {
+ ASSERT(is_immed(msg));
+ ref_term = msg;
+ }
+ tuple = TUPLE4(hp, am_select, resource_term, ref_term, event_atom);
+ hp += 5;
+ ERL_MESSAGE_TERM(mp) = tuple;
+ ASSERT(hp == hp_start + hsz); (void)hp_start;
+ }
+
+ ASSERT(is_not_nil(recipient));
+ e->pid = recipient;
+ e->mp = mp;
+}
+
+static ERTS_INLINE void send_select_msg(struct erts_nif_select_event* e)
+{
+ Process* rp = erts_proc_lookup(e->pid);
+
+ ASSERT(is_internal_pid(e->pid));
+ if (!rp) {
+ erts_cleanup_messages(e->mp);
+ return;
+ }
+
+ erts_queue_message(rp, 0, e->mp, ERL_MESSAGE_TERM(e->mp), am_system);
+}
+
+static void clear_select_event(struct erts_nif_select_event* e)
+{
+ if (is_not_nil(e->pid)) {
+ /* Discard unsent message */
+ ASSERT(e->mp);
+ erts_cleanup_messages(e->mp);
+ e->mp = NULL;
+ e->pid = NIL;
+ }
+}
+
static void
deselect(ErtsDrvEventState *state, int mode)
{
@@ -621,8 +711,8 @@ deselect(ErtsDrvEventState *state, int mode)
erts_io_control(state, ERTS_POLL_OP_DEL, 0);
switch (state->type) {
case ERTS_EV_TYPE_NIF:
- state->driver.nif->in.pid = NIL;
- state->driver.nif->out.pid = NIL;
+ clear_select_event(&state->driver.nif->in);
+ clear_select_event(&state->driver.nif->out);
enif_release_resource(state->driver.stop.resource->data);
state->driver.stop.resource = NULL;
break;
@@ -943,12 +1033,21 @@ done_unknown:
}
int
-enif_select(ErlNifEnv* env,
- ErlNifEvent e,
- enum ErlNifSelectFlags mode,
- void* obj,
- const ErlNifPid* pid,
- Eterm ref)
+enif_select(ErlNifEnv* env, ErlNifEvent e, enum ErlNifSelectFlags mode,
+ void* obj, const ErlNifPid* pid, Eterm msg)
+{
+ return enif_select_x(env, e, mode, obj, pid, msg, NULL);
+}
+
+
+int
+enif_select_x(ErlNifEnv* env,
+ ErlNifEvent e,
+ enum ErlNifSelectFlags mode,
+ void* obj,
+ const ErlNifPid* pid,
+ Eterm msg,
+ ErlNifEnv* msg_env)
{
int on;
ErtsResource* resource = DATA_TO_RESOURCE(obj);
@@ -966,7 +1065,7 @@ enif_select(ErlNifEnv* env,
#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
if (!grow_drv_ev_state(fd)) {
- if (fd > 0) nif_select_large_fd_error(fd, mode, resource, ref);
+ if (fd > 0) nif_select_large_fd_error(fd, mode, resource, msg);
return INT_MIN | ERL_NIF_SELECT_INVALID_EVENT;
}
#endif
@@ -993,7 +1092,7 @@ enif_select(ErlNifEnv* env,
ctl_op = ERTS_POLL_OP_DEL;
}
else {
- on = 1;
+ on = !(mode & ERL_NIF_SELECT_CANCEL);
ASSERT(mode);
if (mode & ERL_DRV_READ) {
ctl_events |= ERTS_POLL_EV_IN;
@@ -1012,21 +1111,21 @@ enif_select(ErlNifEnv* env,
* Changing process and/or ref is ok (I think?).
*/
if (state->driver.stop.resource != resource)
- nif_select_steal(state, ERL_DRV_READ | ERL_DRV_WRITE, resource, ref);
+ nif_select_steal(state, ERL_DRV_READ | ERL_DRV_WRITE, resource, msg);
break;
case ERTS_EV_TYPE_DRV_SEL:
- nif_select_steal(state, mode, resource, ref);
+ nif_select_steal(state, mode, resource, msg);
break;
case ERTS_EV_TYPE_STOP_USE: {
erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
- print_nif_select_op(dsbufp, fd, mode, resource, ref);
+ print_nif_select_op(dsbufp, fd, mode, resource, msg);
steal_pending_stop_use(dsbufp, ERTS_INVALID_ERL_DRV_PORT, state, mode, on);
ASSERT(state->type == ERTS_EV_TYPE_NONE);
break;
}
case ERTS_EV_TYPE_STOP_NIF: {
erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
- print_nif_select_op(dsbufp, fd, mode, resource, ref);
+ print_nif_select_op(dsbufp, fd, mode, resource, msg);
steal_pending_stop_nif(dsbufp, resource, state, mode, on);
if (state->type == ERTS_EV_TYPE_STOP_NIF) {
ret = ERL_NIF_SELECT_STOP_SCHEDULED; /* ?? */
@@ -1082,7 +1181,6 @@ enif_select(ErlNifEnv* env,
if (on) {
const Eterm recipient = pid ? pid->pid : env->proc->common.id;
- Uint32* refn;
if (!state->driver.nif)
state->driver.nif = alloc_nif_select_data();
if (state->type == ERTS_EV_TYPE_NONE) {
@@ -1093,64 +1191,62 @@ enif_select(ErlNifEnv* env,
ASSERT(state->type == ERTS_EV_TYPE_NIF);
ASSERT(state->driver.stop.resource == resource);
if (mode & ERL_DRV_READ) {
- state->driver.nif->in.pid = recipient;
- if (is_immed(ref)) {
- state->driver.nif->in.immed = ref;
- } else {
- ASSERT(is_internal_ref(ref));
- refn = internal_ref_numbers(ref);
- state->driver.nif->in.immed = THE_NON_VALUE;
- sys_memcpy(state->driver.nif->in.refn, refn,
- sizeof(state->driver.nif->in.refn));
- }
+ prepare_select_msg(&state->driver.nif->in, mode, recipient,
+ resource, msg, msg_env, am_ready_input);
+ msg_env = NULL;
}
if (mode & ERL_DRV_WRITE) {
- state->driver.nif->out.pid = recipient;
- if (is_immed(ref)) {
- state->driver.nif->out.immed = ref;
- } else {
- ASSERT(is_internal_ref(ref));
- refn = internal_ref_numbers(ref);
- state->driver.nif->out.immed = THE_NON_VALUE;
- sys_memcpy(state->driver.nif->out.refn, refn,
- sizeof(state->driver.nif->out.refn));
- }
+ prepare_select_msg(&state->driver.nif->out, mode, recipient,
+ resource, msg, msg_env, am_ready_output);
}
ret = 0;
}
else { /* off */
+ ret = 0;
if (state->type == ERTS_EV_TYPE_NIF) {
- state->driver.nif->in.pid = NIL;
- state->driver.nif->out.pid = NIL;
- }
- ASSERT(state->events==0);
- if (!wake_poller) {
- /*
- * Safe to close fd now as it is not in pollset
- * or there was no need to eject fd (kernel poll)
- */
- if (state->type == ERTS_EV_TYPE_NIF) {
- ASSERT(state->driver.stop.resource == resource);
- call_stop = CALL_STOP_AND_RELEASE;
- state->driver.stop.resource = NULL;
+ if (mode & ERL_NIF_SELECT_READ
+ && is_not_nil(state->driver.nif->in.pid)) {
+ clear_select_event(&state->driver.nif->in);
+ ret |= ERL_NIF_SELECT_READ_CANCELLED;
}
- else {
- ASSERT(!state->driver.stop.resource);
- call_stop = CALL_STOP;
+ if (mode & ERL_NIF_SELECT_WRITE
+ && is_not_nil(state->driver.nif->out.pid)) {
+ clear_select_event(&state->driver.nif->out);
+ ret |= ERL_NIF_SELECT_WRITE_CANCELLED;
}
- state->type = ERTS_EV_TYPE_NONE;
- ret = ERL_NIF_SELECT_STOP_CALLED;
}
- else {
- /* Not safe to close fd, postpone stop_select callback. */
- if (state->type == ERTS_EV_TYPE_NONE) {
- ASSERT(!state->driver.stop.resource);
- state->driver.stop.resource = resource;
- enif_keep_resource(resource);
+ if (mode & ERL_NIF_SELECT_STOP) {
+ ASSERT(state->events==0);
+ if (!wake_poller) {
+ /*
+ * Safe to close fd now as it is not in pollset
+ * or there was no need to eject fd (kernel poll)
+ */
+ if (state->type == ERTS_EV_TYPE_NIF) {
+ ASSERT(state->driver.stop.resource == resource);
+ call_stop = CALL_STOP_AND_RELEASE;
+ state->driver.stop.resource = NULL;
+ }
+ else {
+ ASSERT(!state->driver.stop.resource);
+ call_stop = CALL_STOP;
+ }
+ state->type = ERTS_EV_TYPE_NONE;
+ ret |= ERL_NIF_SELECT_STOP_CALLED;
+ }
+ else {
+ /* Not safe to close fd, postpone stop_select callback. */
+ if (state->type == ERTS_EV_TYPE_NONE) {
+ ASSERT(!state->driver.stop.resource);
+ state->driver.stop.resource = resource;
+ enif_keep_resource(resource);
+ }
+ state->type = ERTS_EV_TYPE_STOP_NIF;
+ ret |= ERL_NIF_SELECT_STOP_SCHEDULED;
}
- state->type = ERTS_EV_TYPE_STOP_NIF;
- ret = ERL_NIF_SELECT_STOP_SCHEDULED;
}
+ else
+ ASSERT(mode & ERL_NIF_SELECT_CANCEL);
}
done:
@@ -1328,7 +1424,8 @@ print_nif_select_op(erts_dsprintf_buf_t *dsbufp,
(int) fd,
mode & ERL_NIF_SELECT_READ ? " READ" : "",
mode & ERL_NIF_SELECT_WRITE ? " WRITE" : "",
- mode & ERL_NIF_SELECT_STOP ? " STOP" : "",
+ (mode & ERL_NIF_SELECT_STOP ? " STOP"
+ : (mode & ERL_NIF_SELECT_CANCEL ? " CANCEL" : "")),
resource->type->module,
resource->type->name,
ref);
@@ -1531,53 +1628,6 @@ oready(Eterm id, ErtsDrvEventState *state)
}
}
-static ERTS_INLINE void
-send_event_tuple(struct erts_nif_select_event* e, ErtsResource* resource,
- Eterm event_atom)
-{
- Process* rp = erts_proc_lookup(e->pid);
- ErtsProcLocks rp_locks = 0;
- ErtsMessage* mp;
- ErlOffHeap* ohp;
- ErtsBinary* bin;
- Eterm* hp;
- Uint hsz;
- Eterm resource_term, ref_term, tuple;
-
- if (!rp) {
- return;
- }
-
- bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource);
-
- /* {select, Resource, Ref, EventAtom} */
- if (is_value(e->immed)) {
- hsz = 5 + ERTS_MAGIC_REF_THING_SIZE;
- }
- else {
- hsz = 5 + ERTS_MAGIC_REF_THING_SIZE + ERTS_REF_THING_SIZE;
- }
-
- mp = erts_alloc_message_heap(rp, &rp_locks, hsz, &hp, &ohp);
-
- resource_term = erts_mk_magic_ref(&hp, ohp, &bin->binary);
- if (is_value(e->immed)) {
- ASSERT(is_immed(e->immed));
- ref_term = e->immed;
- }
- else {
- write_ref_thing(hp, e->refn[0], e->refn[1], e->refn[2]);
- ref_term = make_internal_ref(hp);
- hp += ERTS_REF_THING_SIZE;
- }
- tuple = TUPLE4(hp, am_select, resource_term, ref_term, event_atom);
-
- erts_queue_message(rp, rp_locks, mp, tuple, am_system);
-
- if (rp_locks)
- erts_proc_unlock(rp, rp_locks);
-}
-
static void bad_fd_in_pollset(ErtsDrvEventState *, Eterm inport, Eterm outport);
void
@@ -1751,7 +1801,6 @@ erts_check_io(ErtsPollThread *psi, ErtsMonotonicTime timeout_time)
case ERTS_EV_TYPE_NIF: { /* Requested via enif_select()... */
struct erts_nif_select_event in = {NIL};
struct erts_nif_select_event out = {NIL};
- ErtsResource* resource = NULL;
if (revents & (ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT)) {
if (revents & ERTS_POLL_EV_OUT) {
@@ -1759,6 +1808,7 @@ erts_check_io(ErtsPollThread *psi, ErtsMonotonicTime timeout_time)
out = state->driver.nif->out;
resource = state->driver.stop.resource;
state->driver.nif->out.pid = NIL;
+ state->driver.nif->out.mp = NULL;
}
}
if (revents & ERTS_POLL_EV_IN) {
@@ -1766,6 +1816,7 @@ erts_check_io(ErtsPollThread *psi, ErtsMonotonicTime timeout_time)
in = state->driver.nif->in;
resource = state->driver.stop.resource;
state->driver.nif->in.pid = NIL;
+ state->driver.nif->in.mp = NULL;
}
}
state->events &= ~revents;
@@ -1778,10 +1829,10 @@ erts_check_io(ErtsPollThread *psi, ErtsMonotonicTime timeout_time)
erts_mtx_unlock(fd_mtx(fd));
if (is_not_nil(in.pid)) {
- send_event_tuple(&in, resource, am_ready_input);
+ send_select_msg(&in);
}
if (is_not_nil(out.pid)) {
- send_event_tuple(&out, resource, am_ready_output);
+ send_select_msg(&out);
}
continue;
}
@@ -2448,10 +2499,16 @@ drvmode2str(int mode) {
static ERTS_INLINE char *
nifmode2str(enum ErlNifSelectFlags mode) {
+ if (mode & ERL_NIF_SELECT_STOP)
+ return "STOP";
switch (mode) {
case ERL_NIF_SELECT_READ: return "READ";
case ERL_NIF_SELECT_WRITE: return "WRITE";
- case ERL_NIF_SELECT_STOP: return "STOP";
+ case ERL_NIF_SELECT_READ|ERL_NIF_SELECT_WRITE: return "READ|WRITE";
+ case ERL_NIF_SELECT_CANCEL|ERL_NIF_SELECT_READ: return "CANCEL|READ";
+ case ERL_NIF_SELECT_CANCEL|ERL_NIF_SELECT_WRITE: return "CANCEL|WRITE";
+ case ERL_NIF_SELECT_CANCEL|ERL_NIF_SELECT_READ|ERL_NIF_SELECT_WRITE:
+ return "CANCEL|READ|WRITE";
default: return "UNKNOWN";
}
}
diff --git a/erts/emulator/sys/common/erl_check_io.h b/erts/emulator/sys/common/erl_check_io.h
index 31182be5ec..0f3fc4f7a2 100644
--- a/erts/emulator/sys/common/erl_check_io.h
+++ b/erts/emulator/sys/common/erl_check_io.h
@@ -138,8 +138,7 @@ typedef struct {
struct erts_nif_select_event {
Eterm pid;
- Eterm immed;
- Uint32 refn[ERTS_REF_NUMBERS];
+ ErtsMessage *mp;
};
typedef struct {
diff --git a/erts/emulator/sys/common/erl_mmap.h b/erts/emulator/sys/common/erl_mmap.h
index 539daea419..e1ff0fe80a 100644
--- a/erts/emulator/sys/common/erl_mmap.h
+++ b/erts/emulator/sys/common/erl_mmap.h
@@ -176,4 +176,61 @@ void hard_dbg_remove_mseg(void* seg, UWord sz);
#endif /* HAVE_ERTS_MMAP */
+/* Marks the given memory region as unused without freeing it, letting the OS
+ * reclaim its physical memory with the promise that we'll get it back (without
+ * its contents) the next time it's accessed. */
+ERTS_GLB_INLINE void erts_mem_discard(void *p, UWord size);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+#ifdef VALGRIND
+ #include <valgrind/memcheck.h>
+
+ ERTS_GLB_INLINE void erts_mem_discard(void *ptr, UWord size) {
+ VALGRIND_MAKE_MEM_UNDEFINED(ptr, size);
+ }
+#elif defined(DEBUG)
+ /* Try to provoke crashes by filling the discard region with garbage. It's
+ * extremely hard to find bugs where we've discarded too much, as the
+ * region often retains its old contents if it's accessed before the OS
+ * reclaims it. */
+ ERTS_GLB_INLINE void erts_mem_discard(void *ptr, UWord size) {
+ static const char pattern[] = "DISCARDED";
+ char *data;
+ int i;
+
+ for(i = 0, data = ptr; i < size; i++) {
+ data[i] = pattern[i % sizeof(pattern)];
+ }
+ }
+#elif defined(HAVE_SYS_MMAN_H)
+ #include <sys/mman.h>
+
+ ERTS_GLB_INLINE void erts_mem_discard(void *ptr, UWord size) {
+ #ifdef MADV_FREE
+ /* This is preferred as it doesn't necessarily free the pages right
+ * away, which is a bit faster than MADV_DONTNEED. */
+ madvise(ptr, size, MADV_FREE);
+ #else
+ madvise(ptr, size, MADV_DONTNEED);
+ #endif
+ }
+#elif defined(_WIN32)
+ #include <winbase.h>
+
+ /* MEM_RESET is defined on all supported versions of Windows, and has the
+ * same semantics as MADV_FREE. */
+ ERTS_GLB_INLINE void erts_mem_discard(void *ptr, UWord size) {
+ VirtualAlloc(ptr, size, MEM_RESET, PAGE_READWRITE);
+ }
+#else
+ /* Dummy implementation. */
+ ERTS_GLB_INLINE void erts_mem_discard(void *ptr, UWord size) {
+ (void)ptr;
+ (void)size;
+ }
+#endif
+
+#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
+
#endif /* ERL_MMAP_H__ */
diff --git a/erts/emulator/sys/win32/sys.c b/erts/emulator/sys/win32/sys.c
index a1c630d68a..b95aadc9b2 100644
--- a/erts/emulator/sys/win32/sys.c
+++ b/erts/emulator/sys/win32/sys.c
@@ -186,7 +186,9 @@ void sys_primitive_init(HMODULE beam)
UWord
erts_sys_get_page_size(void)
{
- return (UWord) 4*1024; /* Guess 4 KB */
+ SYSTEM_INFO info;
+ GetSystemInfo(&info);
+ return (UWord)info.dwPageSize;
}
Uint