diff options
Diffstat (limited to 'erts')
-rw-r--r-- | erts/emulator/beam/beam_emu.c | 23 | ||||
-rw-r--r-- | erts/emulator/beam/erl_gc.c | 6 | ||||
-rw-r--r-- | erts/emulator/beam/erl_message.c | 17 | ||||
-rw-r--r-- | erts/emulator/beam/erl_message.h | 11 | ||||
-rw-r--r-- | erts/emulator/beam/erl_nif.c | 14 | ||||
-rw-r--r-- | erts/emulator/beam/erl_process.h | 9 | ||||
-rw-r--r-- | erts/emulator/beam/utils.c | 3 | ||||
-rw-r--r-- | erts/emulator/sys/common/erl_poll.c | 14 | ||||
-rw-r--r-- | erts/include/internal/erl_misc_utils.h | 29 | ||||
-rw-r--r-- | erts/lib_src/pthread/ethr_event.c | 74 |
10 files changed, 174 insertions, 26 deletions
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index b3e6ebf651..ef4cdf9d5a 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -1960,6 +1960,8 @@ void process_main(void) ErtsMessage* msgp; PROCESS_MAIN_CHK_LOCKS(c_p); + ERTS_CHK_MBUF_SZ(c_p); + PreFetch(0, next); msgp = PEEK_MESSAGE(c_p); @@ -2047,6 +2049,7 @@ void process_main(void) } ERTS_DBG_CHK_REDS(c_p, FCALLS); + ERTS_CHK_MBUF_SZ(c_p); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); @@ -2583,7 +2586,9 @@ do { \ c_p->fcalls = FCALLS; PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + ERTS_CHK_MBUF_SZ(c_p); result = (*bf)(c_p, tmp_reg); + ERTS_CHK_MBUF_SZ(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); @@ -2614,7 +2619,9 @@ do { \ c_p->fcalls = FCALLS; PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + ERTS_CHK_MBUF_SZ(c_p); result = (*bf)(c_p, tmp_reg); + ERTS_CHK_MBUF_SZ(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); @@ -2644,7 +2651,9 @@ do { \ SWAPOUT; PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + ERTS_CHK_MBUF_SZ(c_p); result = (*bf)(c_p, reg, live); + ERTS_CHK_MBUF_SZ(c_p); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); @@ -2685,7 +2694,9 @@ do { \ SWAPOUT; PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + ERTS_CHK_MBUF_SZ(c_p); result = (*bf)(c_p, reg, live); + ERTS_CHK_MBUF_SZ(c_p); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); @@ -2728,7 +2739,9 @@ do { \ SWAPOUT; PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + ERTS_CHK_MBUF_SZ(c_p); result = (*bf)(c_p, reg, live); + ERTS_CHK_MBUF_SZ(c_p); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); @@ -2766,7 +2779,9 @@ do { \ c_p->fcalls = FCALLS; PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + ERTS_CHK_MBUF_SZ(c_p); result = (*bf)(c_p, tmp_reg); + ERTS_CHK_MBUF_SZ(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); @@ -2793,7 +2808,9 @@ do { \ bf = (BifFunction) Arg(0); PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + ERTS_CHK_MBUF_SZ(c_p); result = (*bf)(c_p, tmp_reg); + ERTS_CHK_MBUF_SZ(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); @@ -2843,7 +2860,9 @@ do { \ ASSERT(!ERTS_PROC_IS_EXITING(c_p)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); live_hf_end = c_p->mbuf; + ERTS_CHK_MBUF_SZ(c_p); result = (*bf)(c_p, reg, I); + ERTS_CHK_MBUF_SZ(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); ERTS_HOLE_CHECK(c_p); @@ -3556,11 +3575,13 @@ do { \ ASSERT(c_p->scheduler_data); #endif live_hf_end = c_p->mbuf; + ERTS_CHK_MBUF_SZ(c_p); erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2], NULL); nif_bif_result = (*fp)(&env, bif_nif_arity, reg); if (env.exception_thrown) nif_bif_result = THE_NON_VALUE; erts_post_nif(&env); + ERTS_CHK_MBUF_SZ(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); @@ -3612,7 +3633,9 @@ do { \ Eterm (*bf)(Process*, Eterm*, BeamInstr*) = vbf; ASSERT(!ERTS_PROC_IS_EXITING(c_p)); live_hf_end = c_p->mbuf; + ERTS_CHK_MBUF_SZ(c_p); nif_bif_result = (*bf)(c_p, reg, I); + ERTS_CHK_MBUF_SZ(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(nif_bif_result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 8b0dd9a5a6..75f8ebefbd 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -517,6 +517,8 @@ delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need, int erts_proc_sched_data((p))->virtual_reds += vreds; } + ERTS_CHK_MBUF_SZ(p); + ASSERT(CONTEXT_REDS >= erts_proc_sched_data(p)->virtual_reds); return reds_left; } @@ -527,6 +529,8 @@ young_gen_usage(Process *p) Uint hsz; Eterm *aheap; + ERTS_CHK_MBUF_SZ(p); + hsz = p->mbuf_sz; if (p->flags & F_ON_HEAP_MSGQ) { @@ -590,6 +594,8 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end, DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE); #endif + ERTS_CHK_MBUF_SZ(p); + ASSERT(CONTEXT_REDS - ERTS_REDS_LEFT(p, fcalls) >= erts_proc_sched_data(p)->virtual_reds); diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index 91e06cde2d..e4c696ae3b 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -1842,3 +1842,20 @@ void erts_factory_undo(ErtsHeapFactory* factory) factory->heap_frags = NULL; #endif } + +Uint +erts_mbuf_size(Process *p) +{ + Uint sz = 0; + ErlHeapFragment* bp; + ErtsMessage* mp; + + for (bp = p->mbuf; bp; bp = bp->next) + sz += bp->used_size; + + for (mp = p->msg_frag; mp; mp = mp->next) + for (bp = erts_message_to_heap_frag(mp); bp; bp = bp->next) + sz += bp->used_size; + + return sz; +} diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index 6df969367b..42ed14e69c 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -499,4 +499,15 @@ erts_msgq_replace_msg_ref(ErlMessageQueue *msgq, ErtsMessage *newp, ErtsMessage #endif +Uint erts_mbuf_size(Process *p); +#if defined(DEBUG) || 0 +# define ERTS_CHK_MBUF_SZ(P) \ + do { \ + Uint actual_mbuf_sz__ = erts_mbuf_size((P)); \ + ERTS_ASSERT((P)->mbuf_sz >= actual_mbuf_sz__); \ + } while (0) +#else +# define ERTS_CHK_MBUF_SZ(P) ((void) 1) +#endif + #endif diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index ef2fb93106..27392a5996 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -156,7 +156,9 @@ static Eterm* alloc_heap_heavy(ErlNifEnv* env, size_t need, Eterm* hp) HEAP_TOP(env->proc) = env->hp; } else { - env->heap_frag->used_size = hp - env->heap_frag->mem; + Uint usz = env->hp - env->heap_frag->mem; + env->proc->mbuf_sz += usz - env->heap_frag->used_size; + env->heap_frag->used_size = usz; ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size); } hp = erts_heap_alloc(env->proc, need, MIN_HEAP_FRAG_SZ); @@ -308,11 +310,14 @@ static void full_flush_env(ErlNifEnv* env) HEAP_TOP(c_p) = env->hp; } else { + Uint usz; ASSERT(env->hp_end != HEAP_LIMIT(c_p)); ASSERT(env->hp_end - env->hp <= env->heap_frag->alloc_size); HEAP_TOP(c_p) = HEAP_TOP(env->proc); - env->heap_frag->used_size = env->hp - env->heap_frag->mem; + usz = env->hp - env->heap_frag->mem; + env->proc->mbuf_sz += usz - env->heap_frag->used_size; + env->heap_frag->used_size = usz; ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size); @@ -394,9 +399,12 @@ static void flush_env(ErlNifEnv* env) HEAP_TOP(env->proc) = env->hp; } else { + Uint usz; ASSERT(env->hp_end != HEAP_LIMIT(env->proc)); ASSERT(env->hp_end - env->hp <= env->heap_frag->alloc_size); - env->heap_frag->used_size = env->hp - env->heap_frag->mem; + usz = env->hp - env->heap_frag->mem; + env->proc->mbuf_sz += usz - env->heap_frag->used_size; + env->heap_frag->used_size = usz; ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size); } } diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 94c13bb948..3347a7a60e 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -1328,10 +1328,13 @@ ERTS_GLB_INLINE void erts_heap_frag_shrink(Process* p, Eterm* hp); ERTS_GLB_INLINE void erts_heap_frag_shrink(Process* p, Eterm* hp) { ErlHeapFragment* hf = MBUF(p); + Uint sz; ASSERT(hf!=NULL && (hp - hf->mem < hf->alloc_size)); - hf->used_size = hp - hf->mem; + sz = hp - hf->mem; + p->mbuf_sz -= hf->used_size - sz; + hf->used_size = sz; } #endif /* inline */ @@ -1549,9 +1552,9 @@ extern int erts_system_profile_ts_type; #define ERTS_SCHEDULER_IS_DIRTY(ESDP) \ ((ESDP)->dirty_no.s.num != 0) #define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP) \ - ((ESDP)->dirty_no.s.type == 0) + (ERTS_SCHEDULER_IS_DIRTY((ESDP)) & ((ESDP)->dirty_no.s.type == 0)) #define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP) \ - ((ESDP)->dirty_no.s.type == 1) + (ERTS_SCHEDULER_IS_DIRTY((ESDP)) & ((ESDP)->dirty_no.s.type == 1)) #else #define ERTS_SCHEDULER_IS_DIRTY(ESDP) 0 #define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP) 0 diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 85647b8500..6786657faf 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -102,6 +102,7 @@ erts_heap_alloc(Process* p, Uint need, Uint xtra) if (bp != NULL && need <= (bp->alloc_size - bp->used_size)) { Eterm* ret = bp->mem + bp->used_size; bp->used_size += need; + p->mbuf_sz += need; return ret; } #ifdef DEBUG @@ -124,7 +125,7 @@ erts_heap_alloc(Process* p, Uint need, Uint xtra) MBUF(p) = bp; bp->alloc_size = n; bp->used_size = need; - MBUF_SIZE(p) += n; + MBUF_SIZE(p) += need; bp->off_heap.first = NULL; bp->off_heap.overhead = 0; return bp->mem; diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c index e394d84f73..b8a28bcc18 100644 --- a/erts/emulator/sys/common/erl_poll.c +++ b/erts/emulator/sys/common/erl_poll.c @@ -75,6 +75,7 @@ #include "erl_driver.h" #include "erl_alloc.h" #include "erl_msacc.h" +#include "erl_misc_utils.h" #if !defined(ERTS_POLL_USE_EPOLL) \ && !defined(ERTS_POLL_USE_DEVPOLL) \ @@ -2132,16 +2133,19 @@ get_timeout(ErtsPollSet ps, if (timeout > (ErtsMonotonicTime) INT_MAX) timeout = (ErtsMonotonicTime) INT_MAX; save_timeout_time += ERTS_MSEC_TO_MONOTONIC(timeout); + timeout -= ERTS_PREMATURE_TIMEOUT(timeout, 1000); break; case 1000000: /* Round up to nearest even micro second */ timeout = ERTS_MONOTONIC_TO_USEC(diff_time - 1) + 1; save_timeout_time += ERTS_USEC_TO_MONOTONIC(timeout); + timeout -= ERTS_PREMATURE_TIMEOUT(timeout, 1000*1000); break; case 1000000000: /* Round up to nearest even nano second */ timeout = ERTS_MONOTONIC_TO_NSEC(diff_time - 1) + 1; save_timeout_time += ERTS_NSEC_TO_MONOTONIC(timeout); + timeout -= ERTS_PREMATURE_TIMEOUT(timeout, 1000*1000*1000); break; default: ERTS_INTERNAL_ERROR("Invalid resolution"); @@ -2452,7 +2456,15 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps, } #endif - res = check_fd_events(ps, to, no_fds); + while (1) { + res = check_fd_events(ps, to, no_fds); + if (res != 0) + break; + if (to == ERTS_POLL_NO_TIMEOUT) + break; + if (erts_get_monotonic_time(NULL) >= timeout_time) + break; + } woke_up(ps); diff --git a/erts/include/internal/erl_misc_utils.h b/erts/include/internal/erl_misc_utils.h index a4a5d1d510..55566ddf74 100644 --- a/erts/include/internal/erl_misc_utils.h +++ b/erts/include/internal/erl_misc_utils.h @@ -56,4 +56,33 @@ int erts_map_win_error_to_errno(DWORD win_error); int erts_get_last_win_errno(void); #endif +#if defined(__APPLE__) && defined(__MACH__) && !defined(__DARWIN__) +#define __DARWIN__ 1 +#endif + +/* + * ERTS_PREMATURE_TIMEOUT() expects time units + * 1000 (millisec), 1000000 (microsec), or + * 1000000000 (nanosec). Might not work properly + * otherwise. + */ +#undef ERTS_USE_PREMATURE_TIMEOUT +#undef ERTS_PREMATURE_TIMEOUT + +#if defined(__DARWIN__) +#define ERTS_USE_PREMATURE_TIMEOUT 1 +#define ERTS_PREMATURE_TIMEOUT(TMO, TU) \ + ((TMO) >= 1 * ((TU) / 1000) \ + ? ((TMO) >= 20 * ((TU) / 1000) \ + ? 15 * ((TU) / 1000) \ + : ((TMO) >= 5 * ((TU) / 1000) \ + ? 3 * ((TU) / 1000) \ + : 5 * ((TU) / 10000))) \ + : 0) + +#else +#define ERTS_USE_PREMATURE_TIMEOUT 0 +#define ERTS_PREMATURE_TIMEOUT(TMO, TU) (0) +#endif + #endif /* #ifndef ERL_MISC_UTILS_H_ */ diff --git a/erts/lib_src/pthread/ethr_event.c b/erts/lib_src/pthread/ethr_event.c index eef88d5002..464875570a 100644 --- a/erts/lib_src/pthread/ethr_event.c +++ b/erts/lib_src/pthread/ethr_event.c @@ -184,6 +184,8 @@ return_event_on: #include <errno.h> #include <string.h> +#include "erl_misc_utils.h" + #ifdef __DARWIN__ struct ethr_event_fdsets___ { @@ -346,12 +348,9 @@ wait__(ethr_event *e, int spincount, ethr_sint64_t timeout) ethr_sint32_t val; int res, ulres; int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; - ethr_sint64_t time = 0; /* SHUT UP annoying faulty warning... */ + ethr_sint64_t time = 0; #ifdef ETHR_HAVE_ETHR_GET_MONOTONIC_TIME - ethr_sint64_t start = 0; /* SHUT UP annoying faulty warning... */ -#endif -#ifdef ETHR_HAVE_PTHREAD_COND_TIMEDWAIT_MONOTONIC - struct timespec cond_timeout; + ethr_sint64_t timeout_time = 0; /* SHUT UP annoying faulty warning... */ #endif val = ethr_atomic32_read(&e->state); @@ -365,30 +364,27 @@ wait__(ethr_event *e, int spincount, ethr_sint64_t timeout) if (timeout == 0) return ETIMEDOUT; else { - time = timeout; - switch (e->fd[0]) { - case ETHR_EVENT_INVALID_FD__: #ifdef ETHR_HAVE_ETHR_GET_MONOTONIC_TIME - start = ethr_get_monotonic_time(); + timeout_time = ethr_get_monotonic_time(); + timeout_time += timeout; #endif + switch (e->fd[0]) { + case ETHR_EVENT_INVALID_FD__: + time = timeout; setup_nonblocking_pipe(e); break; #ifdef ETHR_HAVE_PTHREAD_COND_TIMEDWAIT_MONOTONIC case ETHR_EVENT_COND_TIMEDWAIT__: - time += ethr_get_monotonic_time(); - cond_timeout.tv_sec = time / (1000*1000*1000); - cond_timeout.tv_nsec = time % (1000*1000*1000); + time = -1; if (spincount == 0) goto set_event_off_waiter; break; #endif default: + time = timeout; /* Already initialized pipe... */ if (spincount == 0) goto set_select_timeout; -#ifdef ETHR_HAVE_ETHR_GET_MONOTONIC_TIME - start = ethr_get_monotonic_time(); -#endif break; } } @@ -418,6 +414,9 @@ wait__(ethr_event *e, int spincount, ethr_sint64_t timeout) || e->fd[0] == ETHR_EVENT_COND_TIMEDWAIT__ #endif ) { +#ifdef ETHR_HAVE_PTHREAD_COND_TIMEDWAIT_MONOTONIC + struct timespec cond_timeout; +#endif set_event_off_waiter: @@ -446,9 +445,35 @@ wait__(ethr_event *e, int spincount, ethr_sint64_t timeout) #ifdef ETHR_HAVE_PTHREAD_COND_TIMEDWAIT_MONOTONIC if (timeout > 0) { + if (time != timeout_time) { + time = timeout_time; + +#if ERTS_USE_PREMATURE_TIMEOUT + { + ethr_sint64_t rtmo; + + rtmo = timeout_time - ethr_get_monotonic_time(); + if (rtmo <= 0) { + res = ETIMEDOUT; + break; + } + time = timeout_time; + time -= ERTS_PREMATURE_TIMEOUT(rtmo, 1000*1000*1000); + } +#endif + + cond_timeout.tv_sec = time / (1000*1000*1000); + cond_timeout.tv_nsec = time % (1000*1000*1000); + } res = pthread_cond_timedwait(&e->cnd, &e->mtx, &cond_timeout); - if (res == EINTR || res == ETIMEDOUT) + if (res == EINTR + || (res == ETIMEDOUT +#if ERTS_USE_PREMATURE_TIMEOUT + && time == timeout_time +#endif + )) { break; + } } else #endif @@ -477,7 +502,11 @@ wait__(ethr_event *e, int spincount, ethr_sint64_t timeout) struct timeval select_timeout; #ifdef ETHR_HAVE_ETHR_GET_MONOTONIC_TIME - time -= ethr_get_monotonic_time() - start; +#if ERTS_USE_PREMATURE_TIMEOUT + restart_select: +#endif + + time = timeout_time - ethr_get_monotonic_time(); if (time <= 0) return ETIMEDOUT; #endif @@ -492,6 +521,11 @@ wait__(ethr_event *e, int spincount, ethr_sint64_t timeout) */ time = ((time - 1) / 1000) + 1; +#if defined(ETHR_HAVE_ETHR_GET_MONOTONIC_TIME) \ + && ERTS_USE_PREMATURE_TIMEOUT + time -= ERTS_PREMATURE_TIMEOUT(time, 1000*1000); +#endif + select_timeout.tv_sec = time / (1000*1000); select_timeout.tv_usec = time % (1000*1000); @@ -559,7 +593,11 @@ wait__(ethr_event *e, int spincount, ethr_sint64_t timeout) val = ethr_atomic32_read(&e->state); if (val == ETHR_EVENT_ON__) goto return_event_on; - +#if defined(ETHR_HAVE_ETHR_GET_MONOTONIC_TIME) \ + && ERTS_USE_PREMATURE_TIMEOUT + if (res == ETIMEDOUT) + goto restart_select; /* Verify timeout */ +#endif } return res; |