diff options
Diffstat (limited to 'erts/emulator/beam')
33 files changed, 1836 insertions, 1216 deletions
diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index cb4fab51f1..42a368cdd8 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -124,7 +124,6 @@ atom big atom bif_return_trap atom bif_timer_server atom binary -atom binary_bin_to_list_trap atom binary_copy_trap atom binary_find_trap atom binary_longest_prefix_trap @@ -577,7 +576,8 @@ atom running_procs atom runtime atom safe atom save_calls -atom scheduler +atom sbct +atom scheduler atom scheduler_id atom scheduler_wall_time atom scheduler_wall_time_all diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index f086c434ea..bfd572335f 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -1744,7 +1744,6 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2) else if (BIF_ARG_1 == am_scheduler) { ErtsRunQueue *old, *new, *curr; Sint sched; - erts_aint32_t state; if (!is_small(BIF_ARG_2)) goto error; @@ -1753,23 +1752,23 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2) goto error; if (sched == 0) { + old = erts_bind_runq_proc(BIF_P, 0); new = NULL; - state = erts_atomic32_read_band_mb(&BIF_P->state, - ~ERTS_PSFLG_BOUND); } else { + int bound = !0; new = erts_schedid2runq(sched); - erts_atomic_set_nob(&BIF_P->run_queue, (erts_aint_t) new); - state = erts_atomic32_read_bor_mb(&BIF_P->state, - ERTS_PSFLG_BOUND); + old = erts_set_runq_proc(BIF_P, new, &bound); + if (!bound) + old = NULL; } + old_value = old ? make_small(old->ix+1) : make_small(0); + curr = erts_proc_sched_data(BIF_P)->run_queue; - old = (ERTS_PSFLG_BOUND & state) ? curr : NULL; ASSERT(!old || old == curr); - old_value = old ? make_small(old->ix+1) : make_small(0); if (new && new != curr) ERTS_BIF_YIELD_RETURN_X(BIF_P, old_value, am_scheduler); else @@ -4805,6 +4804,8 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) "scheduled for removal in Erlang/OTP 18. For more\n" "information see the erlang:system_flag/2 documentation.\n"); return erts_bind_schedulers(BIF_P, BIF_ARG_2); + } else if (ERTS_IS_ATOM_STR("erts_alloc", BIF_ARG_1)) { + return erts_alloc_set_dyn_param(BIF_P, BIF_ARG_2); } error: BIF_ERROR(BIF_P, BADARG); diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index c0d5e8ce74..0d1166f6ed 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -551,9 +551,6 @@ bif binary:last/1 bif binary:at/2 bif binary:part/2 binary_binary_part_2 bif binary:part/3 binary_binary_part_3 -bif binary:bin_to_list/1 -bif binary:bin_to_list/2 -bif binary:bin_to_list/3 bif binary:list_to_bin/1 bif binary:copy/1 bif binary:copy/2 @@ -696,3 +693,4 @@ bif erlang:iolist_to_iovec/1 bif erts_internal:new_connection/1 bif erts_internal:abort_connection/2 bif erts_internal:map_next/3 +bif ets:whereis/1 diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c index d4db1a4188..95d324d2c1 100644 --- a/erts/emulator/beam/binary.c +++ b/erts/emulator/beam/binary.c @@ -953,7 +953,10 @@ HIPE_WRAPPER_BIF_DISABLE_GC(iolist_to_binary, 1) BIF_RETTYPE iolist_to_binary_1(BIF_ALIST_1) { if (is_binary(BIF_ARG_1)) { - BIF_RET(BIF_ARG_1); + if (binary_bitsize(BIF_ARG_1) == 0) { + BIF_RET(BIF_ARG_1); + } + BIF_ERROR(BIF_P, BADARG); } return erts_list_to_binary_bif(BIF_P, BIF_ARG_1, bif_export[BIF_iolist_to_binary_1]); } diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index e7acea0c5f..b9b70cd8ef 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -336,6 +336,12 @@ print_process_info(fmtfn_t to, void *to_arg, Process *p) erts_print(to, to_arg, "Heap unused: %bpu\n", (p->hend - p->htop)); erts_print(to, to_arg, "OldHeap unused: %bpu\n", (OLD_HEAP(p) == NULL) ? 0 : (OLD_HEND(p) - OLD_HTOP(p)) ); + erts_print(to, to_arg, "BinVHeap: %b64u\n", p->off_heap.overhead); + erts_print(to, to_arg, "OldBinVHeap: %b64u\n", BIN_OLD_VHEAP(p)); + erts_print(to, to_arg, "BinVHeap unused: %b64u\n", + BIN_VHEAP_SZ(p) - p->off_heap.overhead); + erts_print(to, to_arg, "OldBinVHeap unused: %b64u\n", + BIN_OLD_VHEAP_SZ(p) - BIN_OLD_VHEAP(p)); erts_print(to, to_arg, "Memory: %beu\n", erts_process_memory(p, !0)); if (garbing) { @@ -891,6 +897,21 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) erts_print_scheduler_info(to, to_arg, ERTS_SCHEDULER_IX(i)), erts_cbprintf(to, to_arg, "** crashed **\n")); } + for (i = 0; i < erts_no_dirty_cpu_schedulers; i++) { + ERTS_SYS_TRY_CATCH( + erts_print_scheduler_info(to, to_arg, ERTS_DIRTY_CPU_SCHEDULER_IX(i)), + erts_cbprintf(to, to_arg, "** crashed **\n")); + } + erts_cbprintf(to, to_arg, "=dirty_cpu_run_queue\n"); + erts_print_run_queue_info(to, to_arg, ERTS_DIRTY_CPU_RUNQ); + + for (i = 0; i < erts_no_dirty_io_schedulers; i++) { + ERTS_SYS_TRY_CATCH( + erts_print_scheduler_info(to, to_arg, ERTS_DIRTY_IO_SCHEDULER_IX(i)), + erts_cbprintf(to, to_arg, "** crashed **\n")); + } + erts_cbprintf(to, to_arg, "=dirty_io_run_queue\n"); + erts_print_run_queue_info(to, to_arg, ERTS_DIRTY_IO_RUNQ); #endif diff --git a/erts/emulator/beam/bs_instrs.tab b/erts/emulator/beam/bs_instrs.tab index b11903a47b..94e0000c8b 100644 --- a/erts/emulator/beam/bs_instrs.tab +++ b/erts/emulator/beam/bs_instrs.tab @@ -990,6 +990,9 @@ ctx_to_bin.execute() { Uint hole_size; Uint orig = mb->orig; ErlSubBin* sb = (ErlSubBin *) boxed_val(context); + /* Since we're going to overwrite the match state with the result, an + * ErlBinMatchState must be at least as large as an ErlSubBin. */ + ERTS_CT_ASSERT(sizeof(ErlSubBin) <= sizeof(ErlBinMatchState)); hole_size = 1 + header_arity(sb->thing_word) - ERL_SUB_BIN_SIZE; sb->thing_word = HEADER_SUB_BIN; sb->size = BYTE_OFFSET(size); diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index 10bf197405..7769a914db 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -611,7 +611,7 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint Eterm* htop; Eterm* hbot; Eterm* hp; - Eterm* objp; + Eterm* ERTS_RESTRICT objp; Eterm* tp; Eterm res; Eterm elem; @@ -1821,7 +1821,8 @@ all_clean: * * NOTE: Assumes that term is a tuple (ptr is an untagged tuple ptr). */ -Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) +Eterm copy_shallow(Eterm* ERTS_RESTRICT ptr, Uint sz, Eterm** hpp, + ErlOffHeap* off_heap) { Eterm* tp = ptr; Eterm* hp = *hpp; @@ -1985,7 +1986,7 @@ move_one_frag(Eterm** hpp, ErlHeapFragment* frag, ErlOffHeap* off_heap, int lite if (is_header(val)) { struct erl_off_heap_header* hdr = (struct erl_off_heap_header*)hp; ASSERT(ptr + header_arity(val) < end); - move_boxed(&ptr, val, &hp, &dummy_ref); + ptr = move_boxed(ptr, val, &hp, &dummy_ref); switch (val & _HEADER_SUBTAG_MASK) { case REF_SUBTAG: if (is_ordinary_ref_thing(hdr)) @@ -2002,7 +2003,7 @@ move_one_frag(Eterm** hpp, ErlHeapFragment* frag, ErlOffHeap* off_heap, int lite } else { /* must be a cons cell */ ASSERT(ptr+1 < end); - move_cons(&ptr, val, &hp, &dummy_ref); + move_cons(ptr, val, &hp, &dummy_ref); ptr += 2; } } diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 88285d8be6..239dda6bcf 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -160,7 +160,7 @@ enum allctr_type { GOODFIT, BESTFIT, AFIT, - AOFIRSTFIT + FIRSTFIT }; struct au_init { @@ -500,8 +500,9 @@ set_default_test_alloc_opts(struct au_init *ip) SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = 0; /* Disabled by default */ ip->thr_spec = -1 * erts_no_schedulers; - ip->atype = AOFIRSTFIT; - ip->init.aoff.flavor = AOFF_BF; + ip->atype = FIRSTFIT; + ip->init.aoff.crr_order = FF_AOFF; + ip->init.aoff.blk_order = FF_BF; ip->init.util.name_prefix = "test_"; ip->init.util.alloc_no = ERTS_ALC_A_TEST; ip->init.util.mmbcs = 0; /* Main carrier size */ @@ -599,10 +600,10 @@ static ERTS_INLINE int strategy_support_carrier_migration(struct au_init *auip) { /* - * Currently only aoff, aoffcbf and aoffcaobf support carrier + * Currently only aoff* and ageff* support carrier * migration, i.e, type AOFIRSTFIT. */ - return auip->atype == AOFIRSTFIT; + return auip->atype == FIRSTFIT; } static ERTS_INLINE void @@ -617,8 +618,9 @@ adjust_carrier_migration_support(struct au_init *auip) */ if (!strategy_support_carrier_migration(auip)) { /* Default to aoffcbf */ - auip->atype = AOFIRSTFIT; - auip->init.aoff.flavor = AOFF_BF; + auip->atype = FIRSTFIT; + auip->init.aoff.crr_order = FF_AOFF; + auip->init.aoff.blk_order = FF_BF; } } } @@ -1132,7 +1134,7 @@ start_au_allocator(ErtsAlcType_t alctr_n, &init->init.af, &init->init.util); break; - case AOFIRSTFIT: + case FIRSTFIT: as = erts_aoffalc_start((AOFFAllctr_t *) as0, &init->init.aoff, &init->init.util); @@ -1226,22 +1228,32 @@ get_bool_value(char *param_end, char** argv, int* ip) return -1; } +static Uint kb_to_bytes(Sint kb, Uint *bytes) +{ + const Uint max = ((~((Uint) 0))/1024) + 1; + + if (kb < 0 || (Uint)kb > max) + return 0; + if ((Uint)kb == max) + *bytes = ~((Uint) 0); + else + *bytes = ((Uint) kb)*1024; + return 1; +} + static Uint get_kb_value(char *param_end, char** argv, int* ip) { Sint tmp; - Uint max = ((~((Uint) 0))/1024) + 1; + Uint bytes = 0; char *rest; char *param = argv[*ip]+1; char *value = get_value(param_end, argv, ip); errno = 0; tmp = (Sint) ErtsStrToSint(value, &rest, 10); - if (errno != 0 || rest == value || tmp < 0 || max < ((Uint) tmp)) + if (errno != 0 || rest == value || !kb_to_bytes(tmp, &bytes)) bad_value(param, param_end, value); - if (max == (Uint) tmp) - return ~((Uint) 0); - else - return ((Uint) tmp)*1024; + return bytes; } static UWord @@ -1328,18 +1340,30 @@ handle_au_arg(struct au_init *auip, switch (sub_param[0]) { case 'a': - if (has_prefix("acul", sub_param)) { - if (!auip->carrier_migration_allowed) { - if (!u_switch) - goto bad_switch; - else { - /* ignore */ - (void) get_acul_value(auip, sub_param + 4, argv, ip); - break; - } - } - auip->init.util.acul = get_acul_value(auip, sub_param + 4, argv, ip); - } + if (sub_param[1] == 'c') { /* Migration parameters "ac*" */ + UWord value; + UWord* wp; + if (!auip->carrier_migration_allowed && !u_switch) + goto bad_switch; + + if (has_prefix("acul", sub_param)) { + value = get_acul_value(auip, sub_param + 4, argv, ip); + wp = &auip->init.util.acul; + } + else if (has_prefix("acnl", sub_param)) { + value = get_amount_value(sub_param + 4, argv, ip); + wp = &auip->init.util.acnl; + } + else if (has_prefix("acfml", sub_param)) { + value = get_amount_value(sub_param + 5, argv, ip); + wp = &auip->init.util.acfml; + } + else + goto bad_switch; + + if (auip->carrier_migration_allowed) + *wp = value; + } else if(has_prefix("asbcst", sub_param)) { auip->init.util.asbcst = get_kb_value(sub_param + 6, argv, ip); } @@ -1360,17 +1384,35 @@ handle_au_arg(struct au_init *auip, auip->atype = AFIT; } else if (strcmp("aoff", alg) == 0) { - auip->atype = AOFIRSTFIT; - auip->init.aoff.flavor = AOFF_AOFF; + auip->atype = FIRSTFIT; + auip->init.aoff.crr_order = FF_AOFF; + auip->init.aoff.blk_order = FF_AOFF; } else if (strcmp("aoffcbf", alg) == 0) { - auip->atype = AOFIRSTFIT; - auip->init.aoff.flavor = AOFF_BF; + auip->atype = FIRSTFIT; + auip->init.aoff.crr_order = FF_AOFF; + auip->init.aoff.blk_order = FF_BF; } else if (strcmp("aoffcaobf", alg) == 0) { - auip->atype = AOFIRSTFIT; - auip->init.aoff.flavor = AOFF_AOBF; + auip->atype = FIRSTFIT; + auip->init.aoff.crr_order = FF_AOFF; + auip->init.aoff.blk_order = FF_AOBF; } + else if (strcmp("ageffcaoff", alg) == 0) { + auip->atype = FIRSTFIT; + auip->init.aoff.crr_order = FF_AGEFF; + auip->init.aoff.blk_order = FF_AOFF; + } + else if (strcmp("ageffcbf", alg) == 0) { + auip->atype = FIRSTFIT; + auip->init.aoff.crr_order = FF_AGEFF; + auip->init.aoff.blk_order = FF_BF; + } + else if (strcmp("ageffcaobf", alg) == 0) { + auip->atype = FIRSTFIT; + auip->init.aoff.crr_order = FF_AGEFF; + auip->init.aoff.blk_order = FF_AOBF; + } else { bad_value(param, sub_param + 1, alg); } @@ -3392,6 +3434,65 @@ erts_request_alloc_info(struct process *c_p, return 1; } +Eterm erts_alloc_set_dyn_param(Process* c_p, Eterm tuple) +{ + ErtsAllocatorThrSpec_t *tspec; + ErtsAlcType_t ai; + Allctr_t* allctr; + Eterm* tp; + Eterm res; + + if (!is_tuple_arity(tuple, 3)) + goto badarg; + + tp = tuple_val(tuple); + + /* + * Ex: {ets_alloc, sbct, 256000} + */ + if (!is_atom(tp[1]) || !is_atom(tp[2]) || !is_integer(tp[3])) + goto badarg; + + for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++) + if (erts_is_atom_str(erts_alc_a2ad[ai], tp[1], 0)) + break; + + if (ai > ERTS_ALC_A_MAX) + goto badarg; + + if (!erts_allctrs_info[ai].enabled || + !erts_allctrs_info[ai].alloc_util) { + return am_notsup; + } + + if (tp[2] == am_sbct) { + Uint sbct; + int i, ok; + + if (!term_to_Uint(tp[3], &sbct)) + goto badarg; + + tspec = &erts_allctr_thr_spec[ai]; + if (tspec->enabled) { + ok = 0; + for (i = 0; i < tspec->size; i++) { + allctr = tspec->allctr[i]; + ok |= allctr->try_set_dyn_param(allctr, am_sbct, sbct); + } + } + else { + allctr = erts_allctrs_info[ai].extra; + ok = allctr->try_set_dyn_param(allctr, am_sbct, sbct); + } + return ok ? am_ok : am_notsup; + } + return am_notsup; + +badarg: + ERTS_BIF_PREP_ERROR(res, c_p, EXC_BADARG); + return res; +} + /* * The allocator wrapper prelocking stuff below is about the locking order. * It only affects wrappers (erl_mtrace.c and erl_instrument.c) that keep locks @@ -3528,7 +3629,7 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) &init.init.af, &init.init.util); break; - case AOFIRSTFIT: + case FIRSTFIT: allctr = erts_aoffalc_start((AOFFAllctr_t *) erts_alloc(ERTS_ALC_T_UNDEF, sizeof(AOFFAllctr_t)), @@ -3622,7 +3723,9 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) case 0xf15: erts_free(ERTS_ALC_T_TEST, (void*)a1); return 0; - case 0xf16: { + case 0xf16: return (UWord) erts_realloc(ERTS_ALC_T_TEST, (void*)a1, (Uint)a2); + + case 0xf17: { Uint extra_hdr_sz = UNIT_CEILING((Uint)a1); ErtsAllocatorThrSpec_t* ts = &erts_allctr_thr_spec[ERTS_ALC_A_TEST]; Uint offset = ts->allctr[0]->mbc_header_size; @@ -3649,7 +3752,7 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) *(void**)a3 = orig_destroying_mbc; return offset; } - case 0xf17: { + case 0xf18: { ErtsAllocatorThrSpec_t* ts = &erts_allctr_thr_spec[ERTS_ALC_A_TEST]; return ts->allctr[0]->largest_mbc_size; } diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h index 0438b7cd8c..174bf7a80a 100644 --- a/erts/emulator/beam/erl_alloc.h +++ b/erts/emulator/beam/erl_alloc.h @@ -171,6 +171,8 @@ __decl_noreturn void erts_realloc_n_enomem(ErtsAlcType_t,void*,Uint) __decl_noreturn void erts_alc_fatal_error(int,int,ErtsAlcType_t,...) __noreturn; +Eterm erts_alloc_set_dyn_param(struct process*, Eterm); + #undef ERTS_HAVE_IS_IN_LITERAL_RANGE #if defined(ARCH_32) || defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) # define ERTS_HAVE_IS_IN_LITERAL_RANGE diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 4d4bddb93f..fa97ead908 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -362,8 +362,10 @@ do { \ #define ERTS_CRR_ALCTR_FLG_IN_POOL (((erts_aint_t) 1) << 0) #define ERTS_CRR_ALCTR_FLG_BUSY (((erts_aint_t) 1) << 1) +#define ERTS_CRR_ALCTR_FLG_HOMECOMING (((erts_aint_t) 1) << 2) #define ERTS_CRR_ALCTR_FLG_MASK (ERTS_CRR_ALCTR_FLG_IN_POOL | \ - ERTS_CRR_ALCTR_FLG_BUSY) + ERTS_CRR_ALCTR_FLG_BUSY | \ + ERTS_CRR_ALCTR_FLG_HOMECOMING) #define SBC_HEADER_SIZE \ (UNIT_CEILING(offsetof(Carrier_t, cpool) \ @@ -563,7 +565,7 @@ do { \ DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ } while (0) -#define STAT_MBC_CPOOL_INSERT(AP, CRR) \ +#define STAT_MBC_ABANDON(AP, CRR) \ do { \ UWord csz__ = CARRIER_SZ((CRR)); \ if (IS_MSEG_CARRIER((CRR))) \ @@ -1153,89 +1155,23 @@ unlink_carrier(CarrierList_t *cl, Carrier_t *crr) ASSERT(crr->next); crr->next->prev = crr->prev; } -} - - #ifdef DEBUG -static int is_in_list(ErtsDoubleLink_t* sentinel, ErtsDoubleLink_t* node) -{ - ErtsDoubleLink_t* p; - - ASSERT(node != sentinel); - for (p = sentinel->next; p != sentinel; p = p->next) { - if (p == node) - return 1; - } - return 0; -} -#endif /* DEBUG */ - -static ERTS_INLINE void -link_edl_after(ErtsDoubleLink_t* after_me, ErtsDoubleLink_t* node) -{ - ErtsDoubleLink_t* before_me = after_me->next; - ASSERT(node != after_me && node != before_me); - node->next = before_me; - node->prev = after_me; - before_me->prev = node; - after_me->next = node; -} - -static ERTS_INLINE void -link_edl_before(ErtsDoubleLink_t* before_me, ErtsDoubleLink_t* node) -{ - ErtsDoubleLink_t* after_me = before_me->prev; - ASSERT(node != before_me && node != after_me); - node->next = before_me; - node->prev = after_me; - before_me->prev = node; - after_me->next = node; -} - -static ERTS_INLINE void -unlink_edl(ErtsDoubleLink_t* node) -{ - node->next->prev = node->prev; - node->prev->next = node->next; -} - -static ERTS_INLINE void -relink_edl_before(ErtsDoubleLink_t* before_me, ErtsDoubleLink_t* node) -{ - if (node != before_me && node != before_me->prev) { - unlink_edl(node); - link_edl_before(before_me, node); - } + crr->next = crr; + crr->prev = crr; +#endif } static ERTS_INLINE int is_abandoned(Carrier_t *crr) { - return crr->cpool.abandoned.next != NULL; -} - -static ERTS_INLINE void -link_abandoned_carrier(ErtsDoubleLink_t* list, Carrier_t *crr) -{ - ASSERT(!is_abandoned(crr)); - - link_edl_after(list, &crr->cpool.abandoned); - - ASSERT(crr->cpool.abandoned.next != &crr->cpool.abandoned); - ASSERT(crr->cpool.abandoned.prev != &crr->cpool.abandoned); + return crr->cpool.state != ERTS_MBC_IS_HOME; } static ERTS_INLINE void unlink_abandoned_carrier(Carrier_t *crr) { - ASSERT(is_in_list(&crr->cpool.orig_allctr->cpool.pooled_list, - &crr->cpool.abandoned) || - is_in_list(&crr->cpool.orig_allctr->cpool.traitor_list, - &crr->cpool.abandoned)); - - unlink_edl(&crr->cpool.abandoned); - - crr->cpool.abandoned.next = NULL; - crr->cpool.abandoned.prev = NULL; + if (crr->cpool.state == ERTS_MBC_WAS_POOLED) { + aoff_remove_pooled_mbc(crr->cpool.orig_allctr, crr); + } } static ERTS_INLINE void @@ -1243,24 +1179,19 @@ clear_busy_pool_carrier(Allctr_t *allctr, Carrier_t *crr) { if (crr) { erts_aint_t max_size; - erts_aint_t new_val; + erts_aint_t iallctr; max_size = (erts_aint_t) allctr->largest_fblk_in_mbc(allctr, crr); erts_atomic_set_nob(&crr->cpool.max_size, max_size); - new_val = (((erts_aint_t) allctr)|ERTS_CRR_ALCTR_FLG_IN_POOL); + iallctr = erts_atomic_read_nob(&crr->allctr); + ERTS_ALC_CPOOL_ASSERT((iallctr & ~ERTS_CRR_ALCTR_FLG_HOMECOMING) + == ((erts_aint_t)allctr | + ERTS_CRR_ALCTR_FLG_IN_POOL | + ERTS_CRR_ALCTR_FLG_BUSY)); -#ifdef ERTS_ALC_CPOOL_DEBUG - { - erts_aint_t old_val = new_val|ERTS_CRR_ALCTR_FLG_BUSY; - - ERTS_ALC_CPOOL_ASSERT(old_val - == erts_atomic_xchg_relb(&crr->allctr, - new_val)); - } -#else - erts_atomic_set_relb(&crr->allctr, new_val); -#endif + iallctr &= ~ERTS_CRR_ALCTR_FLG_BUSY; + erts_atomic_set_relb(&crr->allctr, iallctr); } } @@ -1658,6 +1589,11 @@ dealloc_mbc(Allctr_t *allctr, Carrier_t *crr) } +static void set_new_allctr_abandon_limit(Allctr_t*); +static void abandon_carrier(Allctr_t*, Carrier_t*); +static void poolify_my_carrier(Allctr_t*, Carrier_t*); +static void enqueue_homecoming(Allctr_t*, Carrier_t*); + static ERTS_INLINE Allctr_t* get_pref_allctr(void *extra) { @@ -1724,9 +1660,23 @@ get_used_allctr(Allctr_t *pref_allctr, int pref_lock, void *p, UWord *sizep, erts_aint_t act; ERTS_ALC_CPOOL_ASSERT(!(iallctr & ERTS_CRR_ALCTR_FLG_BUSY)); - act = erts_atomic_cmpxchg_ddrb(&crr->allctr, - iallctr|ERTS_CRR_ALCTR_FLG_BUSY, - iallctr); + if (iallctr & ERTS_CRR_ALCTR_FLG_HOMECOMING) { + /* + * This carrier has just been given back to us by writing + * to crr->allctr with a write barrier (see abandon_carrier). + * + * We need a mathing read barrier to guarantee a correct view + * of the carrier for deallocation work. + */ + act = erts_atomic_cmpxchg_rb(&crr->allctr, + iallctr|ERTS_CRR_ALCTR_FLG_BUSY, + iallctr); + } + else { + act = erts_atomic_cmpxchg_ddrb(&crr->allctr, + iallctr|ERTS_CRR_ALCTR_FLG_BUSY, + iallctr); + } if (act == iallctr) { *busy_pcrr_pp = crr; break; @@ -1742,13 +1692,6 @@ get_used_allctr(Allctr_t *pref_allctr, int pref_lock, void *p, UWord *sizep, erts_mtx_unlock(&pref_allctr->mutex); } } - - ERTS_ALC_CPOOL_ASSERT( - (((iallctr & ~ERTS_CRR_ALCTR_FLG_MASK) == (erts_aint_t) pref_allctr) - ? (((iallctr & ERTS_CRR_ALCTR_FLG_MASK) == ERTS_CRR_ALCTR_FLG_IN_POOL) - || ((iallctr & ERTS_CRR_ALCTR_FLG_MASK) == 0)) - : 1)); - return used_allctr; } } @@ -2000,9 +1943,9 @@ handle_delayed_fix_dealloc(Allctr_t *allctr, void *ptr) /* Carrier migrated; need to redirect block to new owner... */ int cinit = used_allctr->dd.ix - allctr->dd.ix; - ERTS_ALC_CPOOL_ASSERT(!busy_pcrr_p); + ERTS_ALC_CPOOL_ASSERT(!busy_pcrr_p); - DEC_CC(allctr->calls.this_free); + DEC_CC(allctr->calls.this_free); ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type = type; if (ddq_enqueue(&used_allctr->dd.q, ptr, cinit)) @@ -2011,8 +1954,9 @@ handle_delayed_fix_dealloc(Allctr_t *allctr, void *ptr) } } -static void -schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr); +static void schedule_dealloc_carrier(Allctr_t*, Carrier_t*); +static void dealloc_my_carrier(Allctr_t*, Carrier_t*); + static ERTS_INLINE int handle_delayed_dealloc(Allctr_t *allctr, @@ -2074,39 +2018,61 @@ handle_delayed_dealloc(Allctr_t *allctr, res = 1; blk = UMEM2BLK(ptr); - if (IS_FREE_LAST_MBC_BLK(blk)) { + if (blk->bhdr == HOMECOMING_MBC_BLK_HDR) { /* * A multiblock carrier that previously has been migrated away - * from us and now is back to be deallocated. For more info - * see schedule_dealloc_carrier(). - * - * Note that we cannot use FBLK_TO_MBC(blk) since it - * data has been overwritten by the queue. + * from us, was sent back to us either because + * - it became empty and we need to deallocated it, or + * - it was inserted into the pool and we need to update our pooled_tree */ - Carrier_t *crr = FIRST_BLK_TO_MBC(allctr, blk); - - /* Restore word overwritten by the dd-queue as it will be read - * if this carrier is pulled from dc_list by cpool_fetch() - */ - ERTS_ALC_CPOOL_ASSERT(FBLK_TO_MBC(blk) != crr); - ERTS_CT_ASSERT(sizeof(ErtsAllctrDDBlock_t) == sizeof(void*)); -#ifdef MBC_ABLK_OFFSET_BITS - blk->u.carrier = crr; -#else - blk->carrier = crr; -#endif + Carrier_t *crr = ErtsContainerStruct(blk, Carrier_t, + cpool.homecoming_dd.blk); + Block_t* first_blk = MBC_TO_FIRST_BLK(allctr, crr); + erts_aint_t iallctr; ERTS_ALC_CPOOL_ASSERT(ERTS_ALC_IS_CPOOL_ENABLED(allctr)); ERTS_ALC_CPOOL_ASSERT(allctr == crr->cpool.orig_allctr); - ERTS_ALC_CPOOL_ASSERT(((erts_aint_t) allctr) - != (erts_atomic_read_nob(&crr->allctr) - & ~ERTS_CRR_ALCTR_FLG_MASK)); - - erts_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr)); - schedule_dealloc_carrier(allctr, crr); + iallctr = erts_atomic_read_nob(&crr->allctr); + ASSERT(iallctr & ERTS_CRR_ALCTR_FLG_HOMECOMING); + while (1) { + if ((iallctr & (~ERTS_CRR_ALCTR_FLG_MASK | + ERTS_CRR_ALCTR_FLG_IN_POOL)) + == (erts_aint_t)allctr) { + /* + * Carrier is home (mine and not in pool) + */ + ASSERT(!(iallctr & ERTS_CRR_ALCTR_FLG_BUSY)); + erts_atomic_set_nob(&crr->allctr, (erts_aint_t)allctr); + if (IS_FREE_LAST_MBC_BLK(first_blk)) + dealloc_my_carrier(allctr, crr); + else + ASSERT(crr->cpool.state == ERTS_MBC_IS_HOME); + } + else { + erts_aint_t exp = iallctr; + erts_aint_t want = iallctr & ~ERTS_CRR_ALCTR_FLG_HOMECOMING; + + iallctr = erts_atomic_cmpxchg_nob(&crr->allctr, + want, + exp); + if (iallctr != exp) + continue; /* retry */ + + ASSERT(crr->cpool.state != ERTS_MBC_IS_HOME); + unlink_abandoned_carrier(crr); + if (iallctr & ERTS_CRR_ALCTR_FLG_IN_POOL) + poolify_my_carrier(allctr, crr); + else + crr->cpool.state = ERTS_MBC_WAS_TRAITOR; + } + break; + } } else { + ASSERT(IS_SBC_BLK(blk) || (ABLK_TO_MBC(blk) != + ErtsContainerStruct(blk, Carrier_t, + cpool.homecoming_dd.blk))); INC_CC(allctr->calls.this_free); @@ -2148,20 +2114,26 @@ enqueue_dealloc_other_instance(ErtsAlcType_t type, erts_alloc_notify_delayed_dealloc(allctr->ix); } - -static void -set_new_allctr_abandon_limit(Allctr_t *allctr); -static void -abandon_carrier(Allctr_t *allctr, Carrier_t *crr); - +static ERTS_INLINE void +update_pooled_tree(Allctr_t *allctr, Carrier_t *crr, Uint blk_sz) +{ + if (allctr == crr->cpool.orig_allctr && crr->cpool.state == ERTS_MBC_WAS_POOLED) { + /* + * Update pooled_tree with a potentially new (larger) max_sz + */ + AOFF_RBTree_t* crr_node = &crr->cpool.pooled; + if (blk_sz > crr_node->hdr.bhdr) { + crr_node->hdr.bhdr = blk_sz; + erts_aoff_larger_max_size(crr_node); + } + } +} static ERTS_INLINE void check_abandon_carrier(Allctr_t *allctr, Block_t *fblk, Carrier_t **busy_pcrr_pp) { Carrier_t *crr; - - if (busy_pcrr_pp && *busy_pcrr_pp) - return; + UWord ncrr_in_pool, largest_fblk; if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) return; @@ -2170,8 +2142,7 @@ check_abandon_carrier(Allctr_t *allctr, Block_t *fblk, Carrier_t **busy_pcrr_pp) if (--allctr->cpool.check_limit_count <= 0) set_new_allctr_abandon_limit(allctr); - if (!erts_thr_progress_is_managed_thread()) - return; + ASSERT(erts_thr_progress_is_managed_thread()); if (allctr->cpool.disable_abandon) return; @@ -2179,6 +2150,9 @@ check_abandon_carrier(Allctr_t *allctr, Block_t *fblk, Carrier_t **busy_pcrr_pp) if (allctr->mbcs.blocks.curr.size > allctr->cpool.abandon_limit) return; + ncrr_in_pool = erts_atomic_read_nob(&allctr->cpool.stat.no_carriers); + if (ncrr_in_pool >= allctr->cpool.in_pool_limit) + return; crr = FBLK_TO_MBC(fblk); @@ -2189,9 +2163,14 @@ check_abandon_carrier(Allctr_t *allctr, Block_t *fblk, Carrier_t **busy_pcrr_pp) return; if (crr->cpool.thr_prgr != ERTS_THR_PRGR_INVALID - && !erts_thr_progress_has_reached(crr->cpool.thr_prgr)) - return; + && !erts_thr_progress_has_reached(crr->cpool.thr_prgr)) + return; + + largest_fblk = allctr->largest_fblk_in_mbc(allctr, crr); + if (largest_fblk < allctr->cpool.fblk_min_limit) + return; + erts_atomic_set_nob(&crr->cpool.max_size, largest_fblk); abandon_carrier(allctr, crr); } @@ -2237,6 +2216,7 @@ dealloc_block(Allctr_t *allctr, void *ptr, ErtsAlcFixList_t *fix, int dec_cc_on_ else { Carrier_t *busy_pcrr_p; Allctr_t *used_allctr; + used_allctr = get_used_allctr(allctr, ERTS_ALC_TS_PREF_LOCK_NO, ptr, NULL, &busy_pcrr_p); if (used_allctr == allctr) { @@ -2253,10 +2233,10 @@ dealloc_block(Allctr_t *allctr, void *ptr, ErtsAlcFixList_t *fix, int dec_cc_on_ /* Carrier migrated; need to redirect block to new owner... */ int cinit = used_allctr->dd.ix - allctr->dd.ix; - ERTS_ALC_CPOOL_ASSERT(!busy_pcrr_p); + ERTS_ALC_CPOOL_ASSERT(!busy_pcrr_p); - if (dec_cc_on_redirect) - DEC_CC(allctr->calls.this_free); + if (dec_cc_on_redirect) + DEC_CC(allctr->calls.this_free); if (ddq_enqueue(&used_allctr->dd.q, ptr, cinit)) erts_alloc_notify_delayed_dealloc(used_allctr->ix); } @@ -2500,15 +2480,16 @@ mbc_free(Allctr_t *allctr, void *p, Carrier_t **busy_pcrr_pp) ASSERT(blk_sz % sizeof(Unit_t) == 0); ASSERT(IS_MBC_BLK(blk)); - if (is_first_blk - && is_last_blk - && allctr->main_carrier != FIRST_BLK_TO_MBC(allctr, blk)) { - destroy_carrier(allctr, blk, busy_pcrr_pp); + if (is_first_blk && is_last_blk && crr != allctr->main_carrier) { + destroy_carrier(allctr, blk, busy_pcrr_pp); } else { (*allctr->link_free_block)(allctr, blk); HARD_CHECK_BLK_CARRIER(allctr, blk); - check_abandon_carrier(allctr, blk, busy_pcrr_pp); + if (busy_pcrr_pp && *busy_pcrr_pp) + update_pooled_tree(allctr, crr, blk_sz); + else + check_abandon_carrier(allctr, blk, busy_pcrr_pp); } } @@ -2542,8 +2523,19 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs, return NULL; #else /* !MBC_REALLOC_ALWAYS_MOVES */ - if (busy_pcrr_pp && *busy_pcrr_pp) - goto realloc_move; /* Don't want to use carrier in pool */ + if (busy_pcrr_pp && *busy_pcrr_pp) { + /* + * Don't want to use carrier in pool + */ + new_p = mbc_alloc(allctr, size); + if (!new_p) + return NULL; + new_blk = UMEM2BLK(new_p); + ASSERT(!(IS_MBC_BLK(new_blk) && ABLK_TO_MBC(new_blk) == *busy_pcrr_pp)); + sys_memcpy(new_p, p, MIN(size, old_blk_sz - ABLK_HDR_SZ)); + mbc_free(allctr, p, busy_pcrr_pp); + return new_p; + } get_blk_sz = blk_sz = UMEMSZ2BLKSZ(allctr, size); @@ -2776,7 +2768,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs, if (cand_blk_sz < get_blk_sz) { /* We wont fit in cand_blk get a new one */ - realloc_move: + #endif /* !MBC_REALLOC_ALWAYS_MOVES */ new_p = mbc_alloc(allctr, size); @@ -2880,8 +2872,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs, #define ERTS_ALC_MAX_DEALLOC_CARRIER 10 -#define ERTS_ALC_CPOOL_MAX_FETCH_INSPECT 20 -#define ERTS_ALC_CPOOL_MAX_TRAITOR_INSPECT 10 +#define ERTS_ALC_CPOOL_MAX_FETCH_INSPECT 100 #define ERTS_ALC_CPOOL_CHECK_LIMIT_COUNT 100 #define ERTS_ALC_CPOOL_MAX_FAILED_STAT_READS 3 @@ -3045,19 +3036,18 @@ cpool_insert(Allctr_t *allctr, Carrier_t *crr) ErtsAlcCPoolData_t *cpd1p, *cpd2p; erts_aint_t val; ErtsAlcCPoolData_t *sentinel = &carrier_pool[allctr->alloc_no].sentinel; + Allctr_t *orig_allctr = crr->cpool.orig_allctr; ERTS_ALC_CPOOL_ASSERT(allctr->alloc_no == ERTS_ALC_A_INVALID /* testcase */ || erts_thr_progress_is_managed_thread()); - ERTS_ALC_CPOOL_ASSERT(erts_atomic_read_nob(&crr->allctr) - == (erts_aint_t) allctr); - erts_atomic_add_nob(&allctr->cpool.stat.blocks_size, + erts_atomic_add_nob(&orig_allctr->cpool.stat.blocks_size, (erts_aint_t) crr->cpool.blocks_size); - erts_atomic_add_nob(&allctr->cpool.stat.no_blocks, + erts_atomic_add_nob(&orig_allctr->cpool.stat.no_blocks, (erts_aint_t) crr->cpool.blocks); - erts_atomic_add_nob(&allctr->cpool.stat.carriers_size, + erts_atomic_add_nob(&orig_allctr->cpool.stat.carriers_size, (erts_aint_t) CARRIER_SZ(crr)); - erts_atomic_inc_nob(&allctr->cpool.stat.no_carriers); + erts_atomic_inc_nob(&orig_allctr->cpool.stat.no_carriers); /* * We search in 'next' direction and begin by passing @@ -3118,8 +3108,6 @@ cpool_insert(Allctr_t *allctr, Carrier_t *crr) (erts_aint_t) &crr->cpool, (erts_aint_t) cpd1p); - erts_atomic_set_wb(&crr->allctr, - ((erts_aint_t) allctr)|ERTS_CRR_ALCTR_FLG_IN_POOL); LTTNG3(carrier_pool_put, ERTS_ALC_A2AD(allctr->alloc_no), allctr->ix, CARRIER_SZ(crr)); } @@ -3221,130 +3209,126 @@ cpool_delete(Allctr_t *allctr, Allctr_t *prev_allctr, Carrier_t *crr) static Carrier_t * cpool_fetch(Allctr_t *allctr, UWord size) { - int i, i_stop, has_passed_sentinel; + enum { IGNORANT, HAS_SEEN_SENTINEL, THE_LAST_ONE } loop_state; + int i; Carrier_t *crr; + Carrier_t *reinsert_crr = NULL; ErtsAlcCPoolData_t *cpdp; - ErtsAlcCPoolData_t *cpool_entrance; + ErtsAlcCPoolData_t *cpool_entrance = NULL; ErtsAlcCPoolData_t *sentinel; - ErtsDoubleLink_t* dl; - ErtsDoubleLink_t* first_old_traitor; ERTS_ALC_CPOOL_ASSERT(allctr->alloc_no == ERTS_ALC_A_INVALID /* testcase */ || erts_thr_progress_is_managed_thread()); i = ERTS_ALC_CPOOL_MAX_FETCH_INSPECT; - first_old_traitor = allctr->cpool.traitor_list.next; - cpool_entrance = NULL; LTTNG3(carrier_pool_get, ERTS_ALC_A2AD(allctr->alloc_no), allctr->ix, (unsigned long)size); /* - * Search my own pooled_list, + * Search my own pooled_tree, * i.e my abandoned carriers that were in the pool last time I checked. */ + do { + erts_aint_t exp, act; + + crr = aoff_lookup_pooled_mbc(allctr, size); + if (!crr) + break; + + ASSERT(crr->cpool.state == ERTS_MBC_WAS_POOLED); + ASSERT(crr->cpool.orig_allctr == allctr); + + aoff_remove_pooled_mbc(allctr, crr); + + exp = erts_atomic_read_nob(&crr->allctr); + if (exp & ERTS_CRR_ALCTR_FLG_IN_POOL) { + ASSERT((exp & ~ERTS_CRR_ALCTR_FLG_MASK) == (erts_aint_t)allctr); + if (erts_atomic_read_nob(&crr->cpool.max_size) < size) { + /* + * This carrier has been fetched and inserted back again + * by a foreign allocator. That's why it has a stale search size. + */ + ASSERT(exp & ERTS_CRR_ALCTR_FLG_HOMECOMING); + crr->cpool.pooled.hdr.bhdr = erts_atomic_read_nob(&crr->cpool.max_size); + aoff_add_pooled_mbc(allctr, crr); + INC_CC(allctr->cpool.stat.skip_size); + continue; + } + else if (exp & ERTS_CRR_ALCTR_FLG_BUSY) { + /* + * This must be our own carrier as part of a realloc call. + * Skip it to make things simpler. + * Must wait to re-insert to not be found again by lookup. + */ + ASSERT(!reinsert_crr); + reinsert_crr = crr; + INC_CC(allctr->cpool.stat.skip_busy); + continue; + } - dl = allctr->cpool.pooled_list.next; - while(dl != &allctr->cpool.pooled_list) { - erts_aint_t exp, act; - crr = (Carrier_t *) (((char *) dl) - offsetof(Carrier_t, cpool.abandoned)); - - ASSERT(!is_in_list(&allctr->cpool.traitor_list, dl)); - ASSERT(crr->cpool.orig_allctr == allctr); - dl = dl->next; - exp = erts_atomic_read_rb(&crr->allctr); - if ((exp & ERTS_CRR_ALCTR_FLG_MASK) == ERTS_CRR_ALCTR_FLG_IN_POOL - && erts_atomic_read_nob(&crr->cpool.max_size) >= size) { - /* Try to fetch it... */ - act = erts_atomic_cmpxchg_mb(&crr->allctr, - (erts_aint_t) allctr, - exp); - if (act == exp) { - cpool_delete(allctr, ((Allctr_t *) (act & ~ERTS_CRR_ALCTR_FLG_MASK)), crr); - unlink_abandoned_carrier(crr); - - /* Move sentinel to continue next search from here */ - relink_edl_before(dl, &allctr->cpool.pooled_list); - return crr; - } - exp = act; - } - if (exp & ERTS_CRR_ALCTR_FLG_IN_POOL) { - if (!cpool_entrance) - cpool_entrance = &crr->cpool; - } - else { /* Not in pool, move to traitor_list */ - unlink_abandoned_carrier(crr); - link_abandoned_carrier(&allctr->cpool.traitor_list, crr); - } - if (--i <= 0) { - /* Move sentinel to continue next search from here */ - relink_edl_before(dl, &allctr->cpool.pooled_list); - return NULL; - } - } + /* Try to fetch it... */ + act = erts_atomic_cmpxchg_mb(&crr->allctr, + exp & ~ERTS_CRR_ALCTR_FLG_IN_POOL, + exp); + if (act == exp) { + cpool_delete(allctr, allctr, crr); + crr->cpool.state = ERTS_MBC_IS_HOME; + + if (reinsert_crr) + aoff_add_pooled_mbc(allctr, reinsert_crr); + return crr; + } + exp = act; + INC_CC(allctr->cpool.stat.skip_race); + } + else + INC_CC(allctr->cpool.stat.skip_not_pooled); - /* Now search traitor_list. - * i.e carriers employed by other allocators last time I checked. - * They might have been abandoned since then. - */ + /* Not in pool anymore */ + ASSERT(!(exp & ERTS_CRR_ALCTR_FLG_BUSY)); + crr->cpool.state = ERTS_MBC_WAS_TRAITOR; - i_stop = (i < ERTS_ALC_CPOOL_MAX_TRAITOR_INSPECT ? - 0 : i - ERTS_ALC_CPOOL_MAX_TRAITOR_INSPECT); - dl = first_old_traitor; - while(dl != &allctr->cpool.traitor_list) { - erts_aint_t exp, act; - crr = (Carrier_t *) (((char *) dl) - offsetof(Carrier_t, cpool.abandoned)); - ASSERT(dl != &allctr->cpool.pooled_list); - ASSERT(crr->cpool.orig_allctr == allctr); - dl = dl->next; - exp = erts_atomic_read_rb(&crr->allctr); - if (exp & ERTS_CRR_ALCTR_FLG_IN_POOL) { - if (!(exp & ERTS_CRR_ALCTR_FLG_BUSY) - && erts_atomic_read_nob(&crr->cpool.max_size) >= size) { - /* Try to fetch it... */ - act = erts_atomic_cmpxchg_mb(&crr->allctr, - (erts_aint_t) allctr, - exp); - if (act == exp) { - cpool_delete(allctr, ((Allctr_t *) (act & ~ERTS_CRR_ALCTR_FLG_MASK)), crr); - unlink_abandoned_carrier(crr); + }while (--i > 0); - /* Move sentinel to continue next search from here */ - relink_edl_before(dl, &allctr->cpool.traitor_list); - return crr; - } - exp = act; - } - if (exp & ERTS_CRR_ALCTR_FLG_IN_POOL) { - if (!cpool_entrance) - cpool_entrance = &crr->cpool; + if (reinsert_crr) + aoff_add_pooled_mbc(allctr, reinsert_crr); - /* Move to pooled_list */ - unlink_abandoned_carrier(crr); - link_abandoned_carrier(&allctr->cpool.pooled_list, crr); - } - } - if (--i <= i_stop) { - /* Move sentinel to continue next search from here */ - relink_edl_before(dl, &allctr->cpool.traitor_list); - if (i > 0) - break; - else - return NULL; - } + /* + * Try find a nice cpool_entrance + */ + while (allctr->cpool.pooled_tree) { + erts_aint_t iallctr; + + crr = ErtsContainerStruct(allctr->cpool.pooled_tree, Carrier_t, cpool.pooled); + iallctr = erts_atomic_read_nob(&crr->allctr); + if (iallctr & ERTS_CRR_ALCTR_FLG_IN_POOL) { + cpool_entrance = &crr->cpool; + break; + } + /* Not in pool anymore */ + ASSERT(!(iallctr & ERTS_CRR_ALCTR_FLG_BUSY)); + aoff_remove_pooled_mbc(allctr, crr); + crr->cpool.state = ERTS_MBC_WAS_TRAITOR; + + if (--i <= 0) { + INC_CC(allctr->cpool.stat.fail_pooled); + return NULL; + } } + /* * Finally search the shared pool and try employ foreign carriers */ - sentinel = &carrier_pool[allctr->alloc_no].sentinel; if (cpool_entrance) { - /* We saw a pooled carried above, use it as entrance into the pool + /* + * We saw a pooled carried above, use it as entrance into the pool */ cpdp = cpool_entrance; } else { - /* No pooled carried seen above. Start search at cpool sentinel, + /* + * No pooled carried seen above. Start search at cpool sentinel, * but begin by passing one element before trying to fetch. * This in order to avoid contention with threads inserting elements. */ @@ -3354,8 +3338,8 @@ cpool_fetch(Allctr_t *allctr, UWord size) goto check_dc_list; } - has_passed_sentinel = 0; - while (1) { + loop_state = IGNORANT; + do { erts_aint_t exp; cpdp = cpool_aint2cpd(cpool_read(&cpdp->prev)); if (cpdp == cpool_entrance) { @@ -3364,38 +3348,52 @@ cpool_fetch(Allctr_t *allctr, UWord size) if (cpdp == sentinel) break; } - i = 0; /* Last one to inspect */ + loop_state = THE_LAST_ONE; } else if (cpdp == sentinel) { - if (has_passed_sentinel) { + if (loop_state == HAS_SEEN_SENTINEL) { /* We been here before. cpool_entrance must have been removed */ + INC_CC(allctr->cpool.stat.entrance_removed); break; } cpdp = cpool_aint2cpd(cpool_read(&cpdp->prev)); if (cpdp == sentinel) break; - has_passed_sentinel = 1; + loop_state = HAS_SEEN_SENTINEL; } - crr = (Carrier_t *)(((char *)cpdp) - offsetof(Carrier_t, cpool)); + crr = ErtsContainerStruct(cpdp, Carrier_t, cpool); exp = erts_atomic_read_rb(&crr->allctr); - if (((exp & (ERTS_CRR_ALCTR_FLG_MASK)) == ERTS_CRR_ALCTR_FLG_IN_POOL) - && (erts_atomic_read_nob(&cpdp->max_size) >= size)) { + + if (erts_atomic_read_nob(&cpdp->max_size) < size) { + INC_CC(allctr->cpool.stat.skip_size); + } + else if ((exp & (ERTS_CRR_ALCTR_FLG_IN_POOL | ERTS_CRR_ALCTR_FLG_BUSY)) + == ERTS_CRR_ALCTR_FLG_IN_POOL) { erts_aint_t act; - /* Try to fetch it... */ - act = erts_atomic_cmpxchg_mb(&crr->allctr, - (erts_aint_t) allctr, - exp); + erts_aint_t want = (((erts_aint_t) allctr) + | (exp & ERTS_CRR_ALCTR_FLG_HOMECOMING)); + /* Try to fetch it... */ + act = erts_atomic_cmpxchg_mb(&crr->allctr, want, exp); if (act == exp) { cpool_delete(allctr, ((Allctr_t *) (act & ~ERTS_CRR_ALCTR_FLG_MASK)), crr); if (crr->cpool.orig_allctr == allctr) { unlink_abandoned_carrier(crr); - } + crr->cpool.state = ERTS_MBC_IS_HOME; + } return crr; } } - if (--i <= 0) + + if (exp & ERTS_CRR_ALCTR_FLG_BUSY) + INC_CC(allctr->cpool.stat.skip_busy); + else + INC_CC(allctr->cpool.stat.skip_race); + + if (--i <= 0) { + INC_CC(allctr->cpool.stat.fail_shared); return NULL; - } + } + }while (loop_state != THE_LAST_ONE); check_dc_list: /* Last; check our own pending dealloc carrier list... */ @@ -3404,23 +3402,23 @@ check_dc_list: if (erts_atomic_read_nob(&crr->cpool.max_size) >= size) { Block_t* blk; unlink_carrier(&allctr->cpool.dc_list, crr); -#ifdef ERTS_ALC_CPOOL_DEBUG - ERTS_ALC_CPOOL_ASSERT(erts_atomic_xchg_nob(&crr->allctr, - ((erts_aint_t) allctr)) - == (((erts_aint_t) allctr) & ~ERTS_CRR_ALCTR_FLG_MASK)); -#else - erts_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr)); -#endif + ERTS_ALC_CPOOL_ASSERT(erts_atomic_read_nob(&crr->allctr) + == ((erts_aint_t) allctr)); blk = MBC_TO_FIRST_BLK(allctr, crr); ASSERT(FBLK_TO_MBC(blk) == crr); allctr->link_free_block(allctr, blk); return crr; } crr = crr->prev; - if (--i <= 0) + if (--i <= 0) { + INC_CC(allctr->cpool.stat.fail_pend_dealloc); return NULL; + } } + if (i != ERTS_ALC_CPOOL_MAX_FETCH_INSPECT) + INC_CC(allctr->cpool.stat.fail); + return NULL; } @@ -3475,9 +3473,6 @@ static void schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr) { Allctr_t *orig_allctr; - Block_t *blk; - int check_pending_dealloc; - erts_aint_t max_size; ASSERT(IS_MB_CARRIER(crr)); @@ -3488,9 +3483,17 @@ schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr) orig_allctr = crr->cpool.orig_allctr; - if (allctr != orig_allctr) { - int cinit = orig_allctr->dd.ix - allctr->dd.ix; - + if (allctr == orig_allctr) { + if (!(erts_atomic_read_nob(&crr->allctr) & ERTS_CRR_ALCTR_FLG_HOMECOMING)) { + dealloc_my_carrier(allctr, crr); + } + /*else + * Carrier was abandoned earlier by other thread and + * is still waiting for us in dd-queue. + * handle_delayed_dealloc() will handle it when crr is dequeued. + */ + } + else { /* * We send the carrier to its origin for deallocation. * This in order: @@ -3499,29 +3502,39 @@ schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr) * - to ensure that we always only reuse empty carriers * originating from our own thread specific mseg_alloc * instance which is beneficial on NUMA systems. - * - * The receiver will recognize that this is a carrier to - * deallocate (and not a block which is the common case) - * since the block is an mbc block that is free and last - * in the carrier. */ - blk = MBC_TO_FIRST_BLK(allctr, crr); - ERTS_ALC_CPOOL_ASSERT(IS_FREE_LAST_MBC_BLK(blk)); - - ERTS_ALC_CPOOL_ASSERT(IS_MBC_FIRST_ABLK(allctr, blk)); - ERTS_ALC_CPOOL_ASSERT(crr == FBLK_TO_MBC(blk)); - ERTS_ALC_CPOOL_ASSERT(crr == FIRST_BLK_TO_MBC(allctr, blk)); - ERTS_ALC_CPOOL_ASSERT(((erts_aint_t) allctr) - == (erts_atomic_read_nob(&crr->allctr) - & ~ERTS_CRR_ALCTR_FLG_MASK)); + erts_aint_t iallctr; +#ifdef ERTS_ALC_CPOOL_DEBUG + Block_t* first_blk = MBC_TO_FIRST_BLK(allctr, crr); + ERTS_ALC_CPOOL_ASSERT(IS_FREE_LAST_MBC_BLK(first_blk)); + + ERTS_ALC_CPOOL_ASSERT(IS_MBC_FIRST_ABLK(allctr, first_blk)); + ERTS_ALC_CPOOL_ASSERT(crr == FBLK_TO_MBC(first_blk)); + ERTS_ALC_CPOOL_ASSERT(crr == FIRST_BLK_TO_MBC(allctr, first_blk)); + ERTS_ALC_CPOOL_ASSERT((erts_atomic_read_nob(&crr->allctr) + & ~ERTS_CRR_ALCTR_FLG_HOMECOMING) + == (erts_aint_t) allctr); +#endif - if (ddq_enqueue(&orig_allctr->dd.q, BLK2UMEM(blk), cinit)) - erts_alloc_notify_delayed_dealloc(orig_allctr->ix); - return; + iallctr = (erts_aint_t)orig_allctr | ERTS_CRR_ALCTR_FLG_HOMECOMING; + if (!(erts_atomic_xchg_nob(&crr->allctr, iallctr) + & ERTS_CRR_ALCTR_FLG_HOMECOMING)) { + enqueue_homecoming(allctr, crr); + } } +} + +static void dealloc_my_carrier(Allctr_t *allctr, Carrier_t *crr) +{ + Block_t *blk; + int check_pending_dealloc; + erts_aint_t max_size; - if (is_abandoned(crr)) - unlink_abandoned_carrier(crr); + ERTS_ALC_CPOOL_ASSERT(allctr == crr->cpool.orig_allctr); + if (is_abandoned(crr)) { + unlink_abandoned_carrier(crr); + crr->cpool.state = ERTS_MBC_IS_HOME; + } if (crr->cpool.thr_prgr == ERTS_THR_PRGR_INVALID || erts_thr_progress_has_reached(crr->cpool.thr_prgr)) { @@ -3553,6 +3566,7 @@ schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr) static ERTS_INLINE void cpool_init_carrier_data(Allctr_t *allctr, Carrier_t *crr) { + crr->cpool.homecoming_dd.blk.bhdr = HOMECOMING_MBC_BLK_HDR; erts_atomic_init_nob(&crr->cpool.next, ERTS_AINT_NULL); erts_atomic_init_nob(&crr->cpool.prev, ERTS_AINT_NULL); crr->cpool.orig_allctr = allctr; @@ -3571,8 +3585,7 @@ cpool_init_carrier_data(Allctr_t *allctr, Carrier_t *crr) limit = (csz/100)*allctr->cpool.util_limit; crr->cpool.abandon_limit = limit; } - crr->cpool.abandoned.next = NULL; - crr->cpool.abandoned.prev = NULL; + crr->cpool.state = ERTS_MBC_IS_HOME; } static void @@ -3598,23 +3611,62 @@ set_new_allctr_abandon_limit(Allctr_t *allctr) static void abandon_carrier(Allctr_t *allctr, Carrier_t *crr) { - erts_aint_t max_size; + erts_aint_t iallctr; - STAT_MBC_CPOOL_INSERT(allctr, crr); + STAT_MBC_ABANDON(allctr, crr); unlink_carrier(&allctr->mbc_list, crr); - if (crr->cpool.orig_allctr == allctr) { - link_abandoned_carrier(&allctr->cpool.pooled_list, crr); + allctr->remove_mbc(allctr, crr); + set_new_allctr_abandon_limit(allctr); + + cpool_insert(allctr, crr); + + + iallctr = erts_atomic_read_nob(&crr->allctr); + if (allctr == crr->cpool.orig_allctr) { + /* preserve HOMECOMING flag */ + ASSERT((iallctr & ~ERTS_CRR_ALCTR_FLG_HOMECOMING) == (erts_aint_t)allctr); + erts_atomic_set_wb(&crr->allctr, iallctr | ERTS_CRR_ALCTR_FLG_IN_POOL); + poolify_my_carrier(allctr, crr); } + else { + ASSERT((iallctr & ~ERTS_CRR_ALCTR_FLG_HOMECOMING) == (erts_aint_t)allctr); + iallctr = ((erts_aint_t)crr->cpool.orig_allctr | + ERTS_CRR_ALCTR_FLG_HOMECOMING | + ERTS_CRR_ALCTR_FLG_IN_POOL); + if (!(erts_atomic_xchg_wb(&crr->allctr, iallctr) + & ERTS_CRR_ALCTR_FLG_HOMECOMING)) { + + enqueue_homecoming(allctr, crr); + } + } +} - allctr->remove_mbc(allctr, crr); +static void +enqueue_homecoming(Allctr_t* allctr, Carrier_t* crr) +{ + Allctr_t* orig_allctr = crr->cpool.orig_allctr; + const int cinit = orig_allctr->dd.ix - allctr->dd.ix; + Block_t* dd_blk = &crr->cpool.homecoming_dd.blk; - max_size = (erts_aint_t) allctr->largest_fblk_in_mbc(allctr, crr); - erts_atomic_set_nob(&crr->cpool.max_size, max_size); + /* + * The receiver will recognize this as a carrier + * (and not a block which is the common case) + * since the block header is HOMECOMING_MBC_BLK_HDR. + */ + ASSERT(dd_blk->bhdr == HOMECOMING_MBC_BLK_HDR); + if (ddq_enqueue(&orig_allctr->dd.q, BLK2UMEM(dd_blk), cinit)) + erts_alloc_notify_delayed_dealloc(orig_allctr->ix); +} - cpool_insert(allctr, crr); +static void +poolify_my_carrier(Allctr_t *allctr, Carrier_t *crr) +{ + ERTS_ALC_CPOOL_ASSERT(allctr == crr->cpool.orig_allctr); - set_new_allctr_abandon_limit(allctr); + crr->cpool.pooled.hdr.bhdr = erts_atomic_read_nob(&crr->cpool.max_size); + aoff_add_pooled_mbc(allctr, crr); + crr->cpool.state = ERTS_MBC_WAS_POOLED; } static void @@ -3771,6 +3823,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) crr = cpool_fetch(allctr, blk_sz); if (crr) { STAT_MBC_CPOOL_FETCH(allctr, crr); + INC_CC(allctr->cpool.stat.fetch); link_carrier(&allctr->mbc_list, crr); (*allctr->add_mbc)(allctr, crr); blk = (*allctr->get_free_block)(allctr, blk_sz, NULL, 0); @@ -4128,13 +4181,18 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp) #endif if (busy_pcrr_pp && *busy_pcrr_pp) { + erts_aint_t iallctr = erts_atomic_read_nob(&crr->allctr); ERTS_ALC_CPOOL_ASSERT(*busy_pcrr_pp == crr); - *busy_pcrr_pp = NULL; - ERTS_ALC_CPOOL_ASSERT(erts_atomic_read_nob(&crr->allctr) - == (((erts_aint_t) allctr) - | ERTS_CRR_ALCTR_FLG_IN_POOL - | ERTS_CRR_ALCTR_FLG_BUSY)); - erts_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr)); + ERTS_ALC_CPOOL_ASSERT((iallctr & ~ERTS_CRR_ALCTR_FLG_HOMECOMING) + == (((erts_aint_t) allctr) + | ERTS_CRR_ALCTR_FLG_IN_POOL + | ERTS_CRR_ALCTR_FLG_BUSY)); + ERTS_ALC_CPOOL_ASSERT(allctr == crr->cpool.orig_allctr); + + *busy_pcrr_pp = NULL; + erts_atomic_set_nob(&crr->allctr, + (iallctr & ~(ERTS_CRR_ALCTR_FLG_IN_POOL | + ERTS_CRR_ALCTR_FLG_BUSY))); cpool_delete(allctr, allctr, crr); } else @@ -4184,7 +4242,6 @@ static struct { Eterm e; Eterm t; Eterm ramv; - Eterm sbct; #if HAVE_ERTS_MSEG Eterm asbcst; Eterm rsbcst; @@ -4201,6 +4258,8 @@ static struct { Eterm smbcs; Eterm mbcgs; Eterm acul; + Eterm acnl; + Eterm acfml; #if HAVE_ERTS_MSEG Eterm mmc; @@ -4212,6 +4271,17 @@ static struct { Eterm mbcs; Eterm mbcs_pool; + Eterm fetch; + Eterm fail_pooled; + Eterm fail_shared; + Eterm fail_pend_dealloc; + Eterm fail; + Eterm skip_size; + Eterm skip_busy; + Eterm skip_not_pooled; + Eterm skip_homecoming; + Eterm skip_race; + Eterm entrance_removed; Eterm sbcs; Eterm sys_alloc_carriers_size; @@ -4272,7 +4342,6 @@ init_atoms(Allctr_t *allctr) AM_INIT(e); AM_INIT(t); AM_INIT(ramv); - AM_INIT(sbct); #if HAVE_ERTS_MSEG AM_INIT(asbcst); AM_INIT(rsbcst); @@ -4289,6 +4358,8 @@ init_atoms(Allctr_t *allctr) AM_INIT(smbcs); AM_INIT(mbcgs); AM_INIT(acul); + AM_INIT(acnl); + AM_INIT(acfml); #if HAVE_ERTS_MSEG AM_INIT(mmc); @@ -4300,6 +4371,17 @@ init_atoms(Allctr_t *allctr) AM_INIT(mbcs); AM_INIT(mbcs_pool); + AM_INIT(fetch); + AM_INIT(fail_pooled); + AM_INIT(fail_shared); + AM_INIT(fail_pend_dealloc); + AM_INIT(fail); + AM_INIT(skip_size); + AM_INIT(skip_busy); + AM_INIT(skip_not_pooled); + AM_INIT(skip_homecoming); + AM_INIT(skip_race); + AM_INIT(entrance_removed); AM_INIT(sbcs); AM_INIT(sys_alloc_carriers_size); @@ -4583,9 +4665,56 @@ info_cpool(Allctr_t *allctr, if (hpp || szp) { res = NIL; + + if (!sz_only) { + add_3tup(hpp, szp, &res, am.fail_pooled, + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.fail_pooled)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.fail_pooled))); + + add_3tup(hpp, szp, &res, am.fail_shared, + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.fail_shared)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.fail_shared))); + + add_3tup(hpp, szp, &res, am.fail_pend_dealloc, + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.fail_pend_dealloc)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.fail_pend_dealloc))); + + add_3tup(hpp, szp, &res, am.fail, + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.fail)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.fail))); + + add_3tup(hpp, szp, &res, am.fetch, + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.fetch)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.fetch))); + + add_3tup(hpp, szp, &res, am.skip_size, + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.skip_size)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.skip_size))); + + add_3tup(hpp, szp, &res, am.skip_busy, + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.skip_busy)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.skip_busy))); + + add_3tup(hpp, szp, &res, am.skip_not_pooled, + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.skip_not_pooled)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.skip_not_pooled))); + + add_3tup(hpp, szp, &res, am.skip_homecoming, + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.skip_homecoming)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.skip_homecoming))); + + add_3tup(hpp, szp, &res, am.skip_race, + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.skip_race)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.skip_race))); + + add_3tup(hpp, szp, &res, am.entrance_removed, + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.entrance_removed)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.entrance_removed))); + add_2tup(hpp, szp, &res, am.carriers_size, bld_unstable_uint(hpp, szp, csz)); + } if (!sz_only) add_2tup(hpp, szp, &res, am.carriers, @@ -4844,7 +4973,7 @@ info_options(Allctr_t *allctr, Uint *szp) { Eterm res = THE_NON_VALUE; - int acul; + UWord acul, acnl, acfml; if (!allctr) { if (print_to_p) @@ -4857,6 +4986,8 @@ info_options(Allctr_t *allctr, } acul = allctr->cpool.util_limit; + acnl = allctr->cpool.in_pool_limit; + acfml = allctr->cpool.fblk_min_limit; if (print_to_p) { char topt[21]; /* Enough for any 64-bit integer */ @@ -4884,7 +5015,7 @@ info_options(Allctr_t *allctr, "option lmbcs: %beu\n" "option smbcs: %beu\n" "option mbcgs: %beu\n" - "option acul: %d\n", + "option acul: %bpu\n", topt, allctr->ramv ? "true" : "false", allctr->sbc_threshold, @@ -4909,9 +5040,15 @@ info_options(Allctr_t *allctr, hpp, szp); if (hpp || szp) { + add_2tup(hpp, szp, &res, + am.acfml, + bld_uint(hpp, szp, acfml)); + add_2tup(hpp, szp, &res, + am.acnl, + bld_uint(hpp, szp, acnl)); add_2tup(hpp, szp, &res, am.acul, - bld_uint(hpp, szp, (UWord) acul)); + bld_uint(hpp, szp, acul)); add_2tup(hpp, szp, &res, am.mbcgs, bld_uint(hpp, szp, allctr->mbc_growth_stages)); @@ -4947,7 +5084,7 @@ info_options(Allctr_t *allctr, bld_uint(hpp, szp, allctr->mseg_opt.abs_shrink_th)); #endif add_2tup(hpp, szp, &res, - am.sbct, + am_sbct, bld_uint(hpp, szp, allctr->sbc_threshold)); add_2tup(hpp, szp, &res, am.ramv, allctr->ramv ? am_true : am_false); add_2tup(hpp, szp, &res, am.t, (allctr->t ? am_true : am_false)); @@ -5481,12 +5618,13 @@ erts_alcu_free_thr_pref(ErtsAlcType_t type, void *extra, void *p) pref_allctr = get_pref_allctr(extra); used_allctr = get_used_allctr(pref_allctr, ERTS_ALC_TS_PREF_LOCK_IF_USED, p, NULL, &busy_pcrr_p); - if (pref_allctr != used_allctr) + if (pref_allctr != used_allctr) { enqueue_dealloc_other_instance(type, - used_allctr, - p, - (used_allctr->dd.ix - - pref_allctr->dd.ix)); + used_allctr, + p, + (used_allctr->dd.ix + - pref_allctr->dd.ix)); + } else { ERTS_ALCU_DBG_CHK_THR_ACCESS(used_allctr); do_erts_alcu_free(type, used_allctr, p, &busy_pcrr_p); @@ -5854,6 +5992,37 @@ erts_alcu_realloc_mv_thr_pref(ErtsAlcType_t type, void *extra, +static Uint adjust_sbct(Allctr_t* allctr, Uint sbct) +{ +#ifndef ARCH_64 + if (sbct > 0) { + Uint max_mbc_block_sz = UNIT_CEILING(sbct - 1 + ABLK_HDR_SZ); + if (max_mbc_block_sz + UNIT_FLOOR(allctr->min_block_size - 1) > MBC_ABLK_SZ_MASK + || max_mbc_block_sz < sbct) { /* wrap around */ + /* + * By limiting sbc_threshold to (hard limit - min_block_size) + * we avoid having to split off free "residue blocks" + * smaller than min_block_size. + */ + max_mbc_block_sz = MBC_ABLK_SZ_MASK - UNIT_FLOOR(allctr->min_block_size - 1); + sbct = max_mbc_block_sz - ABLK_HDR_SZ + 1; + } + } +#endif + return sbct; +} + +int erts_alcu_try_set_dyn_param(Allctr_t* allctr, Eterm param, Uint value) +{ + const Uint MIN_DYN_SBCT = 4000; /* a lame catastrophe prevention */ + + if (param == am_sbct && value >= MIN_DYN_SBCT) { + allctr->sbc_threshold = adjust_sbct(allctr, value); + return 1; + } + return 0; +} + /* ------------------------------------------------------------------------- */ int @@ -5941,10 +6110,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->min_block_size = sz; } - allctr->cpool.pooled_list.next = &allctr->cpool.pooled_list; - allctr->cpool.pooled_list.prev = &allctr->cpool.pooled_list; - allctr->cpool.traitor_list.next = &allctr->cpool.traitor_list; - allctr->cpool.traitor_list.prev = &allctr->cpool.traitor_list; + allctr->cpool.pooled_tree = NULL; allctr->cpool.dc_list.first = NULL; allctr->cpool.dc_list.last = NULL; allctr->cpool.abandon_limit = 0; @@ -5954,24 +6120,18 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) erts_atomic_init_nob(&allctr->cpool.stat.carriers_size, 0); erts_atomic_init_nob(&allctr->cpool.stat.no_carriers, 0); allctr->cpool.check_limit_count = ERTS_ALC_CPOOL_CHECK_LIMIT_COUNT; - allctr->cpool.util_limit = init->ts ? 0 : init->acul; - - allctr->sbc_threshold = init->sbct; -#ifndef ARCH_64 - if (allctr->sbc_threshold > 0) { - Uint max_mbc_block_sz = UNIT_CEILING(allctr->sbc_threshold - 1 + ABLK_HDR_SZ); - if (max_mbc_block_sz + UNIT_FLOOR(allctr->min_block_size - 1) > MBC_ABLK_SZ_MASK - || max_mbc_block_sz < allctr->sbc_threshold) { /* wrap around */ - /* - * By limiting sbc_threshold to (hard limit - min_block_size) - * we avoid having to split off free "residue blocks" - * smaller than min_block_size. - */ - max_mbc_block_sz = MBC_ABLK_SZ_MASK - UNIT_FLOOR(allctr->min_block_size - 1); - allctr->sbc_threshold = max_mbc_block_sz - ABLK_HDR_SZ + 1; - } + if (!init->ts && init->acul && init->acnl) { + allctr->cpool.util_limit = init->acul; + allctr->cpool.in_pool_limit = init->acnl; + allctr->cpool.fblk_min_limit = init->acfml; } -#endif + else { + allctr->cpool.util_limit = 0; + allctr->cpool.in_pool_limit = 0; + allctr->cpool.fblk_min_limit = 0; + } + + allctr->sbc_threshold = adjust_sbct(allctr, init->sbct); #if HAVE_ERTS_MSEG if (allctr->mseg_opt.abs_shrink_th > ~((UWord) 0) / 100) @@ -6022,6 +6182,9 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->sys_realloc = &erts_alcu_sys_realloc; allctr->sys_dealloc = &erts_alcu_sys_dealloc; } + + allctr->try_set_dyn_param = &erts_alcu_try_set_dyn_param; + #if HAVE_ERTS_MSEG if (init->mseg_alloc) { ASSERT(init->mseg_realloc && init->mseg_dealloc); @@ -6036,6 +6199,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->mseg_realloc = &erts_alcu_mseg_realloc; allctr->mseg_dealloc = &erts_alcu_mseg_dealloc; } + /* If a custom carrier alloc function is specified, make sure it's used */ if (init->mseg_alloc && !init->sys_alloc) { allctr->crr_set_flgs = CFLG_FORCE_MSEG; diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index faeb5ef368..9a6de2bb75 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -61,7 +61,9 @@ typedef struct { UWord lmbcs; UWord smbcs; UWord mbcgs; - int acul; + UWord acul; + UWord acnl; + UWord acfml; void *fix; size_t *fix_type_size; @@ -116,6 +118,8 @@ typedef struct { 1024*1024, /* (bytes) smbcs: smallest mbc size */\ 10, /* (amount) mbcgs: mbc growth stages */\ 0, /* (%) acul: abandon carrier utilization limit */\ + 1000, /* (amount) acnl: abandoned carriers number limit */\ + 0, /* (bytes) acfml: abandoned carrier fblk min limit */\ /* --- Data not options -------------------------------------------- */\ NULL, /* (ptr) fix */\ NULL /* (ptr) fix_type_size */\ @@ -149,6 +153,8 @@ typedef struct { 128*1024, /* (bytes) smbcs: smallest mbc size */\ 10, /* (amount) mbcgs: mbc growth stages */\ 0, /* (%) acul: abandon carrier utilization limit */\ + 1000, /* (amount) acnl: abandoned carriers number limit */\ + 0, /* (bytes) acfml: abandoned carrier fblk min limit */\ /* --- Data not options -------------------------------------------- */\ NULL, /* (ptr) fix */\ NULL /* (ptr) fix_type_size */\ @@ -216,6 +222,8 @@ void erts_alcu_literal_32_sys_dealloc(Allctr_t*, void *ptr, Uint size, int supe void erts_lcnt_update_allocator_locks(int enable); #endif +int erts_alcu_try_set_dyn_param(Allctr_t*, Eterm param, Uint value); + #endif /* !ERL_ALLOC_UTIL__ */ #if defined(GET_ERL_ALLOC_UTIL_IMPL) && !defined(ERL_ALLOC_UTIL_IMPL__) @@ -296,41 +304,7 @@ void erts_lcnt_update_allocator_locks(int enable); typedef union {char c[ERTS_ALLOC_ALIGN_BYTES]; long l; double d;} Unit_t; - -typedef struct ErtsDoubleLink_t_ { - struct ErtsDoubleLink_t_ *next; - struct ErtsDoubleLink_t_ *prev; -}ErtsDoubleLink_t; - -typedef struct { - erts_atomic_t next; - erts_atomic_t prev; - Allctr_t *orig_allctr; /* read-only while carrier is alive */ - ErtsThrPrgrVal thr_prgr; - erts_atomic_t max_size; - UWord abandon_limit; - UWord blocks; - UWord blocks_size; - ErtsDoubleLink_t abandoned; /* node in pooled_list or traitor_list */ -} ErtsAlcCPoolData_t; - - typedef struct Carrier_t_ Carrier_t; -struct Carrier_t_ { - UWord chdr; - Carrier_t *next; - Carrier_t *prev; - erts_atomic_t allctr; - ErtsAlcCPoolData_t cpool; /* Overwritten by block if sbc */ -}; - -#define ERTS_ALC_CARRIER_TO_ALLCTR(C) \ - ((Allctr_t *) (erts_atomic_read_nob(&(C)->allctr) & ~FLG_MASK)) - -typedef struct { - Carrier_t *first; - Carrier_t *last; -} CarrierList_t; typedef struct { UWord bhdr; @@ -344,6 +318,22 @@ typedef struct { #endif } Block_t; +typedef union ErtsAllctrDDBlock_t_ ErtsAllctrDDBlock_t; + +union ErtsAllctrDDBlock_t_ { + erts_atomic_t atmc_next; + ErtsAllctrDDBlock_t *ptr_next; +}; + +typedef struct { + Block_t blk; +#if !MBC_ABLK_OFFSET_BITS + ErtsAllctrDDBlock_t umem_; +#endif +} ErtsFakeDDBlock_t; + + + #define THIS_FREE_BLK_HDR_FLG (((UWord) 1) << 0) #define PREV_FREE_BLK_HDR_FLG (((UWord) 1) << 1) #define LAST_BLK_HDR_FLG (((UWord) 1) << 2) @@ -352,14 +342,13 @@ typedef struct { (THIS_FREE_BLK_HDR_FLG | PREV_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG) /* - * FREE_LAST_MBC_BLK_HDR_FLGS is a special flag combo used for - * distinguishing empty mbc's from allocated blocks in - * handle_delayed_dealloc(). + * HOMECOMING_MBC_BLK_HDR is a special block header combo used for + * distinguishing MBC's from allocated blocks in handle_delayed_dealloc(). */ -#define FREE_LAST_MBC_BLK_HDR_FLGS (THIS_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG) +#define HOMECOMING_MBC_BLK_HDR (THIS_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG) #define IS_FREE_LAST_MBC_BLK(B) \ - (((B)->bhdr & FLG_MASK) == FREE_LAST_MBC_BLK_HDR_FLGS) + (((B)->bhdr & FLG_MASK) == (THIS_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG)) #define IS_SBC_BLK(B) (((B)->bhdr & FLG_MASK) == SBC_BLK_HDR_FLG) #define IS_MBC_BLK(B) (!IS_SBC_BLK((B))) @@ -383,6 +372,57 @@ typedef struct { typedef UWord FreeBlkFtr_t; /* Footer of a free block */ +/* This AOFF stuff really belong in erl_ao_firstfit_alloc.h */ +typedef struct AOFF_RBTree_t_ AOFF_RBTree_t; +struct AOFF_RBTree_t_ { + Block_t hdr; + AOFF_RBTree_t *parent; + AOFF_RBTree_t *left; + AOFF_RBTree_t *right; + Uint32 flags; + Uint32 max_sz; /* of all blocks in this sub-tree */ +}; + +void aoff_add_pooled_mbc(Allctr_t*, Carrier_t*); +void aoff_remove_pooled_mbc(Allctr_t*, Carrier_t*); +Carrier_t* aoff_lookup_pooled_mbc(Allctr_t*, Uint size); +void erts_aoff_larger_max_size(AOFF_RBTree_t *node); + +typedef struct { + ErtsFakeDDBlock_t homecoming_dd; + erts_atomic_t next; + erts_atomic_t prev; + Allctr_t *orig_allctr; /* read-only while carrier is alive */ + ErtsThrPrgrVal thr_prgr; + erts_atomic_t max_size; + UWord abandon_limit; + UWord blocks; + UWord blocks_size; + enum { + ERTS_MBC_IS_HOME, + ERTS_MBC_WAS_POOLED, + ERTS_MBC_WAS_TRAITOR + } state; + AOFF_RBTree_t pooled; /* node in pooled_tree */ +} ErtsAlcCPoolData_t; + +struct Carrier_t_ { + UWord chdr; + Carrier_t *next; + Carrier_t *prev; + erts_atomic_t allctr; + ErtsAlcCPoolData_t cpool; /* Overwritten by block if sbc */ +}; + +#define ERTS_ALC_CARRIER_TO_ALLCTR(C) \ + ((Allctr_t *) (erts_atomic_read_nob(&(C)->allctr) & ~FLG_MASK)) + +typedef struct { + Carrier_t *first; + Carrier_t *last; +} CarrierList_t; + + typedef Uint64 CallCounter_t; typedef struct { @@ -419,13 +459,6 @@ typedef struct { #endif -typedef union ErtsAllctrDDBlock_t_ ErtsAllctrDDBlock_t; - -union ErtsAllctrDDBlock_t_ { - erts_atomic_t atmc_next; - ErtsAllctrDDBlock_t *ptr_next; -}; - typedef struct { ErtsAllctrDDBlock_t marker; erts_atomic_t last; @@ -537,25 +570,37 @@ struct Allctr_t_ { UWord crr_set_flgs; UWord crr_clr_flgs; - /* Carriers */ + /* Carriers *employed* by this allocator */ CarrierList_t mbc_list; CarrierList_t sbc_list; struct { - /* pooled_list, traitor list and dc_list contain only - carriers _created_ by this allocator */ - ErtsDoubleLink_t pooled_list; - ErtsDoubleLink_t traitor_list; + /* pooled_tree and dc_list contain only + carriers *created* by this allocator */ + AOFF_RBTree_t* pooled_tree; CarrierList_t dc_list; UWord abandon_limit; int disable_abandon; int check_limit_count; - int util_limit; + UWord util_limit; /* acul */ + UWord in_pool_limit; /* acnl */ + UWord fblk_min_limit; /* acmfl */ struct { erts_atomic_t blocks_size; erts_atomic_t no_blocks; erts_atomic_t carriers_size; erts_atomic_t no_carriers; + CallCounter_t fail_pooled; + CallCounter_t fail_shared; + CallCounter_t fail_pend_dealloc; + CallCounter_t fail; + CallCounter_t fetch; + CallCounter_t skip_size; + CallCounter_t skip_busy; + CallCounter_t skip_not_pooled; + CallCounter_t skip_homecoming; + CallCounter_t skip_race; + CallCounter_t entrance_removed; } stat; } cpool; @@ -589,6 +634,8 @@ struct Allctr_t_ { void* (*sys_realloc)(Allctr_t *allctr, void *ptr, Uint *size_p, Uint old_size, int superalign); void (*sys_dealloc)(Allctr_t *allctr, void *ptr, Uint size, int superalign); + int (*try_set_dyn_param)(Allctr_t*, Eterm param, Uint value); + void (*init_atoms) (void); #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c index 05ba1f9891..f8a6101b95 100644 --- a/erts/emulator/beam/erl_ao_firstfit_alloc.c +++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c @@ -20,7 +20,7 @@ /* - * Description: An "address order first fit" allocator + * Description: A family of "first fit" allocator strategies * based on a Red-Black (binary search) Tree. The search, * insert, and delete operations are all O(log n) operations * on a Red-Black Tree. @@ -40,6 +40,10 @@ * sorting order. Blocks within the same carrier are sorted * wrt size instead of address. The 'max_sz' field is maintained * in order to dismiss entire carriers with too small blocks. + * Age Order: + * Carriers are ordered by creation time instead of address. + * Oldest carrier with a large enough free block is chosen. + * No age order supported for blocks. * * Authors: Rickard Green/Sverker Eriksson */ @@ -53,10 +57,12 @@ #include "erl_ao_firstfit_alloc.h" #ifdef DEBUG +# define IS_DEBUG 1 #if 0 #define HARD_DEBUG #endif #else +# define IS_DEBUG 0 #undef HARD_DEBUG #endif @@ -92,18 +98,6 @@ #define RBT_ASSERT(x) #endif - -/* Types... */ -typedef struct AOFF_RBTree_t_ AOFF_RBTree_t; - -struct AOFF_RBTree_t_ { - Block_t hdr; - AOFF_RBTree_t *parent; - AOFF_RBTree_t *left; - AOFF_RBTree_t *right; - Uint32 flags; - Uint32 max_sz; /* of all blocks in this sub-tree */ -}; #define AOFF_BLK_SZ(B) MBC_FBLK_SZ(&(B)->hdr) /* BF block nodes keeps list of all with equal size @@ -121,6 +115,7 @@ typedef struct AOFF_Carrier_t_ AOFF_Carrier_t; struct AOFF_Carrier_t_ { Carrier_t crr; AOFF_RBTree_t rbt_node; /* My node in the carrier tree */ + Sint64 birth_time; AOFF_RBTree_t* root; /* Root of my block tree */ }; #define RBT_NODE_TO_MBC(PTR) ErtsContainerStruct((PTR), AOFF_Carrier_t, rbt_node) @@ -136,12 +131,12 @@ struct AOFF_Carrier_t_ { */ #ifdef HARD_DEBUG -# define HARD_CHECK_IS_MEMBER(ROOT,NODE) rbt_assert_is_member(ROOT,NODE) -# define HARD_CHECK_TREE(CRR,FLV,ROOT,SZ) check_tree(CRR, FLV, ROOT, SZ) -static AOFF_RBTree_t * check_tree(Carrier_t* within_crr, enum AOFF_Flavor flavor, AOFF_RBTree_t* root, Uint); +# define HARD_CHECK_IS_MEMBER(ROOT,NODE) ASSERT(rbt_is_member(ROOT,NODE)) +# define HARD_CHECK_TREE(CRR,ORDER,ROOT,SZ) check_tree(CRR, ORDER, ROOT, SZ) +static AOFF_RBTree_t * check_tree(Carrier_t*, enum AOFFSortOrder, AOFF_RBTree_t*, Uint); #else # define HARD_CHECK_IS_MEMBER(ROOT,NODE) -# define HARD_CHECK_TREE(CRR,FLV,ROOT,SZ) +# define HARD_CHECK_TREE(CRR,ORDER,ROOT,SZ) #endif @@ -179,25 +174,63 @@ static ERTS_INLINE void lower_max_size(AOFF_RBTree_t *node, else ASSERT(new_max == old_max); } -static ERTS_INLINE SWord cmp_blocks(enum AOFF_Flavor flavor, +/* + * Set possibly new larger 'max_sz' of node and propagate change toward root + */ +void erts_aoff_larger_max_size(AOFF_RBTree_t *node) +{ + AOFF_RBTree_t* x = node; + const Uint new_sz = node->hdr.bhdr; + + ASSERT(!x->left || x->left->max_sz <= x->max_sz); + ASSERT(!x->right || x->right->max_sz <= x->max_sz); + + while (new_sz > x->max_sz) { + x->max_sz = new_sz; + x = x->parent; + if (!x) + break; + } +} + +/* Compare nodes for both carrier and block trees */ +static ERTS_INLINE SWord cmp_blocks(enum AOFFSortOrder order, AOFF_RBTree_t* lhs, AOFF_RBTree_t* rhs) { ASSERT(lhs != rhs); - ASSERT(flavor == AOFF_AOFF || FBLK_TO_MBC(&lhs->hdr) == FBLK_TO_MBC(&rhs->hdr)); - if (flavor != AOFF_AOFF) { - SWord diff = (SWord)AOFF_BLK_SZ(lhs) - (SWord)AOFF_BLK_SZ(rhs); - if (diff || flavor == AOFF_BF) return diff; + if (order == FF_AGEFF) { + AOFF_Carrier_t* lc = RBT_NODE_TO_MBC(lhs); + AOFF_Carrier_t* rc = RBT_NODE_TO_MBC(rhs); + Sint64 diff = lc->birth_time - rc->birth_time; + #ifdef ARCH_64 + if (diff) + return diff; + #else + if (diff < 0) + return -1; + else if (diff > 0) + return 1; + #endif + } + else { + ASSERT(order == FF_AOFF || FBLK_TO_MBC(&lhs->hdr) == FBLK_TO_MBC(&rhs->hdr)); + if (order != FF_AOFF) { + SWord diff = (SWord)AOFF_BLK_SZ(lhs) - (SWord)AOFF_BLK_SZ(rhs); + if (diff || order == FF_BF) return diff; + } } return (char*)lhs - (char*)rhs; } -static ERTS_INLINE SWord cmp_cand_blk(enum AOFF_Flavor flavor, +/* Compare candidate block. Only for block tree */ +static ERTS_INLINE SWord cmp_cand_blk(enum AOFFSortOrder order, Block_t* cand_blk, AOFF_RBTree_t* rhs) { - if (flavor != AOFF_AOFF) { + ASSERT(order != FF_AGEFF); + if (order != FF_AOFF) { if (BLK_TO_MBC(cand_blk) == FBLK_TO_MBC(&rhs->hdr)) { SWord diff = (SWord)MBC_BLK_SZ(cand_blk) - (SWord)MBC_FBLK_SZ(&rhs->hdr); - if (diff || flavor == AOFF_BF) return diff; + if (diff || order == FF_BF) return diff; } } return (char*)cand_blk - (char*)rhs; @@ -218,11 +251,8 @@ static UWord aoff_largest_fblk_in_mbc(Allctr_t*, Carrier_t*); /* Generic tree functions used by both carrier and block trees. */ static void rbt_delete(AOFF_RBTree_t** root, AOFF_RBTree_t* del); -static void rbt_insert(enum AOFF_Flavor flavor, AOFF_RBTree_t** root, AOFF_RBTree_t* blk); +static void rbt_insert(enum AOFFSortOrder, AOFF_RBTree_t** root, AOFF_RBTree_t* blk); static AOFF_RBTree_t* rbt_search(AOFF_RBTree_t* root, Uint size); -#ifdef HARD_DEBUG -static int rbt_assert_is_member(AOFF_RBTree_t* root, AOFF_RBTree_t* node); -#endif static Eterm info_options(Allctr_t *, char *, fmtfn_t *, void *, Uint **, Uint *); static void init_atoms(void); @@ -230,10 +260,17 @@ static void init_atoms(void); static int atoms_initialized = 0; +#ifndef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT +static erts_atomic64_t birth_time_counter; +#endif + void erts_aoffalc_init(void) { atoms_initialized = 0; +#ifndef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + erts_atomic64_init_nob(&birth_time_counter, 0); +#endif } Allctr_t * @@ -254,11 +291,12 @@ erts_aoffalc_start(AOFFAllctr_t *alc, sys_memcpy((void *) alc, (void *) &zero.allctr, sizeof(AOFFAllctr_t)); - alc->flavor = aoffinit->flavor; + alc->blk_order = aoffinit->blk_order; + alc->crr_order = aoffinit->crr_order; allctr->mbc_header_size = sizeof(AOFF_Carrier_t); allctr->min_mbc_size = MIN_MBC_SZ; allctr->min_mbc_first_free_size = MIN_MBC_FIRST_FREE_SZ; - allctr->min_block_size = (aoffinit->flavor == AOFF_BF ? + allctr->min_block_size = (aoffinit->blk_order == FF_BF ? sizeof(AOFF_RBTreeList_t):sizeof(AOFF_RBTree_t)); allctr->vsn_str = ERTS_ALC_AOFF_ALLOC_VSN_STR; @@ -487,9 +525,9 @@ aoff_unlink_free_block(Allctr_t *allctr, Block_t *blk) AOFF_Carrier_t *crr = (AOFF_Carrier_t*) FBLK_TO_MBC(&del->hdr); ASSERT(crr->rbt_node.hdr.bhdr == crr->root->max_sz); - HARD_CHECK_TREE(&crr->crr, alc->flavor, crr->root, 0); + HARD_CHECK_TREE(&crr->crr, alc->blk_order, crr->root, 0); - if (alc->flavor == AOFF_BF) { + if (alc->blk_order == FF_BF) { ASSERT(del->flags & IS_BF_FLG); if (IS_LIST_ELEM(del)) { /* Remove from list */ @@ -510,14 +548,14 @@ aoff_unlink_free_block(Allctr_t *allctr, Block_t *blk) replace(&crr->root, (AOFF_RBTree_t*)del, LIST_NEXT(del)); - HARD_CHECK_TREE(&crr->crr, alc->flavor, crr->root, 0); + HARD_CHECK_TREE(&crr->crr, alc->blk_order, crr->root, 0); return; } } rbt_delete(&crr->root, (AOFF_RBTree_t*)del); - HARD_CHECK_TREE(&crr->crr, alc->flavor, crr->root, 0); + HARD_CHECK_TREE(&crr->crr, alc->blk_order, crr->root, 0); /* Update the carrier tree with a potentially new (lower) max_sz */ @@ -715,32 +753,33 @@ aoff_link_free_block(Allctr_t *allctr, Block_t *block) ASSERT(allctr == ERTS_ALC_CARRIER_TO_ALLCTR(&blk_crr->crr)); ASSERT(blk_crr->rbt_node.hdr.bhdr == (blk_crr->root ? blk_crr->root->max_sz : 0)); - HARD_CHECK_TREE(&blk_crr->crr, alc->flavor, blk_crr->root, 0); + HARD_CHECK_TREE(&blk_crr->crr, alc->blk_order, blk_crr->root, 0); - rbt_insert(alc->flavor, &blk_crr->root, blk); + rbt_insert(alc->blk_order, &blk_crr->root, blk); - /* Update the carrier tree with a potentially new (larger) max_sz - */ + /* + * Update carrier tree with a potentially new (larger) max_sz + */ crr_node = &blk_crr->rbt_node; if (blk_sz > crr_node->hdr.bhdr) { - ASSERT(blk_sz == blk_crr->root->max_sz); - crr_node->hdr.bhdr = blk_sz; - while (blk_sz > crr_node->max_sz) { - crr_node->max_sz = blk_sz; - crr_node = crr_node->parent; - if (!crr_node) break; - } + ASSERT(blk_sz == blk_crr->root->max_sz); + crr_node->hdr.bhdr = blk_sz; + while (blk_sz > crr_node->max_sz) { + crr_node->max_sz = blk_sz; + crr_node = crr_node->parent; + if (!crr_node) break; + } } - HARD_CHECK_TREE(&blk_crr->crr, alc->flavor, blk_crr->root, 0); + HARD_CHECK_TREE(NULL, alc->crr_order, alc->mbc_root, 0); } static void -rbt_insert(enum AOFF_Flavor flavor, AOFF_RBTree_t** root, AOFF_RBTree_t* blk) +rbt_insert(enum AOFFSortOrder order, AOFF_RBTree_t** root, AOFF_RBTree_t* blk) { Uint blk_sz = AOFF_BLK_SZ(blk); #ifdef DEBUG - blk->flags = (flavor == AOFF_BF) ? IS_BF_FLG : 0; + blk->flags = (order == FF_BF) ? IS_BF_FLG : 0; #else blk->flags = 0; #endif @@ -760,7 +799,7 @@ rbt_insert(enum AOFF_Flavor flavor, AOFF_RBTree_t** root, AOFF_RBTree_t* blk) if (x->max_sz < blk_sz) { x->max_sz = blk_sz; } - diff = cmp_blocks(flavor, blk, x); + diff = cmp_blocks(order, blk, x); if (diff < 0) { if (!x->left) { blk->parent = x; @@ -778,7 +817,7 @@ rbt_insert(enum AOFF_Flavor flavor, AOFF_RBTree_t** root, AOFF_RBTree_t* blk) x = x->right; } else { - ASSERT(flavor == AOFF_BF); + ASSERT(order == FF_BF); ASSERT(blk->flags & IS_BF_FLG); ASSERT(x->flags & IS_BF_FLG); SET_LIST_ELEM(blk); @@ -798,7 +837,7 @@ rbt_insert(enum AOFF_Flavor flavor, AOFF_RBTree_t** root, AOFF_RBTree_t* blk) if (IS_RED(blk->parent)) tree_insert_fixup(root, blk); } - if (flavor == AOFF_BF) { + if (order == FF_BF) { SET_TREE_NODE(blk); LIST_NEXT(blk) = NULL; } @@ -826,6 +865,16 @@ rbt_search(AOFF_RBTree_t* root, Uint size) } } +Carrier_t* aoff_lookup_pooled_mbc(Allctr_t* allctr, Uint size) +{ + AOFF_RBTree_t* node; + + if (!allctr->cpool.pooled_tree) + return NULL; + node = rbt_search(allctr->cpool.pooled_tree, size); + return node ? ErtsContainerStruct(node, Carrier_t, cpool.pooled) : NULL; +} + static Block_t * aoff_get_free_block(Allctr_t *allctr, Uint size, Block_t *cand_blk, Uint cand_size) @@ -850,7 +899,7 @@ aoff_get_free_block(Allctr_t *allctr, Uint size, /* Get block within carrier tree */ #ifdef HARD_DEBUG - dbg_blk = HARD_CHECK_TREE(&crr->crr, alc->flavor, crr->root, size); + dbg_blk = HARD_CHECK_TREE(&crr->crr, alc->blk_order, crr->root, size); #endif blk = rbt_search(crr->root, size); @@ -863,7 +912,7 @@ aoff_get_free_block(Allctr_t *allctr, Uint size, if (!blk) return NULL; - if (cand_blk && cmp_cand_blk(alc->flavor, cand_blk, blk) < 0) { + if (cand_blk && cmp_cand_blk(alc->blk_order, cand_blk, blk) < 0) { return NULL; /* cand_blk was better */ } @@ -872,23 +921,32 @@ aoff_get_free_block(Allctr_t *allctr, Uint size, return (Block_t *) blk; } +static ERTS_INLINE Sint64 get_birth_time(void) +{ +#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + return (Sint64) erts_os_monotonic_time(); +#else + return (Sint64) erts_atomic64_inc_read_nob(&birth_time_counter); +#endif +} + static void aoff_creating_mbc(Allctr_t *allctr, Carrier_t *carrier) { AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr; AOFF_Carrier_t *crr = (AOFF_Carrier_t*) carrier; AOFF_RBTree_t **root = &alc->mbc_root; - HARD_CHECK_TREE(NULL, 0, *root, 0); + HARD_CHECK_TREE(NULL, alc->crr_order, *root, 0); - /* Link carrier in address order tree - */ crr->rbt_node.hdr.bhdr = 0; - rbt_insert(AOFF_AOFF, root, &crr->rbt_node); + if (alc->crr_order == FF_AGEFF || IS_DEBUG) + crr->birth_time = get_birth_time(); + rbt_insert(alc->crr_order, root, &crr->rbt_node); /* aoff_link_free_block will add free block later */ crr->root = NULL; - HARD_CHECK_TREE(NULL, 0, *root, 0); + HARD_CHECK_TREE(NULL, alc->crr_order, *root, 0); } #define IS_CRR_IN_TREE(CRR,ROOT) \ @@ -911,27 +969,38 @@ static void aoff_add_mbc(Allctr_t *allctr, Carrier_t *carrier) AOFF_RBTree_t **root = &alc->mbc_root; ASSERT(!IS_CRR_IN_TREE(crr, *root)); - HARD_CHECK_TREE(NULL, 0, *root, 0); + HARD_CHECK_TREE(NULL, alc->crr_order, *root, 0); + + rbt_insert(alc->crr_order, root, &crr->rbt_node); + + HARD_CHECK_TREE(NULL, alc->crr_order, *root, 0); +} + +void aoff_add_pooled_mbc(Allctr_t *allctr, Carrier_t *crr) +{ + AOFF_RBTree_t **root = &allctr->cpool.pooled_tree; + + ASSERT(allctr == crr->cpool.orig_allctr); + HARD_CHECK_TREE(NULL, 0, *root, 0); /* Link carrier in address order tree */ - rbt_insert(AOFF_AOFF, root, &crr->rbt_node); + rbt_insert(FF_AOFF, root, &crr->cpool.pooled); HARD_CHECK_TREE(NULL, 0, *root, 0); } static void aoff_remove_mbc(Allctr_t *allctr, Carrier_t *carrier) { - AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr; - AOFF_Carrier_t *crr = (AOFF_Carrier_t*) carrier; - AOFF_RBTree_t **root = &alc->mbc_root; + AOFF_RBTree_t **root = &((AOFFAllctr_t*)allctr)->mbc_root; + AOFF_Carrier_t *crr = (AOFF_Carrier_t*)carrier; ASSERT(allctr == ERTS_ALC_CARRIER_TO_ALLCTR(carrier)); if (!IS_CRR_IN_TREE(crr,*root)) - return; + return; - HARD_CHECK_TREE(NULL, 0, *root, 0); + HARD_CHECK_TREE(NULL, alc->crr_order, *root, 0); rbt_delete(root, &crr->rbt_node); crr->rbt_node.parent = NULL; @@ -939,9 +1008,27 @@ static void aoff_remove_mbc(Allctr_t *allctr, Carrier_t *carrier) crr->rbt_node.right = NULL; crr->rbt_node.max_sz = crr->rbt_node.hdr.bhdr; - HARD_CHECK_TREE(NULL, 0, *root, 0); + HARD_CHECK_TREE(NULL, alc->crr_order, *root, 0); +} + +void aoff_remove_pooled_mbc(Allctr_t *allctr, Carrier_t *crr) +{ + ASSERT(allctr == crr->cpool.orig_allctr); + + HARD_CHECK_TREE(NULL, 0, allctr->cpool.pooled_tree, 0); + + rbt_delete(&allctr->cpool.pooled_tree, &crr->cpool.pooled); +#ifdef DEBUG + crr->cpool.pooled.parent = NULL; + crr->cpool.pooled.left = NULL; + crr->cpool.pooled.right = NULL; + crr->cpool.pooled.max_sz = 0; +#endif + HARD_CHECK_TREE(NULL, 0, allctr->cpool.pooled_tree, 0); + } + static UWord aoff_largest_fblk_in_mbc(Allctr_t* allctr, Carrier_t* carrier) { AOFF_Carrier_t *crr = (AOFF_Carrier_t*) carrier; @@ -955,17 +1042,17 @@ static UWord aoff_largest_fblk_in_mbc(Allctr_t* allctr, Carrier_t* carrier) * info_options() */ +static const char* flavor_str[2][3] = { + {"ageffcaoff", "ageffcaobf", "ageffcbf"}, + { "aoff", "aoffcaobf", "aoffcbf"} +}; +static Eterm flavor_atoms[2][3]; + static struct { Eterm as; - Eterm aoff; - Eterm aoffcaobf; - Eterm aoffcbf; -#ifdef DEBUG - Eterm end_of_atoms; -#endif } am; -static void ERTS_INLINE atom_init(Eterm *atom, char *name) +static void ERTS_INLINE atom_init(Eterm *atom, const char *name) { *atom = am_atom_put(name, strlen(name)); } @@ -974,28 +1061,16 @@ static void ERTS_INLINE atom_init(Eterm *atom, char *name) static void init_atoms(void) { -#ifdef DEBUG - Eterm *atom; -#endif + int i, j; if (atoms_initialized) return; -#ifdef DEBUG - for (atom = (Eterm *) &am; atom <= &am.end_of_atoms; atom++) { - *atom = THE_NON_VALUE; - } -#endif AM_INIT(as); - AM_INIT(aoff); - AM_INIT(aoffcaobf); - AM_INIT(aoffcbf); -#ifdef DEBUG - for (atom = (Eterm *) &am; atom < &am.end_of_atoms; atom++) { - ASSERT(*atom != THE_NON_VALUE); - } -#endif + for (i = 0; i < 2; i++) + for (j = 0; j < 3; j++) + atom_init(&flavor_atoms[i][j], flavor_str[i][j]); atoms_initialized = 1; } @@ -1021,15 +1096,16 @@ info_options(Allctr_t *allctr, { AOFFAllctr_t* alc = (AOFFAllctr_t*) allctr; Eterm res = THE_NON_VALUE; - const char* flavor_str[3] = {"aoff", "aoffcaobf", "aoffcbf"}; - Eterm flavor_atom[3] = {am.aoff, am.aoffcaobf, am.aoffcbf}; + + ASSERT(alc->crr_order >= 0 && alc->crr_order <= 1); + ASSERT(alc->blk_order >= 1 && alc->blk_order <= 3); if (print_to_p) { erts_print(*print_to_p, print_to_arg, "%sas: %s\n", prefix, - flavor_str[alc->flavor]); + flavor_str[alc->crr_order][alc->blk_order-1]); } if (hpp || szp) { @@ -1039,7 +1115,8 @@ info_options(Allctr_t *allctr, __FILE__, __LINE__);; res = NIL; - add_2tup(hpp, szp, &res, am.as, flavor_atom[alc->flavor]); + add_2tup(hpp, szp, &res, am.as, + flavor_atoms[alc->crr_order][alc->blk_order-1]); } return res; @@ -1057,7 +1134,7 @@ UWord erts_aoffalc_test(UWord op, UWord a1, UWord a2) { switch (op) { - case 0x500: return (UWord) ((AOFFAllctr_t *) a1)->flavor == AOFF_AOBF; + case 0x500: return (UWord) ((AOFFAllctr_t *) a1)->blk_order == FF_AOBF; case 0x501: { AOFF_RBTree_t *node = ((AOFFAllctr_t *) a1)->mbc_root; Uint size = (Uint) a2; @@ -1072,7 +1149,7 @@ erts_aoffalc_test(UWord op, UWord a1, UWord a2) case 0x507: return (UWord) IS_TREE_NODE((AOFF_RBTree_t *) a1); case 0x508: return (UWord) 0; /* IS_BF_ALGO */ case 0x509: return (UWord) ((AOFF_RBTree_t *) a1)->max_sz; - case 0x50a: return (UWord) ((AOFFAllctr_t *) a1)->flavor == AOFF_BF; + case 0x50a: return (UWord) ((AOFFAllctr_t *) a1)->blk_order == FF_BF; case 0x50b: return (UWord) LIST_PREV(a1); default: ASSERT(0); return ~((UWord) 0); } @@ -1085,12 +1162,13 @@ erts_aoffalc_test(UWord op, UWord a1, UWord a2) #ifdef HARD_DEBUG - -static int rbt_assert_is_member(AOFF_RBTree_t* root, AOFF_RBTree_t* node) +static int rbt_is_member(AOFF_RBTree_t* root, AOFF_RBTree_t* node) { while (node != root) { - ASSERT(node->parent); - ASSERT(node->parent->left == node || node->parent->right == node); + if (!node->parent || (node->parent->left != node && + node->parent->right != node)) { + return 0; + } node = node->parent; } return 1; @@ -1132,7 +1210,7 @@ static void print_tree(AOFF_RBTree_t*); */ static AOFF_RBTree_t * -check_tree(Carrier_t* within_crr, enum AOFF_Flavor flavor, AOFF_RBTree_t* root, Uint size) +check_tree(Carrier_t* within_crr, enum AOFFSortOrder order, AOFF_RBTree_t* root, Uint size) { AOFF_RBTree_t *res = NULL; Sint blacks; @@ -1144,7 +1222,8 @@ check_tree(Carrier_t* within_crr, enum AOFF_Flavor flavor, AOFF_RBTree_t* root, #ifdef PRINT_TREE print_tree(root); #endif - ASSERT(within_crr || flavor == AOFF_AOFF); + ASSERT((within_crr && order >= FF_AOFF) || + (!within_crr && order <= FF_AOFF)); if (!root) return res; @@ -1202,7 +1281,7 @@ check_tree(Carrier_t* within_crr, enum AOFF_Flavor flavor, AOFF_RBTree_t* root, ASSERT(((char*)x + AOFF_BLK_SZ(x)) <= ((char*)crr + CARRIER_SZ(crr))); } - if (flavor == AOFF_BF) { + if (order == FF_BF) { AOFF_RBTree_t* y = x; AOFF_RBTree_t* nxt = LIST_NEXT(y); ASSERT(IS_TREE_NODE(x)); @@ -1225,13 +1304,13 @@ check_tree(Carrier_t* within_crr, enum AOFF_Flavor flavor, AOFF_RBTree_t* root, if (x->left) { ASSERT(x->left->parent == x); - ASSERT(cmp_blocks(flavor, x->left, x) < 0); + ASSERT(cmp_blocks(order, x->left, x) < 0); ASSERT(x->left->max_sz <= x->max_sz); } if (x->right) { ASSERT(x->right->parent == x); - ASSERT(cmp_blocks(flavor, x->right, x) > 0); + ASSERT(cmp_blocks(order, x->right, x) > 0); ASSERT(x->right->max_sz <= x->max_sz); } ASSERT(x->max_sz >= AOFF_BLK_SZ(x)); @@ -1240,7 +1319,7 @@ check_tree(Carrier_t* within_crr, enum AOFF_Flavor flavor, AOFF_RBTree_t* root, || x->max_sz == (x->right ? x->right->max_sz : 0)); if (size && AOFF_BLK_SZ(x) >= size) { - if (!res || cmp_blocks(flavor, x, res) < 0) { + if (!res || cmp_blocks(order, x, res) < 0) { res = x; } } diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.h b/erts/emulator/beam/erl_ao_firstfit_alloc.h index 7349c6ab19..9cf4fc81a8 100644 --- a/erts/emulator/beam/erl_ao_firstfit_alloc.h +++ b/erts/emulator/beam/erl_ao_firstfit_alloc.h @@ -28,14 +28,16 @@ typedef struct AOFFAllctr_t_ AOFFAllctr_t; -enum AOFF_Flavor { - AOFF_AOFF = 0, - AOFF_AOBF = 1, - AOFF_BF = 2 +enum AOFFSortOrder { + FF_AGEFF = 0, + FF_AOFF = 1, + FF_AOBF = 2, + FF_BF = 3 }; typedef struct { - enum AOFF_Flavor flavor; + enum AOFFSortOrder blk_order; + enum AOFFSortOrder crr_order; } AOFFAllctrInit_t; #define ERTS_DEFAULT_AOFF_ALLCTR_INIT {0/*dummy*/} @@ -53,12 +55,12 @@ Allctr_t *erts_aoffalc_start(AOFFAllctr_t *, AOFFAllctrInit_t*, AllctrInit_t *); #define GET_ERL_ALLOC_UTIL_IMPL #include "erl_alloc_util.h" - struct AOFFAllctr_t_ { Allctr_t allctr; /* Has to be first! */ struct AOFF_RBTree_t_* mbc_root; - enum AOFF_Flavor flavor; + enum AOFFSortOrder blk_order; + enum AOFFSortOrder crr_order; }; UWord erts_aoffalc_test(UWord, UWord, UWord); diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c index 41c2ae08d3..33bc189182 100644 --- a/erts/emulator/beam/erl_bif_binary.c +++ b/erts/emulator/beam/erl_bif_binary.c @@ -61,8 +61,6 @@ static Export binary_longest_prefix_trap_export; static BIF_RETTYPE binary_longest_prefix_trap(BIF_ALIST_3); static Export binary_longest_suffix_trap_export; static BIF_RETTYPE binary_longest_suffix_trap(BIF_ALIST_3); -static Export binary_bin_to_list_trap_export; -static BIF_RETTYPE binary_bin_to_list_trap(BIF_ALIST_3); static Export binary_copy_trap_export; static BIF_RETTYPE binary_copy_trap(BIF_ALIST_2); static Uint max_loop_limit; @@ -86,10 +84,6 @@ void erts_init_bif_binary(void) am_erlang, am_binary_longest_suffix_trap, 3, &binary_longest_suffix_trap); - erts_init_trap_export(&binary_bin_to_list_trap_export, - am_erlang, am_binary_bin_to_list_trap, 3, - &binary_bin_to_list_trap); - erts_init_trap_export(&binary_copy_trap_export, am_erlang, am_binary_copy_trap, 2, &binary_copy_trap); @@ -2440,191 +2434,6 @@ BIF_RETTYPE binary_at_2(BIF_ALIST_2) BIF_ERROR(BIF_P,BADARG); } -#define BIN_TO_LIST_OK 0 -#define BIN_TO_LIST_TRAP 1 -/* No badarg, checked before call */ - -#define BIN_TO_LIST_LOOP_FACTOR 10 - -static int do_bin_to_list(Process *p, byte *bytes, Uint bit_offs, - Uint start, Sint *lenp, Eterm *termp) -{ - Uint reds = get_reds(p, BIN_TO_LIST_LOOP_FACTOR); /* reds can never be 0 */ - Uint len = *lenp; - Uint loops; - Eterm *hp; - Eterm term = *termp; - Uint n; - - ASSERT(reds > 0); - - loops = MIN(reds,len); - - BUMP_REDS(p, loops / BIN_TO_LIST_LOOP_FACTOR); - - hp = HAlloc(p,2*loops); - while (loops--) { - --len; - if (bit_offs) { - n = ((((Uint) bytes[start+len]) << bit_offs) | - (((Uint) bytes[start+len+1]) >> (8-bit_offs))) & 0xFF; - } else { - n = bytes[start+len]; - } - - term = CONS(hp,make_small(n),term); - hp +=2; - } - *termp = term; - *lenp = len; - if (len) { - BUMP_ALL_REDS(p); - return BIN_TO_LIST_TRAP; - } - return BIN_TO_LIST_OK; -} - - -static BIF_RETTYPE do_trap_bin_to_list(Process *p, Eterm binary, - Uint start, Sint len, Eterm sofar) -{ - Eterm *hp; - Eterm blob; - - hp = HAlloc(p,3); - hp[0] = make_pos_bignum_header(2); - hp[1] = start; - hp[2] = (Uint) len; - blob = make_big(hp); - BIF_TRAP3(&binary_bin_to_list_trap_export, p, binary, blob, sofar); -} - -static BIF_RETTYPE binary_bin_to_list_trap(BIF_ALIST_3) -{ - Eterm *ptr; - Uint start; - Sint len; - byte *bytes; - Uint bit_offs; - Uint bit_size; - Eterm res = BIF_ARG_3; - - ptr = big_val(BIF_ARG_2); - start = ptr[1]; - len = (Sint) ptr[2]; - - ERTS_GET_BINARY_BYTES(BIF_ARG_1,bytes,bit_offs,bit_size); - if (do_bin_to_list(BIF_P, bytes, bit_offs, start, &len, &res) == - BIN_TO_LIST_OK) { - BIF_RET(res); - } - return do_trap_bin_to_list(BIF_P,BIF_ARG_1,start,len,res); -} - -static BIF_RETTYPE binary_bin_to_list_common(Process *p, - Eterm bin, - Eterm epos, - Eterm elen) -{ - Uint pos; - Sint len; - size_t sz; - byte *bytes; - Uint bit_offs; - Uint bit_size; - Eterm res = NIL; - - if (is_not_binary(bin)) { - goto badarg; - } - if (!term_to_Uint(epos, &pos)) { - goto badarg; - } - if (!term_to_Sint(elen, &len)) { - goto badarg; - } - if (len < 0) { - Uint lentmp = -(Uint)len; - /* overflow */ - if ((Sint)lentmp < 0) { - goto badarg; - } - len = lentmp; - if (len > pos) { - goto badarg; - } - pos -= len; - } - /* overflow */ - if ((pos + len) < pos || (len > 0 && (pos + len) == pos)) { - goto badarg; - } - sz = binary_size(bin); - - if (pos+len > sz) { - goto badarg; - } - ERTS_GET_BINARY_BYTES(bin,bytes,bit_offs,bit_size); - if (bit_size != 0) { - goto badarg; - } - if(do_bin_to_list(p, bytes, bit_offs, pos, &len, &res) == - BIN_TO_LIST_OK) { - BIF_RET(res); - } - return do_trap_bin_to_list(p,bin,pos,len,res); - - badarg: - BIF_ERROR(p,BADARG); -} - -BIF_RETTYPE binary_bin_to_list_3(BIF_ALIST_3) -{ - return binary_bin_to_list_common(BIF_P,BIF_ARG_1,BIF_ARG_2,BIF_ARG_3); -} - -BIF_RETTYPE binary_bin_to_list_2(BIF_ALIST_2) -{ - Eterm *tp; - - if (is_not_tuple(BIF_ARG_2)) { - goto badarg; - } - tp = tuple_val(BIF_ARG_2); - if (arityval(*tp) != 2) { - goto badarg; - } - return binary_bin_to_list_common(BIF_P,BIF_ARG_1,tp[1],tp[2]); - badarg: - BIF_ERROR(BIF_P,BADARG); -} - -BIF_RETTYPE binary_bin_to_list_1(BIF_ALIST_1) -{ - Uint pos = 0; - Sint len; - byte *bytes; - Uint bit_offs; - Uint bit_size; - Eterm res = NIL; - - if (is_not_binary(BIF_ARG_1)) { - goto badarg; - } - len = binary_size(BIF_ARG_1); - ERTS_GET_BINARY_BYTES(BIF_ARG_1,bytes,bit_offs,bit_size); - if (bit_size != 0) { - goto badarg; - } - if(do_bin_to_list(BIF_P, bytes, bit_offs, pos, &len, &res) == - BIN_TO_LIST_OK) { - BIF_RET(res); - } - return do_trap_bin_to_list(BIF_P,BIF_ARG_1,pos,len,res); - badarg: - BIF_ERROR(BIF_P,BADARG); -} - HIPE_WRAPPER_BIF_DISABLE_GC(binary_list_to_bin, 1) BIF_RETTYPE binary_list_to_bin_1(BIF_ALIST_1) diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c index 9f0c90ff7b..b184adedee 100644 --- a/erts/emulator/beam/erl_bif_port.c +++ b/erts/emulator/beam/erl_bif_port.c @@ -639,6 +639,27 @@ BIF_RETTYPE port_get_data_1(BIF_ALIST_1) BIF_RET(res); } +Eterm erts_port_data_read(Port* prt) +{ + Eterm res; + erts_aint_t data; + + data = erts_atomic_read_ddrb(&prt->data); + if (data == (erts_aint_t)NULL) + return am_undefined; /* Port terminated by racing thread */ + + if ((data & 0x3) != 0) { + res = (Eterm) (UWord) data; + ASSERT(is_immed(res)); + } + else { + ErtsPortDataHeap *pdhp = (ErtsPortDataHeap *) data; + res = pdhp->data; + } + return res; +} + + /* * Open a port. Most of the work is not done here but rather in * the file io.c. diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 3ba0886464..1ab1c4a363 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -1753,6 +1753,28 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) BIF_RET(ret); } +/* +** Retrieves the tid() of a named ets table. +*/ +BIF_RETTYPE ets_whereis_1(BIF_ALIST_1) +{ + DbTable* tb; + Eterm res; + + if (is_not_atom(BIF_ARG_1)) { + BIF_ERROR(BIF_P, BADARG); + } + + if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_INFO, LCK_READ)) == NULL) { + BIF_RET(am_undefined); + } + + res = make_tid(BIF_P, tb); + db_unlock(tb, LCK_READ); + + BIF_RET(res); +} + /* ** The lookup BIF */ @@ -3126,7 +3148,8 @@ BIF_RETTYPE ets_info_1(BIF_ALIST_1) static Eterm fields[] = {am_protection, am_keypos, am_type, am_named_table, am_node, am_size, am_name, am_heir, am_owner, am_memory, am_compressed, am_write_concurrency, - am_read_concurrency}; + am_read_concurrency, + am_id}; Eterm results[sizeof(fields)/sizeof(Eterm)]; DbTable* tb; Eterm res; @@ -4016,7 +4039,10 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What) ret = is_table_named(tb) ? am_true : am_false; } else if (What == am_compressed) { ret = tb->common.compress ? am_true : am_false; + } else if (What == am_id) { + ret = make_tid(p, tb); } + /* * For debugging purposes */ diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 0da4468f9c..1c64644efc 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -116,6 +116,7 @@ typedef struct { static Uint setup_rootset(Process*, Eterm*, int, Rootset*); static void cleanup_rootset(Rootset *rootset); static Eterm *full_sweep_heaps(Process *p, + ErlHeapFragment *live_hf_end, int hibernate, Eterm *n_heap, Eterm* n_htop, char *oh, Uint oh_size, @@ -142,7 +143,7 @@ static Eterm* sweep_literal_area(Eterm* n_hp, Eterm* n_htop, static Eterm* sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint src_size); static Eterm* collect_live_heap_frags(Process* p, ErlHeapFragment *live_hf_end, - Eterm* heap, Eterm* htop, Eterm* objv, int nobj); + Eterm* htop); static int adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj); static void shrink_new_heap(Process *p, Uint new_sz, Eterm *objv, int nobj); static void grow_new_heap(Process *p, Uint new_sz, Eterm* objv, int nobj); @@ -917,6 +918,7 @@ garbage_collect_hibernate(Process* p, int check_long_gc) htop = heap; htop = full_sweep_heaps(p, + ERTS_INVALID_HFRAG_PTR, 1, heap, htop, @@ -1161,7 +1163,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, roots++; - while (g_sz--) { + for ( ; g_sz--; g_ptr++) { Eterm gval = *g_ptr; switch (primary_tag(gval)) { @@ -1170,26 +1172,21 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, val = *ptr; if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); - *g_ptr++ = val; + *g_ptr = val; } else if (ErtsInArea(ptr, area, area_size)) { - move_boxed(&ptr,val,&old_htop,g_ptr++); - } else { - g_ptr++; + move_boxed(ptr,val,&old_htop,g_ptr); } break; case TAG_PRIMARY_LIST: ptr = list_val(gval); val = *ptr; if (IS_MOVED_CONS(val)) { /* Moved */ - *g_ptr++ = ptr[1]; + *g_ptr = ptr[1]; } else if (ErtsInArea(ptr, area, area_size)) { - move_cons(&ptr,val,&old_htop,g_ptr++); - } else { - g_ptr++; - } + move_cons(ptr,val,&old_htop,g_ptr); + } break; default: - g_ptr++; break; } } @@ -1479,25 +1476,29 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end, n_htop = n_heap = (Eterm*) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP, sizeof(Eterm)*new_sz); + n = setup_rootset(p, objv, nobj, &rootset); + roots = rootset.roots; + + /* + * All allocations done. Start defile heap with move markers. + * A crash dump due to allocation failure above will see a healthy heap. + */ + if (live_hf_end != ERTS_INVALID_HFRAG_PTR) { /* * Move heap frags that we know are completely live * directly into the new young heap generation. */ - n_htop = collect_live_heap_frags(p, live_hf_end, n_heap, n_htop, - objv, nobj); + n_htop = collect_live_heap_frags(p, live_hf_end, n_htop); } - n = setup_rootset(p, objv, nobj, &rootset); - roots = rootset.roots; - GENSWEEP_NSTACK(p, old_htop, n_htop); while (n--) { Eterm* g_ptr = roots->v; Uint g_sz = roots->sz; roots++; - while (g_sz--) { + for ( ; g_sz--; g_ptr++) { gval = *g_ptr; switch (primary_tag(gval)) { @@ -1507,14 +1508,12 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end, val = *ptr; if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); - *g_ptr++ = val; + *g_ptr = val; } else if (ErtsInArea(ptr, mature, mature_size)) { - move_boxed(&ptr,val,&old_htop,g_ptr++); + move_boxed(ptr,val,&old_htop,g_ptr); } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) { - move_boxed(&ptr,val,&n_htop,g_ptr++); - } else { - g_ptr++; - } + move_boxed(ptr,val,&n_htop,g_ptr); + } break; } @@ -1522,19 +1521,15 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end, ptr = list_val(gval); val = *ptr; if (IS_MOVED_CONS(val)) { /* Moved */ - *g_ptr++ = ptr[1]; + *g_ptr = ptr[1]; } else if (ErtsInArea(ptr, mature, mature_size)) { - move_cons(&ptr,val,&old_htop,g_ptr++); + move_cons(ptr,val,&old_htop,g_ptr); } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) { - move_cons(&ptr,val,&n_htop,g_ptr++); - } else { - g_ptr++; - } + move_cons(ptr,val,&n_htop,g_ptr); + } break; } - default: - g_ptr++; break; } } @@ -1568,9 +1563,9 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end, ASSERT(is_boxed(val)); *n_hp++ = val; } else if (ErtsInArea(ptr, mature, mature_size)) { - move_boxed(&ptr,val,&old_htop,n_hp++); + move_boxed(ptr,val,&old_htop,n_hp++); } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) { - move_boxed(&ptr,val,&n_htop,n_hp++); + move_boxed(ptr,val,&n_htop,n_hp++); } else { n_hp++; } @@ -1582,9 +1577,9 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end, if (IS_MOVED_CONS(val)) { *n_hp++ = ptr[1]; } else if (ErtsInArea(ptr, mature, mature_size)) { - move_cons(&ptr,val,&old_htop,n_hp++); + move_cons(ptr,val,&old_htop,n_hp++); } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) { - move_cons(&ptr,val,&n_htop,n_hp++); + move_cons(ptr,val,&n_htop,n_hp++); } else { n_hp++; } @@ -1604,10 +1599,10 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end, *origptr = val; mb->base = binary_bytes(val); } else if (ErtsInArea(ptr, mature, mature_size)) { - move_boxed(&ptr,val,&old_htop,origptr); + move_boxed(ptr,val,&old_htop,origptr); mb->base = binary_bytes(mb->orig); } else if (ErtsInYoungGen(*origptr, ptr, oh, oh_size)) { - move_boxed(&ptr,val,&n_htop,origptr); + move_boxed(ptr,val,&n_htop,origptr); mb->base = binary_bytes(mb->orig); } } @@ -1733,16 +1728,8 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end, n_htop = n_heap = (Eterm *) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP, sizeof(Eterm)*new_sz); - if (live_hf_end != ERTS_INVALID_HFRAG_PTR) { - /* - * Move heap frags that we know are completely live - * directly into the heap. - */ - n_htop = collect_live_heap_frags(p, live_hf_end, n_heap, n_htop, - objv, nobj); - } - - n_htop = full_sweep_heaps(p, 0, n_heap, n_htop, oh, oh_size, objv, nobj); + n_htop = full_sweep_heaps(p, live_hf_end, 0, n_heap, n_htop, oh, oh_size, + objv, nobj); /* Move the stack to the end of the heap */ stk_sz = HEAP_END(p) - p->stop; @@ -1789,6 +1776,7 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end, static Eterm * full_sweep_heaps(Process *p, + ErlHeapFragment *live_hf_end, int hibernate, Eterm *n_heap, Eterm* n_htop, char *oh, Uint oh_size, @@ -1805,6 +1793,19 @@ full_sweep_heaps(Process *p, n = setup_rootset(p, objv, nobj, &rootset); + /* + * All allocations done. Start defile heap with move markers. + * A crash dump due to allocation failure above will see a healthy heap. + */ + + if (live_hf_end != ERTS_INVALID_HFRAG_PTR) { + /* + * Move heap frags that we know are completely live + * directly into the heap. + */ + n_htop = collect_live_heap_frags(p, live_hf_end, n_htop); + } + #ifdef HIPE if (hibernate) hipe_empty_nstack(p); @@ -1818,7 +1819,7 @@ full_sweep_heaps(Process *p, Eterm g_sz = roots->sz; roots++; - while (g_sz--) { + for ( ; g_sz--; g_ptr++) { Eterm* ptr; Eterm val; Eterm gval = *g_ptr; @@ -1830,32 +1831,26 @@ full_sweep_heaps(Process *p, val = *ptr; if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); - *g_ptr++ = val; + *g_ptr = val; } else if (!erts_is_literal(gval, ptr)) { - move_boxed(&ptr,val,&n_htop,g_ptr++); - } else { - g_ptr++; + move_boxed(ptr,val,&n_htop,g_ptr); } - continue; + break; } case TAG_PRIMARY_LIST: { ptr = list_val(gval); val = *ptr; if (IS_MOVED_CONS(val)) { - *g_ptr++ = ptr[1]; + *g_ptr = ptr[1]; } else if (!erts_is_literal(gval, ptr)) { - move_cons(&ptr,val,&n_htop,g_ptr++); - } else { - g_ptr++; + move_cons(ptr,val,&n_htop,g_ptr); } - continue; + break; } - default: { - g_ptr++; - continue; - } + default: + break; } } } @@ -2134,7 +2129,7 @@ sweep(Eterm *n_hp, Eterm *n_htop, ASSERT(is_boxed(val)); *n_hp++ = val; } else if (ERTS_IS_IN_SWEEP_AREA(gval, ptr)) { - move_boxed(&ptr,val,&n_htop,n_hp++); + move_boxed(ptr,val,&n_htop,n_hp++); } else { n_hp++; } @@ -2146,7 +2141,7 @@ sweep(Eterm *n_hp, Eterm *n_htop, if (IS_MOVED_CONS(val)) { *n_hp++ = ptr[1]; } else if (ERTS_IS_IN_SWEEP_AREA(gval, ptr)) { - move_cons(&ptr,val,&n_htop,n_hp++); + move_cons(ptr,val,&n_htop,n_hp++); } else { n_hp++; } @@ -2167,7 +2162,7 @@ sweep(Eterm *n_hp, Eterm *n_htop, *origptr = val; mb->base = binary_bytes(*origptr); } else if (ERTS_IS_IN_SWEEP_AREA(*origptr, ptr)) { - move_boxed(&ptr,val,&n_htop,origptr); + move_boxed(ptr,val,&n_htop,origptr); mb->base = binary_bytes(*origptr); } } @@ -2230,7 +2225,7 @@ sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, ASSERT(is_boxed(val)); *heap_ptr++ = val; } else if (ErtsInArea(ptr, src, src_size)) { - move_boxed(&ptr,val,&htop,heap_ptr++); + move_boxed(ptr,val,&htop,heap_ptr++); } else { heap_ptr++; } @@ -2242,7 +2237,7 @@ sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, if (IS_MOVED_CONS(val)) { *heap_ptr++ = ptr[1]; } else if (ErtsInArea(ptr, src, src_size)) { - move_cons(&ptr,val,&htop,heap_ptr++); + move_cons(ptr,val,&htop,heap_ptr++); } else { heap_ptr++; } @@ -2263,7 +2258,7 @@ sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, *origptr = val; mb->base = binary_bytes(*origptr); } else if (ErtsInArea(ptr, src, src_size)) { - move_boxed(&ptr,val,&htop,origptr); + move_boxed(ptr,val,&htop,origptr); mb->base = binary_bytes(*origptr); } } @@ -2296,11 +2291,11 @@ move_one_area(Eterm* n_htop, char* src, Uint src_size) ASSERT(val != ERTS_HOLE_MARKER); if (is_header(val)) { ASSERT(ptr + header_arity(val) < end); - move_boxed(&ptr, val, &n_htop, &dummy_ref); + ptr = move_boxed(ptr, val, &n_htop, &dummy_ref); } else { /* must be a cons cell */ ASSERT(ptr+1 < end); - move_cons(&ptr, val, &n_htop, &dummy_ref); + move_cons(ptr, val, &n_htop, &dummy_ref); ptr += 2; } } @@ -2313,9 +2308,7 @@ move_one_area(Eterm* n_htop, char* src, Uint src_size) */ static Eterm* -collect_live_heap_frags(Process* p, ErlHeapFragment *live_hf_end, - Eterm* n_hstart, Eterm* n_htop, - Eterm* objv, int nobj) +collect_live_heap_frags(Process* p, ErlHeapFragment *live_hf_end, Eterm* n_htop) { ErlHeapFragment* qb; char* frag_begin; @@ -3281,8 +3274,7 @@ reply_gc_info(void *vgcirp) gcireq_free(vgcirp); } -void erts_sub_binary_to_heap_binary(Eterm **pp, Eterm **hpp, Eterm *orig) { - Eterm *ptr = *pp; +Eterm* erts_sub_binary_to_heap_binary(Eterm *ptr, Eterm **hpp, Eterm *orig) { Eterm *htop = *hpp; Eterm gval; ErlSubBin *sb = (ErlSubBin *)ptr; @@ -3310,7 +3302,7 @@ void erts_sub_binary_to_heap_binary(Eterm **pp, Eterm **hpp, Eterm *orig) { htop += heap_bin_size(sb->size); *hpp = htop; - *pp = ptr; + return ptr; } diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h index 6a529b8443..dec0ab1143 100644 --- a/erts/emulator/beam/erl_gc.h +++ b/erts/emulator/beam/erl_gc.h @@ -33,14 +33,15 @@ #define IS_MOVED_BOXED(x) (!is_header((x))) #define IS_MOVED_CONS(x) (is_non_value((x))) -void erts_sub_binary_to_heap_binary(Eterm **pp, Eterm **hpp, Eterm *orig); +Eterm* erts_sub_binary_to_heap_binary(Eterm *ptr, Eterm **hpp, Eterm *orig); -ERTS_GLB_INLINE void move_cons(Eterm **pp, Eterm car, Eterm **hpp, Eterm *orig); +ERTS_GLB_INLINE void move_cons(Eterm *ERTS_RESTRICT ptr, Eterm car, Eterm **hpp, + Eterm *orig); #if ERTS_GLB_INLINE_INCL_FUNC_DEF -ERTS_GLB_INLINE void move_cons(Eterm **pp, Eterm car, Eterm **hpp, Eterm *orig) +ERTS_GLB_INLINE void move_cons(Eterm *ERTS_RESTRICT ptr, Eterm car, Eterm **hpp, + Eterm *orig) { - Eterm *ptr = *pp; - Eterm *htop = *hpp; + Eterm *ERTS_RESTRICT htop = *hpp; Eterm gval; htop[0] = car; /* copy car */ @@ -53,14 +54,15 @@ ERTS_GLB_INLINE void move_cons(Eterm **pp, Eterm car, Eterm **hpp, Eterm *orig) } #endif -ERTS_GLB_INLINE void move_boxed(Eterm **pp, Eterm hdr, Eterm **hpp, Eterm *orig); +ERTS_GLB_INLINE Eterm* move_boxed(Eterm *ERTS_RESTRICT ptr, Eterm hdr, Eterm **hpp, + Eterm *orig); #if ERTS_GLB_INLINE_INCL_FUNC_DEF -ERTS_GLB_INLINE void move_boxed(Eterm **pp, Eterm hdr, Eterm **hpp, Eterm *orig) +ERTS_GLB_INLINE Eterm* move_boxed(Eterm *ERTS_RESTRICT ptr, Eterm hdr, Eterm **hpp, + Eterm *orig) { Eterm gval; Sint nelts; - Eterm *ptr = *pp; - Eterm *htop = *hpp; + Eterm *ERTS_RESTRICT htop = *hpp; ASSERT(is_header(hdr)); nelts = header_arity(hdr); @@ -71,8 +73,7 @@ ERTS_GLB_INLINE void move_boxed(Eterm **pp, Eterm hdr, Eterm **hpp, Eterm *orig) /* convert sub-binary to heap-binary if applicable */ if (sb->bitsize == 0 && sb->bitoffs == 0 && sb->is_writable == 0 && sb->size <= sizeof(Eterm) * 3) { - erts_sub_binary_to_heap_binary(pp, hpp, orig); - return; + return erts_sub_binary_to_heap_binary(ptr, hpp, orig); } } nelts++; @@ -90,7 +91,7 @@ ERTS_GLB_INLINE void move_boxed(Eterm **pp, Eterm hdr, Eterm **hpp, Eterm *orig) while (nelts--) *htop++ = *ptr++; *hpp = htop; - *pp = ptr; + return ptr; } #endif diff --git a/erts/emulator/beam/erl_goodfit_alloc.c b/erts/emulator/beam/erl_goodfit_alloc.c index 50aa41b4d2..a38f6c7daf 100644 --- a/erts/emulator/beam/erl_goodfit_alloc.c +++ b/erts/emulator/beam/erl_goodfit_alloc.c @@ -170,6 +170,7 @@ static void unlink_free_block (Allctr_t *, Block_t *); static void update_last_aux_mbc (Allctr_t *, Carrier_t *); static Eterm info_options (Allctr_t *, char *, fmtfn_t *, void *, Uint **, Uint *); +static int gfalc_try_set_dyn_param(Allctr_t*, Eterm param, Uint value); static void init_atoms (void); #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG @@ -250,6 +251,8 @@ erts_gfalc_start(GFAllctr_t *gfallctr, if (!erts_alcu_start(allctr, init)) return NULL; + allctr->try_set_dyn_param = gfalc_try_set_dyn_param; + if (allctr->min_block_size != MIN_BLK_SZ) return NULL; @@ -584,6 +587,15 @@ info_options(Allctr_t *allctr, return res; } +static int gfalc_try_set_dyn_param(Allctr_t* allctr, Eterm param, Uint value) +{ + if (param == am_sbct) { + /* Cannot change 'sbct' without rearranging buckets */ + return 0; + } + return erts_alcu_try_set_dyn_param(allctr, param, value); +} + /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ * NOTE: erts_gfalc_test() is only supposed to be used for testing. * * * diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 4846ccd2d3..e8048cfdfc 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -1746,6 +1746,7 @@ erl_start(int argc, char **argv) } else if (has_prefix("ecio", sub_param)) { /* ignore argument, eager check io no longer used */ + arg = get_arg(sub_param+4, argv[i+1], &i); } else if (has_prefix("pp", sub_param)) { arg = get_arg(sub_param+2, argv[i+1], &i); diff --git a/erts/emulator/beam/erl_io_queue.c b/erts/emulator/beam/erl_io_queue.c index c93b5248d9..d779d1031a 100644 --- a/erts/emulator/beam/erl_io_queue.c +++ b/erts/emulator/beam/erl_io_queue.c @@ -801,12 +801,11 @@ static Eterm iol2v_make_sub_bin(iol2v_state_t *state, Eterm bin_term, ERTS_GET_REAL_BIN(bin_term, orig_pb_term, byte_offset, bit_offset, bit_size); - (void)bit_offset; - (void)bit_size; + ASSERT(bit_size == 0); sb->thing_word = HEADER_SUB_BIN; + sb->bitoffs = bit_offset; sb->bitsize = 0; - sb->bitoffs = 0; sb->orig = orig_pb_term; sb->is_writable = 0; @@ -975,7 +974,7 @@ static int iol2v_append_binary(iol2v_state_t *state, Eterm bin_term) { parent_header = binary_val(parent_binary); binary_size = binary_size(bin_term); - if (bit_offset != 0 || bit_size != 0) { + if (bit_size != 0) { return 0; } else if (binary_size == 0) { state->bytereds_spent += 1; @@ -1017,8 +1016,16 @@ static int iol2v_append_binary(iol2v_state_t *state, Eterm bin_term) { * then just copy it into the accumulator. */ iol2v_expand_acc(state, binary_size); - sys_memcpy(&(state->acc)->orig_bytes[state->acc_size], - binary_data, binary_size); + if (ERTS_LIKELY(bit_offset == 0)) { + sys_memcpy(&(state->acc)->orig_bytes[state->acc_size], + binary_data, binary_size); + } else { + ASSERT(binary_size <= ERTS_UWORD_MAX / 8); + + erts_copy_bits(binary_data, bit_offset, 1, + (byte*)&(state->acc)->orig_bytes[state->acc_size], 0, 1, + binary_size * 8); + } state->acc_size += binary_size; } else { @@ -1029,8 +1036,16 @@ static int iol2v_append_binary(iol2v_state_t *state, Eterm bin_term) { iol2v_expand_acc(state, spill); - sys_memcpy(&(state->acc)->orig_bytes[state->acc_size], - binary_data, spill); + if (ERTS_LIKELY(bit_offset == 0)) { + sys_memcpy(&(state->acc)->orig_bytes[state->acc_size], + binary_data, spill); + } else { + ASSERT(binary_size <= ERTS_UWORD_MAX / 8); + + erts_copy_bits(binary_data, bit_offset, 1, + (byte*)&(state->acc)->orig_bytes[state->acc_size], 0, 1, + spill * 8); + } state->acc_size += spill; @@ -1179,7 +1194,10 @@ BIF_RETTYPE iolist_to_iovec_1(BIF_ALIST_1) { if (is_nil(BIF_ARG_1)) { BIF_RET(NIL); } else if (is_binary(BIF_ARG_1)) { - if (binary_size(BIF_ARG_1) != 0) { + if (binary_bitsize(BIF_ARG_1) != 0) { + ASSERT(!(BIF_P->flags & F_DISABLE_GC)); + BIF_ERROR(BIF_P, BADARG); + } else if (binary_size(BIF_ARG_1) != 0) { Eterm *hp = HAlloc(BIF_P, 2); BIF_RET(CONS(hp, BIF_ARG_1, NIL)); diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index abf194cf94..6f7c71ef98 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -616,7 +616,7 @@ erts_try_alloc_message_on_heap(Process *pp, } else { in_message_fragment: - if (!((*psp) & ERTS_PSFLG_ON_HEAP_MSGQ)) { + if ((*psp) & ERTS_PSFLG_OFF_HEAP_MSGQ) { mp = erts_alloc_message(sz, hpp); *ohpp = sz == 0 ? NULL : &mp->hfrag.off_heap; } @@ -1079,8 +1079,6 @@ erts_change_message_queue_management(Process *c_p, Eterm new_state) case am_on_heap: c_p->flags |= F_ON_HEAP_MSGQ; c_p->flags &= ~F_OFF_HEAP_MSGQ; - erts_atomic32_read_bor_nob(&c_p->state, - ERTS_PSFLG_ON_HEAP_MSGQ); /* * We are not allowed to clear ERTS_PSFLG_OFF_HEAP_MSGQ * if a off heap change is ongoing. It will be adjusted @@ -1106,8 +1104,6 @@ erts_change_message_queue_management(Process *c_p, Eterm new_state) break; case am_off_heap: c_p->flags &= ~F_ON_HEAP_MSGQ; - erts_atomic32_read_band_nob(&c_p->state, - ~ERTS_PSFLG_ON_HEAP_MSGQ); goto change_to_off_heap; default: res = THE_NON_VALUE; /* badarg */ diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 94ebf88b56..d2000aa71e 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -237,9 +237,11 @@ static void cache_env(ErlNifEnv* env); static void full_flush_env(ErlNifEnv *env); static void flush_env(ErlNifEnv* env); -/* Temporary object header, auto-deallocated when NIF returns - * or when independent environment is cleared. - */ +/* Temporary object header, auto-deallocated when NIF returns or when + * independent environment is cleared. + * + * The payload can be accessed with &tmp_obj_ptr[1] but keep in mind that its + * first element must not require greater alignment than `next`. */ struct enif_tmp_obj_t { struct enif_tmp_obj_t* next; void (*dtor)(struct enif_tmp_obj_t*); @@ -256,6 +258,46 @@ static ERTS_INLINE void free_tmp_objs(ErlNifEnv* env) } } +/* Whether the given environment is bound to a process and will be cleaned up + * when the NIF returns. It's safe to use temp_alloc for objects in + * env->tmp_obj_list when this is true. */ +static ERTS_INLINE int is_proc_bound(ErlNifEnv *env) +{ + return env->mod_nif != NULL; +} + +/* Allocates and attaches an object to the given environment, running its + * destructor when the environment is cleared. To avoid temporary variables the + * address of the allocated object is returned instead of the enif_tmp_obj_t. + * + * The destructor *must* call `erts_free(tmp_obj->allocator, tmp_obj)` to free + * the object. If the destructor needs to refer to the allocated object its + * address will be &tmp_obj[1]. */ +static ERTS_INLINE void *alloc_tmp_obj(ErlNifEnv *env, size_t size, + void (*dtor)(struct enif_tmp_obj_t*)) { + struct enif_tmp_obj_t *tmp_obj; + ErtsAlcType_t allocator; + + allocator = is_proc_bound(env) ? ERTS_ALC_T_TMP : ERTS_ALC_T_NIF; + + tmp_obj = erts_alloc(allocator, sizeof(struct enif_tmp_obj_t) + MAX(1, size)); + + tmp_obj->next = env->tmp_obj_list; + tmp_obj->allocator = allocator; + tmp_obj->dtor = dtor; + + env->tmp_obj_list = tmp_obj; + + return (void*)&tmp_obj[1]; +} + +/* Generic destructor for objects allocated through alloc_tmp_obj that don't + * care about their payload. */ +static void tmp_alloc_dtor(struct enif_tmp_obj_t *tmp_obj) +{ + erts_free(tmp_obj->allocator, tmp_obj); +} + void erts_post_nif(ErlNifEnv* env) { erts_unblock_fpe(env->fpe_was_unmasked); @@ -446,6 +488,7 @@ static void cache_env(ErlNifEnv* env) env->hp_end = env->heap_frag->mem + env->heap_frag->alloc_size; } } + void* enif_priv_data(ErlNifEnv* env) { return env->mod_nif->priv_data; @@ -1019,11 +1062,6 @@ int enif_is_number(ErlNifEnv* env, ERL_NIF_TERM term) return is_number(term); } -static ERTS_INLINE int is_proc_bound(ErlNifEnv* env) -{ - return env->mod_nif != NULL; -} - static void aligned_binary_dtor(struct enif_tmp_obj_t* obj) { erts_free_aligned_binary_bytes_extra((byte*)obj, obj->allocator); @@ -1058,22 +1096,14 @@ int enif_inspect_binary(ErlNifEnv* env, Eterm bin_term, ErlNifBinary* bin) u.tmp->dtor = &aligned_binary_dtor; env->tmp_obj_list = u.tmp; } - bin->bin_term = bin_term; bin->size = binary_size(bin_term); bin->ref_bin = NULL; ADD_READONLY_CHECK(env, bin->data, bin->size); return 1; } -static void tmp_alloc_dtor(struct enif_tmp_obj_t* obj) -{ - erts_free(obj->allocator, obj); -} - int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin) { - struct enif_tmp_obj_t* tobj; - ErtsAlcType_t allocator; ErlDrvSizeT sz; if (is_binary(term)) { return enif_inspect_binary(env,term,bin); @@ -1081,7 +1111,6 @@ int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin) if (is_nil(term)) { bin->data = (unsigned char*) &bin->data; /* dummy non-NULL */ bin->size = 0; - bin->bin_term = THE_NON_VALUE; bin->ref_bin = NULL; return 1; } @@ -1089,16 +1118,8 @@ int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin) return 0; } - allocator = is_proc_bound(env) ? ERTS_ALC_T_TMP : ERTS_ALC_T_NIF; - tobj = erts_alloc(allocator, sz + sizeof(struct enif_tmp_obj_t)); - tobj->allocator = allocator; - tobj->next = env->tmp_obj_list; - tobj->dtor = &tmp_alloc_dtor; - env->tmp_obj_list = tobj; - - bin->data = (unsigned char*) &tobj[1]; + bin->data = alloc_tmp_obj(env, sz, &tmp_alloc_dtor); bin->size = sz; - bin->bin_term = THE_NON_VALUE; bin->ref_bin = NULL; erts_iolist_to_buf(term, (char*) bin->data, sz); ADD_READONLY_CHECK(env, bin->data, bin->size); @@ -1116,7 +1137,6 @@ int enif_alloc_binary(size_t size, ErlNifBinary* bin) bin->size = size; bin->data = (unsigned char*) refbin->orig_bytes; - bin->bin_term = THE_NON_VALUE; bin->ref_bin = refbin; return 1; } @@ -1150,12 +1170,10 @@ void enif_release_binary(ErlNifBinary* bin) { if (bin->ref_bin != NULL) { Binary* refbin = bin->ref_bin; - ASSERT(bin->bin_term == THE_NON_VALUE); erts_bin_release(refbin); } #ifdef DEBUG bin->data = NULL; - bin->bin_term = THE_NON_VALUE; bin->ref_bin = NULL; #endif } @@ -1312,29 +1330,51 @@ int enif_get_string(ErlNifEnv *env, ERL_NIF_TERM list, char* buf, unsigned len, Eterm enif_make_binary(ErlNifEnv* env, ErlNifBinary* bin) { - if (bin->bin_term != THE_NON_VALUE) { - return bin->bin_term; - } - else if (bin->ref_bin != NULL) { - Binary* bptr = bin->ref_bin; - Eterm bin_term; - - bin_term = erts_build_proc_bin(&MSO(env->proc), - alloc_heap(env, PROC_BIN_SIZE), - bptr); - if (erts_refc_read(&bptr->intern.refc, 1) == 1) { - /* Total ownership transfer */ - bin->ref_bin = NULL; - bin->bin_term = bin_term; - } - return bin_term; - } - else { - flush_env(env); - bin->bin_term = new_binary(env->proc, bin->data, bin->size); - cache_env(env); - return bin->bin_term; + Eterm bin_term; + + if (bin->ref_bin != NULL) { + Binary* binary = bin->ref_bin; + + /* If the binary is smaller than the heap binary limit we'll return a + * heap binary to reduce the number of small refc binaries in the + * system. We can't simply release the refc binary right away however; + * the documentation states that the binary should be considered + * read-only from this point on, which implies that it should still be + * readable. + * + * We could keep it alive until we return by adding it to the temporary + * object list, but that requires an off-heap allocation which is + * potentially quite slow, so we create a dummy ProcBin instead and + * rely on the next minor GC to get rid of it. */ + if (bin->size <= ERL_ONHEAP_BIN_LIMIT) { + ErlHeapBin* hb; + + hb = (ErlHeapBin*)alloc_heap(env, heap_bin_size(bin->size)); + hb->thing_word = header_heap_bin(bin->size); + hb->size = bin->size; + + sys_memcpy(hb->data, bin->data, bin->size); + + erts_build_proc_bin(&MSO(env->proc), + alloc_heap(env, PROC_BIN_SIZE), + binary); + + bin_term = make_binary(hb); + } else { + bin_term = erts_build_proc_bin(&MSO(env->proc), + alloc_heap(env, PROC_BIN_SIZE), + binary); + } + + /* Our (possibly shared) ownership has been transferred to the term. */ + bin->ref_bin = NULL; + } else { + flush_env(env); + bin_term = new_binary(env->proc, bin->data, bin->size); + cache_env(env); } + + return bin_term; } Eterm enif_make_sub_binary(ErlNifEnv* env, ERL_NIF_TERM bin_term, @@ -3284,8 +3324,8 @@ typedef struct { Eterm sublist_start; Eterm sublist_end; - UWord offheap_size; - UWord onheap_size; + UWord referenced_size; + UWord copied_size; UWord iovec_len; } iovec_slice_t; @@ -3295,16 +3335,16 @@ static int examine_iovec_term(Eterm list, UWord max_length, iovec_slice_t *resul result->sublist_start = list; result->sublist_length = 0; - result->offheap_size = 0; - result->onheap_size = 0; + result->referenced_size = 0; + result->copied_size = 0; result->iovec_len = 0; lookahead = result->sublist_start; while (is_list(lookahead)) { - Eterm *binary_header, binary; + UWord byte_size; + Eterm binary; Eterm *cell; - UWord size; cell = list_val(lookahead); binary = CAR(cell); @@ -3313,35 +3353,36 @@ static int examine_iovec_term(Eterm list, UWord max_length, iovec_slice_t *resul return 0; } - size = binary_size(binary); - binary_header = binary_val(binary); + byte_size = binary_size(binary); + + if (byte_size > 0) { + int bit_offset, bit_size; + Eterm parent_binary; + UWord byte_offset; - if (size > 0) { - /* If we're a sub-binary we'll need to check our underlying binary - * to determine whether we're on-heap or not. */ - if (thing_subtag(*binary_header) == SUB_BINARY_SUBTAG) { - ErlSubBin *sb = (ErlSubBin*)binary_header; + int requires_copying; - /* Reject bitstrings */ - if((sb->bitoffs + sb->bitsize) > 0) { - return 0; - } + ERTS_GET_REAL_BIN(binary, parent_binary, byte_offset, + bit_offset, bit_size); - ASSERT(size <= binary_size(sb->orig)); - binary_header = binary_val(sb->orig); + (void)byte_offset; + + if (bit_size != 0) { + return 0; } - if (thing_subtag(*binary_header) == HEAP_BINARY_SUBTAG) { - ASSERT(size <= ERL_ONHEAP_BIN_LIMIT); + /* If we're unaligned or an on-heap binary we'll need to copy + * ourselves over to a temporary buffer. */ + requires_copying = (bit_offset != 0) || + thing_subtag(*binary_val(parent_binary)) == HEAP_BINARY_SUBTAG; - result->iovec_len += 1; - result->onheap_size += size; + if (requires_copying) { + result->copied_size += byte_size; } else { - ASSERT(thing_subtag(*binary_header) == REFC_BINARY_SUBTAG); - - result->iovec_len += 1 + size / MAX_SYSIOVEC_IOVLEN; - result->offheap_size += size; + result->referenced_size += byte_size; } + + result->iovec_len += 1 + byte_size / MAX_SYSIOVEC_IOVLEN; } result->sublist_length += 1; @@ -3361,7 +3402,9 @@ static int examine_iovec_term(Eterm list, UWord max_length, iovec_slice_t *resul return 1; } -static void inspect_raw_binary_data(Eterm binary, ErlNifBinary *result) { +static void marshal_iovec_binary(Eterm binary, ErlNifBinary *copy_buffer, + UWord *copy_offset, ErlNifBinary *result) { + Eterm *parent_header; Eterm parent_binary; @@ -3372,10 +3415,11 @@ static void inspect_raw_binary_data(Eterm binary, ErlNifBinary *result) { ERTS_GET_REAL_BIN(binary, parent_binary, byte_offset, bit_offset, bit_size); + ASSERT(bit_size == 0); + parent_header = binary_val(parent_binary); result->size = binary_size(binary); - result->bin_term = binary; if (thing_subtag(*parent_header) == REFC_BINARY_SUBTAG) { ProcBin *pb = (ProcBin*)parent_header; @@ -3398,24 +3442,48 @@ static void inspect_raw_binary_data(Eterm binary, ErlNifBinary *result) { result->data = &((unsigned char*)&hb->data)[byte_offset]; result->ref_bin = NULL; } + + /* If this isn't an *aligned* refc binary, copy its contents to the buffer + * and reference that instead. */ + + if (result->ref_bin == NULL || bit_offset != 0) { + ASSERT(copy_buffer->ref_bin != NULL && copy_buffer->data != NULL); + ASSERT(result->size <= (copy_buffer->size - *copy_offset)); + + if (bit_offset == 0) { + sys_memcpy(©_buffer->data[*copy_offset], + result->data, result->size); + } else { + erts_copy_bits(result->data, bit_offset, 1, + (byte*)©_buffer->data[*copy_offset], 0, 1, + result->size * 8); + } + + result->data = ©_buffer->data[*copy_offset]; + result->ref_bin = copy_buffer->ref_bin; + + *copy_offset += result->size; + } } static int fill_iovec_with_slice(ErlNifEnv *env, iovec_slice_t *slice, ErlNifIOVec *iovec) { - UWord onheap_offset, iovec_idx; - ErlNifBinary onheap_data; + ErlNifBinary copy_buffer = {0}; + UWord copy_offset, iovec_idx; Eterm sublist_iterator; - /* Set up a common refc binary for all on-heap binaries. */ - if (slice->onheap_size > 0) { - if (!enif_alloc_binary(slice->onheap_size, &onheap_data)) { + /* Set up a common refc binary for all on-heap and unaligned binaries. */ + if (slice->copied_size > 0) { + if (!enif_alloc_binary(slice->copied_size, ©_buffer)) { return 0; } + + ASSERT(copy_buffer.ref_bin != NULL); } sublist_iterator = slice->sublist_start; - onheap_offset = 0; + copy_offset = 0; iovec_idx = 0; while (sublist_iterator != slice->sublist_end) { @@ -3423,20 +3491,7 @@ static int fill_iovec_with_slice(ErlNifEnv *env, Eterm *cell; cell = list_val(sublist_iterator); - inspect_raw_binary_data(CAR(cell), &raw_data); - - /* If this isn't a refc binary, copy its contents to the onheap buffer - * and reference that instead. */ - if (raw_data.size > 0 && raw_data.ref_bin == NULL) { - ASSERT(onheap_offset < onheap_data.size); - ASSERT(slice->onheap_size > 0); - - sys_memcpy(&onheap_data.data[onheap_offset], - raw_data.data, raw_data.size); - - raw_data.data = &onheap_data.data[onheap_offset]; - raw_data.ref_bin = onheap_data.ref_bin; - } + marshal_iovec_binary(CAR(cell), ©_buffer, ©_offset, &raw_data); while (raw_data.size > 0) { UWord chunk_len = MIN(raw_data.size, MAX_SYSIOVEC_IOVLEN); @@ -3467,16 +3522,18 @@ static int fill_iovec_with_slice(ErlNifEnv *env, erts_refc_inc(&refc_binary->intern.refc, 1); } - if (slice->onheap_size > 0) { + if (slice->copied_size > 0) { /* Transfer ownership to the iovec; we've taken references to it in * the above loop. */ - enif_release_binary(&onheap_data); + enif_release_binary(©_buffer); } } else { - if (slice->onheap_size > 0) { - /* Attach the binary to our environment and let the GC take care of - * it after returning. */ - enif_make_binary(env, &onheap_data); + if (slice->copied_size > 0) { + /* Attach the binary to our environment and let the next minor GC + * get rid of it. This is slightly faster than using the tmp object + * list since it avoids off-heap allocations. */ + erts_build_proc_bin(&MSO(env->proc), + alloc_heap(env, PROC_BIN_SIZE), copy_buffer.ref_bin); } } @@ -3502,19 +3559,14 @@ static int create_iovec_from_slice(ErlNifEnv *env, alloc_size = binv_offset; alloc_size += slice->iovec_len * sizeof(Binary*); - /* If we have an environment we'll attach the allocated data to it. The - * GC will take care of releasing it later on. */ + /* When the user passes an environment, we attach the iovec to it so + * the user won't have to bother managing it (similar to + * enif_inspect_binary). It'll disappear once the environment is + * cleaned up. */ if (env != NULL) { - ErlNifBinary gc_bin; - - if (!enif_alloc_binary(alloc_size, &gc_bin)) { - return 0; - } - - alloc_base = (char*)gc_bin.data; - enif_make_binary(env, &gc_bin); + alloc_base = alloc_tmp_obj(env, alloc_size, &tmp_alloc_dtor); } else { - alloc_base = enif_alloc(alloc_size); + alloc_base = erts_alloc(ERTS_ALC_T_NIF, alloc_size); } iovec = (ErlNifIOVec*)alloc_base; @@ -3523,12 +3575,12 @@ static int create_iovec_from_slice(ErlNifEnv *env, iovec->flags = 0; } - iovec->size = slice->offheap_size + slice->onheap_size; + iovec->size = slice->referenced_size + slice->copied_size; iovec->iovcnt = slice->iovec_len; if(!fill_iovec_with_slice(env, slice, iovec)) { if (env == NULL && !(iovec->flags & ERL_NIF_IOVEC_FLAGS_PREALLOC)) { - enif_free(iovec); + erts_free(ERTS_ALC_T_NIF, iovec); } return 0; @@ -4218,34 +4270,31 @@ static unsigned calc_checksum(unsigned char* ptr, unsigned size); struct readonly_check_t { - struct enif_tmp_obj_t hdr; unsigned char* ptr; unsigned size; unsigned checksum; }; static void add_readonly_check(ErlNifEnv* env, unsigned char* ptr, unsigned sz) { - ErtsAlcType_t allocator = is_proc_bound(env) ? ERTS_ALC_T_TMP : ERTS_ALC_T_NIF; - struct readonly_check_t* obj = erts_alloc(allocator, - sizeof(struct readonly_check_t)); - obj->hdr.allocator = allocator; - obj->hdr.next = env->tmp_obj_list; - env->tmp_obj_list = &obj->hdr; - obj->hdr.dtor = &readonly_check_dtor; + struct readonly_check_t* obj; + + obj = alloc_tmp_obj(env, sizeof(struct readonly_check_t), + &readonly_check_dtor); + obj->ptr = ptr; obj->size = sz; - obj->checksum = calc_checksum(ptr, sz); + obj->checksum = calc_checksum(ptr, sz); } -static void readonly_check_dtor(struct enif_tmp_obj_t* o) +static void readonly_check_dtor(struct enif_tmp_obj_t* tmp_obj) { - struct readonly_check_t* obj = (struct readonly_check_t*) o; - unsigned chksum = calc_checksum(obj->ptr, obj->size); - if (chksum != obj->checksum) { + struct readonly_check_t* ro_check = (struct readonly_check_t*)&tmp_obj[1]; + unsigned chksum = calc_checksum(ro_check->ptr, ro_check->size); + if (chksum != ro_check->checksum) { fprintf(stderr, "\r\nReadonly data written by NIF, checksums differ" - " %x != %x\r\nABORTING\r\n", chksum, obj->checksum); + " %x != %x\r\nABORTING\r\n", chksum, ro_check->checksum); abort(); } - erts_free(obj->hdr.allocator, obj); + erts_free(tmp_obj->allocator, tmp_obj); } static unsigned calc_checksum(unsigned char* ptr, unsigned size) { diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index 053f7673c4..a99b4db705 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -137,8 +137,9 @@ typedef struct unsigned char* data; /* Internals (avert your eyes) */ - ERL_NIF_TERM bin_term; void* ref_bin; + /* for future additions to be ABI compatible (same struct size) */ + void* __spare__[2]; }ErlNifBinary; #if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h index 9117eb1f72..0d148ee048 100644 --- a/erts/emulator/beam/erl_port.h +++ b/erts/emulator/beam/erl_port.h @@ -180,6 +180,7 @@ void erts_init_port_data(Port *); void erts_cleanup_port_data(Port *); Uint erts_port_data_size(Port *); ErlOffHeap *erts_port_data_offheap(Port *); +Eterm erts_port_data_read(Port* prt); #define ERTS_PORT_GET_CONNECTED(PRT) \ ((Eterm) erts_atomic_read_nob(&(PRT)->connected)) @@ -195,26 +196,52 @@ struct erl_drv_port_data_lock { Port *prt; }; +ERTS_GLB_INLINE void erts_init_runq_port(Port *prt, ErtsRunQueue *runq); +ERTS_GLB_INLINE void erts_set_runq_port(Port *prt, ErtsRunQueue *runq); +ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_port(Port *prt); ERTS_GLB_INLINE ErtsRunQueue *erts_port_runq(Port *prt); #if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE void +erts_init_runq_port(Port *prt, ErtsRunQueue *runq) +{ + if (!runq) + ERTS_INTERNAL_ERROR("Missing run-queue"); + erts_atomic_init_nob(&prt->run_queue, (erts_aint_t) runq); +} + +ERTS_GLB_INLINE void +erts_set_runq_port(Port *prt, ErtsRunQueue *runq) +{ + if (!runq) + ERTS_INTERNAL_ERROR("Missing run-queue"); + erts_atomic_set_nob(&prt->run_queue, (erts_aint_t) runq); +} + +ERTS_GLB_INLINE ErtsRunQueue * +erts_get_runq_port(Port *prt) +{ + ErtsRunQueue *runq; + runq = (ErtsRunQueue *) erts_atomic_read_nob(&prt->run_queue); + if (!runq) + ERTS_INTERNAL_ERROR("Missing run-queue"); + return runq; +} + + ERTS_GLB_INLINE ErtsRunQueue * erts_port_runq(Port *prt) { ErtsRunQueue *rq1, *rq2; - rq1 = (ErtsRunQueue *) erts_atomic_read_nob(&prt->run_queue); - if (!rq1) - return NULL; + rq1 = erts_get_runq_port(prt); while (1) { erts_runq_lock(rq1); - rq2 = (ErtsRunQueue *) erts_atomic_read_nob(&prt->run_queue); + rq2 = erts_get_runq_port(prt); if (rq1 == rq2) return rq1; erts_runq_unlock(rq1); rq1 = rq2; - if (!rq1) - return NULL; } } diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c index a588477320..4a3671df0c 100644 --- a/erts/emulator/beam/erl_port_task.c +++ b/erts/emulator/beam/erl_port_task.c @@ -84,11 +84,10 @@ static void chk_task_queues(Port *pp, ErtsPortTask *execq, int processing_busy_q #define LTTNG_DRIVER(TRACEPOINT, PP) do {} while(0) #endif -#define ERTS_LC_VERIFY_RQ(RQ, PP) \ - do { \ +#define ERTS_LC_VERIFY_RQ(RQ, PP) \ + do { \ ERTS_LC_ASSERT(erts_lc_runq_is_locked(runq)); \ - ERTS_LC_ASSERT((RQ) == ((ErtsRunQueue *) \ - erts_atomic_read_nob(&(PP)->run_queue))); \ + ERTS_LC_ASSERT((RQ) == erts_get_runq_port((PP))); \ } while (0) #define ERTS_PT_STATE_SCHEDULED 0 @@ -1520,19 +1519,15 @@ erts_port_task_schedule(Eterm id, /* Enqueue port on run-queue */ runq = erts_port_runq(pp); - if (!runq) - ERTS_INTERNAL_ERROR("Missing run-queue"); xrunq = erts_check_emigration_need(runq, ERTS_PORT_PRIO_LEVEL); ERTS_LC_ASSERT(runq != xrunq); ERTS_LC_VERIFY_RQ(runq, pp); if (xrunq) { /* Emigrate port ... */ - erts_atomic_set_nob(&pp->run_queue, (erts_aint_t) xrunq); + erts_set_runq_port(pp, xrunq); erts_runq_unlock(runq); runq = erts_port_runq(pp); - if (!runq) - ERTS_INTERNAL_ERROR("Missing run-queue"); } enqueue_port(runq, pp); @@ -1593,8 +1588,6 @@ erts_port_task_free_port(Port *pp) ASSERT(!(erts_atomic32_read_nob(&pp->state) & ERTS_PORT_SFLGS_DEAD)); runq = erts_port_runq(pp); - if (!runq) - ERTS_INTERNAL_ERROR("Missing run-queue"); erts_port_task_sched_lock(&pp->sched); flags = erts_atomic32_read_bor_relb(&pp->sched.flags, ERTS_PTS_FLG_EXIT); @@ -1805,7 +1798,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) erts_unblock_fpe(fpe_was_unmasked); ERTS_MSACC_POP_STATE_M(); - ASSERT(runq == (ErtsRunQueue *) erts_atomic_read_nob(&pp->run_queue)); + ASSERT(runq == erts_get_runq_port(pp)); active = finalize_exec(pp, &execq, processing_busy_q); @@ -1831,11 +1824,10 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) } else { /* Emigrate port... */ - erts_atomic_set_nob(&pp->run_queue, (erts_aint_t) xrunq); + erts_set_runq_port(pp, xrunq); erts_runq_unlock(runq); xrunq = erts_port_runq(pp); - ASSERT(xrunq); enqueue_port(xrunq, pp); erts_runq_unlock(xrunq); erts_notify_inc_runq(xrunq); @@ -2069,7 +2061,7 @@ void erts_enqueue_port(ErtsRunQueue *rq, Port *pp) { ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq)); - ASSERT(rq == (ErtsRunQueue *) erts_atomic_read_nob(&pp->run_queue)); + ASSERT(rq == erts_get_runq_port(pp)); ASSERT(erts_atomic32_read_nob(&pp->sched.flags) & ERTS_PTS_FLG_IN_RUNQ); enqueue_port(rq, pp); } @@ -2080,8 +2072,7 @@ erts_dequeue_port(ErtsRunQueue *rq) Port *pp; ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq)); pp = pop_port(rq); - ASSERT(!pp - || rq == (ErtsRunQueue *) erts_atomic_read_nob(&pp->run_queue)); + ASSERT(!pp || rq == erts_get_runq_port(pp)); ASSERT(!pp || (erts_atomic32_read_nob(&pp->sched.flags) & ERTS_PTS_FLG_IN_RUNQ)); return pp; diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index a807d60ec7..23fe353495 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -137,36 +137,6 @@ runq_got_work_to_execute(ErtsRunQueue *rq) return runq_got_work_to_execute_flags(ERTS_RUNQ_FLGS_GET_NOB(rq)); } -#undef RUNQ_READ_RQ -#undef RUNQ_SET_RQ -#define RUNQ_READ_RQ(X) ((ErtsRunQueue *) erts_atomic_read_nob((X))) -#define RUNQ_SET_RQ(X, RQ) erts_atomic_set_nob((X), (erts_aint_t) (RQ)) - -#ifdef DEBUG -# if defined(ARCH_64) -# define ERTS_DBG_SET_INVALID_RUNQP(RQP, N) \ - (RUNQ_SET_RQ((RQP), (0xdeadbeefdead0003LL | ((N) << 4))) -# define ERTS_DBG_VERIFY_VALID_RUNQP(RQP) \ -do { \ - ASSERT((RQP) != NULL); \ - ASSERT(((((Uint) (RQP)) & ((Uint) 0x3))) == ((Uint) 0)); \ - ASSERT((((Uint) (RQP)) & ~((Uint) 0xffff)) != ((Uint) 0xdeadbeefdead0000LL));\ -} while (0) -# else -# define ERTS_DBG_SET_INVALID_RUNQP(RQP, N) \ - (RUNQ_SET_RQ((RQP), (0xdead0003 | ((N) << 4)))) -# define ERTS_DBG_VERIFY_VALID_RUNQP(RQP) \ -do { \ - ASSERT((RQP) != NULL); \ - ASSERT(((((UWord) (RQP)) & ((UWord) 1))) == ((UWord) 0)); \ - ASSERT((((UWord) (RQP)) & ~((UWord) 0xffff)) != ((UWord) 0xdead0000)); \ -} while (0) -# endif -#else -# define ERTS_DBG_SET_INVALID_RUNQP(RQP, N) -# define ERTS_DBG_VERIFY_VALID_RUNQP(RQP) -#endif - const Process erts_invalid_process = {{ERTS_INVALID_PID}}; extern BeamInstr beam_apply[]; @@ -1538,6 +1508,20 @@ erts_proclist_destroy(ErtsProcList *plp) proclist_destroy(plp); } +void +erts_proclist_dump(fmtfn_t to, void *to_arg, ErtsProcList *plp) +{ + ErtsProcList *first = plp; + + while (plp) { + erts_print(to, to_arg, "%T", plp->pid); + plp = plp->next; + if (plp == first) + break; + } + erts_print(to, to_arg, "\n"); +} + void * erts_psd_set_init(Process *p, int ix, void *data) { @@ -3926,21 +3910,16 @@ immigrate(ErtsRunQueue *c_rq, ErtsMigrationPath *mp) Port *prt; prt = erts_dequeue_port(rq); if (prt) - RUNQ_SET_RQ(&prt->run_queue, c_rq); + erts_set_runq_port(prt, c_rq); erts_runq_unlock(rq); if (prt) { - /* port might terminate while we have no lock... */ rq = erts_port_runq(prt); - if (rq) { - if (rq != c_rq) - erts_exit(ERTS_ABORT_EXIT, - "%s:%d:%s(): Internal error", - __FILE__, __LINE__, __func__); - erts_enqueue_port(c_rq, prt); - if (!iflag) - return; /* done */ - erts_runq_unlock(c_rq); - } + if (rq != c_rq) + ERTS_INTERNAL_ERROR("Unexpected run-queue"); + erts_enqueue_port(c_rq, prt); + if (!iflag) + return; /* done */ + erts_runq_unlock(c_rq); } } else { @@ -3954,12 +3933,11 @@ immigrate(ErtsRunQueue *c_rq, ErtsMigrationPath *mp) while (proc) { erts_aint32_t state; state = erts_atomic32_read_acqb(&proc->state); - if (!(ERTS_PSFLG_BOUND & state) - && (prio == (int) ERTS_PSFLGS_GET_PRQ_PRIO(state))) { + if (prio == (int) ERTS_PSFLGS_GET_PRQ_PRIO(state) + && erts_try_change_runq_proc(proc, c_rq)) { ErtsRunQueueInfo *rqi = &rq->procs.prio_info[prio]; unqueue_process(rq, rpq, rqi, prio, prev_proc, proc); erts_runq_unlock(rq); - RUNQ_SET_RQ(&proc->run_queue, c_rq); rq_locked = 0; erts_runq_lock(c_rq); @@ -4140,21 +4118,13 @@ evacuate_run_queue(ErtsRunQueue *rq, while (prt) { ErtsRunQueue *prt_rq; prt = erts_dequeue_port(rq); - RUNQ_SET_RQ(&prt->run_queue, to_rq); + erts_set_runq_port(prt, to_rq); erts_runq_unlock(rq); - /* - * The port might terminate while - * we have no lock on it... - */ prt_rq = erts_port_runq(prt); - if (prt_rq) { - if (prt_rq != to_rq) - erts_exit(ERTS_ABORT_EXIT, - "%s:%d:%s() internal error\n", - __FILE__, __LINE__, __func__); - erts_enqueue_port(to_rq, prt); - erts_runq_unlock(to_rq); - } + if (prt_rq != to_rq) + ERTS_INTERNAL_ERROR("Unexpected run-queue"); + erts_enqueue_port(to_rq, prt); + erts_runq_unlock(to_rq); erts_runq_lock(rq); prt = rq->ports.start; } @@ -4165,8 +4135,6 @@ evacuate_run_queue(ErtsRunQueue *rq, for (prio_q = 0; prio_q < ERTS_NO_PROC_PRIO_QUEUES; prio_q++) { erts_aint32_t state; Process *proc; - int notify = 0; - to_rq = NULL; if (!mp->prio[prio_q].runq) return; @@ -4177,14 +4145,13 @@ evacuate_run_queue(ErtsRunQueue *rq, while (proc) { Process *real_proc; int prio; - erts_aint32_t max_qbit, qbit, real_state; + erts_aint32_t max_qbit, qbit; prio = ERTS_PSFLGS_GET_PRQ_PRIO(state); qbit = ((erts_aint32_t) 1) << prio; if (!(state & ERTS_PSFLG_PROXY)) { real_proc = proc; - real_state = state; } else { real_proc = erts_proc_lookup_raw(proc->common.id); @@ -4192,7 +4159,6 @@ evacuate_run_queue(ErtsRunQueue *rq, free_proxy_proc(proc); goto handle_next_proc; } - real_state = erts_atomic32_read_acqb(&real_proc->state); } max_qbit = (state >> ERTS_PSFLGS_IN_PRQ_MASK_OFFSET); @@ -4227,7 +4193,13 @@ evacuate_run_queue(ErtsRunQueue *rq, goto handle_next_proc; } - if (ERTS_PSFLG_BOUND & real_state) { + prio = (int) ERTS_PSFLGS_GET_PRQ_PRIO(state); + to_rq = mp->prio[prio].runq; + + if (!to_rq) + goto handle_next_proc; + + if (!erts_try_change_runq_proc(proc, to_rq)) { /* Bound processes get stuck here... */ proc->next = NULL; if (sbpp->last) @@ -4237,16 +4209,13 @@ evacuate_run_queue(ErtsRunQueue *rq, sbpp->last = proc; } else { - int prio = (int) ERTS_PSFLGS_GET_PRQ_PRIO(state); erts_runq_unlock(rq); - to_rq = mp->prio[prio].runq; - RUNQ_SET_RQ(&proc->run_queue, to_rq); - erts_runq_lock(to_rq); enqueue_process(to_rq, prio, proc); erts_runq_unlock(to_rq); - notify = 1; + + smp_notify_inc_runq(to_rq); erts_runq_lock(rq); } @@ -4254,8 +4223,7 @@ evacuate_run_queue(ErtsRunQueue *rq, handle_next_proc: proc = dequeue_process(rq, prio_q, &state); } - if (notify) - smp_notify_inc_runq(to_rq); + } } @@ -4309,14 +4277,13 @@ try_steal_task_from_victim(ErtsRunQueue *rq, int *rq_lockedp, ErtsRunQueue *vrq, proc = rpq->first; while (proc) { - erts_aint32_t state = erts_atomic32_read_acqb(&proc->state); - if (!(ERTS_PSFLG_BOUND & state)) { + if (erts_try_change_runq_proc(proc, rq)) { + erts_aint32_t state = erts_atomic32_read_acqb(&proc->state); /* Steal process */ int prio = (int) ERTS_PSFLGS_GET_PRQ_PRIO(state); ErtsRunQueueInfo *rqi = &vrq->procs.prio_info[prio]; unqueue_process(vrq, rpq, rqi, prio, prev_proc, proc); erts_runq_unlock(vrq); - RUNQ_SET_RQ(&proc->run_queue, rq); erts_runq_lock(rq); *rq_lockedp = 1; @@ -4341,26 +4308,14 @@ no_procs: if (vrq->ports.start) { ErtsRunQueue *prt_rq; Port *prt = erts_dequeue_port(vrq); - RUNQ_SET_RQ(&prt->run_queue, rq); + erts_set_runq_port(prt, rq); erts_runq_unlock(vrq); - - /* - * The port might terminate while - * we have no lock on it... - */ - prt_rq = erts_port_runq(prt); - if (!prt_rq) - return 0; - else { - if (prt_rq != rq) - erts_exit(ERTS_ABORT_EXIT, - "%s:%d:%s() internal error\n", - __FILE__, __LINE__, __func__); - *rq_lockedp = 1; - erts_enqueue_port(rq, prt); - return !0; - } + if (prt_rq != rq) + ERTS_INTERNAL_ERROR("Unexpected run-queue"); + *rq_lockedp = 1; + erts_enqueue_port(rq, prt); + return !0; } erts_runq_unlock(vrq); @@ -6116,7 +6071,8 @@ make_proxy_proc(Process *prev_proxy, Process *proc, erts_aint32_t prio) { erts_aint32_t state; Process *proxy; - ErtsRunQueue *rq = RUNQ_READ_RQ(&proc->run_queue); + int bound; + ErtsRunQueue *rq = erts_get_runq_proc(proc, &bound); state = (ERTS_PSFLG_PROXY | ERTS_PSFLG_IN_RUNQ @@ -6129,7 +6085,7 @@ make_proxy_proc(Process *prev_proxy, Process *proc, erts_aint32_t prio) proxy = prev_proxy; ASSERT(erts_atomic32_read_nob(&proxy->state) & ERTS_PSFLG_PROXY); erts_atomic32_set_nob(&proxy->state, state); - RUNQ_SET_RQ(&proc->run_queue, rq); + (void) erts_set_runq_proc(proc, rq, &bound); } else { proxy = erts_alloc(ERTS_ALC_T_PROC, sizeof(Process)); @@ -6142,8 +6098,7 @@ make_proxy_proc(Process *prev_proxy, Process *proc, erts_aint32_t prio) } #endif erts_atomic32_init_nob(&proxy->state, state); - erts_atomic_init_nob(&proxy->run_queue, - erts_atomic_read_nob(&proc->run_queue)); + erts_init_runq_proc(proc, rq, bound); } proxy->common.id = proc->common.id; @@ -6334,18 +6289,21 @@ select_enqueue_run_queue(int enqueue, int enq_prio, Process *p, erts_aint32_t st default: { ErtsRunQueue* runq; + int bound; ASSERT(enqueue == ERTS_ENQUEUE_NORMAL_QUEUE || enqueue == -ERTS_ENQUEUE_NORMAL_QUEUE); - runq = erts_get_runq_proc(p); + runq = erts_get_runq_proc(p, &bound); - if (!(ERTS_PSFLG_BOUND & state)) { + if (!bound) { ErtsRunQueue *new_runq = erts_check_emigration_need(runq, enq_prio); - if (new_runq) { - RUNQ_SET_RQ(&p->run_queue, new_runq); - runq = new_runq; - } + if (new_runq) { + if (erts_try_change_runq_proc(p, new_runq)) + runq = new_runq; + else + runq = erts_get_runq_proc(p, NULL); + } } ASSERT(runq); @@ -11462,6 +11420,7 @@ typedef struct { Process *proc; erts_aint32_t state; ErtsRunQueue *run_queue; + int bound; } ErtsEarlyProcInit; static void early_init_process_struct(void *varg, Eterm data) @@ -11472,10 +11431,9 @@ static void early_init_process_struct(void *varg, Eterm data) proc->common.id = make_internal_pid(data); erts_atomic32_init_nob(&proc->dirty_state, 0); proc->dirty_sys_tasks = NULL; + erts_init_runq_proc(proc, arg->run_queue, arg->bound); erts_atomic32_init_relb(&proc->state, arg->state); - RUNQ_SET_RQ(&proc->run_queue, arg->run_queue); - erts_proc_lock_init(proc); /* All locks locked */ } @@ -11484,7 +11442,7 @@ static void early_init_process_struct(void *varg, Eterm data) ** Allocate process and find out where to place next process. */ static Process* -alloc_process(ErtsRunQueue *rq, erts_aint32_t state) +alloc_process(ErtsRunQueue *rq, int bound, erts_aint32_t state) { ErtsEarlyProcInit init_arg; Process *p; @@ -11493,9 +11451,12 @@ alloc_process(ErtsRunQueue *rq, erts_aint32_t state) if (!p) return NULL; + ASSERT(rq); + init_arg.proc = (Process *) p; - init_arg.run_queue = rq; init_arg.state = state; + init_arg.run_queue = rq; + init_arg.bound = bound; ERTS_CT_ASSERT(offsetof(Process,common) == 0); @@ -11530,6 +11491,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). Eterm args, /* Arguments for function (must be well-formed list). */ ErlSpawnOpts* so) /* Options for spawn. */ { + int bound = 0; Uint flags = 0; ErtsRunQueue *rq = NULL; Process *p; @@ -11566,7 +11528,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). ASSERT(0 <= ix && ix < erts_no_run_queues); rq = ERTS_RUNQ_IX(ix); /* Unsupported feature... */ - state |= ERTS_PSFLG_BOUND; + bound = !0; } prio = (erts_aint32_t) so->priority; } @@ -11579,17 +11541,16 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). flags |= F_OFF_HEAP_MSGQ; } else if (so->flags & SPO_ON_HEAP_MSGQ) { - state |= ERTS_PSFLG_ON_HEAP_MSGQ; flags |= F_ON_HEAP_MSGQ; } ASSERT((flags & F_ON_HEAP_MSGQ) || (flags & F_OFF_HEAP_MSGQ)); if (!rq) - rq = erts_get_runq_proc(parent); + rq = erts_get_runq_proc(parent, NULL); - p = alloc_process(rq, state); /* All proc locks are locked by this thread - on success */ + p = alloc_process(rq, bound, state); /* All proc locks are locked by this thread + on success */ if (!p) { erts_send_error_to_logger_str(parent->group_leader, "Too many processes\n"); @@ -11597,11 +11558,6 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). goto error; } - ASSERT((erts_atomic32_read_nob(&p->state) - & ERTS_PSFLG_ON_HEAP_MSGQ) - || (erts_atomic32_read_nob(&p->state) - & ERTS_PSFLG_OFF_HEAP_MSGQ)); - #ifdef SHCOPY_SPAWN arg_size = copy_shared_calculate(args, &info); #else @@ -11976,7 +11932,7 @@ void erts_init_empty_process(Process *p) p->pending_exit.bp = NULL; erts_proc_lock_init(p); erts_proc_unlock(p, ERTS_PROC_LOCKS_ALL); - RUNQ_SET_RQ(&p->run_queue, ERTS_RUNQ_IX(0)); + erts_init_runq_proc(p, ERTS_RUNQ_IX(0), 0); #if !defined(NO_FPE_SIGNALS) || defined(HIPE) p->fp_exception = 0; @@ -12298,7 +12254,7 @@ save_pending_exiter(Process *p, ErtsProcList *plp) ERTS_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); - rq = RUNQ_READ_RQ(&p->run_queue); + rq = erts_get_runq_proc(p, NULL); ASSERT(rq && !ERTS_RUNQ_IX_IS_DIRTY(rq->ix)); if (!plp) @@ -13398,16 +13354,33 @@ stack_element_dump(fmtfn_t to, void *to_arg, Eterm* sp, int yreg) return yreg; } +static void print_current_process_info(fmtfn_t, void *to_arg, ErtsSchedulerData*); + /* * Print scheduler information */ void -erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) { +erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) +{ int i; erts_aint32_t flg; - Process *p; - erts_print(to, to_arg, "=scheduler:%u\n", esdp->no); + switch (esdp->type) { + case ERTS_SCHED_NORMAL: + erts_print(to, to_arg, "=scheduler:%u\n", esdp->no); + break; + case ERTS_SCHED_DIRTY_CPU: + erts_print(to, to_arg, "=dirty_cpu_scheduler:%u\n", + (esdp->dirty_no + erts_no_schedulers)); + break; + case ERTS_SCHED_DIRTY_IO: + erts_print(to, to_arg, "=dirty_io_scheduler:%u\n", + (esdp->dirty_no + erts_no_schedulers + erts_no_dirty_cpu_schedulers)); + break; + default: + erts_print(to, to_arg, "=unknown_scheduler_type:%u\n", esdp->type); + break; + } flg = erts_atomic32_read_dirty(&esdp->ssi->flags); erts_print(to, to_arg, "Scheduler Sleep Info Flags: "); @@ -13453,10 +13426,24 @@ erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) { } erts_print(to, to_arg, "\n"); - erts_print(to, to_arg, "Current Port: "); - if (esdp->current_port) - erts_print(to, to_arg, "%T", esdp->current_port->common.id); - erts_print(to, to_arg, "\n"); + if (esdp->type == ERTS_SCHED_NORMAL) { + erts_print(to, to_arg, "Current Port: "); + if (esdp->current_port) + erts_print(to, to_arg, "%T", esdp->current_port->common.id); + erts_print(to, to_arg, "\n"); + + erts_print_run_queue_info(to, to_arg, esdp->run_queue); + } + + /* This *MUST* to be the last information in scheduler block */ + print_current_process_info(to, to_arg, esdp); +} + +void erts_print_run_queue_info(fmtfn_t to, void *to_arg, + ErtsRunQueue *run_queue) +{ + erts_aint32_t flg; + int i; for (i = 0; i < ERTS_NO_PROC_PRIO_LEVELS; i++) { erts_print(to, to_arg, "Run Queue "); @@ -13478,12 +13465,12 @@ erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) { break; } erts_print(to, to_arg, "Length: %d\n", - erts_atomic32_read_dirty(&esdp->run_queue->procs.prio_info[i].len)); + erts_atomic32_read_dirty(&run_queue->procs.prio_info[i].len)); } erts_print(to, to_arg, "Run Queue Port Length: %d\n", - erts_atomic32_read_dirty(&esdp->run_queue->ports.info.len)); + erts_atomic32_read_dirty(&run_queue->ports.info.len)); - flg = erts_atomic32_read_dirty(&esdp->run_queue->flags); + flg = erts_atomic32_read_dirty(&run_queue->flags); erts_print(to, to_arg, "Run Queue Flags: "); for (i = 0; i < ERTS_RUNQ_FLG_MAX && flg; i++) { erts_aint32_t chk = (1 << i); @@ -13550,9 +13537,15 @@ erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) { } } erts_print(to, to_arg, "\n"); +} + + +static void print_current_process_info(fmtfn_t to, void *to_arg, + ErtsSchedulerData* esdp) +{ + Process *p = esdp->current_process; + erts_aint32_t flg; - /* This *MUST* to be the last information in scheduler block */ - p = esdp->current_process; erts_print(to, to_arg, "Current Process: "); if (esdp->current_process && !(ERTS_TRACE_FLAGS(p) & F_SENSITIVE)) { flg = erts_atomic32_read_dirty(&p->state); diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 55c020d47b..8581c6a639 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -244,6 +244,9 @@ extern int erts_dio_sched_thread_suggested_stack_size; (erts_aint32_t) (MSK), \ (erts_aint32_t) (FLGS))) +#define ERTS_RUNQ_POINTER_MASK (~((erts_aint_t) 3)) +#define ERTS_RUNQ_BOUND_FLAG ((erts_aint_t) 1) + typedef enum { ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED, ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED, @@ -1168,14 +1171,14 @@ void erts_check_for_holes(Process* p); #define ERTS_PSFLG_RUNNING ERTS_PSFLG_BIT(9) #define ERTS_PSFLG_SUSPENDED ERTS_PSFLG_BIT(10) #define ERTS_PSFLG_GC ERTS_PSFLG_BIT(11) -#define ERTS_PSFLG_BOUND ERTS_PSFLG_BIT(12) +/* #define ERTS_PSFLG_ ERTS_PSFLG_BIT(12) */ #define ERTS_PSFLG_TRAP_EXIT ERTS_PSFLG_BIT(13) #define ERTS_PSFLG_ACTIVE_SYS ERTS_PSFLG_BIT(14) #define ERTS_PSFLG_RUNNING_SYS ERTS_PSFLG_BIT(15) #define ERTS_PSFLG_PROXY ERTS_PSFLG_BIT(16) #define ERTS_PSFLG_DELAYED_SYS ERTS_PSFLG_BIT(17) #define ERTS_PSFLG_OFF_HEAP_MSGQ ERTS_PSFLG_BIT(18) -#define ERTS_PSFLG_ON_HEAP_MSGQ ERTS_PSFLG_BIT(19) +/* #define ERTS_PSFLG_ ERTS_PSFLG_BIT(19) */ #define ERTS_PSFLG_DIRTY_CPU_PROC ERTS_PSFLG_BIT(20) #define ERTS_PSFLG_DIRTY_IO_PROC ERTS_PSFLG_BIT(21) #define ERTS_PSFLG_DIRTY_ACTIVE_SYS ERTS_PSFLG_BIT(22) @@ -1546,6 +1549,7 @@ Uint64 erts_step_proc_interval(void); ErtsProcList *erts_proclist_create(Process *); ErtsProcList *erts_proclist_copy(ErtsProcList *); void erts_proclist_destroy(ErtsProcList *); +void erts_proclist_dump(fmtfn_t to, void *to_arg, ErtsProcList*); ERTS_GLB_INLINE int erts_proclist_same(ErtsProcList *, Process *); ERTS_GLB_INLINE void erts_proclist_store_first(ErtsProcList **, ErtsProcList *); @@ -1792,6 +1796,7 @@ void erts_stack_dump(fmtfn_t to, void *to_arg, Process *); void erts_limited_stack_trace(fmtfn_t to, void *to_arg, Process *); void erts_program_counter_info(fmtfn_t to, void *to_arg, Process *); void erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp); +void erts_print_run_queue_info(fmtfn_t, void *to_arg, ErtsRunQueue*); void erts_dump_extended_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg); void erts_dump_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg); @@ -2167,7 +2172,12 @@ ERTS_GLB_INLINE int erts_is_scheduler_bound(ErtsSchedulerData *esdp); ERTS_GLB_INLINE Process *erts_get_current_process(void); ERTS_GLB_INLINE Eterm erts_get_current_pid(void); ERTS_GLB_INLINE Uint erts_get_scheduler_id(void); -ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_proc(Process *p); +ERTS_GLB_INLINE void erts_init_runq_proc(Process *p, ErtsRunQueue *rq, int bnd); +ERTS_GLB_INLINE ErtsRunQueue *erts_set_runq_proc(Process *p, ErtsRunQueue *rq, int *boundp); +ERTS_GLB_INLINE int erts_try_change_runq_proc(Process *p, ErtsRunQueue *rq); +ERTS_GLB_INLINE ErtsRunQueue *erts_bind_runq_proc(Process *p, int bind); +ERTS_GLB_INLINE int erts_proc_runq_is_bound(Process *p); +ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_proc(Process *p, int *boundp); ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_current(ErtsSchedulerData *esdp); ERTS_GLB_INLINE void erts_runq_lock(ErtsRunQueue *rq); ERTS_GLB_INLINE int erts_runq_trylock(ErtsRunQueue *rq); @@ -2247,11 +2257,144 @@ Uint erts_get_scheduler_id(void) return esdp ? esdp->no : (Uint) 0; } +/** + * Init run-queue of process. + * + * @param p[in,out] Process + * @param rq[in] Run-queue that process will be assigned to + * @param bnd[in,out] If non-zero binds process to run-queue. + */ + +ERTS_GLB_INLINE void +erts_init_runq_proc(Process *p, ErtsRunQueue *rq, int bnd) +{ + erts_aint_t rqint = (erts_aint_t) rq; + if (bnd) + rqint |= ERTS_RUNQ_BOUND_FLAG; + erts_atomic_init_nob(&p->run_queue, rqint); +} + +/** + * Forcibly set run-queue of process. + * + * @param p[in,out] Process + * @param rq[in] Run-queue that process will be assigned to + * @param bndp[in,out] Pointer to integer. On input non-zero + * value causes the process to be bound to + * the run-queue. On output, indicating + * wether process previously was bound or + * not. + * @return Previous run-queue. + */ + +ERTS_GLB_INLINE ErtsRunQueue * +erts_set_runq_proc(Process *p, ErtsRunQueue *rq, int *bndp) +{ + erts_aint_t rqint = (erts_aint_t) rq; + ASSERT(bndp); + ASSERT(rq); + if (*bndp) + rqint |= ERTS_RUNQ_BOUND_FLAG; + rqint = erts_atomic_xchg_nob(&p->run_queue, rqint); + *bndp = (int) (rqint & ERTS_RUNQ_BOUND_FLAG); + return (ErtsRunQueue *) (rqint & ERTS_RUNQ_POINTER_MASK); +} + +/** + * Try to change run-queue assignment of a process. + * + * @param p[in,out] Process + * @param rq[int] Run-queue that process will be assigned to + * @return Non-zero if the run-queue assignment was + * successfully changed. + */ + +ERTS_GLB_INLINE int +erts_try_change_runq_proc(Process *p, ErtsRunQueue *rq) +{ + erts_aint_t old_rqint, new_rqint; + + new_rqint = (erts_aint_t) rq; + old_rqint = (erts_aint_t) erts_atomic_read_nob(&p->run_queue); + while (1) { + erts_aint_t act_rqint; + + if (old_rqint & ERTS_RUNQ_BOUND_FLAG) + return 0; + + act_rqint = erts_atomic_cmpxchg_nob(&p->run_queue, + new_rqint, + old_rqint); + if (act_rqint == old_rqint) + return !0; + } +} + +/** + * + * Bind or unbind process to/from currently used run-queue. + * + * @param p Process + * @param bind Bind if non-zero; otherwise unbind + * @return Pointer to previously bound run-queue, + * or NULL if previously unbound + */ + +ERTS_GLB_INLINE ErtsRunQueue * +erts_bind_runq_proc(Process *p, int bind) +{ + erts_aint_t rqint; + if (bind) + rqint = erts_atomic_read_bor_nob(&p->run_queue, + ERTS_RUNQ_BOUND_FLAG); + else + rqint = erts_atomic_read_band_nob(&p->run_queue, + ~ERTS_RUNQ_BOUND_FLAG); + if (rqint & ERTS_RUNQ_BOUND_FLAG) + return (ErtsRunQueue *) (rqint & ERTS_RUNQ_POINTER_MASK); + else + return NULL; +} + +/** + * Determine wether a process is bound to a run-queue or not. + * + * @return Returns a non-zero value if bound, + * and zero of not bound. + */ + +ERTS_GLB_INLINE int +erts_proc_runq_is_bound(Process *p) +{ + erts_aint_t rqint = erts_atomic_read_nob(&p->run_queue); + return (int) (rqint & ERTS_RUNQ_BOUND_FLAG); +} + +/** + * Set run-queue of process. + * + * @param p[in,out] Process + * @param bndp[out] Pointer to integer. If non-NULL pointer, + * the integer will be set to a non-zero + * value if the process is bound to the + * run-queue. + * @return Pointer to the normal run-queue that + * the process currently is assigend to. + * A process is always assigned to a + * normal run-queue. + */ + ERTS_GLB_INLINE ErtsRunQueue * -erts_get_runq_proc(Process *p) +erts_get_runq_proc(Process *p, int *bndp) { - ASSERT(ERTS_AINT_NULL != erts_atomic_read_nob(&p->run_queue)); - return (ErtsRunQueue *) erts_atomic_read_nob(&p->run_queue); + erts_aint_t rqint = erts_atomic_read_nob(&p->run_queue); + ErtsRunQueue *rq; + if (bndp) + *bndp = (int) (rqint & ERTS_RUNQ_BOUND_FLAG); + rqint &= ERTS_RUNQ_POINTER_MASK; + rq = (ErtsRunQueue *) rqint; + ASSERT(rq); + return rq; } ERTS_GLB_INLINE ErtsRunQueue * diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c index f562fc961b..05e7bcdea2 100644 --- a/erts/emulator/beam/erl_process_dump.c +++ b/erts/emulator/beam/erl_process_dump.c @@ -78,8 +78,17 @@ erts_deep_process_dump(fmtfn_t to, void *to_arg) Process *p = erts_pix2proc(i); if (p && p->i != ENULL) { erts_aint32_t state = erts_atomic32_read_acqb(&p->state); - if (!(state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_GC))) - dump_process_info(to, to_arg, p); + if (state & ERTS_PSFLG_EXITING) + continue; + if (state & ERTS_PSFLG_GC) { + ErtsSchedulerData *sdp = erts_get_scheduler_data(); + if (!sdp || p != sdp->current_process) + continue; + + /* We want to dump the garbing process that caused the dump */ + } + + dump_process_info(to, to_arg, p); } } @@ -135,9 +144,12 @@ dump_process_info(fmtfn_t to, void *to_arg, Process *p) ErtsMessage* mp; int yreg = -1; + if (ERTS_TRACE_FLAGS(p) & F_SENSITIVE) + return; + ERTS_MSGQ_MV_INQ2PRIVQ(p); - if ((ERTS_TRACE_FLAGS(p) & F_SENSITIVE) == 0 && p->msg.first) { + if (p->msg.first) { erts_print(to, to_arg, "=proc_messages:%T\n", p->common.id); for (mp = p->msg.first; mp != NULL; mp = mp->next) { Eterm mesg = ERL_MESSAGE_TERM(mp); @@ -152,38 +164,34 @@ dump_process_info(fmtfn_t to, void *to_arg, Process *p) } } - if ((ERTS_TRACE_FLAGS(p) & F_SENSITIVE) == 0) { - if (p->dictionary) { - erts_print(to, to_arg, "=proc_dictionary:%T\n", p->common.id); - erts_deep_dictionary_dump(to, to_arg, - p->dictionary, dump_element_nl); - } + if (p->dictionary) { + erts_print(to, to_arg, "=proc_dictionary:%T\n", p->common.id); + erts_deep_dictionary_dump(to, to_arg, + p->dictionary, dump_element_nl); } - if ((ERTS_TRACE_FLAGS(p) & F_SENSITIVE) == 0) { - erts_print(to, to_arg, "=proc_stack:%T\n", p->common.id); - for (sp = p->stop; sp < STACK_START(p); sp++) { - yreg = stack_element_dump(to, to_arg, sp, yreg); - } + erts_print(to, to_arg, "=proc_stack:%T\n", p->common.id); + for (sp = p->stop; sp < STACK_START(p); sp++) { + yreg = stack_element_dump(to, to_arg, sp, yreg); + } - erts_print(to, to_arg, "=proc_heap:%T\n", p->common.id); - for (sp = p->stop; sp < STACK_START(p); sp++) { - Eterm term = *sp; - - if (!is_catch(term) && !is_CP(term)) { - heap_dump(to, to_arg, term); - } - } - for (mp = p->msg.first; mp != NULL; mp = mp->next) { - Eterm mesg = ERL_MESSAGE_TERM(mp); - if (is_value(mesg)) - heap_dump(to, to_arg, mesg); - mesg = ERL_MESSAGE_TOKEN(mp); - heap_dump(to, to_arg, mesg); - } - if (p->dictionary) { - erts_deep_dictionary_dump(to, to_arg, p->dictionary, heap_dump); - } + erts_print(to, to_arg, "=proc_heap:%T\n", p->common.id); + for (sp = p->stop; sp < STACK_START(p); sp++) { + Eterm term = *sp; + + if (!is_catch(term) && !is_CP(term)) { + heap_dump(to, to_arg, term); + } + } + for (mp = p->msg.first; mp != NULL; mp = mp->next) { + Eterm mesg = ERL_MESSAGE_TERM(mp); + if (is_value(mesg)) + heap_dump(to, to_arg, mesg); + mesg = ERL_MESSAGE_TOKEN(mp); + heap_dump(to, to_arg, mesg); + } + if (p->dictionary) { + erts_deep_dictionary_dump(to, to_arg, p->dictionary, heap_dump); } } @@ -1001,8 +1009,6 @@ erts_dump_extended_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg) erts_print(to, to_arg, "SUSPENDED"); break; case ERTS_PSFLG_GC: erts_print(to, to_arg, "GC"); break; - case ERTS_PSFLG_BOUND: - erts_print(to, to_arg, "BOUND"); break; case ERTS_PSFLG_TRAP_EXIT: erts_print(to, to_arg, "TRAP_EXIT"); break; case ERTS_PSFLG_ACTIVE_SYS: @@ -1015,8 +1021,6 @@ erts_dump_extended_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg) erts_print(to, to_arg, "DELAYED_SYS"); break; case ERTS_PSFLG_OFF_HEAP_MSGQ: erts_print(to, to_arg, "OFF_HEAP_MSGQ"); break; - case ERTS_PSFLG_ON_HEAP_MSGQ: - erts_print(to, to_arg, "ON_HEAP_MSGQ"); break; case ERTS_PSFLG_DIRTY_CPU_PROC: erts_print(to, to_arg, "DIRTY_CPU_PROC"); break; case ERTS_PSFLG_DIRTY_IO_PROC: diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index 4b996d8fc2..d2495479ab 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -2783,24 +2783,28 @@ is_tracer_enabled(Process* c_p, ErtsProcLocks c_p_locks, ASSERT(0); } - /* Only remove tracer on self() and ports */ + /* Only remove tracer on (self() or ports) AND we are on a normal scheduler */ if (is_internal_port(t_p->id) || (c_p && c_p->common.id == t_p->id)) { + ErtsSchedulerData *esdp = erts_get_scheduler_data(); ErtsProcLocks c_p_xlocks = 0; - if (is_internal_pid(t_p->id)) { - ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) & ERTS_PROC_LOCK_MAIN); - if (c_p_locks != ERTS_PROC_LOCKS_ALL) { - c_p_xlocks = ~c_p_locks & ERTS_PROC_LOCKS_ALL; - if (erts_proc_trylock(c_p, c_p_xlocks) == EBUSY) { - erts_proc_unlock(c_p, c_p_locks & ~ERTS_PROC_LOCK_MAIN); - erts_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); + if (esdp && !ERTS_SCHEDULER_IS_DIRTY(esdp)) { + if (is_internal_pid(t_p->id)) { + ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) & ERTS_PROC_LOCK_MAIN); + if (c_p_locks != ERTS_PROC_LOCKS_ALL) { + c_p_xlocks = ~c_p_locks & ERTS_PROC_LOCKS_ALL; + if (erts_proc_trylock(c_p, c_p_xlocks) == EBUSY) { + erts_proc_unlock(c_p, c_p_locks & ~ERTS_PROC_LOCK_MAIN); + erts_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); + } } } - } - erts_tracer_replace(t_p, erts_tracer_nil); - t_p->trace_flags &= ~TRACEE_FLAGS; - if (c_p_xlocks) - erts_proc_unlock(c_p, c_p_xlocks); + erts_tracer_replace(t_p, erts_tracer_nil); + t_p->trace_flags &= ~TRACEE_FLAGS; + + if (c_p_xlocks) + erts_proc_unlock(c_p, c_p_xlocks); + } } return 0; diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index ebe673cd86..d757651933 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1066,7 +1066,7 @@ Eterm copy_struct_x(Eterm, Uint, Eterm**, ErlOffHeap*, Uint*, erts_literal_area_ #define copy_struct_litopt(Obj,Sz,HPP,OH,LitArea) \ copy_struct_x(Obj,Sz,HPP,OH,NULL,LitArea) -Eterm copy_shallow(Eterm*, Uint, Eterm**, ErlOffHeap*); +Eterm copy_shallow(Eterm* ERTS_RESTRICT, Uint, Eterm**, ErlOffHeap*); void erts_move_multi_frags(Eterm** hpp, ErlOffHeap*, ErlHeapFragment* first, Eterm* refs, unsigned nrefs, int literals); diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 2c1b7871c4..3e8f6263bb 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -298,7 +298,6 @@ static Port *create_port(char *name, erts_aint32_t state = ERTS_PORT_SFLG_CONNECTED; erts_aint32_t x_pts_flgs = 0; - ErtsRunQueue *runq; if (!driver_lock) { /* Align size for mutex following port struct */ port_size = size = ERTS_ALC_DATA_ALIGN_SIZE(sizeof(Port)); @@ -347,11 +346,16 @@ static Port *create_port(char *name, p += sizeof(erts_mtx_t); state |= ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK; } - if (erts_get_scheduler_data()) - runq = erts_get_runq_current(NULL); - else - runq = ERTS_RUNQ_IX(0); - erts_atomic_set_nob(&prt->run_queue, (erts_aint_t) runq); + + { + ErtsRunQueue *runq; + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + if (esdp) + runq = erts_get_runq_current(esdp); + else + runq = ERTS_RUNQ_IX(0); + erts_init_runq_port(prt, runq); + } prt->xports = NULL; @@ -5075,6 +5079,93 @@ static void prt_one_lnk(ErtsLink *lnk, void *vprtd) erts_print(prtd->to, prtd->arg, "%T", lnk->pid); } +static void dump_port_state(fmtfn_t to, void *arg, erts_aint32_t state) +{ + erts_aint32_t rest; + int unknown = 0; + char delim = ' '; + + erts_print(to, arg, "State:"); + + rest = state; + while (rest) { + erts_aint32_t chk = (rest ^ (rest-1)) & rest; /* lowest set bit */ + char* s; + + rest &= ~chk; + switch (chk) { + case ERTS_PORT_SFLG_CONNECTED: s = "CONNECTED"; break; + case ERTS_PORT_SFLG_EXITING: s = "EXITING"; break; + case ERTS_PORT_SFLG_DISTRIBUTION: s = "DISTR"; break; + case ERTS_PORT_SFLG_BINARY_IO: s = "BINARY_IO"; break; + case ERTS_PORT_SFLG_SOFT_EOF: s = "SOFT_EOF"; break; + case ERTS_PORT_SFLG_CLOSING: s = "CLOSING"; break; + case ERTS_PORT_SFLG_SEND_CLOSED: s = "SEND_CLOSED"; break; + case ERTS_PORT_SFLG_LINEBUF_IO: s = "LINEBUF_IO"; break; + case ERTS_PORT_SFLG_FREE: s = "FREE"; break; + case ERTS_PORT_SFLG_INITIALIZING: s = "INITIALIZING"; break; + case ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK: s = "PORT_LOCK"; break; + case ERTS_PORT_SFLG_INVALID: s = "INVALID"; break; + case ERTS_PORT_SFLG_HALT: s = "HALT"; break; +#ifdef DEBUG + case ERTS_PORT_SFLG_PORT_DEBUG: s = "DEBUG"; break; +#endif + default: + unknown = 1; + continue; + } + erts_print(to, arg, "%c%s", delim, s); + delim = '|'; + } + if (unknown || !state) + erts_print(to, arg, "%c0x%x\n", delim, state); + else + erts_print(to, arg, "\n"); +} + +static void dump_port_task_flags(fmtfn_t to, void *arg, Port* p) +{ + erts_aint32_t flags = erts_atomic32_read_nob(&p->sched.flags); + erts_aint32_t unknown = 0; + char delim = ' '; + + if (!flags) + return; + + erts_print(to, arg, "Task Flags:"); + + while (flags) { + erts_aint32_t chk = (flags ^ (flags-1)) & flags; /* lowest set bit */ + char* s; + + flags &= ~chk; + switch (chk) { + case ERTS_PTS_FLG_IN_RUNQ: s = "IN_RUNQ"; break; + case ERTS_PTS_FLG_EXEC: s = "EXEC"; break; + case ERTS_PTS_FLG_HAVE_TASKS: s = "HAVE_TASKS"; break; + case ERTS_PTS_FLG_EXIT: s = "EXIT"; break; + case ERTS_PTS_FLG_BUSY_PORT: s = "BUSY_PORT"; break; + case ERTS_PTS_FLG_BUSY_PORT_Q: s = "BUSY_Q"; break; + case ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q: s = "CHK_UNSET_BUSY_Q"; break; + case ERTS_PTS_FLG_HAVE_BUSY_TASKS: s = "BUSY_TASKS"; break; + case ERTS_PTS_FLG_HAVE_NS_TASKS: s = "NS_TASKS"; break; + case ERTS_PTS_FLG_PARALLELISM: s = "PARALLELISM"; break; + case ERTS_PTS_FLG_FORCE_SCHED: s = "FORCE_SCHED"; break; + case ERTS_PTS_FLG_EXITING: s = "EXITING"; break; + case ERTS_PTS_FLG_EXEC_IMM: s = "EXEC_IMM"; break; + default: + unknown |= chk; + continue; + } + erts_print(to, arg, "%c%s", delim, s); + delim = '|'; + } + if (unknown) + erts_print(to, arg, "%cUNKNOWN(0x%x)\n", delim, unknown); + else + erts_print(to, arg, "\n"); +} + void print_port_info(Port *p, fmtfn_t to, void *arg) { @@ -5084,6 +5175,8 @@ print_port_info(Port *p, fmtfn_t to, void *arg) return; erts_print(to, arg, "=port:%T\n", p->common.id); + dump_port_state(to, arg, state); + dump_port_task_flags(to, arg, p); erts_print(to, arg, "Slot: %d\n", internal_port_index(p->common.id)); if (state & ERTS_PORT_SFLG_CONNECTED) { erts_print(to, arg, "Connected: %T", ERTS_PORT_GET_CONNECTED(p)); @@ -5106,6 +5199,10 @@ print_port_info(Port *p, fmtfn_t to, void *arg) erts_doforall_monitors(ERTS_P_MONITORS(p), &prt_one_monitor, &prtd); erts_print(to, arg, "\n"); } + if (p->suspended) { + erts_print(to, arg, "Suspended: "); + erts_proclist_dump(to, arg, p->suspended); + } if (p->common.u.alive.reg != NULL) erts_print(to, arg, "Registered as: %T\n", p->common.u.alive.reg->name); @@ -5123,6 +5220,14 @@ print_port_info(Port *p, fmtfn_t to, void *arg) } else { erts_print(to, arg, "Port controls linked-in driver: %s\n",p->name); } + erts_print(to, arg, "Input: %beu\n", p->bytes_in); + erts_print(to, arg, "Output: %beu\n", p->bytes_out); + erts_print(to, arg, "Queue: %beu\n", erts_ioq_size(&p->ioq)); + { + Eterm port_data = erts_port_data_read(p); + if (port_data != am_undefined) + erts_print(to, arg, "Port Data: %T\n", port_data); + } } void diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index fe9f1c7606..4bf60619ba 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -3152,6 +3152,9 @@ tailrecur_ne: int cmp; byte* a_ptr; byte* b_ptr; + if (eq_only && a_size != b_size) { + RETURN_NEQ(a_size - b_size); + } ERTS_GET_BINARY_BYTES(a, a_ptr, a_bitoffs, a_bitsize); ERTS_GET_BINARY_BYTES(b, b_ptr, b_bitoffs, b_bitsize); if ((a_bitsize | b_bitsize | a_bitoffs | b_bitoffs) == 0) { |