diff options
Diffstat (limited to 'erts/emulator/beam')
28 files changed, 373 insertions, 262 deletions
diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index f9a2f3e33e..190e7817dc 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -321,6 +321,7 @@ atom ldflags atom Le='=<' atom lf atom line +atom line_delimiter atom line_length atom linked_in_driver atom links diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index c774a70d4c..90985e4f53 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -79,7 +79,7 @@ erts_debug_breakpoint_2(BIF_ALIST_2) { Process* p = BIF_P; Eterm MFA = BIF_ARG_1; - Eterm bool = BIF_ARG_2; + Eterm boolean = BIF_ARG_2; Eterm* tp; Eterm mfa[3]; int i; @@ -87,7 +87,7 @@ erts_debug_breakpoint_2(BIF_ALIST_2) Eterm res; BpFunctions f; - if (bool != am_true && bool != am_false) + if (boolean != am_true && boolean != am_false) goto error; if (is_not_tuple(MFA)) { @@ -124,7 +124,7 @@ erts_debug_breakpoint_2(BIF_ALIST_2) erts_smp_thr_progress_block(); erts_bp_match_functions(&f, mfa, specified); - if (bool == am_true) { + if (boolean == am_true) { erts_set_debug_break(&f); erts_install_breakpoints(&f); erts_commit_staged_bp(); diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 4e3a1cef69..a8cc19ee1f 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -616,7 +616,7 @@ erts_queue_monitor_message(Process *p, } static BIF_RETTYPE -local_pid_monitor(Process *p, Eterm target, Eterm mon_ref, int bool) +local_pid_monitor(Process *p, Eterm target, Eterm mon_ref, int boolean) { BIF_RETTYPE ret; Process *rp; @@ -634,7 +634,7 @@ local_pid_monitor(Process *p, Eterm target, Eterm mon_ref, int bool) if (!rp) { erts_smp_proc_unlock(p, ERTS_PROC_LOCK_LINK); p_locks &= ~ERTS_PROC_LOCK_LINK; - if (bool) + if (boolean) ret = am_false; else erts_queue_monitor_message(p, &p_locks, @@ -643,7 +643,7 @@ local_pid_monitor(Process *p, Eterm target, Eterm mon_ref, int bool) else { ASSERT(rp != p); - if (bool) + if (boolean) ret = am_true; erts_add_monitor(&ERTS_P_MONITORS(p), MON_ORIGIN, mon_ref, target, NIL); @@ -4231,6 +4231,7 @@ BIF_RETTYPE list_to_pid_1(BIF_ALIST_1) goto bad; enp = erts_find_or_insert_node(dep->sysname, dep->creation); + ASSERT(enp != erts_this_node); etp = (ExternalThing *) HAlloc(BIF_P, EXTERNAL_THING_HEAD_SIZE + 1); etp->header = make_external_pid_header(1); diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index 23897a49ae..7be2b77a3b 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -45,6 +45,8 @@ #include "erl_thr_progress.h" #include "dtrace-wrapper.h" +#define DIST_CTL_DEFAULT_SIZE 64 + /* Turn this on to get printouts of all distribution messages * which go on the line */ @@ -66,9 +68,13 @@ static void bw(byte *buf, ErlDrvSizeT sz) static void dist_msg_dbg(ErtsDistExternal *edep, char *what, byte *buf, int sz) { + ErtsHeapFactory factory; + DeclareTmpHeapNoproc(ctl_default,DIST_CTL_DEFAULT_SIZE); + Eterm* ctl = ctl_default; byte *extp = edep->extp; Eterm msg; - Sint size = erts_decode_dist_ext_size(edep); + Sint ctl_len; + Sint size = ctl_len = erts_decode_dist_ext_size(edep); if (size < 0) { erts_fprintf(stderr, "DIST MSG DEBUG: erts_decode_dist_ext_size(%s) failed:\n", @@ -76,10 +82,9 @@ dist_msg_dbg(ErtsDistExternal *edep, char *what, byte *buf, int sz) bw(buf, sz); } else { - Eterm *hp; ErlHeapFragment *mbuf = new_message_buffer(size); - hp = mbuf->mem; - msg = erts_decode_dist_ext(&hp, &mbuf->off_heap, edep); + erts_factory_static_init(&factory, ctl, ctl_len, &mbuf->off_heap); + msg = erts_decode_dist_ext(&factory, edep); if (is_value(msg)) erts_fprintf(stderr, " %s: %T\n", what, msg); else { @@ -1136,7 +1141,6 @@ int erts_net_message(Port *prt, byte *buf, ErlDrvSizeT len) { -#define DIST_CTL_DEFAULT_SIZE 64 ErtsDistExternal ede; byte *t; Sint ctl_len; @@ -1149,7 +1153,6 @@ int erts_net_message(Port *prt, Process* rp; DeclareTmpHeapNoproc(ctl_default,DIST_CTL_DEFAULT_SIZE); Eterm* ctl = ctl_default; - ErlOffHeap off_heap; ErtsHeapFactory factory; Eterm* hp; Sint type; @@ -1164,9 +1167,6 @@ int erts_net_message(Port *prt, #endif UseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE); - /* Thanks to Luke Gorrie */ - off_heap.first = NULL; - off_heap.overhead = 0; ERTS_SMP_CHK_NO_PROC_LOCKS; @@ -1227,15 +1227,15 @@ int erts_net_message(Port *prt, } hp = ctl; - erts_factory_static_init(&factory, ctl, ctl_len, &off_heap); + erts_factory_tmp_init(&factory, ctl, ctl_len, ERTS_ALC_T_DCTRL_BUF); arg = erts_decode_dist_ext(&factory, &ede); if (is_non_value(arg)) { #ifdef ERTS_DIST_MSG_DBG - erts_fprintf(stderr, "DIST MSG DEBUG: erts_dist_ext_size(CTL) failed:\n"); + erts_fprintf(stderr, "DIST MSG DEBUG: erts_decode_dist_ext(CTL) failed:\n"); bw(buf, orig_len); #endif PURIFY_MSG("data error"); - goto data_error; + goto decode_error; } ctl_len = t - buf; @@ -1715,7 +1715,7 @@ int erts_net_message(Port *prt, goto invalid_message; } - erts_cleanup_offheap(&off_heap); + erts_factory_close(&factory); if (ctl != ctl_default) { erts_free(ERTS_ALC_T_DCTRL_BUF, (void *) ctl); } @@ -1728,12 +1728,13 @@ int erts_net_message(Port *prt, erts_dsprintf(dsbufp, "Invalid distribution message: %.200T", arg); erts_send_error_to_logger_nogl(dsbufp); } - data_error: +decode_error: PURIFY_MSG("data error"); - erts_cleanup_offheap(&off_heap); + erts_factory_close(&factory); if (ctl != ctl_default) { erts_free(ERTS_ALC_T_DCTRL_BUF, (void *) ctl); } +data_error: UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE); erts_deliver_port_exit(prt, dep->cid, am_killed, 0); ERTS_SMP_CHK_NO_PROC_LOCKS; @@ -1790,8 +1791,8 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx) #ifdef ERTS_DIST_MSG_DBG erts_fprintf(stderr, ">>%s CTL: %T\n", ctx->pass_through_size ? "P" : " ", ctx->ctl); - if (is_value(msg)) - erts_fprintf(stderr, " MSG: %T\n", msg); + if (is_value(ctx->msg)) + erts_fprintf(stderr, " MSG: %T\n", ctx->msg); #endif ctx->data_size = ctx->pass_through_size; @@ -2572,7 +2573,9 @@ int distribution_info(int to, void *arg) /* Called by break handler */ } for (dep = erts_not_connected_dist_entries; dep; dep = dep->next) { - info_dist_entry(to, arg, dep, 0, 0); + if (dep != erts_this_dist_entry) { + info_dist_entry(to, arg, dep, 0, 0); + } } return(0); @@ -2650,13 +2653,8 @@ BIF_RETTYPE setnode_2(BIF_ALIST_2) if (!net_kernel) goto error; - /* By setting dist_entry==erts_this_dist_entry and DISTRIBUTION on - net_kernel do_net_exist will be called when net_kernel - is terminated !! */ - (void) ERTS_PROC_SET_DIST_ENTRY(net_kernel, - ERTS_PROC_LOCK_MAIN, - erts_this_dist_entry); - erts_refc_inc(&erts_this_dist_entry->refc, 2); + /* By setting F_DISTRIBUTION on net_kernel, + * do_net_exist will be called when net_kernel is terminated !! */ net_kernel->flags |= F_DISTRIBUTION; if (net_kernel != BIF_P) @@ -3017,11 +3015,11 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1) erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx); - ASSERT(erts_no_of_not_connected_dist_entries >= 0); + ASSERT(erts_no_of_not_connected_dist_entries > 0); ASSERT(erts_no_of_hidden_dist_entries >= 0); ASSERT(erts_no_of_visible_dist_entries >= 0); if(not_connected) - length += erts_no_of_not_connected_dist_entries; + length += (erts_no_of_not_connected_dist_entries - 1); if(hidden) length += erts_no_of_hidden_dist_entries; if(visible) @@ -3043,8 +3041,10 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1) #endif if(not_connected) for(dep = erts_not_connected_dist_entries; dep; dep = dep->next) { - result = CONS(hp, dep->sysname, result); - hp += 2; + if (dep != erts_this_dist_entry) { + result = CONS(hp, dep->sysname, result); + hp += 2; + } } if(hidden) for(dep = erts_hidden_dist_entries; dep; dep = dep->next) { diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 55c164bf11..c3f4fe5a63 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -134,6 +134,7 @@ static ErtsAllocatorState_t binary_alloc_state; static ErtsAllocatorState_t ets_alloc_state; static ErtsAllocatorState_t driver_alloc_state; static ErtsAllocatorState_t fix_alloc_state; +static ErtsAllocatorState_t test_alloc_state; typedef struct { erts_smp_atomic32_t refc; @@ -233,6 +234,7 @@ typedef struct { struct au_init std_low_alloc; struct au_init ll_low_alloc; #endif + struct au_init test_alloc; } erts_alc_hndl_args_init_t; #define ERTS_AU_INIT__ {0, 0, 1, GOODFIT, DEFAULT_ALLCTR_INIT, {1,1,1,1}} @@ -432,6 +434,33 @@ set_default_fix_alloc_opts(struct au_init *ip, ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL; } +static void +set_default_test_alloc_opts(struct au_init *ip) +{ + SET_DEFAULT_ALLOC_OPTS(ip); + ip->enable = 0; /* Disabled by default */ + ip->thr_spec = -1 * erts_no_schedulers; + ip->atype = AOFIRSTFIT; + ip->init.aoff.flavor = AOFF_BF; + ip->init.util.name_prefix = "test_"; + ip->init.util.alloc_no = ERTS_ALC_A_TEST; + ip->init.util.mmbcs = 0; /* Main carrier size */ + ip->init.util.ts = ERTS_ALC_MTA_TEST; + ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL; + + /* Use a constant minimal MBC size */ +#if ERTS_SA_MB_CARRIERS + ip->init.util.smbcs = ERTS_SACRR_UNIT_SZ; + ip->init.util.lmbcs = ERTS_SACRR_UNIT_SZ; + ip->init.util.sbct = ERTS_SACRR_UNIT_SZ; +#else + ip->init.util.smbcs = 1 << 12; + ip->init.util.lmbcs = 1 << 12; + ip->init.util.sbct = 1 << 12; +#endif +} + + #ifdef ERTS_SMP static void @@ -613,6 +642,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) set_default_driver_alloc_opts(&init.driver_alloc); set_default_fix_alloc_opts(&init.fix_alloc, fix_type_sizes); + set_default_test_alloc_opts(&init.test_alloc); if (argc && argv) handle_args(argc, argv, &init); @@ -772,6 +802,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) set_au_allocator(ERTS_ALC_A_ETS, &init.ets_alloc, ncpu); set_au_allocator(ERTS_ALC_A_DRIVER, &init.driver_alloc, ncpu); set_au_allocator(ERTS_ALC_A_FIXED_SIZE, &init.fix_alloc, ncpu); + set_au_allocator(ERTS_ALC_A_TEST, &init.test_alloc, ncpu); for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { if (!erts_allctrs[i].alloc) @@ -833,6 +864,10 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) &init.fix_alloc, &fix_alloc_state); + start_au_allocator(ERTS_ALC_A_TEST, + &init.test_alloc, + &test_alloc_state); + erts_mtrace_install_wrapper_functions(); extra_block_size += erts_instr_init(init.instr.stat, init.instr.map); @@ -1418,6 +1453,7 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) &init->fix_alloc, &init->sl_alloc, &init->temp_alloc + /* test_alloc not affected by +Mea??? or +Mu??? */ }; int aui_sz = (int) sizeof(aui)/sizeof(aui[0]); char *arg; @@ -1508,6 +1544,9 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) case 'T': handle_au_arg(&init->temp_alloc, &argv[i][3], argv, &i, 0); break; + case 'Z': + handle_au_arg(&init->test_alloc, &argv[i][3], argv, &i, 0); + break; case 'Y': { /* sys_alloc */ if (has_prefix("tt", param+2)) { /* set trim threshold */ @@ -2222,11 +2261,12 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) return am_badarg; } - /* All alloc_util allocators *have* to be enabled */ + /* All alloc_util allocators *have* to be enabled, except test_alloc */ for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++) { switch (ai) { case ERTS_ALC_A_SYSTEM: + case ERTS_ALC_A_TEST: break; default: if (!erts_allctrs_info[ai].enabled @@ -2267,6 +2307,8 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) * contain any allocated memory. */ continue; + case ERTS_ALC_A_TEST: + continue; case ERTS_ALC_A_EHEAP: save = &size.processes; break; @@ -2675,14 +2717,17 @@ erts_alloc_util_allocators(void *proc) /* * Currently all allocators except sys_alloc are * alloc_util allocators. + * Also hide test_alloc which is disabled by default + * and only intended for our own testing. */ - sz = ((ERTS_ALC_A_MAX + 1 - ERTS_ALC_A_MIN) - 1)*2; + sz = ((ERTS_ALC_A_MAX + 1 - ERTS_ALC_A_MIN) - 2)*2; ASSERT(sz > 0); hp = HAlloc((Process *) proc, sz); res = NIL; for (i = ERTS_ALC_A_MAX; i >= ERTS_ALC_A_MIN; i--) { switch (i) { case ERTS_ALC_A_SYSTEM: + case ERTS_ALC_A_TEST: break; default: { char *alc_str = (char *) ERTS_ALC_A2AD(i); @@ -3566,6 +3611,41 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) #else case 0xf13: return (UWord) 0; #endif + case 0xf14: return (UWord) erts_alloc(ERTS_ALC_T_TEST, (Uint)a1); + + case 0xf15: erts_free(ERTS_ALC_T_TEST, (void*)a1); return 0; + + case 0xf16: { + Uint extra_hdr_sz = UNIT_CEILING((Uint)a1); + ErtsAllocatorThrSpec_t* ts = &erts_allctr_thr_spec[ERTS_ALC_A_TEST]; + Uint offset = ts->allctr[0]->mbc_header_size; + void* orig_creating_mbc = ts->allctr[0]->creating_mbc; + void* orig_destroying_mbc = ts->allctr[0]->destroying_mbc; + void* new_creating_mbc = *(void**)a2; /* inout arg */ + void* new_destroying_mbc = *(void**)a3; /* inout arg */ + int i; + + for (i=0; i < ts->size; i++) { + Allctr_t* ap = ts->allctr[i]; + if (ap->mbc_header_size != offset + || ap->creating_mbc != orig_creating_mbc + || ap->destroying_mbc != orig_destroying_mbc + || ap->mbc_list.first != NULL) + return -1; + } + for (i=0; i < ts->size; i++) { + ts->allctr[i]->mbc_header_size += extra_hdr_sz; + ts->allctr[i]->creating_mbc = new_creating_mbc; + ts->allctr[i]->destroying_mbc = new_destroying_mbc; + } + *(void**)a2 = orig_creating_mbc; + *(void**)a3 = orig_destroying_mbc; + return offset; + } + case 0xf17: { + ErtsAllocatorThrSpec_t* ts = &erts_allctr_thr_spec[ERTS_ALC_A_TEST]; + return ts->allctr[0]->largest_mbc_size; + } default: break; } diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index b1d511ab78..1ecebdeb07 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -112,7 +112,7 @@ allocator STANDARD_LOW false std_low_alloc allocator BINARY true binary_alloc allocator DRIVER true driver_alloc - +allocator TEST true test_alloc # --- Class declarations ----------------------------------------------------- # @@ -456,4 +456,7 @@ type CON_VPRINTF_BUF TEMPORARY SYSTEM con_vprintf_buf +endif +# This type should only be used for test +type TEST TEST SYSTEM testing + # ---------------------------------------------------------------------------- diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 236ee35d18..8229a15824 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -1437,6 +1437,16 @@ erts_alcu_fix_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) static void dealloc_carrier(Allctr_t *allctr, Carrier_t *crr, int superaligned); +static ERTS_INLINE void +dealloc_mbc(Allctr_t *allctr, Carrier_t *crr) +{ + ASSERT(IS_MB_CARRIER(crr)); + if (allctr->destroying_mbc) + allctr->destroying_mbc(allctr, crr); + + dealloc_carrier(allctr, crr, 1); +} + #ifdef ERTS_SMP static ERTS_INLINE Allctr_t* @@ -3149,7 +3159,7 @@ cpool_fetch(Allctr_t *allctr, UWord size) cpool_entrance = sentinel; cpdp = cpool_aint2cpd(cpool_read(&cpool_entrance->prev)); if (cpdp == sentinel) - return NULL; + goto check_dc_list; } has_passed_sentinel = 0; @@ -3160,18 +3170,18 @@ cpool_fetch(Allctr_t *allctr, UWord size) if (cpool_entrance == sentinel) { cpdp = cpool_aint2cpd(cpool_read(&cpdp->prev)); if (cpdp == sentinel) - return NULL; + break; } i = 0; /* Last one to inspect */ } else if (cpdp == sentinel) { if (has_passed_sentinel) { /* We been here before. cpool_entrance must have been removed */ - return NULL; + break; } cpdp = cpool_aint2cpd(cpool_read(&cpdp->prev)); if (cpdp == sentinel) - return NULL; + break; has_passed_sentinel = 1; } crr = (Carrier_t *)(((char *)cpdp) - offsetof(Carrier_t, cpool)); @@ -3195,10 +3205,12 @@ cpool_fetch(Allctr_t *allctr, UWord size) return NULL; } +check_dc_list: /* Last; check our own pending dealloc carrier list... */ crr = allctr->cpool.dc_list.last; while (crr) { if (erts_atomic_read_nob(&crr->cpool.max_size) >= size) { + Block_t* blk; unlink_carrier(&allctr->cpool.dc_list, crr); #ifdef ERTS_ALC_CPOOL_DEBUG ERTS_ALC_CPOOL_ASSERT(erts_smp_atomic_xchg_nob(&crr->allctr, @@ -3207,6 +3219,9 @@ cpool_fetch(Allctr_t *allctr, UWord size) #else erts_smp_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr)); #endif + blk = MBC_TO_FIRST_BLK(allctr, crr); + ASSERT(FBLK_TO_MBC(blk) == crr); + allctr->link_free_block(allctr, blk); return crr; } crr = crr->prev; @@ -3237,7 +3252,7 @@ check_pending_dealloc_carrier(Allctr_t *allctr, dcrr = crr; crr = crr->next; - dealloc_carrier(allctr, dcrr, 1); + dealloc_mbc(allctr, dcrr); i++; } while (crr && i < ERTS_ALC_MAX_DEALLOC_CARRIER); @@ -3268,18 +3283,20 @@ static void schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr) { Allctr_t *orig_allctr; + Block_t *blk; int check_pending_dealloc; erts_aint_t max_size; + ASSERT(IS_MB_CARRIER(crr)); + if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) { - dealloc_carrier(allctr, crr, 1); + dealloc_mbc(allctr, crr); return; } orig_allctr = crr->cpool.orig_allctr; if (allctr != orig_allctr) { - Block_t *blk = MBC_TO_FIRST_BLK(allctr, crr); int cinit = orig_allctr->dd.ix - allctr->dd.ix; /* @@ -3296,6 +3313,7 @@ schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr) * since the block is an mbc block that is free and last * in the carrier. */ + blk = MBC_TO_FIRST_BLK(allctr, crr); ERTS_ALC_CPOOL_ASSERT(IS_FREE_LAST_MBC_BLK(blk)); ERTS_ALC_CPOOL_ASSERT(IS_MBC_FIRST_ABLK(allctr, blk)); @@ -3315,11 +3333,13 @@ schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr) if (crr->cpool.thr_prgr == ERTS_THR_PRGR_INVALID || erts_thr_progress_has_reached(crr->cpool.thr_prgr)) { - dealloc_carrier(allctr, crr, 1); + dealloc_mbc(allctr, crr); return; } - max_size = (erts_aint_t) allctr->largest_fblk_in_mbc(allctr, crr); + blk = MBC_TO_FIRST_BLK(allctr, crr); + ASSERT(IS_FREE_LAST_MBC_BLK(blk)); + max_size = (erts_aint_t) MBC_FBLK_SZ(blk); erts_atomic_set_nob(&crr->cpool.max_size, max_size); crr->next = NULL; @@ -3894,9 +3914,6 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp) } #endif - if (allctr->destroying_mbc) - (*allctr->destroying_mbc)(allctr, crr); - #ifdef ERTS_SMP if (busy_pcrr_pp && *busy_pcrr_pp) { ERTS_ALC_CPOOL_ASSERT(*busy_pcrr_pp == crr); @@ -3920,12 +3937,15 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp) else #endif STAT_SYS_ALLOC_MBC_FREE(allctr, crr_sz); + + if (allctr->remove_mbc) + allctr->remove_mbc(allctr, crr); } #ifdef ERTS_SMP schedule_dealloc_carrier(allctr, crr); #else - dealloc_carrier(allctr, crr, 1); + dealloc_mbc(allctr, crr); #endif } } @@ -6054,6 +6074,16 @@ erts_alcu_test(UWord op, UWord a1, UWord a2) case 0x023: return (UWord) 0; case 0x024: return (UWord) 0; #endif + case 0x025: /* UMEM2BLK_TEST*/ +#ifdef DEBUG +# ifdef HARD_DEBUG + return (UWord)UMEM2BLK(a1-3*sizeof(UWord)); +# else + return (UWord)UMEM2BLK(a1-2*sizeof(UWord)); +# endif +#else + return (UWord)UMEM2BLK(a1); +#endif default: ASSERT(0); return ~((UWord) 0); } diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index df1f0aa65a..f4a2ae7ff3 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -277,7 +277,7 @@ typedef struct ErtsDoubleLink_t_ { typedef struct { erts_atomic_t next; erts_atomic_t prev; - Allctr_t *orig_allctr; + Allctr_t *orig_allctr; /* read-only while carrier is alive */ ErtsThrPrgrVal thr_prgr; erts_atomic_t max_size; UWord abandon_limit; diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c index 7c2a5c3323..19420af8ab 100644 --- a/erts/emulator/beam/erl_ao_firstfit_alloc.c +++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c @@ -209,7 +209,9 @@ static Block_t* aoff_get_free_block(Allctr_t *, Uint, Block_t *, Uint); static void aoff_link_free_block(Allctr_t *, Block_t*); static void aoff_unlink_free_block(Allctr_t *allctr, Block_t *del); static void aoff_creating_mbc(Allctr_t*, Carrier_t*); +#ifdef DEBUG static void aoff_destroying_mbc(Allctr_t*, Carrier_t*); +#endif static void aoff_add_mbc(Allctr_t*, Carrier_t*); static void aoff_remove_mbc(Allctr_t*, Carrier_t*); static UWord aoff_largest_fblk_in_mbc(Allctr_t*, Carrier_t*); @@ -271,7 +273,11 @@ erts_aoffalc_start(AOFFAllctr_t *alc, allctr->get_next_mbc_size = NULL; allctr->creating_mbc = aoff_creating_mbc; +#ifdef DEBUG allctr->destroying_mbc = aoff_destroying_mbc; +#else + allctr->destroying_mbc = NULL; +#endif allctr->add_mbc = aoff_add_mbc; allctr->remove_mbc = aoff_remove_mbc; allctr->largest_fblk_in_mbc = aoff_largest_fblk_in_mbc; @@ -885,17 +891,18 @@ static void aoff_creating_mbc(Allctr_t *allctr, Carrier_t *carrier) HARD_CHECK_TREE(NULL, 0, *root, 0); } +#define IS_CRR_IN_TREE(CRR,ROOT) \ + ((CRR)->rbt_node.parent || (ROOT) == &(CRR)->rbt_node) + +#ifdef DEBUG static void aoff_destroying_mbc(Allctr_t *allctr, Carrier_t *carrier) { AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr; AOFF_Carrier_t *crr = (AOFF_Carrier_t*) carrier; - AOFF_RBTree_t *root = alc->mbc_root; - if (crr->rbt_node.parent || &crr->rbt_node == root) { - aoff_remove_mbc(allctr, carrier); - } - /*else already removed */ + ASSERT(!IS_CRR_IN_TREE(crr, alc->mbc_root)); } +#endif static void aoff_add_mbc(Allctr_t *allctr, Carrier_t *carrier) { @@ -903,6 +910,7 @@ static void aoff_add_mbc(Allctr_t *allctr, Carrier_t *carrier) AOFF_Carrier_t *crr = (AOFF_Carrier_t*) carrier; AOFF_RBTree_t **root = &alc->mbc_root; + ASSERT(!IS_CRR_IN_TREE(crr, *root)); HARD_CHECK_TREE(NULL, 0, *root, 0); /* Link carrier in address order tree @@ -919,6 +927,10 @@ static void aoff_remove_mbc(Allctr_t *allctr, Carrier_t *carrier) AOFF_RBTree_t **root = &alc->mbc_root; ASSERT(allctr == ERTS_ALC_CARRIER_TO_ALLCTR(carrier)); + + if (!IS_CRR_IN_TREE(crr,*root)) + return; + HARD_CHECK_TREE(NULL, 0, *root, 0); rbt_delete(root, &crr->rbt_node); diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c index 3ff54c7a60..e47d7bcbbb 100644 --- a/erts/emulator/beam/erl_bif_port.c +++ b/erts/emulator/beam/erl_bif_port.c @@ -1329,7 +1329,8 @@ BIF_RETTYPE decode_packet_3(BIF_ALIST_3) ErlSubBin* rest; Eterm res; Eterm options; - int code; + int code; + char delimiter = '\n'; if (!is_binary(BIF_ARG_2) || (!is_list(BIF_ARG_3) && !is_nil(BIF_ARG_3))) { @@ -1370,6 +1371,11 @@ BIF_RETTYPE decode_packet_3(BIF_ALIST_3) case am_line_length: trunc_len = val; goto next_option; + case am_line_delimiter: + if (type == TCP_PB_LINE_LF && val >= 0 && val <= 255) { + delimiter = (char)val; + goto next_option; + } } } } @@ -1390,7 +1396,7 @@ BIF_RETTYPE decode_packet_3(BIF_ALIST_3) pca.aligned_ptr = bin_ptr; } packet_sz = packet_get_length(type, (char*)pca.aligned_ptr, pca.bin_sz, - max_plen, trunc_len, &http_state); + max_plen, trunc_len, delimiter, &http_state); if (!(packet_sz > 0 && packet_sz <= pca.bin_sz)) { if (packet_sz < 0) { goto error; diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h index 6b406d069c..dbb4d719c1 100644 --- a/erts/emulator/beam/erl_driver.h +++ b/erts/emulator/beam/erl_driver.h @@ -696,8 +696,8 @@ EXTERN int driver_dl_close(void *); EXTERN char *driver_dl_error(void); /* environment */ -EXTERN int erl_drv_putenv(char *key, char *value); -EXTERN int erl_drv_getenv(char *key, char *value, size_t *value_size); +EXTERN int erl_drv_putenv(const char *key, char *value); +EXTERN int erl_drv_getenv(const char *key, char *value, size_t *value_size); #ifdef __OSE__ typedef ErlDrvUInt ErlDrvOseEventId; diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index d2604f1595..2f21111a2e 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -1237,6 +1237,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) Uint oh_size = (char *) OLD_HTOP(p) - oh; Uint n; Uint new_sz; + int done; /* * Do a fullsweep GC. First figure out the size of the heap @@ -1440,6 +1441,8 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) *recl += size_before - (HEAP_TOP(p) - HEAP_START(p)); + remove_message_buffers(p); + { ErlMessage *msgp; @@ -1458,15 +1461,21 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) } } - adjust_after_fullsweep(p, need, objv, nobj); - -#ifdef HARDDEBUG - disallow_heap_frag_ref_in_heap(p); -#endif - remove_message_buffers(p); + if (MBUF(p)) { + /* This is a very rare case when distributed messages copied above + * contained maps so big they did not fit on the heap causing the + * factory to create heap frags. + * Solution: Trigger a minor gc (without tenuring) + */ + HIGH_WATER(p) = HEAP_START(p); + done = 0; + } else { + adjust_after_fullsweep(p, need, objv, nobj); + done = 1; + } ErtsGcQuickSanityCheck(p); - return 1; /* We are done. */ + return done; } static void @@ -1955,7 +1964,18 @@ collect_heap_frags(Process* p, Eterm* n_hstart, Eterm* n_htop, if (p->dictionary != NULL) { disallow_heap_frag_ref(p, n_htop, p->dictionary->data, p->dictionary->used); } - disallow_heap_frag_ref_in_heap(p); + /* OTP-18: Actually we do allow references from heap to heap fragments now. + This can happen when doing "binary_to_term" with a "fat" map contained + in another term. A "fat" map is a hashmap with higher heap demand than + first estimated by "binary_to_term" causing the factory to allocate + additional heap (fragments) for the hashmap tree nodes. + Run map_SUITE:t_gc_rare_map_overflow to provoke this. + + Inverted references like this does not matter however. The copy done + below by move_one_area() with move markers in the fragments and the + sweeping done later by the GC should make everything ok in the end. + */ + /***disallow_heap_frag_ref_in_heap(p);***/ #endif /* diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h index c391de3f11..4d9d74bc37 100644 --- a/erts/emulator/beam/erl_map.h +++ b/erts/emulator/beam/erl_map.h @@ -195,14 +195,17 @@ typedef struct hashmap_head_s { [one cons cell + one list term in parent node] per key [one header + one boxed term in parent node] per inner node [one header + one size word] for root node + Observed average number of nodes per key is about 0.35. */ -#define HASHMAP_HEAP_SIZE(KEYS,NODES) ((KEYS)*3 + (NODES)*2) +#define HASHMAP_WORDS_PER_KEY 3 +#define HASHMAP_WORDS_PER_NODE 2 #ifdef DEBUG -# define HASHMAP_ESTIMATED_NODE_COUNT(KEYS) (KEYS) +# define HASHMAP_ESTIMATED_TOT_NODE_SIZE(KEYS) \ + (HASHMAP_WORDS_PER_NODE * (KEYS) * 3/10) /* slightly under estimated */ #else -# define HASHMAP_ESTIMATED_NODE_COUNT(KEYS) (2*(KEYS)/5) +# define HASHMAP_ESTIMATED_TOT_NODE_SIZE(KEYS) \ + (HASHMAP_WORDS_PER_NODE * (KEYS) * 4/10) /* slightly over estimated */ #endif #define HASHMAP_ESTIMATED_HEAP_SIZE(KEYS) \ - HASHMAP_HEAP_SIZE(KEYS,HASHMAP_ESTIMATED_NODE_COUNT(KEYS)) - + ((KEYS)*HASHMAP_WORDS_PER_KEY + HASHMAP_ESTIMATED_TOT_NODE_SIZE(KEYS)) #endif diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index ef52823287..fa6b2fc613 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -1174,6 +1174,9 @@ void erts_factory_message_init(ErtsHeapFactory* factory, ASSERT(factory->hp >= factory->hp_start && factory->hp <= factory->hp_end); } +/* One static sized heap that must suffice. + No extra heap fragments will be allocated. +*/ void erts_factory_static_init(ErtsHeapFactory* factory, Eterm* hp, Uint size, @@ -1188,6 +1191,23 @@ void erts_factory_static_init(ErtsHeapFactory* factory, factory->off_heap_saved.overhead = factory->off_heap->overhead; } +/* A temporary heap with default buffer allocated/freed by client. + * factory_close is same as factory_undo + */ +void erts_factory_tmp_init(ErtsHeapFactory* factory, Eterm* hp, Uint size, + Uint32 atype) +{ + factory->mode = FACTORY_TMP; + factory->hp_start = hp; + factory->hp = hp; + factory->hp_end = hp + size; + factory->heap_frags = NULL; + factory->off_heap_saved.first = NULL; + factory->off_heap_saved.overhead = 0; + factory->off_heap = &factory->off_heap_saved; + factory->alloc_type = atype; +} + /* When we know the term is an immediate and need no heap. */ void erts_factory_dummy_init(ErtsHeapFactory* factory) @@ -1231,6 +1251,7 @@ static void reserve_heap(ErtsHeapFactory* factory, Uint need, Uint xtra) return; case FACTORY_HEAP_FRAGS: + case FACTORY_TMP: bp = factory->heap_frags; if (bp) { @@ -1280,6 +1301,9 @@ void erts_factory_close(ErtsHeapFactory* factory) bp->used_size = factory->hp - bp->mem; } break; + case FACTORY_TMP: + erts_factory_undo(factory); + break; case FACTORY_STATIC: break; case FACTORY_CLOSED: break; default: @@ -1371,16 +1395,20 @@ void erts_factory_undo(ErtsHeapFactory* factory) } break; + case FACTORY_TMP: case FACTORY_HEAP_FRAGS: + erts_cleanup_offheap(factory->off_heap); + factory->off_heap->first = NULL; + bp = factory->heap_frags; - do { + while (bp != NULL) { ErlHeapFragment* next_bp = bp->next; - erts_cleanup_offheap(&bp->off_heap); + ASSERT(bp->off_heap.first == NULL); ERTS_HEAP_FREE(factory->alloc_type, (void *) bp, - ERTS_HEAP_FRAG_SIZE(bp->size)); + ERTS_HEAP_FRAG_SIZE(bp->alloc_size)); bp = next_bp; - }while (bp != NULL); + } break; case FACTORY_CLOSED: break; diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index fbdf3fb0e2..92ba3e571c 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -58,7 +58,8 @@ typedef struct { FACTORY_CLOSED = 0, FACTORY_HALLOC, FACTORY_HEAP_FRAGS, - FACTORY_STATIC + FACTORY_STATIC, + FACTORY_TMP } mode; Process* p; Eterm* hp_start; @@ -75,6 +76,7 @@ void erts_factory_proc_init(ErtsHeapFactory*, Process*); void erts_factory_proc_prealloc_init(ErtsHeapFactory*, Process*, Sint size); void erts_factory_message_init(ErtsHeapFactory*, Process*, Eterm* hp, struct erl_heap_fragment*); void erts_factory_static_init(ErtsHeapFactory*, Eterm* hp, Uint size, ErlOffHeap*); +void erts_factory_tmp_init(ErtsHeapFactory*, Eterm* hp, Uint size, Uint32 atype); void erts_factory_dummy_init(ErtsHeapFactory*); Eterm* erts_produce_heap(ErtsHeapFactory*, Uint need, Uint xtra); diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index add4a66f90..d7a2076d85 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -1173,6 +1173,7 @@ ErlNifTid enif_thread_self(void) { return erl_drv_thread_self(); } int enif_equal_tids(ErlNifTid tid1, ErlNifTid tid2) { return erl_drv_equal_tids(tid1,tid2); } void enif_thread_exit(void *resp) { erl_drv_thread_exit(resp); } int enif_thread_join(ErlNifTid tid, void **respp) { return erl_drv_thread_join(tid,respp); } +int enif_getenv(const char *key, char *value, size_t *value_size) { return erl_drv_getenv(key, value, value_size); } int enif_fprintf(void* filep, const char* format, ...) { diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index 7d880126f8..5e39343e9b 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -48,9 +48,10 @@ ** add ErlNifEntry options ** add ErlNifFunc flags ** 2.8: 18.0 add enif_has_pending_exception +** 2.9: 18.2 enif_getenv */ #define ERL_NIF_MAJOR_VERSION 2 -#define ERL_NIF_MINOR_VERSION 8 +#define ERL_NIF_MINOR_VERSION 9 /* * The emulator will refuse to load a nif-lib with a major version @@ -231,6 +232,7 @@ typedef enum { # define ERL_NIF_API_FUNC_DECL(RET_TYPE, NAME, ARGS) RET_TYPE (*NAME) ARGS typedef struct { # include "erl_nif_api_funcs.h" + void* erts_alc_test; } TWinDynNifCallbacks; extern TWinDynNifCallbacks WinDynNifCallbacks; # undef ERL_NIF_API_FUNC_DECL diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h index 2f2180e1aa..08b9afc6af 100644 --- a/erts/emulator/beam/erl_nif_api_funcs.h +++ b/erts/emulator/beam/erl_nif_api_funcs.h @@ -159,6 +159,7 @@ ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_get_pair, (ErlNifEnv *env, ErlNifMa ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_schedule_nif,(ErlNifEnv*,const char*,int,ERL_NIF_TERM (*)(ErlNifEnv*,int,const ERL_NIF_TERM[]),int,const ERL_NIF_TERM[])); ERL_NIF_API_FUNC_DECL(int, enif_has_pending_exception, (ErlNifEnv *env, ERL_NIF_TERM* reason)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM, enif_raise_exception, (ErlNifEnv *env, ERL_NIF_TERM reason)); +ERL_NIF_API_FUNC_DECL(int,enif_getenv,(const char* key, char* value, size_t* value_size)); /* ** ADD NEW ENTRIES HERE (before this comment) !!! @@ -310,6 +311,7 @@ ERL_NIF_API_FUNC_DECL(int,enif_is_on_dirty_scheduler,(ErlNifEnv*)); # define enif_schedule_nif ERL_NIF_API_FUNC_MACRO(enif_schedule_nif) # define enif_has_pending_exception ERL_NIF_API_FUNC_MACRO(enif_has_pending_exception) # define enif_raise_exception ERL_NIF_API_FUNC_MACRO(enif_raise_exception) +# define enif_getenv ERL_NIF_API_FUNC_MACRO(enif_getenv) /* ** ADD NEW ENTRIES HERE (before this comment) diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index 2fb790b953..707de39556 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -37,18 +37,18 @@ erts_smp_rwmtx_t erts_node_table_rwmtx; DistEntry *erts_hidden_dist_entries; DistEntry *erts_visible_dist_entries; -DistEntry *erts_not_connected_dist_entries; +DistEntry *erts_not_connected_dist_entries; /* including erts_this_dist_entry */ Sint erts_no_of_hidden_dist_entries; Sint erts_no_of_visible_dist_entries; -Sint erts_no_of_not_connected_dist_entries; +Sint erts_no_of_not_connected_dist_entries; /* including erts_this_dist_entry */ DistEntry *erts_this_dist_entry; ErlNode *erts_this_node; char erts_this_node_sysname_BUFFER[256], *erts_this_node_sysname = "uninitialized yet"; -static Uint node_entries; -static Uint dist_entries; +static Uint node_entries = 0; +static Uint dist_entries = 0; static int references_atoms_need_init = 1; @@ -91,9 +91,6 @@ dist_table_alloc(void *dep_tmpl) erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; - if(((DistEntry *) dep_tmpl) == erts_this_dist_entry) - return dep_tmpl; - sysname = ((DistEntry *) dep_tmpl)->sysname; chnl_nr = make_small((Uint) atom_val(sysname)); dep = (DistEntry *) erts_alloc(ERTS_ALC_T_DIST_ENTRY, sizeof(DistEntry)); @@ -132,7 +129,9 @@ dist_table_alloc(void *dep_tmpl) /* Link in */ - /* All new dist entries are "not connected" */ + /* All new dist entries are "not connected". + * erts_this_dist_entry is also always included among "not connected" + */ dep->next = erts_not_connected_dist_entries; if(erts_not_connected_dist_entries) { ASSERT(erts_not_connected_dist_entries->prev == NULL); @@ -149,9 +148,6 @@ dist_table_free(void *vdep) { DistEntry *dep = (DistEntry *) vdep; - if(dep == erts_this_dist_entry) - return; - ASSERT(is_nil(dep->cid)); ASSERT(dep->nlinks == NULL); ASSERT(dep->node_links == NULL); @@ -186,7 +182,7 @@ dist_table_free(void *vdep) #endif erts_free(ERTS_ALC_T_DIST_ENTRY, (void *) dep); - ASSERT(dist_entries > 1); + ASSERT(dist_entries > 0); dist_entries--; } @@ -306,7 +302,7 @@ static void try_delete_dist_entry(void *vdep) * thread incremented refc twice. Once for the new reference * and once for this thread. * - * If refc reach -1, noone has used the entry since we + * If refc reach -1, no one has used the entry since we * set up the timer. Delete the entry. * * If refc reach 0, the entry is currently not in use @@ -369,8 +365,7 @@ erts_dist_table_size(void) ASSERT(dist_entries == (erts_no_of_visible_dist_entries + erts_no_of_hidden_dist_entries - + erts_no_of_not_connected_dist_entries - + 1 /* erts_this_dist_entry */)); + + erts_no_of_not_connected_dist_entries)); #endif res = (hash_table_sz(&erts_dist_table) @@ -543,9 +538,6 @@ node_table_alloc(void *venp_tmpl) { ErlNode *enp; - if(((ErlNode *) venp_tmpl) == erts_this_node) - return venp_tmpl; - enp = (ErlNode *) erts_alloc(ERTS_ALC_T_NODE_ENTRY, sizeof(ErlNode)); node_entries++; @@ -563,8 +555,7 @@ node_table_free(void *venp) { ErlNode *enp = (ErlNode *) venp; - if(enp == erts_this_node) - return; + ERTS_SMP_LC_ASSERT(enp != erts_this_node || erts_thr_progress_is_blocking()); erts_deref_dist_entry(enp->dist_entry); #ifdef DEBUG @@ -572,7 +563,7 @@ node_table_free(void *venp) #endif erts_free(ERTS_ALC_T_NODE_ENTRY, venp); - ASSERT(node_entries > 1); + ASSERT(node_entries > 0); node_entries--; } @@ -650,7 +641,7 @@ static void try_delete_node(void *venp) * thread incremented refc twice. Once for the new reference * and once for this thread. * - * If refc reach -1, noone has used the entry since we + * If refc reach -1, no one has used the entry since we * set up the timer. Delete the entry. * * If refc reach 0, the entry is currently not in use @@ -747,25 +738,24 @@ void erts_print_node_info(int to, void erts_set_this_node(Eterm sysname, Uint creation) { - erts_smp_rwmtx_rwlock(&erts_node_table_rwmtx); - erts_smp_rwmtx_rwlock(&erts_dist_table_rwmtx); + ERTS_SMP_LC_ASSERT(erts_thr_progress_is_blocking()); + ASSERT(erts_refc_read(&erts_this_dist_entry->refc, 2)); - (void) hash_erase(&erts_dist_table, (void *) erts_this_dist_entry); - erts_this_dist_entry->sysname = sysname; - erts_this_dist_entry->creation = creation; - (void) hash_put(&erts_dist_table, (void *) erts_this_dist_entry); + if (erts_refc_dectest(&erts_this_node->refc, 0) == 0) + try_delete_node(erts_this_node); - (void) hash_erase(&erts_node_table, (void *) erts_this_node); - erts_this_node->sysname = sysname; - erts_this_node->creation = creation; - erts_this_node_sysname = erts_this_node_sysname_BUFFER; - erts_snprintf(erts_this_node_sysname, sizeof(erts_this_node_sysname_BUFFER), - "%T", sysname); - (void) hash_put(&erts_node_table, (void *) erts_this_node); + if (erts_refc_dectest(&erts_this_dist_entry->refc, 0) == 0) + try_delete_dist_entry(erts_this_dist_entry); - erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx); - erts_smp_rwmtx_rwunlock(&erts_node_table_rwmtx); + erts_this_node = NULL; /* to make sure refc is bumped for this node */ + erts_this_node = erts_find_or_insert_node(sysname, creation); + erts_this_dist_entry = erts_this_node->dist_entry; + + erts_refc_inc(&erts_this_dist_entry->refc, 2); + erts_this_node_sysname = erts_this_node_sysname_BUFFER; + erts_snprintf(erts_this_node_sysname, sizeof(erts_this_node_sysname_BUFFER), + "%T", sysname); } Uint @@ -782,6 +772,7 @@ void erts_init_node_tables(int dd_sec) { erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; HashFunctions f; + ErlNode node_tmpl; if (dd_sec == ERTS_NODE_TAB_DELAY_GC_INFINITY) node_tab_delete_delay = (ErtsMonotonicTime) -1; @@ -793,16 +784,21 @@ void erts_init_node_tables(int dd_sec) rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; + erts_smp_rwmtx_init_opt(&erts_node_table_rwmtx, &rwmtx_opt, "node_table"); + erts_smp_rwmtx_init_opt(&erts_dist_table_rwmtx, &rwmtx_opt, "dist_table"); + f.hash = (H_FUN) dist_table_hash; f.cmp = (HCMP_FUN) dist_table_cmp; f.alloc = (HALLOC_FUN) dist_table_alloc; f.free = (HFREE_FUN) dist_table_free; - - erts_this_dist_entry = erts_alloc(ERTS_ALC_T_DIST_ENTRY, sizeof(DistEntry)); - dist_entries = 1; - hash_init(ERTS_ALC_T_DIST_TABLE, &erts_dist_table, "dist_table", 11, f); + f.hash = (H_FUN) node_table_hash; + f.cmp = (HCMP_FUN) node_table_cmp; + f.alloc = (HALLOC_FUN) node_table_alloc; + f.free = (HFREE_FUN) node_table_free; + hash_init(ERTS_ALC_T_NODE_TABLE, &erts_node_table, "node_table", 11, f); + erts_hidden_dist_entries = NULL; erts_visible_dist_entries = NULL; erts_not_connected_dist_entries = NULL; @@ -810,69 +806,23 @@ void erts_init_node_tables(int dd_sec) erts_no_of_visible_dist_entries = 0; erts_no_of_not_connected_dist_entries = 0; - erts_this_dist_entry->next = NULL; - erts_this_dist_entry->prev = NULL; - erts_refc_init(&erts_this_dist_entry->refc, 1); /* erts_this_node */ - - erts_smp_rwmtx_init_opt_x(&erts_this_dist_entry->rwmtx, - &rwmtx_opt, - "dist_entry", - make_small(ERST_INTERNAL_CHANNEL_NO)); - erts_this_dist_entry->sysname = am_Noname; - erts_this_dist_entry->cid = NIL; - erts_this_dist_entry->connection_id = 0; - erts_this_dist_entry->status = 0; - erts_this_dist_entry->flags = 0; - erts_this_dist_entry->version = 0; - - erts_smp_mtx_init_x(&erts_this_dist_entry->lnk_mtx, - "dist_entry_links", - make_small(ERST_INTERNAL_CHANNEL_NO)); - erts_this_dist_entry->node_links = NULL; - erts_this_dist_entry->nlinks = NULL; - erts_this_dist_entry->monitors = NULL; - - erts_smp_mtx_init_x(&erts_this_dist_entry->qlock, - "dist_entry_out_queue", - make_small(ERST_INTERNAL_CHANNEL_NO)); - erts_this_dist_entry->qflgs = 0; - erts_this_dist_entry->qsize = 0; - erts_this_dist_entry->out_queue.first = NULL; - erts_this_dist_entry->out_queue.last = NULL; - erts_this_dist_entry->suspended = NULL; - - erts_this_dist_entry->finalized_out_queue.first = NULL; - erts_this_dist_entry->finalized_out_queue.last = NULL; - erts_smp_atomic_init_nob(&erts_this_dist_entry->dist_cmd_scheduled, 0); - erts_port_task_handle_init(&erts_this_dist_entry->dist_cmd); - erts_this_dist_entry->send = NULL; - erts_this_dist_entry->cache = NULL; - - (void) hash_put(&erts_dist_table, (void *) erts_this_dist_entry); + node_tmpl.sysname = am_Noname; + node_tmpl.creation = 0; + erts_this_node = hash_put(&erts_node_table, &node_tmpl); + /* +1 for erts_this_node */ + erts_refc_init(&erts_this_node->refc, 1); - f.hash = (H_FUN) node_table_hash; - f.cmp = (HCMP_FUN) node_table_cmp; - f.alloc = (HALLOC_FUN) node_table_alloc; - f.free = (HFREE_FUN) node_table_free; - - hash_init(ERTS_ALC_T_NODE_TABLE, &erts_node_table, "node_table", 11, f); + ASSERT(erts_this_node->dist_entry != NULL); + erts_this_dist_entry = erts_this_node->dist_entry; + /* +1 for erts_this_dist_entry */ + /* +1 for erts_this_node->dist_entry */ + erts_refc_init(&erts_this_dist_entry->refc, 2); - erts_this_node = erts_alloc(ERTS_ALC_T_NODE_ENTRY, sizeof(ErlNode)); - node_entries = 1; - erts_refc_init(&erts_this_node->refc, 1); /* The system itself */ - erts_this_node->sysname = am_Noname; - erts_this_node->creation = 0; - erts_this_node->dist_entry = erts_this_dist_entry; erts_this_node_sysname = erts_this_node_sysname_BUFFER; erts_snprintf(erts_this_node_sysname, sizeof(erts_this_node_sysname_BUFFER), "%T", erts_this_node->sysname); - (void) hash_put(&erts_node_table, (void *) erts_this_node); - - erts_smp_rwmtx_init_opt(&erts_node_table_rwmtx, &rwmtx_opt, "node_table"); - erts_smp_rwmtx_init_opt(&erts_dist_table_rwmtx, &rwmtx_opt, "dist_table"); - references_atoms_need_init = 1; } @@ -1410,6 +1360,10 @@ setup_reference_table(void) SYSTEM_REF, TUPLE2(&heap[0], AM_system, am_undefined)); + insert_dist_entry(erts_this_dist_entry, + SYSTEM_REF, + TUPLE2(&heap[0], AM_system, am_undefined), + erts_this_node->creation); UnUseTmpHeapNoproc(3); max = erts_ptab_max(&erts_proc); @@ -1472,12 +1426,6 @@ setup_reference_table(void) insert_links(ERTS_P_LINKS(proc), proc->common.id); if (ERTS_P_MONITORS(proc)) insert_monitors(ERTS_P_MONITORS(proc), proc->common.id); - /* Insert controller */ - { - DistEntry *dep = ERTS_PROC_GET_DIST_ENTRY(proc); - if (dep) - insert_dist_entry(dep, CTRL_REF, proc->common.id, 0); - } } } diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 3b1b593d1c..d583118e7b 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -625,11 +625,6 @@ erts_pre_init_process(void) erts_psd_required_locks[ERTS_PSD_SCHED_ID].set_locks = ERTS_PSD_SCHED_ID_SET_LOCKS; - erts_psd_required_locks[ERTS_PSD_DIST_ENTRY].get_locks - = ERTS_PSD_DIST_ENTRY_GET_LOCKS; - erts_psd_required_locks[ERTS_PSD_DIST_ENTRY].set_locks - = ERTS_PSD_DIST_ENTRY_SET_LOCKS; - erts_psd_required_locks[ERTS_PSD_CALL_TIME_BP].get_locks = ERTS_PSD_CALL_TIME_BP_GET_LOCKS; erts_psd_required_locks[ERTS_PSD_CALL_TIME_BP].set_locks @@ -12373,9 +12368,7 @@ erts_continue_exit_process(Process *p) erts_proc_dec_refc(p); } - dep = ((p->flags & F_DISTRIBUTION) - ? ERTS_PROC_SET_DIST_ENTRY(p, ERTS_PROC_LOCKS_ALL, NULL) - : NULL); + dep = (p->flags & F_DISTRIBUTION) ? erts_this_dist_entry : NULL; scb = ERTS_PROC_SET_SAVED_CALLS_BUF(p, ERTS_PROC_LOCKS_ALL, NULL); pbt = ERTS_PROC_SET_CALL_TIME(p, ERTS_PROC_LOCKS_ALL, NULL); nif_export = ERTS_PROC_SET_NIF_TRAP_EXPORT(p, ERTS_PROC_LOCKS_ALL, NULL); @@ -12387,8 +12380,6 @@ erts_continue_exit_process(Process *p) if (dep) { erts_do_net_exits(dep, reason); - if(dep) - erts_deref_dist_entry(dep); } /* diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 20ffe7ea7c..10c6fa4a67 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -801,12 +801,11 @@ erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi) #define ERTS_PSD_ERROR_HANDLER 0 #define ERTS_PSD_SAVED_CALLS_BUF 1 #define ERTS_PSD_SCHED_ID 2 -#define ERTS_PSD_DIST_ENTRY 3 -#define ERTS_PSD_CALL_TIME_BP 4 -#define ERTS_PSD_DELAYED_GC_TASK_QS 5 -#define ERTS_PSD_NIF_TRAP_EXPORT 6 +#define ERTS_PSD_CALL_TIME_BP 3 +#define ERTS_PSD_DELAYED_GC_TASK_QS 4 +#define ERTS_PSD_NIF_TRAP_EXPORT 5 -#define ERTS_PSD_SIZE 7 +#define ERTS_PSD_SIZE 6 typedef struct { void *data[ERTS_PSD_SIZE]; @@ -824,9 +823,6 @@ typedef struct { #define ERTS_PSD_SCHED_ID_GET_LOCKS ERTS_PROC_LOCK_STATUS #define ERTS_PSD_SCHED_ID_SET_LOCKS ERTS_PROC_LOCK_STATUS -#define ERTS_PSD_DIST_ENTRY_GET_LOCKS ERTS_PROC_LOCK_MAIN -#define ERTS_PSD_DIST_ENTRY_SET_LOCKS ERTS_PROC_LOCK_MAIN - #define ERTS_PSD_CALL_TIME_BP_GET_LOCKS ERTS_PROC_LOCK_MAIN #define ERTS_PSD_CALL_TIME_BP_SET_LOCKS ERTS_PROC_LOCK_MAIN @@ -1887,11 +1883,6 @@ erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *data) #define ERTS_PROC_SCHED_ID(P, L, ID) \ ((UWord) erts_psd_set((P), (L), ERTS_PSD_SCHED_ID, (void *) (ID))) -#define ERTS_PROC_GET_DIST_ENTRY(P) \ - ((DistEntry *) erts_psd_get((P), ERTS_PSD_DIST_ENTRY)) -#define ERTS_PROC_SET_DIST_ENTRY(P, L, D) \ - ((DistEntry *) erts_psd_set((P), (L), ERTS_PSD_DIST_ENTRY, (void *) (D))) - #define ERTS_PROC_GET_SAVED_CALLS_BUF(P) \ ((struct saved_calls *) erts_psd_get((P), ERTS_PSD_SAVED_CALLS_BUF)) #define ERTS_PROC_SET_SAVED_CALLS_BUF(P, L, SCB) \ diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h index 788348e613..a64c993e8f 100644 --- a/erts/emulator/beam/erl_process_lock.h +++ b/erts/emulator/beam/erl_process_lock.h @@ -854,9 +854,6 @@ ERTS_GLB_INLINE void erts_proc_dec_refc(Process *p) #endif if (!referred) { ASSERT(ERTS_PROC_IS_EXITING(p)); - ASSERT(ERTS_AINT_NULL - == erts_ptab_pix2intptr_ddrb(&erts_proc, - internal_pid_index(p->common.id))); erts_free_proc(p); } } @@ -872,9 +869,6 @@ ERTS_GLB_INLINE void erts_proc_add_refc(Process *p, Sint add_refc) #endif if (!referred) { ASSERT(ERTS_PROC_IS_EXITING(p)); - ASSERT(ERTS_AINT_NULL - == erts_ptab_pix2intptr_ddrb(&erts_proc, - internal_pid_index(p->common.id))); erts_free_proc(p); } } diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index c6d7e3fcc5..a85aa15403 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1179,7 +1179,7 @@ typedef struct { ErtsHeapFactory factory; int remaining_n; char* remaining_bytes; - Eterm* maps_list; + ErtsWStack flat_maps; ErtsPStack hamt_array; } B2TDecodeContext; @@ -1519,7 +1519,7 @@ static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binar ctx->u.dc.res = (Eterm) (UWord) NULL; ctx->u.dc.next = &ctx->u.dc.res; erts_factory_proc_prealloc_init(&ctx->u.dc.factory, p, ctx->heap_size); - ctx->u.dc.maps_list = NULL; + ctx->u.dc.flat_maps.wstart = NULL; ctx->u.dc.hamt_array.pstart = NULL; ctx->state = B2TDecode; /*fall through*/ @@ -2938,7 +2938,7 @@ dec_term(ErtsDistExternal *edep, int n; ErtsAtomEncoding char_enc; register Eterm* hp; /* Please don't take the address of hp */ - Eterm *maps_list; /* for preprocessing of small maps */ + DECLARE_WSTACK(flat_maps); /* for preprocessing of small maps */ Eterm* next; SWord reds; #ifdef DEBUG @@ -2950,7 +2950,6 @@ dec_term(ErtsDistExternal *edep, next = ctx->u.dc.next; ep = ctx->u.dc.ep; factory = &ctx->u.dc.factory; - maps_list = ctx->u.dc.maps_list; if (ctx->state != B2TDecode) { int n_limit = reds; @@ -3026,15 +3025,18 @@ dec_term(ErtsDistExternal *edep, } } PSTACK_CHANGE_ALLOCATOR(hamt_array, ERTS_ALC_T_SAVED_ESTACK); + WSTACK_CHANGE_ALLOCATOR(flat_maps, ERTS_ALC_T_SAVED_ESTACK); if (ctx->u.dc.hamt_array.pstart) { PSTACK_RESTORE(hamt_array, &ctx->u.dc.hamt_array); } + if (ctx->u.dc.flat_maps.wstart) { + WSTACK_RESTORE(flat_maps, &ctx->u.dc.flat_maps); + } } else { reds = ERTS_SWORD_MAX; next = objp; *next = (Eterm) (UWord) NULL; - maps_list = NULL; } hp = factory->hp; @@ -3595,14 +3597,8 @@ dec_term_atom_common: * vptr, last word for values */ - /* - * Use thing_word to link through decoded maps. - * The list of maps is for later validation. - */ - - mp->thing_word = (Eterm) COMPRESS_POINTER(maps_list); - maps_list = (Eterm *) mp; - + WSTACK_PUSH(flat_maps, (UWord)mp); + mp->thing_word = MAP_HEADER_FLATMAP; mp->size = size; mp->keys = keys; *objp = make_flatmap(mp); @@ -3851,7 +3847,9 @@ dec_term_atom_common: ctx->u.dc.ep = ep; ctx->u.dc.next = next; ctx->u.dc.factory.hp = hp; - ctx->u.dc.maps_list = maps_list; + if (!WSTACK_ISEMPTY(flat_maps)) { + WSTACK_SAVE(flat_maps, &ctx->u.dc.flat_maps); + } if (!PSTACK_IS_EMPTY(hamt_array)) { PSTACK_SAVE(hamt_array, &ctx->u.dc.hamt_array); } @@ -3865,18 +3863,6 @@ dec_term_atom_common: } } - /* Iterate through all the maps and check for validity and sort keys - * - done here for when we know it is complete. - */ - - while (maps_list) { - next = (Eterm *)(EXPAND_POINTER(*maps_list)); - *maps_list = MAP_HEADER_FLATMAP; - if (!erts_validate_and_sort_flatmap((flatmap_t*)maps_list)) - goto error; - maps_list = next; - } - ASSERT(hp <= factory->hp_end || (factory->mode == FACTORY_CLOSED && is_immed(*dbg_resultp))); factory->hp = hp; @@ -3885,20 +3871,31 @@ dec_term_atom_common: */ if (!PSTACK_IS_EMPTY(hamt_array)) { - do { - struct dec_term_hamt* hamt = PSTACK_TOP(hamt_array); - - *hamt->objp = erts_hashmap_from_array(factory, - hamt->leaf_array, - hamt->size, - 1); - if (is_non_value(*hamt->objp)) - goto error_hamt; - - (void) PSTACK_POP(hamt_array); - } while (!PSTACK_IS_EMPTY(hamt_array)); - PSTACK_DESTROY(hamt_array); + do { + struct dec_term_hamt* hamt = PSTACK_TOP(hamt_array); + + *hamt->objp = erts_hashmap_from_array(factory, + hamt->leaf_array, + hamt->size, + 1); + if (is_non_value(*hamt->objp)) + goto error_hamt; + + (void) PSTACK_POP(hamt_array); + } while (!PSTACK_IS_EMPTY(hamt_array)); + PSTACK_DESTROY(hamt_array); + } + + /* Iterate through all the (flat)maps and check for validity and sort keys + * - done here for when we know it is complete. + */ + + while(!WSTACK_ISEMPTY(flat_maps)) { + next = (Eterm *)WSTACK_POP(flat_maps); + if (!erts_validate_and_sort_flatmap((flatmap_t*)next)) + goto error; } + WSTACK_DESTROY(flat_maps); ASSERT((Eterm*)EXPAND_POINTER(*dbg_resultp) != NULL); @@ -3924,6 +3921,7 @@ error_hamt: ctx->state = B2TDecodeFail; ctx->reds = reds; } + WSTACK_DESTROY(flat_maps); return NULL; } diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 900616c981..c64c8802b9 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -7596,15 +7596,15 @@ int null_func(void) } int -erl_drv_putenv(char *key, char *value) +erl_drv_putenv(const char *key, char *value) { - return erts_sys_putenv_raw(key, value); + return erts_sys_putenv_raw((char*)key, value); } int -erl_drv_getenv(char *key, char *value, size_t *value_size) +erl_drv_getenv(const char *key, char *value, size_t *value_size) { - return erts_sys_getenv_raw(key, value, value_size); + return erts_sys_getenv_raw((char*)key, value, value_size); } /* get heart_port diff --git a/erts/emulator/beam/packet_parser.c b/erts/emulator/beam/packet_parser.c index 2dd421a9e9..a737a86f14 100644 --- a/erts/emulator/beam/packet_parser.c +++ b/erts/emulator/beam/packet_parser.c @@ -256,6 +256,7 @@ int packet_get_length(enum PacketParseType htype, const char* ptr, unsigned n, /* Bytes read so far */ unsigned max_plen, /* Max packet length, 0=no limit */ unsigned trunc_len, /* Truncate (lines) if longer, 0=no limit */ + char delimiter, /* Line delimiting character */ int* statep) /* Protocol specific state */ { unsigned hlen, plen; @@ -299,9 +300,9 @@ int packet_get_length(enum PacketParseType htype, goto remain; case TCP_PB_LINE_LF: { - /* TCP_PB_LINE_LF: [Data ... \n] */ + /* TCP_PB_LINE_LF: [Data ... Delimiter] */ const char* ptr2; - if ((ptr2 = memchr(ptr, '\n', n)) == NULL) { + if ((ptr2 = memchr(ptr, delimiter, n)) == NULL) { if (n > max_plen && max_plen != 0) { /* packet full */ DEBUGF((" => packet full (no NL)=%d\r\n", n)); goto error; diff --git a/erts/emulator/beam/packet_parser.h b/erts/emulator/beam/packet_parser.h index ff158ff8b8..717d905fad 100644 --- a/erts/emulator/beam/packet_parser.h +++ b/erts/emulator/beam/packet_parser.h @@ -105,7 +105,8 @@ int packet_get_length(enum PacketParseType htype, const char* ptr, unsigned n, /* Bytes read so far */ unsigned max_plen, /* Packet max length, 0=no limit */ unsigned trunc_len, /* Truncate (lines) if longer, 0=no limit */ - int* statep); /* Internal protocol state */ + char delimiter, /* Line delimiting character */ + int* statep); /* Internal protocol state */ ERTS_GLB_INLINE void packet_get_body(enum PacketParseType htype, diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index bb871b05ba..ec94e3a596 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -634,7 +634,6 @@ Uint erts_sys_misc_mem_sz(void); /* Io constants to erts_print and erts_putc */ #define ERTS_PRINT_STDERR (2) #define ERTS_PRINT_STDOUT (1) -#define ERTS_PRINT_INVALID (0) /* Don't want to use 0 since CBUF was 0 */ #define ERTS_PRINT_FILE (-1) #define ERTS_PRINT_SBUF (-2) #define ERTS_PRINT_SNBUF (-3) diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index e9d7c91ac9..5286391746 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -399,9 +399,6 @@ erts_print(int to, void *arg, char *format, ...) case ERTS_PRINT_DSBUF: res = erts_vdsprintf((erts_dsprintf_buf_t *) arg, format, arg_list); break; - case ERTS_PRINT_INVALID: - res = -EINVAL; - break; default: res = erts_vfdprintf((int) to, format, arg_list); break; |