diff options
32 files changed, 465 insertions, 821 deletions
diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 596ad9a010..4fc271d41c 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -402,7 +402,7 @@ check_process_code(Process* rp, Module* modp) BeamInstr* end; Eterm* sp; #ifndef HYBRID /* FIND ME! */ - ErlFunThing* funp; + struct erl_off_heap_header* oh; int done_gc = 0; #endif @@ -470,27 +470,30 @@ check_process_code(Process* rp, Module* modp) #ifndef HYBRID /* FIND ME! */ rescan: - for (funp = MSO(rp).funs; funp; funp = funp->next) { - BeamInstr* fun_code; - - fun_code = funp->fe->address; - - if (INSIDE((BeamInstr *) funp->fe->address)) { - if (done_gc) { - return am_true; - } else { - /* - * Try to get rid of this fun by garbage collecting. - * Clear both fvalue and ftrace to make sure they - * don't hold any funs. - */ - rp->freason = EXC_NULL; - rp->fvalue = NIL; - rp->ftrace = NIL; - done_gc = 1; - FLAGS(rp) |= F_NEED_FULLSWEEP; - (void) erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity); - goto rescan; + for (oh = MSO(rp).first; oh; oh = oh->next) { + if (thing_subtag(oh->thing_word) == FUN_SUBTAG) { + ErlFunThing* funp = (ErlFunThing*) oh; + BeamInstr* fun_code; + + fun_code = funp->fe->address; + + if (INSIDE((BeamInstr *) funp->fe->address)) { + if (done_gc) { + return am_true; + } else { + /* + * Try to get rid of this fun by garbage collecting. + * Clear both fvalue and ftrace to make sure they + * don't hold any funs. + */ + rp->freason = EXC_NULL; + rp->fvalue = NIL; + rp->ftrace = NIL; + done_gc = 1; + FLAGS(rp) |= F_NEED_FULLSWEEP; + (void) erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity); + goto rescan; + } } } } diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index c0680086aa..2c37b26a39 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -3385,8 +3385,8 @@ apply_bif_or_nif_epilogue: HTOP += PROC_BIN_SIZE; pb->thing_word = HEADER_PROC_BIN; pb->size = num_bytes; - pb->next = MSO(c_p).mso; - MSO(c_p).mso = pb; + pb->next = MSO(c_p).first; + MSO(c_p).first = (struct erl_off_heap_header*) pb; pb->val = bptr; pb->bytes = (byte*) bptr->orig_bytes; pb->flags = 0; @@ -3486,8 +3486,8 @@ apply_bif_or_nif_epilogue: HTOP += PROC_BIN_SIZE; pb->thing_word = HEADER_PROC_BIN; pb->size = tmp_arg1; - pb->next = MSO(c_p).mso; - MSO(c_p).mso = pb; + pb->next = MSO(c_p).first; + MSO(c_p).first = (struct erl_off_heap_header*) pb; pb->val = bptr; pb->bytes = (byte*) bptr->orig_bytes; pb->flags = 0; @@ -6323,8 +6323,8 @@ new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free) erts_refc_inc(&fe->refc, 2); funp->thing_word = HEADER_FUN; #ifndef HYBRID /* FIND ME! */ - funp->next = MSO(p).funs; - MSO(p).funs = funp; + funp->next = MSO(p).first; + MSO(p).first = (struct erl_off_heap_header*) funp; #endif funp->fe = fe; funp->num_free = num_free; diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 506bf383ca..6e9755ad48 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -3611,11 +3611,11 @@ BIF_RETTYPE list_to_pid_1(BIF_ALIST_1) etp = (ExternalThing *) HAlloc(BIF_P, EXTERNAL_THING_HEAD_SIZE + 1); etp->header = make_external_pid_header(1); - etp->next = MSO(BIF_P).externals; + etp->next = MSO(BIF_P).first; etp->node = enp; etp->data.ui[0] = make_pid_data(c, b); - MSO(BIF_P).externals = etp; + MSO(BIF_P).first = (struct erl_off_heap_header*) etp; erts_deref_dist_entry(dep); BIF_RET(make_external_pid(etp)); } diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c index c68392fad4..3fd714f9c2 100644 --- a/erts/emulator/beam/binary.c +++ b/erts/emulator/beam/binary.c @@ -88,8 +88,8 @@ new_binary(Process *p, byte *buf, int len) pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE); pb->thing_word = HEADER_PROC_BIN; pb->size = len; - pb->next = MSO(p).mso; - MSO(p).mso = pb; + pb->next = MSO(p).first; + MSO(p).first = (struct erl_off_heap_header*)pb; pb->val = bptr; pb->bytes = (byte*) bptr->orig_bytes; pb->flags = 0; @@ -127,8 +127,8 @@ Eterm erts_new_mso_binary(Process *p, byte *buf, int len) pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE); pb->thing_word = HEADER_PROC_BIN; pb->size = len; - pb->next = MSO(p).mso; - MSO(p).mso = pb; + pb->next = MSO(p).first; + MSO(p).first = (struct erl_off_heap_header*)pb; pb->val = bptr; pb->bytes = (byte*) bptr->orig_bytes; pb->flags = 0; @@ -487,16 +487,6 @@ BIF_RETTYPE split_binary_2(BIF_ALIST_2) BIF_ERROR(BIF_P, BADARG); } -void -erts_cleanup_mso(ProcBin* pb) -{ - while (pb != NULL) { - ProcBin* next = pb->next; - if (erts_refc_dectest(&pb->val->refc, 0) == 0) - erts_bin_free(pb->val); - pb = next; - } -} /* * Local functions. diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 857cb177c8..f339e19761 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -611,29 +611,29 @@ static void bin_check(void) { Process *rp; - ProcBin *bp; - int i, printed; + struct erl_off_heap_header* hdr; + int i, printed = 0; for (i=0; i < erts_max_processes; i++) { if ((rp = process_tab[i]) == NULL) continue; - if (!(bp = rp->off_heap.mso)) - continue; - printed = 0; - while (bp) { - if (printed == 0) { - erts_printf("Process %T holding binary data \n", rp->id); - printed = 1; + for (hdr = rp->off_heap.first; hdr; hdr = hdr->next) { + if (hdr->thing_word == HEADER_PROC_BIN) { + ProcBin *bp = (ProcBin*) hdr; + if (!printed) { + erts_printf("Process %T holding binary data \n", rp->id); + printed = 1; + } + erts_printf("0x%08lx orig_size: %ld, norefs = %ld\n", + (unsigned long)bp->val, + (long)bp->val->orig_size, + erts_smp_atomic_read(&bp->val->refc)); } - erts_printf("0x%08lx orig_size: %ld, norefs = %ld\n", - (unsigned long)bp->val, - (long)bp->val->orig_size, - erts_smp_atomic_read(&bp->val->refc)); - - bp = bp->next; } - if (printed == 1) + if (printed) { erts_printf("--------------------------------------\n"); + printed = 0; + } } /* db_bin_check() has to be rewritten for the AVL trees... */ /*db_bin_check();*/ diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index 521a1b1788..57be0169ba 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -314,9 +314,9 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) *argp = make_binary(hbot); pb = (ProcBin*) hbot; erts_refc_inc(&pb->val->refc, 2); - pb->next = off_heap->mso; + pb->next = off_heap->first; pb->flags = 0; - off_heap->mso = pb; + off_heap->first = (struct erl_off_heap_header*) pb; off_heap->overhead += pb->size / sizeof(Eterm); } break; @@ -363,9 +363,9 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) to->val = from->val; erts_refc_inc(&to->val->refc, 2); to->bytes = from->bytes + offset; - to->next = off_heap->mso; + to->next = off_heap->first; to->flags = 0; - off_heap->mso = to; + off_heap->first = (struct erl_off_heap_header*) to; off_heap->overhead += to->size / sizeof(Eterm); } *argp = make_binary(hbot); @@ -396,8 +396,8 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) } #ifndef HYBRID /* FIND ME! */ funp = (ErlFunThing *) tp; - funp->next = off_heap->funs; - off_heap->funs = funp; + funp->next = off_heap->first; + off_heap->first = (struct erl_off_heap_header*) funp; erts_refc_inc(&funp->fe->refc, 2); #endif *argp = make_fun(tp); @@ -416,8 +416,8 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) *htop++ = *objp++; } - etp->next = off_heap->externals; - off_heap->externals = etp; + etp->next = off_heap->first; + off_heap->first = (struct erl_off_heap_header*)etp; erts_refc_inc(&etp->node->refc, 2); *argp = make_external(tp); @@ -650,8 +650,8 @@ Eterm copy_struct_lazy(Process *from, Eterm orig, Uint offs) *hp++ = *objp++; } erts_refc_inc(&pb->val->refc, 2); - pb->next = erts_global_offheap.mso; - erts_global_offheap.mso = pb; + pb->next = erts_global_offheap.first; + erts_global_offheap.first = pb; erts_global_offheap.overhead += pb->size / sizeof(Eterm); continue; } @@ -672,9 +672,9 @@ Eterm copy_struct_lazy(Process *from, Eterm orig, Uint offs) while (i--) { *hp++ = *objp++; } -#ifndef HYBRID // FIND ME! - funp->next = erts_global_offheap.funs; - erts_global_offheap.funs = funp; +#ifndef HYBRID /* FIND ME! */ + funp->next = erts_global_offheap.first; + erts_global_offheap.first = funp; erts_refc_inc(&funp->fe->refc, 2); #endif for (i = k; i < j; i++) { @@ -718,8 +718,8 @@ Eterm copy_struct_lazy(Process *from, Eterm orig, Uint offs) *hp++ = *objp++; } - etp->next = erts_global_offheap.externals; - erts_global_offheap.externals = etp; + etp->next = erts_global_offheap.first; + erts_global_offheap.first = etp; erts_refc_inc(&etp->node->refc, 2); continue; } @@ -775,8 +775,8 @@ Eterm copy_struct_lazy(Process *from, Eterm orig, Uint offs) to_bin->size = real_size; to_bin->val = from_bin->val; to_bin->bytes = from_bin->bytes + sub_offset; - to_bin->next = erts_global_offheap.mso; - erts_global_offheap.mso = to_bin; + to_bin->next = erts_global_offheap.first; + erts_global_offheap.first = to_bin; erts_global_offheap.overhead += to_bin->size / sizeof(Eterm); res_binary=make_binary(to_bin); hp += PROC_BIN_SIZE; @@ -910,57 +910,43 @@ copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) break; case REFC_BINARY_SUBTAG: { - ProcBin* pb = (ProcBin *) (hp-1); - int tari = thing_arityval(val); - - sz -= tari; - while (tari--) { - *hp++ = *tp++; - } + ProcBin* pb = (ProcBin *) (tp-1); erts_refc_inc(&pb->val->refc, 2); - pb->next = off_heap->mso; - off_heap->mso = pb; off_heap->overhead += pb->size / sizeof(Eterm); } - break; + goto off_heap_common; + case FUN_SUBTAG: { -#ifndef HYBRID /* FIND ME! */ - ErlFunThing* funp = (ErlFunThing *) (hp-1); -#endif - int tari = thing_arityval(val); - - sz -= tari; - while (tari--) { - *hp++ = *tp++; - } -#ifndef HYBRID /* FIND ME! */ - funp->next = off_heap->funs; - off_heap->funs = funp; + ErlFunThing* funp = (ErlFunThing *) (tp-1); erts_refc_inc(&funp->fe->refc, 2); -#endif } - break; + goto off_heap_common; + case EXTERNAL_PID_SUBTAG: case EXTERNAL_PORT_SUBTAG: case EXTERNAL_REF_SUBTAG: { - ExternalThing* etp = (ExternalThing *) (hp-1); + ExternalThing* etp = (ExternalThing *) (tp-1); + erts_refc_inc(&etp->node->refc, 2); + } + off_heap_common: + { + struct erl_off_heap_header* ohh = (struct erl_off_heap_header*)(hp-1); int tari = thing_arityval(val); - + sz -= tari; while (tari--) { *hp++ = *tp++; } - etp->next = off_heap->externals; - off_heap->externals = etp; - erts_refc_inc(&etp->node->refc, 2); + ohh->next = off_heap->first; + off_heap->first = ohh; } break; default: { int tari = header_arity(val); - + sz -= tari; while (tari--) { *hp++ = *tp++; @@ -1029,12 +1015,6 @@ void move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first, static void move_one_frag(Eterm** hpp, Eterm* src, Uint src_sz, ErlOffHeap* off_heap) { - union { - Uint *up; - ProcBin *pbp; - ErlFunThing *efp; - ExternalThing *etp; - } ohe; Eterm* ptr = src; Eterm* end = ptr + src_sz; Eterm dummy_ref; @@ -1046,23 +1026,17 @@ move_one_frag(Eterm** hpp, Eterm* src, Uint src_sz, ErlOffHeap* off_heap) val = *ptr; ASSERT(val != ERTS_HOLE_MARKER); if (is_header(val)) { + struct erl_off_heap_header* hdr = (struct erl_off_heap_header*)hp; ASSERT(ptr + header_arity(val) < end); - ohe.up = hp; MOVE_BOXED(ptr, val, hp, &dummy_ref); switch (val & _HEADER_SUBTAG_MASK) { case REFC_BINARY_SUBTAG: - ohe.pbp->next = off_heap->mso; - off_heap->mso = ohe.pbp; - break; case FUN_SUBTAG: - ohe.efp->next = off_heap->funs; - off_heap->funs = ohe.efp; - break; case EXTERNAL_PID_SUBTAG: case EXTERNAL_PORT_SUBTAG: case EXTERNAL_REF_SUBTAG: - ohe.etp->next = off_heap->externals; - off_heap->externals = ohe.etp; + hdr->next = off_heap->first; + off_heap->first = hdr; break; } } diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index d465017949..16b6aeac3f 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -922,12 +922,8 @@ int erts_net_message(Port *prt, UseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE); /* Thanks to Luke Gorrie */ - off_heap.mso = NULL; -#ifndef HYBRID /* FIND ME! */ - off_heap.funs = NULL; -#endif + off_heap.first = NULL; off_heap.overhead = 0; - off_heap.externals = NULL; ERTS_SMP_CHK_NO_PROC_LOCKS; @@ -1434,16 +1430,8 @@ int erts_net_message(Port *prt, } } - if (off_heap.mso) { - erts_cleanup_mso(off_heap.mso); - } - if (off_heap.externals) { - erts_cleanup_externals(off_heap.externals); - } + erts_cleanup_offheap(&off_heap); #ifndef HYBRID /* FIND ME! */ - if (off_heap.funs) { - erts_cleanup_funs(off_heap.funs); - } if (ctl != ctl_default) { erts_free(ERTS_ALC_T_DCTRL_BUF, (void *) ctl); } @@ -1453,16 +1441,8 @@ int erts_net_message(Port *prt, return 0; data_error: - if (off_heap.mso) { - erts_cleanup_mso(off_heap.mso); - } - if (off_heap.externals) { - erts_cleanup_externals(off_heap.externals); - } + erts_cleanup_offheap(&off_heap); #ifndef HYBRID /* FIND ME! */ - if (off_heap.funs) { - erts_cleanup_funs(off_heap.funs); - } if (ctl != ctl_default) { erts_free(ERTS_ALC_T_DCTRL_BUF, (void *) ctl); } diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c index 82f1e06e8e..1d8fd11b7b 100644 --- a/erts/emulator/beam/erl_bif_binary.c +++ b/erts/emulator/beam/erl_bif_binary.c @@ -2511,8 +2511,8 @@ BIF_RETTYPE binary_copy_trap(BIF_ALIST_2) pb = (ProcBin *) HAlloc(BIF_P, PROC_BIN_SIZE); pb->thing_word = HEADER_PROC_BIN; pb->size = target_size; - pb->next = MSO(BIF_P).mso; - MSO(BIF_P).mso = pb; + pb->next = MSO(BIF_P).first; + MSO(BIF_P).first = (struct erl_off_heap_header*) pb; pb->val = save; pb->bytes = t; pb->flags = 0; diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 48cda52612..ca1ab44012 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -122,22 +122,26 @@ static char erts_system_version[] = ("Erlang " ERLANG_OTP_RELEASE #endif static Eterm -bld_bin_list(Uint **hpp, Uint *szp, ProcBin* pb) +bld_bin_list(Uint **hpp, Uint *szp, ErlOffHeap* oh) { + struct erl_off_heap_header* ohh; Eterm res = NIL; Eterm tuple; - for (; pb; pb = pb->next) { - Eterm val = erts_bld_uword(hpp, szp, (UWord) pb->val); - Eterm orig_size = erts_bld_uint(hpp, szp, pb->val->orig_size); - - if (szp) - *szp += 4+2; - if (hpp) { - Uint refc = (Uint) erts_smp_atomic_read(&pb->val->refc); - tuple = TUPLE3(*hpp, val, orig_size, make_small(refc)); - res = CONS(*hpp + 4, tuple, res); - *hpp += 4+2; + for (ohh = oh->first; ohh; ohh = ohh->next) { + if (ohh->thing_word == HEADER_PROC_BIN) { + ProcBin* pb = (ProcBin*) ohh; + Eterm val = erts_bld_uword(hpp, szp, (UWord) pb->val); + Eterm orig_size = erts_bld_uint(hpp, szp, pb->val->orig_size); + + if (szp) + *szp += 4+2; + if (hpp) { + Uint refc = (Uint) erts_smp_atomic_read(&pb->val->refc); + tuple = TUPLE3(*hpp, val, orig_size, make_small(refc)); + res = CONS(*hpp + 4, tuple, res); + *hpp += 4+2; + } } } return res; @@ -176,10 +180,10 @@ static void do_make_one_mon_element(ErtsMonitor *mon, void * vpmlc) Eterm tup; Eterm r = (IS_CONST(mon->ref) ? mon->ref - : STORE_NC(&(pmlc->hp), &MSO(pmlc->p).externals, mon->ref)); + : STORE_NC(&(pmlc->hp), &MSO(pmlc->p), mon->ref)); Eterm p = (IS_CONST(mon->pid) ? mon->pid - : STORE_NC(&(pmlc->hp), &MSO(pmlc->p).externals, mon->pid)); + : STORE_NC(&(pmlc->hp), &MSO(pmlc->p), mon->pid)); tup = TUPLE5(pmlc->hp, pmlc->tag, make_small(mon->type), r, p, mon->name); pmlc->hp += 6; pmlc->res = CONS(pmlc->hp, tup, pmlc->res); @@ -240,7 +244,7 @@ static void do_make_one_lnk_element(ErtsLink *lnk, void * vpllc) Eterm old_res, targets = NIL; Eterm p = (IS_CONST(lnk->pid) ? lnk->pid - : STORE_NC(&(pllc->hp), &MSO(pllc->p).externals, lnk->pid)); + : STORE_NC(&(pllc->hp), &MSO(pllc->p), lnk->pid)); if (lnk->type == LINK_NODE) { targets = make_small(ERTS_LINK_REFC(lnk)); } else if (ERTS_LINK_ROOT(lnk) != NULL) { @@ -1225,7 +1229,7 @@ process_info_aux(Process *BIF_P, hp = HAlloc(BIF_P, 3 + mic.sz); res = NIL; for (i = 0; i < mic.mi_i; i++) { - item = STORE_NC(&hp, &MSO(BIF_P).externals, mic.mi[i].entity); + item = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity); res = CONS(hp, item, res); hp += 2; } @@ -1258,9 +1262,7 @@ process_info_aux(Process *BIF_P, else { /* Monitor by pid. Build {process, Pid} and cons it. */ Eterm t; - Eterm pid = STORE_NC(&hp, - &MSO(BIF_P).externals, - mic.mi[i].entity); + Eterm pid = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity); t = TUPLE2(hp, am_process, pid); hp += 3; res = CONS(hp, t, res); @@ -1282,7 +1284,7 @@ process_info_aux(Process *BIF_P, res = NIL; for (i = 0; i < mic.mi_i; ++i) { - item = STORE_NC(&hp, &MSO(BIF_P).externals, mic.mi[i].entity); + item = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity); res = CONS(hp, item, res); hp += 2; } @@ -1491,7 +1493,7 @@ process_info_aux(Process *BIF_P, case am_group_leader: { int sz = NC_HEAP_SIZE(rp->group_leader); hp = HAlloc(BIF_P, 3 + sz); - res = STORE_NC(&hp, &MSO(BIF_P).externals, rp->group_leader); + res = STORE_NC(&hp, &MSO(BIF_P), rp->group_leader); break; } @@ -1516,9 +1518,9 @@ process_info_aux(Process *BIF_P, case am_binary: { Uint sz = 3; - (void) bld_bin_list(NULL, &sz, MSO(rp).mso); + (void) bld_bin_list(NULL, &sz, &MSO(rp)); hp = HAlloc(BIF_P, sz); - res = bld_bin_list(&hp, NULL, MSO(rp).mso); + res = bld_bin_list(&hp, NULL, &MSO(rp)); break; } @@ -2678,7 +2680,7 @@ BIF_RETTYPE port_info_2(BIF_ALIST_2) hp = HAlloc(BIF_P, 3 + mic.sz); res = NIL; for (i = 0; i < mic.mi_i; i++) { - item = STORE_NC(&hp, &MSO(BIF_P).externals, mic.mi[i].entity); + item = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity); res = CONS(hp, item, res); hp += 2; } @@ -2698,7 +2700,7 @@ BIF_RETTYPE port_info_2(BIF_ALIST_2) res = NIL; for (i = 0; i < mic.mi_i; i++) { Eterm t; - item = STORE_NC(&hp, &MSO(BIF_P).externals, mic.mi[i].entity); + item = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity); t = TUPLE2(hp, am_process, item); hp += 3; res = CONS(hp, t, res); diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c index 443cac9033..0509e51a6f 100644 --- a/erts/emulator/beam/erl_bif_trace.c +++ b/erts/emulator/beam/erl_bif_trace.c @@ -2161,7 +2161,7 @@ trace_delivered_1(BIF_ALIST_1) #ifdef ERTS_SMP bp = new_message_buffer(REF_THING_SIZE + 4); hp = &bp->mem[0]; - msg_ref = STORE_NC(&hp, &bp->off_heap.externals, ref); + msg_ref = STORE_NC(&hp, &bp->off_heap, ref); #else hp = HAlloc(BIF_P, 4); msg_ref = ref; diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c index defe18c92b..96faa673f6 100644 --- a/erts/emulator/beam/erl_bits.c +++ b/erts/emulator/beam/erl_bits.c @@ -1335,8 +1335,8 @@ erts_bs_append(Process* c_p, Eterm* reg, Uint live, Eterm build_size_term, hp += PROC_BIN_SIZE; pb->thing_word = HEADER_PROC_BIN; pb->size = used_size_in_bytes; - pb->next = MSO(c_p).mso; - MSO(c_p).mso = pb; + pb->next = MSO(c_p).first; + MSO(c_p).first = (struct erl_off_heap_header*)pb; pb->val = bptr; pb->bytes = (byte*) bptr->orig_bytes; pb->flags = PB_IS_WRITABLE | PB_ACTIVE_WRITER; @@ -1506,8 +1506,8 @@ erts_bs_init_writable(Process* p, Eterm sz) hp += PROC_BIN_SIZE; pb->thing_word = HEADER_PROC_BIN; pb->size = 0; - pb->next = MSO(p).mso; - MSO(p).mso = pb; + pb->next = MSO(p).first; + MSO(p).first = (struct erl_off_heap_header*) pb; pb->val = bptr; pb->bytes = (byte*) bptr->orig_bytes; pb->flags = PB_IS_WRITABLE | PB_ACTIVE_WRITER; diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 124129a371..96008ccef0 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -2751,11 +2751,7 @@ static void db_finalize_dbterm_hash(DbUpdateHandle* handle) newDbTerm = &newp->dbterm; newDbTerm->size = handle->new_size; - newDbTerm->off_heap.mso = NULL; - newDbTerm->off_heap.externals = NULL; - #ifndef HYBRID /* FIND ME! */ - newDbTerm->off_heap.funs = NULL; - #endif + newDbTerm->off_heap.first = NULL; newDbTerm->off_heap.overhead = 0; /* make a flat copy */ diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index b6b3cabafe..fed5298e44 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -2588,11 +2588,7 @@ static void db_finalize_dbterm_tree(DbUpdateHandle* handle) newDbTerm = &newp->dbterm; newDbTerm->size = handle->new_size; - newDbTerm->off_heap.mso = NULL; - newDbTerm->off_heap.externals = NULL; - #ifndef HYBRID /* FIND ME! */ - newDbTerm->off_heap.funs = NULL; - #endif + newDbTerm->off_heap.first = NULL; newDbTerm->off_heap.overhead = 0; /* make a flat copy */ diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index 93a47eb76f..48e0080525 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -363,12 +363,7 @@ static ErtsMatchPseudoProcess *match_pseudo_process; static ERTS_INLINE void cleanup_match_pseudo_process(ErtsMatchPseudoProcess *mpsp, int keep_heap) { - if (mpsp->process.mbuf - || mpsp->process.off_heap.mso -#ifndef HYBRID /* FIND ME! */ - || mpsp->process.off_heap.funs -#endif - || mpsp->process.off_heap.externals) { + if (mpsp->process.mbuf || mpsp->process.off_heap.first) { erts_cleanup_empty_process(&mpsp->process); } #ifdef DEBUG @@ -2495,11 +2490,7 @@ void* db_get_term(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj) p = (DbTerm*) ((void *)(((char *) structp) + offset)); } p->size = size; - p->off_heap.mso = NULL; - p->off_heap.externals = NULL; -#ifndef HYBRID /* FIND ME! */ - p->off_heap.funs = NULL; -#endif + p->off_heap.first = NULL; p->off_heap.overhead = 0; top = DBTERM_BUF(p); diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c index 15d9538301..1f74393a96 100644 --- a/erts/emulator/beam/erl_fun.c +++ b/erts/emulator/beam/erl_fun.c @@ -192,20 +192,6 @@ erts_erase_fun_entry(ErlFunEntry* fe) erts_fun_write_unlock(); } -#ifndef HYBRID /* FIND ME! */ -void -erts_cleanup_funs(ErlFunThing* funp) -{ - while (funp) { - ErlFunEntry* fe = funp->fe; - if (erts_refc_dectest(&fe->refc, 0) == 0) { - erts_erase_fun_entry(fe); - } - funp = funp->next; - } -} -#endif - void erts_cleanup_funs_on_purge(BeamInstr* start, BeamInstr* end) { diff --git a/erts/emulator/beam/erl_fun.h b/erts/emulator/beam/erl_fun.h index 944d4b3df5..2f165afa06 100644 --- a/erts/emulator/beam/erl_fun.h +++ b/erts/emulator/beam/erl_fun.h @@ -53,10 +53,10 @@ typedef struct erl_fun_entry { typedef struct erl_fun_thing { Eterm thing_word; /* Subtag FUN_SUBTAG. */ + ErlFunEntry* fe; /* Pointer to fun entry. */ #ifndef HYBRID /* FIND ME! */ - struct erl_fun_thing* next; /* Next fun in mso list. */ + struct erl_off_heap_header* next; #endif - ErlFunEntry* fe; /* Pointer to fun entry. */ #ifdef HIPE UWord* native_address; /* Native code for the fun. */ #endif diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index a19e090f1e..e9411389a0 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -110,9 +110,7 @@ static Uint adjust_after_fullsweep(Process *p, int size_before, int need, Eterm *objv, int nobj); static void shrink_new_heap(Process *p, Uint new_sz, Eterm *objv, int nobj); static void grow_new_heap(Process *p, Uint new_sz, Eterm* objv, int nobj); -static void sweep_proc_bins(Process *p, int fullsweep); -static void sweep_proc_funs(Process *p, int fullsweep); -static void sweep_proc_externals(Process *p, int fullsweep); +static void sweep_off_heap(Process *p, int fullsweep); static void offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size); static void offset_heap_ptr(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size); static void offset_rootset(Process *p, Sint offs, char* area, Uint area_size, @@ -145,6 +143,13 @@ erts_init_gc(void) { int i = 0; + ASSERT(offsetof(ProcBin,thing_word) == offsetof(struct erl_off_heap_header,thing_word)); + ASSERT(offsetof(ProcBin,thing_word) == offsetof(ErlFunThing,thing_word)); + ASSERT(offsetof(ProcBin,thing_word) == offsetof(ExternalThing,header)); + ASSERT(offsetof(ProcBin,next) == offsetof(struct erl_off_heap_header,next)); + ASSERT(offsetof(ProcBin,next) == offsetof(ErlFunThing,next)); + ASSERT(offsetof(ProcBin,next) == offsetof(ExternalThing,next)); + erts_smp_spinlock_init(&info_lck, "gc_info"); garbage_cols = 0; reclaimed = 0; @@ -286,25 +291,14 @@ erts_offset_heap_ptr(Eterm* hp, Uint sz, Sint offs, offset_heap_ptr(hp, sz, offs, (char *) low, ((char *)high)-((char *)low)); } + #define ptr_within(ptr, low, high) ((ptr) < (high) && (ptr) >= (low)) void erts_offset_off_heap(ErlOffHeap *ohp, Sint offs, Eterm* low, Eterm* high) { - if (ohp->mso && ptr_within((Eterm *)ohp->mso, low, high)) { - Eterm** uptr = (Eterm**) (void *) &ohp->mso; - *uptr += offs; - } - -#ifndef HYBRID /* FIND ME! */ - if (ohp->funs && ptr_within((Eterm *)ohp->funs, low, high)) { - Eterm** uptr = (Eterm**) (void *) &ohp->funs; - *uptr += offs; - } -#endif - - if (ohp->externals && ptr_within((Eterm *)ohp->externals, low, high)) { - Eterm** uptr = (Eterm**) (void *) &ohp->externals; + if (ohp->first && ptr_within((Eterm *)ohp->first, low, high)) { + Eterm** uptr = (Eterm**) (void *) &ohp->first; *uptr += offs; } } @@ -504,14 +498,8 @@ erts_garbage_collect_hibernate(Process* p) cleanup_rootset(&rootset); - if (MSO(p).mso) { - sweep_proc_bins(p, 1); - } - if (MSO(p).funs) { - sweep_proc_funs(p, 1); - } - if (MSO(p).externals) { - sweep_proc_externals(p, 1); + if (MSO(p).first) { + sweep_off_heap(p, 1); } /* @@ -1041,15 +1029,8 @@ do_minor(Process *p, int new_sz, Eterm* objv, int nobj) OLD_HTOP(p) = old_htop; HIGH_WATER(p) = (HEAP_START(p) != HIGH_WATER(p)) ? n_heap : n_htop; - if (MSO(p).mso) { - sweep_proc_bins(p, 0); - } - - if (MSO(p).funs) { - sweep_proc_funs(p, 0); - } - if (MSO(p).externals) { - sweep_proc_externals(p, 0); + if (MSO(p).first) { + sweep_off_heap(p, 0); } #ifdef HARDDEBUG @@ -1271,17 +1252,11 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) } } - if (MSO(p).mso) { - sweep_proc_bins(p, 1); - } - if (MSO(p).funs) { - sweep_proc_funs(p, 1); - } - if (MSO(p).externals) { - sweep_proc_externals(p, 1); + if (MSO(p).first) { + sweep_off_heap(p, 1); } - if (OLD_HEAP(p) != NULL) { + if (OLD_HEAP(p) != NULL) { ERTS_HEAP_FREE(ERTS_ALC_T_OLD_HEAP, OLD_HEAP(p), (OLD_HEND(p) - OLD_HEAP(p)) * sizeof(Eterm)); @@ -1305,6 +1280,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) HIGH_WATER(p) = HEAP_TOP(p); ErtsGcQuickSanityCheck(p); + /* * Copy newly received message onto the end of the new heap. */ @@ -2044,108 +2020,24 @@ next_vheap_size(Process* p, Uint vheap, Uint vheap_sz) { return vheap_sz < p->min_vheap_size ? p->min_vheap_size : vheap_sz; } -static void -sweep_proc_externals(Process *p, int fullsweep) -{ - ExternalThing** prev; - ExternalThing* ptr; - char* oh = 0; - Uint oh_size = 0; - - if (fullsweep == 0) { - oh = (char *) OLD_HEAP(p); - oh_size = (char *) OLD_HEND(p) - oh; - } - - prev = &MSO(p).externals; - ptr = MSO(p).externals; - - while (ptr) { - Eterm* ppt = (Eterm *) ptr; - - if (IS_MOVED_BOXED(*ppt)) { /* Object is alive */ - ExternalThing* ro = external_thing_ptr(*ppt); - - *prev = ro; /* Patch to moved pos */ - prev = &ro->next; - ptr = ro->next; - } else if (in_area(ppt, oh, oh_size)) { - /* - * Object resides on old heap, and we just did a - * generational collection - keep object in list. - */ - prev = &ptr->next; - ptr = ptr->next; - } else { /* Object has not been moved - deref it */ - erts_deref_node_entry(ptr->node); - *prev = ptr = ptr->next; - } - } - ASSERT(*prev == NULL); -} - -static void -sweep_proc_funs(Process *p, int fullsweep) -{ - ErlFunThing** prev; - ErlFunThing* ptr; - char* oh = 0; - Uint oh_size = 0; - - if (fullsweep == 0) { - oh = (char *) OLD_HEAP(p); - oh_size = (char *) OLD_HEND(p) - oh; - } - - prev = &MSO(p).funs; - ptr = MSO(p).funs; - - while (ptr) { - Eterm* ppt = (Eterm *) ptr; - - if (IS_MOVED_BOXED(*ppt)) { /* Object is alive */ - ErlFunThing* ro = (ErlFunThing *) fun_val(*ppt); - - *prev = ro; /* Patch to moved pos */ - prev = &ro->next; - ptr = ro->next; - } else if (in_area(ppt, oh, oh_size)) { - /* - * Object resides on old heap, and we just did a - * generational collection - keep object in list. - */ - prev = &ptr->next; - ptr = ptr->next; - } else { /* Object has not been moved - deref it */ - ErlFunEntry* fe = ptr->fe; - - *prev = ptr = ptr->next; - if (erts_refc_dectest(&fe->refc, 0) == 0) { - erts_erase_fun_entry(fe); - } - } - } - ASSERT(*prev == NULL); -} - struct shrink_cand_data { - ProcBin* new_candidates; - ProcBin* new_candidates_end; - ProcBin* old_candidates; + struct erl_off_heap_header* new_candidates; + struct erl_off_heap_header* new_candidates_end; + struct erl_off_heap_header* old_candidates; Uint no_of_candidates; Uint no_of_active; }; static ERTS_INLINE void link_live_proc_bin(struct shrink_cand_data *shrink, - ProcBin ***prevppp, - ProcBin **pbpp, + struct erl_off_heap_header*** prevppp, + struct erl_off_heap_header** currpp, int new_heap) { - ProcBin *pbp = *pbpp; - - *pbpp = pbp->next; + ProcBin *pbp = (ProcBin*) *currpp; + ASSERT(**prevppp == *currpp); + *currpp = pbp->next; if (pbp->flags & (PB_ACTIVE_WRITER|PB_IS_WRITABLE)) { ASSERT(((pbp->flags & (PB_ACTIVE_WRITER|PB_IS_WRITABLE)) == (PB_ACTIVE_WRITER|PB_IS_WRITABLE)) @@ -2162,15 +2054,16 @@ link_live_proc_bin(struct shrink_cand_data *shrink, /* Our allocators are 8 byte aligned, i.e., shrinking with less than 8 bytes will have no real effect */ if (unused >= 8) { /* A shrink candidate; save in candidate list */ + **prevppp = pbp->next; if (new_heap) { if (!shrink->new_candidates) - shrink->new_candidates_end = pbp; + shrink->new_candidates_end = (struct erl_off_heap_header*)pbp; pbp->next = shrink->new_candidates; - shrink->new_candidates = pbp; + shrink->new_candidates = (struct erl_off_heap_header*)pbp; } else { pbp->next = shrink->old_candidates; - shrink->old_candidates = pbp; + shrink->old_candidates = (struct erl_off_heap_header*)pbp; } shrink->no_of_candidates++; return; @@ -2178,67 +2071,101 @@ link_live_proc_bin(struct shrink_cand_data *shrink, } } - /* Not a shrink candidate; keep in original mso list */ - **prevppp = pbp; + /* Not a shrink candidate; keep in original mso list */ *prevppp = &pbp->next; - } static void -sweep_proc_bins(Process *p, int fullsweep) +sweep_off_heap(Process *p, int fullsweep) { struct shrink_cand_data shrink = {0}; - ProcBin** prev; - ProcBin* ptr; - Binary* bptr; - char* oh = NULL; - Uint oh_size = 0; + struct erl_off_heap_header* ptr; + struct erl_off_heap_header** prev; + char* oheap = NULL; + Uint oheap_sz = 0; Uint bin_vheap = 0; +#ifdef DEBUG + int seen_mature = 0; +#endif if (fullsweep == 0) { - oh = (char *) OLD_HEAP(p); - oh_size = (char *) OLD_HEND(p) - oh; + oheap = (char *) OLD_HEAP(p); + oheap_sz = (char *) OLD_HEND(p) - oheap; } BIN_OLD_VHEAP(p) = 0; - prev = &MSO(p).mso; - ptr = MSO(p).mso; + prev = &MSO(p).first; + ptr = MSO(p).first; - /* - * Note: In R7 we no longer force a fullsweep when we find binaries - * on the old heap. The reason is that with the introduction of the - * bit syntax we can expect binaries to be used a lot more. Note that - * in earlier releases a brand new binary (or any other term) could - * be put on the old heap during a gen-gc fullsweep, but this is - * no longer the case in R7. + /* Firts part of the list will reside on the (old) new-heap. + * Keep if moved, otherwise deref. */ while (ptr) { - Eterm* ppt = (Eterm *) ptr; - - if (IS_MOVED_BOXED(*ppt)) { /* Object is alive */ - bin_vheap += ptr->size / sizeof(Eterm); - ptr = (ProcBin*) binary_val(*ppt); - link_live_proc_bin(&shrink, - &prev, - &ptr, - !in_area(ptr, oh, oh_size)); - } else if (in_area(ppt, oh, oh_size)) { - /* - * Object resides on old heap, and we just did a - * generational collection - keep object in list. - */ - BIN_OLD_VHEAP(p) += ptr->size / sizeof(Eterm); /* for binary gc (words)*/ - link_live_proc_bin(&shrink, &prev, &ptr, 0); - } else { /* Object has not been moved - deref it */ - - *prev = ptr->next; - bptr = ptr->val; - if (erts_refc_dectest(&bptr->refc, 0) == 0) - erts_bin_free(bptr); - ptr = *prev; - } + if (IS_MOVED_BOXED(ptr->thing_word)) { + ASSERT(!in_area(ptr, oheap, oheap_sz)); + *prev = ptr = (struct erl_off_heap_header*) boxed_val(ptr->thing_word); + ASSERT(!IS_MOVED_BOXED(ptr->thing_word)); + if (ptr->thing_word == HEADER_PROC_BIN) { + int to_new_heap = !in_area(ptr, oheap, oheap_sz); + ASSERT(to_new_heap == !seen_mature || (!to_new_heap && (seen_mature=1))); + if (to_new_heap) { + bin_vheap += ptr->u.size / sizeof(Eterm); + } else { + BIN_OLD_VHEAP(p) += ptr->u.size / sizeof(Eterm); /* for binary gc (words)*/ + } + link_live_proc_bin(&shrink, &prev, &ptr, to_new_heap); + } + else { + prev = &ptr->next; + ptr = ptr->next; + } + } + else if (!in_area(ptr, oheap, oheap_sz)) { + /* garbage */ + switch (thing_subtag(ptr->thing_word)) { + case REFC_BINARY_SUBTAG: + { + Binary* bptr = ((ProcBin*)ptr)->val; + if (erts_refc_dectest(&bptr->refc, 0) == 0) { + erts_bin_free(bptr); + } + break; + } + case FUN_SUBTAG: + { + ErlFunEntry* fe = ((ErlFunThing*)ptr)->fe; + if (erts_refc_dectest(&fe->refc, 0) == 0) { + erts_erase_fun_entry(fe); + } + break; + } + default: + ASSERT(is_external_header(ptr->thing_word)); + erts_deref_node_entry(((ExternalThing*)ptr)->node); + } + *prev = ptr = ptr->next; + } + else break; /* and let old-heap loop continue */ + } + + /* The rest of the list resides on old-heap, and we just did a + * generational collection - keep objects in list. + */ + while (ptr) { + ASSERT(in_area(ptr, oheap, oheap_sz)); + ASSERT(!IS_MOVED_BOXED(ptr->thing_word)); + if (ptr->thing_word == HEADER_PROC_BIN) { + BIN_OLD_VHEAP(p) += ptr->u.size / sizeof(Eterm); /* for binary gc (words)*/ + link_live_proc_bin(&shrink, &prev, &ptr, 0); + } + else { + ASSERT(is_fun_header(ptr->thing_word) || + is_external_header(ptr->thing_word)); + prev = &ptr->next; + ptr = ptr->next; + } } if (BIN_OLD_VHEAP(p) >= BIN_OLD_VHEAP_SZ(p)) { @@ -2254,7 +2181,8 @@ sweep_proc_bins(Process *p, int fullsweep) */ if (shrink.no_of_candidates) { - ProcBin *candlist[] = {shrink.new_candidates, shrink.old_candidates}; + ProcBin *candlist[] = { (ProcBin*)shrink.new_candidates, + (ProcBin*)shrink.old_candidates }; Uint leave_unused = 0; int i; @@ -2266,21 +2194,21 @@ sweep_proc_bins(Process *p, int fullsweep) } for (i = 0; i < sizeof(candlist)/sizeof(candlist[0]); i++) { - - for (ptr = candlist[i]; ptr; ptr = ptr->next) { - Uint new_size = ptr->size; + ProcBin* pb; + for (pb = candlist[i]; pb; pb = (ProcBin*)pb->next) { + Uint new_size = pb->size; if (leave_unused) { new_size += (new_size * 100) / leave_unused; /* Our allocators are 8 byte aligned, i.e., shrinking with less than 8 bytes will have no real effect */ - if (new_size + 8 >= ptr->val->orig_size) + if (new_size + 8 >= pb->val->orig_size) continue; } - ptr->val = erts_bin_realloc(ptr->val, new_size); - ptr->val->orig_size = new_size; - ptr->bytes = (byte *) ptr->val->orig_bytes; + pb->val = erts_bin_realloc(pb->val, new_size); + pb->val->orig_size = new_size; + pb->bytes = (byte *) pb->val->orig_bytes; } } @@ -2289,21 +2217,20 @@ sweep_proc_bins(Process *p, int fullsweep) * We now potentially have the mso list divided into three lists: * - shrink candidates on new heap (inactive writable with unused data) * - shrink candidates on old heap (inactive writable with unused data) - * - other binaries (read only + active writable ...) + * - other binaries (read only + active writable ...) + funs and externals * * Put them back together: new candidates -> other -> old candidates * This order will ensure that the list only refers from new * generation to old and never from old to new *which is important*. */ if (shrink.new_candidates) { - if (prev == &MSO(p).mso) /* empty other binaries list */ + if (prev == &MSO(p).first) /* empty other binaries list */ prev = &shrink.new_candidates_end->next; else - shrink.new_candidates_end->next = MSO(p).mso; - MSO(p).mso = shrink.new_candidates; + shrink.new_candidates_end->next = MSO(p).first; + MSO(p).first = shrink.new_candidates; } } - *prev = shrink.old_candidates; } @@ -2334,15 +2261,17 @@ offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size) tari = thing_arityval(val); switch (thing_subtag(val)) { case REFC_BINARY_SUBTAG: + case FUN_SUBTAG: + case EXTERNAL_PID_SUBTAG: + case EXTERNAL_PORT_SUBTAG: + case EXTERNAL_REF_SUBTAG: { - ProcBin* pb = (ProcBin*) hp; - Eterm** uptr = (Eterm **) (void *) &pb->next; + struct erl_off_heap_header* oh = (struct erl_off_heap_header*) hp; - if (*uptr && in_area((Eterm *)pb->next, area, area_size)) { + if (in_area(oh->next, area, area_size)) { + Eterm** uptr = (Eterm **) (void *) &oh->next; *uptr += offs; /* Patch the mso chain */ } - sz -= tari; - hp += tari + 1; } break; case BIN_MATCHSTATE_SUBTAG: @@ -2353,40 +2282,11 @@ offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size) mb->orig = offset_ptr(mb->orig, offs); mb->base = binary_bytes(mb->orig); } - sz -= tari; - hp += tari + 1; } break; - case FUN_SUBTAG: - { - ErlFunThing* funp = (ErlFunThing *) hp; - Eterm** uptr = (Eterm **) (void *) &funp->next; - - if (*uptr && in_area((Eterm *)funp->next, area, area_size)) { - *uptr += offs; - } - sz -= tari; - hp += tari + 1; - } - break; - case EXTERNAL_PID_SUBTAG: - case EXTERNAL_PORT_SUBTAG: - case EXTERNAL_REF_SUBTAG: - { - ExternalThing* etp = (ExternalThing *) hp; - Eterm** uptr = (Eterm **) (void *) &etp->next; - - if (*uptr && in_area((Eterm *)etp->next, area, area_size)) { - *uptr += offs; - } - sz -= tari; - hp += tari + 1; - } - break; - default: - sz -= tari; - hp += tari + 1; } + sz -= tari; + hp += tari + 1; break; } default: @@ -2423,18 +2323,8 @@ offset_heap_ptr(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size) static void offset_off_heap(Process* p, Sint offs, char* area, Uint area_size) { - if (MSO(p).mso && in_area((Eterm *)MSO(p).mso, area, area_size)) { - Eterm** uptr = (Eterm**) (void *) &MSO(p).mso; - *uptr += offs; - } - - if (MSO(p).funs && in_area((Eterm *)MSO(p).funs, area, area_size)) { - Eterm** uptr = (Eterm**) (void *) &MSO(p).funs; - *uptr += offs; - } - - if (MSO(p).externals && in_area((Eterm *)MSO(p).externals, area, area_size)) { - Eterm** uptr = (Eterm**) (void *) &MSO(p).externals; + if (MSO(p).first && in_area((Eterm *)MSO(p).first, area, area_size)) { + Eterm** uptr = (Eterm**) (void *) &MSO(p).first; *uptr += offs; } } @@ -2555,8 +2445,8 @@ do { \ __FILE__, __LINE__, #EXP); \ } while (0) -#ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_EXTERNAL_LIST -# define ERTS_EXTERNAL_VISITED_BIT ((Eterm) 1 << 31) +#ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_LIST +# define ERTS_OFFHEAP_VISITED_BIT ((Eterm) 1 << 31) #endif @@ -2566,62 +2456,45 @@ erts_check_off_heap2(Process *p, Eterm *htop) Eterm *oheap = (Eterm *) OLD_HEAP(p); Eterm *ohtop = (Eterm *) OLD_HTOP(p); int old; - ProcBin *pb; - ErlFunThing *eft; - ExternalThing *et; + union erl_off_heap_ptr u; old = 0; - for (pb = MSO(p).mso; pb; pb = pb->next) { - Eterm *ptr = (Eterm *) pb; - long refc = erts_refc_read(&pb->val->refc, 1); + for (u.hdr = MSO(p).first; u.hdr; u.hdr = u.hdr->next) { + long refc; + switch (thing_subtag(u.hdr->thing_word)) { + case REFC_BINARY_SUBTAG: + refc = erts_refc_read(&u.pb->val->refc, 1); + break; + case FUN_SUBTAG: + refc = erts_refc_read(&u.fun->fe->refc, 1); + break; + case EXTERNAL_PID_SUBTAG: + case EXTERNAL_PORT_SUBTAG: + case EXTERNAL_REF_SUBTAG: + refc = erts_refc_read(&u.ext->node->refc, 1); + break; + default: + ASSERT(!!"erts_check_off_heap2: Invalid thing_word"); + } ERTS_CHK_OFFHEAP_ASSERT(refc >= 1); +#ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_LIST + ERTS_CHK_OFFHEAP_ASSERT(!(u.hdr->thing_word & ERTS_EXTERNAL_VISITED_BIT)); + u.hdr->thing_word |= ERTS_OFFHEAP_VISITED_BIT; +#endif if (old) { - ERTS_CHK_OFFHEAP_ASSERT(oheap <= ptr && ptr < ohtop); + ERTS_CHK_OFFHEAP_ASSERT(oheap <= u.ep && u.ep < ohtop); } - else if (oheap <= ptr && ptr < ohtop) + else if (oheap <= u.ep && u.ep < ohtop) old = 1; else { - ERTS_CHK_OFFHEAP_ASSERT(within2(ptr, p, htop)); + ERTS_CHK_OFFHEAP_ASSERT(within2(u.ep, p, htop)); } } - old = 0; - for (eft = MSO(p).funs; eft; eft = eft->next) { - Eterm *ptr = (Eterm *) eft; - long refc = erts_refc_read(&eft->fe->refc, 1); - ERTS_CHK_OFFHEAP_ASSERT(refc >= 1); - if (old) - ERTS_CHK_OFFHEAP_ASSERT(oheap <= ptr && ptr < ohtop); - else if (oheap <= ptr && ptr < ohtop) - old = 1; - else - ERTS_CHK_OFFHEAP_ASSERT(within2(ptr, p, htop)); - } - - old = 0; - for (et = MSO(p).externals; et; et = et->next) { - Eterm *ptr = (Eterm *) et; - long refc = erts_refc_read(&et->node->refc, 1); - ERTS_CHK_OFFHEAP_ASSERT(refc >= 1); -#ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_EXTERNAL_LIST - ERTS_CHK_OFFHEAP_ASSERT(!(et->header & ERTS_EXTERNAL_VISITED_BIT)); -#endif - if (old) - ERTS_CHK_OFFHEAP_ASSERT(oheap <= ptr && ptr < ohtop); - else if (oheap <= ptr && ptr < ohtop) - old = 1; - else - ERTS_CHK_OFFHEAP_ASSERT(within2(ptr, p, htop)); -#ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_EXTERNAL_LIST - et->header |= ERTS_EXTERNAL_VISITED_BIT; -#endif - } - #ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_EXTERNAL_LIST - for (et = MSO(p).externals; et; et = et->next) - et->header &= ~ERTS_EXTERNAL_VISITED_BIT; + for (u.hdr = MSO(p).first; u.hdr; u.hdr = u.hdr->next) + u.hdr->thing_word &= ~ERTS_OFFHEAP_VISITED_BIT; #endif - } void diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index b63f3df7df..1f61dca230 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -30,6 +30,7 @@ #include "erl_message.h" #include "erl_process.h" #include "erl_nmgc.h" +#include "erl_binary.h" ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(message, ErlMessage, @@ -164,16 +165,25 @@ erts_resize_message_buffer(ErlHeapFragment *bp, Uint size, void erts_cleanup_offheap(ErlOffHeap *offheap) { - if (offheap->mso) { - erts_cleanup_mso(offheap->mso); - } -#ifndef HYBRID /* FIND ME! */ - if (offheap->funs) { - erts_cleanup_funs(offheap->funs); - } -#endif - if (offheap->externals) { - erts_cleanup_externals(offheap->externals); + union erl_off_heap_ptr u; + + for (u.hdr = offheap->first; u.hdr; u.hdr = u.hdr->next) { + switch (thing_subtag(u.hdr->thing_word)) { + case REFC_BINARY_SUBTAG: + if (erts_refc_dectest(&u.pb->val->refc, 0) == 0) { + erts_bin_free(u.pb->val); + } + break; + case FUN_SUBTAG: + if (erts_refc_dectest(&u.fun->fe->refc, 0) == 0) { + erts_erase_fun_entry(u.fun->fe); + } + break; + default: + ASSERT(is_external_header(u.hdr->thing_word)); + erts_deref_node_entry(u.ext->node); + break; + } } } @@ -201,41 +211,17 @@ link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp) MBUF_SIZE(proc) += bp->used_size; FLAGS(proc) |= F_FORCE_GC; - /* Move any binaries into the process */ - if (bp->off_heap.mso != NULL) { - ProcBin** next_p = &bp->off_heap.mso; + /* Move any off_heap's into the process */ + if (bp->off_heap.first != NULL) { + struct erl_off_heap_header** next_p = &bp->off_heap.first; while (*next_p != NULL) { next_p = &((*next_p)->next); } - *next_p = MSO(proc).mso; - MSO(proc).mso = bp->off_heap.mso; - bp->off_heap.mso = NULL; + *next_p = MSO(proc).first; + MSO(proc).first = bp->off_heap.first; + bp->off_heap.first = NULL; MSO(proc).overhead += bp->off_heap.overhead; } - - /* Move any funs into the process */ -#ifndef HYBRID - if (bp->off_heap.funs != NULL) { - ErlFunThing** next_p = &bp->off_heap.funs; - while (*next_p != NULL) { - next_p = &((*next_p)->next); - } - *next_p = MSO(proc).funs; - MSO(proc).funs = bp->off_heap.funs; - bp->off_heap.funs = NULL; - } -#endif - - /* Move any external things into the process */ - if (bp->off_heap.externals != NULL) { - ExternalThing** next_p = &bp->off_heap.externals; - while (*next_p != NULL) { - next_p = &((*next_p)->next); - } - *next_p = MSO(proc).externals; - MSO(proc).externals = bp->off_heap.externals; - bp->off_heap.externals = NULL; - } } } @@ -506,19 +492,7 @@ erts_link_mbuf_to_proc(struct process *proc, ErlHeapFragment *bp) void erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) { - /* Unions for typecasts avoids warnings about type-punned pointers and aliasing */ - union { - Uint** upp; - ProcBin **pbpp; - ErlFunThing **efpp; - ExternalThing **etpp; - } oh_list_pp, oh_el_next_pp; - union { - Uint *up; - ProcBin *pbp; - ErlFunThing *efp; - ExternalThing *etp; - } oh_el_p; + struct erl_off_heap_header* oh; Eterm term, token, *fhp, *hp; Sint offs; Uint sz; @@ -571,9 +545,7 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) hp = *hpp; offs = hp - fhp; - oh_list_pp.upp = NULL; - oh_el_next_pp.upp = NULL; /* Shut up compiler warning */ - oh_el_p.up = NULL; /* Shut up compiler warning */ + oh = NULL; while (sz--) { Uint cpy_sz; Eterm val = *fhp++; @@ -593,25 +565,11 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) case ARITYVAL_SUBTAG: break; case REFC_BINARY_SUBTAG: - oh_list_pp.pbpp = &off_heap->mso; - oh_el_p.up = (hp-1); - oh_el_next_pp.pbpp = &(oh_el_p.pbp)->next; - cpy_sz = thing_arityval(val); - goto cpy_words; case FUN_SUBTAG: -#ifndef HYBRID - oh_list_pp.efpp = &off_heap->funs; - oh_el_p.up = (hp-1); - oh_el_next_pp.efpp = &(oh_el_p.efp)->next; -#endif - cpy_sz = thing_arityval(val); - goto cpy_words; case EXTERNAL_PID_SUBTAG: case EXTERNAL_PORT_SUBTAG: case EXTERNAL_REF_SUBTAG: - oh_list_pp.etpp = &off_heap->externals; - oh_el_p.up = (hp-1); - oh_el_next_pp.etpp = &(oh_el_p.etp)->next; + oh = (struct erl_off_heap_header*) (hp-1); cpy_sz = thing_arityval(val); goto cpy_words; default: @@ -641,44 +599,13 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) case 1: *hp++ = *fhp++; default: break; } - if (oh_list_pp.upp) { -#ifdef HARD_DEBUG - Uint *dbg_old_oh_list_p = *oh_list_pp.upp; -#endif + if (oh) { /* Add to offheap list */ - *oh_el_next_pp.upp = *oh_list_pp.upp; - *oh_list_pp.upp = oh_el_p.up; - ASSERT(*hpp <= oh_el_p.up); - ASSERT(hp > oh_el_p.up); -#ifdef HARD_DEBUG - switch (val & _HEADER_SUBTAG_MASK) { - case REFC_BINARY_SUBTAG: - ASSERT(off_heap->mso == *oh_list_pp.pbpp); - ASSERT(off_heap->mso->next - == (ProcBin *) dbg_old_oh_list_p); - break; -#ifndef HYBRID - case FUN_SUBTAG: - ASSERT(off_heap->funs == *oh_list_pp.efpp); - ASSERT(off_heap->funs->next - == (ErlFunThing *) dbg_old_oh_list_p); - break; -#endif - case EXTERNAL_PID_SUBTAG: - case EXTERNAL_PORT_SUBTAG: - case EXTERNAL_REF_SUBTAG: - ASSERT(off_heap->externals - == *oh_list_pp.etpp); - ASSERT(off_heap->externals->next - == (ExternalThing *) dbg_old_oh_list_p); - break; - default: - ASSERT(0); - } -#endif - oh_list_pp.upp = NULL; - - + oh->next = off_heap->first; + off_heap->first = oh; + ASSERT(*hpp <= (Eterm*)oh); + ASSERT(hp > (Eterm*)oh); + oh = NULL; } break; } @@ -765,11 +692,7 @@ copy_done: #endif - bp->off_heap.mso = NULL; -#ifndef HYBRID - bp->off_heap.funs = NULL; -#endif - bp->off_heap.externals = NULL; + bp->off_heap.first = NULL; free_message_buffer(bp); msg->data.heap_frag = NULL; diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index f478572ac2..890af2675c 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -28,13 +28,18 @@ struct external_thing_; * but is stored outside of any heap. */ +struct erl_off_heap_header { + Eterm thing_word; + union { + Uint size; + void* ptr; + }u; + struct erl_off_heap_header* next; +}; + typedef struct erl_off_heap { - struct proc_bin* mso; /* List of associated binaries. */ -#ifndef HYBRID /* FIND ME! */ - struct erl_fun_thing* funs; /* List of funs. */ -#endif - struct external_thing_* externals; /* List of external things. */ - int overhead; /* Administrative overhead (used to force GC). */ + struct erl_off_heap_header* first; + int overhead; /* Administrative overhead (used to force GC). */ } ErlOffHeap; #include "external.h" @@ -201,9 +206,7 @@ do { \ (HEAP_FRAG_P)->next = NULL; \ (HEAP_FRAG_P)->alloc_size = (DATA_WORDS); \ (HEAP_FRAG_P)->used_size = (DATA_WORDS); \ - (HEAP_FRAG_P)->off_heap.mso = NULL; \ - (HEAP_FRAG_P)->off_heap.funs = NULL; \ - (HEAP_FRAG_P)->off_heap.externals = NULL; \ + (HEAP_FRAG_P)->off_heap.first = NULL; \ (HEAP_FRAG_P)->off_heap.overhead = 0; \ } while (0) diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 3d63fa1caf..e95d9c4f75 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -258,9 +258,7 @@ void enif_free_env(ErlNifEnv* env) static ERTS_INLINE void clear_offheap(ErlOffHeap* oh) { - oh->mso = NULL; - oh->externals = NULL; - oh->funs = NULL; + oh->first = NULL; oh->overhead = 0; } @@ -364,7 +362,7 @@ ERL_NIF_TERM enif_make_copy(ErlNifEnv* dst_env, ERL_NIF_TERM src_term) #ifdef DEBUG static int is_offheap(const ErlOffHeap* oh) { - return oh->mso != NULL || oh->funs != NULL || oh->externals != NULL; + return oh->first != NULL; } #endif @@ -627,8 +625,8 @@ Eterm enif_make_binary(ErlNifEnv* env, ErlNifBinary* bin) pb = (ProcBin *) alloc_heap(env, PROC_BIN_SIZE); pb->thing_word = HEADER_PROC_BIN; pb->size = bptr->orig_size; - pb->next = MSO(env->proc).mso; - MSO(env->proc).mso = pb; + pb->next = MSO(env->proc).first; + MSO(env->proc).first = (struct erl_off_heap_header*) pb; pb->val = bptr; pb->bytes = (byte*) bptr->orig_bytes; pb->flags = 0; diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index 5865d33138..7482da251e 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -1087,30 +1087,24 @@ insert_offheap2(ErlOffHeap *oh, void *arg) static void insert_offheap(ErlOffHeap *oh, int type, Eterm id) { - if(oh->externals) { - ExternalThing *etp = oh->externals; - while (etp) { - insert_node(etp->node, type, id); - etp = etp->next; - } - } + union erl_off_heap_ptr u; + struct insert_offheap2_arg a; + a.type = BIN_REF; - if(oh->mso) { - ProcBin *pb; - struct insert_offheap2_arg a; - a.type = BIN_REF; - for(pb = oh->mso; pb; pb = pb->next) { - if(IsMatchProgBinary(pb->val)) { + for (u.hdr = oh->first; u.hdr; u.hdr = u.hdr->next) { + switch (thing_subtag(u.hdr->thing_word)) { + case REFC_BINARY_SUBTAG: + if(IsMatchProgBinary(u.pb->val)) { InsertedBin *ib; int insert_bin = 1; for (ib = inserted_bins; ib; ib = ib->next) - if(ib->bin_val == pb->val) { + if(ib->bin_val == u.pb->val) { insert_bin = 0; break; } if (insert_bin) { #if HALFWORD_HEAP - UWord val = (UWord) pb->val; + UWord val = (UWord) u.pb->val; DeclareTmpHeapNoproc(id_heap,BIG_UINT_HEAP_SIZE*2); /* extra place allocated */ #else DeclareTmpHeapNoproc(id_heap,BIG_UINT_HEAP_SIZE); @@ -1124,13 +1118,13 @@ insert_offheap(ErlOffHeap *oh, int type, Eterm id) a.id = erts_bld_uword(&hp, NULL, (UWord) val); #else UseTmpHeapNoproc(BIG_UINT_HEAP_SIZE); - a.id = erts_bld_uint(&hp, NULL, (Uint) pb->val); + a.id = erts_bld_uint(&hp, NULL, (Uint) u.pb->val); #endif - erts_match_prog_foreach_offheap(pb->val, + erts_match_prog_foreach_offheap(u.pb->val, insert_offheap2, (void *) &a); nib = erts_alloc(ERTS_ALC_T_NC_TMP, sizeof(InsertedBin)); - nib->bin_val = pb->val; + nib->bin_val = u.pb->val; nib->next = inserted_bins; inserted_bins = nib; #if HALFWORD_HEAP @@ -1139,15 +1133,16 @@ insert_offheap(ErlOffHeap *oh, int type, Eterm id) UnUseTmpHeapNoproc(BIG_UINT_HEAP_SIZE); #endif } - } + } + break; + case FUN_SUBTAG: + break; /* No need to */ + default: + ASSERT(is_external_header(u.hdr->thing_word)); + insert_node(u.ext->node, type, id); + break; } } - -#if 0 - if(oh->funs) { - /* No need to */ - } -#endif } static void doit_insert_monitor(ErtsMonitor *monitor, void *p) @@ -1289,6 +1284,7 @@ setup_reference_table(void) for (i = 0; i < erts_max_processes; i++) if (process_tab[i]) { ErlMessage *msg; + /* Insert Heap */ insert_offheap(&(process_tab[i]->off_heap), HEAP_REF, @@ -1375,21 +1371,22 @@ setup_reference_table(void) { /* Add binaries stored elsewhere ... */ ErlOffHeap oh; - ProcBin pb[2] = {{0},{0}}; - ProcBin *mso = NULL; + ProcBin pb[2]; int i = 0; Binary *default_match_spec; Binary *default_meta_match_spec; - /* Only the ProcBin members val and next will be inspected + oh.first = NULL; + /* Only the ProcBin members thing_word, val and next will be inspected (by insert_offheap()) */ #undef ADD_BINARY -#define ADD_BINARY(Bin) \ - if ((Bin)) { \ - pb[i].val = (Bin); \ - pb[i].next = mso; \ - mso = &pb[i]; \ - i++; \ +#define ADD_BINARY(Bin) \ + if ((Bin)) { \ + pb[i].thing_word = REFC_BINARY_SUBTAG; \ + pb[i].val = (Bin); \ + pb[i].next = oh.first; \ + oh.first = (struct erl_off_heap_header*) &pb[i]; \ + i++; \ } erts_get_default_trace_pattern(NULL, @@ -1401,11 +1398,6 @@ setup_reference_table(void) ADD_BINARY(default_match_spec); ADD_BINARY(default_meta_match_spec); - oh.mso = mso; - oh.externals = NULL; -#ifndef HYBRID /* FIND ME! */ - oh.funs = NULL; -#endif insert_offheap(&oh, BIN_REF, AM_match_spec); #undef ADD_BINARY } diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 41031f5468..8c88353593 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -6761,11 +6761,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). /* * Must initialize binary lists here before copying binaries to process. */ - p->off_heap.mso = NULL; -#ifndef HYBRID /* FIND ME! */ - p->off_heap.funs = NULL; -#endif - p->off_heap.externals = NULL; + p->off_heap.first = NULL; p->off_heap.overhead = 0; heap_need += @@ -6855,7 +6851,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->group_leader = IS_CONST(parent->group_leader) ? parent->group_leader - : STORE_NC(&p->htop, &p->off_heap.externals, parent->group_leader); + : STORE_NC(&p->htop, &p->off_heap, parent->group_leader); } erts_get_default_tracing(&p->trace_flags, &p->tracer_proc); @@ -7056,11 +7052,7 @@ void erts_init_empty_process(Process *p) memset(&(p->u.tm), 0, sizeof(ErlTimer)); #endif p->next = NULL; - p->off_heap.mso = NULL; -#ifndef HYBRID /* FIND ME! */ - p->off_heap.funs = NULL; -#endif - p->off_heap.externals = NULL; + p->off_heap.first = NULL; p->off_heap.overhead = 0; p->reg = NULL; p->heap_sz = 0; @@ -7207,11 +7199,7 @@ erts_debug_verify_clean_empty_process(Process* p) /* Thing that erts_cleanup_empty_process() cleans up */ - ASSERT(p->off_heap.mso == NULL); -#ifndef HYBRID /* FIND ME! */ - ASSERT(p->off_heap.funs == NULL); -#endif - ASSERT(p->off_heap.externals == NULL); + ASSERT(p->off_heap.first == NULL); ASSERT(p->off_heap.overhead == 0); ASSERT(p->mbuf == NULL); @@ -7225,11 +7213,7 @@ erts_cleanup_empty_process(Process* p) /* We only check fields that are known to be used... */ erts_cleanup_offheap(&p->off_heap); - p->off_heap.mso = NULL; -#ifndef HYBRID /* FIND ME! */ - p->off_heap.funs = NULL; -#endif - p->off_heap.externals = NULL; + p->off_heap.first = NULL; p->off_heap.overhead = 0; if (p->mbuf != NULL) { @@ -7266,7 +7250,7 @@ delete_process(Process* p) * The mso list should not be used anymore, but if it is, make sure that * we'll notice. */ - p->off_heap.mso = (void *) 0x8DEFFACD; + p->off_heap.first = (void *) 0x8DEFFACD; if (p->arg_reg != p->def_arg_reg) { erts_free(ERTS_ALC_T_ARG_REG, p->arg_reg); diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 8f9f7f004e..6ce74a917f 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -828,7 +828,7 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags; #define F_INSLPQUEUE (1 << 1) /* Set if in timer queue */ #define F_TIMO (1 << 2) /* Set if timeout */ #define F_HEAP_GROW (1 << 3) -#define F_NEED_FULLSWEEP (1 << 4) /* If process has old binaries & funs. */ +#define F_NEED_FULLSWEEP (1 << 4) #define F_USING_DB (1 << 5) /* If have created tables */ #define F_DISTRIBUTION (1 << 6) /* Process used in distribution */ #define F_USING_DDLL (1 << 7) /* Process has used the DDLL interface */ diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h index 3a8c30fe6a..0fb2362abd 100644 --- a/erts/emulator/beam/erl_term.h +++ b/erts/emulator/beam/erl_term.h @@ -857,8 +857,8 @@ _ET_DECLARE_CHECKED(struct erl_node_*,internal_ref_node,Eterm) typedef struct external_thing_ { /* ----+ */ Eterm header; /* | */ - struct external_thing_ *next; /* > External thing head */ - struct erl_node_ *node; /* | */ + struct erl_node_* node; /* > External thing head */ + struct erl_off_heap_header* next; /* | */ /* ----+ */ union { Uint32 ui32[1]; diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 466165f26f..310a83200b 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1468,11 +1468,11 @@ dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Ete *hpp += EXTERNAL_THING_HEAD_SIZE + 1; etp->header = make_external_pid_header(1); - etp->next = off_heap->externals; + etp->next = off_heap->first; etp->node = node; etp->data.ui[0] = data; - off_heap->externals = etp; + off_heap->first = (struct erl_off_heap_header*) etp; *objp = make_external_pid(etp); } return ep; @@ -1910,69 +1910,30 @@ is_external_string(Eterm list, int* p_is_string) return len; } -/* Assumes that the ones to undo are preluding the lists. */ +/* Assumes that the ones to undo are preluding the list. */ static void undo_offheap_in_area(ErlOffHeap* off_heap, Eterm* start, Eterm* end) { const Uint area_sz = (end - start) * sizeof(Eterm); - struct proc_bin* mso; - struct proc_bin** mso_nextp = NULL; -#ifndef HYBRID /* FIND ME! */ - struct erl_fun_thing* funs; - struct erl_fun_thing** funs_nextp = NULL; -#endif - struct external_thing_* ext; - struct external_thing_** ext_nextp = NULL; - - for (mso = off_heap->mso; ; mso=mso->next) { - if (!in_area(mso, start, area_sz)) { - if (mso_nextp != NULL) { - *mso_nextp = NULL; - erts_cleanup_mso(off_heap->mso); - off_heap->mso = mso; + struct erl_off_heap_header* hdr; + struct erl_off_heap_header** hdr_nextp = NULL; + + for (hdr = off_heap->first; ; hdr=hdr->next) { + if (!in_area(hdr, start, area_sz)) { + if (hdr_nextp != NULL) { + *hdr_nextp = NULL; + erts_cleanup_offheap(off_heap); + off_heap->first = hdr; } break; } - mso_nextp = &mso->next; + hdr_nextp = &hdr->next; } -#ifndef HYBRID /* FIND ME! */ - for (funs = off_heap->funs; ; funs=funs->next) { - if (!in_area(funs, start, area_sz)) { - if (funs_nextp != NULL) { - *funs_nextp = NULL; - erts_cleanup_funs(off_heap->funs); - off_heap->funs = funs; - } - break; - } - funs_nextp = &funs->next; - } -#endif - for (ext = off_heap->externals; ; ext=ext->next) { - if (!in_area(ext, start, area_sz)) { - if (ext_nextp != NULL) { - *ext_nextp = NULL; - erts_cleanup_externals(off_heap->externals); - off_heap->externals = ext; - } - break; - } - ext_nextp = &ext->next; - } - - /* Assert that the ones to undo were indeed preluding the lists. */ + /* Assert that the ones to undo were indeed preluding the list. */ #ifdef DEBUG - for (mso = off_heap->mso; mso != NULL; mso=mso->next) { - ASSERT(!in_area(mso, start, area_sz)); - } -# ifndef HYBRID /* FIND ME! */ - for (funs = off_heap->funs; funs != NULL; funs=funs->next) { - ASSERT(!in_area(funs, start, area_sz)); - } -# endif - for (ext = off_heap->externals; ext != NULL; ext=ext->next) { - ASSERT(!in_area(ext, start, area_sz)); + for (hdr = off_heap->first; hdr != NULL; hdr = hdr->next) { + ASSERT(!in_area(hdr, start, area_sz)); } #endif /* DEBUG */ } @@ -2216,11 +2177,11 @@ dec_term_atom_common: hp += EXTERNAL_THING_HEAD_SIZE + 1; etp->header = make_external_port_header(1); - etp->next = off_heap->externals; + etp->next = off_heap->first; etp->node = node; etp->data.ui[0] = num; - off_heap->externals = etp; + off_heap->first = (struct erl_off_heap_header*)etp; *objp = make_external_port(etp); } @@ -2298,10 +2259,10 @@ dec_term_atom_common: #else etp->header = make_external_ref_header(ref_words); #endif - etp->next = off_heap->externals; + etp->next = off_heap->first; etp->node = node; - off_heap->externals = etp; + off_heap->first = (struct erl_off_heap_header*)etp; *objp = make_external_ref(etp); ref_num = &(etp->data.ui32[0]); } @@ -2344,8 +2305,8 @@ dec_term_atom_common: hp += PROC_BIN_SIZE; pb->thing_word = HEADER_PROC_BIN; pb->size = n; - pb->next = off_heap->mso; - off_heap->mso = pb; + pb->next = off_heap->first; + off_heap->first = (struct erl_off_heap_header*)pb; pb->val = dbin; pb->bytes = (byte*) dbin->orig_bytes; pb->flags = 0; @@ -2381,8 +2342,8 @@ dec_term_atom_common: pb = (ProcBin *) hp; pb->thing_word = HEADER_PROC_BIN; pb->size = n; - pb->next = off_heap->mso; - off_heap->mso = pb; + pb->next = off_heap->first; + off_heap->first = (struct erl_off_heap_header*)pb; pb->val = dbin; pb->bytes = (byte*) dbin->orig_bytes; pb->flags = 0; @@ -2502,8 +2463,8 @@ dec_term_atom_common: * It is safe to link the fun into the fun list only when * no more validity tests can fail. */ - funp->next = off_heap->funs; - off_heap->funs = funp; + funp->next = off_heap->first; + off_heap->first = (struct erl_off_heap_header*)funp; #endif funp->fe = erts_put_fun_entry2(module, old_uniq, old_index, @@ -2580,8 +2541,8 @@ dec_term_atom_common: * It is safe to link the fun into the fun list only when * no more validity tests can fail. */ - funp->next = off_heap->funs; - off_heap->funs = funp; + funp->next = off_heap->first; + off_heap->first = (struct erl_off_heap_header*)funp; #endif old_uniq = unsigned_val(temp); diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index b4a7a22082..607d3cd14c 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -464,7 +464,10 @@ typedef union { typedef struct proc_bin { Eterm thing_word; /* Subtag REFC_BINARY_SUBTAG. */ Uint size; /* Binary size in bytes. */ - struct proc_bin *next; /* Pointer to next ProcBin. */ +#if HALFWORD_HEAP + char alignment__[SIZEOF_VOID_P - ERTS_SIZEOF_ETERM]; +#endif + struct erl_off_heap_header *next; Binary *val; /* Pointer to Binary structure. */ byte *bytes; /* Pointer to the actual data bytes. */ Uint flags; /* Flag word. */ @@ -494,8 +497,8 @@ erts_mk_magic_binary_term(Eterm **hpp, ErlOffHeap *ohp, Binary *mbp) pb->thing_word = HEADER_PROC_BIN; pb->size = 0; - pb->next = ohp->mso; - ohp->mso = pb; + pb->next = ohp->first; + ohp->first = (struct erl_off_heap_header*) pb; pb->val = mbp; pb->bytes = (byte *) mbp->orig_bytes; pb->flags = 0; @@ -512,6 +515,15 @@ erts_mk_magic_binary_term(Eterm **hpp, ErlOffHeap *ohp, Binary *mbp) && (thing_subtag(*binary_val((T))) == REFC_BINARY_SUBTAG) \ && (((ProcBin *) binary_val((T)))->val->flags & BIN_FLAG_MAGIC)) + +union erl_off_heap_ptr { + struct erl_off_heap_header* hdr; + ProcBin *pb; + struct erl_fun_thing* fun; + struct external_thing_* ext; + Eterm* ep; +}; + /* arrays that get malloced at startup */ extern Port* erts_port; extern erts_smp_atomic_t erts_ports_alive; @@ -823,7 +835,6 @@ Eterm erts_new_heap_binary(Process *p, byte *buf, int len, byte** datap); Eterm erts_new_mso_binary(Process*, byte*, int); Eterm new_binary(Process*, byte*, int); Eterm erts_realloc_binary(Eterm bin, size_t size); -void erts_cleanup_mso(ProcBin* pb); /* erl_bif_info.c */ @@ -1499,7 +1510,6 @@ void p_slpq(_VOID_); void erts_silence_warn_unused_result(long unused); void erts_cleanup_offheap(ErlOffHeap *offheap); -void erts_cleanup_externals(ExternalThing *); Uint erts_fit_in_bits(Uint); int list_length(Eterm); @@ -1532,7 +1542,7 @@ erts_bld_atom_2uint_3tup_list(Uint **hpp, Uint *szp, Sint length, Eterm atoms[], Uint uints1[], Uint uints2[]); Eterm store_external_or_ref_in_proc_(Process *, Eterm); -Eterm store_external_or_ref_(Uint **, ExternalThing **, Eterm); +Eterm store_external_or_ref_(Uint **, ErlOffHeap*, Eterm); #define NC_HEAP_SIZE(NC) \ (ASSERT_EXPR(is_node_container((NC))), \ @@ -1957,5 +1967,5 @@ erts_alloc_message_heap(Uint size, # define UnUseTmpHeap(Size,Proc) /* Nothing */ # define UseTmpHeapNoproc(Size) /* Nothing */ # define UnUseTmpHeapNoproc(Size) /* Nothing */ -#endif -#endif +#endif /* HEAP_ON_C_STACK */ +#endif /* !__GLOBAL_H__ */ diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 68625801cf..b840f65cdd 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -1575,8 +1575,8 @@ static void deliver_read_message(Port* prt, Eterm to, pb = (ProcBin *) hp; pb->thing_word = HEADER_PROC_BIN; pb->size = len; - pb->next = ohp->mso; - ohp->mso = pb; + pb->next = ohp->first; + ohp->first = (struct erl_off_heap_header*)pb; pb->val = bptr; pb->bytes = (byte*) bptr->orig_bytes; pb->flags = 0; @@ -1725,8 +1725,8 @@ deliver_vec_message(Port* prt, /* Port */ } pb->thing_word = HEADER_PROC_BIN; pb->size = iov->iov_len; - pb->next = ohp->mso; - ohp->mso = pb; + pb->next = ohp->first; + ohp->first = (struct erl_off_heap_header*)pb; pb->val = ErlDrvBinary2Binary(b); pb->bytes = base; pb->flags = 0; @@ -2259,8 +2259,8 @@ erts_port_control(Process* p, Port* prt, Uint command, Eterm iolist) ProcBin* pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE); pb->thing_word = HEADER_PROC_BIN; pb->size = dbin->orig_size; - pb->next = MSO(p).mso; - MSO(p).mso = pb; + pb->next = MSO(p).first; + MSO(p).first = (struct erl_off_heap_header*)pb; pb->val = ErlDrvBinary2Binary(dbin); pb->bytes = (byte*) dbin->orig_bytes; pb->flags = 0; @@ -3033,8 +3033,8 @@ driver_deliver_term(ErlDrvPort port, driver_binary_inc_refc(b); /* caller will free binary */ pb->thing_word = HEADER_PROC_BIN; pb->size = size; - pb->next = ohp->mso; - ohp->mso = pb; + pb->next = ohp->first; + ohp->first = (struct erl_off_heap_header*)pb; pb->val = ErlDrvBinary2Binary(b); pb->bytes = ((byte*) b->orig_bytes) + offset; pb->flags = 0; @@ -3072,8 +3072,8 @@ driver_deliver_term(ErlDrvPort port, hp += PROC_BIN_SIZE; pbp->thing_word = HEADER_PROC_BIN; pbp->size = size; - pbp->next = ohp->mso; - ohp->mso = pbp; + pbp->next = ohp->first; + ohp->first = (struct erl_off_heap_header*)pbp; pbp->val = bp; pbp->bytes = (byte*) bp->orig_bytes; pbp->flags = 0; diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index b8d407f5e5..25472ef47a 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -162,13 +162,8 @@ erts_heap_alloc(Process* p, Uint need) bp->alloc_size = n; bp->used_size = n; MBUF_SIZE(p) += n; - bp->off_heap.mso = NULL; -#ifndef HYBRID /* FIND ME! */ - bp->off_heap.funs = NULL; -#endif - bp->off_heap.externals = NULL; + bp->off_heap.first = NULL; bp->off_heap.overhead = 0; - return bp->mem; } @@ -2729,21 +2724,8 @@ not_equal: } -void -erts_cleanup_externals(ExternalThing *etp) -{ - ExternalThing *tetp; - - tetp = etp; - - while(tetp) { - erts_deref_node_entry(tetp->node); - tetp = tetp->next; - } -} - Eterm -store_external_or_ref_(Uint **hpp, ExternalThing **etpp, Eterm ns) +store_external_or_ref_(Uint **hpp, ErlOffHeap* oh, Eterm ns) { Uint i; Uint size; @@ -2762,8 +2744,8 @@ store_external_or_ref_(Uint **hpp, ExternalThing **etpp, Eterm ns) erts_refc_inc(&((ExternalThing *) to_hp)->node->refc, 2); - ((ExternalThing *) to_hp)->next = *etpp; - *etpp = (ExternalThing *) to_hp; + ((struct erl_off_heap_header*) to_hp)->next = oh->first; + oh->first = (struct erl_off_heap_header*) to_hp; return make_external(to_hp); } @@ -2792,7 +2774,7 @@ store_external_or_ref_in_proc_(Process *proc, Eterm ns) sz = NC_HEAP_SIZE(ns); ASSERT(sz > 0); hp = HAlloc(proc, sz); - return store_external_or_ref_(&hp, &MSO(proc).externals, ns); + return store_external_or_ref_(&hp, &MSO(proc), ns); } void bin_write(int to, void *to_arg, byte* buf, int sz) diff --git a/erts/emulator/hipe/hipe_mkliterals.c b/erts/emulator/hipe/hipe_mkliterals.c index a77aec7919..900dfc5a8a 100644 --- a/erts/emulator/hipe/hipe_mkliterals.c +++ b/erts/emulator/hipe/hipe_mkliterals.c @@ -261,7 +261,7 @@ static const struct literal { /* Field offsets in a process struct */ { "P_HP", offsetof(struct process, htop) }, { "P_HP_LIMIT", offsetof(struct process, stop) }, - { "P_OFF_HEAP_MSO", offsetof(struct process, off_heap.mso) }, + { "P_OFF_HEAP_FIRST", offsetof(struct process, off_heap.first) }, { "P_MBUF", offsetof(struct process, mbuf) }, { "P_ID", offsetof(struct process, id) }, { "P_FLAGS", offsetof(struct process, flags) }, @@ -456,7 +456,7 @@ static const struct rts_param { } rts_params[] = { { 1, "P_OFF_HEAP_FUNS", #if !defined(HYBRID) - 1, offsetof(struct process, off_heap.funs) + 1, offsetof(struct process, off_heap.first) #endif }, diff --git a/lib/hipe/rtl/hipe_rtl_primops.erl b/lib/hipe/rtl/hipe_rtl_primops.erl index 560e0259f8..0361053676 100644 --- a/lib/hipe/rtl/hipe_rtl_primops.erl +++ b/lib/hipe/rtl/hipe_rtl_primops.erl @@ -701,9 +701,9 @@ gen_cons(Dst, [Arg1, Arg2]) -> %% Increase refcount %% fe->refc++; %% -%% Link to the process off_heap.funs list -%% funp->next = p->off_heap.funs; -%% p->off_heap.funs = funp; +%% Link to the process off_heap list +%% funp->next = p->off_heap.first; +%% p->off_heap.first = funp; %% %% Tag the thing %% return make_fun(funp); @@ -729,9 +729,9 @@ gen_mkfun([Dst], {_Mod, _FunId, _Arity} = MFidA, MagicNr, Index, FreeVars) -> %% fe->refc++; SkeletonCode = gen_fun_thing_skeleton(HP, MFidA, NumFree, MagicNr, Index), - %% Link to the process off_heap.funs list - %% funp->next = p->off_heap.funs; - %% p->off_heap.funs = funp; + %% Link to the process off_heap list + %% funp->next = p->off_heap.first; + %% p->off_heap.first = funp; LinkCode = gen_link_closure(HP), %% Tag the thing and increase the heap_pointer. @@ -797,14 +797,14 @@ gen_link_closure(FUNP) -> end. gen_link_closure_private(FUNP) -> - %% Link to the process off_heap.funs list - %% funp->next = p->off_heap.funs; - %% p->off_heap.funs = funp; + %% Link fun to the process off_heap list + %% funp->next = p->off_heap.first; + %% p->off_heap.first = funp; FunsVar = hipe_rtl:mk_new_reg(), - [load_p_field(FunsVar,?P_OFF_HEAP_FUNS), + [load_p_field(FunsVar,?P_OFF_HEAP_FIRST), hipe_rtl:mk_store(FUNP, hipe_rtl:mk_imm(?EFT_NEXT), FunsVar), - store_p_field(FUNP,?P_OFF_HEAP_FUNS)]. + store_p_field(FUNP,?P_OFF_HEAP_FIRST)]. gen_link_closure_non_private(_FUNP) -> []. diff --git a/lib/hipe/rtl/hipe_tagscheme.erl b/lib/hipe/rtl/hipe_tagscheme.erl index dc44b803a1..c0b6dfad8a 100644 --- a/lib/hipe/rtl/hipe_tagscheme.erl +++ b/lib/hipe/rtl/hipe_tagscheme.erl @@ -849,9 +849,9 @@ create_refc_binary(Base, Size, Flags, Dst) -> heap_arch_spec(HP) -> Tmp1 = hipe_rtl:mk_new_reg(), % MSO state - [hipe_rtl_arch:pcb_load(Tmp1, ?P_OFF_HEAP_MSO), + [hipe_rtl_arch:pcb_load(Tmp1, ?P_OFF_HEAP_FIRST), set_field_from_pointer({proc_bin, next}, HP, Tmp1), - hipe_rtl_arch:pcb_store(?P_OFF_HEAP_MSO, HP)]. + hipe_rtl_arch:pcb_store(?P_OFF_HEAP_FIRST, HP)]. test_heap_binary(Binary, TrueLblName, FalseLblName) -> Tmp1 = hipe_rtl:mk_new_reg_gcsafe(), diff --git a/lib/stdlib/test/ets_SUITE.erl b/lib/stdlib/test/ets_SUITE.erl index 13c87ca005..677cc09f5e 100644 --- a/lib/stdlib/test/ets_SUITE.erl +++ b/lib/stdlib/test/ets_SUITE.erl @@ -391,7 +391,7 @@ memory(Config) when is_list(Config) -> ?line erts_debug:set_internal_state(available_internal_state, true), ?line ok = chk_normal_tab_struct_size(), ?line L = [T1,T2,T3,T4] = fill_sets_int(1000), - ?line XRes1 = adjust_xmem(L, {16862,16072,16072,16078}), + ?line XRes1 = adjust_xmem(L, {14862,14072,14072,14078}), ?line Res1 = {?S(T1),?S(T2),?S(T3),?S(T4)}, ?line lists:foreach(fun(T) -> Before = ets:info(T,size), @@ -402,7 +402,7 @@ memory(Config) when is_list(Config) -> [Key, ets:info(T,type), Before, ets:info(T,size), Objs]) end, L), - ?line XRes2 = adjust_xmem(L, {16849,16060,16048,16054}), + ?line XRes2 = adjust_xmem(L, {14851,14062,14052,14058}), ?line Res2 = {?S(T1),?S(T2),?S(T3),?S(T4)}, ?line lists:foreach(fun(T) -> Before = ets:info(T,size), @@ -413,7 +413,7 @@ memory(Config) when is_list(Config) -> [Key, ets:info(T,type), Before, ets:info(T,size), Objs]) end, L), - ?line XRes3 = adjust_xmem(L, {16836,16048,16024,16030}), + ?line XRes3 = adjust_xmem(L, {14840,14052,14032,14038}), ?line Res3 = {?S(T1),?S(T2),?S(T3),?S(T4)}, ?line lists:foreach(fun(T) -> ?line ets:delete_all_objects(T) |