diff options
Diffstat (limited to 'erts/emulator/beam/external.c')
-rw-r--r-- | erts/emulator/beam/external.c | 312 |
1 files changed, 172 insertions, 140 deletions
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 099eddd195..d7c8aa84e9 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -65,11 +65,9 @@ # endif #endif -/* - * For backward compatibility reasons, only encode integers that - * fit in 28 bits (signed) using INTEGER_EXT. +/* Does Sint fit in Sint32? */ -#define IS_SSMALL28(x) (((Uint) (((x) >> (28-1)) + 1)) < 2) +#define IS_SSMALL32(x) (((Uint) (((x) >> (32-1)) + 1)) < 2) /* * Valid creations for nodes are 1, 2, or 3. 0 can also be sent @@ -504,7 +502,7 @@ erts_make_dist_ext_copy(ErtsDistExternal *edep, Uint xsize) ASSERT(edep->ext_endp >= edep->extp); ext_sz = edep->ext_endp - edep->extp; - align_sz = ERTS_WORD_ALIGN_PAD_SZ(dist_ext_sz + ext_sz); + align_sz = ERTS_EXTRA_DATA_ALIGN_SZ(dist_ext_sz + ext_sz); new_edep = erts_alloc(ERTS_ALC_T_EXT_TERM_DATA, dist_ext_sz + ext_sz + align_sz + xsize); @@ -1470,11 +1468,11 @@ dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Ete *hpp += EXTERNAL_THING_HEAD_SIZE + 1; etp->header = make_external_pid_header(1); - etp->next = off_heap->externals; + etp->next = off_heap->first; etp->node = node; etp->data.ui[0] = data; - off_heap->externals = etp; + off_heap->first = (struct erl_off_heap_header*) etp; *objp = make_external_pid(etp); } return ep; @@ -1489,20 +1487,28 @@ dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Ete static byte* enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) { - DECLARE_ESTACK(s); + DECLARE_WSTACK(s); Uint n; Uint i; Uint j; Uint* ptr; Eterm val; FloatDef f; +#if HALFWORD_HEAP + UWord wobj; +#endif + goto L_jump_start; outer_loop: - while (!ESTACK_ISEMPTY(s)) { - obj = ESTACK_POP(s); - switch (val = ESTACK_POP(s)) { + while (!WSTACK_ISEMPTY(s)) { +#if HALFWORD_HEAP + obj = (Eterm) (wobj = WSTACK_POP(s)); +#else + obj = WSTACK_POP(s); +#endif + switch (val = WSTACK_POP(s)) { case ENC_TERM: break; case ENC_ONE_CONS: @@ -1513,29 +1519,40 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) obj = CAR(cons); tl = CDR(cons); - ESTACK_PUSH(s, is_list(tl) ? ENC_ONE_CONS : ENC_TERM); - ESTACK_PUSH(s, tl); + WSTACK_PUSH(s, is_list(tl) ? ENC_ONE_CONS : ENC_TERM); + WSTACK_PUSH(s, tl); } break; case ENC_PATCH_FUN_SIZE: { +#if HALFWORD_HEAP + byte* size_p = (byte *) wobj; +#else byte* size_p = (byte *) obj; - +#endif put_int32(ep - size_p, size_p); } goto outer_loop; case ENC_LAST_ARRAY_ELEMENT: { +#if HALFWORD_HEAP + Eterm* ptr = (Eterm *) wobj; +#else Eterm* ptr = (Eterm *) obj; +#endif obj = *ptr; } break; default: /* ENC_LAST_ARRAY_ELEMENT+1 and upwards */ { +#if HALFWORD_HEAP + Eterm* ptr = (Eterm *) wobj; +#else Eterm* ptr = (Eterm *) obj; +#endif obj = *ptr++; - ESTACK_PUSH(s, val-1); - ESTACK_PUSH(s, (Eterm) ptr); + WSTACK_PUSH(s, val-1); + WSTACK_PUSH(s, (UWord) ptr); } break; } @@ -1552,19 +1569,23 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) case SMALL_DEF: { + /* From R14B we no longer restrict INTEGER_EXT to 28 bits, + * as done earlier for backward compatibility reasons. */ Sint val = signed_val(obj); if ((Uint)val < 256) { *ep++ = SMALL_INTEGER_EXT; put_int8(val, ep); ep++; - } else if (sizeof(Sint) == 4 || IS_SSMALL28(val)) { + } else if (sizeof(Sint) == 4 || IS_SSMALL32(val)) { *ep++ = INTEGER_EXT; put_int32(val, ep); ep += 4; } else { - Eterm tmp_big[2]; - Eterm big = small_to_big(val, tmp_big); + DeclareTmpHeapNoproc(tmp_big,2); + Eterm big; + UseTmpHeapNoproc(2); + big = small_to_big(val, tmp_big); *ep++ = SMALL_BIG_EXT; n = big_bytes(big); ASSERT(n < 256); @@ -1572,23 +1593,38 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) ep += 1; *ep++ = big_sign(big); ep = big_to_bytes(big, ep); + UnUseTmpHeapNoproc(2); } } break; case BIG_DEF: - if ((n = big_bytes(obj)) < 256) { - *ep++ = SMALL_BIG_EXT; - put_int8(n, ep); - ep += 1; - } - else { - *ep++ = LARGE_BIG_EXT; - put_int32(n, ep); - ep += 4; + { + int sign = big_sign(obj); + n = big_bytes(obj); + if (sizeof(Sint)==4 && n<=4) { + Uint dig = big_digit(obj,0); + Sint val = sign ? -dig : dig; + if ((val<0) == sign) { + *ep++ = INTEGER_EXT; + put_int32(val, ep); + ep += 4; + break; + } + } + if (n < 256) { + *ep++ = SMALL_BIG_EXT; + put_int8(n, ep); + ep += 1; + } + else { + *ep++ = LARGE_BIG_EXT; + put_int32(n, ep); + ep += 4; + } + *ep++ = sign; + ep = big_to_bytes(obj, ep); } - *ep++ = big_sign(obj); - ep = big_to_bytes(obj, ep); break; case PID_DEF: @@ -1662,8 +1698,8 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) ep += 4; } if (i > 0) { - ESTACK_PUSH(s, ENC_LAST_ARRAY_ELEMENT+i-1); - ESTACK_PUSH(s, (Eterm) ptr); + WSTACK_PUSH(s, ENC_LAST_ARRAY_ELEMENT+i-1); + WSTACK_PUSH(s, (UWord) ptr); } break; @@ -1741,7 +1777,7 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) } case EXPORT_DEF: { - Export* exp = (Export *) (export_val(obj))[1]; + Export* exp = *((Export **) (export_val(obj) + 1)); if ((dflags & DFLAG_EXPORT_PTR_TAG) != 0) { *ep++ = EXPORT_EXT; ep = enc_atom(acmp, exp->code[0], ep, dflags); @@ -1770,8 +1806,8 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) int ei; *ep++ = NEW_FUN_EXT; - ESTACK_PUSH(s, ENC_PATCH_FUN_SIZE); - ESTACK_PUSH(s, (Eterm) ep); /* Position for patching in size */ + WSTACK_PUSH(s, ENC_PATCH_FUN_SIZE); + WSTACK_PUSH(s, (UWord) ep); /* Position for patching in size */ ep += 4; *ep = funp->arity; ep += 1; @@ -1788,8 +1824,8 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) fun_env: for (ei = funp->num_free-1; ei > 0; ei--) { - ESTACK_PUSH(s, ENC_TERM); - ESTACK_PUSH(s, funp->env[ei]); + WSTACK_PUSH(s, ENC_TERM); + WSTACK_PUSH(s, (UWord) funp->env[ei]); } if (funp->num_free != 0) { obj = funp->env[0]; @@ -1832,7 +1868,7 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) break; } } - DESTROY_ESTACK(s); + DESTROY_WSTACK(s); return ep; } @@ -1874,69 +1910,30 @@ is_external_string(Eterm list, int* p_is_string) return len; } -/* Assumes that the ones to undo are preluding the lists. */ +/* Assumes that the ones to undo are preluding the list. */ static void undo_offheap_in_area(ErlOffHeap* off_heap, Eterm* start, Eterm* end) { const Uint area_sz = (end - start) * sizeof(Eterm); - struct proc_bin* mso; - struct proc_bin** mso_nextp = NULL; -#ifndef HYBRID /* FIND ME! */ - struct erl_fun_thing* funs; - struct erl_fun_thing** funs_nextp = NULL; -#endif - struct external_thing_* ext; - struct external_thing_** ext_nextp = NULL; - - for (mso = off_heap->mso; ; mso=mso->next) { - if (!in_area(mso, start, area_sz)) { - if (mso_nextp != NULL) { - *mso_nextp = NULL; - erts_cleanup_mso(off_heap->mso); - off_heap->mso = mso; + struct erl_off_heap_header* hdr; + struct erl_off_heap_header** hdr_nextp = NULL; + + for (hdr = off_heap->first; ; hdr=hdr->next) { + if (!in_area(hdr, start, area_sz)) { + if (hdr_nextp != NULL) { + *hdr_nextp = NULL; + erts_cleanup_offheap(off_heap); + off_heap->first = hdr; } break; } - mso_nextp = &mso->next; + hdr_nextp = &hdr->next; } -#ifndef HYBRID /* FIND ME! */ - for (funs = off_heap->funs; ; funs=funs->next) { - if (!in_area(funs, start, area_sz)) { - if (funs_nextp != NULL) { - *funs_nextp = NULL; - erts_cleanup_funs(off_heap->funs); - off_heap->funs = funs; - } - break; - } - funs_nextp = &funs->next; - } -#endif - for (ext = off_heap->externals; ; ext=ext->next) { - if (!in_area(ext, start, area_sz)) { - if (ext_nextp != NULL) { - *ext_nextp = NULL; - erts_cleanup_externals(off_heap->externals); - off_heap->externals = ext; - } - break; - } - ext_nextp = &ext->next; - } - - /* Assert that the ones to undo were indeed preluding the lists. */ + /* Assert that the ones to undo were indeed preluding the list. */ #ifdef DEBUG - for (mso = off_heap->mso; mso != NULL; mso=mso->next) { - ASSERT(!in_area(mso, start, area_sz)); - } -# ifndef HYBRID /* FIND ME! */ - for (funs = off_heap->funs; funs != NULL; funs=funs->next) { - ASSERT(!in_area(funs, start, area_sz)); - } -# endif - for (ext = off_heap->externals; ext != NULL; ext=ext->next) { - ASSERT(!in_area(ext, start, area_sz)); + for (hdr = off_heap->first; hdr != NULL; hdr = hdr->next) { + ASSERT(!in_area(hdr, start, area_sz)); } #endif /* DEBUG */ } @@ -1952,11 +1949,11 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Et register Eterm* hp = *hpp; /* Please don't take the address of hp */ Eterm* next = objp; - *next = (Eterm) NULL; + *next = (Eterm) (UWord) NULL; while (next != NULL) { objp = next; - next = (Eterm *) (*objp); + next = (Eterm *) EXPAND_POINTER(*objp); switch (*ep++) { case INTEGER_EXT: @@ -1964,7 +1961,7 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Et Sint sn = get_int32(ep); ep += 4; -#if defined(ARCH_64) +#if defined(ARCH_64) && !HALFWORD_HEAP *objp = make_small(sn); #else if (MY_IS_SSMALL(sn)) { @@ -2061,7 +2058,7 @@ dec_term_atom_common: hp += n; objp = hp - 1; while (n-- > 0) { - objp[0] = (Eterm) next; + objp[0] = (Eterm) COMPRESS_POINTER(next); next = objp; objp--; } @@ -2079,12 +2076,12 @@ dec_term_atom_common: *objp = make_list(hp); hp += 2*n; objp = hp - 2; - objp[0] = (Eterm) (objp+1); - objp[1] = (Eterm) next; + objp[0] = (Eterm) COMPRESS_POINTER((objp+1)); + objp[1] = (Eterm) COMPRESS_POINTER(next); next = objp; objp -= 2; while (--n > 0) { - objp[0] = (Eterm) next; + objp[0] = (Eterm) COMPRESS_POINTER(next); objp[1] = make_list(objp + 2); next = objp; objp -= 2; @@ -2180,11 +2177,11 @@ dec_term_atom_common: hp += EXTERNAL_THING_HEAD_SIZE + 1; etp->header = make_external_port_header(1); - etp->next = off_heap->externals; + etp->next = off_heap->first; etp->node = node; etp->data.ui[0] = num; - off_heap->externals = etp; + off_heap->first = (struct erl_off_heap_header*)etp; *objp = make_external_port(etp); } @@ -2238,32 +2235,39 @@ dec_term_atom_common: node = erts_find_or_insert_node(sysname, cre); if(node == erts_this_node) { RefThing *rtp = (RefThing *) hp; - hp += REF_THING_HEAD_SIZE; -#ifdef ARCH_64 + ref_num = (Uint32 *) (hp + REF_THING_HEAD_SIZE); + +#if defined(ARCH_64) && !HALFWORD_HEAP + hp += REF_THING_HEAD_SIZE + ref_words/2 + 1; rtp->header = make_ref_thing_header(ref_words/2 + 1); #else + hp += REF_THING_HEAD_SIZE + ref_words; rtp->header = make_ref_thing_header(ref_words); #endif *objp = make_internal_ref(rtp); } else { ExternalThing *etp = (ExternalThing *) hp; - hp += EXTERNAL_THING_HEAD_SIZE; - -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP + hp += EXTERNAL_THING_HEAD_SIZE + ref_words/2 + 1; +#else + hp += EXTERNAL_THING_HEAD_SIZE + ref_words; +#endif + +#if defined(ARCH_64) && !HALFWORD_HEAP etp->header = make_external_ref_header(ref_words/2 + 1); #else etp->header = make_external_ref_header(ref_words); #endif - etp->next = off_heap->externals; + etp->next = off_heap->first; etp->node = node; - off_heap->externals = etp; + off_heap->first = (struct erl_off_heap_header*)etp; *objp = make_external_ref(etp); + ref_num = &(etp->data.ui32[0]); } - ref_num = (Uint32 *) hp; -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP *(ref_num++) = ref_words /* 32-bit arity */; #endif ref_num[0] = r0; @@ -2271,12 +2275,9 @@ dec_term_atom_common: ref_num[i] = get_int32(ep); ep += 4; } -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP if ((1 + ref_words) % 2) ref_num[ref_words] = 0; - hp += ref_words/2 + 1; -#else - hp += ref_words; #endif break; } @@ -2304,8 +2305,8 @@ dec_term_atom_common: hp += PROC_BIN_SIZE; pb->thing_word = HEADER_PROC_BIN; pb->size = n; - pb->next = off_heap->mso; - off_heap->mso = pb; + pb->next = off_heap->first; + off_heap->first = (struct erl_off_heap_header*)pb; pb->val = dbin; pb->bytes = (byte*) dbin->orig_bytes; pb->flags = 0; @@ -2341,8 +2342,8 @@ dec_term_atom_common: pb = (ProcBin *) hp; pb->thing_word = HEADER_PROC_BIN; pb->size = n; - pb->next = off_heap->mso; - off_heap->mso = pb; + pb->next = off_heap->first; + off_heap->first = (struct erl_off_heap_header*)pb; pb->val = dbin; pb->bytes = (byte*) dbin->orig_bytes; pb->flags = 0; @@ -2398,7 +2399,12 @@ dec_term_atom_common: } *objp = make_export(hp); *hp++ = HEADER_EXPORT; +#if HALFWORD_HEAP + *((UWord *) (UWord) hp) = (UWord) erts_export_get_or_make_stub(mod, name, arity); + hp += 2; +#else *hp++ = (Eterm) erts_export_get_or_make_stub(mod, name, arity); +#endif break; } break; @@ -2457,8 +2463,8 @@ dec_term_atom_common: * It is safe to link the fun into the fun list only when * no more validity tests can fail. */ - funp->next = off_heap->funs; - off_heap->funs = funp; + funp->next = off_heap->first; + off_heap->first = (struct erl_off_heap_header*)funp; #endif funp->fe = erts_put_fun_entry2(module, old_uniq, old_index, @@ -2474,11 +2480,11 @@ dec_term_atom_common: /* Environment */ for (i = num_free-1; i >= 0; i--) { - funp->env[i] = (Eterm) next; + funp->env[i] = (Eterm) COMPRESS_POINTER(next); next = funp->env + i; } /* Creator */ - funp->creator = (Eterm) next; + funp->creator = (Eterm) COMPRESS_POINTER(next); next = &(funp->creator); break; } @@ -2535,8 +2541,8 @@ dec_term_atom_common: * It is safe to link the fun into the fun list only when * no more validity tests can fail. */ - funp->next = off_heap->funs; - off_heap->funs = funp; + funp->next = off_heap->first; + off_heap->first = (struct erl_off_heap_header*)funp; #endif old_uniq = unsigned_val(temp); @@ -2549,7 +2555,7 @@ dec_term_atom_common: /* Environment */ for (i = num_free-1; i >= 0; i--) { - funp->env[i] = (Eterm) next; + funp->env[i] = (Eterm) COMPRESS_POINTER(next); next = funp->env + i; } break; @@ -2580,26 +2586,35 @@ dec_term_atom_common: static Uint encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) { - DECLARE_ESTACK(s); + DECLARE_WSTACK(s); Uint m, i, arity; Uint result = 0; +#if HALFWORD_HEAP + UWord wobj = 0; +#endif goto L_jump_start; outer_loop: - while (!ESTACK_ISEMPTY(s)) { - obj = ESTACK_POP(s); - + while (!WSTACK_ISEMPTY(s)) { +#if HALFWORD_HEAP + obj = (Eterm) (wobj = WSTACK_POP(s)); +#else + obj = WSTACK_POP(s); +#endif handle_popped_obj: - if (is_CP(obj)) { + if (is_CP(obj)) { /* Does not look for CP, looks for "no tag" */ +#if HALFWORD_HEAP + Eterm* ptr = (Eterm *) wobj; +#else Eterm* ptr = (Eterm *) obj; - +#endif /* * Pointer into a tuple. */ obj = *ptr--; if (!is_header(obj)) { - ESTACK_PUSH(s, (Eterm)ptr); + WSTACK_PUSH(s, (UWord)ptr); } else { /* Reached tuple header */ ASSERT(header_is_arityval(obj)); @@ -2611,7 +2626,7 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) tl = CDR(cons); obj = CAR(cons); - ESTACK_PUSH(s, tl); + WSTACK_PUSH(s, tl); } else if (is_nil(obj)) { result++; goto outer_loop; @@ -2647,17 +2662,22 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) if ((Uint)val < 256) result += 1 + 1; /* SMALL_INTEGER_EXT */ - else if (sizeof(Sint) == 4 || IS_SSMALL28(val)) + else if (sizeof(Sint) == 4 || IS_SSMALL32(val)) result += 1 + 4; /* INTEGER_EXT */ else { - Eterm tmp_big[2]; + DeclareTmpHeapNoproc(tmp_big,2); + UseTmpHeapNoproc(2); i = big_bytes(small_to_big(val, tmp_big)); result += 1 + 1 + 1 + i; /* SMALL_BIG_EXT */ + UnUseTmpHeapNoproc(2); } } break; case BIG_DEF: - if ((i = big_bytes(obj)) < 256) + i = big_bytes(obj); + if (sizeof(Sint)==4 && i <= 4 && (big_digit(obj,0)-big_sign(obj)) < (1<<31)) + result += 1 + 4; /* INTEGER_EXT */ + else if (i < 256) result += 1 + 1 + 1 + i; /* tag,size,sign,digits */ else result += 1 + 4 + 1 + i; /* tag,size,sign,digits */ @@ -2698,7 +2718,11 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) result += 1 + 4; } ptr += arity; +#if HALFWORD_HEAP + obj = (Eterm) (wobj = (UWord) ptr); +#else obj = (Eterm) ptr; +#endif goto handle_popped_obj; } break; @@ -2740,14 +2764,14 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) if (is_not_list(obj)) { /* Push any non-list terms on the stack */ - ESTACK_PUSH(s, obj); + WSTACK_PUSH(s, obj); } else { /* Lists must be handled specially. */ if ((m = is_string(obj)) && (m < MAX_STRING_LEN)) { result += m + 2 + 1; } else { result += 5; - ESTACK_PUSH(s, obj); + WSTACK_PUSH(s, obj); } } } @@ -2760,8 +2784,12 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) case EXPORT_DEF: { - Export* ep = (Export *) (export_val(obj))[1]; + Export* ep = *((Export **) (export_val(obj) + 1)); +#if HALFWORD_HEAP + result += 2; +#else result += 1; +#endif result += encode_size_struct2(acmp, ep->code[0], dflags); result += encode_size_struct2(acmp, ep->code[1], dflags); result += encode_size_struct2(acmp, make_small(ep->code[2]), dflags); @@ -2774,7 +2802,7 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) } } - DESTROY_ESTACK(s); + DESTROY_WSTACK(s); return result; } @@ -2886,7 +2914,7 @@ decoded_size(byte *ep, byte* endp, int no_refc_bins) ep += 2; atom_extra_skip = 1 + 4*id_words; /* In case it is an external ref */ -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP heap_size += EXTERNAL_THING_HEAD_SIZE + id_words/2 + 1; #else heap_size += EXTERNAL_THING_HEAD_SIZE + id_words; @@ -2961,7 +2989,11 @@ decoded_size(byte *ep, byte* endp, int no_refc_bins) break; case EXPORT_EXT: terms += 3; +#if HALFWORD_HEAP + heap_size += 3; +#else heap_size += 2; +#endif break; case NEW_FUN_EXT: { |