From 55d6274f67cafe62e4923a6369c99a45822cb767 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Wei=C3=9Fl?= Date: Fri, 14 Jun 2013 15:23:40 +0200 Subject: Fix external term format BIFs on floating point middle-endian machines This complements 933e701 (OTP-10209). Simple error example: 1> <<131,70,63,240,0,0,0,0,0,0>> = term_to_binary(1.0, [{minor_version,1}]). ** exception error: no match of right hand side value <<131,70,0,0,0,0,63,240,0,0>> 2> 1.0 = binary_to_term(<<131,70,63,240,0,0,0,0,0,0>>). ** exception error: no match of right hand side value 5.299808824e-315 But roundtrip always works: 3> 1.0 = binary_to_term(term_to_binary(1.0, [{minor_version,1}])). 1.0 --- erts/emulator/beam/external.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 45025ad631..0d56de49c9 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -2316,7 +2316,7 @@ enc_term_int(Process *p,ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dfla GET_DOUBLE(obj, f); if (dflags & DFLAG_NEW_FLOATS) { *ep++ = NEW_FLOAT_EXT; -#ifdef WORDS_BIGENDIAN +#if defined(WORDS_BIGENDIAN) || defined(DOUBLE_MIDDLE_ENDIAN) put_int32(f.fw[0], ep); ep += 4; put_int32(f.fw[1], ep); @@ -2795,7 +2795,7 @@ dec_term_atom_common: volatile unsigned long *fpexnp = erts_get_current_fp_exception(); #endif -#ifdef WORDS_BIGENDIAN +#if defined(WORDS_BIGENDIAN) || defined(DOUBLE_MIDDLE_ENDIAN) ff.fw[0] = get_int32(ep); ep += 4; ff.fw[1] = get_int32(ep); -- cgit v1.2.3 From ee0ca14382e76d97285e64b3396fbb87f33e23da Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Mon, 18 Nov 2013 16:56:40 +0100 Subject: erts: Fix bugs in binary_to_term for invalid bitstrings <<131, 77, Len:32, Bits:8, Data/binary>> badarg if Bits > 8 Used to return internally inconsistent bitstring badarg if Len==0 and Bits > 0 Used to return invalid *huge* binary (size = (Uint)-1) badarg if Bits==0 and Len > 0 Used to return valid binary as if Bits was 8 --- erts/emulator/beam/external.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 1c88765381..cfdd38df73 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -3007,7 +3007,9 @@ dec_term_atom_common: n = get_int32(ep); bitsize = ep[4]; - ep += 5; + if (((bitsize==0) != (n==0)) || bitsize > 8) + goto error; + ep += 5; if (n <= ERL_ONHEAP_BIN_LIMIT) { ErlHeapBin* hb = (ErlHeapBin *) hp; @@ -3035,10 +3037,10 @@ dec_term_atom_common: hp += PROC_BIN_SIZE; } ep += n; - if (bitsize == 0) { + if (bitsize == 8 || n == 0) { *objp = bin; } else { - sb = (ErlSubBin *) hp; + sb = (ErlSubBin *)hp; sb->thing_word = HEADER_SUB_BIN; sb->orig = bin; sb->size = n - 1; -- cgit v1.2.3 From f32368c4b34c86aa772a372cdb3c306a79127185 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Mon, 18 Nov 2013 16:58:43 +0100 Subject: erts: Fix bug in binary_to_term for binaries larger than 2^31 --- erts/emulator/beam/external.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index cfdd38df73..22b0a02937 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -2970,7 +2970,7 @@ dec_term_atom_common: n = get_int32(ep); ep += 4; - if (n <= ERL_ONHEAP_BIN_LIMIT) { + if ((unsigned)n <= ERL_ONHEAP_BIN_LIMIT) { ErlHeapBin* hb = (ErlHeapBin *) hp; hb->thing_word = header_heap_bin(n); @@ -3010,7 +3010,7 @@ dec_term_atom_common: if (((bitsize==0) != (n==0)) || bitsize > 8) goto error; ep += 5; - if (n <= ERL_ONHEAP_BIN_LIMIT) { + if ((unsigned)n <= ERL_ONHEAP_BIN_LIMIT) { ErlHeapBin* hb = (ErlHeapBin *) hp; hb->thing_word = header_heap_bin(n); -- cgit v1.2.3 From 7f17d16335231b4bedd9d2cdd7507ab88a5bf021 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 8 Oct 2013 12:18:46 +0200 Subject: trapping binary_to_term passing binary_SUITE --- erts/emulator/beam/external.c | 299 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 247 insertions(+), 52 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 1c88765381..b1cdad1343 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -61,6 +61,9 @@ */ # define ERTS_DEBUG_USE_DIST_SEP # endif +# define IF_DEBUG(X) X +#else +# define IF_DEBUG(X) #endif /* Does Sint fit in Sint32? @@ -89,7 +92,8 @@ static int enc_term_int(Process *p,ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, static Uint is_external_string(Eterm obj, int* p_is_string); static byte* enc_atom(ErtsAtomCacheMap *, Eterm, byte*, Uint32); static byte* enc_pid(ErtsAtomCacheMap *, Eterm, byte*, Uint32); -static byte* dec_term(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*); +struct B2TContext_t; +static byte* dec_term(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*, struct B2TContext_t*); static byte* dec_atom(ErtsDistExternal *, byte*, Eterm*); static byte* dec_pid(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*); static Sint decoded_size(byte *ep, byte* endp, int internal_tags); @@ -102,11 +106,19 @@ static Uint encode_size_struct2(ErtsAtomCacheMap *, Eterm, unsigned); static int encode_size_struct_int(Process *p, ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags, Sint *reds, Uint *res); +static Export binary_to_term_trap_export; +static BIF_RETTYPE binary_to_term_trap_1(BIF_ALIST_1); +static Eterm binary_to_term_int(Process* p, Eterm bin, Binary* context_b); + void erts_init_external(void) { #if 1 /* In R16 */ erts_init_trap_export(&term_to_binary_trap_export, am_erlang, am_term_to_binary_trap, 1, &term_to_binary_trap_1); + + erts_init_trap_export(&binary_to_term_trap_export, + am_erlang, am_binary_to_term_trap, 1, + &binary_to_term_trap_1); #else sys_memset((void *) &term_to_binary_trap_export, 0, sizeof(Export)); term_to_binary_trap_export.address = &term_to_binary_trap_export.code[3]; @@ -927,7 +939,7 @@ erts_decode_dist_ext(Eterm** hpp, goto error; ep++; } - ep = dec_term(edep, hpp, ep, off_heap, &obj); + ep = dec_term(edep, hpp, ep, off_heap, &obj, NULL); if (!ep) goto error; @@ -948,7 +960,7 @@ Eterm erts_decode_ext(Eterm **hpp, ErlOffHeap *off_heap, byte **ext) byte *ep = *ext; if (*ep++ != VERSION_MAGIC) return THE_NON_VALUE; - ep = dec_term(NULL, hpp, ep, off_heap, &obj); + ep = dec_term(NULL, hpp, ep, off_heap, &obj, NULL); if (!ep) { #ifdef DEBUG bin_write(ERTS_PRINT_STDERR,NULL,*ext,500); @@ -962,7 +974,7 @@ Eterm erts_decode_ext(Eterm **hpp, ErlOffHeap *off_heap, byte **ext) Eterm erts_decode_ext_ets(Eterm **hpp, ErlOffHeap *off_heap, byte *ext) { Eterm obj; - ext = dec_term(NULL, hpp, ext, off_heap, &obj); + ext = dec_term(NULL, hpp, ext, off_heap, &obj, NULL); ASSERT(ext); return obj; } @@ -1112,6 +1124,46 @@ BIF_RETTYPE term_to_binary_2(BIF_ALIST_2) } } + +enum B2TState { /*B2TUncompress,*/ B2TSize, B2TDecodeInit, B2TDecode, B2TDecodeFail, B2TBadArg, B2TDone }; + +typedef struct { +} B2TSizeContext; + +typedef struct { + byte* ep; + Eterm res; + Eterm* next; + Eterm* hp_start; + Eterm* hp; + Eterm* hp_end; +} B2TDecodeContext; + +typedef struct { + /*Uint real_size; + Uint dest_len; + byte *dbytes; + Binary *result_bin; + Binary *destination_bin; + z_stream stream;*/ +} B2TUncompressContext; + +typedef struct B2TContext_t { + Sint heap_size; + byte* aligned_alloc; + ErtsBinary2TermState b2ts; + //Uint ext_size; + Uint reds; + Eterm trap_bin; + enum B2TState state; + union { + B2TSizeContext sc; + B2TDecodeContext dc; + B2TUncompressContext uc; + } u; +} B2TContext; + + static uLongf binary2term_uncomp_size(byte* data, Sint size) { z_stream stream; @@ -1153,7 +1205,7 @@ binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size) if (size < 1 || *bytes != VERSION_MAGIC) { error: if (state->exttmp) - erts_free(ERTS_ALC_T_TMP, state->extp); + erts_free(ERTS_ALC_T_EXT_TERM_DATA, state->extp); state->extp = NULL; state->exttmp = 0; return -1; @@ -1168,17 +1220,18 @@ binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size) bytes += 5; size -= 5; if (dest_len > 32*1024*1024 - || (state->extp = erts_alloc_fnf(ERTS_ALC_T_TMP, dest_len)) == NULL) { + || (state->extp = erts_alloc_fnf(ERTS_ALC_T_EXT_TERM_DATA, dest_len)) == NULL) { if (dest_len != binary2term_uncomp_size(bytes, size)) { goto error; } - state->extp = erts_alloc(ERTS_ALC_T_TMP, dest_len); + state->extp = erts_alloc(ERTS_ALC_T_EXT_TERM_DATA, dest_len); } state->exttmp = 1; if (erl_zlib_uncompress(state->extp, &dest_len, bytes, size) != Z_OK) goto error; size = (Sint) dest_len; } + state->extsize = size; res = decoded_size(state->extp, state->extp + size, 0); if (res < 0) goto error; @@ -1190,7 +1243,7 @@ binary2term_abort(ErtsBinary2TermState *state) { if (state->exttmp) { state->exttmp = 0; - erts_free(ERTS_ALC_T_TMP, state->extp); + erts_free(ERTS_ALC_T_EXT_TERM_DATA, state->extp); } } @@ -1198,11 +1251,11 @@ static ERTS_INLINE Eterm binary2term_create(ErtsDistExternal *edep, ErtsBinary2TermState *state, Eterm **hpp, ErlOffHeap *ohp) { Eterm res; - if (!dec_term(edep, hpp, state->extp, ohp, &res)) + if (!dec_term(edep, hpp, state->extp, ohp, &res, NULL)) res = THE_NON_VALUE; if (state->exttmp) { state->exttmp = 0; - erts_free(ERTS_ALC_T_TMP, state->extp); + erts_free(ERTS_ALC_T_EXT_TERM_DATA, state->extp); } return res; } @@ -1225,46 +1278,155 @@ erts_binary2term_create(ErtsBinary2TermState *state, Eterm **hpp, ErlOffHeap *oh return binary2term_create(NULL,state, hpp, ohp); } +static void b2t_destroy_context(B2TContext* context) +{ + erts_free_aligned_binary_bytes_extra(context->aligned_alloc, + ERTS_ALC_T_EXT_TERM_DATA); + context->aligned_alloc = NULL; + binary2term_abort(&context->b2ts); +} + +static void b2t_context_destructor(Binary *context_bin) +{ + B2TContext* ctx = (B2TContext*) ERTS_MAGIC_BIN_DATA(context_bin); + ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(context_bin) == b2t_context_destructor); + + b2t_destroy_context(ctx); +} + +static BIF_RETTYPE binary_to_term_trap_1(BIF_ALIST_1) +{ + Binary *context_bin = ((ProcBin *) binary_val(BIF_ARG_1))->val; + ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(context_bin) == b2t_context_destructor); + + return binary_to_term_int(BIF_P, THE_NON_VALUE, context_bin); +} + BIF_RETTYPE binary_to_term_1(BIF_ALIST_1) { - Sint heap_size; - Eterm res; - Eterm* hp; - Eterm* endp; - Sint size; + return binary_to_term_int(BIF_P, BIF_ARG_1, NULL); +} + +#define B2T_BYTES_PER_REDUCTION 100 + + +static Eterm binary_to_term_int(Process* p, Eterm bin, Binary* context_b) +{ byte* bytes; - byte* temp_alloc = NULL; - ErtsBinary2TermState b2ts; + B2TContext c_buff; + B2TContext *ctx; + Eterm* magic_space = NULL; - if ((bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc)) == NULL) { - error: - erts_free_aligned_binary_bytes(temp_alloc); - BIF_ERROR(BIF_P, BADARG); - } - size = binary_size(BIF_ARG_1); + if (context_b == NULL) { + /* Setup enough to get started */ + ctx = &c_buff; + ctx->state = B2TSize; + ctx->aligned_alloc = NULL; + IF_DEBUG(ctx->trap_bin = THE_NON_VALUE;) + } else { + ctx = ERTS_MAGIC_BIN_DATA(context_b); + } + ctx->reds = 1; /*(Uint)(ERTS_BIF_REDS_LEFT(p) * B2T_BYTES_PER_REDUCTION);*/ + + do { + switch (ctx->state) { + case B2TSize: + bytes = erts_get_aligned_binary_bytes_extra(bin, + &ctx->aligned_alloc, + ERTS_ALC_T_EXT_TERM_DATA, + 0); + if (bytes == NULL) { + ctx->b2ts.exttmp = 0; + ctx->state = B2TBadArg; + break; + } - heap_size = binary2term_prepare(&b2ts, bytes, size); - if (heap_size < 0) - goto error; + ctx->heap_size = binary2term_prepare(&ctx->b2ts, bytes, + binary_size(bin)); + if (ctx->heap_size < 0) { + ctx->state = B2TBadArg; + break; + } - hp = HAlloc(BIF_P, heap_size); - endp = hp + heap_size; - res = binary2term_create(NULL, &b2ts, &hp, &MSO(BIF_P)); - erts_free_aligned_binary_bytes(temp_alloc); + ctx->state = B2TDecodeInit; - if (hp > endp) { - erl_exit(1, ":%s, line %d: heap overrun by %d words(s)\n", - __FILE__, __LINE__, hp-endp); - } + if (ctx->b2ts.extsize >= ctx->reds) { + ctx->reds = 0; + break; + } + ctx->reds -= ctx->b2ts.extsize; + /*fall through*/ + case B2TDecodeInit: + if (context_b == NULL && ctx->b2ts.extsize > ctx->reds) { + /* dec_term will probably trap, allocate space for magic bin + before result term to make it easy to trim with HRelease. + */ + magic_space = HAlloc(p, PROC_BIN_SIZE); + *magic_space = make_pos_bignum_header(PROC_BIN_SIZE-1); + } + ctx->u.dc.ep = ctx->b2ts.extp; + ctx->u.dc.res = (Eterm) (UWord) NULL; + ctx->u.dc.next = &ctx->u.dc.res; + ctx->u.dc.hp_start = HAlloc(p, ctx->heap_size); + ctx->u.dc.hp = ctx->u.dc.hp_start; + ctx->u.dc.hp_end = ctx->u.dc.hp_start + ctx->heap_size; + ctx->state = B2TDecode; + /*fall through*/ + case B2TDecode: + dec_term(NULL, NULL, NULL, &MSO(p), NULL, ctx); + break; + + case B2TDecodeFail: + HRelease(p, ctx->u.dc.hp_end, ctx->u.dc.hp_start); + /*fall through*/ + case B2TBadArg: + b2t_destroy_context(ctx); + if (context_b) { + erts_set_gc_state(p, 1); + } + BIF_ERROR(p, BADARG); - HRelease(BIF_P, endp, hp); + case B2TDone: + b2t_destroy_context(ctx); - if (res == THE_NON_VALUE) - goto error; + if (ctx->u.dc.hp > ctx->u.dc.hp_end) { + erl_exit(1, ":%s, line %d: heap overrun by %d words(s)\n", + __FILE__, __LINE__, ctx->u.dc.hp - ctx->u.dc.hp_end); + } + HRelease(p, ctx->u.dc.hp_end, ctx->u.dc.hp); - return res; + if (context_b) { + erts_set_gc_state(p, 1); + } + + return ctx->u.dc.res; + + } + }while (ctx->reds); + + if (context_b == NULL) { + ASSERT(ctx->trap_bin == THE_NON_VALUE); + + context_b = erts_create_magic_binary(sizeof(B2TContext), + b2t_context_destructor); + ctx = ERTS_MAGIC_BIN_DATA(context_b); + sys_memcpy(ctx, &c_buff, sizeof(B2TContext)); + + if (!magic_space) { + ASSERT(ctx->state != B2TDecode); + magic_space = HAlloc(p, PROC_BIN_SIZE); + } + ctx->trap_bin = erts_mk_magic_binary_term(&magic_space, &MSO(p), context_b); + + erts_set_gc_state(p, 0); + } + ASSERT(context_b); + ASSERT(ctx->trap_bin != THE_NON_VALUE); + + BUMP_ALL_REDS(p); + BIF_TRAP1(&binary_to_term_trap_export, p, ctx->trap_bin); } BIF_RETTYPE binary_to_term_2(BIF_ALIST_2) @@ -1514,7 +1676,7 @@ typedef struct { } s; } TTBContext; -static void context_destructor(Binary *context_bin) +static void ttb_context_destructor(Binary *context_bin) { TTBContext *context = ERTS_MAGIC_BIN_DATA(context_bin); if (context->alive) { @@ -1567,7 +1729,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla do { \ if (context_b == NULL) { \ context_b = erts_create_magic_binary(sizeof(TTBContext), \ - context_destructor); \ + ttb_context_destructor); \ context = ERTS_MAGIC_BIN_DATA(context_b); \ memcpy(context,&c_buff,sizeof(TTBContext)); \ } \ @@ -2608,15 +2770,34 @@ undo_offheap_in_area(ErlOffHeap* off_heap, Eterm* start, Eterm* end) ** On failure return NULL and (R13B04) *hpp will be unchanged. */ static byte* -dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Eterm* objp) +dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, + Eterm* objp, B2TContext* ctx) { - Eterm* hp_saved = *hpp; + Eterm* hp_saved; int n; ErtsAtomEncoding char_enc; - register Eterm* hp = *hpp; /* Please don't take the address of hp */ - Eterm* next = objp; - - *next = (Eterm) (UWord) NULL; + register Eterm* hp; /* Please don't take the address of hp */ + Eterm* next; + byte* ep_trap_limit; + + if (ctx) { + hp_saved = ctx->u.dc.hp_start; + ep = ctx->u.dc.ep; + ep_trap_limit = ep + ctx->reds; + if (ep_trap_limit < ep) { + ep_trap_limit = (byte*)ERTS_UWORD_MAX; /*SVERK Is there a safe way to create a "largest ptr" */ + } + next = ctx->u.dc.next; + hpp = &ctx->u.dc.hp; + } + else { + hp_saved = *hpp; + ep_trap_limit = (byte*)ERTS_UWORD_MAX; /*SVERK - " - */ + next = objp; + *next = (Eterm) (UWord) NULL; + } + ASSERT(ep < ep_trap_limit); + hp = *hpp; while (next != NULL) { objp = next; @@ -3065,7 +3246,7 @@ dec_term_atom_common: goto error; } *hpp = hp; - ep = dec_term(edep, hpp, ep, off_heap, &temp); + ep = dec_term(edep, hpp, ep, off_heap, &temp, NULL); hp = *hpp; if (ep == NULL) { goto error; @@ -3125,7 +3306,7 @@ dec_term_atom_common: } *hpp = hp; /* Index */ - if ((ep = dec_term(edep, hpp, ep, off_heap, &temp)) == NULL) { + if ((ep = dec_term(edep, hpp, ep, off_heap, &temp, NULL)) == NULL) { goto error; } if (!is_small(temp)) { @@ -3134,7 +3315,7 @@ dec_term_atom_common: old_index = unsigned_val(temp); /* Uniq */ - if ((ep = dec_term(edep, hpp, ep, off_heap, &temp)) == NULL) { + if ((ep = dec_term(edep, hpp, ep, off_heap, &temp, NULL)) == NULL) { goto error; } if (!is_small(temp)) { @@ -3202,7 +3383,7 @@ dec_term_atom_common: } /* Index */ - if ((ep = dec_term(edep, hpp, ep, off_heap, &temp)) == NULL) { + if ((ep = dec_term(edep, hpp, ep, off_heap, &temp, NULL)) == NULL) { goto error; } if (!is_small(temp)) { @@ -3211,7 +3392,7 @@ dec_term_atom_common: old_index = unsigned_val(temp); /* Uniq */ - if ((ep = dec_term(edep, hpp, ep, off_heap, &temp)) == NULL) { + if ((ep = dec_term(edep, hpp, ep, off_heap, &temp, NULL)) == NULL) { goto error; } if (!is_small(temp)) { @@ -3311,8 +3492,22 @@ dec_term_atom_common: } undo_offheap_in_area(off_heap, hp_saved, hp); *hpp = hp_saved; - return NULL; + if (ctx) { + ctx->state = B2TDecodeFail; + } + return NULL; } + if (ep > ep_trap_limit) { + ASSERT(ctx); + ctx->u.dc.ep = ep; + ctx->u.dc.next = next; + ctx->u.dc.hp = hp; + ctx->reds = 0; + return NULL; + } + } + if (ctx) { + ctx->state = B2TDone; } *hpp = hp; return ep; -- cgit v1.2.3 From 78a5dd69c9ed1749f2ad4fd24151e4aa784b1cba Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 10 Oct 2013 12:11:29 +0200 Subject: trapping lists and tuples --- erts/emulator/beam/external.c | 153 +++++++++++++++++++++++++++++++++++------- 1 file changed, 127 insertions(+), 26 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index b1cdad1343..48536d610b 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1125,7 +1125,18 @@ BIF_RETTYPE term_to_binary_2(BIF_ALIST_2) } -enum B2TState { /*B2TUncompress,*/ B2TSize, B2TDecodeInit, B2TDecode, B2TDecodeFail, B2TBadArg, B2TDone }; +enum B2TState { + /*B2TUncompress,*/ + B2TSize, + B2TDecodeInit, + B2TDecode, + B2TDecodeList, + B2TDecodeTuple, + + B2TDone, + B2TDecodeFail, + B2TBadArg +}; typedef struct { } B2TSizeContext; @@ -1137,6 +1148,10 @@ typedef struct { Eterm* hp_start; Eterm* hp; Eterm* hp_end; + int remaining_n; +#ifdef DEBUG + Eterm* container_start; +#endif } B2TDecodeContext; typedef struct { @@ -1153,7 +1168,7 @@ typedef struct B2TContext_t { byte* aligned_alloc; ErtsBinary2TermState b2ts; //Uint ext_size; - Uint reds; + SWord reds; Eterm trap_bin; enum B2TState state; union { @@ -1309,6 +1324,12 @@ BIF_RETTYPE binary_to_term_1(BIF_ALIST_1) #define B2T_BYTES_PER_REDUCTION 100 +static unsigned sverk_rand(void) +{ + static unsigned prev = 17; + prev = (prev * 214013 + 2531011); + return prev; +} static Eterm binary_to_term_int(Process* p, Eterm bin, Binary* context_b) { @@ -1326,7 +1347,7 @@ static Eterm binary_to_term_int(Process* p, Eterm bin, Binary* context_b) } else { ctx = ERTS_MAGIC_BIN_DATA(context_b); } - ctx->reds = 1; /*(Uint)(ERTS_BIF_REDS_LEFT(p) * B2T_BYTES_PER_REDUCTION);*/ + ctx->reds = 1 + sverk_rand() % 4; /*(Uint)(ERTS_BIF_REDS_LEFT(p) * B2T_BYTES_PER_REDUCTION);*/ do { switch (ctx->state) { @@ -1360,6 +1381,10 @@ static Eterm binary_to_term_int(Process* p, Eterm bin, Binary* context_b) /*fall through*/ case B2TDecodeInit: if (context_b == NULL && ctx->b2ts.extsize > ctx->reds) { + /*SVERK Can we do a better prediction that is still safe + OR is there a way to do HAlloc after result some how... + ... support for mutiple HReleases maybe? + */ /* dec_term will probably trap, allocate space for magic bin before result term to make it easy to trim with HRelease. */ @@ -1374,7 +1399,9 @@ static Eterm binary_to_term_int(Process* p, Eterm bin, Binary* context_b) ctx->u.dc.hp_end = ctx->u.dc.hp_start + ctx->heap_size; ctx->state = B2TDecode; /*fall through*/ - case B2TDecode: + case B2TDecode: + case B2TDecodeList: + case B2TDecodeTuple: dec_term(NULL, NULL, NULL, &MSO(p), NULL, ctx); break; @@ -1404,7 +1431,7 @@ static Eterm binary_to_term_int(Process* p, Eterm bin, Binary* context_b) return ctx->u.dc.res; } - }while (ctx->reds); + }while (ctx->reds || ctx->state >= B2TDone); if (context_b == NULL) { ASSERT(ctx->trap_bin == THE_NON_VALUE); @@ -1415,7 +1442,7 @@ static Eterm binary_to_term_int(Process* p, Eterm bin, Binary* context_b) sys_memcpy(ctx, &c_buff, sizeof(B2TContext)); if (!magic_space) { - ASSERT(ctx->state != B2TDecode); + ASSERT(ctx->state < B2TDecode); magic_space = HAlloc(p, PROC_BIN_SIZE); } ctx->trap_bin = erts_mk_magic_binary_term(&magic_space, &MSO(p), context_b); @@ -2778,28 +2805,81 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, ErtsAtomEncoding char_enc; register Eterm* hp; /* Please don't take the address of hp */ Eterm* next; - byte* ep_trap_limit; + SWord reds; if (ctx) { + reds = ctx->reds; + next = ctx->u.dc.next; + + if (ctx->state != B2TDecode) { + n = ctx->u.dc.remaining_n; + if (reds < n) { + ctx->u.dc.remaining_n -= reds; + n = reds; + } + else { + ctx->u.dc.remaining_n = 0; + } + reds -= n; + + switch (ctx->state) { + case B2TDecodeList: + objp = next - 2; + while (n > 0) { + objp[0] = (Eterm) COMPRESS_POINTER(next); + objp[1] = make_list(next); + next = objp; + objp -= 2; + n--; + } + break; + + case B2TDecodeTuple: + objp = next - 1; + while (n-- > 0) { + objp[0] = (Eterm) COMPRESS_POINTER(next); + next = objp; + objp--; + } + break; + + default: + ASSERT(!"Unknown state"); + } + if (ctx->u.dc.remaining_n) { + ctx->u.dc.next = next; + ctx->reds = 0; + return NULL; + } + ASSERT(next == ctx->u.dc.container_start); + ctx->state = B2TDecode; + } + hp_saved = ctx->u.dc.hp_start; ep = ctx->u.dc.ep; - ep_trap_limit = ep + ctx->reds; - if (ep_trap_limit < ep) { - ep_trap_limit = (byte*)ERTS_UWORD_MAX; /*SVERK Is there a safe way to create a "largest ptr" */ - } - next = ctx->u.dc.next; hpp = &ctx->u.dc.hp; } else { hp_saved = *hpp; - ep_trap_limit = (byte*)ERTS_UWORD_MAX; /*SVERK - " - */ + reds = ERTS_SWORD_MAX; next = objp; *next = (Eterm) (UWord) NULL; } - ASSERT(ep < ep_trap_limit); hp = *hpp; while (next != NULL) { + + if (reds <= 0) { + if (ctx) { + ctx->u.dc.ep = ep; + ctx->u.dc.next = next; + ctx->u.dc.hp = hp; + ctx->reds = 0; + return NULL; + } + reds = ERTS_SWORD_MAX; + } + objp = next; next = (Eterm *) EXPAND_POINTER(*objp); @@ -2918,8 +2998,20 @@ dec_term_atom_common: tuple_loop: *objp = make_tuple(hp); *hp++ = make_arityval(n); + #ifdef DEBUG + if (ctx) ctx->u.dc.container_start = hp; + #endif hp += n; - objp = hp - 1; + objp = hp - 1; + if (ctx) { + if (reds < n) { + ASSERT(reds > 0); + ctx->state = B2TDecodeTuple; + ctx->u.dc.remaining_n = n - reds; + n = reds; + } + reds -= n; + } while (n-- > 0) { objp[0] = (Eterm) COMPRESS_POINTER(next); next = objp; @@ -2937,17 +3029,30 @@ dec_term_atom_common: break; } *objp = make_list(hp); - hp += 2*n; + #ifdef DEBUG + if (ctx) ctx->u.dc.container_start = hp; + #endif + hp += 2 * n; objp = hp - 2; objp[0] = (Eterm) COMPRESS_POINTER((objp+1)); objp[1] = (Eterm) COMPRESS_POINTER(next); next = objp; objp -= 2; - while (--n > 0) { + n--; + if (ctx) { + if (reds < n) { + ctx->state = B2TDecodeList; + ctx->u.dc.remaining_n = n - reds; + n = reds; + } + reds -= n; + } + while (n > 0) { objp[0] = (Eterm) COMPRESS_POINTER(next); - objp[1] = make_list(objp + 2); + objp[1] = make_list(next); next = objp; objp -= 2; + n--; } break; case STRING_EXT: @@ -3494,20 +3599,16 @@ dec_term_atom_common: *hpp = hp_saved; if (ctx) { ctx->state = B2TDecodeFail; + ctx->reds = reds; } return NULL; } - if (ep > ep_trap_limit) { - ASSERT(ctx); - ctx->u.dc.ep = ep; - ctx->u.dc.next = next; - ctx->u.dc.hp = hp; - ctx->reds = 0; - return NULL; - } + + --reds; } if (ctx) { ctx->state = B2TDone; + ctx->reds = reds; } *hpp = hp; return ep; -- cgit v1.2.3 From d42bd3b37d3b741d546d94fd7611afda81bba419 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 10 Oct 2013 12:11:59 +0200 Subject: trapping STRING_EXT --- erts/emulator/beam/external.c | 84 ++++++++++++++++++++++++++++--------------- 1 file changed, 55 insertions(+), 29 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 48536d610b..2116c25cd2 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1125,13 +1125,15 @@ BIF_RETTYPE term_to_binary_2(BIF_ALIST_2) } -enum B2TState { +enum B2TState { /* order is somewhat significant */ /*B2TUncompress,*/ B2TSize, B2TDecodeInit, + B2TDecode, B2TDecodeList, B2TDecodeTuple, + B2TDecodeString, B2TDone, B2TDecodeFail, @@ -1150,7 +1152,7 @@ typedef struct { Eterm* hp_end; int remaining_n; #ifdef DEBUG - Eterm* container_start; + Eterm* container_end; #endif } B2TDecodeContext; @@ -1402,6 +1404,7 @@ static Eterm binary_to_term_int(Process* p, Eterm bin, Binary* context_b) case B2TDecode: case B2TDecodeList: case B2TDecodeTuple: + case B2TDecodeString: dec_term(NULL, NULL, NULL, &MSO(p), NULL, ctx); break; @@ -2808,8 +2811,11 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, SWord reds; if (ctx) { - reds = ctx->reds; - next = ctx->u.dc.next; + hp_saved = ctx->u.dc.hp_start; + reds = ctx->reds; + next = ctx->u.dc.next; + ep = ctx->u.dc.ep; + hpp = &ctx->u.dc.hp; if (ctx->state != B2TDecode) { n = ctx->u.dc.remaining_n; @@ -2832,6 +2838,7 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, objp -= 2; n--; } + ASSERT(ctx->u.dc.remaining_n || next == ctx->u.dc.container_end); break; case B2TDecodeTuple: @@ -2841,23 +2848,35 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, next = objp; objp--; } + ASSERT(ctx->u.dc.remaining_n || next == ctx->u.dc.container_end); + break; + + case B2TDecodeString: + hp = *hpp; + hp[-1] = make_list(hp); /* overwrite the premature NIL */ + while (n-- > 0) { + hp[0] = make_small(*ep++); + hp[1] = make_list(hp+2); + hp += 2; + } + hp[-1] = NIL; + *hpp = hp; + ASSERT(ctx->u.dc.remaining_n || hp == ctx->u.dc.container_end); break; default: ASSERT(!"Unknown state"); } - if (ctx->u.dc.remaining_n) { + if (!ctx->u.dc.remaining_n) { + ctx->state = B2TDecode; + } + if (reds <= 0) { ctx->u.dc.next = next; + ctx->u.dc.ep = ep; ctx->reds = 0; return NULL; } - ASSERT(next == ctx->u.dc.container_start); - ctx->state = B2TDecode; } - - hp_saved = ctx->u.dc.hp_start; - ep = ctx->u.dc.ep; - hpp = &ctx->u.dc.hp; } else { hp_saved = *hpp; @@ -2869,17 +2888,6 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, while (next != NULL) { - if (reds <= 0) { - if (ctx) { - ctx->u.dc.ep = ep; - ctx->u.dc.next = next; - ctx->u.dc.hp = hp; - ctx->reds = 0; - return NULL; - } - reds = ERTS_SWORD_MAX; - } - objp = next; next = (Eterm *) EXPAND_POINTER(*objp); @@ -2998,13 +3006,11 @@ dec_term_atom_common: tuple_loop: *objp = make_tuple(hp); *hp++ = make_arityval(n); - #ifdef DEBUG - if (ctx) ctx->u.dc.container_start = hp; - #endif hp += n; objp = hp - 1; if (ctx) { if (reds < n) { + IF_DEBUG(ctx->u.dc.container_end = hp - n;) ASSERT(reds > 0); ctx->state = B2TDecodeTuple; ctx->u.dc.remaining_n = n - reds; @@ -3029,9 +3035,6 @@ dec_term_atom_common: break; } *objp = make_list(hp); - #ifdef DEBUG - if (ctx) ctx->u.dc.container_start = hp; - #endif hp += 2 * n; objp = hp - 2; objp[0] = (Eterm) COMPRESS_POINTER((objp+1)); @@ -3041,6 +3044,7 @@ dec_term_atom_common: n--; if (ctx) { if (reds < n) { + IF_DEBUG(ctx->u.dc.container_end = hp - 2*(n+1);) ctx->state = B2TDecodeList; ctx->u.dc.remaining_n = n - reds; n = reds; @@ -3063,6 +3067,15 @@ dec_term_atom_common: break; } *objp = make_list(hp); + if (ctx) { + if (reds < n) { + IF_DEBUG(ctx->u.dc.container_end = hp + n*2;) + ctx->state = B2TDecodeString; + ctx->u.dc.remaining_n = n - reds; + n = reds; + } + reds -= n; + } while (n-- > 0) { hp[0] = make_small(*ep++); hp[1] = make_list(hp+2); @@ -3604,7 +3617,20 @@ dec_term_atom_common: return NULL; } - --reds; + if (--reds <= 0) { + if (ctx) { + if (next || ctx->state != B2TDecode) { + ctx->u.dc.ep = ep; + ctx->u.dc.next = next; + ctx->u.dc.hp = hp; + ctx->reds = 0; + return NULL; + } + } + else { + reds = ERTS_SWORD_MAX; + } + } } if (ctx) { ctx->state = B2TDone; -- cgit v1.2.3 From 74c5c0314d4cf85365d89c1fdf41a5d286a7191a Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 10 Oct 2013 19:59:09 +0200 Subject: trapping binary_to_term/2 --- erts/emulator/beam/external.c | 73 +++++++++++++++---------------------------- 1 file changed, 26 insertions(+), 47 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 2116c25cd2..38280b7a99 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -108,7 +108,7 @@ static int encode_size_struct_int(Process *p, ErtsAtomCacheMap *acmp, Eterm obj, static Export binary_to_term_trap_export; static BIF_RETTYPE binary_to_term_trap_1(BIF_ALIST_1); -static Eterm binary_to_term_int(Process* p, Eterm bin, Binary* context_b); +static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* context_b); void erts_init_external(void) { #if 1 /* In R16 */ @@ -1169,7 +1169,7 @@ typedef struct B2TContext_t { Sint heap_size; byte* aligned_alloc; ErtsBinary2TermState b2ts; - //Uint ext_size; + Uint32 flags; SWord reds; Eterm trap_bin; enum B2TState state; @@ -1316,15 +1316,22 @@ static BIF_RETTYPE binary_to_term_trap_1(BIF_ALIST_1) Binary *context_bin = ((ProcBin *) binary_val(BIF_ARG_1))->val; ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(context_bin) == b2t_context_destructor); - return binary_to_term_int(BIF_P, THE_NON_VALUE, context_bin); + return binary_to_term_int(BIF_P, 0, THE_NON_VALUE, context_bin); } BIF_RETTYPE binary_to_term_1(BIF_ALIST_1) { - return binary_to_term_int(BIF_P, BIF_ARG_1, NULL); +/*SVERK if (++sverk_cnt % 1000 == 0) { + erts_fprintf(stderr, "Call #%u to binary_to_term_int()\n", sverk_cnt); + } + if (sverk_cnt == 1767) { + sverk_break(); + } +*/ + return binary_to_term_int(BIF_P, 0, BIF_ARG_1, NULL); } -#define B2T_BYTES_PER_REDUCTION 100 +#define B2T_BYTES_PER_REDUCTION 128 static unsigned sverk_rand(void) { @@ -1333,11 +1340,13 @@ static unsigned sverk_rand(void) return prev; } -static Eterm binary_to_term_int(Process* p, Eterm bin, Binary* context_b) +static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* context_b) { + SWord initial_reds = 1 + sverk_rand() % 4; /*(Uint)(ERTS_BIF_REDS_LEFT(p) * B2T_BYTES_PER_REDUCTION);*/ byte* bytes; B2TContext c_buff; B2TContext *ctx; + ErtsDistExternal fakedep; Eterm* magic_space = NULL; if (context_b == NULL) { @@ -1345,11 +1354,12 @@ static Eterm binary_to_term_int(Process* p, Eterm bin, Binary* context_b) ctx = &c_buff; ctx->state = B2TSize; ctx->aligned_alloc = NULL; + ctx->flags = flags; IF_DEBUG(ctx->trap_bin = THE_NON_VALUE;) } else { ctx = ERTS_MAGIC_BIN_DATA(context_b); } - ctx->reds = 1 + sverk_rand() % 4; /*(Uint)(ERTS_BIF_REDS_LEFT(p) * B2T_BYTES_PER_REDUCTION);*/ + ctx->reds = initial_reds; do { switch (ctx->state) { @@ -1405,7 +1415,8 @@ static Eterm binary_to_term_int(Process* p, Eterm bin, Binary* context_b) case B2TDecodeList: case B2TDecodeTuple: case B2TDecodeString: - dec_term(NULL, NULL, NULL, &MSO(p), NULL, ctx); + fakedep.flags = ctx->flags; + dec_term(&fakedep, NULL, NULL, &MSO(p), NULL, ctx); break; case B2TDecodeFail: @@ -1416,6 +1427,7 @@ static Eterm binary_to_term_int(Process* p, Eterm bin, Binary* context_b) if (context_b) { erts_set_gc_state(p, 1); } + BUMP_REDS(p, (initial_reds - ctx->reds) / B2T_BYTES_PER_REDUCTION); BIF_ERROR(p, BADARG); case B2TDone: @@ -1431,6 +1443,7 @@ static Eterm binary_to_term_int(Process* p, Eterm bin, Binary* context_b) erts_set_gc_state(p, 1); } + BUMP_REDS(p, (initial_reds - ctx->reds) / B2T_BYTES_PER_REDUCTION); return ctx->u.dc.res; } @@ -1461,24 +1474,15 @@ static Eterm binary_to_term_int(Process* p, Eterm bin, Binary* context_b) BIF_RETTYPE binary_to_term_2(BIF_ALIST_2) { - Sint heap_size; - Eterm res; Eterm opts; Eterm opt; - Eterm* hp; - Eterm* endp; - Sint size; - byte* bytes; - byte* temp_alloc = NULL; - ErtsBinary2TermState b2ts; - ErtsDistExternal fakedep; + Uint32 flags = 0; - fakedep.flags = 0; opts = BIF_ARG_2; while (is_list(opts)) { opt = CAR(list_val(opts)); if (opt == am_safe) { - fakedep.flags |= ERTS_DIST_EXT_BTT_SAFE; + flags |= ERTS_DIST_EXT_BTT_SAFE; } else { goto error; @@ -1489,35 +1493,10 @@ BIF_RETTYPE binary_to_term_2(BIF_ALIST_2) if (is_not_nil(opts)) goto error; - if ((bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc)) == NULL) { - error: - erts_free_aligned_binary_bytes(temp_alloc); - BIF_ERROR(BIF_P, BADARG); - } - size = binary_size(BIF_ARG_1); - - heap_size = binary2term_prepare(&b2ts, bytes, size); - if (heap_size < 0) - goto error; - - hp = HAlloc(BIF_P, heap_size); - endp = hp + heap_size; - - res = binary2term_create(&fakedep, &b2ts, &hp, &MSO(BIF_P)); - - erts_free_aligned_binary_bytes(temp_alloc); - - if (hp > endp) { - erl_exit(1, ":%s, line %d: heap overrun by %d words(s)\n", - __FILE__, __LINE__, hp-endp); - } - - HRelease(BIF_P, endp, hp); - - if (res == THE_NON_VALUE) - goto error; + return binary_to_term_int(BIF_P, flags, BIF_ARG_1, NULL); - return res; +error: + BIF_ERROR(BIF_P, BADARG); } Eterm -- cgit v1.2.3 From 38f1140d197e14f6cb4d0040859b0b19876d2a14 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 11 Oct 2013 16:28:59 +0200 Subject: trapping size calculation --- erts/emulator/beam/external.c | 175 +++++++++++++++++++++++++++--------------- 1 file changed, 114 insertions(+), 61 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 38280b7a99..d4ffc23536 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -96,7 +96,7 @@ struct B2TContext_t; static byte* dec_term(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*, struct B2TContext_t*); static byte* dec_atom(ErtsDistExternal *, byte*, Eterm*); static byte* dec_pid(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*); -static Sint decoded_size(byte *ep, byte* endp, int internal_tags); +static Sint decoded_size(byte *ep, byte* endp, int internal_tags, struct B2TContext_t*); static BIF_RETTYPE term_to_binary_trap_1(BIF_ALIST_1); static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint flags, @@ -889,7 +889,7 @@ erts_decode_dist_ext_size(ErtsDistExternal *edep) goto fail; ep = edep->extp+1; } - res = decoded_size(ep, edep->ext_endp, 0); + res = decoded_size(ep, edep->ext_endp, 0, NULL); if (res >= 0) return res; fail: @@ -901,12 +901,12 @@ Sint erts_decode_ext_size(byte *ext, Uint size) { if (size == 0 || *ext != VERSION_MAGIC) return -1; - return decoded_size(ext+1, ext+size, 0); + return decoded_size(ext+1, ext+size, 0, NULL); } Sint erts_decode_ext_size_ets(byte *ext, Uint size) { - Sint sz = decoded_size(ext, ext+size, 1); + Sint sz = decoded_size(ext, ext+size, 1, NULL); ASSERT(sz >= 0); return sz; } @@ -1126,7 +1126,7 @@ BIF_RETTYPE term_to_binary_2(BIF_ALIST_2) enum B2TState { /* order is somewhat significant */ - /*B2TUncompress,*/ + B2TPrepare, B2TSize, B2TDecodeInit, @@ -1141,6 +1141,10 @@ enum B2TState { /* order is somewhat significant */ }; typedef struct { + int heap_size; + int terms; + byte* ep; + int atom_extra_skip; } B2TSizeContext; typedef struct { @@ -1210,21 +1214,15 @@ static uLongf binary2term_uncomp_size(byte* data, Sint size) return err == Z_STREAM_END ? uncomp_size : 0; } -static ERTS_INLINE Sint +static ERTS_INLINE int binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size) { - Sint res; byte *bytes = data; Sint size = data_size; state->exttmp = 0; if (size < 1 || *bytes != VERSION_MAGIC) { - error: - if (state->exttmp) - erts_free(ERTS_ALC_T_EXT_TERM_DATA, state->extp); - state->extp = NULL; - state->exttmp = 0; return -1; } bytes++; @@ -1239,20 +1237,17 @@ binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size) if (dest_len > 32*1024*1024 || (state->extp = erts_alloc_fnf(ERTS_ALC_T_EXT_TERM_DATA, dest_len)) == NULL) { if (dest_len != binary2term_uncomp_size(bytes, size)) { - goto error; + return -1; } state->extp = erts_alloc(ERTS_ALC_T_EXT_TERM_DATA, dest_len); } state->exttmp = 1; if (erl_zlib_uncompress(state->extp, &dest_len, bytes, size) != Z_OK) - goto error; + return -1; size = (Sint) dest_len; } state->extsize = size; - res = decoded_size(state->extp, state->extp + size, 0); - if (res < 0) - goto error; - return res; + return 0; } static ERTS_INLINE void @@ -1280,7 +1275,18 @@ binary2term_create(ErtsDistExternal *edep, ErtsBinary2TermState *state, Eterm ** Sint erts_binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size) { - return binary2term_prepare(state, data, data_size); + Sint res; + + if (binary2term_prepare(state, data, data_size) < 0 || + (res=decoded_size(state->extp, state->extp + state->extsize, 0, NULL)) < 0) { + + if (state->exttmp) + erts_free(ERTS_ALC_T_EXT_TERM_DATA, state->extp); + state->extp = NULL; + state->exttmp = 0; + return -1; + } + return res; } void @@ -1319,17 +1325,6 @@ static BIF_RETTYPE binary_to_term_trap_1(BIF_ALIST_1) return binary_to_term_int(BIF_P, 0, THE_NON_VALUE, context_bin); } -BIF_RETTYPE binary_to_term_1(BIF_ALIST_1) -{ -/*SVERK if (++sverk_cnt % 1000 == 0) { - erts_fprintf(stderr, "Call #%u to binary_to_term_int()\n", sverk_cnt); - } - if (sverk_cnt == 1767) { - sverk_break(); - } -*/ - return binary_to_term_int(BIF_P, 0, BIF_ARG_1, NULL); -} #define B2T_BYTES_PER_REDUCTION 128 @@ -1352,18 +1347,19 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con if (context_b == NULL) { /* Setup enough to get started */ ctx = &c_buff; - ctx->state = B2TSize; + ctx->state = B2TPrepare; ctx->aligned_alloc = NULL; ctx->flags = flags; IF_DEBUG(ctx->trap_bin = THE_NON_VALUE;) } else { ctx = ERTS_MAGIC_BIN_DATA(context_b); + ASSERT(ctx->state != B2TPrepare); } ctx->reds = initial_reds; do { switch (ctx->state) { - case B2TSize: + case B2TPrepare: bytes = erts_get_aligned_binary_bytes_extra(bin, &ctx->aligned_alloc, ERTS_ALC_T_EXT_TERM_DATA, @@ -1381,16 +1377,19 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con break; } + ctx->reds = 0; /*SVERK*/ + ctx->u.sc.heap_size = 0; + ctx->u.sc.terms = 1; + ctx->u.sc.ep = ctx->b2ts.extp; + ctx->u.sc.atom_extra_skip = 0; + ctx->state = B2TSize; + break; + case B2TSize: + ctx->heap_size = decoded_size(NULL, ctx->b2ts.extp + ctx->b2ts.extsize, + 0, ctx); + break; - ctx->state = B2TDecodeInit; - - if (ctx->b2ts.extsize >= ctx->reds) { - ctx->reds = 0; - break; - } - ctx->reds -= ctx->b2ts.extsize; - /*fall through*/ case B2TDecodeInit: if (context_b == NULL && ctx->b2ts.extsize > ctx->reds) { /*SVERK Can we do a better prediction that is still safe @@ -1472,6 +1471,18 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con BIF_TRAP1(&binary_to_term_trap_export, p, ctx->trap_bin); } +BIF_RETTYPE binary_to_term_1(BIF_ALIST_1) +{ +/*SVERK if (++sverk_cnt % 1000 == 0) { + erts_fprintf(stderr, "Call #%u to binary_to_term_int()\n", sverk_cnt); + } + if (sverk_cnt == 1767) { + sverk_break(); + } +*/ + return binary_to_term_int(BIF_P, 0, BIF_ARG_1, NULL); +} + BIF_RETTYPE binary_to_term_2(BIF_ALIST_2) { Eterm opts; @@ -3901,18 +3912,32 @@ encode_size_struct_int(Process *p, ErtsAtomCacheMap *acmp, Eterm obj, } static Sint -decoded_size(byte *ep, byte* endp, int internal_tags) +decoded_size(byte *ep, byte* endp, int internal_tags, B2TContext* ctx) { int heap_size = 0; int terms; int atom_extra_skip = 0; Uint n; + SWord reds; + if (ctx) { + heap_size = ctx->u.sc.heap_size; + terms = ctx->u.sc.terms; + ep = ctx->u.sc.ep; + atom_extra_skip = ctx->u.sc.atom_extra_skip; + reds = ctx->reds; + } + else { + heap_size = 0; + terms = 1; + atom_extra_skip = 0; + reds = ERTS_SWORD_MAX; + } #define SKIP(sz) \ do { \ if ((sz) <= endp-ep) { \ ep += (sz); \ - } else { return -1; }; \ + } else { goto error; }; \ } while (0) #define SKIP2(sz1, sz2) \ @@ -3920,25 +3945,27 @@ decoded_size(byte *ep, byte* endp, int internal_tags) Uint sz = (sz1) + (sz2); \ if (sz1 < sz && (sz) <= endp-ep) { \ ep += (sz); \ - } else { return -1; } \ + } else { goto error; } \ } while (0) #define CHKSIZE(sz) \ do { \ - if ((sz) > endp-ep) { return -1; } \ + if ((sz) > endp-ep) { goto error; } \ } while (0) #define ADDTERMS(n) \ do { \ int before = terms; \ terms += (n); \ - if (terms < before) return -1; \ + if (terms < before) goto error; \ } while (0) - - for (terms=1; terms > 0; terms--) { - int tag; - + ASSERT(terms > 0); + do { + int tag; + /*SVERK + erts_fprintf(stderr, "SVERK tag=%d ep=%p terms=%d heap_size=%d endp=%p\n", + ep[0], ep, terms, heap_size, endp); */ CHKSIZE(1); tag = ep++[0]; switch (tag) { @@ -3959,7 +3986,7 @@ decoded_size(byte *ep, byte* endp, int internal_tags) CHKSIZE(4); n = get_int32(ep); if (n > BIG_ARITY_MAX*sizeof(ErtsDigit)) { - return -1; + goto error; } SKIP2(n,4+1); /* skip, size,sign,digits */ heap_size += 1+1+(n+sizeof(Eterm)-1)/sizeof(Eterm); /* XXX: 1 too much? */ @@ -3968,7 +3995,7 @@ decoded_size(byte *ep, byte* endp, int internal_tags) CHKSIZE(2); n = get_int16(ep); if (n > MAX_ATOM_CHARACTERS) { - return -1; + goto error; } SKIP(n+2+atom_extra_skip); atom_extra_skip = 0; @@ -3978,7 +4005,7 @@ decoded_size(byte *ep, byte* endp, int internal_tags) n = get_int16(ep); ep += 2; if (n > MAX_ATOM_SZ_LIMIT) { - return -1; + goto error; } SKIP(n+atom_extra_skip); atom_extra_skip = 0; @@ -3987,7 +4014,7 @@ decoded_size(byte *ep, byte* endp, int internal_tags) CHKSIZE(1); n = get_int8(ep); if (n > MAX_ATOM_CHARACTERS) { - return -1; + goto error; } SKIP(n+1+atom_extra_skip); atom_extra_skip = 0; @@ -3997,7 +4024,7 @@ decoded_size(byte *ep, byte* endp, int internal_tags) n = get_int8(ep); ep++; if (n > MAX_ATOM_SZ_LIMIT) { - return -1; + goto error; } SKIP(n+atom_extra_skip); atom_extra_skip = 0; @@ -4026,7 +4053,7 @@ decoded_size(byte *ep, byte* endp, int internal_tags) id_words = get_int16(ep); if (id_words > ERTS_MAX_REF_NUMBERS) - return -1; + goto error; ep += 2; atom_extra_skip = 1 + 4*id_words; @@ -4128,7 +4155,7 @@ decoded_size(byte *ep, byte* endp, int internal_tags) num_free = get_int32(ep); ep += 4; if (num_free > MAX_ARG) { - return -1; + goto error; } terms += 4 + num_free; heap_size += ERL_FUN_SIZE + num_free; @@ -4145,24 +4172,50 @@ decoded_size(byte *ep, byte* endp, int internal_tags) case BINARY_INTERNAL_REF: if (!internal_tags) { - return -1; + goto error; } SKIP(sizeof(ProcBin)); heap_size += PROC_BIN_SIZE; break; case BIT_BINARY_INTERNAL_REF: if (!internal_tags) { - return -1; + goto error; } SKIP(2+sizeof(ProcBin)); heap_size += PROC_BIN_SIZE + ERL_SUB_BIN_SIZE; break; default: - return -1; + goto error; } - } + terms--; + + if (--reds <= 0) { + if (ctx && terms > 0) { + ctx->u.sc.heap_size = heap_size; + ctx->u.sc.terms = terms; + ctx->u.sc.ep = ep; + ctx->u.sc.atom_extra_skip = atom_extra_skip; + ctx->reds = 0; + return 0; + } + reds = ERTS_SWORD_MAX; + } + }while (terms > 0); + /* 'terms' may be non-zero if it has wrapped around */ - return terms==0 ? heap_size : -1; + if (terms == 0) { + if (ctx) { + ctx->state = B2TDecodeInit; + ctx->reds = reds; + } + return heap_size; + } + +error: + if (ctx) { + ctx->state = B2TBadArg; + } + return -1; #undef SKIP #undef SKIP2 #undef CHKSIZE -- cgit v1.2.3 From f10ea68ce28e9b93ce614b5f829b1ca7f4cc753f Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Mon, 14 Oct 2013 17:46:28 +0200 Subject: trapping uncompress --- erts/emulator/beam/external.c | 146 ++++++++++++++++++++++++++++-------------- 1 file changed, 98 insertions(+), 48 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index d4ffc23536..0d2de1b199 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1127,6 +1127,8 @@ BIF_RETTYPE term_to_binary_2(BIF_ALIST_2) enum B2TState { /* order is somewhat significant */ B2TPrepare, + B2TUncompressChunk, + B2TSizeInit, B2TSize, B2TDecodeInit, @@ -1161,12 +1163,9 @@ typedef struct { } B2TDecodeContext; typedef struct { - /*Uint real_size; - Uint dest_len; - byte *dbytes; - Binary *result_bin; - Binary *destination_bin; - z_stream stream;*/ + z_stream stream; + byte* dbytes; + Uint dleft; } B2TUncompressContext; typedef struct B2TContext_t { @@ -1215,7 +1214,8 @@ static uLongf binary2term_uncomp_size(byte* data, Sint size) } static ERTS_INLINE int -binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size) +binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size, + B2TContext* ctx) { byte *bytes = data; Sint size = data_size; @@ -1229,6 +1229,8 @@ binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size) size--; if (size < 5 || *bytes != COMPRESSED) { state->extp = bytes; + if (ctx) + ctx->state = B2TSizeInit; } else { uLongf dest_len = (Uint32) get_int32(bytes+1); @@ -1236,14 +1238,33 @@ binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size) size -= 5; if (dest_len > 32*1024*1024 || (state->extp = erts_alloc_fnf(ERTS_ALC_T_EXT_TERM_DATA, dest_len)) == NULL) { + /* + * Try avoid out-of-memory crash due to corrupted 'dest_len' + * by checking the actual length of the uncompressed data. + * The only way to do that is to uncompress it. Sad but true. + */ if (dest_len != binary2term_uncomp_size(bytes, size)) { return -1; } state->extp = erts_alloc(ERTS_ALC_T_EXT_TERM_DATA, dest_len); + ctx->reds -= dest_len; } state->exttmp = 1; - if (erl_zlib_uncompress(state->extp, &dest_len, bytes, size) != Z_OK) - return -1; + if (ctx) { + if (erl_zlib_inflate_start(&ctx->u.uc.stream, bytes, size) != Z_OK) + return -1; + + ctx->u.uc.dbytes = state->extp; + ctx->u.uc.dleft = dest_len; + ctx->state = B2TUncompressChunk; + } + else { + uLongf dlen = dest_len; + if (erl_zlib_uncompress(state->extp, &dlen, bytes, size) != Z_OK + || dlen != dest_len) { + return -1; + } + } size = (Sint) dest_len; } state->extsize = size; @@ -1277,7 +1298,7 @@ erts_binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size { Sint res; - if (binary2term_prepare(state, data, data_size) < 0 || + if (binary2term_prepare(state, data, data_size, NULL) < 0 || (res=decoded_size(state->extp, state->extp + state->extsize, 0, NULL)) < 0) { if (state->exttmp) @@ -1307,6 +1328,9 @@ static void b2t_destroy_context(B2TContext* context) ERTS_ALC_T_EXT_TERM_DATA); context->aligned_alloc = NULL; binary2term_abort(&context->b2ts); + if (context->state == B2TUncompressChunk) { + erl_zlib_inflate_finish(&context->u.uc.stream); + } } static void b2t_context_destructor(Binary *context_bin) @@ -1359,7 +1383,8 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con do { switch (ctx->state) { - case B2TPrepare: + case B2TPrepare: { + Uint bin_size; bytes = erts_get_aligned_binary_bytes_extra(bin, &ctx->aligned_alloc, ERTS_ALC_T_EXT_TERM_DATA, @@ -1369,24 +1394,48 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con ctx->state = B2TBadArg; break; } - - ctx->heap_size = binary2term_prepare(&ctx->b2ts, bytes, - binary_size(bin)); - if (ctx->heap_size < 0) { - ctx->state = B2TBadArg; - break; + bin_size = binary_size(bin); + if (ctx->aligned_alloc) { + ctx->reds -= bin_size / 8; } - - ctx->reds = 0; /*SVERK*/ - ctx->u.sc.heap_size = 0; - ctx->u.sc.terms = 1; - ctx->u.sc.ep = ctx->b2ts.extp; - ctx->u.sc.atom_extra_skip = 0; - ctx->state = B2TSize; + if (binary2term_prepare(&ctx->b2ts, bytes, bin_size, ctx) < 0) { + ctx->state = B2TBadArg; + } break; + } + case B2TUncompressChunk: + { + Uint chunk = ctx->reds; + int zret; + if (chunk > ctx->u.uc.dleft) + chunk = ctx->u.uc.dleft; + + zret = erl_zlib_inflate_chunk(&ctx->u.uc.stream, + ctx->u.uc.dbytes, &chunk); + ctx->u.uc.dbytes += chunk; + ctx->u.uc.dleft -= chunk; + if (zret == Z_OK && ctx->u.uc.dleft > 0) { + ctx->reds = 0; + } + else if (erl_zlib_inflate_finish(&ctx->u.uc.stream) == Z_OK + && zret == Z_STREAM_END + && ctx->u.uc.dleft == 0) { + ctx->reds -= chunk; + ctx->state = B2TSizeInit; + } + else { + ctx->state = B2TBadArg; + } + break; + } + case B2TSizeInit: + ctx->u.sc.ep = NULL; + ctx->state = B2TSize; + /*fall through*/ case B2TSize: - ctx->heap_size = decoded_size(NULL, ctx->b2ts.extp + ctx->b2ts.extsize, + ctx->heap_size = decoded_size(ctx->b2ts.extp, + ctx->b2ts.extp + ctx->b2ts.extsize, 0, ctx); break; @@ -1455,7 +1504,9 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con b2t_context_destructor); ctx = ERTS_MAGIC_BIN_DATA(context_b); sys_memcpy(ctx, &c_buff, sizeof(B2TContext)); - + if (ctx->state >= B2TDecode && ctx->u.dc.next == &c_buff.u.dc.res) { + ctx->u.dc.next = &ctx->u.dc.res; + } if (!magic_space) { ASSERT(ctx->state < B2TDecode); magic_space = HAlloc(p, PROC_BIN_SIZE); @@ -3914,25 +3965,27 @@ encode_size_struct_int(Process *p, ErtsAtomCacheMap *acmp, Eterm obj, static Sint decoded_size(byte *ep, byte* endp, int internal_tags, B2TContext* ctx) { - int heap_size = 0; + int heap_size; int terms; - int atom_extra_skip = 0; + int atom_extra_skip; Uint n; SWord reds; if (ctx) { - heap_size = ctx->u.sc.heap_size; - terms = ctx->u.sc.terms; - ep = ctx->u.sc.ep; - atom_extra_skip = ctx->u.sc.atom_extra_skip; reds = ctx->reds; + if (ctx->u.sc.ep) { + heap_size = ctx->u.sc.heap_size; + terms = ctx->u.sc.terms; + ep = ctx->u.sc.ep; + atom_extra_skip = ctx->u.sc.atom_extra_skip; + goto init_done; + } } - else { - heap_size = 0; - terms = 1; - atom_extra_skip = 0; - reds = ERTS_SWORD_MAX; - } + heap_size = 0; + terms = 1; + atom_extra_skip = 0; +init_done: + #define SKIP(sz) \ do { \ if ((sz) <= endp-ep) { \ @@ -4189,16 +4242,13 @@ decoded_size(byte *ep, byte* endp, int internal_tags, B2TContext* ctx) } terms--; - if (--reds <= 0) { - if (ctx && terms > 0) { - ctx->u.sc.heap_size = heap_size; - ctx->u.sc.terms = terms; - ctx->u.sc.ep = ep; - ctx->u.sc.atom_extra_skip = atom_extra_skip; - ctx->reds = 0; - return 0; - } - reds = ERTS_SWORD_MAX; + if (ctx && --reds <= 0 && terms > 0) { + ctx->u.sc.heap_size = heap_size; + ctx->u.sc.terms = terms; + ctx->u.sc.ep = ep; + ctx->u.sc.atom_extra_skip = atom_extra_skip; + ctx->reds = 0; + return 0; } }while (terms > 0); -- cgit v1.2.3 From d5b6c6f0bd96108d788cdfb9be15059125b3d87f Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 16 Oct 2013 16:28:21 +0200 Subject: erts: Add erlang wrappers to binary_to_term to not expose the trapping BIF in the stacktrace when it throws badarg. --- erts/emulator/beam/external.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 0d2de1b199..e3e199b198 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1476,7 +1476,7 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con erts_set_gc_state(p, 1); } BUMP_REDS(p, (initial_reds - ctx->reds) / B2T_BYTES_PER_REDUCTION); - BIF_ERROR(p, BADARG); + BIF_ERROR(p, BADARG & ~EXF_SAVETRACE); case B2TDone: b2t_destroy_context(ctx); @@ -1522,7 +1522,7 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con BIF_TRAP1(&binary_to_term_trap_export, p, ctx->trap_bin); } -BIF_RETTYPE binary_to_term_1(BIF_ALIST_1) +BIF_RETTYPE erts_internal_binary_to_term_1(BIF_ALIST_1) { /*SVERK if (++sverk_cnt % 1000 == 0) { erts_fprintf(stderr, "Call #%u to binary_to_term_int()\n", sverk_cnt); @@ -1534,7 +1534,7 @@ BIF_RETTYPE binary_to_term_1(BIF_ALIST_1) return binary_to_term_int(BIF_P, 0, BIF_ARG_1, NULL); } -BIF_RETTYPE binary_to_term_2(BIF_ALIST_2) +BIF_RETTYPE erts_internal_binary_to_term_2(BIF_ALIST_2) { Eterm opts; Eterm opt; -- cgit v1.2.3 From abbc9c2755396d4050db8f15e02c21032a528049 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 16 Oct 2013 16:28:40 +0200 Subject: erts: Cleanup code for trapping binary_to_term --- erts/emulator/beam/external.c | 103 ++++++++++++++++++------------------------ 1 file changed, 45 insertions(+), 58 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index e3e199b198..651aac03a2 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1131,7 +1131,6 @@ enum B2TState { /* order is somewhat significant */ B2TSizeInit, B2TSize, B2TDecodeInit, - B2TDecode, B2TDecodeList, B2TDecodeTuple, @@ -1157,9 +1156,6 @@ typedef struct { Eterm* hp; Eterm* hp_end; int remaining_n; -#ifdef DEBUG - Eterm* container_end; -#endif } B2TDecodeContext; typedef struct { @@ -1352,20 +1348,28 @@ static BIF_RETTYPE binary_to_term_trap_1(BIF_ALIST_1) #define B2T_BYTES_PER_REDUCTION 128 -static unsigned sverk_rand(void) +/* Define for testing */ +/*#define EXTREME_B2T_TRAPPING 1*/ + +#ifdef EXTREME_B2T_TRAPPING +static unsigned b2t_rand(void) { static unsigned prev = 17; prev = (prev * 214013 + 2531011); return prev; } +#endif + static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* context_b) { - SWord initial_reds = 1 + sverk_rand() % 4; /*(Uint)(ERTS_BIF_REDS_LEFT(p) * B2T_BYTES_PER_REDUCTION);*/ - byte* bytes; +#ifdef EXTREME_B2T_TRAPPING + SWord initial_reds = 1 + b2t_rand() % 4; +#else + SWord initial_reds = (Uint)(ERTS_BIF_REDS_LEFT(p) * B2T_BYTES_PER_REDUCTION); +#endif B2TContext c_buff; B2TContext *ctx; - ErtsDistExternal fakedep; Eterm* magic_space = NULL; if (context_b == NULL) { @@ -1384,6 +1388,7 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con do { switch (ctx->state) { case B2TPrepare: { + byte* bytes; Uint bin_size; bytes = erts_get_aligned_binary_bytes_extra(bin, &ctx->aligned_alloc, @@ -1403,32 +1408,30 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con } break; } - case B2TUncompressChunk: - { - Uint chunk = ctx->reds; - int zret; - if (chunk > ctx->u.uc.dleft) - chunk = ctx->u.uc.dleft; - - zret = erl_zlib_inflate_chunk(&ctx->u.uc.stream, - ctx->u.uc.dbytes, &chunk); - ctx->u.uc.dbytes += chunk; - ctx->u.uc.dleft -= chunk; - if (zret == Z_OK && ctx->u.uc.dleft > 0) { - ctx->reds = 0; - } - else if (erl_zlib_inflate_finish(&ctx->u.uc.stream) == Z_OK - && zret == Z_STREAM_END - && ctx->u.uc.dleft == 0) { - ctx->reds -= chunk; - ctx->state = B2TSizeInit; - } - else { - ctx->state = B2TBadArg; - } - break; - } - + case B2TUncompressChunk: { + Uint chunk = ctx->reds; + int zret; + + if (chunk > ctx->u.uc.dleft) + chunk = ctx->u.uc.dleft; + zret = erl_zlib_inflate_chunk(&ctx->u.uc.stream, + ctx->u.uc.dbytes, &chunk); + ctx->u.uc.dbytes += chunk; + ctx->u.uc.dleft -= chunk; + if (zret == Z_OK && ctx->u.uc.dleft > 0) { + ctx->reds = 0; + } + else if (erl_zlib_inflate_finish(&ctx->u.uc.stream) == Z_OK + && zret == Z_STREAM_END + && ctx->u.uc.dleft == 0) { + ctx->reds -= chunk; + ctx->state = B2TSizeInit; + } + else { + ctx->state = B2TBadArg; + } + break; + } case B2TSizeInit: ctx->u.sc.ep = NULL; ctx->state = B2TSize; @@ -1441,11 +1444,7 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con case B2TDecodeInit: if (context_b == NULL && ctx->b2ts.extsize > ctx->reds) { - /*SVERK Can we do a better prediction that is still safe - OR is there a way to do HAlloc after result some how... - ... support for mutiple HReleases maybe? - */ - /* dec_term will probably trap, allocate space for magic bin + /* dec_term will maybe trap, allocate space for magic bin before result term to make it easy to trim with HRelease. */ magic_space = HAlloc(p, PROC_BIN_SIZE); @@ -1462,11 +1461,12 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con case B2TDecode: case B2TDecodeList: case B2TDecodeTuple: - case B2TDecodeString: + case B2TDecodeString: { + ErtsDistExternal fakedep; fakedep.flags = ctx->flags; dec_term(&fakedep, NULL, NULL, &MSO(p), NULL, ctx); break; - + } case B2TDecodeFail: HRelease(p, ctx->u.dc.hp_end, ctx->u.dc.hp_start); /*fall through*/ @@ -1495,7 +1495,7 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con return ctx->u.dc.res; } - }while (ctx->reds || ctx->state >= B2TDone); + }while (ctx->reds > 0 || ctx->state >= B2TDone); if (context_b == NULL) { ASSERT(ctx->trap_bin == THE_NON_VALUE); @@ -1524,13 +1524,6 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con BIF_RETTYPE erts_internal_binary_to_term_1(BIF_ALIST_1) { -/*SVERK if (++sverk_cnt % 1000 == 0) { - erts_fprintf(stderr, "Call #%u to binary_to_term_int()\n", sverk_cnt); - } - if (sverk_cnt == 1767) { - sverk_break(); - } -*/ return binary_to_term_int(BIF_P, 0, BIF_ARG_1, NULL); } @@ -2879,7 +2872,6 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, objp -= 2; n--; } - ASSERT(ctx->u.dc.remaining_n || next == ctx->u.dc.container_end); break; case B2TDecodeTuple: @@ -2889,7 +2881,6 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, next = objp; objp--; } - ASSERT(ctx->u.dc.remaining_n || next == ctx->u.dc.container_end); break; case B2TDecodeString: @@ -2902,7 +2893,6 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, } hp[-1] = NIL; *hpp = hp; - ASSERT(ctx->u.dc.remaining_n || hp == ctx->u.dc.container_end); break; default: @@ -3051,7 +3041,6 @@ dec_term_atom_common: objp = hp - 1; if (ctx) { if (reds < n) { - IF_DEBUG(ctx->u.dc.container_end = hp - n;) ASSERT(reds > 0); ctx->state = B2TDecodeTuple; ctx->u.dc.remaining_n = n - reds; @@ -3085,7 +3074,6 @@ dec_term_atom_common: n--; if (ctx) { if (reds < n) { - IF_DEBUG(ctx->u.dc.container_end = hp - 2*(n+1);) ctx->state = B2TDecodeList; ctx->u.dc.remaining_n = n - reds; n = reds; @@ -3110,7 +3098,6 @@ dec_term_atom_common: *objp = make_list(hp); if (ctx) { if (reds < n) { - IF_DEBUG(ctx->u.dc.container_end = hp + n*2;) ctx->state = B2TDecodeString; ctx->u.dc.remaining_n = n - reds; n = reds; @@ -3981,6 +3968,9 @@ decoded_size(byte *ep, byte* endp, int internal_tags, B2TContext* ctx) goto init_done; } } + else + reds = 0; /* not used but compiler warns anyway */ + heap_size = 0; terms = 1; atom_extra_skip = 0; @@ -4016,9 +4006,6 @@ init_done: ASSERT(terms > 0); do { int tag; - /*SVERK - erts_fprintf(stderr, "SVERK tag=%d ep=%p terms=%d heap_size=%d endp=%p\n", - ep[0], ep, terms, heap_size, endp); */ CHKSIZE(1); tag = ep++[0]; switch (tag) { -- cgit v1.2.3 From bcb227d1f467439fa5f901233e8ad382e75373cb Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Mon, 4 Nov 2013 17:53:43 +0100 Subject: erts: Trapping memcpy in binary_to_term --- erts/emulator/beam/external.c | 73 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 62 insertions(+), 11 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 651aac03a2..a73734d487 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1135,6 +1135,7 @@ enum B2TState { /* order is somewhat significant */ B2TDecodeList, B2TDecodeTuple, B2TDecodeString, + B2TDecodeBinary, B2TDone, B2TDecodeFail, @@ -1156,6 +1157,7 @@ typedef struct { Eterm* hp; Eterm* hp_end; int remaining_n; + char* remaining_bytes; } B2TDecodeContext; typedef struct { @@ -1347,6 +1349,7 @@ static BIF_RETTYPE binary_to_term_trap_1(BIF_ALIST_1) #define B2T_BYTES_PER_REDUCTION 128 +#define B2T_MEMCPY_FACTOR 8 /* Define for testing */ /*#define EXTREME_B2T_TRAPPING 1*/ @@ -1461,7 +1464,8 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con case B2TDecode: case B2TDecodeList: case B2TDecodeTuple: - case B2TDecodeString: { + case B2TDecodeString: + case B2TDecodeBinary: { ErtsDistExternal fakedep; fakedep.flags = ctx->flags; dec_term(&fakedep, NULL, NULL, &MSO(p), NULL, ctx); @@ -1494,6 +1498,8 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con BUMP_REDS(p, (initial_reds - ctx->reds) / B2T_BYTES_PER_REDUCTION); return ctx->u.dc.res; + default: + ASSERT(!"Unknown state in binary_to_term"); } }while (ctx->reds > 0 || ctx->state >= B2TDone); @@ -2830,6 +2836,7 @@ undo_offheap_in_area(ErlOffHeap* off_heap, Eterm* start, Eterm* end) #endif /* DEBUG */ } + /* Decode term from external format into *objp. ** On failure return NULL and (R13B04) *hpp will be unchanged. */ @@ -2852,15 +2859,25 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, hpp = &ctx->u.dc.hp; if (ctx->state != B2TDecode) { - n = ctx->u.dc.remaining_n; - if (reds < n) { - ctx->u.dc.remaining_n -= reds; - n = reds; + int n_limit = reds; + + n = ctx->u.dc.remaining_n; + if (ctx->state == B2TDecodeBinary) { + n_limit *= B2T_MEMCPY_FACTOR; + ASSERT(n_limit >= reds); + reds -= n / B2T_MEMCPY_FACTOR; + } + else + reds -= n; + + if (n > n_limit) { + ctx->u.dc.remaining_n -= n_limit; + n = n_limit; + reds = 0; } else { ctx->u.dc.remaining_n = 0; } - reds -= n; switch (ctx->state) { case B2TDecodeList: @@ -2895,6 +2912,12 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, *hpp = hp; break; + case B2TDecodeBinary: + sys_memcpy(ctx->u.dc.remaining_bytes, ep, n); + ctx->u.dc.remaining_bytes += n; + ep += n; + break; + default: ASSERT(!"Unknown state"); } @@ -3311,7 +3334,6 @@ dec_term_atom_common: dbin->flags = 0; dbin->orig_size = n; erts_refc_init(&dbin->refc, 1); - sys_memcpy(dbin->orig_bytes, ep, n); pb = (ProcBin *) hp; hp += PROC_BIN_SIZE; pb->thing_word = HEADER_PROC_BIN; @@ -3322,7 +3344,20 @@ dec_term_atom_common: pb->bytes = (byte*) dbin->orig_bytes; pb->flags = 0; *objp = make_binary(pb); - } + if (ctx) { + int n_limit = reds * B2T_MEMCPY_FACTOR; + if (n > n_limit) { + ctx->state = B2TDecodeBinary; + ctx->u.dc.remaining_n = n - n_limit; + ctx->u.dc.remaining_bytes = dbin->orig_bytes + n_limit; + n = n_limit; + reds = 0; + } + else + reds -= n / B2T_MEMCPY_FACTOR; + } + sys_memcpy(dbin->orig_bytes, ep, n); + } ep += n; break; } @@ -3343,13 +3378,14 @@ dec_term_atom_common: sys_memcpy(hb->data, ep, n); bin = make_binary(hb); hp += heap_bin_size(n); + ep += n; } else { Binary* dbin = erts_bin_nrml_alloc(n); ProcBin* pb; + dbin->flags = 0; dbin->orig_size = n; erts_refc_init(&dbin->refc, 1); - sys_memcpy(dbin->orig_bytes, ep, n); pb = (ProcBin *) hp; pb->thing_word = HEADER_PROC_BIN; pb->size = n; @@ -3360,8 +3396,23 @@ dec_term_atom_common: pb->flags = 0; bin = make_binary(pb); hp += PROC_BIN_SIZE; - } - ep += n; + if (ctx) { + int n_limit = reds * B2T_MEMCPY_FACTOR; + if (n > n_limit) { + ctx->state = B2TDecodeBinary; + ctx->u.dc.remaining_n = n - n_limit; + ctx->u.dc.remaining_bytes = dbin->orig_bytes + n_limit; + n = n_limit; + reds = 0; + } + else + reds -= n / B2T_MEMCPY_FACTOR; + } + sys_memcpy(dbin->orig_bytes, ep, n); + ep += n; + n = pb->size; + } + if (bitsize == 0) { *objp = bin; } else { -- cgit v1.2.3 From adcdcb2c7985ad6076a2bedebdfce08d14465b43 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 5 Nov 2013 18:44:39 +0100 Subject: erts: Fix crash when binary_to_term throws badarg after it has built off_heap data and then done at least one trap call. The undo mechanism in dec_term does not work if we build the magic binary after any other off_heap data. --- erts/emulator/beam/external.c | 51 +++++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 24 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index a73734d487..6a7d15cc88 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1364,6 +1364,21 @@ static unsigned b2t_rand(void) #endif +static B2TContext* b2t_export_context(Process* p, B2TContext* src) +{ + Binary* context_b = erts_create_magic_binary(sizeof(B2TContext), + b2t_context_destructor); + B2TContext* ctx = ERTS_MAGIC_BIN_DATA(context_b); + Eterm* hp; + sys_memcpy(ctx, src, sizeof(B2TContext)); + if (ctx->state >= B2TDecode && ctx->u.dc.next == &src->u.dc.res) { + ctx->u.dc.next = &ctx->u.dc.res; + } + hp = HAlloc(p, PROC_BIN_SIZE); + ctx->trap_bin = erts_mk_magic_binary_term(&hp, &MSO(p), context_b); + return ctx; +} + static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* context_b) { #ifdef EXTREME_B2T_TRAPPING @@ -1373,16 +1388,18 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con #endif B2TContext c_buff; B2TContext *ctx; - Eterm* magic_space = NULL; + int is_first_call; if (context_b == NULL) { /* Setup enough to get started */ + is_first_call = 1; ctx = &c_buff; ctx->state = B2TPrepare; ctx->aligned_alloc = NULL; ctx->flags = flags; IF_DEBUG(ctx->trap_bin = THE_NON_VALUE;) } else { + is_first_call = 0; ctx = ERTS_MAGIC_BIN_DATA(context_b); ASSERT(ctx->state != B2TPrepare); } @@ -1446,12 +1463,11 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con break; case B2TDecodeInit: - if (context_b == NULL && ctx->b2ts.extsize > ctx->reds) { + if (ctx == &c_buff && ctx->b2ts.extsize > ctx->reds) { /* dec_term will maybe trap, allocate space for magic bin before result term to make it easy to trim with HRelease. */ - magic_space = HAlloc(p, PROC_BIN_SIZE); - *magic_space = make_pos_bignum_header(PROC_BIN_SIZE-1); + ctx = b2t_export_context(p, &c_buff); } ctx->u.dc.ep = ctx->b2ts.extp; ctx->u.dc.res = (Eterm) (UWord) NULL; @@ -1476,7 +1492,7 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con /*fall through*/ case B2TBadArg: b2t_destroy_context(ctx); - if (context_b) { + if (!is_first_call) { erts_set_gc_state(p, 1); } BUMP_REDS(p, (initial_reds - ctx->reds) / B2T_BYTES_PER_REDUCTION); @@ -1491,10 +1507,9 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con } HRelease(p, ctx->u.dc.hp_end, ctx->u.dc.hp); - if (context_b) { + if (!is_first_call) { erts_set_gc_state(p, 1); } - BUMP_REDS(p, (initial_reds - ctx->reds) / B2T_BYTES_PER_REDUCTION); return ctx->u.dc.res; @@ -1503,27 +1518,15 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con } }while (ctx->reds > 0 || ctx->state >= B2TDone); - if (context_b == NULL) { + if (ctx == &c_buff) { ASSERT(ctx->trap_bin == THE_NON_VALUE); - - context_b = erts_create_magic_binary(sizeof(B2TContext), - b2t_context_destructor); - ctx = ERTS_MAGIC_BIN_DATA(context_b); - sys_memcpy(ctx, &c_buff, sizeof(B2TContext)); - if (ctx->state >= B2TDecode && ctx->u.dc.next == &c_buff.u.dc.res) { - ctx->u.dc.next = &ctx->u.dc.res; - } - if (!magic_space) { - ASSERT(ctx->state < B2TDecode); - magic_space = HAlloc(p, PROC_BIN_SIZE); - } - ctx->trap_bin = erts_mk_magic_binary_term(&magic_space, &MSO(p), context_b); - - erts_set_gc_state(p, 0); + ctx = b2t_export_context(p, &c_buff); } - ASSERT(context_b); ASSERT(ctx->trap_bin != THE_NON_VALUE); + if (is_first_call) { + erts_set_gc_state(p, 0); + } BUMP_ALL_REDS(p); BIF_TRAP1(&binary_to_term_trap_export, p, ctx->trap_bin); } -- cgit v1.2.3 From 53c26db17f6bf89783e0dd48f7777cfeff18f756 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 5 Nov 2013 21:59:53 +0100 Subject: erts: Fix bug in binary_to_term for compressed on halfword --- erts/emulator/beam/external.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 6a7d15cc88..28c6caf5fe 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1429,7 +1429,7 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con break; } case B2TUncompressChunk: { - Uint chunk = ctx->reds; + uLongf chunk = ctx->reds; int zret; if (chunk > ctx->u.uc.dleft) -- cgit v1.2.3 From 522a29666088d5d96956d2752ffb1596d778cffd Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 13 Nov 2013 18:07:33 +0100 Subject: erts: Let term_to_binary disable gc while trapping as an attempt to improve performance --- erts/emulator/beam/external.c | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 28c6caf5fe..d5f3b19b82 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1055,8 +1055,10 @@ static BIF_RETTYPE term_to_binary_trap_1(BIF_ALIST_1) Binary *bin = ((ProcBin *) binary_val(bt))->val; Eterm res = erts_term_to_binary_int(BIF_P, Term, 0, 0,bin); if (is_tuple(res)) { + ASSERT(BIF_P->flags & F_DISABLE_GC); BIF_TRAP1(&term_to_binary_trap_export,BIF_P,res); } else { + erts_set_gc_state(BIF_P, 1); BIF_RET(res); } } @@ -1065,8 +1067,10 @@ BIF_RETTYPE term_to_binary_1(BIF_ALIST_1) { Eterm res = erts_term_to_binary_int(BIF_P, BIF_ARG_1, 0, TERM_TO_BINARY_DFLAGS, NULL); if (is_tuple(res)) { + erts_set_gc_state(BIF_P, 0); BIF_TRAP1(&term_to_binary_trap_export,BIF_P,res); } else { + ASSERT(!(BIF_P->flags & F_DISABLE_GC)); BIF_RET(res); } } @@ -1118,8 +1122,10 @@ BIF_RETTYPE term_to_binary_2(BIF_ALIST_2) res = erts_term_to_binary_int(p, Term, level, flags, bin); if (is_tuple(res)) { + erts_set_gc_state(p, 0); BIF_TRAP1(&term_to_binary_trap_export,BIF_P,res); } else { + ASSERT(!(BIF_P->flags & F_DISABLE_GC)); BIF_RET(res); } } -- cgit v1.2.3 From 1f09936f34f5daee534bbfde4f16e5bbb434b6c4 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 13 Nov 2013 18:46:17 +0100 Subject: erts: Yield after trapping term_to_binary if gc has been ordered or if "too much" offheap binaries has been built --- erts/emulator/beam/external.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index d5f3b19b82..7dc7ba6f98 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1058,8 +1058,11 @@ static BIF_RETTYPE term_to_binary_trap_1(BIF_ALIST_1) ASSERT(BIF_P->flags & F_DISABLE_GC); BIF_TRAP1(&term_to_binary_trap_export,BIF_P,res); } else { - erts_set_gc_state(BIF_P, 1); - BIF_RET(res); + if (erts_set_gc_state(BIF_P, 1) + || MSO(BIF_P).overhead > BIN_VHEAP_SZ(BIF_P)) + ERTS_BIF_YIELD_RETURN(BIF_P, res); + else + BIF_RET(res); } } -- cgit v1.2.3 From 6cff38512b753172a7dfa2bedd60e8987156736d Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 10 Oct 2013 20:54:12 +0200 Subject: erts: Adjust term_to_binary reduction factors Made them powers of 2 for faster calculations. 500 encoded terms per reductions seemed a bit much, lowered to 32. TERM_TO_BINARY_SIZE_FACTOR was not used in practice as it was only applied to small binaries. Lowered from 500kb to 256kb compressed bytes per trap call. --- erts/emulator/beam/external.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 7dc7ba6f98..4600d691c7 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1717,12 +1717,10 @@ erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags) { /* #define EXTREME_TTB_TRAPPING 1 */ #ifndef EXTREME_TTB_TRAPPING -#define TERM_TO_BINARY_LOOP_FACTOR 500 -#define TERM_TO_BINARY_SIZE_FACTOR 500000 -#define TERM_TO_BINARY_COMPRESS_CHUNK 500000 +#define TERM_TO_BINARY_LOOP_FACTOR 32 +#define TERM_TO_BINARY_COMPRESS_CHUNK (1 << 18) #else #define TERM_TO_BINARY_LOOP_FACTOR 1 -#define TERM_TO_BINARY_SIZE_FACTOR 10 #define TERM_TO_BINARY_COMPRESS_CHUNK 10 #endif @@ -1859,7 +1857,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla /* Finish in one go */ res = erts_term_to_binary_simple(p, Term, size, level, flags); - BUMP_REDS(p, size / TERM_TO_BINARY_SIZE_FACTOR); + BUMP_REDS(p, 1); return res; } -- cgit v1.2.3 From 99fbb7cffd40562907487278ae5f88b2e76d4923 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 8 Jan 2014 19:19:33 +0100 Subject: erts: Remove overestimation of heap space in binary_to_term for 32-bit integers (INTEGER_EXT) on 64-bit architectures. --- erts/emulator/beam/external.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 2cb44a5b64..bccbedc7bd 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2013. All Rights Reserved. + * Copyright Ericsson AB 1996-2014. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -4074,7 +4074,9 @@ init_done: switch (tag) { case INTEGER_EXT: SKIP(4); +#if !defined(ARCH_64) || HALFWORD_HEAP heap_size += BIG_UINT_HEAP_SIZE; +#endif break; case SMALL_INTEGER_EXT: SKIP(1); -- cgit v1.2.3 From b49efb826e191a0ef18da7ebdb3dd9e56fe654da Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 10 Jan 2014 15:41:48 +0100 Subject: erts: Simplify term_to_binary by removing saved ESTACK from root set We disabled GC (in 522a29666088d5) during trapping and don't need to include the saved ESTACK as part of root set. --- erts/emulator/beam/external.c | 219 ++++++++++++++++++++---------------------- 1 file changed, 104 insertions(+), 115 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 2cb44a5b64..50831e848e 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2013. All Rights Reserved. + * Copyright Ericsson AB 1996-2014. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -87,7 +87,8 @@ static Export term_to_binary_trap_export; static byte* enc_term(ErtsAtomCacheMap *, Eterm, byte*, Uint32, struct erl_off_heap_header** off_heap); -static int enc_term_int(Process *p,ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, +struct TTBEncodeContext_; +static int enc_term_int(struct TTBEncodeContext_*,ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, struct erl_off_heap_header** off_heap, Sint *reds, byte **res); static Uint is_external_string(Eterm obj, int* p_is_string); static byte* enc_atom(ErtsAtomCacheMap *, Eterm, byte*, Uint32); @@ -103,7 +104,8 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla Binary *context_b); static Uint encode_size_struct2(ErtsAtomCacheMap *, Eterm, unsigned); -static int encode_size_struct_int(Process *p, ErtsAtomCacheMap *acmp, Eterm obj, +struct TTBSizeContext_; +static int encode_size_struct_int(struct TTBSizeContext_*, ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags, Sint *reds, Uint *res); static Export binary_to_term_trap_export; @@ -1086,7 +1088,6 @@ BIF_RETTYPE term_to_binary_2(BIF_ALIST_2) int level = 0; Uint flags = TERM_TO_BINARY_DFLAGS; Eterm res; - Binary *bin = NULL; while (is_list(Flags)) { Eterm arg = CAR(list_val(Flags)); @@ -1123,7 +1124,7 @@ BIF_RETTYPE term_to_binary_2(BIF_ALIST_2) goto error; } - res = erts_term_to_binary_int(p, Term, level, flags, bin); + res = erts_term_to_binary_int(p, Term, level, flags, NULL); if (is_tuple(res)) { erts_set_gc_state(p, 0); BIF_TRAP1(&term_to_binary_trap_export,BIF_P,res); @@ -1726,14 +1727,22 @@ erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags) { typedef enum { TTBSize, TTBEncode, TTBCompress } TTBState; -typedef struct { +typedef struct TTBSizeContext_ { Uint flags; int level; + Uint result; + Eterm obj; + UWord* stack; + Uint stack_sz; } TTBSizeContext; -typedef struct { +typedef struct TTBEncodeContext_ { Uint flags; int level; + byte* ep; + Eterm obj; + UWord* stack; + Uint stack_sz; Binary *result_bin; } TTBEncodeContext; @@ -1763,8 +1772,16 @@ static void ttb_context_destructor(Binary *context_bin) context->alive = 0; switch (context->state) { case TTBSize: + if (context->s.sc.stack) { + erts_free(ERTS_ALC_T_SAVED_ESTACK, context->s.sc.stack); + context->s.sc.stack = NULL; + } break; case TTBEncode: + if (context->s.ec.stack) { + erts_free(ERTS_ALC_T_SAVED_ESTACK, context->s.ec.stack); + context->s.ec.stack = NULL; + } if (context->s.ec.result_bin != NULL) { /* Set to NULL if ever made alive! */ ASSERT(erts_refc_read(&(context->s.ec.result_bin->refc),0) == 0); erts_bin_free(context->s.ec.result_bin); @@ -1829,6 +1846,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla /* Setup enough to get started */ context->state = TTBSize; context->alive = 1; + context->s.sc.stack = NULL; context->s.sc.flags = flags; context->s.sc.level = level; } else { @@ -1844,7 +1862,8 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla int level; Uint flags; /* Try for fast path */ - if (encode_size_struct_int(p, NULL, Term, context->s.sc.flags, &reds, &size) < 0) { + if (encode_size_struct_int(&context->s.sc, NULL, Term, + context->s.sc.flags, &reds, &size) < 0) { EXPORT_CONTEXT(); /* Same state */ RETURN_STATE(); @@ -1870,6 +1889,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla context->state = TTBEncode; context->s.ec.flags = flags; context->s.ec.level = level; + context->s.ec.stack = NULL; context->s.ec.result_bin = result_bin; break; } @@ -1881,7 +1901,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla Binary *result_bin; flags = context->s.ec.flags; - if (enc_term_int(p,NULL,Term, bytes+1, flags, NULL, &reds, &endp) < 0) { + if (enc_term_int(&context->s.ec, NULL,Term, bytes+1, flags, NULL, &reds, &endp) < 0) { EXPORT_CONTEXT(); RETURN_STATE(); } @@ -2289,27 +2309,6 @@ dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Ete #define ENC_PATCH_FUN_SIZE ((Eterm) 2) #define ENC_LAST_ARRAY_ELEMENT ((Eterm) 3) -/* Free extra rootset (used when trapping) */ -static void cleanup_ttb_extra_root(ErlExtraRootSet *rs) -{ - if (rs->objv != NULL) { - erts_free(ERTS_ALC_T_EXTRA_ROOT, rs->objv); - } - erts_free(ERTS_ALC_T_EXTRA_ROOT, rs); -} - -/* Same as above, but we have an extra "stack" beyond GC reach, i.e. an array of two extra roots */ -static void cleanup_ttb_extra_root_2(ErlExtraRootSet *rs) -{ - if (rs->objv != NULL) { - erts_free(ERTS_ALC_T_EXTRA_ROOT, rs->objv); - } - if (rs[1].objv != NULL) { - erts_free(ERTS_ALC_T_EXTRA_ROOT, rs[1].objv); - } - - erts_free(ERTS_ALC_T_EXTRA_ROOT, rs); -} static byte* enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, @@ -2321,39 +2320,43 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, } static int -enc_term_int(Process *p,ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, +enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, struct erl_off_heap_header** off_heap, Sint *reds, byte **res) { - DECLARE_ESTACK(s); - DECLARE_WSTACK(com); + DECLARE_WSTACK(s); Uint n; Uint i; Uint j; Uint* ptr; Eterm val; FloatDef f; - int count_reds = (p != NULL && reds != NULL); Sint r = 0; +#if HALFWORD_HEAP + UWord wobj; +#endif - if (count_reds) { - ESTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_EXTRA_ROOT); - WSTACK_CHANGE_ALLOCATOR(com, ERTS_ALC_T_EXTRA_ROOT); + + if (ctx) { + WSTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK); r = *reds; - } - if (p && p->extra_root) { /* restore saved stacks and byte pointer */ - ESTACK_RESTORE(s,p->extra_root[0].objv, p->extra_root[0].sz); - obj = ESTACK_POP(s); - WSTACK_RESTORE(com, p->extra_root[1].objv, p->extra_root[1].sz); - ep = (byte *) WSTACK_POP(com); + if (ctx->stack) { /* restore saved stacks and byte pointer */ + WSTACK_RESTORE(s, ctx->stack, ctx->stack_sz); + ep = ctx->ep; + obj = ctx->obj; + } } goto L_jump_start; outer_loop: - while (!ESTACK_ISEMPTY(s)) { - obj = ESTACK_POP(s); - switch (val = WSTACK_POP(com)) { + while (!WSTACK_ISEMPTY(s)) { +#if HALFWORD_HEAP + obj = (Eterm) (wobj = WSTACK_POP(s)); +#else + obj = WSTACK_POP(s); +#endif + switch (val = WSTACK_POP(s)) { case ENC_TERM: break; case ENC_ONE_CONS: @@ -2364,55 +2367,52 @@ enc_term_int(Process *p,ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dfla obj = CAR(cons); tl = CDR(cons); - WSTACK_PUSH(com, is_list(tl) ? ENC_ONE_CONS : ENC_TERM); - ESTACK_PUSH(s, tl); + WSTACK_PUSH(s, is_list(tl) ? ENC_ONE_CONS : ENC_TERM); + WSTACK_PUSH(s, tl); } break; case ENC_PATCH_FUN_SIZE: - /* obj will be discarded, it was NIL */ { - byte* size_p = (byte *) WSTACK_POP(com); +#if HALFWORD_HEAP + byte* size_p = (byte *) wobj; +#else + byte* size_p = (byte *) obj; +#endif put_int32(ep - size_p, size_p); } goto outer_loop; case ENC_LAST_ARRAY_ELEMENT: /* obj is the tuple */ { - Eterm* ptr = tuple_val(obj); - i = arityval(*ptr); - obj = ptr[i]; +#if HALFWORD_HEAP + Eterm* ptr = (Eterm *) wobj; +#else + Eterm* ptr = (Eterm *) obj; +#endif + obj = *ptr; } break; default: /* ENC_LAST_ARRAY_ELEMENT+1 and upwards */ { - Eterm* ptr = tuple_val(obj); - i = arityval(*ptr); - ESTACK_PUSH(s, obj); /* put back tuple and next element index */ - WSTACK_PUSH(com, val-1); - obj = ptr[i - (val - ENC_LAST_ARRAY_ELEMENT)]; /* the index is counting down */ +#if HALFWORD_HEAP + Eterm* ptr = (Eterm *) wobj; +#else + Eterm* ptr = (Eterm *) obj; +#endif + WSTACK_PUSH(s, val-1); + obj = *ptr++; + WSTACK_PUSH(s, (UWord)ptr); } break; } L_jump_start: - if (count_reds && --r == 0) { + if (ctx && --r == 0) { *reds = r; - ESTACK_PUSH(s,obj); /* push back current object, to be popped on restore */ - WSTACK_PUSH(com,((UWord) ep)); - if (p->extra_root == NULL) { - /* NB. Allocate an array of two "extra-roots", of which only the first element - is seen and handled by the GC. Index 1 holds the Wstack. */ - p->extra_root = erts_alloc(ERTS_ALC_T_EXTRA_ROOT, sizeof(ErlExtraRootSet)*2); - p->extra_root->objv = NULL; - p->extra_root->sz = 0; - p->extra_root->cleanup = cleanup_ttb_extra_root_2; - p->extra_root[1].objv = NULL; - p->extra_root[1].sz = 0; - p->extra_root[1].cleanup = NULL; /* Never used */ - } - ESTACK_SAVE(s, p->extra_root[0].objv, p->extra_root[0].sz); - WSTACK_SAVE(com, p->extra_root[1].objv, (p->extra_root[1].sz)); + ctx->obj = obj; + ctx->ep = ep; + WSTACK_SAVE(s, ctx->stack, ctx->stack_sz); return -1; } switch(tag_val_def(obj)) { @@ -2558,8 +2558,8 @@ enc_term_int(Process *p,ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dfla ep += 4; } if (i > 0) { - WSTACK_PUSH(com, ENC_LAST_ARRAY_ELEMENT+i-1); - ESTACK_PUSH(s, obj); + WSTACK_PUSH(s, ENC_LAST_ARRAY_ELEMENT+i-1); + WSTACK_PUSH(s, (UWord)ptr); } break; @@ -2703,9 +2703,8 @@ enc_term_int(Process *p,ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dfla int ei; *ep++ = NEW_FUN_EXT; - WSTACK_PUSH(com, (UWord) ep); /* Position for patching in size */ - WSTACK_PUSH(com, ENC_PATCH_FUN_SIZE); - ESTACK_PUSH(s,NIL); /* Will be thrown away */ + WSTACK_PUSH(s, ENC_PATCH_FUN_SIZE); + WSTACK_PUSH(s, (UWord) ep); /* Position for patching in size */ ep += 4; *ep = funp->arity; ep += 1; @@ -2722,8 +2721,8 @@ enc_term_int(Process *p,ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dfla fun_env: for (ei = funp->num_free-1; ei > 0; ei--) { - WSTACK_PUSH(com, ENC_TERM); - ESTACK_PUSH(s, (UWord) funp->env[ei]); + WSTACK_PUSH(s, ENC_TERM); + WSTACK_PUSH(s, (UWord) funp->env[ei]); } if (funp->num_free != 0) { obj = funp->env[0]; @@ -2766,13 +2765,12 @@ enc_term_int(Process *p,ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dfla break; } } - DESTROY_ESTACK(s); - DESTROY_WSTACK(com); - if (p && p->extra_root) { - cleanup_ttb_extra_root_2(p->extra_root); - p->extra_root = NULL; - } - if (count_reds) { + DESTROY_WSTACK(s); + if (ctx) { + if (ctx->stack) { + erts_free(ERTS_ALC_T_SAVED_ESTACK, ctx->stack); + ctx->stack = NULL; + } *reds = r; } *res = ep; @@ -3742,26 +3740,24 @@ static Uint encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dfla } static int -encode_size_struct_int(Process *p, ErtsAtomCacheMap *acmp, Eterm obj, +encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags, Sint *reds, Uint *res) { DECLARE_ESTACK(s); Uint m, i, arity; Uint result = 0; - int count_reds = (p != NULL && reds != 0); Sint r = 0; - if (count_reds) { - ESTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_EXTRA_ROOT); + if (ctx) { + ESTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK); r = *reds; - } - - if (p && p->extra_root) { /* restore saved stack */ - ESTACK_RESTORE(s,p->extra_root->objv, p->extra_root->sz + 1); - result = ESTACK_POP(s); /*Untagged, beyond p->extra_root->sz */ - obj = ESTACK_POP(s); - } + if (ctx->stack) { /* restore saved stack */ + ESTACK_RESTORE(s, ctx->stack, ctx->stack_sz); + result = ctx->result; + obj = ctx->obj; + } + } goto L_jump_start; @@ -3787,18 +3783,11 @@ encode_size_struct_int(Process *p, ErtsAtomCacheMap *acmp, Eterm obj, } L_jump_start: - if (count_reds && --r == 0) { + if (ctx && --r == 0) { *reds = r; - ESTACK_PUSH(s,obj); /* push back current object */ - ESTACK_PUSH(s,result); /* Untagged, will be out of GC reach */ - if (p->extra_root == NULL) { - p->extra_root = erts_alloc(ERTS_ALC_T_EXTRA_ROOT, sizeof(ErlExtraRootSet)); - p->extra_root->objv = NULL; - p->extra_root->sz = 0; - p->extra_root->cleanup = cleanup_ttb_extra_root; - } - ESTACK_SAVE(s, p->extra_root->objv, p->extra_root->sz); - --p->extra_root->sz; /* Hide result from GC */ + ctx->obj = obj; + ctx->result = result; + ESTACK_SAVE(s, ctx->stack, ctx->stack_sz); return -1; } switch (tag_val_def(obj)) { @@ -4001,11 +3990,11 @@ encode_size_struct_int(Process *p, ErtsAtomCacheMap *acmp, Eterm obj, } DESTROY_ESTACK(s); - if (p && p->extra_root) { - cleanup_ttb_extra_root(p->extra_root); - p->extra_root = NULL; - } - if (count_reds) { + if (ctx) { + if (ctx->stack) { + erts_free(ERTS_ALC_T_SAVED_ESTACK, ctx->stack); + ctx->stack = NULL; + } *reds = r; } *res = result; -- cgit v1.2.3 From 172ebf11dc455e22b87f742c06fa6344995d7db5 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 16 Jan 2014 11:10:34 +0100 Subject: erts: Refactor ESTACK & WSTACK to use a struct easy to "export" This is not a clean refactor. It changes the behaviour slightly of E/WSTACK_RESTORE. The allocated stack from E/WSTACK_SAVE is used as-is and not copied into default_stack. This will hopefully fix an illusive memory leak that valgrind is reporting. --- erts/emulator/beam/external.c | 42 ++++++++++++++---------------------------- 1 file changed, 14 insertions(+), 28 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 50831e848e..94acceaaf1 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1732,8 +1732,7 @@ typedef struct TTBSizeContext_ { int level; Uint result; Eterm obj; - UWord* stack; - Uint stack_sz; + ErtsEStack estack; } TTBSizeContext; typedef struct TTBEncodeContext_ { @@ -1741,8 +1740,7 @@ typedef struct TTBEncodeContext_ { int level; byte* ep; Eterm obj; - UWord* stack; - Uint stack_sz; + ErtsWStack wstack; Binary *result_bin; } TTBEncodeContext; @@ -1772,16 +1770,10 @@ static void ttb_context_destructor(Binary *context_bin) context->alive = 0; switch (context->state) { case TTBSize: - if (context->s.sc.stack) { - erts_free(ERTS_ALC_T_SAVED_ESTACK, context->s.sc.stack); - context->s.sc.stack = NULL; - } + DESTROY_SAVED_ESTACK(&context->s.sc.estack); break; case TTBEncode: - if (context->s.ec.stack) { - erts_free(ERTS_ALC_T_SAVED_ESTACK, context->s.ec.stack); - context->s.ec.stack = NULL; - } + DESTROY_SAVED_WSTACK(&context->s.ec.wstack); if (context->s.ec.result_bin != NULL) { /* Set to NULL if ever made alive! */ ASSERT(erts_refc_read(&(context->s.ec.result_bin->refc),0) == 0); erts_bin_free(context->s.ec.result_bin); @@ -1846,7 +1838,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla /* Setup enough to get started */ context->state = TTBSize; context->alive = 1; - context->s.sc.stack = NULL; + context->s.sc.estack.start = NULL; context->s.sc.flags = flags; context->s.sc.level = level; } else { @@ -1889,7 +1881,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla context->state = TTBEncode; context->s.ec.flags = flags; context->s.ec.level = level; - context->s.ec.stack = NULL; + context->s.ec.wstack.wstart = NULL; context->s.ec.result_bin = result_bin; break; } @@ -2340,8 +2332,8 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, WSTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK); r = *reds; - if (ctx->stack) { /* restore saved stacks and byte pointer */ - WSTACK_RESTORE(s, ctx->stack, ctx->stack_sz); + if (ctx->wstack.wstart) { /* restore saved stacks and byte pointer */ + WSTACK_RESTORE(s, &ctx->wstack); ep = ctx->ep; obj = ctx->obj; } @@ -2412,7 +2404,7 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, *reds = r; ctx->obj = obj; ctx->ep = ep; - WSTACK_SAVE(s, ctx->stack, ctx->stack_sz); + WSTACK_SAVE(s, &ctx->wstack); return -1; } switch(tag_val_def(obj)) { @@ -2767,10 +2759,7 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, } DESTROY_WSTACK(s); if (ctx) { - if (ctx->stack) { - erts_free(ERTS_ALC_T_SAVED_ESTACK, ctx->stack); - ctx->stack = NULL; - } + ASSERT(ctx->wstack.wstart == NULL); *reds = r; } *res = ep; @@ -3752,8 +3741,8 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, ESTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK); r = *reds; - if (ctx->stack) { /* restore saved stack */ - ESTACK_RESTORE(s, ctx->stack, ctx->stack_sz); + if (ctx->estack.start) { /* restore saved stack */ + ESTACK_RESTORE(s, &ctx->estack); result = ctx->result; obj = ctx->obj; } @@ -3787,7 +3776,7 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, *reds = r; ctx->obj = obj; ctx->result = result; - ESTACK_SAVE(s, ctx->stack, ctx->stack_sz); + ESTACK_SAVE(s, &ctx->estack); return -1; } switch (tag_val_def(obj)) { @@ -3991,10 +3980,7 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, DESTROY_ESTACK(s); if (ctx) { - if (ctx->stack) { - erts_free(ERTS_ALC_T_SAVED_ESTACK, ctx->stack); - ctx->stack = NULL; - } + ASSERT(ctx->estack.start == NULL); *reds = r; } *res = result; -- cgit v1.2.3 From eb218eb375d353bcd78ae895805d1d45f927b3a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Wed, 2 Oct 2013 16:19:22 +0200 Subject: erts: Teach term_to_binary/1 and binary_to_term/1 Map encoding --- erts/emulator/beam/external.c | 197 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 182 insertions(+), 15 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 5e7a5cab6e..a31b2731be 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -42,6 +42,7 @@ #include "erl_binary.h" #include "erl_bits.h" #include "erl_zlib.h" +#include "erl_map.h" #ifdef HIPE #include "hipe_mode_switch.h" @@ -2555,6 +2556,38 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, } break; + case MAP_DEF: + { + map_t *mp = (map_t*)map_val(obj); + Uint size = map_get_size(mp); + Eterm *mptr; + + *ep++ = MAP_EXT; + put_int32(size, ep); ep += 4; + + /* Push values first */ + if (size > 0) { + mptr = map_get_values(mp); + for (i = size-1; i >= 1; i--) { + WSTACK_PUSH(s, ENC_TERM); + WSTACK_PUSH(s, (UWord) mptr[i]); + } + + WSTACK_PUSH(s, ENC_TERM); + WSTACK_PUSH(s, (UWord) mptr[0]); + + mptr = map_get_keys(mp); + for (i = size-1; i >= 1; i--) { + WSTACK_PUSH(s, ENC_TERM); + WSTACK_PUSH(s, (UWord) mptr[i]); + } + + obj = mptr[0]; + goto L_jump_start; + } + } + break; + case FLOAT_DEF: GET_DOUBLE(obj, f); if (dflags & DFLAG_NEW_FLOATS) { @@ -2845,6 +2878,7 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, int n; ErtsAtomEncoding char_enc; register Eterm* hp; /* Please don't take the address of hp */ + Eterm *maps_head = NULL; /* for validation of maps */ Eterm* next; SWord reds; @@ -3469,6 +3503,65 @@ dec_term_atom_common: break; } break; + case MAP_EXT: + { + map_t *mp; + Uint32 size,n; + Eterm *kptr,*vptr; + Eterm keys; + + size = get_int32(ep); ep += 4; + + keys = make_tuple(hp); + *hp++ = make_arityval(size); + kptr = hp; + hp += size; + + mp = (map_t*)hp; + hp += MAP_HEADER_SIZE; + vptr = hp; + hp += size; + + /* kptr, first word for keys + * vptr, first word for values + */ + + /* + * Use thing_word to link through decoded maps. + * The list of maps is for later validation. + */ + + mp->thing_word = (Eterm) COMPRESS_POINTER(maps_head); + maps_head = (Eterm *) mp; + + mp->size = size; + mp->keys = keys; + *objp = make_map(mp); + + /* We assume the map is wellformed, meaning: + * - ascending key order + * - unique keys + */ + + objp = vptr + size - 1; + n = size; + + while (n-- > 0) { + *objp = (Eterm) COMPRESS_POINTER(next); + next = objp; + objp--; + } + + objp = kptr + size - 1; + n = size; + + while (n-- > 0) { + *objp = (Eterm) COMPRESS_POINTER(next); + next = objp; + objp--; + } + } + break; case NEW_FUN_EXT: { ErlFunThing* funp = (ErlFunThing *) hp; @@ -3678,21 +3771,7 @@ dec_term_atom_common: } default: - error: - /* UNDO: - * Must unlink all off-heap objects that may have been - * linked into the process. - */ - if (hp < *hpp) { /* Sometimes we used hp and sometimes *hpp */ - hp = *hpp; /* the largest must be the freshest */ - } - undo_offheap_in_area(off_heap, hp_saved, hp); - *hpp = hp_saved; - if (ctx) { - ctx->state = B2TDecodeFail; - ctx->reds = reds; - } - return NULL; + goto error; } if (--reds <= 0) { @@ -3710,12 +3789,53 @@ dec_term_atom_common: } } } + + /* Iterate through all the maps and check for validity + * - done here for when we know it is complete. + */ + + while (maps_head) { + Eterm *keys; + Sint arity; + + next = (Eterm *)(EXPAND_POINTER(*maps_head)); + keys = tuple_val(*(maps_head + 2)); + arity = arityval(*keys++); + + while(arity-- > 1) { + if (CMP(keys[arity-1],keys[arity]) >= 0) { + goto error; + } + } + + *maps_head = MAP_HEADER; + maps_head = next; + } + if (ctx) { ctx->state = B2TDone; ctx->reds = reds; } + *hpp = hp; return ep; + +error: + /* UNDO: + * Must unlink all off-heap objects that may have been + * linked into the process. + */ + if (hp < *hpp) { /* Sometimes we used hp and sometimes *hpp */ + hp = *hpp; /* the largest must be the freshest */ + } + undo_offheap_in_area(off_heap, hp_saved, hp); + *hpp = hp_saved; + if (ctx) { + ctx->state = B2TDecodeFail; + ctx->reds = reds; + } + + return NULL; } /* returns the number of bytes needed to encode an object @@ -3885,6 +4005,46 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, goto outer_loop; } break; + case MAP_DEF: + { + map_t *mp = (map_t*)map_val(obj); + Uint size = map_get_size(mp); + Uint i; + Eterm *ptr; + + result += 1 + 4; /* tag + 4 bytes size */ + + /* push values first */ + ptr = map_get_values(mp); + i = size; + while(i--) { + if (is_list(*ptr)) { + if ((m = is_string(*ptr)) && (m < MAX_STRING_LEN)) { + result += m + 2 + 1; + } else { + result += 5; + } + } + ESTACK_PUSH(s,*ptr); + ++ptr; + } + + ptr = map_get_keys(mp); + i = size; + while(i--) { + if (is_list(*ptr)) { + if ((m = is_string(*ptr)) && (m < MAX_STRING_LEN)) { + result += m + 2 + 1; + } else { + result += 5; + } + } + ESTACK_PUSH(s,*ptr); + ++ptr; + } + goto outer_loop; + } + break; case FLOAT_DEF: if (dflags & DFLAG_NEW_FLOATS) { result += 9; @@ -4175,6 +4335,13 @@ init_done: ADDTERMS(n); heap_size += n + 1; break; + case MAP_EXT: + CHKSIZE(4); + n = get_int32(ep); + ep += 4; + ADDTERMS(2*n); + heap_size += 3 + n + 1 + n; + break; case STRING_EXT: CHKSIZE(2); n = get_int16(ep); -- cgit v1.2.3 From c334d4841d9d600237184233518ee13f7421f91c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Fri, 10 Jan 2014 18:04:00 +0100 Subject: erts: Update maps serializing to use term order * erlang:term_to_binary/1 * erlang:binary_to_term/1 --- erts/emulator/beam/external.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index a31b2731be..e8b77b9f37 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -3803,7 +3803,7 @@ dec_term_atom_common: arity = arityval(*keys++); while(arity-- > 1) { - if (CMP(keys[arity-1],keys[arity]) >= 0) { + if (CMP_TERM(keys[arity-1],keys[arity]) >= 0) { goto error; } } -- cgit v1.2.3 From 39c35199f5118a59f337b695a934c6bfcbf0813b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Mon, 13 Jan 2014 16:23:56 +0100 Subject: erts: Let erlang:binary_to_term/1 handle unsorted Maps Maps may be encoded with keys in arbitrary order. This is fine, as long as keys are unique. --- erts/emulator/beam/external.c | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index e8b77b9f37..57251286c8 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -3790,24 +3790,41 @@ dec_term_atom_common: } } - /* Iterate through all the maps and check for validity + /* Iterate through all the maps and check for validity and sort keys * - done here for when we know it is complete. */ while (maps_head) { - Eterm *keys; - Sint arity; + map_t *mp = (map_t*)maps_head; + Eterm *ks = map_get_keys(mp); + Eterm *vs = map_get_values(mp); + Uint sz = map_get_size(mp); + Uint ix,jx; + Eterm tmp; + int c; next = (Eterm *)(EXPAND_POINTER(*maps_head)); - keys = tuple_val(*(maps_head + 2)); - arity = arityval(*keys++); - while(arity-- > 1) { - if (CMP_TERM(keys[arity-1],keys[arity]) >= 0) { - goto error; + /* sort */ + + for ( ix = 1; ix < sz; ix++) { + jx = ix; + while( jx > 0 && (c = CMP_TERM(ks[jx],ks[jx-1])) <= 0 ) { + /* identical key -> error */ + if (c == 0) goto error; + + tmp = ks[jx]; + ks[jx] = ks[jx - 1]; + ks[jx - 1] = tmp; + + tmp = vs[jx]; + vs[jx] = vs[jx - 1]; + vs[jx - 1] = tmp; + + jx--; } - } + } *maps_head = MAP_HEADER; maps_head = next; } -- cgit v1.2.3 From bc34b5dafaebf07d7185900310246f5752c0a6c4 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 21 Jan 2014 12:21:11 +0100 Subject: erts: Add map construction to driver API erl_drv_output_term() and erl_drv_send_term() can send messages containing maps with the use of the new ERL_DRV_MAP. The driver API minor version is updated as new functionality is added. --- erts/emulator/beam/external.c | 31 ++----------------------------- 1 file changed, 2 insertions(+), 29 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 57251286c8..a4cc3435c3 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -3795,37 +3795,10 @@ dec_term_atom_common: */ while (maps_head) { - map_t *mp = (map_t*)maps_head; - Eterm *ks = map_get_keys(mp); - Eterm *vs = map_get_values(mp); - Uint sz = map_get_size(mp); - Uint ix,jx; - Eterm tmp; - int c; - next = (Eterm *)(EXPAND_POINTER(*maps_head)); - - /* sort */ - - for ( ix = 1; ix < sz; ix++) { - jx = ix; - while( jx > 0 && (c = CMP_TERM(ks[jx],ks[jx-1])) <= 0 ) { - /* identical key -> error */ - if (c == 0) goto error; - - tmp = ks[jx]; - ks[jx] = ks[jx - 1]; - ks[jx - 1] = tmp; - - tmp = vs[jx]; - vs[jx] = vs[jx - 1]; - vs[jx - 1] = tmp; - - jx--; - } - - } *maps_head = MAP_HEADER; + if (!erts_validate_and_sort_map((map_t*)maps_head)) + goto error; maps_head = next; } -- cgit v1.2.3 From 9ae4a522ea841a880d94e2b1e683e470cf2c5037 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Mon, 3 Feb 2014 17:58:24 +0100 Subject: erts: Fix bug in binary_to_term for maps 'maps_head' was not restored when yielding. Risk for crash increases with size and number of maps in term. --- erts/emulator/beam/external.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index a4cc3435c3..9fb2dbd8bf 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1169,6 +1169,7 @@ typedef struct { Eterm* hp_end; int remaining_n; char* remaining_bytes; + Eterm* maps_head; } B2TDecodeContext; typedef struct { @@ -1486,6 +1487,7 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con ctx->u.dc.hp_start = HAlloc(p, ctx->heap_size); ctx->u.dc.hp = ctx->u.dc.hp_start; ctx->u.dc.hp_end = ctx->u.dc.hp_start + ctx->heap_size; + ctx->u.dc.maps_head = NULL; ctx->state = B2TDecode; /*fall through*/ case B2TDecode: @@ -2878,7 +2880,7 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, int n; ErtsAtomEncoding char_enc; register Eterm* hp; /* Please don't take the address of hp */ - Eterm *maps_head = NULL; /* for validation of maps */ + Eterm *maps_head; /* for validation of maps */ Eterm* next; SWord reds; @@ -2888,6 +2890,7 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, next = ctx->u.dc.next; ep = ctx->u.dc.ep; hpp = &ctx->u.dc.hp; + maps_head = ctx->u.dc.maps_head; if (ctx->state != B2TDecode) { int n_limit = reds; @@ -2968,6 +2971,7 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, reds = ERTS_SWORD_MAX; next = objp; *next = (Eterm) (UWord) NULL; + maps_head = NULL; } hp = *hpp; @@ -3780,6 +3784,7 @@ dec_term_atom_common: ctx->u.dc.ep = ep; ctx->u.dc.next = next; ctx->u.dc.hp = hp; + ctx->u.dc.maps_head = maps_head; ctx->reds = 0; return NULL; } -- cgit v1.2.3 From a3af5f4a5c4568225ef91ee4493da6bf659f7161 Mon Sep 17 00:00:00 2001 From: Lukas Larsson Date: Mon, 2 Sep 2013 17:27:58 +0200 Subject: erts: Set default external enc to use new float scheme This change was triggered by the OSE float printing function not working exactly the same way as linux/win32. But it is also a good one in general as it cuts size in more than half for floats. --- erts/emulator/beam/external.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 9fb2dbd8bf..b8e6b3b072 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -529,7 +529,7 @@ Uint erts_encode_ext_size(Eterm term) Uint erts_encode_ext_size_2(Eterm term, unsigned dflags) { - return encode_size_struct2(NULL, term, TERM_TO_BINARY_DFLAGS|dflags) + return encode_size_struct2(NULL, term, dflags) + 1 /* VERSION_MAGIC */; } @@ -1099,10 +1099,10 @@ BIF_RETTYPE term_to_binary_2(BIF_ALIST_2) if (tp[1] == am_minor_version && is_small(tp[2])) { switch (signed_val(tp[2])) { case 0: - flags = TERM_TO_BINARY_DFLAGS; + flags = TERM_TO_BINARY_DFLAGS & ~DFLAG_NEW_FLOATS; break; case 1: - flags = TERM_TO_BINARY_DFLAGS|DFLAG_NEW_FLOATS; + flags = TERM_TO_BINARY_DFLAGS; break; default: goto error; @@ -1605,9 +1605,9 @@ external_size_2(BIF_ALIST_2) if (tp[1] == am_minor_version && is_small(tp[2])) { switch (signed_val(tp[2])) { case 0: + flags &= ~DFLAG_NEW_FLOATS; break; case 1: - flags |= DFLAG_NEW_FLOATS; break; default: goto error; -- cgit v1.2.3 From eb53a3f0b7a7d6c4d0a877fe71bc0b0ca11d1597 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 20 Feb 2014 14:34:41 +0100 Subject: erts: Fix heap overwrite by hipe "trap frames" when GC is disabled by trapping BIFs like term_to_binary and binary_to_term. --- erts/emulator/beam/external.c | 65 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 9fb2dbd8bf..2ca52c8025 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1068,7 +1068,7 @@ static BIF_RETTYPE term_to_binary_trap_1(BIF_ALIST_1) BIF_RET(res); } } - + BIF_RETTYPE term_to_binary_1(BIF_ALIST_1) { Eterm res = erts_term_to_binary_int(BIF_P, BIF_ARG_1, 0, TERM_TO_BINARY_DFLAGS, NULL); @@ -4459,3 +4459,66 @@ error: #undef SKIP2 #undef CHKSIZE } + + +#ifdef HIPE +BIF_RETTYPE hipe_wrapper_term_to_binary_1(BIF_ALIST_1); +BIF_RETTYPE hipe_wrapper_term_to_binary_2(BIF_ALIST_2); +BIF_RETTYPE hipe_wrapper_erts_internal_binary_to_term_1(BIF_ALIST_1); +BIF_RETTYPE hipe_wrapper_erts_internal_binary_to_term_2(BIF_ALIST_2); + +/* Hipe wrappers used by native code for BIFs that disable GC while trapping. + * + * Problem: + * When native code calls a BIF that traps, hipe_mode_switch will push a + * "trap frame" on the Erlang stack in order to find its way back from beam_emu + * back to native caller when finally done. If GC is disabled and stack/heap + * is full there is no place to push the "trap frame". + * + * Solution: + * We reserve space on stack for the "trap frame" here before the BIF is called. + * If the BIF does not trap, the space is reclaimed here before returning. + * If the BIF traps, hipe_push_beam_trap_frame() will detect that a "trap frame" + * already is reserved and use it. + */ +BIF_RETTYPE hipe_wrapper_term_to_binary_1(BIF_ALIST_1) +{ + Eterm res; + hipe_reserve_beam_trap_frame(BIF_P, BIF__ARGS, 1); + res = term_to_binary_1(BIF_P, BIF__ARGS); + if (is_value(res) || BIF_P->freason != TRAP) { + hipe_unreserve_beam_trap_frame(BIF_P); + } + return res; +} +BIF_RETTYPE hipe_wrapper_term_to_binary_2(BIF_ALIST_2) +{ + Eterm res; + hipe_reserve_beam_trap_frame(BIF_P, BIF__ARGS, 2); + res = term_to_binary_2(BIF_P, BIF__ARGS); + if (is_value(res) || BIF_P->freason != TRAP) { + hipe_unreserve_beam_trap_frame(BIF_P); + } + return res; +} +BIF_RETTYPE hipe_wrapper_erts_internal_binary_to_term_1(BIF_ALIST_1) +{ + Eterm res; + hipe_reserve_beam_trap_frame(BIF_P, BIF__ARGS, 1); + res = erts_internal_binary_to_term_1(BIF_P, BIF__ARGS); + if (is_value(res) || BIF_P->freason != TRAP) { + hipe_unreserve_beam_trap_frame(BIF_P); + } + return res; +} +BIF_RETTYPE hipe_wrapper_erts_internal_binary_to_term_2(BIF_ALIST_2) +{ + Eterm res; + hipe_reserve_beam_trap_frame(BIF_P, BIF__ARGS, 2); + res = erts_internal_binary_to_term_2(BIF_P, BIF__ARGS); + if (is_value(res) || BIF_P->freason != TRAP) { + hipe_unreserve_beam_trap_frame(BIF_P); + } + return res; +} +#endif /*HIPE*/ -- cgit v1.2.3 From c543d5bff7fb23c3f44cc4817c0654117de78919 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 12 Mar 2014 20:11:10 +0100 Subject: erts: Change external format for maps to be: 116,Arity, K1,V1,K2,V2,...,Kn,Vn instead of: 116,Arity, K1,K2,...,Kn, V1,V2,....,Vn We think this will be better for future internal map structures like HAMT. Would be bad if we need to iterate twice over HAMT in term_to_binary, one for keys and one for values. --- erts/emulator/beam/external.c | 55 ++++++++++++++----------------------------- 1 file changed, 18 insertions(+), 37 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 9671cde228..656de7c49a 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -2562,29 +2562,25 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, { map_t *mp = (map_t*)map_val(obj); Uint size = map_get_size(mp); - Eterm *mptr; *ep++ = MAP_EXT; put_int32(size, ep); ep += 4; - /* Push values first */ if (size > 0) { - mptr = map_get_values(mp); + Eterm *kptr = map_get_keys(mp); + Eterm *vptr = map_get_values(mp); + for (i = size-1; i >= 1; i--) { WSTACK_PUSH(s, ENC_TERM); - WSTACK_PUSH(s, (UWord) mptr[i]); + WSTACK_PUSH(s, (UWord) vptr[i]); + WSTACK_PUSH(s, ENC_TERM); + WSTACK_PUSH(s, (UWord) kptr[i]); } WSTACK_PUSH(s, ENC_TERM); - WSTACK_PUSH(s, (UWord) mptr[0]); - - mptr = map_get_keys(mp); - for (i = size-1; i >= 1; i--) { - WSTACK_PUSH(s, ENC_TERM); - WSTACK_PUSH(s, (UWord) mptr[i]); - } + WSTACK_PUSH(s, (UWord) vptr[0]); - obj = mptr[0]; + obj = kptr[0]; goto L_jump_start; } } @@ -3518,16 +3514,16 @@ dec_term_atom_common: keys = make_tuple(hp); *hp++ = make_arityval(size); - kptr = hp; hp += size; + kptr = hp - 1; mp = (map_t*)hp; hp += MAP_HEADER_SIZE; - vptr = hp; hp += size; + vptr = hp - 1; - /* kptr, first word for keys - * vptr, first word for values + /* kptr, last word for keys + * vptr, last word for values */ /* @@ -3542,27 +3538,12 @@ dec_term_atom_common: mp->keys = keys; *objp = make_map(mp); - /* We assume the map is wellformed, meaning: - * - ascending key order - * - unique keys - */ - - objp = vptr + size - 1; - n = size; - - while (n-- > 0) { - *objp = (Eterm) COMPRESS_POINTER(next); - next = objp; - objp--; - } - - objp = kptr + size - 1; - n = size; - - while (n-- > 0) { - *objp = (Eterm) COMPRESS_POINTER(next); - next = objp; - objp--; + for (n = size; n; n--) { + *vptr = (Eterm) COMPRESS_POINTER(next); + *kptr = (Eterm) COMPRESS_POINTER(vptr); + next = kptr; + vptr--; + kptr--; } } break; -- cgit v1.2.3 From 94d877a95de7f22fc66f0737da54cc6500a7231c Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 2 May 2014 14:57:35 +0200 Subject: HiPE wrappers for BIFs disabling GC --- erts/emulator/beam/external.c | 74 +++++-------------------------------------- 1 file changed, 8 insertions(+), 66 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 656de7c49a..e376253400 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -44,9 +44,6 @@ #include "erl_zlib.h" #include "erl_map.h" -#ifdef HIPE -#include "hipe_mode_switch.h" -#endif #define in_area(ptr,start,nbytes) ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes)) #define MAX_STRING_LEN 0xffff @@ -1069,6 +1066,8 @@ static BIF_RETTYPE term_to_binary_trap_1(BIF_ALIST_1) } } +HIPE_WRAPPER_BIF_DISABLE_GC(term_to_binary, 1) + BIF_RETTYPE term_to_binary_1(BIF_ALIST_1) { Eterm res = erts_term_to_binary_int(BIF_P, BIF_ARG_1, 0, TERM_TO_BINARY_DFLAGS, NULL); @@ -1081,6 +1080,8 @@ BIF_RETTYPE term_to_binary_1(BIF_ALIST_1) } } +HIPE_WRAPPER_BIF_DISABLE_GC(term_to_binary, 2) + BIF_RETTYPE term_to_binary_2(BIF_ALIST_2) { Process* p = BIF_P; @@ -1544,11 +1545,15 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con BIF_TRAP1(&binary_to_term_trap_export, p, ctx->trap_bin); } +HIPE_WRAPPER_BIF_DISABLE_GC(erts_internal_binary_to_term, 1) + BIF_RETTYPE erts_internal_binary_to_term_1(BIF_ALIST_1) { return binary_to_term_int(BIF_P, 0, BIF_ARG_1, NULL); } +HIPE_WRAPPER_BIF_DISABLE_GC(erts_internal_binary_to_term, 2) + BIF_RETTYPE erts_internal_binary_to_term_2(BIF_ALIST_2) { Eterm opts; @@ -4440,66 +4445,3 @@ error: #undef SKIP2 #undef CHKSIZE } - - -#ifdef HIPE -BIF_RETTYPE hipe_wrapper_term_to_binary_1(BIF_ALIST_1); -BIF_RETTYPE hipe_wrapper_term_to_binary_2(BIF_ALIST_2); -BIF_RETTYPE hipe_wrapper_erts_internal_binary_to_term_1(BIF_ALIST_1); -BIF_RETTYPE hipe_wrapper_erts_internal_binary_to_term_2(BIF_ALIST_2); - -/* Hipe wrappers used by native code for BIFs that disable GC while trapping. - * - * Problem: - * When native code calls a BIF that traps, hipe_mode_switch will push a - * "trap frame" on the Erlang stack in order to find its way back from beam_emu - * back to native caller when finally done. If GC is disabled and stack/heap - * is full there is no place to push the "trap frame". - * - * Solution: - * We reserve space on stack for the "trap frame" here before the BIF is called. - * If the BIF does not trap, the space is reclaimed here before returning. - * If the BIF traps, hipe_push_beam_trap_frame() will detect that a "trap frame" - * already is reserved and use it. - */ -BIF_RETTYPE hipe_wrapper_term_to_binary_1(BIF_ALIST_1) -{ - Eterm res; - hipe_reserve_beam_trap_frame(BIF_P, BIF__ARGS, 1); - res = term_to_binary_1(BIF_P, BIF__ARGS); - if (is_value(res) || BIF_P->freason != TRAP) { - hipe_unreserve_beam_trap_frame(BIF_P); - } - return res; -} -BIF_RETTYPE hipe_wrapper_term_to_binary_2(BIF_ALIST_2) -{ - Eterm res; - hipe_reserve_beam_trap_frame(BIF_P, BIF__ARGS, 2); - res = term_to_binary_2(BIF_P, BIF__ARGS); - if (is_value(res) || BIF_P->freason != TRAP) { - hipe_unreserve_beam_trap_frame(BIF_P); - } - return res; -} -BIF_RETTYPE hipe_wrapper_erts_internal_binary_to_term_1(BIF_ALIST_1) -{ - Eterm res; - hipe_reserve_beam_trap_frame(BIF_P, BIF__ARGS, 1); - res = erts_internal_binary_to_term_1(BIF_P, BIF__ARGS); - if (is_value(res) || BIF_P->freason != TRAP) { - hipe_unreserve_beam_trap_frame(BIF_P); - } - return res; -} -BIF_RETTYPE hipe_wrapper_erts_internal_binary_to_term_2(BIF_ALIST_2) -{ - Eterm res; - hipe_reserve_beam_trap_frame(BIF_P, BIF__ARGS, 2); - res = erts_internal_binary_to_term_2(BIF_P, BIF__ARGS); - if (is_value(res) || BIF_P->freason != TRAP) { - hipe_unreserve_beam_trap_frame(BIF_P); - } - return res; -} -#endif /*HIPE*/ -- cgit v1.2.3 From 82c048745efcb5d811ec0489858b821ef39ea387 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Mon, 19 May 2014 18:09:13 +0200 Subject: Replace erlang:binary_to_term() Erlang wrappers Replace the 'erlang:binary_to_term/1' and 'erlang:binary_to_term/2' Erlang wrappers taking care of failure after yield with management of this in the hidden yield BIF. --- erts/emulator/beam/external.c | 74 +++++++++++++++++++++++++++---------------- 1 file changed, 46 insertions(+), 28 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index e376253400..8d240355b0 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -108,26 +108,17 @@ static int encode_size_struct_int(struct TTBSizeContext_*, ErtsAtomCacheMap *acm static Export binary_to_term_trap_export; static BIF_RETTYPE binary_to_term_trap_1(BIF_ALIST_1); -static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* context_b); +static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* context_b, + Export *bif, Eterm arg0, Eterm arg1); void erts_init_external(void) { -#if 1 /* In R16 */ erts_init_trap_export(&term_to_binary_trap_export, - am_erlang, am_term_to_binary_trap, 1, + am_erts_internal, am_term_to_binary_trap, 1, &term_to_binary_trap_1); erts_init_trap_export(&binary_to_term_trap_export, - am_erlang, am_binary_to_term_trap, 1, + am_erts_internal, am_binary_to_term_trap, 1, &binary_to_term_trap_1); -#else - sys_memset((void *) &term_to_binary_trap_export, 0, sizeof(Export)); - term_to_binary_trap_export.address = &term_to_binary_trap_export.code[3]; - term_to_binary_trap_export.code[0] = am_erlang; - term_to_binary_trap_export.code[1] = am_term_to_binary_trap; - term_to_binary_trap_export.code[2] = 1; - term_to_binary_trap_export.code[3] = (BeamInstr) em_apply_bif; - term_to_binary_trap_export.code[4] = (BeamInstr) &term_to_binary_trap_1; -#endif return; } @@ -1186,6 +1177,8 @@ typedef struct B2TContext_t { Uint32 flags; SWord reds; Eterm trap_bin; + Export *bif; + Eterm arg[2]; enum B2TState state; union { B2TSizeContext sc; @@ -1357,7 +1350,8 @@ static BIF_RETTYPE binary_to_term_trap_1(BIF_ALIST_1) Binary *context_bin = ((ProcBin *) binary_val(BIF_ARG_1))->val; ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(context_bin) == b2t_context_destructor); - return binary_to_term_int(BIF_P, 0, THE_NON_VALUE, context_bin); + return binary_to_term_int(BIF_P, 0, THE_NON_VALUE, context_bin, NULL, + THE_NON_VALUE, THE_NON_VALUE); } @@ -1392,8 +1386,10 @@ static B2TContext* b2t_export_context(Process* p, B2TContext* src) return ctx; } -static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* context_b) +static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* context_b, + Export *bif_init, Eterm arg0, Eterm arg1) { + BIF_RETTYPE ret_val; #ifdef EXTREME_B2T_TRAPPING SWord initial_reds = 1 + b2t_rand() % 4; #else @@ -1410,6 +1406,9 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con ctx->state = B2TPrepare; ctx->aligned_alloc = NULL; ctx->flags = flags; + ctx->bif = bif_init; + ctx->arg[0] = arg0; + ctx->arg[1] = arg1; IF_DEBUG(ctx->trap_bin = THE_NON_VALUE;) } else { is_first_call = 0; @@ -1505,12 +1504,24 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con HRelease(p, ctx->u.dc.hp_end, ctx->u.dc.hp_start); /*fall through*/ case B2TBadArg: - b2t_destroy_context(ctx); - if (!is_first_call) { - erts_set_gc_state(p, 1); - } BUMP_REDS(p, (initial_reds - ctx->reds) / B2T_BYTES_PER_REDUCTION); - BIF_ERROR(p, BADARG & ~EXF_SAVETRACE); + + ASSERT(ctx->bif == bif_export[BIF_binary_to_term_1] + || ctx->bif == bif_export[BIF_binary_to_term_2]); + + if (is_first_call) + ERTS_BIF_PREP_ERROR(ret_val, p, BADARG); + else { + erts_set_gc_state(p, 1); + if (is_non_value(ctx->arg[1])) + ERTS_BIF_PREP_ERROR_TRAPPED1(ret_val, p, BADARG, ctx->bif, + ctx->arg[0]); + else + ERTS_BIF_PREP_ERROR_TRAPPED2(ret_val, p, BADARG, ctx->bif, + ctx->arg[0], ctx->arg[1]); + } + b2t_destroy_context(ctx); + return ret_val; case B2TDone: b2t_destroy_context(ctx); @@ -1525,7 +1536,8 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con erts_set_gc_state(p, 1); } BUMP_REDS(p, (initial_reds - ctx->reds) / B2T_BYTES_PER_REDUCTION); - return ctx->u.dc.res; + ERTS_BIF_PREP_RET(ret_val, ctx->u.dc.res); + return ret_val; default: ASSERT(!"Unknown state in binary_to_term"); @@ -1542,19 +1554,24 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con erts_set_gc_state(p, 0); } BUMP_ALL_REDS(p); - BIF_TRAP1(&binary_to_term_trap_export, p, ctx->trap_bin); + + ERTS_BIF_PREP_TRAP1(ret_val, &binary_to_term_trap_export, + p, ctx->trap_bin); + + return ret_val; } -HIPE_WRAPPER_BIF_DISABLE_GC(erts_internal_binary_to_term, 1) +HIPE_WRAPPER_BIF_DISABLE_GC(binary_to_term, 1) -BIF_RETTYPE erts_internal_binary_to_term_1(BIF_ALIST_1) +BIF_RETTYPE binary_to_term_1(BIF_ALIST_1) { - return binary_to_term_int(BIF_P, 0, BIF_ARG_1, NULL); + return binary_to_term_int(BIF_P, 0, BIF_ARG_1, NULL, bif_export[BIF_binary_to_term_1], + BIF_ARG_1, THE_NON_VALUE); } -HIPE_WRAPPER_BIF_DISABLE_GC(erts_internal_binary_to_term, 2) +HIPE_WRAPPER_BIF_DISABLE_GC(binary_to_term, 2) -BIF_RETTYPE erts_internal_binary_to_term_2(BIF_ALIST_2) +BIF_RETTYPE binary_to_term_2(BIF_ALIST_2) { Eterm opts; Eterm opt; @@ -1575,7 +1592,8 @@ BIF_RETTYPE erts_internal_binary_to_term_2(BIF_ALIST_2) if (is_not_nil(opts)) goto error; - return binary_to_term_int(BIF_P, flags, BIF_ARG_1, NULL); + return binary_to_term_int(BIF_P, flags, BIF_ARG_1, NULL, bif_export[BIF_binary_to_term_2], + BIF_ARG_1, BIF_ARG_2); error: BIF_ERROR(BIF_P, BADARG); -- cgit v1.2.3 From 88b094b6439737b61c117cd6873beea4518757a8 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 1 Jul 2014 22:04:55 +0200 Subject: erts: Implement yielding for distributed send of large messages Use same mechanism as term_to_binary to yield while encoding large messages for distributed send. --- erts/emulator/beam/external.c | 97 +++++++++++++++++-------------------------- 1 file changed, 38 insertions(+), 59 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 8d240355b0..564657a13b 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -498,15 +498,37 @@ byte *erts_encode_ext_dist_header_finalize(byte *ext, ErtsAtomCache *cache, Uint return ep; } -Uint erts_encode_dist_ext_size(Eterm term, Uint32 flags, ErtsAtomCacheMap *acmp) +int erts_encode_dist_ext_size(Eterm term, Uint32 flags, ErtsAtomCacheMap *acmp, + Uint* szp) { - Uint sz = 0; + Uint sz; + if (encode_size_struct_int(NULL, acmp, term, flags, NULL, &sz)) { + return -1; + } else { #ifndef ERTS_DEBUG_USE_DIST_SEP - if (!(flags & DFLAG_DIST_HDR_ATOM_CACHE)) + if (!(flags & DFLAG_DIST_HDR_ATOM_CACHE)) #endif - sz++ /* VERSION_MAGIC */; - sz += encode_size_struct2(acmp, term, flags); - return sz; + sz++ /* VERSION_MAGIC */; + + *szp += sz; + return 0; + } +} + +int erts_encode_dist_ext_size_int(Eterm term, struct erts_dsig_send_context* ctx, Uint* szp) +{ + Uint sz; + if (encode_size_struct_int(&ctx->u.sc, ctx->acmp, term, ctx->flags, &ctx->reds, &sz)) { + return -1; + } else { +#ifndef ERTS_DEBUG_USE_DIST_SEP + if (!(ctx->flags & DFLAG_DIST_HDR_ATOM_CACHE)) +#endif + sz++ /* VERSION_MAGIC */; + + *szp += sz; + return 0; + } } Uint erts_encode_ext_size(Eterm term) @@ -527,19 +549,16 @@ Uint erts_encode_ext_size_ets(Eterm term) } -void erts_encode_dist_ext(Eterm term, byte **ext, Uint32 flags, ErtsAtomCacheMap *acmp) +int erts_encode_dist_ext(Eterm term, byte **ext, Uint32 flags, ErtsAtomCacheMap *acmp, + TTBEncodeContext* ctx, Sint* reds) { - byte *ep = *ext; -#ifndef ERTS_DEBUG_USE_DIST_SEP - if (!(flags & DFLAG_DIST_HDR_ATOM_CACHE)) -#endif - *ep++ = VERSION_MAGIC; - ep = enc_term(acmp, term, ep, flags, NULL); - if (!ep) - erl_exit(ERTS_ABORT_EXIT, - "%s:%d:erts_encode_dist_ext(): Internal data structure error\n", - __FILE__, __LINE__); - *ext = ep; + if (!ctx || !ctx->wstack.wstart) { + #ifndef ERTS_DEBUG_USE_DIST_SEP + if (!(flags & DFLAG_DIST_HDR_ATOM_CACHE)) + #endif + *(*ext)++ = VERSION_MAGIC; + } + return enc_term_int(ctx, acmp, term, *ext, flags, NULL, reds, ext); } void erts_encode_ext(Eterm term, byte **ext) @@ -1740,55 +1759,15 @@ erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags) { return erts_term_to_binary_simple(p, Term, size, level, flags); } -/* Define for testing */ -/* #define EXTREME_TTB_TRAPPING 1 */ +/* Define EXTREME_TTB_TRAPPING for testing in dist.h */ #ifndef EXTREME_TTB_TRAPPING -#define TERM_TO_BINARY_LOOP_FACTOR 32 #define TERM_TO_BINARY_COMPRESS_CHUNK (1 << 18) #else -#define TERM_TO_BINARY_LOOP_FACTOR 1 #define TERM_TO_BINARY_COMPRESS_CHUNK 10 #endif -typedef enum { TTBSize, TTBEncode, TTBCompress } TTBState; -typedef struct TTBSizeContext_ { - Uint flags; - int level; - Uint result; - Eterm obj; - ErtsEStack estack; -} TTBSizeContext; - -typedef struct TTBEncodeContext_ { - Uint flags; - int level; - byte* ep; - Eterm obj; - ErtsWStack wstack; - Binary *result_bin; -} TTBEncodeContext; - -typedef struct { - Uint real_size; - Uint dest_len; - byte *dbytes; - Binary *result_bin; - Binary *destination_bin; - z_stream stream; -} TTBCompressContext; - -typedef struct { - int alive; - TTBState state; - union { - TTBSizeContext sc; - TTBEncodeContext ec; - TTBCompressContext cc; - } s; -} TTBContext; - static void ttb_context_destructor(Binary *context_bin) { TTBContext *context = ERTS_MAGIC_BIN_DATA(context_bin); -- cgit v1.2.3 From ac2eba5ea214152d806a73a8c992911cdf3aae1d Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Mon, 18 Aug 2014 17:48:42 +0200 Subject: erts: Yield in term_to_binary while copying large binaries Applies also to distributed send. --- erts/emulator/beam/external.c | 45 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 7 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 564657a13b..c6e6a8a954 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -2304,8 +2304,8 @@ dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Ete #define ENC_TERM ((Eterm) 0) #define ENC_ONE_CONS ((Eterm) 1) #define ENC_PATCH_FUN_SIZE ((Eterm) 2) -#define ENC_LAST_ARRAY_ELEMENT ((Eterm) 3) - +#define ENC_BIN_COPY ((Eterm) 3) +#define ENC_LAST_ARRAY_ELEMENT ((Eterm) 4) static byte* enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, @@ -2341,6 +2341,9 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, WSTACK_RESTORE(s, &ctx->wstack); ep = ctx->ep; obj = ctx->obj; + if (is_non_value(obj)) { + goto outer_loop; + } } } @@ -2378,6 +2381,26 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, put_int32(ep - size_p, size_p); } goto outer_loop; + case ENC_BIN_COPY: { + Uint bits = (Uint)obj; + Uint bitoffs = WSTACK_POP(s); + byte* bytes = (byte*) WSTACK_POP(s); + byte* dst = (byte*) WSTACK_POP(s); + if (bits > r * (B2T_MEMCPY_FACTOR * 8)) { + Uint n = r * B2T_MEMCPY_FACTOR; + WSTACK_PUSH3(s, (UWord)(dst + n), (UWord)(bytes + n), bitoffs); + WSTACK_PUSH2(s, ENC_BIN_COPY, bits - 8*n); + bits = 8*n; + copy_binary_to_buffer(dst, 0, bytes, bitoffs, bits); + obj = THE_NON_VALUE; + r = 0; /* yield */ + break; + } else { + copy_binary_to_buffer(dst, 0, bytes, bitoffs, bits); + r -= bits / (B2T_MEMCPY_FACTOR * 8); + goto outer_loop; + } + } case ENC_LAST_ARRAY_ELEMENT: /* obj is the tuple */ { @@ -2405,8 +2428,8 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, L_jump_start: - if (ctx && --r == 0) { - *reds = r; + if (ctx && --r <= 0) { + *reds = 0; ctx->obj = obj; ctx->ep = ep; WSTACK_SAVE(s, &ctx->wstack); @@ -2621,6 +2644,7 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint bitoffs; Uint bitsize; byte* bytes; + byte* data_dst; ERTS_GET_BINARY_BYTES(obj, bytes, bitoffs, bitsize); if (dflags & DFLAG_INTERNAL_TAGS) { @@ -2666,7 +2690,7 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, j = binary_size(obj); put_int32(j, ep); ep += 4; - copy_binary_to_buffer(ep, 0, bytes, bitoffs, 8*j); + data_dst = ep; ep += j; } else if (dflags & DFLAG_BIT_BINARIES) { /* Bit-level binary. */ @@ -2676,7 +2700,7 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, ep += 4; *ep++ = bitsize; ep[j] = 0; /* Zero unused bits at end of binary */ - copy_binary_to_buffer(ep, 0, bytes, bitoffs, 8*j+bitsize); + data_dst = ep; ep += j + 1; } else { /* @@ -2690,11 +2714,18 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, put_int32((j+1), ep); ep += 4; ep[j] = 0; /* Zero unused bits at end of binary */ - copy_binary_to_buffer(ep, 0, bytes, bitoffs, 8*j+bitsize); + data_dst = ep; ep += j+1; *ep++ = SMALL_INTEGER_EXT; *ep++ = bitsize; } + if (ctx && j > r * B2T_MEMCPY_FACTOR) { + WSTACK_PUSH3(s, (UWord)data_dst, (UWord)bytes, bitoffs); + WSTACK_PUSH2(s, ENC_BIN_COPY, 8*j + bitsize); + } else { + copy_binary_to_buffer(data_dst, 0, bytes, bitoffs, + 8 * j + bitsize); + } } break; case EXPORT_DEF: -- cgit v1.2.3 From fb25e93d290138251d0305bd2698ad29575c661c Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Mon, 18 Aug 2014 20:17:07 +0200 Subject: erts: Remove unnecessary goto for fun encoding --- erts/emulator/beam/external.c | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index c6e6a8a954..914214c5b5 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -2754,10 +2754,9 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, case FUN_DEF: { ErlFunThing* funp = (ErlFunThing *) fun_val(obj); + int ei; if ((dflags & DFLAG_NEW_FUN_TAGS) != 0) { - int ei; - *ep++ = NEW_FUN_EXT; WSTACK_PUSH(s, ENC_PATCH_FUN_SIZE); WSTACK_PUSH(s, (UWord) ep); /* Position for patching in size */ @@ -2774,16 +2773,6 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, ep = enc_term(acmp, make_small(funp->fe->old_index), ep, dflags, off_heap); ep = enc_term(acmp, make_small(funp->fe->old_uniq), ep, dflags, off_heap); ep = enc_pid(acmp, funp->creator, ep, dflags); - - fun_env: - for (ei = funp->num_free-1; ei > 0; ei--) { - WSTACK_PUSH(s, ENC_TERM); - WSTACK_PUSH(s, (UWord) funp->env[ei]); - } - if (funp->num_free != 0) { - obj = funp->env[0]; - goto L_jump_start; - } } else { /* * Communicating with an obsolete erl_interface or @@ -2815,7 +2804,14 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, *ep++ = SMALL_TUPLE_EXT; put_int8(funp->num_free, ep); ep += 1; - goto fun_env; + } + for (ei = funp->num_free-1; ei > 0; ei--) { + WSTACK_PUSH(s, ENC_TERM); + WSTACK_PUSH(s, (UWord) funp->env[ei]); + } + if (funp->num_free != 0) { + obj = funp->env[0]; + goto L_jump_start; } } break; -- cgit v1.2.3 From c1676f09ea1176116b503de88604816a2da36ac5 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 19 Aug 2014 16:40:52 +0200 Subject: erts: Yield in term_to_binary when encoding big maps --- erts/emulator/beam/external.c | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 914214c5b5..fb2fe146f3 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -2305,7 +2305,8 @@ dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Ete #define ENC_ONE_CONS ((Eterm) 1) #define ENC_PATCH_FUN_SIZE ((Eterm) 2) #define ENC_BIN_COPY ((Eterm) 3) -#define ENC_LAST_ARRAY_ELEMENT ((Eterm) 4) +#define ENC_MAP_PAIR ((Eterm) 4) +#define ENC_LAST_ARRAY_ELEMENT ((Eterm) 5) static byte* enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, @@ -2388,8 +2389,8 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, byte* dst = (byte*) WSTACK_POP(s); if (bits > r * (B2T_MEMCPY_FACTOR * 8)) { Uint n = r * B2T_MEMCPY_FACTOR; - WSTACK_PUSH3(s, (UWord)(dst + n), (UWord)(bytes + n), bitoffs); - WSTACK_PUSH2(s, ENC_BIN_COPY, bits - 8*n); + WSTACK_PUSH5(s, (UWord)(dst + n), (UWord)(bytes + n), bitoffs, + ENC_BIN_COPY, bits - 8*n); bits = 8*n; copy_binary_to_buffer(dst, 0, bytes, bitoffs, bits); obj = THE_NON_VALUE; @@ -2401,6 +2402,19 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, goto outer_loop; } } + case ENC_MAP_PAIR: { + Uint pairs_left = obj; + Eterm *vptr = (Eterm*) WSTACK_POP(s); + Eterm *kptr = (Eterm*) WSTACK_POP(s); + + obj = *kptr; + if (--pairs_left > 0) { + WSTACK_PUSH4(s, (UWord)(kptr+1), (UWord)(vptr+1), + ENC_MAP_PAIR, pairs_left); + } + WSTACK_PUSH2(s, ENC_TERM, *vptr); + break; + } case ENC_LAST_ARRAY_ELEMENT: /* obj is the tuple */ { @@ -2595,18 +2609,8 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Eterm *kptr = map_get_keys(mp); Eterm *vptr = map_get_values(mp); - for (i = size-1; i >= 1; i--) { - WSTACK_PUSH(s, ENC_TERM); - WSTACK_PUSH(s, (UWord) vptr[i]); - WSTACK_PUSH(s, ENC_TERM); - WSTACK_PUSH(s, (UWord) kptr[i]); - } - - WSTACK_PUSH(s, ENC_TERM); - WSTACK_PUSH(s, (UWord) vptr[0]); - - obj = kptr[0]; - goto L_jump_start; + WSTACK_PUSH4(s, (UWord)kptr, (UWord)vptr, + ENC_MAP_PAIR, size); } } break; @@ -2720,8 +2724,8 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, *ep++ = bitsize; } if (ctx && j > r * B2T_MEMCPY_FACTOR) { - WSTACK_PUSH3(s, (UWord)data_dst, (UWord)bytes, bitoffs); - WSTACK_PUSH2(s, ENC_BIN_COPY, 8*j + bitsize); + WSTACK_PUSH5(s, (UWord)data_dst, (UWord)bytes, bitoffs, + ENC_BIN_COPY, 8*j + bitsize); } else { copy_binary_to_buffer(data_dst, 0, bytes, bitoffs, 8 * j + bitsize); -- cgit v1.2.3 From 5d66bc37e4583074abd00cb05f9a377f39e2081f Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 19 Aug 2014 16:45:45 +0200 Subject: erts: Optimize some repeated calls to {E,W}STACK_PUSH --- erts/emulator/beam/external.c | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index fb2fe146f3..5e1f5c56e1 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -2368,8 +2368,8 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, obj = CAR(cons); tl = CDR(cons); - WSTACK_PUSH(s, is_list(tl) ? ENC_ONE_CONS : ENC_TERM); - WSTACK_PUSH(s, tl); + WSTACK_PUSH2(s, (is_list(tl) ? ENC_ONE_CONS : ENC_TERM), + tl); } break; case ENC_PATCH_FUN_SIZE: @@ -2433,9 +2433,8 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, #else Eterm* ptr = (Eterm *) obj; #endif - WSTACK_PUSH(s, val-1); obj = *ptr++; - WSTACK_PUSH(s, (UWord)ptr); + WSTACK_PUSH2(s, val-1, (UWord)ptr); } break; } @@ -2592,8 +2591,7 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, ep += 4; } if (i > 0) { - WSTACK_PUSH(s, ENC_LAST_ARRAY_ELEMENT+i-1); - WSTACK_PUSH(s, (UWord)ptr); + WSTACK_PUSH2(s, ENC_LAST_ARRAY_ELEMENT+i-1, (UWord)ptr); } break; @@ -2762,8 +2760,8 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, if ((dflags & DFLAG_NEW_FUN_TAGS) != 0) { *ep++ = NEW_FUN_EXT; - WSTACK_PUSH(s, ENC_PATCH_FUN_SIZE); - WSTACK_PUSH(s, (UWord) ep); /* Position for patching in size */ + WSTACK_PUSH2(s, ENC_PATCH_FUN_SIZE, + (UWord) ep); /* Position for patching in size */ ep += 4; *ep = funp->arity; ep += 1; @@ -2810,8 +2808,7 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, ep += 1; } for (ei = funp->num_free-1; ei > 0; ei--) { - WSTACK_PUSH(s, ENC_TERM); - WSTACK_PUSH(s, (UWord) funp->env[ei]); + WSTACK_PUSH2(s, ENC_TERM, (UWord) funp->env[ei]); } if (funp->num_free != 0) { obj = funp->env[0]; -- cgit v1.2.3 From 05f748bfdaa4297ae45264ae9297cadb5f33966e Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 2 Sep 2014 21:16:58 +0200 Subject: erts: Fix bug in term_to_binary that reallocates binary with wrong size --- erts/emulator/beam/external.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 8d240355b0..9b9b4b2a62 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1925,6 +1925,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla } real_size = endp - bytes; result_bin = erts_bin_realloc(context->s.ec.result_bin,real_size); + result_bin->orig_size = real_size; level = context->s.ec.level; BUMP_REDS(p, (initial_reds - reds) / TERM_TO_BINARY_LOOP_FACTOR); if (level == 0 || real_size < 6) { /* We are done */ @@ -2004,6 +2005,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla erl_zlib_deflate_finish(&(context->s.cc.stream)); result_bin = erts_bin_realloc(context->s.cc.destination_bin, context->s.cc.dest_len+6); + result_bin->orig_size = context->s.cc.dest_len+6; context->s.cc.destination_bin = NULL; pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE); pb->thing_word = HEADER_PROC_BIN; -- cgit v1.2.3 From 2b53f2b1514b0d405ab92c767277483c761ad928 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 4 Sep 2014 20:10:54 +0200 Subject: erts: Refactor binary allocation interface to also initialize Binary except the reference counter 'refc', as different callers have different strategies regarding the lifetime of the binary. --- erts/emulator/beam/external.c | 10 ---------- 1 file changed, 10 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 9b9b4b2a62..196913a741 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1899,8 +1899,6 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla } result_bin = erts_bin_nrml_alloc(size); - result_bin->flags = 0; - result_bin->orig_size = size; erts_refc_init(&result_bin->refc, 0); result_bin->orig_bytes[0] = VERSION_MAGIC; /* Next state immediately, no need to export context */ @@ -1925,7 +1923,6 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla } real_size = endp - bytes; result_bin = erts_bin_realloc(context->s.ec.result_bin,real_size); - result_bin->orig_size = real_size; level = context->s.ec.level; BUMP_REDS(p, (initial_reds - reds) / TERM_TO_BINARY_LOOP_FACTOR); if (level == 0 || real_size < 6) { /* We are done */ @@ -1962,8 +1959,6 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla context->s.cc.result_bin = result_bin; result_bin = erts_bin_nrml_alloc(real_size); - result_bin->flags = 0; - result_bin->orig_size = real_size; erts_refc_init(&result_bin->refc, 0); result_bin->orig_bytes[0] = VERSION_MAGIC; @@ -2005,7 +2000,6 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla erl_zlib_deflate_finish(&(context->s.cc.stream)); result_bin = erts_bin_realloc(context->s.cc.destination_bin, context->s.cc.dest_len+6); - result_bin->orig_size = context->s.cc.dest_len+6; context->s.cc.destination_bin = NULL; pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE); pb->thing_word = HEADER_PROC_BIN; @@ -3387,8 +3381,6 @@ dec_term_atom_common: } else { Binary* dbin = erts_bin_nrml_alloc(n); ProcBin* pb; - dbin->flags = 0; - dbin->orig_size = n; erts_refc_init(&dbin->refc, 1); pb = (ProcBin *) hp; hp += PROC_BIN_SIZE; @@ -3441,8 +3433,6 @@ dec_term_atom_common: Binary* dbin = erts_bin_nrml_alloc(n); ProcBin* pb; - dbin->flags = 0; - dbin->orig_size = n; erts_refc_init(&dbin->refc, 1); pb = (ProcBin *) hp; pb->thing_word = HEADER_PROC_BIN; -- cgit v1.2.3 From e45a4a8a3797a637a23ad5d660138c021f3ba1b1 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 29 Aug 2014 16:06:20 +0200 Subject: erts: Make hipe_{un}reserve_beam_trap_frame INLINE --- erts/emulator/beam/external.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 9b9b4b2a62..c95479bc0b 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -36,7 +36,9 @@ #include "erl_process.h" #include "error.h" #include "external.h" +#define ERL_WANT_HIPE_BIF_WRAPPER__ #include "bif.h" +#undef ERL_WANT_HIPE_BIF_WRAPPER__ #include "big.h" #include "dist.h" #include "erl_binary.h" -- cgit v1.2.3 From d3f0abb4daf5c823f05c3908fdc4f6ee01035937 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Mon, 3 Nov 2014 18:04:26 +0100 Subject: erts: Add constant TERM_TO_BINARY_MEMCPY_FACTOR and do not piggyback on B2T_MEMCPY_FACTOR --- erts/emulator/beam/external.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 5e1f5c56e1..831af2be4a 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1766,7 +1766,7 @@ erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags) { #else #define TERM_TO_BINARY_COMPRESS_CHUNK 10 #endif - +#define TERM_TO_BINARY_MEMCPY_FACTOR 8 static void ttb_context_destructor(Binary *context_bin) { @@ -2387,8 +2387,8 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint bitoffs = WSTACK_POP(s); byte* bytes = (byte*) WSTACK_POP(s); byte* dst = (byte*) WSTACK_POP(s); - if (bits > r * (B2T_MEMCPY_FACTOR * 8)) { - Uint n = r * B2T_MEMCPY_FACTOR; + if (bits > r * (TERM_TO_BINARY_MEMCPY_FACTOR * 8)) { + Uint n = r * TERM_TO_BINARY_MEMCPY_FACTOR; WSTACK_PUSH5(s, (UWord)(dst + n), (UWord)(bytes + n), bitoffs, ENC_BIN_COPY, bits - 8*n); bits = 8*n; @@ -2398,7 +2398,7 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, break; } else { copy_binary_to_buffer(dst, 0, bytes, bitoffs, bits); - r -= bits / (B2T_MEMCPY_FACTOR * 8); + r -= bits / (TERM_TO_BINARY_MEMCPY_FACTOR * 8); goto outer_loop; } } @@ -2721,7 +2721,7 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, *ep++ = SMALL_INTEGER_EXT; *ep++ = bitsize; } - if (ctx && j > r * B2T_MEMCPY_FACTOR) { + if (ctx && j > r * TERM_TO_BINARY_MEMCPY_FACTOR) { WSTACK_PUSH5(s, (UWord)data_dst, (UWord)bytes, bitoffs, ENC_BIN_COPY, 8*j + bitsize); } else { -- cgit v1.2.3 From 7e147a05683c709128b6777d0c360fcde067f567 Mon Sep 17 00:00:00 2001 From: Mikael Pettersson Date: Wed, 4 Feb 2015 20:27:37 +0100 Subject: don't create oversize bignums in binary matching Bignums are artifically restricted in size. Arithmetic and logical operations check the sizes of resulting bignums, and turn oversize results into system_limit exceptions. However, this check is not performed when bignums are constructed by binary matching. The consequence is that such matchings can construct oversize bignums that satisfy is_integer/1 yet don't work. Performing arithmetic such as Term - 0 fails with a system_limit exception. Worse, performing a logical operation such as Term band Term results in []. The latter occurs because the size checking (e.g. in erts_band()) is a simple ASSERT(is_not_nil(...)) on the result of the bignum operation, which internally is [] (NIL) in the case of oversize results. However, ASSERT is a no-op in release builds, so the error goes unnoticed and [] is returned as the result of the band/2. This patch addresses this by preventing oversize bignums from entering the VM via binary matching: - the internal bytes_to_big() procedure is augmented to return NIL for oversize results, just like big_norm() - callers of bytes_to_big() are augmented to check for NIL returns and signal errors in those cases - erts_bs_get_integer_2() can only fail with badmatch, so that is the Erlang-level result of oversize bignums from binary matches - big_SUITE.erl is extended with a test case that fails without this fix (no error signalled) and passes with it (badmatch occurs) Credit goes to Nico Kruber for the initial bug report. --- erts/emulator/beam/external.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 9b9b4b2a62..45d1f7514e 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -3056,6 +3056,8 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, big = make_small(0); } else { big = bytes_to_big(first, n, neg, hp); + if (is_nil(big)) + goto error; if (is_big(big)) { hp += big_arity(big) + 1; } -- cgit v1.2.3 From 10a1f3a2d5d9343bd4d2b8ec77c32ec18da21666 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 12 Mar 2015 13:42:15 +0100 Subject: erts: Add missing binary offheap overhead in binary_to_term Binary offheap overhead is used to trigger GC when a process is referring "too much" binary offheap data. Offheap binaries created from external format (binary_to_term, distributed messages or compacted ets tables) were not accounted for. Example: A process receiving a lot of binary data in distributed messages, while not building much terms on its heap, could cause an extensive memory consumption for garbage binaries. --- erts/emulator/beam/external.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 9b9b4b2a62..ae011a34bd 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -3396,6 +3396,7 @@ dec_term_atom_common: pb->size = n; pb->next = off_heap->first; off_heap->first = (struct erl_off_heap_header*)pb; + OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm)); pb->val = dbin; pb->bytes = (byte*) dbin->orig_bytes; pb->flags = 0; @@ -3449,6 +3450,7 @@ dec_term_atom_common: pb->size = n; pb->next = off_heap->first; off_heap->first = (struct erl_off_heap_header*)pb; + OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm)); pb->val = dbin; pb->bytes = (byte*) dbin->orig_bytes; pb->flags = 0; @@ -3747,6 +3749,7 @@ dec_term_atom_common: hp += PROC_BIN_SIZE; pb->next = off_heap->first; off_heap->first = (struct erl_off_heap_header*)pb; + OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm)); pb->flags = 0; *objp = make_binary(pb); break; @@ -3764,6 +3767,7 @@ dec_term_atom_common: hp += PROC_BIN_SIZE; pb->next = off_heap->first; off_heap->first = (struct erl_off_heap_header*)pb; + OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm)); pb->flags = 0; sub = (ErlSubBin*)hp; -- cgit v1.2.3 From e5d514d5adb230bbc832db5c6cf344a69a8df3bf Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 23 Jan 2015 17:50:57 +0100 Subject: erts: Add term_to_binary for hashmap --- erts/emulator/beam/external.c | 80 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 79 insertions(+), 1 deletion(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index e5fb2d3ec1..af8db4c265 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -2304,7 +2304,8 @@ dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Ete #define ENC_PATCH_FUN_SIZE ((Eterm) 2) #define ENC_BIN_COPY ((Eterm) 3) #define ENC_MAP_PAIR ((Eterm) 4) -#define ENC_LAST_ARRAY_ELEMENT ((Eterm) 5) +#define ENC_HASHMAP_NODE ((Eterm) 5) +#define ENC_LAST_ARRAY_ELEMENT ((Eterm) 6) static byte* enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, @@ -2413,6 +2414,13 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, WSTACK_PUSH2(s, ENC_TERM, *vptr); break; } + case ENC_HASHMAP_NODE: + if (is_list(obj)) { /* leaf node [K|V] */ + ptr = list_val(obj); + WSTACK_PUSH2(s, ENC_TERM, CDR(ptr)); + obj = CAR(ptr); + } + break; case ENC_LAST_ARRAY_ELEMENT: /* obj is the tuple */ { @@ -2611,6 +2619,44 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, } break; + case HASHMAP_DEF: + { + Eterm hdr; + Uint node_sz; + ptr = boxed_val(obj); + hdr = *ptr; + ASSERT(is_header(hdr)); + switch(hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: + *ep++ = MAP_EXT; + ptr++; + put_int32(*ptr, ep); ep += 4; + /*fall through*/ + case HAMT_SUBTAG_NODE_ARRAY: + node_sz = 16; + break; + case HAMT_SUBTAG_HEAD_BITMAP: + *ep++ = MAP_EXT; + ptr++; + put_int32(*ptr, ep); ep += 4; + /*fall through*/ + case HAMT_SUBTAG_NODE_BITMAP: + node_sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + ASSERT(node_sz < 17); + break; + default: + erl_exit(1, "bad header\r\n"); + } + + ptr++; + WSTACK_RESERVE(s, node_sz*2); + while(node_sz--) { + WSTACK_FAST_PUSH(s, ENC_HASHMAP_NODE); + WSTACK_FAST_PUSH(s, *ptr++); + } + } + break; + case FLOAT_DEF: GET_DOUBLE(obj, f); if (dflags & DFLAG_NEW_FLOATS) { @@ -4047,6 +4093,38 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, goto outer_loop; } break; + + case HASHMAP_DEF: + { + Eterm *ptr; + Eterm hdr; + Uint node_sz; + ptr = boxed_val(obj); + hdr = *ptr; + ASSERT(is_header(hdr)); + switch(hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: ptr++; + case HAMT_SUBTAG_NODE_ARRAY: + node_sz = 16; + break; + case HAMT_SUBTAG_HEAD_BITMAP: ptr++; + case HAMT_SUBTAG_NODE_BITMAP: + node_sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + ASSERT(node_sz < 17); + break; + default: + erl_exit(1, "bad header\r\n"); + } + + ptr++; + ESTACK_RESERVE(s, node_sz); + while(node_sz--) { + ESTACK_FAST_PUSH(s, *ptr++); + } + result += 1 + 4; /* tag + 4 bytes size */ + } + + break; case FLOAT_DEF: if (dflags & DFLAG_NEW_FLOATS) { result += 9; -- cgit v1.2.3 From 7a12c43da25e3dcad54212f538ebae3dc13f5c2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Tue, 17 Feb 2015 14:35:14 +0100 Subject: erts: Refactor erl_hashmap header includes --- erts/emulator/beam/external.c | 1 + 1 file changed, 1 insertion(+) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index af8db4c265..9030b528a4 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -45,6 +45,7 @@ #include "erl_bits.h" #include "erl_zlib.h" #include "erl_map.h" +#include "erl_hashmap.h" #define in_area(ptr,start,nbytes) ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes)) -- cgit v1.2.3 From efb521c69baccb8ab905595c222abf353c5c3283 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Mon, 23 Feb 2015 17:47:26 +0100 Subject: erts: Remove erl_hashmap.[ch] files --- erts/emulator/beam/external.c | 1 - 1 file changed, 1 deletion(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 9030b528a4..af8db4c265 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -45,7 +45,6 @@ #include "erl_bits.h" #include "erl_zlib.h" #include "erl_map.h" -#include "erl_hashmap.h" #define in_area(ptr,start,nbytes) ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes)) -- cgit v1.2.3 From f8dbf0c0ff3bd68d720faca230356d281e4e3e42 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 3 Mar 2015 19:32:00 +0100 Subject: First stab at binary_to_term for hamt with over estimation of heap size. --- erts/emulator/beam/external.c | 163 ++++++++++++++++++++++++++++++------------ 1 file changed, 116 insertions(+), 47 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index af8db4c265..0e012e9eec 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1182,7 +1182,8 @@ typedef struct { Eterm* hp_end; int remaining_n; char* remaining_bytes; - Eterm* maps_head; + Eterm* maps_list; + struct dec_term_hamt_placeholder* hamt_list; } B2TDecodeContext; typedef struct { @@ -1508,7 +1509,8 @@ static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binar ctx->u.dc.hp_start = HAlloc(p, ctx->heap_size); ctx->u.dc.hp = ctx->u.dc.hp_start; ctx->u.dc.hp_end = ctx->u.dc.hp_start + ctx->heap_size; - ctx->u.dc.maps_head = NULL; + ctx->u.dc.maps_list = NULL; + ctx->u.dc.hamt_list = NULL; ctx->state = B2TDecode; /*fall through*/ case B2TDecode: @@ -2938,9 +2940,21 @@ undo_offheap_in_area(ErlOffHeap* off_heap, Eterm* start, Eterm* end) #endif /* DEBUG */ } +struct dec_term_hamt_placeholder +{ + struct dec_term_hamt_placeholder* next; + Eterm* objp; /* write result here */ + Uint size; /* nr of leafs */ + Eterm* leaf_array; + + Eterm _heap_capacity_[1]; +}; + +#define DEC_TERM_HAMT_PLACEHOLDER_SIZE(SZ) \ + (offsetof(struct dec_term_hamt_placeholder, _heap_capacity_) + (SZ)) /* Decode term from external format into *objp. -** On failure return NULL and (R13B04) *hpp will be unchanged. +** On failure return NULL and *hpp will be unchanged. */ static byte* dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, @@ -2950,7 +2964,8 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, int n; ErtsAtomEncoding char_enc; register Eterm* hp; /* Please don't take the address of hp */ - Eterm *maps_head; /* for validation of maps */ + Eterm *maps_list; /* for preprocessing of small maps */ + struct dec_term_hamt_placeholder* hamt_list; /* for preprocessing of big maps */ Eterm* next; SWord reds; @@ -2960,7 +2975,8 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, next = ctx->u.dc.next; ep = ctx->u.dc.ep; hpp = &ctx->u.dc.hp; - maps_head = ctx->u.dc.maps_head; + maps_list = ctx->u.dc.maps_list; + hamt_list = ctx->u.dc.hamt_list; if (ctx->state != B2TDecode) { int n_limit = reds; @@ -3041,7 +3057,8 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, reds = ERTS_SWORD_MAX; next = objp; *next = (Eterm) (UWord) NULL; - maps_head = NULL; + maps_list = NULL; + hamt_list = NULL; } hp = *hpp; @@ -3577,46 +3594,68 @@ dec_term_atom_common: break; case MAP_EXT: { - map_t *mp; Uint32 size,n; Eterm *kptr,*vptr; Eterm keys; size = get_int32(ep); ep += 4; - keys = make_tuple(hp); - *hp++ = make_arityval(size); - hp += size; - kptr = hp - 1; - - mp = (map_t*)hp; - hp += MAP_HEADER_SIZE; - hp += size; - vptr = hp - 1; - - /* kptr, last word for keys - * vptr, last word for values - */ - - /* - * Use thing_word to link through decoded maps. - * The list of maps is for later validation. - */ - - mp->thing_word = (Eterm) COMPRESS_POINTER(maps_head); - maps_head = (Eterm *) mp; - - mp->size = size; - mp->keys = keys; - *objp = make_map(mp); - - for (n = size; n; n--) { - *vptr = (Eterm) COMPRESS_POINTER(next); - *kptr = (Eterm) COMPRESS_POINTER(vptr); - next = kptr; - vptr--; - kptr--; - } + if (size <= MAP_SMALL_MAP_LIMIT) { + map_t *mp; + + keys = make_tuple(hp); + *hp++ = make_arityval(size); + hp += size; + kptr = hp - 1; + + mp = (map_t*)hp; + hp += MAP_HEADER_SIZE; + hp += size; + vptr = hp - 1; + + /* kptr, last word for keys + * vptr, last word for values + */ + + /* + * Use thing_word to link through decoded maps. + * The list of maps is for later validation. + */ + + mp->thing_word = (Eterm) COMPRESS_POINTER(maps_list); + maps_list = (Eterm *) mp; + + mp->size = size; + mp->keys = keys; + *objp = make_map(mp); + + for (n = size; n; n--) { + *vptr = (Eterm) COMPRESS_POINTER(next); + *kptr = (Eterm) COMPRESS_POINTER(vptr); + next = kptr; + vptr--; + kptr--; + } + } + else { /* Make hamt */ + struct dec_term_hamt_placeholder* holder = + (struct dec_term_hamt_placeholder*) hp; + + holder->next = hamt_list; + hamt_list = holder; + holder->objp = objp; + holder->size = size; + + hp += DEC_TERM_HAMT_PLACEHOLDER_SIZE(size); + holder->leaf_array = hp; + + for (n = size; n; n--) { + CDR(hp) = (Eterm) COMPRESS_POINTER(next); + CAR(hp) = (Eterm) COMPRESS_POINTER(&CDR(hp)); + next = &CAR(hp); + hp += 2; + } + } } break; case NEW_FUN_EXT: @@ -3837,7 +3876,7 @@ dec_term_atom_common: ctx->u.dc.ep = ep; ctx->u.dc.next = next; ctx->u.dc.hp = hp; - ctx->u.dc.maps_head = maps_head; + ctx->u.dc.maps_list = maps_list; ctx->reds = 0; return NULL; } @@ -3852,12 +3891,38 @@ dec_term_atom_common: * - done here for when we know it is complete. */ - while (maps_head) { - next = (Eterm *)(EXPAND_POINTER(*maps_head)); - *maps_head = MAP_HEADER; - if (!erts_validate_and_sort_map((map_t*)maps_head)) + while (maps_list) { + next = (Eterm *)(EXPAND_POINTER(*maps_list)); + *maps_list = MAP_HEADER; + if (!erts_validate_and_sort_map((map_t*)maps_list)) goto error; - maps_head = next; + maps_list = next; + } + + while (hamt_list) { + ErtsHeapFactory factory; + int residue; + + factory.p = NULL; + factory.hp = (Eterm*) hamt_list; + factory.hp_end = hamt_list->leaf_array; + + next = (Eterm *) hamt_list->next; + objp = hamt_list->objp; + + /*SVERK Make it reject duplicate keys */ + *objp = erts_hashmap_from_array(&factory, + hamt_list->leaf_array, + hamt_list->size); + if (is_non_value(*objp)) + goto error; + + residue = factory.hp_end - factory.hp; + if (residue) { + ASSERT(residue > 0); + *factory.hp = make_pos_bignum_header(residue-1); + } + hamt_list = (struct dec_term_hamt_placeholder*) next; } if (ctx) { @@ -4420,7 +4485,11 @@ init_done: n = get_int32(ep); ep += 4; ADDTERMS(2*n); - heap_size += 3 + n + 1 + n; + if (n <= MAP_SMALL_MAP_LIMIT) { + heap_size += 3 + n + 1 + n; + } else { + heap_size += DEC_TERM_HAMT_PLACEHOLDER_SIZE(n) + 2*n; + } break; case STRING_EXT: CHKSIZE(2); -- cgit v1.2.3 From 01e843722aa39b3411d349c6fbea80ad87a6e9ef Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 5 Mar 2015 18:48:14 +0100 Subject: erts: Reject duplicate keys for hamt in binary_to_term --- erts/emulator/beam/external.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 0e012e9eec..9bcd9a7a50 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -3910,10 +3910,10 @@ dec_term_atom_common: next = (Eterm *) hamt_list->next; objp = hamt_list->objp; - /*SVERK Make it reject duplicate keys */ *objp = erts_hashmap_from_array(&factory, hamt_list->leaf_array, - hamt_list->size); + hamt_list->size, + 1); if (is_non_value(*objp)) goto error; -- cgit v1.2.3 From 7478569d0a7d619d600816f3a75e56922d8ed428 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 10 Mar 2015 18:07:11 +0100 Subject: erts: Tweak over estimation of hashmap size for binary_to_term Strategy: Calculate an over estimation of heap size that will give such a low probability for overflow, that "it will not happen". Scary assumption 1: Uniformly distributed hash values. Scary assumption 2: Tree size is normally distributed (right?) --- erts/emulator/beam/external.c | 60 ++++++++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 29 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 9bcd9a7a50..bd9ad65086 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -2945,13 +2945,11 @@ struct dec_term_hamt_placeholder struct dec_term_hamt_placeholder* next; Eterm* objp; /* write result here */ Uint size; /* nr of leafs */ - Eterm* leaf_array; - - Eterm _heap_capacity_[1]; + Eterm leafs[1]; }; -#define DEC_TERM_HAMT_PLACEHOLDER_SIZE(SZ) \ - (offsetof(struct dec_term_hamt_placeholder, _heap_capacity_) + (SZ)) +#define DEC_TERM_HAMT_PLACEHOLDER_SIZE \ + (offsetof(struct dec_term_hamt_placeholder, leafs) / sizeof(Eterm)) /* Decode term from external format into *objp. ** On failure return NULL and *hpp will be unchanged. @@ -3646,8 +3644,7 @@ dec_term_atom_common: holder->objp = objp; holder->size = size; - hp += DEC_TERM_HAMT_PLACEHOLDER_SIZE(size); - holder->leaf_array = hp; + hp += DEC_TERM_HAMT_PLACEHOLDER_SIZE; for (n = size; n; n--) { CDR(hp) = (Eterm) COMPRESS_POINTER(next); @@ -3899,30 +3896,33 @@ dec_term_atom_common: maps_list = next; } - while (hamt_list) { - ErtsHeapFactory factory; - int residue; - - factory.p = NULL; - factory.hp = (Eterm*) hamt_list; - factory.hp_end = hamt_list->leaf_array; + /* Iterate through all the hamts and build tree nodes. + */ + if (hamt_list) { + struct dec_term_hamt_placeholder* hamt = hamt_list; + ErtsHeapFactory factory; + + factory.p = NULL; + factory.hp = hp; + /* We assume heap will suffice (see hashmap_over_estimated_heap_size) */ + factory.hp_end = hp + (ERTS_SWORD_MAX / sizeof(Eterm)); + + do { + *hamt->objp = erts_hashmap_from_array(&factory, + hamt->leafs, + hamt->size, + 1); + if (is_non_value(*hamt->objp)) + goto error; - next = (Eterm *) hamt_list->next; - objp = hamt_list->objp; + hamt_list = hamt->next; - *objp = erts_hashmap_from_array(&factory, - hamt_list->leaf_array, - hamt_list->size, - 1); - if (is_non_value(*objp)) - goto error; + /* Yes, we waste a couple of heap words per hamt + for the temporary placeholder */ + *(Eterm*)hamt = make_pos_bignum_header(DEC_TERM_HAMT_PLACEHOLDER_SIZE-1); + } while (hamt_list); - residue = factory.hp_end - factory.hp; - if (residue) { - ASSERT(residue > 0); - *factory.hp = make_pos_bignum_header(residue-1); - } - hamt_list = (struct dec_term_hamt_placeholder*) next; + hp = factory.hp; } if (ctx) { @@ -4292,6 +4292,8 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, return 0; } + + static Sint decoded_size(byte *ep, byte* endp, int internal_tags, B2TContext* ctx) { @@ -4488,7 +4490,7 @@ init_done: if (n <= MAP_SMALL_MAP_LIMIT) { heap_size += 3 + n + 1 + n; } else { - heap_size += DEC_TERM_HAMT_PLACEHOLDER_SIZE(n) + 2*n; + heap_size += hashmap_over_estimated_heap_size(n); } break; case STRING_EXT: -- cgit v1.2.3 From 27e57aa05354b743b735a41716c0e3af18f2843e Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 10 Mar 2015 19:03:19 +0100 Subject: erts: Refactor maps naming convention flatmap: Small map hashmap: Large map map: flatmap or hashmap --- erts/emulator/beam/external.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index bd9ad65086..65b4ae5412 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -2605,15 +2605,15 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, case MAP_DEF: { - map_t *mp = (map_t*)map_val(obj); - Uint size = map_get_size(mp); + flatmap_t *mp = (flatmap_t*)flatmap_val(obj); + Uint size = flatmap_get_size(mp); *ep++ = MAP_EXT; put_int32(size, ep); ep += 4; if (size > 0) { - Eterm *kptr = map_get_keys(mp); - Eterm *vptr = map_get_values(mp); + Eterm *kptr = flatmap_get_keys(mp); + Eterm *vptr = flatmap_get_values(mp); WSTACK_PUSH4(s, (UWord)kptr, (UWord)vptr, ENC_MAP_PAIR, size); @@ -3599,14 +3599,14 @@ dec_term_atom_common: size = get_int32(ep); ep += 4; if (size <= MAP_SMALL_MAP_LIMIT) { - map_t *mp; + flatmap_t *mp; keys = make_tuple(hp); *hp++ = make_arityval(size); hp += size; kptr = hp - 1; - mp = (map_t*)hp; + mp = (flatmap_t*)hp; hp += MAP_HEADER_SIZE; hp += size; vptr = hp - 1; @@ -3625,7 +3625,7 @@ dec_term_atom_common: mp->size = size; mp->keys = keys; - *objp = make_map(mp); + *objp = make_flatmap(mp); for (n = size; n; n--) { *vptr = (Eterm) COMPRESS_POINTER(next); @@ -3891,7 +3891,7 @@ dec_term_atom_common: while (maps_list) { next = (Eterm *)(EXPAND_POINTER(*maps_list)); *maps_list = MAP_HEADER; - if (!erts_validate_and_sort_map((map_t*)maps_list)) + if (!erts_validate_and_sort_flatmap((flatmap_t*)maps_list)) goto error; maps_list = next; } @@ -4120,15 +4120,15 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, break; case MAP_DEF: { - map_t *mp = (map_t*)map_val(obj); - Uint size = map_get_size(mp); + flatmap_t *mp = (flatmap_t*)flatmap_val(obj); + Uint size = flatmap_get_size(mp); Uint i; Eterm *ptr; result += 1 + 4; /* tag + 4 bytes size */ /* push values first */ - ptr = map_get_values(mp); + ptr = flatmap_get_values(mp); i = size; while(i--) { if (is_list(*ptr)) { @@ -4142,7 +4142,7 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, ++ptr; } - ptr = map_get_keys(mp); + ptr = flatmap_get_keys(mp); i = size; while(i--) { if (is_list(*ptr)) { -- cgit v1.2.3 From 12fc63bcaf68b4a9e89ce91e1235aafb8bcdaee5 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 13 Mar 2015 18:00:33 +0100 Subject: erts: Fix map bug in dec_term for 32-bit debug VM Adding ERTS_SWORD_MAX to a pointer does not work as a way to disable a bound check. Remove the hp_end from ErtsHeapFactory as it isn't really used anyway. --- erts/emulator/beam/external.c | 1 - 1 file changed, 1 deletion(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 65b4ae5412..9a5ef56c47 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -3905,7 +3905,6 @@ dec_term_atom_common: factory.p = NULL; factory.hp = hp; /* We assume heap will suffice (see hashmap_over_estimated_heap_size) */ - factory.hp_end = hp + (ERTS_SWORD_MAX / sizeof(Eterm)); do { *hamt->objp = erts_hashmap_from_array(&factory, -- cgit v1.2.3 From 65c7116bb38013c3d829f21b00bf094606e46731 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 19 Mar 2015 15:40:29 +0100 Subject: erts: Fix bug in binary_to_term with more than one big map --- erts/emulator/beam/external.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 9a5ef56c47..458ebd8aa0 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -3899,7 +3899,6 @@ dec_term_atom_common: /* Iterate through all the hamts and build tree nodes. */ if (hamt_list) { - struct dec_term_hamt_placeholder* hamt = hamt_list; ErtsHeapFactory factory; factory.p = NULL; @@ -3907,6 +3906,7 @@ dec_term_atom_common: /* We assume heap will suffice (see hashmap_over_estimated_heap_size) */ do { + struct dec_term_hamt_placeholder* hamt = hamt_list; *hamt->objp = erts_hashmap_from_array(&factory, hamt->leafs, hamt->size, -- cgit v1.2.3 From 1f23b603f022c7345dd573bf917926a575ae030d Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 24 Mar 2015 20:01:11 +0100 Subject: erts: Fix bug in binary_to_term for hamt when yielding Must save hamt_list in context. --- erts/emulator/beam/external.c | 1 + 1 file changed, 1 insertion(+) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 82c60840e5..c99b60ed09 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -3878,6 +3878,7 @@ dec_term_atom_common: ctx->u.dc.next = next; ctx->u.dc.hp = hp; ctx->u.dc.maps_list = maps_list; + ctx->u.dc.hamt_list = hamt_list; ctx->reds = 0; return NULL; } -- cgit v1.2.3 From db54eaa94562b49c81b677948a8e9139ebdb010e Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 24 Mar 2015 15:27:11 +0100 Subject: erts: Remove HAMT_SUBTAG_NODE_ARRAY This will also fix a bug in term_to_binary treating full nodes as tuples and emiting LIST_EXT for leafs. --- erts/emulator/beam/external.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index c99b60ed09..b0b232f185 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -2633,8 +2633,6 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, *ep++ = MAP_EXT; ptr++; put_int32(*ptr, ep); ep += 4; - /*fall through*/ - case HAMT_SUBTAG_NODE_ARRAY: node_sz = 16; break; case HAMT_SUBTAG_HEAD_BITMAP: @@ -4172,8 +4170,8 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, hdr = *ptr; ASSERT(is_header(hdr)); switch(hdr & _HEADER_MAP_SUBTAG_MASK) { - case HAMT_SUBTAG_HEAD_ARRAY: ptr++; - case HAMT_SUBTAG_NODE_ARRAY: + case HAMT_SUBTAG_HEAD_ARRAY: + ptr++; node_sz = 16; break; case HAMT_SUBTAG_HEAD_BITMAP: ptr++; -- cgit v1.2.3 From c157dce842bf78080c533472fcec74df01ed8fdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Fri, 20 Mar 2015 18:01:50 +0100 Subject: erts: Combine flat and hash maps under one unifying tag --- erts/emulator/beam/external.c | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index b0b232f185..2a9189b51e 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -2604,7 +2604,7 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, break; case MAP_DEF: - { + if (is_flatmap(obj)) { flatmap_t *mp = (flatmap_t*)flatmap_val(obj); Uint size = flatmap_get_size(mp); @@ -2618,11 +2618,7 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, WSTACK_PUSH4(s, (UWord)kptr, (UWord)vptr, ENC_MAP_PAIR, size); } - } - break; - - case HASHMAP_DEF: - { + } else { Eterm hdr; Uint node_sz; ptr = boxed_val(obj); @@ -2656,7 +2652,6 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, } } break; - case FLOAT_DEF: GET_DOUBLE(obj, f); if (dflags & DFLAG_NEW_FLOATS) { @@ -3607,7 +3602,7 @@ dec_term_atom_common: kptr = hp - 1; mp = (flatmap_t*)hp; - hp += MAP_HEADER_SIZE; + hp += MAP_HEADER_FLATMAP_SZ; hp += size; vptr = hp - 1; @@ -3893,7 +3888,7 @@ dec_term_atom_common: while (maps_list) { next = (Eterm *)(EXPAND_POINTER(*maps_list)); - *maps_list = MAP_HEADER; + *maps_list = MAP_HEADER_FLATMAP; if (!erts_validate_and_sort_flatmap((flatmap_t*)maps_list)) goto error; maps_list = next; @@ -4121,7 +4116,7 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, } break; case MAP_DEF: - { + if (is_flatmap(obj)) { flatmap_t *mp = (flatmap_t*)flatmap_val(obj); Uint size = flatmap_get_size(mp); Uint i; @@ -4158,11 +4153,7 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, ++ptr; } goto outer_loop; - } - break; - - case HASHMAP_DEF: - { + } else { Eterm *ptr; Eterm hdr; Uint node_sz; @@ -4190,7 +4181,6 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, } result += 1 + 4; /* tag + 4 bytes size */ } - break; case FLOAT_DEF: if (dflags & DFLAG_NEW_FLOATS) { -- cgit v1.2.3 From 5d5543fb1e1a30e4572e90bac2ec194a61ca4e30 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 25 Mar 2015 19:55:43 +0100 Subject: erts: Optimize term_to_binary size estimation for tuples and maps containing ascii strings (lists). --- erts/emulator/beam/external.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index b0b232f185..4cd57d8aeb 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -4109,8 +4109,9 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, } for (i = 1; i <= arity; ++i) { if (is_list(ptr[i])) { - if ((m = is_string(obj)) && (m < MAX_STRING_LEN)) { + if ((m = is_string(ptr[i])) && (m < MAX_STRING_LEN)) { result += m + 2 + 1; + continue; } else { result += 5; } @@ -4131,31 +4132,29 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, /* push values first */ ptr = flatmap_get_values(mp); - i = size; - while(i--) { + for (i = size; i; i--, ptr++) { if (is_list(*ptr)) { if ((m = is_string(*ptr)) && (m < MAX_STRING_LEN)) { result += m + 2 + 1; + continue; } else { result += 5; } } ESTACK_PUSH(s,*ptr); - ++ptr; } ptr = flatmap_get_keys(mp); - i = size; - while(i--) { + for (i = size; i; i--, ptr++) { if (is_list(*ptr)) { if ((m = is_string(*ptr)) && (m < MAX_STRING_LEN)) { result += m + 2 + 1; + continue; } else { result += 5; } } ESTACK_PUSH(s,*ptr); - ++ptr; } goto outer_loop; } -- cgit v1.2.3 From d09ec9ee97816bd987e6322797b0e28c27f8590a Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 25 Mar 2015 20:50:12 +0100 Subject: erts: Fix bug in term_to_binary size estimation for hamt --- erts/emulator/beam/external.c | 39 +++++++++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 4cd57d8aeb..8f0e19d27d 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -4172,8 +4172,12 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, case HAMT_SUBTAG_HEAD_ARRAY: ptr++; node_sz = 16; + result += 1 + 4; /* tag + 4 bytes size */ break; - case HAMT_SUBTAG_HEAD_BITMAP: ptr++; + case HAMT_SUBTAG_HEAD_BITMAP: + ptr++; + result += 1 + 4; /* tag + 4 bytes size */ + /*fall through*/ case HAMT_SUBTAG_NODE_BITMAP: node_sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); ASSERT(node_sz < 17); @@ -4183,11 +4187,38 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, } ptr++; - ESTACK_RESERVE(s, node_sz); + ESTACK_RESERVE(s, node_sz*2); while(node_sz--) { - ESTACK_FAST_PUSH(s, *ptr++); + if (is_list(*ptr)) { + Eterm* leaf = list_val(*ptr); + if (is_not_list(CAR(leaf))) { + ESTACK_FAST_PUSH(s, CAR(leaf)); + } + else { + if ((m = is_string(CAR(leaf))) && (m < MAX_STRING_LEN)) { + result += m + 2 + 1; + } else { + result += 5; + ESTACK_FAST_PUSH(s, CAR(leaf)); + } + } + if (is_not_list(CDR(leaf))) { + ESTACK_FAST_PUSH(s, CDR(leaf)); + } + else { + if ((m = is_string(CDR(leaf))) && (m < MAX_STRING_LEN)) { + result += m + 2 + 1; + } else { + result += 5; + ESTACK_FAST_PUSH(s, CDR(leaf)); + } + } + } + else { + ESTACK_FAST_PUSH(s, *ptr); + } + ptr++; } - result += 1 + 4; /* tag + 4 bytes size */ } break; -- cgit v1.2.3 From e4da94bee94bdd7e0dbb2e5021ab4cc5f89f8256 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Mon, 30 Mar 2015 19:54:23 +0200 Subject: erts: Refactor encode_size_struct_int to handle the "start of list" case in one place and not seven. Note that this commit reverts (47d6fd3ccf35) back to using WSTACK and pushing raw pointers. We disable GC while yielding, so this should not be a problem. --- erts/emulator/beam/external.c | 187 ++++++++++++++++-------------------------- 1 file changed, 70 insertions(+), 117 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 2117dbec62..fe48298155 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1779,7 +1779,7 @@ static void ttb_context_destructor(Binary *context_bin) context->alive = 0; switch (context->state) { case TTBSize: - DESTROY_SAVED_ESTACK(&context->s.sc.estack); + DESTROY_SAVED_WSTACK(&context->s.sc.wstack); break; case TTBEncode: DESTROY_SAVED_WSTACK(&context->s.ec.wstack); @@ -1847,7 +1847,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla /* Setup enough to get started */ context->state = TTBSize; context->alive = 1; - context->s.sc.estack.start = NULL; + context->s.sc.wstack.wstart = NULL; context->s.sc.flags = flags; context->s.sc.level = level; } else { @@ -3962,51 +3962,35 @@ static int encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags, Sint *reds, Uint *res) { - DECLARE_ESTACK(s); + DECLARE_WSTACK(s); Uint m, i, arity; Uint result = 0; Sint r = 0; if (ctx) { - ESTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK); + WSTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK); r = *reds; - if (ctx->estack.start) { /* restore saved stack */ - ESTACK_RESTORE(s, &ctx->estack); + if (ctx->wstack.wstart) { /* restore saved stack */ + WSTACK_RESTORE(s, &ctx->wstack); result = ctx->result; obj = ctx->obj; } } - goto L_jump_start; +#define LIST_TAIL_OP ((0 << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER) +#define TERM_ARRAY_OP(N) (((N) << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER) +#define TERM_ARRAY_OP_DEC(OP) ((OP) - (1 << _TAG_PRIMARY_SIZE)) + + + for (;;) { + ASSERT(!is_header(obj)); - outer_loop: - while (!ESTACK_ISEMPTY(s)) { - obj = ESTACK_POP(s); - handle_popped_obj: - if (is_list(obj)) { - Eterm* cons = list_val(obj); - Eterm tl; - - tl = CDR(cons); - obj = CAR(cons); - ESTACK_PUSH(s, tl); - } else if (is_nil(obj)) { - result++; - goto outer_loop; - } else { - /* - * Other term (in the tail of a non-proper list or - * in a fun's environment). - */ - } - - L_jump_start: if (ctx && --r == 0) { *reds = r; ctx->obj = obj; ctx->result = result; - ESTACK_SAVE(s, &ctx->estack); + WSTACK_SAVE(s, &ctx->wstack); return -1; } switch (tag_val_def(obj)) { @@ -4089,69 +4073,43 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, result += m + 2 + 1; } else { result += 5; - goto handle_popped_obj; + WSTACK_PUSH2(s, (UWord)CDR(list_val(obj)), (UWord)LIST_TAIL_OP); + obj = CAR(list_val(obj)); + continue; /* big loop */ } break; case TUPLE_DEF: { Eterm* ptr = tuple_val(obj); - Uint i; arity = arityval(*ptr); if (arity <= 0xff) { result += 1 + 1; } else { result += 1 + 4; } - for (i = 1; i <= arity; ++i) { - if (is_list(ptr[i])) { - if ((m = is_string(ptr[i])) && (m < MAX_STRING_LEN)) { - result += m + 2 + 1; - continue; - } else { - result += 5; - } - } - ESTACK_PUSH(s,ptr[i]); + if (arity > 1) { + WSTACK_PUSH2(s, (UWord) (ptr + 2), + (UWord) TERM_ARRAY_OP(arity-1)); } - goto outer_loop; + else if (arity == 0) { + break; + } + obj = ptr[1]; + continue; /* big loop */ } - break; case MAP_DEF: if (is_flatmap(obj)) { flatmap_t *mp = (flatmap_t*)flatmap_val(obj); Uint size = flatmap_get_size(mp); - Uint i; - Eterm *ptr; result += 1 + 4; /* tag + 4 bytes size */ - /* push values first */ - ptr = flatmap_get_values(mp); - for (i = size; i; i--, ptr++) { - if (is_list(*ptr)) { - if ((m = is_string(*ptr)) && (m < MAX_STRING_LEN)) { - result += m + 2 + 1; - continue; - } else { - result += 5; - } - } - ESTACK_PUSH(s,*ptr); + if (size) { + WSTACK_PUSH4(s, (UWord) flatmap_get_values(mp), + (UWord) TERM_ARRAY_OP(size), + (UWord) flatmap_get_keys(mp), + (UWord) TERM_ARRAY_OP(size)); } - - ptr = flatmap_get_keys(mp); - for (i = size; i; i--, ptr++) { - if (is_list(*ptr)) { - if ((m = is_string(*ptr)) && (m < MAX_STRING_LEN)) { - result += m + 2 + 1; - continue; - } else { - result += 5; - } - } - ESTACK_PUSH(s,*ptr); - } - goto outer_loop; } else { Eterm *ptr; Eterm hdr; @@ -4178,35 +4136,13 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, } ptr++; - ESTACK_RESERVE(s, node_sz*2); + WSTACK_RESERVE(s, node_sz*2); while(node_sz--) { if (is_list(*ptr)) { - Eterm* leaf = list_val(*ptr); - if (is_not_list(CAR(leaf))) { - ESTACK_FAST_PUSH(s, CAR(leaf)); - } - else { - if ((m = is_string(CAR(leaf))) && (m < MAX_STRING_LEN)) { - result += m + 2 + 1; - } else { - result += 5; - ESTACK_FAST_PUSH(s, CAR(leaf)); - } - } - if (is_not_list(CDR(leaf))) { - ESTACK_FAST_PUSH(s, CDR(leaf)); - } - else { - if ((m = is_string(CDR(leaf))) && (m < MAX_STRING_LEN)) { - result += m + 2 + 1; - } else { - result += 5; - ESTACK_FAST_PUSH(s, CDR(leaf)); - } - } - } - else { - ESTACK_FAST_PUSH(s, *ptr); + WSTACK_FAST_PUSH(s, CAR(list_val(*ptr))); + WSTACK_FAST_PUSH(s, CDR(list_val(*ptr))); + } else { + WSTACK_FAST_PUSH(s, *ptr); } ptr++; } @@ -4262,25 +4198,13 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, result += 2 * (1 + 4); /* Index + Uniq */ result += 1 + (funp->num_free < 0x100 ? 1 : 4); } - for (i = 1; i < funp->num_free; i++) { - obj = funp->env[i]; - - if (is_not_list(obj)) { - /* Push any non-list terms on the stack */ - ESTACK_PUSH(s, obj); - } else { - /* Lists must be handled specially. */ - if ((m = is_string(obj)) && (m < MAX_STRING_LEN)) { - result += m + 2 + 1; - } else { - result += 5; - ESTACK_PUSH(s, obj); - } - } + if (funp->num_free > 1) { + WSTACK_PUSH2(s, (UWord) (funp->env + 1), + (UWord) TERM_ARRAY_OP(funp->num_free-1)); } if (funp->num_free != 0) { obj = funp->env[0]; - goto L_jump_start; + continue; /* big loop */ } break; } @@ -4303,11 +4227,40 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, erl_exit(1,"Internal data structure error (in encode_size_struct2)%x\n", obj); } + + if (WSTACK_ISEMPTY(s)) { + break; + } + obj = (Eterm) WSTACK_POP(s); + + if (is_header(obj)) { + switch (obj) { + case LIST_TAIL_OP: + obj = (Eterm) WSTACK_POP(s); + if (is_list(obj)) { + Eterm* cons = list_val(obj); + + WSTACK_PUSH2(s, (UWord)CDR(cons), (UWord)LIST_TAIL_OP); + obj = CAR(cons); + } + break; + + case TERM_ARRAY_OP(1): + obj = *(Eterm*)WSTACK_POP(s); + break; + default: { /* TERM_ARRAY_OP(N) when N > 1 */ + Eterm* ptr = (Eterm*) WSTACK_POP(s); + WSTACK_PUSH2(s, (UWord) (ptr+1), + (UWord) TERM_ARRAY_OP_DEC(obj)); + obj = *ptr; + } + } + } } - DESTROY_ESTACK(s); + WSTACK_DESTROY(s); if (ctx) { - ASSERT(ctx->estack.start == NULL); + ASSERT(ctx->wstack.wstart == NULL); *reds = r; } *res = result; -- cgit v1.2.3 From 02d0ce034598297565f2b35ecc3d1af121787f33 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 2 Jun 2015 19:23:49 +0200 Subject: erts: Remove hashmap probabilistic heap overestimation by adding a dynamic heap factory. "binary_to_term" is now a hybrid solution with both a call to decoded_size() to calculate needed heap space AND possible dynamic allocation of more heap space if needed for big maps. The heap size returned from decoded_size() is guaranteed to be sufficient for all term heap data except for hashmap nodes. All hashmap nodes are created at the end of dec_term() by invoking the heap factory interface that may allocate more heap space on process heap or in fragments. With this commit it is no longer guaranteed that a message is confined to only one heap fragment. --- erts/emulator/beam/external.c | 291 ++++++++++++++++++++---------------------- 1 file changed, 135 insertions(+), 156 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index fe48298155..0a69172980 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -94,9 +94,9 @@ static Uint is_external_string(Eterm obj, int* p_is_string); static byte* enc_atom(ErtsAtomCacheMap *, Eterm, byte*, Uint32); static byte* enc_pid(ErtsAtomCacheMap *, Eterm, byte*, Uint32); struct B2TContext_t; -static byte* dec_term(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*, struct B2TContext_t*); +static byte* dec_term(ErtsDistExternal*, ErtsHeapFactory*, byte*, Eterm*, struct B2TContext_t*); static byte* dec_atom(ErtsDistExternal *, byte*, Eterm*); -static byte* dec_pid(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*); +static byte* dec_pid(ErtsDistExternal *, ErtsHeapFactory*, byte*, Eterm*); static Sint decoded_size(byte *ep, byte* endp, int internal_tags, struct B2TContext_t*); static BIF_RETTYPE term_to_binary_trap_1(BIF_ALIST_1); @@ -930,8 +930,7 @@ Sint erts_decode_ext_size_ets(byte *ext, Uint size) ** on return hpp is updated to point after allocated data */ Eterm -erts_decode_dist_ext(Eterm** hpp, - ErlOffHeap* off_heap, +erts_decode_dist_ext(ErtsHeapFactory* factory, ErtsDistExternal *edep) { Eterm obj; @@ -951,7 +950,7 @@ erts_decode_dist_ext(Eterm** hpp, goto error; ep++; } - ep = dec_term(edep, hpp, ep, off_heap, &obj, NULL); + ep = dec_term(edep, factory, ep, &obj, NULL); if (!ep) goto error; @@ -960,19 +959,22 @@ erts_decode_dist_ext(Eterm** hpp, return obj; error: + erts_factory_undo(factory); bad_dist_ext(edep); return THE_NON_VALUE; } -Eterm erts_decode_ext(Eterm **hpp, ErlOffHeap *off_heap, byte **ext) +Eterm erts_decode_ext(ErtsHeapFactory* factory, byte **ext) { Eterm obj; byte *ep = *ext; - if (*ep++ != VERSION_MAGIC) + if (*ep++ != VERSION_MAGIC) { + erts_factory_undo(factory); return THE_NON_VALUE; - ep = dec_term(NULL, hpp, ep, off_heap, &obj, NULL); + } + ep = dec_term(NULL, factory, ep, &obj, NULL); if (!ep) { #ifdef DEBUG bin_write(ERTS_PRINT_STDERR,NULL,*ext,500); @@ -983,10 +985,10 @@ Eterm erts_decode_ext(Eterm **hpp, ErlOffHeap *off_heap, byte **ext) return obj; } -Eterm erts_decode_ext_ets(Eterm **hpp, ErlOffHeap *off_heap, byte *ext) +Eterm erts_decode_ext_ets(ErtsHeapFactory* factory, byte *ext) { Eterm obj; - ext = dec_term(NULL, hpp, ext, off_heap, &obj, NULL); + ext = dec_term(NULL, factory, ext, &obj, NULL); ASSERT(ext); return obj; } @@ -995,9 +997,8 @@ Eterm erts_decode_ext_ets(Eterm **hpp, ErlOffHeap *off_heap, byte *ext) BIF_RETTYPE erts_debug_dist_ext_to_term_2(BIF_ALIST_2) { + ErtsHeapFactory factory; Eterm res; - Eterm *hp; - Eterm *hendp; Sint hsz; ErtsDistExternal ede; Eterm *tp; @@ -1044,12 +1045,9 @@ BIF_RETTYPE erts_debug_dist_ext_to_term_2(BIF_ALIST_2) if (hsz < 0) goto badarg; - hp = HAlloc(BIF_P, (Uint) hsz); - hendp = hp + hsz; - - res = erts_decode_dist_ext(&hp, &MSO(BIF_P), &ede); - - HRelease(BIF_P, hendp, hp); + erts_factory_proc_prealloc_init(&factory, BIF_P, hsz); + res = erts_decode_dist_ext(&factory, &ede); + erts_factory_close(&factory); if (is_value(res)) BIF_RET(res); @@ -1177,13 +1175,11 @@ typedef struct { byte* ep; Eterm res; Eterm* next; - Eterm* hp_start; - Eterm* hp; - Eterm* hp_end; + ErtsHeapFactory factory; int remaining_n; char* remaining_bytes; Eterm* maps_list; - struct dec_term_hamt_placeholder* hamt_list; + ErtsPStack hamt_array; } B2TDecodeContext; typedef struct { @@ -1307,10 +1303,12 @@ binary2term_abort(ErtsBinary2TermState *state) } static ERTS_INLINE Eterm -binary2term_create(ErtsDistExternal *edep, ErtsBinary2TermState *state, Eterm **hpp, ErlOffHeap *ohp) +binary2term_create(ErtsDistExternal *edep, ErtsBinary2TermState *state, + ErtsHeapFactory* factory) { Eterm res; - if (!dec_term(edep, hpp, state->extp, ohp, &res, NULL)) + + if (!dec_term(edep, factory, state->extp, &res, NULL)) res = THE_NON_VALUE; if (state->exttmp) { state->exttmp = 0; @@ -1343,9 +1341,9 @@ erts_binary2term_abort(ErtsBinary2TermState *state) } Eterm -erts_binary2term_create(ErtsBinary2TermState *state, Eterm **hpp, ErlOffHeap *ohp) +erts_binary2term_create(ErtsBinary2TermState *state, ErtsHeapFactory* factory) { - return binary2term_create(NULL,state, hpp, ohp); + return binary2term_create(NULL,state, factory); } static void b2t_destroy_context(B2TContext* context) @@ -1354,8 +1352,21 @@ static void b2t_destroy_context(B2TContext* context) ERTS_ALC_T_EXT_TERM_DATA); context->aligned_alloc = NULL; binary2term_abort(&context->b2ts); - if (context->state == B2TUncompressChunk) { + switch (context->state) { + case B2TUncompressChunk: erl_zlib_inflate_finish(&context->u.uc.stream); + break; + case B2TDecode: + case B2TDecodeList: + case B2TDecodeTuple: + case B2TDecodeString: + case B2TDecodeBinary: + if (context->u.dc.hamt_array.pstart) { + erts_free(context->u.dc.hamt_array.alloc_type, + context->u.dc.hamt_array.pstart); + } + break; + default:; } } @@ -1506,11 +1517,9 @@ static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binar ctx->u.dc.ep = ctx->b2ts.extp; ctx->u.dc.res = (Eterm) (UWord) NULL; ctx->u.dc.next = &ctx->u.dc.res; - ctx->u.dc.hp_start = HAlloc(p, ctx->heap_size); - ctx->u.dc.hp = ctx->u.dc.hp_start; - ctx->u.dc.hp_end = ctx->u.dc.hp_start + ctx->heap_size; + erts_factory_proc_prealloc_init(&ctx->u.dc.factory, p, ctx->heap_size); ctx->u.dc.maps_list = NULL; - ctx->u.dc.hamt_list = NULL; + ctx->u.dc.hamt_array.pstart = NULL; ctx->state = B2TDecode; /*fall through*/ case B2TDecode: @@ -1520,11 +1529,10 @@ static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binar case B2TDecodeBinary: { ErtsDistExternal fakedep; fakedep.flags = ctx->flags; - dec_term(&fakedep, NULL, NULL, &MSO(p), NULL, ctx); + dec_term(&fakedep, NULL, NULL, NULL, ctx); break; } case B2TDecodeFail: - HRelease(p, ctx->u.dc.hp_end, ctx->u.dc.hp_start); /*fall through*/ case B2TBadArg: BUMP_REDS(p, (initial_reds - ctx->reds) / B2T_BYTES_PER_REDUCTION); @@ -1549,11 +1557,11 @@ static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binar case B2TDone: b2t_destroy_context(ctx); - if (ctx->u.dc.hp > ctx->u.dc.hp_end) { + if (ctx->u.dc.factory.hp > ctx->u.dc.factory.hp_end) { erl_exit(1, ":%s, line %d: heap overrun by %d words(s)\n", - __FILE__, __LINE__, ctx->u.dc.hp - ctx->u.dc.hp_end); + __FILE__, __LINE__, ctx->u.dc.factory.hp - ctx->u.dc.factory.hp_end); } - HRelease(p, ctx->u.dc.hp_end, ctx->u.dc.hp); + erts_factory_close(&ctx->u.dc.factory); if (!is_first_call) { erts_set_gc_state(p, 1); @@ -2247,7 +2255,7 @@ static ERTS_INLINE ErlNode* dec_get_node(Eterm sysname, Uint creation) } static byte* -dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Eterm* objp) +dec_pid(ErtsDistExternal *edep, ErtsHeapFactory* factory, byte* ep, Eterm* objp) { Eterm sysname; Uint data; @@ -2286,15 +2294,15 @@ dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Ete if(node == erts_this_node) { *objp = make_internal_pid(data); } else { - ExternalThing *etp = (ExternalThing *) *hpp; - *hpp += EXTERNAL_THING_HEAD_SIZE + 1; + ExternalThing *etp = (ExternalThing *) factory->hp; + factory->hp += EXTERNAL_THING_HEAD_SIZE + 1; etp->header = make_external_pid_header(1); - etp->next = off_heap->first; + etp->next = factory->off_heap->first; etp->node = node; etp->data.ui[0] = data; - off_heap->first = (struct erl_off_heap_header*) etp; + factory->off_heap->first = (struct erl_off_heap_header*) etp; *objp = make_external_pid(etp); } return ep; @@ -2905,69 +2913,43 @@ is_external_string(Eterm list, int* p_is_string) return len; } -/* Assumes that the ones to undo are preluding the list. */ -static void -undo_offheap_in_area(ErlOffHeap* off_heap, Eterm* start, Eterm* end) -{ - const Uint area_sz = (end - start) * sizeof(Eterm); - struct erl_off_heap_header* hdr; - struct erl_off_heap_header** hdr_nextp = NULL; - - for (hdr = off_heap->first; ; hdr=hdr->next) { - if (!in_area(hdr, start, area_sz)) { - if (hdr_nextp != NULL) { - *hdr_nextp = NULL; - erts_cleanup_offheap(off_heap); - off_heap->first = hdr; - } - break; - } - hdr_nextp = &hdr->next; - } - - /* Assert that the ones to undo were indeed preluding the list. */ -#ifdef DEBUG - for (hdr = off_heap->first; hdr != NULL; hdr = hdr->next) { - ASSERT(!in_area(hdr, start, area_sz)); - } -#endif /* DEBUG */ -} -struct dec_term_hamt_placeholder +struct dec_term_hamt { - struct dec_term_hamt_placeholder* next; Eterm* objp; /* write result here */ Uint size; /* nr of leafs */ - Eterm leafs[1]; + Eterm* leaf_array; }; -#define DEC_TERM_HAMT_PLACEHOLDER_SIZE \ - (offsetof(struct dec_term_hamt_placeholder, leafs) / sizeof(Eterm)) /* Decode term from external format into *objp. -** On failure return NULL and *hpp will be unchanged. +** On failure calls erts_factory_undo() and returns NULL */ static byte* -dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, - Eterm* objp, B2TContext* ctx) +dec_term(ErtsDistExternal *edep, + ErtsHeapFactory* factory, + byte* ep, + Eterm* objp, + B2TContext* ctx) { - Eterm* hp_saved; +#define PSTACK_TYPE struct dec_term_hamt + PSTACK_DECLARE(hamt_array, 5); int n; ErtsAtomEncoding char_enc; register Eterm* hp; /* Please don't take the address of hp */ Eterm *maps_list; /* for preprocessing of small maps */ - struct dec_term_hamt_placeholder* hamt_list; /* for preprocessing of big maps */ Eterm* next; SWord reds; +#ifdef DEBUG + Eterm* dbg_resultp = ctx ? &ctx->u.dc.res : objp; +#endif if (ctx) { - hp_saved = ctx->u.dc.hp_start; reds = ctx->reds; next = ctx->u.dc.next; ep = ctx->u.dc.ep; - hpp = &ctx->u.dc.hp; + factory = &ctx->u.dc.factory; maps_list = ctx->u.dc.maps_list; - hamt_list = ctx->u.dc.hamt_list; if (ctx->state != B2TDecode) { int n_limit = reds; @@ -3012,7 +2994,7 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, break; case B2TDecodeString: - hp = *hpp; + hp = factory->hp; hp[-1] = make_list(hp); /* overwrite the premature NIL */ while (n-- > 0) { hp[0] = make_small(*ep++); @@ -3020,7 +3002,7 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, hp += 2; } hp[-1] = NIL; - *hpp = hp; + factory->hp = hp; break; case B2TDecodeBinary: @@ -3042,16 +3024,18 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, return NULL; } } + PSTACK_CHANGE_ALLOCATOR(hamt_array, ERTS_ALC_T_SAVED_ESTACK); + if (ctx->u.dc.hamt_array.pstart) { + PSTACK_RESTORE(hamt_array, &ctx->u.dc.hamt_array); + } } else { - hp_saved = *hpp; reds = ERTS_SWORD_MAX; next = objp; *next = (Eterm) (UWord) NULL; maps_list = NULL; - hamt_list = NULL; } - hp = *hpp; + hp = factory->hp; while (next != NULL) { @@ -3288,9 +3272,9 @@ dec_term_atom_common: break; } case PID_EXT: - *hpp = hp; - ep = dec_pid(edep, hpp, ep, off_heap, objp); - hp = *hpp; + factory->hp = hp; + ep = dec_pid(edep, factory, ep, objp); + hp = factory->hp; if (ep == NULL) { goto error; } @@ -3324,11 +3308,11 @@ dec_term_atom_common: hp += EXTERNAL_THING_HEAD_SIZE + 1; etp->header = make_external_port_header(1); - etp->next = off_heap->first; + etp->next = factory->off_heap->first; etp->node = node; etp->data.ui[0] = num; - off_heap->first = (struct erl_off_heap_header*)etp; + factory->off_heap->first = (struct erl_off_heap_header*)etp; *objp = make_external_port(etp); } @@ -3408,10 +3392,10 @@ dec_term_atom_common: #else etp->header = make_external_ref_header(ref_words); #endif - etp->next = off_heap->first; + etp->next = factory->off_heap->first; etp->node = node; - off_heap->first = (struct erl_off_heap_header*)etp; + factory->off_heap->first = (struct erl_off_heap_header*)etp; *objp = make_external_ref(etp); ref_num = &(etp->data.ui32[0]); } @@ -3451,9 +3435,9 @@ dec_term_atom_common: hp += PROC_BIN_SIZE; pb->thing_word = HEADER_PROC_BIN; pb->size = n; - pb->next = off_heap->first; - off_heap->first = (struct erl_off_heap_header*)pb; - OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm)); + pb->next = factory->off_heap->first; + factory->off_heap->first = (struct erl_off_heap_header*)pb; + OH_OVERHEAD(factory->off_heap, pb->size / sizeof(Eterm)); pb->val = dbin; pb->bytes = (byte*) dbin->orig_bytes; pb->flags = 0; @@ -3503,9 +3487,9 @@ dec_term_atom_common: pb = (ProcBin *) hp; pb->thing_word = HEADER_PROC_BIN; pb->size = n; - pb->next = off_heap->first; - off_heap->first = (struct erl_off_heap_header*)pb; - OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm)); + pb->next = factory->off_heap->first; + factory->off_heap->first = (struct erl_off_heap_header*)pb; + OH_OVERHEAD(factory->off_heap, pb->size / sizeof(Eterm)); pb->val = dbin; pb->bytes = (byte*) dbin->orig_bytes; pb->flags = 0; @@ -3557,9 +3541,9 @@ dec_term_atom_common: if ((ep = dec_atom(edep, ep, &name)) == NULL) { goto error; } - *hpp = hp; - ep = dec_term(edep, hpp, ep, off_heap, &temp, NULL); - hp = *hpp; + factory->hp = hp; + ep = dec_term(edep, factory, ep, &temp, NULL); + hp = factory->hp; if (ep == NULL) { goto error; } @@ -3631,15 +3615,11 @@ dec_term_atom_common: } } else { /* Make hamt */ - struct dec_term_hamt_placeholder* holder = - (struct dec_term_hamt_placeholder*) hp; - - holder->next = hamt_list; - hamt_list = holder; - holder->objp = objp; - holder->size = size; + struct dec_term_hamt* hamt = PSTACK_PUSH(hamt_array); - hp += DEC_TERM_HAMT_PLACEHOLDER_SIZE; + hamt->objp = objp; + hamt->size = size; + hamt->leaf_array = hp; for (n = size; n; n--) { CDR(hp) = (Eterm) COMPRESS_POINTER(next); @@ -3681,9 +3661,9 @@ dec_term_atom_common: if ((ep = dec_atom(edep, ep, &module)) == NULL) { goto error; } - *hpp = hp; + factory->hp = hp; /* Index */ - if ((ep = dec_term(edep, hpp, ep, off_heap, &temp, NULL)) == NULL) { + if ((ep = dec_term(edep, factory, ep, &temp, NULL)) == NULL) { goto error; } if (!is_small(temp)) { @@ -3692,7 +3672,7 @@ dec_term_atom_common: old_index = unsigned_val(temp); /* Uniq */ - if ((ep = dec_term(edep, hpp, ep, off_heap, &temp, NULL)) == NULL) { + if ((ep = dec_term(edep, factory, ep, &temp, NULL)) == NULL) { goto error; } if (!is_small(temp)) { @@ -3704,8 +3684,8 @@ dec_term_atom_common: * It is safe to link the fun into the fun list only when * no more validity tests can fail. */ - funp->next = off_heap->first; - off_heap->first = (struct erl_off_heap_header*)funp; + funp->next = factory->off_heap->first; + factory->off_heap->first = (struct erl_off_heap_header*)funp; funp->fe = erts_put_fun_entry2(module, old_uniq, old_index, uniq, index, arity); @@ -3716,7 +3696,7 @@ dec_term_atom_common: } funp->native_address = funp->fe->native_address; #endif - hp = *hpp; + hp = factory->hp; /* Environment */ for (i = num_free-1; i >= 0; i--) { @@ -3742,14 +3722,14 @@ dec_term_atom_common: ep += 4; hp += ERL_FUN_SIZE; hp += num_free; - *hpp = hp; + factory->hp = hp; funp->thing_word = HEADER_FUN; funp->num_free = num_free; *objp = make_fun(funp); /* Creator pid */ if (*ep != PID_EXT - || (ep = dec_pid(edep, hpp, ++ep, off_heap, + || (ep = dec_pid(edep, factory, ++ep, &funp->creator))==NULL) { goto error; } @@ -3760,7 +3740,7 @@ dec_term_atom_common: } /* Index */ - if ((ep = dec_term(edep, hpp, ep, off_heap, &temp, NULL)) == NULL) { + if ((ep = dec_term(edep, factory, ep, &temp, NULL)) == NULL) { goto error; } if (!is_small(temp)) { @@ -3769,7 +3749,7 @@ dec_term_atom_common: old_index = unsigned_val(temp); /* Uniq */ - if ((ep = dec_term(edep, hpp, ep, off_heap, &temp, NULL)) == NULL) { + if ((ep = dec_term(edep, factory, ep, &temp, NULL)) == NULL) { goto error; } if (!is_small(temp)) { @@ -3780,8 +3760,8 @@ dec_term_atom_common: * It is safe to link the fun into the fun list only when * no more validity tests can fail. */ - funp->next = off_heap->first; - off_heap->first = (struct erl_off_heap_header*)funp; + funp->next = factory->off_heap->first; + factory->off_heap->first = (struct erl_off_heap_header*)funp; old_uniq = unsigned_val(temp); funp->fe = erts_put_fun_entry(module, old_uniq, old_index); @@ -3789,7 +3769,7 @@ dec_term_atom_common: #ifdef HIPE funp->native_address = funp->fe->native_address; #endif - hp = *hpp; + hp = factory->hp; /* Environment */ for (i = num_free-1; i >= 0; i--) { @@ -3823,9 +3803,9 @@ dec_term_atom_common: erts_refc_inc(&pb->val->refc, 1); hp += PROC_BIN_SIZE; - pb->next = off_heap->first; - off_heap->first = (struct erl_off_heap_header*)pb; - OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm)); + pb->next = factory->off_heap->first; + factory->off_heap->first = (struct erl_off_heap_header*)pb; + OH_OVERHEAD(factory->off_heap, pb->size / sizeof(Eterm)); pb->flags = 0; *objp = make_binary(pb); break; @@ -3841,9 +3821,9 @@ dec_term_atom_common: erts_refc_inc(&pb->val->refc, 1); hp += PROC_BIN_SIZE; - pb->next = off_heap->first; - off_heap->first = (struct erl_off_heap_header*)pb; - OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm)); + pb->next = factory->off_heap->first; + factory->off_heap->first = (struct erl_off_heap_header*)pb; + OH_OVERHEAD(factory->off_heap, pb->size / sizeof(Eterm)); pb->flags = 0; sub = (ErlSubBin*)hp; @@ -3869,9 +3849,11 @@ dec_term_atom_common: if (next || ctx->state != B2TDecode) { ctx->u.dc.ep = ep; ctx->u.dc.next = next; - ctx->u.dc.hp = hp; + ctx->u.dc.factory.hp = hp; ctx->u.dc.maps_list = maps_list; - ctx->u.dc.hamt_list = hamt_list; + if (!PSTACK_IS_EMPTY(hamt_array)) { + PSTACK_SAVE(hamt_array, &ctx->u.dc.hamt_array); + } ctx->reds = 0; return NULL; } @@ -3894,40 +3876,36 @@ dec_term_atom_common: maps_list = next; } - /* Iterate through all the hamts and build tree nodes. + ASSERT(hp <= factory->hp_end + || (factory->mode == FACTORY_CLOSED && is_immed(*dbg_resultp))); + factory->hp = hp; + /* + * From here on factory may produce (more) heap fragments */ - if (hamt_list) { - ErtsHeapFactory factory; - factory.p = NULL; - factory.hp = hp; - /* We assume heap will suffice (see hashmap_over_estimated_heap_size) */ + if (!PSTACK_IS_EMPTY(hamt_array)) { + do { + struct dec_term_hamt* hamt = PSTACK_TOP(hamt_array); - do { - struct dec_term_hamt_placeholder* hamt = hamt_list; - *hamt->objp = erts_hashmap_from_array(&factory, - hamt->leafs, + *hamt->objp = erts_hashmap_from_array(factory, + hamt->leaf_array, hamt->size, 1); if (is_non_value(*hamt->objp)) - goto error; + goto error_hamt; - hamt_list = hamt->next; - - /* Yes, we waste a couple of heap words per hamt - for the temporary placeholder */ - *(Eterm*)hamt = make_pos_bignum_header(DEC_TERM_HAMT_PLACEHOLDER_SIZE-1); - } while (hamt_list); - - hp = factory.hp; + (void) PSTACK_POP(hamt_array); + } while (!PSTACK_IS_EMPTY(hamt_array)); + PSTACK_DESTROY(hamt_array); } + ASSERT((Eterm*)EXPAND_POINTER(*dbg_resultp) != NULL); + if (ctx) { ctx->state = B2TDone; ctx->reds = reds; } - *hpp = hp; return ep; error: @@ -3935,11 +3913,12 @@ error: * Must unlink all off-heap objects that may have been * linked into the process. */ - if (hp < *hpp) { /* Sometimes we used hp and sometimes *hpp */ - hp = *hpp; /* the largest must be the freshest */ + if (factory->hp < hp) { /* Sometimes we used hp and sometimes factory->hp */ + factory->hp = hp; /* the largest must be the freshest */ } - undo_offheap_in_area(off_heap, hp_saved, hp); - *hpp = hp_saved; +error_hamt: + erts_factory_undo(factory); + PSTACK_DESTROY(hamt_array); if (ctx) { ctx->state = B2TDecodeFail; ctx->reds = reds; @@ -4465,7 +4444,7 @@ init_done: if (n <= MAP_SMALL_MAP_LIMIT) { heap_size += 3 + n + 1 + n; } else { - heap_size += hashmap_over_estimated_heap_size(n); + heap_size += HASHMAP_ESTIMATED_HEAP_SIZE(n); } break; case STRING_EXT: -- cgit v1.2.3 From 738c34d4bb8f1a3811acd00af8c6c12107f8315b Mon Sep 17 00:00:00 2001 From: Bruce Yinhe Date: Thu, 18 Jun 2015 11:31:02 +0200 Subject: Change license text to APLv2 --- erts/emulator/beam/external.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 0a69172980..c6d7e3fcc5 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ -- cgit v1.2.3 From 1c86a620d74f4f9383c4956dafd3e2486300dc0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Wed, 17 Jun 2015 17:33:29 +0200 Subject: erts: Remove HALFWORD_HEAP definition --- erts/emulator/beam/external.c | 50 ++++++++----------------------------------- 1 file changed, 9 insertions(+), 41 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index c6d7e3fcc5..f7b372d294 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -2339,10 +2339,6 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Eterm val; FloatDef f; Sint r = 0; -#if HALFWORD_HEAP - UWord wobj; -#endif - if (ctx) { WSTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK); @@ -2362,11 +2358,8 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, outer_loop: while (!WSTACK_ISEMPTY(s)) { -#if HALFWORD_HEAP - obj = (Eterm) (wobj = WSTACK_POP(s)); -#else obj = WSTACK_POP(s); -#endif + switch (val = WSTACK_POP(s)) { case ENC_TERM: break; @@ -2384,11 +2377,7 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, break; case ENC_PATCH_FUN_SIZE: { -#if HALFWORD_HEAP - byte* size_p = (byte *) wobj; -#else byte* size_p = (byte *) obj; -#endif put_int32(ep - size_p, size_p); } goto outer_loop; @@ -2435,21 +2424,13 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, case ENC_LAST_ARRAY_ELEMENT: /* obj is the tuple */ { -#if HALFWORD_HEAP - Eterm* ptr = (Eterm *) wobj; -#else Eterm* ptr = (Eterm *) obj; -#endif obj = *ptr; } break; default: /* ENC_LAST_ARRAY_ELEMENT+1 and upwards */ { -#if HALFWORD_HEAP - Eterm* ptr = (Eterm *) wobj; -#else Eterm* ptr = (Eterm *) obj; -#endif obj = *ptr++; WSTACK_PUSH2(s, val-1, (UWord)ptr); } @@ -3049,7 +3030,7 @@ dec_term(ErtsDistExternal *edep, Sint sn = get_int32(ep); ep += 4; -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) *objp = make_small(sn); #else if (MY_IS_SSMALL(sn)) { @@ -3371,7 +3352,7 @@ dec_term_atom_common: RefThing *rtp = (RefThing *) hp; ref_num = (Uint32 *) (hp + REF_THING_HEAD_SIZE); -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) hp += REF_THING_HEAD_SIZE + ref_words/2 + 1; rtp->header = make_ref_thing_header(ref_words/2 + 1); #else @@ -3382,13 +3363,13 @@ dec_term_atom_common: } else { ExternalThing *etp = (ExternalThing *) hp; -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) hp += EXTERNAL_THING_HEAD_SIZE + ref_words/2 + 1; #else hp += EXTERNAL_THING_HEAD_SIZE + ref_words; #endif -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) etp->header = make_external_ref_header(ref_words/2 + 1); #else etp->header = make_external_ref_header(ref_words); @@ -3401,7 +3382,7 @@ dec_term_atom_common: ref_num = &(etp->data.ui32[0]); } -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) *(ref_num++) = ref_words /* 32-bit arity */; #endif ref_num[0] = r0; @@ -3409,7 +3390,7 @@ dec_term_atom_common: ref_num[i] = get_int32(ep); ep += 4; } -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) if ((1 + ref_words) % 2) ref_num[ref_words] = 0; #endif @@ -3561,12 +3542,7 @@ dec_term_atom_common: } *objp = make_export(hp); *hp++ = HEADER_EXPORT; -#if HALFWORD_HEAP - *((UWord *) (UWord) hp) = (UWord) erts_export_get_or_make_stub(mod, name, arity); - hp += 2; -#else *hp++ = (Eterm) erts_export_get_or_make_stub(mod, name, arity); -#endif break; } break; @@ -4192,11 +4168,7 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, case EXPORT_DEF: { Export* ep = *((Export **) (export_val(obj) + 1)); -#if HALFWORD_HEAP - result += 2; -#else result += 1; -#endif result += encode_size_struct2(acmp, ep->code[0], dflags); result += encode_size_struct2(acmp, ep->code[1], dflags); result += encode_size_struct2(acmp, make_small(ep->code[2]), dflags); @@ -4311,7 +4283,7 @@ init_done: switch (tag) { case INTEGER_EXT: SKIP(4); -#if !defined(ARCH_64) || HALFWORD_HEAP +#if !defined(ARCH_64) heap_size += BIG_UINT_HEAP_SIZE; #endif break; @@ -4400,7 +4372,7 @@ init_done: ep += 2; atom_extra_skip = 1 + 4*id_words; /* In case it is an external ref */ -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) heap_size += EXTERNAL_THING_HEAD_SIZE + id_words/2 + 1; #else heap_size += EXTERNAL_THING_HEAD_SIZE + id_words; @@ -4486,11 +4458,7 @@ init_done: break; case EXPORT_EXT: terms += 3; -#if HALFWORD_HEAP - heap_size += 3; -#else heap_size += 2; -#endif break; case NEW_FUN_EXT: { -- cgit v1.2.3 From 17bcc73e511eee06ca64d51edb401f8340fe9abc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Thu, 18 Jun 2015 14:59:28 +0200 Subject: erts: Remove halfword pointer compression * Removed COMPRESS_POINTER and EXPAND_POINTER --- erts/emulator/beam/external.c | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index f7b372d294..c3e93d1ad2 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -2958,7 +2958,7 @@ dec_term(ErtsDistExternal *edep, case B2TDecodeList: objp = next - 2; while (n > 0) { - objp[0] = (Eterm) COMPRESS_POINTER(next); + objp[0] = (Eterm) next; objp[1] = make_list(next); next = objp; objp -= 2; @@ -2969,7 +2969,7 @@ dec_term(ErtsDistExternal *edep, case B2TDecodeTuple: objp = next - 1; while (n-- > 0) { - objp[0] = (Eterm) COMPRESS_POINTER(next); + objp[0] = (Eterm) next; next = objp; objp--; } @@ -3022,7 +3022,7 @@ dec_term(ErtsDistExternal *edep, while (next != NULL) { objp = next; - next = (Eterm *) EXPAND_POINTER(*objp); + next = (Eterm *) *objp; switch (*ep++) { case INTEGER_EXT: @@ -3153,7 +3153,7 @@ dec_term_atom_common: reds -= n; } while (n-- > 0) { - objp[0] = (Eterm) COMPRESS_POINTER(next); + objp[0] = (Eterm) next; next = objp; objp--; } @@ -3171,8 +3171,8 @@ dec_term_atom_common: *objp = make_list(hp); hp += 2 * n; objp = hp - 2; - objp[0] = (Eterm) COMPRESS_POINTER((objp+1)); - objp[1] = (Eterm) COMPRESS_POINTER(next); + objp[0] = (Eterm) (objp+1); + objp[1] = (Eterm) next; next = objp; objp -= 2; n--; @@ -3185,7 +3185,7 @@ dec_term_atom_common: reds -= n; } while (n > 0) { - objp[0] = (Eterm) COMPRESS_POINTER(next); + objp[0] = (Eterm) next; objp[1] = make_list(next); next = objp; objp -= 2; @@ -3576,7 +3576,7 @@ dec_term_atom_common: * The list of maps is for later validation. */ - mp->thing_word = (Eterm) COMPRESS_POINTER(maps_list); + mp->thing_word = (Eterm) maps_list; maps_list = (Eterm *) mp; mp->size = size; @@ -3584,8 +3584,8 @@ dec_term_atom_common: *objp = make_flatmap(mp); for (n = size; n; n--) { - *vptr = (Eterm) COMPRESS_POINTER(next); - *kptr = (Eterm) COMPRESS_POINTER(vptr); + *vptr = (Eterm) next; + *kptr = (Eterm) vptr; next = kptr; vptr--; kptr--; @@ -3599,8 +3599,8 @@ dec_term_atom_common: hamt->leaf_array = hp; for (n = size; n; n--) { - CDR(hp) = (Eterm) COMPRESS_POINTER(next); - CAR(hp) = (Eterm) COMPRESS_POINTER(&CDR(hp)); + CDR(hp) = (Eterm) next; + CAR(hp) = (Eterm) &CDR(hp); next = &CAR(hp); hp += 2; } @@ -3677,11 +3677,11 @@ dec_term_atom_common: /* Environment */ for (i = num_free-1; i >= 0; i--) { - funp->env[i] = (Eterm) COMPRESS_POINTER(next); + funp->env[i] = (Eterm) next; next = funp->env + i; } /* Creator */ - funp->creator = (Eterm) COMPRESS_POINTER(next); + funp->creator = (Eterm) next; next = &(funp->creator); break; } @@ -3750,7 +3750,7 @@ dec_term_atom_common: /* Environment */ for (i = num_free-1; i >= 0; i--) { - funp->env[i] = (Eterm) COMPRESS_POINTER(next); + funp->env[i] = (Eterm) next; next = funp->env + i; } break; @@ -3846,11 +3846,11 @@ dec_term_atom_common: */ while (maps_list) { - next = (Eterm *)(EXPAND_POINTER(*maps_list)); + next = (Eterm *) *maps_list; *maps_list = MAP_HEADER_FLATMAP; if (!erts_validate_and_sort_flatmap((flatmap_t*)maps_list)) goto error; - maps_list = next; + maps_list = next; } ASSERT(hp <= factory->hp_end @@ -3876,7 +3876,7 @@ dec_term_atom_common: PSTACK_DESTROY(hamt_array); } - ASSERT((Eterm*)EXPAND_POINTER(*dbg_resultp) != NULL); + ASSERT((Eterm*)*dbg_resultp != NULL); if (ctx) { ctx->state = B2TDone; -- cgit v1.2.3 From 7dd3fcb287c6d307ce28800d79d1e4c86a533952 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Fri, 20 Nov 2015 16:07:41 +0100 Subject: erts: Fix maps decode in erlang:binary_to_term/1 Decoding a term with a large (HAMT) map in an small (FLAT) map could cause a critical error if the external format was not produced by beam. --- erts/emulator/beam/external.c | 76 +++++++++++++++++++++---------------------- 1 file changed, 37 insertions(+), 39 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index c6d7e3fcc5..a85aa15403 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1179,7 +1179,7 @@ typedef struct { ErtsHeapFactory factory; int remaining_n; char* remaining_bytes; - Eterm* maps_list; + ErtsWStack flat_maps; ErtsPStack hamt_array; } B2TDecodeContext; @@ -1519,7 +1519,7 @@ static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binar ctx->u.dc.res = (Eterm) (UWord) NULL; ctx->u.dc.next = &ctx->u.dc.res; erts_factory_proc_prealloc_init(&ctx->u.dc.factory, p, ctx->heap_size); - ctx->u.dc.maps_list = NULL; + ctx->u.dc.flat_maps.wstart = NULL; ctx->u.dc.hamt_array.pstart = NULL; ctx->state = B2TDecode; /*fall through*/ @@ -2938,7 +2938,7 @@ dec_term(ErtsDistExternal *edep, int n; ErtsAtomEncoding char_enc; register Eterm* hp; /* Please don't take the address of hp */ - Eterm *maps_list; /* for preprocessing of small maps */ + DECLARE_WSTACK(flat_maps); /* for preprocessing of small maps */ Eterm* next; SWord reds; #ifdef DEBUG @@ -2950,7 +2950,6 @@ dec_term(ErtsDistExternal *edep, next = ctx->u.dc.next; ep = ctx->u.dc.ep; factory = &ctx->u.dc.factory; - maps_list = ctx->u.dc.maps_list; if (ctx->state != B2TDecode) { int n_limit = reds; @@ -3026,15 +3025,18 @@ dec_term(ErtsDistExternal *edep, } } PSTACK_CHANGE_ALLOCATOR(hamt_array, ERTS_ALC_T_SAVED_ESTACK); + WSTACK_CHANGE_ALLOCATOR(flat_maps, ERTS_ALC_T_SAVED_ESTACK); if (ctx->u.dc.hamt_array.pstart) { PSTACK_RESTORE(hamt_array, &ctx->u.dc.hamt_array); } + if (ctx->u.dc.flat_maps.wstart) { + WSTACK_RESTORE(flat_maps, &ctx->u.dc.flat_maps); + } } else { reds = ERTS_SWORD_MAX; next = objp; *next = (Eterm) (UWord) NULL; - maps_list = NULL; } hp = factory->hp; @@ -3595,14 +3597,8 @@ dec_term_atom_common: * vptr, last word for values */ - /* - * Use thing_word to link through decoded maps. - * The list of maps is for later validation. - */ - - mp->thing_word = (Eterm) COMPRESS_POINTER(maps_list); - maps_list = (Eterm *) mp; - + WSTACK_PUSH(flat_maps, (UWord)mp); + mp->thing_word = MAP_HEADER_FLATMAP; mp->size = size; mp->keys = keys; *objp = make_flatmap(mp); @@ -3851,7 +3847,9 @@ dec_term_atom_common: ctx->u.dc.ep = ep; ctx->u.dc.next = next; ctx->u.dc.factory.hp = hp; - ctx->u.dc.maps_list = maps_list; + if (!WSTACK_ISEMPTY(flat_maps)) { + WSTACK_SAVE(flat_maps, &ctx->u.dc.flat_maps); + } if (!PSTACK_IS_EMPTY(hamt_array)) { PSTACK_SAVE(hamt_array, &ctx->u.dc.hamt_array); } @@ -3865,18 +3863,6 @@ dec_term_atom_common: } } - /* Iterate through all the maps and check for validity and sort keys - * - done here for when we know it is complete. - */ - - while (maps_list) { - next = (Eterm *)(EXPAND_POINTER(*maps_list)); - *maps_list = MAP_HEADER_FLATMAP; - if (!erts_validate_and_sort_flatmap((flatmap_t*)maps_list)) - goto error; - maps_list = next; - } - ASSERT(hp <= factory->hp_end || (factory->mode == FACTORY_CLOSED && is_immed(*dbg_resultp))); factory->hp = hp; @@ -3885,20 +3871,31 @@ dec_term_atom_common: */ if (!PSTACK_IS_EMPTY(hamt_array)) { - do { - struct dec_term_hamt* hamt = PSTACK_TOP(hamt_array); - - *hamt->objp = erts_hashmap_from_array(factory, - hamt->leaf_array, - hamt->size, - 1); - if (is_non_value(*hamt->objp)) - goto error_hamt; - - (void) PSTACK_POP(hamt_array); - } while (!PSTACK_IS_EMPTY(hamt_array)); - PSTACK_DESTROY(hamt_array); + do { + struct dec_term_hamt* hamt = PSTACK_TOP(hamt_array); + + *hamt->objp = erts_hashmap_from_array(factory, + hamt->leaf_array, + hamt->size, + 1); + if (is_non_value(*hamt->objp)) + goto error_hamt; + + (void) PSTACK_POP(hamt_array); + } while (!PSTACK_IS_EMPTY(hamt_array)); + PSTACK_DESTROY(hamt_array); + } + + /* Iterate through all the (flat)maps and check for validity and sort keys + * - done here for when we know it is complete. + */ + + while(!WSTACK_ISEMPTY(flat_maps)) { + next = (Eterm *)WSTACK_POP(flat_maps); + if (!erts_validate_and_sort_flatmap((flatmap_t*)next)) + goto error; } + WSTACK_DESTROY(flat_maps); ASSERT((Eterm*)EXPAND_POINTER(*dbg_resultp) != NULL); @@ -3924,6 +3921,7 @@ error_hamt: ctx->state = B2TDecodeFail; ctx->reds = reds; } + WSTACK_DESTROY(flat_maps); return NULL; } -- cgit v1.2.3 From 56090db3ea417157a749bdd810fc61d117493f1f Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 23 Feb 2016 19:45:01 +0100 Subject: erts: Change erl_exit into erts_exit This is mostly a pure refactoring. Except for the buggy cases when calling erlang:halt() with a positive integer in the range -(INT_MIN+2) to -INT_MIN that got confused with ERTS_ABORT_EXIT, ERTS_DUMP_EXIT and ERTS_INTR_EXIT. Outcome OLD erl_exit(n, ) NEW erts_exit(n, ) ------- ------------------- ------------------------------------------- exit(Status) n = -Status <= 0 n = Status >= 0 crashdump+abort n > 0, ignore n n = ERTS_ERROR_EXIT < 0 The outcome of the old ERTS_ABORT_EXIT, ERTS_INTR_EXIT and ERTS_DUMP_EXIT are the same as before (even though their values have changed). --- erts/emulator/beam/external.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index a85aa15403..ffe3303796 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -570,7 +570,7 @@ void erts_encode_ext(Eterm term, byte **ext) *ep++ = VERSION_MAGIC; ep = enc_term(NULL, term, ep, TERM_TO_BINARY_DFLAGS, NULL); if (!ep) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d:erts_encode_ext(): Internal data structure error\n", __FILE__, __LINE__); *ext = ep; @@ -1559,7 +1559,7 @@ static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binar b2t_destroy_context(ctx); if (ctx->u.dc.factory.hp > ctx->u.dc.factory.hp_end) { - erl_exit(1, ":%s, line %d: heap overrun by %d words(s)\n", + erts_exit(ERTS_ERROR_EXIT, ":%s, line %d: heap overrun by %d words(s)\n", __FILE__, __LINE__, ctx->u.dc.factory.hp - ctx->u.dc.factory.hp_end); } erts_factory_close(&ctx->u.dc.factory); @@ -1708,12 +1708,12 @@ erts_term_to_binary_simple(Process* p, Eterm Term, Uint size, int level, Uint fl if ((endp = enc_term(NULL, Term, bytes, flags, NULL)) == NULL) { - erl_exit(1, "%s, line %d: bad term: %x\n", + erts_exit(ERTS_ERROR_EXIT, "%s, line %d: bad term: %x\n", __FILE__, __LINE__, Term); } real_size = endp - bytes; if (real_size > size) { - erl_exit(1, "%s, line %d: buffer overflow: %d word(s)\n", + erts_exit(ERTS_ERROR_EXIT, "%s, line %d: buffer overflow: %d word(s)\n", __FILE__, __LINE__, real_size - size); } @@ -1753,12 +1753,12 @@ erts_term_to_binary_simple(Process* p, Eterm Term, Uint size, int level, Uint fl bytes[0] = VERSION_MAGIC; if ((endp = enc_term(NULL, Term, bytes+1, flags, NULL)) == NULL) { - erl_exit(1, "%s, line %d: bad term: %x\n", + erts_exit(ERTS_ERROR_EXIT, "%s, line %d: bad term: %x\n", __FILE__, __LINE__, Term); } real_size = endp - bytes; if (real_size > size) { - erl_exit(1, "%s, line %d: buffer overflow: %d word(s)\n", + erts_exit(ERTS_ERROR_EXIT, "%s, line %d: buffer overflow: %d word(s)\n", __FILE__, __LINE__, endp - (bytes + size)); } return erts_realloc_binary(bin, real_size); @@ -2650,7 +2650,7 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, ASSERT(node_sz < 17); break; default: - erl_exit(1, "bad header\r\n"); + erts_exit(ERTS_ERROR_EXIT, "bad header\r\n"); } ptr++; @@ -4110,7 +4110,7 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, ASSERT(node_sz < 17); break; default: - erl_exit(1, "bad header\r\n"); + erts_exit(ERTS_ERROR_EXIT, "bad header\r\n"); } ptr++; @@ -4202,7 +4202,7 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, break; default: - erl_exit(1,"Internal data structure error (in encode_size_struct2)%x\n", + erts_exit(ERTS_ERROR_EXIT,"Internal data structure error (in encode_size_struct2)%x\n", obj); } -- cgit v1.2.3 From 6664eed554974336909d3ffe03f20349cc4c38fd Mon Sep 17 00:00:00 2001 From: Henrik Nord Date: Tue, 15 Mar 2016 15:19:56 +0100 Subject: update copyright-year --- erts/emulator/beam/external.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index aac1490f0c..6bb62d1040 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2014. All Rights Reserved. + * Copyright Ericsson AB 1996-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. -- cgit v1.2.3 From a2a86dadc648dda68b5221a7c1d83b9238be1e25 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 12 Feb 2016 18:52:20 +0100 Subject: erts: Improve enif_binary_to_term * Accept a raw data buffer instead of ErlNifBinary * Accept option ERL_NIF_BIN2TERM_SAFE * Return number of read bytes --- erts/emulator/beam/external.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index aac1490f0c..f5eb13421d 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -967,15 +967,23 @@ erts_decode_dist_ext(ErtsHeapFactory* factory, return THE_NON_VALUE; } -Eterm erts_decode_ext(ErtsHeapFactory* factory, byte **ext) +Eterm erts_decode_ext(ErtsHeapFactory* factory, byte **ext, Uint32 flags) { + ErtsDistExternal ede, *edep; Eterm obj; byte *ep = *ext; if (*ep++ != VERSION_MAGIC) { erts_factory_undo(factory); return THE_NON_VALUE; } - ep = dec_term(NULL, factory, ep, &obj, NULL); + if (flags) { + ASSERT(flags == ERTS_DIST_EXT_BTT_SAFE); + ede.flags = flags; /* a dummy struct just for the flags */ + edep = &ede; + } else { + edep = NULL; + } + ep = dec_term(edep, factory, ep, &obj, NULL); if (!ep) { #ifdef DEBUG bin_write(ERTS_PRINT_STDERR,NULL,*ext,500); -- cgit v1.2.3 From 043ab9055917a4f6d89f1fa2788079e0618928c4 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Mon, 15 Feb 2016 19:23:12 +0100 Subject: erts: Remove printout when dec_term fails in DEBUG --- erts/emulator/beam/external.c | 3 --- 1 file changed, 3 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index f5eb13421d..10f03636ec 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -985,9 +985,6 @@ Eterm erts_decode_ext(ErtsHeapFactory* factory, byte **ext, Uint32 flags) } ep = dec_term(edep, factory, ep, &obj, NULL); if (!ep) { -#ifdef DEBUG - bin_write(ERTS_PRINT_STDERR,NULL,*ext,500); -#endif return THE_NON_VALUE; } *ext = ep; -- cgit v1.2.3 From 995f81ed452f95c065bd94d62cb69dbf5289be76 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 11 Nov 2015 16:14:33 +0100 Subject: erts: Refactor ETS compressed encoding of local node Instead of INTERNAL_CREATION (255), use empty atom for node name to mean the local node (regardless of node name or creation). The purpose is to get rid of special value 255, for future expansion of creation to 32-bit. --- erts/emulator/beam/external.c | 43 +++++++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 22 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 10f03636ec..de0c155fc0 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -51,7 +51,7 @@ #define MAX_STRING_LEN 0xffff -#define is_valid_creation(Cre) ((unsigned)(Cre) < MAX_CREATION || (Cre) == INTERNAL_CREATION) +#define is_valid_creation(Cre) ((unsigned)(Cre) < MAX_CREATION) #undef ERTS_DEBUG_USE_DIST_SEP #ifdef DEBUG @@ -2152,12 +2152,12 @@ static byte* enc_pid(ErtsAtomCacheMap *acmp, Eterm pid, byte* ep, Uint32 dflags) { Uint on, os; + Eterm sysname = ((is_internal_pid(pid) && (dflags & DFLAG_INTERNAL_TAGS)) + ? am_Empty : pid_node_name(pid)); *ep++ = PID_EXT; /* insert atom here containing host and sysname */ - ep = enc_atom(acmp, pid_node_name(pid), ep, dflags); - - /* two bytes for each number and serial */ + ep = enc_atom(acmp, sysname, ep, dflags); on = pid_number(pid); os = pid_serial(pid); @@ -2166,8 +2166,7 @@ enc_pid(ErtsAtomCacheMap *acmp, Eterm pid, byte* ep, Uint32 dflags) ep += 4; put_int32(os, ep); ep += 4; - *ep++ = (is_internal_pid(pid) && (dflags & DFLAG_INTERNAL_TAGS)) ? - INTERNAL_CREATION : pid_creation(pid); + *ep++ = pid_creation(pid); return ep; } @@ -2249,14 +2248,13 @@ dec_atom(ErtsDistExternal *edep, byte* ep, Eterm* objp) static ERTS_INLINE ErlNode* dec_get_node(Eterm sysname, Uint creation) { - switch (creation) { - case INTERNAL_CREATION: + if (sysname == am_Empty) /* && DFLAG_INTERNAL_TAGS */ return erts_this_node; - case ORIG_CREATION: - if (sysname == erts_this_node->sysname) { - creation = erts_this_node->creation; - } - } + + if (sysname == erts_this_node->sysname + && (creation == erts_this_node->creation || creation == ORIG_CREATION)) + return erts_this_node; + return erts_find_or_insert_node(sysname,creation); } @@ -2528,6 +2526,8 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, case REF_DEF: case EXTERNAL_REF_DEF: { Uint32 *ref_num; + Eterm sysname = (((dflags & DFLAG_INTERNAL_TAGS) && is_internal_ref(obj)) + ? am_Empty : ref_node_name(obj)); ASSERT(dflags & DFLAG_EXTENDED_REFERENCES); @@ -2535,9 +2535,8 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, i = ref_no_of_numbers(obj); put_int16(i, ep); ep += 2; - ep = enc_atom(acmp,ref_node_name(obj),ep,dflags); - *ep++ = ((dflags & DFLAG_INTERNAL_TAGS) && is_internal_ref(obj)) ? - INTERNAL_CREATION : ref_creation(obj); + ep = enc_atom(acmp, sysname, ep, dflags); + *ep++ = ref_creation(obj); ref_num = ref_numbers(obj); for (j = 0; j < i; j++) { put_int32(ref_num[j], ep); @@ -2546,17 +2545,17 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, break; } case PORT_DEF: - case EXTERNAL_PORT_DEF: - + case EXTERNAL_PORT_DEF: { + Eterm sysname = (((dflags & DFLAG_INTERNAL_TAGS) && is_internal_port(obj)) + ? am_Empty : port_node_name(obj)); *ep++ = PORT_EXT; - ep = enc_atom(acmp,port_node_name(obj),ep,dflags); + ep = enc_atom(acmp, sysname, ep, dflags); j = port_number(obj); put_int32(j, ep); ep += 4; - *ep++ = ((dflags & DFLAG_INTERNAL_TAGS) && is_internal_port(obj)) ? - INTERNAL_CREATION : port_creation(obj); + *ep++ = port_creation(obj); break; - + } case LIST_DEF: { int is_str; -- cgit v1.2.3 From bf0bf9cf009fa8ccc7fc364fdbbdeb6f491efe43 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 13 Nov 2015 16:47:40 +0100 Subject: erts: Support 32-bit creation for external pid,port,refs from future nodes. --- erts/emulator/beam/external.c | 157 +++++++++++++++++++++++++++++++----------- 1 file changed, 118 insertions(+), 39 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index de0c155fc0..5ea155f83f 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -51,7 +51,18 @@ #define MAX_STRING_LEN 0xffff -#define is_valid_creation(Cre) ((unsigned)(Cre) < MAX_CREATION) +/* MAX value for the creation field in pid, port and reference + for the local node and for the current external format. + + Larger creation values than this are allowed in external pid, port and refs + encoded with NEW_PID_EXT, NEW_PORT_EXT and NEWER_REFERENCE_EXT. + The point here is to prepare for future upgrade to 32-bit creation. + OTP-19 (erts-8.0) can handle big creation values from other (newer) nodes, + but do not use big creation values for the local node yet, + as we still may have to communicate with older nodes. +*/ +#define ERTS_MAX_LOCAL_CREATION (3) +#define is_valid_creation(Cre) ((unsigned)(Cre) <= ERTS_MAX_LOCAL_CREATION) #undef ERTS_DEBUG_USE_DIST_SEP #ifdef DEBUG @@ -97,7 +108,7 @@ static byte* enc_pid(ErtsAtomCacheMap *, Eterm, byte*, Uint32); struct B2TContext_t; static byte* dec_term(ErtsDistExternal*, ErtsHeapFactory*, byte*, Eterm*, struct B2TContext_t*); static byte* dec_atom(ErtsDistExternal *, byte*, Eterm*); -static byte* dec_pid(ErtsDistExternal *, ErtsHeapFactory*, byte*, Eterm*); +static byte* dec_pid(ErtsDistExternal *, ErtsHeapFactory*, byte*, Eterm*, byte tag); static Sint decoded_size(byte *ep, byte* endp, int internal_tags, struct B2TContext_t*); static BIF_RETTYPE term_to_binary_trap_1(BIF_ALIST_1); @@ -2154,8 +2165,9 @@ enc_pid(ErtsAtomCacheMap *acmp, Eterm pid, byte* ep, Uint32 dflags) Uint on, os; Eterm sysname = ((is_internal_pid(pid) && (dflags & DFLAG_INTERNAL_TAGS)) ? am_Empty : pid_node_name(pid)); + Uint32 creation = pid_creation(pid); + byte* tagp = ep++; - *ep++ = PID_EXT; /* insert atom here containing host and sysname */ ep = enc_atom(acmp, sysname, ep, dflags); @@ -2166,7 +2178,15 @@ enc_pid(ErtsAtomCacheMap *acmp, Eterm pid, byte* ep, Uint32 dflags) ep += 4; put_int32(os, ep); ep += 4; - *ep++ = pid_creation(pid); + if (creation <= ERTS_MAX_LOCAL_CREATION) { + *tagp = PID_EXT; + *ep++ = creation; + } else { + ASSERT(is_external_pid(pid)); + *tagp = NEW_PID_EXT; + put_int32(creation, ep); + ep += 4; + } return ep; } @@ -2246,7 +2266,7 @@ dec_atom(ErtsDistExternal *edep, byte* ep, Eterm* objp) return ep; } -static ERTS_INLINE ErlNode* dec_get_node(Eterm sysname, Uint creation) +static ERTS_INLINE ErlNode* dec_get_node(Eterm sysname, Uint32 creation) { if (sysname == am_Empty) /* && DFLAG_INTERNAL_TAGS */ return erts_this_node; @@ -2259,13 +2279,14 @@ static ERTS_INLINE ErlNode* dec_get_node(Eterm sysname, Uint creation) } static byte* -dec_pid(ErtsDistExternal *edep, ErtsHeapFactory* factory, byte* ep, Eterm* objp) +dec_pid(ErtsDistExternal *edep, ErtsHeapFactory* factory, byte* ep, + Eterm* objp, byte tag) { Eterm sysname; Uint data; Uint num; Uint ser; - Uint cre; + Uint32 cre; ErlNode *node; *objp = NIL; /* In case we fail, don't leave a hole in the heap */ @@ -2281,12 +2302,19 @@ dec_pid(ErtsDistExternal *edep, ErtsHeapFactory* factory, byte* ep, Eterm* objp) ep += 4; if (ser > ERTS_MAX_PID_SERIAL) return NULL; - cre = get_int8(ep); - ep += 1; - if (!is_valid_creation(cre)) { - return NULL; + if (tag == PID_EXT) { + cre = get_int8(ep); + ep += 1; + if (!is_valid_creation(cre)) { + return NULL; + } + } else { + ASSERT(tag == NEW_PID_EXT); + cre = get_int32(ep); + ep += 4; } + data = make_pid_data(ser, num); /* @@ -2528,15 +2556,24 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 *ref_num; Eterm sysname = (((dflags & DFLAG_INTERNAL_TAGS) && is_internal_ref(obj)) ? am_Empty : ref_node_name(obj)); + Uint32 creation = ref_creation(obj); + byte* tagp = ep++; ASSERT(dflags & DFLAG_EXTENDED_REFERENCES); - *ep++ = NEW_REFERENCE_EXT; i = ref_no_of_numbers(obj); put_int16(i, ep); ep += 2; ep = enc_atom(acmp, sysname, ep, dflags); - *ep++ = ref_creation(obj); + if (creation <= ERTS_MAX_LOCAL_CREATION) { + *tagp = NEW_REFERENCE_EXT; + *ep++ = creation; + } else { + ASSERT(is_external_ref(obj)); + *tagp = NEWER_REFERENCE_EXT; + put_int32(creation, ep); + ep += 4; + } ref_num = ref_numbers(obj); for (j = 0; j < i; j++) { put_int32(ref_num[j], ep); @@ -2548,12 +2585,22 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, case EXTERNAL_PORT_DEF: { Eterm sysname = (((dflags & DFLAG_INTERNAL_TAGS) && is_internal_port(obj)) ? am_Empty : port_node_name(obj)); - *ep++ = PORT_EXT; + Uint32 creation = port_creation(obj); + byte* tagp = ep++; + ep = enc_atom(acmp, sysname, ep, dflags); j = port_number(obj); put_int32(j, ep); ep += 4; - *ep++ = port_creation(obj); + if (creation <= ERTS_MAX_LOCAL_CREATION) { + *tagp = PORT_EXT; + *ep++ = creation; + } else { + ASSERT(is_external_port(obj)); + *tagp = NEW_PORT_EXT; + put_int32(creation, ep); + ep += 4; + } break; } case LIST_DEF: @@ -3259,20 +3306,23 @@ dec_term_atom_common: hp += FLOAT_SIZE_OBJECT; break; } - case PID_EXT: + case PID_EXT: + case NEW_PID_EXT: factory->hp = hp; - ep = dec_pid(edep, factory, ep, objp); + ep = dec_pid(edep, factory, ep, objp, ep[-1]); hp = factory->hp; if (ep == NULL) { goto error; } break; - case PORT_EXT: + case PORT_EXT: + case NEW_PORT_EXT: { Eterm sysname; ErlNode *node; Uint num; - Uint cre; + Uint32 cre; + byte tag = ep[-1]; if ((ep = dec_atom(edep, ep, &sysname)) == NULL) { goto error; @@ -3281,12 +3331,17 @@ dec_term_atom_common: goto error; } ep += 4; - cre = get_int8(ep); - ep++; - if (!is_valid_creation(cre)) { - goto error; - } - + if (tag == PORT_EXT) { + cre = get_int8(ep); + ep++; + if (!is_valid_creation(cre)) { + goto error; + } + } + else { + cre = get_int32(ep); + ep += 4; + } node = dec_get_node(sysname, cre); if(node == erts_this_node) { *objp = make_internal_port(num); @@ -3311,7 +3366,7 @@ dec_term_atom_common: Eterm sysname; ErlNode *node; int i; - Uint cre; + Uint32 cre; Uint32 *ref_num; Uint32 r0; Uint ref_words; @@ -3335,9 +3390,6 @@ dec_term_atom_common: ref_words = get_int16(ep); ep += 2; - if (ref_words > ERTS_MAX_REF_NUMBERS) - goto error; - if ((ep = dec_atom(edep, ep, &sysname)) == NULL) goto error; @@ -3350,8 +3402,23 @@ dec_term_atom_common: ep += 4; if (r0 >= MAX_REFERENCE) goto error; + goto ref_ext_common; + + case NEWER_REFERENCE_EXT: + ref_words = get_int16(ep); + ep += 2; + + if ((ep = dec_atom(edep, ep, &sysname)) == NULL) + goto error; + + cre = get_int32(ep); + ep += 4; + r0 = get_int32(ep); /* allow full word */ + ep += 4; ref_ext_common: + if (ref_words > ERTS_MAX_REF_NUMBERS) + goto error; node = dec_get_node(sysname, cre); if(node == erts_this_node) { @@ -3705,9 +3772,9 @@ dec_term_atom_common: *objp = make_fun(funp); /* Creator pid */ - if (*ep != PID_EXT - || (ep = dec_pid(edep, factory, ++ep, - &funp->creator))==NULL) { + if ((*ep != PID_EXT && *ep != NEW_PID_EXT) + || (ep = dec_pid(edep, factory, ep+1, + &funp->creator, *ep))==NULL) { goto error; } @@ -4009,20 +4076,29 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, else result += 1 + 4 + 1 + i; /* tag,size,sign,digits */ break; + case EXTERNAL_PID_DEF: + if (external_pid_creation(obj) > ERTS_MAX_LOCAL_CREATION) + result += 3; + /*fall through*/ case PID_DEF: - case EXTERNAL_PID_DEF: result += (1 + encode_size_struct2(acmp, pid_node_name(obj), dflags) + 4 + 4 + 1); break; + case EXTERNAL_REF_DEF: + if (external_ref_creation(obj) > ERTS_MAX_LOCAL_CREATION) + result += 3; + /*fall through*/ case REF_DEF: - case EXTERNAL_REF_DEF: ASSERT(dflags & DFLAG_EXTENDED_REFERENCES); i = ref_no_of_numbers(obj); result += (1 + 2 + encode_size_struct2(acmp, ref_node_name(obj), dflags) + 1 + 4*i); break; - case PORT_DEF: - case EXTERNAL_PORT_DEF: + case EXTERNAL_PORT_DEF: + if (external_port_creation(obj) > ERTS_MAX_LOCAL_CREATION) + result += 3; + /*fall through*/ + case PORT_DEF: result += (1 + encode_size_struct2(acmp, port_node_name(obj), dflags) + 4 + 1); break; @@ -4349,19 +4425,22 @@ init_done: SKIP(1+atom_extra_skip); atom_extra_skip = 0; break; - case PID_EXT: + case PID_EXT: + case NEW_PID_EXT: atom_extra_skip = 9; /* In case it is an external pid */ heap_size += EXTERNAL_THING_HEAD_SIZE + 1; terms++; break; - case PORT_EXT: + case PORT_EXT: + case NEW_PORT_EXT: atom_extra_skip = 5; /* In case it is an external port */ heap_size += EXTERNAL_THING_HEAD_SIZE + 1; terms++; break; - case NEW_REFERENCE_EXT: + case NEW_REFERENCE_EXT: + case NEWER_REFERENCE_EXT: { int id_words; -- cgit v1.2.3 From 350c2beb2ed583aa6b0345e8118615bd3c5ccc9d Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 13 Apr 2016 20:02:21 +0200 Subject: erts: Fix decoding of pids, ports and refs with big creations Must skip 3 extra bytes after node name atom. --- erts/emulator/beam/external.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 5ea155f83f..243ba8d9f3 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -4425,22 +4425,32 @@ init_done: SKIP(1+atom_extra_skip); atom_extra_skip = 0; break; - case PID_EXT: case NEW_PID_EXT: + atom_extra_skip = 12; + goto case_PID; + case PID_EXT: atom_extra_skip = 9; + case_PID: /* In case it is an external pid */ heap_size += EXTERNAL_THING_HEAD_SIZE + 1; terms++; break; - case PORT_EXT: case NEW_PORT_EXT: + atom_extra_skip = 8; + goto case_PORT; + case PORT_EXT: atom_extra_skip = 5; + case_PORT: /* In case it is an external port */ heap_size += EXTERNAL_THING_HEAD_SIZE + 1; terms++; break; - case NEW_REFERENCE_EXT: case NEWER_REFERENCE_EXT: + atom_extra_skip = 4; + goto case_NEW_REFERENCE; + case NEW_REFERENCE_EXT: + atom_extra_skip = 1; + case_NEW_REFERENCE: { int id_words; @@ -4451,7 +4461,7 @@ init_done: goto error; ep += 2; - atom_extra_skip = 1 + 4*id_words; + atom_extra_skip += 4*id_words; /* In case it is an external ref */ #if defined(ARCH_64) heap_size += EXTERNAL_THING_HEAD_SIZE + id_words/2 + 1; -- cgit v1.2.3 From d698386e5b101a02786c85af70f1513c6beb191b Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 3 Mar 2016 16:33:22 +0100 Subject: erts: Silence harmless valgrind warning in dec_term provoked by nif_SUITE:nif_binary_to_term. If we fail to decode an immediate (unsafe atom for example) with a dummy factory then hp and factory->hp will both be uninitialized and valgrind will complain about comparing them. --- erts/emulator/beam/external.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 9f43240b7e..723c25ff77 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -3958,9 +3958,13 @@ error: * Must unlink all off-heap objects that may have been * linked into the process. */ - if (factory->hp < hp) { /* Sometimes we used hp and sometimes factory->hp */ - factory->hp = hp; /* the largest must be the freshest */ + if (factory->mode != FACTORY_CLOSED) { + if (factory->hp < hp) { /* Sometimes we used hp and sometimes factory->hp */ + factory->hp = hp; /* the largest must be the freshest */ + } } + else ASSERT(factory->hp == hp); + error_hamt: erts_factory_undo(factory); PSTACK_DESTROY(hamt_array); -- cgit v1.2.3 From e3a07b08709bd7eacae0bcb0353c347475134ad7 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 15 Jun 2016 16:32:16 +0200 Subject: erts: Change local sysname for ETS compressed Yes this is an ugly workaround. One approach for a better solution could be to introduce an internal secret atom tagged as an atom with a unique index, but impossible to find by string hash lookup/insert. --- erts/emulator/beam/external.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) (limited to 'erts/emulator/beam/external.c') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 3c002d43a7..beed847578 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -2159,12 +2159,23 @@ enc_atom(ErtsAtomCacheMap *acmp, Eterm atom, byte *ep, Uint32 dflags) return ep; } +/* + * We use this atom as sysname in local pid/port/refs + * for the ETS compressed format (DFLAG_INTERNAL_TAGS). + * + * We used atom '' earlier but that turned out to cause problems + * for buggy erl_interface/ic usage of c-nodes with empty node names. + * A long atom reduces risk of nodes actually called this and the length + * does not matter anyway as it's encoded with atom index (ATOM_INTERNAL_REF2). + */ +#define INTERNAL_LOCAL_SYSNAME am_await_microstate_accounting_modifications + static byte* enc_pid(ErtsAtomCacheMap *acmp, Eterm pid, byte* ep, Uint32 dflags) { Uint on, os; Eterm sysname = ((is_internal_pid(pid) && (dflags & DFLAG_INTERNAL_TAGS)) - ? am_Empty : pid_node_name(pid)); + ? INTERNAL_LOCAL_SYSNAME : pid_node_name(pid)); Uint32 creation = pid_creation(pid); byte* tagp = ep++; @@ -2268,7 +2279,7 @@ dec_atom(ErtsDistExternal *edep, byte* ep, Eterm* objp) static ERTS_INLINE ErlNode* dec_get_node(Eterm sysname, Uint32 creation) { - if (sysname == am_Empty) /* && DFLAG_INTERNAL_TAGS */ + if (sysname == INTERNAL_LOCAL_SYSNAME) /* && DFLAG_INTERNAL_TAGS */ return erts_this_node; if (sysname == erts_this_node->sysname @@ -2555,7 +2566,7 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, case EXTERNAL_REF_DEF: { Uint32 *ref_num; Eterm sysname = (((dflags & DFLAG_INTERNAL_TAGS) && is_internal_ref(obj)) - ? am_Empty : ref_node_name(obj)); + ? INTERNAL_LOCAL_SYSNAME : ref_node_name(obj)); Uint32 creation = ref_creation(obj); byte* tagp = ep++; @@ -2584,7 +2595,7 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, case PORT_DEF: case EXTERNAL_PORT_DEF: { Eterm sysname = (((dflags & DFLAG_INTERNAL_TAGS) && is_internal_port(obj)) - ? am_Empty : port_node_name(obj)); + ? INTERNAL_LOCAL_SYSNAME : port_node_name(obj)); Uint32 creation = port_creation(obj); byte* tagp = ep++; -- cgit v1.2.3