aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator/beam/external.c
diff options
context:
space:
mode:
Diffstat (limited to 'erts/emulator/beam/external.c')
-rw-r--r--erts/emulator/beam/external.c1929
1 files changed, 1536 insertions, 393 deletions
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 5ce0d97c74..ffe3303796 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1,18 +1,19 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 1996-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2014. All Rights Reserved.
*
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*
* %CopyrightEnd%
*/
@@ -36,16 +37,16 @@
#include "erl_process.h"
#include "error.h"
#include "external.h"
+#define ERL_WANT_HIPE_BIF_WRAPPER__
#include "bif.h"
+#undef ERL_WANT_HIPE_BIF_WRAPPER__
#include "big.h"
#include "dist.h"
#include "erl_binary.h"
#include "erl_bits.h"
#include "erl_zlib.h"
+#include "erl_map.h"
-#ifdef HIPE
-#include "hipe_mode_switch.h"
-#endif
#define in_area(ptr,start,nbytes) ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes))
#define MAX_STRING_LEN 0xffff
@@ -61,6 +62,9 @@
*/
# define ERTS_DEBUG_USE_DIST_SEP
# endif
+# define IF_DEBUG(X) X
+#else
+# define IF_DEBUG(X)
#endif
/* Does Sint fit in Sint32?
@@ -81,17 +85,45 @@
*
*/
+static Export term_to_binary_trap_export;
+
static byte* enc_term(ErtsAtomCacheMap *, Eterm, byte*, Uint32, struct erl_off_heap_header** off_heap);
+struct TTBEncodeContext_;
+static int enc_term_int(struct TTBEncodeContext_*,ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags,
+ struct erl_off_heap_header** off_heap, Sint *reds, byte **res);
static Uint is_external_string(Eterm obj, int* p_is_string);
static byte* enc_atom(ErtsAtomCacheMap *, Eterm, byte*, Uint32);
static byte* enc_pid(ErtsAtomCacheMap *, Eterm, byte*, Uint32);
-static byte* dec_term(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*);
+struct B2TContext_t;
+static byte* dec_term(ErtsDistExternal*, ErtsHeapFactory*, byte*, Eterm*, struct B2TContext_t*);
static byte* dec_atom(ErtsDistExternal *, byte*, Eterm*);
-static byte* dec_pid(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*);
-static Sint decoded_size(byte *ep, byte* endp, int internal_tags);
+static byte* dec_pid(ErtsDistExternal *, ErtsHeapFactory*, byte*, Eterm*);
+static Sint decoded_size(byte *ep, byte* endp, int internal_tags, struct B2TContext_t*);
+static BIF_RETTYPE term_to_binary_trap_1(BIF_ALIST_1);
+static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint flags,
+ Binary *context_b);
static Uint encode_size_struct2(ErtsAtomCacheMap *, Eterm, unsigned);
+struct TTBSizeContext_;
+static int encode_size_struct_int(struct TTBSizeContext_*, ErtsAtomCacheMap *acmp, Eterm obj,
+ unsigned dflags, Sint *reds, Uint *res);
+
+static Export binary_to_term_trap_export;
+static BIF_RETTYPE binary_to_term_trap_1(BIF_ALIST_1);
+static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* context_b,
+ Export *bif, Eterm arg0, Eterm arg1);
+
+void erts_init_external(void) {
+ erts_init_trap_export(&term_to_binary_trap_export,
+ am_erts_internal, am_term_to_binary_trap, 1,
+ &term_to_binary_trap_1);
+
+ erts_init_trap_export(&binary_to_term_trap_export,
+ am_erts_internal, am_binary_to_term_trap, 1,
+ &binary_to_term_trap_1);
+ return;
+}
#define ERTS_MAX_INTERNAL_ATOM_CACHE_ENTRIES 255
@@ -469,15 +501,37 @@ byte *erts_encode_ext_dist_header_finalize(byte *ext, ErtsAtomCache *cache, Uint
return ep;
}
-Uint erts_encode_dist_ext_size(Eterm term, Uint32 flags, ErtsAtomCacheMap *acmp)
+int erts_encode_dist_ext_size(Eterm term, Uint32 flags, ErtsAtomCacheMap *acmp,
+ Uint* szp)
{
- Uint sz = 0;
+ Uint sz;
+ if (encode_size_struct_int(NULL, acmp, term, flags, NULL, &sz)) {
+ return -1;
+ } else {
#ifndef ERTS_DEBUG_USE_DIST_SEP
- if (!(flags & DFLAG_DIST_HDR_ATOM_CACHE))
+ if (!(flags & DFLAG_DIST_HDR_ATOM_CACHE))
#endif
- sz++ /* VERSION_MAGIC */;
- sz += encode_size_struct2(acmp, term, flags);
- return sz;
+ sz++ /* VERSION_MAGIC */;
+
+ *szp += sz;
+ return 0;
+ }
+}
+
+int erts_encode_dist_ext_size_int(Eterm term, struct erts_dsig_send_context* ctx, Uint* szp)
+{
+ Uint sz;
+ if (encode_size_struct_int(&ctx->u.sc, ctx->acmp, term, ctx->flags, &ctx->reds, &sz)) {
+ return -1;
+ } else {
+#ifndef ERTS_DEBUG_USE_DIST_SEP
+ if (!(ctx->flags & DFLAG_DIST_HDR_ATOM_CACHE))
+#endif
+ sz++ /* VERSION_MAGIC */;
+
+ *szp += sz;
+ return 0;
+ }
}
Uint erts_encode_ext_size(Eterm term)
@@ -488,7 +542,7 @@ Uint erts_encode_ext_size(Eterm term)
Uint erts_encode_ext_size_2(Eterm term, unsigned dflags)
{
- return encode_size_struct2(NULL, term, TERM_TO_BINARY_DFLAGS|dflags)
+ return encode_size_struct2(NULL, term, dflags)
+ 1 /* VERSION_MAGIC */;
}
@@ -498,19 +552,16 @@ Uint erts_encode_ext_size_ets(Eterm term)
}
-void erts_encode_dist_ext(Eterm term, byte **ext, Uint32 flags, ErtsAtomCacheMap *acmp)
+int erts_encode_dist_ext(Eterm term, byte **ext, Uint32 flags, ErtsAtomCacheMap *acmp,
+ TTBEncodeContext* ctx, Sint* reds)
{
- byte *ep = *ext;
-#ifndef ERTS_DEBUG_USE_DIST_SEP
- if (!(flags & DFLAG_DIST_HDR_ATOM_CACHE))
-#endif
- *ep++ = VERSION_MAGIC;
- ep = enc_term(acmp, term, ep, flags, NULL);
- if (!ep)
- erl_exit(ERTS_ABORT_EXIT,
- "%s:%d:erts_encode_dist_ext(): Internal data structure error\n",
- __FILE__, __LINE__);
- *ext = ep;
+ if (!ctx || !ctx->wstack.wstart) {
+ #ifndef ERTS_DEBUG_USE_DIST_SEP
+ if (!(flags & DFLAG_DIST_HDR_ATOM_CACHE))
+ #endif
+ *(*ext)++ = VERSION_MAGIC;
+ }
+ return enc_term_int(ctx, acmp, term, *ext, flags, NULL, reds, ext);
}
void erts_encode_ext(Eterm term, byte **ext)
@@ -519,7 +570,7 @@ void erts_encode_ext(Eterm term, byte **ext)
*ep++ = VERSION_MAGIC;
ep = enc_term(NULL, term, ep, TERM_TO_BINARY_DFLAGS, NULL);
if (!ep)
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"%s:%d:erts_encode_ext(): Internal data structure error\n",
__FILE__, __LINE__);
*ext = ep;
@@ -851,7 +902,7 @@ erts_decode_dist_ext_size(ErtsDistExternal *edep)
goto fail;
ep = edep->extp+1;
}
- res = decoded_size(ep, edep->ext_endp, 0);
+ res = decoded_size(ep, edep->ext_endp, 0, NULL);
if (res >= 0)
return res;
fail:
@@ -863,12 +914,12 @@ Sint erts_decode_ext_size(byte *ext, Uint size)
{
if (size == 0 || *ext != VERSION_MAGIC)
return -1;
- return decoded_size(ext+1, ext+size, 0);
+ return decoded_size(ext+1, ext+size, 0, NULL);
}
Sint erts_decode_ext_size_ets(byte *ext, Uint size)
{
- Sint sz = decoded_size(ext, ext+size, 1);
+ Sint sz = decoded_size(ext, ext+size, 1, NULL);
ASSERT(sz >= 0);
return sz;
}
@@ -880,8 +931,7 @@ Sint erts_decode_ext_size_ets(byte *ext, Uint size)
** on return hpp is updated to point after allocated data
*/
Eterm
-erts_decode_dist_ext(Eterm** hpp,
- ErlOffHeap* off_heap,
+erts_decode_dist_ext(ErtsHeapFactory* factory,
ErtsDistExternal *edep)
{
Eterm obj;
@@ -901,7 +951,7 @@ erts_decode_dist_ext(Eterm** hpp,
goto error;
ep++;
}
- ep = dec_term(edep, hpp, ep, off_heap, &obj);
+ ep = dec_term(edep, factory, ep, &obj, NULL);
if (!ep)
goto error;
@@ -910,19 +960,22 @@ erts_decode_dist_ext(Eterm** hpp,
return obj;
error:
+ erts_factory_undo(factory);
bad_dist_ext(edep);
return THE_NON_VALUE;
}
-Eterm erts_decode_ext(Eterm **hpp, ErlOffHeap *off_heap, byte **ext)
+Eterm erts_decode_ext(ErtsHeapFactory* factory, byte **ext)
{
Eterm obj;
byte *ep = *ext;
- if (*ep++ != VERSION_MAGIC)
+ if (*ep++ != VERSION_MAGIC) {
+ erts_factory_undo(factory);
return THE_NON_VALUE;
- ep = dec_term(NULL, hpp, ep, off_heap, &obj);
+ }
+ ep = dec_term(NULL, factory, ep, &obj, NULL);
if (!ep) {
#ifdef DEBUG
bin_write(ERTS_PRINT_STDERR,NULL,*ext,500);
@@ -933,10 +986,10 @@ Eterm erts_decode_ext(Eterm **hpp, ErlOffHeap *off_heap, byte **ext)
return obj;
}
-Eterm erts_decode_ext_ets(Eterm **hpp, ErlOffHeap *off_heap, byte *ext)
+Eterm erts_decode_ext_ets(ErtsHeapFactory* factory, byte *ext)
{
Eterm obj;
- ext = dec_term(NULL, hpp, ext, off_heap, &obj);
+ ext = dec_term(NULL, factory, ext, &obj, NULL);
ASSERT(ext);
return obj;
}
@@ -945,9 +998,8 @@ Eterm erts_decode_ext_ets(Eterm **hpp, ErlOffHeap *off_heap, byte *ext)
BIF_RETTYPE erts_debug_dist_ext_to_term_2(BIF_ALIST_2)
{
+ ErtsHeapFactory factory;
Eterm res;
- Eterm *hp;
- Eterm *hendp;
Sint hsz;
ErtsDistExternal ede;
Eterm *tp;
@@ -994,12 +1046,9 @@ BIF_RETTYPE erts_debug_dist_ext_to_term_2(BIF_ALIST_2)
if (hsz < 0)
goto badarg;
- hp = HAlloc(BIF_P, (Uint) hsz);
- hendp = hp + hsz;
-
- res = erts_decode_dist_ext(&hp, &MSO(BIF_P), &ede);
-
- HRelease(BIF_P, hendp, hp);
+ erts_factory_proc_prealloc_init(&factory, BIF_P, hsz);
+ res = erts_decode_dist_ext(&factory, &ede);
+ erts_factory_close(&factory);
if (is_value(res))
BIF_RET(res);
@@ -1009,12 +1058,41 @@ BIF_RETTYPE erts_debug_dist_ext_to_term_2(BIF_ALIST_2)
BIF_ERROR(BIF_P, BADARG);
}
+static BIF_RETTYPE term_to_binary_trap_1(BIF_ALIST_1)
+{
+ Eterm *tp = tuple_val(BIF_ARG_1);
+ Eterm Term = tp[1];
+ Eterm bt = tp[2];
+ Binary *bin = ((ProcBin *) binary_val(bt))->val;
+ Eterm res = erts_term_to_binary_int(BIF_P, Term, 0, 0,bin);
+ if (is_tuple(res)) {
+ ASSERT(BIF_P->flags & F_DISABLE_GC);
+ BIF_TRAP1(&term_to_binary_trap_export,BIF_P,res);
+ } else {
+ if (erts_set_gc_state(BIF_P, 1)
+ || MSO(BIF_P).overhead > BIN_VHEAP_SZ(BIF_P))
+ ERTS_BIF_YIELD_RETURN(BIF_P, res);
+ else
+ BIF_RET(res);
+ }
+}
+
+HIPE_WRAPPER_BIF_DISABLE_GC(term_to_binary, 1)
BIF_RETTYPE term_to_binary_1(BIF_ALIST_1)
{
- return erts_term_to_binary(BIF_P, BIF_ARG_1, 0, TERM_TO_BINARY_DFLAGS);
+ Eterm res = erts_term_to_binary_int(BIF_P, BIF_ARG_1, 0, TERM_TO_BINARY_DFLAGS, NULL);
+ if (is_tuple(res)) {
+ erts_set_gc_state(BIF_P, 0);
+ BIF_TRAP1(&term_to_binary_trap_export,BIF_P,res);
+ } else {
+ ASSERT(!(BIF_P->flags & F_DISABLE_GC));
+ BIF_RET(res);
+ }
}
+HIPE_WRAPPER_BIF_DISABLE_GC(term_to_binary, 2)
+
BIF_RETTYPE term_to_binary_2(BIF_ALIST_2)
{
Process* p = BIF_P;
@@ -1022,6 +1100,7 @@ BIF_RETTYPE term_to_binary_2(BIF_ALIST_2)
Eterm Flags = BIF_ARG_2;
int level = 0;
Uint flags = TERM_TO_BINARY_DFLAGS;
+ Eterm res;
while (is_list(Flags)) {
Eterm arg = CAR(list_val(Flags));
@@ -1032,10 +1111,10 @@ BIF_RETTYPE term_to_binary_2(BIF_ALIST_2)
if (tp[1] == am_minor_version && is_small(tp[2])) {
switch (signed_val(tp[2])) {
case 0:
- flags = TERM_TO_BINARY_DFLAGS;
+ flags = TERM_TO_BINARY_DFLAGS & ~DFLAG_NEW_FLOATS;
break;
case 1:
- flags = TERM_TO_BINARY_DFLAGS|DFLAG_NEW_FLOATS;
+ flags = TERM_TO_BINARY_DFLAGS;
break;
default:
goto error;
@@ -1058,9 +1137,76 @@ BIF_RETTYPE term_to_binary_2(BIF_ALIST_2)
goto error;
}
- return erts_term_to_binary(p, Term, level, flags);
+ res = erts_term_to_binary_int(p, Term, level, flags, NULL);
+ if (is_tuple(res)) {
+ erts_set_gc_state(p, 0);
+ BIF_TRAP1(&term_to_binary_trap_export,BIF_P,res);
+ } else {
+ ASSERT(!(BIF_P->flags & F_DISABLE_GC));
+ BIF_RET(res);
+ }
}
+
+enum B2TState { /* order is somewhat significant */
+ B2TPrepare,
+ B2TUncompressChunk,
+ B2TSizeInit,
+ B2TSize,
+ B2TDecodeInit,
+ B2TDecode,
+ B2TDecodeList,
+ B2TDecodeTuple,
+ B2TDecodeString,
+ B2TDecodeBinary,
+
+ B2TDone,
+ B2TDecodeFail,
+ B2TBadArg
+};
+
+typedef struct {
+ int heap_size;
+ int terms;
+ byte* ep;
+ int atom_extra_skip;
+} B2TSizeContext;
+
+typedef struct {
+ byte* ep;
+ Eterm res;
+ Eterm* next;
+ ErtsHeapFactory factory;
+ int remaining_n;
+ char* remaining_bytes;
+ ErtsWStack flat_maps;
+ ErtsPStack hamt_array;
+} B2TDecodeContext;
+
+typedef struct {
+ z_stream stream;
+ byte* dbytes;
+ Uint dleft;
+} B2TUncompressContext;
+
+typedef struct B2TContext_t {
+ Sint heap_size;
+ byte* aligned_alloc;
+ ErtsBinary2TermState b2ts;
+ Uint32 flags;
+ SWord reds;
+ Eterm trap_bin;
+ Export *bif;
+ Eterm arg[2];
+ enum B2TState state;
+ union {
+ B2TSizeContext sc;
+ B2TDecodeContext dc;
+ B2TUncompressContext uc;
+ } u;
+} B2TContext;
+
+
static uLongf binary2term_uncomp_size(byte* data, Sint size)
{
z_stream stream;
@@ -1090,48 +1236,62 @@ static uLongf binary2term_uncomp_size(byte* data, Sint size)
return err == Z_STREAM_END ? uncomp_size : 0;
}
-static ERTS_INLINE Sint
-binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size)
+static ERTS_INLINE int
+binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size,
+ B2TContext* ctx)
{
- Sint res;
byte *bytes = data;
Sint size = data_size;
state->exttmp = 0;
if (size < 1 || *bytes != VERSION_MAGIC) {
- error:
- if (state->exttmp)
- erts_free(ERTS_ALC_T_TMP, state->extp);
- state->extp = NULL;
- state->exttmp = 0;
return -1;
}
bytes++;
size--;
if (size < 5 || *bytes != COMPRESSED) {
state->extp = bytes;
+ if (ctx)
+ ctx->state = B2TSizeInit;
}
else {
uLongf dest_len = (Uint32) get_int32(bytes+1);
bytes += 5;
size -= 5;
if (dest_len > 32*1024*1024
- || (state->extp = erts_alloc_fnf(ERTS_ALC_T_TMP, dest_len)) == NULL) {
+ || (state->extp = erts_alloc_fnf(ERTS_ALC_T_EXT_TERM_DATA, dest_len)) == NULL) {
+ /*
+ * Try avoid out-of-memory crash due to corrupted 'dest_len'
+ * by checking the actual length of the uncompressed data.
+ * The only way to do that is to uncompress it. Sad but true.
+ */
if (dest_len != binary2term_uncomp_size(bytes, size)) {
- goto error;
+ return -1;
}
- state->extp = erts_alloc(ERTS_ALC_T_TMP, dest_len);
+ state->extp = erts_alloc(ERTS_ALC_T_EXT_TERM_DATA, dest_len);
+ ctx->reds -= dest_len;
}
state->exttmp = 1;
- if (erl_zlib_uncompress(state->extp, &dest_len, bytes, size) != Z_OK)
- goto error;
+ if (ctx) {
+ if (erl_zlib_inflate_start(&ctx->u.uc.stream, bytes, size) != Z_OK)
+ return -1;
+
+ ctx->u.uc.dbytes = state->extp;
+ ctx->u.uc.dleft = dest_len;
+ ctx->state = B2TUncompressChunk;
+ }
+ else {
+ uLongf dlen = dest_len;
+ if (erl_zlib_uncompress(state->extp, &dlen, bytes, size) != Z_OK
+ || dlen != dest_len) {
+ return -1;
+ }
+ }
size = (Sint) dest_len;
}
- res = decoded_size(state->extp, state->extp + size, 0);
- if (res < 0)
- goto error;
- return res;
+ state->extsize = size;
+ return 0;
}
static ERTS_INLINE void
@@ -1139,19 +1299,21 @@ binary2term_abort(ErtsBinary2TermState *state)
{
if (state->exttmp) {
state->exttmp = 0;
- erts_free(ERTS_ALC_T_TMP, state->extp);
+ erts_free(ERTS_ALC_T_EXT_TERM_DATA, state->extp);
}
}
static ERTS_INLINE Eterm
-binary2term_create(ErtsDistExternal *edep, ErtsBinary2TermState *state, Eterm **hpp, ErlOffHeap *ohp)
+binary2term_create(ErtsDistExternal *edep, ErtsBinary2TermState *state,
+ ErtsHeapFactory* factory)
{
Eterm res;
- if (!dec_term(edep, hpp, state->extp, ohp, &res))
+
+ if (!dec_term(edep, factory, state->extp, &res, NULL))
res = THE_NON_VALUE;
if (state->exttmp) {
state->exttmp = 0;
- erts_free(ERTS_ALC_T_TMP, state->extp);
+ erts_free(ERTS_ALC_T_EXT_TERM_DATA, state->extp);
}
return res;
}
@@ -1159,7 +1321,18 @@ binary2term_create(ErtsDistExternal *edep, ErtsBinary2TermState *state, Eterm **
Sint
erts_binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size)
{
- return binary2term_prepare(state, data, data_size);
+ Sint res;
+
+ if (binary2term_prepare(state, data, data_size, NULL) < 0 ||
+ (res=decoded_size(state->extp, state->extp + state->extsize, 0, NULL)) < 0) {
+
+ if (state->exttmp)
+ erts_free(ERTS_ALC_T_EXT_TERM_DATA, state->extp);
+ state->extp = NULL;
+ state->exttmp = 0;
+ return -1;
+ }
+ return res;
}
void
@@ -1169,73 +1342,278 @@ erts_binary2term_abort(ErtsBinary2TermState *state)
}
Eterm
-erts_binary2term_create(ErtsBinary2TermState *state, Eterm **hpp, ErlOffHeap *ohp)
+erts_binary2term_create(ErtsBinary2TermState *state, ErtsHeapFactory* factory)
{
- return binary2term_create(NULL,state, hpp, ohp);
+ return binary2term_create(NULL,state, factory);
}
-BIF_RETTYPE binary_to_term_1(BIF_ALIST_1)
+static void b2t_destroy_context(B2TContext* context)
{
- Sint heap_size;
- Eterm res;
+ erts_free_aligned_binary_bytes_extra(context->aligned_alloc,
+ ERTS_ALC_T_EXT_TERM_DATA);
+ context->aligned_alloc = NULL;
+ binary2term_abort(&context->b2ts);
+ switch (context->state) {
+ case B2TUncompressChunk:
+ erl_zlib_inflate_finish(&context->u.uc.stream);
+ break;
+ case B2TDecode:
+ case B2TDecodeList:
+ case B2TDecodeTuple:
+ case B2TDecodeString:
+ case B2TDecodeBinary:
+ if (context->u.dc.hamt_array.pstart) {
+ erts_free(context->u.dc.hamt_array.alloc_type,
+ context->u.dc.hamt_array.pstart);
+ }
+ break;
+ default:;
+ }
+}
+
+static void b2t_context_destructor(Binary *context_bin)
+{
+ B2TContext* ctx = (B2TContext*) ERTS_MAGIC_BIN_DATA(context_bin);
+ ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(context_bin) == b2t_context_destructor);
+
+ b2t_destroy_context(ctx);
+}
+
+static BIF_RETTYPE binary_to_term_trap_1(BIF_ALIST_1)
+{
+ Binary *context_bin = ((ProcBin *) binary_val(BIF_ARG_1))->val;
+ ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(context_bin) == b2t_context_destructor);
+
+ return binary_to_term_int(BIF_P, 0, THE_NON_VALUE, context_bin, NULL,
+ THE_NON_VALUE, THE_NON_VALUE);
+}
+
+
+#define B2T_BYTES_PER_REDUCTION 128
+#define B2T_MEMCPY_FACTOR 8
+
+/* Define for testing */
+/*#define EXTREME_B2T_TRAPPING 1*/
+
+#ifdef EXTREME_B2T_TRAPPING
+static unsigned b2t_rand(void)
+{
+ static unsigned prev = 17;
+ prev = (prev * 214013 + 2531011);
+ return prev;
+}
+#endif
+
+
+static B2TContext* b2t_export_context(Process* p, B2TContext* src)
+{
+ Binary* context_b = erts_create_magic_binary(sizeof(B2TContext),
+ b2t_context_destructor);
+ B2TContext* ctx = ERTS_MAGIC_BIN_DATA(context_b);
Eterm* hp;
- Eterm* endp;
- Sint size;
- byte* bytes;
- byte* temp_alloc = NULL;
- ErtsBinary2TermState b2ts;
+ sys_memcpy(ctx, src, sizeof(B2TContext));
+ if (ctx->state >= B2TDecode && ctx->u.dc.next == &src->u.dc.res) {
+ ctx->u.dc.next = &ctx->u.dc.res;
+ }
+ hp = HAlloc(p, PROC_BIN_SIZE);
+ ctx->trap_bin = erts_mk_magic_binary_term(&hp, &MSO(p), context_b);
+ return ctx;
+}
- if ((bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc)) == NULL) {
- error:
- erts_free_aligned_binary_bytes(temp_alloc);
- BIF_ERROR(BIF_P, BADARG);
+static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* context_b,
+ Export *bif_init, Eterm arg0, Eterm arg1)
+{
+ BIF_RETTYPE ret_val;
+#ifdef EXTREME_B2T_TRAPPING
+ SWord initial_reds = 1 + b2t_rand() % 4;
+#else
+ SWord initial_reds = (Uint)(ERTS_BIF_REDS_LEFT(p) * B2T_BYTES_PER_REDUCTION);
+#endif
+ B2TContext c_buff;
+ B2TContext *ctx;
+ int is_first_call;
+
+ if (context_b == NULL) {
+ /* Setup enough to get started */
+ is_first_call = 1;
+ ctx = &c_buff;
+ ctx->state = B2TPrepare;
+ ctx->aligned_alloc = NULL;
+ ctx->flags = flags;
+ ctx->bif = bif_init;
+ ctx->arg[0] = arg0;
+ ctx->arg[1] = arg1;
+ IF_DEBUG(ctx->trap_bin = THE_NON_VALUE;)
+ } else {
+ is_first_call = 0;
+ ctx = ERTS_MAGIC_BIN_DATA(context_b);
+ ASSERT(ctx->state != B2TPrepare);
}
- size = binary_size(BIF_ARG_1);
+ ctx->reds = initial_reds;
+
+ do {
+ switch (ctx->state) {
+ case B2TPrepare: {
+ byte* bytes;
+ Uint bin_size;
+ bytes = erts_get_aligned_binary_bytes_extra(bin,
+ &ctx->aligned_alloc,
+ ERTS_ALC_T_EXT_TERM_DATA,
+ 0);
+ if (bytes == NULL) {
+ ctx->b2ts.exttmp = 0;
+ ctx->state = B2TBadArg;
+ break;
+ }
+ bin_size = binary_size(bin);
+ if (ctx->aligned_alloc) {
+ ctx->reds -= bin_size / 8;
+ }
+ if (binary2term_prepare(&ctx->b2ts, bytes, bin_size, ctx) < 0) {
+ ctx->state = B2TBadArg;
+ }
+ break;
+ }
+ case B2TUncompressChunk: {
+ uLongf chunk = ctx->reds;
+ int zret;
+
+ if (chunk > ctx->u.uc.dleft)
+ chunk = ctx->u.uc.dleft;
+ zret = erl_zlib_inflate_chunk(&ctx->u.uc.stream,
+ ctx->u.uc.dbytes, &chunk);
+ ctx->u.uc.dbytes += chunk;
+ ctx->u.uc.dleft -= chunk;
+ if (zret == Z_OK && ctx->u.uc.dleft > 0) {
+ ctx->reds = 0;
+ }
+ else if (erl_zlib_inflate_finish(&ctx->u.uc.stream) == Z_OK
+ && zret == Z_STREAM_END
+ && ctx->u.uc.dleft == 0) {
+ ctx->reds -= chunk;
+ ctx->state = B2TSizeInit;
+ }
+ else {
+ ctx->state = B2TBadArg;
+ }
+ break;
+ }
+ case B2TSizeInit:
+ ctx->u.sc.ep = NULL;
+ ctx->state = B2TSize;
+ /*fall through*/
+ case B2TSize:
+ ctx->heap_size = decoded_size(ctx->b2ts.extp,
+ ctx->b2ts.extp + ctx->b2ts.extsize,
+ 0, ctx);
+ break;
+
+ case B2TDecodeInit:
+ if (ctx == &c_buff && ctx->b2ts.extsize > ctx->reds) {
+ /* dec_term will maybe trap, allocate space for magic bin
+ before result term to make it easy to trim with HRelease.
+ */
+ ctx = b2t_export_context(p, &c_buff);
+ }
+ ctx->u.dc.ep = ctx->b2ts.extp;
+ ctx->u.dc.res = (Eterm) (UWord) NULL;
+ ctx->u.dc.next = &ctx->u.dc.res;
+ erts_factory_proc_prealloc_init(&ctx->u.dc.factory, p, ctx->heap_size);
+ ctx->u.dc.flat_maps.wstart = NULL;
+ ctx->u.dc.hamt_array.pstart = NULL;
+ ctx->state = B2TDecode;
+ /*fall through*/
+ case B2TDecode:
+ case B2TDecodeList:
+ case B2TDecodeTuple:
+ case B2TDecodeString:
+ case B2TDecodeBinary: {
+ ErtsDistExternal fakedep;
+ fakedep.flags = ctx->flags;
+ dec_term(&fakedep, NULL, NULL, NULL, ctx);
+ break;
+ }
+ case B2TDecodeFail:
+ /*fall through*/
+ case B2TBadArg:
+ BUMP_REDS(p, (initial_reds - ctx->reds) / B2T_BYTES_PER_REDUCTION);
- heap_size = binary2term_prepare(&b2ts, bytes, size);
- if (heap_size < 0)
- goto error;
+ ASSERT(ctx->bif == bif_export[BIF_binary_to_term_1]
+ || ctx->bif == bif_export[BIF_binary_to_term_2]);
- hp = HAlloc(BIF_P, heap_size);
- endp = hp + heap_size;
+ if (is_first_call)
+ ERTS_BIF_PREP_ERROR(ret_val, p, BADARG);
+ else {
+ erts_set_gc_state(p, 1);
+ if (is_non_value(ctx->arg[1]))
+ ERTS_BIF_PREP_ERROR_TRAPPED1(ret_val, p, BADARG, ctx->bif,
+ ctx->arg[0]);
+ else
+ ERTS_BIF_PREP_ERROR_TRAPPED2(ret_val, p, BADARG, ctx->bif,
+ ctx->arg[0], ctx->arg[1]);
+ }
+ b2t_destroy_context(ctx);
+ return ret_val;
- res = binary2term_create(NULL, &b2ts, &hp, &MSO(BIF_P));
+ case B2TDone:
+ b2t_destroy_context(ctx);
- erts_free_aligned_binary_bytes(temp_alloc);
+ if (ctx->u.dc.factory.hp > ctx->u.dc.factory.hp_end) {
+ erts_exit(ERTS_ERROR_EXIT, ":%s, line %d: heap overrun by %d words(s)\n",
+ __FILE__, __LINE__, ctx->u.dc.factory.hp - ctx->u.dc.factory.hp_end);
+ }
+ erts_factory_close(&ctx->u.dc.factory);
- if (hp > endp) {
- erl_exit(1, ":%s, line %d: heap overrun by %d words(s)\n",
- __FILE__, __LINE__, hp-endp);
+ if (!is_first_call) {
+ erts_set_gc_state(p, 1);
+ }
+ BUMP_REDS(p, (initial_reds - ctx->reds) / B2T_BYTES_PER_REDUCTION);
+ ERTS_BIF_PREP_RET(ret_val, ctx->u.dc.res);
+ return ret_val;
+
+ default:
+ ASSERT(!"Unknown state in binary_to_term");
+ }
+ }while (ctx->reds > 0 || ctx->state >= B2TDone);
+
+ if (ctx == &c_buff) {
+ ASSERT(ctx->trap_bin == THE_NON_VALUE);
+ ctx = b2t_export_context(p, &c_buff);
}
+ ASSERT(ctx->trap_bin != THE_NON_VALUE);
- HRelease(BIF_P, endp, hp);
+ if (is_first_call) {
+ erts_set_gc_state(p, 0);
+ }
+ BUMP_ALL_REDS(p);
- if (res == THE_NON_VALUE)
- goto error;
+ ERTS_BIF_PREP_TRAP1(ret_val, &binary_to_term_trap_export,
+ p, ctx->trap_bin);
- return res;
+ return ret_val;
}
+HIPE_WRAPPER_BIF_DISABLE_GC(binary_to_term, 1)
+
+BIF_RETTYPE binary_to_term_1(BIF_ALIST_1)
+{
+ return binary_to_term_int(BIF_P, 0, BIF_ARG_1, NULL, bif_export[BIF_binary_to_term_1],
+ BIF_ARG_1, THE_NON_VALUE);
+}
+
+HIPE_WRAPPER_BIF_DISABLE_GC(binary_to_term, 2)
+
BIF_RETTYPE binary_to_term_2(BIF_ALIST_2)
{
- Sint heap_size;
- Eterm res;
Eterm opts;
Eterm opt;
- Eterm* hp;
- Eterm* endp;
- Sint size;
- byte* bytes;
- byte* temp_alloc = NULL;
- ErtsBinary2TermState b2ts;
- ErtsDistExternal fakedep;
+ Uint32 flags = 0;
- fakedep.flags = 0;
opts = BIF_ARG_2;
while (is_list(opts)) {
opt = CAR(list_val(opts));
if (opt == am_safe) {
- fakedep.flags |= ERTS_DIST_EXT_BTT_SAFE;
+ flags |= ERTS_DIST_EXT_BTT_SAFE;
}
else {
goto error;
@@ -1246,35 +1624,11 @@ BIF_RETTYPE binary_to_term_2(BIF_ALIST_2)
if (is_not_nil(opts))
goto error;
- if ((bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc)) == NULL) {
- error:
- erts_free_aligned_binary_bytes(temp_alloc);
- BIF_ERROR(BIF_P, BADARG);
- }
- size = binary_size(BIF_ARG_1);
-
- heap_size = binary2term_prepare(&b2ts, bytes, size);
- if (heap_size < 0)
- goto error;
-
- hp = HAlloc(BIF_P, heap_size);
- endp = hp + heap_size;
+ return binary_to_term_int(BIF_P, flags, BIF_ARG_1, NULL, bif_export[BIF_binary_to_term_2],
+ BIF_ARG_1, BIF_ARG_2);
- res = binary2term_create(&fakedep, &b2ts, &hp, &MSO(BIF_P));
-
- erts_free_aligned_binary_bytes(temp_alloc);
-
- if (hp > endp) {
- erl_exit(1, ":%s, line %d: heap overrun by %d words(s)\n",
- __FILE__, __LINE__, hp-endp);
- }
-
- HRelease(BIF_P, endp, hp);
-
- if (res == THE_NON_VALUE)
- goto error;
-
- return res;
+error:
+ BIF_ERROR(BIF_P, BADARG);
}
Eterm
@@ -1306,9 +1660,9 @@ external_size_2(BIF_ALIST_2)
if (tp[1] == am_minor_version && is_small(tp[2])) {
switch (signed_val(tp[2])) {
case 0:
+ flags &= ~DFLAG_NEW_FLOATS;
break;
case 1:
- flags |= DFLAG_NEW_FLOATS;
break;
default:
goto error;
@@ -1335,16 +1689,13 @@ external_size_2(BIF_ALIST_2)
}
}
-Eterm
-erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags)
+static Eterm
+erts_term_to_binary_simple(Process* p, Eterm Term, Uint size, int level, Uint flags)
{
- Uint size;
Eterm bin;
size_t real_size;
byte* endp;
- size = encode_size_struct2(NULL, Term, flags) + 1 /* VERSION_MAGIC */;
-
if (level != 0) {
byte buf[256];
byte* bytes = buf;
@@ -1357,12 +1708,12 @@ erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags)
if ((endp = enc_term(NULL, Term, bytes, flags, NULL))
== NULL) {
- erl_exit(1, "%s, line %d: bad term: %x\n",
+ erts_exit(ERTS_ERROR_EXIT, "%s, line %d: bad term: %x\n",
__FILE__, __LINE__, Term);
}
real_size = endp - bytes;
if (real_size > size) {
- erl_exit(1, "%s, line %d: buffer overflow: %d word(s)\n",
+ erts_exit(ERTS_ERROR_EXIT, "%s, line %d: buffer overflow: %d word(s)\n",
__FILE__, __LINE__, real_size - size);
}
@@ -1402,18 +1753,304 @@ erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags)
bytes[0] = VERSION_MAGIC;
if ((endp = enc_term(NULL, Term, bytes+1, flags, NULL))
== NULL) {
- erl_exit(1, "%s, line %d: bad term: %x\n",
+ erts_exit(ERTS_ERROR_EXIT, "%s, line %d: bad term: %x\n",
__FILE__, __LINE__, Term);
}
real_size = endp - bytes;
if (real_size > size) {
- erl_exit(1, "%s, line %d: buffer overflow: %d word(s)\n",
+ erts_exit(ERTS_ERROR_EXIT, "%s, line %d: buffer overflow: %d word(s)\n",
__FILE__, __LINE__, endp - (bytes + size));
}
return erts_realloc_binary(bin, real_size);
}
}
+Eterm
+erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags) {
+ Uint size;
+ size = encode_size_struct2(NULL, Term, flags) + 1 /* VERSION_MAGIC */;
+ return erts_term_to_binary_simple(p, Term, size, level, flags);
+}
+
+/* Define EXTREME_TTB_TRAPPING for testing in dist.h */
+
+#ifndef EXTREME_TTB_TRAPPING
+#define TERM_TO_BINARY_COMPRESS_CHUNK (1 << 18)
+#else
+#define TERM_TO_BINARY_COMPRESS_CHUNK 10
+#endif
+#define TERM_TO_BINARY_MEMCPY_FACTOR 8
+
+static void ttb_context_destructor(Binary *context_bin)
+{
+ TTBContext *context = ERTS_MAGIC_BIN_DATA(context_bin);
+ if (context->alive) {
+ context->alive = 0;
+ switch (context->state) {
+ case TTBSize:
+ DESTROY_SAVED_WSTACK(&context->s.sc.wstack);
+ break;
+ case TTBEncode:
+ DESTROY_SAVED_WSTACK(&context->s.ec.wstack);
+ if (context->s.ec.result_bin != NULL) { /* Set to NULL if ever made alive! */
+ ASSERT(erts_refc_read(&(context->s.ec.result_bin->refc),0) == 0);
+ erts_bin_free(context->s.ec.result_bin);
+ context->s.ec.result_bin = NULL;
+ }
+ break;
+ case TTBCompress:
+ erl_zlib_deflate_finish(&(context->s.cc.stream));
+
+ if (context->s.cc.destination_bin != NULL) { /* Set to NULL if ever made alive! */
+ ASSERT(erts_refc_read(&(context->s.cc.destination_bin->refc),0) == 0);
+ erts_bin_free(context->s.cc.destination_bin);
+ context->s.cc.destination_bin = NULL;
+ }
+
+ if (context->s.cc.result_bin != NULL) { /* Set to NULL if ever made alive! */
+ ASSERT(erts_refc_read(&(context->s.cc.result_bin->refc),0) == 0);
+ erts_bin_free(context->s.cc.result_bin);
+ context->s.cc.result_bin = NULL;
+ }
+ break;
+ }
+ }
+}
+
+static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint flags,
+ Binary *context_b)
+{
+ Eterm *hp;
+ Eterm res;
+ Eterm c_term;
+#ifndef EXTREME_TTB_TRAPPING
+ Sint reds = (Sint) (ERTS_BIF_REDS_LEFT(p) * TERM_TO_BINARY_LOOP_FACTOR);
+#else
+ Sint reds = 20; /* For testing */
+#endif
+ Sint initial_reds = reds;
+ TTBContext c_buff;
+ TTBContext *context = &c_buff;
+
+#define EXPORT_CONTEXT() \
+ do { \
+ if (context_b == NULL) { \
+ context_b = erts_create_magic_binary(sizeof(TTBContext), \
+ ttb_context_destructor); \
+ context = ERTS_MAGIC_BIN_DATA(context_b); \
+ memcpy(context,&c_buff,sizeof(TTBContext)); \
+ } \
+ } while (0)
+
+#define RETURN_STATE() \
+ do { \
+ hp = HAlloc(p, PROC_BIN_SIZE+3); \
+ c_term = erts_mk_magic_binary_term(&hp, &MSO(p), context_b); \
+ res = TUPLE2(hp, Term, c_term); \
+ BUMP_ALL_REDS(p); \
+ return res; \
+ } while (0);
+
+
+ if (context_b == NULL) {
+ /* Setup enough to get started */
+ context->state = TTBSize;
+ context->alive = 1;
+ context->s.sc.wstack.wstart = NULL;
+ context->s.sc.flags = flags;
+ context->s.sc.level = level;
+ } else {
+ context = ERTS_MAGIC_BIN_DATA(context_b);
+ }
+ /* Initialization done, now we will go through the states */
+ for (;;) {
+ switch (context->state) {
+ case TTBSize:
+ {
+ Uint size;
+ Binary *result_bin;
+ int level;
+ Uint flags;
+ /* Try for fast path */
+ if (encode_size_struct_int(&context->s.sc, NULL, Term,
+ context->s.sc.flags, &reds, &size) < 0) {
+ EXPORT_CONTEXT();
+ /* Same state */
+ RETURN_STATE();
+ }
+ ++size; /* VERSION_MAGIC */
+ /* Move these to next state */
+ flags = context->s.sc.flags;
+ level = context->s.sc.level;
+ if (size <= ERL_ONHEAP_BIN_LIMIT) {
+ /* Finish in one go */
+ res = erts_term_to_binary_simple(p, Term, size,
+ level, flags);
+ BUMP_REDS(p, 1);
+ return res;
+ }
+
+ result_bin = erts_bin_nrml_alloc(size);
+ erts_refc_init(&result_bin->refc, 0);
+ result_bin->orig_bytes[0] = VERSION_MAGIC;
+ /* Next state immediately, no need to export context */
+ context->state = TTBEncode;
+ context->s.ec.flags = flags;
+ context->s.ec.level = level;
+ context->s.ec.wstack.wstart = NULL;
+ context->s.ec.result_bin = result_bin;
+ break;
+ }
+ case TTBEncode:
+ {
+ byte *endp;
+ byte *bytes = (byte *) context->s.ec.result_bin->orig_bytes;
+ size_t real_size;
+ Binary *result_bin;
+
+ flags = context->s.ec.flags;
+ if (enc_term_int(&context->s.ec, NULL,Term, bytes+1, flags, NULL, &reds, &endp) < 0) {
+ EXPORT_CONTEXT();
+ RETURN_STATE();
+ }
+ real_size = endp - bytes;
+ result_bin = erts_bin_realloc(context->s.ec.result_bin,real_size);
+ level = context->s.ec.level;
+ BUMP_REDS(p, (initial_reds - reds) / TERM_TO_BINARY_LOOP_FACTOR);
+ if (level == 0 || real_size < 6) { /* We are done */
+ ProcBin* pb;
+ return_normal:
+ context->s.ec.result_bin = NULL;
+ context->alive = 0;
+ pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE);
+ pb->thing_word = HEADER_PROC_BIN;
+ pb->size = real_size;
+ pb->next = MSO(p).first;
+ MSO(p).first = (struct erl_off_heap_header*)pb;
+ pb->val = result_bin;
+ pb->bytes = (byte*) result_bin->orig_bytes;
+ pb->flags = 0;
+ OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm));
+ erts_refc_inc(&result_bin->refc, 1);
+ if (context_b && erts_refc_read(&context_b->refc,0) == 0) {
+ erts_bin_free(context_b);
+ }
+ return make_binary(pb);
+ }
+ /* Continue with compression... */
+ /* To make absolutely sure that zlib does not barf on a reallocated context,
+ we make sure it's "exported" before doing anything compession-like */
+ EXPORT_CONTEXT();
+ bytes = (byte *) result_bin->orig_bytes; /* result_bin is reallocated */
+ if (erl_zlib_deflate_start(&(context->s.cc.stream),bytes+1,real_size-1,level)
+ != Z_OK) {
+ goto return_normal;
+ }
+ context->state = TTBCompress;
+ context->s.cc.real_size = real_size;
+ context->s.cc.result_bin = result_bin;
+
+ result_bin = erts_bin_nrml_alloc(real_size);
+ erts_refc_init(&result_bin->refc, 0);
+ result_bin->orig_bytes[0] = VERSION_MAGIC;
+
+ context->s.cc.destination_bin = result_bin;
+ context->s.cc.dest_len = 0;
+ context->s.cc.dbytes = (byte *) result_bin->orig_bytes+6;
+ break;
+ }
+ case TTBCompress:
+ {
+ uLongf tot_dest_len = context->s.cc.real_size - 6;
+ uLongf left = (tot_dest_len - context->s.cc.dest_len);
+ uLongf this_time = (left > TERM_TO_BINARY_COMPRESS_CHUNK) ?
+ TERM_TO_BINARY_COMPRESS_CHUNK :
+ left;
+ Binary *result_bin;
+ ProcBin *pb;
+ Uint max = (ERTS_BIF_REDS_LEFT(p) * TERM_TO_BINARY_COMPRESS_CHUNK) / CONTEXT_REDS;
+
+ if (max < this_time) {
+ this_time = max + 1; /* do not set this_time to 0 */
+ }
+
+ res = erl_zlib_deflate_chunk(&(context->s.cc.stream), context->s.cc.dbytes, &this_time);
+ context->s.cc.dbytes += this_time;
+ context->s.cc.dest_len += this_time;
+ switch (res) {
+ case Z_OK:
+ if (context->s.cc.dest_len >= tot_dest_len) {
+ goto no_use_compressing;
+ }
+ RETURN_STATE();
+ case Z_STREAM_END:
+ {
+ byte *dbytes = (byte *) context->s.cc.destination_bin->orig_bytes + 1;
+
+ dbytes[0] = COMPRESSED;
+ put_int32(context->s.cc.real_size-1,dbytes+1);
+ erl_zlib_deflate_finish(&(context->s.cc.stream));
+ result_bin = erts_bin_realloc(context->s.cc.destination_bin,
+ context->s.cc.dest_len+6);
+ context->s.cc.destination_bin = NULL;
+ pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE);
+ pb->thing_word = HEADER_PROC_BIN;
+ pb->size = context->s.cc.dest_len+6;
+ pb->next = MSO(p).first;
+ MSO(p).first = (struct erl_off_heap_header*)pb;
+ pb->val = result_bin;
+ pb->bytes = (byte*) result_bin->orig_bytes;
+ pb->flags = 0;
+ OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm));
+ erts_refc_inc(&result_bin->refc, 1);
+ erts_bin_free(context->s.cc.result_bin);
+ context->s.cc.result_bin = NULL;
+ context->alive = 0;
+ BUMP_REDS(p, (this_time * CONTEXT_REDS) / TERM_TO_BINARY_COMPRESS_CHUNK);
+ if (context_b && erts_refc_read(&context_b->refc,0) == 0) {
+ erts_bin_free(context_b);
+ }
+ return make_binary(pb);
+ }
+ default: /* Compression error, revert to uncompressed binary (still in
+ context) */
+ no_use_compressing:
+ result_bin = context->s.cc.result_bin;
+ context->s.cc.result_bin = NULL;
+ pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE);
+ pb->thing_word = HEADER_PROC_BIN;
+ pb->size = context->s.cc.real_size;
+ pb->next = MSO(p).first;
+ MSO(p).first = (struct erl_off_heap_header*)pb;
+ pb->val = result_bin;
+ pb->bytes = (byte*) result_bin->orig_bytes;
+ pb->flags = 0;
+ OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm));
+ erts_refc_inc(&result_bin->refc, 1);
+ erl_zlib_deflate_finish(&(context->s.cc.stream));
+ erts_bin_free(context->s.cc.destination_bin);
+ context->s.cc.destination_bin = NULL;
+ context->alive = 0;
+ BUMP_REDS(p, (this_time * CONTEXT_REDS) / TERM_TO_BINARY_COMPRESS_CHUNK);
+ if (context_b && erts_refc_read(&context_b->refc,0) == 0) {
+ erts_bin_free(context_b);
+ }
+ return make_binary(pb);
+ }
+ }
+ }
+ }
+#undef EXPORT_CONTEXT
+#undef RETURN_STATE
+}
+
+
+
+
+
+
+
+
/*
* This function fills ext with the external format of atom.
* If it's an old atom we just supply an index, otherwise
@@ -1619,7 +2256,7 @@ static ERTS_INLINE ErlNode* dec_get_node(Eterm sysname, Uint creation)
}
static byte*
-dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Eterm* objp)
+dec_pid(ErtsDistExternal *edep, ErtsHeapFactory* factory, byte* ep, Eterm* objp)
{
Eterm sysname;
Uint data;
@@ -1658,15 +2295,15 @@ dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Ete
if(node == erts_this_node) {
*objp = make_internal_pid(data);
} else {
- ExternalThing *etp = (ExternalThing *) *hpp;
- *hpp += EXTERNAL_THING_HEAD_SIZE + 1;
+ ExternalThing *etp = (ExternalThing *) factory->hp;
+ factory->hp += EXTERNAL_THING_HEAD_SIZE + 1;
etp->header = make_external_pid_header(1);
- etp->next = off_heap->first;
+ etp->next = factory->off_heap->first;
etp->node = node;
etp->data.ui[0] = data;
- off_heap->first = (struct erl_off_heap_header*) etp;
+ factory->off_heap->first = (struct erl_off_heap_header*) etp;
*objp = make_external_pid(etp);
}
return ep;
@@ -1676,12 +2313,24 @@ dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Ete
#define ENC_TERM ((Eterm) 0)
#define ENC_ONE_CONS ((Eterm) 1)
#define ENC_PATCH_FUN_SIZE ((Eterm) 2)
-#define ENC_LAST_ARRAY_ELEMENT ((Eterm) 3)
+#define ENC_BIN_COPY ((Eterm) 3)
+#define ENC_MAP_PAIR ((Eterm) 4)
+#define ENC_HASHMAP_NODE ((Eterm) 5)
+#define ENC_LAST_ARRAY_ELEMENT ((Eterm) 6)
static byte*
enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags,
struct erl_off_heap_header** off_heap)
{
+ byte *res;
+ (void) enc_term_int(NULL, acmp, obj, ep, dflags, off_heap, NULL, &res);
+ return res;
+}
+
+static int
+enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags,
+ struct erl_off_heap_header** off_heap, Sint *reds, byte **res)
+{
DECLARE_WSTACK(s);
Uint n;
Uint i;
@@ -1689,11 +2338,26 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags,
Uint* ptr;
Eterm val;
FloatDef f;
+ Sint r = 0;
#if HALFWORD_HEAP
UWord wobj;
#endif
+ if (ctx) {
+ WSTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK);
+ r = *reds;
+
+ if (ctx->wstack.wstart) { /* restore saved stacks and byte pointer */
+ WSTACK_RESTORE(s, &ctx->wstack);
+ ep = ctx->ep;
+ obj = ctx->obj;
+ if (is_non_value(obj)) {
+ goto outer_loop;
+ }
+ }
+ }
+
goto L_jump_start;
outer_loop:
@@ -1714,8 +2378,8 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags,
obj = CAR(cons);
tl = CDR(cons);
- WSTACK_PUSH(s, is_list(tl) ? ENC_ONE_CONS : ENC_TERM);
- WSTACK_PUSH(s, tl);
+ WSTACK_PUSH2(s, (is_list(tl) ? ENC_ONE_CONS : ENC_TERM),
+ tl);
}
break;
case ENC_PATCH_FUN_SIZE:
@@ -1728,7 +2392,48 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags,
put_int32(ep - size_p, size_p);
}
goto outer_loop;
+ case ENC_BIN_COPY: {
+ Uint bits = (Uint)obj;
+ Uint bitoffs = WSTACK_POP(s);
+ byte* bytes = (byte*) WSTACK_POP(s);
+ byte* dst = (byte*) WSTACK_POP(s);
+ if (bits > r * (TERM_TO_BINARY_MEMCPY_FACTOR * 8)) {
+ Uint n = r * TERM_TO_BINARY_MEMCPY_FACTOR;
+ WSTACK_PUSH5(s, (UWord)(dst + n), (UWord)(bytes + n), bitoffs,
+ ENC_BIN_COPY, bits - 8*n);
+ bits = 8*n;
+ copy_binary_to_buffer(dst, 0, bytes, bitoffs, bits);
+ obj = THE_NON_VALUE;
+ r = 0; /* yield */
+ break;
+ } else {
+ copy_binary_to_buffer(dst, 0, bytes, bitoffs, bits);
+ r -= bits / (TERM_TO_BINARY_MEMCPY_FACTOR * 8);
+ goto outer_loop;
+ }
+ }
+ case ENC_MAP_PAIR: {
+ Uint pairs_left = obj;
+ Eterm *vptr = (Eterm*) WSTACK_POP(s);
+ Eterm *kptr = (Eterm*) WSTACK_POP(s);
+
+ obj = *kptr;
+ if (--pairs_left > 0) {
+ WSTACK_PUSH4(s, (UWord)(kptr+1), (UWord)(vptr+1),
+ ENC_MAP_PAIR, pairs_left);
+ }
+ WSTACK_PUSH2(s, ENC_TERM, *vptr);
+ break;
+ }
+ case ENC_HASHMAP_NODE:
+ if (is_list(obj)) { /* leaf node [K|V] */
+ ptr = list_val(obj);
+ WSTACK_PUSH2(s, ENC_TERM, CDR(ptr));
+ obj = CAR(ptr);
+ }
+ break;
case ENC_LAST_ARRAY_ELEMENT:
+ /* obj is the tuple */
{
#if HALFWORD_HEAP
Eterm* ptr = (Eterm *) wobj;
@@ -1746,13 +2451,20 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags,
Eterm* ptr = (Eterm *) obj;
#endif
obj = *ptr++;
- WSTACK_PUSH(s, val-1);
- WSTACK_PUSH(s, (UWord) ptr);
+ WSTACK_PUSH2(s, val-1, (UWord)ptr);
}
break;
}
L_jump_start:
+
+ if (ctx && --r <= 0) {
+ *reds = 0;
+ ctx->obj = obj;
+ ctx->ep = ep;
+ WSTACK_SAVE(s, &ctx->wstack);
+ return -1;
+ }
switch(tag_val_def(obj)) {
case NIL_DEF:
*ep++ = NIL_EXT;
@@ -1896,16 +2608,64 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags,
ep += 4;
}
if (i > 0) {
- WSTACK_PUSH(s, ENC_LAST_ARRAY_ELEMENT+i-1);
- WSTACK_PUSH(s, (UWord) ptr);
+ WSTACK_PUSH2(s, ENC_LAST_ARRAY_ELEMENT+i-1, (UWord)ptr);
}
break;
+ case MAP_DEF:
+ if (is_flatmap(obj)) {
+ flatmap_t *mp = (flatmap_t*)flatmap_val(obj);
+ Uint size = flatmap_get_size(mp);
+
+ *ep++ = MAP_EXT;
+ put_int32(size, ep); ep += 4;
+
+ if (size > 0) {
+ Eterm *kptr = flatmap_get_keys(mp);
+ Eterm *vptr = flatmap_get_values(mp);
+
+ WSTACK_PUSH4(s, (UWord)kptr, (UWord)vptr,
+ ENC_MAP_PAIR, size);
+ }
+ } else {
+ Eterm hdr;
+ Uint node_sz;
+ ptr = boxed_val(obj);
+ hdr = *ptr;
+ ASSERT(is_header(hdr));
+ switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
+ case HAMT_SUBTAG_HEAD_ARRAY:
+ *ep++ = MAP_EXT;
+ ptr++;
+ put_int32(*ptr, ep); ep += 4;
+ node_sz = 16;
+ break;
+ case HAMT_SUBTAG_HEAD_BITMAP:
+ *ep++ = MAP_EXT;
+ ptr++;
+ put_int32(*ptr, ep); ep += 4;
+ /*fall through*/
+ case HAMT_SUBTAG_NODE_BITMAP:
+ node_sz = hashmap_bitcount(MAP_HEADER_VAL(hdr));
+ ASSERT(node_sz < 17);
+ break;
+ default:
+ erts_exit(ERTS_ERROR_EXIT, "bad header\r\n");
+ }
+
+ ptr++;
+ WSTACK_RESERVE(s, node_sz*2);
+ while(node_sz--) {
+ WSTACK_FAST_PUSH(s, ENC_HASHMAP_NODE);
+ WSTACK_FAST_PUSH(s, *ptr++);
+ }
+ }
+ break;
case FLOAT_DEF:
GET_DOUBLE(obj, f);
if (dflags & DFLAG_NEW_FLOATS) {
*ep++ = NEW_FLOAT_EXT;
-#ifdef WORDS_BIGENDIAN
+#if defined(WORDS_BIGENDIAN) || defined(DOUBLE_MIDDLE_ENDIAN)
put_int32(f.fw[0], ep);
ep += 4;
put_int32(f.fw[1], ep);
@@ -1934,6 +2694,7 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags,
Uint bitoffs;
Uint bitsize;
byte* bytes;
+ byte* data_dst;
ERTS_GET_BINARY_BYTES(obj, bytes, bitoffs, bitsize);
if (dflags & DFLAG_INTERNAL_TAGS) {
@@ -1979,7 +2740,7 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags,
j = binary_size(obj);
put_int32(j, ep);
ep += 4;
- copy_binary_to_buffer(ep, 0, bytes, bitoffs, 8*j);
+ data_dst = ep;
ep += j;
} else if (dflags & DFLAG_BIT_BINARIES) {
/* Bit-level binary. */
@@ -1989,7 +2750,7 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags,
ep += 4;
*ep++ = bitsize;
ep[j] = 0; /* Zero unused bits at end of binary */
- copy_binary_to_buffer(ep, 0, bytes, bitoffs, 8*j+bitsize);
+ data_dst = ep;
ep += j + 1;
} else {
/*
@@ -2003,11 +2764,18 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags,
put_int32((j+1), ep);
ep += 4;
ep[j] = 0; /* Zero unused bits at end of binary */
- copy_binary_to_buffer(ep, 0, bytes, bitoffs, 8*j+bitsize);
+ data_dst = ep;
ep += j+1;
*ep++ = SMALL_INTEGER_EXT;
*ep++ = bitsize;
}
+ if (ctx && j > r * TERM_TO_BINARY_MEMCPY_FACTOR) {
+ WSTACK_PUSH5(s, (UWord)data_dst, (UWord)bytes, bitoffs,
+ ENC_BIN_COPY, 8*j + bitsize);
+ } else {
+ copy_binary_to_buffer(data_dst, 0, bytes, bitoffs,
+ 8 * j + bitsize);
+ }
}
break;
case EXPORT_DEF:
@@ -2036,13 +2804,12 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags,
case FUN_DEF:
{
ErlFunThing* funp = (ErlFunThing *) fun_val(obj);
+ int ei;
if ((dflags & DFLAG_NEW_FUN_TAGS) != 0) {
- int ei;
-
*ep++ = NEW_FUN_EXT;
- WSTACK_PUSH(s, ENC_PATCH_FUN_SIZE);
- WSTACK_PUSH(s, (UWord) ep); /* Position for patching in size */
+ WSTACK_PUSH2(s, ENC_PATCH_FUN_SIZE,
+ (UWord) ep); /* Position for patching in size */
ep += 4;
*ep = funp->arity;
ep += 1;
@@ -2056,16 +2823,6 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags,
ep = enc_term(acmp, make_small(funp->fe->old_index), ep, dflags, off_heap);
ep = enc_term(acmp, make_small(funp->fe->old_uniq), ep, dflags, off_heap);
ep = enc_pid(acmp, funp->creator, ep, dflags);
-
- fun_env:
- for (ei = funp->num_free-1; ei > 0; ei--) {
- WSTACK_PUSH(s, ENC_TERM);
- WSTACK_PUSH(s, (UWord) funp->env[ei]);
- }
- if (funp->num_free != 0) {
- obj = funp->env[0];
- goto L_jump_start;
- }
} else {
/*
* Communicating with an obsolete erl_interface or
@@ -2097,14 +2854,25 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags,
*ep++ = SMALL_TUPLE_EXT;
put_int8(funp->num_free, ep);
ep += 1;
- goto fun_env;
+ }
+ for (ei = funp->num_free-1; ei > 0; ei--) {
+ WSTACK_PUSH2(s, ENC_TERM, (UWord) funp->env[ei]);
+ }
+ if (funp->num_free != 0) {
+ obj = funp->env[0];
+ goto L_jump_start;
}
}
break;
}
}
DESTROY_WSTACK(s);
- return ep;
+ if (ctx) {
+ ASSERT(ctx->wstack.wstart == NULL);
+ *reds = r;
+ }
+ *res = ep;
+ return 0;
}
static
@@ -2146,49 +2914,134 @@ is_external_string(Eterm list, int* p_is_string)
return len;
}
-/* Assumes that the ones to undo are preluding the list. */
-static void
-undo_offheap_in_area(ErlOffHeap* off_heap, Eterm* start, Eterm* end)
-{
- const Uint area_sz = (end - start) * sizeof(Eterm);
- struct erl_off_heap_header* hdr;
- struct erl_off_heap_header** hdr_nextp = NULL;
- for (hdr = off_heap->first; ; hdr=hdr->next) {
- if (!in_area(hdr, start, area_sz)) {
- if (hdr_nextp != NULL) {
- *hdr_nextp = NULL;
- erts_cleanup_offheap(off_heap);
- off_heap->first = hdr;
- }
- break;
- }
- hdr_nextp = &hdr->next;
- }
+struct dec_term_hamt
+{
+ Eterm* objp; /* write result here */
+ Uint size; /* nr of leafs */
+ Eterm* leaf_array;
+};
- /* Assert that the ones to undo were indeed preluding the list. */
-#ifdef DEBUG
- for (hdr = off_heap->first; hdr != NULL; hdr = hdr->next) {
- ASSERT(!in_area(hdr, start, area_sz));
- }
-#endif /* DEBUG */
-}
/* Decode term from external format into *objp.
-** On failure return NULL and (R13B04) *hpp will be unchanged.
+** On failure calls erts_factory_undo() and returns NULL
*/
static byte*
-dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Eterm* objp)
+dec_term(ErtsDistExternal *edep,
+ ErtsHeapFactory* factory,
+ byte* ep,
+ Eterm* objp,
+ B2TContext* ctx)
{
- Eterm* hp_saved = *hpp;
+#define PSTACK_TYPE struct dec_term_hamt
+ PSTACK_DECLARE(hamt_array, 5);
int n;
ErtsAtomEncoding char_enc;
- register Eterm* hp = *hpp; /* Please don't take the address of hp */
- Eterm* next = objp;
+ register Eterm* hp; /* Please don't take the address of hp */
+ DECLARE_WSTACK(flat_maps); /* for preprocessing of small maps */
+ Eterm* next;
+ SWord reds;
+#ifdef DEBUG
+ Eterm* dbg_resultp = ctx ? &ctx->u.dc.res : objp;
+#endif
- *next = (Eterm) (UWord) NULL;
+ if (ctx) {
+ reds = ctx->reds;
+ next = ctx->u.dc.next;
+ ep = ctx->u.dc.ep;
+ factory = &ctx->u.dc.factory;
+
+ if (ctx->state != B2TDecode) {
+ int n_limit = reds;
+
+ n = ctx->u.dc.remaining_n;
+ if (ctx->state == B2TDecodeBinary) {
+ n_limit *= B2T_MEMCPY_FACTOR;
+ ASSERT(n_limit >= reds);
+ reds -= n / B2T_MEMCPY_FACTOR;
+ }
+ else
+ reds -= n;
+
+ if (n > n_limit) {
+ ctx->u.dc.remaining_n -= n_limit;
+ n = n_limit;
+ reds = 0;
+ }
+ else {
+ ctx->u.dc.remaining_n = 0;
+ }
+
+ switch (ctx->state) {
+ case B2TDecodeList:
+ objp = next - 2;
+ while (n > 0) {
+ objp[0] = (Eterm) COMPRESS_POINTER(next);
+ objp[1] = make_list(next);
+ next = objp;
+ objp -= 2;
+ n--;
+ }
+ break;
+
+ case B2TDecodeTuple:
+ objp = next - 1;
+ while (n-- > 0) {
+ objp[0] = (Eterm) COMPRESS_POINTER(next);
+ next = objp;
+ objp--;
+ }
+ break;
+
+ case B2TDecodeString:
+ hp = factory->hp;
+ hp[-1] = make_list(hp); /* overwrite the premature NIL */
+ while (n-- > 0) {
+ hp[0] = make_small(*ep++);
+ hp[1] = make_list(hp+2);
+ hp += 2;
+ }
+ hp[-1] = NIL;
+ factory->hp = hp;
+ break;
+
+ case B2TDecodeBinary:
+ sys_memcpy(ctx->u.dc.remaining_bytes, ep, n);
+ ctx->u.dc.remaining_bytes += n;
+ ep += n;
+ break;
+
+ default:
+ ASSERT(!"Unknown state");
+ }
+ if (!ctx->u.dc.remaining_n) {
+ ctx->state = B2TDecode;
+ }
+ if (reds <= 0) {
+ ctx->u.dc.next = next;
+ ctx->u.dc.ep = ep;
+ ctx->reds = 0;
+ return NULL;
+ }
+ }
+ PSTACK_CHANGE_ALLOCATOR(hamt_array, ERTS_ALC_T_SAVED_ESTACK);
+ WSTACK_CHANGE_ALLOCATOR(flat_maps, ERTS_ALC_T_SAVED_ESTACK);
+ if (ctx->u.dc.hamt_array.pstart) {
+ PSTACK_RESTORE(hamt_array, &ctx->u.dc.hamt_array);
+ }
+ if (ctx->u.dc.flat_maps.wstart) {
+ WSTACK_RESTORE(flat_maps, &ctx->u.dc.flat_maps);
+ }
+ }
+ else {
+ reds = ERTS_SWORD_MAX;
+ next = objp;
+ *next = (Eterm) (UWord) NULL;
+ }
+ hp = factory->hp;
while (next != NULL) {
+
objp = next;
next = (Eterm *) EXPAND_POINTER(*objp);
@@ -2247,6 +3100,8 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Et
big = make_small(0);
} else {
big = bytes_to_big(first, n, neg, hp);
+ if (is_nil(big))
+ goto error;
if (is_big(big)) {
hp += big_arity(big) + 1;
}
@@ -2308,7 +3163,16 @@ dec_term_atom_common:
*objp = make_tuple(hp);
*hp++ = make_arityval(n);
hp += n;
- objp = hp - 1;
+ objp = hp - 1;
+ if (ctx) {
+ if (reds < n) {
+ ASSERT(reds > 0);
+ ctx->state = B2TDecodeTuple;
+ ctx->u.dc.remaining_n = n - reds;
+ n = reds;
+ }
+ reds -= n;
+ }
while (n-- > 0) {
objp[0] = (Eterm) COMPRESS_POINTER(next);
next = objp;
@@ -2326,17 +3190,27 @@ dec_term_atom_common:
break;
}
*objp = make_list(hp);
- hp += 2*n;
+ hp += 2 * n;
objp = hp - 2;
objp[0] = (Eterm) COMPRESS_POINTER((objp+1));
objp[1] = (Eterm) COMPRESS_POINTER(next);
next = objp;
objp -= 2;
- while (--n > 0) {
+ n--;
+ if (ctx) {
+ if (reds < n) {
+ ctx->state = B2TDecodeList;
+ ctx->u.dc.remaining_n = n - reds;
+ n = reds;
+ }
+ reds -= n;
+ }
+ while (n > 0) {
objp[0] = (Eterm) COMPRESS_POINTER(next);
- objp[1] = make_list(objp + 2);
+ objp[1] = make_list(next);
next = objp;
objp -= 2;
+ n--;
}
break;
case STRING_EXT:
@@ -2347,6 +3221,14 @@ dec_term_atom_common:
break;
}
*objp = make_list(hp);
+ if (ctx) {
+ if (reds < n) {
+ ctx->state = B2TDecodeString;
+ ctx->u.dc.remaining_n = n - reds;
+ n = reds;
+ }
+ reds -= n;
+ }
while (n-- > 0) {
hp[0] = make_small(*ep++);
hp[1] = make_list(hp+2);
@@ -2374,7 +3256,7 @@ dec_term_atom_common:
volatile unsigned long *fpexnp = erts_get_current_fp_exception();
#endif
-#ifdef WORDS_BIGENDIAN
+#if defined(WORDS_BIGENDIAN) || defined(DOUBLE_MIDDLE_ENDIAN)
ff.fw[0] = get_int32(ep);
ep += 4;
ff.fw[1] = get_int32(ep);
@@ -2393,9 +3275,9 @@ dec_term_atom_common:
break;
}
case PID_EXT:
- *hpp = hp;
- ep = dec_pid(edep, hpp, ep, off_heap, objp);
- hp = *hpp;
+ factory->hp = hp;
+ ep = dec_pid(edep, factory, ep, objp);
+ hp = factory->hp;
if (ep == NULL) {
goto error;
}
@@ -2429,11 +3311,11 @@ dec_term_atom_common:
hp += EXTERNAL_THING_HEAD_SIZE + 1;
etp->header = make_external_port_header(1);
- etp->next = off_heap->first;
+ etp->next = factory->off_heap->first;
etp->node = node;
etp->data.ui[0] = num;
- off_heap->first = (struct erl_off_heap_header*)etp;
+ factory->off_heap->first = (struct erl_off_heap_header*)etp;
*objp = make_external_port(etp);
}
@@ -2513,10 +3395,10 @@ dec_term_atom_common:
#else
etp->header = make_external_ref_header(ref_words);
#endif
- etp->next = off_heap->first;
+ etp->next = factory->off_heap->first;
etp->node = node;
- off_heap->first = (struct erl_off_heap_header*)etp;
+ factory->off_heap->first = (struct erl_off_heap_header*)etp;
*objp = make_external_ref(etp);
ref_num = &(etp->data.ui32[0]);
}
@@ -2540,7 +3422,7 @@ dec_term_atom_common:
n = get_int32(ep);
ep += 4;
- if (n <= ERL_ONHEAP_BIN_LIMIT) {
+ if ((unsigned)n <= ERL_ONHEAP_BIN_LIMIT) {
ErlHeapBin* hb = (ErlHeapBin *) hp;
hb->thing_word = header_heap_bin(n);
@@ -2551,21 +3433,32 @@ dec_term_atom_common:
} else {
Binary* dbin = erts_bin_nrml_alloc(n);
ProcBin* pb;
- dbin->flags = 0;
- dbin->orig_size = n;
erts_refc_init(&dbin->refc, 1);
- sys_memcpy(dbin->orig_bytes, ep, n);
pb = (ProcBin *) hp;
hp += PROC_BIN_SIZE;
pb->thing_word = HEADER_PROC_BIN;
pb->size = n;
- pb->next = off_heap->first;
- off_heap->first = (struct erl_off_heap_header*)pb;
+ pb->next = factory->off_heap->first;
+ factory->off_heap->first = (struct erl_off_heap_header*)pb;
+ OH_OVERHEAD(factory->off_heap, pb->size / sizeof(Eterm));
pb->val = dbin;
pb->bytes = (byte*) dbin->orig_bytes;
pb->flags = 0;
*objp = make_binary(pb);
- }
+ if (ctx) {
+ int n_limit = reds * B2T_MEMCPY_FACTOR;
+ if (n > n_limit) {
+ ctx->state = B2TDecodeBinary;
+ ctx->u.dc.remaining_n = n - n_limit;
+ ctx->u.dc.remaining_bytes = dbin->orig_bytes + n_limit;
+ n = n_limit;
+ reds = 0;
+ }
+ else
+ reds -= n / B2T_MEMCPY_FACTOR;
+ }
+ sys_memcpy(dbin->orig_bytes, ep, n);
+ }
ep += n;
break;
}
@@ -2577,8 +3470,10 @@ dec_term_atom_common:
n = get_int32(ep);
bitsize = ep[4];
- ep += 5;
- if (n <= ERL_ONHEAP_BIN_LIMIT) {
+ if (((bitsize==0) != (n==0)) || bitsize > 8)
+ goto error;
+ ep += 5;
+ if ((unsigned)n <= ERL_ONHEAP_BIN_LIMIT) {
ErlHeapBin* hb = (ErlHeapBin *) hp;
hb->thing_word = header_heap_bin(n);
@@ -2586,29 +3481,44 @@ dec_term_atom_common:
sys_memcpy(hb->data, ep, n);
bin = make_binary(hb);
hp += heap_bin_size(n);
+ ep += n;
} else {
Binary* dbin = erts_bin_nrml_alloc(n);
ProcBin* pb;
- dbin->flags = 0;
- dbin->orig_size = n;
+
erts_refc_init(&dbin->refc, 1);
- sys_memcpy(dbin->orig_bytes, ep, n);
pb = (ProcBin *) hp;
pb->thing_word = HEADER_PROC_BIN;
pb->size = n;
- pb->next = off_heap->first;
- off_heap->first = (struct erl_off_heap_header*)pb;
+ pb->next = factory->off_heap->first;
+ factory->off_heap->first = (struct erl_off_heap_header*)pb;
+ OH_OVERHEAD(factory->off_heap, pb->size / sizeof(Eterm));
pb->val = dbin;
pb->bytes = (byte*) dbin->orig_bytes;
pb->flags = 0;
bin = make_binary(pb);
hp += PROC_BIN_SIZE;
- }
- ep += n;
- if (bitsize == 0) {
+ if (ctx) {
+ int n_limit = reds * B2T_MEMCPY_FACTOR;
+ if (n > n_limit) {
+ ctx->state = B2TDecodeBinary;
+ ctx->u.dc.remaining_n = n - n_limit;
+ ctx->u.dc.remaining_bytes = dbin->orig_bytes + n_limit;
+ n = n_limit;
+ reds = 0;
+ }
+ else
+ reds -= n / B2T_MEMCPY_FACTOR;
+ }
+ sys_memcpy(dbin->orig_bytes, ep, n);
+ ep += n;
+ n = pb->size;
+ }
+
+ if (bitsize == 8 || n == 0) {
*objp = bin;
} else {
- sb = (ErlSubBin *) hp;
+ sb = (ErlSubBin *)hp;
sb->thing_word = HEADER_SUB_BIN;
sb->orig = bin;
sb->size = n - 1;
@@ -2634,9 +3544,9 @@ dec_term_atom_common:
if ((ep = dec_atom(edep, ep, &name)) == NULL) {
goto error;
}
- *hpp = hp;
- ep = dec_term(edep, hpp, ep, off_heap, &temp);
- hp = *hpp;
+ factory->hp = hp;
+ ep = dec_term(edep, factory, ep, &temp, NULL);
+ hp = factory->hp;
if (ep == NULL) {
goto error;
}
@@ -2662,6 +3572,61 @@ dec_term_atom_common:
break;
}
break;
+ case MAP_EXT:
+ {
+ Uint32 size,n;
+ Eterm *kptr,*vptr;
+ Eterm keys;
+
+ size = get_int32(ep); ep += 4;
+
+ if (size <= MAP_SMALL_MAP_LIMIT) {
+ flatmap_t *mp;
+
+ keys = make_tuple(hp);
+ *hp++ = make_arityval(size);
+ hp += size;
+ kptr = hp - 1;
+
+ mp = (flatmap_t*)hp;
+ hp += MAP_HEADER_FLATMAP_SZ;
+ hp += size;
+ vptr = hp - 1;
+
+ /* kptr, last word for keys
+ * vptr, last word for values
+ */
+
+ WSTACK_PUSH(flat_maps, (UWord)mp);
+ mp->thing_word = MAP_HEADER_FLATMAP;
+ mp->size = size;
+ mp->keys = keys;
+ *objp = make_flatmap(mp);
+
+ for (n = size; n; n--) {
+ *vptr = (Eterm) COMPRESS_POINTER(next);
+ *kptr = (Eterm) COMPRESS_POINTER(vptr);
+ next = kptr;
+ vptr--;
+ kptr--;
+ }
+ }
+ else { /* Make hamt */
+ struct dec_term_hamt* hamt = PSTACK_PUSH(hamt_array);
+
+ hamt->objp = objp;
+ hamt->size = size;
+ hamt->leaf_array = hp;
+
+ for (n = size; n; n--) {
+ CDR(hp) = (Eterm) COMPRESS_POINTER(next);
+ CAR(hp) = (Eterm) COMPRESS_POINTER(&CDR(hp));
+ next = &CAR(hp);
+ hp += 2;
+ }
+ }
+ }
+ break;
case NEW_FUN_EXT:
{
ErlFunThing* funp = (ErlFunThing *) hp;
@@ -2693,9 +3658,9 @@ dec_term_atom_common:
if ((ep = dec_atom(edep, ep, &module)) == NULL) {
goto error;
}
- *hpp = hp;
+ factory->hp = hp;
/* Index */
- if ((ep = dec_term(edep, hpp, ep, off_heap, &temp)) == NULL) {
+ if ((ep = dec_term(edep, factory, ep, &temp, NULL)) == NULL) {
goto error;
}
if (!is_small(temp)) {
@@ -2704,7 +3669,7 @@ dec_term_atom_common:
old_index = unsigned_val(temp);
/* Uniq */
- if ((ep = dec_term(edep, hpp, ep, off_heap, &temp)) == NULL) {
+ if ((ep = dec_term(edep, factory, ep, &temp, NULL)) == NULL) {
goto error;
}
if (!is_small(temp)) {
@@ -2716,8 +3681,8 @@ dec_term_atom_common:
* It is safe to link the fun into the fun list only when
* no more validity tests can fail.
*/
- funp->next = off_heap->first;
- off_heap->first = (struct erl_off_heap_header*)funp;
+ funp->next = factory->off_heap->first;
+ factory->off_heap->first = (struct erl_off_heap_header*)funp;
funp->fe = erts_put_fun_entry2(module, old_uniq, old_index,
uniq, index, arity);
@@ -2728,7 +3693,7 @@ dec_term_atom_common:
}
funp->native_address = funp->fe->native_address;
#endif
- hp = *hpp;
+ hp = factory->hp;
/* Environment */
for (i = num_free-1; i >= 0; i--) {
@@ -2754,14 +3719,14 @@ dec_term_atom_common:
ep += 4;
hp += ERL_FUN_SIZE;
hp += num_free;
- *hpp = hp;
+ factory->hp = hp;
funp->thing_word = HEADER_FUN;
funp->num_free = num_free;
*objp = make_fun(funp);
/* Creator pid */
if (*ep != PID_EXT
- || (ep = dec_pid(edep, hpp, ++ep, off_heap,
+ || (ep = dec_pid(edep, factory, ++ep,
&funp->creator))==NULL) {
goto error;
}
@@ -2772,7 +3737,7 @@ dec_term_atom_common:
}
/* Index */
- if ((ep = dec_term(edep, hpp, ep, off_heap, &temp)) == NULL) {
+ if ((ep = dec_term(edep, factory, ep, &temp, NULL)) == NULL) {
goto error;
}
if (!is_small(temp)) {
@@ -2781,7 +3746,7 @@ dec_term_atom_common:
old_index = unsigned_val(temp);
/* Uniq */
- if ((ep = dec_term(edep, hpp, ep, off_heap, &temp)) == NULL) {
+ if ((ep = dec_term(edep, factory, ep, &temp, NULL)) == NULL) {
goto error;
}
if (!is_small(temp)) {
@@ -2792,8 +3757,8 @@ dec_term_atom_common:
* It is safe to link the fun into the fun list only when
* no more validity tests can fail.
*/
- funp->next = off_heap->first;
- off_heap->first = (struct erl_off_heap_header*)funp;
+ funp->next = factory->off_heap->first;
+ factory->off_heap->first = (struct erl_off_heap_header*)funp;
old_uniq = unsigned_val(temp);
funp->fe = erts_put_fun_entry(module, old_uniq, old_index);
@@ -2801,7 +3766,7 @@ dec_term_atom_common:
#ifdef HIPE
funp->native_address = funp->fe->native_address;
#endif
- hp = *hpp;
+ hp = factory->hp;
/* Environment */
for (i = num_free-1; i >= 0; i--) {
@@ -2835,8 +3800,9 @@ dec_term_atom_common:
erts_refc_inc(&pb->val->refc, 1);
hp += PROC_BIN_SIZE;
- pb->next = off_heap->first;
- off_heap->first = (struct erl_off_heap_header*)pb;
+ pb->next = factory->off_heap->first;
+ factory->off_heap->first = (struct erl_off_heap_header*)pb;
+ OH_OVERHEAD(factory->off_heap, pb->size / sizeof(Eterm));
pb->flags = 0;
*objp = make_binary(pb);
break;
@@ -2852,8 +3818,9 @@ dec_term_atom_common:
erts_refc_inc(&pb->val->refc, 1);
hp += PROC_BIN_SIZE;
- pb->next = off_heap->first;
- off_heap->first = (struct erl_off_heap_header*)pb;
+ pb->next = factory->off_heap->first;
+ factory->off_heap->first = (struct erl_off_heap_header*)pb;
+ OH_OVERHEAD(factory->off_heap, pb->size / sizeof(Eterm));
pb->flags = 0;
sub = (ErlSubBin*)hp;
@@ -2871,83 +3838,139 @@ dec_term_atom_common:
}
default:
- error:
- /* UNDO:
- * Must unlink all off-heap objects that may have been
- * linked into the process.
- */
- if (hp < *hpp) { /* Sometimes we used hp and sometimes *hpp */
- hp = *hpp; /* the largest must be the freshest */
- }
- undo_offheap_in_area(off_heap, hp_saved, hp);
- *hpp = hp_saved;
- return NULL;
+ goto error;
}
+
+ if (--reds <= 0) {
+ if (ctx) {
+ if (next || ctx->state != B2TDecode) {
+ ctx->u.dc.ep = ep;
+ ctx->u.dc.next = next;
+ ctx->u.dc.factory.hp = hp;
+ if (!WSTACK_ISEMPTY(flat_maps)) {
+ WSTACK_SAVE(flat_maps, &ctx->u.dc.flat_maps);
+ }
+ if (!PSTACK_IS_EMPTY(hamt_array)) {
+ PSTACK_SAVE(hamt_array, &ctx->u.dc.hamt_array);
+ }
+ ctx->reds = 0;
+ return NULL;
+ }
+ }
+ else {
+ reds = ERTS_SWORD_MAX;
+ }
+ }
+ }
+
+ ASSERT(hp <= factory->hp_end
+ || (factory->mode == FACTORY_CLOSED && is_immed(*dbg_resultp)));
+ factory->hp = hp;
+ /*
+ * From here on factory may produce (more) heap fragments
+ */
+
+ if (!PSTACK_IS_EMPTY(hamt_array)) {
+ do {
+ struct dec_term_hamt* hamt = PSTACK_TOP(hamt_array);
+
+ *hamt->objp = erts_hashmap_from_array(factory,
+ hamt->leaf_array,
+ hamt->size,
+ 1);
+ if (is_non_value(*hamt->objp))
+ goto error_hamt;
+
+ (void) PSTACK_POP(hamt_array);
+ } while (!PSTACK_IS_EMPTY(hamt_array));
+ PSTACK_DESTROY(hamt_array);
+ }
+
+ /* Iterate through all the (flat)maps and check for validity and sort keys
+ * - done here for when we know it is complete.
+ */
+
+ while(!WSTACK_ISEMPTY(flat_maps)) {
+ next = (Eterm *)WSTACK_POP(flat_maps);
+ if (!erts_validate_and_sort_flatmap((flatmap_t*)next))
+ goto error;
+ }
+ WSTACK_DESTROY(flat_maps);
+
+ ASSERT((Eterm*)EXPAND_POINTER(*dbg_resultp) != NULL);
+
+ if (ctx) {
+ ctx->state = B2TDone;
+ ctx->reds = reds;
}
- *hpp = hp;
+
return ep;
+
+error:
+ /* UNDO:
+ * Must unlink all off-heap objects that may have been
+ * linked into the process.
+ */
+ if (factory->hp < hp) { /* Sometimes we used hp and sometimes factory->hp */
+ factory->hp = hp; /* the largest must be the freshest */
+ }
+error_hamt:
+ erts_factory_undo(factory);
+ PSTACK_DESTROY(hamt_array);
+ if (ctx) {
+ ctx->state = B2TDecodeFail;
+ ctx->reds = reds;
+ }
+ WSTACK_DESTROY(flat_maps);
+
+ return NULL;
}
/* returns the number of bytes needed to encode an object
to a sequence of bytes
N.B. That this must agree with to_external2() above!!!
(except for cached atoms) */
+static Uint encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) {
+ Uint res;
+ (void) encode_size_struct_int(NULL, acmp, obj, dflags, NULL, &res);
+ return res;
+}
-static Uint
-encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags)
+static int
+encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj,
+ unsigned dflags, Sint *reds, Uint *res)
{
DECLARE_WSTACK(s);
Uint m, i, arity;
Uint result = 0;
-#if HALFWORD_HEAP
- UWord wobj = 0;
-#endif
+ Sint r = 0;
- goto L_jump_start;
+ if (ctx) {
+ WSTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK);
+ r = *reds;
- outer_loop:
- while (!WSTACK_ISEMPTY(s)) {
-#if HALFWORD_HEAP
- obj = (Eterm) (wobj = WSTACK_POP(s));
-#else
- obj = WSTACK_POP(s);
-#endif
- handle_popped_obj:
- if (is_CP(obj)) { /* Does not look for CP, looks for "no tag" */
-#if HALFWORD_HEAP
- Eterm* ptr = (Eterm *) wobj;
-#else
- Eterm* ptr = (Eterm *) obj;
-#endif
- /*
- * Pointer into a tuple.
- */
- obj = *ptr--;
- if (!is_header(obj)) {
- WSTACK_PUSH(s, (UWord)ptr);
- } else {
- /* Reached tuple header */
- ASSERT(header_is_arityval(obj));
- goto outer_loop;
- }
- } else if (is_list(obj)) {
- Eterm* cons = list_val(obj);
- Eterm tl;
+ if (ctx->wstack.wstart) { /* restore saved stack */
+ WSTACK_RESTORE(s, &ctx->wstack);
+ result = ctx->result;
+ obj = ctx->obj;
+ }
+ }
- tl = CDR(cons);
- obj = CAR(cons);
- WSTACK_PUSH(s, tl);
- } else if (is_nil(obj)) {
- result++;
- goto outer_loop;
- } else {
- /*
- * Other term (in the tail of a non-proper list or
- * in a fun's environment).
- */
+#define LIST_TAIL_OP ((0 << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER)
+#define TERM_ARRAY_OP(N) (((N) << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER)
+#define TERM_ARRAY_OP_DEC(OP) ((OP) - (1 << _TAG_PRIMARY_SIZE))
+
+
+ for (;;) {
+ ASSERT(!is_header(obj));
+
+ if (ctx && --r == 0) {
+ *reds = r;
+ ctx->obj = obj;
+ ctx->result = result;
+ WSTACK_SAVE(s, &ctx->wstack);
+ return -1;
}
-
- L_jump_start:
switch (tag_val_def(obj)) {
case NIL_DEF:
result++;
@@ -3028,26 +4051,79 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags)
result += m + 2 + 1;
} else {
result += 5;
- goto handle_popped_obj;
+ WSTACK_PUSH2(s, (UWord)CDR(list_val(obj)), (UWord)LIST_TAIL_OP);
+ obj = CAR(list_val(obj));
+ continue; /* big loop */
}
break;
case TUPLE_DEF:
{
Eterm* ptr = tuple_val(obj);
-
arity = arityval(*ptr);
if (arity <= 0xff) {
result += 1 + 1;
} else {
result += 1 + 4;
}
- ptr += arity;
-#if HALFWORD_HEAP
- obj = (Eterm) (wobj = (UWord) ptr);
-#else
- obj = (Eterm) ptr;
-#endif
- goto handle_popped_obj;
+ if (arity > 1) {
+ WSTACK_PUSH2(s, (UWord) (ptr + 2),
+ (UWord) TERM_ARRAY_OP(arity-1));
+ }
+ else if (arity == 0) {
+ break;
+ }
+ obj = ptr[1];
+ continue; /* big loop */
+ }
+ case MAP_DEF:
+ if (is_flatmap(obj)) {
+ flatmap_t *mp = (flatmap_t*)flatmap_val(obj);
+ Uint size = flatmap_get_size(mp);
+
+ result += 1 + 4; /* tag + 4 bytes size */
+
+ if (size) {
+ WSTACK_PUSH4(s, (UWord) flatmap_get_values(mp),
+ (UWord) TERM_ARRAY_OP(size),
+ (UWord) flatmap_get_keys(mp),
+ (UWord) TERM_ARRAY_OP(size));
+ }
+ } else {
+ Eterm *ptr;
+ Eterm hdr;
+ Uint node_sz;
+ ptr = boxed_val(obj);
+ hdr = *ptr;
+ ASSERT(is_header(hdr));
+ switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
+ case HAMT_SUBTAG_HEAD_ARRAY:
+ ptr++;
+ node_sz = 16;
+ result += 1 + 4; /* tag + 4 bytes size */
+ break;
+ case HAMT_SUBTAG_HEAD_BITMAP:
+ ptr++;
+ result += 1 + 4; /* tag + 4 bytes size */
+ /*fall through*/
+ case HAMT_SUBTAG_NODE_BITMAP:
+ node_sz = hashmap_bitcount(MAP_HEADER_VAL(hdr));
+ ASSERT(node_sz < 17);
+ break;
+ default:
+ erts_exit(ERTS_ERROR_EXIT, "bad header\r\n");
+ }
+
+ ptr++;
+ WSTACK_RESERVE(s, node_sz*2);
+ while(node_sz--) {
+ if (is_list(*ptr)) {
+ WSTACK_FAST_PUSH(s, CAR(list_val(*ptr)));
+ WSTACK_FAST_PUSH(s, CDR(list_val(*ptr)));
+ } else {
+ WSTACK_FAST_PUSH(s, *ptr);
+ }
+ ptr++;
+ }
}
break;
case FLOAT_DEF:
@@ -3100,25 +4176,13 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags)
result += 2 * (1 + 4); /* Index + Uniq */
result += 1 + (funp->num_free < 0x100 ? 1 : 4);
}
- for (i = 1; i < funp->num_free; i++) {
- obj = funp->env[i];
-
- if (is_not_list(obj)) {
- /* Push any non-list terms on the stack */
- WSTACK_PUSH(s, obj);
- } else {
- /* Lists must be handled specially. */
- if ((m = is_string(obj)) && (m < MAX_STRING_LEN)) {
- result += m + 2 + 1;
- } else {
- result += 5;
- WSTACK_PUSH(s, obj);
- }
- }
+ if (funp->num_free > 1) {
+ WSTACK_PUSH2(s, (UWord) (funp->env + 1),
+ (UWord) TERM_ARRAY_OP(funp->num_free-1));
}
if (funp->num_free != 0) {
obj = funp->env[0];
- goto L_jump_start;
+ continue; /* big loop */
}
break;
}
@@ -3138,39 +4202,83 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags)
break;
default:
- erl_exit(1,"Internal data structure error (in encode_size_struct2)%x\n",
+ erts_exit(ERTS_ERROR_EXIT,"Internal data structure error (in encode_size_struct2)%x\n",
obj);
}
+
+ if (WSTACK_ISEMPTY(s)) {
+ break;
+ }
+ obj = (Eterm) WSTACK_POP(s);
+
+ if (is_header(obj)) {
+ switch (obj) {
+ case LIST_TAIL_OP:
+ obj = (Eterm) WSTACK_POP(s);
+ if (is_list(obj)) {
+ Eterm* cons = list_val(obj);
+
+ WSTACK_PUSH2(s, (UWord)CDR(cons), (UWord)LIST_TAIL_OP);
+ obj = CAR(cons);
+ }
+ break;
+
+ case TERM_ARRAY_OP(1):
+ obj = *(Eterm*)WSTACK_POP(s);
+ break;
+ default: { /* TERM_ARRAY_OP(N) when N > 1 */
+ Eterm* ptr = (Eterm*) WSTACK_POP(s);
+ WSTACK_PUSH2(s, (UWord) (ptr+1),
+ (UWord) TERM_ARRAY_OP_DEC(obj));
+ obj = *ptr;
+ }
+ }
+ }
}
- DESTROY_WSTACK(s);
- return result;
+ WSTACK_DESTROY(s);
+ if (ctx) {
+ ASSERT(ctx->wstack.wstart == NULL);
+ *reds = r;
+ }
+ *res = result;
+ return 0;
}
-static int is_valid_utf8_atom(byte* bytes, Uint nbytes)
-{
- byte* err_pos;
- Uint num_chars;
- /*SVERK Do we really need to validate correct utf8? */
- return nbytes <= MAX_ATOM_SZ_LIMIT
- && erts_analyze_utf8(bytes, nbytes, &err_pos, &num_chars, NULL) == ERTS_UTF8_OK
- && num_chars <= MAX_ATOM_CHARACTERS;
-}
static Sint
-decoded_size(byte *ep, byte* endp, int internal_tags)
+decoded_size(byte *ep, byte* endp, int internal_tags, B2TContext* ctx)
{
- int heap_size = 0;
+ int heap_size;
int terms;
- int atom_extra_skip = 0;
+ int atom_extra_skip;
Uint n;
+ SWord reds;
+
+ if (ctx) {
+ reds = ctx->reds;
+ if (ctx->u.sc.ep) {
+ heap_size = ctx->u.sc.heap_size;
+ terms = ctx->u.sc.terms;
+ ep = ctx->u.sc.ep;
+ atom_extra_skip = ctx->u.sc.atom_extra_skip;
+ goto init_done;
+ }
+ }
+ else
+ reds = 0; /* not used but compiler warns anyway */
+
+ heap_size = 0;
+ terms = 1;
+ atom_extra_skip = 0;
+init_done:
#define SKIP(sz) \
do { \
if ((sz) <= endp-ep) { \
ep += (sz); \
- } else { return -1; }; \
+ } else { goto error; }; \
} while (0)
#define SKIP2(sz1, sz2) \
@@ -3178,31 +4286,32 @@ decoded_size(byte *ep, byte* endp, int internal_tags)
Uint sz = (sz1) + (sz2); \
if (sz1 < sz && (sz) <= endp-ep) { \
ep += (sz); \
- } else { return -1; } \
+ } else { goto error; } \
} while (0)
#define CHKSIZE(sz) \
do { \
- if ((sz) > endp-ep) { return -1; } \
+ if ((sz) > endp-ep) { goto error; } \
} while (0)
#define ADDTERMS(n) \
do { \
int before = terms; \
terms += (n); \
- if (terms < before) return -1; \
+ if (terms < before) goto error; \
} while (0)
-
- for (terms=1; terms > 0; terms--) {
- int tag;
-
+ ASSERT(terms > 0);
+ do {
+ int tag;
CHKSIZE(1);
tag = ep++[0];
switch (tag) {
case INTEGER_EXT:
SKIP(4);
+#if !defined(ARCH_64) || HALFWORD_HEAP
heap_size += BIG_UINT_HEAP_SIZE;
+#endif
break;
case SMALL_INTEGER_EXT:
SKIP(1);
@@ -3217,7 +4326,7 @@ decoded_size(byte *ep, byte* endp, int internal_tags)
CHKSIZE(4);
n = get_int32(ep);
if (n > BIG_ARITY_MAX*sizeof(ErtsDigit)) {
- return -1;
+ goto error;
}
SKIP2(n,4+1); /* skip, size,sign,digits */
heap_size += 1+1+(n+sizeof(Eterm)-1)/sizeof(Eterm); /* XXX: 1 too much? */
@@ -3226,7 +4335,7 @@ decoded_size(byte *ep, byte* endp, int internal_tags)
CHKSIZE(2);
n = get_int16(ep);
if (n > MAX_ATOM_CHARACTERS) {
- return -1;
+ goto error;
}
SKIP(n+2+atom_extra_skip);
atom_extra_skip = 0;
@@ -3235,8 +4344,8 @@ decoded_size(byte *ep, byte* endp, int internal_tags)
CHKSIZE(2);
n = get_int16(ep);
ep += 2;
- if (!is_valid_utf8_atom(ep, n)) {
- return -1;
+ if (n > MAX_ATOM_SZ_LIMIT) {
+ goto error;
}
SKIP(n+atom_extra_skip);
atom_extra_skip = 0;
@@ -3245,7 +4354,7 @@ decoded_size(byte *ep, byte* endp, int internal_tags)
CHKSIZE(1);
n = get_int8(ep);
if (n > MAX_ATOM_CHARACTERS) {
- return -1;
+ goto error;
}
SKIP(n+1+atom_extra_skip);
atom_extra_skip = 0;
@@ -3254,8 +4363,8 @@ decoded_size(byte *ep, byte* endp, int internal_tags)
CHKSIZE(1);
n = get_int8(ep);
ep++;
- if (!is_valid_utf8_atom(ep, n)) {
- return -1;
+ if (n > MAX_ATOM_SZ_LIMIT) {
+ goto error;
}
SKIP(n+atom_extra_skip);
atom_extra_skip = 0;
@@ -3284,7 +4393,7 @@ decoded_size(byte *ep, byte* endp, int internal_tags)
id_words = get_int16(ep);
if (id_words > ERTS_MAX_REF_NUMBERS)
- return -1;
+ goto error;
ep += 2;
atom_extra_skip = 1 + 4*id_words;
@@ -3326,6 +4435,17 @@ decoded_size(byte *ep, byte* endp, int internal_tags)
ADDTERMS(n);
heap_size += n + 1;
break;
+ case MAP_EXT:
+ CHKSIZE(4);
+ n = get_int32(ep);
+ ep += 4;
+ ADDTERMS(2*n);
+ if (n <= MAP_SMALL_MAP_LIMIT) {
+ heap_size += 3 + n + 1 + n;
+ } else {
+ heap_size += HASHMAP_ESTIMATED_HEAP_SIZE(n);
+ }
+ break;
case STRING_EXT:
CHKSIZE(2);
n = get_int16(ep);
@@ -3386,7 +4506,7 @@ decoded_size(byte *ep, byte* endp, int internal_tags)
num_free = get_int32(ep);
ep += 4;
if (num_free > MAX_ARG) {
- return -1;
+ goto error;
}
terms += 4 + num_free;
heap_size += ERL_FUN_SIZE + num_free;
@@ -3403,24 +4523,47 @@ decoded_size(byte *ep, byte* endp, int internal_tags)
case BINARY_INTERNAL_REF:
if (!internal_tags) {
- return -1;
+ goto error;
}
SKIP(sizeof(ProcBin));
heap_size += PROC_BIN_SIZE;
break;
case BIT_BINARY_INTERNAL_REF:
if (!internal_tags) {
- return -1;
+ goto error;
}
SKIP(2+sizeof(ProcBin));
heap_size += PROC_BIN_SIZE + ERL_SUB_BIN_SIZE;
break;
default:
- return -1;
+ goto error;
}
- }
+ terms--;
+
+ if (ctx && --reds <= 0 && terms > 0) {
+ ctx->u.sc.heap_size = heap_size;
+ ctx->u.sc.terms = terms;
+ ctx->u.sc.ep = ep;
+ ctx->u.sc.atom_extra_skip = atom_extra_skip;
+ ctx->reds = 0;
+ return 0;
+ }
+ }while (terms > 0);
+
/* 'terms' may be non-zero if it has wrapped around */
- return terms==0 ? heap_size : -1;
+ if (terms == 0) {
+ if (ctx) {
+ ctx->state = B2TDecodeInit;
+ ctx->reds = reds;
+ }
+ return heap_size;
+ }
+
+error:
+ if (ctx) {
+ ctx->state = B2TBadArg;
+ }
+ return -1;
#undef SKIP
#undef SKIP2
#undef CHKSIZE
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-16 05:36:00 +0000