Merge tag 'OTP_R16B03_yielding_binary_to_term'

Yielding binary_to_term.

OTP-11535

* tag 'OTP_R16B03_yielding_binary_to_term':
  Increase versions for OTP_R16B03_yielding_binary_to_term
  erts: Adjust term_to_binary reduction factors
  erts: Yield after trapping term_to_binary if gc has been ordered
  erts: Let term_to_binary disable gc while trapping
  erts: Improve stress of binary_to_term in binary_SUITE
  erts: Fix bug in binary_to_term for compressed on halfword
  erts: Fix crash when binary_to_term throws badarg
  erts: Trapping memcpy in binary_to_term
  erts: Cleanup code for trapping binary_to_term
  erts: Add erlang wrappers to binary_to_term
  trapping uncompress
  trapping size calculation
  trapping binary_to_term/2
  trapping STRING_EXT
  trapping lists and tuples
  trapping binary_to_term passing binary_SUITE

Conflicts:
	erts/preloaded/ebin/erlang.beam
	erts/preloaded/ebin/erts_internal.beam
	erts/vsn.mk
	lib/kernel/vsn.mk
	lib/stdlib/vsn.mk

1 files changed, 600 insertions, 148 deletions

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 22b0a02937..2cb44a5b64 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -61,6 +61,9 @@
  */
 #    define ERTS_DEBUG_USE_DIST_SEP
 #  endif
+#  define IF_DEBUG(X) X
+#else
+#  define IF_DEBUG(X)
 #endif
 
 /* Does Sint fit in Sint32?
@@ -89,10 +92,11 @@ static int enc_term_int(Process *p,ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 static Uint is_external_string(Eterm obj, int* p_is_string);
 static byte* enc_atom(ErtsAtomCacheMap *, Eterm, byte*, Uint32);
 static byte* enc_pid(ErtsAtomCacheMap *, Eterm, byte*, Uint32);
-static byte* dec_term(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*);
+struct B2TContext_t;
+static byte* dec_term(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*, struct B2TContext_t*);
 static byte* dec_atom(ErtsDistExternal *, byte*, Eterm*);
 static byte* dec_pid(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*);
-static Sint decoded_size(byte *ep, byte* endp, int internal_tags);
+static Sint decoded_size(byte *ep, byte* endp, int internal_tags, struct B2TContext_t*);
 static BIF_RETTYPE term_to_binary_trap_1(BIF_ALIST_1);
 
 static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint flags, 
@@ -102,11 +106,19 @@ static Uint encode_size_struct2(ErtsAtomCacheMap *, Eterm, unsigned);
 static int encode_size_struct_int(Process *p, ErtsAtomCacheMap *acmp, Eterm obj, 
 				  unsigned dflags, Sint *reds, Uint *res);
 
+static Export binary_to_term_trap_export;
+static BIF_RETTYPE binary_to_term_trap_1(BIF_ALIST_1);
+static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* context_b);
+
 void erts_init_external(void) {
 #if 1 /* In R16 */
     erts_init_trap_export(&term_to_binary_trap_export,
 			  am_erlang, am_term_to_binary_trap, 1,
 			  &term_to_binary_trap_1);
+
+    erts_init_trap_export(&binary_to_term_trap_export,
+			  am_erlang, am_binary_to_term_trap, 1,
+			  &binary_to_term_trap_1);
 #else
     sys_memset((void *) &term_to_binary_trap_export, 0, sizeof(Export));
     term_to_binary_trap_export.address = &term_to_binary_trap_export.code[3];
@@ -877,7 +889,7 @@ erts_decode_dist_ext_size(ErtsDistExternal *edep)
 	    goto fail;
 	ep = edep->extp+1;
     }
-    res = decoded_size(ep, edep->ext_endp, 0);
+    res = decoded_size(ep, edep->ext_endp, 0, NULL);
     if (res >= 0)
 	return res;
  fail:
@@ -889,12 +901,12 @@ Sint erts_decode_ext_size(byte *ext, Uint size)
 {
     if (size == 0 || *ext != VERSION_MAGIC)
 	return -1;
-    return decoded_size(ext+1, ext+size, 0);
+    return decoded_size(ext+1, ext+size, 0, NULL);
 }
 
 Sint erts_decode_ext_size_ets(byte *ext, Uint size)
 {
-    Sint sz = decoded_size(ext, ext+size, 1);
+    Sint sz = decoded_size(ext, ext+size, 1, NULL);
     ASSERT(sz >= 0);
     return sz;
 }
@@ -927,7 +939,7 @@ erts_decode_dist_ext(Eterm** hpp,
 	    goto error;
 	ep++;
     }
-    ep = dec_term(edep, hpp, ep, off_heap, &obj);
+    ep = dec_term(edep, hpp, ep, off_heap, &obj, NULL);
     if (!ep)
 	goto error;
 
@@ -948,7 +960,7 @@ Eterm erts_decode_ext(Eterm **hpp, ErlOffHeap *off_heap, byte **ext)
     byte *ep = *ext;
     if (*ep++ != VERSION_MAGIC)
 	return THE_NON_VALUE;
-    ep = dec_term(NULL, hpp, ep, off_heap, &obj);
+    ep = dec_term(NULL, hpp, ep, off_heap, &obj, NULL);
     if (!ep) {
 #ifdef DEBUG
 	bin_write(ERTS_PRINT_STDERR,NULL,*ext,500);
@@ -962,7 +974,7 @@ Eterm erts_decode_ext(Eterm **hpp, ErlOffHeap *off_heap, byte **ext)
 Eterm erts_decode_ext_ets(Eterm **hpp, ErlOffHeap *off_heap, byte *ext)
 {
     Eterm obj;
-    ext = dec_term(NULL, hpp, ext, off_heap, &obj);
+    ext = dec_term(NULL, hpp, ext, off_heap, &obj, NULL);
     ASSERT(ext);
     return obj;
 }
@@ -1043,9 +1055,14 @@ static BIF_RETTYPE term_to_binary_trap_1(BIF_ALIST_1)
     Binary *bin = ((ProcBin *) binary_val(bt))->val;
     Eterm res = erts_term_to_binary_int(BIF_P, Term, 0, 0,bin);
     if (is_tuple(res)) {
+	ASSERT(BIF_P->flags & F_DISABLE_GC);
 	BIF_TRAP1(&term_to_binary_trap_export,BIF_P,res);
     } else {
-	BIF_RET(res);
+        if (erts_set_gc_state(BIF_P, 1)
+            || MSO(BIF_P).overhead > BIN_VHEAP_SZ(BIF_P))
+            ERTS_BIF_YIELD_RETURN(BIF_P, res);
+        else
+            BIF_RET(res);
     }
 }
     
@@ -1053,8 +1070,10 @@ BIF_RETTYPE term_to_binary_1(BIF_ALIST_1)
 {
     Eterm res = erts_term_to_binary_int(BIF_P, BIF_ARG_1, 0, TERM_TO_BINARY_DFLAGS, NULL);
     if (is_tuple(res)) {
+	erts_set_gc_state(BIF_P, 0);
 	BIF_TRAP1(&term_to_binary_trap_export,BIF_P,res);
     } else {
+	ASSERT(!(BIF_P->flags & F_DISABLE_GC));
 	BIF_RET(res);
     }
 }
@@ -1106,12 +1125,72 @@ BIF_RETTYPE term_to_binary_2(BIF_ALIST_2)
 
     res = erts_term_to_binary_int(p, Term, level, flags, bin);
     if (is_tuple(res)) {
+	erts_set_gc_state(p, 0);
 	BIF_TRAP1(&term_to_binary_trap_export,BIF_P,res);
     } else {
+	ASSERT(!(BIF_P->flags & F_DISABLE_GC));
 	BIF_RET(res);
     }
 }
 
+
+enum B2TState { /* order is somewhat significant */
+    B2TPrepare,
+    B2TUncompressChunk,
+    B2TSizeInit,
+    B2TSize,
+    B2TDecodeInit,
+    B2TDecode,
+    B2TDecodeList,
+    B2TDecodeTuple,
+    B2TDecodeString,
+    B2TDecodeBinary,
+
+    B2TDone,
+    B2TDecodeFail,
+    B2TBadArg
+};
+
+typedef struct {
+    int heap_size;
+    int terms;
+    byte* ep;
+    int atom_extra_skip;
+} B2TSizeContext;
+
+typedef struct {
+    byte*  ep;
+    Eterm  res;
+    Eterm* next;
+    Eterm* hp_start;
+    Eterm* hp;
+    Eterm* hp_end;
+    int remaining_n;
+    char* remaining_bytes;
+} B2TDecodeContext;
+
+typedef struct {
+    z_stream stream;
+    byte* dbytes;
+    Uint dleft;
+} B2TUncompressContext;
+
+typedef struct B2TContext_t {
+    Sint heap_size;
+    byte* aligned_alloc;
+    ErtsBinary2TermState b2ts;
+    Uint32 flags;
+    SWord reds;
+    Eterm trap_bin;
+    enum B2TState state;
+    union {
+	B2TSizeContext sc;
+	B2TDecodeContext dc;
+	B2TUncompressContext uc;
+    } u;
+} B2TContext;
+
+
 static uLongf binary2term_uncomp_size(byte* data, Sint size)
 {
     z_stream stream;
@@ -1141,48 +1220,62 @@ static uLongf binary2term_uncomp_size(byte* data, Sint size)
     return err == Z_STREAM_END ? uncomp_size : 0;
 }
 
-static ERTS_INLINE Sint
-binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size)
+static ERTS_INLINE int
+binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size,
+		    B2TContext* ctx)
 {
-    Sint res;
     byte *bytes = data;
     Sint size = data_size;
 
     state->exttmp = 0;
 
     if (size < 1 || *bytes != VERSION_MAGIC) {
-    error:
-	if (state->exttmp)
-	    erts_free(ERTS_ALC_T_TMP, state->extp);
-	state->extp = NULL;
-	state->exttmp = 0;
 	return -1;
     }
     bytes++;
     size--;
     if (size < 5 || *bytes != COMPRESSED) {
 	state->extp = bytes;
+        if (ctx)
+	    ctx->state = B2TSizeInit;
     }
     else  {
 	uLongf dest_len = (Uint32) get_int32(bytes+1);
 	bytes += 5;
 	size -= 5;	
 	if (dest_len > 32*1024*1024
-	    || (state->extp = erts_alloc_fnf(ERTS_ALC_T_TMP, dest_len)) == NULL) {
+	    || (state->extp = erts_alloc_fnf(ERTS_ALC_T_EXT_TERM_DATA, dest_len)) == NULL) {
+            /*
+             * Try avoid out-of-memory crash due to corrupted 'dest_len'
+             * by checking the actual length of the uncompressed data.
+             * The only way to do that is to uncompress it. Sad but true.
+             */
 	    if (dest_len != binary2term_uncomp_size(bytes, size)) {
-		goto error;
+                return -1;
 	    }
-	    state->extp = erts_alloc(ERTS_ALC_T_TMP, dest_len);
+	    state->extp = erts_alloc(ERTS_ALC_T_EXT_TERM_DATA, dest_len);
+            ctx->reds -= dest_len;
 	}
 	state->exttmp = 1;
-	if (erl_zlib_uncompress(state->extp, &dest_len, bytes, size) != Z_OK)
-	    goto error;
+        if (ctx) {
+	    if (erl_zlib_inflate_start(&ctx->u.uc.stream, bytes, size) != Z_OK)
+		return -1;
+
+	    ctx->u.uc.dbytes = state->extp;
+	    ctx->u.uc.dleft = dest_len;
+	    ctx->state = B2TUncompressChunk;
+        }
+	else {
+	    uLongf dlen = dest_len;
+	    if (erl_zlib_uncompress(state->extp, &dlen, bytes, size) != Z_OK
+		|| dlen != dest_len) {
+		return -1;
+	    }
+        }
 	size = (Sint) dest_len;
     }
-    res = decoded_size(state->extp, state->extp + size, 0);
-    if (res < 0)
-	goto error;
-    return res;
+    state->extsize = size;
+    return 0;
 }
 
 static ERTS_INLINE void
@@ -1190,7 +1283,7 @@ binary2term_abort(ErtsBinary2TermState *state)
 {
     if (state->exttmp) {
 	state->exttmp = 0;
-	erts_free(ERTS_ALC_T_TMP, state->extp);
+	erts_free(ERTS_ALC_T_EXT_TERM_DATA, state->extp);
     }
 }
 
@@ -1198,11 +1291,11 @@ static ERTS_INLINE Eterm
 binary2term_create(ErtsDistExternal *edep, ErtsBinary2TermState *state, Eterm **hpp, ErlOffHeap *ohp)
 {
     Eterm res;
-    if (!dec_term(edep, hpp, state->extp, ohp, &res))
+    if (!dec_term(edep, hpp, state->extp, ohp, &res, NULL))
 	res = THE_NON_VALUE;
     if (state->exttmp) {
 	state->exttmp = 0;
-	erts_free(ERTS_ALC_T_TMP, state->extp);
+	erts_free(ERTS_ALC_T_EXT_TERM_DATA, state->extp);
     }
     return res;
 }
@@ -1210,7 +1303,18 @@ binary2term_create(ErtsDistExternal *edep, ErtsBinary2TermState *state, Eterm **
 Sint
 erts_binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size)
 {
-    return binary2term_prepare(state, data, data_size);
+    Sint res;
+
+    if (binary2term_prepare(state, data, data_size, NULL) < 0 ||
+        (res=decoded_size(state->extp, state->extp + state->extsize, 0, NULL)) < 0) {
+
+        if (state->exttmp)
+            erts_free(ERTS_ALC_T_EXT_TERM_DATA, state->extp);
+        state->extp = NULL;
+	state->exttmp = 0;
+	return -1;
+    }
+    return res;
 }
 
 void
@@ -1225,68 +1329,233 @@ erts_binary2term_create(ErtsBinary2TermState *state, Eterm **hpp, ErlOffHeap *oh
     return binary2term_create(NULL,state, hpp, ohp);
 }
 
-BIF_RETTYPE binary_to_term_1(BIF_ALIST_1)
+static void b2t_destroy_context(B2TContext* context)
 {
-    Sint heap_size;
-    Eterm res;
+    erts_free_aligned_binary_bytes_extra(context->aligned_alloc,
+                                         ERTS_ALC_T_EXT_TERM_DATA);
+    context->aligned_alloc = NULL;
+    binary2term_abort(&context->b2ts);
+    if (context->state == B2TUncompressChunk) {
+	erl_zlib_inflate_finish(&context->u.uc.stream);
+    }
+}
+
+static void b2t_context_destructor(Binary *context_bin)
+{
+    B2TContext* ctx = (B2TContext*) ERTS_MAGIC_BIN_DATA(context_bin);
+    ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(context_bin) == b2t_context_destructor);
+
+    b2t_destroy_context(ctx);
+}
+
+static BIF_RETTYPE binary_to_term_trap_1(BIF_ALIST_1)
+{
+    Binary *context_bin = ((ProcBin *) binary_val(BIF_ARG_1))->val;
+    ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(context_bin) == b2t_context_destructor);
+
+    return binary_to_term_int(BIF_P, 0, THE_NON_VALUE, context_bin);
+}
+
+
+#define B2T_BYTES_PER_REDUCTION 128
+#define B2T_MEMCPY_FACTOR 8
+
+/* Define for testing */
+/*#define EXTREME_B2T_TRAPPING 1*/
+
+#ifdef EXTREME_B2T_TRAPPING
+static unsigned b2t_rand(void)
+{
+    static unsigned prev = 17;
+    prev = (prev * 214013 + 2531011);
+    return prev;
+}
+#endif
+
+
+static B2TContext* b2t_export_context(Process* p, B2TContext* src)
+{
+    Binary* context_b = erts_create_magic_binary(sizeof(B2TContext),
+                                                 b2t_context_destructor);
+    B2TContext* ctx = ERTS_MAGIC_BIN_DATA(context_b);
     Eterm* hp;
-    Eterm* endp;
-    Sint size;
-    byte* bytes;
-    byte* temp_alloc = NULL;
-    ErtsBinary2TermState b2ts;
+    sys_memcpy(ctx, src, sizeof(B2TContext));
+    if (ctx->state >= B2TDecode && ctx->u.dc.next == &src->u.dc.res) {
+        ctx->u.dc.next = &ctx->u.dc.res;
+    }
+    hp = HAlloc(p, PROC_BIN_SIZE);
+    ctx->trap_bin = erts_mk_magic_binary_term(&hp, &MSO(p), context_b);
+    return ctx;
+}
 
-    if ((bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc)) == NULL) {
-    error:
-	erts_free_aligned_binary_bytes(temp_alloc);
-	BIF_ERROR(BIF_P, BADARG);
+static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* context_b)
+{
+#ifdef EXTREME_B2T_TRAPPING
+    SWord initial_reds = 1 + b2t_rand() % 4;
+#else
+    SWord initial_reds = (Uint)(ERTS_BIF_REDS_LEFT(p) * B2T_BYTES_PER_REDUCTION);
+#endif
+    B2TContext c_buff;
+    B2TContext *ctx;
+    int is_first_call;
+
+    if (context_b == NULL) {
+	/* Setup enough to get started */
+        is_first_call = 1;
+        ctx = &c_buff;
+	ctx->state = B2TPrepare;
+        ctx->aligned_alloc = NULL;
+        ctx->flags = flags;
+        IF_DEBUG(ctx->trap_bin = THE_NON_VALUE;)
+    } else {
+        is_first_call = 0;
+	ctx = ERTS_MAGIC_BIN_DATA(context_b);
+        ASSERT(ctx->state != B2TPrepare);
     }
-    size = binary_size(BIF_ARG_1);
+    ctx->reds = initial_reds;
+
+    do {
+        switch (ctx->state) {
+        case B2TPrepare: {
+	    byte* bytes;
+            Uint bin_size;
+            bytes = erts_get_aligned_binary_bytes_extra(bin,
+                                                        &ctx->aligned_alloc,
+                                                        ERTS_ALC_T_EXT_TERM_DATA,
+                                                        0);
+            if (bytes == NULL) {
+                ctx->b2ts.exttmp = 0;
+                ctx->state = B2TBadArg;
+                break;
+            }
+            bin_size = binary_size(bin);
+            if (ctx->aligned_alloc) {
+                ctx->reds -= bin_size / 8;
+            }
+            if (binary2term_prepare(&ctx->b2ts, bytes, bin_size, ctx) < 0) {
+		ctx->state = B2TBadArg;
+	    }
+            break;
+        }
+	case B2TUncompressChunk: {
+            uLongf chunk = ctx->reds;
+            int zret;
+
+            if (chunk > ctx->u.uc.dleft)
+                chunk = ctx->u.uc.dleft;
+            zret = erl_zlib_inflate_chunk(&ctx->u.uc.stream,
+                                          ctx->u.uc.dbytes, &chunk);
+            ctx->u.uc.dbytes += chunk;
+            ctx->u.uc.dleft  -= chunk;
+            if (zret == Z_OK && ctx->u.uc.dleft > 0) {
+                ctx->reds = 0;
+            }
+            else if (erl_zlib_inflate_finish(&ctx->u.uc.stream) == Z_OK
+                     && zret == Z_STREAM_END
+                     && ctx->u.uc.dleft == 0) {
+                ctx->reds -= chunk;
+                ctx->state = B2TSizeInit;
+            }
+            else {
+                ctx->state = B2TBadArg;
+            }
+            break;
+        }
+	case B2TSizeInit:
+	    ctx->u.sc.ep = NULL;
+	    ctx->state = B2TSize;
+	    /*fall through*/
+        case B2TSize:
+            ctx->heap_size = decoded_size(ctx->b2ts.extp,
+					  ctx->b2ts.extp + ctx->b2ts.extsize,
+                                          0, ctx);
+            break;
+
+        case B2TDecodeInit:
+            if (ctx == &c_buff && ctx->b2ts.extsize > ctx->reds) {
+                /* dec_term will maybe trap, allocate space for magic bin
+                   before result term to make it easy to trim with HRelease.
+                 */
+                ctx = b2t_export_context(p, &c_buff);
+            }
+            ctx->u.dc.ep = ctx->b2ts.extp;
+            ctx->u.dc.res = (Eterm) (UWord) NULL;
+            ctx->u.dc.next = &ctx->u.dc.res;
+            ctx->u.dc.hp_start = HAlloc(p, ctx->heap_size);
+            ctx->u.dc.hp       = ctx->u.dc.hp_start;
+            ctx->u.dc.hp_end   = ctx->u.dc.hp_start + ctx->heap_size;
+            ctx->state = B2TDecode;
+            /*fall through*/
+	case B2TDecode:
+        case B2TDecodeList:
+        case B2TDecodeTuple:
+        case B2TDecodeString:
+        case B2TDecodeBinary: {
+	    ErtsDistExternal fakedep;
+            fakedep.flags = ctx->flags;
+            dec_term(&fakedep, NULL, NULL, &MSO(p), NULL, ctx);
+            break;
+	}
+        case B2TDecodeFail:
+            HRelease(p, ctx->u.dc.hp_end, ctx->u.dc.hp_start);
+            /*fall through*/
+        case B2TBadArg:
+            b2t_destroy_context(ctx);
+            if (!is_first_call) {
+                erts_set_gc_state(p, 1);
+            }
+            BUMP_REDS(p, (initial_reds - ctx->reds) / B2T_BYTES_PER_REDUCTION);
+            BIF_ERROR(p, BADARG & ~EXF_SAVETRACE);
 
-    heap_size = binary2term_prepare(&b2ts, bytes, size);
-    if (heap_size < 0)
-	goto error;
+        case B2TDone:
+            b2t_destroy_context(ctx);
 
-    hp = HAlloc(BIF_P, heap_size);
-    endp = hp + heap_size;
+            if (ctx->u.dc.hp > ctx->u.dc.hp_end) {
+                erl_exit(1, ":%s, line %d: heap overrun by %d words(s)\n",
+                         __FILE__, __LINE__, ctx->u.dc.hp - ctx->u.dc.hp_end);
+            }
+            HRelease(p, ctx->u.dc.hp_end, ctx->u.dc.hp);
 
-    res = binary2term_create(NULL, &b2ts, &hp, &MSO(BIF_P));
+            if (!is_first_call) {
+                erts_set_gc_state(p, 1);
+            }
+            BUMP_REDS(p, (initial_reds - ctx->reds) / B2T_BYTES_PER_REDUCTION);
+            return ctx->u.dc.res;
 
-    erts_free_aligned_binary_bytes(temp_alloc);
+        default:
+            ASSERT(!"Unknown state in binary_to_term");
+        }
+    }while (ctx->reds > 0 || ctx->state >= B2TDone);
 
-    if (hp > endp) {
-	erl_exit(1, ":%s, line %d: heap overrun by %d words(s)\n",
-		 __FILE__, __LINE__, hp-endp);
+    if (ctx == &c_buff) {
+        ASSERT(ctx->trap_bin == THE_NON_VALUE);
+        ctx = b2t_export_context(p, &c_buff);
     }
+    ASSERT(ctx->trap_bin != THE_NON_VALUE);
 
-    HRelease(BIF_P, endp, hp);
-
-    if (res == THE_NON_VALUE)
-	goto error;
+    if (is_first_call) {
+        erts_set_gc_state(p, 0);
+    }
+    BUMP_ALL_REDS(p);
+    BIF_TRAP1(&binary_to_term_trap_export, p, ctx->trap_bin);
+}
 
-    return res;
+BIF_RETTYPE erts_internal_binary_to_term_1(BIF_ALIST_1)
+{
+    return binary_to_term_int(BIF_P, 0, BIF_ARG_1, NULL);
 }
 
-BIF_RETTYPE binary_to_term_2(BIF_ALIST_2)
+BIF_RETTYPE erts_internal_binary_to_term_2(BIF_ALIST_2)
 {
-    Sint heap_size;
-    Eterm res;
     Eterm opts;
     Eterm opt;
-    Eterm* hp;
-    Eterm* endp;
-    Sint size;
-    byte* bytes;
-    byte* temp_alloc = NULL;
-    ErtsBinary2TermState b2ts;
-    ErtsDistExternal fakedep;
+    Uint32 flags = 0;
 
-    fakedep.flags = 0;
     opts = BIF_ARG_2;
     while (is_list(opts)) {
         opt = CAR(list_val(opts));
         if (opt == am_safe) {
-	    fakedep.flags |= ERTS_DIST_EXT_BTT_SAFE;
+            flags |= ERTS_DIST_EXT_BTT_SAFE;
         }
 	else {
             goto error;
@@ -1297,35 +1566,10 @@ BIF_RETTYPE binary_to_term_2(BIF_ALIST_2)
     if (is_not_nil(opts))
         goto error;
 
-    if ((bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc)) == NULL) {
-    error:
-	erts_free_aligned_binary_bytes(temp_alloc);
-	BIF_ERROR(BIF_P, BADARG);
-    }
-    size = binary_size(BIF_ARG_1);
-
-    heap_size = binary2term_prepare(&b2ts, bytes, size);
-    if (heap_size < 0)
-	goto error;
-
-    hp = HAlloc(BIF_P, heap_size);
-    endp = hp + heap_size;
-
-    res = binary2term_create(&fakedep, &b2ts, &hp, &MSO(BIF_P));
-
-    erts_free_aligned_binary_bytes(temp_alloc);
-
-    if (hp > endp) {
-	erl_exit(1, ":%s, line %d: heap overrun by %d words(s)\n",
-		 __FILE__, __LINE__, hp-endp);
-    }
-
-    HRelease(BIF_P, endp, hp);
-
-    if (res == THE_NON_VALUE)
-	goto error;
+    return binary_to_term_int(BIF_P, flags, BIF_ARG_1, NULL);
 
-    return res;
+error:
+    BIF_ERROR(BIF_P, BADARG);
 }
 
 Eterm
@@ -1473,12 +1717,10 @@ erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags) {
 /* #define EXTREME_TTB_TRAPPING 1 */
 
 #ifndef EXTREME_TTB_TRAPPING
-#define TERM_TO_BINARY_LOOP_FACTOR 500
-#define TERM_TO_BINARY_SIZE_FACTOR 500000
-#define TERM_TO_BINARY_COMPRESS_CHUNK 500000
+#define TERM_TO_BINARY_LOOP_FACTOR 32
+#define TERM_TO_BINARY_COMPRESS_CHUNK (1 << 18)
 #else
 #define TERM_TO_BINARY_LOOP_FACTOR 1
-#define TERM_TO_BINARY_SIZE_FACTOR 10
 #define TERM_TO_BINARY_COMPRESS_CHUNK 10
 #endif
 
@@ -1514,7 +1756,7 @@ typedef struct {
     } s;
 } TTBContext;
 
-static void context_destructor(Binary *context_bin) 
+static void ttb_context_destructor(Binary *context_bin)
 {
     TTBContext *context = ERTS_MAGIC_BIN_DATA(context_bin);
     if (context->alive) {
@@ -1567,7 +1809,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla
     do {								\
 	if (context_b == NULL) {					\
 	    context_b = erts_create_magic_binary(sizeof(TTBContext),    \
-                                                 context_destructor);   \
+                                                 ttb_context_destructor);   \
 	    context =  ERTS_MAGIC_BIN_DATA(context_b);			\
 	    memcpy(context,&c_buff,sizeof(TTBContext));			\
 	}								\
@@ -1615,7 +1857,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla
 		    /* Finish in one go */
 		    res = erts_term_to_binary_simple(p, Term, size, 
 						     level, flags);
-		    BUMP_REDS(p, size / TERM_TO_BINARY_SIZE_FACTOR);
+		    BUMP_REDS(p, 1);
 		    return res;
 		}
 
@@ -2604,21 +2846,112 @@ undo_offheap_in_area(ErlOffHeap* off_heap, Eterm* start, Eterm* end)
 #endif /* DEBUG */
 }
 
+
 /* Decode term from external format into *objp.
 ** On failure return NULL and (R13B04) *hpp will be unchanged.
 */
 static byte*
-dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Eterm* objp)
+dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap,
+         Eterm* objp, B2TContext* ctx)
 {
-    Eterm* hp_saved = *hpp;
+    Eterm* hp_saved;
     int n;
     ErtsAtomEncoding char_enc;
-    register Eterm* hp = *hpp;	/* Please don't take the address of hp */
-    Eterm* next = objp;
+    register Eterm* hp;        /* Please don't take the address of hp */
+    Eterm* next;
+    SWord reds;
+
+    if (ctx) {
+        hp_saved = ctx->u.dc.hp_start;
+        reds     = ctx->reds;
+        next     = ctx->u.dc.next;
+        ep       = ctx->u.dc.ep;
+        hpp      = &ctx->u.dc.hp;
+
+        if (ctx->state != B2TDecode) {
+            int n_limit = reds;
+
+	    n = ctx->u.dc.remaining_n;
+            if (ctx->state == B2TDecodeBinary) {
+                n_limit *= B2T_MEMCPY_FACTOR;
+                ASSERT(n_limit >= reds);
+		reds -= n / B2T_MEMCPY_FACTOR;
+            }
+	    else
+		reds -= n;
 
-    *next = (Eterm) (UWord) NULL;
+            if (n > n_limit) {
+                ctx->u.dc.remaining_n -= n_limit;
+                n = n_limit;
+                reds = 0;
+            }
+            else {
+                ctx->u.dc.remaining_n = 0;
+            }
+
+            switch (ctx->state) {
+            case B2TDecodeList:
+                objp = next - 2;
+                while (n > 0) {
+                    objp[0] = (Eterm) COMPRESS_POINTER(next);
+                    objp[1] = make_list(next);
+                    next = objp;
+                    objp -= 2;
+                    n--;
+                }
+                break;
+
+            case B2TDecodeTuple:
+                objp = next - 1;
+                while (n-- > 0) {
+                    objp[0] = (Eterm) COMPRESS_POINTER(next);
+                    next = objp;
+                    objp--;
+                }
+                break;
+
+            case B2TDecodeString:
+                hp = *hpp;
+                hp[-1] = make_list(hp);  /* overwrite the premature NIL */
+                while (n-- > 0) {
+                    hp[0] = make_small(*ep++);
+                    hp[1] = make_list(hp+2);
+                    hp += 2;
+                }
+                hp[-1] = NIL;
+                *hpp = hp;
+                break;
+
+            case B2TDecodeBinary:
+                sys_memcpy(ctx->u.dc.remaining_bytes, ep, n);
+                ctx->u.dc.remaining_bytes += n;
+                ep += n;
+                break;
+
+            default:
+                ASSERT(!"Unknown state");
+            }
+            if (!ctx->u.dc.remaining_n) {
+                ctx->state = B2TDecode;
+            }
+            if (reds <= 0) {
+                ctx->u.dc.next = next;
+                ctx->u.dc.ep = ep;
+                ctx->reds = 0;
+                return NULL;
+            }
+        }
+    }
+    else {
+        hp_saved = *hpp;
+        reds = ERTS_SWORD_MAX;
+        next = objp;
+        *next = (Eterm) (UWord) NULL;
+    }
+    hp = *hpp;
 
     while (next != NULL) {
+
 	objp = next;
 	next = (Eterm *) EXPAND_POINTER(*objp);
 
@@ -2738,7 +3071,16 @@ dec_term_atom_common:
 	    *objp = make_tuple(hp);
 	    *hp++ = make_arityval(n);
 	    hp += n;
-	    objp = hp - 1;
+            objp = hp - 1;
+            if (ctx) {
+                if (reds < n) {
+                    ASSERT(reds > 0);
+                    ctx->state = B2TDecodeTuple;
+                    ctx->u.dc.remaining_n = n - reds;
+                    n = reds;
+                }
+		reds -= n;
+	    }
 	    while (n-- > 0) {
 		objp[0] = (Eterm) COMPRESS_POINTER(next);
 		next = objp;
@@ -2756,17 +3098,27 @@ dec_term_atom_common:
 		break;
 	    }
 	    *objp = make_list(hp);
-	    hp += 2*n;
+            hp += 2 * n;
 	    objp = hp - 2;
 	    objp[0] = (Eterm) COMPRESS_POINTER((objp+1));
 	    objp[1] = (Eterm) COMPRESS_POINTER(next);
 	    next = objp;
 	    objp -= 2;
-	    while (--n > 0) {
+            n--;
+	    if (ctx) {
+                if (reds < n) {
+		    ctx->state = B2TDecodeList;
+		    ctx->u.dc.remaining_n = n - reds;
+		    n = reds;
+		}
+		reds -= n;
+	    }
+            while (n > 0) {
 		objp[0] = (Eterm) COMPRESS_POINTER(next);
-		objp[1] = make_list(objp + 2);
+		objp[1] = make_list(next);
 		next = objp;
 		objp -= 2;
+                n--;
 	    }
 	    break;
 	case STRING_EXT:
@@ -2777,6 +3129,14 @@ dec_term_atom_common:
 		break;
 	    }
 	    *objp = make_list(hp);
+            if (ctx) {
+                if (reds < n) {
+                    ctx->state = B2TDecodeString;
+                    ctx->u.dc.remaining_n = n - reds;
+                    n = reds;
+		}
+                reds -= n;
+            }
 	    while (n-- > 0) {
 		hp[0] = make_small(*ep++);
 		hp[1] = make_list(hp+2);
@@ -2984,7 +3344,6 @@ dec_term_atom_common:
 		    dbin->flags = 0;
 		    dbin->orig_size = n;
 		    erts_refc_init(&dbin->refc, 1);
-		    sys_memcpy(dbin->orig_bytes, ep, n);
 		    pb = (ProcBin *) hp;
 		    hp += PROC_BIN_SIZE;
 		    pb->thing_word = HEADER_PROC_BIN;
@@ -2995,7 +3354,20 @@ dec_term_atom_common:
 		    pb->bytes = (byte*) dbin->orig_bytes;
 		    pb->flags = 0;
 		    *objp = make_binary(pb);
-		}
+                    if (ctx) {
+                        int n_limit = reds * B2T_MEMCPY_FACTOR;
+                        if (n > n_limit) {
+                            ctx->state = B2TDecodeBinary;
+                            ctx->u.dc.remaining_n = n - n_limit;
+                            ctx->u.dc.remaining_bytes = dbin->orig_bytes + n_limit;
+                            n = n_limit;
+                            reds = 0;
+                        }
+                        else
+                            reds -= n / B2T_MEMCPY_FACTOR;
+                    }
+                    sys_memcpy(dbin->orig_bytes, ep, n);
+                }
 		ep += n;
 		break;
 	    }
@@ -3018,13 +3390,14 @@ dec_term_atom_common:
 		    sys_memcpy(hb->data, ep, n);
 		    bin = make_binary(hb);
 		    hp += heap_bin_size(n);
+                    ep += n;
 		} else {
 		    Binary* dbin = erts_bin_nrml_alloc(n);
 		    ProcBin* pb;
+
 		    dbin->flags = 0;
 		    dbin->orig_size = n;
 		    erts_refc_init(&dbin->refc, 1);
-		    sys_memcpy(dbin->orig_bytes, ep, n);
 		    pb = (ProcBin *) hp;
 		    pb->thing_word = HEADER_PROC_BIN;
 		    pb->size = n;
@@ -3035,8 +3408,23 @@ dec_term_atom_common:
 		    pb->flags = 0;
 		    bin = make_binary(pb);
 		    hp += PROC_BIN_SIZE;
-		}
-		ep += n;
+                    if (ctx) {
+                        int n_limit = reds * B2T_MEMCPY_FACTOR;
+                        if (n > n_limit) {
+                            ctx->state = B2TDecodeBinary;
+                            ctx->u.dc.remaining_n = n - n_limit;
+                            ctx->u.dc.remaining_bytes = dbin->orig_bytes + n_limit;
+                            n = n_limit;
+                            reds = 0;
+                        }
+                        else
+                            reds -= n / B2T_MEMCPY_FACTOR;
+                    }
+                    sys_memcpy(dbin->orig_bytes, ep, n);
+                    ep += n;
+                    n = pb->size;
+                }
+
 		if (bitsize == 8 || n == 0) {
 		    *objp = bin;
 		} else {
@@ -3067,7 +3455,7 @@ dec_term_atom_common:
 		    goto error;
 		}
 		*hpp = hp;
-		ep = dec_term(edep, hpp, ep, off_heap, &temp);
+		ep = dec_term(edep, hpp, ep, off_heap, &temp, NULL);
 		hp = *hpp;
 		if (ep == NULL) {
 		    goto error;
@@ -3127,7 +3515,7 @@ dec_term_atom_common:
 		}
 		*hpp = hp;
 		/* Index */
-		if ((ep = dec_term(edep, hpp, ep, off_heap, &temp)) == NULL) {
+		if ((ep = dec_term(edep, hpp, ep, off_heap, &temp, NULL)) == NULL) {
 		    goto error;
 		}
 		if (!is_small(temp)) {
@@ -3136,7 +3524,7 @@ dec_term_atom_common:
 		old_index = unsigned_val(temp);
 
 		/* Uniq */
-		if ((ep = dec_term(edep, hpp, ep, off_heap, &temp)) == NULL) {
+		if ((ep = dec_term(edep, hpp, ep, off_heap, &temp, NULL)) == NULL) {
 		    goto error;
 		}
 		if (!is_small(temp)) {
@@ -3204,7 +3592,7 @@ dec_term_atom_common:
 		}
 
 		/* Index */
-		if ((ep = dec_term(edep, hpp, ep, off_heap, &temp)) == NULL) {
+		if ((ep = dec_term(edep, hpp, ep, off_heap, &temp, NULL)) == NULL) {
 		    goto error;
 		}
 		if (!is_small(temp)) {
@@ -3213,7 +3601,7 @@ dec_term_atom_common:
 		old_index = unsigned_val(temp);
 
 		/* Uniq */
-		if ((ep = dec_term(edep, hpp, ep, off_heap, &temp)) == NULL) {
+		if ((ep = dec_term(edep, hpp, ep, off_heap, &temp, NULL)) == NULL) {
 		    goto error;
 		}
 		if (!is_small(temp)) {
@@ -3313,8 +3701,31 @@ dec_term_atom_common:
 	    }
 	    undo_offheap_in_area(off_heap, hp_saved, hp);
 	    *hpp = hp_saved;
-	    return NULL;
+            if (ctx) {
+                ctx->state = B2TDecodeFail;
+		ctx->reds = reds;
+            }
+            return NULL;
 	}
+
+        if (--reds <= 0) {
+            if (ctx) {
+                if (next || ctx->state != B2TDecode) {
+                    ctx->u.dc.ep = ep;
+                    ctx->u.dc.next = next;
+                    ctx->u.dc.hp = hp;
+                    ctx->reds = 0;
+                    return NULL;
+                }
+            }
+            else {
+                reds = ERTS_SWORD_MAX;
+            }
+        }
+    }
+    if (ctx) {
+        ctx->state = B2TDone;
+	ctx->reds = reds;
     }
     *hpp = hp;
     return ep;
@@ -3602,18 +4013,37 @@ encode_size_struct_int(Process *p, ErtsAtomCacheMap *acmp, Eterm obj,
 }
 
 static Sint
-decoded_size(byte *ep, byte* endp, int internal_tags)
+decoded_size(byte *ep, byte* endp, int internal_tags, B2TContext* ctx)
 {
-    int heap_size = 0;
+    int heap_size;
     int terms;
-    int atom_extra_skip = 0;
+    int atom_extra_skip;
     Uint n;
+    SWord reds;
+
+    if (ctx) {
+        reds = ctx->reds;
+        if (ctx->u.sc.ep) {
+            heap_size = ctx->u.sc.heap_size;
+            terms = ctx->u.sc.terms;
+            ep = ctx->u.sc.ep;
+            atom_extra_skip = ctx->u.sc.atom_extra_skip;
+            goto init_done;
+        }
+    }
+    else
+        reds = 0; /* not used but compiler warns anyway */
+
+    heap_size = 0;
+    terms = 1;
+    atom_extra_skip = 0;
+init_done:
 
 #define SKIP(sz)				\
     do {					\
 	if ((sz) <= endp-ep) {			\
 	    ep += (sz);				\
-        } else { return -1; };			\
+        } else { goto error; };			\
     } while (0)
 
 #define SKIP2(sz1, sz2)				\
@@ -3621,25 +4051,24 @@ decoded_size(byte *ep, byte* endp, int internal_tags)
 	Uint sz = (sz1) + (sz2);		\
 	if (sz1 < sz && (sz) <= endp-ep) {	\
 	    ep += (sz);				\
-        } else { return -1; }			\
+        } else { goto error; }			\
     } while (0)
 
 #define CHKSIZE(sz)				\
     do {					\
-	 if ((sz) > endp-ep) { return -1; }	\
+	 if ((sz) > endp-ep) { goto error; }	\
     } while (0)
 
 #define ADDTERMS(n)				\
     do {					\
         int before = terms;		        \
 	terms += (n);                           \
-	if (terms < before) return -1;     	\
+	if (terms < before) goto error;     	\
     } while (0)
 
-
-    for (terms=1; terms > 0; terms--) {
-	int tag;
-
+    ASSERT(terms > 0);
+    do {
+        int tag;
 	CHKSIZE(1);
 	tag = ep++[0];
 	switch (tag) {
@@ -3660,7 +4089,7 @@ decoded_size(byte *ep, byte* endp, int internal_tags)
 	    CHKSIZE(4);
 	    n = get_int32(ep);
 	    if (n > BIG_ARITY_MAX*sizeof(ErtsDigit)) {
-		return -1;
+		goto error;
 	    }
 	    SKIP2(n,4+1);		/* skip, size,sign,digits */
 	    heap_size += 1+1+(n+sizeof(Eterm)-1)/sizeof(Eterm); /* XXX: 1 too much? */
@@ -3669,7 +4098,7 @@ decoded_size(byte *ep, byte* endp, int internal_tags)
 	    CHKSIZE(2);
 	    n = get_int16(ep);
 	    if (n > MAX_ATOM_CHARACTERS) {
-		return -1;
+		goto error;
 	    }
 	    SKIP(n+2+atom_extra_skip);
 	    atom_extra_skip = 0;
@@ -3679,7 +4108,7 @@ decoded_size(byte *ep, byte* endp, int internal_tags)
 	    n = get_int16(ep);
 	    ep += 2;
 	    if (n > MAX_ATOM_SZ_LIMIT) {
-		return -1;
+		goto error;
 	    }
 	    SKIP(n+atom_extra_skip);
 	    atom_extra_skip = 0;
@@ -3688,7 +4117,7 @@ decoded_size(byte *ep, byte* endp, int internal_tags)
 	    CHKSIZE(1);
 	    n = get_int8(ep);
 	    if (n > MAX_ATOM_CHARACTERS) {
-		return -1;
+		goto error;
 	    }
 	    SKIP(n+1+atom_extra_skip);
 	    atom_extra_skip = 0;
@@ -3698,7 +4127,7 @@ decoded_size(byte *ep, byte* endp, int internal_tags)
 	    n = get_int8(ep);
 	    ep++;
 	    if (n > MAX_ATOM_SZ_LIMIT) {
-		return -1;
+		goto error;
 	    }
 	    SKIP(n+atom_extra_skip);
 	    atom_extra_skip = 0;
@@ -3727,7 +4156,7 @@ decoded_size(byte *ep, byte* endp, int internal_tags)
 		id_words = get_int16(ep);
 		    
 		if (id_words > ERTS_MAX_REF_NUMBERS)
-		    return -1;
+		    goto error;
 
 		ep += 2;
 		atom_extra_skip = 1 + 4*id_words;
@@ -3829,7 +4258,7 @@ decoded_size(byte *ep, byte* endp, int internal_tags)
 		num_free = get_int32(ep);
 		ep += 4;
 		if (num_free > MAX_ARG) {
-		    return -1;
+		    goto error;
 		}
 		terms += 4 + num_free;
 		heap_size += ERL_FUN_SIZE + num_free;
@@ -3846,24 +4275,47 @@ decoded_size(byte *ep, byte* endp, int internal_tags)
 
 	case BINARY_INTERNAL_REF:
 	    if (!internal_tags) {
-		return -1;
+		goto error;
 	    }
 	    SKIP(sizeof(ProcBin));
 	    heap_size += PROC_BIN_SIZE;
 	    break;
 	case BIT_BINARY_INTERNAL_REF:
 	    if (!internal_tags) {
-		return -1;
+		goto error;
 	    }
 	    SKIP(2+sizeof(ProcBin));
 	    heap_size += PROC_BIN_SIZE + ERL_SUB_BIN_SIZE;
 	    break;
 	default:
-	    return -1;
+	    goto error;
 	}
-    }
+        terms--;
+
+        if (ctx && --reds <= 0 && terms > 0) {
+            ctx->u.sc.heap_size = heap_size;
+            ctx->u.sc.terms = terms;
+            ctx->u.sc.ep = ep;
+            ctx->u.sc.atom_extra_skip = atom_extra_skip;
+            ctx->reds = 0;
+            return 0;
+        }
+    }while (terms > 0);
+
     /* 'terms' may be non-zero if it has wrapped around */
-    return terms==0 ? heap_size : -1;
+    if (terms == 0) {
+        if (ctx) {
+            ctx->state = B2TDecodeInit;
+            ctx->reds = reds;
+        }
+        return heap_size;
+    }
+
+error:
+    if (ctx) {
+        ctx->state = B2TBadArg;
+    }
+    return -1;
 #undef SKIP
 #undef SKIP2
 #undef CHKSIZE