Merge branch 'rickard/dist-system-limit/OTP-15708'

* rickard/dist-system-limit/OTP-15708:
  Fail when we cannot encode term in binary

10 files changed, 389 insertions, 72 deletions

diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 30fe13fad3..27599f38e0 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -2307,8 +2307,18 @@ erts_dsig_send(ErtsDSigSendContext *ctx)
 
 	    ctx->data_size = ctx->max_finalize_prepend;
 	    erts_reset_atom_cache_map(ctx->acmp);
-	    erts_encode_dist_ext_size(ctx->ctl, ctx->flags, ctx->acmp, &ctx->data_size);
 
+	    switch (erts_encode_dist_ext_size(ctx->ctl, ctx->flags,
+                                              ctx->acmp, &ctx->data_size)) {
+            case ERTS_EXT_SZ_OK:
+                break;
+            case ERTS_EXT_SZ_SYSTEM_LIMIT:
+                retval = ERTS_DSIG_SEND_TOO_LRG;
+                goto done;
+            case ERTS_EXT_SZ_YIELD:
+                ERTS_INTERNAL_ERROR("Unexpected yield result");
+                break;
+            }
 	    if (is_non_value(ctx->msg)) {
                 ctx->phase = ERTS_DSIG_SEND_PHASE_ALLOC;
                 break;
@@ -2318,17 +2328,31 @@ erts_dsig_send(ErtsDSigSendContext *ctx)
             ctx->u.sc.level = 0;
 
             ctx->phase = ERTS_DSIG_SEND_PHASE_MSG_SIZE;
-	case ERTS_DSIG_SEND_PHASE_MSG_SIZE:
-            if (!ctx->no_trap) {
-                if (erts_encode_dist_ext_size_int(ctx->msg, ctx, &ctx->data_size)) {
-                    retval = ERTS_DSIG_SEND_CONTINUE;
-                    goto done;
-                }
-            } else {
-                erts_encode_dist_ext_size(ctx->msg, ctx->flags, ctx->acmp, &ctx->data_size);
+	case ERTS_DSIG_SEND_PHASE_MSG_SIZE: {
+            ErtsExtSzRes sz_res;
+            sz_res = (!ctx->no_trap
+                      ? erts_encode_dist_ext_size_ctx(ctx->msg,
+                                                      ctx,
+                                                      &ctx->data_size)
+                      : erts_encode_dist_ext_size(ctx->msg,
+                                                  ctx->flags,
+                                                  ctx->acmp,
+                                                  &ctx->data_size));
+            switch (sz_res) {
+            case ERTS_EXT_SZ_OK:
+                break;
+            case ERTS_EXT_SZ_SYSTEM_LIMIT:
+                retval = ERTS_DSIG_SEND_TOO_LRG;
+                goto done;
+            case ERTS_EXT_SZ_YIELD:
+                if (ctx->no_trap)
+                    ERTS_INTERNAL_ERROR("Unexpected yield result");
+                retval = ERTS_DSIG_SEND_CONTINUE;
+                goto done;
             }
 
 	    ctx->phase = ERTS_DSIG_SEND_PHASE_ALLOC;
+        }
 	case ERTS_DSIG_SEND_PHASE_ALLOC:
 	    erts_finalize_atom_cache_map(ctx->acmp, ctx->flags);
 
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index c3d0ea40f6..0339589b79 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -4236,7 +4236,10 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
 		Uint dflags = (TERM_TO_BINARY_DFLAGS
                                & ~DFLAG_EXPORT_PTR_TAG
                                & ~DFLAG_BIT_BINARIES);
-		BIF_RET(erts_term_to_binary(BIF_P, tp[2], 0, dflags));
+		Eterm res = erts_term_to_binary(BIF_P, tp[2], 0, dflags);
+                if (is_value(res))
+                    BIF_RET(res);
+                BIF_ERROR(BIF_P, SYSTEM_LIMIT);
 	    }
 	    else if (ERTS_IS_ATOM_STR("dist_ctrl", tp[1])) {
 		Eterm res = am_undefined;
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index deaf35c2a1..1fbe362330 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1344,11 +1344,18 @@ unsigned char* enif_make_new_binary(ErlNifEnv* env, size_t size,
 int enif_term_to_binary(ErlNifEnv *dst_env, ERL_NIF_TERM term,
                         ErlNifBinary *bin)
 {
-    Sint size;
+    Uint size;
     byte *bp;
     Binary* refbin;
 
-    size = erts_encode_ext_size(term);
+    switch (erts_encode_ext_size(term, &size)) {
+    case ERTS_EXT_SZ_SYSTEM_LIMIT:
+        return 0; /* system limit */
+    case ERTS_EXT_SZ_YIELD:
+        ERTS_INTERNAL_ERROR("Unexpected yield");
+    case ERTS_EXT_SZ_OK:
+        break;
+    }
     if (!enif_alloc_binary(size, bin))
         return 0;
 
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 76eec96372..1f6adb98ef 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -12151,6 +12151,7 @@ erts_proc_exit_handle_dist_monitor(ErtsMonitor *mon, void *vctxt, Sint reds)
         case ERTS_DSIG_SEND_OK:
             break;
         case ERTS_DSIG_SEND_TOO_LRG:
+            erts_kill_dist_connection(dep, dist->connection_id);
             erts_set_gc_state(c_p, 1);
             break;
         default:
@@ -12401,6 +12402,7 @@ erts_proc_exit_handle_dist_link(ErtsLink *lnk, void *vctxt, Sint reds)
         case ERTS_DSIG_SEND_OK:
             break;
         case ERTS_DSIG_SEND_TOO_LRG:
+            erts_kill_dist_connection(dep, dist->connection_id);
             erts_set_gc_state(c_p, 1);
             break;
         default:
@@ -12897,7 +12899,9 @@ restart:
 
             switch (result) {
             case ERTS_DSIG_SEND_OK:
+                break;
             case ERTS_DSIG_SEND_TOO_LRG: /*SEND_SYSTEM_LIMIT*/
+                erts_kill_dist_connection(ctx->dep, ctx->connection_id);
                 break;
             case ERTS_DSIG_SEND_YIELD: /*SEND_YIELD_RETURN*/
             case ERTS_DSIG_SEND_CONTINUE: { /*SEND_YIELD_CONTINUE*/
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index ae7084b7f4..c85a7df5ec 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -635,9 +635,11 @@ write_sys_msg_to_port(Eterm unused_to,
 		      Eterm message) {
     byte *buffer;
     byte *ptr;
-    unsigned size;
+    Uint size;
+
+    if (erts_encode_ext_size(message, &size) != ERTS_EXT_SZ_OK)
+	erts_exit(ERTS_ERROR_EXIT, "Internal error: System limit\n");
 
-    size = erts_encode_ext_size(message);
     buffer = (byte *) erts_alloc(ERTS_ALC_T_TMP, size);
 
     ptr = buffer;
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 395ff51ad3..ec67ab2aed 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -112,13 +112,13 @@ static byte* dec_pid(ErtsDistExternal *, ErtsHeapFactory*, byte*, Eterm*, byte t
 static Sint decoded_size(byte *ep, byte* endp, int internal_tags, struct B2TContext_t*);
 static BIF_RETTYPE term_to_binary_trap_1(BIF_ALIST_1);
 
-static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint flags, 
-				     Binary *context_b);
+static Eterm erts_term_to_binary_int(Process* p, Eterm Term, Eterm opts, int level,
+                                     Uint flags, Binary *context_b);
 
 static Uint encode_size_struct2(ErtsAtomCacheMap *, Eterm, unsigned);
 struct TTBSizeContext_;
-static int encode_size_struct_int(struct TTBSizeContext_*, ErtsAtomCacheMap *acmp, Eterm obj,
-				  unsigned dflags, Sint *reds, Uint *res);
+static ErtsExtSzRes encode_size_struct_int(struct TTBSizeContext_*, ErtsAtomCacheMap *acmp,
+                                           Eterm obj, unsigned dflags, Sint *reds, Uint *res);
 
 static Export binary_to_term_trap_export;
 static BIF_RETTYPE binary_to_term_trap_1(BIF_ALIST_1);
@@ -603,49 +603,50 @@ done:
     return reds < 0 ? 0 : reds;
 }
 
-int erts_encode_dist_ext_size(Eterm term, Uint32 flags, ErtsAtomCacheMap *acmp,
-			      Uint* szp)
+ErtsExtSzRes
+erts_encode_dist_ext_size(Eterm term, Uint32 flags, ErtsAtomCacheMap *acmp, Uint* szp)
 {
     Uint sz;
-    if (encode_size_struct_int(NULL, acmp, term, flags, NULL, &sz)) {
-	return -1;
-    } else {
+    ErtsExtSzRes res = encode_size_struct_int(NULL, acmp, term, flags, NULL, &sz);
+    if (res == ERTS_EXT_SZ_OK) {
 #ifndef ERTS_DEBUG_USE_DIST_SEP
 	if (!(flags & (DFLAG_DIST_HDR_ATOM_CACHE | DFLAG_NO_MAGIC)))
 #endif
 	    sz++ /* VERSION_MAGIC */;
 
 	*szp += sz;
-	return 0;
     }
+    return res;
 }
 
-int erts_encode_dist_ext_size_int(Eterm term, ErtsDSigSendContext *ctx, Uint* szp)
+ErtsExtSzRes
+erts_encode_dist_ext_size_ctx(Eterm term, ErtsDSigSendContext *ctx, Uint* szp)
 {
     Uint sz;
-    if (encode_size_struct_int(&ctx->u.sc, ctx->acmp, term, ctx->flags, &ctx->reds, &sz)) {
-	return -1;
-    } else {
+    ErtsExtSzRes res = encode_size_struct_int(&ctx->u.sc, ctx->acmp, term,
+                                              ctx->flags, &ctx->reds, &sz);
+    if (res == ERTS_EXT_SZ_OK) {
 #ifndef ERTS_DEBUG_USE_DIST_SEP
 	if (!(ctx->flags & (DFLAG_DIST_HDR_ATOM_CACHE | DFLAG_NO_MAGIC)))
 #endif
 	    sz++ /* VERSION_MAGIC */;
 
 	*szp += sz;
-	return 0;
     }
+    return res;
 }
 
-Uint erts_encode_ext_size(Eterm term)
+ErtsExtSzRes erts_encode_ext_size_2(Eterm term, unsigned dflags, Uint *szp)
 {
-    return encode_size_struct2(NULL, term, TERM_TO_BINARY_DFLAGS)
-	+ 1 /* VERSION_MAGIC */;
+    ErtsExtSzRes res = encode_size_struct_int(NULL, NULL, term, dflags,
+                                              NULL, szp);
+    (*szp)++ /* VERSION_MAGIC */;
+    return res;
 }
 
-Uint erts_encode_ext_size_2(Eterm term, unsigned dflags)
+ErtsExtSzRes erts_encode_ext_size(Eterm term, Uint *szp)
 {
-    return encode_size_struct2(NULL, term, dflags)
-        + 1 /* VERSION_MAGIC */;
+    return erts_encode_ext_size_2(term, TERM_TO_BINARY_DFLAGS, szp);
 }
 
 Uint erts_encode_ext_size_ets(Eterm term)
@@ -1253,9 +1254,22 @@ static BIF_RETTYPE term_to_binary_trap_1(BIF_ALIST_1)
 {
     Eterm *tp = tuple_val(BIF_ARG_1);
     Eterm Term = tp[1];
-    Eterm bt = tp[2];
+    Eterm Opts = tp[2];
+    Eterm bt = tp[3];
     Binary *bin = erts_magic_ref2bin(bt);
-    Eterm res = erts_term_to_binary_int(BIF_P, Term, 0, 0,bin);
+    Eterm res = erts_term_to_binary_int(BIF_P, Term, Opts, 0, 0,bin);
+    if (is_non_value(res)) {
+        if (erts_set_gc_state(BIF_P, 1)
+            || MSO(BIF_P).overhead > BIN_VHEAP_SZ(BIF_P)) {
+            ERTS_VBUMP_ALL_REDS(BIF_P);
+        }
+        if (Opts == am_undefined)
+            ERTS_BIF_ERROR_TRAPPED1(BIF_P, SYSTEM_LIMIT,
+                                    bif_export[BIF_term_to_binary_1], Term);
+        else
+            ERTS_BIF_ERROR_TRAPPED2(BIF_P, SYSTEM_LIMIT,
+                                    bif_export[BIF_term_to_binary_2], Term, Opts);
+    }
     if (is_tuple(res)) {
 	ASSERT(BIF_P->flags & F_DISABLE_GC);
 	BIF_TRAP1(&term_to_binary_trap_export,BIF_P,res);
@@ -1272,7 +1286,12 @@ HIPE_WRAPPER_BIF_DISABLE_GC(term_to_binary, 1)
 
 BIF_RETTYPE term_to_binary_1(BIF_ALIST_1)
 {
-    Eterm res = erts_term_to_binary_int(BIF_P, BIF_ARG_1, 0, TERM_TO_BINARY_DFLAGS, NULL);
+    Eterm res = erts_term_to_binary_int(BIF_P, BIF_ARG_1, am_undefined,
+                                        0, TERM_TO_BINARY_DFLAGS, NULL);
+    if (is_non_value(res)) {
+	ASSERT(!(BIF_P->flags & F_DISABLE_GC));
+        BIF_ERROR(BIF_P, SYSTEM_LIMIT);
+    }
     if (is_tuple(res)) {
 	erts_set_gc_state(BIF_P, 0);
 	BIF_TRAP1(&term_to_binary_trap_export,BIF_P,res);
@@ -1331,7 +1350,12 @@ BIF_RETTYPE term_to_binary_2(BIF_ALIST_2)
 	goto error;
     }
 
-    res = erts_term_to_binary_int(p, Term, level, flags, NULL);
+    res = erts_term_to_binary_int(p, Term, BIF_ARG_2,
+                                  level, flags, NULL);
+    if (is_non_value(res)) {
+	ASSERT(!(BIF_P->flags & F_DISABLE_GC));
+        BIF_ERROR(BIF_P, SYSTEM_LIMIT);
+    }
     if (is_tuple(res)) {
 	erts_set_gc_state(p, 0);
 	BIF_TRAP1(&term_to_binary_trap_export,BIF_P,res);
@@ -1880,8 +1904,17 @@ external_size_1(BIF_ALIST_1)
 {
     Process* p = BIF_P;
     Eterm Term = BIF_ARG_1;
+    Uint size;
+
+    switch (erts_encode_ext_size(Term, &size)) {
+    case ERTS_EXT_SZ_SYSTEM_LIMIT:
+        BIF_ERROR(BIF_P, SYSTEM_LIMIT);
+    case ERTS_EXT_SZ_YIELD:
+        ERTS_INTERNAL_ERROR("Unexpected yield");
+    case ERTS_EXT_SZ_OK:
+        break;
+    }
 
-    Uint size = erts_encode_ext_size(Term);
     if (IS_USMALL(0, size)) {
 	BIF_RET(make_small(size));
     } else {
@@ -1924,7 +1957,15 @@ external_size_2(BIF_ALIST_2)
         goto error;
     }
 
-    size = erts_encode_ext_size_2(BIF_ARG_1, flags);
+    switch (erts_encode_ext_size_2(BIF_ARG_1, flags, &size)) {
+    case ERTS_EXT_SZ_SYSTEM_LIMIT:
+        BIF_ERROR(BIF_P, SYSTEM_LIMIT);
+    case ERTS_EXT_SZ_YIELD:
+        ERTS_INTERNAL_ERROR("Unexpected yield");
+    case ERTS_EXT_SZ_OK:
+        break;
+    }
+
     if (IS_USMALL(0, size)) {
         BIF_RET(make_small(size));
     } else {
@@ -2012,7 +2053,15 @@ erts_term_to_binary_simple(Process* p, Eterm Term, Uint size, int level, Uint fl
 Eterm
 erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags) {
     Uint size;
-    size = encode_size_struct2(NULL, Term, flags) + 1 /* VERSION_MAGIC */;
+    switch (encode_size_struct_int(NULL, NULL, Term, flags, NULL, &size)) {
+    case ERTS_EXT_SZ_SYSTEM_LIMIT:
+        return THE_NON_VALUE;
+    case ERTS_EXT_SZ_YIELD:
+        ERTS_INTERNAL_ERROR("Unexpected yield");
+    case ERTS_EXT_SZ_OK:
+        break;
+    }
+    size++; /* VERSION_MAGIC */;
     return erts_term_to_binary_simple(p, Term, size, level, flags);
 }
 
@@ -2062,8 +2111,8 @@ static int ttb_context_destructor(Binary *context_bin)
     return 1;
 }
 
-static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint flags, 
-				     Binary *context_b) 
+static Eterm erts_term_to_binary_int(Process* p, Eterm Term, Eterm opts, int level,
+                                     Uint flags, Binary *context_b) 
 {
     Eterm *hp;
     Eterm res;
@@ -2081,18 +2130,17 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla
     do {								\
 	if (context_b == NULL) {					\
 	    context_b = erts_create_magic_binary(sizeof(TTBContext),    \
-                                                 ttb_context_destructor);   \
+                                                 ttb_context_destructor);\
 	    context =  ERTS_MAGIC_BIN_DATA(context_b);			\
-	    sys_memcpy(context,&c_buff,sizeof(TTBContext));			\
+	    sys_memcpy(context,&c_buff,sizeof(TTBContext));		\
 	}								\
     } while (0)
 
 #define RETURN_STATE()							\
     do {								\
-	static const int TUPLE2_SIZE = 2 + 1;				\
-	hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE + TUPLE2_SIZE);        \
+	hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE + 1 + 3);              \
 	c_term = erts_mk_magic_ref(&hp, &MSO(p), context_b);            \
-	res = TUPLE2(hp, Term, c_term);					\
+	res = TUPLE3(hp, Term, opts, c_term);                           \
 	BUMP_ALL_REDS(p);                                               \
 	return res;							\
     } while (0);
@@ -2118,11 +2166,17 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla
 		int level;
 		Uint flags;
 		/* Try for fast path */
-		if (encode_size_struct_int(&context->s.sc, NULL, Term,
-					   context->s.sc.flags, &reds, &size) < 0) {
+                switch (encode_size_struct_int(&context->s.sc, NULL, Term,
+                                               context->s.sc.flags, &reds, &size)) {
+                case ERTS_EXT_SZ_SYSTEM_LIMIT:
+                    BUMP_REDS(p, (initial_reds - reds) / TERM_TO_BINARY_LOOP_FACTOR);
+                    return THE_NON_VALUE;
+                case ERTS_EXT_SZ_YIELD:
 		    EXPORT_CONTEXT();
 		    /* Same state */
 		    RETURN_STATE();
+                case ERTS_EXT_SZ_OK:
+                    break;
 		}
 		++size; /* VERSION_MAGIC */
 		/* Move these to next state */
@@ -4184,13 +4238,21 @@ error_hamt:
    to a sequence of bytes
    N.B. That this must agree with to_external2() above!!!
    (except for cached atoms) */
-static Uint encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) {
-    Uint res;
-    (void) encode_size_struct_int(NULL, acmp, obj, dflags, NULL, &res);
-    return res;
+static Uint encode_size_struct2(ErtsAtomCacheMap *acmp,
+                                Eterm obj,
+                                unsigned dflags) {
+    Uint size;
+    ErtsExtSzRes res = encode_size_struct_int(NULL, acmp, obj,
+                                              dflags, NULL, &size);
+    /*
+     * encode_size_struct2() only allowed when
+     * we know the result will always be OK!
+     */ 
+    ASSERT(res == ERTS_EXT_SZ_OK); (void) res;
+    return (Uint) size;
 }
 
-static int
+static ErtsExtSzRes
 encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj,
 		       unsigned dflags, Sint *reds, Uint *res)
 {
@@ -4223,7 +4285,7 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj,
 	    ctx->obj = obj;
 	    ctx->result = result;
 	    WSTACK_SAVE(s, &ctx->wstack);
-	    return -1;
+	    return ERTS_EXT_SZ_YIELD;
 	}
 	switch (tag_val_def(obj)) {
 	case NIL_DEF:
@@ -4399,11 +4461,26 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj,
 		result += 32;   /* Yes, including the tag */
 	    }
 	    break;
-	case BINARY_DEF:
-	    if (dflags & DFLAG_INTERNAL_TAGS) {
+	case BINARY_DEF: {
+            ProcBin* pb = (ProcBin*) binary_val(obj);
+            Uint tot_bytes = pb->size;
+            if (!(dflags & DFLAG_INTERNAL_TAGS)) {
+#ifdef ARCH_64
+                if (tot_bytes >= (Uint) 0xffffffff) {
+                    if (pb->thing_word == HEADER_SUB_BIN) {
+                        ErlSubBin* sub = (ErlSubBin*) pb;
+                        tot_bytes += (sub->bitoffs + sub->bitsize+ 7) / 8;
+                    }
+                    if (tot_bytes > (Uint) 0xffffffff) {
+                        WSTACK_DESTROY(s);
+                        return ERTS_EXT_SZ_SYSTEM_LIMIT;
+                    }
+                }
+#endif
+            }
+            else {
 		ProcBin* pb = (ProcBin*) binary_val(obj);
 		Uint sub_extra = 0;
-		Uint tot_bytes = pb->size;
 		if (pb->thing_word == HEADER_SUB_BIN) {
 		    ErlSubBin* sub = (ErlSubBin*) pb;
 		    pb = (ProcBin*) binary_val(sub->orig);
@@ -4420,6 +4497,7 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj,
 	    result += 1 + 4 + binary_size(obj) +
 		    5;			/* For unaligned binary */
 	    break;
+        }
 	case FUN_DEF:
 	    {
 		ErlFunThing* funp = (ErlFunThing *) fun_val(obj);
@@ -4452,7 +4530,7 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj,
 	    break;
 
 	default:
-	    erts_exit(ERTS_ERROR_EXIT,"Internal data structure error (in encode_size_struct2)%x\n",
+	    erts_exit(ERTS_ERROR_EXIT,"Internal data structure error (in encode_size_struct_int) %x\n",
 		     obj);
 	}
 
@@ -4492,7 +4570,7 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj,
 	*reds = r < 0 ? 0 : r;
     }
     *res = result;
-    return 0;
+    return ERTS_EXT_SZ_OK;
 }
 
 
diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h
index f2cc9bf98f..b556c9076c 100644
--- a/erts/emulator/beam/external.h
+++ b/erts/emulator/beam/external.h
@@ -164,14 +164,21 @@ byte *erts_encode_ext_dist_header_setup(byte *, ErtsAtomCacheMap *, Uint, Eterm)
 byte *erts_encode_ext_dist_header_fragment(byte **, Uint, Eterm);
 Sint erts_encode_ext_dist_header_finalize(ErtsDistOutputBuf*, DistEntry *, Uint32 dflags, Sint reds);
 struct erts_dsig_send_context;
-int erts_encode_dist_ext_size(Eterm, Uint32, ErtsAtomCacheMap*, Uint* szp);
-int erts_encode_dist_ext_size_int(Eterm term, struct erts_dsig_send_context* ctx, Uint* szp);
+
+typedef enum {
+    ERTS_EXT_SZ_OK,
+    ERTS_EXT_SZ_YIELD,
+    ERTS_EXT_SZ_SYSTEM_LIMIT
+} ErtsExtSzRes;
+
+ErtsExtSzRes erts_encode_dist_ext_size(Eterm, Uint32, ErtsAtomCacheMap*, Uint* szp);
+ErtsExtSzRes erts_encode_dist_ext_size_ctx(Eterm term, struct erts_dsig_send_context* ctx, Uint* szp);
 struct TTBEncodeContext_;
 int erts_encode_dist_ext(Eterm, byte **, Uint32, ErtsAtomCacheMap *,
 			  struct TTBEncodeContext_ *, Sint* reds);
 
-Uint erts_encode_ext_size(Eterm);
-Uint erts_encode_ext_size_2(Eterm, unsigned);
+ErtsExtSzRes erts_encode_ext_size(Eterm, Uint *szp);
+ErtsExtSzRes erts_encode_ext_size_2(Eterm, unsigned, Uint *szp);
 Uint erts_encode_ext_size_ets(Eterm);
 void erts_encode_ext(Eterm, byte **);
 byte* erts_encode_ext_ets(Eterm, byte *, struct erl_off_heap_header** ext_off_heap);
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index b961c639f5..31c941337f 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -4450,6 +4450,7 @@ erts_port_call(Process* c_p,
     char input_buf[256];
     char *bufp;
     byte *endp;
+    Uint uintsz;
     ErlDrvSizeT size;
     int try_call;
     erts_aint32_t sched_flags;
@@ -4462,7 +4463,9 @@ erts_port_call(Process* c_p,
 
     try_call = !(sched_flags & ERTS_PTS_FLGS_FORCE_SCHEDULE_OP);
 
-    size = erts_encode_ext_size(data);
+    if (erts_encode_ext_size(data, &uintsz) != ERTS_EXT_SZ_OK)
+        return ERTS_PORT_OP_BADARG;
+    size = (ErlDrvSizeT) uintsz;
 
     if (!try_call)
 	bufp = erts_alloc(ERTS_ALC_T_DRV_CALL_DATA, size);
diff --git a/erts/emulator/test/binary_SUITE.erl b/erts/emulator/test/binary_SUITE.erl
index 563d60cc3f..4fb339926e 100644
--- a/erts/emulator/test/binary_SUITE.erl
+++ b/erts/emulator/test/binary_SUITE.erl
@@ -71,7 +71,8 @@
          term2bin_tuple_fallbacks/1,
          robustness/1,otp_8117/1,
 	 otp_8180/1, trapping/1, large/1,
-	 error_after_yield/1, cmp_old_impl/1]).
+	 error_after_yield/1, cmp_old_impl/1,
+         t2b_system_limit/1]).
 
 %% Internal exports.
 -export([sleeper/0,trapping_loop/4]).
@@ -79,7 +80,7 @@
 suite() -> [{ct_hooks,[ts_install_cth]},
 	    {timetrap,{minutes,4}}].
 
-all() -> 
+all() ->
     [copy_terms, conversions, deep_lists, deep_bitstr_lists,
      t_split_binary, bad_split,
      bad_list_to_binary, bad_binary_to_list, terms,
@@ -90,7 +91,8 @@ all() ->
      b2t_used_big,
      bad_binary_to_term_2, safe_binary_to_term2,
      bad_binary_to_term, bad_terms, t_hash, bad_size,
-     bad_term_to_binary, more_bad_terms, otp_5484, otp_5933,
+     bad_term_to_binary, t2b_system_limit, more_bad_terms,
+     otp_5484, otp_5933,
      ordering, unaligned_order, gc_test,
      bit_sized_binary_sizes, otp_6817, otp_8117, deep,
      term2bin_tuple_fallbacks,
@@ -462,6 +464,54 @@ bad_term_to_binary(Config) when is_list(Config) ->
 
     ok.
 
+t2b_system_limit(Config) when is_list(Config) ->
+    case erlang:system_info(wordsize) of
+        8 ->
+            case proplists:get_value(system_total_memory,
+                                     memsup:get_system_memory_data()) of
+                Memory when is_integer(Memory),
+                            Memory > 6*1024*1024*1024 ->
+                    test_t2b_system_limit(),
+                    garbage_collect(),
+                    ok;
+                _ ->
+                    {skipped, "Not enough memory on this machine"}
+            end;
+        4 ->
+            {skipped, "Only interesting on 64-bit builds"}
+    end.
+
+test_t2b_system_limit() ->
+    io:format("Creating HugeBin~n", []),
+    Bits = ((1 bsl 32)+1)*8,
+    HugeBin = <<0:Bits>>,
+
+    io:format("Testing term_to_binary(HugeBin)~n", []),
+    {'EXIT',{system_limit,[{erlang,term_to_binary,
+                            [HugeBin],
+                            _} |_]}} = (catch term_to_binary(HugeBin)),
+
+    io:format("Testing term_to_binary(HugeBin, [compressed])~n", []),
+    {'EXIT',{system_limit,[{erlang,term_to_binary,
+                            [HugeBin, [compressed]],
+                            _} |_]}} = (catch term_to_binary(HugeBin, [compressed])),
+
+    %% Check that it works also after we have trapped...
+    io:format("Creating HugeListBin~n", []),
+    HugeListBin = [lists:duplicate(2000000,2000000), HugeBin],
+
+    io:format("Testing term_to_binary(HugeListBin)~n", []),
+    {'EXIT',{system_limit,[{erlang,term_to_binary,
+                            [HugeListBin],
+                            _} |_]}} = (catch term_to_binary(HugeListBin)),
+
+    io:format("Testing term_to_binary(HugeListBin, [compressed])~n", []),
+    {'EXIT',{system_limit,[{erlang,term_to_binary,
+                            [HugeListBin, [compressed]],
+                            _} |_]}} = (catch term_to_binary(HugeListBin, [compressed])),
+
+    ok.
+
 %% Tests binary_to_term/1 and term_to_binary/1.
 
 terms(Config) when is_list(Config) ->
diff --git a/erts/emulator/test/distribution_SUITE.erl b/erts/emulator/test/distribution_SUITE.erl
index 973a8f4717..7885d35d9d 100644
--- a/erts/emulator/test/distribution_SUITE.erl
+++ b/erts/emulator/test/distribution_SUITE.erl
@@ -67,7 +67,8 @@
          message_latency_large_message/1,
          message_latency_large_link_exit/1,
          message_latency_large_monitor_exit/1,
-         message_latency_large_exit2/1]).
+         message_latency_large_exit2/1,
+         system_limit/1]).
 
 %% Internal exports.
 -export([sender/3, receiver2/2, dummy_waiter/0, dead_process/0,
@@ -96,7 +97,7 @@ all() ->
      contended_atom_cache_entry, contended_unicode_atom_cache_entry,
      {group, message_latency},
      {group, bad_dist}, {group, bad_dist_ext},
-     start_epmd_false, epmd_module].
+     start_epmd_false, epmd_module, system_limit].
 
 groups() ->
     [{bulk_send, [], [bulk_send_small, bulk_send_big, bulk_send_bigbig]},
@@ -1537,6 +1538,144 @@ flush() ->
             ok
     end.
 
+system_limit(Config) when is_list(Config) ->
+    case erlang:system_info(wordsize) of
+        8 ->
+            case proplists:get_value(system_total_memory,
+                                     memsup:get_system_memory_data()) of
+                Memory when is_integer(Memory),
+                            Memory > 6*1024*1024*1024 ->
+                    test_system_limit(Config),
+                    garbage_collect(),
+                    ok;
+                _ ->
+                    {skipped, "Not enough memory on this machine"}
+            end;
+        4 ->
+            {skipped, "Only interesting on 64-bit builds"}
+    end.
+
+test_system_limit(Config) when is_list(Config) ->
+    Bits = ((1 bsl 32)+1)*8,
+    HugeBin = <<0:Bits>>,
+    HugeListBin = [lists:duplicate(2000000,2000000), HugeBin],
+    {ok, N1} = start_node(Config),
+    monitor_node(N1, true),
+    receive
+        {nodedown, N1} ->
+            ct:fail({unexpected_nodedown, N1})
+    after 0 ->
+            ok
+    end,
+    P1 = spawn(N1,
+               fun () ->
+                       receive after infinity -> ok end
+               end),
+
+    io:format("~n** distributed send **~n~n", []),
+    try
+        P1 ! HugeBin,
+        exit(oops1)
+    catch
+        error:system_limit -> ok
+    end,
+    try
+        P1 ! HugeListBin,
+        exit(oops2)
+    catch
+        error:system_limit -> ok
+    end,
+
+    io:format("~n** distributed exit **~n~n", []),
+    try
+        exit(P1, HugeBin),
+        exit(oops3)
+    catch
+        error:system_limit -> ok
+    end,
+    try
+        exit(P1, HugeListBin),
+        exit(oops4)
+    catch
+        error:system_limit -> ok
+    end,
+
+    io:format("~n** distributed registered send **~n~n", []),
+    try
+        {missing_proc, N1} ! HugeBin,
+        exit(oops5)
+    catch
+        error:system_limit -> ok
+    end,
+    try
+        {missing_proc, N1} ! HugeListBin,
+        exit(oops6)
+    catch
+        error:system_limit -> ok
+    end,
+    receive
+        {nodedown, N1} ->
+            ct:fail({unexpected_nodedown, N1})
+    after 0 ->
+            ok
+    end,
+
+    %%
+    %% system_limit in exit reasons brings the
+    %% connection down...
+    %%
+
+    io:format("~n** distributed link exit **~n~n", []),
+    spawn(fun () ->
+                  link(P1),
+                  exit(HugeBin)
+          end),
+    receive {nodedown, N1} -> ok end,
+
+    {ok, N2} = start_node(Config),
+    monitor_node(N2, true),
+    P2 = spawn(N2,
+               fun () ->
+                       receive after infinity -> ok end
+               end),
+    spawn(fun () ->
+                  link(P2),
+                  exit(HugeListBin)
+          end),
+    receive {nodedown, N2} -> ok end,
+
+    io:format("~n** distributed monitor down **~n~n", []),
+    {ok, N3} = start_node(Config),
+    monitor_node(N3, true),
+    Go1 = make_ref(),
+    LP1 = spawn(fun () ->
+                        receive Go1 -> ok end,
+                        exit(HugeBin)
+                end),
+    _ = spawn(N3,
+               fun () ->
+                       _ = erlang:monitor(process, LP1),
+                       LP1 ! Go1,
+                       receive after infinity -> ok end
+               end),
+    receive {nodedown, N3} -> ok end,
+
+    {ok, N4} = start_node(Config),
+    monitor_node(N4, true),
+    Go2 = make_ref(),
+    LP2 = spawn(fun () ->
+                        receive Go2 -> ok end,
+                        exit(HugeListBin)
+                end),
+    _ = spawn(N4,
+              fun () ->
+                      _ = erlang:monitor(process, LP2),
+                      LP2 ! Go2,
+                      receive after infinity -> ok end
+              end),
+    receive {nodedown, N4} -> ok end,
+    ok.
+
 -define(COOKIE, '').
 -define(DOP_LINK,		1).
 -define(DOP_SEND,		2).