Merge branch 'max-au/dist_msg_too_long'

* max-au/dist_msg_too_long:
  Cleanup unused dist output buf immediately instead of at GC
  Throw 'system_limit' when distribution message size exceed INT_MAX instead of crashing emulator with 'Absurdly large distribution data buffer'

3 files changed, 20 insertions, 0 deletions

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index f18af8bcd7..56ac072449 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -1803,6 +1803,7 @@ ebif_bang_2(BIF_ALIST_2)
 #define SEND_INTERNAL_ERROR	(-6)
 #define SEND_AWAIT_RESULT	(-7)
 #define SEND_YIELD_CONTINUE     (-8)
+#define SEND_SYSTEM_LIMIT		(-9)
 
 
 static Sint remote_send(Process *p, DistEntry *dep,
@@ -1842,6 +1843,8 @@ static Sint remote_send(Process *p, DistEntry *dep,
 	    res = SEND_YIELD_RETURN;
 	else if (code == ERTS_DSIG_SEND_CONTINUE)
 	    res = SEND_YIELD_CONTINUE;
+	else if (code == ERTS_DSIG_SEND_TOO_LRG)
+	    res = SEND_SYSTEM_LIMIT;
 	else
 	    res = 0;
 	break;
@@ -2162,6 +2165,9 @@ BIF_RETTYPE send_3(BIF_ALIST_3)
     case SEND_BADARG:
 	ERTS_BIF_PREP_ERROR(retval, p, BADARG);
 	break;
+    case SEND_SYSTEM_LIMIT:
+	ERTS_BIF_PREP_ERROR(retval, p, SYSTEM_LIMIT);
+	break;
     case SEND_USER_ERROR:
 	ERTS_BIF_PREP_ERROR(retval, p, EXC_ERROR);
 	break;
@@ -2218,6 +2224,10 @@ static BIF_RETTYPE dsend_continue_trap_1(BIF_ALIST_1)
 	BUMP_ALL_REDS(BIF_P);
 	BIF_TRAP1(&dsend_continue_trap_export, BIF_P, BIF_ARG_1);
     }
+    case ERTS_DSIG_SEND_TOO_LRG: { /*SEND_SYSTEM_LIMIT*/
+	erts_set_gc_state(BIF_P, 1);
+	BIF_ERROR(BIF_P, SYSTEM_LIMIT);
+    }
     default:
 	erts_exit(ERTS_ABORT_EXIT, "dsend_continue_trap invalid result %d\n", (int)result);
 	break;
@@ -2275,6 +2285,9 @@ Eterm erl_send(Process *p, Eterm to, Eterm msg)
     case SEND_BADARG:
 	ERTS_BIF_PREP_ERROR(retval, p, BADARG);
 	break;
+    case SEND_SYSTEM_LIMIT:
+	ERTS_BIF_PREP_ERROR(retval, p, SYSTEM_LIMIT);
+	break;
     case SEND_USER_ERROR:
 	ERTS_BIF_PREP_ERROR(retval, p, EXC_ERROR);
 	break;
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 146c00b07d..db594a23a0 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -1924,6 +1924,12 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx)
 	    ASSERT(ctx->obuf->ext_endp <= &ctx->obuf->data[0] + ctx->data_size);
 
 	    ctx->data_size = ctx->obuf->ext_endp - ctx->obuf->extp;
+	    if (ctx->data_size > (Uint) INT_MAX) {
+		free_dist_obuf(ctx->obuf);
+                ctx->obuf = NULL;
+		retval = ERTS_DSIG_SEND_TOO_LRG;
+		goto done;
+	    }
 
             ctx->obuf->hopefull_flags = ctx->u.ec.hopefull_flags;
 	    /*
diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index dda2029a4c..55204eb83d 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -376,6 +376,7 @@ typedef struct {
 #define ERTS_DSIG_SEND_OK	0
 #define ERTS_DSIG_SEND_YIELD	1
 #define ERTS_DSIG_SEND_CONTINUE 2
+#define ERTS_DSIG_SEND_TOO_LRG  3
 
 extern int erts_dsig_send_link(ErtsDSigData *, Eterm, Eterm);
 extern int erts_dsig_send_msg(Eterm, Eterm, ErtsSendContext*);