Fragmented young heap generation and off_heap_message_queue option

* The youngest generation of the heap can now consist of multiple
  blocks. Heap fragments and message fragments are added to the
  youngest generation when needed without triggering a GC. After
  a GC the youngest generation is contained in one single block.
* The off_heap_message_queue process flag has been added. When
  enabled all message data in the queue is kept off heap. When
  a message is selected from the queue, the message fragment (or
  heap fragment) containing the actual message is attached to the
  youngest generation. Messages stored off heap is not part of GC.

2 files changed, 38 insertions, 32 deletions

diff --git a/erts/emulator/hipe/hipe_gc.c b/erts/emulator/hipe/hipe_gc.c
index 2c747771ac..2e19bf88bf 100644
--- a/erts/emulator/hipe/hipe_gc.c
+++ b/erts/emulator/hipe/hipe_gc.c
@@ -46,10 +46,6 @@ Eterm *fullsweep_nstack(Process *p, Eterm *n_htop)
     /* arch-specific nstack walk state */
     struct nstack_walk_state walk_state;
 
-    /* fullsweep-specific state */
-    char *src, *oh;
-    Uint src_size, oh_size;
-
     if (!p->hipe.nstack) {
 	ASSERT(!p->hipe.nsp && !p->hipe.nstend);
 	return n_htop;
@@ -66,11 +62,6 @@ Eterm *fullsweep_nstack(Process *p, Eterm *n_htop)
 
     sdesc = nstack_walk_init_sdesc(p, &walk_state);
 
-    src = (char*)HEAP_START(p);
-    src_size = (char*)HEAP_TOP(p) - src;
-    oh = (char*)OLD_HEAP(p);
-    oh_size = (char*)OLD_HTOP(p) - oh;
-
     for (;;) {
 	if (nstack_walk_nsp_reached_end(nsp, nsp_end)) {
 	    if (nsp == nsp_end) {
@@ -97,8 +88,7 @@ Eterm *fullsweep_nstack(Process *p, Eterm *n_htop)
 		    if (IS_MOVED_BOXED(val)) {
 			ASSERT(is_boxed(val));
 			*nsp_i = val;
-		    } else if (in_area(ptr, src, src_size) ||
-			       in_area(ptr, oh, oh_size)) {
+		    } else if (!erts_is_literal(gval, ptr)) {
 			MOVE_BOXED(ptr, val, n_htop, nsp_i);
 		    }
 		} else if (is_list(gval)) {
@@ -106,8 +96,7 @@ Eterm *fullsweep_nstack(Process *p, Eterm *n_htop)
 		    Eterm val = *ptr;
 		    if (IS_MOVED_CONS(val)) {
 			*nsp_i = ptr[1];
-		    } else if (in_area(ptr, src, src_size) ||
-			       in_area(ptr, oh, oh_size)) {
+		    } else if (!erts_is_literal(gval, ptr)) {
 			ASSERT(within(ptr, p));
 			MOVE_CONS(ptr, val, n_htop, nsp_i);
 		    }
@@ -139,11 +128,13 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop)
     unsigned int mask;
     /* arch-specific nstack walk state */
     struct nstack_walk_state walk_state;
+    char *oh;
+    Uint oh_size;
 
     /* gensweep-specific state */
     Eterm *old_htop, *n_htop;
-    char *heap;
-    Uint heap_size, mature_size;
+    char *mature;
+    Uint mature_size;
 
     if (!p->hipe.nstack) {
 	ASSERT(!p->hipe.nsp && !p->hipe.nstend);
@@ -168,9 +159,10 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop)
 
     old_htop = *ptr_old_htop;
     n_htop = *ptr_n_htop;
-    heap = (char*)HEAP_START(p);
-    heap_size = (char*)HEAP_TOP(p) - heap;
-    mature_size = (char*)HIGH_WATER(p) - heap;
+    mature = (char *) (p->abandoned_heap ? p->abandoned_heap : p->heap);
+    mature_size = (char*)HIGH_WATER(p) - mature;
+    oh = (char*)OLD_HEAP(p);
+    oh_size = (char*)OLD_HTOP(p) - oh;
 
     for (;;) {
 	if (nstack_walk_nsp_reached_end(nsp, nsp_end)) {
@@ -209,9 +201,9 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop)
 		    if (IS_MOVED_BOXED(val)) {
 			ASSERT(is_boxed(val));
 			*nsp_i = val;
-		    } else if (in_area(ptr, heap, mature_size)) {
+		    } else if (ErtsInArea(ptr, mature, mature_size)) {
 			MOVE_BOXED(ptr, val, old_htop, nsp_i);
-		    } else if (in_area(ptr, heap, heap_size)) {
+		    } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) {
 			ASSERT(within(ptr, p));
 			MOVE_BOXED(ptr, val, n_htop, nsp_i);
 		    }
@@ -220,9 +212,9 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop)
 		    Eterm val = *ptr;
 		    if (IS_MOVED_CONS(val)) {
 			*nsp_i = ptr[1];
-		    } else if (in_area(ptr, heap, mature_size)) {
+		    } else if (ErtsInArea(ptr, mature, mature_size)) {
 			MOVE_CONS(ptr, val, old_htop, nsp_i);
-		    } else if (in_area(ptr, heap, heap_size)) {
+		    } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) {
 			ASSERT(within(ptr, p));
 			MOVE_CONS(ptr, val, n_htop, nsp_i);
 		    }
diff --git a/erts/emulator/hipe/hipe_native_bif.c b/erts/emulator/hipe/hipe_native_bif.c
index 98bda43f0e..ceae3497c5 100644
--- a/erts/emulator/hipe/hipe_native_bif.c
+++ b/erts/emulator/hipe/hipe_native_bif.c
@@ -160,13 +160,22 @@ BIF_RETTYPE hipe_set_timeout(BIF_ALIST_1)
  */
 void hipe_select_msg(Process *p)
 {
-    ErlMessage *msgp;
+    ErtsMessage *msgp;
 
     msgp = PEEK_MESSAGE(p);
     UNLINK_MESSAGE(p, msgp);	/* decrements global 'erts_proc_tot_mem' variable */
     JOIN_MESSAGE(p);
     CANCEL_TIMER(p);		/* calls erts_cancel_proc_timer() */
-    free_message(msgp);
+    erts_save_message_in_proc(p, msgp);
+    p->flags &= ~F_DISABLE_GC;
+    if (ERTS_IS_GC_DESIRED(p)) {
+	/*
+	 * We want to GC soon but we leave a few
+	 * reductions giving the message some time
+	 * to turn into garbage.
+	 */
+	ERTS_VBUMP_LEAVE_REDS(p, 5);
+    }
 }
 
 void hipe_fclearerror_error(Process *p)
@@ -511,8 +520,9 @@ int hipe_bs_validate_unicode_retract(ErlBinMatchBuffer* mb, Eterm arg)
  */
 Eterm hipe_check_get_msg(Process *c_p)
 {
-    Eterm ret;
-    ErlMessage *msgp;
+    ErtsMessage *msgp;
+
+    c_p->flags |= F_DISABLE_GC;
 
  next_message:
 
@@ -534,25 +544,29 @@ Eterm hipe_check_get_msg(Process *c_p)
 	    /* XXX: BEAM doesn't need this */
 	    c_p->hipe_smp.have_receive_locks = 1;
 #endif
+	    c_p->flags &= ~F_DISABLE_GC;
 	    return THE_NON_VALUE;
 #ifdef ERTS_SMP
 	}
 #endif
     }
-    ErtsMoveMsgAttachmentIntoProc(msgp, c_p, c_p->stop, HEAP_TOP(c_p),
-				  c_p->fcalls, (void) 0, (void) 0);
-    ret = ERL_MESSAGE_TERM(msgp);
-    if (is_non_value(ret)) {
+
+    if (is_non_value(ERL_MESSAGE_TERM(msgp))
+	&& !erts_decode_dist_message(c_p, ERTS_PROC_LOCK_MAIN, msgp, 0)) {
 	/*
 	 * A corrupt distribution message that we weren't able to decode;
 	 * remove it...
 	 */
 	ASSERT(!msgp->data.attached);
 	UNLINK_MESSAGE(c_p, msgp);
-	free_message(msgp);
+	msgp->next = NULL;
+	erts_cleanup_messages(msgp);
 	goto next_message;
     }
-    return ret;
+
+    ASSERT(is_value(ERL_MESSAGE_TERM(msgp)));
+
+    return ERL_MESSAGE_TERM(msgp);
 }
 
 /*