One off-heap list, to eliminate two words per ETS object.

Merging the three off-heap lists (binaries, funs and externals) into
one list. This reduces memory consumption by two words (pointers) per
ETS object.

1 files changed, 36 insertions, 62 deletions

diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index 521a1b1788..57be0169ba 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -314,9 +314,9 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 		    *argp = make_binary(hbot);
 		    pb = (ProcBin*) hbot;
 		    erts_refc_inc(&pb->val->refc, 2);
-		    pb->next = off_heap->mso;
+		    pb->next = off_heap->first;
 		    pb->flags = 0;
-		    off_heap->mso = pb;
+		    off_heap->first = (struct erl_off_heap_header*) pb;
 		    off_heap->overhead += pb->size / sizeof(Eterm);
 		}
 		break;
@@ -363,9 +363,9 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 			to->val = from->val;
 			erts_refc_inc(&to->val->refc, 2);
 			to->bytes = from->bytes + offset;
-			to->next = off_heap->mso;
+			to->next = off_heap->first;
 			to->flags = 0;
-			off_heap->mso = to;
+			off_heap->first = (struct erl_off_heap_header*) to;
 			off_heap->overhead += to->size / sizeof(Eterm);
 		    }
 		    *argp = make_binary(hbot);
@@ -396,8 +396,8 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 		    }
 #ifndef HYBRID /* FIND ME! */
 		    funp = (ErlFunThing *) tp;
-		    funp->next = off_heap->funs;
-		    off_heap->funs = funp;
+		    funp->next = off_heap->first;
+		    off_heap->first = (struct erl_off_heap_header*) funp;
 		    erts_refc_inc(&funp->fe->refc, 2);
 #endif
 		    *argp = make_fun(tp);
@@ -416,8 +416,8 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 		    *htop++ = *objp++;
 		  }
 
-		  etp->next = off_heap->externals;
-		  off_heap->externals = etp;
+		  etp->next = off_heap->first;
+		  off_heap->first = (struct erl_off_heap_header*)etp;
 		  erts_refc_inc(&etp->node->refc, 2);
 
 		  *argp = make_external(tp);
@@ -650,8 +650,8 @@ Eterm copy_struct_lazy(Process *from, Eterm orig, Uint offs)
                     *hp++ = *objp++;
                 }
                 erts_refc_inc(&pb->val->refc, 2);
-                pb->next = erts_global_offheap.mso;
-                erts_global_offheap.mso = pb;
+                pb->next = erts_global_offheap.first;
+                erts_global_offheap.first = pb;
                 erts_global_offheap.overhead += pb->size / sizeof(Eterm);
                 continue;
             }
@@ -672,9 +672,9 @@ Eterm copy_struct_lazy(Process *from, Eterm orig, Uint offs)
                 while (i--) {
                     *hp++ = *objp++;
                 }
-#ifndef HYBRID // FIND ME!
-                funp->next = erts_global_offheap.funs;
-                erts_global_offheap.funs = funp;
+#ifndef HYBRID /* FIND ME! */
+                funp->next = erts_global_offheap.first;
+                erts_global_offheap.first = funp;
                 erts_refc_inc(&funp->fe->refc, 2);
 #endif
                 for (i = k; i < j; i++) {
@@ -718,8 +718,8 @@ Eterm copy_struct_lazy(Process *from, Eterm orig, Uint offs)
                     *hp++ = *objp++;
                 }
 
-                etp->next = erts_global_offheap.externals;
-                erts_global_offheap.externals = etp;
+                etp->next = erts_global_offheap.first;
+                erts_global_offheap.first = etp;
 		erts_refc_inc(&etp->node->refc, 2);
                 continue;
             }
@@ -775,8 +775,8 @@ Eterm copy_struct_lazy(Process *from, Eterm orig, Uint offs)
                     to_bin->size = real_size;
                     to_bin->val = from_bin->val;
                     to_bin->bytes = from_bin->bytes + sub_offset;
-                    to_bin->next = erts_global_offheap.mso;
-                    erts_global_offheap.mso = to_bin;
+                    to_bin->next = erts_global_offheap.first;
+                    erts_global_offheap.first = to_bin;
                     erts_global_offheap.overhead += to_bin->size / sizeof(Eterm);
 		    res_binary=make_binary(to_bin);
 		    hp += PROC_BIN_SIZE;
@@ -910,57 +910,43 @@ copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 		break;
 	    case REFC_BINARY_SUBTAG:
 		{
-		    ProcBin* pb = (ProcBin *) (hp-1);
-		    int tari = thing_arityval(val);
-
-		    sz -= tari;
-		    while (tari--) {
-			*hp++ = *tp++;
-		    }
+		    ProcBin* pb = (ProcBin *) (tp-1);
 		    erts_refc_inc(&pb->val->refc, 2);
-		    pb->next = off_heap->mso;
-		    off_heap->mso = pb;
 		    off_heap->overhead += pb->size / sizeof(Eterm);
 		}
-		break;
+		goto off_heap_common;
+
 	    case FUN_SUBTAG:
 		{
-#ifndef HYBRID /* FIND ME! */
-		    ErlFunThing* funp = (ErlFunThing *) (hp-1);
-#endif
-		    int tari = thing_arityval(val);
-
-		    sz -= tari;
-		    while (tari--) {
-			*hp++ = *tp++;
-		    }
-#ifndef HYBRID /* FIND ME! */
-		    funp->next = off_heap->funs;
-		    off_heap->funs = funp;
+		    ErlFunThing* funp = (ErlFunThing *) (tp-1);
 		    erts_refc_inc(&funp->fe->refc, 2);
-#endif
 		}
-		break;
+		goto off_heap_common;
+
 	    case EXTERNAL_PID_SUBTAG:
 	    case EXTERNAL_PORT_SUBTAG:
 	    case EXTERNAL_REF_SUBTAG:
 		{
-		    ExternalThing* etp = (ExternalThing *) (hp-1);
+		    ExternalThing* etp = (ExternalThing *) (tp-1);
+		    erts_refc_inc(&etp->node->refc, 2);
+		}
+	    off_heap_common:
+		{
+		    struct erl_off_heap_header* ohh = (struct erl_off_heap_header*)(hp-1);
 		    int tari = thing_arityval(val);
-
+		    
 		    sz -= tari;
 		    while (tari--) {
 			*hp++ = *tp++;
 		    }
-		    etp->next = off_heap->externals;
-		    off_heap->externals = etp;
-		    erts_refc_inc(&etp->node->refc, 2);
+		    ohh->next = off_heap->first;
+		    off_heap->first = ohh;
 		}
 		break;
 	    default:
 		{
 		    int tari = header_arity(val);
-
+    
 		    sz -= tari;
 		    while (tari--) {
 			*hp++ = *tp++;
@@ -1029,12 +1015,6 @@ void move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first,
 static void
 move_one_frag(Eterm** hpp, Eterm* src, Uint src_sz, ErlOffHeap* off_heap)
 {
-    union {
-	Uint *up;
-	ProcBin *pbp;
-	ErlFunThing *efp;
-	ExternalThing *etp;
-    } ohe;
     Eterm* ptr = src;
     Eterm* end = ptr + src_sz;
     Eterm dummy_ref;
@@ -1046,23 +1026,17 @@ move_one_frag(Eterm** hpp, Eterm* src, Uint src_sz, ErlOffHeap* off_heap)
 	val = *ptr;
 	ASSERT(val != ERTS_HOLE_MARKER);
 	if (is_header(val)) {
+	    struct erl_off_heap_header* hdr = (struct erl_off_heap_header*)hp;
 	    ASSERT(ptr + header_arity(val) < end);
-	    ohe.up = hp;
 	    MOVE_BOXED(ptr, val, hp, &dummy_ref);	    
 	    switch (val & _HEADER_SUBTAG_MASK) {
 	    case REFC_BINARY_SUBTAG:
-		ohe.pbp->next = off_heap->mso;
-		off_heap->mso = ohe.pbp;
-		break;
 	    case FUN_SUBTAG:
-		ohe.efp->next = off_heap->funs;
-		off_heap->funs = ohe.efp;
-		break;
 	    case EXTERNAL_PID_SUBTAG:
 	    case EXTERNAL_PORT_SUBTAG:
 	    case EXTERNAL_REF_SUBTAG:
-		ohe.etp->next = off_heap->externals;
-		off_heap->externals = ohe.etp;
+		hdr->next = off_heap->first;
+		off_heap->first = hdr;
 		break;
 	    }
 	}