1 files changed, 139 insertions, 69 deletions

diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index 0a5050b1fe..8bee47232e 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -37,6 +37,8 @@ MA_STACK_DECLARE(dst);
 MA_STACK_DECLARE(offset);
 #endif
 
+static void move_one_frag(Eterm** hpp, Eterm* src, Uint src_sz, ErlOffHeap*);
+
 void
 init_copy(void)
 {
@@ -86,7 +88,7 @@ size_object(Eterm obj)
 	    obj = *ptr++;
 	    if (!IS_CONST(obj)) {
 		ESTACK_PUSH(s, obj);
-	    }
+	    }	    
 	    obj = *ptr;
 	    break;
 	case TAG_PRIMARY_BOXED:
@@ -99,7 +101,7 @@ size_object(Eterm obj)
 		    arity = header_arity(hdr);
 		    sum += arity + 1;
 		    if (arity == 0) { /* Empty tuple -- unusual. */
-			goto size_common;
+			goto pop_next;
 		    }
 		    while (arity-- > 1) {
 			obj = *++ptr;
@@ -115,7 +117,6 @@ size_object(Eterm obj)
 			ErlFunThing* funp = (ErlFunThing *) bptr;
 			unsigned eterms = 1 /* creator */ + funp->num_free;
 			unsigned sz = thing_arityval(hdr);
-
 			sum += 1 /* header */ + sz + eterms;
 			bptr += 1 /* header */ + sz;
 			while (eterms-- > 1) {
@@ -151,7 +152,7 @@ size_object(Eterm obj)
 			} else {
 			    sum += heap_bin_size(binary_size(obj)+extra_bytes);
 			}
-			goto size_common;
+			goto pop_next;
 		    }
 		    break;
 		case BIN_MATCHSTATE_SUBTAG:
@@ -159,18 +160,12 @@ size_object(Eterm obj)
 			     "size_object: matchstate term not allowed");
 		default:
 		    sum += thing_arityval(hdr) + 1;
-		    /* Fall through */
-		size_common:
-		    if (ESTACK_ISEMPTY(s)) {
-			DESTROY_ESTACK(s);
-			return sum;
-		    }
-		    obj = ESTACK_POP(s);
-		    break;
+		    goto pop_next;
 		}
 	    }
 	    break;
 	case TAG_PRIMARY_IMMED1:
+	pop_next:
 	    if (ESTACK_ISEMPTY(s)) {
 		DESTROY_ESTACK(s);
 		return sum;
@@ -319,10 +314,10 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 		    *argp = make_binary(hbot);
 		    pb = (ProcBin*) hbot;
 		    erts_refc_inc(&pb->val->refc, 2);
-		    pb->next = off_heap->mso;
+		    pb->next = off_heap->first;
 		    pb->flags = 0;
-		    off_heap->mso = pb;
-		    off_heap->overhead += pb->size / sizeof(Eterm);
+		    off_heap->first = (struct erl_off_heap_header*) pb;
+		    OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm));
 		}
 		break;
 	    case SUB_BINARY_SUBTAG:
@@ -368,10 +363,10 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 			to->val = from->val;
 			erts_refc_inc(&to->val->refc, 2);
 			to->bytes = from->bytes + offset;
-			to->next = off_heap->mso;
+			to->next = off_heap->first;
 			to->flags = 0;
-			off_heap->mso = to;
-			off_heap->overhead += to->size / sizeof(Eterm);
+			off_heap->first = (struct erl_off_heap_header*) to;
+			OH_OVERHEAD(off_heap, to->size / sizeof(Eterm));
 		    }
 		    *argp = make_binary(hbot);
 		    if (extra_bytes != 0) {
@@ -401,8 +396,8 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 		    }
 #ifndef HYBRID /* FIND ME! */
 		    funp = (ErlFunThing *) tp;
-		    funp->next = off_heap->funs;
-		    off_heap->funs = funp;
+		    funp->next = off_heap->first;
+		    off_heap->first = (struct erl_off_heap_header*) funp;
 		    erts_refc_inc(&funp->fe->refc, 2);
 #endif
 		    *argp = make_fun(tp);
@@ -421,8 +416,8 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 		    *htop++ = *objp++;
 		  }
 
-		  etp->next = off_heap->externals;
-		  off_heap->externals = etp;
+		  etp->next = off_heap->first;
+		  off_heap->first = (struct erl_off_heap_header*)etp;
 		  erts_refc_inc(&etp->node->refc, 2);
 
 		  *argp = make_external(tp);
@@ -655,9 +650,9 @@ Eterm copy_struct_lazy(Process *from, Eterm orig, Uint offs)
                     *hp++ = *objp++;
                 }
                 erts_refc_inc(&pb->val->refc, 2);
-                pb->next = erts_global_offheap.mso;
-                erts_global_offheap.mso = pb;
-                erts_global_offheap.overhead += pb->size / sizeof(Eterm);
+                pb->next = erts_global_offheap.first;
+                erts_global_offheap.first = pb;
+		OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm));
                 continue;
             }
 
@@ -677,9 +672,9 @@ Eterm copy_struct_lazy(Process *from, Eterm orig, Uint offs)
                 while (i--) {
                     *hp++ = *objp++;
                 }
-#ifndef HYBRID // FIND ME!
-                funp->next = erts_global_offheap.funs;
-                erts_global_offheap.funs = funp;
+#ifndef HYBRID /* FIND ME! */
+                funp->next = erts_global_offheap.first;
+                erts_global_offheap.first = funp;
                 erts_refc_inc(&funp->fe->refc, 2);
 #endif
                 for (i = k; i < j; i++) {
@@ -723,8 +718,8 @@ Eterm copy_struct_lazy(Process *from, Eterm orig, Uint offs)
                     *hp++ = *objp++;
                 }
 
-                etp->next = erts_global_offheap.externals;
-                erts_global_offheap.externals = etp;
+                etp->next = erts_global_offheap.first;
+                erts_global_offheap.first = etp;
 		erts_refc_inc(&etp->node->refc, 2);
                 continue;
             }
@@ -780,9 +775,9 @@ Eterm copy_struct_lazy(Process *from, Eterm orig, Uint offs)
                     to_bin->size = real_size;
                     to_bin->val = from_bin->val;
                     to_bin->bytes = from_bin->bytes + sub_offset;
-                    to_bin->next = erts_global_offheap.mso;
-                    erts_global_offheap.mso = to_bin;
-                    erts_global_offheap.overhead += to_bin->size / sizeof(Eterm);
+                    to_bin->next = erts_global_offheap.first;
+                    erts_global_offheap.first = to_bin;
+		    OH_OVERHEAD(&erts_global_offheap, to_bin->size / sizeof(Eterm));
 		    res_binary=make_binary(to_bin);
 		    hp += PROC_BIN_SIZE;
                 }
@@ -915,57 +910,43 @@ copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 		break;
 	    case REFC_BINARY_SUBTAG:
 		{
-		    ProcBin* pb = (ProcBin *) (hp-1);
-		    int tari = thing_arityval(val);
-
-		    sz -= tari;
-		    while (tari--) {
-			*hp++ = *tp++;
-		    }
+		    ProcBin* pb = (ProcBin *) (tp-1);
 		    erts_refc_inc(&pb->val->refc, 2);
-		    pb->next = off_heap->mso;
-		    off_heap->mso = pb;
-		    off_heap->overhead += pb->size / sizeof(Eterm);
+		    OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm));
 		}
-		break;
+		goto off_heap_common;
+
 	    case FUN_SUBTAG:
 		{
-#ifndef HYBRID /* FIND ME! */
-		    ErlFunThing* funp = (ErlFunThing *) (hp-1);
-#endif
-		    int tari = thing_arityval(val);
-
-		    sz -= tari;
-		    while (tari--) {
-			*hp++ = *tp++;
-		    }
-#ifndef HYBRID /* FIND ME! */
-		    funp->next = off_heap->funs;
-		    off_heap->funs = funp;
+		    ErlFunThing* funp = (ErlFunThing *) (tp-1);
 		    erts_refc_inc(&funp->fe->refc, 2);
-#endif
 		}
-		break;
+		goto off_heap_common;
+
 	    case EXTERNAL_PID_SUBTAG:
 	    case EXTERNAL_PORT_SUBTAG:
 	    case EXTERNAL_REF_SUBTAG:
 		{
-		    ExternalThing* etp = (ExternalThing *) (hp-1);
+		    ExternalThing* etp = (ExternalThing *) (tp-1);
+		    erts_refc_inc(&etp->node->refc, 2);
+		}
+	    off_heap_common:
+		{
+		    struct erl_off_heap_header* ohh = (struct erl_off_heap_header*)(hp-1);
 		    int tari = thing_arityval(val);
-
+		    
 		    sz -= tari;
 		    while (tari--) {
 			*hp++ = *tp++;
 		    }
-		    etp->next = off_heap->externals;
-		    off_heap->externals = etp;
-		    erts_refc_inc(&etp->node->refc, 2);
+		    ohh->next = off_heap->first;
+		    off_heap->first = ohh;
 		}
 		break;
 	    default:
 		{
 		    int tari = header_arity(val);
-
+    
 		    sz -= tari;
 		    while (tari--) {
 			*hp++ = *tp++;
@@ -979,3 +960,92 @@ copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
     *hpp = hp;
     return make_tuple(ptr + offs); 
 }
+
+/* Move all terms in heap fragments into heap. The terms must be guaranteed to 
+ * be contained within the fragments. The source terms are destructed with
+ * move markers.
+ * Typically used to copy a multi-fragmented message (from NIF).
+ */
+void move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first,
+		      Eterm* refs, unsigned nrefs)
+{
+    ErlHeapFragment* bp;
+    Eterm* hp_start = *hpp;
+    Eterm* hp_end;
+    Eterm* hp;
+    unsigned i;
+
+    for (bp=first; bp!=NULL; bp=bp->next) {
+	move_one_frag(hpp, bp->mem, bp->used_size, off_heap);
+	OH_OVERHEAD(off_heap, bp->off_heap.overhead);
+    }
+    hp_end = *hpp;
+    for (hp=hp_start; hp<hp_end; ++hp) {
+	Eterm* ptr;
+	Eterm val;
+	Eterm gval = *hp;
+	switch (primary_tag(gval)) {
+	case TAG_PRIMARY_BOXED:
+	    ptr = boxed_val(gval);
+	    val = *ptr;
+	    if (IS_MOVED_BOXED(val)) {
+		ASSERT(is_boxed(val));
+		*hp = val;
+	    }
+	    break;
+	case TAG_PRIMARY_LIST:
+	    ptr = list_val(gval);
+	    val = *ptr;
+	    if (IS_MOVED_CONS(val)) {
+		*hp = ptr[1];
+	    }
+	    break;
+	case TAG_PRIMARY_HEADER:
+	    if (header_is_thing(gval)) {
+		hp += thing_arityval(gval);
+	    }
+	    break;
+	}
+    }
+    for (i=0; i<nrefs; ++i) {
+	refs[i] = follow_moved(refs[i]);
+    }
+}
+
+static void
+move_one_frag(Eterm** hpp, Eterm* src, Uint src_sz, ErlOffHeap* off_heap)
+{
+    Eterm* ptr = src;
+    Eterm* end = ptr + src_sz;
+    Eterm dummy_ref;
+    Eterm* hp = *hpp;
+
+    while (ptr != end) {
+	Eterm val;
+	ASSERT(ptr < end);
+	val = *ptr;
+	ASSERT(val != ERTS_HOLE_MARKER);
+	if (is_header(val)) {
+	    struct erl_off_heap_header* hdr = (struct erl_off_heap_header*)hp;
+	    ASSERT(ptr + header_arity(val) < end);
+	    MOVE_BOXED(ptr, val, hp, &dummy_ref);	    
+	    switch (val & _HEADER_SUBTAG_MASK) {
+	    case REFC_BINARY_SUBTAG:
+	    case FUN_SUBTAG:
+	    case EXTERNAL_PID_SUBTAG:
+	    case EXTERNAL_PORT_SUBTAG:
+	    case EXTERNAL_REF_SUBTAG:
+		hdr->next = off_heap->first;
+		off_heap->first = hdr;
+		break;
+	    }
+	}
+	else { /* must be a cons cell */
+	    ASSERT(ptr+1 < end);
+	    MOVE_CONS(ptr, val, hp, &dummy_ref);
+	    ptr += 2;
+	}
+    }
+    *hpp = hp;
+}
+