erts: Remove the need for copying of literals

* Literals are not copied between processes for messages or spawn

Increases performance of message sent and processes spawned when
literals are involved in messages or arguments.

5 files changed, 88 insertions, 49 deletions

diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index ccc4cbad43..7d3777f37e 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -40,7 +40,8 @@ static void move_one_frag(Eterm** hpp, ErlHeapFragment*, ErlOffHeap*, int);
 /*
  *  Copy object "obj" to process p.
  */
-Eterm copy_object_x(Eterm obj, Process* to, Uint extra) {
+Eterm copy_object_x(Eterm obj, Process* to, Uint extra)
+{
     if (!is_immed(obj)) {
         Uint size = size_object(obj);
         Eterm* hp = HAllocX(to, size, extra);
@@ -70,7 +71,12 @@ Eterm copy_object_x(Eterm obj, Process* to, Uint extra) {
  * Return the "flat" size of the object.
  */
 
-Uint size_object(Eterm obj)
+#define in_literal_purge_area(PTR)                   \
+    (lit_purge_ptr && (                              \
+        (lit_purge_ptr <= (PTR) &&                   \
+        (PTR) < (lit_purge_ptr + lit_purge_sz))))
+
+Uint size_object_x(Eterm obj, Eterm *lit_purge_ptr, Uint lit_purge_sz, Uint litopt)
 {
     Uint sum = 0;
     Eterm* ptr;
@@ -78,7 +84,6 @@ Uint size_object(Eterm obj)
 #ifdef DEBUG
     Eterm mypid = erts_get_current_pid();
 #endif
-
     DECLARE_ESTACK(s);
 
     VERBOSE(DEBUG_SHCOPY, ("[pid=%T] size_object %p\n", mypid, obj));
@@ -86,17 +91,25 @@ Uint size_object(Eterm obj)
     for (;;) {
 	switch (primary_tag(obj)) {
 	case TAG_PRIMARY_LIST:
-	    sum += 2;
 	    ptr = list_val(obj);
+            if (litopt && erts_is_literal(obj,ptr) && !in_literal_purge_area(ptr)) {
+                goto pop_next;
+            }
+	    sum += 2;
 	    obj = *ptr++;
 	    if (!IS_CONST(obj)) {
 		ESTACK_PUSH(s, obj);
-	    }	    
+	    }
 	    obj = *ptr;
 	    break;
 	case TAG_PRIMARY_BOXED:
 	    {
-		Eterm hdr = *boxed_val(obj);
+		Eterm hdr;
+                ptr = boxed_val(obj);
+                if (litopt && erts_is_literal(obj,ptr) && !in_literal_purge_area(ptr)) {
+                    goto pop_next;
+                }
+                hdr = *ptr;
 		ASSERT(is_header(hdr));
 		switch (hdr & _TAG_HEADER_MASK) {
 		case ARITYVAL_SUBTAG:
@@ -279,10 +292,6 @@ do {                                                                    \
 
 #define COUNT_OFF_HEAP (0)
 
-#define IN_LITERAL_PURGE_AREA(info, ptr)                 \
-    ((info)->range_ptr && (                              \
-        (info)->range_ptr <= (ptr) &&                    \
-        (ptr) < ((info)->range_ptr + (info)->range_sz)))
 /*
  *  Return the real size of an object and find sharing information
  *  This currently returns the same as erts_debug:size/1.
@@ -599,7 +608,8 @@ cleanup:
 /*
  *  Copy a structure to a heap.
  */
-Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint *bsz)
+Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint *bsz,
+                    Eterm *lit_purge_ptr, Uint lit_purge_sz, Uint litopt)
 {
     char* hstart;
     Uint hsize;
@@ -651,7 +661,6 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint
  L_copy:
     while (hp != htop) {
 	obj = *hp;
-
 	switch (primary_tag(obj)) {
 	case TAG_PRIMARY_IMMED1:
 	    hp++;
@@ -667,6 +676,10 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint
 
 	L_copy_list:
 	    tailp = argp;
+            if (litopt && erts_is_literal(obj,objp) && !in_literal_purge_area(objp)) {
+                *tailp = obj;
+                goto L_copy;
+            }
 	    for (;;) {
 		tp = tailp;
 		elem = CAR(objp);
@@ -674,18 +687,23 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint
 		    hbot -= 2;
 		    CAR(hbot) = elem;
 		    tailp = &CDR(hbot);
-		}
-		else {
+		} else {
 		    CAR(htop) = elem;
 		    tailp = &CDR(htop);
 		    htop += 2;
 		}
 		*tp = make_list(tailp - 1);
 		obj = CDR(objp);
+
 		if (!is_list(obj)) {
 		    break;
 		}
 		objp = list_val(obj);
+
+                if (litopt && erts_is_literal(obj,objp) && !in_literal_purge_area(objp)) {
+                    *tailp = obj;
+                    goto L_copy;
+                }
 	    }
 	    switch (primary_tag(obj)) {
 	    case TAG_PRIMARY_IMMED1: *tailp = obj; goto L_copy;
@@ -695,7 +713,7 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint
 			 "%s, line %d: Internal error in copy_struct: 0x%08x\n",
 			 __FILE__, __LINE__,obj);
 	    }
-	    
+
 	case TAG_PRIMARY_BOXED:
 	    if (ErtsInArea(boxed_val(obj),hstart,hsize)) {
 		hp++;
@@ -705,6 +723,10 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint
 
 	L_copy_boxed:
 	    objp = boxed_val(obj);
+            if (litopt && erts_is_literal(obj,objp) && !in_literal_purge_area(objp)) {
+                *argp = obj;
+                break;
+            }
 	    hdr = *objp;
 	    switch (hdr & _TAG_HEADER_MASK) {
 	    case ARITYVAL_SUBTAG:
@@ -765,7 +787,7 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint
 			extra_bytes = 1;
 		    } else {
 			extra_bytes = 0;
-		    } 
+		    }
 		    real_size = size+extra_bytes;
 		    objp = binary_val(real_bin);
 		    if (thing_subtag(*objp) == HEAP_BINARY_SUBTAG) {
@@ -780,7 +802,7 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint
 		    } else {
 			ProcBin* from = (ProcBin *) objp;
 			ProcBin* to;
-			
+
 			ASSERT(thing_subtag(*objp) == REFC_BINARY_SUBTAG);
 			if (from->flags) {
 			    erts_emasculate_writable_binary(from);
@@ -900,6 +922,12 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint
         *bsz = hend - hbot;
     } else {
 #ifdef DEBUG
+        if (!eq(org_obj, res)) {
+            erts_exit(ERTS_ABORT_EXIT,
+                    "Internal error in copy_struct() when copying %T:"
+                    " not equal to copy %T\n",
+                    org_obj, res);
+        }
         if (htop != hbot)
             erts_exit(ERTS_ABORT_EXIT,
                     "Internal error in copy_struct() when copying %T:"
@@ -1036,6 +1064,8 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info)
     Uint e;
     unsigned sz;
     Eterm* ptr;
+    Eterm *lit_purge_ptr = info->lit_purge_ptr;
+    Uint lit_purge_sz = info->lit_purge_sz;
 #ifdef DEBUG
     Eterm mypid = erts_get_current_pid();
 #endif
@@ -1081,7 +1111,7 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info)
 	    /* off heap list pointers are copied verbatim */
 	    if (erts_is_literal(obj,ptr)) {
 		VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", mypid, ptr, obj));
-                if (IN_LITERAL_PURGE_AREA(info,ptr))
+                if (in_literal_purge_area(ptr))
                     info->literal_size += size_object(obj);
 		goto pop_next;
 	    }
@@ -1132,7 +1162,7 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info)
 	    /* off heap pointers to boxes are copied verbatim */
 	    if (erts_is_literal(obj,ptr)) {
 		VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", mypid, ptr, obj));
-                if (IN_LITERAL_PURGE_AREA(info,ptr))
+                if (in_literal_purge_area(ptr))
                     info->literal_size += size_object(obj);
 		goto pop_next;
 	    }
@@ -1298,6 +1328,8 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info,
     Eterm* resp;
     Eterm *hbot, *hend;
     unsigned remaining;
+    Eterm *lit_purge_ptr = info->lit_purge_ptr;
+    Uint lit_purge_sz = info->lit_purge_sz;
 #ifdef DEBUG
     Eterm mypid = erts_get_current_pid();
     Eterm saved_obj = obj;
@@ -1347,11 +1379,11 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info,
 	    ptr = list_val(obj);
 	    /* off heap list pointers are copied verbatim */
 	    if (erts_is_literal(obj,ptr)) {
-                if (!IN_LITERAL_PURGE_AREA(info,ptr)) {
+                if (!in_literal_purge_area(ptr)) {
                     *resp = obj;
                 } else {
                     Uint bsz = 0;
-                    *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz);
+                    *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz, NULL, 0, 0); /* copy literal */
                     hbot -= bsz;
                 }
 		goto cleanup_next;
@@ -1415,11 +1447,11 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info,
 	    ptr = boxed_val(obj);
 	    /* off heap pointers to boxes are copied verbatim */
 	    if (erts_is_literal(obj,ptr)) {
-                if (!IN_LITERAL_PURGE_AREA(info,ptr)) {
+                if (!in_literal_purge_area(ptr)) {
                     *resp = obj;
                 } else {
                     Uint bsz = 0;
-                    *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz);
+                    *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz, NULL, 0, 0); /* copy literal */
                     hbot -= bsz;
                 }
 		goto cleanup_next;
@@ -1923,7 +1955,7 @@ move_one_frag(Eterm** hpp, ErlHeapFragment* frag, ErlOffHeap* off_heap, int lite
 	if (is_header(val)) {
 	    struct erl_off_heap_header* hdr = (struct erl_off_heap_header*)hp;
 	    ASSERT(ptr + header_arity(val) < end);
-	    MOVE_BOXED(ptr, val, hp, &dummy_ref);	    
+	    MOVE_BOXED(ptr, val, hp, &dummy_ref);
 	    switch (val & _HEADER_SUBTAG_MASK) {
 	    case REFC_BINARY_SUBTAG:
 	    case FUN_SUBTAG:
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index a224383493..22a7d430c8 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -2155,26 +2155,18 @@ copy_one_frag(Eterm** hpp, ErlOffHeap* off_heap,
 	    *hp++ = val;
 	    break;
 	case TAG_PRIMARY_LIST:
-#ifdef SHCOPY_SEND
             if (erts_is_literal(val,list_val(val))) {
                 *hp++ = val;
             } else {
                 *hp++ = offset_ptr(val, offs);
             }
-#else
-            *hp++ = offset_ptr(val, offs);
-#endif
             break;
 	case TAG_PRIMARY_BOXED:
-#ifdef SHCOPY_SEND
             if (erts_is_literal(val,boxed_val(val))) {
                 *hp++ = val;
             } else {
                 *hp++ = offset_ptr(val, offs);
             }
-#else
-            *hp++ = offset_ptr(val, offs);
-#endif
 	    break;
 	case TAG_PRIMARY_HEADER:
 	    *hp++ = val;
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index 91e06cde2d..14ee07a304 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -696,6 +696,9 @@ erts_send_message(Process* sender,
     erts_aint32_t receiver_state;
 #ifdef SHCOPY_SEND
     erts_shcopy_t info;
+#else
+    Eterm *lit_purge_ptr = erts_clrange.ptr;
+    Uint lit_purge_sz = erts_clrange.sz;
 #endif
 
 #ifdef USE_VM_PROBES
@@ -725,7 +728,7 @@ erts_send_message(Process* sender,
          */
         if (have_seqtrace(stoken)) {
 	    seq_trace_update_send(sender);
-	    seq_trace_output(stoken, message, SEQ_TRACE_SEND, 
+	    seq_trace_output(stoken, message, SEQ_TRACE_SEND,
 			     receiver->common.id, sender);
 	    seq_trace_size = 6; /* TUPLE5 */
 	}
@@ -741,7 +744,7 @@ erts_send_message(Process* sender,
         INITIALIZE_SHCOPY(info);
         msize = copy_shared_calculate(message, &info);
 #else
-        msize = size_object(message);
+        msize = size_object_litopt(message, lit_purge_ptr, lit_purge_sz);
 #endif
         mp = erts_alloc_message_heap_state(receiver,
                                            &receiver_state,
@@ -760,7 +763,8 @@ erts_send_message(Process* sender,
         DESTROY_SHCOPY(info);
 #else
 	if (is_not_immed(message))
-            message = copy_struct(message, msize, &hp, ohp);
+            message = copy_struct_litopt(message, msize, &hp, ohp,
+                                         lit_purge_ptr, lit_purge_sz);
 #endif
 	if (is_immed(stoken))
 	    token = stoken;
@@ -796,7 +800,7 @@ erts_send_message(Process* sender,
             INITIALIZE_SHCOPY(info);
             msize = copy_shared_calculate(message, &info);
 #else
-            msize = size_object(message);
+            msize = size_object_litopt(message, lit_purge_ptr, lit_purge_sz);
 #endif
 	    mp = erts_alloc_message_heap_state(receiver,
 					       &receiver_state,
@@ -810,7 +814,8 @@ erts_send_message(Process* sender,
             DESTROY_SHCOPY(info);
 #else
             if (is_not_immed(message))
-                message = copy_struct(message, msize, &hp, ohp);
+                message = copy_struct_litopt(message, msize, &hp, ohp,
+                                             lit_purge_ptr, lit_purge_sz);
 #endif
 	}
 #ifdef USE_VM_PROBES
@@ -998,8 +1003,8 @@ erts_move_messages_off_heap(Process *c_p)
 	hp = hfrag->mem;
 	if (is_not_immed(ERL_MESSAGE_TERM(mp)))
 	    ERL_MESSAGE_TERM(mp) = copy_struct(ERL_MESSAGE_TERM(mp),
-					       msg_sz, &hp,
-					       &hfrag->off_heap);
+                                               msg_sz, &hp,
+                                               &hfrag->off_heap);
 	if (is_not_immed(ERL_MESSAGE_TOKEN(mp)))
 	    ERL_MESSAGE_TOKEN(mp) = copy_struct(ERL_MESSAGE_TOKEN(mp),
 						token_sz, &hp,
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index b5d8c5bc75..d6037e9f5d 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -11099,6 +11099,9 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 #ifdef SHCOPY_SPAWN
     erts_shcopy_t info;
     INITIALIZE_SHCOPY(info);
+#else
+    Eterm *lit_purge_ptr = erts_clrange.ptr;
+    Uint lit_purge_sz = erts_clrange.sz;
 #endif
 
     erts_smp_proc_lock(parent, ERTS_PROC_LOCKS_ALL_MINOR);
@@ -11157,7 +11160,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 #ifdef SHCOPY_SPAWN
     arg_size = copy_shared_calculate(args, &info);
 #else
-    arg_size = size_object(args);
+    arg_size = size_object_litopt(args, lit_purge_ptr, lit_purge_sz);
 #endif
     heap_need = arg_size;
 
@@ -11181,7 +11184,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     }
     p->schedule_count = 0;
     ASSERT(p->min_heap_size == erts_next_heap_size(p->min_heap_size, 0));
-    
+
     p->u.initial[INITIAL_MOD] = mod;
     p->u.initial[INITIAL_FUN] = func;
     p->u.initial[INITIAL_ARI] = (Uint) arity;
@@ -11239,7 +11242,8 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     p->arg_reg[2] = copy_shared_perform(args, arg_size, &info, &p->htop, &p->off_heap);
     DESTROY_SHCOPY(info);
 #else
-    p->arg_reg[2] = copy_struct(args, arg_size, &p->htop, &p->off_heap);
+    p->arg_reg[2] = copy_struct_litopt(args, arg_size, &p->htop, &p->off_heap,
+                                       lit_purge_ptr, lit_purge_sz);
 #endif
     p->arity = 3;
 
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index bf2e50e52f..67ce0d0723 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1085,8 +1085,8 @@ typedef struct {
     Eterm* shtable_start;
     ErtsAlcType_t shtable_alloc_type;
     Uint literal_size;
-    Eterm *range_ptr;
-    Uint  range_sz;
+    Eterm *lit_purge_ptr;
+    Uint lit_purge_sz;
 } erts_shcopy_t;
 
 #define INITIALIZE_SHCOPY(info)                         \
@@ -1095,8 +1095,8 @@ do {                                                    \
     info.bitstore_start = info.bitstore_default;        \
     info.shtable_start = info.shtable_default;          \
     info.literal_size = 0;                              \
-    info.range_ptr = erts_clrange.ptr;                  \
-    info.range_sz  = erts_clrange.sz;                   \
+    info.lit_purge_ptr = erts_clrange.ptr;              \
+    info.lit_purge_sz  = erts_clrange.sz;               \
 } while(0)
 
 #define DESTROY_SHCOPY(info)                                            \
@@ -1116,15 +1116,21 @@ do {                                                                    \
 Eterm copy_object_x(Eterm, Process*, Uint);
 #define copy_object(Term, Proc) copy_object_x(Term,Proc,0)
 
-Uint size_object(Eterm);
+Uint size_object_x(Eterm,Eterm*,Uint,Uint);
+#define size_object(Term) size_object_x(Term,NULL,0,0)
+#define size_object_litopt(Term,LitPtr,LitSz) size_object_x(Term,LitPtr,LitSz,1)
+
 Uint copy_shared_calculate(Eterm, erts_shcopy_t*);
 Eterm copy_shared_perform(Eterm, Uint, erts_shcopy_t*, Eterm**, ErlOffHeap*);
 
 Uint size_shared(Eterm);
 
-Eterm copy_struct_x(Eterm, Uint, Eterm**, ErlOffHeap*, Uint* bsz);
+Eterm copy_struct_x(Eterm, Uint, Eterm**, ErlOffHeap*, Uint* bsz, Eterm *lit_ptr, Uint lit_sz, Uint litopt);
 #define copy_struct(Obj,Sz,HPP,OH) \
-    copy_struct_x(Obj,Sz,HPP,OH,NULL)
+    copy_struct_x(Obj,Sz,HPP,OH,NULL,NULL,0,0)
+#define copy_struct_litopt(Obj,Sz,HPP,OH,LitPtr,LitSz) \
+    copy_struct_x(Obj,Sz,HPP,OH,NULL,LitPtr,LitSz,1)
+
 Eterm copy_shallow(Eterm*, Uint, Eterm**, ErlOffHeap*);
 
 void erts_move_multi_frags(Eterm** hpp, ErlOffHeap*, ErlHeapFragment* first,