Introduce literal tag

12 files changed, 206 insertions, 39 deletions

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 6e4c8ecee3..4d19f52a52 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -2963,6 +2963,9 @@ do {						\
 		     }
 		 }
 		 Op1 = small_to_big(ires, tmp_big);
+#ifdef TAG_LITERAL_PTR
+		 Op1 |= TAG_LITERAL_PTR;
+#endif
 
 	     big_shift:
 		 if (i > 0) {	/* Left shift. */
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 1c598601c6..7a1a563be2 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -4439,12 +4439,13 @@ freeze_code(LoaderState* stp)
 	code_hdr->literals_start = ptr;
 	code_hdr->literals_end = ptr + stp->total_literal_size;
 	for (i = 0; i < stp->num_literals; i++) {
-            if (stp->literals[i].heap_frags) {
-                move_multi_frags(&ptr, &code_off_heap, stp->literals[i].heap_frags,
-                                 &stp->literals[i].term, 1);
-                ASSERT(erts_is_literal(ptr_val(stp->literals[i].term)));
+            if (is_not_immed(stp->literals[i].term)) {
+                erts_move_multi_frags(&ptr, &code_off_heap,
+				      stp->literals[i].heap_frags,
+				      &stp->literals[i].term, 1, 1);
+                ASSERT(erts_is_literal(stp->literals[i].term,
+				       ptr_val(stp->literals[i].term)));
             }
-            else ASSERT(is_immed(stp->literals[i].term));
 	}
 	code_hdr->literals_off_heap = code_off_heap.first;
 	lp = stp->literal_patches;
diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index ec769c3b49..b185758b1d 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -35,7 +35,7 @@
 #include "erl_bits.h"
 #include "dtrace-wrapper.h"
 
-static void move_one_frag(Eterm** hpp, ErlHeapFragment*, ErlOffHeap*);
+static void move_one_frag(Eterm** hpp, ErlHeapFragment*, ErlOffHeap*, int);
 
 /*
  *  Copy object "obj" to process p.
@@ -621,17 +621,24 @@ Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
  * move markers.
  * Typically used to copy a multi-fragmented message (from NIF).
  */
-void move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first,
-		      Eterm* refs, unsigned nrefs)
+void erts_move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first,
+			   Eterm* refs, unsigned nrefs, int literals)
 {
     ErlHeapFragment* bp;
     Eterm* hp_start = *hpp;
     Eterm* hp_end;
     Eterm* hp;
     unsigned i;
+    Eterm literal_tag;
+
+#ifdef TAG_LITERAL_PTR
+    literal_tag = (Eterm) literals ? TAG_LITERAL_PTR : 0;
+#else
+    literal_tag = (Eterm) 0;
+#endif
 
     for (bp=first; bp!=NULL; bp=bp->next) {
-	move_one_frag(hpp, bp, off_heap);
+	move_one_frag(hpp, bp, off_heap, literals);
     }
     hp_end = *hpp;
     for (hp=hp_start; hp<hp_end; ++hp) {
@@ -644,6 +651,9 @@ void move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first,
 	    val = *ptr;
 	    if (IS_MOVED_BOXED(val)) {
 		ASSERT(is_boxed(val));
+#ifdef TAG_LITERAL_PTR
+		val |= literal_tag;
+#endif
 		*hp = val;
 	    }
 	    break;
@@ -651,7 +661,11 @@ void move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first,
 	    ptr = list_val(gval);
 	    val = *ptr;
 	    if (IS_MOVED_CONS(val)) {
-		*hp = ptr[1];
+		val = ptr[1];
+#ifdef TAG_LITERAL_PTR
+		val |= literal_tag;
+#endif
+		*hp = val;
 	    }
 	    break;
 	case TAG_PRIMARY_HEADER:
@@ -662,12 +676,12 @@ void move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first,
 	}
     }
     for (i=0; i<nrefs; ++i) {
-	refs[i] = follow_moved(refs[i]);
+	refs[i] = follow_moved(refs[i], literal_tag);
     }
 }
 
 static void
-move_one_frag(Eterm** hpp, ErlHeapFragment* frag, ErlOffHeap* off_heap)
+move_one_frag(Eterm** hpp, ErlHeapFragment* frag, ErlOffHeap* off_heap, int literals)
 {
     Eterm* ptr = frag->mem;
     Eterm* end = ptr + frag->used_size;
@@ -704,4 +718,3 @@ move_one_frag(Eterm** hpp, ErlHeapFragment* frag, ErlOffHeap* off_heap)
     OH_OVERHEAD(off_heap, frag->off_heap.overhead);
     frag->off_heap.first = NULL;
 }
-
diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h
index a0d8561e84..9da9c823f7 100644
--- a/erts/emulator/beam/erl_alloc.h
+++ b/erts/emulator/beam/erl_alloc.h
@@ -178,6 +178,12 @@ void  sys_free(void *)               __deprecated; /* erts_free()        */
 void *sys_alloc(Uint )               __deprecated; /* erts_alloc_fnf()   */
 void *sys_realloc(void *, Uint)      __deprecated; /* erts_realloc_fnf() */
 
+#undef ERTS_HAVE_IS_IN_LITERAL_RANGE
+#if defined(ARCH_32) || defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
+#  define ERTS_HAVE_IS_IN_LITERAL_RANGE
+#endif
+
+
 /*
  * erts_alloc[_fnf](), erts_realloc[_fnf](), erts_free() works as
  * malloc(), realloc(), and free() with the following exceptions:
@@ -205,7 +211,9 @@ void erts_free(ErtsAlcType_t type, void *ptr);
 void *erts_alloc_fnf(ErtsAlcType_t type, Uint size);
 void *erts_realloc_fnf(ErtsAlcType_t type, void *ptr, Uint size);
 int erts_is_allctr_wrapper_prelocked(void);
-int erts_is_literal(void* ptr);
+#ifdef ERTS_HAVE_IS_IN_LITERAL_RANGE
+int erts_is_in_literal_range(void* ptr);
+#endif
 
 #endif /* #if !ERTS_ALC_DO_INLINE */
 
@@ -283,8 +291,10 @@ int erts_is_allctr_wrapper_prelocked(void)
 	&& !!erts_tsd_get(erts_allctr_prelock_tsd_key);  /* by me  */
 }
 
+#ifdef ERTS_HAVE_IS_IN_LITERAL_RANGE
+
 ERTS_ALC_INLINE
-int erts_is_literal(void* ptr)
+int erts_is_in_literal_range(void* ptr)
 {
 #if defined(ARCH_32)
     Uint ix = (UWord)ptr >> ERTS_MMAP_SUPERALIGNED_BITS;
@@ -293,18 +303,16 @@ int erts_is_literal(void* ptr)
                   & ((UWord)1 << (ix % ERTS_VSPACE_WORD_BITS));
 
 #elif defined(ARCH_64)
-# if defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
     extern char* erts_literals_start;
     extern UWord erts_literals_size;
     return ErtsInArea(ptr, erts_literals_start, erts_literals_size);
-# else
-#  error Do the tag thing
-# endif
 #else
 # error No ARCH_xx
 #endif
 }
 
+#endif /* ERTS_HAVE_IS_IN_LITERAL_RANGE */
+
 #endif /* #if ERTS_ALC_DO_INLINE || defined(ERTS_ALC_INTERNAL__) */
 
 #define ERTS_ALC_GET_THR_IX() ((int) erts_get_scheduler_id())
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index a73ad826db..a684c81445 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -2766,6 +2766,20 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
     else if (ERTS_IS_ATOM_STR("eager_check_io",BIF_ARG_1)) {
 	BIF_RET(erts_eager_check_io ? am_true : am_false);
     }
+    else if (ERTS_IS_ATOM_STR("literal_test",BIF_ARG_1)) {
+#ifdef ERTS_HAVE_IS_IN_LITERAL_RANGE
+#ifdef ARCH_64
+	DECL_AM(range);
+	BIF_RET(AM_range);
+#else /* ARCH_32 */
+	DECL_AM(range_bitmask);
+	BIF_RET(AM_range_bitmask);
+#endif /* ARCH_32 */
+#else  /* ! ERTS_HAVE_IS_IN_LITERAL_RANGE */
+	DECL_AM(tag);
+	BIF_RET(AM_tag);
+#endif
+    }
 
     BIF_ERROR(BIF_P, BADARG);
 }
@@ -4323,12 +4337,15 @@ static void os_info_init(void)
     erts_free(ERTS_ALC_T_TMP, (void *) buf);
     hp = erts_alloc(ERTS_ALC_T_LITERAL, (3+4)*sizeof(Eterm));
     os_type_tuple = TUPLE2(hp, type, flav);
+    erts_set_literal_tag(&os_type_tuple, hp, 3);
+
     hp += 3;
     os_version(&major, &minor, &build);
     os_version_tuple = TUPLE3(hp,
 			      make_small(major),
 			      make_small(minor),
 			      make_small(build));
+    erts_set_literal_tag(&os_version_tuple, hp, 4);
 }
 
 void
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 89fabde67c..e316ab95ab 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -1004,6 +1004,8 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
     Uint mature_size = (char *) HIGH_WATER(p) - heap;
     Eterm* old_htop = OLD_HTOP(p);
     Eterm* n_heap;
+    char* oh = (char *) OLD_HEAP(p);
+    Uint oh_size = (char *) OLD_HTOP(p) - oh;
 
     n_htop = n_heap = (Eterm*) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP,
 					       sizeof(Eterm)*new_sz);
@@ -1035,8 +1037,12 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
                 } else if (in_area(ptr, heap, mature_size)) {
                     MOVE_BOXED(ptr,val,old_htop,g_ptr++);
                 } else if (in_area(ptr, heap, heap_size)) {
+		    ASSERT(!erts_is_literal(gval, ptr)
+			   && !in_area(ptr, oh, oh_size));
                     MOVE_BOXED(ptr,val,n_htop,g_ptr++);
                 } else {
+		    ASSERT(erts_is_literal(gval, ptr)
+			   || in_area(ptr, oh, oh_size));
 		    g_ptr++;
 		}
                 break;
@@ -1050,8 +1056,12 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
                 } else if (in_area(ptr, heap, mature_size)) {
                     MOVE_CONS(ptr,val,old_htop,g_ptr++);
                 } else if (in_area(ptr, heap, heap_size)) {
+		    ASSERT(!erts_is_literal(gval, ptr)
+			   && !in_area(ptr, oh, oh_size));
                     MOVE_CONS(ptr,val,n_htop,g_ptr++);
                 } else {
+		    ASSERT(erts_is_literal(gval, ptr)
+			   || in_area(ptr, oh, oh_size));
 		    g_ptr++;
 		}
 		break;
@@ -1094,8 +1104,12 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
 		} else if (in_area(ptr, heap, mature_size)) {
 		    MOVE_BOXED(ptr,val,old_htop,n_hp++);
 		} else if (in_area(ptr, heap, heap_size)) {
+		    ASSERT(!erts_is_literal(gval, ptr)
+			   && !in_area(ptr, oh, oh_size));
 		    MOVE_BOXED(ptr,val,n_htop,n_hp++);
 		} else {
+		    ASSERT(erts_is_literal(gval, ptr)
+			   || in_area(ptr, oh, oh_size));
 		    n_hp++;
 		}
 		break;
@@ -1108,8 +1122,12 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
 		} else if (in_area(ptr, heap, mature_size)) {
 		    MOVE_CONS(ptr,val,old_htop,n_hp++);
 		} else if (in_area(ptr, heap, heap_size)) {
+		    ASSERT(!erts_is_literal(gval, ptr)
+			   && !in_area(ptr, oh, oh_size));
 		    MOVE_CONS(ptr,val,n_htop,n_hp++);
 		} else {
+		    ASSERT(erts_is_literal(gval, ptr)
+			   || in_area(ptr, oh, oh_size));
 		    n_hp++;
 		}
 		break;
@@ -1131,9 +1149,15 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
 			    MOVE_BOXED(ptr,val,old_htop,origptr);
 			    mb->base = binary_bytes(mb->orig);
 			} else if (in_area(ptr, heap, heap_size)) {
+			    ASSERT(!erts_is_literal(*origptr, origptr)
+				   && !in_area(ptr, oh, oh_size));
 			    MOVE_BOXED(ptr,val,n_htop,origptr);
 			    mb->base = binary_bytes(mb->orig);
 			}
+			else {
+			    ASSERT(erts_is_literal(*origptr, origptr)
+				   || in_area(ptr, oh, oh_size));
+			}
 		    }
 		    n_hp += (thing_arityval(gval)+1);
 		}
@@ -1275,8 +1299,10 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 		    ASSERT(is_boxed(val));
 		    *g_ptr++ = val;
 		} else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) {
+		    ASSERT(!erts_is_literal(gval, ptr));
 		    MOVE_BOXED(ptr,val,n_htop,g_ptr++);
 		} else {
+		    ASSERT(erts_is_literal(gval, ptr));
 		    g_ptr++;
 		}
 		continue;
@@ -1288,8 +1314,10 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 		if (IS_MOVED_CONS(val)) {
 		    *g_ptr++ = ptr[1];
 		} else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) {
+		    ASSERT(!erts_is_literal(gval, ptr));
 		    MOVE_CONS(ptr,val,n_htop,g_ptr++);
 		} else {
+		    ASSERT(erts_is_literal(gval, ptr));
 		    g_ptr++;
 		}
 		continue;
@@ -1330,8 +1358,10 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 		    ASSERT(is_boxed(val));
 		    *n_hp++ = val;
 		} else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) {
+		    ASSERT(!erts_is_literal(gval, ptr));
 		    MOVE_BOXED(ptr,val,n_htop,n_hp++);
 		} else {
+		    ASSERT(erts_is_literal(gval, ptr));
 		    n_hp++;
 		}
 		break;
@@ -1342,8 +1372,10 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 		if (IS_MOVED_CONS(val)) {
 		    *n_hp++ = ptr[1];
 		} else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) {
+		    ASSERT(!erts_is_literal(gval, ptr));
 		    MOVE_CONS(ptr,val,n_htop,n_hp++);
 		} else {
+		    ASSERT(erts_is_literal(gval, ptr));
 		    n_hp++;
 		}
 		break;
@@ -1363,12 +1395,16 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 			    *origptr = val;
 			    mb->base = binary_bytes(*origptr);
 			} else if (in_area(ptr, src, src_size) ||
-				   in_area(ptr, oh, oh_size)) {
+-				   in_area(ptr, oh, oh_size)) {
+			    ASSERT(!erts_is_literal(*origptr, origptr));
 			    MOVE_BOXED(ptr,val,n_htop,origptr); 
 			    mb->base = binary_bytes(*origptr);
 			    ptr = boxed_val(*origptr);
 			    val = *ptr;
 			}
+			else {
+			    ASSERT(erts_is_literal(*origptr, origptr));
+			}
 		    }
 		    n_hp += (thing_arityval(gval)+1);
 		}
@@ -1985,7 +2021,7 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset)
     }
 
     ASSERT((is_nil(p->seq_trace_token) ||
-	    is_tuple(follow_moved(p->seq_trace_token)) ||
+	    is_tuple(follow_moved(p->seq_trace_token, (Eterm) 0)) ||
 	    is_atom(p->seq_trace_token)));
     if (is_not_immed(p->seq_trace_token)) {
 	roots[n].v = &p->seq_trace_token;
@@ -2003,7 +2039,7 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset)
 	   is_internal_pid(ERTS_TRACER_PROC(p)) ||
 	   is_internal_port(ERTS_TRACER_PROC(p)));
 
-    ASSERT(is_pid(follow_moved(p->group_leader)));
+    ASSERT(is_pid(follow_moved(p->group_leader, (Eterm) 0)));
     if (is_not_immed(p->group_leader)) {
 	roots[n].v  = &p->group_leader;
 	roots[n].sz = 1;
diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h
index ecd1bf4d22..fdbf948f9d 100644
--- a/erts/emulator/beam/erl_gc.h
+++ b/erts/emulator/beam/erl_gc.h
@@ -76,10 +76,10 @@ do {                                                                    \
 int within(Eterm *ptr, Process *p);
 #endif
 
-ERTS_GLB_INLINE Eterm follow_moved(Eterm term);
+ERTS_GLB_INLINE Eterm follow_moved(Eterm term, Eterm xptr_tag);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
-ERTS_GLB_INLINE Eterm follow_moved(Eterm term)
+ERTS_GLB_INLINE Eterm follow_moved(Eterm term, Eterm xptr_tag)
 {
     Eterm* ptr;
     switch (primary_tag(term)) {
@@ -87,11 +87,11 @@ ERTS_GLB_INLINE Eterm follow_moved(Eterm term)
 	break;
     case TAG_PRIMARY_BOXED:
 	ptr = boxed_val(term);
-	if (IS_MOVED_BOXED(*ptr)) term = *ptr;
+	if (IS_MOVED_BOXED(*ptr)) term = (*ptr) | xptr_tag;
 	break;
     case TAG_PRIMARY_LIST:
 	ptr = list_val(term);
-	if (IS_MOVED_CONS(ptr[0])) term = ptr[1];
+	if (IS_MOVED_CONS(ptr[0])) term = (ptr[1]) | xptr_tag;
 	break;
     default:
 	ASSERT(!"strange tag in follow_moved");
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index ef52823287..e23c79d301 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -642,13 +642,13 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg)
 #endif
 
     if (bp->next != NULL) {
-	move_multi_frags(hpp, off_heap, bp, msg->m, 
+	erts_move_multi_frags(hpp, off_heap, bp, msg->m, 
 #ifdef USE_VM_PROBES
-			 3
+			      3,
 #else
-			 2
+			      2,
 #endif
-			 );
+			      0);
 	goto copy_done;
     }
 
diff --git a/erts/emulator/beam/erl_term.c b/erts/emulator/beam/erl_term.c
index 3a5fbcc284..8efbb8c554 100644
--- a/erts/emulator/beam/erl_term.c
+++ b/erts/emulator/beam/erl_term.c
@@ -28,6 +28,37 @@
 #include <stdlib.h>
 #include <stdio.h>
 
+void
+erts_set_literal_tag(Eterm *term, Eterm *hp_start, Eterm hsz)
+{
+#ifdef TAG_LITERAL_PTR
+    Eterm *hp_end, *hp;
+    
+    hp_end = hp_start + hsz;
+    hp = hp_start;
+
+    while (hp < hp_end) {
+	switch (primary_tag(*hp)) {
+	case TAG_PRIMARY_BOXED:
+	case TAG_PRIMARY_LIST:
+	    *hp |= TAG_LITERAL_PTR;
+	    break;
+	case TAG_PRIMARY_HEADER:
+	    if (header_is_thing(*hp)) {
+		hp += thing_arityval(*hp);
+	    }
+	    break;
+	default:
+	    break;
+	}
+	
+	hp++;
+    }
+    if (is_boxed(*term) || is_list(*term))
+	*term |= TAG_LITERAL_PTR;
+#endif
+}
+
 __decl_noreturn static void __noreturn
 et_abort(const char *expr, const char *file, unsigned line)
 {
diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h
index 089eecf024..a80716fb0f 100644
--- a/erts/emulator/beam/erl_term.h
+++ b/erts/emulator/beam/erl_term.h
@@ -21,6 +21,8 @@
 #ifndef __ERL_TERM_H
 #define __ERL_TERM_H
 
+#include "erl_mmap.h"
+
 typedef UWord Wterm;  /* Full word terms */
 
 struct erl_node_; /* Declared in erl_node_tables.h */
@@ -48,6 +50,24 @@ struct erl_node_; /* Declared in erl_node_tables.h */
 #define _ET_APPLY(F,X)	_unchecked_##F(X)
 #endif
 
+#if defined(ARCH_64)
+#  define TAG_PTR_MASK__	0x7
+#  if !defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
+#    ifdef HIPE
+#      error Hipe on 64-bit needs a real mmap as it does not support the literal tag
+#    endif
+#    define TAG_LITERAL_PTR	0x4
+#  else
+#    undef TAG_LITERAL_PTR
+#  endif
+#elif defined(ARCH_32)
+#  define TAG_PTR_MASK__	0x3
+#  undef TAG_LITERAL_PTR
+#else
+#  error Not supported arch
+#endif
+
+
 #define _TAG_PRIMARY_SIZE	2
 #define _TAG_PRIMARY_MASK	0x3
 #define TAG_PRIMARY_HEADER	0x0
@@ -165,10 +185,11 @@ struct erl_node_; /* Declared in erl_node_tables.h */
 
 /* boxed object access methods */
 
-#define _is_taggable_pointer(x)	 (((Uint)(x) & 0x3) == 0)
+#define _is_taggable_pointer(x)	 (((Uint)(x) & TAG_PTR_MASK__) == 0)
+
 #define  _boxed_precond(x)       (is_boxed(x))
 
-#define _is_aligned(x)		(((Uint)(x) & 0x3) == 0)
+#define _is_aligned(x)		(((Uint)(x) & TAG_PTR_MASK__) == 0)
 #define _unchecked_make_boxed(x) ((Uint)(x) + TAG_PRIMARY_BOXED)
 _ET_DECLARE_CHECKED(Eterm,make_boxed,const Eterm*)
 #define make_boxed(x)		_ET_APPLY(make_boxed,(x))
@@ -180,7 +201,11 @@ _ET_DECLARE_CHECKED(int,is_boxed,Eterm)
 #else
 #define is_boxed(x)		(((x) & _TAG_PRIMARY_MASK) == TAG_PRIMARY_BOXED)
 #endif
+#ifdef TAG_LITERAL_PTR
+#define _unchecked_boxed_val(x) _unchecked_ptr_val(x)
+#else
 #define _unchecked_boxed_val(x) ((Eterm*) ((x) - TAG_PRIMARY_BOXED))
+#endif
 _ET_DECLARE_CHECKED(Eterm*,boxed_val,Wterm)
 #define boxed_val(x)		_ET_APPLY(boxed_val,(x))
 
@@ -198,7 +223,11 @@ _ET_DECLARE_CHECKED(int,is_not_list,Eterm)
 #define is_not_list(x)		(!is_list((x)))
 #endif
 #define _list_precond(x)        (is_list(x))
+#ifdef TAG_LITERAL_PTR
+#define _unchecked_list_val(x) _unchecked_ptr_val(x)
+#else
 #define _unchecked_list_val(x) ((Eterm*) ((x) - TAG_PRIMARY_LIST))
+#endif
 _ET_DECLARE_CHECKED(Eterm*,list_val,Wterm)
 #define list_val(x)		_ET_APPLY(list_val,(x))
 
@@ -209,13 +238,20 @@ _ET_DECLARE_CHECKED(Eterm*,list_val,Wterm)
 #define CDR(x)  ((x)[1])
 
 /* generic tagged pointer (boxed or list) access methods */
-#define _unchecked_ptr_val(x)	((Eterm*) ((x) & ~((Uint) 0x3)))
+#define _unchecked_ptr_val(x)	((Eterm*) ((x) & ~((Uint) TAG_PTR_MASK__)))
 #define ptr_val(x)		_unchecked_ptr_val((x))	/*XXX*/
 #define _unchecked_offset_ptr(x,offs)	((x)+((offs)*sizeof(Eterm)))
 #define offset_ptr(x,offs)	_unchecked_offset_ptr(x,offs)	/*XXX*/
 #define _unchecked_byte_offset_ptr(x,byte_offs)	((x)+(offs))
 #define byte_offset_ptr(x,offs) _unchecked_byte_offset_ptr(x,offs)  /*XXX*/
 
+#ifdef TAG_LITERAL_PTR
+#define _unchecked_is_not_literal_ptr(x) (!((x) & TAG_LITERAL_PTR))
+#define is_not_literal_ptr(x) _unchecked_is_not_literal_ptr((x)) /*XXX*/
+#define is_literal_ptr(x) (!is_not_literal_ptr((x))) /*XXX*/
+#endif
+
+
 /* fixnum ("small") access methods */
 #if defined(ARCH_64)
 #define SMALL_BITS	(64-4)
@@ -1113,5 +1149,7 @@ extern unsigned tag_val_def(Wterm);
 
 #define is_same(A,B) ((A)==(B))
 
+void erts_set_literal_tag(Eterm *term, Eterm *hp_start, Eterm hsz);
+
 #endif	/* __ERL_TERM_H */
 
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index b4d02dd1dd..052994b972 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -961,8 +961,8 @@ Uint size_object(Eterm);
 Eterm copy_struct(Eterm, Uint, Eterm**, ErlOffHeap*);
 Eterm copy_shallow(Eterm*, Uint, Eterm**, ErlOffHeap*);
 
-void move_multi_frags(Eterm** hpp, ErlOffHeap*, ErlHeapFragment* first,
-		      Eterm* refs, unsigned nrefs);
+void erts_move_multi_frags(Eterm** hpp, ErlOffHeap*, ErlHeapFragment* first,
+			   Eterm* refs, unsigned nrefs, int literals);
 
 /* Utilities */
 extern void erts_delete_nodes_monitors(Process *, ErtsProcLocks);
@@ -1274,6 +1274,27 @@ int erts_print_system_version(int to, void *arg, Process *c_p);
 
 int erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg);
 
+ERTS_GLB_INLINE int erts_is_literal(Eterm tptr, Eterm *ptr);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE int erts_is_literal(Eterm tptr, Eterm *ptr)
+{
+    ASSERT(is_boxed(tptr) || is_list(tptr));
+    ASSERT(ptr == ptr_val(tptr));
+
+#if defined(ERTS_HAVE_IS_IN_LITERAL_RANGE)
+    return erts_is_in_literal_range(ptr);
+#elif defined(TAG_LITERAL_PTR)
+    return is_literal_ptr(tptr);
+#else
+#  error Not able to detect literals...
+#endif
+
+}
+
+#endif
+
 /*
 ** Call_trace uses this API for the parameter matching functions
 */
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 46577bd07f..d63e81cb5e 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -72,6 +72,9 @@
 
 #define ERTS_I64_LITERAL(X) X##LL
 
+#define ErtsInArea(ptr,start,nbytes) \
+    ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes))
+
 #if defined (__WIN32__)
 #  include "erl_win_sys.h"
 #else
@@ -1044,10 +1047,6 @@ extern int erts_use_kernel_poll;
 
 #define put_int8(i, s) do {((unsigned char*)(s))[0] = (i) & 0xff;} while (0)
 
-
-#define ErtsInArea(PTR,START,NBYTES) \
-    ((UWord)((char*)(PTR) - (char*)(START)) < (NBYTES))
-
 /*
  * Use DEBUGF as you would use printf, but use double parentheses:
  *