From b21b604137c5cb5f5039a40994e429871e5b707b Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Wed, 26 Aug 2015 19:47:10 +0200 Subject: Introduce literal tag --- erts/configure.in | 2 +- erts/emulator/beam/beam_emu.c | 3 +++ erts/emulator/beam/beam_load.c | 11 +++++----- erts/emulator/beam/copy.c | 29 +++++++++++++++++++------- erts/emulator/beam/erl_alloc.h | 20 ++++++++++++------ erts/emulator/beam/erl_bif_info.c | 17 +++++++++++++++ erts/emulator/beam/erl_gc.c | 42 ++++++++++++++++++++++++++++++++++--- erts/emulator/beam/erl_gc.h | 8 +++---- erts/emulator/beam/erl_message.c | 8 +++---- erts/emulator/beam/erl_term.c | 31 +++++++++++++++++++++++++++ erts/emulator/beam/erl_term.h | 44 ++++++++++++++++++++++++++++++++++++--- erts/emulator/beam/global.h | 25 ++++++++++++++++++++-- erts/emulator/beam/sys.h | 7 +++---- erts/emulator/hipe/hipe_bif0.c | 2 ++ 14 files changed, 209 insertions(+), 40 deletions(-) diff --git a/erts/configure.in b/erts/configure.in index 25256bcac9..a93d1af0bf 100644 --- a/erts/configure.in +++ b/erts/configure.in @@ -3702,7 +3702,7 @@ dnl crypto # #-------------------------------------------------------------------- -DED_SYS_INCLUDE="-I${ERL_TOP}/erts/emulator/beam -I${ERL_TOP}/erts/include -I${ERL_TOP}/erts/include/$host -I${ERL_TOP}/erts/include/internal -I${ERL_TOP}/erts/include/internal/$host -I${ERL_TOP}/erts/emulator/sys/$ERLANG_OSTYPE" +DED_SYS_INCLUDE="-I${ERL_TOP}/erts/emulator/beam -I${ERL_TOP}/erts/include -I${ERL_TOP}/erts/include/$host -I${ERL_TOP}/erts/include/internal -I${ERL_TOP}/erts/include/internal/$host -I${ERL_TOP}/erts/emulator/sys/$ERLANG_OSTYPE -I${ERL_TOP}/erts/emulator/sys/common" if test "X$ETHR_DEFS" = "X"; then DED_THR_DEFS="-D_THREAD_SAFE -D_REENTRANT" diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 6e4c8ecee3..4d19f52a52 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -2963,6 +2963,9 @@ do { \ } } Op1 = small_to_big(ires, tmp_big); +#ifdef TAG_LITERAL_PTR + Op1 |= TAG_LITERAL_PTR; +#endif big_shift: if (i > 0) { /* Left shift. */ diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 1c598601c6..7a1a563be2 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -4439,12 +4439,13 @@ freeze_code(LoaderState* stp) code_hdr->literals_start = ptr; code_hdr->literals_end = ptr + stp->total_literal_size; for (i = 0; i < stp->num_literals; i++) { - if (stp->literals[i].heap_frags) { - move_multi_frags(&ptr, &code_off_heap, stp->literals[i].heap_frags, - &stp->literals[i].term, 1); - ASSERT(erts_is_literal(ptr_val(stp->literals[i].term))); + if (is_not_immed(stp->literals[i].term)) { + erts_move_multi_frags(&ptr, &code_off_heap, + stp->literals[i].heap_frags, + &stp->literals[i].term, 1, 1); + ASSERT(erts_is_literal(stp->literals[i].term, + ptr_val(stp->literals[i].term))); } - else ASSERT(is_immed(stp->literals[i].term)); } code_hdr->literals_off_heap = code_off_heap.first; lp = stp->literal_patches; diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index ec769c3b49..b185758b1d 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -35,7 +35,7 @@ #include "erl_bits.h" #include "dtrace-wrapper.h" -static void move_one_frag(Eterm** hpp, ErlHeapFragment*, ErlOffHeap*); +static void move_one_frag(Eterm** hpp, ErlHeapFragment*, ErlOffHeap*, int); /* * Copy object "obj" to process p. @@ -621,17 +621,24 @@ Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) * move markers. * Typically used to copy a multi-fragmented message (from NIF). */ -void move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first, - Eterm* refs, unsigned nrefs) +void erts_move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first, + Eterm* refs, unsigned nrefs, int literals) { ErlHeapFragment* bp; Eterm* hp_start = *hpp; Eterm* hp_end; Eterm* hp; unsigned i; + Eterm literal_tag; + +#ifdef TAG_LITERAL_PTR + literal_tag = (Eterm) literals ? TAG_LITERAL_PTR : 0; +#else + literal_tag = (Eterm) 0; +#endif for (bp=first; bp!=NULL; bp=bp->next) { - move_one_frag(hpp, bp, off_heap); + move_one_frag(hpp, bp, off_heap, literals); } hp_end = *hpp; for (hp=hp_start; hpmem; Eterm* end = ptr + frag->used_size; @@ -704,4 +718,3 @@ move_one_frag(Eterm** hpp, ErlHeapFragment* frag, ErlOffHeap* off_heap) OH_OVERHEAD(off_heap, frag->off_heap.overhead); frag->off_heap.first = NULL; } - diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h index a0d8561e84..9da9c823f7 100644 --- a/erts/emulator/beam/erl_alloc.h +++ b/erts/emulator/beam/erl_alloc.h @@ -178,6 +178,12 @@ void sys_free(void *) __deprecated; /* erts_free() */ void *sys_alloc(Uint ) __deprecated; /* erts_alloc_fnf() */ void *sys_realloc(void *, Uint) __deprecated; /* erts_realloc_fnf() */ +#undef ERTS_HAVE_IS_IN_LITERAL_RANGE +#if defined(ARCH_32) || defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) +# define ERTS_HAVE_IS_IN_LITERAL_RANGE +#endif + + /* * erts_alloc[_fnf](), erts_realloc[_fnf](), erts_free() works as * malloc(), realloc(), and free() with the following exceptions: @@ -205,7 +211,9 @@ void erts_free(ErtsAlcType_t type, void *ptr); void *erts_alloc_fnf(ErtsAlcType_t type, Uint size); void *erts_realloc_fnf(ErtsAlcType_t type, void *ptr, Uint size); int erts_is_allctr_wrapper_prelocked(void); -int erts_is_literal(void* ptr); +#ifdef ERTS_HAVE_IS_IN_LITERAL_RANGE +int erts_is_in_literal_range(void* ptr); +#endif #endif /* #if !ERTS_ALC_DO_INLINE */ @@ -283,8 +291,10 @@ int erts_is_allctr_wrapper_prelocked(void) && !!erts_tsd_get(erts_allctr_prelock_tsd_key); /* by me */ } +#ifdef ERTS_HAVE_IS_IN_LITERAL_RANGE + ERTS_ALC_INLINE -int erts_is_literal(void* ptr) +int erts_is_in_literal_range(void* ptr) { #if defined(ARCH_32) Uint ix = (UWord)ptr >> ERTS_MMAP_SUPERALIGNED_BITS; @@ -293,18 +303,16 @@ int erts_is_literal(void* ptr) & ((UWord)1 << (ix % ERTS_VSPACE_WORD_BITS)); #elif defined(ARCH_64) -# if defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) extern char* erts_literals_start; extern UWord erts_literals_size; return ErtsInArea(ptr, erts_literals_start, erts_literals_size); -# else -# error Do the tag thing -# endif #else # error No ARCH_xx #endif } +#endif /* ERTS_HAVE_IS_IN_LITERAL_RANGE */ + #endif /* #if ERTS_ALC_DO_INLINE || defined(ERTS_ALC_INTERNAL__) */ #define ERTS_ALC_GET_THR_IX() ((int) erts_get_scheduler_id()) diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index a73ad826db..a684c81445 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -2766,6 +2766,20 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) else if (ERTS_IS_ATOM_STR("eager_check_io",BIF_ARG_1)) { BIF_RET(erts_eager_check_io ? am_true : am_false); } + else if (ERTS_IS_ATOM_STR("literal_test",BIF_ARG_1)) { +#ifdef ERTS_HAVE_IS_IN_LITERAL_RANGE +#ifdef ARCH_64 + DECL_AM(range); + BIF_RET(AM_range); +#else /* ARCH_32 */ + DECL_AM(range_bitmask); + BIF_RET(AM_range_bitmask); +#endif /* ARCH_32 */ +#else /* ! ERTS_HAVE_IS_IN_LITERAL_RANGE */ + DECL_AM(tag); + BIF_RET(AM_tag); +#endif + } BIF_ERROR(BIF_P, BADARG); } @@ -4323,12 +4337,15 @@ static void os_info_init(void) erts_free(ERTS_ALC_T_TMP, (void *) buf); hp = erts_alloc(ERTS_ALC_T_LITERAL, (3+4)*sizeof(Eterm)); os_type_tuple = TUPLE2(hp, type, flav); + erts_set_literal_tag(&os_type_tuple, hp, 3); + hp += 3; os_version(&major, &minor, &build); os_version_tuple = TUPLE3(hp, make_small(major), make_small(minor), make_small(build)); + erts_set_literal_tag(&os_version_tuple, hp, 4); } void diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 89fabde67c..e316ab95ab 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -1004,6 +1004,8 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj) Uint mature_size = (char *) HIGH_WATER(p) - heap; Eterm* old_htop = OLD_HTOP(p); Eterm* n_heap; + char* oh = (char *) OLD_HEAP(p); + Uint oh_size = (char *) OLD_HTOP(p) - oh; n_htop = n_heap = (Eterm*) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP, sizeof(Eterm)*new_sz); @@ -1035,8 +1037,12 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj) } else if (in_area(ptr, heap, mature_size)) { MOVE_BOXED(ptr,val,old_htop,g_ptr++); } else if (in_area(ptr, heap, heap_size)) { + ASSERT(!erts_is_literal(gval, ptr) + && !in_area(ptr, oh, oh_size)); MOVE_BOXED(ptr,val,n_htop,g_ptr++); } else { + ASSERT(erts_is_literal(gval, ptr) + || in_area(ptr, oh, oh_size)); g_ptr++; } break; @@ -1050,8 +1056,12 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj) } else if (in_area(ptr, heap, mature_size)) { MOVE_CONS(ptr,val,old_htop,g_ptr++); } else if (in_area(ptr, heap, heap_size)) { + ASSERT(!erts_is_literal(gval, ptr) + && !in_area(ptr, oh, oh_size)); MOVE_CONS(ptr,val,n_htop,g_ptr++); } else { + ASSERT(erts_is_literal(gval, ptr) + || in_area(ptr, oh, oh_size)); g_ptr++; } break; @@ -1094,8 +1104,12 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj) } else if (in_area(ptr, heap, mature_size)) { MOVE_BOXED(ptr,val,old_htop,n_hp++); } else if (in_area(ptr, heap, heap_size)) { + ASSERT(!erts_is_literal(gval, ptr) + && !in_area(ptr, oh, oh_size)); MOVE_BOXED(ptr,val,n_htop,n_hp++); } else { + ASSERT(erts_is_literal(gval, ptr) + || in_area(ptr, oh, oh_size)); n_hp++; } break; @@ -1108,8 +1122,12 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj) } else if (in_area(ptr, heap, mature_size)) { MOVE_CONS(ptr,val,old_htop,n_hp++); } else if (in_area(ptr, heap, heap_size)) { + ASSERT(!erts_is_literal(gval, ptr) + && !in_area(ptr, oh, oh_size)); MOVE_CONS(ptr,val,n_htop,n_hp++); } else { + ASSERT(erts_is_literal(gval, ptr) + || in_area(ptr, oh, oh_size)); n_hp++; } break; @@ -1131,9 +1149,15 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj) MOVE_BOXED(ptr,val,old_htop,origptr); mb->base = binary_bytes(mb->orig); } else if (in_area(ptr, heap, heap_size)) { + ASSERT(!erts_is_literal(*origptr, origptr) + && !in_area(ptr, oh, oh_size)); MOVE_BOXED(ptr,val,n_htop,origptr); mb->base = binary_bytes(mb->orig); } + else { + ASSERT(erts_is_literal(*origptr, origptr) + || in_area(ptr, oh, oh_size)); + } } n_hp += (thing_arityval(gval)+1); } @@ -1275,8 +1299,10 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) ASSERT(is_boxed(val)); *g_ptr++ = val; } else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) { + ASSERT(!erts_is_literal(gval, ptr)); MOVE_BOXED(ptr,val,n_htop,g_ptr++); } else { + ASSERT(erts_is_literal(gval, ptr)); g_ptr++; } continue; @@ -1288,8 +1314,10 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) if (IS_MOVED_CONS(val)) { *g_ptr++ = ptr[1]; } else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) { + ASSERT(!erts_is_literal(gval, ptr)); MOVE_CONS(ptr,val,n_htop,g_ptr++); } else { + ASSERT(erts_is_literal(gval, ptr)); g_ptr++; } continue; @@ -1330,8 +1358,10 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) ASSERT(is_boxed(val)); *n_hp++ = val; } else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) { + ASSERT(!erts_is_literal(gval, ptr)); MOVE_BOXED(ptr,val,n_htop,n_hp++); } else { + ASSERT(erts_is_literal(gval, ptr)); n_hp++; } break; @@ -1342,8 +1372,10 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) if (IS_MOVED_CONS(val)) { *n_hp++ = ptr[1]; } else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) { + ASSERT(!erts_is_literal(gval, ptr)); MOVE_CONS(ptr,val,n_htop,n_hp++); } else { + ASSERT(erts_is_literal(gval, ptr)); n_hp++; } break; @@ -1363,12 +1395,16 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) *origptr = val; mb->base = binary_bytes(*origptr); } else if (in_area(ptr, src, src_size) || - in_area(ptr, oh, oh_size)) { +- in_area(ptr, oh, oh_size)) { + ASSERT(!erts_is_literal(*origptr, origptr)); MOVE_BOXED(ptr,val,n_htop,origptr); mb->base = binary_bytes(*origptr); ptr = boxed_val(*origptr); val = *ptr; } + else { + ASSERT(erts_is_literal(*origptr, origptr)); + } } n_hp += (thing_arityval(gval)+1); } @@ -1985,7 +2021,7 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset) } ASSERT((is_nil(p->seq_trace_token) || - is_tuple(follow_moved(p->seq_trace_token)) || + is_tuple(follow_moved(p->seq_trace_token, (Eterm) 0)) || is_atom(p->seq_trace_token))); if (is_not_immed(p->seq_trace_token)) { roots[n].v = &p->seq_trace_token; @@ -2003,7 +2039,7 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset) is_internal_pid(ERTS_TRACER_PROC(p)) || is_internal_port(ERTS_TRACER_PROC(p))); - ASSERT(is_pid(follow_moved(p->group_leader))); + ASSERT(is_pid(follow_moved(p->group_leader, (Eterm) 0))); if (is_not_immed(p->group_leader)) { roots[n].v = &p->group_leader; roots[n].sz = 1; diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h index ecd1bf4d22..fdbf948f9d 100644 --- a/erts/emulator/beam/erl_gc.h +++ b/erts/emulator/beam/erl_gc.h @@ -76,10 +76,10 @@ do { \ int within(Eterm *ptr, Process *p); #endif -ERTS_GLB_INLINE Eterm follow_moved(Eterm term); +ERTS_GLB_INLINE Eterm follow_moved(Eterm term, Eterm xptr_tag); #if ERTS_GLB_INLINE_INCL_FUNC_DEF -ERTS_GLB_INLINE Eterm follow_moved(Eterm term) +ERTS_GLB_INLINE Eterm follow_moved(Eterm term, Eterm xptr_tag) { Eterm* ptr; switch (primary_tag(term)) { @@ -87,11 +87,11 @@ ERTS_GLB_INLINE Eterm follow_moved(Eterm term) break; case TAG_PRIMARY_BOXED: ptr = boxed_val(term); - if (IS_MOVED_BOXED(*ptr)) term = *ptr; + if (IS_MOVED_BOXED(*ptr)) term = (*ptr) | xptr_tag; break; case TAG_PRIMARY_LIST: ptr = list_val(term); - if (IS_MOVED_CONS(ptr[0])) term = ptr[1]; + if (IS_MOVED_CONS(ptr[0])) term = (ptr[1]) | xptr_tag; break; default: ASSERT(!"strange tag in follow_moved"); diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index ef52823287..e23c79d301 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -642,13 +642,13 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) #endif if (bp->next != NULL) { - move_multi_frags(hpp, off_heap, bp, msg->m, + erts_move_multi_frags(hpp, off_heap, bp, msg->m, #ifdef USE_VM_PROBES - 3 + 3, #else - 2 + 2, #endif - ); + 0); goto copy_done; } diff --git a/erts/emulator/beam/erl_term.c b/erts/emulator/beam/erl_term.c index 3a5fbcc284..8efbb8c554 100644 --- a/erts/emulator/beam/erl_term.c +++ b/erts/emulator/beam/erl_term.c @@ -28,6 +28,37 @@ #include #include +void +erts_set_literal_tag(Eterm *term, Eterm *hp_start, Eterm hsz) +{ +#ifdef TAG_LITERAL_PTR + Eterm *hp_end, *hp; + + hp_end = hp_start + hsz; + hp = hp_start; + + while (hp < hp_end) { + switch (primary_tag(*hp)) { + case TAG_PRIMARY_BOXED: + case TAG_PRIMARY_LIST: + *hp |= TAG_LITERAL_PTR; + break; + case TAG_PRIMARY_HEADER: + if (header_is_thing(*hp)) { + hp += thing_arityval(*hp); + } + break; + default: + break; + } + + hp++; + } + if (is_boxed(*term) || is_list(*term)) + *term |= TAG_LITERAL_PTR; +#endif +} + __decl_noreturn static void __noreturn et_abort(const char *expr, const char *file, unsigned line) { diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h index 089eecf024..a80716fb0f 100644 --- a/erts/emulator/beam/erl_term.h +++ b/erts/emulator/beam/erl_term.h @@ -21,6 +21,8 @@ #ifndef __ERL_TERM_H #define __ERL_TERM_H +#include "erl_mmap.h" + typedef UWord Wterm; /* Full word terms */ struct erl_node_; /* Declared in erl_node_tables.h */ @@ -48,6 +50,24 @@ struct erl_node_; /* Declared in erl_node_tables.h */ #define _ET_APPLY(F,X) _unchecked_##F(X) #endif +#if defined(ARCH_64) +# define TAG_PTR_MASK__ 0x7 +# if !defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) +# ifdef HIPE +# error Hipe on 64-bit needs a real mmap as it does not support the literal tag +# endif +# define TAG_LITERAL_PTR 0x4 +# else +# undef TAG_LITERAL_PTR +# endif +#elif defined(ARCH_32) +# define TAG_PTR_MASK__ 0x3 +# undef TAG_LITERAL_PTR +#else +# error Not supported arch +#endif + + #define _TAG_PRIMARY_SIZE 2 #define _TAG_PRIMARY_MASK 0x3 #define TAG_PRIMARY_HEADER 0x0 @@ -165,10 +185,11 @@ struct erl_node_; /* Declared in erl_node_tables.h */ /* boxed object access methods */ -#define _is_taggable_pointer(x) (((Uint)(x) & 0x3) == 0) +#define _is_taggable_pointer(x) (((Uint)(x) & TAG_PTR_MASK__) == 0) + #define _boxed_precond(x) (is_boxed(x)) -#define _is_aligned(x) (((Uint)(x) & 0x3) == 0) +#define _is_aligned(x) (((Uint)(x) & TAG_PTR_MASK__) == 0) #define _unchecked_make_boxed(x) ((Uint)(x) + TAG_PRIMARY_BOXED) _ET_DECLARE_CHECKED(Eterm,make_boxed,const Eterm*) #define make_boxed(x) _ET_APPLY(make_boxed,(x)) @@ -180,7 +201,11 @@ _ET_DECLARE_CHECKED(int,is_boxed,Eterm) #else #define is_boxed(x) (((x) & _TAG_PRIMARY_MASK) == TAG_PRIMARY_BOXED) #endif +#ifdef TAG_LITERAL_PTR +#define _unchecked_boxed_val(x) _unchecked_ptr_val(x) +#else #define _unchecked_boxed_val(x) ((Eterm*) ((x) - TAG_PRIMARY_BOXED)) +#endif _ET_DECLARE_CHECKED(Eterm*,boxed_val,Wterm) #define boxed_val(x) _ET_APPLY(boxed_val,(x)) @@ -198,7 +223,11 @@ _ET_DECLARE_CHECKED(int,is_not_list,Eterm) #define is_not_list(x) (!is_list((x))) #endif #define _list_precond(x) (is_list(x)) +#ifdef TAG_LITERAL_PTR +#define _unchecked_list_val(x) _unchecked_ptr_val(x) +#else #define _unchecked_list_val(x) ((Eterm*) ((x) - TAG_PRIMARY_LIST)) +#endif _ET_DECLARE_CHECKED(Eterm*,list_val,Wterm) #define list_val(x) _ET_APPLY(list_val,(x)) @@ -209,13 +238,20 @@ _ET_DECLARE_CHECKED(Eterm*,list_val,Wterm) #define CDR(x) ((x)[1]) /* generic tagged pointer (boxed or list) access methods */ -#define _unchecked_ptr_val(x) ((Eterm*) ((x) & ~((Uint) 0x3))) +#define _unchecked_ptr_val(x) ((Eterm*) ((x) & ~((Uint) TAG_PTR_MASK__))) #define ptr_val(x) _unchecked_ptr_val((x)) /*XXX*/ #define _unchecked_offset_ptr(x,offs) ((x)+((offs)*sizeof(Eterm))) #define offset_ptr(x,offs) _unchecked_offset_ptr(x,offs) /*XXX*/ #define _unchecked_byte_offset_ptr(x,byte_offs) ((x)+(offs)) #define byte_offset_ptr(x,offs) _unchecked_byte_offset_ptr(x,offs) /*XXX*/ +#ifdef TAG_LITERAL_PTR +#define _unchecked_is_not_literal_ptr(x) (!((x) & TAG_LITERAL_PTR)) +#define is_not_literal_ptr(x) _unchecked_is_not_literal_ptr((x)) /*XXX*/ +#define is_literal_ptr(x) (!is_not_literal_ptr((x))) /*XXX*/ +#endif + + /* fixnum ("small") access methods */ #if defined(ARCH_64) #define SMALL_BITS (64-4) @@ -1113,5 +1149,7 @@ extern unsigned tag_val_def(Wterm); #define is_same(A,B) ((A)==(B)) +void erts_set_literal_tag(Eterm *term, Eterm *hp_start, Eterm hsz); + #endif /* __ERL_TERM_H */ diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index b4d02dd1dd..052994b972 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -961,8 +961,8 @@ Uint size_object(Eterm); Eterm copy_struct(Eterm, Uint, Eterm**, ErlOffHeap*); Eterm copy_shallow(Eterm*, Uint, Eterm**, ErlOffHeap*); -void move_multi_frags(Eterm** hpp, ErlOffHeap*, ErlHeapFragment* first, - Eterm* refs, unsigned nrefs); +void erts_move_multi_frags(Eterm** hpp, ErlOffHeap*, ErlHeapFragment* first, + Eterm* refs, unsigned nrefs, int literals); /* Utilities */ extern void erts_delete_nodes_monitors(Process *, ErtsProcLocks); @@ -1274,6 +1274,27 @@ int erts_print_system_version(int to, void *arg, Process *c_p); int erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg); +ERTS_GLB_INLINE int erts_is_literal(Eterm tptr, Eterm *ptr); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE int erts_is_literal(Eterm tptr, Eterm *ptr) +{ + ASSERT(is_boxed(tptr) || is_list(tptr)); + ASSERT(ptr == ptr_val(tptr)); + +#if defined(ERTS_HAVE_IS_IN_LITERAL_RANGE) + return erts_is_in_literal_range(ptr); +#elif defined(TAG_LITERAL_PTR) + return is_literal_ptr(tptr); +#else +# error Not able to detect literals... +#endif + +} + +#endif + /* ** Call_trace uses this API for the parameter matching functions */ diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index 46577bd07f..d63e81cb5e 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -72,6 +72,9 @@ #define ERTS_I64_LITERAL(X) X##LL +#define ErtsInArea(ptr,start,nbytes) \ + ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes)) + #if defined (__WIN32__) # include "erl_win_sys.h" #else @@ -1044,10 +1047,6 @@ extern int erts_use_kernel_poll; #define put_int8(i, s) do {((unsigned char*)(s))[0] = (i) & 0xff;} while (0) - -#define ErtsInArea(PTR,START,NBYTES) \ - ((UWord)((char*)(PTR) - (char*)(START)) < (NBYTES)) - /* * Use DEBUGF as you would use printf, but use double parentheses: * diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c index c6ea8a5132..00936b6b8a 100644 --- a/erts/emulator/hipe/hipe_bif0.c +++ b/erts/emulator/hipe/hipe_bif0.c @@ -497,6 +497,8 @@ static void *const_term_alloc(void *tmpl) hp = &p->mem[0]; p->val = copy_struct(obj, size, &hp, &const_term_table_off_heap); + erts_set_literal_tag(&p->val, &p->mem[0], size); + return &p->bucket; } -- cgit v1.2.3