From 1c86a620d74f4f9383c4956dafd3e2486300dc0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Wed, 17 Jun 2015 17:33:29 +0200 Subject: erts: Remove HALFWORD_HEAP definition --- erts/emulator/beam/copy.c | 32 -------------------------------- 1 file changed, 32 deletions(-) (limited to 'erts/emulator/beam/copy.c') diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index 8849dadd00..94048f4c59 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -69,11 +69,7 @@ copy_object(Eterm obj, Process* to) * Return the "flat" size of the object. */ -#if HALFWORD_HEAP -Uint size_object_rel(Eterm obj, Eterm* base) -#else Uint size_object(Eterm obj) -#endif { Uint sum = 0; Eterm* ptr; @@ -227,12 +223,7 @@ Uint size_object(Eterm obj) /* * Copy a structure to a heap. */ -#if HALFWORD_HEAP -Eterm copy_struct_rel(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, - Eterm* src_base, Eterm* dst_base) -#else Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) -#endif { char* hstart; Uint hsize; @@ -287,13 +278,10 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) break; case TAG_PRIMARY_LIST: objp = list_val_rel(obj,src_base); - #if !HALFWORD_HEAP || defined(DEBUG) if (in_area(objp,hstart,hsize)) { - ASSERT(!HALFWORD_HEAP); hp++; break; } - #endif argp = hp++; /* Fall through */ @@ -309,17 +297,9 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) } else { CAR(htop) = elem; - #if HALFWORD_HEAP - CDR(htop) = CDR(objp); - *tailp = make_list_rel(htop,dst_base); - htop += 2; - goto L_copy; - #else tailp = &CDR(htop); htop += 2; - #endif } - ASSERT(!HALFWORD_HEAP || tp < hp || tp >= hbot); *tp = make_list_rel(tailp - 1, dst_base); obj = CDR(objp); if (!is_list(obj)) { @@ -337,13 +317,10 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) } case TAG_PRIMARY_BOXED: - #if !HALFWORD_HEAP || defined(DEBUG) if (in_area(boxed_val_rel(obj,src_base),hstart,hsize)) { - ASSERT(!HALFWORD_HEAP); hp++; break; } - #endif argp = hp++; L_copy_boxed: @@ -563,21 +540,12 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) * * NOTE: Assumes that term is a tuple (ptr is an untagged tuple ptr). */ -#if HALFWORD_HEAP -Eterm copy_shallow_rel(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, - Eterm* src_base) -#else Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) -#endif { Eterm* tp = ptr; Eterm* hp = *hpp; const Eterm res = make_tuple(hp); -#if HALFWORD_HEAP - const Sint offs = COMPRESS_POINTER(hp - (tp - src_base)); -#else const Sint offs = (hp - tp) * sizeof(Eterm); -#endif while (sz--) { Eterm val = *tp++; -- cgit v1.2.3 From 4f9c6bd8fc7090dcbc4b5b3cf595e1689fdaff7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Thu, 18 Jun 2015 14:30:28 +0200 Subject: erts: Remove halfword basic relative heap operations --- erts/emulator/beam/copy.c | 48 +++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) (limited to 'erts/emulator/beam/copy.c') diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index 94048f4c59..38c40f4e7d 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -80,7 +80,7 @@ Uint size_object(Eterm obj) switch (primary_tag(obj)) { case TAG_PRIMARY_LIST: sum += 2; - ptr = list_val_rel(obj,base); + ptr = list_val(obj); obj = *ptr++; if (!IS_CONST(obj)) { ESTACK_PUSH(s, obj); @@ -89,11 +89,11 @@ Uint size_object(Eterm obj) break; case TAG_PRIMARY_BOXED: { - Eterm hdr = *boxed_val_rel(obj,base); + Eterm hdr = *boxed_val(obj); ASSERT(is_header(hdr)); switch (hdr & _TAG_HEADER_MASK) { case ARITYVAL_SUBTAG: - ptr = tuple_val_rel(obj,base); + ptr = tuple_val(obj); arity = header_arity(hdr); sum += arity + 1; if (arity == 0) { /* Empty tuple -- unusual. */ @@ -109,7 +109,7 @@ Uint size_object(Eterm obj) break; case FUN_SUBTAG: { - Eterm* bptr = fun_val_rel(obj,base); + Eterm* bptr = fun_val(obj); ErlFunThing* funp = (ErlFunThing *) bptr; unsigned eterms = 1 /* creator */ + funp->num_free; unsigned sz = thing_arityval(hdr); @@ -130,7 +130,7 @@ Uint size_object(Eterm obj) { Uint n; flatmap_t *mp; - mp = (flatmap_t*)flatmap_val_rel(obj,base); + mp = (flatmap_t*)flatmap_val(obj); ptr = (Eterm *)mp; n = flatmap_get_size(mp) + 1; sum += n + 2; @@ -149,7 +149,7 @@ Uint size_object(Eterm obj) { Eterm *head; Uint sz; - head = hashmap_val_rel(obj, base); + head = hashmap_val(obj); sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); sum += 1 + sz + header_arity(hdr); head += 1 + header_arity(hdr); @@ -188,11 +188,11 @@ Uint size_object(Eterm obj) } else { extra_bytes = 0; } - hdr = *binary_val_rel(real_bin,base); + hdr = *binary_val(real_bin); if (thing_subtag(hdr) == REFC_BINARY_SUBTAG) { sum += PROC_BIN_SIZE; } else { - sum += heap_bin_size(binary_size_rel(obj,base)+extra_bytes); + sum += heap_bin_size(binary_size(obj)+extra_bytes); } goto pop_next; } @@ -259,7 +259,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) switch (primary_tag(obj)) { case TAG_PRIMARY_LIST: argp = &res; - objp = list_val_rel(obj,src_base); + objp = list_val(obj); goto L_copy_list; case TAG_PRIMARY_BOXED: argp = &res; goto L_copy_boxed; default: @@ -277,7 +277,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) hp++; break; case TAG_PRIMARY_LIST: - objp = list_val_rel(obj,src_base); + objp = list_val(obj); if (in_area(objp,hstart,hsize)) { hp++; break; @@ -300,12 +300,12 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) tailp = &CDR(htop); htop += 2; } - *tp = make_list_rel(tailp - 1, dst_base); + *tp = make_list(tailp - 1); obj = CDR(objp); if (!is_list(obj)) { break; } - objp = list_val_rel(obj,src_base); + objp = list_val(obj); } switch (primary_tag(obj)) { case TAG_PRIMARY_IMMED1: *tailp = obj; goto L_copy; @@ -317,21 +317,21 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) } case TAG_PRIMARY_BOXED: - if (in_area(boxed_val_rel(obj,src_base),hstart,hsize)) { + if (in_area(boxed_val(obj),hstart,hsize)) { hp++; break; } argp = hp++; L_copy_boxed: - objp = boxed_val_rel(obj, src_base); + objp = boxed_val(obj); hdr = *objp; switch (hdr & _TAG_HEADER_MASK) { case ARITYVAL_SUBTAG: { int const_flag = 1; /* assume constant tuple */ i = arityval(hdr); - *argp = make_tuple_rel(htop, dst_base); + *argp = make_tuple(htop); tp = htop; /* tp is pointer to new arity value */ *htop++ = *objp++; /* copy arity value */ while (i--) { @@ -360,7 +360,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) while (i--) { *tp++ = *objp++; } - *argp = make_binary_rel(hbot, dst_base); + *argp = make_binary(hbot); pb = (ProcBin*) hbot; erts_refc_inc(&pb->val->refc, 2); pb->next = off_heap->first; @@ -387,7 +387,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) extra_bytes = 0; } real_size = size+extra_bytes; - objp = binary_val_rel(real_bin,src_base); + objp = binary_val(real_bin); if (thing_subtag(*objp) == HEAP_BINARY_SUBTAG) { ErlHeapBin* from = (ErlHeapBin *) objp; ErlHeapBin* to; @@ -417,7 +417,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) off_heap->first = (struct erl_off_heap_header*) to; OH_OVERHEAD(off_heap, to->size / sizeof(Eterm)); } - *argp = make_binary_rel(hbot, dst_base); + *argp = make_binary(hbot); if (extra_bytes != 0) { ErlSubBin* res; hbot -= ERL_SUB_BIN_SIZE; @@ -429,7 +429,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) res->offs = 0; res->is_writable = 0; res->orig = *argp; - *argp = make_binary_rel(hbot, dst_base); + *argp = make_binary(hbot); } break; } @@ -447,7 +447,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) funp->next = off_heap->first; off_heap->first = (struct erl_off_heap_header*) funp; erts_refc_inc(&funp->fe->refc, 2); - *argp = make_fun_rel(tp, dst_base); + *argp = make_fun(tp); } break; case EXTERNAL_PID_SUBTAG: @@ -467,7 +467,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) off_heap->first = (struct erl_off_heap_header*)etp; erts_refc_inc(&etp->node->refc, 2); - *argp = make_external_rel(tp, dst_base); + *argp = make_external(tp); } break; case MAP_SUBTAG: @@ -475,7 +475,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) switch (MAP_HEADER_TYPE(hdr)) { case MAP_HEADER_TAG_FLATMAP_HEAD : i = flatmap_get_size(objp) + 3; - *argp = make_flatmap_rel(htop, dst_base); + *argp = make_flatmap(htop); while (i--) { *htop++ = *objp++; } @@ -486,7 +486,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) case MAP_HEADER_TAG_HAMT_NODE_BITMAP : i = 1 + hashmap_bitcount(MAP_HEADER_VAL(hdr)); while (i--) { *htop++ = *objp++; } - *argp = make_hashmap_rel(tp, dst_base); + *argp = make_hashmap(tp); break; default: erl_exit(ERTS_ABORT_EXIT, "copy_struct: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); @@ -499,7 +499,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) i = thing_arityval(hdr)+1; hbot -= i; tp = hbot; - *argp = make_boxed_rel(hbot, dst_base); + *argp = make_boxed(hbot); while (i--) { *tp++ = *objp++; } -- cgit v1.2.3 From d6712b1c54de471beb1784bb329ea217767f70ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Thu, 18 Jun 2015 17:03:48 +0200 Subject: erts: Reinstate copy_object over-allocation optimization --- erts/emulator/beam/copy.c | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) (limited to 'erts/emulator/beam/copy.c') diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index 38c40f4e7d..3de2c10203 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -40,29 +40,30 @@ static void move_one_frag(Eterm** hpp, ErlHeapFragment*, ErlOffHeap*); /* * Copy object "obj" to process p. */ -Eterm -copy_object(Eterm obj, Process* to) -{ - Uint size = size_object(obj); - Eterm* hp = HAlloc(to, size); - Eterm res; +Eterm copy_object_x(Eterm obj, Process* to, Uint extra) { + if (!is_immed(obj)) { + Uint size = size_object(obj); + Eterm* hp = HAllocX(to, size, extra); + Eterm res; #ifdef USE_VM_PROBES - if (DTRACE_ENABLED(copy_object)) { - DTRACE_CHARBUF(proc_name, 64); + if (DTRACE_ENABLED(copy_object)) { + DTRACE_CHARBUF(proc_name, 64); - erts_snprintf(proc_name, sizeof(DTRACE_CHARBUF_NAME(proc_name)), - "%T", to->common.id); - DTRACE2(copy_object, proc_name, size); - } + erts_snprintf(proc_name, sizeof(DTRACE_CHARBUF_NAME(proc_name)), + "%T", to->common.id); + DTRACE2(copy_object, proc_name, size); + } #endif - res = copy_struct(obj, size, &hp, &to->off_heap); + res = copy_struct(obj, size, &hp, &to->off_heap); #ifdef DEBUG - if (eq(obj, res) == 0) { - erl_exit(ERTS_ABORT_EXIT, "copy not equal to source\n"); - } + if (eq(obj, res) == 0) { + erl_exit(ERTS_ABORT_EXIT, "copy not equal to source\n"); + } #endif - return res; + return res; + } + return obj; } /* -- cgit v1.2.3 From af8bb5e30b021b7b8e80ed96f3f3e16c18b865e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Thu, 18 Jun 2015 17:43:59 +0200 Subject: erts: Remove halfword BINARY RELs * ERTS_GET_BINARY_BYTES_REL * ERTS_GET_REAL_BIN_REL --- erts/emulator/beam/copy.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'erts/emulator/beam/copy.c') diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index 3de2c10203..ec769c3b49 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -179,7 +179,7 @@ Uint size_object(Eterm obj) Uint bitoffs; Uint extra_bytes; Eterm hdr; - ERTS_GET_REAL_BIN_REL(obj, real_bin, offset, bitoffs, bitsize, base); + ERTS_GET_REAL_BIN(obj, real_bin, offset, bitoffs, bitsize); if ((bitsize + bitoffs) > 8) { sum += ERL_SUB_BIN_SIZE; extra_bytes = 2; -- cgit v1.2.3 From b21b604137c5cb5f5039a40994e429871e5b707b Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Wed, 26 Aug 2015 19:47:10 +0200 Subject: Introduce literal tag --- erts/emulator/beam/copy.c | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) (limited to 'erts/emulator/beam/copy.c') diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index ec769c3b49..b185758b1d 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -35,7 +35,7 @@ #include "erl_bits.h" #include "dtrace-wrapper.h" -static void move_one_frag(Eterm** hpp, ErlHeapFragment*, ErlOffHeap*); +static void move_one_frag(Eterm** hpp, ErlHeapFragment*, ErlOffHeap*, int); /* * Copy object "obj" to process p. @@ -621,17 +621,24 @@ Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) * move markers. * Typically used to copy a multi-fragmented message (from NIF). */ -void move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first, - Eterm* refs, unsigned nrefs) +void erts_move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first, + Eterm* refs, unsigned nrefs, int literals) { ErlHeapFragment* bp; Eterm* hp_start = *hpp; Eterm* hp_end; Eterm* hp; unsigned i; + Eterm literal_tag; + +#ifdef TAG_LITERAL_PTR + literal_tag = (Eterm) literals ? TAG_LITERAL_PTR : 0; +#else + literal_tag = (Eterm) 0; +#endif for (bp=first; bp!=NULL; bp=bp->next) { - move_one_frag(hpp, bp, off_heap); + move_one_frag(hpp, bp, off_heap, literals); } hp_end = *hpp; for (hp=hp_start; hpmem; Eterm* end = ptr + frag->used_size; @@ -704,4 +718,3 @@ move_one_frag(Eterm** hpp, ErlHeapFragment* frag, ErlOffHeap* off_heap) OH_OVERHEAD(off_heap, frag->off_heap.overhead); frag->off_heap.first = NULL; } - -- cgit v1.2.3 From 3ac08f9b668613a4292436979eacc61863c2ab94 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Wed, 16 Sep 2015 15:02:50 +0200 Subject: Fragmented young heap generation and off_heap_message_queue option * The youngest generation of the heap can now consist of multiple blocks. Heap fragments and message fragments are added to the youngest generation when needed without triggering a GC. After a GC the youngest generation is contained in one single block. * The off_heap_message_queue process flag has been added. When enabled all message data in the queue is kept off heap. When a message is selected from the queue, the message fragment (or heap fragment) containing the actual message is attached to the youngest generation. Messages stored off heap is not part of GC. --- erts/emulator/beam/copy.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'erts/emulator/beam/copy.c') diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index b185758b1d..f27c526413 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -279,7 +279,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) break; case TAG_PRIMARY_LIST: objp = list_val(obj); - if (in_area(objp,hstart,hsize)) { + if (ErtsInArea(objp,hstart,hsize)) { hp++; break; } @@ -318,7 +318,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) } case TAG_PRIMARY_BOXED: - if (in_area(boxed_val(obj),hstart,hsize)) { + if (ErtsInArea(boxed_val(obj),hstart,hsize)) { hp++; break; } -- cgit v1.2.3 From d5711cb70be9ad2e5c78839e8a368900ba248a4a Mon Sep 17 00:00:00 2001 From: "Nikolaos S. Papaspyrou" Date: Fri, 8 Jun 2012 23:13:50 +0300 Subject: Add all the main machinery Add functions size_shared, copy_shared_calculate and copy_shared_perform. Add the infrastructure for making these communicate with each other. Add debug information to other places in the VM, to watch interaction with the sharing-preserving copy. CAUTION: If you define the SHCOPY_DEBUG macro (after SHCOPY is actually used in the VM) and make the whole OTP, there will be a lot of debugging messages during make (it will also be enabled in erlc). You have been warned... --- erts/emulator/beam/copy.c | 1131 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1131 insertions(+) (limited to 'erts/emulator/beam/copy.c') diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index f27c526413..b31e043f08 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -75,8 +75,17 @@ Uint size_object(Eterm obj) Uint sum = 0; Eterm* ptr; int arity; +#ifdef SHCOPY_DEBUG + Eterm mypid; +#endif DECLARE_ESTACK(s); + +#ifdef SHCOPY_DEBUG + mypid = erts_get_current_pid(); + VERBOSE_DEBUG("[pid=%T] size_object %p\n", mypid, obj); +#endif + for (;;) { switch (primary_tag(obj)) { case TAG_PRIMARY_LIST: @@ -211,6 +220,7 @@ Uint size_object(Eterm obj) pop_next: if (ESTACK_ISEMPTY(s)) { DESTROY_ESTACK(s); + VERBOSE_DEBUG("[pid=%T] size was: %u\n", mypid, sum); return sum; } obj = ESTACK_POP(s); @@ -221,6 +231,350 @@ Uint size_object(Eterm obj) } } +/* + * Machinery for sharing preserving information + * Using a WSTACK but not very transparently; consider refactoring + */ + +#define DECLARE_BITSTORE(s) \ + DECLARE_WSTACK(s); \ + int WSTK_CONCAT(s,_bitoffs) = 0; \ + int WSTK_CONCAT(s,_offset) = 0; \ + UWord WSTK_CONCAT(s,_buffer) = 0 + +#define DESTROY_BITSTORE(s) DESTROY_WSTACK(s) +#define BITSTORE_PUT(s,i) \ +do { \ + WSTK_CONCAT(s,_buffer) |= i << WSTK_CONCAT(s,_bitoffs); \ + WSTK_CONCAT(s,_bitoffs) += 2; \ + if (WSTK_CONCAT(s,_bitoffs) >= 8*sizeof(UWord)) { \ + WSTACK_PUSH(s, WSTK_CONCAT(s,_buffer)); \ + WSTK_CONCAT(s,_bitoffs) = 0; \ + WSTK_CONCAT(s,_buffer) = 0; \ + } \ +} while(0) +#define BITSTORE_CLOSE(s) \ +do { \ + if (WSTK_CONCAT(s,_bitoffs) > 0) { \ + WSTACK_PUSH(s, WSTK_CONCAT(s,_buffer)); \ + WSTK_CONCAT(s,_bitoffs) = 0; \ + } \ +} while(0) + +#define BITSTORE_GET(s) ({ \ + UWord result; \ + if (WSTK_CONCAT(s,_bitoffs) <= 0) { \ + WSTK_CONCAT(s,_buffer) = s.wstart[WSTK_CONCAT(s,_offset)]; \ + WSTK_CONCAT(s,_offset)++; \ + WSTK_CONCAT(s,_bitoffs) = 8*sizeof(UWord); \ + } \ + WSTK_CONCAT(s,_bitoffs) -= 2; \ + result = WSTK_CONCAT(s,_buffer) & 3; \ + WSTK_CONCAT(s,_buffer) >>= 2; \ + result; \ +}) + +#define BOXED_VISITED_MASK ((Eterm) 3) +#define BOXED_VISITED ((Eterm) 1) +#define BOXED_SHARED_UNPROCESSED ((Eterm) 2) +#define BOXED_SHARED_PROCESSED ((Eterm) 3) + + +/* + * Is an object in the local heap of a process? + */ + +#define INHEAP_SIMPLE(p, ptr) ( \ + (OLD_HEAP(p) && OLD_HEAP(p) <= ptr && ptr < OLD_HEND(p)) || \ + (HEAP_START(p) <= ptr && ptr < HEAP_END(p)) \ + ) +#define INHEAP(p, ptr) ( \ + INHEAP_SIMPLE(p, ptr) || \ + (force_local ? (force_local = 0, 1) : 0) \ + ) +#define COUNT_OFF_HEAP 0 + + +/* + * Return the real size of an object and find sharing information + * This currently returns the same as erts_debug:size/1. + * It is argued whether the size of subterms in constant pools + * should be counted or not. + */ +#if HALFWORD_HEAP +Uint size_shared_rel(Eterm obj, Eterm* base) +#else +Uint size_shared(Eterm obj) +#endif +{ + Eterm saved_obj = obj; + Uint sum = 0; + Eterm* ptr; + Process* myself; + + DECLARE_EQUEUE(s); + DECLARE_BITSTORE(b); + + myself = erts_get_current_process(); + if (myself == NULL) + return size_object(obj); + + for (;;) { + VERBOSE_DEBUG("[size] visiting: %x ", obj); + switch (primary_tag(obj)) { + case TAG_PRIMARY_LIST: { + Eterm head, tail; + VERBOSE_DEBUG("L"); + ptr = list_val_rel(obj, base); + /* we're not counting anything that's outside our heap */ + if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) { + goto pop_next; + } + head = CAR(ptr); + tail = CDR(ptr); + /* if it's visited, don't count it */ + if (primary_tag(tail) == TAG_PRIMARY_HEADER || + primary_tag(head) == TAG_PRIMARY_HEADER) { + VERBOSE_DEBUG("!"); + goto pop_next; + } + /* else make it visited now */ + switch (primary_tag(tail)) { + case TAG_PRIMARY_LIST: + VERBOSE_DEBUG("/L"); + ptr[1] = (tail - TAG_PRIMARY_LIST) | TAG_PRIMARY_HEADER; + break; + case TAG_PRIMARY_IMMED1: + VERBOSE_DEBUG("/I"); + CAR(ptr) = (head - primary_tag(head)) | TAG_PRIMARY_HEADER; + CDR(ptr) = (tail - TAG_PRIMARY_IMMED1) | primary_tag(head); + break; + case TAG_PRIMARY_BOXED: + VERBOSE_DEBUG("/B saved %d", primary_tag(head)); + BITSTORE_PUT(b, primary_tag(head)); + CAR(ptr) = (head - primary_tag(head)) | TAG_PRIMARY_HEADER; + CDR(ptr) = (tail - TAG_PRIMARY_BOXED) | TAG_PRIMARY_HEADER; + break; + } + /* and count it */ + sum += 2; + if (!IS_CONST(head)) { + EQUEUE_PUT(s, head); + } + obj = tail; + break; + } + case TAG_PRIMARY_BOXED: { + Eterm hdr; + VERBOSE_DEBUG("B"); + ptr = boxed_val_rel(obj, base); + /* we're not counting anything that's outside our heap */ + if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) { + goto pop_next; + } + hdr = *ptr; + /* if it's visited, don't count it */ + if (primary_tag(hdr) != TAG_PRIMARY_HEADER) { + VERBOSE_DEBUG("!"); + goto pop_next; + } + /* else make it visited now */ + *ptr = (hdr - primary_tag(hdr)) + BOXED_VISITED; + /* and count it */ + ASSERT(is_header(hdr)); + switch (hdr & _TAG_HEADER_MASK) { + case ARITYVAL_SUBTAG: { + int arity = header_arity(hdr); + VERBOSE_DEBUG("/T"); + sum += arity + 1; + if (arity == 0) { /* Empty tuple -- unusual. */ + VERBOSE_DEBUG("e"); + goto pop_next; + } + while (arity-- > 0) { + obj = *++ptr; + if (!IS_CONST(obj)) { + EQUEUE_PUT(s, obj); + } + } + goto pop_next; + } + case FUN_SUBTAG: { + ErlFunThing* funp = (ErlFunThing *) ptr; + unsigned eterms = 1 /* creator */ + funp->num_free; + unsigned sz = thing_arityval(hdr); + VERBOSE_DEBUG("/F"); + sum += 1 /* header */ + sz + eterms; + ptr += 1 /* header */ + sz; + while (eterms-- > 0) { + obj = *ptr++; + if (!IS_CONST(obj)) { + EQUEUE_PUT(s, obj); + } + } + goto pop_next; + } + case SUB_BINARY_SUBTAG: { + ErlSubBin* sb = (ErlSubBin *) ptr; + Uint extra_bytes; + Eterm hdr; + ASSERT((sb->thing_word & ~BOXED_VISITED_MASK) == HEADER_SUB_BIN); + if (sb->bitsize + sb->bitoffs > 8) { + sum += ERL_SUB_BIN_SIZE; + extra_bytes = 2; + } else if (sb->bitsize + sb->bitoffs > 0) { + sum += ERL_SUB_BIN_SIZE; + extra_bytes = 1; + } else { + extra_bytes = 0; + } + ptr = binary_val_rel(sb->orig, base); + hdr = (*ptr) & ~BOXED_VISITED_MASK; + if (thing_subtag(hdr) == REFC_BINARY_SUBTAG) { + sum += PROC_BIN_SIZE; + } else { + ASSERT(thing_subtag(hdr) == HEAP_BINARY_SUBTAG); + sum += heap_bin_size(binary_size_rel(obj, base) + extra_bytes); + } + goto pop_next; + } + case BIN_MATCHSTATE_SUBTAG: + erl_exit(ERTS_ABORT_EXIT, + "size_shared: matchstate term not allowed"); + default: + VERBOSE_DEBUG("/D"); + sum += thing_arityval(hdr) + 1; + goto pop_next; + } + break; + } + case TAG_PRIMARY_IMMED1: + VERBOSE_DEBUG("I"); + pop_next: + if (EQUEUE_ISEMPTY(s)) { + goto cleanup; + } + obj = EQUEUE_GET(s); + break; + default: + erl_exit(ERTS_ABORT_EXIT, "size_shared: bad tag for %#x\n", obj); + } + VERBOSE_DEBUG("\n"); + } + +cleanup: + VERBOSE_DEBUG("\n"); + obj = saved_obj; + BITSTORE_CLOSE(b); + for (;;) { + VERBOSE_DEBUG("[size] revisiting: %x ", obj); + switch (primary_tag(obj)) { + case TAG_PRIMARY_LIST: { + Eterm head, tail; + VERBOSE_DEBUG("L"); + ptr = list_val_rel(obj, base); + if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) { + goto cleanup_next; + } + head = CAR(ptr); + tail = CDR(ptr); + /* if not already clean, clean it up */ + if (primary_tag(tail) == TAG_PRIMARY_HEADER) { + if (primary_tag(head) == TAG_PRIMARY_HEADER) { + Eterm saved = BITSTORE_GET(b); + VERBOSE_DEBUG("/B restoring %d", saved); + CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) | saved; + CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) | TAG_PRIMARY_BOXED; + } else { + VERBOSE_DEBUG("/L"); + CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) | TAG_PRIMARY_LIST; + } + } else if (primary_tag(head) == TAG_PRIMARY_HEADER) { + VERBOSE_DEBUG("/I"); + CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) | primary_tag(tail); + CDR(ptr) = tail = (tail - primary_tag(tail)) | TAG_PRIMARY_IMMED1; + } else { + VERBOSE_DEBUG("!"); + goto cleanup_next; + } + /* and its children too */ + if (!IS_CONST(head)) { + EQUEUE_PUT_UNCHECKED(s, head); + } + obj = tail; + break; + } + case TAG_PRIMARY_BOXED: { + Eterm hdr; + VERBOSE_DEBUG("B"); + ptr = boxed_val_rel(obj, base); + if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) { + goto cleanup_next; + } + hdr = *ptr; + /* if not already clean, clean it up */ + if (primary_tag(hdr) == TAG_PRIMARY_HEADER) { + goto cleanup_next; + } + else { + ASSERT(primary_tag(hdr) == BOXED_VISITED); + *ptr = hdr = (hdr - BOXED_VISITED) + TAG_PRIMARY_HEADER; + } + /* and its children too */ + switch (hdr & _TAG_HEADER_MASK) { + case ARITYVAL_SUBTAG: { + int arity = header_arity(hdr); + if (arity == 0) { /* Empty tuple -- unusual. */ + goto cleanup_next; + } + while (arity-- > 0) { + obj = *++ptr; + if (!IS_CONST(obj)) { + EQUEUE_PUT_UNCHECKED(s, obj); + } + } + goto cleanup_next; + } + case FUN_SUBTAG: { + ErlFunThing* funp = (ErlFunThing *) ptr; + unsigned eterms = 1 /* creator */ + funp->num_free; + unsigned sz = thing_arityval(hdr); + ptr += 1 /* header */ + sz; + while (eterms-- > 0) { + obj = *ptr++; + if (!IS_CONST(obj)) { + EQUEUE_PUT_UNCHECKED(s, obj); + } + } + goto cleanup_next; + } + default: + goto cleanup_next; + } + break; + } + case TAG_PRIMARY_IMMED1: + cleanup_next: + if (EQUEUE_ISEMPTY(s)) { + goto all_clean; + } + obj = EQUEUE_GET(s); + break; + default: + erl_exit(ERTS_ABORT_EXIT, "size_shared: bad tag for %#x\n", obj); + } + VERBOSE_DEBUG("\n"); + } + + all_clean: + VERBOSE_DEBUG("\n"); + /* Return the result */ + DESTROY_EQUEUE(s); + DESTROY_BITSTORE(b); + return sum; +} + + /* * Copy a structure to a heap. */ @@ -244,10 +598,18 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) Eterm org_obj = obj; Uint org_sz = sz; #endif +#ifdef SHCOPY_DEBUG + Eterm mypid; +#endif if (IS_CONST(obj)) return obj; +#ifdef SHCOPY_DEBUG + mypid = erts_get_current_pid(); + VERBOSE_DEBUG("[pid=%T] copy_struct %p\n", mypid, obj); +#endif + DTRACE1(copy_struct, (int32_t)sz); hp = htop = *hpp; @@ -529,9 +891,778 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) } #endif *hpp = (Eterm *) (hstart+hsize); + VERBOSE_DEBUG("[pid=%T] result is at %p\n", mypid, res); return res; } + +/* + * Machinery for the table used by the sharing preserving copier + * Using an ESTACK but not very transparently; consider refactoring + */ + +#define DECLARE_SHTABLE(s) \ + DECLARE_ESTACK(s); \ + Uint ESTK_CONCAT(s,_offset) = 0 +#define DESTROY_SHTABLE(s) DESTROY_ESTACK(s) +#define SHTABLE_INCR 4 +#define SHTABLE_NEXT(s) ESTK_CONCAT(s,_offset) +#define SHTABLE_PUSH(s,x,y,b) \ +do { \ + if (s.sp > s.end - SHTABLE_INCR) { \ + erl_grow_estack(&s, ESTK_DEF_STACK(s)); \ + } \ + *s.sp++ = (x); \ + *s.sp++ = (y); \ + *s.sp++ = (Eterm) NULL; \ + *s.sp++ = (Eterm) (b); /* bad in HALF_WORD */ \ + ESTK_CONCAT(s,_offset) += SHTABLE_INCR; \ +} while(0) +#define SHTABLE_X(s,e) (s.start[e]) +#define SHTABLE_Y(s,e) (s.start[(e)+1]) +#define SHTABLE_FWD(s,e) ((Eterm *) (s.start[(e)+2])) +#define SHTABLE_FWD_UPD(s,e,p) (s.start[(e)+2] = (Eterm) (p)) +#define SHTABLE_REV(s,e) ((Eterm *) (s.start[(e)+3])) + +#define LIST_SHARED_UNPROCESSED ((Eterm) 0) +#define LIST_SHARED_PROCESSED ((Eterm) 1) + +#define HEAP_ELEM_TO_BE_FILLED _unchecked_make_list(NULL) + + +/* + * Specialized macros for using/reusing the persistent state + */ + +#define DECLARE_EQUEUE_INIT_INFO(q, info) \ + UWord* EQUE_DEF_QUEUE(q) = info->queue_default; \ + ErtsEQueue q = { \ + EQUE_DEF_QUEUE(q), /* start */ \ + EQUE_DEF_QUEUE(q), /* front */ \ + EQUE_DEF_QUEUE(q), /* back */ \ + 1, /* possibly_empty */ \ + EQUE_DEF_QUEUE(q) + DEF_EQUEUE_SIZE, /* end */ \ + ERTS_ALC_T_ESTACK /* alloc_type */ \ + } + +#define DECLARE_EQUEUE_FROM_INFO(q, info) \ + /* no EQUE_DEF_QUEUE(q), read-only */ \ + ErtsEQueue q = { \ + info->queue_start, /* start */ \ + info->queue_start, /* front */ \ + info->queue_start, /* back */ \ + 1, /* possibly_empty */ \ + info->queue_end, /* end */ \ + info->queue_alloc_type /* alloc_type */ \ + } + +#define DECLARE_BITSTORE_INIT_INFO(s, info) \ + UWord* WSTK_DEF_STACK(s) = info->bitstore_default; \ + ErtsWStack s = { \ + WSTK_DEF_STACK(s), /* wstart */ \ + WSTK_DEF_STACK(s), /* wsp */ \ + WSTK_DEF_STACK(s) + DEF_WSTACK_SIZE, /* wend */ \ + ERTS_ALC_T_ESTACK /* alloc_type */ \ + }; \ + int WSTK_CONCAT(s,_bitoffs) = 0; \ + /* no WSTK_CONCAT(s,_offset), write-only */ \ + UWord WSTK_CONCAT(s,_buffer) = 0 + +#define DECLARE_BITSTORE_FROM_INFO(s, info) \ + /* no WSTK_DEF_STACK(s), read-only */ \ + ErtsWStack s = { \ + info->bitstore_start, /* wstart */ \ + NULL, /* wsp, read-only */ \ + NULL, /* wend, read-only */ \ + info->bitstore_alloc_type /* alloc_type */ \ + }; \ + int WSTK_CONCAT(s,_bitoffs) = 0; \ + int WSTK_CONCAT(s,_offset) = 0; \ + UWord WSTK_CONCAT(s,_buffer) = 0 + +#define DECLARE_SHTABLE_INIT_INFO(s, info) \ + Eterm* ESTK_DEF_STACK(s) = info->shtable_default; \ + ErtsEStack s = { \ + ESTK_DEF_STACK(s), /* start */ \ + ESTK_DEF_STACK(s), /* sp */ \ + ESTK_DEF_STACK(s) + DEF_ESTACK_SIZE, /* end */ \ + ERTS_ALC_T_ESTACK /* alloc_type */ \ + }; \ + Uint ESTK_CONCAT(s,_offset) = 0 + +#define DECLARE_SHTABLE_FROM_INFO(s, info) \ + /* no ESTK_DEF_STACK(s), read-only */ \ + ErtsEStack s = { \ + info->shtable_start, /* start */ \ + NULL, /* sp, read-only */ \ + NULL, /* end, read-only */ \ + info->shtable_alloc_type /* alloc_type */ \ + }; \ + /* no ESTK_CONCAT(s,_offset), read-only */ + +/* + * Copy object "obj" preserving sharing. + * First half: count size and calculate sharing. + * NOTE: We do not support HALF_WORD (yet?). + */ +Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) +{ + Uint sum; + Uint e; + unsigned sz; + Eterm* ptr; + Process* myself; + int force_local = flags & ERTS_SHCOPY_FLG_TMP_BUF; + + DECLARE_EQUEUE_INIT_INFO(s, info); + DECLARE_BITSTORE_INIT_INFO(b, info); + DECLARE_SHTABLE_INIT_INFO(t, info); + + /* step #0: + ------------------------------------------------------- + get rid of the easy cases first: + - copying constants + - if not a proper process, do flat copy + */ + + if (IS_CONST(obj)) + return 0; + + myself = erts_get_current_process(); + if (myself == NULL || (flags & ERTS_SHCOPY_FLG_NONE)) + return size_object(obj); + + VERBOSE_DEBUG("[pid=%T] copy_shared_calculate %p\n", myself->common.id, obj); + VERBOSE_DEBUG("[pid=%T] message is %T\n", myself->common.id, obj); + + /* step #1: + ------------------------------------------------------- + traverse the term and calculate the size; + when traversing, transform as you do in size_shared + but when you find shared objects: + + a. add entry in the table, indexed by i + b. mark them: + b1. boxed terms, set header to (i | 11) + store (old header, NONV, NULL, backptr) in the entry + b2. cons cells, set CDR to NONV, set CAR to i + store (old CAR, old CDR, NULL, backptr) in the entry + */ + + sum = 0; + + for (;;) { + VERBOSE_DEBUG("[copy] visiting: %x ", obj); + switch (primary_tag(obj)) { + case TAG_PRIMARY_LIST: { + Eterm head, tail; + VERBOSE_DEBUG("L"); + ptr = list_val_rel(obj, base); + /* off heap list pointers are copied verbatim */ + if (!INHEAP(myself, ptr)) { + VERBOSE_DEBUG("[pid=%T] bypassed copying %p is %T\n", myself->common.id, ptr, obj); + if (myself->mbuf != NULL) + VERBOSE_DEBUG("[pid=%T] BUT !!! there are message buffers!\n", myself->common.id); + VERBOSE_DEBUG("#"); + goto pop_next; + } + head = CAR(ptr); + tail = CDR(ptr); + /* if it's visited, don't count it; + if not already shared, make it shared and store it in the table */ + if (primary_tag(tail) == TAG_PRIMARY_HEADER || + primary_tag(head) == TAG_PRIMARY_HEADER) { + VERBOSE_DEBUG("!"); + if (tail != THE_NON_VALUE) { + e = SHTABLE_NEXT(t); + VERBOSE_DEBUG("[pid=%T] tabling L %p\n", myself->common.id, ptr); + SHTABLE_PUSH(t, head, tail, ptr); + CAR(ptr) = (e << _TAG_PRIMARY_SIZE) | LIST_SHARED_UNPROCESSED; + CDR(ptr) = THE_NON_VALUE; + } + goto pop_next; + } + /* else make it visited now */ + switch (primary_tag(tail)) { + case TAG_PRIMARY_LIST: + VERBOSE_DEBUG("/L"); + VERBOSE_DEBUG("[pid=%T] mangling L/L %p\n", myself->common.id, ptr); + CDR(ptr) = (tail - TAG_PRIMARY_LIST) | TAG_PRIMARY_HEADER; + break; + case TAG_PRIMARY_IMMED1: + VERBOSE_DEBUG("/I"); + VERBOSE_DEBUG("[pid=%T] mangling L/I %p\n", myself->common.id, ptr); + CAR(ptr) = (head - primary_tag(head)) | TAG_PRIMARY_HEADER; + CDR(ptr) = (tail - TAG_PRIMARY_IMMED1) | primary_tag(head); + break; + case TAG_PRIMARY_BOXED: + VERBOSE_DEBUG("/B saved %d", primary_tag(head)); + BITSTORE_PUT(b, primary_tag(head)); + VERBOSE_DEBUG("[pid=%T] mangling L/B %p\n", myself->common.id, ptr); + CAR(ptr) = (head - primary_tag(head)) | TAG_PRIMARY_HEADER; + CDR(ptr) = (tail - TAG_PRIMARY_BOXED) | TAG_PRIMARY_HEADER; + break; + } + /* and count it */ + sum += 2; + if (!IS_CONST(head)) { + EQUEUE_PUT(s, head); + } + obj = tail; + break; + } + case TAG_PRIMARY_BOXED: { + Eterm hdr; + VERBOSE_DEBUG("B"); + ptr = boxed_val_rel(obj, base); + /* off heap pointers to boxes are copied verbatim */ + if (!INHEAP(myself, ptr)) { + VERBOSE_DEBUG("[pid=%T] bypassed copying %p is %T\n", myself->common.id, ptr, obj); + VERBOSE_DEBUG("#"); + goto pop_next; + } + hdr = *ptr; + /* if it's visited, don't count it; + if not already shared, make it shared and store it in the table */ + if (primary_tag(hdr) != TAG_PRIMARY_HEADER) { + VERBOSE_DEBUG("!"); + if (primary_tag(hdr) == BOXED_VISITED) { + e = SHTABLE_NEXT(t); + VERBOSE_DEBUG("[pid=%T] tabling B %p\n", myself->common.id, ptr); + SHTABLE_PUSH(t, hdr, THE_NON_VALUE, ptr); + *ptr = (e << _TAG_PRIMARY_SIZE) | BOXED_SHARED_UNPROCESSED; + } + goto pop_next; + } + /* else make it visited now */ + VERBOSE_DEBUG("[pid=%T] mangling B %p\n", myself->common.id, ptr); + *ptr = (hdr - primary_tag(hdr)) + BOXED_VISITED; + /* and count it */ + ASSERT(is_header(hdr)); + switch (hdr & _TAG_HEADER_MASK) { + case ARITYVAL_SUBTAG: { + int arity = header_arity(hdr); + VERBOSE_DEBUG("/T"); + sum += arity + 1; + if (arity == 0) { /* Empty tuple -- unusual. */ + VERBOSE_DEBUG("e"); + goto pop_next; + } + while (arity-- > 0) { + obj = *++ptr; + if (!IS_CONST(obj)) { + EQUEUE_PUT(s, obj); + } + } + goto pop_next; + } + case FUN_SUBTAG: { + ErlFunThing* funp = (ErlFunThing *) ptr; + unsigned eterms = 1 /* creator */ + funp->num_free; + sz = thing_arityval(hdr); + VERBOSE_DEBUG("/F"); + sum += 1 /* header */ + sz + eterms; + ptr += 1 /* header */ + sz; + while (eterms-- > 0) { + obj = *ptr++; + if (!IS_CONST(obj)) { + EQUEUE_PUT(s, obj); + } + } + goto pop_next; + } + case SUB_BINARY_SUBTAG: { + ErlSubBin* sb = (ErlSubBin *) ptr; + Eterm real_bin = sb->orig; + Uint bit_offset = sb->bitoffs; + Uint bit_size = sb->bitsize; + size_t size = sb->size; + Uint extra_bytes; + Eterm hdr; + if (bit_size + bit_offset > 8) { + sum += ERL_SUB_BIN_SIZE; + extra_bytes = 2; + } else if (bit_size + bit_offset > 0) { + sum += ERL_SUB_BIN_SIZE; + extra_bytes = 1; + } else { + extra_bytes = 0; + } + ASSERT(is_boxed(rterm2wterm(real_bin, base)) && + (((*boxed_val(rterm2wterm(real_bin, base))) & + (_TAG_HEADER_MASK - _BINARY_XXX_MASK - BOXED_VISITED_MASK)) + == _TAG_HEADER_REFC_BIN)); + hdr = *_unchecked_binary_val(rterm2wterm(real_bin, base)) & ~BOXED_VISITED_MASK; + if (thing_subtag(hdr) == HEAP_BINARY_SUBTAG) { + sum += heap_bin_size(size+extra_bytes); + } else { + ASSERT(thing_subtag(hdr) == REFC_BINARY_SUBTAG); + sum += PROC_BIN_SIZE; + } + goto pop_next; + } + case BIN_MATCHSTATE_SUBTAG: + erl_exit(ERTS_ABORT_EXIT, + "size_shared: matchstate term not allowed"); + default: + VERBOSE_DEBUG("/D"); + sum += thing_arityval(hdr) + 1; + goto pop_next; + } + break; + } + case TAG_PRIMARY_IMMED1: + VERBOSE_DEBUG("I"); + pop_next: + if (EQUEUE_ISEMPTY(s)) { + VERBOSE_DEBUG("\n"); + // add sentinel to the table + SHTABLE_PUSH(t, THE_NON_VALUE, THE_NON_VALUE, NULL); + // store persistent info + BITSTORE_CLOSE(b); + info->queue_start = s.start; + info->queue_end = s.end; + info->queue_alloc_type = s.alloc_type; + info->bitstore_start = b.wstart; + info->bitstore_alloc_type = b.alloc_type; + info->shtable_start = t.start; + info->shtable_alloc_type = t.alloc_type; + // single point of return: the size of the object + VERBOSE_DEBUG("[pid=%T] size was: %u\n", myself->common.id, sum); + return sum; + } + obj = EQUEUE_GET(s); + break; + default: + erl_exit(ERTS_ABORT_EXIT, "[pid=%T] size_shared: bad tag for %#x\n", obj); + } + VERBOSE_DEBUG("\n"); + } +} + + +/* + * Copy object "obj" preserving sharing. + * Second half: copy and restore the object. + * NOTE: We do not support HALF_WORD (yet?). + */ +Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, ErlOffHeap* off_heap, unsigned flags) +{ + Uint e; + unsigned sz; + Eterm* ptr; + Eterm* hp; + Eterm* hscan; + Eterm result; + Eterm* resp; + unsigned remaining; + Process* myself; + int force_local = flags & ERTS_SHCOPY_FLG_TMP_BUF; +#if defined(DEBUG) || defined(SHCOPY_DEBUG) + Eterm saved_obj = obj; +#endif + + DECLARE_EQUEUE_FROM_INFO(s, info); + DECLARE_BITSTORE_FROM_INFO(b, info); + DECLARE_SHTABLE_FROM_INFO(t, info); + + /* step #0: + ------------------------------------------------------- + get rid of the easy cases first: + - copying constants + - if not a proper process, do flat copy + */ + + if (IS_CONST(obj)) + return obj; + + myself = erts_get_current_process(); + if (myself == NULL || (flags & ERTS_SHCOPY_FLG_NONE)) + return copy_struct(obj, size, hpp, off_heap); + + VERBOSE_DEBUG("[pid=%T] copy_shared_perform %p\n", myself->common.id, obj); + + /* step #2: was performed before this function was called + ------------------------------------------------------- + allocate new space + */ + + hscan = hp = *hpp; + + /* step #3: + ------------------------------------------------------- + traverse the term a second time and when traversing: + a. if the object is marked as shared + a1. if the entry contains a forwarding ptr, use that + a2. otherwise, copy it to the new space and store the + forwarding ptr to the entry + b. otherwise, reverse-transform as you do in size_shared + and copy to the new space + */ + + resp = &result; + remaining = 0; + for (;;) { + VERBOSE_DEBUG("[copy] revisiting: %x ", obj); + switch (primary_tag(obj)) { + case TAG_PRIMARY_LIST: { + Eterm head, tail; + VERBOSE_DEBUG("L"); + ptr = list_val_rel(obj, base); + /* off heap list pointers are copied verbatim */ + if (!INHEAP(myself, ptr)) { + VERBOSE_DEBUG("#"); + *resp = obj; + goto cleanup_next; + } + head = CAR(ptr); + tail = CDR(ptr); + /* if it is shared */ + if (tail == THE_NON_VALUE) { + e = head >> _TAG_PRIMARY_SIZE; + /* if it has been processed, just use the forwarding pointer */ + if (primary_tag(head) == LIST_SHARED_PROCESSED) { + VERBOSE_DEBUG("!"); + *resp = make_list(SHTABLE_FWD(t, e)); + goto cleanup_next; + } + /* else, let's process it now, + copy it and keep the forwarding pointer */ + else { + VERBOSE_DEBUG("$"); + CAR(ptr) = (head - primary_tag(head)) + LIST_SHARED_PROCESSED; + head = SHTABLE_X(t, e); + tail = SHTABLE_Y(t, e); + ptr = &(SHTABLE_X(t, e)); + VERBOSE_DEBUG("[pid=%T] tabled L %p is %p\n", myself->common.id, ptr, SHTABLE_REV(t, e)); + SHTABLE_FWD_UPD(t, e, hp); + } + } + /* if not already clean, clean it up and copy it */ + if (primary_tag(tail) == TAG_PRIMARY_HEADER) { + if (primary_tag(head) == TAG_PRIMARY_HEADER) { + Eterm saved = BITSTORE_GET(b); + VERBOSE_DEBUG("/B restoring %d", saved); + VERBOSE_DEBUG("[pid=%T] unmangling L/B %p\n", myself->common.id, ptr); + CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) + saved; + CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) + TAG_PRIMARY_BOXED; + } else { + VERBOSE_DEBUG("/L"); + VERBOSE_DEBUG("[pid=%T] unmangling L/L %p\n", myself->common.id, ptr); + CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) + TAG_PRIMARY_LIST; + } + } else if (primary_tag(head) == TAG_PRIMARY_HEADER) { + VERBOSE_DEBUG("/I"); + VERBOSE_DEBUG("[pid=%T] unmangling L/I %p\n", myself->common.id, ptr); + CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) | primary_tag(tail); + CDR(ptr) = tail = (tail - primary_tag(tail)) | TAG_PRIMARY_IMMED1; + } else { + ASSERT(0 && "cannot come here"); + goto cleanup_next; + } + /* and its children too */ + if (IS_CONST(head)) { + CAR(hp) = head; + } else { + EQUEUE_PUT_UNCHECKED(s, head); + CAR(hp) = HEAP_ELEM_TO_BE_FILLED; + } + *resp = make_list(hp); + resp = &(CDR(hp)); + hp += 2; + obj = tail; + break; + } + case TAG_PRIMARY_BOXED: { + Eterm hdr; + VERBOSE_DEBUG("B"); + ptr = boxed_val_rel(obj, base); + /* off heap pointers to boxes are copied verbatim */ + if (!INHEAP(myself, ptr)) { + VERBOSE_DEBUG("#"); + *resp = obj; + goto cleanup_next; + } + hdr = *ptr; + /* clean it up, unless it's already clean or shared and processed */ + switch (primary_tag(hdr)) { + case TAG_PRIMARY_HEADER: + ASSERT(0 && "cannot come here"); + /* if it is shared and has been processed, + just use the forwarding pointer */ + case BOXED_SHARED_PROCESSED: + VERBOSE_DEBUG("!"); + e = hdr >> _TAG_PRIMARY_SIZE; + *resp = make_boxed(SHTABLE_FWD(t, e)); + goto cleanup_next; + /* if it is shared but has not been processed yet, let's process + it now: copy it and keep the forwarding pointer */ + case BOXED_SHARED_UNPROCESSED: + e = hdr >> _TAG_PRIMARY_SIZE; + VERBOSE_DEBUG("$"); + *ptr = (hdr - primary_tag(hdr)) + BOXED_SHARED_PROCESSED; + hdr = SHTABLE_X(t, e); + ASSERT(primary_tag(hdr) == BOXED_VISITED); + VERBOSE_DEBUG("[pid=%T] tabled B %p is %p\n", myself->common.id, ptr, SHTABLE_REV(t, e)); + VERBOSE_DEBUG("[pid=%T] unmangling B %p\n", myself->common.id, ptr); + SHTABLE_X(t, e) = hdr = (hdr - BOXED_VISITED) + TAG_PRIMARY_HEADER; + SHTABLE_FWD_UPD(t, e, hp); + break; + case BOXED_VISITED: + VERBOSE_DEBUG("[pid=%T] unmangling B %p\n", myself->common.id, ptr); + *ptr = hdr = (hdr - BOXED_VISITED) + TAG_PRIMARY_HEADER; + break; + } + /* and its children too */ + switch (hdr & _TAG_HEADER_MASK) { + case ARITYVAL_SUBTAG: { + int arity = header_arity(hdr); + *resp = make_boxed(hp); + *hp++ = hdr; + while (arity-- > 0) { + obj = *++ptr; + if (IS_CONST(obj)) { + *hp++ = obj; + } else { + EQUEUE_PUT_UNCHECKED(s, obj); + *hp++ = HEAP_ELEM_TO_BE_FILLED; + } + } + goto cleanup_next; + } + case FUN_SUBTAG: { + ErlFunThing* funp = (ErlFunThing *) ptr; + unsigned eterms = 1 /* creator */ + funp->num_free; + sz = thing_arityval(hdr); + funp = (ErlFunThing *) hp; + *resp = make_fun(hp); + *hp++ = hdr; + ptr++; + while (sz-- > 0) { + *hp++ = *ptr++; + } + while (eterms-- > 0) { + obj = *ptr++; + if (IS_CONST(obj)) { + *hp++ = obj; + } else { + EQUEUE_PUT_UNCHECKED(s, obj); + *hp++ = HEAP_ELEM_TO_BE_FILLED; + } + } + funp->next = off_heap->first; + off_heap->first = (struct erl_off_heap_header*) funp; + erts_refc_inc(&funp->fe->refc, 2); + goto cleanup_next; + } + case REFC_BINARY_SUBTAG: { + ProcBin* pb = (ProcBin *) ptr; + sz = thing_arityval(hdr); + if (pb->flags) { + erts_emasculate_writable_binary(pb); + } + pb = (ProcBin *) hp; + *resp = make_binary(hp); + *hp++ = hdr; + ptr++; + while (sz-- > 0) { + *hp++ = *ptr++; + } + erts_refc_inc(&pb->val->refc, 2); + pb->next = off_heap->first; + pb->flags = 0; + off_heap->first = (struct erl_off_heap_header*) pb; + OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm)); + goto cleanup_next; + } + case SUB_BINARY_SUBTAG: { + ErlSubBin* sb = (ErlSubBin *) ptr; + Eterm real_bin = sb->orig; + Uint bit_offset = sb->bitoffs; + Uint bit_size = sb->bitsize; + Uint offset = sb->offs; + size_t size = sb->size; + Uint extra_bytes; + Uint real_size; + if ((bit_size + bit_offset) > 8) { + extra_bytes = 2; + } else if ((bit_size + bit_offset) > 0) { + extra_bytes = 1; + } else { + extra_bytes = 0; + } + real_size = size+extra_bytes; + ASSERT(is_boxed(rterm2wterm(real_bin, base)) && + (((*boxed_val(rterm2wterm(real_bin, base))) & + (_TAG_HEADER_MASK - _BINARY_XXX_MASK - BOXED_VISITED_MASK)) + == _TAG_HEADER_REFC_BIN)); + ptr = _unchecked_binary_val(rterm2wterm(real_bin, base)); + *resp = make_binary(hp); + if (extra_bytes != 0) { + ErlSubBin* res = (ErlSubBin *) hp; + hp += ERL_SUB_BIN_SIZE; + res->thing_word = HEADER_SUB_BIN; + res->size = size; + res->bitsize = bit_size; + res->bitoffs = bit_offset; + res->offs = 0; + res->is_writable = 0; + res->orig = make_binary(hp); + } + if (thing_subtag(*ptr & ~BOXED_VISITED_MASK) == HEAP_BINARY_SUBTAG) { + ErlHeapBin* from = (ErlHeapBin *) ptr; + ErlHeapBin* to = (ErlHeapBin *) hp; + hp += heap_bin_size(real_size); + to->thing_word = header_heap_bin(real_size); + to->size = real_size; + sys_memcpy(to->data, ((byte *)from->data)+offset, real_size); + } else { + ProcBin* from = (ProcBin *) ptr; + ProcBin* to = (ProcBin *) hp; + ASSERT(thing_subtag(*ptr & ~BOXED_VISITED_MASK) == REFC_BINARY_SUBTAG); + if (from->flags) { + erts_emasculate_writable_binary(from); + } + hp += PROC_BIN_SIZE; + to->thing_word = HEADER_PROC_BIN; + to->size = real_size; + to->val = from->val; + erts_refc_inc(&to->val->refc, 2); + to->bytes = from->bytes + offset; + to->next = off_heap->first; + to->flags = 0; + off_heap->first = (struct erl_off_heap_header*) to; + OH_OVERHEAD(off_heap, to->size / sizeof(Eterm)); + } + goto cleanup_next; + } + case EXTERNAL_PID_SUBTAG: + case EXTERNAL_PORT_SUBTAG: + case EXTERNAL_REF_SUBTAG: { + ExternalThing *etp = (ExternalThing *) hp; + sz = thing_arityval(hdr); + *resp = make_external(hp); + *hp++ = hdr; + ptr++; + while (sz-- > 0) { + *hp++ = *ptr++; + } + etp->next = off_heap->first; + off_heap->first = (struct erl_off_heap_header*) etp; + erts_refc_inc(&etp->node->refc, 2); + goto cleanup_next; + } + default: + sz = thing_arityval(hdr); + *resp = make_boxed(hp); + *hp++ = hdr; + ptr++; + while (sz-- > 0) { + *hp++ = *ptr++; + } + goto cleanup_next; + } + break; + } + case TAG_PRIMARY_IMMED1: + *resp = obj; + cleanup_next: + if (EQUEUE_ISEMPTY(s)) { + goto all_clean; + } + obj = EQUEUE_GET(s); + for (;;) { + ASSERT(hscan < hp); + if (remaining == 0) { + if (*hscan == HEAP_ELEM_TO_BE_FILLED) { + resp = hscan; + hscan += 2; + break; /* scanning loop */ + } else if (primary_tag(*hscan) == TAG_PRIMARY_HEADER) { + switch (*hscan & _TAG_HEADER_MASK) { + case ARITYVAL_SUBTAG: + remaining = header_arity(*hscan); + hscan++; + break; + case FUN_SUBTAG: { + ErlFunThing* funp = (ErlFunThing *) hscan; + hscan += 1 + thing_arityval(*hscan); + remaining = 1 + funp->num_free; + break; + } + case SUB_BINARY_SUBTAG: + ASSERT(((ErlSubBin *) hscan)->bitoffs + + ((ErlSubBin *) hscan)->bitsize > 0); + hscan += ERL_SUB_BIN_SIZE; + break; + default: + hscan += 1 + thing_arityval(*hscan); + break; + } + } else { + hscan++; + } + } else if (*hscan == HEAP_ELEM_TO_BE_FILLED) { + resp = hscan++; + remaining--; + break; /* scanning loop */ + } else { + hscan++; + remaining--; + } + } + ASSERT(resp < hp); + break; + default: + erl_exit(ERTS_ABORT_EXIT, "size_shared: bad tag for %#x\n", obj); + } + VERBOSE_DEBUG("\n"); + } + + /* step #4: + ------------------------------------------------------- + traverse the table and reverse-transform all stored entries + */ + +all_clean: + VERBOSE_DEBUG("\n"); + for (e = 0; ; e += SHTABLE_INCR) { + ptr = SHTABLE_REV(t, e); + if (ptr == NULL) + break; + VERBOSE_DEBUG("[copy] restoring shared: %x\n", ptr); + /* entry was a list */ + if (SHTABLE_Y(t, e) != THE_NON_VALUE) { + VERBOSE_DEBUG("[pid=%T] untabling L %p\n", myself->common.id, ptr); + CAR(ptr) = SHTABLE_X(t, e); + CDR(ptr) = SHTABLE_Y(t, e); + } + /* entry was boxed */ + else { + VERBOSE_DEBUG("[pid=%T] untabling B %p\n", myself->common.id, ptr); + *ptr = SHTABLE_X(t, e); + ASSERT(primary_tag(*ptr) == TAG_PRIMARY_HEADER); + } + } + +#ifdef DEBUG + if (eq(saved_obj, result) == 0) { + erts_fprintf(stderr, "original = %T\n", saved_obj); + erts_fprintf(stderr, "copy = %T\n", result); + erl_exit(ERTS_ABORT_EXIT, "copy (shared) not equal to source\n"); + } +#endif + + VERBOSE_DEBUG("[pid=%T] original was %T\n", myself->common.id, saved_obj); + VERBOSE_DEBUG("[pid=%T] copy is %T\n", myself->common.id, result); + VERBOSE_DEBUG("[pid=%T] result is at %p\n", myself->common.id, result); + + ASSERT(hp == *hpp + size); + *hpp = hp; + return result; +} + + /* * Copy a term that is guaranteed to be contained in a single * heap block. The heap block is copied word by word, and any -- cgit v1.2.3 From 8d02ad60c88aa060ff83ff3179bc8c0ab66868ee Mon Sep 17 00:00:00 2001 From: "Nikolaos S. Papaspyrou" Date: Fri, 14 Mar 2014 11:32:46 +0200 Subject: Add machinery to enable SHCOPY dynamically This commit is just for debugging purposes, will probably be reverted. It comes with a the erts_debug:copy_shared/1 BIF. If SHCOPY_DISABLE is defined, SHCOPY starts disabled and is dynamically enabled the first time that the BIF is called. --- erts/emulator/beam/copy.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'erts/emulator/beam/copy.c') diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index b31e043f08..2566707717 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -901,6 +901,10 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) * Using an ESTACK but not very transparently; consider refactoring */ +#ifdef SHCOPY_DISABLE +int disable_copy_shared = ERTS_SHCOPY_FLG_NONE; +#endif + #define DECLARE_SHTABLE(s) \ DECLARE_ESTACK(s); \ Uint ESTK_CONCAT(s,_offset) = 0 @@ -1028,6 +1032,10 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) if (IS_CONST(obj)) return 0; +#ifdef SHCOPY_DISABLE + flags |= disable_copy_shared; +#endif + myself = erts_get_current_process(); if (myself == NULL || (flags & ERTS_SHCOPY_FLG_NONE)) return size_object(obj); @@ -1276,6 +1284,10 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E if (IS_CONST(obj)) return obj; +#ifdef SHCOPY_DISABLE + flags |= disable_copy_shared; +#endif + myself = erts_get_current_process(); if (myself == NULL || (flags & ERTS_SHCOPY_FLG_NONE)) return copy_struct(obj, size, hpp, off_heap); -- cgit v1.2.3 From 277e8e77384ed6628009243e63d62f0555d10c69 Mon Sep 17 00:00:00 2001 From: "Nikolaos S. Papaspyrou" Date: Mon, 11 Jun 2012 15:17:01 +0300 Subject: Add -debug +vc flag for debuging SHCOPY This is very verbose, you have been warned. It should work with the copy-spy.py script, which may be a bit outdated. --- erts/emulator/beam/copy.c | 131 ++++++++++++---------------------------------- 1 file changed, 33 insertions(+), 98 deletions(-) (limited to 'erts/emulator/beam/copy.c') diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index 2566707717..f74c6b1c89 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -75,16 +75,13 @@ Uint size_object(Eterm obj) Uint sum = 0; Eterm* ptr; int arity; -#ifdef SHCOPY_DEBUG - Eterm mypid; +#ifdef DEBUG + Eterm mypid = erts_get_current_pid(); #endif DECLARE_ESTACK(s); -#ifdef SHCOPY_DEBUG - mypid = erts_get_current_pid(); - VERBOSE_DEBUG("[pid=%T] size_object %p\n", mypid, obj); -#endif + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] size_object %p\n", mypid, obj)); for (;;) { switch (primary_tag(obj)) { @@ -220,7 +217,7 @@ Uint size_object(Eterm obj) pop_next: if (ESTACK_ISEMPTY(s)) { DESTROY_ESTACK(s); - VERBOSE_DEBUG("[pid=%T] size was: %u\n", mypid, sum); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] size was: %u\n", mypid, sum)); return sum; } obj = ESTACK_POP(s); @@ -320,11 +317,9 @@ Uint size_shared(Eterm obj) return size_object(obj); for (;;) { - VERBOSE_DEBUG("[size] visiting: %x ", obj); switch (primary_tag(obj)) { case TAG_PRIMARY_LIST: { Eterm head, tail; - VERBOSE_DEBUG("L"); ptr = list_val_rel(obj, base); /* we're not counting anything that's outside our heap */ if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) { @@ -335,22 +330,18 @@ Uint size_shared(Eterm obj) /* if it's visited, don't count it */ if (primary_tag(tail) == TAG_PRIMARY_HEADER || primary_tag(head) == TAG_PRIMARY_HEADER) { - VERBOSE_DEBUG("!"); goto pop_next; } /* else make it visited now */ switch (primary_tag(tail)) { case TAG_PRIMARY_LIST: - VERBOSE_DEBUG("/L"); ptr[1] = (tail - TAG_PRIMARY_LIST) | TAG_PRIMARY_HEADER; break; case TAG_PRIMARY_IMMED1: - VERBOSE_DEBUG("/I"); CAR(ptr) = (head - primary_tag(head)) | TAG_PRIMARY_HEADER; CDR(ptr) = (tail - TAG_PRIMARY_IMMED1) | primary_tag(head); break; case TAG_PRIMARY_BOXED: - VERBOSE_DEBUG("/B saved %d", primary_tag(head)); BITSTORE_PUT(b, primary_tag(head)); CAR(ptr) = (head - primary_tag(head)) | TAG_PRIMARY_HEADER; CDR(ptr) = (tail - TAG_PRIMARY_BOXED) | TAG_PRIMARY_HEADER; @@ -366,7 +357,6 @@ Uint size_shared(Eterm obj) } case TAG_PRIMARY_BOXED: { Eterm hdr; - VERBOSE_DEBUG("B"); ptr = boxed_val_rel(obj, base); /* we're not counting anything that's outside our heap */ if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) { @@ -375,7 +365,6 @@ Uint size_shared(Eterm obj) hdr = *ptr; /* if it's visited, don't count it */ if (primary_tag(hdr) != TAG_PRIMARY_HEADER) { - VERBOSE_DEBUG("!"); goto pop_next; } /* else make it visited now */ @@ -385,10 +374,8 @@ Uint size_shared(Eterm obj) switch (hdr & _TAG_HEADER_MASK) { case ARITYVAL_SUBTAG: { int arity = header_arity(hdr); - VERBOSE_DEBUG("/T"); sum += arity + 1; if (arity == 0) { /* Empty tuple -- unusual. */ - VERBOSE_DEBUG("e"); goto pop_next; } while (arity-- > 0) { @@ -403,7 +390,6 @@ Uint size_shared(Eterm obj) ErlFunThing* funp = (ErlFunThing *) ptr; unsigned eterms = 1 /* creator */ + funp->num_free; unsigned sz = thing_arityval(hdr); - VERBOSE_DEBUG("/F"); sum += 1 /* header */ + sz + eterms; ptr += 1 /* header */ + sz; while (eterms-- > 0) { @@ -442,14 +428,12 @@ Uint size_shared(Eterm obj) erl_exit(ERTS_ABORT_EXIT, "size_shared: matchstate term not allowed"); default: - VERBOSE_DEBUG("/D"); sum += thing_arityval(hdr) + 1; goto pop_next; } break; } case TAG_PRIMARY_IMMED1: - VERBOSE_DEBUG("I"); pop_next: if (EQUEUE_ISEMPTY(s)) { goto cleanup; @@ -459,19 +443,15 @@ Uint size_shared(Eterm obj) default: erl_exit(ERTS_ABORT_EXIT, "size_shared: bad tag for %#x\n", obj); } - VERBOSE_DEBUG("\n"); } cleanup: - VERBOSE_DEBUG("\n"); obj = saved_obj; BITSTORE_CLOSE(b); for (;;) { - VERBOSE_DEBUG("[size] revisiting: %x ", obj); switch (primary_tag(obj)) { case TAG_PRIMARY_LIST: { Eterm head, tail; - VERBOSE_DEBUG("L"); ptr = list_val_rel(obj, base); if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) { goto cleanup_next; @@ -482,19 +462,15 @@ cleanup: if (primary_tag(tail) == TAG_PRIMARY_HEADER) { if (primary_tag(head) == TAG_PRIMARY_HEADER) { Eterm saved = BITSTORE_GET(b); - VERBOSE_DEBUG("/B restoring %d", saved); CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) | saved; CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) | TAG_PRIMARY_BOXED; } else { - VERBOSE_DEBUG("/L"); CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) | TAG_PRIMARY_LIST; } } else if (primary_tag(head) == TAG_PRIMARY_HEADER) { - VERBOSE_DEBUG("/I"); CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) | primary_tag(tail); CDR(ptr) = tail = (tail - primary_tag(tail)) | TAG_PRIMARY_IMMED1; } else { - VERBOSE_DEBUG("!"); goto cleanup_next; } /* and its children too */ @@ -506,7 +482,6 @@ cleanup: } case TAG_PRIMARY_BOXED: { Eterm hdr; - VERBOSE_DEBUG("B"); ptr = boxed_val_rel(obj, base); if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) { goto cleanup_next; @@ -563,11 +538,9 @@ cleanup: default: erl_exit(ERTS_ABORT_EXIT, "size_shared: bad tag for %#x\n", obj); } - VERBOSE_DEBUG("\n"); } all_clean: - VERBOSE_DEBUG("\n"); /* Return the result */ DESTROY_EQUEUE(s); DESTROY_BITSTORE(b); @@ -597,18 +570,13 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) #ifdef DEBUG Eterm org_obj = obj; Uint org_sz = sz; -#endif -#ifdef SHCOPY_DEBUG - Eterm mypid; + Eterm mypid = erts_get_current_pid(); #endif if (IS_CONST(obj)) return obj; -#ifdef SHCOPY_DEBUG - mypid = erts_get_current_pid(); - VERBOSE_DEBUG("[pid=%T] copy_struct %p\n", mypid, obj); -#endif + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy_struct %p\n", mypid, obj)); DTRACE1(copy_struct, (int32_t)sz); @@ -891,7 +859,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) } #endif *hpp = (Eterm *) (hstart+hsize); - VERBOSE_DEBUG("[pid=%T] result is at %p\n", mypid, res); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] result is at %p\n", mypid, res)); return res; } @@ -1040,8 +1008,8 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) if (myself == NULL || (flags & ERTS_SHCOPY_FLG_NONE)) return size_object(obj); - VERBOSE_DEBUG("[pid=%T] copy_shared_calculate %p\n", myself->common.id, obj); - VERBOSE_DEBUG("[pid=%T] message is %T\n", myself->common.id, obj); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy_shared_calculate %p\n", myself->common.id, obj)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] message is %T\n", myself->common.id, obj)); /* step #1: ------------------------------------------------------- @@ -1060,18 +1028,13 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) sum = 0; for (;;) { - VERBOSE_DEBUG("[copy] visiting: %x ", obj); switch (primary_tag(obj)) { case TAG_PRIMARY_LIST: { Eterm head, tail; - VERBOSE_DEBUG("L"); ptr = list_val_rel(obj, base); /* off heap list pointers are copied verbatim */ if (!INHEAP(myself, ptr)) { - VERBOSE_DEBUG("[pid=%T] bypassed copying %p is %T\n", myself->common.id, ptr, obj); - if (myself->mbuf != NULL) - VERBOSE_DEBUG("[pid=%T] BUT !!! there are message buffers!\n", myself->common.id); - VERBOSE_DEBUG("#"); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", myself->common.id, ptr, obj)); goto pop_next; } head = CAR(ptr); @@ -1080,10 +1043,9 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) if not already shared, make it shared and store it in the table */ if (primary_tag(tail) == TAG_PRIMARY_HEADER || primary_tag(head) == TAG_PRIMARY_HEADER) { - VERBOSE_DEBUG("!"); if (tail != THE_NON_VALUE) { e = SHTABLE_NEXT(t); - VERBOSE_DEBUG("[pid=%T] tabling L %p\n", myself->common.id, ptr); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] tabling L %p\n", myself->common.id, ptr)); SHTABLE_PUSH(t, head, tail, ptr); CAR(ptr) = (e << _TAG_PRIMARY_SIZE) | LIST_SHARED_UNPROCESSED; CDR(ptr) = THE_NON_VALUE; @@ -1093,20 +1055,17 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) /* else make it visited now */ switch (primary_tag(tail)) { case TAG_PRIMARY_LIST: - VERBOSE_DEBUG("/L"); - VERBOSE_DEBUG("[pid=%T] mangling L/L %p\n", myself->common.id, ptr); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] mangling L/L %p\n", myself->common.id, ptr)); CDR(ptr) = (tail - TAG_PRIMARY_LIST) | TAG_PRIMARY_HEADER; break; case TAG_PRIMARY_IMMED1: - VERBOSE_DEBUG("/I"); - VERBOSE_DEBUG("[pid=%T] mangling L/I %p\n", myself->common.id, ptr); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] mangling L/I %p\n", myself->common.id, ptr)); CAR(ptr) = (head - primary_tag(head)) | TAG_PRIMARY_HEADER; CDR(ptr) = (tail - TAG_PRIMARY_IMMED1) | primary_tag(head); break; case TAG_PRIMARY_BOXED: - VERBOSE_DEBUG("/B saved %d", primary_tag(head)); BITSTORE_PUT(b, primary_tag(head)); - VERBOSE_DEBUG("[pid=%T] mangling L/B %p\n", myself->common.id, ptr); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] mangling L/B %p\n", myself->common.id, ptr)); CAR(ptr) = (head - primary_tag(head)) | TAG_PRIMARY_HEADER; CDR(ptr) = (tail - TAG_PRIMARY_BOXED) | TAG_PRIMARY_HEADER; break; @@ -1121,39 +1080,34 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) } case TAG_PRIMARY_BOXED: { Eterm hdr; - VERBOSE_DEBUG("B"); ptr = boxed_val_rel(obj, base); /* off heap pointers to boxes are copied verbatim */ if (!INHEAP(myself, ptr)) { - VERBOSE_DEBUG("[pid=%T] bypassed copying %p is %T\n", myself->common.id, ptr, obj); - VERBOSE_DEBUG("#"); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", myself->common.id, ptr, obj)); goto pop_next; } hdr = *ptr; /* if it's visited, don't count it; if not already shared, make it shared and store it in the table */ if (primary_tag(hdr) != TAG_PRIMARY_HEADER) { - VERBOSE_DEBUG("!"); if (primary_tag(hdr) == BOXED_VISITED) { e = SHTABLE_NEXT(t); - VERBOSE_DEBUG("[pid=%T] tabling B %p\n", myself->common.id, ptr); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] tabling B %p\n", myself->common.id, ptr)); SHTABLE_PUSH(t, hdr, THE_NON_VALUE, ptr); *ptr = (e << _TAG_PRIMARY_SIZE) | BOXED_SHARED_UNPROCESSED; } goto pop_next; } /* else make it visited now */ - VERBOSE_DEBUG("[pid=%T] mangling B %p\n", myself->common.id, ptr); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] mangling B %p\n", myself->common.id, ptr)); *ptr = (hdr - primary_tag(hdr)) + BOXED_VISITED; /* and count it */ ASSERT(is_header(hdr)); switch (hdr & _TAG_HEADER_MASK) { case ARITYVAL_SUBTAG: { int arity = header_arity(hdr); - VERBOSE_DEBUG("/T"); sum += arity + 1; if (arity == 0) { /* Empty tuple -- unusual. */ - VERBOSE_DEBUG("e"); goto pop_next; } while (arity-- > 0) { @@ -1168,7 +1122,6 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) ErlFunThing* funp = (ErlFunThing *) ptr; unsigned eterms = 1 /* creator */ + funp->num_free; sz = thing_arityval(hdr); - VERBOSE_DEBUG("/F"); sum += 1 /* header */ + sz + eterms; ptr += 1 /* header */ + sz; while (eterms-- > 0) { @@ -1213,17 +1166,14 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) erl_exit(ERTS_ABORT_EXIT, "size_shared: matchstate term not allowed"); default: - VERBOSE_DEBUG("/D"); sum += thing_arityval(hdr) + 1; goto pop_next; } break; } case TAG_PRIMARY_IMMED1: - VERBOSE_DEBUG("I"); pop_next: if (EQUEUE_ISEMPTY(s)) { - VERBOSE_DEBUG("\n"); // add sentinel to the table SHTABLE_PUSH(t, THE_NON_VALUE, THE_NON_VALUE, NULL); // store persistent info @@ -1236,7 +1186,7 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) info->shtable_start = t.start; info->shtable_alloc_type = t.alloc_type; // single point of return: the size of the object - VERBOSE_DEBUG("[pid=%T] size was: %u\n", myself->common.id, sum); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] size was: %u\n", myself->common.id, sum)); return sum; } obj = EQUEUE_GET(s); @@ -1244,7 +1194,6 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) default: erl_exit(ERTS_ABORT_EXIT, "[pid=%T] size_shared: bad tag for %#x\n", obj); } - VERBOSE_DEBUG("\n"); } } @@ -1266,7 +1215,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E unsigned remaining; Process* myself; int force_local = flags & ERTS_SHCOPY_FLG_TMP_BUF; -#if defined(DEBUG) || defined(SHCOPY_DEBUG) +#ifdef DEBUG Eterm saved_obj = obj; #endif @@ -1292,7 +1241,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E if (myself == NULL || (flags & ERTS_SHCOPY_FLG_NONE)) return copy_struct(obj, size, hpp, off_heap); - VERBOSE_DEBUG("[pid=%T] copy_shared_perform %p\n", myself->common.id, obj); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy_shared_perform %p\n", myself->common.id, obj)); /* step #2: was performed before this function was called ------------------------------------------------------- @@ -1315,15 +1264,12 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E resp = &result; remaining = 0; for (;;) { - VERBOSE_DEBUG("[copy] revisiting: %x ", obj); switch (primary_tag(obj)) { case TAG_PRIMARY_LIST: { Eterm head, tail; - VERBOSE_DEBUG("L"); ptr = list_val_rel(obj, base); /* off heap list pointers are copied verbatim */ if (!INHEAP(myself, ptr)) { - VERBOSE_DEBUG("#"); *resp = obj; goto cleanup_next; } @@ -1334,19 +1280,17 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E e = head >> _TAG_PRIMARY_SIZE; /* if it has been processed, just use the forwarding pointer */ if (primary_tag(head) == LIST_SHARED_PROCESSED) { - VERBOSE_DEBUG("!"); *resp = make_list(SHTABLE_FWD(t, e)); goto cleanup_next; } /* else, let's process it now, copy it and keep the forwarding pointer */ else { - VERBOSE_DEBUG("$"); CAR(ptr) = (head - primary_tag(head)) + LIST_SHARED_PROCESSED; head = SHTABLE_X(t, e); tail = SHTABLE_Y(t, e); ptr = &(SHTABLE_X(t, e)); - VERBOSE_DEBUG("[pid=%T] tabled L %p is %p\n", myself->common.id, ptr, SHTABLE_REV(t, e)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] tabled L %p is %p\n", myself->common.id, ptr, SHTABLE_REV(t, e))); SHTABLE_FWD_UPD(t, e, hp); } } @@ -1354,18 +1298,15 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E if (primary_tag(tail) == TAG_PRIMARY_HEADER) { if (primary_tag(head) == TAG_PRIMARY_HEADER) { Eterm saved = BITSTORE_GET(b); - VERBOSE_DEBUG("/B restoring %d", saved); - VERBOSE_DEBUG("[pid=%T] unmangling L/B %p\n", myself->common.id, ptr); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling L/B %p\n", myself->common.id, ptr)); CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) + saved; CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) + TAG_PRIMARY_BOXED; } else { - VERBOSE_DEBUG("/L"); - VERBOSE_DEBUG("[pid=%T] unmangling L/L %p\n", myself->common.id, ptr); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling L/L %p\n", myself->common.id, ptr)); CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) + TAG_PRIMARY_LIST; } } else if (primary_tag(head) == TAG_PRIMARY_HEADER) { - VERBOSE_DEBUG("/I"); - VERBOSE_DEBUG("[pid=%T] unmangling L/I %p\n", myself->common.id, ptr); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling L/I %p\n", myself->common.id, ptr)); CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) | primary_tag(tail); CDR(ptr) = tail = (tail - primary_tag(tail)) | TAG_PRIMARY_IMMED1; } else { @@ -1387,11 +1328,9 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E } case TAG_PRIMARY_BOXED: { Eterm hdr; - VERBOSE_DEBUG("B"); ptr = boxed_val_rel(obj, base); /* off heap pointers to boxes are copied verbatim */ if (!INHEAP(myself, ptr)) { - VERBOSE_DEBUG("#"); *resp = obj; goto cleanup_next; } @@ -1403,7 +1342,6 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E /* if it is shared and has been processed, just use the forwarding pointer */ case BOXED_SHARED_PROCESSED: - VERBOSE_DEBUG("!"); e = hdr >> _TAG_PRIMARY_SIZE; *resp = make_boxed(SHTABLE_FWD(t, e)); goto cleanup_next; @@ -1411,17 +1349,16 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E it now: copy it and keep the forwarding pointer */ case BOXED_SHARED_UNPROCESSED: e = hdr >> _TAG_PRIMARY_SIZE; - VERBOSE_DEBUG("$"); *ptr = (hdr - primary_tag(hdr)) + BOXED_SHARED_PROCESSED; hdr = SHTABLE_X(t, e); ASSERT(primary_tag(hdr) == BOXED_VISITED); - VERBOSE_DEBUG("[pid=%T] tabled B %p is %p\n", myself->common.id, ptr, SHTABLE_REV(t, e)); - VERBOSE_DEBUG("[pid=%T] unmangling B %p\n", myself->common.id, ptr); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] tabled B %p is %p\n", myself->common.id, ptr, SHTABLE_REV(t, e))); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling B %p\n", myself->common.id, ptr)); SHTABLE_X(t, e) = hdr = (hdr - BOXED_VISITED) + TAG_PRIMARY_HEADER; SHTABLE_FWD_UPD(t, e, hp); break; case BOXED_VISITED: - VERBOSE_DEBUG("[pid=%T] unmangling B %p\n", myself->common.id, ptr); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling B %p\n", myself->common.id, ptr)); *ptr = hdr = (hdr - BOXED_VISITED) + TAG_PRIMARY_HEADER; break; } @@ -1628,7 +1565,6 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E default: erl_exit(ERTS_ABORT_EXIT, "size_shared: bad tag for %#x\n", obj); } - VERBOSE_DEBUG("\n"); } /* step #4: @@ -1637,21 +1573,20 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E */ all_clean: - VERBOSE_DEBUG("\n"); for (e = 0; ; e += SHTABLE_INCR) { ptr = SHTABLE_REV(t, e); if (ptr == NULL) break; - VERBOSE_DEBUG("[copy] restoring shared: %x\n", ptr); + VERBOSE(DEBUG_SHCOPY, ("[copy] restoring shared: %x\n", ptr)); /* entry was a list */ if (SHTABLE_Y(t, e) != THE_NON_VALUE) { - VERBOSE_DEBUG("[pid=%T] untabling L %p\n", myself->common.id, ptr); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] untabling L %p\n", myself->common.id, ptr)); CAR(ptr) = SHTABLE_X(t, e); CDR(ptr) = SHTABLE_Y(t, e); } /* entry was boxed */ else { - VERBOSE_DEBUG("[pid=%T] untabling B %p\n", myself->common.id, ptr); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] untabling B %p\n", myself->common.id, ptr)); *ptr = SHTABLE_X(t, e); ASSERT(primary_tag(*ptr) == TAG_PRIMARY_HEADER); } @@ -1665,9 +1600,9 @@ all_clean: } #endif - VERBOSE_DEBUG("[pid=%T] original was %T\n", myself->common.id, saved_obj); - VERBOSE_DEBUG("[pid=%T] copy is %T\n", myself->common.id, result); - VERBOSE_DEBUG("[pid=%T] result is at %p\n", myself->common.id, result); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] original was %T\n", myself->common.id, saved_obj)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy is %T\n", myself->common.id, result)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] result is at %p\n", myself->common.id, result)); ASSERT(hp == *hpp + size); *hpp = hp; -- cgit v1.2.3 From bcdf32795a3f5a6aad95567cffba78fcecb9f40f Mon Sep 17 00:00:00 2001 From: "Nikolaos S. Papaspyrou" Date: Thu, 24 Apr 2014 12:59:39 +0300 Subject: Add support for maps in preserved copy --- erts/emulator/beam/copy.c | 61 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) (limited to 'erts/emulator/beam/copy.c') diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index f74c6b1c89..cb1f38429a 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -424,6 +424,19 @@ Uint size_shared(Eterm obj) } goto pop_next; } + case MAP_SUBTAG: { + map_t *mp = (map_t *) ptr; + Uint n = map_get_size(mp) + 1; + sum += n + 2; + ptr += 2; /* hdr + size words */ + while (n-- > 0) { + obj = *ptr++; + if (!IS_CONST(obj)) { + EQUEUE_PUT(s, obj); + } + } + goto pop_next; + } case BIN_MATCHSTATE_SUBTAG: erl_exit(ERTS_ABORT_EXIT, "size_shared: matchstate term not allowed"); @@ -523,6 +536,18 @@ cleanup: } goto cleanup_next; } + case MAP_SUBTAG: { + map_t *mp = (map_t *) ptr; + Uint n = map_get_size(mp) + 1; + ptr += 2; /* hdr + size words */ + while (n-- > 0) { + obj = *ptr++; + if (!IS_CONST(obj)) { + EQUEUE_PUT_UNCHECKED(s, obj); + } + } + goto cleanup_next; + } default: goto cleanup_next; } @@ -1162,6 +1187,19 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) } goto pop_next; } + case MAP_SUBTAG: { + map_t *mp = (map_t *) ptr; + Uint n = map_get_size(mp) + 1; + sum += n + 2; + ptr += 2; /* hdr + size words */ + while (n-- > 0) { + obj = *ptr++; + if (!IS_CONST(obj)) { + EQUEUE_PUT(s, obj); + } + } + goto pop_next; + } case BIN_MATCHSTATE_SUBTAG: erl_exit(ERTS_ABORT_EXIT, "size_shared: matchstate term not allowed"); @@ -1404,6 +1442,23 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E erts_refc_inc(&funp->fe->refc, 2); goto cleanup_next; } + case MAP_SUBTAG: { + map_t *mp = (map_t *) ptr; + Uint n = map_get_size(mp) + 1; + *resp = make_map(hp); + *hp++ = hdr; + *hp++ = *++ptr; + while (n-- > 0) { + obj = *++ptr; + if (IS_CONST(obj)) { + *hp++ = obj; + } else { + EQUEUE_PUT_UNCHECKED(s, obj); + *hp++ = HEAP_ELEM_TO_BE_FILLED; + } + } + goto cleanup_next; + } case REFC_BINARY_SUBTAG: { ProcBin* pb = (ProcBin *) ptr; sz = thing_arityval(hdr); @@ -1539,6 +1594,12 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E remaining = 1 + funp->num_free; break; } + case MAP_SUBTAG: { + map_t *mp = (map_t *) hscan; + remaining = map_get_size(mp) + 1; + hscan += 2; + break; + } case SUB_BINARY_SUBTAG: ASSERT(((ErlSubBin *) hscan)->bitoffs + ((ErlSubBin *) hscan)->bitsize > 0); -- cgit v1.2.3 From 99de9eb01d8884ac293fdda20ab65c045d215d46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Tue, 15 Sep 2015 17:47:36 +0200 Subject: Fix internal stacks The internal stacks has changed between releases. --- erts/emulator/beam/copy.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'erts/emulator/beam/copy.c') diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index cb1f38429a..added85b46 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -907,7 +907,7 @@ int disable_copy_shared = ERTS_SHCOPY_FLG_NONE; #define SHTABLE_PUSH(s,x,y,b) \ do { \ if (s.sp > s.end - SHTABLE_INCR) { \ - erl_grow_estack(&s, ESTK_DEF_STACK(s)); \ + erl_grow_estack(&(s), SHTABLE_INCR); \ } \ *s.sp++ = (x); \ *s.sp++ = (y); \ @@ -959,6 +959,7 @@ do { \ WSTK_DEF_STACK(s), /* wstart */ \ WSTK_DEF_STACK(s), /* wsp */ \ WSTK_DEF_STACK(s) + DEF_WSTACK_SIZE, /* wend */ \ + WSTK_DEF_STACK(s), /* wdflt */ \ ERTS_ALC_T_ESTACK /* alloc_type */ \ }; \ int WSTK_CONCAT(s,_bitoffs) = 0; \ @@ -969,8 +970,9 @@ do { \ /* no WSTK_DEF_STACK(s), read-only */ \ ErtsWStack s = { \ info->bitstore_start, /* wstart */ \ - NULL, /* wsp, read-only */ \ + NULL, /* wsp, read-only */ \ NULL, /* wend, read-only */ \ + NULL, /* wdef, read-only */ \ info->bitstore_alloc_type /* alloc_type */ \ }; \ int WSTK_CONCAT(s,_bitoffs) = 0; \ @@ -983,6 +985,7 @@ do { \ ESTK_DEF_STACK(s), /* start */ \ ESTK_DEF_STACK(s), /* sp */ \ ESTK_DEF_STACK(s) + DEF_ESTACK_SIZE, /* end */ \ + ESTK_DEF_STACK(s), /* default */ \ ERTS_ALC_T_ESTACK /* alloc_type */ \ }; \ Uint ESTK_CONCAT(s,_offset) = 0 @@ -991,8 +994,9 @@ do { \ /* no ESTK_DEF_STACK(s), read-only */ \ ErtsEStack s = { \ info->shtable_start, /* start */ \ - NULL, /* sp, read-only */ \ + NULL, /* sp, read-only */ \ NULL, /* end, read-only */ \ + NULL, /* def, read-only */ \ info->shtable_alloc_type /* alloc_type */ \ }; \ /* no ESTK_CONCAT(s,_offset), read-only */ -- cgit v1.2.3 From 2ab76f7f1fe6ec4c94541ba5a2f0cd9dc1b9c79a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Tue, 15 Sep 2015 16:49:28 +0200 Subject: Fix Halfword removal Halfword is no longer present in the runtime system. --- erts/emulator/beam/copy.c | 40 +++++++++++++++++----------------------- 1 file changed, 17 insertions(+), 23 deletions(-) (limited to 'erts/emulator/beam/copy.c') diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index added85b46..42b4bee29d 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -298,11 +298,7 @@ do { \ * It is argued whether the size of subterms in constant pools * should be counted or not. */ -#if HALFWORD_HEAP -Uint size_shared_rel(Eterm obj, Eterm* base) -#else Uint size_shared(Eterm obj) -#endif { Eterm saved_obj = obj; Uint sum = 0; @@ -320,7 +316,7 @@ Uint size_shared(Eterm obj) switch (primary_tag(obj)) { case TAG_PRIMARY_LIST: { Eterm head, tail; - ptr = list_val_rel(obj, base); + ptr = list_val(obj); /* we're not counting anything that's outside our heap */ if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) { goto pop_next; @@ -357,7 +353,7 @@ Uint size_shared(Eterm obj) } case TAG_PRIMARY_BOXED: { Eterm hdr; - ptr = boxed_val_rel(obj, base); + ptr = boxed_val(obj); /* we're not counting anything that's outside our heap */ if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) { goto pop_next; @@ -414,13 +410,13 @@ Uint size_shared(Eterm obj) } else { extra_bytes = 0; } - ptr = binary_val_rel(sb->orig, base); + ptr = binary_val(sb->orig); hdr = (*ptr) & ~BOXED_VISITED_MASK; if (thing_subtag(hdr) == REFC_BINARY_SUBTAG) { sum += PROC_BIN_SIZE; } else { ASSERT(thing_subtag(hdr) == HEAP_BINARY_SUBTAG); - sum += heap_bin_size(binary_size_rel(obj, base) + extra_bytes); + sum += heap_bin_size(binary_size(obj) + extra_bytes); } goto pop_next; } @@ -465,7 +461,7 @@ cleanup: switch (primary_tag(obj)) { case TAG_PRIMARY_LIST: { Eterm head, tail; - ptr = list_val_rel(obj, base); + ptr = list_val(obj); if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) { goto cleanup_next; } @@ -495,7 +491,7 @@ cleanup: } case TAG_PRIMARY_BOXED: { Eterm hdr; - ptr = boxed_val_rel(obj, base); + ptr = boxed_val(obj); if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) { goto cleanup_next; } @@ -912,7 +908,7 @@ do { \ *s.sp++ = (x); \ *s.sp++ = (y); \ *s.sp++ = (Eterm) NULL; \ - *s.sp++ = (Eterm) (b); /* bad in HALF_WORD */ \ + *s.sp++ = (Eterm) (b); \ ESTK_CONCAT(s,_offset) += SHTABLE_INCR; \ } while(0) #define SHTABLE_X(s,e) (s.start[e]) @@ -1004,7 +1000,6 @@ do { \ /* * Copy object "obj" preserving sharing. * First half: count size and calculate sharing. - * NOTE: We do not support HALF_WORD (yet?). */ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) { @@ -1060,7 +1055,7 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) switch (primary_tag(obj)) { case TAG_PRIMARY_LIST: { Eterm head, tail; - ptr = list_val_rel(obj, base); + ptr = list_val(obj); /* off heap list pointers are copied verbatim */ if (!INHEAP(myself, ptr)) { VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", myself->common.id, ptr, obj)); @@ -1109,7 +1104,7 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) } case TAG_PRIMARY_BOXED: { Eterm hdr; - ptr = boxed_val_rel(obj, base); + ptr = boxed_val(obj); /* off heap pointers to boxes are copied verbatim */ if (!INHEAP(myself, ptr)) { VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", myself->common.id, ptr, obj)); @@ -1178,11 +1173,11 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) } else { extra_bytes = 0; } - ASSERT(is_boxed(rterm2wterm(real_bin, base)) && - (((*boxed_val(rterm2wterm(real_bin, base))) & + ASSERT(is_boxed(real_bin) && + (((*boxed_val(real_bin)) & (_TAG_HEADER_MASK - _BINARY_XXX_MASK - BOXED_VISITED_MASK)) == _TAG_HEADER_REFC_BIN)); - hdr = *_unchecked_binary_val(rterm2wterm(real_bin, base)) & ~BOXED_VISITED_MASK; + hdr = *_unchecked_binary_val(real_bin) & ~BOXED_VISITED_MASK; if (thing_subtag(hdr) == HEAP_BINARY_SUBTAG) { sum += heap_bin_size(size+extra_bytes); } else { @@ -1243,7 +1238,6 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) /* * Copy object "obj" preserving sharing. * Second half: copy and restore the object. - * NOTE: We do not support HALF_WORD (yet?). */ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, ErlOffHeap* off_heap, unsigned flags) { @@ -1309,7 +1303,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E switch (primary_tag(obj)) { case TAG_PRIMARY_LIST: { Eterm head, tail; - ptr = list_val_rel(obj, base); + ptr = list_val(obj); /* off heap list pointers are copied verbatim */ if (!INHEAP(myself, ptr)) { *resp = obj; @@ -1370,7 +1364,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E } case TAG_PRIMARY_BOXED: { Eterm hdr; - ptr = boxed_val_rel(obj, base); + ptr = boxed_val(obj); /* off heap pointers to boxes are copied verbatim */ if (!INHEAP(myself, ptr)) { *resp = obj; @@ -1500,11 +1494,11 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E extra_bytes = 0; } real_size = size+extra_bytes; - ASSERT(is_boxed(rterm2wterm(real_bin, base)) && - (((*boxed_val(rterm2wterm(real_bin, base))) & + ASSERT(is_boxed(real_bin) && + (((*boxed_val(real_bin)) & (_TAG_HEADER_MASK - _BINARY_XXX_MASK - BOXED_VISITED_MASK)) == _TAG_HEADER_REFC_BIN)); - ptr = _unchecked_binary_val(rterm2wterm(real_bin, base)); + ptr = _unchecked_binary_val(real_bin); *resp = make_binary(hp); if (extra_bytes != 0) { ErlSubBin* res = (ErlSubBin *) hp; -- cgit v1.2.3 From 95b25f7a3a33c43912079af713bf1f386a764071 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Tue, 15 Sep 2015 17:36:22 +0200 Subject: Fix Map preserved sharing copy implementation The Map implementation has changed since initial preserved copy implementation. --- erts/emulator/beam/copy.c | 135 +++++++++++++++++++++++++++++----------------- 1 file changed, 85 insertions(+), 50 deletions(-) (limited to 'erts/emulator/beam/copy.c') diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index 42b4bee29d..c7806ed8ff 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -420,19 +420,28 @@ Uint size_shared(Eterm obj) } goto pop_next; } - case MAP_SUBTAG: { - map_t *mp = (map_t *) ptr; - Uint n = map_get_size(mp) + 1; - sum += n + 2; - ptr += 2; /* hdr + size words */ - while (n-- > 0) { - obj = *ptr++; - if (!IS_CONST(obj)) { - EQUEUE_PUT(s, obj); + case MAP_SUBTAG: + switch (MAP_HEADER_TYPE(hdr)) { + case MAP_HEADER_TAG_FLATMAP_HEAD : { + flatmap_t *mp = (flatmap_t*)flatmap_val(obj); + Uint n = flatmap_get_size(mp) + 1; + ptr = (Eterm *)mp; + sum += n + 2; + ptr += 2; /* hdr + size words */ + while (n--) { + obj = *ptr++; + if (!IS_CONST(obj)) { + EQUEUE_PUT(s, obj); + } + } + goto pop_next; } + case MAP_HEADER_TAG_HAMT_HEAD_BITMAP : + case MAP_HEADER_TAG_HAMT_HEAD_ARRAY : + case MAP_HEADER_TAG_HAMT_NODE_BITMAP : + default: + erl_exit(ERTS_ABORT_EXIT, "size_shared: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); } - goto pop_next; - } case BIN_MATCHSTATE_SUBTAG: erl_exit(ERTS_ABORT_EXIT, "size_shared: matchstate term not allowed"); @@ -533,10 +542,10 @@ cleanup: goto cleanup_next; } case MAP_SUBTAG: { - map_t *mp = (map_t *) ptr; - Uint n = map_get_size(mp) + 1; + flatmap_t *mp = (flatmap_t *) ptr; + Uint n = flatmap_get_size(mp) + 1; ptr += 2; /* hdr + size words */ - while (n-- > 0) { + while (n--) { obj = *ptr++; if (!IS_CONST(obj)) { EQUEUE_PUT_UNCHECKED(s, obj); @@ -1186,20 +1195,28 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) } goto pop_next; } - case MAP_SUBTAG: { - map_t *mp = (map_t *) ptr; - Uint n = map_get_size(mp) + 1; - sum += n + 2; - ptr += 2; /* hdr + size words */ - while (n-- > 0) { - obj = *ptr++; - if (!IS_CONST(obj)) { - EQUEUE_PUT(s, obj); + case MAP_SUBTAG: + switch (MAP_HEADER_TYPE(hdr)) { + case MAP_HEADER_TAG_FLATMAP_HEAD : { + flatmap_t *mp = (flatmap_t *) ptr; + Uint n = flatmap_get_size(mp) + 1; + sum += n + 2; + ptr += 2; /* hdr + size words */ + while (n--) { + obj = *ptr++; + if (!IS_CONST(obj)) { + EQUEUE_PUT(s, obj); + } + } + goto pop_next; } + case MAP_HEADER_TAG_HAMT_HEAD_BITMAP : + case MAP_HEADER_TAG_HAMT_HEAD_ARRAY : + case MAP_HEADER_TAG_HAMT_NODE_BITMAP : + default: + erl_exit(ERTS_ABORT_EXIT, "copy_shared_calculate: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); } - goto pop_next; - } - case BIN_MATCHSTATE_SUBTAG: + case BIN_MATCHSTATE_SUBTAG: erl_exit(ERTS_ABORT_EXIT, "size_shared: matchstate term not allowed"); default: @@ -1234,7 +1251,6 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) } } - /* * Copy object "obj" preserving sharing. * Second half: copy and restore the object. @@ -1440,23 +1456,31 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E erts_refc_inc(&funp->fe->refc, 2); goto cleanup_next; } - case MAP_SUBTAG: { - map_t *mp = (map_t *) ptr; - Uint n = map_get_size(mp) + 1; - *resp = make_map(hp); - *hp++ = hdr; - *hp++ = *++ptr; - while (n-- > 0) { - obj = *++ptr; - if (IS_CONST(obj)) { - *hp++ = obj; - } else { - EQUEUE_PUT_UNCHECKED(s, obj); - *hp++ = HEAP_ELEM_TO_BE_FILLED; - } - } - goto cleanup_next; - } + case MAP_SUBTAG: + switch (MAP_HEADER_TYPE(hdr)) { + case MAP_HEADER_TAG_FLATMAP_HEAD : { + flatmap_t *mp = (flatmap_t *) ptr; + Uint n = flatmap_get_size(mp) + 1; + *resp = make_flatmap(hp); + *hp++ = hdr; + *hp++ = *++ptr; + while (n--) { + obj = *++ptr; + if (IS_CONST(obj)) { + *hp++ = obj; + } else { + EQUEUE_PUT_UNCHECKED(s, obj); + *hp++ = HEAP_ELEM_TO_BE_FILLED; + } + } + goto cleanup_next; + } + case MAP_HEADER_TAG_HAMT_HEAD_BITMAP : + case MAP_HEADER_TAG_HAMT_HEAD_ARRAY : + case MAP_HEADER_TAG_HAMT_NODE_BITMAP : + default: + erl_exit(ERTS_ABORT_EXIT, "copy_shared_perform: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); + } case REFC_BINARY_SUBTAG: { ProcBin* pb = (ProcBin *) ptr; sz = thing_arityval(hdr); @@ -1592,12 +1616,23 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E remaining = 1 + funp->num_free; break; } - case MAP_SUBTAG: { - map_t *mp = (map_t *) hscan; - remaining = map_get_size(mp) + 1; - hscan += 2; - break; - } + case MAP_SUBTAG: + switch (MAP_HEADER_TYPE(*hscan)) { + case MAP_HEADER_TAG_FLATMAP_HEAD : { + flatmap_t *mp = (flatmap_t *) hscan; + remaining = flatmap_get_size(mp) + 1; + hscan += 2; + break; + } + case MAP_HEADER_TAG_HAMT_HEAD_BITMAP : + case MAP_HEADER_TAG_HAMT_HEAD_ARRAY : + case MAP_HEADER_TAG_HAMT_NODE_BITMAP : + default: + erl_exit(ERTS_ABORT_EXIT, + "copy_shared_perform: bad hashmap type %d\n", + MAP_HEADER_TYPE(*hscan)); + } + break; case SUB_BINARY_SUBTAG: ASSERT(((ErlSubBin *) hscan)->bitoffs + ((ErlSubBin *) hscan)->bitsize > 0); -- cgit v1.2.3 From 2e10fe29f61bbe3246902e8eaf1636dd6457979f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Wed, 16 Sep 2015 16:21:43 +0200 Subject: Add support for HAMT maps in preserved copy --- erts/emulator/beam/copy.c | 115 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 88 insertions(+), 27 deletions(-) (limited to 'erts/emulator/beam/copy.c') diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index c7806ed8ff..aa17713d07 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -438,7 +438,18 @@ Uint size_shared(Eterm obj) } case MAP_HEADER_TAG_HAMT_HEAD_BITMAP : case MAP_HEADER_TAG_HAMT_HEAD_ARRAY : - case MAP_HEADER_TAG_HAMT_NODE_BITMAP : + case MAP_HEADER_TAG_HAMT_NODE_BITMAP : { + Uint n = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + sum += 1 + n + header_arity(hdr); + ptr += 1 + header_arity(hdr); + while (n--) { + obj = *ptr++; + if (!IS_CONST(obj)) { + EQUEUE_PUT(s, obj); + } + } + goto pop_next; + } default: erl_exit(ERTS_ABORT_EXIT, "size_shared: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); } @@ -541,18 +552,37 @@ cleanup: } goto cleanup_next; } - case MAP_SUBTAG: { - flatmap_t *mp = (flatmap_t *) ptr; - Uint n = flatmap_get_size(mp) + 1; - ptr += 2; /* hdr + size words */ - while (n--) { - obj = *ptr++; - if (!IS_CONST(obj)) { - EQUEUE_PUT_UNCHECKED(s, obj); + case MAP_SUBTAG: + switch (MAP_HEADER_TYPE(hdr)) { + case MAP_HEADER_TAG_FLATMAP_HEAD : { + flatmap_t *mp = (flatmap_t *) ptr; + Uint n = flatmap_get_size(mp) + 1; + ptr += 2; /* hdr + size words */ + while (n--) { + obj = *ptr++; + if (!IS_CONST(obj)) { + EQUEUE_PUT_UNCHECKED(s, obj); + } + } + goto cleanup_next; + } + case MAP_HEADER_TAG_HAMT_HEAD_BITMAP : + case MAP_HEADER_TAG_HAMT_HEAD_ARRAY : + case MAP_HEADER_TAG_HAMT_NODE_BITMAP : { + Uint n = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + sum += 1 + n + header_arity(hdr); + ptr += 1 + header_arity(hdr); + while (n--) { + obj = *ptr++; + if (!IS_CONST(obj)) { + EQUEUE_PUT_UNCHECKED(s, obj); + } + } + goto cleanup_next; } + default: + erl_exit(ERTS_ABORT_EXIT, "size_shared: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); } - goto cleanup_next; - } default: goto cleanup_next; } @@ -1212,7 +1242,22 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) } case MAP_HEADER_TAG_HAMT_HEAD_BITMAP : case MAP_HEADER_TAG_HAMT_HEAD_ARRAY : - case MAP_HEADER_TAG_HAMT_NODE_BITMAP : + case MAP_HEADER_TAG_HAMT_NODE_BITMAP : { + Uint n = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + sum += 1 + n + header_arity(hdr); + ptr += 1 + header_arity(hdr); + + if (n == 0) { + goto pop_next; + } + while(n--) { + obj = *ptr++; + if (!IS_CONST(obj)) { + EQUEUE_PUT(s, obj); + } + } + goto pop_next; + } default: erl_exit(ERTS_ABORT_EXIT, "copy_shared_calculate: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); } @@ -1228,20 +1273,20 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) case TAG_PRIMARY_IMMED1: pop_next: if (EQUEUE_ISEMPTY(s)) { - // add sentinel to the table - SHTABLE_PUSH(t, THE_NON_VALUE, THE_NON_VALUE, NULL); - // store persistent info - BITSTORE_CLOSE(b); - info->queue_start = s.start; - info->queue_end = s.end; + /* add sentinel to the table */ + SHTABLE_PUSH(t, THE_NON_VALUE, THE_NON_VALUE, NULL); + /* store persistent info */ + BITSTORE_CLOSE(b); + info->queue_start = s.start; + info->queue_end = s.end; info->queue_alloc_type = s.alloc_type; - info->bitstore_start = b.wstart; + info->bitstore_start = b.wstart; info->bitstore_alloc_type = b.alloc_type; - info->shtable_start = t.start; + info->shtable_start = t.start; info->shtable_alloc_type = t.alloc_type; - // single point of return: the size of the object - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] size was: %u\n", myself->common.id, sum)); - return sum; + /* single point of return: the size of the object */ + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] size was: %u\n", myself->common.id, sum)); + return sum; } obj = EQUEUE_GET(s); break; @@ -1457,13 +1502,13 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E goto cleanup_next; } case MAP_SUBTAG: + *resp = make_flatmap(hp); + *hp++ = hdr; switch (MAP_HEADER_TYPE(hdr)) { case MAP_HEADER_TAG_FLATMAP_HEAD : { flatmap_t *mp = (flatmap_t *) ptr; Uint n = flatmap_get_size(mp) + 1; - *resp = make_flatmap(hp); - *hp++ = hdr; - *hp++ = *++ptr; + *hp++ = *++ptr; /* keys */ while (n--) { obj = *++ptr; if (IS_CONST(obj)) { @@ -1477,7 +1522,20 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E } case MAP_HEADER_TAG_HAMT_HEAD_BITMAP : case MAP_HEADER_TAG_HAMT_HEAD_ARRAY : - case MAP_HEADER_TAG_HAMT_NODE_BITMAP : + *hp++ = *++ptr; /* total map size */ + case MAP_HEADER_TAG_HAMT_NODE_BITMAP : { + Uint n = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + while (n--) { + obj = *++ptr; + if (IS_CONST(obj)) { + *hp++ = obj; + } else { + EQUEUE_PUT_UNCHECKED(s, obj); + *hp++ = HEAP_ELEM_TO_BE_FILLED; + } + } + goto cleanup_next; + } default: erl_exit(ERTS_ABORT_EXIT, "copy_shared_perform: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); } @@ -1627,6 +1685,9 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E case MAP_HEADER_TAG_HAMT_HEAD_BITMAP : case MAP_HEADER_TAG_HAMT_HEAD_ARRAY : case MAP_HEADER_TAG_HAMT_NODE_BITMAP : + remaining = hashmap_bitcount(MAP_HEADER_VAL(*hscan)); + hscan += MAP_HEADER_ARITY(*hscan) + 1; + break; default: erl_exit(ERTS_ABORT_EXIT, "copy_shared_perform: bad hashmap type %d\n", -- cgit v1.2.3 From 748c73f1687b2375d4c607487f40036ba990c4c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Thu, 17 Sep 2015 16:17:10 +0200 Subject: Refactor copy sharing --- erts/emulator/beam/copy.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'erts/emulator/beam/copy.c') diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index aa17713d07..f2dd73d862 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -1040,14 +1040,14 @@ do { \ * Copy object "obj" preserving sharing. * First half: count size and calculate sharing. */ -Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) +Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info, Uint32 flags) { Uint sum; Uint e; unsigned sz; Eterm* ptr; Process* myself; - int force_local = flags & ERTS_SHCOPY_FLG_TMP_BUF; + int force_local = flags & ERTS_SHCOPY_FLG_TMPBUF; DECLARE_EQUEUE_INIT_INFO(s, info); DECLARE_BITSTORE_INIT_INFO(b, info); @@ -1300,8 +1300,8 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) * Copy object "obj" preserving sharing. * Second half: copy and restore the object. */ -Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, ErlOffHeap* off_heap, unsigned flags) -{ +Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, + Eterm** hpp, ErlOffHeap* off_heap, Uint32 flags) { Uint e; unsigned sz; Eterm* ptr; @@ -1311,7 +1311,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E Eterm* resp; unsigned remaining; Process* myself; - int force_local = flags & ERTS_SHCOPY_FLG_TMP_BUF; + int force_local = flags & ERTS_SHCOPY_FLG_TMPBUF; #ifdef DEBUG Eterm saved_obj = obj; #endif -- cgit v1.2.3 From 5d2d888f368ba8ca0098d8bd9936ca9d67df7d3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Thu, 17 Sep 2015 19:30:08 +0200 Subject: Copy literals in copy sharing --- erts/emulator/beam/copy.c | 49 +++++++++++++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 17 deletions(-) (limited to 'erts/emulator/beam/copy.c') diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index f2dd73d862..4f4d20cb83 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -611,7 +611,7 @@ cleanup: /* * Copy a structure to a heap. */ -Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) +Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint *bsz) { char* hstart; Uint hsize; @@ -626,6 +626,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) Eterm* argp; Eterm* const_tuple; Eterm hdr; + Eterm *hend; int i; #ifdef DEBUG Eterm org_obj = obj; @@ -641,7 +642,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) DTRACE1(copy_struct, (int32_t)sz); hp = htop = *hpp; - hbot = htop + sz; + hbot = hend = htop + sz; hstart = (char *)htop; hsize = (char*) hbot - hstart; const_tuple = 0; @@ -906,19 +907,24 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) } } + if (bsz) { + *hpp = htop; + *bsz = hend - hbot; + } else { #ifdef DEBUG - if (htop != hbot) - erl_exit(ERTS_ABORT_EXIT, - "Internal error in copy_struct() when copying %T:" - " htop=%p != hbot=%p (sz=%beu)\n", - org_obj, htop, hbot, org_sz); + if (htop != hbot) + erl_exit(ERTS_ABORT_EXIT, + "Internal error in copy_struct() when copying %T:" + " htop=%p != hbot=%p (sz=%beu)\n", + org_obj, htop, hbot, org_sz); #else - if (htop > hbot) { - erl_exit(ERTS_ABORT_EXIT, - "Internal error in copy_struct(): htop, hbot overrun\n"); - } + if (htop > hbot) { + erl_exit(ERTS_ABORT_EXIT, + "Internal error in copy_struct(): htop, hbot overrun\n"); + } #endif - *hpp = (Eterm *) (hstart+hsize); + *hpp = (Eterm *) (hstart+hsize); + } VERBOSE(DEBUG_SHCOPY, ("[pid=%T] result is at %p\n", mypid, res)); return res; } @@ -1098,6 +1104,7 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info, Uint32 flags) /* off heap list pointers are copied verbatim */ if (!INHEAP(myself, ptr)) { VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", myself->common.id, ptr, obj)); + info->literal_size += size_object(obj); goto pop_next; } head = CAR(ptr); @@ -1147,6 +1154,7 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info, Uint32 flags) /* off heap pointers to boxes are copied verbatim */ if (!INHEAP(myself, ptr)) { VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", myself->common.id, ptr, obj)); + info->literal_size += size_object(obj); goto pop_next; } hdr = *ptr; @@ -1286,7 +1294,7 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info, Uint32 flags) info->shtable_alloc_type = t.alloc_type; /* single point of return: the size of the object */ VERBOSE(DEBUG_SHCOPY, ("[pid=%T] size was: %u\n", myself->common.id, sum)); - return sum; + return sum + info->literal_size; } obj = EQUEUE_GET(s); break; @@ -1309,6 +1317,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, Eterm* hscan; Eterm result; Eterm* resp; + Eterm *hbot, *hend; unsigned remaining; Process* myself; int force_local = flags & ERTS_SHCOPY_FLG_TMPBUF; @@ -1346,6 +1355,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, */ hscan = hp = *hpp; + hbot = hend = hp + size; /* step #3: ------------------------------------------------------- @@ -1367,7 +1377,9 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, ptr = list_val(obj); /* off heap list pointers are copied verbatim */ if (!INHEAP(myself, ptr)) { - *resp = obj; + Uint bsz; + *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz); + hbot -= bsz; goto cleanup_next; } head = CAR(ptr); @@ -1428,7 +1440,9 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, ptr = boxed_val(obj); /* off heap pointers to boxes are copied verbatim */ if (!INHEAP(myself, ptr)) { - *resp = obj; + Uint bsz; + *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz); + hbot -= bsz; goto cleanup_next; } hdr = *ptr; @@ -1759,8 +1773,9 @@ all_clean: VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy is %T\n", myself->common.id, result)); VERBOSE(DEBUG_SHCOPY, ("[pid=%T] result is at %p\n", myself->common.id, result)); - ASSERT(hp == *hpp + size); - *hpp = hp; + ASSERT(hbot == hp); + ASSERT(size == ((hp - *hpp) + (hend - hbot))); + *hpp = hend; return result; } -- cgit v1.2.3 From 1391715d8bbba315e1509e60e6245159a009bd9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Fri, 25 Sep 2015 17:03:02 +0200 Subject: Use copy literal range check in message passing and purging --- erts/emulator/beam/copy.c | 136 ++++++++++++++++++++++------------------------ 1 file changed, 65 insertions(+), 71 deletions(-) (limited to 'erts/emulator/beam/copy.c') diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index 4f4d20cb83..5bca3877b5 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -276,49 +276,35 @@ do { \ #define BOXED_SHARED_UNPROCESSED ((Eterm) 2) #define BOXED_SHARED_PROCESSED ((Eterm) 3) +#define COUNT_OFF_HEAP (0) -/* - * Is an object in the local heap of a process? - */ - -#define INHEAP_SIMPLE(p, ptr) ( \ - (OLD_HEAP(p) && OLD_HEAP(p) <= ptr && ptr < OLD_HEND(p)) || \ - (HEAP_START(p) <= ptr && ptr < HEAP_END(p)) \ - ) -#define INHEAP(p, ptr) ( \ - INHEAP_SIMPLE(p, ptr) || \ - (force_local ? (force_local = 0, 1) : 0) \ - ) -#define COUNT_OFF_HEAP 0 - - +#define IN_LITERAL_PURGE_AREA(info, ptr) \ + ((info)->range_ptr && ( \ + (info)->range_ptr <= (ptr) && \ + (ptr) < ((info)->range_ptr + (info)->range_sz))) /* * Return the real size of an object and find sharing information * This currently returns the same as erts_debug:size/1. * It is argued whether the size of subterms in constant pools * should be counted or not. */ + Uint size_shared(Eterm obj) { Eterm saved_obj = obj; Uint sum = 0; Eterm* ptr; - Process* myself; DECLARE_EQUEUE(s); DECLARE_BITSTORE(b); - myself = erts_get_current_process(); - if (myself == NULL) - return size_object(obj); - for (;;) { switch (primary_tag(obj)) { case TAG_PRIMARY_LIST: { Eterm head, tail; ptr = list_val(obj); /* we're not counting anything that's outside our heap */ - if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) { + if (!COUNT_OFF_HEAP && erts_is_literal(obj,ptr)) { goto pop_next; } head = CAR(ptr); @@ -355,7 +341,7 @@ Uint size_shared(Eterm obj) Eterm hdr; ptr = boxed_val(obj); /* we're not counting anything that's outside our heap */ - if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) { + if (!COUNT_OFF_HEAP && erts_is_literal(obj,ptr)) { goto pop_next; } hdr = *ptr; @@ -482,7 +468,7 @@ cleanup: case TAG_PRIMARY_LIST: { Eterm head, tail; ptr = list_val(obj); - if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) { + if (!COUNT_OFF_HEAP && erts_is_literal(obj,ptr)) { goto cleanup_next; } head = CAR(ptr); @@ -512,7 +498,7 @@ cleanup: case TAG_PRIMARY_BOXED: { Eterm hdr; ptr = boxed_val(obj); - if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) { + if (!COUNT_OFF_HEAP && erts_is_literal(obj,ptr)) { goto cleanup_next; } hdr = *ptr; @@ -1052,8 +1038,9 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info, Uint32 flags) Uint e; unsigned sz; Eterm* ptr; - Process* myself; - int force_local = flags & ERTS_SHCOPY_FLG_TMPBUF; +#ifdef DEBUG + Eterm mypid = erts_get_current_pid(); +#endif DECLARE_EQUEUE_INIT_INFO(s, info); DECLARE_BITSTORE_INIT_INFO(b, info); @@ -1073,12 +1060,11 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info, Uint32 flags) flags |= disable_copy_shared; #endif - myself = erts_get_current_process(); - if (myself == NULL || (flags & ERTS_SHCOPY_FLG_NONE)) + if (flags & ERTS_SHCOPY_FLG_NONE) return size_object(obj); - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy_shared_calculate %p\n", myself->common.id, obj)); - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] message is %T\n", myself->common.id, obj)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy_shared_calculate %p\n", mypid, obj)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] message is %T\n", mypid, obj)); /* step #1: ------------------------------------------------------- @@ -1102,9 +1088,10 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info, Uint32 flags) Eterm head, tail; ptr = list_val(obj); /* off heap list pointers are copied verbatim */ - if (!INHEAP(myself, ptr)) { - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", myself->common.id, ptr, obj)); - info->literal_size += size_object(obj); + if (erts_is_literal(obj,ptr)) { + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", mypid, ptr, obj)); + if (IN_LITERAL_PURGE_AREA(info,ptr)) + info->literal_size += size_object(obj); goto pop_next; } head = CAR(ptr); @@ -1115,7 +1102,7 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info, Uint32 flags) primary_tag(head) == TAG_PRIMARY_HEADER) { if (tail != THE_NON_VALUE) { e = SHTABLE_NEXT(t); - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] tabling L %p\n", myself->common.id, ptr)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] tabling L %p\n", mypid, ptr)); SHTABLE_PUSH(t, head, tail, ptr); CAR(ptr) = (e << _TAG_PRIMARY_SIZE) | LIST_SHARED_UNPROCESSED; CDR(ptr) = THE_NON_VALUE; @@ -1125,17 +1112,17 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info, Uint32 flags) /* else make it visited now */ switch (primary_tag(tail)) { case TAG_PRIMARY_LIST: - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] mangling L/L %p\n", myself->common.id, ptr)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] mangling L/L %p\n", mypid, ptr)); CDR(ptr) = (tail - TAG_PRIMARY_LIST) | TAG_PRIMARY_HEADER; break; case TAG_PRIMARY_IMMED1: - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] mangling L/I %p\n", myself->common.id, ptr)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] mangling L/I %p\n", mypid, ptr)); CAR(ptr) = (head - primary_tag(head)) | TAG_PRIMARY_HEADER; CDR(ptr) = (tail - TAG_PRIMARY_IMMED1) | primary_tag(head); break; case TAG_PRIMARY_BOXED: BITSTORE_PUT(b, primary_tag(head)); - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] mangling L/B %p\n", myself->common.id, ptr)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] mangling L/B %p\n", mypid, ptr)); CAR(ptr) = (head - primary_tag(head)) | TAG_PRIMARY_HEADER; CDR(ptr) = (tail - TAG_PRIMARY_BOXED) | TAG_PRIMARY_HEADER; break; @@ -1152,9 +1139,10 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info, Uint32 flags) Eterm hdr; ptr = boxed_val(obj); /* off heap pointers to boxes are copied verbatim */ - if (!INHEAP(myself, ptr)) { - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", myself->common.id, ptr, obj)); - info->literal_size += size_object(obj); + if (erts_is_literal(obj,ptr)) { + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", mypid, ptr, obj)); + if (IN_LITERAL_PURGE_AREA(info,ptr)) + info->literal_size += size_object(obj); goto pop_next; } hdr = *ptr; @@ -1163,14 +1151,14 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info, Uint32 flags) if (primary_tag(hdr) != TAG_PRIMARY_HEADER) { if (primary_tag(hdr) == BOXED_VISITED) { e = SHTABLE_NEXT(t); - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] tabling B %p\n", myself->common.id, ptr)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] tabling B %p\n", mypid, ptr)); SHTABLE_PUSH(t, hdr, THE_NON_VALUE, ptr); *ptr = (e << _TAG_PRIMARY_SIZE) | BOXED_SHARED_UNPROCESSED; } goto pop_next; } /* else make it visited now */ - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] mangling B %p\n", myself->common.id, ptr)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] mangling B %p\n", mypid, ptr)); *ptr = (hdr - primary_tag(hdr)) + BOXED_VISITED; /* and count it */ ASSERT(is_header(hdr)); @@ -1293,7 +1281,7 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info, Uint32 flags) info->shtable_start = t.start; info->shtable_alloc_type = t.alloc_type; /* single point of return: the size of the object */ - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] size was: %u\n", myself->common.id, sum)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] size was: %u\n", mypid, sum)); return sum + info->literal_size; } obj = EQUEUE_GET(s); @@ -1319,9 +1307,8 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, Eterm* resp; Eterm *hbot, *hend; unsigned remaining; - Process* myself; - int force_local = flags & ERTS_SHCOPY_FLG_TMPBUF; #ifdef DEBUG + Eterm mypid = erts_get_current_pid(); Eterm saved_obj = obj; #endif @@ -1343,11 +1330,10 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, flags |= disable_copy_shared; #endif - myself = erts_get_current_process(); - if (myself == NULL || (flags & ERTS_SHCOPY_FLG_NONE)) + if (flags & ERTS_SHCOPY_FLG_NONE) return copy_struct(obj, size, hpp, off_heap); - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy_shared_perform %p\n", myself->common.id, obj)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy_shared_perform %p\n", mypid, obj)); /* step #2: was performed before this function was called ------------------------------------------------------- @@ -1376,10 +1362,14 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, Eterm head, tail; ptr = list_val(obj); /* off heap list pointers are copied verbatim */ - if (!INHEAP(myself, ptr)) { - Uint bsz; - *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz); - hbot -= bsz; + if (erts_is_literal(obj,ptr)) { + if (!IN_LITERAL_PURGE_AREA(info,ptr)) { + *resp = obj; + } else { + Uint bsz = 0; + *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz); + hbot -= bsz; + } goto cleanup_next; } head = CAR(ptr); @@ -1399,23 +1389,23 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, head = SHTABLE_X(t, e); tail = SHTABLE_Y(t, e); ptr = &(SHTABLE_X(t, e)); - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] tabled L %p is %p\n", myself->common.id, ptr, SHTABLE_REV(t, e))); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] tabled L %p is %p\n", mypid, ptr, SHTABLE_REV(t, e))); SHTABLE_FWD_UPD(t, e, hp); } } /* if not already clean, clean it up and copy it */ if (primary_tag(tail) == TAG_PRIMARY_HEADER) { if (primary_tag(head) == TAG_PRIMARY_HEADER) { - Eterm saved = BITSTORE_GET(b); - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling L/B %p\n", myself->common.id, ptr)); - CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) + saved; - CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) + TAG_PRIMARY_BOXED; + Eterm saved = BITSTORE_GET(b); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling L/B %p\n", mypid, ptr)); + CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) + saved; + CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) + TAG_PRIMARY_BOXED; } else { - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling L/L %p\n", myself->common.id, ptr)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling L/L %p\n", mypid, ptr)); CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) + TAG_PRIMARY_LIST; } } else if (primary_tag(head) == TAG_PRIMARY_HEADER) { - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling L/I %p\n", myself->common.id, ptr)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling L/I %p\n", mypid, ptr)); CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) | primary_tag(tail); CDR(ptr) = tail = (tail - primary_tag(tail)) | TAG_PRIMARY_IMMED1; } else { @@ -1439,10 +1429,14 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, Eterm hdr; ptr = boxed_val(obj); /* off heap pointers to boxes are copied verbatim */ - if (!INHEAP(myself, ptr)) { - Uint bsz; - *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz); - hbot -= bsz; + if (erts_is_literal(obj,ptr)) { + if (!IN_LITERAL_PURGE_AREA(info,ptr)) { + *resp = obj; + } else { + Uint bsz = 0; + *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz); + hbot -= bsz; + } goto cleanup_next; } hdr = *ptr; @@ -1463,13 +1457,13 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, *ptr = (hdr - primary_tag(hdr)) + BOXED_SHARED_PROCESSED; hdr = SHTABLE_X(t, e); ASSERT(primary_tag(hdr) == BOXED_VISITED); - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] tabled B %p is %p\n", myself->common.id, ptr, SHTABLE_REV(t, e))); - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling B %p\n", myself->common.id, ptr)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] tabled B %p is %p\n", mypid, ptr, SHTABLE_REV(t, e))); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling B %p\n", mypid, ptr)); SHTABLE_X(t, e) = hdr = (hdr - BOXED_VISITED) + TAG_PRIMARY_HEADER; SHTABLE_FWD_UPD(t, e, hp); break; case BOXED_VISITED: - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling B %p\n", myself->common.id, ptr)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling B %p\n", mypid, ptr)); *ptr = hdr = (hdr - BOXED_VISITED) + TAG_PRIMARY_HEADER; break; } @@ -1749,13 +1743,13 @@ all_clean: VERBOSE(DEBUG_SHCOPY, ("[copy] restoring shared: %x\n", ptr)); /* entry was a list */ if (SHTABLE_Y(t, e) != THE_NON_VALUE) { - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] untabling L %p\n", myself->common.id, ptr)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] untabling L %p\n", mypid, ptr)); CAR(ptr) = SHTABLE_X(t, e); CDR(ptr) = SHTABLE_Y(t, e); } /* entry was boxed */ else { - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] untabling B %p\n", myself->common.id, ptr)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] untabling B %p\n", mypid, ptr)); *ptr = SHTABLE_X(t, e); ASSERT(primary_tag(*ptr) == TAG_PRIMARY_HEADER); } @@ -1769,9 +1763,9 @@ all_clean: } #endif - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] original was %T\n", myself->common.id, saved_obj)); - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy is %T\n", myself->common.id, result)); - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] result is at %p\n", myself->common.id, result)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] original was %T\n", mypid, saved_obj)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy is %T\n", mypid, result)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] result is at %p\n", mypid, result)); ASSERT(hbot == hp); ASSERT(size == ((hp - *hpp) + (hend - hbot))); -- cgit v1.2.3 From 0d5ee7a4ad7bc41b7cca878990926fb5ba57f6a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Thu, 1 Oct 2015 12:19:50 +0200 Subject: Do not use GCC extensions in copy --- erts/emulator/beam/copy.c | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) (limited to 'erts/emulator/beam/copy.c') diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index 5bca3877b5..83ca527334 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -258,18 +258,19 @@ do { \ } \ } while(0) -#define BITSTORE_GET(s) ({ \ - UWord result; \ - if (WSTK_CONCAT(s,_bitoffs) <= 0) { \ - WSTK_CONCAT(s,_buffer) = s.wstart[WSTK_CONCAT(s,_offset)]; \ - WSTK_CONCAT(s,_offset)++; \ - WSTK_CONCAT(s,_bitoffs) = 8*sizeof(UWord); \ - } \ - WSTK_CONCAT(s,_bitoffs) -= 2; \ - result = WSTK_CONCAT(s,_buffer) & 3; \ - WSTK_CONCAT(s,_buffer) >>= 2; \ - result; \ -}) +#define BITSTORE_FETCH(s,dst) \ +do { \ + UWord result; \ + if (WSTK_CONCAT(s,_bitoffs) <= 0) { \ + WSTK_CONCAT(s,_buffer) = s.wstart[WSTK_CONCAT(s,_offset)]; \ + WSTK_CONCAT(s,_offset)++; \ + WSTK_CONCAT(s,_bitoffs) = 8*sizeof(UWord); \ + } \ + WSTK_CONCAT(s,_bitoffs) -= 2; \ + result = WSTK_CONCAT(s,_buffer) & 3; \ + WSTK_CONCAT(s,_buffer) >>= 2; \ + (dst) = result; \ +} while(0) #define BOXED_VISITED_MASK ((Eterm) 3) #define BOXED_VISITED ((Eterm) 1) @@ -476,7 +477,8 @@ cleanup: /* if not already clean, clean it up */ if (primary_tag(tail) == TAG_PRIMARY_HEADER) { if (primary_tag(head) == TAG_PRIMARY_HEADER) { - Eterm saved = BITSTORE_GET(b); + Eterm saved; + BITSTORE_FETCH(b, saved); CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) | saved; CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) | TAG_PRIMARY_BOXED; } else { @@ -1396,10 +1398,11 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, /* if not already clean, clean it up and copy it */ if (primary_tag(tail) == TAG_PRIMARY_HEADER) { if (primary_tag(head) == TAG_PRIMARY_HEADER) { - Eterm saved = BITSTORE_GET(b); - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling L/B %p\n", mypid, ptr)); - CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) + saved; - CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) + TAG_PRIMARY_BOXED; + Eterm saved; + BITSTORE_FETCH(b, saved); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling L/B %p\n", mypid, ptr)); + CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) + saved; + CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) + TAG_PRIMARY_BOXED; } else { VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling L/L %p\n", mypid, ptr)); CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) + TAG_PRIMARY_LIST; -- cgit v1.2.3 From 5d764f988ab09326d24e39a172083b09ab364c6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Wed, 18 Nov 2015 17:58:14 +0100 Subject: Refactor sharing preserved copy flags The TMPBUF option is no longer needed due to is_literal test and NONE was only used for initial debugging. So we remove the entire option. --- erts/emulator/beam/copy.c | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) (limited to 'erts/emulator/beam/copy.c') diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index 83ca527334..67a96f6442 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -923,10 +923,6 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint * Using an ESTACK but not very transparently; consider refactoring */ -#ifdef SHCOPY_DISABLE -int disable_copy_shared = ERTS_SHCOPY_FLG_NONE; -#endif - #define DECLARE_SHTABLE(s) \ DECLARE_ESTACK(s); \ Uint ESTK_CONCAT(s,_offset) = 0 @@ -1034,7 +1030,7 @@ do { \ * Copy object "obj" preserving sharing. * First half: count size and calculate sharing. */ -Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info, Uint32 flags) +Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info) { Uint sum; Uint e; @@ -1058,13 +1054,6 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info, Uint32 flags) if (IS_CONST(obj)) return 0; -#ifdef SHCOPY_DISABLE - flags |= disable_copy_shared; -#endif - - if (flags & ERTS_SHCOPY_FLG_NONE) - return size_object(obj); - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy_shared_calculate %p\n", mypid, obj)); VERBOSE(DEBUG_SHCOPY, ("[pid=%T] message is %T\n", mypid, obj)); @@ -1299,7 +1288,7 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info, Uint32 flags) * Second half: copy and restore the object. */ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, - Eterm** hpp, ErlOffHeap* off_heap, Uint32 flags) { + Eterm** hpp, ErlOffHeap* off_heap) { Uint e; unsigned sz; Eterm* ptr; @@ -1328,13 +1317,6 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, if (IS_CONST(obj)) return obj; -#ifdef SHCOPY_DISABLE - flags |= disable_copy_shared; -#endif - - if (flags & ERTS_SHCOPY_FLG_NONE) - return copy_struct(obj, size, hpp, off_heap); - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy_shared_perform %p\n", mypid, obj)); /* step #2: was performed before this function was called -- cgit v1.2.3