From dad527f55b51d60e75a0d19aa0f4f42c1065777f Mon Sep 17 00:00:00 2001 From: "Nikolaos S. Papaspyrou" Date: Fri, 8 Jun 2012 22:21:02 +0300 Subject: An implementation of lightweight unbounded queues --- erts/emulator/beam/global.h | 84 ++++++++++++++++++++++++++++++++++++++++++++- erts/emulator/beam/utils.c | 25 ++++++++++++++ 2 files changed, 108 insertions(+), 1 deletion(-) (limited to 'erts') diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 594c0ccf94..20f100b427 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -380,7 +380,7 @@ extern int bif_reductions; /* reductions + fcalls (when doing call_bif) */ extern int stackdump_on_exit; /* - * Here is an implementation of a lightweiht stack. + * Here is an implementation of a lightweight stack. * * Use it like this: * @@ -845,6 +845,88 @@ do {\ } while(0) +/* + * An implementation of lightweight unbounded queues, + * using a circular dynamic array. + * It does not include support for change_allocator. + * + * Use it like this: + * + * DECLARE_EQUEUE(Queue) (At the start of a block) + * ... + * EQUEUE_PUT(Queue, Term) + * ... + * if (EQUEUE_ISEMPTY(Queue)) { + * Queue is empty + * } else { + * Term = EQUEUE_GET(Stack); + * Process popped Term here + * } + * ... + * DESTROY_EQUEUE(Queue) + */ + +typedef struct { + Eterm* start; + Eterm* front; + Eterm* back; + int possibly_empty; + Eterm* end; + ErtsAlcType_t alloc_type; +}ErtsEQueue; + +#define DEF_EQUEUE_SIZE (16) + +void erl_grow_equeue(ErtsEQueue*, Eterm* def_queue); +#define EQUE_CONCAT(a,b) a##b +#define EQUE_DEF_QUEUE(q) EQUE_CONCAT(q,_default_equeue) + +#define DECLARE_EQUEUE(q) \ + UWord EQUE_DEF_QUEUE(q)[DEF_EQUEUE_SIZE]; \ + ErtsEQueue q = { \ + EQUE_DEF_QUEUE(q), /* start */ \ + EQUE_DEF_QUEUE(q), /* front */ \ + EQUE_DEF_QUEUE(q), /* back */ \ + 1, /* possibly_empty */ \ + EQUE_DEF_QUEUE(q) + DEF_EQUEUE_SIZE, /* end */ \ + ERTS_ALC_T_ESTACK /* alloc_type */ \ + } + +#define DESTROY_EQUEUE(q) \ +do { \ + if (q.start != EQUE_DEF_QUEUE(q)) { \ + erts_free(q.alloc_type, q.start); \ + } \ +} while(0) + +#define EQUEUE_PUT_UNCHECKED(q, x) \ +do { \ + q.possibly_empty = 0; \ + *(q.back) = (x); \ + if (++(q.back) == q.end) { \ + q.back = q.start; \ + } \ +} while(0) + +#define EQUEUE_PUT(q, x) \ +do { \ + if (q.back == q.front && !q.possibly_empty) { \ + erl_grow_equeue(&q, EQUE_DEF_QUEUE(q)); \ + } \ + EQUEUE_PUT_UNCHECKED(q, x); \ +} while(0) + +#define EQUEUE_ISEMPTY(q) (q.back == q.front && q.possibly_empty) + +#define EQUEUE_GET(q) ({ \ + q.possibly_empty = 1; \ + UWord x = *(q.front); \ + if (++(q.front) == q.end) { \ + q.front = q.start; \ + } \ + x; \ +}) + /* binary.c */ void erts_emasculate_writable_binary(ProcBin* pb); diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index c3735683bb..184477c36b 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -258,6 +258,31 @@ erl_grow_pstack(ErtsPStack* s, void* default_pstack, unsigned need_bytes) s->psp = s->pstart + sp_offs; } +/* + * Helper function for the EQUEUE macros defined in global.h. + */ + +void +erl_grow_equeue(ErtsEQueue* q, Eterm* default_equeue) +{ + Uint old_size = (q->end - q->start); + Uint new_size = old_size * 2; + Uint first_part = (q->end - q->front); + Uint second_part = (q->back - q->start); + Eterm* new_ptr = erts_alloc(q->alloc_type, new_size*sizeof(Eterm)); + ASSERT(q->back == q->front); // of course the queue is full now! + if (first_part > 0) + sys_memcpy(new_ptr, q->front, first_part*sizeof(Eterm)); + if (second_part > 0) + sys_memcpy(new_ptr+first_part, q->start, second_part*sizeof(Eterm)); + if (q->start != default_equeue) + erts_free(q->alloc_type, q->start); + q->start = new_ptr; + q->end = q->start + new_size; + q->front = q->start; + q->back = q->start + old_size; +} + /* CTYPE macros */ #define LATIN1 -- cgit v1.2.3 From d5711cb70be9ad2e5c78839e8a368900ba248a4a Mon Sep 17 00:00:00 2001 From: "Nikolaos S. Papaspyrou" Date: Fri, 8 Jun 2012 23:13:50 +0300 Subject: Add all the main machinery Add functions size_shared, copy_shared_calculate and copy_shared_perform. Add the infrastructure for making these communicate with each other. Add debug information to other places in the VM, to watch interaction with the sharing-preserving copy. CAUTION: If you define the SHCOPY_DEBUG macro (after SHCOPY is actually used in the VM) and make the whole OTP, there will be a lot of debugging messages during make (it will also be enabled in erlc). You have been warned... --- erts/emulator/beam/copy.c | 1131 +++++++++++++++++++++++++++++++++++++++++++ erts/emulator/beam/erl_gc.c | 10 + erts/emulator/beam/global.h | 56 ++- 3 files changed, 1196 insertions(+), 1 deletion(-) (limited to 'erts') diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index f27c526413..b31e043f08 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -75,8 +75,17 @@ Uint size_object(Eterm obj) Uint sum = 0; Eterm* ptr; int arity; +#ifdef SHCOPY_DEBUG + Eterm mypid; +#endif DECLARE_ESTACK(s); + +#ifdef SHCOPY_DEBUG + mypid = erts_get_current_pid(); + VERBOSE_DEBUG("[pid=%T] size_object %p\n", mypid, obj); +#endif + for (;;) { switch (primary_tag(obj)) { case TAG_PRIMARY_LIST: @@ -211,6 +220,7 @@ Uint size_object(Eterm obj) pop_next: if (ESTACK_ISEMPTY(s)) { DESTROY_ESTACK(s); + VERBOSE_DEBUG("[pid=%T] size was: %u\n", mypid, sum); return sum; } obj = ESTACK_POP(s); @@ -221,6 +231,350 @@ Uint size_object(Eterm obj) } } +/* + * Machinery for sharing preserving information + * Using a WSTACK but not very transparently; consider refactoring + */ + +#define DECLARE_BITSTORE(s) \ + DECLARE_WSTACK(s); \ + int WSTK_CONCAT(s,_bitoffs) = 0; \ + int WSTK_CONCAT(s,_offset) = 0; \ + UWord WSTK_CONCAT(s,_buffer) = 0 + +#define DESTROY_BITSTORE(s) DESTROY_WSTACK(s) +#define BITSTORE_PUT(s,i) \ +do { \ + WSTK_CONCAT(s,_buffer) |= i << WSTK_CONCAT(s,_bitoffs); \ + WSTK_CONCAT(s,_bitoffs) += 2; \ + if (WSTK_CONCAT(s,_bitoffs) >= 8*sizeof(UWord)) { \ + WSTACK_PUSH(s, WSTK_CONCAT(s,_buffer)); \ + WSTK_CONCAT(s,_bitoffs) = 0; \ + WSTK_CONCAT(s,_buffer) = 0; \ + } \ +} while(0) +#define BITSTORE_CLOSE(s) \ +do { \ + if (WSTK_CONCAT(s,_bitoffs) > 0) { \ + WSTACK_PUSH(s, WSTK_CONCAT(s,_buffer)); \ + WSTK_CONCAT(s,_bitoffs) = 0; \ + } \ +} while(0) + +#define BITSTORE_GET(s) ({ \ + UWord result; \ + if (WSTK_CONCAT(s,_bitoffs) <= 0) { \ + WSTK_CONCAT(s,_buffer) = s.wstart[WSTK_CONCAT(s,_offset)]; \ + WSTK_CONCAT(s,_offset)++; \ + WSTK_CONCAT(s,_bitoffs) = 8*sizeof(UWord); \ + } \ + WSTK_CONCAT(s,_bitoffs) -= 2; \ + result = WSTK_CONCAT(s,_buffer) & 3; \ + WSTK_CONCAT(s,_buffer) >>= 2; \ + result; \ +}) + +#define BOXED_VISITED_MASK ((Eterm) 3) +#define BOXED_VISITED ((Eterm) 1) +#define BOXED_SHARED_UNPROCESSED ((Eterm) 2) +#define BOXED_SHARED_PROCESSED ((Eterm) 3) + + +/* + * Is an object in the local heap of a process? + */ + +#define INHEAP_SIMPLE(p, ptr) ( \ + (OLD_HEAP(p) && OLD_HEAP(p) <= ptr && ptr < OLD_HEND(p)) || \ + (HEAP_START(p) <= ptr && ptr < HEAP_END(p)) \ + ) +#define INHEAP(p, ptr) ( \ + INHEAP_SIMPLE(p, ptr) || \ + (force_local ? (force_local = 0, 1) : 0) \ + ) +#define COUNT_OFF_HEAP 0 + + +/* + * Return the real size of an object and find sharing information + * This currently returns the same as erts_debug:size/1. + * It is argued whether the size of subterms in constant pools + * should be counted or not. + */ +#if HALFWORD_HEAP +Uint size_shared_rel(Eterm obj, Eterm* base) +#else +Uint size_shared(Eterm obj) +#endif +{ + Eterm saved_obj = obj; + Uint sum = 0; + Eterm* ptr; + Process* myself; + + DECLARE_EQUEUE(s); + DECLARE_BITSTORE(b); + + myself = erts_get_current_process(); + if (myself == NULL) + return size_object(obj); + + for (;;) { + VERBOSE_DEBUG("[size] visiting: %x ", obj); + switch (primary_tag(obj)) { + case TAG_PRIMARY_LIST: { + Eterm head, tail; + VERBOSE_DEBUG("L"); + ptr = list_val_rel(obj, base); + /* we're not counting anything that's outside our heap */ + if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) { + goto pop_next; + } + head = CAR(ptr); + tail = CDR(ptr); + /* if it's visited, don't count it */ + if (primary_tag(tail) == TAG_PRIMARY_HEADER || + primary_tag(head) == TAG_PRIMARY_HEADER) { + VERBOSE_DEBUG("!"); + goto pop_next; + } + /* else make it visited now */ + switch (primary_tag(tail)) { + case TAG_PRIMARY_LIST: + VERBOSE_DEBUG("/L"); + ptr[1] = (tail - TAG_PRIMARY_LIST) | TAG_PRIMARY_HEADER; + break; + case TAG_PRIMARY_IMMED1: + VERBOSE_DEBUG("/I"); + CAR(ptr) = (head - primary_tag(head)) | TAG_PRIMARY_HEADER; + CDR(ptr) = (tail - TAG_PRIMARY_IMMED1) | primary_tag(head); + break; + case TAG_PRIMARY_BOXED: + VERBOSE_DEBUG("/B saved %d", primary_tag(head)); + BITSTORE_PUT(b, primary_tag(head)); + CAR(ptr) = (head - primary_tag(head)) | TAG_PRIMARY_HEADER; + CDR(ptr) = (tail - TAG_PRIMARY_BOXED) | TAG_PRIMARY_HEADER; + break; + } + /* and count it */ + sum += 2; + if (!IS_CONST(head)) { + EQUEUE_PUT(s, head); + } + obj = tail; + break; + } + case TAG_PRIMARY_BOXED: { + Eterm hdr; + VERBOSE_DEBUG("B"); + ptr = boxed_val_rel(obj, base); + /* we're not counting anything that's outside our heap */ + if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) { + goto pop_next; + } + hdr = *ptr; + /* if it's visited, don't count it */ + if (primary_tag(hdr) != TAG_PRIMARY_HEADER) { + VERBOSE_DEBUG("!"); + goto pop_next; + } + /* else make it visited now */ + *ptr = (hdr - primary_tag(hdr)) + BOXED_VISITED; + /* and count it */ + ASSERT(is_header(hdr)); + switch (hdr & _TAG_HEADER_MASK) { + case ARITYVAL_SUBTAG: { + int arity = header_arity(hdr); + VERBOSE_DEBUG("/T"); + sum += arity + 1; + if (arity == 0) { /* Empty tuple -- unusual. */ + VERBOSE_DEBUG("e"); + goto pop_next; + } + while (arity-- > 0) { + obj = *++ptr; + if (!IS_CONST(obj)) { + EQUEUE_PUT(s, obj); + } + } + goto pop_next; + } + case FUN_SUBTAG: { + ErlFunThing* funp = (ErlFunThing *) ptr; + unsigned eterms = 1 /* creator */ + funp->num_free; + unsigned sz = thing_arityval(hdr); + VERBOSE_DEBUG("/F"); + sum += 1 /* header */ + sz + eterms; + ptr += 1 /* header */ + sz; + while (eterms-- > 0) { + obj = *ptr++; + if (!IS_CONST(obj)) { + EQUEUE_PUT(s, obj); + } + } + goto pop_next; + } + case SUB_BINARY_SUBTAG: { + ErlSubBin* sb = (ErlSubBin *) ptr; + Uint extra_bytes; + Eterm hdr; + ASSERT((sb->thing_word & ~BOXED_VISITED_MASK) == HEADER_SUB_BIN); + if (sb->bitsize + sb->bitoffs > 8) { + sum += ERL_SUB_BIN_SIZE; + extra_bytes = 2; + } else if (sb->bitsize + sb->bitoffs > 0) { + sum += ERL_SUB_BIN_SIZE; + extra_bytes = 1; + } else { + extra_bytes = 0; + } + ptr = binary_val_rel(sb->orig, base); + hdr = (*ptr) & ~BOXED_VISITED_MASK; + if (thing_subtag(hdr) == REFC_BINARY_SUBTAG) { + sum += PROC_BIN_SIZE; + } else { + ASSERT(thing_subtag(hdr) == HEAP_BINARY_SUBTAG); + sum += heap_bin_size(binary_size_rel(obj, base) + extra_bytes); + } + goto pop_next; + } + case BIN_MATCHSTATE_SUBTAG: + erl_exit(ERTS_ABORT_EXIT, + "size_shared: matchstate term not allowed"); + default: + VERBOSE_DEBUG("/D"); + sum += thing_arityval(hdr) + 1; + goto pop_next; + } + break; + } + case TAG_PRIMARY_IMMED1: + VERBOSE_DEBUG("I"); + pop_next: + if (EQUEUE_ISEMPTY(s)) { + goto cleanup; + } + obj = EQUEUE_GET(s); + break; + default: + erl_exit(ERTS_ABORT_EXIT, "size_shared: bad tag for %#x\n", obj); + } + VERBOSE_DEBUG("\n"); + } + +cleanup: + VERBOSE_DEBUG("\n"); + obj = saved_obj; + BITSTORE_CLOSE(b); + for (;;) { + VERBOSE_DEBUG("[size] revisiting: %x ", obj); + switch (primary_tag(obj)) { + case TAG_PRIMARY_LIST: { + Eterm head, tail; + VERBOSE_DEBUG("L"); + ptr = list_val_rel(obj, base); + if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) { + goto cleanup_next; + } + head = CAR(ptr); + tail = CDR(ptr); + /* if not already clean, clean it up */ + if (primary_tag(tail) == TAG_PRIMARY_HEADER) { + if (primary_tag(head) == TAG_PRIMARY_HEADER) { + Eterm saved = BITSTORE_GET(b); + VERBOSE_DEBUG("/B restoring %d", saved); + CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) | saved; + CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) | TAG_PRIMARY_BOXED; + } else { + VERBOSE_DEBUG("/L"); + CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) | TAG_PRIMARY_LIST; + } + } else if (primary_tag(head) == TAG_PRIMARY_HEADER) { + VERBOSE_DEBUG("/I"); + CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) | primary_tag(tail); + CDR(ptr) = tail = (tail - primary_tag(tail)) | TAG_PRIMARY_IMMED1; + } else { + VERBOSE_DEBUG("!"); + goto cleanup_next; + } + /* and its children too */ + if (!IS_CONST(head)) { + EQUEUE_PUT_UNCHECKED(s, head); + } + obj = tail; + break; + } + case TAG_PRIMARY_BOXED: { + Eterm hdr; + VERBOSE_DEBUG("B"); + ptr = boxed_val_rel(obj, base); + if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) { + goto cleanup_next; + } + hdr = *ptr; + /* if not already clean, clean it up */ + if (primary_tag(hdr) == TAG_PRIMARY_HEADER) { + goto cleanup_next; + } + else { + ASSERT(primary_tag(hdr) == BOXED_VISITED); + *ptr = hdr = (hdr - BOXED_VISITED) + TAG_PRIMARY_HEADER; + } + /* and its children too */ + switch (hdr & _TAG_HEADER_MASK) { + case ARITYVAL_SUBTAG: { + int arity = header_arity(hdr); + if (arity == 0) { /* Empty tuple -- unusual. */ + goto cleanup_next; + } + while (arity-- > 0) { + obj = *++ptr; + if (!IS_CONST(obj)) { + EQUEUE_PUT_UNCHECKED(s, obj); + } + } + goto cleanup_next; + } + case FUN_SUBTAG: { + ErlFunThing* funp = (ErlFunThing *) ptr; + unsigned eterms = 1 /* creator */ + funp->num_free; + unsigned sz = thing_arityval(hdr); + ptr += 1 /* header */ + sz; + while (eterms-- > 0) { + obj = *ptr++; + if (!IS_CONST(obj)) { + EQUEUE_PUT_UNCHECKED(s, obj); + } + } + goto cleanup_next; + } + default: + goto cleanup_next; + } + break; + } + case TAG_PRIMARY_IMMED1: + cleanup_next: + if (EQUEUE_ISEMPTY(s)) { + goto all_clean; + } + obj = EQUEUE_GET(s); + break; + default: + erl_exit(ERTS_ABORT_EXIT, "size_shared: bad tag for %#x\n", obj); + } + VERBOSE_DEBUG("\n"); + } + + all_clean: + VERBOSE_DEBUG("\n"); + /* Return the result */ + DESTROY_EQUEUE(s); + DESTROY_BITSTORE(b); + return sum; +} + + /* * Copy a structure to a heap. */ @@ -244,10 +598,18 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) Eterm org_obj = obj; Uint org_sz = sz; #endif +#ifdef SHCOPY_DEBUG + Eterm mypid; +#endif if (IS_CONST(obj)) return obj; +#ifdef SHCOPY_DEBUG + mypid = erts_get_current_pid(); + VERBOSE_DEBUG("[pid=%T] copy_struct %p\n", mypid, obj); +#endif + DTRACE1(copy_struct, (int32_t)sz); hp = htop = *hpp; @@ -529,9 +891,778 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) } #endif *hpp = (Eterm *) (hstart+hsize); + VERBOSE_DEBUG("[pid=%T] result is at %p\n", mypid, res); return res; } + +/* + * Machinery for the table used by the sharing preserving copier + * Using an ESTACK but not very transparently; consider refactoring + */ + +#define DECLARE_SHTABLE(s) \ + DECLARE_ESTACK(s); \ + Uint ESTK_CONCAT(s,_offset) = 0 +#define DESTROY_SHTABLE(s) DESTROY_ESTACK(s) +#define SHTABLE_INCR 4 +#define SHTABLE_NEXT(s) ESTK_CONCAT(s,_offset) +#define SHTABLE_PUSH(s,x,y,b) \ +do { \ + if (s.sp > s.end - SHTABLE_INCR) { \ + erl_grow_estack(&s, ESTK_DEF_STACK(s)); \ + } \ + *s.sp++ = (x); \ + *s.sp++ = (y); \ + *s.sp++ = (Eterm) NULL; \ + *s.sp++ = (Eterm) (b); /* bad in HALF_WORD */ \ + ESTK_CONCAT(s,_offset) += SHTABLE_INCR; \ +} while(0) +#define SHTABLE_X(s,e) (s.start[e]) +#define SHTABLE_Y(s,e) (s.start[(e)+1]) +#define SHTABLE_FWD(s,e) ((Eterm *) (s.start[(e)+2])) +#define SHTABLE_FWD_UPD(s,e,p) (s.start[(e)+2] = (Eterm) (p)) +#define SHTABLE_REV(s,e) ((Eterm *) (s.start[(e)+3])) + +#define LIST_SHARED_UNPROCESSED ((Eterm) 0) +#define LIST_SHARED_PROCESSED ((Eterm) 1) + +#define HEAP_ELEM_TO_BE_FILLED _unchecked_make_list(NULL) + + +/* + * Specialized macros for using/reusing the persistent state + */ + +#define DECLARE_EQUEUE_INIT_INFO(q, info) \ + UWord* EQUE_DEF_QUEUE(q) = info->queue_default; \ + ErtsEQueue q = { \ + EQUE_DEF_QUEUE(q), /* start */ \ + EQUE_DEF_QUEUE(q), /* front */ \ + EQUE_DEF_QUEUE(q), /* back */ \ + 1, /* possibly_empty */ \ + EQUE_DEF_QUEUE(q) + DEF_EQUEUE_SIZE, /* end */ \ + ERTS_ALC_T_ESTACK /* alloc_type */ \ + } + +#define DECLARE_EQUEUE_FROM_INFO(q, info) \ + /* no EQUE_DEF_QUEUE(q), read-only */ \ + ErtsEQueue q = { \ + info->queue_start, /* start */ \ + info->queue_start, /* front */ \ + info->queue_start, /* back */ \ + 1, /* possibly_empty */ \ + info->queue_end, /* end */ \ + info->queue_alloc_type /* alloc_type */ \ + } + +#define DECLARE_BITSTORE_INIT_INFO(s, info) \ + UWord* WSTK_DEF_STACK(s) = info->bitstore_default; \ + ErtsWStack s = { \ + WSTK_DEF_STACK(s), /* wstart */ \ + WSTK_DEF_STACK(s), /* wsp */ \ + WSTK_DEF_STACK(s) + DEF_WSTACK_SIZE, /* wend */ \ + ERTS_ALC_T_ESTACK /* alloc_type */ \ + }; \ + int WSTK_CONCAT(s,_bitoffs) = 0; \ + /* no WSTK_CONCAT(s,_offset), write-only */ \ + UWord WSTK_CONCAT(s,_buffer) = 0 + +#define DECLARE_BITSTORE_FROM_INFO(s, info) \ + /* no WSTK_DEF_STACK(s), read-only */ \ + ErtsWStack s = { \ + info->bitstore_start, /* wstart */ \ + NULL, /* wsp, read-only */ \ + NULL, /* wend, read-only */ \ + info->bitstore_alloc_type /* alloc_type */ \ + }; \ + int WSTK_CONCAT(s,_bitoffs) = 0; \ + int WSTK_CONCAT(s,_offset) = 0; \ + UWord WSTK_CONCAT(s,_buffer) = 0 + +#define DECLARE_SHTABLE_INIT_INFO(s, info) \ + Eterm* ESTK_DEF_STACK(s) = info->shtable_default; \ + ErtsEStack s = { \ + ESTK_DEF_STACK(s), /* start */ \ + ESTK_DEF_STACK(s), /* sp */ \ + ESTK_DEF_STACK(s) + DEF_ESTACK_SIZE, /* end */ \ + ERTS_ALC_T_ESTACK /* alloc_type */ \ + }; \ + Uint ESTK_CONCAT(s,_offset) = 0 + +#define DECLARE_SHTABLE_FROM_INFO(s, info) \ + /* no ESTK_DEF_STACK(s), read-only */ \ + ErtsEStack s = { \ + info->shtable_start, /* start */ \ + NULL, /* sp, read-only */ \ + NULL, /* end, read-only */ \ + info->shtable_alloc_type /* alloc_type */ \ + }; \ + /* no ESTK_CONCAT(s,_offset), read-only */ + +/* + * Copy object "obj" preserving sharing. + * First half: count size and calculate sharing. + * NOTE: We do not support HALF_WORD (yet?). + */ +Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) +{ + Uint sum; + Uint e; + unsigned sz; + Eterm* ptr; + Process* myself; + int force_local = flags & ERTS_SHCOPY_FLG_TMP_BUF; + + DECLARE_EQUEUE_INIT_INFO(s, info); + DECLARE_BITSTORE_INIT_INFO(b, info); + DECLARE_SHTABLE_INIT_INFO(t, info); + + /* step #0: + ------------------------------------------------------- + get rid of the easy cases first: + - copying constants + - if not a proper process, do flat copy + */ + + if (IS_CONST(obj)) + return 0; + + myself = erts_get_current_process(); + if (myself == NULL || (flags & ERTS_SHCOPY_FLG_NONE)) + return size_object(obj); + + VERBOSE_DEBUG("[pid=%T] copy_shared_calculate %p\n", myself->common.id, obj); + VERBOSE_DEBUG("[pid=%T] message is %T\n", myself->common.id, obj); + + /* step #1: + ------------------------------------------------------- + traverse the term and calculate the size; + when traversing, transform as you do in size_shared + but when you find shared objects: + + a. add entry in the table, indexed by i + b. mark them: + b1. boxed terms, set header to (i | 11) + store (old header, NONV, NULL, backptr) in the entry + b2. cons cells, set CDR to NONV, set CAR to i + store (old CAR, old CDR, NULL, backptr) in the entry + */ + + sum = 0; + + for (;;) { + VERBOSE_DEBUG("[copy] visiting: %x ", obj); + switch (primary_tag(obj)) { + case TAG_PRIMARY_LIST: { + Eterm head, tail; + VERBOSE_DEBUG("L"); + ptr = list_val_rel(obj, base); + /* off heap list pointers are copied verbatim */ + if (!INHEAP(myself, ptr)) { + VERBOSE_DEBUG("[pid=%T] bypassed copying %p is %T\n", myself->common.id, ptr, obj); + if (myself->mbuf != NULL) + VERBOSE_DEBUG("[pid=%T] BUT !!! there are message buffers!\n", myself->common.id); + VERBOSE_DEBUG("#"); + goto pop_next; + } + head = CAR(ptr); + tail = CDR(ptr); + /* if it's visited, don't count it; + if not already shared, make it shared and store it in the table */ + if (primary_tag(tail) == TAG_PRIMARY_HEADER || + primary_tag(head) == TAG_PRIMARY_HEADER) { + VERBOSE_DEBUG("!"); + if (tail != THE_NON_VALUE) { + e = SHTABLE_NEXT(t); + VERBOSE_DEBUG("[pid=%T] tabling L %p\n", myself->common.id, ptr); + SHTABLE_PUSH(t, head, tail, ptr); + CAR(ptr) = (e << _TAG_PRIMARY_SIZE) | LIST_SHARED_UNPROCESSED; + CDR(ptr) = THE_NON_VALUE; + } + goto pop_next; + } + /* else make it visited now */ + switch (primary_tag(tail)) { + case TAG_PRIMARY_LIST: + VERBOSE_DEBUG("/L"); + VERBOSE_DEBUG("[pid=%T] mangling L/L %p\n", myself->common.id, ptr); + CDR(ptr) = (tail - TAG_PRIMARY_LIST) | TAG_PRIMARY_HEADER; + break; + case TAG_PRIMARY_IMMED1: + VERBOSE_DEBUG("/I"); + VERBOSE_DEBUG("[pid=%T] mangling L/I %p\n", myself->common.id, ptr); + CAR(ptr) = (head - primary_tag(head)) | TAG_PRIMARY_HEADER; + CDR(ptr) = (tail - TAG_PRIMARY_IMMED1) | primary_tag(head); + break; + case TAG_PRIMARY_BOXED: + VERBOSE_DEBUG("/B saved %d", primary_tag(head)); + BITSTORE_PUT(b, primary_tag(head)); + VERBOSE_DEBUG("[pid=%T] mangling L/B %p\n", myself->common.id, ptr); + CAR(ptr) = (head - primary_tag(head)) | TAG_PRIMARY_HEADER; + CDR(ptr) = (tail - TAG_PRIMARY_BOXED) | TAG_PRIMARY_HEADER; + break; + } + /* and count it */ + sum += 2; + if (!IS_CONST(head)) { + EQUEUE_PUT(s, head); + } + obj = tail; + break; + } + case TAG_PRIMARY_BOXED: { + Eterm hdr; + VERBOSE_DEBUG("B"); + ptr = boxed_val_rel(obj, base); + /* off heap pointers to boxes are copied verbatim */ + if (!INHEAP(myself, ptr)) { + VERBOSE_DEBUG("[pid=%T] bypassed copying %p is %T\n", myself->common.id, ptr, obj); + VERBOSE_DEBUG("#"); + goto pop_next; + } + hdr = *ptr; + /* if it's visited, don't count it; + if not already shared, make it shared and store it in the table */ + if (primary_tag(hdr) != TAG_PRIMARY_HEADER) { + VERBOSE_DEBUG("!"); + if (primary_tag(hdr) == BOXED_VISITED) { + e = SHTABLE_NEXT(t); + VERBOSE_DEBUG("[pid=%T] tabling B %p\n", myself->common.id, ptr); + SHTABLE_PUSH(t, hdr, THE_NON_VALUE, ptr); + *ptr = (e << _TAG_PRIMARY_SIZE) | BOXED_SHARED_UNPROCESSED; + } + goto pop_next; + } + /* else make it visited now */ + VERBOSE_DEBUG("[pid=%T] mangling B %p\n", myself->common.id, ptr); + *ptr = (hdr - primary_tag(hdr)) + BOXED_VISITED; + /* and count it */ + ASSERT(is_header(hdr)); + switch (hdr & _TAG_HEADER_MASK) { + case ARITYVAL_SUBTAG: { + int arity = header_arity(hdr); + VERBOSE_DEBUG("/T"); + sum += arity + 1; + if (arity == 0) { /* Empty tuple -- unusual. */ + VERBOSE_DEBUG("e"); + goto pop_next; + } + while (arity-- > 0) { + obj = *++ptr; + if (!IS_CONST(obj)) { + EQUEUE_PUT(s, obj); + } + } + goto pop_next; + } + case FUN_SUBTAG: { + ErlFunThing* funp = (ErlFunThing *) ptr; + unsigned eterms = 1 /* creator */ + funp->num_free; + sz = thing_arityval(hdr); + VERBOSE_DEBUG("/F"); + sum += 1 /* header */ + sz + eterms; + ptr += 1 /* header */ + sz; + while (eterms-- > 0) { + obj = *ptr++; + if (!IS_CONST(obj)) { + EQUEUE_PUT(s, obj); + } + } + goto pop_next; + } + case SUB_BINARY_SUBTAG: { + ErlSubBin* sb = (ErlSubBin *) ptr; + Eterm real_bin = sb->orig; + Uint bit_offset = sb->bitoffs; + Uint bit_size = sb->bitsize; + size_t size = sb->size; + Uint extra_bytes; + Eterm hdr; + if (bit_size + bit_offset > 8) { + sum += ERL_SUB_BIN_SIZE; + extra_bytes = 2; + } else if (bit_size + bit_offset > 0) { + sum += ERL_SUB_BIN_SIZE; + extra_bytes = 1; + } else { + extra_bytes = 0; + } + ASSERT(is_boxed(rterm2wterm(real_bin, base)) && + (((*boxed_val(rterm2wterm(real_bin, base))) & + (_TAG_HEADER_MASK - _BINARY_XXX_MASK - BOXED_VISITED_MASK)) + == _TAG_HEADER_REFC_BIN)); + hdr = *_unchecked_binary_val(rterm2wterm(real_bin, base)) & ~BOXED_VISITED_MASK; + if (thing_subtag(hdr) == HEAP_BINARY_SUBTAG) { + sum += heap_bin_size(size+extra_bytes); + } else { + ASSERT(thing_subtag(hdr) == REFC_BINARY_SUBTAG); + sum += PROC_BIN_SIZE; + } + goto pop_next; + } + case BIN_MATCHSTATE_SUBTAG: + erl_exit(ERTS_ABORT_EXIT, + "size_shared: matchstate term not allowed"); + default: + VERBOSE_DEBUG("/D"); + sum += thing_arityval(hdr) + 1; + goto pop_next; + } + break; + } + case TAG_PRIMARY_IMMED1: + VERBOSE_DEBUG("I"); + pop_next: + if (EQUEUE_ISEMPTY(s)) { + VERBOSE_DEBUG("\n"); + // add sentinel to the table + SHTABLE_PUSH(t, THE_NON_VALUE, THE_NON_VALUE, NULL); + // store persistent info + BITSTORE_CLOSE(b); + info->queue_start = s.start; + info->queue_end = s.end; + info->queue_alloc_type = s.alloc_type; + info->bitstore_start = b.wstart; + info->bitstore_alloc_type = b.alloc_type; + info->shtable_start = t.start; + info->shtable_alloc_type = t.alloc_type; + // single point of return: the size of the object + VERBOSE_DEBUG("[pid=%T] size was: %u\n", myself->common.id, sum); + return sum; + } + obj = EQUEUE_GET(s); + break; + default: + erl_exit(ERTS_ABORT_EXIT, "[pid=%T] size_shared: bad tag for %#x\n", obj); + } + VERBOSE_DEBUG("\n"); + } +} + + +/* + * Copy object "obj" preserving sharing. + * Second half: copy and restore the object. + * NOTE: We do not support HALF_WORD (yet?). + */ +Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, ErlOffHeap* off_heap, unsigned flags) +{ + Uint e; + unsigned sz; + Eterm* ptr; + Eterm* hp; + Eterm* hscan; + Eterm result; + Eterm* resp; + unsigned remaining; + Process* myself; + int force_local = flags & ERTS_SHCOPY_FLG_TMP_BUF; +#if defined(DEBUG) || defined(SHCOPY_DEBUG) + Eterm saved_obj = obj; +#endif + + DECLARE_EQUEUE_FROM_INFO(s, info); + DECLARE_BITSTORE_FROM_INFO(b, info); + DECLARE_SHTABLE_FROM_INFO(t, info); + + /* step #0: + ------------------------------------------------------- + get rid of the easy cases first: + - copying constants + - if not a proper process, do flat copy + */ + + if (IS_CONST(obj)) + return obj; + + myself = erts_get_current_process(); + if (myself == NULL || (flags & ERTS_SHCOPY_FLG_NONE)) + return copy_struct(obj, size, hpp, off_heap); + + VERBOSE_DEBUG("[pid=%T] copy_shared_perform %p\n", myself->common.id, obj); + + /* step #2: was performed before this function was called + ------------------------------------------------------- + allocate new space + */ + + hscan = hp = *hpp; + + /* step #3: + ------------------------------------------------------- + traverse the term a second time and when traversing: + a. if the object is marked as shared + a1. if the entry contains a forwarding ptr, use that + a2. otherwise, copy it to the new space and store the + forwarding ptr to the entry + b. otherwise, reverse-transform as you do in size_shared + and copy to the new space + */ + + resp = &result; + remaining = 0; + for (;;) { + VERBOSE_DEBUG("[copy] revisiting: %x ", obj); + switch (primary_tag(obj)) { + case TAG_PRIMARY_LIST: { + Eterm head, tail; + VERBOSE_DEBUG("L"); + ptr = list_val_rel(obj, base); + /* off heap list pointers are copied verbatim */ + if (!INHEAP(myself, ptr)) { + VERBOSE_DEBUG("#"); + *resp = obj; + goto cleanup_next; + } + head = CAR(ptr); + tail = CDR(ptr); + /* if it is shared */ + if (tail == THE_NON_VALUE) { + e = head >> _TAG_PRIMARY_SIZE; + /* if it has been processed, just use the forwarding pointer */ + if (primary_tag(head) == LIST_SHARED_PROCESSED) { + VERBOSE_DEBUG("!"); + *resp = make_list(SHTABLE_FWD(t, e)); + goto cleanup_next; + } + /* else, let's process it now, + copy it and keep the forwarding pointer */ + else { + VERBOSE_DEBUG("$"); + CAR(ptr) = (head - primary_tag(head)) + LIST_SHARED_PROCESSED; + head = SHTABLE_X(t, e); + tail = SHTABLE_Y(t, e); + ptr = &(SHTABLE_X(t, e)); + VERBOSE_DEBUG("[pid=%T] tabled L %p is %p\n", myself->common.id, ptr, SHTABLE_REV(t, e)); + SHTABLE_FWD_UPD(t, e, hp); + } + } + /* if not already clean, clean it up and copy it */ + if (primary_tag(tail) == TAG_PRIMARY_HEADER) { + if (primary_tag(head) == TAG_PRIMARY_HEADER) { + Eterm saved = BITSTORE_GET(b); + VERBOSE_DEBUG("/B restoring %d", saved); + VERBOSE_DEBUG("[pid=%T] unmangling L/B %p\n", myself->common.id, ptr); + CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) + saved; + CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) + TAG_PRIMARY_BOXED; + } else { + VERBOSE_DEBUG("/L"); + VERBOSE_DEBUG("[pid=%T] unmangling L/L %p\n", myself->common.id, ptr); + CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) + TAG_PRIMARY_LIST; + } + } else if (primary_tag(head) == TAG_PRIMARY_HEADER) { + VERBOSE_DEBUG("/I"); + VERBOSE_DEBUG("[pid=%T] unmangling L/I %p\n", myself->common.id, ptr); + CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) | primary_tag(tail); + CDR(ptr) = tail = (tail - primary_tag(tail)) | TAG_PRIMARY_IMMED1; + } else { + ASSERT(0 && "cannot come here"); + goto cleanup_next; + } + /* and its children too */ + if (IS_CONST(head)) { + CAR(hp) = head; + } else { + EQUEUE_PUT_UNCHECKED(s, head); + CAR(hp) = HEAP_ELEM_TO_BE_FILLED; + } + *resp = make_list(hp); + resp = &(CDR(hp)); + hp += 2; + obj = tail; + break; + } + case TAG_PRIMARY_BOXED: { + Eterm hdr; + VERBOSE_DEBUG("B"); + ptr = boxed_val_rel(obj, base); + /* off heap pointers to boxes are copied verbatim */ + if (!INHEAP(myself, ptr)) { + VERBOSE_DEBUG("#"); + *resp = obj; + goto cleanup_next; + } + hdr = *ptr; + /* clean it up, unless it's already clean or shared and processed */ + switch (primary_tag(hdr)) { + case TAG_PRIMARY_HEADER: + ASSERT(0 && "cannot come here"); + /* if it is shared and has been processed, + just use the forwarding pointer */ + case BOXED_SHARED_PROCESSED: + VERBOSE_DEBUG("!"); + e = hdr >> _TAG_PRIMARY_SIZE; + *resp = make_boxed(SHTABLE_FWD(t, e)); + goto cleanup_next; + /* if it is shared but has not been processed yet, let's process + it now: copy it and keep the forwarding pointer */ + case BOXED_SHARED_UNPROCESSED: + e = hdr >> _TAG_PRIMARY_SIZE; + VERBOSE_DEBUG("$"); + *ptr = (hdr - primary_tag(hdr)) + BOXED_SHARED_PROCESSED; + hdr = SHTABLE_X(t, e); + ASSERT(primary_tag(hdr) == BOXED_VISITED); + VERBOSE_DEBUG("[pid=%T] tabled B %p is %p\n", myself->common.id, ptr, SHTABLE_REV(t, e)); + VERBOSE_DEBUG("[pid=%T] unmangling B %p\n", myself->common.id, ptr); + SHTABLE_X(t, e) = hdr = (hdr - BOXED_VISITED) + TAG_PRIMARY_HEADER; + SHTABLE_FWD_UPD(t, e, hp); + break; + case BOXED_VISITED: + VERBOSE_DEBUG("[pid=%T] unmangling B %p\n", myself->common.id, ptr); + *ptr = hdr = (hdr - BOXED_VISITED) + TAG_PRIMARY_HEADER; + break; + } + /* and its children too */ + switch (hdr & _TAG_HEADER_MASK) { + case ARITYVAL_SUBTAG: { + int arity = header_arity(hdr); + *resp = make_boxed(hp); + *hp++ = hdr; + while (arity-- > 0) { + obj = *++ptr; + if (IS_CONST(obj)) { + *hp++ = obj; + } else { + EQUEUE_PUT_UNCHECKED(s, obj); + *hp++ = HEAP_ELEM_TO_BE_FILLED; + } + } + goto cleanup_next; + } + case FUN_SUBTAG: { + ErlFunThing* funp = (ErlFunThing *) ptr; + unsigned eterms = 1 /* creator */ + funp->num_free; + sz = thing_arityval(hdr); + funp = (ErlFunThing *) hp; + *resp = make_fun(hp); + *hp++ = hdr; + ptr++; + while (sz-- > 0) { + *hp++ = *ptr++; + } + while (eterms-- > 0) { + obj = *ptr++; + if (IS_CONST(obj)) { + *hp++ = obj; + } else { + EQUEUE_PUT_UNCHECKED(s, obj); + *hp++ = HEAP_ELEM_TO_BE_FILLED; + } + } + funp->next = off_heap->first; + off_heap->first = (struct erl_off_heap_header*) funp; + erts_refc_inc(&funp->fe->refc, 2); + goto cleanup_next; + } + case REFC_BINARY_SUBTAG: { + ProcBin* pb = (ProcBin *) ptr; + sz = thing_arityval(hdr); + if (pb->flags) { + erts_emasculate_writable_binary(pb); + } + pb = (ProcBin *) hp; + *resp = make_binary(hp); + *hp++ = hdr; + ptr++; + while (sz-- > 0) { + *hp++ = *ptr++; + } + erts_refc_inc(&pb->val->refc, 2); + pb->next = off_heap->first; + pb->flags = 0; + off_heap->first = (struct erl_off_heap_header*) pb; + OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm)); + goto cleanup_next; + } + case SUB_BINARY_SUBTAG: { + ErlSubBin* sb = (ErlSubBin *) ptr; + Eterm real_bin = sb->orig; + Uint bit_offset = sb->bitoffs; + Uint bit_size = sb->bitsize; + Uint offset = sb->offs; + size_t size = sb->size; + Uint extra_bytes; + Uint real_size; + if ((bit_size + bit_offset) > 8) { + extra_bytes = 2; + } else if ((bit_size + bit_offset) > 0) { + extra_bytes = 1; + } else { + extra_bytes = 0; + } + real_size = size+extra_bytes; + ASSERT(is_boxed(rterm2wterm(real_bin, base)) && + (((*boxed_val(rterm2wterm(real_bin, base))) & + (_TAG_HEADER_MASK - _BINARY_XXX_MASK - BOXED_VISITED_MASK)) + == _TAG_HEADER_REFC_BIN)); + ptr = _unchecked_binary_val(rterm2wterm(real_bin, base)); + *resp = make_binary(hp); + if (extra_bytes != 0) { + ErlSubBin* res = (ErlSubBin *) hp; + hp += ERL_SUB_BIN_SIZE; + res->thing_word = HEADER_SUB_BIN; + res->size = size; + res->bitsize = bit_size; + res->bitoffs = bit_offset; + res->offs = 0; + res->is_writable = 0; + res->orig = make_binary(hp); + } + if (thing_subtag(*ptr & ~BOXED_VISITED_MASK) == HEAP_BINARY_SUBTAG) { + ErlHeapBin* from = (ErlHeapBin *) ptr; + ErlHeapBin* to = (ErlHeapBin *) hp; + hp += heap_bin_size(real_size); + to->thing_word = header_heap_bin(real_size); + to->size = real_size; + sys_memcpy(to->data, ((byte *)from->data)+offset, real_size); + } else { + ProcBin* from = (ProcBin *) ptr; + ProcBin* to = (ProcBin *) hp; + ASSERT(thing_subtag(*ptr & ~BOXED_VISITED_MASK) == REFC_BINARY_SUBTAG); + if (from->flags) { + erts_emasculate_writable_binary(from); + } + hp += PROC_BIN_SIZE; + to->thing_word = HEADER_PROC_BIN; + to->size = real_size; + to->val = from->val; + erts_refc_inc(&to->val->refc, 2); + to->bytes = from->bytes + offset; + to->next = off_heap->first; + to->flags = 0; + off_heap->first = (struct erl_off_heap_header*) to; + OH_OVERHEAD(off_heap, to->size / sizeof(Eterm)); + } + goto cleanup_next; + } + case EXTERNAL_PID_SUBTAG: + case EXTERNAL_PORT_SUBTAG: + case EXTERNAL_REF_SUBTAG: { + ExternalThing *etp = (ExternalThing *) hp; + sz = thing_arityval(hdr); + *resp = make_external(hp); + *hp++ = hdr; + ptr++; + while (sz-- > 0) { + *hp++ = *ptr++; + } + etp->next = off_heap->first; + off_heap->first = (struct erl_off_heap_header*) etp; + erts_refc_inc(&etp->node->refc, 2); + goto cleanup_next; + } + default: + sz = thing_arityval(hdr); + *resp = make_boxed(hp); + *hp++ = hdr; + ptr++; + while (sz-- > 0) { + *hp++ = *ptr++; + } + goto cleanup_next; + } + break; + } + case TAG_PRIMARY_IMMED1: + *resp = obj; + cleanup_next: + if (EQUEUE_ISEMPTY(s)) { + goto all_clean; + } + obj = EQUEUE_GET(s); + for (;;) { + ASSERT(hscan < hp); + if (remaining == 0) { + if (*hscan == HEAP_ELEM_TO_BE_FILLED) { + resp = hscan; + hscan += 2; + break; /* scanning loop */ + } else if (primary_tag(*hscan) == TAG_PRIMARY_HEADER) { + switch (*hscan & _TAG_HEADER_MASK) { + case ARITYVAL_SUBTAG: + remaining = header_arity(*hscan); + hscan++; + break; + case FUN_SUBTAG: { + ErlFunThing* funp = (ErlFunThing *) hscan; + hscan += 1 + thing_arityval(*hscan); + remaining = 1 + funp->num_free; + break; + } + case SUB_BINARY_SUBTAG: + ASSERT(((ErlSubBin *) hscan)->bitoffs + + ((ErlSubBin *) hscan)->bitsize > 0); + hscan += ERL_SUB_BIN_SIZE; + break; + default: + hscan += 1 + thing_arityval(*hscan); + break; + } + } else { + hscan++; + } + } else if (*hscan == HEAP_ELEM_TO_BE_FILLED) { + resp = hscan++; + remaining--; + break; /* scanning loop */ + } else { + hscan++; + remaining--; + } + } + ASSERT(resp < hp); + break; + default: + erl_exit(ERTS_ABORT_EXIT, "size_shared: bad tag for %#x\n", obj); + } + VERBOSE_DEBUG("\n"); + } + + /* step #4: + ------------------------------------------------------- + traverse the table and reverse-transform all stored entries + */ + +all_clean: + VERBOSE_DEBUG("\n"); + for (e = 0; ; e += SHTABLE_INCR) { + ptr = SHTABLE_REV(t, e); + if (ptr == NULL) + break; + VERBOSE_DEBUG("[copy] restoring shared: %x\n", ptr); + /* entry was a list */ + if (SHTABLE_Y(t, e) != THE_NON_VALUE) { + VERBOSE_DEBUG("[pid=%T] untabling L %p\n", myself->common.id, ptr); + CAR(ptr) = SHTABLE_X(t, e); + CDR(ptr) = SHTABLE_Y(t, e); + } + /* entry was boxed */ + else { + VERBOSE_DEBUG("[pid=%T] untabling B %p\n", myself->common.id, ptr); + *ptr = SHTABLE_X(t, e); + ASSERT(primary_tag(*ptr) == TAG_PRIMARY_HEADER); + } + } + +#ifdef DEBUG + if (eq(saved_obj, result) == 0) { + erts_fprintf(stderr, "original = %T\n", saved_obj); + erts_fprintf(stderr, "copy = %T\n", result); + erl_exit(ERTS_ABORT_EXIT, "copy (shared) not equal to source\n"); + } +#endif + + VERBOSE_DEBUG("[pid=%T] original was %T\n", myself->common.id, saved_obj); + VERBOSE_DEBUG("[pid=%T] copy is %T\n", myself->common.id, result); + VERBOSE_DEBUG("[pid=%T] result is at %p\n", myself->common.id, result); + + ASSERT(hp == *hpp + size); + *hpp = hp; + return result; +} + + /* * Copy a term that is guaranteed to be contained in a single * heap block. The heap block is copied word by word, and any diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 6cb37752bc..ec96a7563a 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -42,6 +42,8 @@ #include "dtrace-wrapper.h" #include "erl_bif_unique.h" +#undef SHCOPY_DEBUG + #define ERTS_INACT_WR_PB_LEAVE_MUCH_LIMIT 1 #define ERTS_INACT_WR_PB_LEAVE_MUCH_PERCENTAGE 20 #define ERTS_INACT_WR_PB_LEAVE_LIMIT 10 @@ -1166,6 +1168,10 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end, char* oh = (char *) OLD_HEAP(p); Uint oh_size = (char *) OLD_HTOP(p) - oh; +#ifdef SHCOPY_DEBUG + VERBOSE_DEBUG("[pid=%T] MINOR GC: %p %p %p %p\n", p->common.id, HEAP_START(p), HEAP_END(p), OLD_HEAP(p), OLD_HEND(p)); +#endif + n_htop = n_heap = (Eterm*) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP, sizeof(Eterm)*new_sz); @@ -1383,6 +1389,10 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end, Uint new_sz, stk_sz; int adjusted; +#ifdef SHCOPY_DEBUG + VERBOSE_DEBUG("[pid=%T] MAJOR GC: %p %p %p %p\n", p->common.id, HEAP_START(p), HEAP_END(p), OLD_HEAP(p), OLD_HEND(p)); +#endif + /* * Do a fullsweep GC. First figure out the size of the heap * to receive all live data. diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 20f100b427..d99f6548f9 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -919,8 +919,9 @@ do { \ #define EQUEUE_ISEMPTY(q) (q.back == q.front && q.possibly_empty) #define EQUEUE_GET(q) ({ \ + UWord x; \ q.possibly_empty = 1; \ - UWord x = *(q.front); \ + x = *(q.front); \ if (++(q.front) == q.end) { \ q.front = q.start; \ } \ @@ -1035,11 +1036,64 @@ __decl_noreturn void __noreturn erl_exit(int n, char*, ...); __decl_noreturn void __noreturn erl_exit_flush_async(int n, char*, ...); void erl_error(char*, va_list); +/* This controls whether sharing-preserving copy is used by Erlang */ + +#define SHCOPY_SEND +#define SHCOPY_SPAWN + +#if defined(SHCOPY_SEND) \ + || defined(SHCOPY_SPAWN) +#define SHCOPY +/* Use this with care, it is *very* verbose! */ +#undef SHCOPY_DEBUG +#endif + +#define VERBOSE_DEBUG(...) do { \ + erts_fprintf(stderr, __VA_ARGS__); \ + } while(0) + +#define ERTS_SHCOPY_FLG_MASK (((unsigned) 3) << 0) +#define ERTS_SHCOPY_FLG_NONE (((unsigned) 1) << 0) +#define ERTS_SHCOPY_FLG_TMP_BUF (((unsigned) 1) << 1) + +/* The persistent state while the sharing-preserving copier works */ + +typedef struct shcopy_info { + Eterm queue_default[DEF_EQUEUE_SIZE]; + Eterm* queue_start; + Eterm* queue_end; + ErtsAlcType_t queue_alloc_type; + UWord bitstore_default[DEF_WSTACK_SIZE]; + UWord* bitstore_start; + ErtsAlcType_t bitstore_alloc_type; + Eterm shtable_default[DEF_ESTACK_SIZE]; + Eterm* shtable_start; + ErtsAlcType_t shtable_alloc_type; +} shcopy_info; + +#define DESTROY_INFO(info) \ +do { \ + if (info.queue_start != info.queue_default) { \ + erts_free(info.queue_alloc_type, info.queue_start); \ + } \ + if (info.bitstore_start != info.bitstore_default) { \ + erts_free(info.bitstore_alloc_type, info.bitstore_start); \ + } \ + if (info.shtable_start != info.shtable_default) { \ + erts_free(info.shtable_alloc_type, info.shtable_start); \ + } \ +} while(0) + /* copy.c */ Eterm copy_object_x(Eterm, Process*, Uint); #define copy_object(Term, Proc) copy_object_x(Term,Proc,0) Uint size_object(Eterm); +Uint copy_shared_calculate(Eterm, shcopy_info*, unsigned); +Eterm copy_shared_perform(Eterm, Uint, shcopy_info*, Eterm**, ErlOffHeap*, unsigned); + +Uint size_shared(Eterm); + Eterm copy_struct(Eterm, Uint, Eterm**, ErlOffHeap*); Eterm copy_shallow(Eterm*, Uint, Eterm**, ErlOffHeap*); -- cgit v1.2.3 From cfe998988c942bed0fbf026c00a2531fdf5aba7c Mon Sep 17 00:00:00 2001 From: "Nikolaos S. Papaspyrou" Date: Sat, 9 Jun 2012 00:19:28 +0300 Subject: Add the BIF size_shared/1 and debug cleanup --- erts/emulator/beam/beam_debug.c | 14 ++++++++++++++ erts/emulator/beam/bif.tab | 12 +++++++----- erts/emulator/beam/erl_gc.c | 2 -- erts/emulator/beam/global.h | 4 ++++ 4 files changed, 25 insertions(+), 7 deletions(-) (limited to 'erts') diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index e989310789..40e3f4db4e 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -73,6 +73,20 @@ erts_debug_flat_size_1(BIF_ALIST_1) } } +BIF_RETTYPE +erts_debug_size_shared_1(BIF_ALIST_1) +{ + Process* p = BIF_P; + Eterm term = BIF_ARG_1; + Uint size = size_shared(term); + + if (IS_USMALL(0, size)) { + BIF_RET(make_small(size)); + } else { + Eterm* hp = HAlloc(p, BIG_UINT_HEAP_SIZE); + BIF_RET(uint_to_big(size, hp)); + } +} BIF_RETTYPE erts_debug_breakpoint_2(BIF_ALIST_2) diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 65f8d6f1f5..f45f886395 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -639,14 +639,16 @@ bif ets:update_counter/4 bif erts_debug:map_info/1 # -# Obsolete +# New in 19.0 # -bif erlang:hash/2 +bif binary:split/2 +bif binary:split/3 +bif erts_debug:size_shared/1 +bif 'erl.system.debug':size_shared/1 ebif_erts_debug_size_shared_1 # -# New in 19.0 +# Obsolete # -bif binary:split/2 -bif binary:split/3 +bif erlang:hash/2 diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index ec96a7563a..1cf6509012 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -42,8 +42,6 @@ #include "dtrace-wrapper.h" #include "erl_bif_unique.h" -#undef SHCOPY_DEBUG - #define ERTS_INACT_WR_PB_LEAVE_MUCH_LIMIT 1 #define ERTS_INACT_WR_PB_LEAVE_MUCH_PERCENTAGE 20 #define ERTS_INACT_WR_PB_LEAVE_LIMIT 10 diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index d99f6548f9..c2056dafaa 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1048,9 +1048,13 @@ void erl_error(char*, va_list); #undef SHCOPY_DEBUG #endif +#ifdef SHCOPY_DEBUG #define VERBOSE_DEBUG(...) do { \ erts_fprintf(stderr, __VA_ARGS__); \ } while(0) +#else +#define VERBOSE_DEBUG(...) +#endif #define ERTS_SHCOPY_FLG_MASK (((unsigned) 3) << 0) #define ERTS_SHCOPY_FLG_NONE (((unsigned) 1) << 0) -- cgit v1.2.3 From f3ae60838a5e8c83cfdd3000e25562dfcbbd438d Mon Sep 17 00:00:00 2001 From: "Nikolaos S. Papaspyrou" Date: Sat, 9 Jun 2012 01:12:50 +0300 Subject: Enable shcopy for sending messages --- erts/emulator/beam/erl_db.c | 4 +- erts/emulator/beam/erl_message.c | 114 +++++++++++++++++++++++++-------------- erts/emulator/beam/erl_message.h | 5 ++ erts/emulator/beam/global.h | 7 +++ 4 files changed, 87 insertions(+), 43 deletions(-) (limited to 'erts') diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 9ec14ab5ae..6119a9225f 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -1832,7 +1832,7 @@ BIF_RETTYPE ets_give_away_3(BIF_ALIST_3) tb->common.id, from_pid, BIF_ARG_3), - 0); + ERTS_SND_FLG_SHCOPY_TMP_BUF); erts_smp_proc_unlock(to_proc, to_locks); UnUseTmpHeap(5,BIF_P); BIF_RET(am_true); @@ -3211,7 +3211,7 @@ retry: tb->common.id, p->common.id, heir_data), - 0); + ERTS_SND_FLG_SHCOPY_TMP_BUF); erts_smp_proc_unlock(to_proc, to_locks); return !0; } diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index 79739501a8..b1fca1df0c 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -68,6 +68,9 @@ new_message_buffer(Uint size) bp = (ErlHeapFragment*) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP_FRAG, ERTS_HEAP_FRAG_SIZE(size)); ERTS_INIT_HEAP_FRAG(bp, size, size); +#ifdef SHCOPY_DEBUG + VERBOSE_DEBUG("[pid=%T] new message buffer %p\n", erts_get_current_pid(), bp->mem); +#endif return bp; } @@ -248,36 +251,6 @@ erts_realloc_shrink_message(ErtsMessage *mp, Uint sz, Eterm *brefs, Uint brefs_s return nmp; } -void -erts_link_mbuf_to_proc(Process *proc, ErlHeapFragment *first_bp) -{ - if (first_bp) { - ErlHeapFragment *bp = first_bp; - - while (1) { - /* Move any off_heap's into the process */ - if (bp->off_heap.first != NULL) { - struct erl_off_heap_header** next_p = &bp->off_heap.first; - while (*next_p != NULL) { - next_p = &((*next_p)->next); - } - *next_p = MSO(proc).first; - MSO(proc).first = bp->off_heap.first; - bp->off_heap.first = NULL; - OH_OVERHEAD(&(MSO(proc)), bp->off_heap.overhead); - } - MBUF_SIZE(proc) += bp->used_size; - if (!bp->next) - break; - bp = bp->next; - } - - /* Link the message buffer */ - bp->next = MBUF(proc); - MBUF(proc) = first_bp; - } -} - void erts_queue_dist_message(Process *rcvr, ErtsProcLocks *rcvr_locks, @@ -540,6 +513,36 @@ erts_queue_message(Process* receiver, ErtsProcLocks *receiver_locks, ); } +void +erts_link_mbuf_to_proc(Process *proc, ErlHeapFragment *first_bp) +{ + if (first_bp) { + ErlHeapFragment *bp = first_bp; + + while (1) { + /* Move any off_heap's into the process */ + if (bp->off_heap.first != NULL) { + struct erl_off_heap_header** next_p = &bp->off_heap.first; + while (*next_p != NULL) { + next_p = &((*next_p)->next); + } + *next_p = MSO(proc).first; + MSO(proc).first = bp->off_heap.first; + bp->off_heap.first = NULL; + OH_OVERHEAD(&(MSO(proc)), bp->off_heap.overhead); + } + MBUF_SIZE(proc) += bp->used_size; + if (!bp->next) + break; + bp = bp->next; + } + + /* Link the message buffer */ + bp->next = MBUF(proc); + MBUF(proc) = first_bp; + } +} + Uint erts_msg_attached_data_size_aux(ErtsMessage *msg) { @@ -663,11 +666,15 @@ erts_send_message(Process* sender, Eterm utag = NIL; #endif erts_aint32_t receiver_state; +#ifdef SHCOPY_SEND + unsigned shflags = (flags & ERTS_SND_FLG_SHCOPY_MASK) >> ERTS_SND_FLG_SHCOPY_SHIFT; + erts_shcopy_t info; +#endif BM_STOP_TIMER(system); BM_MESSAGE(message,sender,receiver); BM_START_TIMER(send); - #ifdef USE_VM_PROBES +#ifdef USE_VM_PROBES *sender_name = *receiver_name = '\0'; if (DTRACE_ENABLED(message_send)) { erts_snprintf(sender_name, sizeof(DTRACE_CHARBUF_NAME(sender_name)), @@ -686,15 +693,23 @@ erts_send_message(Process* sender, #ifdef USE_VM_PROBES Uint dt_utag_size = 0; #endif - +#ifdef SHCOPY_SEND + unsigned shflags = (flags & ERTS_SND_FLG_SHCOPY_MASK) >> ERTS_SND_FLG_SHCOPY_SHIFT; + erts_shcopy_t info; + INITIALIZE_SHCOPY(info); +#endif BM_SWAP_TIMER(send,size); - msize = size_object(message); +#ifdef SHCOPY_SEND + INITIALIZE_SHCOPY(info); + msize = copy_shared_calculate(message, &info, shflags); +#else + msize = size_object(message); +#endif BM_SWAP_TIMER(size,send); #ifdef USE_VM_PROBES if (stoken != am_have_dt_utag) { #endif - seq_trace_update_send(sender); seq_trace_output(stoken, message, SEQ_TRACE_SEND, receiver->common.id, sender); @@ -720,14 +735,20 @@ erts_send_message(Process* sender, &ohp); BM_SWAP_TIMER(send,copy); + +#ifdef SHCOPY_SEND + if (is_not_immed(message)) + message = copy_shared_perform(message, msize, &info, &hp, ohp, shflags); + DESTROY_SHCOPY(info); +#else + if (is_not_immed(message)) + message = copy_struct(message, msize, &hp, ohp); +#endif if (is_immed(stoken)) token = stoken; else token = copy_struct(stoken, seq_trace_size, &hp, ohp); - if (is_not_immed(message)) - message = copy_struct(message, msize, &hp, ohp); - #ifdef USE_VM_PROBES if (DT_UTAG_FLAGS(sender) & DT_UTAG_SPREADING) { if (is_immed(DT_UTAG(sender))) @@ -764,7 +785,12 @@ erts_send_message(Process* sender, } else { BM_SWAP_TIMER(send,size); - msize = size_object(message); +#ifdef SHCOPY_SEND + INITIALIZE_SHCOPY(info); + msize = copy_shared_calculate(message, &info, shflags); +#else + msize = size_object(message); +#endif BM_SWAP_TIMER(size,send); mp = erts_alloc_message_heap_state(receiver, @@ -774,8 +800,14 @@ erts_send_message(Process* sender, &hp, &ohp); BM_SWAP_TIMER(send,copy); - if (is_not_immed(message)) - message = copy_struct(message, msize, &hp, ohp); +#ifdef SHCOPY_SEND + if (is_not_immed(message)) + message = copy_shared_perform(message, msize, &info, &hp, ohp, shflags); + DESTROY_SHCOPY(info); +#else + if (is_not_immed(message)) + message = copy_struct(message, msize, &hp, ohp); +#endif BM_MESSAGE_COPIED(msz); BM_SWAP_TIMER(copy,send); } @@ -800,6 +832,7 @@ erts_send_message(Process* sender, return res; } + /* * This function delivers an EXIT message to a process * which is trapping EXITs. @@ -1725,4 +1758,3 @@ void erts_factory_undo(ErtsHeapFactory* factory) factory->heap_frags = NULL; #endif } - diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index 740ae46a0f..0de36c3199 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -238,6 +238,11 @@ do { \ #define ERTS_SND_FLG_NO_SEQ_TRACE (((unsigned) 1) << 0) +#define ERTS_SND_FLG_SHCOPY_SHIFT 1 +#define ERTS_SND_FLG_SHCOPY_MASK (ERTS_SHCOPY_FLG_MASK << ERTS_SND_FLG_SHCOPY_SHIFT) +#define ERTS_SND_FLG_SHCOPY_NONE (ERTS_SHCOPY_FLG_NONE << ERTS_SND_FLG_SHCOPY_SHIFT) +#define ERTS_SND_FLG_SHCOPY_TMP_BUF (ERTS_SHCOPY_FLG_TMP_BUF << ERTS_SND_FLG_SHCOPY_SHIFT) + #define ERTS_HEAP_FRAG_SIZE(DATA_WORDS) \ (sizeof(ErlHeapFragment) - sizeof(Eterm) + (DATA_WORDS)*sizeof(Eterm)) diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index c2056dafaa..a5e6ff6f1c 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1075,6 +1075,13 @@ typedef struct shcopy_info { ErtsAlcType_t shtable_alloc_type; } shcopy_info; +#define INITIALIZE_INFO(info) \ +do { \ + info.queue_start = info.queue_default; \ + info.bitstore_start = info.bitstore_default; \ + info.shtable_start = info.shtable_default; \ +} while(0) + #define DESTROY_INFO(info) \ do { \ if (info.queue_start != info.queue_default) { \ -- cgit v1.2.3 From 7141fa8e4534ab7e4dcc3cad6c46872efa0e6e66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Tue, 17 Nov 2015 14:44:52 +0100 Subject: Remove DTrace Harddebug clutter --- erts/emulator/beam/beam_emu.c | 19 ------------------- 1 file changed, 19 deletions(-) (limited to 'erts') diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 208a16dfd0..af97cba61c 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -1914,20 +1914,7 @@ void process_main(void) if (DT_UTAG(c_p) != NIL) { if (DT_UTAG_FLAGS(c_p) & DT_UTAG_PERMANENT) { SEQ_TRACE_TOKEN(c_p) = am_have_dt_utag; -#ifdef DTRACE_TAG_HARDDEBUG - if (DT_UTAG_FLAGS(c_p) & DT_UTAG_SPREADING) - erts_fprintf(stderr, - "Dtrace -> (%T) stop spreading " - "tag %T with message %T\r\n", - c_p->common.id,DT_UTAG(c_p),ERL_MESSAGE_TERM(msgp)); -#endif } else { -#ifdef DTRACE_TAG_HARDDEBUG - erts_fprintf(stderr, - "Dtrace -> (%T) kill tag %T with " - "message %T\r\n", - c_p->common.id,DT_UTAG(c_p),ERL_MESSAGE_TERM(msgp)); -#endif DT_UTAG(c_p) = NIL; SEQ_TRACE_TOKEN(c_p) = NIL; } @@ -1947,12 +1934,6 @@ void process_main(void) DT_UTAG(c_p) = ERL_MESSAGE_DT_UTAG(msgp); } DT_UTAG_FLAGS(c_p) |= DT_UTAG_SPREADING; -#ifdef DTRACE_TAG_HARDDEBUG - erts_fprintf(stderr, - "Dtrace -> (%T) receive tag (%T) " - "with message %T\r\n", - c_p->common.id, DT_UTAG(c_p), ERL_MESSAGE_TERM(msgp)); -#endif } else { #endif ASSERT(is_tuple(SEQ_TRACE_TOKEN(c_p))); -- cgit v1.2.3 From 8d02ad60c88aa060ff83ff3179bc8c0ab66868ee Mon Sep 17 00:00:00 2001 From: "Nikolaos S. Papaspyrou" Date: Fri, 14 Mar 2014 11:32:46 +0200 Subject: Add machinery to enable SHCOPY dynamically This commit is just for debugging purposes, will probably be reverted. It comes with a the erts_debug:copy_shared/1 BIF. If SHCOPY_DISABLE is defined, SHCOPY starts disabled and is dynamically enabled the first time that the BIF is called. --- erts/emulator/beam/beam_debug.c | 26 ++++++++++++++++++++++++++ erts/emulator/beam/copy.c | 12 ++++++++++++ erts/emulator/beam/global.h | 2 ++ 3 files changed, 40 insertions(+) (limited to 'erts') diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index 40e3f4db4e..36f3cfabbc 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -88,6 +88,32 @@ erts_debug_size_shared_1(BIF_ALIST_1) } } +BIF_RETTYPE +erts_debug_copy_shared_1(BIF_ALIST_1) +{ + Process* p = BIF_P; + Eterm term = BIF_ARG_1; + Uint size; + Eterm* hp; + Eterm copy; + shcopy_info info; +#ifdef SHCOPY_DISABLE + extern int disable_copy_shared; +#endif + INITIALIZE_INFO(info); + + size = copy_shared_calculate(term, &info, 0); + if (size > 0) { + hp = HAlloc(p, size); + } + copy = copy_shared_perform(term, size, &info, &hp, &p->off_heap, 0); + DESTROY_INFO(info); +#ifdef SHCOPY_DISABLE + disable_copy_shared = 0; +#endif + BIF_RET(copy); +} + BIF_RETTYPE erts_debug_breakpoint_2(BIF_ALIST_2) { diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index b31e043f08..2566707717 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -901,6 +901,10 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) * Using an ESTACK but not very transparently; consider refactoring */ +#ifdef SHCOPY_DISABLE +int disable_copy_shared = ERTS_SHCOPY_FLG_NONE; +#endif + #define DECLARE_SHTABLE(s) \ DECLARE_ESTACK(s); \ Uint ESTK_CONCAT(s,_offset) = 0 @@ -1028,6 +1032,10 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) if (IS_CONST(obj)) return 0; +#ifdef SHCOPY_DISABLE + flags |= disable_copy_shared; +#endif + myself = erts_get_current_process(); if (myself == NULL || (flags & ERTS_SHCOPY_FLG_NONE)) return size_object(obj); @@ -1276,6 +1284,10 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E if (IS_CONST(obj)) return obj; +#ifdef SHCOPY_DISABLE + flags |= disable_copy_shared; +#endif + myself = erts_get_current_process(); if (myself == NULL || (flags & ERTS_SHCOPY_FLG_NONE)) return copy_struct(obj, size, hpp, off_heap); diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index a5e6ff6f1c..3e1f3664bc 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1044,6 +1044,8 @@ void erl_error(char*, va_list); #if defined(SHCOPY_SEND) \ || defined(SHCOPY_SPAWN) #define SHCOPY +/* Use this if you want sharing-preserving copy to be initially disabled */ +#undef SHCOPY_DISABLE /* Use this with care, it is *very* verbose! */ #undef SHCOPY_DEBUG #endif -- cgit v1.2.3 From 244e9d5855d1b1f160d667b5cf369defee72829d Mon Sep 17 00:00:00 2001 From: "Nikolaos S. Papaspyrou" Date: Sat, 9 Jun 2012 01:37:29 +0300 Subject: Enable shcopy for spawning processes --- erts/emulator/beam/erl_process.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'erts') diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 9acce8acb6..f2da5289d3 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -10747,6 +10747,11 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). Eterm res = THE_NON_VALUE; erts_aint32_t state = 0; erts_aint32_t prio = (erts_aint32_t) PRIORITY_NORMAL; +#ifdef SHCOPY_SPAWN + unsigned shflags = 0; /* could be taken from so->flags, if necessary */ + shcopy_info info; + INITIALIZE_INFO(info); +#endif #ifdef ERTS_SMP erts_smp_proc_lock(parent, ERTS_PROC_LOCKS_ALL_MINOR); @@ -10798,7 +10803,11 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). BM_COUNT(processes_spawned); BM_SWAP_TIMER(system,size); +#ifdef SHCOPY_SPAWN + arg_size = copy_shared_calculate(args, &info, shflags); +#else arg_size = size_object(args); +#endif BM_SWAP_TIMER(size,system); heap_need = arg_size; @@ -10876,7 +10885,12 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). BM_MESSAGE(args,p,parent); BM_START_TIMER(system); BM_SWAP_TIMER(system,copy); +#ifdef SHCOPY_SPAWN + p->arg_reg[2] = copy_shared_perform(args, arg_size, &info, &p->htop, &p->off_heap, shflags); + DESTROY_INFO(info); +#else p->arg_reg[2] = copy_struct(args, arg_size, &p->htop, &p->off_heap); +#endif BM_MESSAGE_COPIED(arg_size); BM_SWAP_TIMER(copy,system); p->arity = 3; @@ -11260,6 +11274,8 @@ static void delete_process(Process* p) { VERBOSE(DEBUG_PROCESSES, ("Removing process: %T\n",p->common.id)); + VERBOSE_DEBUG("[pid=%T] delete process: %p %p %p %p\n", p->common.id, + HEAP_START(p), HEAP_END(p), OLD_HEAP(p), OLD_HEND(p)); /* Cleanup psd */ -- cgit v1.2.3 From 277e8e77384ed6628009243e63d62f0555d10c69 Mon Sep 17 00:00:00 2001 From: "Nikolaos S. Papaspyrou" Date: Mon, 11 Jun 2012 15:17:01 +0300 Subject: Add -debug +vc flag for debuging SHCOPY This is very verbose, you have been warned. It should work with the copy-spy.py script, which may be a bit outdated. --- erts/emulator/beam/copy.c | 131 ++++++++++----------------------------- erts/emulator/beam/erl_debug.h | 1 + erts/emulator/beam/erl_gc.c | 10 ++- erts/emulator/beam/erl_init.c | 2 + erts/emulator/beam/erl_message.c | 4 +- erts/emulator/beam/erl_process.c | 4 +- erts/emulator/beam/global.h | 10 --- 7 files changed, 43 insertions(+), 119 deletions(-) (limited to 'erts') diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index 2566707717..f74c6b1c89 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -75,16 +75,13 @@ Uint size_object(Eterm obj) Uint sum = 0; Eterm* ptr; int arity; -#ifdef SHCOPY_DEBUG - Eterm mypid; +#ifdef DEBUG + Eterm mypid = erts_get_current_pid(); #endif DECLARE_ESTACK(s); -#ifdef SHCOPY_DEBUG - mypid = erts_get_current_pid(); - VERBOSE_DEBUG("[pid=%T] size_object %p\n", mypid, obj); -#endif + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] size_object %p\n", mypid, obj)); for (;;) { switch (primary_tag(obj)) { @@ -220,7 +217,7 @@ Uint size_object(Eterm obj) pop_next: if (ESTACK_ISEMPTY(s)) { DESTROY_ESTACK(s); - VERBOSE_DEBUG("[pid=%T] size was: %u\n", mypid, sum); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] size was: %u\n", mypid, sum)); return sum; } obj = ESTACK_POP(s); @@ -320,11 +317,9 @@ Uint size_shared(Eterm obj) return size_object(obj); for (;;) { - VERBOSE_DEBUG("[size] visiting: %x ", obj); switch (primary_tag(obj)) { case TAG_PRIMARY_LIST: { Eterm head, tail; - VERBOSE_DEBUG("L"); ptr = list_val_rel(obj, base); /* we're not counting anything that's outside our heap */ if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) { @@ -335,22 +330,18 @@ Uint size_shared(Eterm obj) /* if it's visited, don't count it */ if (primary_tag(tail) == TAG_PRIMARY_HEADER || primary_tag(head) == TAG_PRIMARY_HEADER) { - VERBOSE_DEBUG("!"); goto pop_next; } /* else make it visited now */ switch (primary_tag(tail)) { case TAG_PRIMARY_LIST: - VERBOSE_DEBUG("/L"); ptr[1] = (tail - TAG_PRIMARY_LIST) | TAG_PRIMARY_HEADER; break; case TAG_PRIMARY_IMMED1: - VERBOSE_DEBUG("/I"); CAR(ptr) = (head - primary_tag(head)) | TAG_PRIMARY_HEADER; CDR(ptr) = (tail - TAG_PRIMARY_IMMED1) | primary_tag(head); break; case TAG_PRIMARY_BOXED: - VERBOSE_DEBUG("/B saved %d", primary_tag(head)); BITSTORE_PUT(b, primary_tag(head)); CAR(ptr) = (head - primary_tag(head)) | TAG_PRIMARY_HEADER; CDR(ptr) = (tail - TAG_PRIMARY_BOXED) | TAG_PRIMARY_HEADER; @@ -366,7 +357,6 @@ Uint size_shared(Eterm obj) } case TAG_PRIMARY_BOXED: { Eterm hdr; - VERBOSE_DEBUG("B"); ptr = boxed_val_rel(obj, base); /* we're not counting anything that's outside our heap */ if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) { @@ -375,7 +365,6 @@ Uint size_shared(Eterm obj) hdr = *ptr; /* if it's visited, don't count it */ if (primary_tag(hdr) != TAG_PRIMARY_HEADER) { - VERBOSE_DEBUG("!"); goto pop_next; } /* else make it visited now */ @@ -385,10 +374,8 @@ Uint size_shared(Eterm obj) switch (hdr & _TAG_HEADER_MASK) { case ARITYVAL_SUBTAG: { int arity = header_arity(hdr); - VERBOSE_DEBUG("/T"); sum += arity + 1; if (arity == 0) { /* Empty tuple -- unusual. */ - VERBOSE_DEBUG("e"); goto pop_next; } while (arity-- > 0) { @@ -403,7 +390,6 @@ Uint size_shared(Eterm obj) ErlFunThing* funp = (ErlFunThing *) ptr; unsigned eterms = 1 /* creator */ + funp->num_free; unsigned sz = thing_arityval(hdr); - VERBOSE_DEBUG("/F"); sum += 1 /* header */ + sz + eterms; ptr += 1 /* header */ + sz; while (eterms-- > 0) { @@ -442,14 +428,12 @@ Uint size_shared(Eterm obj) erl_exit(ERTS_ABORT_EXIT, "size_shared: matchstate term not allowed"); default: - VERBOSE_DEBUG("/D"); sum += thing_arityval(hdr) + 1; goto pop_next; } break; } case TAG_PRIMARY_IMMED1: - VERBOSE_DEBUG("I"); pop_next: if (EQUEUE_ISEMPTY(s)) { goto cleanup; @@ -459,19 +443,15 @@ Uint size_shared(Eterm obj) default: erl_exit(ERTS_ABORT_EXIT, "size_shared: bad tag for %#x\n", obj); } - VERBOSE_DEBUG("\n"); } cleanup: - VERBOSE_DEBUG("\n"); obj = saved_obj; BITSTORE_CLOSE(b); for (;;) { - VERBOSE_DEBUG("[size] revisiting: %x ", obj); switch (primary_tag(obj)) { case TAG_PRIMARY_LIST: { Eterm head, tail; - VERBOSE_DEBUG("L"); ptr = list_val_rel(obj, base); if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) { goto cleanup_next; @@ -482,19 +462,15 @@ cleanup: if (primary_tag(tail) == TAG_PRIMARY_HEADER) { if (primary_tag(head) == TAG_PRIMARY_HEADER) { Eterm saved = BITSTORE_GET(b); - VERBOSE_DEBUG("/B restoring %d", saved); CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) | saved; CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) | TAG_PRIMARY_BOXED; } else { - VERBOSE_DEBUG("/L"); CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) | TAG_PRIMARY_LIST; } } else if (primary_tag(head) == TAG_PRIMARY_HEADER) { - VERBOSE_DEBUG("/I"); CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) | primary_tag(tail); CDR(ptr) = tail = (tail - primary_tag(tail)) | TAG_PRIMARY_IMMED1; } else { - VERBOSE_DEBUG("!"); goto cleanup_next; } /* and its children too */ @@ -506,7 +482,6 @@ cleanup: } case TAG_PRIMARY_BOXED: { Eterm hdr; - VERBOSE_DEBUG("B"); ptr = boxed_val_rel(obj, base); if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) { goto cleanup_next; @@ -563,11 +538,9 @@ cleanup: default: erl_exit(ERTS_ABORT_EXIT, "size_shared: bad tag for %#x\n", obj); } - VERBOSE_DEBUG("\n"); } all_clean: - VERBOSE_DEBUG("\n"); /* Return the result */ DESTROY_EQUEUE(s); DESTROY_BITSTORE(b); @@ -597,18 +570,13 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) #ifdef DEBUG Eterm org_obj = obj; Uint org_sz = sz; -#endif -#ifdef SHCOPY_DEBUG - Eterm mypid; + Eterm mypid = erts_get_current_pid(); #endif if (IS_CONST(obj)) return obj; -#ifdef SHCOPY_DEBUG - mypid = erts_get_current_pid(); - VERBOSE_DEBUG("[pid=%T] copy_struct %p\n", mypid, obj); -#endif + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy_struct %p\n", mypid, obj)); DTRACE1(copy_struct, (int32_t)sz); @@ -891,7 +859,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) } #endif *hpp = (Eterm *) (hstart+hsize); - VERBOSE_DEBUG("[pid=%T] result is at %p\n", mypid, res); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] result is at %p\n", mypid, res)); return res; } @@ -1040,8 +1008,8 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) if (myself == NULL || (flags & ERTS_SHCOPY_FLG_NONE)) return size_object(obj); - VERBOSE_DEBUG("[pid=%T] copy_shared_calculate %p\n", myself->common.id, obj); - VERBOSE_DEBUG("[pid=%T] message is %T\n", myself->common.id, obj); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy_shared_calculate %p\n", myself->common.id, obj)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] message is %T\n", myself->common.id, obj)); /* step #1: ------------------------------------------------------- @@ -1060,18 +1028,13 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) sum = 0; for (;;) { - VERBOSE_DEBUG("[copy] visiting: %x ", obj); switch (primary_tag(obj)) { case TAG_PRIMARY_LIST: { Eterm head, tail; - VERBOSE_DEBUG("L"); ptr = list_val_rel(obj, base); /* off heap list pointers are copied verbatim */ if (!INHEAP(myself, ptr)) { - VERBOSE_DEBUG("[pid=%T] bypassed copying %p is %T\n", myself->common.id, ptr, obj); - if (myself->mbuf != NULL) - VERBOSE_DEBUG("[pid=%T] BUT !!! there are message buffers!\n", myself->common.id); - VERBOSE_DEBUG("#"); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", myself->common.id, ptr, obj)); goto pop_next; } head = CAR(ptr); @@ -1080,10 +1043,9 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) if not already shared, make it shared and store it in the table */ if (primary_tag(tail) == TAG_PRIMARY_HEADER || primary_tag(head) == TAG_PRIMARY_HEADER) { - VERBOSE_DEBUG("!"); if (tail != THE_NON_VALUE) { e = SHTABLE_NEXT(t); - VERBOSE_DEBUG("[pid=%T] tabling L %p\n", myself->common.id, ptr); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] tabling L %p\n", myself->common.id, ptr)); SHTABLE_PUSH(t, head, tail, ptr); CAR(ptr) = (e << _TAG_PRIMARY_SIZE) | LIST_SHARED_UNPROCESSED; CDR(ptr) = THE_NON_VALUE; @@ -1093,20 +1055,17 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) /* else make it visited now */ switch (primary_tag(tail)) { case TAG_PRIMARY_LIST: - VERBOSE_DEBUG("/L"); - VERBOSE_DEBUG("[pid=%T] mangling L/L %p\n", myself->common.id, ptr); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] mangling L/L %p\n", myself->common.id, ptr)); CDR(ptr) = (tail - TAG_PRIMARY_LIST) | TAG_PRIMARY_HEADER; break; case TAG_PRIMARY_IMMED1: - VERBOSE_DEBUG("/I"); - VERBOSE_DEBUG("[pid=%T] mangling L/I %p\n", myself->common.id, ptr); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] mangling L/I %p\n", myself->common.id, ptr)); CAR(ptr) = (head - primary_tag(head)) | TAG_PRIMARY_HEADER; CDR(ptr) = (tail - TAG_PRIMARY_IMMED1) | primary_tag(head); break; case TAG_PRIMARY_BOXED: - VERBOSE_DEBUG("/B saved %d", primary_tag(head)); BITSTORE_PUT(b, primary_tag(head)); - VERBOSE_DEBUG("[pid=%T] mangling L/B %p\n", myself->common.id, ptr); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] mangling L/B %p\n", myself->common.id, ptr)); CAR(ptr) = (head - primary_tag(head)) | TAG_PRIMARY_HEADER; CDR(ptr) = (tail - TAG_PRIMARY_BOXED) | TAG_PRIMARY_HEADER; break; @@ -1121,39 +1080,34 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) } case TAG_PRIMARY_BOXED: { Eterm hdr; - VERBOSE_DEBUG("B"); ptr = boxed_val_rel(obj, base); /* off heap pointers to boxes are copied verbatim */ if (!INHEAP(myself, ptr)) { - VERBOSE_DEBUG("[pid=%T] bypassed copying %p is %T\n", myself->common.id, ptr, obj); - VERBOSE_DEBUG("#"); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", myself->common.id, ptr, obj)); goto pop_next; } hdr = *ptr; /* if it's visited, don't count it; if not already shared, make it shared and store it in the table */ if (primary_tag(hdr) != TAG_PRIMARY_HEADER) { - VERBOSE_DEBUG("!"); if (primary_tag(hdr) == BOXED_VISITED) { e = SHTABLE_NEXT(t); - VERBOSE_DEBUG("[pid=%T] tabling B %p\n", myself->common.id, ptr); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] tabling B %p\n", myself->common.id, ptr)); SHTABLE_PUSH(t, hdr, THE_NON_VALUE, ptr); *ptr = (e << _TAG_PRIMARY_SIZE) | BOXED_SHARED_UNPROCESSED; } goto pop_next; } /* else make it visited now */ - VERBOSE_DEBUG("[pid=%T] mangling B %p\n", myself->common.id, ptr); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] mangling B %p\n", myself->common.id, ptr)); *ptr = (hdr - primary_tag(hdr)) + BOXED_VISITED; /* and count it */ ASSERT(is_header(hdr)); switch (hdr & _TAG_HEADER_MASK) { case ARITYVAL_SUBTAG: { int arity = header_arity(hdr); - VERBOSE_DEBUG("/T"); sum += arity + 1; if (arity == 0) { /* Empty tuple -- unusual. */ - VERBOSE_DEBUG("e"); goto pop_next; } while (arity-- > 0) { @@ -1168,7 +1122,6 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) ErlFunThing* funp = (ErlFunThing *) ptr; unsigned eterms = 1 /* creator */ + funp->num_free; sz = thing_arityval(hdr); - VERBOSE_DEBUG("/F"); sum += 1 /* header */ + sz + eterms; ptr += 1 /* header */ + sz; while (eterms-- > 0) { @@ -1213,17 +1166,14 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) erl_exit(ERTS_ABORT_EXIT, "size_shared: matchstate term not allowed"); default: - VERBOSE_DEBUG("/D"); sum += thing_arityval(hdr) + 1; goto pop_next; } break; } case TAG_PRIMARY_IMMED1: - VERBOSE_DEBUG("I"); pop_next: if (EQUEUE_ISEMPTY(s)) { - VERBOSE_DEBUG("\n"); // add sentinel to the table SHTABLE_PUSH(t, THE_NON_VALUE, THE_NON_VALUE, NULL); // store persistent info @@ -1236,7 +1186,7 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) info->shtable_start = t.start; info->shtable_alloc_type = t.alloc_type; // single point of return: the size of the object - VERBOSE_DEBUG("[pid=%T] size was: %u\n", myself->common.id, sum); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] size was: %u\n", myself->common.id, sum)); return sum; } obj = EQUEUE_GET(s); @@ -1244,7 +1194,6 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) default: erl_exit(ERTS_ABORT_EXIT, "[pid=%T] size_shared: bad tag for %#x\n", obj); } - VERBOSE_DEBUG("\n"); } } @@ -1266,7 +1215,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E unsigned remaining; Process* myself; int force_local = flags & ERTS_SHCOPY_FLG_TMP_BUF; -#if defined(DEBUG) || defined(SHCOPY_DEBUG) +#ifdef DEBUG Eterm saved_obj = obj; #endif @@ -1292,7 +1241,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E if (myself == NULL || (flags & ERTS_SHCOPY_FLG_NONE)) return copy_struct(obj, size, hpp, off_heap); - VERBOSE_DEBUG("[pid=%T] copy_shared_perform %p\n", myself->common.id, obj); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy_shared_perform %p\n", myself->common.id, obj)); /* step #2: was performed before this function was called ------------------------------------------------------- @@ -1315,15 +1264,12 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E resp = &result; remaining = 0; for (;;) { - VERBOSE_DEBUG("[copy] revisiting: %x ", obj); switch (primary_tag(obj)) { case TAG_PRIMARY_LIST: { Eterm head, tail; - VERBOSE_DEBUG("L"); ptr = list_val_rel(obj, base); /* off heap list pointers are copied verbatim */ if (!INHEAP(myself, ptr)) { - VERBOSE_DEBUG("#"); *resp = obj; goto cleanup_next; } @@ -1334,19 +1280,17 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E e = head >> _TAG_PRIMARY_SIZE; /* if it has been processed, just use the forwarding pointer */ if (primary_tag(head) == LIST_SHARED_PROCESSED) { - VERBOSE_DEBUG("!"); *resp = make_list(SHTABLE_FWD(t, e)); goto cleanup_next; } /* else, let's process it now, copy it and keep the forwarding pointer */ else { - VERBOSE_DEBUG("$"); CAR(ptr) = (head - primary_tag(head)) + LIST_SHARED_PROCESSED; head = SHTABLE_X(t, e); tail = SHTABLE_Y(t, e); ptr = &(SHTABLE_X(t, e)); - VERBOSE_DEBUG("[pid=%T] tabled L %p is %p\n", myself->common.id, ptr, SHTABLE_REV(t, e)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] tabled L %p is %p\n", myself->common.id, ptr, SHTABLE_REV(t, e))); SHTABLE_FWD_UPD(t, e, hp); } } @@ -1354,18 +1298,15 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E if (primary_tag(tail) == TAG_PRIMARY_HEADER) { if (primary_tag(head) == TAG_PRIMARY_HEADER) { Eterm saved = BITSTORE_GET(b); - VERBOSE_DEBUG("/B restoring %d", saved); - VERBOSE_DEBUG("[pid=%T] unmangling L/B %p\n", myself->common.id, ptr); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling L/B %p\n", myself->common.id, ptr)); CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) + saved; CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) + TAG_PRIMARY_BOXED; } else { - VERBOSE_DEBUG("/L"); - VERBOSE_DEBUG("[pid=%T] unmangling L/L %p\n", myself->common.id, ptr); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling L/L %p\n", myself->common.id, ptr)); CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) + TAG_PRIMARY_LIST; } } else if (primary_tag(head) == TAG_PRIMARY_HEADER) { - VERBOSE_DEBUG("/I"); - VERBOSE_DEBUG("[pid=%T] unmangling L/I %p\n", myself->common.id, ptr); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling L/I %p\n", myself->common.id, ptr)); CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) | primary_tag(tail); CDR(ptr) = tail = (tail - primary_tag(tail)) | TAG_PRIMARY_IMMED1; } else { @@ -1387,11 +1328,9 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E } case TAG_PRIMARY_BOXED: { Eterm hdr; - VERBOSE_DEBUG("B"); ptr = boxed_val_rel(obj, base); /* off heap pointers to boxes are copied verbatim */ if (!INHEAP(myself, ptr)) { - VERBOSE_DEBUG("#"); *resp = obj; goto cleanup_next; } @@ -1403,7 +1342,6 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E /* if it is shared and has been processed, just use the forwarding pointer */ case BOXED_SHARED_PROCESSED: - VERBOSE_DEBUG("!"); e = hdr >> _TAG_PRIMARY_SIZE; *resp = make_boxed(SHTABLE_FWD(t, e)); goto cleanup_next; @@ -1411,17 +1349,16 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E it now: copy it and keep the forwarding pointer */ case BOXED_SHARED_UNPROCESSED: e = hdr >> _TAG_PRIMARY_SIZE; - VERBOSE_DEBUG("$"); *ptr = (hdr - primary_tag(hdr)) + BOXED_SHARED_PROCESSED; hdr = SHTABLE_X(t, e); ASSERT(primary_tag(hdr) == BOXED_VISITED); - VERBOSE_DEBUG("[pid=%T] tabled B %p is %p\n", myself->common.id, ptr, SHTABLE_REV(t, e)); - VERBOSE_DEBUG("[pid=%T] unmangling B %p\n", myself->common.id, ptr); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] tabled B %p is %p\n", myself->common.id, ptr, SHTABLE_REV(t, e))); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling B %p\n", myself->common.id, ptr)); SHTABLE_X(t, e) = hdr = (hdr - BOXED_VISITED) + TAG_PRIMARY_HEADER; SHTABLE_FWD_UPD(t, e, hp); break; case BOXED_VISITED: - VERBOSE_DEBUG("[pid=%T] unmangling B %p\n", myself->common.id, ptr); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling B %p\n", myself->common.id, ptr)); *ptr = hdr = (hdr - BOXED_VISITED) + TAG_PRIMARY_HEADER; break; } @@ -1628,7 +1565,6 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E default: erl_exit(ERTS_ABORT_EXIT, "size_shared: bad tag for %#x\n", obj); } - VERBOSE_DEBUG("\n"); } /* step #4: @@ -1637,21 +1573,20 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E */ all_clean: - VERBOSE_DEBUG("\n"); for (e = 0; ; e += SHTABLE_INCR) { ptr = SHTABLE_REV(t, e); if (ptr == NULL) break; - VERBOSE_DEBUG("[copy] restoring shared: %x\n", ptr); + VERBOSE(DEBUG_SHCOPY, ("[copy] restoring shared: %x\n", ptr)); /* entry was a list */ if (SHTABLE_Y(t, e) != THE_NON_VALUE) { - VERBOSE_DEBUG("[pid=%T] untabling L %p\n", myself->common.id, ptr); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] untabling L %p\n", myself->common.id, ptr)); CAR(ptr) = SHTABLE_X(t, e); CDR(ptr) = SHTABLE_Y(t, e); } /* entry was boxed */ else { - VERBOSE_DEBUG("[pid=%T] untabling B %p\n", myself->common.id, ptr); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] untabling B %p\n", myself->common.id, ptr)); *ptr = SHTABLE_X(t, e); ASSERT(primary_tag(*ptr) == TAG_PRIMARY_HEADER); } @@ -1665,9 +1600,9 @@ all_clean: } #endif - VERBOSE_DEBUG("[pid=%T] original was %T\n", myself->common.id, saved_obj); - VERBOSE_DEBUG("[pid=%T] copy is %T\n", myself->common.id, result); - VERBOSE_DEBUG("[pid=%T] result is at %p\n", myself->common.id, result); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] original was %T\n", myself->common.id, saved_obj)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy is %T\n", myself->common.id, result)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] result is at %p\n", myself->common.id, result)); ASSERT(hp == *hpp + size); *hpp = hp; diff --git a/erts/emulator/beam/erl_debug.h b/erts/emulator/beam/erl_debug.h index f4259e7dae..029320691d 100644 --- a/erts/emulator/beam/erl_debug.h +++ b/erts/emulator/beam/erl_debug.h @@ -48,6 +48,7 @@ #define DEBUG_THREADS 0x0010 /* Thread-related stuff */ #define DEBUG_PROCESSES 0x0020 /* Process creation and removal */ #define DEBUG_MEMORY 0x0040 /* Display results of memory checks */ +#define DEBUG_SHCOPY 0x0080 /* Sharing-preserving copying of terms */ extern Uint32 verbose; diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 1cf6509012..c50756d56b 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -1166,9 +1166,8 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end, char* oh = (char *) OLD_HEAP(p); Uint oh_size = (char *) OLD_HTOP(p) - oh; -#ifdef SHCOPY_DEBUG - VERBOSE_DEBUG("[pid=%T] MINOR GC: %p %p %p %p\n", p->common.id, HEAP_START(p), HEAP_END(p), OLD_HEAP(p), OLD_HEND(p)); -#endif + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] MINOR GC: %p %p %p %p\n", p->common.id, + HEAP_START(p), HEAP_END(p), OLD_HEAP(p), OLD_HEND(p))); n_htop = n_heap = (Eterm*) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP, sizeof(Eterm)*new_sz); @@ -1387,9 +1386,8 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end, Uint new_sz, stk_sz; int adjusted; -#ifdef SHCOPY_DEBUG - VERBOSE_DEBUG("[pid=%T] MAJOR GC: %p %p %p %p\n", p->common.id, HEAP_START(p), HEAP_END(p), OLD_HEAP(p), OLD_HEND(p)); -#endif + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] MAJOR GC: %p %p %p %p\n", p->common.id, + HEAP_START(p), HEAP_END(p), OLD_HEAP(p), OLD_HEND(p))); /* * Do a fullsweep GC. First figure out the size of the heap diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index f396a0a156..296cfdabc3 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -1401,6 +1401,7 @@ erl_start(int argc, char **argv) case 't': verbose |= DEBUG_THREADS; break; case 'p': verbose |= DEBUG_PROCESSES; break; case 'm': verbose |= DEBUG_MESSAGES; break; + case 'c': verbose |= DEBUG_SHCOPY; break; default : erts_fprintf(stderr,"Unknown verbose option: %c\n",*ch); } } @@ -1413,6 +1414,7 @@ erl_start(int argc, char **argv) if (verbose & DEBUG_THREADS) erts_printf("THREADS "); if (verbose & DEBUG_PROCESSES) erts_printf("PROCESSES "); if (verbose & DEBUG_MESSAGES) erts_printf("MESSAGES "); + if (verbose & DEBUG_SHCOPY) erts_printf("SHCOPY "); erts_printf("\n"); #else erts_fprintf(stderr, "warning: -v (only in debug compiled code)\n"); diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index b1fca1df0c..11890a756d 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -68,9 +68,7 @@ new_message_buffer(Uint size) bp = (ErlHeapFragment*) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP_FRAG, ERTS_HEAP_FRAG_SIZE(size)); ERTS_INIT_HEAP_FRAG(bp, size, size); -#ifdef SHCOPY_DEBUG - VERBOSE_DEBUG("[pid=%T] new message buffer %p\n", erts_get_current_pid(), bp->mem); -#endif + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] new message buffer %p\n", erts_get_current_pid(), bp->mem)); return bp; } diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index f2da5289d3..96d17306a5 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -11274,8 +11274,8 @@ static void delete_process(Process* p) { VERBOSE(DEBUG_PROCESSES, ("Removing process: %T\n",p->common.id)); - VERBOSE_DEBUG("[pid=%T] delete process: %p %p %p %p\n", p->common.id, - HEAP_START(p), HEAP_END(p), OLD_HEAP(p), OLD_HEND(p)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] delete process: %p %p %p %p\n", p->common.id, + HEAP_START(p), HEAP_END(p), OLD_HEAP(p), OLD_HEND(p))); /* Cleanup psd */ diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 3e1f3664bc..303b9ee51b 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1046,16 +1046,6 @@ void erl_error(char*, va_list); #define SHCOPY /* Use this if you want sharing-preserving copy to be initially disabled */ #undef SHCOPY_DISABLE -/* Use this with care, it is *very* verbose! */ -#undef SHCOPY_DEBUG -#endif - -#ifdef SHCOPY_DEBUG -#define VERBOSE_DEBUG(...) do { \ - erts_fprintf(stderr, __VA_ARGS__); \ - } while(0) -#else -#define VERBOSE_DEBUG(...) #endif #define ERTS_SHCOPY_FLG_MASK (((unsigned) 3) << 0) -- cgit v1.2.3 From eebdde01b149ea45966c7412bc2a062136457b54 Mon Sep 17 00:00:00 2001 From: Yiannis Tsiouris Date: Tue, 22 Jan 2013 18:16:33 +0200 Subject: Add --enable-sharing-preserving configure flag --- erts/configure.in | 12 ++++++++++++ erts/emulator/beam/erl_bif_info.c | 3 +++ erts/emulator/beam/global.h | 5 +---- 3 files changed, 16 insertions(+), 4 deletions(-) (limited to 'erts') diff --git a/erts/configure.in b/erts/configure.in index 2419925c33..9ad1588b6c 100644 --- a/erts/configure.in +++ b/erts/configure.in @@ -788,6 +788,18 @@ esac AC_SUBST(LIBCARBON) +dnl Check if we should/can build a sharing-preserving emulator + +AC_MSG_CHECKING(if we are building a sharing-preserving emulator) +if test "$enable_sharing_preserving" = "yes"; then + AC_DEFINE(SHCOPY, [1], + [Define if building a sharing-preserving emulator]) + AC_MSG_RESULT([yes]) +else + AC_MSG_RESULT([no]) +fi + + dnl some tests below will call this if we haven't already - and autoconf dnl can't handle those tests being done conditionally at runtime AC_PROG_CPP diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 1eb106a551..82c2aa4b9e 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -129,6 +129,9 @@ static char erts_system_version[] = ("Erlang/OTP " ERLANG_OTP_RELEASE #endif #ifdef USE_SYSTEMTAP " [systemtap]" +#endif +#ifdef SHCOPY + " [sharing-preserving]" #endif "\n"); diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 303b9ee51b..3c59df5f41 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1038,12 +1038,9 @@ void erl_error(char*, va_list); /* This controls whether sharing-preserving copy is used by Erlang */ +#ifdef SHCOPY #define SHCOPY_SEND #define SHCOPY_SPAWN - -#if defined(SHCOPY_SEND) \ - || defined(SHCOPY_SPAWN) -#define SHCOPY /* Use this if you want sharing-preserving copy to be initially disabled */ #undef SHCOPY_DISABLE #endif -- cgit v1.2.3 From bcdf32795a3f5a6aad95567cffba78fcecb9f40f Mon Sep 17 00:00:00 2001 From: "Nikolaos S. Papaspyrou" Date: Thu, 24 Apr 2014 12:59:39 +0300 Subject: Add support for maps in preserved copy --- erts/emulator/beam/copy.c | 61 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) (limited to 'erts') diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index f74c6b1c89..cb1f38429a 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -424,6 +424,19 @@ Uint size_shared(Eterm obj) } goto pop_next; } + case MAP_SUBTAG: { + map_t *mp = (map_t *) ptr; + Uint n = map_get_size(mp) + 1; + sum += n + 2; + ptr += 2; /* hdr + size words */ + while (n-- > 0) { + obj = *ptr++; + if (!IS_CONST(obj)) { + EQUEUE_PUT(s, obj); + } + } + goto pop_next; + } case BIN_MATCHSTATE_SUBTAG: erl_exit(ERTS_ABORT_EXIT, "size_shared: matchstate term not allowed"); @@ -523,6 +536,18 @@ cleanup: } goto cleanup_next; } + case MAP_SUBTAG: { + map_t *mp = (map_t *) ptr; + Uint n = map_get_size(mp) + 1; + ptr += 2; /* hdr + size words */ + while (n-- > 0) { + obj = *ptr++; + if (!IS_CONST(obj)) { + EQUEUE_PUT_UNCHECKED(s, obj); + } + } + goto cleanup_next; + } default: goto cleanup_next; } @@ -1162,6 +1187,19 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) } goto pop_next; } + case MAP_SUBTAG: { + map_t *mp = (map_t *) ptr; + Uint n = map_get_size(mp) + 1; + sum += n + 2; + ptr += 2; /* hdr + size words */ + while (n-- > 0) { + obj = *ptr++; + if (!IS_CONST(obj)) { + EQUEUE_PUT(s, obj); + } + } + goto pop_next; + } case BIN_MATCHSTATE_SUBTAG: erl_exit(ERTS_ABORT_EXIT, "size_shared: matchstate term not allowed"); @@ -1404,6 +1442,23 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E erts_refc_inc(&funp->fe->refc, 2); goto cleanup_next; } + case MAP_SUBTAG: { + map_t *mp = (map_t *) ptr; + Uint n = map_get_size(mp) + 1; + *resp = make_map(hp); + *hp++ = hdr; + *hp++ = *++ptr; + while (n-- > 0) { + obj = *++ptr; + if (IS_CONST(obj)) { + *hp++ = obj; + } else { + EQUEUE_PUT_UNCHECKED(s, obj); + *hp++ = HEAP_ELEM_TO_BE_FILLED; + } + } + goto cleanup_next; + } case REFC_BINARY_SUBTAG: { ProcBin* pb = (ProcBin *) ptr; sz = thing_arityval(hdr); @@ -1539,6 +1594,12 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E remaining = 1 + funp->num_free; break; } + case MAP_SUBTAG: { + map_t *mp = (map_t *) hscan; + remaining = map_get_size(mp) + 1; + hscan += 2; + break; + } case SUB_BINARY_SUBTAG: ASSERT(((ErlSubBin *) hscan)->bitoffs + ((ErlSubBin *) hscan)->bitsize > 0); -- cgit v1.2.3 From 99de9eb01d8884ac293fdda20ab65c045d215d46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Tue, 15 Sep 2015 17:47:36 +0200 Subject: Fix internal stacks The internal stacks has changed between releases. --- erts/emulator/beam/copy.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'erts') diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index cb1f38429a..added85b46 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -907,7 +907,7 @@ int disable_copy_shared = ERTS_SHCOPY_FLG_NONE; #define SHTABLE_PUSH(s,x,y,b) \ do { \ if (s.sp > s.end - SHTABLE_INCR) { \ - erl_grow_estack(&s, ESTK_DEF_STACK(s)); \ + erl_grow_estack(&(s), SHTABLE_INCR); \ } \ *s.sp++ = (x); \ *s.sp++ = (y); \ @@ -959,6 +959,7 @@ do { \ WSTK_DEF_STACK(s), /* wstart */ \ WSTK_DEF_STACK(s), /* wsp */ \ WSTK_DEF_STACK(s) + DEF_WSTACK_SIZE, /* wend */ \ + WSTK_DEF_STACK(s), /* wdflt */ \ ERTS_ALC_T_ESTACK /* alloc_type */ \ }; \ int WSTK_CONCAT(s,_bitoffs) = 0; \ @@ -969,8 +970,9 @@ do { \ /* no WSTK_DEF_STACK(s), read-only */ \ ErtsWStack s = { \ info->bitstore_start, /* wstart */ \ - NULL, /* wsp, read-only */ \ + NULL, /* wsp, read-only */ \ NULL, /* wend, read-only */ \ + NULL, /* wdef, read-only */ \ info->bitstore_alloc_type /* alloc_type */ \ }; \ int WSTK_CONCAT(s,_bitoffs) = 0; \ @@ -983,6 +985,7 @@ do { \ ESTK_DEF_STACK(s), /* start */ \ ESTK_DEF_STACK(s), /* sp */ \ ESTK_DEF_STACK(s) + DEF_ESTACK_SIZE, /* end */ \ + ESTK_DEF_STACK(s), /* default */ \ ERTS_ALC_T_ESTACK /* alloc_type */ \ }; \ Uint ESTK_CONCAT(s,_offset) = 0 @@ -991,8 +994,9 @@ do { \ /* no ESTK_DEF_STACK(s), read-only */ \ ErtsEStack s = { \ info->shtable_start, /* start */ \ - NULL, /* sp, read-only */ \ + NULL, /* sp, read-only */ \ NULL, /* end, read-only */ \ + NULL, /* def, read-only */ \ info->shtable_alloc_type /* alloc_type */ \ }; \ /* no ESTK_CONCAT(s,_offset), read-only */ -- cgit v1.2.3 From 2ab76f7f1fe6ec4c94541ba5a2f0cd9dc1b9c79a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Tue, 15 Sep 2015 16:49:28 +0200 Subject: Fix Halfword removal Halfword is no longer present in the runtime system. --- erts/emulator/beam/copy.c | 40 +++++++++++++++++----------------------- 1 file changed, 17 insertions(+), 23 deletions(-) (limited to 'erts') diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index added85b46..42b4bee29d 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -298,11 +298,7 @@ do { \ * It is argued whether the size of subterms in constant pools * should be counted or not. */ -#if HALFWORD_HEAP -Uint size_shared_rel(Eterm obj, Eterm* base) -#else Uint size_shared(Eterm obj) -#endif { Eterm saved_obj = obj; Uint sum = 0; @@ -320,7 +316,7 @@ Uint size_shared(Eterm obj) switch (primary_tag(obj)) { case TAG_PRIMARY_LIST: { Eterm head, tail; - ptr = list_val_rel(obj, base); + ptr = list_val(obj); /* we're not counting anything that's outside our heap */ if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) { goto pop_next; @@ -357,7 +353,7 @@ Uint size_shared(Eterm obj) } case TAG_PRIMARY_BOXED: { Eterm hdr; - ptr = boxed_val_rel(obj, base); + ptr = boxed_val(obj); /* we're not counting anything that's outside our heap */ if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) { goto pop_next; @@ -414,13 +410,13 @@ Uint size_shared(Eterm obj) } else { extra_bytes = 0; } - ptr = binary_val_rel(sb->orig, base); + ptr = binary_val(sb->orig); hdr = (*ptr) & ~BOXED_VISITED_MASK; if (thing_subtag(hdr) == REFC_BINARY_SUBTAG) { sum += PROC_BIN_SIZE; } else { ASSERT(thing_subtag(hdr) == HEAP_BINARY_SUBTAG); - sum += heap_bin_size(binary_size_rel(obj, base) + extra_bytes); + sum += heap_bin_size(binary_size(obj) + extra_bytes); } goto pop_next; } @@ -465,7 +461,7 @@ cleanup: switch (primary_tag(obj)) { case TAG_PRIMARY_LIST: { Eterm head, tail; - ptr = list_val_rel(obj, base); + ptr = list_val(obj); if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) { goto cleanup_next; } @@ -495,7 +491,7 @@ cleanup: } case TAG_PRIMARY_BOXED: { Eterm hdr; - ptr = boxed_val_rel(obj, base); + ptr = boxed_val(obj); if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) { goto cleanup_next; } @@ -912,7 +908,7 @@ do { \ *s.sp++ = (x); \ *s.sp++ = (y); \ *s.sp++ = (Eterm) NULL; \ - *s.sp++ = (Eterm) (b); /* bad in HALF_WORD */ \ + *s.sp++ = (Eterm) (b); \ ESTK_CONCAT(s,_offset) += SHTABLE_INCR; \ } while(0) #define SHTABLE_X(s,e) (s.start[e]) @@ -1004,7 +1000,6 @@ do { \ /* * Copy object "obj" preserving sharing. * First half: count size and calculate sharing. - * NOTE: We do not support HALF_WORD (yet?). */ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) { @@ -1060,7 +1055,7 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) switch (primary_tag(obj)) { case TAG_PRIMARY_LIST: { Eterm head, tail; - ptr = list_val_rel(obj, base); + ptr = list_val(obj); /* off heap list pointers are copied verbatim */ if (!INHEAP(myself, ptr)) { VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", myself->common.id, ptr, obj)); @@ -1109,7 +1104,7 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) } case TAG_PRIMARY_BOXED: { Eterm hdr; - ptr = boxed_val_rel(obj, base); + ptr = boxed_val(obj); /* off heap pointers to boxes are copied verbatim */ if (!INHEAP(myself, ptr)) { VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", myself->common.id, ptr, obj)); @@ -1178,11 +1173,11 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) } else { extra_bytes = 0; } - ASSERT(is_boxed(rterm2wterm(real_bin, base)) && - (((*boxed_val(rterm2wterm(real_bin, base))) & + ASSERT(is_boxed(real_bin) && + (((*boxed_val(real_bin)) & (_TAG_HEADER_MASK - _BINARY_XXX_MASK - BOXED_VISITED_MASK)) == _TAG_HEADER_REFC_BIN)); - hdr = *_unchecked_binary_val(rterm2wterm(real_bin, base)) & ~BOXED_VISITED_MASK; + hdr = *_unchecked_binary_val(real_bin) & ~BOXED_VISITED_MASK; if (thing_subtag(hdr) == HEAP_BINARY_SUBTAG) { sum += heap_bin_size(size+extra_bytes); } else { @@ -1243,7 +1238,6 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) /* * Copy object "obj" preserving sharing. * Second half: copy and restore the object. - * NOTE: We do not support HALF_WORD (yet?). */ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, ErlOffHeap* off_heap, unsigned flags) { @@ -1309,7 +1303,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E switch (primary_tag(obj)) { case TAG_PRIMARY_LIST: { Eterm head, tail; - ptr = list_val_rel(obj, base); + ptr = list_val(obj); /* off heap list pointers are copied verbatim */ if (!INHEAP(myself, ptr)) { *resp = obj; @@ -1370,7 +1364,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E } case TAG_PRIMARY_BOXED: { Eterm hdr; - ptr = boxed_val_rel(obj, base); + ptr = boxed_val(obj); /* off heap pointers to boxes are copied verbatim */ if (!INHEAP(myself, ptr)) { *resp = obj; @@ -1500,11 +1494,11 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E extra_bytes = 0; } real_size = size+extra_bytes; - ASSERT(is_boxed(rterm2wterm(real_bin, base)) && - (((*boxed_val(rterm2wterm(real_bin, base))) & + ASSERT(is_boxed(real_bin) && + (((*boxed_val(real_bin)) & (_TAG_HEADER_MASK - _BINARY_XXX_MASK - BOXED_VISITED_MASK)) == _TAG_HEADER_REFC_BIN)); - ptr = _unchecked_binary_val(rterm2wterm(real_bin, base)); + ptr = _unchecked_binary_val(real_bin); *resp = make_binary(hp); if (extra_bytes != 0) { ErlSubBin* res = (ErlSubBin *) hp; -- cgit v1.2.3 From 95b25f7a3a33c43912079af713bf1f386a764071 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Tue, 15 Sep 2015 17:36:22 +0200 Subject: Fix Map preserved sharing copy implementation The Map implementation has changed since initial preserved copy implementation. --- erts/emulator/beam/copy.c | 135 +++++++++++++++++++++++++++++----------------- 1 file changed, 85 insertions(+), 50 deletions(-) (limited to 'erts') diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index 42b4bee29d..c7806ed8ff 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -420,19 +420,28 @@ Uint size_shared(Eterm obj) } goto pop_next; } - case MAP_SUBTAG: { - map_t *mp = (map_t *) ptr; - Uint n = map_get_size(mp) + 1; - sum += n + 2; - ptr += 2; /* hdr + size words */ - while (n-- > 0) { - obj = *ptr++; - if (!IS_CONST(obj)) { - EQUEUE_PUT(s, obj); + case MAP_SUBTAG: + switch (MAP_HEADER_TYPE(hdr)) { + case MAP_HEADER_TAG_FLATMAP_HEAD : { + flatmap_t *mp = (flatmap_t*)flatmap_val(obj); + Uint n = flatmap_get_size(mp) + 1; + ptr = (Eterm *)mp; + sum += n + 2; + ptr += 2; /* hdr + size words */ + while (n--) { + obj = *ptr++; + if (!IS_CONST(obj)) { + EQUEUE_PUT(s, obj); + } + } + goto pop_next; } + case MAP_HEADER_TAG_HAMT_HEAD_BITMAP : + case MAP_HEADER_TAG_HAMT_HEAD_ARRAY : + case MAP_HEADER_TAG_HAMT_NODE_BITMAP : + default: + erl_exit(ERTS_ABORT_EXIT, "size_shared: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); } - goto pop_next; - } case BIN_MATCHSTATE_SUBTAG: erl_exit(ERTS_ABORT_EXIT, "size_shared: matchstate term not allowed"); @@ -533,10 +542,10 @@ cleanup: goto cleanup_next; } case MAP_SUBTAG: { - map_t *mp = (map_t *) ptr; - Uint n = map_get_size(mp) + 1; + flatmap_t *mp = (flatmap_t *) ptr; + Uint n = flatmap_get_size(mp) + 1; ptr += 2; /* hdr + size words */ - while (n-- > 0) { + while (n--) { obj = *ptr++; if (!IS_CONST(obj)) { EQUEUE_PUT_UNCHECKED(s, obj); @@ -1186,20 +1195,28 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) } goto pop_next; } - case MAP_SUBTAG: { - map_t *mp = (map_t *) ptr; - Uint n = map_get_size(mp) + 1; - sum += n + 2; - ptr += 2; /* hdr + size words */ - while (n-- > 0) { - obj = *ptr++; - if (!IS_CONST(obj)) { - EQUEUE_PUT(s, obj); + case MAP_SUBTAG: + switch (MAP_HEADER_TYPE(hdr)) { + case MAP_HEADER_TAG_FLATMAP_HEAD : { + flatmap_t *mp = (flatmap_t *) ptr; + Uint n = flatmap_get_size(mp) + 1; + sum += n + 2; + ptr += 2; /* hdr + size words */ + while (n--) { + obj = *ptr++; + if (!IS_CONST(obj)) { + EQUEUE_PUT(s, obj); + } + } + goto pop_next; } + case MAP_HEADER_TAG_HAMT_HEAD_BITMAP : + case MAP_HEADER_TAG_HAMT_HEAD_ARRAY : + case MAP_HEADER_TAG_HAMT_NODE_BITMAP : + default: + erl_exit(ERTS_ABORT_EXIT, "copy_shared_calculate: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); } - goto pop_next; - } - case BIN_MATCHSTATE_SUBTAG: + case BIN_MATCHSTATE_SUBTAG: erl_exit(ERTS_ABORT_EXIT, "size_shared: matchstate term not allowed"); default: @@ -1234,7 +1251,6 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) } } - /* * Copy object "obj" preserving sharing. * Second half: copy and restore the object. @@ -1440,23 +1456,31 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E erts_refc_inc(&funp->fe->refc, 2); goto cleanup_next; } - case MAP_SUBTAG: { - map_t *mp = (map_t *) ptr; - Uint n = map_get_size(mp) + 1; - *resp = make_map(hp); - *hp++ = hdr; - *hp++ = *++ptr; - while (n-- > 0) { - obj = *++ptr; - if (IS_CONST(obj)) { - *hp++ = obj; - } else { - EQUEUE_PUT_UNCHECKED(s, obj); - *hp++ = HEAP_ELEM_TO_BE_FILLED; - } - } - goto cleanup_next; - } + case MAP_SUBTAG: + switch (MAP_HEADER_TYPE(hdr)) { + case MAP_HEADER_TAG_FLATMAP_HEAD : { + flatmap_t *mp = (flatmap_t *) ptr; + Uint n = flatmap_get_size(mp) + 1; + *resp = make_flatmap(hp); + *hp++ = hdr; + *hp++ = *++ptr; + while (n--) { + obj = *++ptr; + if (IS_CONST(obj)) { + *hp++ = obj; + } else { + EQUEUE_PUT_UNCHECKED(s, obj); + *hp++ = HEAP_ELEM_TO_BE_FILLED; + } + } + goto cleanup_next; + } + case MAP_HEADER_TAG_HAMT_HEAD_BITMAP : + case MAP_HEADER_TAG_HAMT_HEAD_ARRAY : + case MAP_HEADER_TAG_HAMT_NODE_BITMAP : + default: + erl_exit(ERTS_ABORT_EXIT, "copy_shared_perform: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); + } case REFC_BINARY_SUBTAG: { ProcBin* pb = (ProcBin *) ptr; sz = thing_arityval(hdr); @@ -1592,12 +1616,23 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E remaining = 1 + funp->num_free; break; } - case MAP_SUBTAG: { - map_t *mp = (map_t *) hscan; - remaining = map_get_size(mp) + 1; - hscan += 2; - break; - } + case MAP_SUBTAG: + switch (MAP_HEADER_TYPE(*hscan)) { + case MAP_HEADER_TAG_FLATMAP_HEAD : { + flatmap_t *mp = (flatmap_t *) hscan; + remaining = flatmap_get_size(mp) + 1; + hscan += 2; + break; + } + case MAP_HEADER_TAG_HAMT_HEAD_BITMAP : + case MAP_HEADER_TAG_HAMT_HEAD_ARRAY : + case MAP_HEADER_TAG_HAMT_NODE_BITMAP : + default: + erl_exit(ERTS_ABORT_EXIT, + "copy_shared_perform: bad hashmap type %d\n", + MAP_HEADER_TYPE(*hscan)); + } + break; case SUB_BINARY_SUBTAG: ASSERT(((ErlSubBin *) hscan)->bitoffs + ((ErlSubBin *) hscan)->bitsize > 0); -- cgit v1.2.3 From 2e10fe29f61bbe3246902e8eaf1636dd6457979f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Wed, 16 Sep 2015 16:21:43 +0200 Subject: Add support for HAMT maps in preserved copy --- erts/emulator/beam/copy.c | 115 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 88 insertions(+), 27 deletions(-) (limited to 'erts') diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index c7806ed8ff..aa17713d07 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -438,7 +438,18 @@ Uint size_shared(Eterm obj) } case MAP_HEADER_TAG_HAMT_HEAD_BITMAP : case MAP_HEADER_TAG_HAMT_HEAD_ARRAY : - case MAP_HEADER_TAG_HAMT_NODE_BITMAP : + case MAP_HEADER_TAG_HAMT_NODE_BITMAP : { + Uint n = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + sum += 1 + n + header_arity(hdr); + ptr += 1 + header_arity(hdr); + while (n--) { + obj = *ptr++; + if (!IS_CONST(obj)) { + EQUEUE_PUT(s, obj); + } + } + goto pop_next; + } default: erl_exit(ERTS_ABORT_EXIT, "size_shared: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); } @@ -541,18 +552,37 @@ cleanup: } goto cleanup_next; } - case MAP_SUBTAG: { - flatmap_t *mp = (flatmap_t *) ptr; - Uint n = flatmap_get_size(mp) + 1; - ptr += 2; /* hdr + size words */ - while (n--) { - obj = *ptr++; - if (!IS_CONST(obj)) { - EQUEUE_PUT_UNCHECKED(s, obj); + case MAP_SUBTAG: + switch (MAP_HEADER_TYPE(hdr)) { + case MAP_HEADER_TAG_FLATMAP_HEAD : { + flatmap_t *mp = (flatmap_t *) ptr; + Uint n = flatmap_get_size(mp) + 1; + ptr += 2; /* hdr + size words */ + while (n--) { + obj = *ptr++; + if (!IS_CONST(obj)) { + EQUEUE_PUT_UNCHECKED(s, obj); + } + } + goto cleanup_next; + } + case MAP_HEADER_TAG_HAMT_HEAD_BITMAP : + case MAP_HEADER_TAG_HAMT_HEAD_ARRAY : + case MAP_HEADER_TAG_HAMT_NODE_BITMAP : { + Uint n = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + sum += 1 + n + header_arity(hdr); + ptr += 1 + header_arity(hdr); + while (n--) { + obj = *ptr++; + if (!IS_CONST(obj)) { + EQUEUE_PUT_UNCHECKED(s, obj); + } + } + goto cleanup_next; } + default: + erl_exit(ERTS_ABORT_EXIT, "size_shared: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); } - goto cleanup_next; - } default: goto cleanup_next; } @@ -1212,7 +1242,22 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) } case MAP_HEADER_TAG_HAMT_HEAD_BITMAP : case MAP_HEADER_TAG_HAMT_HEAD_ARRAY : - case MAP_HEADER_TAG_HAMT_NODE_BITMAP : + case MAP_HEADER_TAG_HAMT_NODE_BITMAP : { + Uint n = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + sum += 1 + n + header_arity(hdr); + ptr += 1 + header_arity(hdr); + + if (n == 0) { + goto pop_next; + } + while(n--) { + obj = *ptr++; + if (!IS_CONST(obj)) { + EQUEUE_PUT(s, obj); + } + } + goto pop_next; + } default: erl_exit(ERTS_ABORT_EXIT, "copy_shared_calculate: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); } @@ -1228,20 +1273,20 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) case TAG_PRIMARY_IMMED1: pop_next: if (EQUEUE_ISEMPTY(s)) { - // add sentinel to the table - SHTABLE_PUSH(t, THE_NON_VALUE, THE_NON_VALUE, NULL); - // store persistent info - BITSTORE_CLOSE(b); - info->queue_start = s.start; - info->queue_end = s.end; + /* add sentinel to the table */ + SHTABLE_PUSH(t, THE_NON_VALUE, THE_NON_VALUE, NULL); + /* store persistent info */ + BITSTORE_CLOSE(b); + info->queue_start = s.start; + info->queue_end = s.end; info->queue_alloc_type = s.alloc_type; - info->bitstore_start = b.wstart; + info->bitstore_start = b.wstart; info->bitstore_alloc_type = b.alloc_type; - info->shtable_start = t.start; + info->shtable_start = t.start; info->shtable_alloc_type = t.alloc_type; - // single point of return: the size of the object - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] size was: %u\n", myself->common.id, sum)); - return sum; + /* single point of return: the size of the object */ + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] size was: %u\n", myself->common.id, sum)); + return sum; } obj = EQUEUE_GET(s); break; @@ -1457,13 +1502,13 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E goto cleanup_next; } case MAP_SUBTAG: + *resp = make_flatmap(hp); + *hp++ = hdr; switch (MAP_HEADER_TYPE(hdr)) { case MAP_HEADER_TAG_FLATMAP_HEAD : { flatmap_t *mp = (flatmap_t *) ptr; Uint n = flatmap_get_size(mp) + 1; - *resp = make_flatmap(hp); - *hp++ = hdr; - *hp++ = *++ptr; + *hp++ = *++ptr; /* keys */ while (n--) { obj = *++ptr; if (IS_CONST(obj)) { @@ -1477,7 +1522,20 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E } case MAP_HEADER_TAG_HAMT_HEAD_BITMAP : case MAP_HEADER_TAG_HAMT_HEAD_ARRAY : - case MAP_HEADER_TAG_HAMT_NODE_BITMAP : + *hp++ = *++ptr; /* total map size */ + case MAP_HEADER_TAG_HAMT_NODE_BITMAP : { + Uint n = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + while (n--) { + obj = *++ptr; + if (IS_CONST(obj)) { + *hp++ = obj; + } else { + EQUEUE_PUT_UNCHECKED(s, obj); + *hp++ = HEAP_ELEM_TO_BE_FILLED; + } + } + goto cleanup_next; + } default: erl_exit(ERTS_ABORT_EXIT, "copy_shared_perform: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); } @@ -1627,6 +1685,9 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E case MAP_HEADER_TAG_HAMT_HEAD_BITMAP : case MAP_HEADER_TAG_HAMT_HEAD_ARRAY : case MAP_HEADER_TAG_HAMT_NODE_BITMAP : + remaining = hashmap_bitcount(MAP_HEADER_VAL(*hscan)); + hscan += MAP_HEADER_ARITY(*hscan) + 1; + break; default: erl_exit(ERTS_ABORT_EXIT, "copy_shared_perform: bad hashmap type %d\n", -- cgit v1.2.3 From 748c73f1687b2375d4c607487f40036ba990c4c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Thu, 17 Sep 2015 16:17:10 +0200 Subject: Refactor copy sharing --- erts/emulator/beam/beam_debug.c | 6 +++--- erts/emulator/beam/copy.c | 10 ++++----- erts/emulator/beam/erl_db.c | 4 ++-- erts/emulator/beam/erl_message.h | 6 +++--- erts/emulator/beam/erl_process.c | 6 +++--- erts/emulator/beam/global.h | 46 ++++++++++++++++++++-------------------- 6 files changed, 39 insertions(+), 39 deletions(-) (limited to 'erts') diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index 36f3cfabbc..b007d000bf 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -96,18 +96,18 @@ erts_debug_copy_shared_1(BIF_ALIST_1) Uint size; Eterm* hp; Eterm copy; - shcopy_info info; + erts_shcopy_t info; #ifdef SHCOPY_DISABLE extern int disable_copy_shared; #endif - INITIALIZE_INFO(info); + INITIALIZE_SHCOPY(info); size = copy_shared_calculate(term, &info, 0); if (size > 0) { hp = HAlloc(p, size); } copy = copy_shared_perform(term, size, &info, &hp, &p->off_heap, 0); - DESTROY_INFO(info); + DESTROY_SHCOPY(info); #ifdef SHCOPY_DISABLE disable_copy_shared = 0; #endif diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index aa17713d07..f2dd73d862 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -1040,14 +1040,14 @@ do { \ * Copy object "obj" preserving sharing. * First half: count size and calculate sharing. */ -Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) +Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info, Uint32 flags) { Uint sum; Uint e; unsigned sz; Eterm* ptr; Process* myself; - int force_local = flags & ERTS_SHCOPY_FLG_TMP_BUF; + int force_local = flags & ERTS_SHCOPY_FLG_TMPBUF; DECLARE_EQUEUE_INIT_INFO(s, info); DECLARE_BITSTORE_INIT_INFO(b, info); @@ -1300,8 +1300,8 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags) * Copy object "obj" preserving sharing. * Second half: copy and restore the object. */ -Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, ErlOffHeap* off_heap, unsigned flags) -{ +Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, + Eterm** hpp, ErlOffHeap* off_heap, Uint32 flags) { Uint e; unsigned sz; Eterm* ptr; @@ -1311,7 +1311,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E Eterm* resp; unsigned remaining; Process* myself; - int force_local = flags & ERTS_SHCOPY_FLG_TMP_BUF; + int force_local = flags & ERTS_SHCOPY_FLG_TMPBUF; #ifdef DEBUG Eterm saved_obj = obj; #endif diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 6119a9225f..59a0c0b808 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -1832,7 +1832,7 @@ BIF_RETTYPE ets_give_away_3(BIF_ALIST_3) tb->common.id, from_pid, BIF_ARG_3), - ERTS_SND_FLG_SHCOPY_TMP_BUF); + ERTS_SND_FLG_SHCOPY_TMPBUF); erts_smp_proc_unlock(to_proc, to_locks); UnUseTmpHeap(5,BIF_P); BIF_RET(am_true); @@ -3211,7 +3211,7 @@ retry: tb->common.id, p->common.id, heir_data), - ERTS_SND_FLG_SHCOPY_TMP_BUF); + ERTS_SND_FLG_SHCOPY_TMPBUF); erts_smp_proc_unlock(to_proc, to_locks); return !0; } diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index 0de36c3199..52a648fe0b 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -239,9 +239,9 @@ do { \ #define ERTS_SND_FLG_NO_SEQ_TRACE (((unsigned) 1) << 0) #define ERTS_SND_FLG_SHCOPY_SHIFT 1 -#define ERTS_SND_FLG_SHCOPY_MASK (ERTS_SHCOPY_FLG_MASK << ERTS_SND_FLG_SHCOPY_SHIFT) -#define ERTS_SND_FLG_SHCOPY_NONE (ERTS_SHCOPY_FLG_NONE << ERTS_SND_FLG_SHCOPY_SHIFT) -#define ERTS_SND_FLG_SHCOPY_TMP_BUF (ERTS_SHCOPY_FLG_TMP_BUF << ERTS_SND_FLG_SHCOPY_SHIFT) +#define ERTS_SND_FLG_SHCOPY_MASK (ERTS_SHCOPY_FLG_MASK << ERTS_SND_FLG_SHCOPY_SHIFT) +#define ERTS_SND_FLG_SHCOPY_NONE (ERTS_SHCOPY_FLG_NONE << ERTS_SND_FLG_SHCOPY_SHIFT) +#define ERTS_SND_FLG_SHCOPY_TMPBUF (ERTS_SHCOPY_FLG_TMPBUF << ERTS_SND_FLG_SHCOPY_SHIFT) #define ERTS_HEAP_FRAG_SIZE(DATA_WORDS) \ (sizeof(ErlHeapFragment) - sizeof(Eterm) + (DATA_WORDS)*sizeof(Eterm)) diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 96d17306a5..a691a3c773 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -10749,8 +10749,8 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). erts_aint32_t prio = (erts_aint32_t) PRIORITY_NORMAL; #ifdef SHCOPY_SPAWN unsigned shflags = 0; /* could be taken from so->flags, if necessary */ - shcopy_info info; - INITIALIZE_INFO(info); + erts_shcopy_t info; + INITIALIZE_SHCOPY(info); #endif #ifdef ERTS_SMP @@ -10887,7 +10887,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). BM_SWAP_TIMER(system,copy); #ifdef SHCOPY_SPAWN p->arg_reg[2] = copy_shared_perform(args, arg_size, &info, &p->htop, &p->off_heap, shflags); - DESTROY_INFO(info); + DESTROY_SHCOPY(info); #else p->arg_reg[2] = copy_struct(args, arg_size, &p->htop, &p->off_heap); #endif diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 3c59df5f41..bdee20969d 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1045,13 +1045,13 @@ void erl_error(char*, va_list); #undef SHCOPY_DISABLE #endif -#define ERTS_SHCOPY_FLG_MASK (((unsigned) 3) << 0) -#define ERTS_SHCOPY_FLG_NONE (((unsigned) 1) << 0) -#define ERTS_SHCOPY_FLG_TMP_BUF (((unsigned) 1) << 1) +#define ERTS_SHCOPY_FLG_MASK (((Uint32) 3) << 0) +#define ERTS_SHCOPY_FLG_NONE (((Uint32) 1) << 0) +#define ERTS_SHCOPY_FLG_TMPBUF (((Uint32) 1) << 1) /* forces INHEAP to true */ /* The persistent state while the sharing-preserving copier works */ -typedef struct shcopy_info { +typedef struct { Eterm queue_default[DEF_EQUEUE_SIZE]; Eterm* queue_start; Eterm* queue_end; @@ -1062,26 +1062,26 @@ typedef struct shcopy_info { Eterm shtable_default[DEF_ESTACK_SIZE]; Eterm* shtable_start; ErtsAlcType_t shtable_alloc_type; -} shcopy_info; +} erts_shcopy_t; -#define INITIALIZE_INFO(info) \ -do { \ - info.queue_start = info.queue_default; \ - info.bitstore_start = info.bitstore_default; \ - info.shtable_start = info.shtable_default; \ +#define INITIALIZE_SHCOPY(info) \ +do { \ + info.queue_start = info.queue_default; \ + info.bitstore_start = info.bitstore_default; \ + info.shtable_start = info.shtable_default; \ } while(0) -#define DESTROY_INFO(info) \ -do { \ - if (info.queue_start != info.queue_default) { \ - erts_free(info.queue_alloc_type, info.queue_start); \ - } \ - if (info.bitstore_start != info.bitstore_default) { \ - erts_free(info.bitstore_alloc_type, info.bitstore_start); \ - } \ - if (info.shtable_start != info.shtable_default) { \ - erts_free(info.shtable_alloc_type, info.shtable_start); \ - } \ +#define DESTROY_SHCOPY(info) \ +do { \ + if (info.queue_start != info.queue_default) { \ + erts_free(info.queue_alloc_type, info.queue_start); \ + } \ + if (info.bitstore_start != info.bitstore_default) { \ + erts_free(info.bitstore_alloc_type, info.bitstore_start); \ + } \ + if (info.shtable_start != info.shtable_default) { \ + erts_free(info.shtable_alloc_type, info.shtable_start); \ + } \ } while(0) /* copy.c */ @@ -1089,8 +1089,8 @@ Eterm copy_object_x(Eterm, Process*, Uint); #define copy_object(Term, Proc) copy_object_x(Term,Proc,0) Uint size_object(Eterm); -Uint copy_shared_calculate(Eterm, shcopy_info*, unsigned); -Eterm copy_shared_perform(Eterm, Uint, shcopy_info*, Eterm**, ErlOffHeap*, unsigned); +Uint copy_shared_calculate(Eterm, erts_shcopy_t*, Uint32); +Eterm copy_shared_perform(Eterm, Uint, erts_shcopy_t*, Eterm**, ErlOffHeap*, Uint32); Uint size_shared(Eterm); -- cgit v1.2.3 From 5d2d888f368ba8ca0098d8bd9936ca9d67df7d3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Thu, 17 Sep 2015 19:30:08 +0200 Subject: Copy literals in copy sharing --- erts/emulator/beam/copy.c | 49 +++++++++++++++++++++++++++++---------------- erts/emulator/beam/global.h | 6 +++++- 2 files changed, 37 insertions(+), 18 deletions(-) (limited to 'erts') diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index f2dd73d862..4f4d20cb83 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -611,7 +611,7 @@ cleanup: /* * Copy a structure to a heap. */ -Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) +Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint *bsz) { char* hstart; Uint hsize; @@ -626,6 +626,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) Eterm* argp; Eterm* const_tuple; Eterm hdr; + Eterm *hend; int i; #ifdef DEBUG Eterm org_obj = obj; @@ -641,7 +642,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) DTRACE1(copy_struct, (int32_t)sz); hp = htop = *hpp; - hbot = htop + sz; + hbot = hend = htop + sz; hstart = (char *)htop; hsize = (char*) hbot - hstart; const_tuple = 0; @@ -906,19 +907,24 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) } } + if (bsz) { + *hpp = htop; + *bsz = hend - hbot; + } else { #ifdef DEBUG - if (htop != hbot) - erl_exit(ERTS_ABORT_EXIT, - "Internal error in copy_struct() when copying %T:" - " htop=%p != hbot=%p (sz=%beu)\n", - org_obj, htop, hbot, org_sz); + if (htop != hbot) + erl_exit(ERTS_ABORT_EXIT, + "Internal error in copy_struct() when copying %T:" + " htop=%p != hbot=%p (sz=%beu)\n", + org_obj, htop, hbot, org_sz); #else - if (htop > hbot) { - erl_exit(ERTS_ABORT_EXIT, - "Internal error in copy_struct(): htop, hbot overrun\n"); - } + if (htop > hbot) { + erl_exit(ERTS_ABORT_EXIT, + "Internal error in copy_struct(): htop, hbot overrun\n"); + } #endif - *hpp = (Eterm *) (hstart+hsize); + *hpp = (Eterm *) (hstart+hsize); + } VERBOSE(DEBUG_SHCOPY, ("[pid=%T] result is at %p\n", mypid, res)); return res; } @@ -1098,6 +1104,7 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info, Uint32 flags) /* off heap list pointers are copied verbatim */ if (!INHEAP(myself, ptr)) { VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", myself->common.id, ptr, obj)); + info->literal_size += size_object(obj); goto pop_next; } head = CAR(ptr); @@ -1147,6 +1154,7 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info, Uint32 flags) /* off heap pointers to boxes are copied verbatim */ if (!INHEAP(myself, ptr)) { VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", myself->common.id, ptr, obj)); + info->literal_size += size_object(obj); goto pop_next; } hdr = *ptr; @@ -1286,7 +1294,7 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info, Uint32 flags) info->shtable_alloc_type = t.alloc_type; /* single point of return: the size of the object */ VERBOSE(DEBUG_SHCOPY, ("[pid=%T] size was: %u\n", myself->common.id, sum)); - return sum; + return sum + info->literal_size; } obj = EQUEUE_GET(s); break; @@ -1309,6 +1317,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, Eterm* hscan; Eterm result; Eterm* resp; + Eterm *hbot, *hend; unsigned remaining; Process* myself; int force_local = flags & ERTS_SHCOPY_FLG_TMPBUF; @@ -1346,6 +1355,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, */ hscan = hp = *hpp; + hbot = hend = hp + size; /* step #3: ------------------------------------------------------- @@ -1367,7 +1377,9 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, ptr = list_val(obj); /* off heap list pointers are copied verbatim */ if (!INHEAP(myself, ptr)) { - *resp = obj; + Uint bsz; + *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz); + hbot -= bsz; goto cleanup_next; } head = CAR(ptr); @@ -1428,7 +1440,9 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, ptr = boxed_val(obj); /* off heap pointers to boxes are copied verbatim */ if (!INHEAP(myself, ptr)) { - *resp = obj; + Uint bsz; + *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz); + hbot -= bsz; goto cleanup_next; } hdr = *ptr; @@ -1759,8 +1773,9 @@ all_clean: VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy is %T\n", myself->common.id, result)); VERBOSE(DEBUG_SHCOPY, ("[pid=%T] result is at %p\n", myself->common.id, result)); - ASSERT(hp == *hpp + size); - *hpp = hp; + ASSERT(hbot == hp); + ASSERT(size == ((hp - *hpp) + (hend - hbot))); + *hpp = hend; return result; } diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index bdee20969d..f106b941ef 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1062,6 +1062,7 @@ typedef struct { Eterm shtable_default[DEF_ESTACK_SIZE]; Eterm* shtable_start; ErtsAlcType_t shtable_alloc_type; + Uint literal_size; } erts_shcopy_t; #define INITIALIZE_SHCOPY(info) \ @@ -1069,6 +1070,7 @@ do { \ info.queue_start = info.queue_default; \ info.bitstore_start = info.bitstore_default; \ info.shtable_start = info.shtable_default; \ + info.literal_size = 0; \ } while(0) #define DESTROY_SHCOPY(info) \ @@ -1094,7 +1096,9 @@ Eterm copy_shared_perform(Eterm, Uint, erts_shcopy_t*, Eterm**, ErlOffHeap*, Uin Uint size_shared(Eterm); -Eterm copy_struct(Eterm, Uint, Eterm**, ErlOffHeap*); +Eterm copy_struct_x(Eterm, Uint, Eterm**, ErlOffHeap*, Uint* bsz); +#define copy_struct(Obj,Sz,HPP,OH) \ + copy_struct_x(Obj,Sz,HPP,OH,NULL) Eterm copy_shallow(Eterm*, Uint, Eterm**, ErlOffHeap*); void erts_move_multi_frags(Eterm** hpp, ErlOffHeap*, ErlHeapFragment* first, -- cgit v1.2.3 From 8f8aa9c5c4e26e563c935e06f8346175fa15d876 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Fri, 18 Sep 2015 19:20:19 +0200 Subject: Add BIF for setting internal copy literal range --- erts/emulator/beam/beam_bif_load.c | 104 ++++++++++++++++++++++++++++++++++--- erts/emulator/beam/bif.tab | 1 + 2 files changed, 99 insertions(+), 6 deletions(-) (limited to 'erts') diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index c3ebf71a01..e61d9ab0a6 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -154,7 +154,7 @@ static struct /* Protected by code_write_permission */ { Process* stager; ErtsThrPrgrLaterOp lop; -}commiter_state; +} committer_state; #endif static Eterm @@ -367,9 +367,9 @@ staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking, * schedulers to read active code_ix in a safe way while executing * without any memory barriers at all. */ - ASSERT(commiter_state.stager == NULL); - commiter_state.stager = c_p; - erts_schedule_thr_prgr_later_op(smp_code_ix_commiter, NULL, &commiter_state.lop); + ASSERT(committer_state.stager == NULL); + committer_state.stager = c_p; + erts_schedule_thr_prgr_later_op(smp_code_ix_commiter, NULL, &committer_state.lop); erts_proc_inc_refc(c_p); erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL); /* @@ -385,11 +385,11 @@ staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking, #ifdef ERTS_SMP static void smp_code_ix_commiter(void* null) { - Process* p = commiter_state.stager; + Process* p = committer_state.stager; erts_commit_staging_code_ix(); #ifdef DEBUG - commiter_state.stager = NULL; + committer_state.stager = NULL; #endif erts_release_code_write_permission(); erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); @@ -1007,6 +1007,98 @@ any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size) return 0; } +#undef in_area + +#ifdef ERTS_SMP +static void copy_literals_commit(void*); +#endif + +typedef struct { + Eterm *ptr; + Uint sz; +} copy_literals_t; + +copy_literals_t erts_clrange = {NULL, 0}; + +/* copy literals + * + * copy_literals.ptr = LitPtr + * copy_literals.sz = LitSz + * ------ THR PROG COMMIT ----- + * + * - check process code + * - check process code + * ... + * copy_literals.ptr = NULL + * copy_literals.sz = 0 + * ------ THR PROG COMMIT ----- + * ... + */ + + +BIF_RETTYPE copy_literals_2(BIF_ALIST_2) +{ + Module* modp; + ErtsCodeIndex code_ix; + Eterm res = am_true; + + if (is_not_atom(BIF_ARG_1) || (am_true != BIF_ARG_2 && am_false != BIF_ARG_2)) { + BIF_ERROR(BIF_P, BADARG); + } + + if (!erts_try_seize_code_write_permission(BIF_P)) { + ERTS_BIF_YIELD2(bif_export[BIF_copy_literals_2], BIF_P, BIF_ARG_1, BIF_ARG_2); + } + + code_ix = erts_active_code_ix(); + + if ((modp = erts_get_module(BIF_ARG_1, code_ix)) == NULL || !modp->old.code_hdr) { + res = am_false; + goto done; + } + + if (BIF_ARG_2 == am_true) { + if (erts_clrange.ptr != NULL) { + res = am_aborted; + goto done; + } + erts_clrange.ptr = (Eterm*) modp->old.code_hdr->literals_start; + erts_clrange.sz = (Eterm*) modp->old.code_hdr->literals_end - erts_clrange.ptr; + } else if (BIF_ARG_2 == am_false) { + erts_clrange.ptr = NULL; + erts_clrange.sz = 0; + } + +#ifdef ERTS_SMP + ASSERT(committer_state.stager == NULL); + committer_state.stager = BIF_P; + erts_schedule_thr_prgr_later_op(copy_literals_commit, NULL, &committer_state.lop); + erts_proc_inc_refc(BIF_P); + erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL); + ERTS_BIF_YIELD_RETURN(BIF_P, am_true); +#endif +done: + erts_release_code_write_permission(); + BIF_RET(res); +} + +#ifdef ERTS_SMP +static void copy_literals_commit(void* null) { + Process* p = committer_state.stager; +#ifdef DEBUG + committer_state.stager = NULL; +#endif + erts_release_code_write_permission(); + erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + if (!ERTS_PROC_IS_EXITING(p)) { + erts_resume(p, ERTS_PROC_LOCK_STATUS); + } + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + erts_proc_dec_refc(p); +} +#endif /* ERTS_SMP */ + + BIF_RETTYPE purge_module_1(BIF_ALIST_1) { ErtsCodeIndex code_ix; diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index f45f886395..63a0d0b1f0 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -642,6 +642,7 @@ bif erts_debug:map_info/1 # New in 19.0 # +bif erlang:copy_literals/2 bif binary:split/2 bif binary:split/3 bif erts_debug:size_shared/1 -- cgit v1.2.3 From 1391715d8bbba315e1509e60e6245159a009bd9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Fri, 25 Sep 2015 17:03:02 +0200 Subject: Use copy literal range check in message passing and purging --- erts/emulator/beam/beam_bif_load.c | 5 -- erts/emulator/beam/copy.c | 136 ++++++++++++++++++------------------- erts/emulator/beam/global.h | 10 +++ 3 files changed, 75 insertions(+), 76 deletions(-) (limited to 'erts') diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index e61d9ab0a6..6b6c066211 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -1013,11 +1013,6 @@ any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size) static void copy_literals_commit(void*); #endif -typedef struct { - Eterm *ptr; - Uint sz; -} copy_literals_t; - copy_literals_t erts_clrange = {NULL, 0}; /* copy literals diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index 4f4d20cb83..5bca3877b5 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -276,49 +276,35 @@ do { \ #define BOXED_SHARED_UNPROCESSED ((Eterm) 2) #define BOXED_SHARED_PROCESSED ((Eterm) 3) +#define COUNT_OFF_HEAP (0) -/* - * Is an object in the local heap of a process? - */ - -#define INHEAP_SIMPLE(p, ptr) ( \ - (OLD_HEAP(p) && OLD_HEAP(p) <= ptr && ptr < OLD_HEND(p)) || \ - (HEAP_START(p) <= ptr && ptr < HEAP_END(p)) \ - ) -#define INHEAP(p, ptr) ( \ - INHEAP_SIMPLE(p, ptr) || \ - (force_local ? (force_local = 0, 1) : 0) \ - ) -#define COUNT_OFF_HEAP 0 - - +#define IN_LITERAL_PURGE_AREA(info, ptr) \ + ((info)->range_ptr && ( \ + (info)->range_ptr <= (ptr) && \ + (ptr) < ((info)->range_ptr + (info)->range_sz))) /* * Return the real size of an object and find sharing information * This currently returns the same as erts_debug:size/1. * It is argued whether the size of subterms in constant pools * should be counted or not. */ + Uint size_shared(Eterm obj) { Eterm saved_obj = obj; Uint sum = 0; Eterm* ptr; - Process* myself; DECLARE_EQUEUE(s); DECLARE_BITSTORE(b); - myself = erts_get_current_process(); - if (myself == NULL) - return size_object(obj); - for (;;) { switch (primary_tag(obj)) { case TAG_PRIMARY_LIST: { Eterm head, tail; ptr = list_val(obj); /* we're not counting anything that's outside our heap */ - if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) { + if (!COUNT_OFF_HEAP && erts_is_literal(obj,ptr)) { goto pop_next; } head = CAR(ptr); @@ -355,7 +341,7 @@ Uint size_shared(Eterm obj) Eterm hdr; ptr = boxed_val(obj); /* we're not counting anything that's outside our heap */ - if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) { + if (!COUNT_OFF_HEAP && erts_is_literal(obj,ptr)) { goto pop_next; } hdr = *ptr; @@ -482,7 +468,7 @@ cleanup: case TAG_PRIMARY_LIST: { Eterm head, tail; ptr = list_val(obj); - if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) { + if (!COUNT_OFF_HEAP && erts_is_literal(obj,ptr)) { goto cleanup_next; } head = CAR(ptr); @@ -512,7 +498,7 @@ cleanup: case TAG_PRIMARY_BOXED: { Eterm hdr; ptr = boxed_val(obj); - if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) { + if (!COUNT_OFF_HEAP && erts_is_literal(obj,ptr)) { goto cleanup_next; } hdr = *ptr; @@ -1052,8 +1038,9 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info, Uint32 flags) Uint e; unsigned sz; Eterm* ptr; - Process* myself; - int force_local = flags & ERTS_SHCOPY_FLG_TMPBUF; +#ifdef DEBUG + Eterm mypid = erts_get_current_pid(); +#endif DECLARE_EQUEUE_INIT_INFO(s, info); DECLARE_BITSTORE_INIT_INFO(b, info); @@ -1073,12 +1060,11 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info, Uint32 flags) flags |= disable_copy_shared; #endif - myself = erts_get_current_process(); - if (myself == NULL || (flags & ERTS_SHCOPY_FLG_NONE)) + if (flags & ERTS_SHCOPY_FLG_NONE) return size_object(obj); - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy_shared_calculate %p\n", myself->common.id, obj)); - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] message is %T\n", myself->common.id, obj)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy_shared_calculate %p\n", mypid, obj)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] message is %T\n", mypid, obj)); /* step #1: ------------------------------------------------------- @@ -1102,9 +1088,10 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info, Uint32 flags) Eterm head, tail; ptr = list_val(obj); /* off heap list pointers are copied verbatim */ - if (!INHEAP(myself, ptr)) { - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", myself->common.id, ptr, obj)); - info->literal_size += size_object(obj); + if (erts_is_literal(obj,ptr)) { + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", mypid, ptr, obj)); + if (IN_LITERAL_PURGE_AREA(info,ptr)) + info->literal_size += size_object(obj); goto pop_next; } head = CAR(ptr); @@ -1115,7 +1102,7 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info, Uint32 flags) primary_tag(head) == TAG_PRIMARY_HEADER) { if (tail != THE_NON_VALUE) { e = SHTABLE_NEXT(t); - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] tabling L %p\n", myself->common.id, ptr)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] tabling L %p\n", mypid, ptr)); SHTABLE_PUSH(t, head, tail, ptr); CAR(ptr) = (e << _TAG_PRIMARY_SIZE) | LIST_SHARED_UNPROCESSED; CDR(ptr) = THE_NON_VALUE; @@ -1125,17 +1112,17 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info, Uint32 flags) /* else make it visited now */ switch (primary_tag(tail)) { case TAG_PRIMARY_LIST: - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] mangling L/L %p\n", myself->common.id, ptr)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] mangling L/L %p\n", mypid, ptr)); CDR(ptr) = (tail - TAG_PRIMARY_LIST) | TAG_PRIMARY_HEADER; break; case TAG_PRIMARY_IMMED1: - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] mangling L/I %p\n", myself->common.id, ptr)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] mangling L/I %p\n", mypid, ptr)); CAR(ptr) = (head - primary_tag(head)) | TAG_PRIMARY_HEADER; CDR(ptr) = (tail - TAG_PRIMARY_IMMED1) | primary_tag(head); break; case TAG_PRIMARY_BOXED: BITSTORE_PUT(b, primary_tag(head)); - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] mangling L/B %p\n", myself->common.id, ptr)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] mangling L/B %p\n", mypid, ptr)); CAR(ptr) = (head - primary_tag(head)) | TAG_PRIMARY_HEADER; CDR(ptr) = (tail - TAG_PRIMARY_BOXED) | TAG_PRIMARY_HEADER; break; @@ -1152,9 +1139,10 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info, Uint32 flags) Eterm hdr; ptr = boxed_val(obj); /* off heap pointers to boxes are copied verbatim */ - if (!INHEAP(myself, ptr)) { - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", myself->common.id, ptr, obj)); - info->literal_size += size_object(obj); + if (erts_is_literal(obj,ptr)) { + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", mypid, ptr, obj)); + if (IN_LITERAL_PURGE_AREA(info,ptr)) + info->literal_size += size_object(obj); goto pop_next; } hdr = *ptr; @@ -1163,14 +1151,14 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info, Uint32 flags) if (primary_tag(hdr) != TAG_PRIMARY_HEADER) { if (primary_tag(hdr) == BOXED_VISITED) { e = SHTABLE_NEXT(t); - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] tabling B %p\n", myself->common.id, ptr)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] tabling B %p\n", mypid, ptr)); SHTABLE_PUSH(t, hdr, THE_NON_VALUE, ptr); *ptr = (e << _TAG_PRIMARY_SIZE) | BOXED_SHARED_UNPROCESSED; } goto pop_next; } /* else make it visited now */ - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] mangling B %p\n", myself->common.id, ptr)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] mangling B %p\n", mypid, ptr)); *ptr = (hdr - primary_tag(hdr)) + BOXED_VISITED; /* and count it */ ASSERT(is_header(hdr)); @@ -1293,7 +1281,7 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info, Uint32 flags) info->shtable_start = t.start; info->shtable_alloc_type = t.alloc_type; /* single point of return: the size of the object */ - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] size was: %u\n", myself->common.id, sum)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] size was: %u\n", mypid, sum)); return sum + info->literal_size; } obj = EQUEUE_GET(s); @@ -1319,9 +1307,8 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, Eterm* resp; Eterm *hbot, *hend; unsigned remaining; - Process* myself; - int force_local = flags & ERTS_SHCOPY_FLG_TMPBUF; #ifdef DEBUG + Eterm mypid = erts_get_current_pid(); Eterm saved_obj = obj; #endif @@ -1343,11 +1330,10 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, flags |= disable_copy_shared; #endif - myself = erts_get_current_process(); - if (myself == NULL || (flags & ERTS_SHCOPY_FLG_NONE)) + if (flags & ERTS_SHCOPY_FLG_NONE) return copy_struct(obj, size, hpp, off_heap); - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy_shared_perform %p\n", myself->common.id, obj)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy_shared_perform %p\n", mypid, obj)); /* step #2: was performed before this function was called ------------------------------------------------------- @@ -1376,10 +1362,14 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, Eterm head, tail; ptr = list_val(obj); /* off heap list pointers are copied verbatim */ - if (!INHEAP(myself, ptr)) { - Uint bsz; - *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz); - hbot -= bsz; + if (erts_is_literal(obj,ptr)) { + if (!IN_LITERAL_PURGE_AREA(info,ptr)) { + *resp = obj; + } else { + Uint bsz = 0; + *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz); + hbot -= bsz; + } goto cleanup_next; } head = CAR(ptr); @@ -1399,23 +1389,23 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, head = SHTABLE_X(t, e); tail = SHTABLE_Y(t, e); ptr = &(SHTABLE_X(t, e)); - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] tabled L %p is %p\n", myself->common.id, ptr, SHTABLE_REV(t, e))); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] tabled L %p is %p\n", mypid, ptr, SHTABLE_REV(t, e))); SHTABLE_FWD_UPD(t, e, hp); } } /* if not already clean, clean it up and copy it */ if (primary_tag(tail) == TAG_PRIMARY_HEADER) { if (primary_tag(head) == TAG_PRIMARY_HEADER) { - Eterm saved = BITSTORE_GET(b); - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling L/B %p\n", myself->common.id, ptr)); - CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) + saved; - CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) + TAG_PRIMARY_BOXED; + Eterm saved = BITSTORE_GET(b); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling L/B %p\n", mypid, ptr)); + CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) + saved; + CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) + TAG_PRIMARY_BOXED; } else { - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling L/L %p\n", myself->common.id, ptr)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling L/L %p\n", mypid, ptr)); CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) + TAG_PRIMARY_LIST; } } else if (primary_tag(head) == TAG_PRIMARY_HEADER) { - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling L/I %p\n", myself->common.id, ptr)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling L/I %p\n", mypid, ptr)); CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) | primary_tag(tail); CDR(ptr) = tail = (tail - primary_tag(tail)) | TAG_PRIMARY_IMMED1; } else { @@ -1439,10 +1429,14 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, Eterm hdr; ptr = boxed_val(obj); /* off heap pointers to boxes are copied verbatim */ - if (!INHEAP(myself, ptr)) { - Uint bsz; - *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz); - hbot -= bsz; + if (erts_is_literal(obj,ptr)) { + if (!IN_LITERAL_PURGE_AREA(info,ptr)) { + *resp = obj; + } else { + Uint bsz = 0; + *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz); + hbot -= bsz; + } goto cleanup_next; } hdr = *ptr; @@ -1463,13 +1457,13 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, *ptr = (hdr - primary_tag(hdr)) + BOXED_SHARED_PROCESSED; hdr = SHTABLE_X(t, e); ASSERT(primary_tag(hdr) == BOXED_VISITED); - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] tabled B %p is %p\n", myself->common.id, ptr, SHTABLE_REV(t, e))); - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling B %p\n", myself->common.id, ptr)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] tabled B %p is %p\n", mypid, ptr, SHTABLE_REV(t, e))); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling B %p\n", mypid, ptr)); SHTABLE_X(t, e) = hdr = (hdr - BOXED_VISITED) + TAG_PRIMARY_HEADER; SHTABLE_FWD_UPD(t, e, hp); break; case BOXED_VISITED: - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling B %p\n", myself->common.id, ptr)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling B %p\n", mypid, ptr)); *ptr = hdr = (hdr - BOXED_VISITED) + TAG_PRIMARY_HEADER; break; } @@ -1749,13 +1743,13 @@ all_clean: VERBOSE(DEBUG_SHCOPY, ("[copy] restoring shared: %x\n", ptr)); /* entry was a list */ if (SHTABLE_Y(t, e) != THE_NON_VALUE) { - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] untabling L %p\n", myself->common.id, ptr)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] untabling L %p\n", mypid, ptr)); CAR(ptr) = SHTABLE_X(t, e); CDR(ptr) = SHTABLE_Y(t, e); } /* entry was boxed */ else { - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] untabling B %p\n", myself->common.id, ptr)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] untabling B %p\n", mypid, ptr)); *ptr = SHTABLE_X(t, e); ASSERT(primary_tag(*ptr) == TAG_PRIMARY_HEADER); } @@ -1769,9 +1763,9 @@ all_clean: } #endif - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] original was %T\n", myself->common.id, saved_obj)); - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy is %T\n", myself->common.id, result)); - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] result is at %p\n", myself->common.id, result)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] original was %T\n", mypid, saved_obj)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy is %T\n", mypid, result)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] result is at %p\n", mypid, result)); ASSERT(hbot == hp); ASSERT(size == ((hp - *hpp) + (hend - hbot))); diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index f106b941ef..71d072fa7e 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -982,6 +982,12 @@ Eterm erl_is_function(Process* p, Eterm arg1, Eterm arg2); /* beam_bif_load.c */ Eterm erts_check_process_code(Process *c_p, Eterm module, int allow_gc, int *redsp); +typedef struct { + Eterm *ptr; + Uint sz; +} copy_literals_t; + +extern copy_literals_t erts_clrange; /* beam_load.c */ typedef struct { @@ -1063,6 +1069,8 @@ typedef struct { Eterm* shtable_start; ErtsAlcType_t shtable_alloc_type; Uint literal_size; + Eterm *range_ptr; + Uint range_sz; } erts_shcopy_t; #define INITIALIZE_SHCOPY(info) \ @@ -1071,6 +1079,8 @@ do { \ info.bitstore_start = info.bitstore_default; \ info.shtable_start = info.shtable_default; \ info.literal_size = 0; \ + info.range_ptr = erts_clrange.ptr; \ + info.range_sz = erts_clrange.sz; \ } while(0) #define DESTROY_SHCOPY(info) \ -- cgit v1.2.3 From 0d5ee7a4ad7bc41b7cca878990926fb5ba57f6a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Thu, 1 Oct 2015 12:19:50 +0200 Subject: Do not use GCC extensions in copy --- erts/emulator/beam/copy.c | 37 ++++++++++++++++++++----------------- erts/emulator/beam/global.h | 25 +++++++++++++++---------- 2 files changed, 35 insertions(+), 27 deletions(-) (limited to 'erts') diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index 5bca3877b5..83ca527334 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -258,18 +258,19 @@ do { \ } \ } while(0) -#define BITSTORE_GET(s) ({ \ - UWord result; \ - if (WSTK_CONCAT(s,_bitoffs) <= 0) { \ - WSTK_CONCAT(s,_buffer) = s.wstart[WSTK_CONCAT(s,_offset)]; \ - WSTK_CONCAT(s,_offset)++; \ - WSTK_CONCAT(s,_bitoffs) = 8*sizeof(UWord); \ - } \ - WSTK_CONCAT(s,_bitoffs) -= 2; \ - result = WSTK_CONCAT(s,_buffer) & 3; \ - WSTK_CONCAT(s,_buffer) >>= 2; \ - result; \ -}) +#define BITSTORE_FETCH(s,dst) \ +do { \ + UWord result; \ + if (WSTK_CONCAT(s,_bitoffs) <= 0) { \ + WSTK_CONCAT(s,_buffer) = s.wstart[WSTK_CONCAT(s,_offset)]; \ + WSTK_CONCAT(s,_offset)++; \ + WSTK_CONCAT(s,_bitoffs) = 8*sizeof(UWord); \ + } \ + WSTK_CONCAT(s,_bitoffs) -= 2; \ + result = WSTK_CONCAT(s,_buffer) & 3; \ + WSTK_CONCAT(s,_buffer) >>= 2; \ + (dst) = result; \ +} while(0) #define BOXED_VISITED_MASK ((Eterm) 3) #define BOXED_VISITED ((Eterm) 1) @@ -476,7 +477,8 @@ cleanup: /* if not already clean, clean it up */ if (primary_tag(tail) == TAG_PRIMARY_HEADER) { if (primary_tag(head) == TAG_PRIMARY_HEADER) { - Eterm saved = BITSTORE_GET(b); + Eterm saved; + BITSTORE_FETCH(b, saved); CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) | saved; CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) | TAG_PRIMARY_BOXED; } else { @@ -1396,10 +1398,11 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, /* if not already clean, clean it up and copy it */ if (primary_tag(tail) == TAG_PRIMARY_HEADER) { if (primary_tag(head) == TAG_PRIMARY_HEADER) { - Eterm saved = BITSTORE_GET(b); - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling L/B %p\n", mypid, ptr)); - CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) + saved; - CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) + TAG_PRIMARY_BOXED; + Eterm saved; + BITSTORE_FETCH(b, saved); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling L/B %p\n", mypid, ptr)); + CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) + saved; + CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) + TAG_PRIMARY_BOXED; } else { VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling L/L %p\n", mypid, ptr)); CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) + TAG_PRIMARY_LIST; diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 71d072fa7e..e9f7901a51 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -873,7 +873,7 @@ typedef struct { int possibly_empty; Eterm* end; ErtsAlcType_t alloc_type; -}ErtsEQueue; +} ErtsEQueue; #define DEF_EQUEUE_SIZE (16) @@ -918,15 +918,20 @@ do { \ #define EQUEUE_ISEMPTY(q) (q.back == q.front && q.possibly_empty) -#define EQUEUE_GET(q) ({ \ - UWord x; \ - q.possibly_empty = 1; \ - x = *(q.front); \ - if (++(q.front) == q.end) { \ - q.front = q.start; \ - } \ - x; \ -}) +ERTS_GLB_INLINE Eterm erts_equeue_get(ErtsEQueue *q); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE Eterm erts_equeue_get(ErtsEQueue *q) { + Eterm x; + q->possibly_empty = 1; + x = *(q->front); + if (++(q->front) == q->end) { + q->front = q->start; + } + return x; +} +#endif +#define EQUEUE_GET(q) erts_equeue_get(&(q)); /* binary.c */ -- cgit v1.2.3 From 7f215c6db8caf0760161437e9d1c3c0a06cf5841 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Fri, 25 Sep 2015 18:20:10 +0200 Subject: Add copy_literals testcase Repeatedly reload a literals module while sending the references literals around in a process ring. This will smoke test the non-copying literals message sending does not corrupt code unloading. --- erts/emulator/test/code_SUITE.erl | 82 ++++++++++++++++++++++++- erts/emulator/test/code_SUITE_data/literals.erl | 7 +++ 2 files changed, 87 insertions(+), 2 deletions(-) (limited to 'erts') diff --git a/erts/emulator/test/code_SUITE.erl b/erts/emulator/test/code_SUITE.erl index 9f318a38be..1acc4538fb 100644 --- a/erts/emulator/test/code_SUITE.erl +++ b/erts/emulator/test/code_SUITE.erl @@ -26,7 +26,8 @@ t_check_process_code_ets/1, external_fun/1,get_chunk/1,module_md5/1,make_stub/1, make_stub_many_funs/1,constant_pools/1,constant_refc_binaries/1, - false_dependency/1,coverage/1,fun_confusion/1]). + false_dependency/1,coverage/1,fun_confusion/1, + t_copy_literals/1]). -define(line_trace, 1). -include_lib("test_server/include/test_server.hrl"). @@ -38,7 +39,7 @@ all() -> t_check_process_code_ets, t_check_old_code, external_fun, get_chunk, module_md5, make_stub, make_stub_many_funs, constant_pools, constant_refc_binaries, false_dependency, - coverage, fun_confusion]. + coverage, fun_confusion, t_copy_literals]. groups() -> []. @@ -753,6 +754,80 @@ compile_load(Mod, Src, Ver) -> {module,Mod} = code:load_binary(Mod, "fun_confusion.beam", Code1), ok. + +t_copy_literals(Config) when is_list(Config) -> + %% Compile the the literals module. + Data = ?config(data_dir, Config), + File = filename:join(Data, "literals"), + {ok,literals,Code} = compile:file(File, [report,binary]), + {module,literals} = erlang:load_module(literals, Code), + + N = 30, + Me = self(), + %% reload literals code every 567 ms + Rel = spawn_link(fun() -> reloader(literals,Code,567) end), + %% add new literal msgs to the loop every 789 ms + Sat = spawn_link(fun() -> saturate(Me,789) end), + %% run for 10s + _ = spawn_link(fun() -> receive after 10000 -> Me ! done end end), + ok = chase_msg(N, Me), + %% cleanup + Rel ! done, + Sat ! done, + ok = flush(), + ok. + + +chase_msg(0, Pid) -> + chase_loop(Pid); +chase_msg(N, Master) -> + Pid = spawn_link(fun() -> chase_msg(N - 1,Master) end), + chase_loop(Pid). + +chase_loop(Pid) -> + receive + done -> + Pid ! done, + ok; + {_From,Msg} -> + Pid ! {self(), Msg}, + ok = traverse(Msg), + chase_loop(Pid) + end. + +saturate(Pid,Time) -> + Es = [msg1,msg2,msg3,msg4,msg5], + Msg = [literals:E()||E <- Es], + Pid ! {self(), Msg}, + receive + done -> ok + after Time -> + saturate(Pid,Time) + end. + +traverse([]) -> ok; +traverse([H|T]) -> + ok = traverse(H), + traverse(T); +traverse(T) when is_tuple(T) -> ok; +traverse(B) when is_binary(B) -> ok; +traverse(I) when is_integer(I) -> ok; +traverse(#{ 1 := V1, b := V2 }) -> + ok = traverse(V1), + ok = traverse(V2), + ok. + + +reloader(Mod,Code,Time) -> + receive + done -> ok + after Time -> + code:purge(Mod), + {module,Mod} = erlang:load_module(Mod, Code), + reloader(Mod,Code,Time) + end. + + %% Utilities. make_sub_binary(Bin) when is_binary(Bin) -> @@ -775,4 +850,7 @@ bit_sized_binary(Bin0) -> BitSize = 8*size(Bin) + 1, Bin. +flush() -> + receive _ -> flush() after 0 -> ok end. + id(I) -> I. diff --git a/erts/emulator/test/code_SUITE_data/literals.erl b/erts/emulator/test/code_SUITE_data/literals.erl index 9802d9d3f9..a36bfe09dd 100644 --- a/erts/emulator/test/code_SUITE_data/literals.erl +++ b/erts/emulator/test/code_SUITE_data/literals.erl @@ -20,6 +20,7 @@ -module(literals). -export([a/0,b/0,huge_bignum/0,binary/0,unused_binaries/0,bits/0]). +-export([msg1/0,msg2/0,msg3/0,msg4/0,msg5/0]). a() -> {a,42.0,[7,38877938333399637266518333334747]}. @@ -101,3 +102,9 @@ unused_binaries() -> bits() -> {bits,<<42:13,?MB_1>>}. + +msg1() -> "halloj". +msg2() -> {"hello","world"}. +msg3() -> <<"halloj">>. +msg4() -> #{ 1=> "hello", b => "world"}. +msg5() -> {1,2,3,4,5,6}. -- cgit v1.2.3 From 85491375da14ab087ec99532ee3a896ff6fe0371 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Tue, 29 Sep 2015 15:33:18 +0200 Subject: Add erlang:copy_literals/2 spec --- erts/preloaded/src/erlang.erl | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'erts') diff --git a/erts/preloaded/src/erlang.erl b/erts/preloaded/src/erlang.erl index 6a9ec9c915..0d5176019f 100644 --- a/erts/preloaded/src/erlang.erl +++ b/erts/preloaded/src/erlang.erl @@ -91,7 +91,7 @@ -export([bit_size/1, bitsize/1, bitstring_to_list/1]). -export([bump_reductions/1, byte_size/1, call_on_load_function/1]). -export([cancel_timer/1, cancel_timer/2, check_old_code/1, check_process_code/2, - check_process_code/3, crc32/1]). + check_process_code/3, copy_literals/2, crc32/1]). -export([crc32/2, crc32_combine/3, date/0, decode_packet/3]). -export([delete_element/2]). -export([delete_module/1, demonitor/1, demonitor/2, display/1]). @@ -520,6 +520,13 @@ get_cpc_opts([{allow_gc, AllowGC} | Options], Async, _OldAllowGC) -> get_cpc_opts([], Async, AllowGC) -> {Async, AllowGC}. +%% copy_literals/2 +-spec erlang:copy_literals(Module,Bool) -> 'true' | 'false' | 'aborted' when + Module :: module(), + Bool :: boolean(). +copy_literals(_Mod, _Bool) -> + erlang:nif_error(undefined). + %% crc32/1 -spec erlang:crc32(Data) -> non_neg_integer() when Data :: iodata(). -- cgit v1.2.3 From da8220501710184f3ee80337bac57d215560ea64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Tue, 29 Sep 2015 15:35:54 +0200 Subject: Update preloaded module erlang.beam --- erts/preloaded/ebin/erlang.beam | Bin 101544 -> 101816 bytes 1 file changed, 0 insertions(+), 0 deletions(-) (limited to 'erts') diff --git a/erts/preloaded/ebin/erlang.beam b/erts/preloaded/ebin/erlang.beam index 641fac2d26..4f35928db2 100644 Binary files a/erts/preloaded/ebin/erlang.beam and b/erts/preloaded/ebin/erlang.beam differ -- cgit v1.2.3 From 15c26af0a35fb482408885ea90b3e669d64d71f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Tue, 17 Nov 2015 16:26:35 +0100 Subject: Fix erts_debug:copy_shared/1 prototype --- erts/emulator/beam/bif.tab | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'erts') diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 63a0d0b1f0..c49a3ff313 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -646,7 +646,7 @@ bif erlang:copy_literals/2 bif binary:split/2 bif binary:split/3 bif erts_debug:size_shared/1 -bif 'erl.system.debug':size_shared/1 ebif_erts_debug_size_shared_1 +bif erts_debug:copy_shared/1 # # Obsolete -- cgit v1.2.3 From 3f29cf1f72f50d20ce1864f76e1a298602429ca4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Wed, 18 Nov 2015 16:59:08 +0100 Subject: Fix rebase of SHCOPY seq_tokens --- erts/emulator/beam/erl_message.c | 55 ++++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 27 deletions(-) (limited to 'erts') diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index 11890a756d..66b337402d 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -691,22 +691,15 @@ erts_send_message(Process* sender, #ifdef USE_VM_PROBES Uint dt_utag_size = 0; #endif -#ifdef SHCOPY_SEND - unsigned shflags = (flags & ERTS_SND_FLG_SHCOPY_MASK) >> ERTS_SND_FLG_SHCOPY_SHIFT; - erts_shcopy_t info; - INITIALIZE_SHCOPY(info); -#endif - BM_SWAP_TIMER(send,size); -#ifdef SHCOPY_SEND - INITIALIZE_SHCOPY(info); - msize = copy_shared_calculate(message, &info, shflags); -#else - msize = size_object(message); -#endif - BM_SWAP_TIMER(size,send); + BM_SWAP_TIMER(send,size); + /* SHCOPY corrupts the heap between + * copy_shared_calculate, and + * copy_shared_perform. (it inserts move_markers like the gc). + * Make sure we don't use the heap between those instances. + */ #ifdef USE_VM_PROBES - if (stoken != am_have_dt_utag) { + if (stoken != am_have_dt_utag) { #endif seq_trace_update_send(sender); seq_trace_output(stoken, message, SEQ_TRACE_SEND, @@ -714,23 +707,31 @@ erts_send_message(Process* sender, seq_trace_size = 6; /* TUPLE5 */ #ifdef USE_VM_PROBES } - if (DT_UTAG_FLAGS(sender) & DT_UTAG_SPREADING) { - dt_utag_size = size_object(DT_UTAG(sender)); - } else if (stoken == am_have_dt_utag ) { - stoken = NIL; - } + if (DT_UTAG_FLAGS(sender) & DT_UTAG_SPREADING) { + dt_utag_size = size_object(DT_UTAG(sender)); + } else if (stoken == am_have_dt_utag ) { + stoken = NIL; + } +#endif + +#ifdef SHCOPY_SEND + INITIALIZE_SHCOPY(info); + msize = copy_shared_calculate(message, &info, shflags); +#else + msize = size_object(message); #endif + BM_SWAP_TIMER(size,send); - mp = erts_alloc_message_heap_state(receiver, - &receiver_state, - receiver_locks, - (msize + mp = erts_alloc_message_heap_state(receiver, + &receiver_state, + receiver_locks, + (msize #ifdef USE_VM_PROBES - + dt_utag_size + + dt_utag_size #endif - + seq_trace_size), - &hp, - &ohp); + + seq_trace_size), + &hp, + &ohp); BM_SWAP_TIMER(send,copy); -- cgit v1.2.3 From 5d764f988ab09326d24e39a172083b09ab364c6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Wed, 18 Nov 2015 17:58:14 +0100 Subject: Refactor sharing preserved copy flags The TMPBUF option is no longer needed due to is_literal test and NONE was only used for initial debugging. So we remove the entire option. --- erts/emulator/beam/beam_debug.c | 10 ++-------- erts/emulator/beam/copy.c | 22 ++-------------------- erts/emulator/beam/erl_db.c | 4 ++-- erts/emulator/beam/erl_message.c | 9 ++++----- erts/emulator/beam/erl_message.h | 5 ----- erts/emulator/beam/erl_process.c | 5 ++--- erts/emulator/beam/global.h | 10 ++-------- 7 files changed, 14 insertions(+), 51 deletions(-) (limited to 'erts') diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index b007d000bf..e37bd4d78c 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -97,20 +97,14 @@ erts_debug_copy_shared_1(BIF_ALIST_1) Eterm* hp; Eterm copy; erts_shcopy_t info; -#ifdef SHCOPY_DISABLE - extern int disable_copy_shared; -#endif INITIALIZE_SHCOPY(info); - size = copy_shared_calculate(term, &info, 0); + size = copy_shared_calculate(term, &info); if (size > 0) { hp = HAlloc(p, size); } - copy = copy_shared_perform(term, size, &info, &hp, &p->off_heap, 0); + copy = copy_shared_perform(term, size, &info, &hp, &p->off_heap); DESTROY_SHCOPY(info); -#ifdef SHCOPY_DISABLE - disable_copy_shared = 0; -#endif BIF_RET(copy); } diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index 83ca527334..67a96f6442 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -923,10 +923,6 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint * Using an ESTACK but not very transparently; consider refactoring */ -#ifdef SHCOPY_DISABLE -int disable_copy_shared = ERTS_SHCOPY_FLG_NONE; -#endif - #define DECLARE_SHTABLE(s) \ DECLARE_ESTACK(s); \ Uint ESTK_CONCAT(s,_offset) = 0 @@ -1034,7 +1030,7 @@ do { \ * Copy object "obj" preserving sharing. * First half: count size and calculate sharing. */ -Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info, Uint32 flags) +Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info) { Uint sum; Uint e; @@ -1058,13 +1054,6 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info, Uint32 flags) if (IS_CONST(obj)) return 0; -#ifdef SHCOPY_DISABLE - flags |= disable_copy_shared; -#endif - - if (flags & ERTS_SHCOPY_FLG_NONE) - return size_object(obj); - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy_shared_calculate %p\n", mypid, obj)); VERBOSE(DEBUG_SHCOPY, ("[pid=%T] message is %T\n", mypid, obj)); @@ -1299,7 +1288,7 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info, Uint32 flags) * Second half: copy and restore the object. */ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, - Eterm** hpp, ErlOffHeap* off_heap, Uint32 flags) { + Eterm** hpp, ErlOffHeap* off_heap) { Uint e; unsigned sz; Eterm* ptr; @@ -1328,13 +1317,6 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, if (IS_CONST(obj)) return obj; -#ifdef SHCOPY_DISABLE - flags |= disable_copy_shared; -#endif - - if (flags & ERTS_SHCOPY_FLG_NONE) - return copy_struct(obj, size, hpp, off_heap); - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy_shared_perform %p\n", mypid, obj)); /* step #2: was performed before this function was called diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 59a0c0b808..3030c1c91a 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -1832,7 +1832,7 @@ BIF_RETTYPE ets_give_away_3(BIF_ALIST_3) tb->common.id, from_pid, BIF_ARG_3), - ERTS_SND_FLG_SHCOPY_TMPBUF); + 0); erts_smp_proc_unlock(to_proc, to_locks); UnUseTmpHeap(5,BIF_P); BIF_RET(am_true); @@ -3211,7 +3211,7 @@ retry: tb->common.id, p->common.id, heir_data), - ERTS_SND_FLG_SHCOPY_TMPBUF); + 0); erts_smp_proc_unlock(to_proc, to_locks); return !0; } diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index 66b337402d..a964f1968b 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -665,7 +665,6 @@ erts_send_message(Process* sender, #endif erts_aint32_t receiver_state; #ifdef SHCOPY_SEND - unsigned shflags = (flags & ERTS_SND_FLG_SHCOPY_MASK) >> ERTS_SND_FLG_SHCOPY_SHIFT; erts_shcopy_t info; #endif BM_STOP_TIMER(system); @@ -716,7 +715,7 @@ erts_send_message(Process* sender, #ifdef SHCOPY_SEND INITIALIZE_SHCOPY(info); - msize = copy_shared_calculate(message, &info, shflags); + msize = copy_shared_calculate(message, &info); #else msize = size_object(message); #endif @@ -737,7 +736,7 @@ erts_send_message(Process* sender, #ifdef SHCOPY_SEND if (is_not_immed(message)) - message = copy_shared_perform(message, msize, &info, &hp, ohp, shflags); + message = copy_shared_perform(message, msize, &info, &hp, ohp); DESTROY_SHCOPY(info); #else if (is_not_immed(message)) @@ -786,7 +785,7 @@ erts_send_message(Process* sender, BM_SWAP_TIMER(send,size); #ifdef SHCOPY_SEND INITIALIZE_SHCOPY(info); - msize = copy_shared_calculate(message, &info, shflags); + msize = copy_shared_calculate(message, &info); #else msize = size_object(message); #endif @@ -801,7 +800,7 @@ erts_send_message(Process* sender, BM_SWAP_TIMER(send,copy); #ifdef SHCOPY_SEND if (is_not_immed(message)) - message = copy_shared_perform(message, msize, &info, &hp, ohp, shflags); + message = copy_shared_perform(message, msize, &info, &hp, ohp); DESTROY_SHCOPY(info); #else if (is_not_immed(message)) diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index 52a648fe0b..740ae46a0f 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -238,11 +238,6 @@ do { \ #define ERTS_SND_FLG_NO_SEQ_TRACE (((unsigned) 1) << 0) -#define ERTS_SND_FLG_SHCOPY_SHIFT 1 -#define ERTS_SND_FLG_SHCOPY_MASK (ERTS_SHCOPY_FLG_MASK << ERTS_SND_FLG_SHCOPY_SHIFT) -#define ERTS_SND_FLG_SHCOPY_NONE (ERTS_SHCOPY_FLG_NONE << ERTS_SND_FLG_SHCOPY_SHIFT) -#define ERTS_SND_FLG_SHCOPY_TMPBUF (ERTS_SHCOPY_FLG_TMPBUF << ERTS_SND_FLG_SHCOPY_SHIFT) - #define ERTS_HEAP_FRAG_SIZE(DATA_WORDS) \ (sizeof(ErlHeapFragment) - sizeof(Eterm) + (DATA_WORDS)*sizeof(Eterm)) diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index a691a3c773..23616f36bf 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -10748,7 +10748,6 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). erts_aint32_t state = 0; erts_aint32_t prio = (erts_aint32_t) PRIORITY_NORMAL; #ifdef SHCOPY_SPAWN - unsigned shflags = 0; /* could be taken from so->flags, if necessary */ erts_shcopy_t info; INITIALIZE_SHCOPY(info); #endif @@ -10804,7 +10803,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). BM_SWAP_TIMER(system,size); #ifdef SHCOPY_SPAWN - arg_size = copy_shared_calculate(args, &info, shflags); + arg_size = copy_shared_calculate(args, &info); #else arg_size = size_object(args); #endif @@ -10886,7 +10885,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). BM_START_TIMER(system); BM_SWAP_TIMER(system,copy); #ifdef SHCOPY_SPAWN - p->arg_reg[2] = copy_shared_perform(args, arg_size, &info, &p->htop, &p->off_heap, shflags); + p->arg_reg[2] = copy_shared_perform(args, arg_size, &info, &p->htop, &p->off_heap); DESTROY_SHCOPY(info); #else p->arg_reg[2] = copy_struct(args, arg_size, &p->htop, &p->off_heap); diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index e9f7901a51..98c275a20c 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1052,14 +1052,8 @@ void erl_error(char*, va_list); #ifdef SHCOPY #define SHCOPY_SEND #define SHCOPY_SPAWN -/* Use this if you want sharing-preserving copy to be initially disabled */ -#undef SHCOPY_DISABLE #endif -#define ERTS_SHCOPY_FLG_MASK (((Uint32) 3) << 0) -#define ERTS_SHCOPY_FLG_NONE (((Uint32) 1) << 0) -#define ERTS_SHCOPY_FLG_TMPBUF (((Uint32) 1) << 1) /* forces INHEAP to true */ - /* The persistent state while the sharing-preserving copier works */ typedef struct { @@ -1106,8 +1100,8 @@ Eterm copy_object_x(Eterm, Process*, Uint); #define copy_object(Term, Proc) copy_object_x(Term,Proc,0) Uint size_object(Eterm); -Uint copy_shared_calculate(Eterm, erts_shcopy_t*, Uint32); -Eterm copy_shared_perform(Eterm, Uint, erts_shcopy_t*, Eterm**, ErlOffHeap*, Uint32); +Uint copy_shared_calculate(Eterm, erts_shcopy_t*); +Eterm copy_shared_perform(Eterm, Uint, erts_shcopy_t*, Eterm**, ErlOffHeap*); Uint size_shared(Eterm); -- cgit v1.2.3 From 8b93e77a6d3df65e45a3ca3e2ec9dd4c52464f63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Wed, 18 Nov 2015 18:47:55 +0100 Subject: Use sharing preserving copy in enif_make_copy --- erts/emulator/beam/erl_nif.c | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'erts') diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index a37cda93ef..2ff509e6d0 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -379,9 +379,19 @@ ERL_NIF_TERM enif_make_copy(ErlNifEnv* dst_env, ERL_NIF_TERM src_term) { Uint sz; Eterm* hp; +#ifdef SHCOPY + erts_shcopy_t info; + INITIALIZE_SHCOPY(info); + sz = copy_shared_calculate(src_term, &info); + hp = alloc_heap(dst_env, sz); + src_term = copy_shared_perform(src_term, sz, &info, &hp, &MSO(dst_env->proc)); + DESTROY_SHCOPY(info); + return src_term; +#else sz = size_object(src_term); hp = alloc_heap(dst_env, sz); return copy_struct(src_term, sz, &hp, &MSO(dst_env->proc)); +#endif } -- cgit v1.2.3 From 02f038355d16e9b6474837727878f58e4ca669c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Wed, 18 Nov 2015 20:12:38 +0100 Subject: Use sharing preserving copy error messages and exceptions --- erts/emulator/beam/erl_message.c | 28 +++++++++++++++++++++++++--- erts/emulator/beam/erl_process.c | 33 +++++++++++++++++++++++---------- 2 files changed, 48 insertions(+), 13 deletions(-) (limited to 'erts') diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index a964f1968b..f64385d92b 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -850,6 +850,9 @@ erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp, Eterm temptoken; ErtsMessage* mp; ErlOffHeap *ohp; +#ifdef SHCOPY_SEND + erts_shcopy_t info; +#endif if (token != NIL #ifdef USE_VM_PROBES @@ -858,13 +861,23 @@ erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp, ) { ASSERT(is_tuple(token)); - sz_reason = size_object(reason); sz_token = size_object(token); sz_from = size_object(from); +#ifdef SHCOPY_SEND + INITIALIZE_SHCOPY(info); + sz_reason = copy_shared_calculate(reason, &info); +#else + sz_reason = size_object(reason); +#endif mp = erts_alloc_message_heap(to, to_locksp, sz_reason + sz_from + sz_token + 4, &hp, &ohp); +#ifdef SHCOPY_SEND + mess = copy_shared_perform(reason, sz_reason, &info, &hp, ohp); + DESTROY_SHCOPY(info); +#else mess = copy_struct(reason, sz_reason, &hp, ohp); +#endif from_copy = copy_struct(from, sz_from, &hp, ohp); save = TUPLE3(hp, am_EXIT, from_copy, mess); hp += 4; @@ -873,13 +886,22 @@ erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp, temptoken = copy_struct(token, sz_token, &hp, ohp); erts_queue_message(to, to_locksp, mp, save, temptoken); } else { - sz_reason = size_object(reason); sz_from = IS_CONST(from) ? 0 : size_object(from); - +#ifdef SHCOPY_SEND + INITIALIZE_SHCOPY(info); + sz_reason = copy_shared_calculate(reason, &info); +#else + sz_reason = size_object(reason); +#endif mp = erts_alloc_message_heap(to, to_locksp, sz_reason+sz_from+4, &hp, &ohp); +#ifdef SHCOPY_SEND + mess = copy_shared_perform(reason, sz_reason, &info, &hp, ohp); + DESTROY_SHCOPY(info); +#else mess = copy_struct(reason, sz_reason, &hp, ohp); +#endif from_copy = (IS_CONST(from) ? from : copy_struct(from, sz_from, &hp, ohp)); diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 23616f36bf..f347d57a9d 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -11520,31 +11520,44 @@ send_exit_message(Process *to, ErtsProcLocks *to_locksp, { ErtsMessage *mp; ErlOffHeap *ohp; + Eterm* hp; + Eterm mess; +#ifdef SHCOPY_SEND + erts_shcopy_t info; +#endif if (token == NIL #ifdef USE_VM_PROBES || token == am_have_dt_utag #endif ) { - Eterm* hp; - Eterm mess; - - mp = erts_alloc_message_heap(to, to_locksp, - term_size, &hp, &ohp); +#ifdef SHCOPY_SEND + INITIALIZE_SHCOPY(info); + term_size = copy_shared_calculate(exit_term, &info); + mp = erts_alloc_message_heap(to, to_locksp, term_size, &hp, &ohp); + mess = copy_shared_perform(exit_term, term_size, &info, &hp, ohp); + DESTROY_SHCOPY(info); +#else + mp = erts_alloc_message_heap(to, to_locksp, term_size, &hp, &ohp); mess = copy_struct(exit_term, term_size, &hp, ohp); +#endif erts_queue_message(to, to_locksp, mp, mess, NIL); } else { - Eterm* hp; - Eterm mess; Eterm temp_token; Uint sz_token; ASSERT(is_tuple(token)); sz_token = size_object(token); - - mp = erts_alloc_message_heap(to, to_locksp, - term_size+sz_token, &hp, &ohp); +#ifdef SHCOPY_SEND + INITIALIZE_SHCOPY(info); + term_size = copy_shared_calculate(exit_term, &info); + mp = erts_alloc_message_heap(to, to_locksp, term_size, &hp, &ohp); + mess = copy_shared_perform(exit_term, term_size, &info, &hp, ohp); + DESTROY_SHCOPY(info); +#else + mp = erts_alloc_message_heap(to, to_locksp, term_size+sz_token, &hp, &ohp); mess = copy_struct(exit_term, term_size, &hp, ohp); +#endif /* the trace token must in this case be updated by the caller */ seq_trace_output(token, mess, SEQ_TRACE_SEND, to->common.id, NULL); temp_token = copy_struct(token, sz_token, &hp, ohp); -- cgit v1.2.3 From 6ff15f23c68db356bbad6ab5f939c191b58d453d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Thu, 19 Nov 2015 19:36:08 +0100 Subject: Refactor have seq_trace token test --- erts/emulator/beam/beam_emu.c | 2 +- erts/emulator/beam/bif.c | 6 +----- erts/emulator/beam/dist.c | 24 ++++++------------------ erts/emulator/beam/erl_bif_trace.c | 26 +++++--------------------- erts/emulator/beam/erl_db_util.c | 12 ++---------- erts/emulator/beam/erl_message.c | 19 ++++++------------- erts/emulator/beam/erl_message.h | 7 +++++++ erts/emulator/beam/erl_process.c | 12 ++---------- erts/emulator/beam/erl_trace.c | 6 +----- 9 files changed, 31 insertions(+), 83 deletions(-) (limited to 'erts') diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index af97cba61c..1a4133bceb 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -1963,7 +1963,7 @@ void process_main(void) dtrace_proc_str(c_p, receiver_name); token2 = SEQ_TRACE_TOKEN(c_p); - if (token2 != NIL && token2 != am_have_dt_utag) { + if (have_seqtrace(token2)) { tok_label = signed_val(SEQ_TRACE_T_LABEL(token2)); tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token2)); tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token2)); diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index f0340540cb..14ab113b32 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -2038,11 +2038,7 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext* ctx) if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) save_calls(p, &exp_send); - if (SEQ_TRACE_TOKEN(p) != NIL -#ifdef USE_VM_PROBES - && SEQ_TRACE_TOKEN(p) != am_have_dt_utag -#endif - ) { + if (have_seqtrace(SEQ_TRACE_TOKEN(p))) { seq_trace_update_send(p); seq_trace_output(SEQ_TRACE_TOKEN(p), msg, SEQ_TRACE_SEND, portid, p); diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index 68745fc448..f480915256 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -881,11 +881,7 @@ erts_dsig_send_msg(Eterm remote, Eterm message, ErtsSendContext* ctx) DTRACE_CHARBUF(receiver_name, 64); #endif - if (SEQ_TRACE_TOKEN(sender) != NIL -#ifdef USE_VM_PROBES - && SEQ_TRACE_TOKEN(sender) != am_have_dt_utag -#endif - ) { + if (have_seqtrace(SEQ_TRACE_TOKEN(sender))) { seq_trace_update_send(sender); token = SEQ_TRACE_TOKEN(sender); seq_trace_output(token, message, SEQ_TRACE_SEND, remote, sender); @@ -900,7 +896,7 @@ erts_dsig_send_msg(Eterm remote, Eterm message, ErtsSendContext* ctx) erts_snprintf(receiver_name, sizeof(DTRACE_CHARBUF_NAME(receiver_name)), "%T", remote); msize = size_object(message); - if (token != NIL && token != am_have_dt_utag) { + if (have_seqtrace(token)) { tok_label = signed_val(SEQ_TRACE_T_LABEL(token)); tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token)); tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token)); @@ -942,11 +938,7 @@ erts_dsig_send_reg_msg(Eterm remote_name, Eterm message, DTRACE_CHARBUF(receiver_name, 128); #endif - if (SEQ_TRACE_TOKEN(sender) != NIL -#ifdef USE_VM_PROBES - && SEQ_TRACE_TOKEN(sender) != am_have_dt_utag -#endif - ) { + if (have_seqtrace(SEQ_TRACE_TOKEN(sender))) { seq_trace_update_send(sender); token = SEQ_TRACE_TOKEN(sender); seq_trace_output(token, message, SEQ_TRACE_SEND, remote_name, sender); @@ -961,7 +953,7 @@ erts_dsig_send_reg_msg(Eterm remote_name, Eterm message, erts_snprintf(receiver_name, sizeof(DTRACE_CHARBUF_NAME(receiver_name)), "{%T,%s}", remote_name, node_name); msize = size_object(message); - if (token != NIL && token != am_have_dt_utag) { + if (have_seqtrace(token)) { tok_label = signed_val(SEQ_TRACE_T_LABEL(token)); tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token)); tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token)); @@ -1006,11 +998,7 @@ erts_dsig_send_exit_tt(ErtsDSigData *dsdp, Eterm local, Eterm remote, #endif UseTmpHeapNoproc(6); - if (token != NIL -#ifdef USE_VM_PROBES - && token != am_have_dt_utag -#endif - ) { + if (have_seqtrace(token)) { seq_trace_update_send(dsdp->proc); seq_trace_output_exit(token, reason, SEQ_TRACE_SEND, remote, local); ctl = TUPLE5(&ctl_heap[0], @@ -1029,7 +1017,7 @@ erts_dsig_send_exit_tt(ErtsDSigData *dsdp, Eterm local, Eterm remote, "{%T,%s}", remote, node_name); erts_snprintf(reason_str, sizeof(DTRACE_CHARBUF_NAME(reason_str)), "%T", reason); - if (token != NIL && token != am_have_dt_utag) { + if (have_seqtrace(token)) { tok_label = signed_val(SEQ_TRACE_T_LABEL(token)); tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token)); tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token)); diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c index 03f51132b1..4d67e39e7e 100644 --- a/erts/emulator/beam/erl_bif_trace.c +++ b/erts/emulator/beam/erl_bif_trace.c @@ -1879,11 +1879,7 @@ new_seq_trace_token(Process* p) { Eterm* hp; - if (SEQ_TRACE_TOKEN(p) == NIL -#ifdef USE_VM_PROBES - || SEQ_TRACE_TOKEN(p) == am_have_dt_utag -#endif - ) { + if (have_no_seqtrace(SEQ_TRACE_TOKEN(p))) { hp = HAlloc(p, 6); SEQ_TRACE_TOKEN(p) = TUPLE5(hp, make_small(0), /* Flags */ make_small(0), /* Label */ @@ -1903,12 +1899,8 @@ BIF_RETTYPE erl_seq_trace_info(Process *p, Eterm item) BIF_ERROR(p, BADARG); } - if (SEQ_TRACE_TOKEN(p) == NIL -#ifdef USE_VM_PROBES - || SEQ_TRACE_TOKEN(p) == am_have_dt_utag -#endif - ) { - if ((item == am_send) || (item == am_receive) || + if (have_no_seqtrace(SEQ_TRACE_TOKEN(p))) { + if ((item == am_send) || (item == am_receive) || (item == am_print) || (item == am_timestamp)) { hp = HAlloc(p,3); res = TUPLE2(hp, item, am_false); @@ -1964,11 +1956,7 @@ BIF_RETTYPE seq_trace_info_1(BIF_ALIST_1) */ BIF_RETTYPE seq_trace_print_1(BIF_ALIST_1) { - if (SEQ_TRACE_TOKEN(BIF_P) == NIL -#ifdef USE_VM_PROBES - || SEQ_TRACE_TOKEN(BIF_P) == am_have_dt_utag -#endif - ) { + if (have_no_seqtrace(SEQ_TRACE_TOKEN(BIF_P))) { BIF_RET(am_false); } seq_trace_update_send(BIF_P); @@ -1987,11 +1975,7 @@ BIF_RETTYPE seq_trace_print_1(BIF_ALIST_1) */ BIF_RETTYPE seq_trace_print_2(BIF_ALIST_2) { - if (SEQ_TRACE_TOKEN(BIF_P) == NIL -#ifdef USE_VM_PROBES - || SEQ_TRACE_TOKEN(BIF_P) == am_have_dt_utag -#endif - ) { + if (have_no_seqtrace(SEQ_TRACE_TOKEN(BIF_P))) { BIF_RET(am_false); } if (!(is_atom(BIF_ARG_1) || is_small(BIF_ARG_1))) { diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index 0bf9558ac9..399be6058c 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -2308,11 +2308,7 @@ restart: *esp++ = am_true; break; case matchIsSeqTrace: - if (SEQ_TRACE_TOKEN(c_p) != NIL -#ifdef USE_VM_PROBES - && SEQ_TRACE_TOKEN(c_p) != am_have_dt_utag -#endif - ) + if (have_seqtrace(SEQ_TRACE_TOKEN(c_p))) *esp++ = am_true; else *esp++ = am_false; @@ -2336,11 +2332,7 @@ restart: --esp; break; case matchGetSeqToken: - if (SEQ_TRACE_TOKEN(c_p) == NIL -#ifdef USE_VM_PROBES - || SEQ_TRACE_TOKEN(c_p) == am_have_dt_utag -#endif - ) + if (have_no_seqtrace(SEQ_TRACE_TOKEN(c_p))) *esp++ = NIL; else { Eterm sender = SEQ_TRACE_TOKEN_SENDER(c_p); diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index f64385d92b..1811651a58 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -314,7 +314,7 @@ erts_queue_dist_message(Process *rcvr, DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE); dtrace_proc_str(rcvr, receiver_name); - if (token != NIL && token != am_have_dt_utag) { + if (have_seqtrace(token)) { tok_label = signed_val(SEQ_TRACE_T_LABEL(token)); tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token)); tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token)); @@ -335,7 +335,7 @@ erts_queue_dist_message(Process *rcvr, DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE); dtrace_proc_str(rcvr, receiver_name); - if (token != NIL && token != am_have_dt_utag) { + if (have_seqtrace(token)) { tok_label = signed_val(SEQ_TRACE_T_LABEL(token)); tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token)); tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token)); @@ -697,15 +697,13 @@ erts_send_message(Process* sender, * copy_shared_perform. (it inserts move_markers like the gc). * Make sure we don't use the heap between those instances. */ -#ifdef USE_VM_PROBES - if (stoken != am_have_dt_utag) { -#endif + if (have_seqtrace(stoken)) { seq_trace_update_send(sender); seq_trace_output(stoken, message, SEQ_TRACE_SEND, receiver->common.id, sender); seq_trace_size = 6; /* TUPLE5 */ -#ifdef USE_VM_PROBES } +#ifdef USE_VM_PROBES if (DT_UTAG_FLAGS(sender) & DT_UTAG_SPREADING) { dt_utag_size = size_object(DT_UTAG(sender)); } else if (stoken == am_have_dt_utag ) { @@ -765,7 +763,7 @@ erts_send_message(Process* sender, #ifdef USE_VM_PROBES if (DTRACE_ENABLED(message_send)) { - if (stoken != NIL && stoken != am_have_dt_utag) { + if (have_seqtrace(stoken)) { tok_label = signed_val(SEQ_TRACE_T_LABEL(stoken)); tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(stoken)); tok_serial = signed_val(SEQ_TRACE_T_SERIAL(stoken)); @@ -854,12 +852,7 @@ erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp, erts_shcopy_t info; #endif - if (token != NIL -#ifdef USE_VM_PROBES - && token != am_have_dt_utag -#endif - ) { - + if (have_seqtrace(token)) { ASSERT(is_tuple(token)); sz_token = size_object(token); sz_from = size_object(from); diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index 740ae46a0f..76387bc34c 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -128,6 +128,13 @@ struct erl_heap_fragment { #else #endif +#ifdef USE_VM_PROBES +#define have_no_seqtrace(T) ((T) == NIL || (T) == am_have_dt_utag) +#else +#define have_no_seqtrace(T) ((T) == NIL) +#endif +#define have_seqtrace(T) (!have_no_seqtrace(T)) + #define ERL_MESSAGE_REF_FIELDS__ \ ErtsMessage *next; /* Next message */ \ union { \ diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index f347d57a9d..5907dd4567 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -11526,11 +11526,7 @@ send_exit_message(Process *to, ErtsProcLocks *to_locksp, erts_shcopy_t info; #endif - if (token == NIL -#ifdef USE_VM_PROBES - || token == am_have_dt_utag -#endif - ) { + if (!have_seqtrace(token)) { #ifdef SHCOPY_SEND INITIALIZE_SHCOPY(info); term_size = copy_shared_calculate(exit_term, &info); @@ -11670,11 +11666,7 @@ send_exit_signal(Process *c_p, /* current process if and only if ((state & ERTS_PSFLG_TRAP_EXIT) && (reason != am_kill || (flags & ERTS_XSIG_FLG_IGN_KILL))) { - if (is_not_nil(token) -#ifdef USE_VM_PROBES - && token != am_have_dt_utag -#endif - && token_update) + if (have_seqtrace(token)) seq_trace_update_send(token_update); if (is_value(exit_tuple)) send_exit_message(rp, rp_locks, exit_tuple, exit_tuple_sz, token); diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index d02f1f7213..b774ba97af 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -1046,11 +1046,7 @@ seq_trace_update_send(Process *p) { Eterm seq_tracer = erts_get_system_seq_tracer(); ASSERT((is_tuple(SEQ_TRACE_TOKEN(p)) || is_nil(SEQ_TRACE_TOKEN(p)))); - if ( (p->common.id == seq_tracer) || (SEQ_TRACE_TOKEN(p) == NIL) -#ifdef USE_VM_PROBES - || (SEQ_TRACE_TOKEN(p) == am_have_dt_utag) -#endif - ) { + if ((p->common.id == seq_tracer) || have_no_seqtrace(SEQ_TRACE_TOKEN(p))) { return 0; } SEQ_TRACE_TOKEN_SENDER(p) = p->common.id; -- cgit v1.2.3