From a2359927ca4e7c315e9849deec3375b1f2ae4170 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Tue, 26 Jul 2016 10:19:31 +0200 Subject: erts: Remove the need for copying of literals * Literals are not copied between processes for messages or spawn Increases performance of message sent and processes spawned when literals are involved in messages or arguments. --- erts/emulator/beam/copy.c | 80 ++++++++++++++++++++++++++++------------ erts/emulator/beam/erl_gc.c | 8 ---- erts/emulator/beam/erl_message.c | 19 ++++++---- erts/emulator/beam/erl_process.c | 10 +++-- erts/emulator/beam/global.h | 20 ++++++---- 5 files changed, 88 insertions(+), 49 deletions(-) (limited to 'erts/emulator') diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index ccc4cbad43..7d3777f37e 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -40,7 +40,8 @@ static void move_one_frag(Eterm** hpp, ErlHeapFragment*, ErlOffHeap*, int); /* * Copy object "obj" to process p. */ -Eterm copy_object_x(Eterm obj, Process* to, Uint extra) { +Eterm copy_object_x(Eterm obj, Process* to, Uint extra) +{ if (!is_immed(obj)) { Uint size = size_object(obj); Eterm* hp = HAllocX(to, size, extra); @@ -70,7 +71,12 @@ Eterm copy_object_x(Eterm obj, Process* to, Uint extra) { * Return the "flat" size of the object. */ -Uint size_object(Eterm obj) +#define in_literal_purge_area(PTR) \ + (lit_purge_ptr && ( \ + (lit_purge_ptr <= (PTR) && \ + (PTR) < (lit_purge_ptr + lit_purge_sz)))) + +Uint size_object_x(Eterm obj, Eterm *lit_purge_ptr, Uint lit_purge_sz, Uint litopt) { Uint sum = 0; Eterm* ptr; @@ -78,7 +84,6 @@ Uint size_object(Eterm obj) #ifdef DEBUG Eterm mypid = erts_get_current_pid(); #endif - DECLARE_ESTACK(s); VERBOSE(DEBUG_SHCOPY, ("[pid=%T] size_object %p\n", mypid, obj)); @@ -86,17 +91,25 @@ Uint size_object(Eterm obj) for (;;) { switch (primary_tag(obj)) { case TAG_PRIMARY_LIST: - sum += 2; ptr = list_val(obj); + if (litopt && erts_is_literal(obj,ptr) && !in_literal_purge_area(ptr)) { + goto pop_next; + } + sum += 2; obj = *ptr++; if (!IS_CONST(obj)) { ESTACK_PUSH(s, obj); - } + } obj = *ptr; break; case TAG_PRIMARY_BOXED: { - Eterm hdr = *boxed_val(obj); + Eterm hdr; + ptr = boxed_val(obj); + if (litopt && erts_is_literal(obj,ptr) && !in_literal_purge_area(ptr)) { + goto pop_next; + } + hdr = *ptr; ASSERT(is_header(hdr)); switch (hdr & _TAG_HEADER_MASK) { case ARITYVAL_SUBTAG: @@ -279,10 +292,6 @@ do { \ #define COUNT_OFF_HEAP (0) -#define IN_LITERAL_PURGE_AREA(info, ptr) \ - ((info)->range_ptr && ( \ - (info)->range_ptr <= (ptr) && \ - (ptr) < ((info)->range_ptr + (info)->range_sz))) /* * Return the real size of an object and find sharing information * This currently returns the same as erts_debug:size/1. @@ -599,7 +608,8 @@ cleanup: /* * Copy a structure to a heap. */ -Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint *bsz) +Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint *bsz, + Eterm *lit_purge_ptr, Uint lit_purge_sz, Uint litopt) { char* hstart; Uint hsize; @@ -651,7 +661,6 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint L_copy: while (hp != htop) { obj = *hp; - switch (primary_tag(obj)) { case TAG_PRIMARY_IMMED1: hp++; @@ -667,6 +676,10 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint L_copy_list: tailp = argp; + if (litopt && erts_is_literal(obj,objp) && !in_literal_purge_area(objp)) { + *tailp = obj; + goto L_copy; + } for (;;) { tp = tailp; elem = CAR(objp); @@ -674,18 +687,23 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint hbot -= 2; CAR(hbot) = elem; tailp = &CDR(hbot); - } - else { + } else { CAR(htop) = elem; tailp = &CDR(htop); htop += 2; } *tp = make_list(tailp - 1); obj = CDR(objp); + if (!is_list(obj)) { break; } objp = list_val(obj); + + if (litopt && erts_is_literal(obj,objp) && !in_literal_purge_area(objp)) { + *tailp = obj; + goto L_copy; + } } switch (primary_tag(obj)) { case TAG_PRIMARY_IMMED1: *tailp = obj; goto L_copy; @@ -695,7 +713,7 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint "%s, line %d: Internal error in copy_struct: 0x%08x\n", __FILE__, __LINE__,obj); } - + case TAG_PRIMARY_BOXED: if (ErtsInArea(boxed_val(obj),hstart,hsize)) { hp++; @@ -705,6 +723,10 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint L_copy_boxed: objp = boxed_val(obj); + if (litopt && erts_is_literal(obj,objp) && !in_literal_purge_area(objp)) { + *argp = obj; + break; + } hdr = *objp; switch (hdr & _TAG_HEADER_MASK) { case ARITYVAL_SUBTAG: @@ -765,7 +787,7 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint extra_bytes = 1; } else { extra_bytes = 0; - } + } real_size = size+extra_bytes; objp = binary_val(real_bin); if (thing_subtag(*objp) == HEAP_BINARY_SUBTAG) { @@ -780,7 +802,7 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint } else { ProcBin* from = (ProcBin *) objp; ProcBin* to; - + ASSERT(thing_subtag(*objp) == REFC_BINARY_SUBTAG); if (from->flags) { erts_emasculate_writable_binary(from); @@ -900,6 +922,12 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint *bsz = hend - hbot; } else { #ifdef DEBUG + if (!eq(org_obj, res)) { + erts_exit(ERTS_ABORT_EXIT, + "Internal error in copy_struct() when copying %T:" + " not equal to copy %T\n", + org_obj, res); + } if (htop != hbot) erts_exit(ERTS_ABORT_EXIT, "Internal error in copy_struct() when copying %T:" @@ -1036,6 +1064,8 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info) Uint e; unsigned sz; Eterm* ptr; + Eterm *lit_purge_ptr = info->lit_purge_ptr; + Uint lit_purge_sz = info->lit_purge_sz; #ifdef DEBUG Eterm mypid = erts_get_current_pid(); #endif @@ -1081,7 +1111,7 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info) /* off heap list pointers are copied verbatim */ if (erts_is_literal(obj,ptr)) { VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", mypid, ptr, obj)); - if (IN_LITERAL_PURGE_AREA(info,ptr)) + if (in_literal_purge_area(ptr)) info->literal_size += size_object(obj); goto pop_next; } @@ -1132,7 +1162,7 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info) /* off heap pointers to boxes are copied verbatim */ if (erts_is_literal(obj,ptr)) { VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", mypid, ptr, obj)); - if (IN_LITERAL_PURGE_AREA(info,ptr)) + if (in_literal_purge_area(ptr)) info->literal_size += size_object(obj); goto pop_next; } @@ -1298,6 +1328,8 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, Eterm* resp; Eterm *hbot, *hend; unsigned remaining; + Eterm *lit_purge_ptr = info->lit_purge_ptr; + Uint lit_purge_sz = info->lit_purge_sz; #ifdef DEBUG Eterm mypid = erts_get_current_pid(); Eterm saved_obj = obj; @@ -1347,11 +1379,11 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, ptr = list_val(obj); /* off heap list pointers are copied verbatim */ if (erts_is_literal(obj,ptr)) { - if (!IN_LITERAL_PURGE_AREA(info,ptr)) { + if (!in_literal_purge_area(ptr)) { *resp = obj; } else { Uint bsz = 0; - *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz); + *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz, NULL, 0, 0); /* copy literal */ hbot -= bsz; } goto cleanup_next; @@ -1415,11 +1447,11 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, ptr = boxed_val(obj); /* off heap pointers to boxes are copied verbatim */ if (erts_is_literal(obj,ptr)) { - if (!IN_LITERAL_PURGE_AREA(info,ptr)) { + if (!in_literal_purge_area(ptr)) { *resp = obj; } else { Uint bsz = 0; - *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz); + *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz, NULL, 0, 0); /* copy literal */ hbot -= bsz; } goto cleanup_next; @@ -1923,7 +1955,7 @@ move_one_frag(Eterm** hpp, ErlHeapFragment* frag, ErlOffHeap* off_heap, int lite if (is_header(val)) { struct erl_off_heap_header* hdr = (struct erl_off_heap_header*)hp; ASSERT(ptr + header_arity(val) < end); - MOVE_BOXED(ptr, val, hp, &dummy_ref); + MOVE_BOXED(ptr, val, hp, &dummy_ref); switch (val & _HEADER_SUBTAG_MASK) { case REFC_BINARY_SUBTAG: case FUN_SUBTAG: diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index a224383493..22a7d430c8 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -2155,26 +2155,18 @@ copy_one_frag(Eterm** hpp, ErlOffHeap* off_heap, *hp++ = val; break; case TAG_PRIMARY_LIST: -#ifdef SHCOPY_SEND if (erts_is_literal(val,list_val(val))) { *hp++ = val; } else { *hp++ = offset_ptr(val, offs); } -#else - *hp++ = offset_ptr(val, offs); -#endif break; case TAG_PRIMARY_BOXED: -#ifdef SHCOPY_SEND if (erts_is_literal(val,boxed_val(val))) { *hp++ = val; } else { *hp++ = offset_ptr(val, offs); } -#else - *hp++ = offset_ptr(val, offs); -#endif break; case TAG_PRIMARY_HEADER: *hp++ = val; diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index 91e06cde2d..14ee07a304 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -696,6 +696,9 @@ erts_send_message(Process* sender, erts_aint32_t receiver_state; #ifdef SHCOPY_SEND erts_shcopy_t info; +#else + Eterm *lit_purge_ptr = erts_clrange.ptr; + Uint lit_purge_sz = erts_clrange.sz; #endif #ifdef USE_VM_PROBES @@ -725,7 +728,7 @@ erts_send_message(Process* sender, */ if (have_seqtrace(stoken)) { seq_trace_update_send(sender); - seq_trace_output(stoken, message, SEQ_TRACE_SEND, + seq_trace_output(stoken, message, SEQ_TRACE_SEND, receiver->common.id, sender); seq_trace_size = 6; /* TUPLE5 */ } @@ -741,7 +744,7 @@ erts_send_message(Process* sender, INITIALIZE_SHCOPY(info); msize = copy_shared_calculate(message, &info); #else - msize = size_object(message); + msize = size_object_litopt(message, lit_purge_ptr, lit_purge_sz); #endif mp = erts_alloc_message_heap_state(receiver, &receiver_state, @@ -760,7 +763,8 @@ erts_send_message(Process* sender, DESTROY_SHCOPY(info); #else if (is_not_immed(message)) - message = copy_struct(message, msize, &hp, ohp); + message = copy_struct_litopt(message, msize, &hp, ohp, + lit_purge_ptr, lit_purge_sz); #endif if (is_immed(stoken)) token = stoken; @@ -796,7 +800,7 @@ erts_send_message(Process* sender, INITIALIZE_SHCOPY(info); msize = copy_shared_calculate(message, &info); #else - msize = size_object(message); + msize = size_object_litopt(message, lit_purge_ptr, lit_purge_sz); #endif mp = erts_alloc_message_heap_state(receiver, &receiver_state, @@ -810,7 +814,8 @@ erts_send_message(Process* sender, DESTROY_SHCOPY(info); #else if (is_not_immed(message)) - message = copy_struct(message, msize, &hp, ohp); + message = copy_struct_litopt(message, msize, &hp, ohp, + lit_purge_ptr, lit_purge_sz); #endif } #ifdef USE_VM_PROBES @@ -998,8 +1003,8 @@ erts_move_messages_off_heap(Process *c_p) hp = hfrag->mem; if (is_not_immed(ERL_MESSAGE_TERM(mp))) ERL_MESSAGE_TERM(mp) = copy_struct(ERL_MESSAGE_TERM(mp), - msg_sz, &hp, - &hfrag->off_heap); + msg_sz, &hp, + &hfrag->off_heap); if (is_not_immed(ERL_MESSAGE_TOKEN(mp))) ERL_MESSAGE_TOKEN(mp) = copy_struct(ERL_MESSAGE_TOKEN(mp), token_sz, &hp, diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index b5d8c5bc75..d6037e9f5d 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -11099,6 +11099,9 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). #ifdef SHCOPY_SPAWN erts_shcopy_t info; INITIALIZE_SHCOPY(info); +#else + Eterm *lit_purge_ptr = erts_clrange.ptr; + Uint lit_purge_sz = erts_clrange.sz; #endif erts_smp_proc_lock(parent, ERTS_PROC_LOCKS_ALL_MINOR); @@ -11157,7 +11160,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). #ifdef SHCOPY_SPAWN arg_size = copy_shared_calculate(args, &info); #else - arg_size = size_object(args); + arg_size = size_object_litopt(args, lit_purge_ptr, lit_purge_sz); #endif heap_need = arg_size; @@ -11181,7 +11184,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). } p->schedule_count = 0; ASSERT(p->min_heap_size == erts_next_heap_size(p->min_heap_size, 0)); - + p->u.initial[INITIAL_MOD] = mod; p->u.initial[INITIAL_FUN] = func; p->u.initial[INITIAL_ARI] = (Uint) arity; @@ -11239,7 +11242,8 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->arg_reg[2] = copy_shared_perform(args, arg_size, &info, &p->htop, &p->off_heap); DESTROY_SHCOPY(info); #else - p->arg_reg[2] = copy_struct(args, arg_size, &p->htop, &p->off_heap); + p->arg_reg[2] = copy_struct_litopt(args, arg_size, &p->htop, &p->off_heap, + lit_purge_ptr, lit_purge_sz); #endif p->arity = 3; diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index bf2e50e52f..67ce0d0723 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1085,8 +1085,8 @@ typedef struct { Eterm* shtable_start; ErtsAlcType_t shtable_alloc_type; Uint literal_size; - Eterm *range_ptr; - Uint range_sz; + Eterm *lit_purge_ptr; + Uint lit_purge_sz; } erts_shcopy_t; #define INITIALIZE_SHCOPY(info) \ @@ -1095,8 +1095,8 @@ do { \ info.bitstore_start = info.bitstore_default; \ info.shtable_start = info.shtable_default; \ info.literal_size = 0; \ - info.range_ptr = erts_clrange.ptr; \ - info.range_sz = erts_clrange.sz; \ + info.lit_purge_ptr = erts_clrange.ptr; \ + info.lit_purge_sz = erts_clrange.sz; \ } while(0) #define DESTROY_SHCOPY(info) \ @@ -1116,15 +1116,21 @@ do { \ Eterm copy_object_x(Eterm, Process*, Uint); #define copy_object(Term, Proc) copy_object_x(Term,Proc,0) -Uint size_object(Eterm); +Uint size_object_x(Eterm,Eterm*,Uint,Uint); +#define size_object(Term) size_object_x(Term,NULL,0,0) +#define size_object_litopt(Term,LitPtr,LitSz) size_object_x(Term,LitPtr,LitSz,1) + Uint copy_shared_calculate(Eterm, erts_shcopy_t*); Eterm copy_shared_perform(Eterm, Uint, erts_shcopy_t*, Eterm**, ErlOffHeap*); Uint size_shared(Eterm); -Eterm copy_struct_x(Eterm, Uint, Eterm**, ErlOffHeap*, Uint* bsz); +Eterm copy_struct_x(Eterm, Uint, Eterm**, ErlOffHeap*, Uint* bsz, Eterm *lit_ptr, Uint lit_sz, Uint litopt); #define copy_struct(Obj,Sz,HPP,OH) \ - copy_struct_x(Obj,Sz,HPP,OH,NULL) + copy_struct_x(Obj,Sz,HPP,OH,NULL,NULL,0,0) +#define copy_struct_litopt(Obj,Sz,HPP,OH,LitPtr,LitSz) \ + copy_struct_x(Obj,Sz,HPP,OH,NULL,LitPtr,LitSz,1) + Eterm copy_shallow(Eterm*, Uint, Eterm**, ErlOffHeap*); void erts_move_multi_frags(Eterm** hpp, ErlOffHeap*, ErlHeapFragment* first, -- cgit v1.2.3 From a9368b3c1ee78a3c83fcff83590a3d94ea4ddbe0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Wed, 27 Jul 2016 17:00:32 +0200 Subject: erts: Fix literal size calculation in check_process_code We want to know the total size of literals of all heap fragmens. --- erts/emulator/beam/beam_bif_load.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'erts/emulator') diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 15e878ba65..2a165bed00 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -873,12 +873,12 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls continue; { ErlHeapFragment *hf; - Uint lit_sz; + Uint lit_sz = 0; for (hf=hfrag; hf; hf = hf->next) { if (check_mod_funs(rp, &hfrag->off_heap, mod_start, mod_size)) return am_true; - lit_sz = hfrag_literal_size(&hf->mem[0], &hf->mem[hf->used_size], - literals, lit_bsize); + lit_sz += hfrag_literal_size(&hf->mem[0], &hf->mem[hf->used_size], + literals, lit_bsize); } if (lit_sz > 0) { ErlHeapFragment *bp = new_message_buffer(lit_sz); -- cgit v1.2.3 From 13ec43880b8565a5ddd7da590c36a520d4ff4b26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Fri, 29 Jul 2016 12:19:11 +0200 Subject: erts: Don't copy literals in enif_send --- erts/emulator/beam/erl_nif.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'erts/emulator') diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index c6127a4967..e1944fff29 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -627,7 +627,9 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, MBUF(&menv->phony_proc) = NULL; } } else { - Uint sz = size_object(msg); + Eterm *lit_purge_ptr = erts_clrange.ptr; + Uint lit_purge_sz = erts_clrange.sz; + Uint sz = size_object_litopt(msg, lit_purge_ptr, lit_purge_sz); ErlOffHeap *ohp; Eterm *hp; if (env && !env->tracee) { @@ -649,7 +651,8 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, ohp = &bp->off_heap; } } - msg = copy_struct(msg, sz, &hp, ohp); + msg = copy_struct_litopt(msg, sz, &hp, ohp, + lit_purge_ptr, lit_purge_sz); } ERL_MESSAGE_TERM(mp) = msg; -- cgit v1.2.3 From dbea5c4023ec6ac59bb1988711532682a032536d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Fri, 29 Jul 2016 15:59:16 +0200 Subject: erts: No need to literal opt message token --- erts/emulator/beam/erl_gc.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'erts/emulator') diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 22a7d430c8..cddcc847a3 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -2252,10 +2252,12 @@ move_msgq_to_heap(Process *p) ASSERT(mp->data.dist_ext->heap_size >= 0); if (is_not_nil(ERL_MESSAGE_TOKEN(mp))) { bp = erts_dist_ext_trailer(mp->data.dist_ext); + /* Tokens does not use literal optimization */ ERL_MESSAGE_TOKEN(mp) = copy_struct(ERL_MESSAGE_TOKEN(mp), bp->used_size, - &factory.hp, - factory.off_heap); + &factory.hp, + factory.off_heap); + erts_cleanup_offheap(&bp->off_heap); } ERL_MESSAGE_TERM(mp) = erts_decode_dist_ext(&factory, -- cgit v1.2.3 From e52fa24451306080be39a35444cf52210982b41a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Fri, 29 Jul 2016 17:16:53 +0200 Subject: erts: Refactor literal purge area arguments --- erts/emulator/beam/copy.c | 14 ++++++++------ erts/emulator/beam/erl_message.c | 14 ++++++-------- erts/emulator/beam/erl_nif.c | 10 +++++----- erts/emulator/beam/erl_process.c | 9 ++++----- erts/emulator/beam/global.h | 25 ++++++++++++++++++------- 5 files changed, 41 insertions(+), 31 deletions(-) (limited to 'erts/emulator') diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index 7d3777f37e..c4dcd6a3cc 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -76,16 +76,17 @@ Eterm copy_object_x(Eterm obj, Process* to, Uint extra) (lit_purge_ptr <= (PTR) && \ (PTR) < (lit_purge_ptr + lit_purge_sz)))) -Uint size_object_x(Eterm obj, Eterm *lit_purge_ptr, Uint lit_purge_sz, Uint litopt) +Uint size_object_x(Eterm obj, erts_literal_area_t *litopt) { Uint sum = 0; Eterm* ptr; int arity; + Eterm *lit_purge_ptr = litopt ? litopt->lit_purge_ptr : NULL; + Uint lit_purge_sz = litopt ? litopt->lit_purge_sz : 0; #ifdef DEBUG Eterm mypid = erts_get_current_pid(); #endif DECLARE_ESTACK(s); - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] size_object %p\n", mypid, obj)); for (;;) { @@ -608,8 +609,7 @@ cleanup: /* * Copy a structure to a heap. */ -Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint *bsz, - Eterm *lit_purge_ptr, Uint lit_purge_sz, Uint litopt) +Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint *bsz, erts_literal_area_t *litopt) { char* hstart; Uint hsize; @@ -626,6 +626,8 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint Eterm hdr; Eterm *hend; int i; + Eterm *lit_purge_ptr = litopt ? litopt->lit_purge_ptr : NULL; + Uint lit_purge_sz = litopt ? litopt->lit_purge_sz : 0; #ifdef DEBUG Eterm org_obj = obj; Uint org_sz = sz; @@ -1383,7 +1385,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, *resp = obj; } else { Uint bsz = 0; - *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz, NULL, 0, 0); /* copy literal */ + *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz, NULL); /* copy literal */ hbot -= bsz; } goto cleanup_next; @@ -1451,7 +1453,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, *resp = obj; } else { Uint bsz = 0; - *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz, NULL, 0, 0); /* copy literal */ + *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz, NULL); /* copy literal */ hbot -= bsz; } goto cleanup_next; diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index 14ee07a304..e9f0586edd 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -697,8 +697,8 @@ erts_send_message(Process* sender, #ifdef SHCOPY_SEND erts_shcopy_t info; #else - Eterm *lit_purge_ptr = erts_clrange.ptr; - Uint lit_purge_sz = erts_clrange.sz; + erts_literal_area_t litarea; + INITIALIZE_LITERAL_PURGE_AREA(litarea); #endif #ifdef USE_VM_PROBES @@ -744,7 +744,7 @@ erts_send_message(Process* sender, INITIALIZE_SHCOPY(info); msize = copy_shared_calculate(message, &info); #else - msize = size_object_litopt(message, lit_purge_ptr, lit_purge_sz); + msize = size_object_litopt(message, &litarea); #endif mp = erts_alloc_message_heap_state(receiver, &receiver_state, @@ -763,8 +763,7 @@ erts_send_message(Process* sender, DESTROY_SHCOPY(info); #else if (is_not_immed(message)) - message = copy_struct_litopt(message, msize, &hp, ohp, - lit_purge_ptr, lit_purge_sz); + message = copy_struct_litopt(message, msize, &hp, ohp, &litarea); #endif if (is_immed(stoken)) token = stoken; @@ -800,7 +799,7 @@ erts_send_message(Process* sender, INITIALIZE_SHCOPY(info); msize = copy_shared_calculate(message, &info); #else - msize = size_object_litopt(message, lit_purge_ptr, lit_purge_sz); + msize = size_object_litopt(message, &litarea); #endif mp = erts_alloc_message_heap_state(receiver, &receiver_state, @@ -814,8 +813,7 @@ erts_send_message(Process* sender, DESTROY_SHCOPY(info); #else if (is_not_immed(message)) - message = copy_struct_litopt(message, msize, &hp, ohp, - lit_purge_ptr, lit_purge_sz); + message = copy_struct_litopt(message, msize, &hp, ohp, &litarea); #endif } #ifdef USE_VM_PROBES diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index e1944fff29..6bd27d11f4 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -627,11 +627,12 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, MBUF(&menv->phony_proc) = NULL; } } else { - Eterm *lit_purge_ptr = erts_clrange.ptr; - Uint lit_purge_sz = erts_clrange.sz; - Uint sz = size_object_litopt(msg, lit_purge_ptr, lit_purge_sz); + erts_literal_area_t litarea; ErlOffHeap *ohp; Eterm *hp; + Uint sz; + INITIALIZE_LITERAL_PURGE_AREA(litarea); + sz = size_object_litopt(msg, &litarea); if (env && !env->tracee) { flush_env(env); mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp); @@ -651,8 +652,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, ohp = &bp->off_heap; } } - msg = copy_struct_litopt(msg, sz, &hp, ohp, - lit_purge_ptr, lit_purge_sz); + msg = copy_struct_litopt(msg, sz, &hp, ohp, &litarea); } ERL_MESSAGE_TERM(mp) = msg; diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index d6037e9f5d..85c37e08be 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -11100,8 +11100,8 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). erts_shcopy_t info; INITIALIZE_SHCOPY(info); #else - Eterm *lit_purge_ptr = erts_clrange.ptr; - Uint lit_purge_sz = erts_clrange.sz; + erts_literal_area_t litarea; + INITIALIZE_LITERAL_PURGE_AREA(litarea); #endif erts_smp_proc_lock(parent, ERTS_PROC_LOCKS_ALL_MINOR); @@ -11160,7 +11160,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). #ifdef SHCOPY_SPAWN arg_size = copy_shared_calculate(args, &info); #else - arg_size = size_object_litopt(args, lit_purge_ptr, lit_purge_sz); + arg_size = size_object_litopt(args, &litarea); #endif heap_need = arg_size; @@ -11242,8 +11242,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->arg_reg[2] = copy_shared_perform(args, arg_size, &info, &p->htop, &p->off_heap); DESTROY_SHCOPY(info); #else - p->arg_reg[2] = copy_struct_litopt(args, arg_size, &p->htop, &p->off_heap, - lit_purge_ptr, lit_purge_sz); + p->arg_reg[2] = copy_struct_litopt(args, arg_size, &p->htop, &p->off_heap, &litarea); #endif p->arity = 3; diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 67ce0d0723..1423973739 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1113,23 +1113,34 @@ do { \ } while(0) /* copy.c */ +typedef struct { + Eterm *lit_purge_ptr; + Uint lit_purge_sz; +} erts_literal_area_t; + +#define INITIALIZE_LITERAL_PURGE_AREA(Area) \ + do { \ + (Area).lit_purge_ptr = erts_clrange.ptr; \ + (Area).lit_purge_sz = erts_clrange.sz; \ + } while(0) + Eterm copy_object_x(Eterm, Process*, Uint); #define copy_object(Term, Proc) copy_object_x(Term,Proc,0) -Uint size_object_x(Eterm,Eterm*,Uint,Uint); -#define size_object(Term) size_object_x(Term,NULL,0,0) -#define size_object_litopt(Term,LitPtr,LitSz) size_object_x(Term,LitPtr,LitSz,1) +Uint size_object_x(Eterm, erts_literal_area_t*); +#define size_object(Term) size_object_x(Term,NULL) +#define size_object_litopt(Term,LitArea) size_object_x(Term,LitArea) Uint copy_shared_calculate(Eterm, erts_shcopy_t*); Eterm copy_shared_perform(Eterm, Uint, erts_shcopy_t*, Eterm**, ErlOffHeap*); Uint size_shared(Eterm); -Eterm copy_struct_x(Eterm, Uint, Eterm**, ErlOffHeap*, Uint* bsz, Eterm *lit_ptr, Uint lit_sz, Uint litopt); +Eterm copy_struct_x(Eterm, Uint, Eterm**, ErlOffHeap*, Uint*, erts_literal_area_t*); #define copy_struct(Obj,Sz,HPP,OH) \ - copy_struct_x(Obj,Sz,HPP,OH,NULL,NULL,0,0) -#define copy_struct_litopt(Obj,Sz,HPP,OH,LitPtr,LitSz) \ - copy_struct_x(Obj,Sz,HPP,OH,NULL,LitPtr,LitSz,1) + copy_struct_x(Obj,Sz,HPP,OH,NULL,NULL) +#define copy_struct_litopt(Obj,Sz,HPP,OH,LitArea) \ + copy_struct_x(Obj,Sz,HPP,OH,NULL,LitArea) Eterm copy_shallow(Eterm*, Uint, Eterm**, ErlOffHeap*); -- cgit v1.2.3