diff options
| author | Lukas Larsson <[email protected]> | 2019-02-22 11:16:29 +0100 |
|---|---|---|
| committer | Lukas Larsson <[email protected]> | 2019-02-22 11:16:29 +0100 |
| commit | 305204ffb816a93885e4919b0cd691fcaa8ed440 (patch) | |
| tree | b8569b97492930d427c06d4cb6d3667da1418c8b /erts/emulator/beam | |
| parent | a1e787d2eed6223f3b1c0175cfc379f61922c349 (diff) | |
| parent | 3ee8e06e8102b63c24df0b443027853cdf99549f (diff) | |
| download | otp-305204ffb816a93885e4919b0cd691fcaa8ed440.tar.gz otp-305204ffb816a93885e4919b0cd691fcaa8ed440.tar.bz2 otp-305204ffb816a93885e4919b0cd691fcaa8ed440.zip | |
Merge branch 'lukas/erts/fragment-dist-messages/OTP-13397/OTP-15610/OTP-15611/OTP-15612/OTP-15613'
* lukas/erts/fragment-dist-messages/OTP-13397/OTP-15610/OTP-15611/OTP-15612/OTP-15613:
erts: Add debug dist obuf memory leak check
win32: Fix ./otp_build debuginfo_win32
Make ld.sh on windows print better error reason
erts: Fix so that externals with creation 0 compare equal to all
erts: Expand etp to look for free processes
erts: Implement trapping while sending distr exit/down
erts: Add ERL_NODE_BOOKKEEP to node tables refc
erts: Refactor ErtsSendContext to be ErtsDSigSendContext
erts: Add distr testcases for fragmentation
erts: Make remote send of exit/2 trap
erts: Implement fragmentation of distrubution messages
erts: Expand distribution protocol documentation
erts: Move reason in dist messages to payload
erts: Remove a copy of distribution data payload
erts: Yield later during process exit and allow free procs to run
erts: Refactor rbt _yielding to use reductions
erts: Limit binary printout for %.XT in erts_print
Diffstat (limited to 'erts/emulator/beam')
41 files changed, 3448 insertions, 1911 deletions
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 4ef06464f4..90162a6543 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -885,19 +885,22 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) #include "beam_warm.h" OpCase(normal_exit): { - SWAPOUT; + HEAVY_SWAPOUT; c_p->freason = EXC_NORMAL; c_p->arity = 0; /* In case this process will ever be garbed again. */ ERTS_UNREQ_PROC_MAIN_LOCK(c_p); erts_do_exit_process(c_p, am_normal); ERTS_REQ_PROC_MAIN_LOCK(c_p); + HEAVY_SWAPIN; goto do_schedule; } OpCase(continue_exit): { + HEAVY_SWAPOUT; ERTS_UNREQ_PROC_MAIN_LOCK(c_p); erts_continue_exit_process(c_p); ERTS_REQ_PROC_MAIN_LOCK(c_p); + HEAVY_SWAPIN; goto do_schedule; } diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 7ff55e8927..e9e294cd59 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -6108,7 +6108,8 @@ erts_release_literal_area(ErtsLiteralArea* literal_area) } default: ASSERT(is_external_header(oh->thing_word)); - erts_deref_node_entry(((ExternalThing*)oh)->node); + erts_deref_node_entry(((ExternalThing*)oh)->node, + make_boxed(&oh->thing_word)); } oh = oh->next; } diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index ff7db0e742..a4c7af9a24 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -184,7 +184,7 @@ BIF_RETTYPE link_1(BIF_ALIST_1) DistEntry *dep; ErtsLink *lnk; int code; - ErtsDSigData dsd; + ErtsDSigSendContext ctx; dep = external_pid_dist_entry(BIF_ARG_1); if (dep == erts_this_dist_entry) @@ -201,9 +201,9 @@ BIF_RETTYPE link_1(BIF_ALIST_1) ldp = erts_link_to_data(lnk); - code = erts_dsig_prepare(&dsd, dep, BIF_P, + code = erts_dsig_prepare(&ctx, dep, BIF_P, ERTS_PROC_LOCK_MAIN, - ERTS_DSP_RLOCK, 0, 1); + ERTS_DSP_RLOCK, 0, 1, 1); switch (code) { case ERTS_DSIG_PREP_NOT_ALIVE: case ERTS_DSIG_PREP_NOT_CONNECTED: @@ -222,9 +222,10 @@ BIF_RETTYPE link_1(BIF_ALIST_1) ASSERT(inserted); (void)inserted; erts_de_runlock(dep); - code = erts_dsig_send_link(&dsd, BIF_P->common.id, BIF_ARG_1); + code = erts_dsig_send_link(&ctx, BIF_P->common.id, BIF_ARG_1); if (code == ERTS_DSIG_SEND_YIELD) ERTS_BIF_YIELD_RETURN(BIF_P, am_true); + ASSERT(code == ERTS_DSIG_SEND_OK); BIF_RET(am_true); break; } @@ -306,7 +307,7 @@ demonitor(Process *c_p, Eterm ref, Eterm *multip) DistEntry *dep; int code = ERTS_DSIG_SEND_OK; int deleted; - ErtsDSigData dsd; + ErtsDSigSendContext ctx; ASSERT(is_external_pid(to) || is_node_name_atom(to)); @@ -322,8 +323,8 @@ demonitor(Process *c_p, Eterm ref, Eterm *multip) } } - code = erts_dsig_prepare(&dsd, dep, c_p, ERTS_PROC_LOCK_MAIN, - ERTS_DSP_RLOCK, 0, 0); + code = erts_dsig_prepare(&ctx, dep, c_p, ERTS_PROC_LOCK_MAIN, + ERTS_DSP_RLOCK, 0, 1, 0); deleted = erts_monitor_dist_delete(&mdp->target); @@ -352,8 +353,8 @@ demonitor(Process *c_p, Eterm ref, Eterm *multip) * monitor list since in case of monitor name * the atom is stored there. Yield if necessary. */ - code = erts_dsig_send_demonitor(&dsd, c_p->common.id, - watched, mdp->ref, 0); + code = erts_dsig_send_demonitor(&ctx, c_p->common.id, + watched, mdp->ref); break; } @@ -533,7 +534,7 @@ BIF_RETTYPE monitor_2(BIF_ALIST_2) } if (is_external_pid(target)) { - ErtsDSigData dsd; + ErtsDSigSendContext ctx; int code; dep = external_pid_dist_entry(target); @@ -551,9 +552,9 @@ BIF_RETTYPE monitor_2(BIF_ALIST_2) BIF_P->common.id, id, name); erts_monitor_tree_insert(&ERTS_P_MONITORS(BIF_P), &mdp->origin); - code = erts_dsig_prepare(&dsd, dep, + code = erts_dsig_prepare(&ctx, dep, BIF_P, ERTS_PROC_LOCK_MAIN, - ERTS_DSP_RLOCK, 0, 1); + ERTS_DSP_RLOCK, 0, 1, 1); switch (code) { case ERTS_DSIG_PREP_NOT_ALIVE: case ERTS_DSIG_PREP_NOT_CONNECTED: @@ -568,7 +569,7 @@ BIF_RETTYPE monitor_2(BIF_ALIST_2) ASSERT(inserted); (void)inserted; erts_de_runlock(dep); - code = erts_dsig_send_monitor(&dsd, BIF_P->common.id, target, ref); + code = erts_dsig_send_monitor(&ctx, BIF_P->common.id, target, ref); break; } @@ -914,7 +915,7 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1) ErtsLinkData *ldp; DistEntry *dep; int code; - ErtsDSigData dsd; + ErtsDSigSendContext ctx; dep = external_pid_dist_entry(BIF_ARG_1); if (dep == erts_this_dist_entry) @@ -932,15 +933,15 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1) else erts_link_release(lnk); - code = erts_dsig_prepare(&dsd, dep, BIF_P, ERTS_PROC_LOCK_MAIN, - ERTS_DSP_NO_LOCK, 0, 0); + code = erts_dsig_prepare(&ctx, dep, BIF_P, ERTS_PROC_LOCK_MAIN, + ERTS_DSP_NO_LOCK, 0, 1, 0); switch (code) { case ERTS_DSIG_PREP_NOT_ALIVE: case ERTS_DSIG_PREP_NOT_CONNECTED: BIF_RET(am_true); case ERTS_DSIG_PREP_PENDING: case ERTS_DSIG_PREP_CONNECTED: - code = erts_dsig_send_unlink(&dsd, BIF_P->common.id, BIF_ARG_1); + code = erts_dsig_send_unlink(&ctx, BIF_P->common.id, BIF_ARG_1); if (code == ERTS_DSIG_SEND_YIELD) ERTS_BIF_YIELD_RETURN(BIF_P, am_true); break; @@ -1301,10 +1302,11 @@ static BIF_RETTYPE send_exit_signal_bif(Process *c_p, Eterm id, Eterm reason, in ERTS_BIF_PREP_RET(ret_val, am_true); /* Old incarnation of this node... */ else { int code; - ErtsDSigData dsd; + ErtsDSigSendContext ctx; + + code = erts_dsig_prepare(&ctx, dep, c_p, ERTS_PROC_LOCK_MAIN, + ERTS_DSP_NO_LOCK, 0, 0, 1); - code = erts_dsig_prepare(&dsd, dep, c_p, ERTS_PROC_LOCK_MAIN, - ERTS_DSP_NO_LOCK, 0, 1); switch (code) { case ERTS_DSIG_PREP_NOT_ALIVE: case ERTS_DSIG_PREP_NOT_CONNECTED: @@ -1312,11 +1314,29 @@ static BIF_RETTYPE send_exit_signal_bif(Process *c_p, Eterm id, Eterm reason, in break; case ERTS_DSIG_PREP_PENDING: case ERTS_DSIG_PREP_CONNECTED: - code = erts_dsig_send_exit2(&dsd, c_p->common.id, id, reason); - if (code == ERTS_DSIG_SEND_YIELD) + code = erts_dsig_send_exit2(&ctx, c_p->common.id, id, reason); + switch (code) { + case ERTS_DSIG_SEND_YIELD: ERTS_BIF_PREP_YIELD_RETURN(ret_val, c_p, am_true); - else + break; + case ERTS_DSIG_SEND_CONTINUE: + BUMP_ALL_REDS(c_p); + erts_set_gc_state(c_p, 0); + ERTS_BIF_PREP_TRAP1(ret_val, &dsend_continue_trap_export, c_p, + erts_dsend_export_trap_context(c_p, &ctx)); + break; + case ERTS_DSIG_SEND_OK: ERTS_BIF_PREP_RET(ret_val, am_true); + break; + case ERTS_DSIG_SEND_TOO_LRG: + erts_set_gc_state(c_p, 1); + ERTS_BIF_PREP_ERROR(ret_val, c_p, SYSTEM_LIMIT); + break; + default: + ASSERT(! "Invalid dsig send exit2 result"); + ERTS_BIF_PREP_ERROR(ret_val, c_p, EXC_INTERNAL_ERROR); + break; + } break; default: ASSERT(! "Invalid dsig prepare result"); @@ -1800,33 +1820,36 @@ ebif_bang_2(BIF_ALIST_2) static Sint remote_send(Process *p, DistEntry *dep, - Eterm to, Eterm full_to, Eterm msg, - ErtsSendContext* ctx) + Eterm to, Eterm node, Eterm full_to, Eterm msg, + Eterm return_term, Eterm *ctxpp, + int connect, int suspend) { Sint res; int code; + ErtsDSigSendContext ctx; ASSERT(is_atom(to) || is_external_pid(to)); - ctx->dep = dep; - code = erts_dsig_prepare(&ctx->dsd, dep, p, ERTS_PROC_LOCK_MAIN, + code = erts_dsig_prepare(&ctx, dep, p, ERTS_PROC_LOCK_MAIN, ERTS_DSP_NO_LOCK, - !ctx->suspend, ctx->connect); + !suspend, 0, connect); + ctx.return_term = return_term; + ctx.node = node; switch (code) { case ERTS_DSIG_PREP_NOT_ALIVE: case ERTS_DSIG_PREP_NOT_CONNECTED: res = SEND_NOCONNECT; break; case ERTS_DSIG_PREP_WOULD_SUSPEND: - ASSERT(!ctx->suspend); + ASSERT(!suspend); res = SEND_YIELD; break; case ERTS_DSIG_PREP_PENDING: case ERTS_DSIG_PREP_CONNECTED: { if (is_atom(to)) - code = erts_dsig_send_reg_msg(to, msg, ctx); + code = erts_dsig_send_reg_msg(&ctx, to, msg); else - code = erts_dsig_send_msg(to, msg, ctx); + code = erts_dsig_send_msg(&ctx, to, msg); /* * Note that reductions have been bumped on calling * process by erts_dsig_send_reg_msg() or @@ -1834,9 +1857,19 @@ static Sint remote_send(Process *p, DistEntry *dep, */ if (code == ERTS_DSIG_SEND_YIELD) res = SEND_YIELD_RETURN; - else if (code == ERTS_DSIG_SEND_CONTINUE) + else if (code == ERTS_DSIG_SEND_CONTINUE) { + erts_set_gc_state(p, 0); + + /* Keep a reference to the dist entry if the + name is an not a pid. */ + if (is_atom(to)) { + erts_ref_dist_entry(ctx.dep); + ctx.deref_dep = 1; + } + + *ctxpp = erts_dsend_export_trap_context(p, &ctx); res = SEND_YIELD_CONTINUE; - else if (code == ERTS_DSIG_SEND_TOO_LRG) + } else if (code == ERTS_DSIG_SEND_TOO_LRG) res = SEND_SYSTEM_LIMIT; else res = 0; @@ -1858,7 +1891,8 @@ static Sint remote_send(Process *p, DistEntry *dep, } static Sint -do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext *ctx) +do_send(Process *p, Eterm to, Eterm msg, Eterm return_term, Eterm *refp, + Eterm *dist_ctx, int connect, int suspend) { Eterm portid; Port *pt; @@ -1890,7 +1924,8 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext *ctx) erts_send_error_to_logger(p->group_leader, dsbufp); return 0; } - return remote_send(p, dep, to, to, msg, ctx); + return remote_send(p, dep, to, dep->sysname, to, msg, return_term, + dist_ctx, connect, suspend); } else if (is_atom(to)) { Eterm id = erts_whereis_name_to_id(p, to); @@ -1945,7 +1980,7 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext *ctx) ret_val = 0; if (pt) { - int ps_flags = ctx->suspend ? 0 : ERTS_PORT_SIG_FLG_NOSUSPEND; + int ps_flags = suspend ? 0 : ERTS_PORT_SIG_FLG_NOSUSPEND; *refp = NIL; if (IS_TRACED_FL(p, F_TRACE_SEND)) /* trace once only !! */ @@ -1960,12 +1995,12 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext *ctx) switch (erts_port_command(p, ps_flags, pt, msg, refp)) { case ERTS_PORT_OP_BUSY: /* Nothing has been sent */ - if (ctx->suspend) + if (suspend) erts_suspend(p, ERTS_PROC_LOCK_MAIN, pt); return SEND_YIELD; case ERTS_PORT_OP_BUSY_SCHEDULED: /* Message was sent */ - if (ctx->suspend) { + if (suspend) { erts_suspend(p, ERTS_PROC_LOCK_MAIN, pt); ret_val = SEND_YIELD_RETURN; break; @@ -2036,13 +2071,10 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext *ctx) ASSERT(dep != erts_this_dist_entry); deref_dep = 1; } - ctx->dsd.node = tp[2]; - ret = remote_send(p, dep, tp[1], to, msg, ctx); - if (ret == SEND_YIELD_CONTINUE) { - erts_ref_dist_entry(ctx->dep); - ctx->deref_dep = 1; - } + ret = remote_send(p, dep, tp[1], tp[2], to, msg, return_term, + dist_ctx, connect, suspend); + if (deref_dep) erts_deref_dist_entry(dep); return ret; @@ -2081,25 +2113,16 @@ BIF_RETTYPE send_3(BIF_ALIST_3) Eterm l = opts; Sint result; - - DeclareTypedTmpHeap(ErtsSendContext, ctx, BIF_P); + int connect = 1, suspend = 1; + Eterm ctx; ERTS_MSACC_PUSH_STATE_M_X(); - UseTmpHeap(sizeof(ErtsSendContext)/sizeof(Eterm), BIF_P); - - ctx->suspend = !0; - ctx->connect = !0; - ctx->deref_dep = 0; - ctx->return_term = am_ok; - ctx->dss.reds = (Sint) (ERTS_BIF_REDS_LEFT(p) * TERM_TO_BINARY_LOOP_FACTOR); - ctx->dss.phase = ERTS_DSIG_SEND_PHASE_INIT; - while (is_list(l)) { if (CAR(list_val(l)) == am_noconnect) { - ctx->connect = 0; + connect = 0; } else if (CAR(list_val(l)) == am_nosuspend) { - ctx->suspend = 0; + suspend = 0; } else { ERTS_BIF_PREP_ERROR(retval, p, BADARG); goto done; @@ -2116,7 +2139,7 @@ BIF_RETTYPE send_3(BIF_ALIST_3) #endif ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_SEND); - result = do_send(p, to, msg, &ref, ctx); + result = do_send(p, to, msg, am_ok, &ref, &ctx, connect, suspend); ERTS_MSACC_POP_STATE_M_X(); if (result >= 0) { @@ -2129,22 +2152,21 @@ BIF_RETTYPE send_3(BIF_ALIST_3) switch (result) { case SEND_NOCONNECT: - if (ctx->connect) { + if (connect) { ERTS_BIF_PREP_RET(retval, am_ok); } else { ERTS_BIF_PREP_RET(retval, am_noconnect); } break; case SEND_YIELD: - if (ctx->suspend) { - ERTS_BIF_PREP_YIELD3(retval, - bif_export[BIF_send_3], p, to, msg, opts); + if (suspend) { + ERTS_BIF_PREP_YIELD3(retval, bif_export[BIF_send_3], p, to, msg, opts); } else { ERTS_BIF_PREP_RET(retval, am_nosuspend); } break; case SEND_YIELD_RETURN: - if (!ctx->suspend) { + if (!suspend) { ERTS_BIF_PREP_RET(retval, am_nosuspend); break; } @@ -2169,9 +2191,7 @@ BIF_RETTYPE send_3(BIF_ALIST_3) break; case SEND_YIELD_CONTINUE: BUMP_ALL_REDS(p); - erts_set_gc_state(p, 0); - ERTS_BIF_PREP_TRAP1(retval, &dsend_continue_trap_export, p, - erts_dsend_export_trap_context(p, ctx)); + ERTS_BIF_PREP_TRAP1(retval, &dsend_continue_trap_export, p, ctx); break; default: erts_exit(ERTS_ABORT_EXIT, "send_3 invalid result %d\n", (int)result); @@ -2179,7 +2199,6 @@ BIF_RETTYPE send_3(BIF_ALIST_3) } done: - UnUseTmpHeap(sizeof(ErtsSendContext)/sizeof(Eterm), BIF_P); return retval; } @@ -2193,14 +2212,14 @@ BIF_RETTYPE send_2(BIF_ALIST_2) static BIF_RETTYPE dsend_continue_trap_1(BIF_ALIST_1) { Binary* bin = erts_magic_ref2bin(BIF_ARG_1); - ErtsSendContext* ctx = (ErtsSendContext*) ERTS_MAGIC_BIN_DATA(bin); + ErtsDSigSendContext *ctx = (ErtsDSigSendContext*) ERTS_MAGIC_BIN_DATA(bin); Sint initial_reds = (Sint) (ERTS_BIF_REDS_LEFT(BIF_P) * TERM_TO_BINARY_LOOP_FACTOR); int result; ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == erts_dsend_context_dtor); - ctx->dss.reds = initial_reds; - result = erts_dsig_send(&ctx->dsd, &ctx->dss); + ctx->reds = initial_reds; + result = erts_dsig_send(ctx); switch (result) { case ERTS_DSIG_SEND_OK: @@ -2209,7 +2228,7 @@ static BIF_RETTYPE dsend_continue_trap_1(BIF_ALIST_1) break; case ERTS_DSIG_SEND_YIELD: /*SEND_YIELD_RETURN*/ erts_set_gc_state(BIF_P, 1); - if (!ctx->suspend) + if (ctx->no_suspend) BIF_RET(am_nosuspend); ERTS_BIF_YIELD_RETURN(BIF_P, ctx->return_term); @@ -2234,20 +2253,14 @@ Eterm erl_send(Process *p, Eterm to, Eterm msg) Eterm retval; Eterm ref; Sint result; - DeclareTypedTmpHeap(ErtsSendContext, ctx, p); + Eterm ctx; ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_SEND); - UseTmpHeap(sizeof(ErtsSendContext)/sizeof(Eterm), p); + #ifdef DEBUG ref = NIL; #endif - ctx->suspend = !0; - ctx->connect = !0; - ctx->deref_dep = 0; - ctx->return_term = msg; - ctx->dss.reds = (Sint) (ERTS_BIF_REDS_LEFT(p) * TERM_TO_BINARY_LOOP_FACTOR); - ctx->dss.phase = ERTS_DSIG_SEND_PHASE_INIT; - result = do_send(p, to, msg, &ref, ctx); + result = do_send(p, to, msg, msg, &ref, &ctx, 1, 1); ERTS_MSACC_POP_STATE_M_X(); @@ -2289,9 +2302,7 @@ Eterm erl_send(Process *p, Eterm to, Eterm msg) break; case SEND_YIELD_CONTINUE: BUMP_ALL_REDS(p); - erts_set_gc_state(p, 0); - ERTS_BIF_PREP_TRAP1(retval, &dsend_continue_trap_export, p, - erts_dsend_export_trap_context(p, ctx)); + ERTS_BIF_PREP_TRAP1(retval, &dsend_continue_trap_export, p, ctx); break; default: erts_exit(ERTS_ABORT_EXIT, "invalid send result %d\n", (int)result); @@ -2299,7 +2310,6 @@ Eterm erl_send(Process *p, Eterm to, Eterm msg) } done: - UnUseTmpHeap(sizeof(ErtsSendContext)/sizeof(Eterm), p); return retval; } @@ -4053,10 +4063,12 @@ BIF_RETTYPE list_to_pid_1(BIF_ALIST_1) if (is_nil(dep->cid)) goto bad; - enp = erts_find_or_insert_node(dep->sysname, dep->creation); + etp = (ExternalThing *) HAlloc(BIF_P, EXTERNAL_THING_HEAD_SIZE + 1); + + enp = erts_find_or_insert_node(dep->sysname, dep->creation, + make_boxed(&etp->header)); ASSERT(enp != erts_this_node); - etp = (ExternalThing *) HAlloc(BIF_P, EXTERNAL_THING_HEAD_SIZE + 1); etp->header = make_external_pid_header(1); etp->next = MSO(BIF_P).first; etp->node = enp; @@ -4120,10 +4132,11 @@ BIF_RETTYPE list_to_port_1(BIF_ALIST_1) if (is_nil(dep->cid)) goto bad; - enp = erts_find_or_insert_node(dep->sysname, dep->creation); + etp = (ExternalThing *) HAlloc(BIF_P, EXTERNAL_THING_HEAD_SIZE + 1); + enp = erts_find_or_insert_node(dep->sysname, dep->creation, + make_boxed(&etp->header)); ASSERT(enp != erts_this_node); - etp = (ExternalThing *) HAlloc(BIF_P, EXTERNAL_THING_HEAD_SIZE + 1); etp->header = make_external_port_header(1); etp->next = MSO(BIF_P).first; etp->node = enp; @@ -4226,9 +4239,6 @@ BIF_RETTYPE list_to_ref_1(BIF_ALIST_1) if (is_nil(dep->cid)) goto bad; - enp = erts_find_or_insert_node(dep->sysname, dep->creation); - ASSERT(enp != erts_this_node); - hsz = EXTERNAL_THING_HEAD_SIZE; #if defined(ARCH_64) hsz += n/2 + 1; @@ -4237,6 +4247,11 @@ BIF_RETTYPE list_to_ref_1(BIF_ALIST_1) #endif etp = (ExternalThing *) HAlloc(BIF_P, hsz); + + enp = erts_find_or_insert_node(dep->sysname, dep->creation, + make_boxed(&etp->header)); + ASSERT(enp != erts_this_node); + etp->header = make_external_ref_header(n/2); etp->next = BIF_P->off_heap.first; etp->node = enp; @@ -4356,21 +4371,21 @@ BIF_RETTYPE erts_internal_group_leader_2(BIF_ALIST_2) if (is_external_pid(BIF_ARG_2)) { DistEntry *dep; int code; - ErtsDSigData dsd; + ErtsDSigSendContext ctx; dep = external_pid_dist_entry(BIF_ARG_2); ERTS_ASSERT(dep); if(dep == erts_this_dist_entry) BIF_ERROR(BIF_P, BADARG); - code = erts_dsig_prepare(&dsd, dep, BIF_P, ERTS_PROC_LOCK_MAIN, - ERTS_DSP_NO_LOCK, 0, 1); + code = erts_dsig_prepare(&ctx, dep, BIF_P, ERTS_PROC_LOCK_MAIN, + ERTS_DSP_NO_LOCK, 0, 1, 1); switch (code) { case ERTS_DSIG_PREP_NOT_ALIVE: case ERTS_DSIG_PREP_NOT_CONNECTED: BIF_RET(am_true); case ERTS_DSIG_PREP_PENDING: case ERTS_DSIG_PREP_CONNECTED: - code = erts_dsig_send_group_leader(&dsd, BIF_ARG_1, BIF_ARG_2); + code = erts_dsig_send_group_leader(&ctx, BIF_ARG_1, BIF_ARG_2); if (code == ERTS_DSIG_SEND_YIELD) ERTS_BIF_YIELD_RETURN(BIF_P, am_true); BIF_RET(am_true); diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 273b598562..27bf2187c2 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -129,7 +129,7 @@ typedef struct { void *to_arg; } PrintMonitorContext; -static void doit_print_link(ErtsLink *lnk, void *vpcontext) +static int doit_print_link(ErtsLink *lnk, void *vpcontext, Sint reds) { PrintMonitorContext *pcontext = vpcontext; fmtfn_t to = pcontext->to; @@ -141,10 +141,11 @@ static void doit_print_link(ErtsLink *lnk, void *vpcontext) } else { erts_print(to, to_arg, ", %T", lnk->other.item); } + return 1; } -static void doit_print_monitor(ErtsMonitor *mon, void *vpcontext) +static int doit_print_monitor(ErtsMonitor *mon, void *vpcontext, Sint reds) { ErtsMonitorData *mdp; PrintMonitorContext *pcontext = vpcontext; @@ -196,6 +197,7 @@ static void doit_print_monitor(ErtsMonitor *mon, void *vpcontext) /* ignore other monitors... */ break; } + return 1; } /* Display info about an individual Erlang process */ diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index e7bd046e18..db74b06cc5 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -854,7 +854,7 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint case EXTERNAL_REF_SUBTAG: { ExternalThing *etp = (ExternalThing *) objp; - erts_refc_inc(&etp->node->refc, 2); + erts_ref_node_entry(etp->node, 2, make_boxed(htop)); } L_off_heap_node_container_common: { @@ -1660,7 +1660,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, case EXTERNAL_REF_SUBTAG: { ExternalThing *etp = (ExternalThing *) ptr; - erts_refc_inc(&etp->node->refc, 2); + erts_ref_node_entry(etp->node, 2, make_boxed(hp)); } off_heap_node_container_common: { @@ -1866,7 +1866,7 @@ Eterm copy_shallow(Eterm* ERTS_RESTRICT ptr, Uint sz, Eterm** hpp, case EXTERNAL_REF_SUBTAG: { ExternalThing* etp = (ExternalThing *) (tp-1); - erts_refc_inc(&etp->node->refc, 2); + erts_ref_node_entry(etp->node, 2, make_boxed(hp-1)); } off_heap_common: { diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index 15642e1669..b50c8273b1 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -50,19 +50,26 @@ #define DIST_CTL_DEFAULT_SIZE 64 /* Turn this on to get printouts of all distribution messages - * which go on the line + * which go on the line. Enabling this may make some testcases + * fail. Especially the broken dist testcases in distribution_SUITE. */ #if 0 #define ERTS_DIST_MSG_DBG +FILE *dbg_file; #endif #if 0 +/* Enable this to print the dist debug messages to a file instead */ +#define ERTS_DIST_MSG_DBG_FILE "/tmp/dist_dbg.%d" +#endif +#if 0 +/* Enable this to print the raw bytes sent and received */ #define ERTS_RAW_DIST_MSG_DBG #endif #if defined(ERTS_DIST_MSG_DBG) || defined(ERTS_RAW_DIST_MSG_DBG) static void bw(byte *buf, ErlDrvSizeT sz) { - bin_write(ERTS_PRINT_STDERR, NULL, buf, sz); + bin_write(ERTS_PRINT_FILE, dbg_file, buf, sz); } #endif @@ -70,39 +77,93 @@ static void bw(byte *buf, ErlDrvSizeT sz) static void dist_msg_dbg(ErtsDistExternal *edep, char *what, byte *buf, int sz) { - ErtsHeapFactory factory; - DeclareTmpHeapNoproc(ctl_default,DIST_CTL_DEFAULT_SIZE); - Eterm* ctl = ctl_default; - byte *extp = edep->extp; + byte *extp = edep->data->extp; Eterm msg; Sint ctl_len; - Sint size = ctl_len = erts_decode_dist_ext_size(edep); + Sint size = ctl_len = erts_decode_dist_ext_size(edep, 0); if (size < 0) { - erts_fprintf(stderr, + erts_fprintf(dbg_file, "DIST MSG DEBUG: erts_decode_dist_ext_size(%s) failed:\n", what); bw(buf, sz); } else { - ErlHeapFragment *mbuf = new_message_buffer(size); - erts_factory_static_init(&factory, ctl, ctl_len, &mbuf->off_heap); - msg = erts_decode_dist_ext(&factory, edep); + ErtsHeapFactory factory; + ErtsMessage *mbuf = erts_factory_message_create(&factory, NULL, 0, ctl_len); + /* Set mbuf msg to NIL as erts_factory_undo will fail otherwise */ + ERL_MESSAGE_TERM(mbuf) = NIL; + msg = erts_decode_dist_ext(&factory, edep, 0); if (is_value(msg)) - erts_fprintf(stderr, " %s: %T\n", what, msg); + erts_fprintf(dbg_file, " %s: %.80T\n", what, msg); else { - erts_fprintf(stderr, + erts_fprintf(dbg_file, "DIST MSG DEBUG: erts_decode_dist_ext(%s) failed:\n", what); bw(buf, sz); } - free_message_buffer(mbuf); - edep->extp = extp; + erts_factory_undo(&factory); + edep->data->extp = extp; } } +static char *erts_dop_to_string(enum dop dop) { + if (dop == DOP_LINK) + return "LINK"; + if (dop == DOP_SEND) + return "SEND"; + if (dop == DOP_EXIT) + return "EXIT"; + if (dop == DOP_UNLINK) + return "UNLINK"; + if (dop == DOP_REG_SEND) + return "REG_SEND"; + if (dop == DOP_GROUP_LEADER) + return "GROUP_LEADER"; + if (dop == DOP_EXIT2) + return "EXIT2"; + if (dop == DOP_SEND_TT) + return "SEND_TT"; + if (dop == DOP_EXIT_TT) + return "EXIT_TT"; + if (dop == DOP_REG_SEND_TT) + return "REG_SEND_TT"; + if (dop == DOP_EXIT2_TT) + return "EXIT2_TT"; + if (dop == DOP_MONITOR_P) + return "MONITOR_P"; + if (dop == DOP_DEMONITOR_P) + return "DEMONITOR_P"; + if (dop == DOP_MONITOR_P_EXIT) + return "MONITOR_P_EXIT"; + if (dop == DOP_SEND_SENDER) + return "SEND_SENDER"; + if (dop == DOP_SEND_SENDER_TT) + return "SEND_SENDER_TT"; + if (dop == DOP_PAYLOAD_EXIT) + return "PAYLOAD_EXIT"; + if (dop == DOP_PAYLOAD_EXIT_TT) + return "PAYLOAD_EXIT_TT"; + if (dop == DOP_PAYLOAD_EXIT2) + return "PAYLOAD_EXIT2"; + if (dop == DOP_PAYLOAD_EXIT2_TT) + return "PAYLOAD_EXIT2_TT"; + if (dop == DOP_PAYLOAD_MONITOR_P_EXIT) + return "PAYLOAD_MONITOR_P_EXIT"; + ASSERT(0); + return "UNKNOWN"; +} + #endif +#if defined(VALGRIND) +#include <valgrind/valgrind.h> +#include <valgrind/memcheck.h> +# define PURIFY_MSG(msg) \ + VALGRIND_PRINTF("%s, line %d: %s", __FILE__, __LINE__, msg) +#else +# define PURIFY_MSG(msg) +#endif int erts_is_alive; /* System must be blocked on change */ int erts_dist_buf_busy_limit; @@ -116,12 +177,17 @@ static Export *dist_ctrl_put_data_trap; /* forward declarations */ -static int dsig_send_ctl(ErtsDSigData* dsdp, Eterm ctl, int force_busy); +static void erts_schedule_dist_command(Port *, DistEntry *); +static int dsig_send_exit(ErtsDSigSendContext *ctx, Eterm ctl, Eterm msg); +static int dsig_send_ctl(ErtsDSigSendContext *ctx, Eterm ctl); static void send_nodes_mon_msgs(Process *, Eterm, Eterm, Eterm, Eterm); static void init_nodes_monitors(void); static Sint abort_connection(DistEntry* dep, Uint32 conn_id); static ErtsDistOutputBuf* clear_de_out_queues(DistEntry*); static void free_de_out_queues(DistEntry*, ErtsDistOutputBuf*); +int erts_dist_seq_tree_foreach_delete_yielding(DistSeqNode **root, + void **vyspp, + Sint limit); static erts_atomic_t no_caches; static erts_atomic_t no_nodes; @@ -142,7 +208,6 @@ delete_cache(ErtsAtomCache *cache) } } - static void create_cache(DistEntry *dep) { @@ -185,32 +250,36 @@ get_suspended_on_de(DistEntry *dep, erts_aint32_t unset_qflgs) } } -#define ERTS_MON_LNK_FIRE_LIMIT 100 +#define ERTS_MON_LNK_FIRE_REDS 40 -static void monitor_connection_down(ErtsMonitor *mon, void *unused) +static int monitor_connection_down(ErtsMonitor *mon, void *unused, Sint reds) { if (erts_monitor_is_origin(mon)) erts_proc_sig_send_demonitor(mon); else erts_proc_sig_send_monitor_down(mon, am_noconnection); + return ERTS_MON_LNK_FIRE_REDS; } -static void link_connection_down(ErtsLink *lnk, void *vdist) +static int link_connection_down(ErtsLink *lnk, void *vdist, Sint reds) { erts_proc_sig_send_link_exit(NULL, THE_NON_VALUE, lnk, am_noconnection, NIL); + return ERTS_MON_LNK_FIRE_REDS; } typedef enum { ERTS_CML_CLEANUP_STATE_LINKS, ERTS_CML_CLEANUP_STATE_MONITORS, ERTS_CML_CLEANUP_STATE_ONAME_MONITORS, + ERTS_CML_CLEANUP_STATE_SEQUENCES, ERTS_CML_CLEANUP_STATE_NODE_MONITORS -} ErtsConMonLnkCleaupState; +} ErtsConMonLnkSeqCleanupState; typedef struct { - ErtsConMonLnkCleaupState state; + ErtsConMonLnkSeqCleanupState state; ErtsMonLnkDist *dist; + DistSeqNode *seq; void *yield_state; int trigger_node_monitors; Eterm nodename; @@ -218,49 +287,58 @@ typedef struct { Eterm reason; ErlOffHeap oh; Eterm heap[1]; -} ErtsConMonLnkCleanup; +} ErtsConMonLnkSeqCleanup; static void -con_monitor_link_cleanup(void *vcmlcp) +con_monitor_link_seq_cleanup(void *vcmlcp) { - ErtsConMonLnkCleanup *cmlcp = vcmlcp; + ErtsConMonLnkSeqCleanup *cmlcp = vcmlcp; ErtsMonLnkDist *dist = cmlcp->dist; ErtsSchedulerData *esdp; - int yield; + int reds = CONTEXT_REDS; switch (cmlcp->state) { case ERTS_CML_CLEANUP_STATE_LINKS: - yield = erts_link_list_foreach_delete_yielding(&dist->links, - link_connection_down, - NULL, &cmlcp->yield_state, - ERTS_MON_LNK_FIRE_LIMIT); - if (yield) + reds = erts_link_list_foreach_delete_yielding(&dist->links, + link_connection_down, + NULL, &cmlcp->yield_state, + reds); + if (reds <= 0) break; ASSERT(!cmlcp->yield_state); cmlcp->state = ERTS_CML_CLEANUP_STATE_MONITORS; case ERTS_CML_CLEANUP_STATE_MONITORS: - yield = erts_monitor_list_foreach_delete_yielding(&dist->monitors, - monitor_connection_down, - NULL, &cmlcp->yield_state, - ERTS_MON_LNK_FIRE_LIMIT); - if (yield) + reds = erts_monitor_list_foreach_delete_yielding(&dist->monitors, + monitor_connection_down, + NULL, &cmlcp->yield_state, + reds); + if (reds <= 0) break; ASSERT(!cmlcp->yield_state); cmlcp->state = ERTS_CML_CLEANUP_STATE_ONAME_MONITORS; case ERTS_CML_CLEANUP_STATE_ONAME_MONITORS: - yield = erts_monitor_tree_foreach_delete_yielding(&dist->orig_name_monitors, - monitor_connection_down, - NULL, &cmlcp->yield_state, - ERTS_MON_LNK_FIRE_LIMIT/2); - if (yield) + reds = erts_monitor_tree_foreach_delete_yielding(&dist->orig_name_monitors, + monitor_connection_down, + NULL, &cmlcp->yield_state, + reds); + if (reds <= 0) break; cmlcp->dist = NULL; erts_mon_link_dist_dec_refc(dist); ASSERT(!cmlcp->yield_state); + cmlcp->state = ERTS_CML_CLEANUP_STATE_SEQUENCES; + case ERTS_CML_CLEANUP_STATE_SEQUENCES: + reds = erts_dist_seq_tree_foreach_delete_yielding(&cmlcp->seq, + &cmlcp->yield_state, + reds); + if (reds <= 0) + break; + + ASSERT(!cmlcp->yield_state); cmlcp->state = ERTS_CML_CLEANUP_STATE_NODE_MONITORS; case ERTS_CML_CLEANUP_STATE_NODE_MONITORS: if (cmlcp->trigger_node_monitors) { @@ -280,22 +358,23 @@ con_monitor_link_cleanup(void *vcmlcp) esdp = erts_get_scheduler_data(); ASSERT(esdp && esdp->type == ERTS_SCHED_NORMAL); erts_schedule_misc_aux_work((int) esdp->no, - con_monitor_link_cleanup, + con_monitor_link_seq_cleanup, (void *) cmlcp); } static void -schedule_con_monitor_link_cleanup(ErtsMonLnkDist *dist, - Eterm nodename, - Eterm visability, - Eterm reason) +schedule_con_monitor_link_seq_cleanup(ErtsMonLnkDist *dist, + DistSeqNode *seq, + Eterm nodename, + Eterm visability, + Eterm reason) { - if (dist || is_value(nodename)) { + if (dist || is_value(nodename) || seq) { ErtsSchedulerData *esdp; - ErtsConMonLnkCleanup *cmlcp; + ErtsConMonLnkSeqCleanup *cmlcp; Uint rsz, size; - size = sizeof(ErtsConMonLnkCleanup); + size = sizeof(ErtsConMonLnkSeqCleanup); if (is_non_value(reason) || is_immed(reason)) { rsz = 0; @@ -322,6 +401,8 @@ schedule_con_monitor_link_cleanup(ErtsMonLnkDist *dist, erts_mtx_unlock(&dist->mtx); } + cmlcp->seq = seq; + cmlcp->trigger_node_monitors = is_value(nodename); cmlcp->nodename = nodename; cmlcp->visability = visability; @@ -335,13 +416,13 @@ schedule_con_monitor_link_cleanup(ErtsMonLnkDist *dist, esdp = erts_get_scheduler_data(); ASSERT(esdp && esdp->type == ERTS_SCHED_NORMAL); erts_schedule_misc_aux_work((int) esdp->no, - con_monitor_link_cleanup, + con_monitor_link_seq_cleanup, (void *) cmlcp); } } /* -** A full node name constists of a "n@h" +** A full node name consists of a "n@h" ** ** n must be a valid node name: string of ([a-z][A-Z][0-9]_-)+ ** @@ -558,6 +639,7 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason) } else { /* Call from distribution controller (port/process) */ ErtsMonLnkDist *mld; + DistSeqNode *sequences; ErtsAtomCache *cache; ErtsProcList *suspendees; ErtsDistOutputBuf *obuf; @@ -587,6 +669,9 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason) mld = dep->mld; dep->mld = NULL; + sequences = dep->sequences; + dep->sequences = NULL; + nodename = dep->sysname; flags = dep->flags; @@ -610,14 +695,15 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason) erts_de_rwunlock(dep); - schedule_con_monitor_link_cleanup(mld, - nodename, - (flags & DFLAG_PUBLISHED - ? am_visible - : am_hidden), - (reason == am_normal - ? am_connection_closed - : reason)); + schedule_con_monitor_link_seq_cleanup(mld, + sequences, + nodename, + (flags & DFLAG_PUBLISHED + ? am_visible + : am_hidden), + (reason == am_normal + ? am_connection_closed + : reason)); erts_resume_processes(suspendees); @@ -651,6 +737,16 @@ void init_dist(void) { init_nodes_monitors(); +#ifdef ERTS_DIST_MSG_DBG_FILE + { + char buff[255]; + sprintf(buff, ERTS_DIST_MSG_DBG_FILE, getpid()); + dbg_file = fopen(buff,"w+"); + } +#elif defined (ERTS_DIST_MSG_DBG) + dbg_file = stderr; +#endif + nodedown.reason = NIL; nodedown.bp = NULL; @@ -674,21 +770,79 @@ void init_dist(void) } } -#define ErtsDistOutputBuf2Binary(OB) \ - ((Binary *) (((char *) (OB)) - offsetof(Binary, orig_bytes))) +#define ErtsDistOutputBuf2Binary(OB) OB->bin + +#ifdef DEBUG + +struct obuf_list; +struct obuf_list { + erts_refc_t refc; + struct obuf_list *next; + struct obuf_list *prev; +}; +#define obuf_list_size sizeof(struct obuf_list) +static struct obuf_list *erts_obuf_list = NULL; +static erts_mtx_t erts_obuf_list_mtx; + +static void +insert_obuf(struct obuf_list *obuf, erts_aint_t initial) { + erts_mtx_lock(&erts_obuf_list_mtx); + obuf->next = erts_obuf_list; + obuf->prev = NULL; + erts_refc_init(&obuf->refc, initial); + if (erts_obuf_list) + erts_obuf_list->prev = obuf; + erts_obuf_list = obuf; + erts_mtx_unlock(&erts_obuf_list_mtx); +} + +static void +remove_obuf(struct obuf_list *obuf) { + if (erts_refc_dectest(&obuf->refc, 0) == 0) { + erts_mtx_lock(&erts_obuf_list_mtx); + if (obuf->prev) { + obuf->prev->next = obuf->next; + } else { + erts_obuf_list = obuf->next; + } + if (obuf->next) obuf->next->prev = obuf->prev; + erts_mtx_unlock(&erts_obuf_list_mtx); + } +} + +void check_obuf(void); +void check_obuf(void) { + erts_mtx_lock(&erts_obuf_list_mtx); + ERTS_ASSERT(erts_obuf_list == NULL); + erts_mtx_unlock(&erts_obuf_list_mtx); +} +#else +#define insert_obuf(...) +#define remove_obuf(...) +#define obuf_list_size 0 +#endif static ERTS_INLINE ErtsDistOutputBuf * -alloc_dist_obuf(Uint size) +alloc_dist_obuf(Uint size, Uint headers) { + int i; ErtsDistOutputBuf *obuf; - Uint obuf_size = sizeof(ErtsDistOutputBuf)+sizeof(byte)*(size-1); + Uint obuf_size = sizeof(ErtsDistOutputBuf)*(headers) + + sizeof(byte)*size + obuf_list_size; Binary *bin = erts_bin_drv_alloc(obuf_size); - obuf = (ErtsDistOutputBuf *) &bin->orig_bytes[0]; + size += obuf_list_size; + obuf = (ErtsDistOutputBuf *) &bin->orig_bytes[size]; + erts_refc_add(&bin->intern.refc, headers - 1, 1); + for (i = 0; i < headers; i++) { + obuf[i].bin = bin; + obuf[i].extp = (byte *)&bin->orig_bytes[0] + obuf_list_size; #ifdef DEBUG - obuf->dbg_pattern = ERTS_DIST_OUTPUT_BUF_DBG_PATTERN; - obuf->alloc_endp = obuf->data + size; - ASSERT(bin == ErtsDistOutputBuf2Binary(obuf)); + obuf[i].dbg_pattern = ERTS_DIST_OUTPUT_BUF_DBG_PATTERN; + obuf[i].alloc_endp = obuf->extp + size; + ASSERT(bin == ErtsDistOutputBuf2Binary(obuf)); #endif + } + insert_obuf((struct obuf_list*)&bin->orig_bytes[0], headers); return obuf; } @@ -697,14 +851,17 @@ free_dist_obuf(ErtsDistOutputBuf *obuf) { Binary *bin = ErtsDistOutputBuf2Binary(obuf); ASSERT(obuf->dbg_pattern == ERTS_DIST_OUTPUT_BUF_DBG_PATTERN); - erts_bin_release(bin); + remove_obuf((struct obuf_list*)&bin->orig_bytes[0]); + if (erts_refc_dectest(&bin->intern.refc, 0) == 0) { + erts_bin_free(bin); + } } static ERTS_INLINE Sint size_obuf(ErtsDistOutputBuf *obuf) { - Binary *bin = ErtsDistOutputBuf2Binary(obuf); - return bin->orig_size; + return sizeof(ErtsDistOutputBuf) + (obuf->ext_endp - obuf->ext_start) + + (obuf->hdr_endp - obuf->hdrp); } static ErtsDistOutputBuf* clear_de_out_queues(DistEntry* dep) @@ -758,18 +915,20 @@ static void free_de_out_queues(DistEntry* dep, ErtsDistOutputBuf *obuf) int erts_dsend_context_dtor(Binary* ctx_bin) { - ErtsSendContext* ctx = ERTS_MAGIC_BIN_DATA(ctx_bin); - switch (ctx->dss.phase) { + ErtsDSigSendContext* ctx = ERTS_MAGIC_BIN_DATA(ctx_bin); + switch (ctx->phase) { case ERTS_DSIG_SEND_PHASE_MSG_SIZE: - DESTROY_SAVED_WSTACK(&ctx->dss.u.sc.wstack); + DESTROY_SAVED_WSTACK(&ctx->u.sc.wstack); break; case ERTS_DSIG_SEND_PHASE_MSG_ENCODE: - DESTROY_SAVED_WSTACK(&ctx->dss.u.ec.wstack); + DESTROY_SAVED_WSTACK(&ctx->u.ec.wstack); break; default:; } - if (ctx->dss.phase >= ERTS_DSIG_SEND_PHASE_ALLOC && ctx->dss.obuf) { - free_dist_obuf(ctx->dss.obuf); + if (ctx->phase >= ERTS_DSIG_SEND_PHASE_ALLOC && ctx->obuf) { + int i; + for (i = 0; i < ctx->fragments; i++) + free_dist_obuf(&ctx->obuf[i]); } if (ctx->deref_dep) erts_deref_dist_entry(ctx->dep); @@ -777,10 +936,10 @@ int erts_dsend_context_dtor(Binary* ctx_bin) return 1; } -Eterm erts_dsend_export_trap_context(Process* p, ErtsSendContext* ctx) +Eterm erts_dsend_export_trap_context(Process* p, ErtsDSigSendContext* ctx) { struct exported_ctx { - ErtsSendContext ctx; + ErtsDSigSendContext ctx; ErtsAtomCacheMap acm; }; Binary* ctx_bin = erts_create_magic_binary(sizeof(struct exported_ctx), @@ -788,12 +947,12 @@ Eterm erts_dsend_export_trap_context(Process* p, ErtsSendContext* ctx) struct exported_ctx* dst = ERTS_MAGIC_BIN_DATA(ctx_bin); Eterm* hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE); - sys_memcpy(&dst->ctx, ctx, sizeof(ErtsSendContext)); - ASSERT(ctx->dss.ctl == make_tuple(ctx->ctl_heap)); - dst->ctx.dss.ctl = make_tuple(dst->ctx.ctl_heap); - if (ctx->dss.acmp) { - sys_memcpy(&dst->acm, ctx->dss.acmp, sizeof(ErtsAtomCacheMap)); - dst->ctx.dss.acmp = &dst->acm; + sys_memcpy(&dst->ctx, ctx, sizeof(ErtsDSigSendContext)); + ASSERT(ctx->ctl == make_tuple(ctx->ctl_heap)); + dst->ctx.ctl = make_tuple(dst->ctx.ctl_heap); + if (ctx->acmp) { + sys_memcpy(&dst->acm, ctx->acmp, sizeof(ErtsAtomCacheMap)); + dst->ctx.acmp = &dst->acm; } return erts_mk_magic_ref(&hp, &MSO(p), ctx_bin); } @@ -814,71 +973,58 @@ Eterm erts_dsend_export_trap_context(Process* p, ErtsSendContext* ctx) ** Send a DOP_LINK link message */ int -erts_dsig_send_link(ErtsDSigData *dsdp, Eterm local, Eterm remote) +erts_dsig_send_link(ErtsDSigSendContext *ctx, Eterm local, Eterm remote) { - DeclareTmpHeapNoproc(ctl_heap,4); - Eterm ctl = TUPLE3(&ctl_heap[0], make_small(DOP_LINK), local, remote); - int res; - UseTmpHeapNoproc(4); - - res = dsig_send_ctl(dsdp, ctl, 0); - UnUseTmpHeapNoproc(4); - return res; + Eterm ctl = TUPLE3(&ctx->ctl_heap[0], make_small(DOP_LINK), local, remote); + return dsig_send_ctl(ctx, ctl); } int -erts_dsig_send_unlink(ErtsDSigData *dsdp, Eterm local, Eterm remote) +erts_dsig_send_unlink(ErtsDSigSendContext *ctx, Eterm local, Eterm remote) { - DeclareTmpHeapNoproc(ctl_heap,4); - Eterm ctl = TUPLE3(&ctl_heap[0], make_small(DOP_UNLINK), local, remote); - int res; - - UseTmpHeapNoproc(4); - res = dsig_send_ctl(dsdp, ctl, 0); - UnUseTmpHeapNoproc(4); - return res; + Eterm ctl = TUPLE3(&ctx->ctl_heap[0], make_small(DOP_UNLINK), local, remote); + return dsig_send_ctl(ctx, ctl); } /* A local process that's being monitored by a remote one exits. We send: {DOP_MONITOR_P_EXIT, Local pid or name, Remote pid, ref, reason} */ int -erts_dsig_send_m_exit(ErtsDSigData *dsdp, Eterm watcher, Eterm watched, - Eterm ref, Eterm reason) +erts_dsig_send_m_exit(ErtsDSigSendContext *ctx, Eterm watcher, Eterm watched, + Eterm ref, Eterm reason) { - Eterm ctl; - DeclareTmpHeapNoproc(ctl_heap,6); - int res; + Eterm ctl, msg; - if (~dsdp->flags & (DFLAG_DIST_MONITOR | DFLAG_DIST_MONITOR_NAME)) { + if (~ctx->flags & (DFLAG_DIST_MONITOR | DFLAG_DIST_MONITOR_NAME)) { /* * Receiver does not support DOP_MONITOR_P_EXIT (see dsig_send_monitor) */ return ERTS_DSIG_SEND_OK; } - UseTmpHeapNoproc(6); - - ctl = TUPLE5(&ctl_heap[0], make_small(DOP_MONITOR_P_EXIT), - watched, watcher, ref, reason); + if (ctx->dep->flags & DFLAG_EXIT_PAYLOAD) { + ctl = TUPLE4(&ctx->ctl_heap[0], make_small(DOP_PAYLOAD_MONITOR_P_EXIT), + watched, watcher, ref); + msg = reason; + } else { + ctl = TUPLE5(&ctx->ctl_heap[0], make_small(DOP_MONITOR_P_EXIT), + watched, watcher, ref, reason); + msg = THE_NON_VALUE; + } - res = dsig_send_ctl(dsdp, ctl, 1); - UnUseTmpHeapNoproc(6); - return res; + return dsig_send_exit(ctx, ctl, msg); } /* We want to monitor a process (named or unnamed) on another node, we send: {DOP_MONITOR_P, Local pid, Remote pid or name, Ref}, which is exactly what's needed on the other side... */ int -erts_dsig_send_monitor(ErtsDSigData *dsdp, Eterm watcher, Eterm watched, +erts_dsig_send_monitor(ErtsDSigSendContext *ctx, Eterm watcher, Eterm watched, Eterm ref) { Eterm ctl; - DeclareTmpHeapNoproc(ctl_heap,5); - int res; - if (~dsdp->flags & (DFLAG_DIST_MONITOR | DFLAG_DIST_MONITOR_NAME)) { + if (~ctx->flags & (DFLAG_DIST_MONITOR | DFLAG_DIST_MONITOR_NAME)) { /* * Receiver does not support DOP_MONITOR_P. * Just avoid sending it and by doing that reduce this monitor @@ -888,48 +1034,40 @@ erts_dsig_send_monitor(ErtsDSigData *dsdp, Eterm watcher, Eterm watched, return ERTS_DSIG_SEND_OK; } - UseTmpHeapNoproc(5); - ctl = TUPLE4(&ctl_heap[0], + ctl = TUPLE4(&ctx->ctl_heap[0], make_small(DOP_MONITOR_P), watcher, watched, ref); - res = dsig_send_ctl(dsdp, ctl, 0); - UnUseTmpHeapNoproc(5); - return res; + return dsig_send_ctl(ctx, ctl); } /* A local process monitoring a remote one wants to stop monitoring, either because of a demonitor bif call or because the local process died. We send {DOP_DEMONITOR_P, Local pid, Remote pid or name, ref} */ int -erts_dsig_send_demonitor(ErtsDSigData *dsdp, Eterm watcher, - Eterm watched, Eterm ref, int force) +erts_dsig_send_demonitor(ErtsDSigSendContext *ctx, Eterm watcher, + Eterm watched, Eterm ref) { Eterm ctl; - DeclareTmpHeapNoproc(ctl_heap,5); - int res; - if (~dsdp->flags & (DFLAG_DIST_MONITOR | DFLAG_DIST_MONITOR_NAME)) { + if (~ctx->flags & (DFLAG_DIST_MONITOR | DFLAG_DIST_MONITOR_NAME)) { /* * Receiver does not support DOP_DEMONITOR_P (see dsig_send_monitor) */ return ERTS_DSIG_SEND_OK; } - UseTmpHeapNoproc(5); - ctl = TUPLE4(&ctl_heap[0], + ctl = TUPLE4(&ctx->ctl_heap[0], make_small(DOP_DEMONITOR_P), watcher, watched, ref); - res = dsig_send_ctl(dsdp, ctl, force); - UnUseTmpHeapNoproc(5); - return res; + return dsig_send_ctl(ctx, ctl); } -static int can_send_seqtrace_token(ErtsSendContext* ctx, Eterm token) { +static int can_send_seqtrace_token(ErtsDSigSendContext* ctx, Eterm token) { Eterm label; - if (ctx->dsd.flags & DFLAG_BIG_SEQTRACE_LABELS) { + if (ctx->flags & DFLAG_BIG_SEQTRACE_LABELS) { /* The other end is capable of handling arbitrary seq_trace labels. */ return 1; } @@ -945,11 +1083,11 @@ static int can_send_seqtrace_token(ErtsSendContext* ctx, Eterm token) { } int -erts_dsig_send_msg(Eterm remote, Eterm message, ErtsSendContext* ctx) +erts_dsig_send_msg(ErtsDSigSendContext* ctx, Eterm remote, Eterm message) { Eterm ctl; Eterm token = NIL; - Process *sender = ctx->dsd.proc; + Process *sender = ctx->c_p; int res; #ifdef USE_VM_PROBES Sint tok_label = 0; @@ -970,7 +1108,7 @@ erts_dsig_send_msg(Eterm remote, Eterm message, ErtsSendContext* ctx) *node_name = *sender_name = *receiver_name = '\0'; if (DTRACE_ENABLED(message_send) || DTRACE_ENABLED(message_send_remote)) { erts_snprintf(node_name, sizeof(DTRACE_CHARBUF_NAME(node_name)), - "%T", ctx->dsd.dep->sysname); + "%T", ctx->dep->sysname); erts_snprintf(sender_name, sizeof(DTRACE_CHARBUF_NAME(sender_name)), "%T", sender->common.id); erts_snprintf(receiver_name, sizeof(DTRACE_CHARBUF_NAME(receiver_name)), @@ -990,7 +1128,7 @@ erts_dsig_send_msg(Eterm remote, Eterm message, ErtsSendContext* ctx) send_token = (token != NIL && can_send_seqtrace_token(ctx, token)); - if (ctx->dsd.flags & DFLAG_SEND_SENDER) { + if (ctx->flags & DFLAG_SEND_SENDER) { dist_op = make_small(send_token ? DOP_SEND_SENDER_TT : DOP_SEND_SENDER); @@ -1013,21 +1151,18 @@ erts_dsig_send_msg(Eterm remote, Eterm message, ErtsSendContext* ctx) msize, tok_label, tok_lastcnt, tok_serial); DTRACE7(message_send_remote, sender_name, node_name, receiver_name, msize, tok_label, tok_lastcnt, tok_serial); - ctx->dss.ctl = ctl; - ctx->dss.msg = message; - ctx->dss.force_busy = 0; - res = erts_dsig_send(&ctx->dsd, &ctx->dss); + ctx->ctl = ctl; + ctx->msg = message; + res = erts_dsig_send(ctx); return res; } int -erts_dsig_send_reg_msg(Eterm remote_name, Eterm message, - ErtsSendContext* ctx) +erts_dsig_send_reg_msg(ErtsDSigSendContext* ctx, Eterm remote_name, Eterm message) { Eterm ctl; Eterm token = NIL; - Process *sender = ctx->dsd.proc; - int res; + Process *sender = ctx->c_p; #ifdef USE_VM_PROBES Sint tok_label = 0; Sint tok_lastcnt = 0; @@ -1047,7 +1182,7 @@ erts_dsig_send_reg_msg(Eterm remote_name, Eterm message, *node_name = *sender_name = *receiver_name = '\0'; if (DTRACE_ENABLED(message_send) || DTRACE_ENABLED(message_send_remote)) { erts_snprintf(node_name, sizeof(DTRACE_CHARBUF_NAME(node_name)), - "%T", ctx->dsd.dep->sysname); + "%T", ctx->dep->sysname); erts_snprintf(sender_name, sizeof(DTRACE_CHARBUF_NAME(sender_name)), "%T", sender->common.id); erts_snprintf(receiver_name, sizeof(DTRACE_CHARBUF_NAME(receiver_name)), @@ -1072,23 +1207,19 @@ erts_dsig_send_reg_msg(Eterm remote_name, Eterm message, msize, tok_label, tok_lastcnt, tok_serial); DTRACE7(message_send_remote, sender_name, node_name, receiver_name, msize, tok_label, tok_lastcnt, tok_serial); - ctx->dss.ctl = ctl; - ctx->dss.msg = message; - ctx->dss.force_busy = 0; - res = erts_dsig_send(&ctx->dsd, &ctx->dss); - return res; + ctx->ctl = ctl; + ctx->msg = message; + return erts_dsig_send(ctx); } /* local has died, deliver the exit signal to remote */ int -erts_dsig_send_exit_tt(ErtsDSigData *dsdp, Eterm local, Eterm remote, +erts_dsig_send_exit_tt(ErtsDSigSendContext *ctx, Eterm local, Eterm remote, Eterm reason, Eterm token) { - Eterm ctl; - DeclareTmpHeapNoproc(ctl_heap,6); - int res; + Eterm ctl, msg = THE_NON_VALUE; #ifdef USE_VM_PROBES - Process *sender = dsdp->proc; + Process *sender = ctx->c_p; Sint tok_label = 0; Sint tok_lastcnt = 0; Sint tok_serial = 0; @@ -1098,20 +1229,29 @@ erts_dsig_send_exit_tt(ErtsDSigData *dsdp, Eterm local, Eterm remote, DTRACE_CHARBUF(reason_str, 128); #endif - UseTmpHeapNoproc(6); + if (ctx->dep->flags & DFLAG_EXIT_PAYLOAD) + msg = reason; + if (have_seqtrace(token)) { - seq_trace_update_send(dsdp->proc); + seq_trace_update_send(ctx->c_p); seq_trace_output_exit(token, reason, SEQ_TRACE_SEND, remote, local); - ctl = TUPLE5(&ctl_heap[0], - make_small(DOP_EXIT_TT), local, remote, token, reason); + if (ctx->dep->flags & DFLAG_EXIT_PAYLOAD) { + ctl = TUPLE4(&ctx->ctl_heap[0], + make_small(DOP_PAYLOAD_EXIT_TT), local, remote, token); + } else + ctl = TUPLE5(&ctx->ctl_heap[0], + make_small(DOP_EXIT_TT), local, remote, token, reason); } else { - ctl = TUPLE4(&ctl_heap[0], make_small(DOP_EXIT), local, remote, reason); + if (ctx->dep->flags & DFLAG_EXIT_PAYLOAD) + ctl = TUPLE3(&ctx->ctl_heap[0], make_small(DOP_PAYLOAD_EXIT), local, remote); + else + ctl = TUPLE4(&ctx->ctl_heap[0], make_small(DOP_EXIT), local, remote, reason); } #ifdef USE_VM_PROBES *node_name = *sender_name = *remote_name = '\0'; if (DTRACE_ENABLED(process_exit_signal_remote)) { erts_snprintf(node_name, sizeof(DTRACE_CHARBUF_NAME(node_name)), - "%T", dsdp->dep->sysname); + "%T", ctx->dep->sysname); erts_snprintf(sender_name, sizeof(DTRACE_CHARBUF_NAME(sender_name)), "%T", sender->common.id); erts_snprintf(remote_name, sizeof(DTRACE_CHARBUF_NAME(remote_name)), @@ -1127,73 +1267,171 @@ erts_dsig_send_exit_tt(ErtsDSigData *dsdp, Eterm local, Eterm remote, #endif DTRACE7(process_exit_signal_remote, sender_name, node_name, remote_name, reason_str, tok_label, tok_lastcnt, tok_serial); - /* forced, i.e ignore busy */ - res = dsig_send_ctl(dsdp, ctl, 1); - UnUseTmpHeapNoproc(6); - return res; + return dsig_send_exit(ctx, ctl, msg); } int -erts_dsig_send_exit(ErtsDSigData *dsdp, Eterm local, Eterm remote, Eterm reason) +erts_dsig_send_exit(ErtsDSigSendContext *ctx, Eterm local, Eterm remote, Eterm reason) { - DeclareTmpHeapNoproc(ctl_heap,5); - int res; - Eterm ctl; + Eterm ctl, msg = ctx->dep->flags & DFLAG_EXIT_PAYLOAD ? reason : THE_NON_VALUE; - UseTmpHeapNoproc(5); - ctl = TUPLE4(&ctl_heap[0], - make_small(DOP_EXIT), local, remote, reason); - /* forced, i.e ignore busy */ - res = dsig_send_ctl(dsdp, ctl, 1); - UnUseTmpHeapNoproc(5); - return res; + if (ctx->dep->flags & DFLAG_EXIT_PAYLOAD) { + ctl = TUPLE3(&ctx->ctl_heap[0], make_small(DOP_PAYLOAD_EXIT), local, remote); + msg = reason; + } else { + ctl = TUPLE4(&ctx->ctl_heap[0], make_small(DOP_EXIT), local, remote, reason); + msg = THE_NON_VALUE; + } + return dsig_send_exit(ctx, ctl, msg); } int -erts_dsig_send_exit2(ErtsDSigData *dsdp, Eterm local, Eterm remote, Eterm reason) +erts_dsig_send_exit2(ErtsDSigSendContext *ctx, Eterm local, Eterm remote, Eterm reason) { - DeclareTmpHeapNoproc(ctl_heap,5); - int res; - Eterm ctl; + Eterm ctl, msg; - UseTmpHeapNoproc(5); - ctl = TUPLE4(&ctl_heap[0], - make_small(DOP_EXIT2), local, remote, reason); + if (ctx->dep->flags & DFLAG_EXIT_PAYLOAD) { + ctl = TUPLE3(&ctx->ctl_heap[0], + make_small(DOP_PAYLOAD_EXIT2), local, remote); + msg = reason; + } else { + ctl = TUPLE4(&ctx->ctl_heap[0], + make_small(DOP_EXIT2), local, remote, reason); + msg = THE_NON_VALUE; + } - res = dsig_send_ctl(dsdp, ctl, 0); - UnUseTmpHeapNoproc(5); - return res; + return dsig_send_exit(ctx, ctl, msg); } int -erts_dsig_send_group_leader(ErtsDSigData *dsdp, Eterm leader, Eterm remote) +erts_dsig_send_group_leader(ErtsDSigSendContext *ctx, Eterm leader, Eterm remote) { - DeclareTmpHeapNoproc(ctl_heap,4); - int res; Eterm ctl; - UseTmpHeapNoproc(4); - ctl = TUPLE3(&ctl_heap[0], + ctl = TUPLE3(&ctx->ctl_heap[0], make_small(DOP_GROUP_LEADER), leader, remote); - res = dsig_send_ctl(dsdp, ctl, 0); - UnUseTmpHeapNoproc(4); - return res; + return dsig_send_ctl(ctx, ctl); } -#if defined(PURIFY) -# define PURIFY_MSG(msg) \ - purify_printf("%s, line %d: %s", __FILE__, __LINE__, msg) -#elif defined(VALGRIND) -#include <valgrind/valgrind.h> -#include <valgrind/memcheck.h> +struct dist_sequences { + ErlHeapFragment hfrag; + struct dist_sequences *parent; + struct dist_sequences *left; + struct dist_sequences *right; + char is_red; -# define PURIFY_MSG(msg) \ - VALGRIND_PRINTF("%s, line %d: %s", __FILE__, __LINE__, msg) -#else -# define PURIFY_MSG(msg) -#endif + Uint64 seq_id; + int cnt; + Sint ctl_len; +}; + +#define ERTS_RBT_PREFIX dist_seq +#define ERTS_RBT_T DistSeqNode +#define ERTS_RBT_KEY_T Uint +#define ERTS_RBT_FLAGS_T int +#define ERTS_RBT_INIT_EMPTY_TNODE(T) \ + do { \ + (T)->parent = NULL; \ + (T)->left = NULL; \ + (T)->right = NULL; \ + (T)->is_red = 0; \ + } while(0) +#define ERTS_RBT_IS_RED(T) ((T)->is_red) +#define ERTS_RBT_SET_RED(T) ((T)->is_red = 1) +#define ERTS_RBT_IS_BLACK(T) (!ERTS_RBT_IS_RED(T)) +#define ERTS_RBT_SET_BLACK(T) ((T)->is_red = 0) +#define ERTS_RBT_GET_FLAGS(T) ((T)->is_red) +#define ERTS_RBT_SET_FLAGS(T, F) ((T)->is_red = F) +#define ERTS_RBT_GET_PARENT(T) ((T)->parent) +#define ERTS_RBT_SET_PARENT(T, P) ((T)->parent = P) +#define ERTS_RBT_GET_RIGHT(T) ((T)->right) +#define ERTS_RBT_SET_RIGHT(T, R) ((T)->right = (R)) +#define ERTS_RBT_GET_LEFT(T) ((T)->left) +#define ERTS_RBT_SET_LEFT(T, L) ((T)->left = (L)) +#define ERTS_RBT_GET_KEY(T) ((T)->seq_id) +#define ERTS_RBT_IS_LT(KX, KY) (KX < KY) +#define ERTS_RBT_IS_EQ(KX, KY) (KX == KY) +#define ERTS_RBT_WANT_DELETE +#define ERTS_RBT_WANT_LOOKUP_INSERT +#define ERTS_RBT_WANT_LOOKUP +#define ERTS_RBT_WANT_FOREACH +#define ERTS_RBT_WANT_FOREACH_DESTROY_YIELDING + +#include "erl_rbtree.h" + +struct erts_dist_seq_tree_foreach_iter_arg { + int (*func)(ErtsDistExternal *, void *, Sint); + void *arg; +}; + +static int +erts_dist_seq_tree_foreach_iter(DistSeqNode *seq, void *arg, Sint reds) +{ + struct erts_dist_seq_tree_foreach_iter_arg *state = arg; + return state->func(erts_get_dist_ext(&seq->hfrag), state->arg, reds); +} + +void +erts_dist_seq_tree_foreach(DistEntry *dep, int (*func)(ErtsDistExternal *, void *, Sint), void *arg) +{ + struct erts_dist_seq_tree_foreach_iter_arg state; + state.func = func; + state.arg = arg; + dist_seq_rbt_foreach(dep->sequences, erts_dist_seq_tree_foreach_iter, &state); +} + +static int dist_seq_cleanup(DistSeqNode *seq, void *unused, Sint reds) +{ + erts_free_dist_ext_copy(erts_get_dist_ext(&seq->hfrag)); + free_message_buffer(&seq->hfrag); + return 1; +} + +typedef struct { + DistSeqNode *root; + dist_seq_rbt_yield_state_t rbt_ystate; +} DistSeqNodeYieldState; + +int +erts_dist_seq_tree_foreach_delete_yielding(DistSeqNode **root, + void **vyspp, + Sint limit) +{ + DistSeqNodeYieldState ys = {*root, ERTS_RBT_YIELD_STAT_INITER}; + DistSeqNodeYieldState *ysp; + int res; + + ysp = (DistSeqNodeYieldState *) *vyspp; + if (!ysp) { + *root = NULL; + ysp = &ys; + } + res = dist_seq_rbt_foreach_destroy_yielding(&ysp->root, + dist_seq_cleanup, + NULL, + &ysp->rbt_ystate, + limit); + if (res > 0) { + if (ysp != &ys) + erts_free(ERTS_ALC_T_ML_YIELD_STATE, ysp); + *vyspp = NULL; + } + else { + + if (ysp == &ys) { + ysp = erts_alloc(ERTS_ALC_T_SEQ_YIELD_STATE, + sizeof(DistSeqNodeYieldState)); + sys_memcpy((void *) ysp, (void *) &ys, + sizeof(DistSeqNodeYieldState)); + } + + *vyspp = (void *) ysp; + } + + return res; +} /* ** Input from distribution port. @@ -1210,10 +1448,13 @@ int erts_net_message(Port *prt, Uint32 conn_id, byte *hbuf, ErlDrvSizeT hlen, + Binary *bin, byte *buf, ErlDrvSizeT len) { - ErtsDistExternal ede; + ErtsDistExternal ede, *edep = &ede; + ErtsDistExternalData ede_data; + ErlHeapFragment *ede_hfrag = NULL; Sint ctl_len; Eterm arg; Eterm from, to; @@ -1225,10 +1466,8 @@ int erts_net_message(Port *prt, DeclareTmpHeapNoproc(ctl_default,DIST_CTL_DEFAULT_SIZE); Eterm* ctl = ctl_default; ErtsHeapFactory factory; - Eterm* hp; Sint type; Eterm token; - Eterm token_size; Uint tuple_arity; int res; #ifdef ERTS_DIST_MSG_DBG @@ -1254,69 +1493,182 @@ int erts_net_message(Port *prt, } #ifdef ERTS_RAW_DIST_MSG_DBG - erts_fprintf(stderr, "<< "); + erts_fprintf(dbg_file, "RECV: "); bw(buf, len); #endif - res = erts_prepare_dist_ext(&ede, buf, len, dep, conn_id, dep->cache); + ede.data = &ede_data; + + res = erts_prepare_dist_ext(&ede, buf, len, bin, dep, conn_id, dep->cache); switch (res) { case ERTS_PREP_DIST_EXT_CLOSED: return 0; /* Connection not alive; ignore signal... */ case ERTS_PREP_DIST_EXT_FAILED: #ifdef ERTS_DIST_MSG_DBG - erts_fprintf(stderr, "DIST MSG DEBUG: erts_prepare_dist_ext() failed:\n"); + erts_fprintf(dbg_file, "DIST MSG DEBUG: erts_prepare_dist_ext() failed:\n"); bw(buf, orig_len); #endif goto data_error; case ERTS_PREP_DIST_EXT_SUCCESS: - ctl_len = erts_decode_dist_ext_size(&ede); + ctl_len = erts_decode_dist_ext_size(&ede, 1); if (ctl_len < 0) { #ifdef ERTS_DIST_MSG_DBG - erts_fprintf(stderr, "DIST MSG DEBUG: erts_decode_dist_ext_size(CTL) failed:\n"); + erts_fprintf(dbg_file, "DIST MSG DEBUG: erts_decode_dist_ext_size(CTL) failed:\n"); bw(buf, orig_len); #endif PURIFY_MSG("data error"); goto data_error; } + + /* A non-fragmented message */ + if (!ede.data->seq_id) { + if (ctl_len > DIST_CTL_DEFAULT_SIZE) { + ctl = erts_alloc(ERTS_ALC_T_DCTRL_BUF, ctl_len * sizeof(Eterm)); + } + + erts_factory_tmp_init(&factory, ctl, ctl_len, ERTS_ALC_T_DCTRL_BUF); + break; + } else { + DistSeqNode *seq; + Uint sz = erts_dist_ext_size(&ede); + Uint used_sz = ctl_len * sizeof(Eterm); + + /* We calculate the size of the heap fragment to be allocated. + The used_size part has to be larger that the ctl data and the + DistSeqNode. */ + if (used_sz + (sizeof(ErlHeapFragment) - sizeof(Eterm)) < sizeof(DistSeqNode)) + used_sz = sizeof(DistSeqNode) - (sizeof(ErlHeapFragment) - sizeof(Eterm)); + + seq = (DistSeqNode *)new_message_buffer((sz + used_sz) / sizeof(Eterm)); + seq->hfrag.used_size = used_sz / sizeof(Eterm); + + seq->ctl_len = ctl_len; + seq->seq_id = ede.data->seq_id; + seq->cnt = ede.data->frag_id; + if (dist_seq_rbt_lookup_insert(&dep->sequences, seq) != NULL) { + free_message_buffer(&seq->hfrag); + goto data_error; + } + + erts_make_dist_ext_copy(&ede, erts_get_dist_ext(&seq->hfrag)); + + if (ede.data->frag_id > 1) { + seq->cnt--; + return 0; + } + } + + /* fall through, the first fragment in the sequence was the last fragment */ + case ERTS_PREP_DIST_EXT_FRAG_CONT: { + DistSeqNode *seq = dist_seq_rbt_lookup(dep->sequences, ede.data->seq_id); + + if (!seq) + goto data_error; + + /* If we did a fall-though we already did this */ + if (res == ERTS_PREP_DIST_EXT_FRAG_CONT) + erts_dist_ext_frag(&ede_data, erts_get_dist_ext(&seq->hfrag)); + + /* Verify that the fragments have arrived in the correct order */ + if (seq->cnt != ede.data->frag_id) + goto data_error; + + seq->cnt--; + + /* Check if this was the last fragment */ + if (ede.data->frag_id > 1) + return 0; + + /* Last fragment arrived, time to dispatch the signal */ + dist_seq_rbt_delete(&dep->sequences, seq); + ctl_len = seq->ctl_len; + + /* Now that we no longer need the DistSeqNode we re-use the heapfragment + to decode the ctl msg into. We don't need the ctl message to be in + the heapfragment, but we decode into the heapfragment speculatively + in case there is a trace token that we need. */ + erts_factory_heap_frag_init(&factory, &seq->hfrag); + edep = erts_get_dist_ext(&seq->hfrag); + ede_hfrag = &seq->hfrag; + + /* If the sequence consisted of more than 1 fragment we create one large + binary out of all of the fragments. This because erts_decode_ext + cannot handle a segmented buffer. + TODO: Move this copy to as late as possible, preferably in in the + erts_decode_dist_ext in the receiving process. + */ + if (edep->data->frag_id > 1) { + Uint sz = 0; + Binary *bin; + int i; + byte *ep; + + for (i = 0; i < edep->data->frag_id; i++) + sz += edep->data[i].ext_endp - edep->data[i].extp; + + bin = erts_bin_nrml_alloc(sz); + ep = (byte*)bin->orig_bytes; + + for (i = 0; i < edep->data->frag_id; i++) { + sys_memcpy(ep, edep->data[i].extp, edep->data[i].ext_endp - edep->data[i].extp); + ep += edep->data[i].ext_endp - edep->data[i].extp; + erts_bin_release(edep->data[i].binp); + edep->data[i].binp = NULL; + edep->data[i].extp = NULL; + edep->data[i].ext_endp = NULL; + } + + edep->data->frag_id = 1; + edep->data->extp = (byte*)bin->orig_bytes; + edep->data->ext_endp = ep; + edep->data->binp = bin; + } + break; + } default: ERTS_INTERNAL_ERROR("Unexpected result from erts_prepare_dist_ext()"); break; } - if (ctl_len > DIST_CTL_DEFAULT_SIZE) { - ctl = erts_alloc(ERTS_ALC_T_DCTRL_BUF, ctl_len * sizeof(Eterm)); - } - hp = ctl; - - erts_factory_tmp_init(&factory, ctl, ctl_len, ERTS_ALC_T_DCTRL_BUF); - arg = erts_decode_dist_ext(&factory, &ede); + arg = erts_decode_dist_ext(&factory, edep, 1); if (is_non_value(arg)) { #ifdef ERTS_DIST_MSG_DBG - erts_fprintf(stderr, "DIST MSG DEBUG: erts_decode_dist_ext(CTL) failed:\n"); + erts_fprintf(dbg_file, "DIST MSG DEBUG: erts_decode_dist_ext(CTL) failed:\n"); bw(buf, orig_len); #endif PURIFY_MSG("data error"); goto decode_error; } -#ifdef ERTS_DIST_MSG_DBG - erts_fprintf(stderr, "<< CTL: %T\n", arg); -#endif + /* Fill the unused part of the hfrag with a bignum header */ + if (ede_hfrag && ede_hfrag->mem + ede_hfrag->used_size > factory.hp) { + Uint slot = factory.hp - ede_hfrag->mem; + ede_hfrag->mem[slot] = make_pos_bignum_header(ede_hfrag->used_size - slot - 1); + } if (is_not_tuple(arg) || (tuple = tuple_val(arg), (tuple_arity = arityval(*tuple)) < 1) || is_not_small(tuple[1])) { +#ifdef ERTS_DIST_MSG_DBG + if (is_tuple(arg) && arityval(*tuple) > 1) + erts_fprintf(dbg_file, "RECV: CTL: %s: %.80T\n", + erts_dop_to_string(unsigned_val(tuple[1])), arg); +#endif goto invalid_message; } - token_size = 0; +#ifdef ERTS_DIST_MSG_DBG + erts_fprintf(dbg_file, "RECV: CTL: %s: %.80T\n", + erts_dop_to_string(unsigned_val(tuple[1])), arg); +#endif + token = NIL; switch (type = unsigned_val(tuple[1])) { case DOP_LINK: { - ErtsDSigData dsd; + ErtsDSigSendContext ctx; int code; if (tuple_arity != 3) { @@ -1354,9 +1706,9 @@ int erts_net_message(Port *prt, erts_link_release_both(ldp); } - code = erts_dsig_prepare(&dsd, dep, NULL, 0, ERTS_DSP_NO_LOCK, 0, 0); + code = erts_dsig_prepare(&ctx, dep, NULL, 0, ERTS_DSP_NO_LOCK, 1, 1, 0); if (code == ERTS_DSIG_PREP_CONNECTED) { - code = erts_dsig_send_exit(&dsd, to, from, am_noproc); + code = erts_dsig_send_exit(&ctx, to, from, am_noproc); ASSERT(code == ERTS_DSIG_SEND_OK); } @@ -1389,7 +1741,7 @@ int erts_net_message(Port *prt, /* A remote process wants to monitor us, we get: {DOP_MONITOR_P, Remote pid, local pid or name, ref} */ Eterm pid, name; - ErtsDSigData dsd; + ErtsDSigSendContext ctx; int code; if (tuple_arity != 4) { @@ -1444,10 +1796,9 @@ int erts_net_message(Port *prt, } - code = erts_dsig_prepare(&dsd, dep, NULL, 0, ERTS_DSP_NO_LOCK, 0, 0); + code = erts_dsig_prepare(&ctx, dep, NULL, 0, ERTS_DSP_NO_LOCK, 1, 1, 0); if (code == ERTS_DSIG_PREP_CONNECTED) { - code = erts_dsig_send_m_exit(&dsd, watcher, watched, ref, - am_noproc); + code = erts_dsig_send_m_exit(&ctx, watcher, watched, ref, am_noproc); ASSERT(code == ERTS_DSIG_SEND_OK); } @@ -1506,7 +1857,6 @@ int erts_net_message(Port *prt, goto invalid_message; } - token_size = size_object(tuple[5]); /* Fall through ... */ case DOP_REG_SEND: /* {DOP_REG_SEND, From, Cookie, ToName} -- Message */ @@ -1521,7 +1871,7 @@ int erts_net_message(Port *prt, } #ifdef ERTS_DIST_MSG_DBG - dist_msg_dbg(&ede, "MSG", buf, orig_len); + dist_msg_dbg(edep, "MSG", buf, orig_len); #endif from = tuple[2]; @@ -1531,35 +1881,25 @@ int erts_net_message(Port *prt, } rp = erts_whereis_process(NULL, 0, to, 0, 0); if (rp) { - Uint xsize = (type == DOP_REG_SEND - ? 0 - : ERTS_HEAP_FRAG_SIZE(token_size)); ErtsProcLocks locks = 0; - ErtsDistExternal *ede_copy; - ede_copy = erts_make_dist_ext_copy(&ede, xsize); if (type == DOP_REG_SEND) { token = NIL; } else { - ErlHeapFragment *heap_frag; - ErlOffHeap *ohp; - ASSERT(xsize); - heap_frag = erts_dist_ext_trailer(ede_copy); - ERTS_INIT_HEAP_FRAG(heap_frag, token_size, token_size); - hp = heap_frag->mem; - ohp = &heap_frag->off_heap; token = tuple[5]; - token = copy_struct(token, token_size, &hp, ohp); } - erts_queue_dist_message(rp, locks, ede_copy, token, from); + erts_queue_dist_message(rp, locks, edep, ede_hfrag, token, from); + if (locks) erts_proc_unlock(rp, locks); - } + } else if (ede_hfrag) { + erts_free_dist_ext_copy(erts_get_dist_ext(ede_hfrag)); + free_message_buffer(ede_hfrag); + } break; case DOP_SEND_SENDER_TT: { - Uint xsize; case DOP_SEND_TT: if (tuple_arity != 4) { @@ -1567,15 +1907,12 @@ int erts_net_message(Port *prt, } token = tuple[4]; - token_size = size_object(token); - xsize = ERTS_HEAP_FRAG_SIZE(token_size); goto send_common; case DOP_SEND_SENDER: case DOP_SEND: token = NIL; - xsize = 0; if (tuple_arity != 3) goto invalid_message; @@ -1591,7 +1928,7 @@ int erts_net_message(Port *prt, : tuple[2] == am_Empty); #ifdef ERTS_DIST_MSG_DBG - dist_msg_dbg(&ede, "MSG", buf, orig_len); + dist_msg_dbg(edep, "MSG", buf, orig_len); #endif to = tuple[3]; if (is_not_pid(to)) { @@ -1600,39 +1937,38 @@ int erts_net_message(Port *prt, rp = erts_proc_lookup(to); if (rp) { ErtsProcLocks locks = 0; - ErtsDistExternal *ede_copy; - - ede_copy = erts_make_dist_ext_copy(&ede, xsize); - if (is_not_nil(token)) { - ErlHeapFragment *heap_frag; - ErlOffHeap *ohp; - ASSERT(xsize); - heap_frag = erts_dist_ext_trailer(ede_copy); - ERTS_INIT_HEAP_FRAG(heap_frag, token_size, token_size); - hp = heap_frag->mem; - ohp = &heap_frag->off_heap; - token = copy_struct(token, token_size, &hp, ohp); - } - erts_queue_dist_message(rp, locks, ede_copy, token, am_Empty); + erts_queue_dist_message(rp, locks, edep, ede_hfrag, token, am_Empty); if (locks) erts_proc_unlock(rp, locks); - } + } else if (ede_hfrag) { + erts_free_dist_ext_copy(erts_get_dist_ext(ede_hfrag)); + free_message_buffer(ede_hfrag); + } break; } + case DOP_PAYLOAD_MONITOR_P_EXIT: case DOP_MONITOR_P_EXIT: { + /* We are monitoring a process on the remote node which dies, we get {DOP_MONITOR_P_EXIT, Remote pid or name, Local pid, ref, reason} */ - - if (tuple_arity != 5) { - goto invalid_message; - } - watched = tuple[2]; /* remote proc or name which died */ - watcher = tuple[3]; + watched = tuple[2]; /* remote proc or name which died */ + watcher = tuple[3]; ref = tuple[4]; - reason = tuple[5]; + + if (type == DOP_PAYLOAD_MONITOR_P_EXIT) { + if (tuple_arity != 4) { + goto invalid_message; + } + reason = THE_NON_VALUE; + } else { + if (tuple_arity != 5) { + goto invalid_message; + } + reason = tuple[5]; + } if (is_not_ref(ref)) goto invalid_message; @@ -1648,70 +1984,125 @@ int erts_net_message(Port *prt, goto invalid_message; } - erts_proc_sig_send_dist_monitor_down(dep, ref, watched, - watcher, reason); + if (!erts_proc_lookup(watcher)) break; /* Process not alive */ + + if (reason == THE_NON_VALUE) { + +#ifdef ERTS_DIST_MSG_DBG + dist_msg_dbg(edep, "MSG", buf, orig_len); +#endif + + } + + erts_proc_sig_send_dist_monitor_down( + dep, ref, watched, watcher, edep, ede_hfrag, reason); break; } + case DOP_PAYLOAD_EXIT: + case DOP_PAYLOAD_EXIT_TT: case DOP_EXIT_TT: case DOP_EXIT: { + /* 'from', which 'to' is linked to, died */ + from = tuple[2]; + to = tuple[3]; + if (type == DOP_EXIT) { if (tuple_arity != 4) { goto invalid_message; } - - from = tuple[2]; - to = tuple[3]; - reason = tuple[4]; token = NIL; - } else { + reason = tuple[4]; + } else if (type == DOP_EXIT_TT){ if (tuple_arity != 5) { goto invalid_message; } - from = tuple[2]; - to = tuple[3]; token = tuple[4]; reason = tuple[5]; - } + } else if (type == DOP_PAYLOAD_EXIT) { + if (tuple_arity != 3) { + goto invalid_message; + } + token = NIL; + reason = THE_NON_VALUE; + } else { + if (tuple_arity != 4) { + goto invalid_message; + } + token = tuple[4]; + reason = THE_NON_VALUE; + } if (is_not_external_pid(from) || dep != external_pid_dist_entry(from) || is_not_internal_pid(to)) { goto invalid_message; } + if (!erts_proc_lookup(to)) break; /* Process not alive */ + + if (reason == THE_NON_VALUE) { +#ifdef ERTS_DIST_MSG_DBG + dist_msg_dbg(edep, "MSG", buf, orig_len); +#endif + } + erts_proc_sig_send_dist_link_exit(dep, - from, to, + from, to, edep, ede_hfrag, reason, token); break; } + case DOP_PAYLOAD_EXIT2_TT: + case DOP_PAYLOAD_EXIT2: case DOP_EXIT2_TT: - case DOP_EXIT2: + case DOP_EXIT2: { + /* 'from' is send an exit signal to 'to' */ + from = tuple[2]; + to = tuple[3]; + if (type == DOP_EXIT2) { if (tuple_arity != 4) { goto invalid_message; } - from = tuple[2]; - to = tuple[3]; reason = tuple[4]; token = NIL; - } else { + } else if (type == DOP_EXIT2_TT) { if (tuple_arity != 5) { goto invalid_message; } - from = tuple[2]; - to = tuple[3]; token = tuple[4]; reason = tuple[5]; - } - if (is_not_pid(from) || is_not_internal_pid(to)) { + } else if (type == DOP_PAYLOAD_EXIT2) { + if (tuple_arity != 3) { + goto invalid_message; + } + reason = THE_NON_VALUE; + token = NIL; + } else { + if (tuple_arity != 4) { + goto invalid_message; + } + reason = THE_NON_VALUE; + token = tuple[4]; + } + if (is_not_pid(from) + || dep != external_pid_dist_entry(from) + || is_not_internal_pid(to)) { goto invalid_message; } - erts_proc_sig_send_exit(NULL, from, to, reason, token, 0); - break; + if (!erts_proc_lookup(to)) break; /* Process not alive */ + if (reason == THE_NON_VALUE) { +#ifdef ERTS_DIST_MSG_DBG + dist_msg_dbg(edep, "MSG", buf, orig_len); +#endif + } + + erts_proc_sig_send_dist_exit(dep, from, to, edep, ede_hfrag, reason, token); + break; + } case DOP_GROUP_LEADER: if (tuple_arity != 3) { goto invalid_message; @@ -1725,13 +2116,15 @@ int erts_net_message(Port *prt, (void) erts_proc_sig_send_group_leader(NULL, to, from, NIL); break; - default: + default: goto invalid_message; } - erts_factory_close(&factory); - if (ctl != ctl_default) { - erts_free(ERTS_ALC_T_DCTRL_BUF, (void *) ctl); + if (ede_hfrag == NULL) { + erts_factory_close(&factory); + if (ctl != ctl_default) { + erts_free(ERTS_ALC_T_DCTRL_BUF, (void *) ctl); + } } UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE); ERTS_CHK_NO_PROC_LOCKS; @@ -1744,9 +2137,14 @@ int erts_net_message(Port *prt, } decode_error: PURIFY_MSG("data error"); - erts_factory_close(&factory); - if (ctl != ctl_default) { - erts_free(ERTS_ALC_T_DCTRL_BUF, (void *) ctl); + if (ede_hfrag == NULL) { + erts_factory_close(&factory); + if (ctl != ctl_default) { + erts_free(ERTS_ALC_T_DCTRL_BUF, (void *) ctl); + } + } else { + erts_free_dist_ext_copy(erts_get_dist_ext(ede_hfrag)); + free_message_buffer(ede_hfrag); } data_error: UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE); @@ -1755,18 +2153,21 @@ data_error: return -1; } -static int dsig_send_ctl(ErtsDSigData* dsdp, Eterm ctl, int force_busy) +static int dsig_send_exit(ErtsDSigSendContext *ctx, Eterm ctl, Eterm msg) +{ + ctx->ctl = ctl; + ctx->msg = msg; + return erts_dsig_send(ctx); +} + +static int dsig_send_ctl(ErtsDSigSendContext *ctx, Eterm ctl) { - struct erts_dsig_send_context ctx; int ret; - ctx.ctl = ctl; - ctx.msg = THE_NON_VALUE; - ctx.force_busy = force_busy; - ctx.phase = ERTS_DSIG_SEND_PHASE_INIT; -#ifdef DEBUG - ctx.reds = 1; /* provoke assert below (no reduction count without msg) */ -#endif - ret = erts_dsig_send(dsdp, &ctx); + ctx->ctl = ctl; + ctx->msg = THE_NON_VALUE; + ctx->from = THE_NON_VALUE; + ctx->reds = 1; /* provoke assert below (no reduction count without msg) */ + ret = erts_dsig_send(ctx); ASSERT(ret != ERTS_DSIG_SEND_CONTINUE); return ret; } @@ -1797,7 +2198,117 @@ notify_dist_data(Process *c_p, Eterm pid) } int -erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx) +erts_dsig_prepare(ErtsDSigSendContext *ctx, + DistEntry *dep, + Process *proc, + ErtsProcLocks proc_locks, + ErtsDSigPrepLock dspl, + int no_suspend, + int no_trap, + int connect) +{ + int res; + + if (!erts_is_alive) + return ERTS_DSIG_PREP_NOT_ALIVE; + if (!dep) { + ASSERT(!connect); + return ERTS_DSIG_PREP_NOT_CONNECTED; + } + +#ifdef ERTS_ENABLE_LOCK_CHECK + if (connect) { + erts_proc_lc_might_unlock(proc, proc_locks); + } +#endif + +retry: + erts_de_rlock(dep); + + if (dep->state == ERTS_DE_STATE_CONNECTED) { + res = ERTS_DSIG_PREP_CONNECTED; + } + else if (dep->state == ERTS_DE_STATE_PENDING) { + res = ERTS_DSIG_PREP_PENDING; + } + else if (dep->state == ERTS_DE_STATE_EXITING) { + res = ERTS_DSIG_PREP_NOT_CONNECTED; + goto fail; + } + else if (connect) { + ASSERT(dep->state == ERTS_DE_STATE_IDLE); + erts_de_runlock(dep); + if (!erts_auto_connect(dep, proc, proc_locks)) { + return ERTS_DSIG_PREP_NOT_ALIVE; + } + goto retry; + } + else { + ASSERT(dep->state == ERTS_DE_STATE_IDLE); + res = ERTS_DSIG_PREP_NOT_CONNECTED; + goto fail; + } + + if (no_suspend) { + if (erts_atomic32_read_acqb(&dep->qflgs) & ERTS_DE_QFLG_BUSY) { + res = ERTS_DSIG_PREP_WOULD_SUSPEND; + goto fail; + } + } + + ctx->c_p = proc; + ctx->dep = dep; + ctx->deref_dep = 0; + ctx->cid = dep->cid; + ctx->connection_id = dep->connection_id; + ctx->no_suspend = no_suspend; + ctx->no_trap = no_trap; + ctx->flags = dep->flags; + ctx->return_term = am_true; + ctx->phase = ERTS_DSIG_SEND_PHASE_INIT; + ctx->from = proc ? proc->common.id : am_undefined; + ctx->reds = no_trap ? 1 : (Sint) (ERTS_BIF_REDS_LEFT(proc) * TERM_TO_BINARY_LOOP_FACTOR); + if (dspl == ERTS_DSP_NO_LOCK) + erts_de_runlock(dep); + return res; + + fail: + erts_de_runlock(dep); + return res; +} + +static +void erts_schedule_dist_command(Port *prt, DistEntry *dist_entry) +{ + DistEntry *dep; + Eterm id; + + if (prt) { + ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); + ASSERT((erts_atomic32_read_nob(&prt->state) + & ERTS_PORT_SFLGS_DEAD) == 0); + + dep = (DistEntry*) erts_prtsd_get(prt, ERTS_PRTSD_DIST_ENTRY); + ASSERT(dep); + id = prt->common.id; + } + else { + ASSERT(dist_entry); + ERTS_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&dist_entry->rwmtx) + || erts_lc_rwmtx_is_rwlocked(&dist_entry->rwmtx)); + ASSERT(is_internal_port(dist_entry->cid)); + + dep = dist_entry; + id = dep->cid; + } + + if (!erts_atomic_xchg_mb(&dep->dist_cmd_scheduled, 1)) + erts_port_task_schedule(id, &dep->dist_cmd, ERTS_PORT_TASK_DIST_CMD); +} + + +int +erts_dsig_send(ErtsDSigSendContext *ctx) { int retval; Sint initial_reds = ctx->reds; @@ -1806,11 +2317,13 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx) while (1) { switch (ctx->phase) { case ERTS_DSIG_SEND_PHASE_INIT: - ctx->flags = dsdp->flags; - ctx->c_p = dsdp->proc; + ctx->flags = ctx->flags; + ctx->c_p = ctx->c_p; - if (!ctx->c_p || dsdp->no_suspend) - ctx->force_busy = 1; + if (!ctx->c_p) { + ctx->no_trap = 1; + ctx->no_suspend = 1; + } ERTS_LC_ASSERT(!ctx->c_p || (ERTS_PROC_LOCK_MAIN @@ -1829,9 +2342,11 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx) } #ifdef ERTS_DIST_MSG_DBG - erts_fprintf(stderr, ">> CTL: %T\n", ctx->ctl); + erts_fprintf(dbg_file, "SEND: CTL: %s: %.80T\n", + erts_dop_to_string(unsigned_val(tuple_val(ctx->ctl)[1])), + ctx->ctl); if (is_value(ctx->msg)) - erts_fprintf(stderr, " MSG: %T\n", ctx->msg); + erts_fprintf(dbg_file, " MSG: %.160T\n", ctx->msg); #endif ctx->data_size = ctx->max_finalize_prepend; @@ -1848,25 +2363,45 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx) ctx->phase = ERTS_DSIG_SEND_PHASE_MSG_SIZE; case ERTS_DSIG_SEND_PHASE_MSG_SIZE: - if (erts_encode_dist_ext_size_int(ctx->msg, ctx, &ctx->data_size)) { - retval = ERTS_DSIG_SEND_CONTINUE; - goto done; - } + if (!ctx->no_trap) { + if (erts_encode_dist_ext_size_int(ctx->msg, ctx, &ctx->data_size)) { + retval = ERTS_DSIG_SEND_CONTINUE; + goto done; + } + } else { + erts_encode_dist_ext_size(ctx->msg, ctx->flags, ctx->acmp, &ctx->data_size); + } ctx->phase = ERTS_DSIG_SEND_PHASE_ALLOC; case ERTS_DSIG_SEND_PHASE_ALLOC: erts_finalize_atom_cache_map(ctx->acmp, ctx->flags); - ctx->dhdr_ext_size = erts_encode_ext_dist_header_size(ctx->acmp); - ctx->data_size += ctx->dhdr_ext_size; + if (ctx->flags & DFLAG_FRAGMENTS && is_value(ctx->msg) && is_not_immed(ctx->msg)) { + /* Calculate the max number of fragments that are needed */ + ASSERT(is_pid(ctx->from) && + "from has to be a pid because it is used as sequence id"); + ctx->fragments = ctx->data_size / ERTS_DIST_FRAGMENT_SIZE + 1; + } else + ctx->fragments = 1; + + ctx->dhdr_ext_size = erts_encode_ext_dist_header_size(ctx->acmp, ctx->fragments); - ctx->obuf = alloc_dist_obuf(ctx->data_size); - ctx->obuf->ext_endp = &ctx->obuf->data[0] + ctx->max_finalize_prepend + ctx->dhdr_ext_size; + ctx->obuf = alloc_dist_obuf( + ctx->dhdr_ext_size + ctx->data_size + + (ctx->fragments-1) * ERTS_DIST_FRAGMENT_HEADER_SIZE, + ctx->fragments); + ctx->obuf->ext_start = &ctx->obuf->extp[0]; + ctx->obuf->ext_endp = &ctx->obuf->extp[0] + ctx->max_finalize_prepend + ctx->dhdr_ext_size; /* Encode internal version of dist header */ - ctx->obuf->extp = erts_encode_ext_dist_header_setup(ctx->obuf->ext_endp, ctx->acmp); + ctx->obuf->extp = erts_encode_ext_dist_header_setup( + ctx->obuf->ext_endp, ctx->acmp, ctx->fragments, ctx->from); /* Encode control message */ erts_encode_dist_ext(ctx->ctl, &ctx->obuf->ext_endp, ctx->flags, ctx->acmp, NULL, NULL); + + ctx->obuf->hdrp = NULL; + ctx->obuf->hdr_endp = NULL; + if (is_non_value(ctx->msg)) { ctx->obuf->msg_start = NULL; ctx->phase = ERTS_DSIG_SEND_PHASE_FIN; @@ -1880,50 +2415,137 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx) ctx->phase = ERTS_DSIG_SEND_PHASE_MSG_ENCODE; case ERTS_DSIG_SEND_PHASE_MSG_ENCODE: - if (erts_encode_dist_ext(ctx->msg, &ctx->obuf->ext_endp, ctx->flags, ctx->acmp, &ctx->u.ec, &ctx->reds)) { - retval = ERTS_DSIG_SEND_CONTINUE; - goto done; - } + if (!ctx->no_trap) { + if (erts_encode_dist_ext(ctx->msg, &ctx->obuf->ext_endp, ctx->flags, + ctx->acmp, &ctx->u.ec, &ctx->reds)) { + retval = ERTS_DSIG_SEND_CONTINUE; + goto done; + } + } else { + erts_encode_dist_ext(ctx->msg, &ctx->obuf->ext_endp, ctx->flags, + ctx->acmp, NULL, NULL); + } - ctx->phase = ERTS_DSIG_SEND_PHASE_FIN; + ctx->phase = ERTS_DSIG_SEND_PHASE_FIN; case ERTS_DSIG_SEND_PHASE_FIN: { - DistEntry *dep = dsdp->dep; - int suspended = 0; - int resume = 0; ASSERT(ctx->obuf->extp < ctx->obuf->ext_endp); - ASSERT(&ctx->obuf->data[0] <= ctx->obuf->extp - ctx->max_finalize_prepend); - ASSERT(ctx->obuf->ext_endp <= &ctx->obuf->data[0] + ctx->data_size); + ASSERT(((byte*)&ctx->obuf->bin->orig_bytes[0]+obuf_list_size) <= ctx->obuf->extp - ctx->max_finalize_prepend); + ASSERT(ctx->obuf->ext_endp <= ((byte*)ctx->obuf->bin->orig_bytes+obuf_list_size) + ctx->data_size + ctx->dhdr_ext_size); ctx->data_size = ctx->obuf->ext_endp - ctx->obuf->extp; - if (ctx->data_size > (Uint) INT_MAX) { - free_dist_obuf(ctx->obuf); - ctx->obuf = NULL; - retval = ERTS_DSIG_SEND_TOO_LRG; - goto done; - } ctx->obuf->hopefull_flags = ctx->u.ec.hopefull_flags; - /* + + if (ctx->fragments > 1) { + int fin_fragments; + int i; + byte *msg = ctx->obuf->msg_start, + *msg_end = ctx->obuf->ext_endp, + *hdrp = msg_end; + + ASSERT((ctx->obuf->hopefull_flags & ctx->flags) == ctx->obuf->hopefull_flags); + ASSERT(get_int64(ctx->obuf->extp + 1 + 1 + 8) == ctx->fragments); + + /* Now that encoding is done we know how large the term will + be so we adjust the number of fragments to send. Note that + this can mean that only 1 fragment is sent. */ + fin_fragments = (ctx->obuf->ext_endp - ctx->obuf->msg_start + ERTS_DIST_FRAGMENT_SIZE-1) / + ERTS_DIST_FRAGMENT_SIZE - 1; + + /* Update the frag_id in the DIST_FRAG_HEADER */ + put_int64(fin_fragments+1, ctx->obuf->extp + 1 + 1 + 8); + + if (fin_fragments > 0) + msg += ERTS_DIST_FRAGMENT_SIZE; + else + msg = msg_end; + ctx->obuf->next = &ctx->obuf[1]; + ctx->obuf->ext_endp = msg; + + /* Loop through all fragments, updating the output buffers + to be correct and also writing the DIST_FRAG_CONT header. */ + for (i = 1; i < fin_fragments + 1; i++) { + ctx->obuf[i].hopefull_flags = 0; + ctx->obuf[i].extp = msg; + ctx->obuf[i].ext_start = msg; + if (msg + ERTS_DIST_FRAGMENT_SIZE > msg_end) + ctx->obuf[i].ext_endp = msg_end; + else { + msg += ERTS_DIST_FRAGMENT_SIZE; + ctx->obuf[i].ext_endp = msg; + } + ASSERT(ctx->obuf[i].ext_endp > ctx->obuf[i].extp); + ctx->obuf[i].hdrp = erts_encode_ext_dist_header_fragment( + &hdrp, fin_fragments - i + 1, ctx->from); + ctx->obuf[i].hdr_endp = hdrp; + ctx->obuf[i].next = &ctx->obuf[i+1]; + } + /* If the initial fragment calculation was incorrect we free the + remaining output buffers. */ + for (; i < ctx->fragments; i++) { + free_dist_obuf(&ctx->obuf[i]); + } + if (!ctx->no_trap && !ctx->no_suspend) + ctx->reds -= ctx->fragments; + ctx->fragments = fin_fragments + 1; + } + + ctx->phase = ERTS_DSIG_SEND_PHASE_SEND; + + if (ctx->reds <= 0) { + retval = ERTS_DSIG_SEND_CONTINUE; + goto done; + } + } + case ERTS_DSIG_SEND_PHASE_SEND: { + /* * Signal encoded; now verify that the connection still exists, * and if so enqueue the signal and schedule it for send. */ - ctx->obuf->next = NULL; + DistEntry *dep = ctx->dep; + int suspended = 0; + int resume = 0; + int i; erts_de_rlock(dep); cid = dep->cid; if (dep->state == ERTS_DE_STATE_EXITING || dep->state == ERTS_DE_STATE_IDLE - || dep->connection_id != dsdp->connection_id) { + || dep->connection_id != ctx->connection_id) { /* Not the same connection as when we started; drop message... */ erts_de_runlock(dep); - free_dist_obuf(ctx->obuf); + for (i = 0; i < ctx->fragments; i++) + free_dist_obuf(&ctx->obuf[i]); + ctx->fragments = 0; } else { - Sint qsize; + Sint qsize = erts_atomic_read_nob(&dep->qsize); erts_aint32_t qflgs; ErtsProcList *plp = NULL; Eterm notify_proc = NIL; - Sint obsz = size_obuf(ctx->obuf); + Sint obsz; + int fragments; + + /* Calculate how many fragments to send. This depends on + the available space in the distr queue and the amount + of remaining reductions. */ + for (fragments = 0, obsz = 0; + fragments < ctx->fragments && + ((ctx->reds > 0 && (qsize + obsz) < erts_dist_buf_busy_limit) || + ctx->no_trap || ctx->no_suspend); + fragments++) { +#ifdef DEBUG + int reds = 100; +#else + int reds = 10; +#endif + if (!ctx->no_trap && !ctx->no_suspend) + ctx->reds -= reds; + obsz += size_obuf(&ctx->obuf[fragments]); + } + + ASSERT(fragments == ctx->fragments || + (!ctx->no_trap && !ctx->no_suspend)); erts_mtx_lock(&dep->qlock); qsize = erts_atomic_add_read_nob(&dep->qsize, (erts_aint_t) obsz); @@ -1944,7 +2566,7 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx) /* else: requester will send itself the message... */ qflgs &= ~ERTS_DE_QFLG_REQ_INFO; } - if (!ctx->force_busy && (qflgs & ERTS_DE_QFLG_BUSY)) { + if (!ctx->no_suspend && (qflgs & ERTS_DE_QFLG_BUSY)) { erts_mtx_unlock(&dep->qlock); plp = erts_proclist_create(ctx->c_p); @@ -1953,14 +2575,27 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx) erts_mtx_lock(&dep->qlock); } - /* Enqueue obuf on dist entry */ - if (dep->out_queue.last) - dep->out_queue.last->next = ctx->obuf; - else - dep->out_queue.first = ctx->obuf; - dep->out_queue.last = ctx->obuf; + if (fragments > 1) { + if (!ctx->obuf->hdrp) { + ASSERT(get_int64(ctx->obuf->extp + 10) == ctx->fragments); + } else { + ASSERT(get_int64(ctx->obuf->hdrp + 10) == ctx->fragments); + } + } + + if (fragments) { + ctx->obuf[fragments-1].next = NULL; + if (dep->out_queue.last) + dep->out_queue.last->next = ctx->obuf; + else + dep->out_queue.first = ctx->obuf; + dep->out_queue.last = &ctx->obuf[fragments-1]; - if (!ctx->force_busy) { + ctx->fragments -= fragments; + ctx->obuf = &ctx->obuf[fragments]; + } + + if (!ctx->no_suspend) { qflgs = erts_atomic32_read_nob(&dep->qflgs); if (!(qflgs & ERTS_DE_QFLG_BUSY)) { if (suspended) @@ -2007,6 +2642,13 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx) * erroneously scheduled when it shouldn't be. */ } + /* More fragments left to be sent, yield and re-schedule */ + if (ctx->fragments) { + retval = ERTS_DSIG_SEND_CONTINUE; + if (!resume && erts_system_monitor_flags.busy_dist_port) + monitor_generic(ctx->c_p, am_busy_dist_port, cid); + goto done; + } } ctx->obuf = NULL; @@ -2091,9 +2733,9 @@ static Uint dist_port_commandv(Port *prt, ErtsDistOutputBuf *obuf) { int fpe_was_unmasked; - ErlDrvSizeT size; - SysIOVec iov[2]; - ErlDrvBinary* bv[2]; + ErlDrvSizeT size = 0; + SysIOVec iov[3]; + ErlDrvBinary* bv[3]; ErlIOVec eiov; ERTS_CHK_NO_PROC_LOCKS; @@ -2108,12 +2750,31 @@ dist_port_commandv(Port *prt, ErtsDistOutputBuf *obuf) eiov.vsize = 1; } else { - size = obuf->ext_endp - obuf->extp; + int i = 1; eiov.vsize = 2; - iov[1].iov_base = obuf->extp; - iov[1].iov_len = size; - bv[1] = Binary2ErlDrvBinary(ErtsDistOutputBuf2Binary(obuf)); + if (obuf->hdrp) { + eiov.vsize = 3; + iov[i].iov_base = obuf->hdrp; + iov[i].iov_len = obuf->hdr_endp - obuf->hdrp; + size += iov[i].iov_len; + bv[i] = Binary2ErlDrvBinary(ErtsDistOutputBuf2Binary(obuf)); +#ifdef ERTS_RAW_DIST_MSG_DBG + erts_fprintf(dbg_file, "SEND: "); + bw(iov[i].iov_base, iov[i].iov_len); +#endif + i++; + + } + + iov[i].iov_base = obuf->extp; + iov[i].iov_len = obuf->ext_endp - obuf->extp; +#ifdef ERTS_RAW_DIST_MSG_DBG + erts_fprintf(dbg_file, "SEND: "); + bw(iov[i].iov_base, iov[i].iov_len); +#endif + size += iov[i].iov_len; + bv[i] = Binary2ErlDrvBinary(ErtsDistOutputBuf2Binary(obuf)); } eiov.size = size; @@ -2220,6 +2881,23 @@ erts_dist_command(Port *prt, int initial_reds) dep->finalized_out_queue.first = NULL; dep->finalized_out_queue.last = NULL; +#ifdef DEBUG + { + Uint sz = 0; + ErtsDistOutputBuf *curr = oq.first; + while (curr) { + sz += size_obuf(curr); + curr = curr->next; + } + curr = foq.first; + while (curr) { + sz += size_obuf(curr); + curr = curr->next; + } + ASSERT(sz <= erts_atomic_read_nob(&dep->qsize)); + } +#endif + sched_flags = erts_atomic32_read_nob(&prt->sched.flags); if (reds < 0) @@ -2233,10 +2911,6 @@ erts_dist_command(Port *prt, int initial_reds) size = (*send)(prt, foq.first); erts_atomic64_inc_nob(&dep->out); esdp->io.out += (Uint64) size; -#ifdef ERTS_RAW_DIST_MSG_DBG - erts_fprintf(stderr, ">> "); - bw(foq.first->extp, size); -#endif reds -= ERTS_PORT_REDS_DIST_CMD_DATA(size); fob = foq.first; obufsize += size_obuf(fob); @@ -2305,15 +2979,11 @@ erts_dist_command(Port *prt, int initial_reds) preempt = 1; break; } - ASSERT(&oq.first->data[0] <= oq.first->extp - && oq.first->extp <= oq.first->ext_endp); + ASSERT(oq.first->bin->orig_bytes <= (char*)oq.first->extp + && oq.first->extp <= oq.first->ext_endp); size = (*send)(prt, oq.first); erts_atomic64_inc_nob(&dep->out); esdp->io.out += (Uint64) size; -#ifdef ERTS_RAW_DIST_MSG_DBG - erts_fprintf(stderr, ">> "); - bw(oq.first->extp, size); -#endif reds -= ERTS_PORT_REDS_DIST_CMD_DATA(size); fob = oq.first; obufsize += size_obuf(fob); @@ -2450,100 +3120,6 @@ erts_dist_command(Port *prt, int initial_reds) goto done; } -#if 0 - -int -dist_data_finalize(Process *c_p, int reds_limit) -{ - int reds = 5; - DistEntry *dep = ; - ErtsDistOutputQueue oq, foq; - ErtsDistOutputBuf *ob; - int preempt; - - - erts_mtx_lock(&dep->qlock); - flags = dep->flags; - oq.first = dep->out_queue.first; - oq.last = dep->out_queue.last; - dep->out_queue.first = NULL; - dep->out_queue.last = NULL; - erts_mtx_unlock(&dep->qlock); - - if (!oq.first) { - ASSERT(!oq.last); - oq.first = dep->tmp_out_queue.first; - oq.last = dep->tmp_out_queue.last; - } - else { - ErtsDistOutputBuf *f, *l; - ASSERT(oq.last); - if (dep->tmp_out_queue.last) { - dep->tmp_out_queue.last->next = oq.first; - oq.first = dep->tmp_out_queue.first; - } - } - - if (!oq.first) { - /* Nothing to do... */ - ASSERT(!oq.last); - return reds; - } - - foq.first = dep->finalized_out_queue.first; - foq.last = dep->finalized_out_queue.last; - - preempt = 0; - ob = oq.first; - ASSERT(ob); - - do { - ob->extp = erts_encode_ext_dist_header_finalize(ob->extp, - dep->cache, - flags); - if (!(flags & DFLAG_DIST_HDR_ATOM_CACHE)) - *--ob->extp = PASS_THROUGH; /* Old node; 'pass through' - needed */ - ASSERT(&ob->data[0] <= ob->extp && ob->extp < ob->ext_endp); - reds += ERTS_PORT_REDS_DIST_CMD_FINALIZE; - preempt = reds > reds_limit; - if (preempt) - break; - ob = ob->next; - } while (ob); - /* - * At least one buffer was finalized; if we got preempted, - * ob points to the last buffer that we finalized. - */ - if (foq.last) - foq.last->next = oq.first; - else - foq.first = oq.first; - if (!preempt) { - /* All buffers finalized */ - foq.last = oq.last; - oq.first = oq.last = NULL; - } - else { - /* Not all buffers finalized; split oq. */ - foq.last = ob; - oq.first = ob->next; - if (oq.first) - ob->next = NULL; - else - oq.last = NULL; - } - - dep->finalized_out_queue.first = foq.first; - dep->finalized_out_queue.last = foq.last; - dep->tmp_out_queue.first = oq.first; - dep->tmp_out_queue.last = oq.last; - - return reds; -} - -#endif - BIF_RETTYPE dist_ctrl_get_data_notification_1(BIF_ALIST_1) { @@ -2612,6 +3188,7 @@ dist_ctrl_put_data_2(BIF_ALIST_2) ErlDrvSizeT size; Eterm input_handler; Uint32 conn_id; + Binary *bin = NULL; if (is_binary(BIF_ARG_2)) size = binary_size(BIF_ARG_2); @@ -2637,13 +3214,27 @@ dist_ctrl_put_data_2(BIF_ALIST_2) if (size != 0) { byte *data, *temp_alloc = NULL; - data = (byte *) erts_get_aligned_binary_bytes(BIF_ARG_2, &temp_alloc); + if (binary_bitoffset(BIF_ARG_2)) + data = (byte *) erts_get_aligned_binary_bytes(BIF_ARG_2, &temp_alloc); + else { + Eterm real_bin; + ProcBin *proc_bin; + Uint offset, bitoffs, bitsize; + + ERTS_GET_REAL_BIN(BIF_ARG_2, real_bin, offset, bitoffs, bitsize); + ASSERT(bitoffs == 0); + data = binary_bytes(real_bin) + offset; + proc_bin = (ProcBin *)binary_val(real_bin); + if (proc_bin->thing_word == HEADER_PROC_BIN) + bin = proc_bin->val; + } + if (!data) BIF_ERROR(BIF_P, BADARG); erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - (void) erts_net_message(NULL, dep, conn_id, NULL, 0, data, size); + (void) erts_net_message(NULL, dep, conn_id, NULL, 0, bin, data, size); /* * We ignore any decode failures. On fatal failures the * connection will be taken down by killing the @@ -2877,6 +3468,10 @@ static void kill_connection(DistEntry *dep) void erts_kill_dist_connection(DistEntry *dep, Uint32 conn_id) { +#ifdef ERTS_DIST_MSG_DBG + erts_fprintf(dbg_file, "INTR: kill dist conn to %T:%u\n", + dep->sysname, conn_id); +#endif erts_de_rwlock(dep); if (conn_id == dep->connection_id && dep->state == ERTS_DE_STATE_CONNECTED) { @@ -2891,7 +3486,7 @@ struct print_to_data { void *arg; }; -static void doit_print_monitor_info(ErtsMonitor *mon, void *vptdp) +static int doit_print_monitor_info(ErtsMonitor *mon, void *vptdp, Sint reds) { fmtfn_t to = ((struct print_to_data *) vptdp)->to; void *arg = ((struct print_to_data *) vptdp)->arg; @@ -2914,6 +3509,7 @@ static void doit_print_monitor_info(ErtsMonitor *mon, void *vptdp) else erts_print(to, arg, "%T\n", mdep->md.origin.other.item); } + return 1; } static void print_monitor_info(fmtfn_t to, void *arg, DistEntry *dep) @@ -2929,12 +3525,13 @@ static void print_monitor_info(fmtfn_t to, void *arg, DistEntry *dep) } } -static void doit_print_link_info(ErtsLink *lnk, void *vptdp) +static int doit_print_link_info(ErtsLink *lnk, void *vptdp, Sint reds) { struct print_to_data *ptdp = vptdp; ErtsLink *lnk2 = erts_link_to_other(lnk, NULL); erts_print(ptdp->to, ptdp->arg, "Remote link: %T %T\n", lnk2->other.item, lnk->other.item); + return 1; } static void print_link_info(fmtfn_t to, void *arg, DistEntry *dep) @@ -3536,8 +4133,9 @@ Sint erts_abort_connection_rwunlock(DistEntry* dep) erts_set_dist_entry_not_connected(dep); erts_de_rwunlock(dep); - schedule_con_monitor_link_cleanup(mld, THE_NON_VALUE, - THE_NON_VALUE, THE_NON_VALUE); + schedule_con_monitor_link_seq_cleanup( + mld, NULL, THE_NON_VALUE, + THE_NON_VALUE, THE_NON_VALUE); if (resume_procs) { int resumed = erts_resume_processes(resume_procs); @@ -3843,16 +4441,16 @@ monitor_node(Process* p, Eterm Node, Eterm Bool, Eterm Options) } case am_true: { - ErtsDSigData dsd; - dsd.node = Node; + ErtsDSigSendContext ctx; + ctx.node = Node; dep = erts_find_or_insert_dist_entry(Node); if (dep == erts_this_dist_entry) break; - switch (erts_dsig_prepare(&dsd, dep, p, + switch (erts_dsig_prepare(&ctx, dep, p, ERTS_PROC_LOCK_MAIN, - ERTS_DSP_RLOCK, 0, async_connect)) { + ERTS_DSP_RLOCK, 0, 0, async_connect)) { case ERTS_DSIG_PREP_NOT_ALIVE: case ERTS_DSIG_PREP_NOT_CONNECTED: /* Trap to either send 'nodedown' or do passive connection attempt */ @@ -4009,6 +4607,10 @@ init_nodes_monitors(void) { erts_mtx_init(&nodes_monitors_mtx, "nodes_monitors", NIL, ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION); +#ifdef DEBUG + erts_mtx_init(&erts_obuf_list_mtx, "sad", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION); +#endif nodes_monitors = NULL; no_nodes_monitors = 0; } @@ -4150,8 +4752,8 @@ typedef struct { Uint i; } ErtsNodesMonitorContext; -static void -save_nodes_monitor(ErtsMonitor *mon, void *vctxt) +static int +save_nodes_monitor(ErtsMonitor *mon, void *vctxt, Sint reds) { ErtsNodesMonitorContext *ctxt = vctxt; ErtsMonitorData *mdp = erts_monitor_to_data(mon); @@ -4163,6 +4765,7 @@ save_nodes_monitor(ErtsMonitor *mon, void *vctxt) ctxt->nmdp[ctxt->i].options = mdp->origin.other.item; ctxt->i++; + return 1; } static void @@ -4295,8 +4898,8 @@ typedef struct { } ErtsNodesMonitorInfoContext; -static void -nodes_monitor_info(ErtsMonitor *mon, void *vctxt) +static int +nodes_monitor_info(ErtsMonitor *mon, void *vctxt, Sint reds) { ErtsMonitorDataExtended *mdep; ErtsNodesMonitorInfoContext *ctxt = vctxt; @@ -4343,6 +4946,7 @@ nodes_monitor_info(ErtsMonitor *mon, void *vctxt) ctxt->hpp = hpp; ctxt->szp = szp; ctxt->res = res; + return 1; } Eterm diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h index 845fab229a..c4bb967592 100644 --- a/erts/emulator/beam/dist.h +++ b/erts/emulator/beam/dist.h @@ -47,6 +47,8 @@ #define DFLAG_SEND_SENDER 0x80000 #define DFLAG_BIG_SEQTRACE_LABELS 0x100000 #define DFLAG_NO_MAGIC 0x200000 /* internal for pending connection */ +#define DFLAG_EXIT_PAYLOAD 0x400000 +#define DFLAG_FRAGMENTS 0x800000 /* Mandatory flags for distribution */ #define DFLAG_DIST_MANDATORY (DFLAG_EXTENDED_REFERENCES \ @@ -75,7 +77,9 @@ | DFLAG_MAP_TAG \ | DFLAG_BIG_CREATION \ | DFLAG_SEND_SENDER \ - | DFLAG_BIG_SEQTRACE_LABELS) + | DFLAG_BIG_SEQTRACE_LABELS \ + | DFLAG_EXIT_PAYLOAD \ + | DFLAG_FRAGMENTS) /* Flags addable by local distr implementations */ #define DFLAG_DIST_ADDABLE DFLAG_DIST_DEFAULT @@ -99,26 +103,35 @@ | DFLAG_BIG_CREATION) /* opcodes used in distribution messages */ -#define DOP_LINK 1 -#define DOP_SEND 2 -#define DOP_EXIT 3 -#define DOP_UNLINK 4 +enum dop { + DOP_LINK = 1, + DOP_SEND = 2, + DOP_EXIT = 3, + DOP_UNLINK = 4, /* Ancient DOP_NODE_LINK (5) was here, can be reused */ -#define DOP_REG_SEND 6 -#define DOP_GROUP_LEADER 7 -#define DOP_EXIT2 8 - -#define DOP_SEND_TT 12 -#define DOP_EXIT_TT 13 -#define DOP_REG_SEND_TT 16 -#define DOP_EXIT2_TT 18 - -#define DOP_MONITOR_P 19 -#define DOP_DEMONITOR_P 20 -#define DOP_MONITOR_P_EXIT 21 - -#define DOP_SEND_SENDER 22 -#define DOP_SEND_SENDER_TT 23 + DOP_REG_SEND = 6, + DOP_GROUP_LEADER = 7, + DOP_EXIT2 = 8, + + DOP_SEND_TT = 12, + DOP_EXIT_TT = 13, + DOP_REG_SEND_TT = 16, + DOP_EXIT2_TT = 18, + + DOP_MONITOR_P = 19, + DOP_DEMONITOR_P = 20, + DOP_MONITOR_P_EXIT = 21, + + DOP_SEND_SENDER = 22, + DOP_SEND_SENDER_TT = 23, + + /* These are used when DFLAG_EXIT_PAYLOAD is detected */ + DOP_PAYLOAD_EXIT = 24, + DOP_PAYLOAD_EXIT_TT = 25, + DOP_PAYLOAD_EXIT2 = 26, + DOP_PAYLOAD_EXIT2_TT = 27, + DOP_PAYLOAD_MONITOR_P_EXIT = 28 +}; /* distribution trap functions */ extern Export* dmonitor_node_trap; @@ -129,15 +142,15 @@ typedef enum { } ErtsDSigPrepLock; -typedef struct { - Process *proc; - DistEntry *dep; - Eterm node; /* used if dep == NULL */ - Eterm cid; - Eterm connection_id; - int no_suspend; - Uint32 flags; -} ErtsDSigData; +/* Must be larger or equal to 16 */ +#ifdef DEBUG +#define ERTS_DIST_FRAGMENT_SIZE 16 +#else +/* This should be made configurable */ +#define ERTS_DIST_FRAGMENT_SIZE (64 * 1024) +#endif + +#define ERTS_DIST_FRAGMENT_HEADER_SIZE (1 + 1 + 8 + 8) /* magic, header, seq id, frag id*/ #define ERTS_DE_BUSY_LIMIT (1024*1024) extern int erts_dist_buf_busy_limit; @@ -159,124 +172,6 @@ extern int erts_is_alive; /* Pending connection; signals can be enqueued */ #define ERTS_DSIG_PREP_PENDING 4 -ERTS_GLB_INLINE int erts_dsig_prepare(ErtsDSigData *, - DistEntry*, - Process *, - ErtsProcLocks, - ErtsDSigPrepLock, - int, - int); - -ERTS_GLB_INLINE -void erts_schedule_dist_command(Port *, DistEntry *); - -int erts_auto_connect(DistEntry* dep, Process *proc, ErtsProcLocks proc_locks); - -#if ERTS_GLB_INLINE_INCL_FUNC_DEF - -ERTS_GLB_INLINE int -erts_dsig_prepare(ErtsDSigData *dsdp, - DistEntry *dep, - Process *proc, - ErtsProcLocks proc_locks, - ErtsDSigPrepLock dspl, - int no_suspend, - int connect) -{ - int res; - - if (!erts_is_alive) - return ERTS_DSIG_PREP_NOT_ALIVE; - if (!dep) { - ASSERT(!connect); - return ERTS_DSIG_PREP_NOT_CONNECTED; - } - -#ifdef ERTS_ENABLE_LOCK_CHECK - if (connect) { - erts_proc_lc_might_unlock(proc, proc_locks); - } -#endif - -retry: - erts_de_rlock(dep); - - if (dep->state == ERTS_DE_STATE_CONNECTED) { - res = ERTS_DSIG_PREP_CONNECTED; - } - else if (dep->state == ERTS_DE_STATE_PENDING) { - res = ERTS_DSIG_PREP_PENDING; - } - else if (dep->state == ERTS_DE_STATE_EXITING) { - res = ERTS_DSIG_PREP_NOT_CONNECTED; - goto fail; - } - else if (connect) { - ASSERT(dep->state == ERTS_DE_STATE_IDLE); - erts_de_runlock(dep); - if (!erts_auto_connect(dep, proc, proc_locks)) { - return ERTS_DSIG_PREP_NOT_ALIVE; - } - goto retry; - } - else { - ASSERT(dep->state == ERTS_DE_STATE_IDLE); - res = ERTS_DSIG_PREP_NOT_CONNECTED; - goto fail; - } - - if (no_suspend) { - if (erts_atomic32_read_acqb(&dep->qflgs) & ERTS_DE_QFLG_BUSY) { - res = ERTS_DSIG_PREP_WOULD_SUSPEND; - goto fail; - } - } - dsdp->proc = proc; - dsdp->dep = dep; - dsdp->cid = dep->cid; - dsdp->connection_id = dep->connection_id; - dsdp->no_suspend = no_suspend; - dsdp->flags = dep->flags; - if (dspl == ERTS_DSP_NO_LOCK) - erts_de_runlock(dep); - return res; - - fail: - erts_de_runlock(dep); - return res; -} - -ERTS_GLB_INLINE -void erts_schedule_dist_command(Port *prt, DistEntry *dist_entry) -{ - DistEntry *dep; - Eterm id; - - if (prt) { - ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); - ASSERT((erts_atomic32_read_nob(&prt->state) - & ERTS_PORT_SFLGS_DEAD) == 0); - - dep = (DistEntry*) erts_prtsd_get(prt, ERTS_PRTSD_DIST_ENTRY); - ASSERT(dep); - id = prt->common.id; - } - else { - ASSERT(dist_entry); - ERTS_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&dist_entry->rwmtx) - || erts_lc_rwmtx_is_rwlocked(&dist_entry->rwmtx)); - ASSERT(is_internal_port(dist_entry->cid)); - - dep = dist_entry; - id = dep->cid; - } - - if (!erts_atomic_xchg_mb(&dep->dist_cmd_scheduled, 1)) - erts_port_task_schedule(id, &dep->dist_cmd, ERTS_PORT_TASK_DIST_CMD); -} - -#endif - #ifdef DEBUG #define ERTS_DBG_CHK_NO_DIST_LNK(D, R, L) \ erts_dbg_chk_no_dist_proc_link((D), (R), (L)) @@ -336,41 +231,45 @@ enum erts_dsig_send_phase { ERTS_DSIG_SEND_PHASE_MSG_SIZE, ERTS_DSIG_SEND_PHASE_ALLOC, ERTS_DSIG_SEND_PHASE_MSG_ENCODE, - ERTS_DSIG_SEND_PHASE_FIN + ERTS_DSIG_SEND_PHASE_FIN, + ERTS_DSIG_SEND_PHASE_SEND }; -struct erts_dsig_send_context { - enum erts_dsig_send_phase phase; - Sint reds; +typedef struct erts_dsig_send_context { + int connect; + int no_suspend; + int no_trap; Eterm ctl; Eterm msg; - int force_busy; + Eterm from; + Eterm ctl_heap[6]; + Eterm return_term; + + DistEntry *dep; + Eterm node; /* used if dep == NULL */ + Eterm cid; + Eterm connection_id; + int deref_dep; + + enum erts_dsig_send_phase phase; + Sint reds; + Uint32 max_finalize_prepend; Uint data_size, dhdr_ext_size; ErtsAtomCacheMap *acmp; ErtsDistOutputBuf *obuf; + Uint fragments; Uint32 flags; Process *c_p; union { TTBSizeContext sc; TTBEncodeContext ec; }u; -}; -typedef struct { - int suspend; - int connect; - - Eterm ctl_heap[6]; - ErtsDSigData dsd; - DistEntry *dep; - int deref_dep; - struct erts_dsig_send_context dss; - - Eterm return_term; -}ErtsSendContext; +} ErtsDSigSendContext; +typedef struct dist_sequences DistSeqNode; /* * erts_dsig_send_* return values. @@ -380,21 +279,21 @@ typedef struct { #define ERTS_DSIG_SEND_CONTINUE 2 #define ERTS_DSIG_SEND_TOO_LRG 3 -extern int erts_dsig_send_link(ErtsDSigData *, Eterm, Eterm); -extern int erts_dsig_send_msg(Eterm, Eterm, ErtsSendContext*); -extern int erts_dsig_send_exit_tt(ErtsDSigData *, Eterm, Eterm, Eterm, Eterm); -extern int erts_dsig_send_unlink(ErtsDSigData *, Eterm, Eterm); -extern int erts_dsig_send_reg_msg(Eterm, Eterm, ErtsSendContext*); -extern int erts_dsig_send_group_leader(ErtsDSigData *, Eterm, Eterm); -extern int erts_dsig_send_exit(ErtsDSigData *, Eterm, Eterm, Eterm); -extern int erts_dsig_send_exit2(ErtsDSigData *, Eterm, Eterm, Eterm); -extern int erts_dsig_send_demonitor(ErtsDSigData *, Eterm, Eterm, Eterm, int); -extern int erts_dsig_send_monitor(ErtsDSigData *, Eterm, Eterm, Eterm); -extern int erts_dsig_send_m_exit(ErtsDSigData *, Eterm, Eterm, Eterm, Eterm); - -extern int erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx); +extern int erts_dsig_send_msg(ErtsDSigSendContext*, Eterm, Eterm); +extern int erts_dsig_send_reg_msg(ErtsDSigSendContext*, Eterm, Eterm); +extern int erts_dsig_send_link(ErtsDSigSendContext *, Eterm, Eterm); +extern int erts_dsig_send_exit_tt(ErtsDSigSendContext *, Eterm, Eterm, Eterm, Eterm); +extern int erts_dsig_send_unlink(ErtsDSigSendContext *, Eterm, Eterm); +extern int erts_dsig_send_group_leader(ErtsDSigSendContext *, Eterm, Eterm); +extern int erts_dsig_send_exit(ErtsDSigSendContext *, Eterm, Eterm, Eterm); +extern int erts_dsig_send_exit2(ErtsDSigSendContext *, Eterm, Eterm, Eterm); +extern int erts_dsig_send_demonitor(ErtsDSigSendContext *, Eterm, Eterm, Eterm); +extern int erts_dsig_send_monitor(ErtsDSigSendContext *, Eterm, Eterm, Eterm); +extern int erts_dsig_send_m_exit(ErtsDSigSendContext *, Eterm, Eterm, Eterm, Eterm); + +extern int erts_dsig_send(ErtsDSigSendContext *dsdp); extern int erts_dsend_context_dtor(Binary*); -extern Eterm erts_dsend_export_trap_context(Process* p, ErtsSendContext* ctx); +extern Eterm erts_dsend_export_trap_context(Process* p, ErtsDSigSendContext* ctx); extern int erts_dist_command(Port *prt, int reds); extern void erts_dist_port_not_busy(Port *prt); @@ -404,5 +303,18 @@ extern Uint erts_dist_cache_size(void); extern Sint erts_abort_connection_rwunlock(DistEntry *dep); +extern void erts_dist_seq_tree_foreach( + DistEntry *dep, + int (*func)(ErtsDistExternal *, void*, Sint), void *args); + +extern int erts_dsig_prepare(ErtsDSigSendContext *, + DistEntry*, + Process *, + ErtsProcLocks, + ErtsDSigPrepLock, + int, + int, + int); +int erts_auto_connect(DistEntry* dep, Process *proc, ErtsProcLocks proc_locks); #endif diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 08dcc5ea9a..490a033b8a 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -275,6 +275,8 @@ type ML_DIST STANDARD SYSTEM monitor_link_dist type PF3_ARGS SHORT_LIVED PROCESSES process_flag_3_arguments type SETUP_CONN_ARG SHORT_LIVED PROCESSES setup_connection_argument type LIST_TRAP SHORT_LIVED PROCESSES list_bif_trap_state +type CONT_EXIT_TRAP SHORT_LIVED PROCESSES continue_exit_trap_state +type SEQ_YIELD_STATE SHORT_LIVED SYSTEM dist_seq_yield_state type ENVIRONMENT SYSTEM SYSTEM environment diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index d238d38d27..8d4464969a 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -7503,7 +7503,7 @@ static int gather_ahist_scan(Allctr_t *allocator, return blocks_scanned; } -static void gather_ahist_append_result(hist_tree_t *node, void *arg) +static int gather_ahist_append_result(hist_tree_t *node, void *arg, Sint reds) { gather_ahist_t *state = (gather_ahist_t*)arg; @@ -7537,6 +7537,7 @@ static void gather_ahist_append_result(hist_tree_t *node, void *arg) /* Plain free is intentional. */ free(node); + return 1; } static void gather_ahist_send(gather_ahist_t *state) @@ -7595,11 +7596,11 @@ static int gather_ahist_finish(void *arg) state->building_result = 1; } - if (hist_tree_rbt_foreach_destroy_yielding(&state->hist_tree, - &gather_ahist_append_result, - state, - &state->hist_tree_yield, - BLOCKSCAN_REDUCTIONS)) { + if (!hist_tree_rbt_foreach_destroy_yielding(&state->hist_tree, + &gather_ahist_append_result, + state, + &state->hist_tree_yield, + BLOCKSCAN_REDUCTIONS)) { return 1; } @@ -7608,10 +7609,11 @@ static int gather_ahist_finish(void *arg) return 0; } -static void gather_ahist_destroy_result(hist_tree_t *node, void *arg) +static int gather_ahist_destroy_result(hist_tree_t *node, void *arg, Sint reds) { (void)arg; free(node); + return 1; } static void gather_ahist_abort(void *arg) diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 6f4e34e1a8..74708b2caa 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -227,7 +227,7 @@ bld_magic_ref_bin_list(Uint **hpp, Uint *szp, ErlOffHeap* oh) }). */ -static void do_calc_mon_size(ErtsMonitor *mon, void *vpsz) +static int do_calc_mon_size(ErtsMonitor *mon, void *vpsz, Sint reds) { ErtsMonitorData *mdp = erts_monitor_to_data(mon); Uint *psz = vpsz; @@ -238,7 +238,8 @@ static void do_calc_mon_size(ErtsMonitor *mon, void *vpsz) else *psz += is_immed(mon->other.item) ? 0 : NC_HEAP_SIZE(mon->other.item); - *psz += 9; /* CONS + 6-tuple */ + *psz += 9; /* CONS + 6-tuple */ + return 1; } typedef struct { @@ -248,7 +249,7 @@ typedef struct { Eterm tag; } MonListContext; -static void do_make_one_mon_element(ErtsMonitor *mon, void * vpmlc) +static int do_make_one_mon_element(ErtsMonitor *mon, void * vpmlc, Sint reds) { ErtsMonitorData *mdp = erts_monitor_to_data(mon); MonListContext *pmlc = vpmlc; @@ -319,6 +320,7 @@ static void do_make_one_mon_element(ErtsMonitor *mon, void * vpmlc) pmlc->hp += 7; pmlc->res = CONS(pmlc->hp, tup, pmlc->res); pmlc->hp += 2; + return 1; } static Eterm @@ -328,7 +330,7 @@ make_monitor_list(Process *p, int tree, ErtsMonitor *root, Eterm tail) Uint sz = 0; MonListContext mlc; void (*foreach)(ErtsMonitor *, - void (*)(ErtsMonitor *, void *), + ErtsMonitorFunc, void *); foreach = tree ? erts_monitor_tree_foreach : erts_monitor_list_foreach; @@ -354,7 +356,7 @@ make_monitor_list(Process *p, int tree, ErtsMonitor *root, Eterm tail) }). */ -static void calc_lnk_size(ErtsLink *lnk, void *vpsz) +static int calc_lnk_size(ErtsLink *lnk, void *vpsz, Sint reds) { Uint *psz = vpsz; Uint sz = 0; @@ -364,7 +366,8 @@ static void calc_lnk_size(ErtsLink *lnk, void *vpsz) *psz += sz; *psz += is_immed(lnk->other.item) ? 0 : size_object(lnk->other.item); - *psz += 7; /* CONS + 4-tuple */ + *psz += 7; /* CONS + 4-tuple */ + return 1; } typedef struct { @@ -374,7 +377,7 @@ typedef struct { Eterm tag; } LnkListContext; -static void make_one_lnk_element(ErtsLink *lnk, void * vpllc) +static int make_one_lnk_element(ErtsLink *lnk, void * vpllc, Sint reds) { LnkListContext *pllc = vpllc; Eterm tup, t, pid, id; @@ -411,6 +414,7 @@ static void make_one_lnk_element(ErtsLink *lnk, void * vpllc) pllc->hp += 5; pllc->res = CONS(pllc->hp, tup, pllc->res); pllc->hp += 2; + return 1; } static Eterm @@ -420,7 +424,7 @@ make_link_list(Process *p, int tree, ErtsLink *root, Eterm tail) Uint sz = 0; LnkListContext llc; void (*foreach)(ErtsLink *, - void (*)(ErtsLink *, void *), + ErtsLinkFunc, void *); foreach = tree ? erts_link_tree_foreach : erts_link_list_foreach; @@ -519,16 +523,17 @@ do { \ } \ } while (0) -static void collect_one_link(ErtsLink *lnk, void *vmicp) +static int collect_one_link(ErtsLink *lnk, void *vmicp, Sint reds) { MonitorInfoCollection *micp = vmicp; EXTEND_MONITOR_INFOS(micp); micp->mi[micp->mi_i].entity.term = lnk->other.item; micp->sz += 2 + NC_HEAP_SIZE(lnk->other.item); micp->mi_i++; + return 1; } -static void collect_one_origin_monitor(ErtsMonitor *mon, void *vmicp) +static int collect_one_origin_monitor(ErtsMonitor *mon, void *vmicp, Sint reds) { if (erts_monitor_is_origin(mon)) { MonitorInfoCollection *micp = vmicp; @@ -573,9 +578,10 @@ static void collect_one_origin_monitor(ErtsMonitor *mon, void *vmicp) break; } } + return 1; } -static void collect_one_target_monitor(ErtsMonitor *mon, void *vmicp) +static int collect_one_target_monitor(ErtsMonitor *mon, void *vmicp, Sint reds) { MonitorInfoCollection *micp = vmicp; @@ -612,8 +618,8 @@ static void collect_one_target_monitor(ErtsMonitor *mon, void *vmicp) default: break; } - } + return 1; } typedef struct { @@ -653,8 +659,8 @@ do { \ } \ } while (0) -static void -collect_one_suspend_monitor(ErtsMonitor *mon, void *vsmicp) +static int +collect_one_suspend_monitor(ErtsMonitor *mon, void *vsmicp, Sint reds) { if (mon->type == ERTS_MON_TYPE_SUSPEND) { Sint count; @@ -678,6 +684,7 @@ collect_one_suspend_monitor(ErtsMonitor *mon, void *vsmicp) smicp->smi_i++; } + return 1; } /* @@ -3144,14 +3151,16 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_ERROR(BIF_P, BADARG); } -static void monitor_size(ErtsMonitor *mon, void *vsz) +static int monitor_size(ErtsMonitor *mon, void *vsz, Sint reds) { *((Uint *) vsz) = erts_monitor_size(mon); + return 1; } -static void link_size(ErtsMonitor *lnk, void *vsz) +static int link_size(ErtsMonitor *lnk, void *vsz, Sint reds) { *((Uint *) vsz) = erts_link_size(lnk); + return 1; } /**********************************************************************/ diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index df6f42edd3..4132a54934 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -3701,7 +3701,7 @@ static SWord proc_cleanup_fixed_table(Process* p, DbFixation* fix) /* * erts_db_process_exiting() is called when a process terminates. * It returns 0 when completely done, and !0 when it wants to - * yield. c_p->u.terminate can hold a pointer to a state while + * yield. *yield_state can hold a pointer to a state while * yielding. */ #define ERTS_DB_INTERNAL_ERROR(LSTR) \ @@ -3709,7 +3709,7 @@ static SWord proc_cleanup_fixed_table(Process* p, DbFixation* fix) __FILE__, __LINE__) int -erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks) +erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks, void **yield_state) { typedef struct { enum { @@ -3719,7 +3719,7 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks) }op; DbTable *tb; } CleanupState; - CleanupState *state = (CleanupState *) c_p->u.terminate; + CleanupState *state = (CleanupState *) *yield_state; Eterm pid = c_p->common.id; CleanupState default_state; SWord initial_reds = ERTS_BIF_REDS_LEFT(c_p); @@ -3792,7 +3792,7 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks) if (state != &default_state) erts_free(ERTS_ALC_T_DB_PROC_CLEANUP, state); - c_p->u.terminate = NULL; + *yield_state = NULL; BUMP_REDS(c_p, (initial_reds - reds)); return 0; @@ -3812,12 +3812,12 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks) yield: if (state == &default_state) { - c_p->u.terminate = erts_alloc(ERTS_ALC_T_DB_PROC_CLEANUP, - sizeof(CleanupState)); - sys_memcpy(c_p->u.terminate, (void*) state, sizeof(CleanupState)); + *yield_state = erts_alloc(ERTS_ALC_T_DB_PROC_CLEANUP, + sizeof(CleanupState)); + sys_memcpy(*yield_state, (void*) state, sizeof(CleanupState)); } else - ASSERT(state == c_p->u.terminate); + ASSERT(state == *yield_state); return !0; } @@ -3912,7 +3912,7 @@ struct free_fixations_ctx SWord cnt; }; -static void free_fixations_op(DbFixation* fix, void* vctx) +static int free_fixations_op(DbFixation* fix, void* vctx, Sint reds) { struct free_fixations_ctx* ctx = (struct free_fixations_ctx*) vctx; erts_aint_t diff; @@ -3949,6 +3949,7 @@ static void free_fixations_op(DbFixation* fix, void* vctx) ERTS_ETS_MISC_MEM_ADD(-sizeof(DbFixation)); } ctx->cnt++; + return 1; } int erts_db_execute_free_fixation(Process* p, DbFixation* fix) @@ -4093,7 +4094,7 @@ struct fixing_procs_info_ctx Eterm list; }; -static void fixing_procs_info_op(DbFixation* fix, void* vctx) +static int fixing_procs_info_op(DbFixation* fix, void* vctx, Sint reds) { struct fixing_procs_info_ctx* ctx = (struct fixing_procs_info_ctx*) vctx; Eterm* hp; @@ -4103,6 +4104,7 @@ static void fixing_procs_info_op(DbFixation* fix, void* vctx) tpl = TUPLE2(hp, fix->procs.p->common.id, make_small(fix->counter)); hp += 3; ctx->list = CONS(hp, tpl, ctx->list); + return 1; } static Eterm table_info(Process* p, DbTable* tb, Eterm What) diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h index 5955d42aae..dc77fbb60c 100644 --- a/erts/emulator/beam/erl_db.h +++ b/erts/emulator/beam/erl_db.h @@ -111,7 +111,7 @@ typedef enum { } ErtsDbSpinCount; void init_db(ErtsDbSpinCount); -int erts_db_process_exiting(Process *, ErtsProcLocks); +int erts_db_process_exiting(Process *, ErtsProcLocks, void **); int erts_db_execute_free_fixation(Process*, DbFixation*); void db_info(fmtfn_t, void *, int); void erts_db_foreach_table(void (*)(DbTable *, void *), void *); diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index ec7442aeeb..1ea7074d21 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -3312,7 +3312,7 @@ void db_cleanup_offheap_comp(DbTerm* obj) default: ASSERT(is_external_header(u.hdr->thing_word)); ASSERT(u.pb != &tmp); - erts_deref_node_entry(u.ext->node); + erts_deref_node_entry(u.ext->node, make_boxed(u.ep)); break; } } diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 3a50b294d1..9317850d96 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -1303,7 +1303,8 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, ExternalThing *etp; ASSERT(is_external_header(ptr->thing_word)); etp = (ExternalThing *) ptr; - erts_refc_inc(&etp->node->refc, 1); + erts_ref_node_entry(etp->node, 1, + make_boxed(&oh->thing_word)); break; } } @@ -2836,7 +2837,11 @@ sweep_off_heap(Process *p, int fullsweep) while (ptr) { if (IS_MOVED_BOXED(ptr->thing_word)) { ASSERT(!ErtsInArea(ptr, oheap, oheap_sz)); - *prev = ptr = (struct erl_off_heap_header*) boxed_val(ptr->thing_word); + if (is_external_header(((struct erl_off_heap_header*) boxed_val(ptr->thing_word))->thing_word)) + erts_node_bookkeep(((ExternalThing*)ptr)->node, + make_boxed(&ptr->thing_word), + ERL_NODE_DEC); + *prev = ptr = (struct erl_off_heap_header*) boxed_val(ptr->thing_word); ASSERT(!IS_MOVED_BOXED(ptr->thing_word)); switch (ptr->thing_word) { case HEADER_PROC_BIN: { @@ -2863,6 +2868,11 @@ sweep_off_heap(Process *p, int fullsweep) /* fall through... */ } default: + if (is_external_header(ptr->thing_word)) { + erts_node_bookkeep(((ExternalThing*)ptr)->node, + make_boxed(&ptr->thing_word), + ERL_NODE_INC); + } prev = &ptr->next; ptr = ptr->next; } @@ -2896,7 +2906,8 @@ sweep_off_heap(Process *p, int fullsweep) } default: ASSERT(is_external_header(ptr->thing_word)); - erts_deref_node_entry(((ExternalThing*)ptr)->node); + erts_deref_node_entry(((ExternalThing*)ptr)->node, + make_boxed(&ptr->thing_word)); } *prev = ptr = ptr->next; } @@ -3029,6 +3040,13 @@ offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size) { struct erl_off_heap_header* oh = (struct erl_off_heap_header*) hp; + if (is_external_header(oh->thing_word)) { + erts_node_bookkeep(((ExternalThing*)oh)->node, + make_boxed(((Eterm*)oh)-offs), ERL_NODE_DEC); + erts_node_bookkeep(((ExternalThing*)oh)->node, + make_boxed((Eterm*)oh), ERL_NODE_INC); + } + if (ErtsInArea(oh->next, area, area_size)) { Eterm** uptr = (Eterm **) (void *) &oh->next; *uptr += offs; /* Patch the mso chain */ diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c index 75ad6de2c9..b0eb0e85c0 100644 --- a/erts/emulator/beam/erl_hl_timer.c +++ b/erts/emulator/beam/erl_hl_timer.c @@ -2263,9 +2263,10 @@ parse_bif_timer_options(Eterm option_list, int *async, return 1; } -static void -exit_cancel_bif_timer(ErtsBifTimer *tmr, void *vesdp) +static int +exit_cancel_bif_timer(ErtsBifTimer *tmr, void *vesdp, Sint reds) { +#define ERTS_BTM_CANCEL_REDS 80 ErtsSchedulerData *esdp = (ErtsSchedulerData *) vesdp; Uint32 sid, roflgs; erts_aint_t state; @@ -2290,7 +2291,7 @@ exit_cancel_bif_timer(ErtsBifTimer *tmr, void *vesdp) if (sid != (Uint32) esdp->no) { queue_canceled_timer(esdp, sid, (ErtsTimer *) tmr); - return; + return ERTS_BTM_CANCEL_REDS; } if (tmr->btm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { @@ -2306,14 +2307,9 @@ exit_cancel_bif_timer(ErtsBifTimer *tmr, void *vesdp) hl_timer_dec_refc(&tmr->type.hlt, roflgs); else tw_timer_dec_refc(&tmr->type.twt); + return ERTS_BTM_CANCEL_REDS; } -#ifdef ERTS_HLT_DEBUG -# define ERTS_BTM_MAX_DESTROY_LIMIT 2 -#else -# define ERTS_BTM_MAX_DESTROY_LIMIT 50 -#endif - typedef struct { ErtsBifTimers *bif_timers; union { @@ -2321,10 +2317,9 @@ typedef struct { } u; } ErtsBifTimerYieldState; -int erts_cancel_bif_timers(Process *p, ErtsBifTimers **btm, void **vyspp) +int erts_cancel_bif_timers(Process *p, ErtsBifTimers **btm, void **vyspp, int reds) { ErtsSchedulerData *esdp = erts_proc_sched_data(p); - ErtsBifTimerYieldState ys = {*btm, {ERTS_RBT_YIELD_STAT_INITER}}; ErtsBifTimerYieldState *ysp; int res; @@ -2337,9 +2332,9 @@ int erts_cancel_bif_timers(Process *p, ErtsBifTimers **btm, void **vyspp) exit_cancel_bif_timer, (void *) esdp, &ysp->u.proc_btm_yield_state, - ERTS_BTM_MAX_DESTROY_LIMIT); + reds); - if (res == 0) { + if (res > 0) { if (ysp != &ys) erts_free(ERTS_ALC_T_BTM_YIELD_STATE, ysp); *vyspp = NULL; @@ -2819,8 +2814,8 @@ btm_print(ErtsBifTimer *tmr, void *vbtmp, ErtsMonotonicTime tpos, int is_hlt) (Sint64) left); } -static void -btm_tree_print(ErtsBifTimer *tmr, void *vbtmp) +static int +btm_tree_print(ErtsBifTimer *tmr, void *vbtmp, Sint reds) { int is_hlt = !!(tmr->type.head.roflgs & ERTS_TMR_ROFLG_HLT); ErtsMonotonicTime tpos; @@ -2829,6 +2824,7 @@ btm_tree_print(ErtsBifTimer *tmr, void *vbtmp) else tpos = erts_tweel_read_timeout(&tmr->type.twt.u.tw_tmr); btm_print(tmr, vbtmp, tpos, is_hlt); + return 1; } void @@ -2860,8 +2856,8 @@ typedef struct { void *arg; } ErtsBTMForeachDebug; -static void -debug_btm_foreach(ErtsBifTimer *tmr, void *vbtmfd) +static int +debug_btm_foreach(ErtsBifTimer *tmr, void *vbtmfd, Sint reds) { if (erts_atomic32_read_nob(&tmr->btm.state) == ERTS_TMR_STATE_ACTIVE) { ErtsBTMForeachDebug *btmfd = (ErtsBTMForeachDebug *) vbtmfd; @@ -2870,6 +2866,7 @@ debug_btm_foreach(ErtsBifTimer *tmr, void *vbtmfd) : tmr->type.head.receiver.proc->common.id); (*btmfd->func)(id, tmr->btm.message, tmr->btm.bp, btmfd->arg); } + return 1; } void @@ -2918,8 +2915,8 @@ debug_callback_timer_foreach_list(ErtsHLTimer *tmr, void *vdfct) tmr->head.u.arg); } -static void -debug_callback_timer_foreach(ErtsHLTimer *tmr, void *vdfct) +static int +debug_callback_timer_foreach(ErtsHLTimer *tmr, void *vdfct, Sint reds) { ErtsDebugForeachCallbackTimer *dfct = (ErtsDebugForeachCallbackTimer *) vdfct; @@ -2934,6 +2931,7 @@ debug_callback_timer_foreach(ErtsHLTimer *tmr, void *vdfct) (*dfct->func)(dfct->arg, tmr->timeout, tmr->head.u.arg); + return 1; } static void @@ -2981,7 +2979,8 @@ erts_debug_callback_timer_foreach(void (*tclbk)(void *), if (srv->yield.root) debug_callback_timer_foreach(srv->yield.root, - (void *) &dfct); + (void *) &dfct, + -1); time_rbt_foreach(srv->time_tree, debug_callback_timer_foreach, diff --git a/erts/emulator/beam/erl_hl_timer.h b/erts/emulator/beam/erl_hl_timer.h index e6f5e8b67d..29c873868b 100644 --- a/erts/emulator/beam/erl_hl_timer.h +++ b/erts/emulator/beam/erl_hl_timer.h @@ -56,7 +56,7 @@ void erts_cancel_proc_timer(Process *); void erts_set_port_timer(Port *, Sint64); void erts_cancel_port_timer(Port *); Sint64 erts_read_port_timer(Port *); -int erts_cancel_bif_timers(Process *, ErtsBifTimers **, void **); +int erts_cancel_bif_timers(Process *, ErtsBifTimers **, void **, int); int erts_detach_accessor_bif_timers(Process *, ErtsBifTimers *, void **); ErtsHLTimerService *erts_create_timer_service(void); void erts_hl_timer_init(void); diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 12750b9aa6..82d5140d1c 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -2417,12 +2417,17 @@ erts_exit_vv(int n, int flush_async, char *fmt, va_list args1, va_list args2) erts_exit_epilogue(); } +void check_obuf(void); __decl_noreturn void __noreturn erts_exit_epilogue(void) { int n = erts_exit_code; sys_tty_reset(n); +#ifdef DEBUG + check_obuf(); +#endif + if (n == ERTS_INTR_EXIT) exit(0); else if (n == ERTS_DUMP_EXIT) diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 3aab4828cc..39eabb6710 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -164,7 +164,8 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "os_monotonic_time", NULL }, { "erts_alloc_hard_debug", NULL }, { "hard_dbg_mseg", NULL }, - { "erts_mmap", NULL } + { "erts_mmap", NULL }, + { "sad", NULL} }; #define ERTS_LOCK_ORDER_SIZE \ diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index 942bec84cf..9d40754d2d 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -181,7 +181,7 @@ erts_cleanup_offheap(ErlOffHeap *offheap) break; default: ASSERT(is_external_header(u.hdr->thing_word)); - erts_deref_node_entry(u.ext->node); + erts_deref_node_entry(u.ext->node, make_boxed(u.ep)); break; } } @@ -201,34 +201,44 @@ free_message_buffer(ErlHeapFragment* bp) }while (bp != NULL); } +static void +erts_cleanup_message(ErtsMessage *mp) +{ + ErlHeapFragment *bp; + if (ERTS_SIG_IS_EXTERNAL_MSG(mp) || ERTS_SIG_IS_NON_MSG(mp)) { + ErtsDistExternal *edep = erts_proc_sig_get_external(mp); + if (edep) { + erts_free_dist_ext_copy(edep); + if (mp->data.heap_frag == &mp->hfrag) { + ASSERT(ERTS_SIG_IS_EXTERNAL_MSG(mp)); + mp->data.heap_frag = ERTS_MSG_COMBINED_HFRAG; + } + } + } + + if (ERTS_SIG_IS_MSG(mp) && mp->data.attached != ERTS_MSG_COMBINED_HFRAG) { + bp = mp->data.heap_frag; + } else { + /* All non msg signals are combined HFRAG messages, + but we overwrite the mp->data field with the + nm_signal queue ptr so have to fix that here + before freeing it. */ + mp->data.attached = ERTS_MSG_COMBINED_HFRAG; + bp = mp->hfrag.next; + erts_cleanup_offheap(&mp->hfrag.off_heap); + } + + if (bp) + free_message_buffer(bp); +} + void erts_cleanup_messages(ErtsMessage *msgp) { ErtsMessage *mp = msgp; while (mp) { ErtsMessage *fmp; - ErlHeapFragment *bp; - if (ERTS_SIG_IS_EXTERNAL_MSG(mp)) { - if (is_not_immed(ERL_MESSAGE_TOKEN(mp))) { - bp = (ErlHeapFragment *) mp->data.dist_ext->ext_endp; - erts_cleanup_offheap(&bp->off_heap); - } - if (mp->data.dist_ext) - erts_free_dist_ext_copy(mp->data.dist_ext); - } - else { - if (ERTS_SIG_IS_INTERNAL_MSG(mp) - && mp->data.attached != ERTS_MSG_COMBINED_HFRAG) { - bp = mp->data.heap_frag; - } - else { - mp->data.attached = ERTS_MSG_COMBINED_HFRAG; - bp = mp->hfrag.next; - erts_cleanup_offheap(&mp->hfrag.off_heap); - } - if (bp) - free_message_buffer(bp); - } + erts_cleanup_message(mp); fmp = mp; mp = mp->next; erts_free_message(fmp); @@ -260,6 +270,7 @@ void erts_queue_dist_message(Process *rcvr, ErtsProcLocks rcvr_locks, ErtsDistExternal *dist_ext, + ErlHeapFragment *hfrag, Eterm token, Eterm from) { @@ -268,8 +279,26 @@ erts_queue_dist_message(Process *rcvr, ERTS_LC_ASSERT(rcvr_locks == erts_proc_lc_my_proc_locks(rcvr)); - mp = erts_alloc_message(0, NULL); - mp->data.dist_ext = dist_ext; + if (hfrag) { + /* Fragmented message, allocate a message reference */ + mp = erts_alloc_message(0, NULL); + mp->data.heap_frag = hfrag; + } else { + /* Un-fragmented message, allocate space for + token and dist_ext in message. */ + Uint dist_ext_sz = erts_dist_ext_size(dist_ext) / sizeof(Eterm); + Uint token_sz = size_object(token); + Uint sz = token_sz + dist_ext_sz; + Eterm *hp; + + mp = erts_alloc_message(sz, &hp); + mp->data.heap_frag = &mp->hfrag; + mp->hfrag.used_size = token_sz; + + erts_make_dist_ext_copy(dist_ext, erts_get_dist_ext(mp->data.heap_frag)); + + token = copy_struct(token, token_sz, &hp, &mp->data.heap_frag->off_heap); + } ERL_MESSAGE_FROM(mp) = dist_ext->dep->sysname; ERL_MESSAGE_TERM(mp) = THE_NON_VALUE; @@ -493,25 +522,27 @@ Uint erts_msg_attached_data_size_aux(ErtsMessage *msg) { Sint sz; - ASSERT(is_non_value(ERL_MESSAGE_TERM(msg))); - ASSERT(msg->data.dist_ext); - ASSERT(msg->data.dist_ext->heap_size < 0); - - sz = erts_decode_dist_ext_size(msg->data.dist_ext); - if (sz < 0) { - /* Bad external - * We leave the message intact in this case as it's not worth the trouble - * to make all callers remove it from queue. It will be detected again - * and removed from message queue later anyway. - */ - return 0; - } + ErtsDistExternal *edep = erts_get_dist_ext(msg->data.heap_frag); + ASSERT(ERTS_SIG_IS_EXTERNAL_MSG(msg)); - msg->data.dist_ext->heap_size = sz; - if (is_not_nil(msg->m[1])) { - ErlHeapFragment *heap_frag; - heap_frag = erts_dist_ext_trailer(msg->data.dist_ext); - sz += heap_frag->used_size; + if (edep->heap_size < 0) { + + sz = erts_decode_dist_ext_size(edep, 1); + if (sz < 0) { + /* Bad external + * We leave the message intact in this case as it's not worth the trouble + * to make all callers remove it from queue. It will be detected again + * and removed from message queue later anyway. + */ + return 0; + } + + edep->heap_size = sz; + } else { + sz = edep->heap_size; + } + if (is_not_nil(ERL_MESSAGE_TOKEN(msg))) { + sz += msg->data.heap_frag->used_size; } return sz; } @@ -1099,80 +1130,6 @@ change_to_off_heap: return res; } -int -erts_decode_dist_message(Process *proc, ErtsProcLocks proc_locks, - ErtsMessage *msgp, int force_off_heap) -{ - ErtsHeapFactory factory; - Eterm msg; - ErlHeapFragment *bp; - Sint need; - int decode_in_heap_frag; - - decode_in_heap_frag = (force_off_heap - || !(proc_locks & ERTS_PROC_LOCK_MAIN) - || (proc->flags & F_OFF_HEAP_MSGQ)); - - if (msgp->data.dist_ext->heap_size >= 0) - need = msgp->data.dist_ext->heap_size; - else { - need = erts_decode_dist_ext_size(msgp->data.dist_ext); - if (need < 0) { - /* bad msg; remove it... */ - if (is_not_immed(ERL_MESSAGE_TOKEN(msgp))) { - bp = erts_dist_ext_trailer(msgp->data.dist_ext); - erts_cleanup_offheap(&bp->off_heap); - } - erts_free_dist_ext_copy(msgp->data.dist_ext); - msgp->data.dist_ext = NULL; - return 0; - } - - msgp->data.dist_ext->heap_size = need; - } - - if (is_not_immed(ERL_MESSAGE_TOKEN(msgp))) { - bp = erts_dist_ext_trailer(msgp->data.dist_ext); - need += bp->used_size; - } - - if (decode_in_heap_frag) - erts_factory_heap_frag_init(&factory, new_message_buffer(need)); - else - erts_factory_proc_prealloc_init(&factory, proc, need); - - ASSERT(msgp->data.dist_ext->heap_size >= 0); - if (is_not_immed(ERL_MESSAGE_TOKEN(msgp))) { - ErlHeapFragment *heap_frag; - heap_frag = erts_dist_ext_trailer(msgp->data.dist_ext); - ERL_MESSAGE_TOKEN(msgp) = copy_struct(ERL_MESSAGE_TOKEN(msgp), - heap_frag->used_size, - &factory.hp, - factory.off_heap); - erts_cleanup_offheap(&heap_frag->off_heap); - } - - msg = erts_decode_dist_ext(&factory, msgp->data.dist_ext); - ERL_MESSAGE_TERM(msgp) = msg; - erts_free_dist_ext_copy(msgp->data.dist_ext); - msgp->data.attached = NULL; - - if (is_non_value(msg)) { - erts_factory_undo(&factory); - return 0; - } - - erts_factory_trim_and_close(&factory, msgp->m, - ERL_MESSAGE_REF_ARRAY_SZ); - - ASSERT(!msgp->data.heap_frag); - - if (decode_in_heap_frag) - msgp->data.heap_frag = factory.heap_frags; - - return 1; -} - void erts_factory_proc_init(ErtsHeapFactory* factory, Process* p) { @@ -1233,7 +1190,7 @@ erts_factory_message_create(ErtsHeapFactory* factory, int on_heap; erts_aint32_t state; - state = proc ? erts_atomic32_read_nob(&proc->state) : 0; + state = proc ? erts_atomic32_read_nob(&proc->state) : ERTS_PSFLG_OFF_HEAP_MSGQ; if (state & ERTS_PSFLG_OFF_HEAP_MSGQ) { msgp = erts_alloc_message(sz, &hp); @@ -1466,8 +1423,8 @@ void erts_factory_close(ErtsHeapFactory* factory) else factory->message->data.heap_frag = factory->heap_frags; - /* Fall through */ - case FACTORY_HEAP_FRAGS: + /* Fall through */ + case FACTORY_HEAP_FRAGS: bp = factory->heap_frags; } diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index b2550814fd..4c2674394e 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -138,7 +138,7 @@ typedef struct erl_heap_fragment ErlHeapFragment; struct erl_heap_fragment { ErlHeapFragment* next; /* Next heap fragment */ ErlOffHeap off_heap; /* Offset heap data. */ - Uint alloc_size; /* Size in (half)words of mem */ + Uint alloc_size; /* Size in words of mem */ Uint used_size; /* With terms to be moved to heap by GC */ Eterm mem[1]; /* Data */ }; @@ -167,7 +167,6 @@ struct erl_heap_fragment { #define ERL_MESSAGE_REF_FIELDS__ \ ErtsMessage *next; /* Next message */ \ union { \ - ErtsDistExternal *dist_ext; \ ErlHeapFragment *heap_frag; \ void *attached; \ } data; \ @@ -438,7 +437,8 @@ ErlHeapFragment* new_message_buffer(Uint); ErlHeapFragment* erts_resize_message_buffer(ErlHeapFragment *, Uint, Eterm *, Uint); void free_message_buffer(ErlHeapFragment *); -void erts_queue_dist_message(Process*, ErtsProcLocks, ErtsDistExternal *, Eterm, Eterm); +void erts_queue_dist_message(Process*, ErtsProcLocks, ErtsDistExternal *, + ErlHeapFragment *, Eterm, Eterm); void erts_queue_message(Process*, ErtsProcLocks,ErtsMessage*, Eterm, Eterm); void erts_queue_proc_message(Process* from,Process* to, ErtsProcLocks,ErtsMessage*, Eterm); void erts_queue_proc_messages(Process* from, Process* to, ErtsProcLocks, @@ -455,8 +455,6 @@ Sint erts_move_messages_off_heap(Process *c_p); Sint erts_complete_off_heap_message_queue_change(Process *c_p); Eterm erts_change_message_queue_management(Process *c_p, Eterm new_state); -int erts_decode_dist_message(Process *, ErtsProcLocks, ErtsMessage *, int); - void erts_cleanup_messages(ErtsMessage *mp); void *erts_alloc_message_ref(void); @@ -585,22 +583,11 @@ ERTS_GLB_INLINE Uint erts_used_frag_sz(const ErlHeapFragment* bp) ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErtsMessage *msg) { ASSERT(msg->data.attached); - if (is_value(ERL_MESSAGE_TERM(msg))) { - ErlHeapFragment *bp; - bp = erts_message_to_heap_frag(msg); - return erts_used_frag_sz(bp); - } - else if (msg->data.dist_ext->heap_size < 0) - return erts_msg_attached_data_size_aux(msg); - else { - Uint sz = msg->data.dist_ext->heap_size; - if (is_not_nil(ERL_MESSAGE_TOKEN(msg))) { - ErlHeapFragment *heap_frag; - heap_frag = erts_dist_ext_trailer(msg->data.dist_ext); - sz += heap_frag->used_size; - } - return sz; - } + + if (ERTS_SIG_IS_INTERNAL_MSG(msg)) + return erts_used_frag_sz(erts_message_to_heap_frag(msg)); + + return erts_msg_attached_data_size_aux(msg); } #endif diff --git a/erts/emulator/beam/erl_monitor_link.c b/erts/emulator/beam/erl_monitor_link.c index 48d9bd4ca5..1c6b4afaa3 100644 --- a/erts/emulator/beam/erl_monitor_link.c +++ b/erts/emulator/beam/erl_monitor_link.c @@ -191,7 +191,8 @@ ml_cmp_keys(Eterm key1, Eterm key2) if (n1->sysname != n2->sysname) return n1->sysname < n2->sysname ? -1 : 1; ASSERT(n1->creation != n2->creation); - return n1->creation < n2->creation ? -1 : 1; + if (n1->creation != 0 && n2->creation != 0) + return n1->creation < n2->creation ? -1 : 1; } ndw1 = external_thing_data_words(et1); @@ -335,7 +336,7 @@ ml_rbt_delete(ErtsMonLnkNode **root, ErtsMonLnkNode *ml) static void ml_rbt_foreach(ErtsMonLnkNode *root, - void (*func)(ErtsMonLnkNode *, void *), + ErtsMonLnkNodeFunc func, void *arg) { mon_lnk_rbt_foreach(root, func, arg); @@ -348,7 +349,7 @@ typedef struct { static int ml_rbt_foreach_yielding(ErtsMonLnkNode *root, - void (*func)(ErtsMonLnkNode *, void *), + ErtsMonLnkNodeFunc func, void *arg, void **vyspp, Sint limit) @@ -362,7 +363,7 @@ ml_rbt_foreach_yielding(ErtsMonLnkNode *root, ysp = &ys; res = mon_lnk_rbt_foreach_yielding(ysp->root, func, arg, &ysp->rbt_ystate, limit); - if (res == 0) { + if (res > 0) { if (ysp != &ys) erts_free(ERTS_ALC_T_ML_YIELD_STATE, ysp); *vyspp = NULL; @@ -383,22 +384,22 @@ ml_rbt_foreach_yielding(ErtsMonLnkNode *root, } typedef struct { - void (*func)(ErtsMonLnkNode *, void *); + ErtsMonLnkNodeFunc func; void *arg; } ErtsMonLnkForeachDeleteContext; -static void -rbt_wrap_foreach_delete(ErtsMonLnkNode *ml, void *vctxt) +static int +rbt_wrap_foreach_delete(ErtsMonLnkNode *ml, void *vctxt, Sint reds) { ErtsMonLnkForeachDeleteContext *ctxt = vctxt; ERTS_ML_ASSERT(ml->flags & ERTS_ML_FLG_IN_TABLE); ml->flags &= ~ERTS_ML_FLG_IN_TABLE; - ctxt->func(ml, ctxt->arg); + return ctxt->func(ml, ctxt->arg, reds); } static void ml_rbt_foreach_delete(ErtsMonLnkNode **root, - void (*func)(ErtsMonLnkNode *, void *), + ErtsMonLnkNodeFunc func, void *arg) { ErtsMonLnkForeachDeleteContext ctxt; @@ -411,7 +412,7 @@ ml_rbt_foreach_delete(ErtsMonLnkNode **root, static int ml_rbt_foreach_delete_yielding(ErtsMonLnkNode **root, - void (*func)(ErtsMonLnkNode *, void *), + ErtsMonLnkNodeFunc func, void *arg, void **vyspp, Sint limit) @@ -433,7 +434,7 @@ ml_rbt_foreach_delete_yielding(ErtsMonLnkNode **root, (void *) &ctxt, &ysp->rbt_ystate, limit); - if (res == 0) { + if (res > 0) { if (ysp != &ys) erts_free(ERTS_ALC_T_ML_YIELD_STATE, ysp); *vyspp = NULL; @@ -459,12 +460,11 @@ ml_rbt_foreach_delete_yielding(ErtsMonLnkNode **root, static int ml_dl_list_foreach_yielding(ErtsMonLnkNode *list, - void (*func)(ErtsMonLnkNode *, void *), + ErtsMonLnkNodeFunc func, void *arg, void **vyspp, - Sint limit) + Sint reds) { - Sint cnt = 0; ErtsMonLnkNode *ml = (ErtsMonLnkNode *) *vyspp; ERTS_ML_ASSERT(!ml || list); @@ -475,28 +475,26 @@ ml_dl_list_foreach_yielding(ErtsMonLnkNode *list, if (ml) { do { ERTS_ML_ASSERT(ml->flags & ERTS_ML_FLG_IN_TABLE); - func(ml, arg); + reds -= func(ml, arg, reds); ml = ml->node.list.next; - cnt++; - } while (ml != list && cnt < limit); + } while (ml != list && reds > 0); if (ml != list) { *vyspp = (void *) ml; - return 1; /* yield */ + return 0; /* yield */ } } *vyspp = NULL; - return 0; /* done */ + return reds <= 0 ? 1 : reds; /* done */ } static int ml_dl_list_foreach_delete_yielding(ErtsMonLnkNode **list, - void (*func)(ErtsMonLnkNode *, void *), + ErtsMonLnkNodeFunc func, void *arg, void **vyspp, - Sint limit) + Sint reds) { - Sint cnt = 0; ErtsMonLnkNode *first = *list; ErtsMonLnkNode *ml = (ErtsMonLnkNode *) *vyspp; @@ -510,19 +508,18 @@ ml_dl_list_foreach_delete_yielding(ErtsMonLnkNode **list, ErtsMonLnkNode *next = ml->node.list.next; ERTS_ML_ASSERT(ml->flags & ERTS_ML_FLG_IN_TABLE); ml->flags &= ~ERTS_ML_FLG_IN_TABLE; - func(ml, arg); + reds -= func(ml, arg, reds); ml = next; - cnt++; - } while (ml != first && cnt < limit); + } while (ml != first && reds > 0); if (ml != first) { *vyspp = (void *) ml; - return 1; /* yield */ + return 0; /* yield */ } } *vyspp = NULL; *list = NULL; - return 0; /* done */ + return reds <= 0 ? 1 : reds; /* done */ } /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ @@ -666,91 +663,91 @@ erts_monitor_tree_delete(ErtsMonitor **root, ErtsMonitor *mon) void erts_monitor_tree_foreach(ErtsMonitor *root, - void (*func)(ErtsMonitor *, void *), + ErtsMonitorFunc func, void *arg) { ml_rbt_foreach((ErtsMonLnkNode *) root, - (void (*)(ErtsMonLnkNode*, void*)) func, + (ErtsMonLnkNodeFunc) func, arg); } int erts_monitor_tree_foreach_yielding(ErtsMonitor *root, - void (*func)(ErtsMonitor *, void *), + ErtsMonitorFunc func, void *arg, void **vyspp, Sint limit) { return ml_rbt_foreach_yielding((ErtsMonLnkNode *) root, - (void (*)(ErtsMonLnkNode*, void*)) func, + (int (*)(ErtsMonLnkNode*, void*, Sint)) func, arg, vyspp, limit); } void erts_monitor_tree_foreach_delete(ErtsMonitor **root, - void (*func)(ErtsMonitor *, void *), + ErtsMonitorFunc func, void *arg) { ml_rbt_foreach_delete((ErtsMonLnkNode **) root, - (void (*)(ErtsMonLnkNode*, void*)) func, + (int (*)(ErtsMonLnkNode*, void*, Sint)) func, arg); } int erts_monitor_tree_foreach_delete_yielding(ErtsMonitor **root, - void (*func)(ErtsMonitor *, void *), + ErtsMonitorFunc func, void *arg, void **vyspp, Sint limit) { return ml_rbt_foreach_delete_yielding((ErtsMonLnkNode **) root, - (void (*)(ErtsMonLnkNode*, void*)) func, + (int (*)(ErtsMonLnkNode*, void*, Sint)) func, arg, vyspp, limit); } void erts_monitor_list_foreach(ErtsMonitor *list, - void (*func)(ErtsMonitor *, void *), + ErtsMonitorFunc func, void *arg) { void *ystate = NULL; - while (ml_dl_list_foreach_yielding((ErtsMonLnkNode *) list, - (void (*)(ErtsMonLnkNode *, void *)) func, - arg, &ystate, (Sint) INT_MAX)); + while (!ml_dl_list_foreach_yielding((ErtsMonLnkNode *) list, + (int (*)(ErtsMonLnkNode *, void *, Sint)) func, + arg, &ystate, (Sint) INT_MAX)); } int erts_monitor_list_foreach_yielding(ErtsMonitor *list, - void (*func)(ErtsMonitor *, void *), + ErtsMonitorFunc func, void *arg, void **vyspp, Sint limit) { return ml_dl_list_foreach_yielding((ErtsMonLnkNode *) list, - (void (*)(ErtsMonLnkNode *, void *)) func, + (int (*)(ErtsMonLnkNode *, void *, Sint)) func, arg, vyspp, limit); } void erts_monitor_list_foreach_delete(ErtsMonitor **list, - void (*func)(ErtsMonitor *, void *), + ErtsMonitorFunc func, void *arg) { void *ystate = NULL; - while (ml_dl_list_foreach_delete_yielding((ErtsMonLnkNode **) list, - (void (*)(ErtsMonLnkNode*, void*)) func, - arg, &ystate, (Sint) INT_MAX)); + while (!ml_dl_list_foreach_delete_yielding((ErtsMonLnkNode **) list, + (int (*)(ErtsMonLnkNode*, void*, Sint)) func, + arg, &ystate, (Sint) INT_MAX)); } int erts_monitor_list_foreach_delete_yielding(ErtsMonitor **list, - void (*func)(ErtsMonitor *, void *), + ErtsMonitorFunc func, void *arg, void **vyspp, Sint limit) { return ml_dl_list_foreach_delete_yielding((ErtsMonLnkNode **) list, - (void (*)(ErtsMonLnkNode*, void*)) func, + (int (*)(ErtsMonLnkNode*, void*, Sint)) func, arg, vyspp, limit); } @@ -1074,92 +1071,92 @@ erts_link_tree_delete(ErtsLink **root, ErtsLink *lnk) void erts_link_tree_foreach(ErtsLink *root, - void (*func)(ErtsLink *, void *), + ErtsLinkFunc func, void *arg) { ml_rbt_foreach((ErtsMonLnkNode *) root, - (void (*)(ErtsMonLnkNode*, void*)) func, + (ErtsMonLnkNodeFunc) func, arg); } int erts_link_tree_foreach_yielding(ErtsLink *root, - void (*func)(ErtsLink *, void *), + ErtsLinkFunc func, void *arg, void **vyspp, Sint limit) { return ml_rbt_foreach_yielding((ErtsMonLnkNode *) root, - (void (*)(ErtsMonLnkNode*, void*)) func, + (ErtsMonLnkNodeFunc) func, arg, vyspp, limit); } void erts_link_tree_foreach_delete(ErtsLink **root, - void (*func)(ErtsLink *, void *), + ErtsLinkFunc func, void *arg) { ml_rbt_foreach_delete((ErtsMonLnkNode **) root, - (void (*)(ErtsMonLnkNode*, void*)) func, + (ErtsMonLnkNodeFunc) func, arg); } int erts_link_tree_foreach_delete_yielding(ErtsLink **root, - void (*func)(ErtsLink *, void *), + ErtsLinkFunc func, void *arg, void **vyspp, Sint limit) { return ml_rbt_foreach_delete_yielding((ErtsMonLnkNode **) root, - (void (*)(ErtsMonLnkNode*, void*)) func, + (ErtsMonLnkNodeFunc) func, arg, vyspp, limit); } void erts_link_list_foreach(ErtsLink *list, - void (*func)(ErtsLink *, void *), + ErtsLinkFunc func, void *arg) { void *ystate = NULL; - while (ml_dl_list_foreach_yielding((ErtsMonLnkNode *) list, - (void (*)(ErtsMonLnkNode *, void *)) func, - arg, &ystate, (Sint) INT_MAX)); + while (!ml_dl_list_foreach_yielding((ErtsMonLnkNode *) list, + (ErtsMonLnkNodeFunc) func, + arg, &ystate, (Sint) INT_MAX)); } int erts_link_list_foreach_yielding(ErtsLink *list, - void (*func)(ErtsLink *, void *), + ErtsLinkFunc func, void *arg, void **vyspp, Sint limit) { return ml_dl_list_foreach_yielding((ErtsMonLnkNode *) list, - (void (*)(ErtsMonLnkNode *, void *)) func, + (ErtsMonLnkNodeFunc) func, arg, vyspp, limit); } void erts_link_list_foreach_delete(ErtsLink **list, - void (*func)(ErtsLink *, void *), + ErtsLinkFunc func, void *arg) { void *ystate = NULL; - while (ml_dl_list_foreach_delete_yielding((ErtsMonLnkNode **) list, - (void (*)(ErtsMonLnkNode*, void*)) func, - arg, &ystate, (Sint) INT_MAX)); + while (!ml_dl_list_foreach_delete_yielding((ErtsMonLnkNode **) list, + (ErtsMonLnkNodeFunc) func, + arg, &ystate, (Sint) INT_MAX)); } int erts_link_list_foreach_delete_yielding(ErtsLink **list, - void (*func)(ErtsLink *, void *), + int (*func)(ErtsLink *, void *, Sint), void *arg, void **vyspp, Sint limit) { return ml_dl_list_foreach_delete_yielding((ErtsMonLnkNode **) list, - (void (*)(ErtsMonLnkNode*, void*)) func, + (ErtsMonLnkNodeFunc) func, arg, vyspp, limit); } diff --git a/erts/emulator/beam/erl_monitor_link.h b/erts/emulator/beam/erl_monitor_link.h index ed7bf7d54a..eff861fce8 100644 --- a/erts/emulator/beam/erl_monitor_link.h +++ b/erts/emulator/beam/erl_monitor_link.h @@ -439,6 +439,7 @@ (ERTS_ML_FLG_EXTENDED|ERTS_ML_FLG_NAME) typedef struct ErtsMonLnkNode__ ErtsMonLnkNode; +typedef int (*ErtsMonLnkNodeFunc)(ErtsMonLnkNode *, void *, Sint); typedef struct { UWord parent; /* Parent ptr and flags... */ @@ -622,6 +623,7 @@ erts_ml_dl_list_last__(ErtsMonLnkNode *list) typedef struct ErtsMonLnkNode__ ErtsMonitor; +typedef int (*ErtsMonitorFunc)(ErtsMonitor *, void *, Sint); typedef struct { ErtsMonitor origin; @@ -653,6 +655,7 @@ struct ErtsMonitorDataExtended__ { typedef struct ErtsMonitorSuspend__ ErtsMonitorSuspend; + struct ErtsMonitorSuspend__ { ErtsMonitorData md; /* origin = suspender; target = suspendee */ ErtsMonitorSuspend *next; @@ -685,7 +688,7 @@ ErtsMonitor *erts_monitor_tree_lookup(ErtsMonitor *root, Eterm key); * * @brief Lookup or insert a monitor in a monitor tree * - * When the funcion is called it is assumed that: + * When the function is called it is assumed that: * - 'mon' monitor is not part of any tree or list * If the above is not true, bad things will happen. * @@ -711,7 +714,7 @@ ErtsMonitor *erts_monotor_tree_lookup_insert(ErtsMonitor **root, * If it is not found, creates a monitor and returns a pointer to the * origin monitor. * - * When the funcion is called it is assumed that: + * When the function is called it is assumed that: * - no target monitors with the key 'target' exists in the tree. * If the above is not true, bad things will happen. * @@ -738,7 +741,7 @@ ErtsMonitor *erts_monitor_tree_lookup_create(ErtsMonitor **root, int *created, * * @brief Insert a monitor in a monitor tree * - * When the funcion is called it is assumed that: + * When the function is called it is assumed that: * - no monitors with the same key that 'mon' exist in the tree * - 'mon' is not part of any list of tree * If the above are not true, bad things will happen. @@ -754,7 +757,7 @@ void erts_monitor_tree_insert(ErtsMonitor **root, ErtsMonitor *mon); * * @brief Replace a monitor in a monitor tree * - * When the funcion is called it is assumed that: + * When the function is called it is assumed that: * - 'old' monitor and 'new' monitor have exactly the same key * - 'old' monitor is part of the tree * - 'new' monitor is not part of any tree or list @@ -774,7 +777,7 @@ void erts_monitor_tree_replace(ErtsMonitor **root, ErtsMonitor *old, * * @brief Delete a monitor from a monitor tree * - * When the funcion is called it is assumed that: + * When the function is called it is assumed that: * - 'mon' monitor is part of the tree * If the above is not true, bad things will happen. * @@ -789,7 +792,7 @@ void erts_monitor_tree_delete(ErtsMonitor **root, ErtsMonitor *mon); * * @brief Call a function for each monitor in a monitor tree * - * The funcion 'func' will be called with a pointer to a monitor + * The function 'func' will be called with a pointer to a monitor * as first argument and 'arg' as second argument for each monitor * in the tree referred to by 'root'. * @@ -802,7 +805,7 @@ void erts_monitor_tree_delete(ErtsMonitor **root, ErtsMonitor *mon); * */ void erts_monitor_tree_foreach(ErtsMonitor *root, - void (*func)(ErtsMonitor *, void *), + ErtsMonitorFunc func, void *arg); /** @@ -810,9 +813,10 @@ void erts_monitor_tree_foreach(ErtsMonitor *root, * @brief Call a function for each monitor in a monitor tree. Yield * if lots of monitors exist. * - * The funcion 'func' will be called with a pointer to a monitor + * The function 'func' will be called with a pointer to a monitor * as first argument and 'arg' as second argument for each monitor - * in the tree referred to by 'root'. + * in the tree referred to by 'root'. It should return the number of + * reductions the operator took to perform. * * It is assumed that: * - *yspp equals NULL on first call @@ -835,27 +839,28 @@ void erts_monitor_tree_foreach(ErtsMonitor *root, * *yspp should be NULL. When done *yspp * will be NULL. * - * @param[in] limit Maximum amount of monitors to process - * before yielding. + * @param[in] reds Reductions available to execute before yielding. * - * @returns A non-zero value when all monitors has been - * processed, and zero when more work is needed. + * @returns The unconsumed reductions when all monitors + * have been processed, and zero when more work + * is needed. * */ int erts_monitor_tree_foreach_yielding(ErtsMonitor *root, - void (*func)(ErtsMonitor *, void *), + ErtsMonitorFunc func, void *arg, void **vyspp, - Sint limit); + Sint reds); /** * * @brief Delete all monitors from a monitor tree and call a function for * each monitor * - * The funcion 'func' will be called with a pointer to a monitor + * The function 'func' will be called with a pointer to a monitor * as first argument and 'arg' as second argument for each monitor - * in the tree referred to by 'root'. + * in the tree referred to by 'root'. It should return the number of + * reductions the operator took to perform. * * @param[in,out] root Pointer to pointer to root of monitor tree * @@ -866,7 +871,7 @@ int erts_monitor_tree_foreach_yielding(ErtsMonitor *root, * */ void erts_monitor_tree_foreach_delete(ErtsMonitor **root, - void (*func)(ErtsMonitor *, void *), + ErtsMonitorFunc func, void *arg); /** @@ -874,9 +879,10 @@ void erts_monitor_tree_foreach_delete(ErtsMonitor **root, * @brief Delete all monitors from a monitor tree and call a function for * each monitor * - * The funcion 'func' will be called with a pointer to a monitor + * The function 'func' will be called with a pointer to a monitor * as first argument and 'arg' as second argument for each monitor - * in the tree referred to by 'root'. + * in the tree referred to by 'root'. It should return the number of + * reductions the operator took to perform. * * It is assumed that: * - *yspp equals NULL on first call @@ -899,18 +905,18 @@ void erts_monitor_tree_foreach_delete(ErtsMonitor **root, * *yspp should be NULL. When done *yspp * will be NULL. * - * @param[in] limit Maximum amount of monitors to process - * before yielding. + * @param[in] reds Reductions available to execute before yielding. * - * @returns A non-zero value when all monitors has been - * processed, and zero when more work is needed. + * @returns The unconsumed reductions when all monitors + * have been processed, and zero when more work + * is needed. * */ int erts_monitor_tree_foreach_delete_yielding(ErtsMonitor **root, - void (*func)(ErtsMonitor *, void *), + ErtsMonitorFunc func, void *arg, void **vyspp, - Sint limit); + Sint reds); /* * --- Monitor list operations -- @@ -920,7 +926,7 @@ int erts_monitor_tree_foreach_delete_yielding(ErtsMonitor **root, * * @brief Insert a monitor in a monitor list * - * When the funcion is called it is assumed that: + * When the function is called it is assumed that: * - 'mon' monitor is not part of any list or tree * If the above is not true, bad things will happen. * @@ -935,7 +941,7 @@ ERTS_GLB_INLINE void erts_monitor_list_insert(ErtsMonitor **list, ErtsMonitor *m * * @brief Delete a monitor from a monitor list * - * When the funcion is called it is assumed that: + * When the function is called it is assumed that: * - 'mon' monitor is part of the list * If the above is not true, bad things will happen. * @@ -980,7 +986,7 @@ ERTS_GLB_INLINE ErtsMonitor *erts_monitor_list_last(ErtsMonitor *list); * * @brief Call a function for each monitor in a monitor list * - * The funcion 'func' will be called with a pointer to a monitor + * The function 'func' will be called with a pointer to a monitor * as first argument and 'arg' as second argument for each monitor * in the tree referred to by 'list'. * @@ -993,7 +999,7 @@ ERTS_GLB_INLINE ErtsMonitor *erts_monitor_list_last(ErtsMonitor *list); * */ void erts_monitor_list_foreach(ErtsMonitor *list, - void (*func)(ErtsMonitor *, void *), + ErtsMonitorFunc func, void *arg); /** @@ -1001,9 +1007,10 @@ void erts_monitor_list_foreach(ErtsMonitor *list, * @brief Call a function for each monitor in a monitor list. Yield * if lots of monitors exist. * - * The funcion 'func' will be called with a pointer to a monitor + * The function 'func' will be called with a pointer to a monitor * as first argument and 'arg' as second argument for each monitor - * in the tree referred to by 'root'. + * in the tree referred to by 'root'. It should return the number of + * reductions the operator took to perform. * * It is assumed that: * - *yspp equals NULL on first call @@ -1026,25 +1033,25 @@ void erts_monitor_list_foreach(ErtsMonitor *list, * *yspp should be NULL. When done *yspp * will be NULL. * - * @param[in] limit Maximum amount of monitors to process - * before yielding. + * @param[in] reds Reductions available to execute before yielding. * - * @returns A non-zero value when all monitors has been - * processed, and zero when more work is needed. + * @returns The unconsumed reductions when all monitors + * have been processed, and zero when more work + * is needed. * */ int erts_monitor_list_foreach_yielding(ErtsMonitor *list, - void (*func)(ErtsMonitor *, void *), + ErtsMonitorFunc func, void *arg, void **vyspp, - Sint limit); + Sint reds); /** * * @brief Delete all monitors from a monitor list and call a function for * each monitor * - * The funcion 'func' will be called with a pointer to a monitor + * The function 'func' will be called with a pointer to a monitor * as first argument and 'arg' as second argument for each monitor * in the tree referred to by 'root'. * @@ -1057,7 +1064,7 @@ int erts_monitor_list_foreach_yielding(ErtsMonitor *list, * */ void erts_monitor_list_foreach_delete(ErtsMonitor **list, - void (*func)(ErtsMonitor *, void *), + ErtsMonitorFunc func, void *arg); /** @@ -1065,9 +1072,10 @@ void erts_monitor_list_foreach_delete(ErtsMonitor **list, * @brief Delete all monitors from a monitor list and call a function for * each monitor * - * The funcion 'func' will be called with a pointer to a monitor + * The function 'func' will be called with a pointer to a monitor * as first argument and 'arg' as second argument for each monitor - * in the tree referred to by 'root'. + * in the tree referred to by 'root'. It should return the number of + * reductions the operator took to perform. * * It is assumed that: * - *yspp equals NULL on first call @@ -1090,18 +1098,18 @@ void erts_monitor_list_foreach_delete(ErtsMonitor **list, * *yspp should be NULL. When done *yspp * will be NULL. * - * @param[in] limit Maximum amount of monitors to process - * before yielding. + * @param[in] reds Reductions available to execute before yielding. * - * @returns A non-zero value when all monitors has been - * processed, and zero when more work is needed. + * @returns The unconsumed reductions when all monitors + * have been processed, and zero when more work + * is needed. * */ int erts_monitor_list_foreach_delete_yielding(ErtsMonitor **list, - void (*func)(ErtsMonitor *, void *), + ErtsMonitorFunc func, void *arg, void **vyspp, - Sint limit); + Sint reds); /* * --- Misc monitor operations --- @@ -1113,7 +1121,7 @@ int erts_monitor_list_foreach_delete_yielding(ErtsMonitor **list, * * Can create all types of monitors * - * When the funcion is called it is assumed that: + * When the function is called it is assumed that: * - 'ref' is an internal ordinary reference if type is ERTS_MON_TYPE_PROC, * ERTS_MON_TYPE_PORT, ERTS_MON_TYPE_TIME_OFFSET, or ERTS_MON_TYPE_RESOURCE * - 'ref' is NIL if type is ERTS_MON_TYPE_NODE, ERTS_MON_TYPE_NODES, or @@ -1199,7 +1207,7 @@ ERTS_GLB_INLINE int erts_monitor_is_in_table(ErtsMonitor *mon); * When both the origin and the target part of the monitor have * been released the monitor structure will be deallocated. * - * When the funcion is called it is assumed that: + * When the function is called it is assumed that: * - 'mon' monitor is not part of any list or tree * - 'mon' is not referred to by any other structures * If the above are not true, bad things will happen. @@ -1216,7 +1224,7 @@ ERTS_GLB_INLINE void erts_monitor_release(ErtsMonitor *mon); * Release both the origin and target parts of the monitor * simultaneously and deallocate the structure. * - * When the funcion is called it is assumed that: + * When the function is called it is assumed that: * - Neither the origin part nor the target part of the monitor * are not part of any list or tree * - Neither the origin part nor the target part of the monitor @@ -1232,7 +1240,7 @@ ERTS_GLB_INLINE void erts_monitor_release_both(ErtsMonitorData *mdp); * * @brief Insert monitor in dist monitor tree or list * - * When the funcion is called it is assumed that: + * When the function is called it is assumed that: * - 'mon' monitor is not part of any list or tree * If the above is not true, bad things will happen. * @@ -1253,7 +1261,7 @@ ERTS_GLB_INLINE int erts_monitor_dist_insert(ErtsMonitor *mon, ErtsMonLnkDist *d * * @brief Delete monitor from dist monitor tree or list * - * When the funcion is called it is assumed that: + * When the function is called it is assumed that: * - 'mon' monitor earler has been inserted into 'dist' * If the above is not true, bad things will happen. * @@ -1291,7 +1299,7 @@ erts_monitor_set_dead_dist(ErtsMonitor *mon, Eterm nodename); * whole size of the monitor data structure is returned; otherwise, * half of the size is returned. * - * When the funcion is called it is assumed that: + * When the function is called it is assumed that: * - 'mon' has not been released * If the above is not true, bad things will happen. * @@ -1507,6 +1515,8 @@ ERTS_GLB_INLINE ErtsMonitorSuspend *erts_monitor_suspend(ErtsMonitor *mon) typedef struct ErtsMonLnkNode__ ErtsLink; +typedef int (*ErtsLinkFunc)(ErtsLink *, void *, Sint); + typedef struct { ErtsLink a; ErtsLink b; @@ -1544,7 +1554,7 @@ ErtsLink *erts_link_tree_lookup(ErtsLink *root, Eterm item); * * @brief Lookup or insert a link in a link tree * - * When the funcion is called it is assumed that: + * When the function is called it is assumed that: * - 'lnk' link is not part of any tree or list * If the above is not true, bad things will happen. * @@ -1590,7 +1600,7 @@ ErtsLink *erts_link_tree_lookup_create(ErtsLink **root, int *created, * * @brief Insert a link in a link tree * - * When the funcion is called it is assumed that: + * When the function is called it is assumed that: * - no links with the same key that 'lnk' exist in the tree * - 'lnk' is not part of any list of tree * If the above are not true, bad things will happen. @@ -1606,7 +1616,7 @@ void erts_link_tree_insert(ErtsLink **root, ErtsLink *lnk); * * @brief Replace a link in a link tree * - * When the funcion is called it is assumed that: + * When the function is called it is assumed that: * - 'old' link and 'new' link have exactly the same key * - 'old' link is part of the tree * - 'new' link is not part of any tree or list @@ -1630,7 +1640,7 @@ void erts_link_tree_replace(ErtsLink **root, ErtsLink *old, ErtsLink *new); * the tree and 'lnk' has a lower address than the link in the * tree, the existing link in the tree is replaced by 'lnk'. * - * When the funcion is called it is assumed that: + * When the function is called it is assumed that: * - 'lnk' link is not part of any tree or list * If the above are not true, bad things will happen. * @@ -1649,7 +1659,7 @@ ERTS_GLB_INLINE ErtsLink *erts_link_tree_insert_addr_replace(ErtsLink **root, * * @brief Delete a link from a link tree * - * When the funcion is called it is assumed that: + * When the function is called it is assumed that: * - 'lnk' link is part of the tree * If the above is not true, bad things will happen. * @@ -1668,7 +1678,7 @@ void erts_link_tree_delete(ErtsLink **root, ErtsLink *lnk); * If link 'lnk' is not in the tree, another link with the same * key as 'lnk' is deleted from the tree if such a link exist. * - * When the funcion is called it is assumed that: + * When the function is called it is assumed that: * - if 'lnk' link is part of a tree or list, it is part of this tree * If the above is not true, bad things will happen. * @@ -1687,7 +1697,7 @@ ERTS_GLB_INLINE ErtsLink *erts_link_tree_key_delete(ErtsLink **root, ErtsLink *l * * @brief Call a function for each link in a link tree * - * The funcion 'func' will be called with a pointer to a link + * The function 'func' will be called with a pointer to a link * as first argument and 'arg' as second argument for each link * in the tree referred to by 'root'. * @@ -1700,7 +1710,7 @@ ERTS_GLB_INLINE ErtsLink *erts_link_tree_key_delete(ErtsLink **root, ErtsLink *l * */ void erts_link_tree_foreach(ErtsLink *root, - void (*func)(ErtsLink *, void *), + ErtsLinkFunc, void *arg); /** @@ -1708,9 +1718,10 @@ void erts_link_tree_foreach(ErtsLink *root, * @brief Call a function for each link in a link tree. Yield if lots * of links exist. * - * The funcion 'func' will be called with a pointer to a link + * The function 'func' will be called with a pointer to a link * as first argument and 'arg' as second argument for each link - * in the tree referred to by 'root'. + * in the tree referred to by 'root'. It should return the number of + * reductions the operator took to perform. * * It is assumed that: * - *yspp equals NULL on first call @@ -1733,25 +1744,25 @@ void erts_link_tree_foreach(ErtsLink *root, * *yspp should be NULL. When done *yspp * will be NULL. * - * @param[in] limit Maximum amount of links to process - * before yielding. + * @param[in] reds Reductions available to execute before yielding. * - * @returns A non-zero value when all links has been - * processed, and zero when more work is needed. + * @returns The unconsumed reductions when all links + * have been processed, and zero when more work + * is needed. * */ int erts_link_tree_foreach_yielding(ErtsLink *root, - void (*func)(ErtsLink *, void *), + ErtsLinkFunc func, void *arg, void **vyspp, - Sint limit); + Sint reds); /** * * @brief Delete all links from a link tree and call a function for * each link * - * The funcion 'func' will be called with a pointer to a link + * The function 'func' will be called with a pointer to a link * as first argument and 'arg' as second argument for each link * in the tree referred to by 'root'. * @@ -1764,7 +1775,7 @@ int erts_link_tree_foreach_yielding(ErtsLink *root, * */ void erts_link_tree_foreach_delete(ErtsLink **root, - void (*func)(ErtsLink *, void *), + ErtsLinkFunc func, void *arg); /** @@ -1772,9 +1783,10 @@ void erts_link_tree_foreach_delete(ErtsLink **root, * @brief Delete all links from a link tree and call a function for * each link * - * The funcion 'func' will be called with a pointer to a link + * The function 'func' will be called with a pointer to a link * as first argument and 'arg' as second argument for each link - * in the tree referred to by 'root'. + * in the tree referred to by 'root'. It should return the number of + * reductions the operator took to perform. * * It is assumed that: * - *yspp equals NULL on first call @@ -1797,18 +1809,18 @@ void erts_link_tree_foreach_delete(ErtsLink **root, * *yspp should be NULL. When done *yspp * will be NULL. * - * @param[in] limit Maximum amount of links to process - * before yielding. + * @param[in] reds Reductions available to execute before yielding. * - * @returns A non-zero value when all links has been - * processed, and zero when more work is needed. + * @returns The unconsumed reductions when all links + * have been processed, and zero when more work + * is needed. * */ int erts_link_tree_foreach_delete_yielding(ErtsLink **root, - void (*func)(ErtsLink *, void *), + ErtsLinkFunc func, void *arg, void **vyspp, - Sint limit); + Sint reds); /* * --- Link list operations --- @@ -1818,7 +1830,7 @@ int erts_link_tree_foreach_delete_yielding(ErtsLink **root, * * @brief Insert a link in a link list * - * When the funcion is called it is assumed that: + * When the function is called it is assumed that: * - 'lnk' link is not part of any list or tree * If the above is not true, bad things will happen. * @@ -1833,7 +1845,7 @@ ERTS_GLB_INLINE void erts_link_list_insert(ErtsLink **list, ErtsLink *lnk); * * @brief Delete a link from a link list * - * When the funcion is called it is assumed that: + * When the function is called it is assumed that: * - 'lnk' link is part of the list * If the above is not true, bad things will happen. * @@ -1878,7 +1890,7 @@ ERTS_GLB_INLINE ErtsLink *erts_link_list_last(ErtsLink *list); * * @brief Call a function for each link in a link list * - * The funcion 'func' will be called with a pointer to a link + * The function 'func' will be called with a pointer to a link * as first argument and 'arg' as second argument for each link * in the tree referred to by 'list'. * @@ -1891,7 +1903,7 @@ ERTS_GLB_INLINE ErtsLink *erts_link_list_last(ErtsLink *list); * */ void erts_link_list_foreach(ErtsLink *list, - void (*func)(ErtsLink *, void *), + ErtsLinkFunc func, void *arg); /** @@ -1899,9 +1911,10 @@ void erts_link_list_foreach(ErtsLink *list, * @brief Call a function for each link in a link list. Yield * if lots of links exist. * - * The funcion 'func' will be called with a pointer to a link + * The function 'func' will be called with a pointer to a link * as first argument and 'arg' as second argument for each link - * in the tree referred to by 'root'. + * in the tree referred to by 'root'. It should return the number of + * reductions the operator took to perform. * * It is assumed that: * - *yspp equals NULL on first call @@ -1924,25 +1937,25 @@ void erts_link_list_foreach(ErtsLink *list, * *yspp should be NULL. When done *yspp * will be NULL. * - * @param[in] limit Maximum amount of links to process - * before yielding. + * @param[in] reds Reductions available to execute before yielding. * - * @returns A non-zero value when all links has been - * processed, and zero when more work is needed. + * @returns The unconsumed reductions when all links + * have been processed, and zero when more work + * is needed. * */ int erts_link_list_foreach_yielding(ErtsLink *list, - void (*func)(ErtsLink *, void *), + ErtsLinkFunc func, void *arg, void **vyspp, - Sint limit); + Sint reds); /** * * @brief Delete all links from a link list and call a function for * each link * - * The funcion 'func' will be called with a pointer to a link + * The function 'func' will be called with a pointer to a link * as first argument and 'arg' as second argument for each link * in the tree referred to by 'root'. * @@ -1955,7 +1968,7 @@ int erts_link_list_foreach_yielding(ErtsLink *list, * */ void erts_link_list_foreach_delete(ErtsLink **list, - void (*func)(ErtsLink *, void *), + ErtsLinkFunc func, void *arg); /** @@ -1963,9 +1976,10 @@ void erts_link_list_foreach_delete(ErtsLink **list, * @brief Delete all links from a link list and call a function for * each link * - * The funcion 'func' will be called with a pointer to a link + * The function 'func' will be called with a pointer to a link * as first argument and 'arg' as second argument for each link - * in the tree referred to by 'root'. + * in the tree referred to by 'root'. It should return the number of + * reductions the operator took to perform. * * It is assumed that: * - *yspp equals NULL on first call @@ -1988,18 +2002,18 @@ void erts_link_list_foreach_delete(ErtsLink **list, * *yspp should be NULL. When done *yspp * will be NULL. * - * @param[in] limit Maximum amount of links to process - * before yielding. + * @param[in] reds Reductions available to execute before yielding. * - * @returns A non-zero value when all links has been - * processed, and zero when more work is needed. + * @returns The unconsumed reductions when all links + * have been processed, and zero when more work + * is needed. * */ int erts_link_list_foreach_delete_yielding(ErtsLink **list, - void (*func)(ErtsLink *, void *), + ErtsLinkFunc func, void *arg, void **vyspp, - Sint limit); + Sint reds); /* * --- Misc link operations --- @@ -2011,7 +2025,7 @@ int erts_link_list_foreach_delete_yielding(ErtsLink **list, * * Can create all types of links * - * When the funcion is called it is assumed that: + * When the function is called it is assumed that: * - 'ref' is an internal ordinary reference if type is ERTS_MON_TYPE_PROC, * ERTS_MON_TYPE_PORT, ERTS_MON_TYPE_TIME_OFFSET, or ERTS_MON_TYPE_RESOURCE * - 'ref' is NIL if type is ERTS_MON_TYPE_NODE or ERTS_MON_TYPE_NODES @@ -2081,7 +2095,7 @@ ERTS_GLB_INLINE int erts_link_is_in_table(ErtsLink *lnk); * When both link halves part of the link have been released the link * structure will be deallocated. * - * When the funcion is called it is assumed that: + * When the function is called it is assumed that: * - 'lnk' link is not part of any list or tree * - 'lnk' is not referred to by any other structures * If the above are not true, bad things will happen. @@ -2098,7 +2112,7 @@ ERTS_GLB_INLINE void erts_link_release(ErtsLink *lnk); * Release both halves of a link simultaneously and deallocate * the structure. * - * When the funcion is called it is assumed that: + * When the function is called it is assumed that: * - Neither of the parts of the link are part of any list or tree * - Neither of the parts of the link or the link data structure * are referred to by any other structures @@ -2113,7 +2127,7 @@ ERTS_GLB_INLINE void erts_link_release_both(ErtsLinkData *ldp); * * @brief Insert link in dist link list * - * When the funcion is called it is assumed that: + * When the function is called it is assumed that: * - 'lnk' link is not part of any list or tree * If the above is not true, bad things will happen. * @@ -2134,7 +2148,7 @@ ERTS_GLB_INLINE int erts_link_dist_insert(ErtsLink *lnk, ErtsMonLnkDist *dist); * * @brief Delete link from dist link list * - * When the funcion is called it is assumed that: + * When the function is called it is assumed that: * - 'lnk' link earler has been inserted into 'dist' * If the above is not true, bad things will happen. * @@ -2172,7 +2186,7 @@ erts_link_set_dead_dist(ErtsLink *lnk, Eterm nodename); * whole size of the link data structure is returned; otherwise, * half of the size is returned. * - * When the funcion is called it is assumed that: + * When the function is called it is assumed that: * - 'lnk' has not been released * If the above is not true, bad things will happen. * diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 06f3438caf..b2bd28fcbb 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -2357,12 +2357,13 @@ rmon_refc_read(ErtsResourceMonitors *rms) return rms->refc & ERTS_RESOURCE_REFC_MASK; } -static void dtor_demonitor(ErtsMonitor* mon, void* context) +static int dtor_demonitor(ErtsMonitor* mon, void* context, Sint reds) { ASSERT(erts_monitor_is_origin(mon)); ASSERT(is_internal_pid(mon->other.item)); erts_proc_sig_send_demonitor(mon); + return 1; } #ifdef DEBUG diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index 18ed782ae3..afafaf48dc 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -201,6 +201,7 @@ dist_table_alloc(void *dep_tmpl) dep->send = NULL; dep->cache = NULL; dep->transcode_ctx = NULL; + dep->sequences = NULL; /* Link in */ @@ -801,8 +802,9 @@ node_table_hash(void *venp) static int node_table_cmp(void *venp1, void *venp2) { - return ((((ErlNode *) venp1)->sysname == ((ErlNode *) venp2)->sysname - && ((ErlNode *) venp1)->creation == ((ErlNode *) venp2)->creation) + return ((((ErlNode *) venp1)->sysname == ((ErlNode *) venp2)->sysname) && + ((((ErlNode *) venp1)->creation == ((ErlNode *) venp2)->creation) || + (((ErlNode *) venp1)->creation == 0 || ((ErlNode *) venp2)->creation == 0)) ? 0 : 1); } @@ -816,11 +818,16 @@ node_table_alloc(void *venp_tmpl) node_entries++; - erts_refc_init(&enp->refc, -1); + erts_init_node_entry(enp, -1); enp->creation = ((ErlNode *) venp_tmpl)->creation; enp->sysname = ((ErlNode *) venp_tmpl)->sysname; enp->dist_entry = erts_find_or_insert_dist_entry(((ErlNode *) venp_tmpl)->sysname); +#ifdef ERL_NODE_BOOKKEEP + erts_atomic_init_nob(&enp->slot, 0); + sys_memzero(enp->books, sizeof(struct erl_node_bookkeeping) * 1024); +#endif + return (void *) enp; } @@ -873,7 +880,7 @@ erts_node_table_info(fmtfn_t to, void *to_arg) } -ErlNode *erts_find_or_insert_node(Eterm sysname, Uint32 creation) +ErlNode *erts_find_or_insert_node(Eterm sysname, Uint32 creation, Eterm book) { ErlNode *res; ErlNode ne; @@ -883,9 +890,9 @@ ErlNode *erts_find_or_insert_node(Eterm sysname, Uint32 creation) erts_rwmtx_rlock(&erts_node_table_rwmtx); res = hash_get(&erts_node_table, (void *) &ne); if (res && res != erts_this_node) { - erts_aint_t refc = erts_refc_inctest(&res->refc, 0); + erts_aint_t refc = erts_ref_node_entry(res, 0, book); if (refc < 2) /* New or pending delete */ - erts_refc_inc(&res->refc, 1); + erts_ref_node_entry(res, 1, THE_NON_VALUE); } erts_rwmtx_runlock(&erts_node_table_rwmtx); if (res) @@ -895,9 +902,9 @@ ErlNode *erts_find_or_insert_node(Eterm sysname, Uint32 creation) res = hash_put(&erts_node_table, (void *) &ne); ASSERT(res); if (res != erts_this_node) { - erts_aint_t refc = erts_refc_inctest(&res->refc, 0); + erts_aint_t refc = erts_ref_node_entry(res, 0, book); if (refc < 2) /* New or pending delete */ - erts_refc_inc(&res->refc, 1); + erts_ref_node_entry(res, 1, THE_NON_VALUE); } erts_rwmtx_rwunlock(&erts_node_table_rwmtx); return res; @@ -924,6 +931,7 @@ static void try_delete_node(void *venp) * * If refc > 0, the entry is in use. Keep the entry. */ + erts_node_bookkeep(enp, THE_NON_VALUE, ERL_NODE_DEC); refc = erts_refc_dectest(&enp->refc, -1); if (refc == -1) (void) hash_erase(&erts_node_table, (void *) enp); @@ -1021,7 +1029,7 @@ erts_set_this_node(Eterm sysname, Uint creation) erts_deref_dist_entry(erts_this_dist_entry); erts_this_node = NULL; /* to make sure refc is bumped for this node */ - erts_this_node = erts_find_or_insert_node(sysname, creation); + erts_this_node = erts_find_or_insert_node(sysname, creation, THE_NON_VALUE); erts_this_dist_entry = erts_this_node->dist_entry; erts_ref_dist_entry(erts_this_dist_entry); @@ -1090,7 +1098,7 @@ void erts_init_node_tables(int dd_sec) node_tmpl.creation = 0; erts_this_node = hash_put(&erts_node_table, &node_tmpl); /* +1 for erts_this_node */ - erts_refc_init(&erts_this_node->refc, 1); + erts_init_node_entry(erts_this_node, 1); ASSERT(erts_this_node->dist_entry != NULL); erts_this_dist_entry = erts_this_node->dist_entry; @@ -1177,6 +1185,7 @@ static Eterm AM_system; static Eterm AM_timer; static Eterm AM_delayed_delete_timer; static Eterm AM_thread_progress_delete_timer; +static Eterm AM_sequence; static Eterm AM_signal; static void setup_reference_table(void); @@ -1218,6 +1227,7 @@ typedef struct dist_referrer_ { int ctrl_ref; int system_ref; int signal_ref; + int sequence_ref; Eterm id; Uint creation; Uint id_heap[ID_HEAP_SIZE]; @@ -1272,6 +1282,7 @@ erts_get_node_and_dist_references(struct process *proc) INIT_AM(delayed_delete_timer); INIT_AM(thread_progress_delete_timer); INIT_AM(signal); + INIT_AM(sequence); references_atoms_need_init = 0; } @@ -1309,8 +1320,9 @@ erts_get_node_and_dist_references(struct process *proc) #define TIMER_REF 8 #define SYSTEM_REF 9 #define SIGNAL_REF 10 +#define SEQUENCE_REF 11 -#define INC_TAB_SZ 10 +#define INC_TAB_SZ 11 static void insert_dist_referrer(ReferredDist *referred_dist, @@ -1344,6 +1356,7 @@ insert_dist_referrer(ReferredDist *referred_dist, drp->ctrl_ref = 0; drp->system_ref = 0; drp->signal_ref = 0; + drp->sequence_ref = 0; } switch (type) { @@ -1353,6 +1366,7 @@ insert_dist_referrer(ReferredDist *referred_dist, case ETS_REF: drp->ets_ref++; break; case SYSTEM_REF: drp->system_ref++; break; case SIGNAL_REF: drp->signal_ref++; break; + case SEQUENCE_REF: drp->sequence_ref++; break; default: ASSERT(0); } } @@ -1509,7 +1523,7 @@ insert_offheap(ErlOffHeap *oh, int type, Eterm id) } } else if (IsSendCtxBinary(u.mref->mb)) { - ErtsSendContext* ctx = ERTS_MAGIC_BIN_DATA(u.mref->mb); + ErtsDSigSendContext* ctx = ERTS_MAGIC_BIN_DATA(u.mref->mb); if (ctx->deref_dep) insert_dist_entry(ctx->dep, type, id, 0); } @@ -1544,16 +1558,18 @@ static void insert_monitor_data(ErtsMonitor *mon, int type, Eterm id) mdp->origin.flags |= ERTS_ML_FLG_DBG_VISITED; } -static void insert_monitor(ErtsMonitor *mon, void *idp) +static int insert_monitor(ErtsMonitor *mon, void *idp, Sint reds) { Eterm id = *((Eterm *) idp); insert_monitor_data(mon, MONITOR_REF, id); + return 1; } -static void clear_visited_monitor(ErtsMonitor *mon, void *p) +static int clear_visited_monitor(ErtsMonitor *mon, void *p, Sint reds) { ErtsMonitorData *mdp = erts_monitor_to_data(mon); mdp->origin.flags &= ~ERTS_ML_FLG_DBG_VISITED; + return 1; } static void @@ -1581,6 +1597,20 @@ insert_dist_monitors(DistEntry *dep) } } + +static int +insert_sequence(ErtsDistExternal *edep, void *arg, Sint reds) +{ + insert_dist_entry(edep->dep, SEQUENCE_REF, *(Eterm*)arg, 0); + return 1; +} + +static void +insert_dist_sequences(DistEntry *dep) +{ + erts_dist_seq_tree_foreach(dep, insert_sequence, (void *) &dep->sysname); +} + static void clear_visited_p_monitors(ErtsPTabElementCommon *p) { @@ -1621,16 +1651,18 @@ static void insert_link_data(ErtsLink *lnk, int type, Eterm id) ldp->a.flags |= ERTS_ML_FLG_DBG_VISITED; } -static void insert_link(ErtsLink *lnk, void *idp) +static int insert_link(ErtsLink *lnk, void *idp, Sint reds) { Eterm id = *((Eterm *) idp); insert_link_data(lnk, LINK_REF, id); + return 1; } -static void clear_visited_link(ErtsLink *lnk, void *p) +static int clear_visited_link(ErtsLink *lnk, void *p, Sint reds) { ErtsLinkData *ldp = erts_link_to_data(lnk); ldp->a.flags &= ~ERTS_ML_FLG_DBG_VISITED; + return 1; } static void @@ -1770,11 +1802,9 @@ insert_message(ErtsMessage *msg, int type, Process *proc) else if (ERTS_SIG_IS_INTERNAL_MSG(msg)) heap_frag = msg->data.heap_frag; else { - if (msg->data.dist_ext->dep) - insert_dist_entry(msg->data.dist_ext->dep, - type, proc->common.id, 0); - if (is_not_nil(ERL_MESSAGE_TOKEN(msg))) - heap_frag = erts_dist_ext_trailer(msg->data.dist_ext); + heap_frag = msg->data.heap_frag; + insert_dist_entry(erts_get_dist_ext(heap_frag)->dep, + type, proc->common.id, 0); } } while (heap_frag) { @@ -1798,24 +1828,115 @@ insert_sig_offheap(ErlOffHeap *ohp, void *arg) insert_offheap(ohp, SIGNAL_REF, proc->common.id); } -static void -insert_sig_monitor(ErtsMonitor *mon, void *arg) +static int +insert_sig_monitor(ErtsMonitor *mon, void *arg, Sint reds) { Process *proc = arg; insert_monitor_data(mon, SIGNAL_REF, proc->common.id); + return 1; } -static void -insert_sig_link(ErtsLink *lnk, void *arg) +static int +insert_sig_link(ErtsLink *lnk, void *arg, Sint reds) { Process *proc = arg; insert_link_data(lnk, SIGNAL_REF, proc->common.id); + return 1; } static void -setup_reference_table(void) +insert_sig_ext(ErtsDistExternal *edep, void *arg) { + Process *proc = arg; + insert_dist_entry(edep->dep, SIGNAL_REF, proc->common.id, 0); +} + +static void +insert_process(Process *proc) +{ + int mli; + ErtsMessage *msg_list[] = {proc->msg_frag}; ErlHeapFragment *hfp; + + /* Insert Heap */ + insert_offheap(&(proc->off_heap), + HEAP_REF, + proc->common.id); + /* Insert heap fragments buffers */ + for(hfp = proc->mbuf; hfp; hfp = hfp->next) + insert_offheap(&(hfp->off_heap), + HEAP_REF, + proc->common.id); + + /* Insert msg buffers */ + for (mli = 0; mli < sizeof(msg_list)/sizeof(msg_list[0]); mli++) { + ErtsMessage *msg; + for (msg = msg_list[mli]; msg; msg = msg->next) + insert_message(msg, HEAP_REF, proc); + } + + /* Insert signal queue */ + erts_proc_sig_debug_foreach_sig(proc, + insert_sig_msg, + insert_sig_offheap, + insert_sig_monitor, + insert_sig_link, + insert_sig_ext, + (void *) proc); + + /* If the process is FREE, the proc->common field has been + re-used by the ptab delete, so we cannot trust it. */ + if (!(erts_atomic32_read_nob(&proc->state) & ERTS_PSFLG_FREE)) { + /* Insert links */ + insert_p_links(&proc->common); + + /* Insert monitors */ + insert_p_monitors(&proc->common); + } + + { + DistEntry *dep = ERTS_PROC_GET_DIST_ENTRY(proc); + if (dep) + insert_dist_entry(dep, + CTRL_REF, + proc->common.id, + 0); + } +} + +static void +insert_dist_suspended_procs(DistEntry *dep) +{ + ErtsProcList *plist = erts_proclist_peek_first(dep->suspended); + while (plist) { + if (is_not_immed(plist->u.pid)) + insert_process(plist->u.p); + plist = erts_proclist_peek_next(dep->suspended, plist); + } +} + +#ifdef ERL_NODE_BOOKKEEP +void +erts_node_bookkeep(ErlNode *np, Eterm term, int what) +{ + erts_aint_t slot = (erts_atomic_inc_read_nob(&np->slot) - 1) % 1024; + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + Eterm who = THE_NON_VALUE; + ASSERT(np); + np->books[slot].what = what; + np->books[slot].term = term; + if (esdp->current_process) { + who = esdp->current_process->common.id; + } else if (esdp->current_port) { + who = esdp->current_port->common.id; + } + np->books[slot].who = who; +} +#endif + +static void +setup_reference_table(void) +{ DistEntry *dep; HashInfo hi; int i, max; @@ -1865,52 +1986,10 @@ setup_reference_table(void) /* Insert all processes */ for (i = 0; i < max; i++) { Process *proc = erts_pix2proc(i); - if (proc) { - int mli; - ErtsMessage *msg_list[] = {proc->msg_frag}; - - /* Insert Heap */ - insert_offheap(&(proc->off_heap), - HEAP_REF, - proc->common.id); - /* Insert heap fragments buffers */ - for(hfp = proc->mbuf; hfp; hfp = hfp->next) - insert_offheap(&(hfp->off_heap), - HEAP_REF, - proc->common.id); - - /* Insert msg buffers */ - for (mli = 0; mli < sizeof(msg_list)/sizeof(msg_list[0]); mli++) { - ErtsMessage *msg; - for (msg = msg_list[mli]; msg; msg = msg->next) - insert_message(msg, HEAP_REF, proc); - } - - /* Insert signal queue */ - erts_proc_sig_debug_foreach_sig(proc, - insert_sig_msg, - insert_sig_offheap, - insert_sig_monitor, - insert_sig_link, - (void *) proc); - - /* Insert links */ - insert_p_links(&proc->common); - - /* Insert monitors */ - insert_p_monitors(&proc->common); - - { - DistEntry *dep = ERTS_PROC_GET_DIST_ENTRY(proc); - if (dep) - insert_dist_entry(dep, - CTRL_REF, - proc->common.id, - 0); - } - } + if (proc) + insert_process(proc); } - + erts_foreach_sys_msg_in_q(insert_sys_msg); /* Insert all ports */ @@ -1983,16 +2062,22 @@ setup_reference_table(void) for(dep = erts_visible_dist_entries; dep; dep = dep->next) { insert_dist_links(dep); insert_dist_monitors(dep); + insert_dist_sequences(dep); + insert_dist_suspended_procs(dep); } for(dep = erts_hidden_dist_entries; dep; dep = dep->next) { insert_dist_links(dep); insert_dist_monitors(dep); + insert_dist_sequences(dep); + insert_dist_suspended_procs(dep); } for(dep = erts_pending_dist_entries; dep; dep = dep->next) { insert_dist_links(dep); insert_dist_monitors(dep); + insert_dist_sequences(dep); + insert_dist_suspended_procs(dep); } /* Not connected dist entries should not have any links, @@ -2000,6 +2085,8 @@ setup_reference_table(void) for(dep = erts_not_connected_dist_entries; dep; dep = dep->next) { insert_dist_links(dep); insert_dist_monitors(dep); + insert_dist_sequences(dep); + insert_dist_suspended_procs(dep); } /* Insert all ets tables */ @@ -2173,6 +2260,10 @@ reference_table_term(Uint **hpp, ErlOffHeap *ohp, Uint *szp) tup = MK_2TUP(AM_system, MK_UINT(drp->system_ref)); drl = MK_CONS(tup, drl); } + if(drp->sequence_ref) { + tup = MK_2TUP(AM_sequence, MK_UINT(drp->sequence_ref)); + drl = MK_CONS(tup, drl); + } if(drp->signal_ref) { tup = MK_2TUP(AM_signal, MK_UINT(drp->signal_ref)); drl = MK_CONS(tup, drl); @@ -2247,6 +2338,12 @@ static void noop_sig_offheap(ErlOffHeap *oh, void *arg) } +static void noop_sig_ext(ErtsDistExternal *ext, void *arg) +{ + +} + + static void delete_reference_table(void) { @@ -2297,6 +2394,7 @@ delete_reference_table(void) noop_sig_offheap, clear_visited_monitor, clear_visited_link, + noop_sig_ext, (void *) proc); } } diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h index c44f1f8991..d5daf0c2df 100644 --- a/erts/emulator/beam/erl_node_tables.h +++ b/erts/emulator/beam/erl_node_tables.h @@ -98,11 +98,22 @@ struct ErtsDistOutputBuf_ { byte *alloc_endp; #endif ErtsDistOutputBuf *next; - Uint hopefull_flags; + Binary *bin; + /* Pointers to the distribution header, + if NULL the distr header is in the extp */ + byte *hdrp; + byte *hdr_endp; + /* Pointers to the ctl + payload */ byte *extp; byte *ext_endp; + /* Start of payload and hopefull_flags, used by transcode */ + Uint hopefull_flags; byte *msg_start; - byte data[1]; + /* start of the ext buffer, this is not always the same as extp + as the atom cache handling can use less then the allotted buffer. + This value is needed to calculate the size of this output buffer.*/ + byte *ext_start; + }; typedef struct { @@ -161,7 +172,46 @@ struct dist_entry_ { ErtsThrPrgrLaterOp later_op; struct transcode_context* transcode_ctx; + + struct dist_sequences *sequences; /* Ongoing distribution sequences */ +}; + +/* +#define ERL_NODE_BOOKKEEP + * Bookkeeping of ErlNode inc and dec operations to help debug refc problems. + * This is best used together with cerl -rr. Type the below into gdb: + * gdb: +set pagination off +set $i = 0 +set $node = referred_nodes[$node_ix].node +while $i < $node->slot.counter + printf "%p: ", $node->books[$i].term + etp-1 $node->books[$i].who + printf " " + p $node->books[$i].what + set $i++ +end + + * Then save that into a file called test.txt and run the below in + * an erlang shell in order to get all inc/dec that do not have a + * match. + +f(), {ok, B} = file:read_file("test.txt"). +Vs = [begin [Val, _, _, _, What] = All = string:lexemes(Ln, " "),{Val,What,All} end || Ln <- string:lexemes(B,"\n")]. +Accs = lists:foldl(fun({V,<<"ERL_NODE_INC">>,_},M) -> Val = maps:get(V,M,0), M#{ V => Val + 1 }; ({V,<<"ERL_NODE_DEC">>,_},M) -> Val = maps:get(V,M,0), M#{ V => Val - 1 } end, #{}, Vs). +lists:usort(lists:filter(fun({V,N}) -> N /= 0 end, maps:to_list(Accs))). + + * There are bound to be bugs in the the instrumentation code, but + * atleast this is a place to start when hunting refc bugs. + * + */ +#ifdef ERL_NODE_BOOKKEEP +struct erl_node_bookkeeping { + Eterm who; + Eterm term; + enum { ERL_NODE_INC, ERL_NODE_DEC } what; }; +#endif typedef struct erl_node_ { HashBucket hash_bucket; /* Hash bucket */ @@ -169,6 +219,10 @@ typedef struct erl_node_ { Eterm sysname; /* name@host atom for efficiency */ Uint32 creation; /* Creation */ DistEntry *dist_entry; /* Corresponding dist entry */ +#ifdef ERL_NODE_BOOKKEEP + struct erl_node_bookkeeping books[1024]; + erts_atomic_t slot; +#endif } ErlNode; @@ -201,7 +255,7 @@ void erts_dist_table_info(fmtfn_t, void *); void erts_set_dist_entry_not_connected(DistEntry *); void erts_set_dist_entry_pending(DistEntry *); void erts_set_dist_entry_connected(DistEntry *, Eterm, Uint); -ErlNode *erts_find_or_insert_node(Eterm, Uint32); +ErlNode *erts_find_or_insert_node(Eterm, Uint32, Eterm); void erts_schedule_delete_node(ErlNode *); void erts_set_this_node(Eterm, Uint); Uint erts_node_table_size(void); @@ -219,18 +273,38 @@ DistEntry *erts_dhandle_to_dist_entry(Eterm dhandle, Uint32* connection_id); Eterm erts_build_dhandle(Eterm **hpp, ErlOffHeap*, DistEntry*, Uint32 conn_id); Eterm erts_make_dhandle(Process *c_p, DistEntry*, Uint32 conn_id); -ERTS_GLB_INLINE void erts_deref_node_entry(ErlNode *np); +ERTS_GLB_INLINE void erts_init_node_entry(ErlNode *np, erts_aint_t val); +ERTS_GLB_INLINE erts_aint_t erts_ref_node_entry(ErlNode *np, int min_val, Eterm term); +ERTS_GLB_INLINE void erts_deref_node_entry(ErlNode *np, Eterm term); ERTS_GLB_INLINE void erts_de_rlock(DistEntry *dep); ERTS_GLB_INLINE void erts_de_runlock(DistEntry *dep); ERTS_GLB_INLINE void erts_de_rwlock(DistEntry *dep); ERTS_GLB_INLINE void erts_de_rwunlock(DistEntry *dep); +#ifdef ERL_NODE_BOOKKEEP +void erts_node_bookkeep(ErlNode *, Eterm , int); +#else +#define erts_node_bookkeep(...) +#endif #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE void -erts_deref_node_entry(ErlNode *np) +erts_init_node_entry(ErlNode *np, erts_aint_t val) +{ + erts_refc_init(&np->refc, val); +} + +ERTS_GLB_INLINE erts_aint_t +erts_ref_node_entry(ErlNode *np, int min_val, Eterm term) +{ + erts_node_bookkeep(np, term, ERL_NODE_INC); + return erts_refc_inctest(&np->refc, min_val); +} + +ERTS_GLB_INLINE void +erts_deref_node_entry(ErlNode *np, Eterm term) { - ASSERT(np); + erts_node_bookkeep(np, term, ERL_NODE_DEC); if (erts_refc_dectest(&np->refc, 0) == 0) erts_schedule_delete_node(np); } diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c index c8f2e88127..30a7875387 100644 --- a/erts/emulator/beam/erl_port_task.c +++ b/erts/emulator/beam/erl_port_task.c @@ -2094,7 +2094,7 @@ begin_port_cleanup(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p) erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), "%T", pp->common.id); while (plp2 != NULL) { - erts_snprintf(pid_str, sizeof(DTRACE_CHARBUF_NAME(pid_str)), "%T", plp2->pid); + erts_snprintf(pid_str, sizeof(DTRACE_CHARBUF_NAME(pid_str)), "%T", plp2->u.pid); DTRACE2(process_port_unblocked, pid_str, port_str); } } diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c index 9cfb7fc681..2e33a8a782 100644 --- a/erts/emulator/beam/erl_printf_term.c +++ b/erts/emulator/beam/erl_printf_term.c @@ -487,6 +487,8 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) { PRINT_UWORD(res, fn, arg, 'u', 0, 1, octet); ++bytep; --bytesize; + if ((*dcount)-- <= 0) + goto L_done; } if (bitsize) { Uint bits = bitoffs + bitsize; @@ -521,6 +523,8 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) { PRINT_CHAR(res, fn, arg, octet); ++bytep; --bytesize; + if ((*dcount)-- <= 0) + goto L_done; } PRINT_STRING(res, fn, arg, "\">>"); } diff --git a/erts/emulator/beam/erl_proc_sig_queue.c b/erts/emulator/beam/erl_proc_sig_queue.c index 18418a76e1..9c74a2c355 100644 --- a/erts/emulator/beam/erl_proc_sig_queue.c +++ b/erts/emulator/beam/erl_proc_sig_queue.c @@ -36,6 +36,7 @@ #include "erl_port_task.h" #include "erl_trace.h" #include "beam_bp.h" +#include "erl_binary.h" #include "big.h" #include "erl_gc.h" #include "bif.h" @@ -80,6 +81,11 @@ #define ERTS_SIG_Q_TYPE_ADJUST_TRACE_INFO \ ERTS_SIG_Q_TYPE_MAX +#define ERTS_SIG_IS_GEN_EXIT(sig) \ + (ERTS_PROC_SIG_TYPE(((ErtsSignal *) sig)->common.tag) == ERTS_SIG_Q_TYPE_GEN_EXIT) +#define ERTS_SIG_IS_GEN_EXIT_EXTERNAL(sig) \ + (ASSERT(ERTS_SIG_IS_GEN_EXIT(sig)),is_non_value(get_exit_signal_data(sig)->reason)) + Process *ERTS_WRITE_UNLIKELY(erts_dirty_process_signal_handler); Process *ERTS_WRITE_UNLIKELY(erts_dirty_process_signal_handler_high); Process *ERTS_WRITE_UNLIKELY(erts_dirty_process_signal_handler_max); @@ -259,7 +265,7 @@ destroy_dist_proc_demonitor(ErtsSigDistProcDemonitor *dmon) Eterm ref = dmon->ref; if (is_external(ref)) { ExternalThing *etp = external_thing_ptr(ref); - erts_deref_node_entry(etp->node); + erts_deref_node_entry(etp->node, ref); } erts_free(ERTS_ALC_T_DIST_DEMONITOR, dmon); } @@ -294,7 +300,8 @@ destroy_sig_dist_link_op(ErtsSigDistLinkOp *sdlnk) { ASSERT(is_external_pid(sdlnk->remote)); ASSERT(boxed_val(sdlnk->remote) == &sdlnk->heap[0]); - erts_deref_node_entry(((ExternalThing *) &sdlnk->heap[0])->node); + erts_deref_node_entry(((ExternalThing *) &sdlnk->heap[0])->node, + make_boxed(&sdlnk->heap[0])); erts_free(ERTS_ALC_T_SIG_DATA, sdlnk); } @@ -936,29 +943,54 @@ erts_proc_sig_privqs_len(Process *c_p) return proc_sig_privqs_len(c_p, 0); } +ErtsDistExternal * +erts_proc_sig_get_external(ErtsMessage *msgp) +{ + if (ERTS_SIG_IS_EXTERNAL_MSG(msgp)) { + return erts_get_dist_ext(msgp->data.heap_frag); + } else if (ERTS_SIG_IS_NON_MSG(msgp) && + ERTS_SIG_IS_GEN_EXIT(msgp) && + ERTS_SIG_IS_GEN_EXIT_EXTERNAL(msgp)) { + ErtsDistExternal *edep; + ErtsExitSignalData *xsigd = get_exit_signal_data(msgp); + ASSERT(ERTS_PROC_SIG_TYPE(((ErtsSignal *) msgp)->common.tag) == ERTS_SIG_Q_TYPE_GEN_EXIT); + ASSERT(is_non_value(xsigd->reason)); + if (msgp->hfrag.next == NULL) + edep = (ErtsDistExternal*)(xsigd + 1); + else + edep = erts_get_dist_ext(msgp->hfrag.next); + return edep; + } + return NULL; +} + static void do_seq_trace_output(Eterm to, Eterm token, Eterm msg); static void send_gen_exit_signal(Process *c_p, Eterm from_tag, Eterm from, Eterm to, - Sint16 op, Eterm reason, Eterm ref, - Eterm token, int normal_kills) + Sint16 op, Eterm reason, ErtsDistExternal *dist_ext, + ErlHeapFragment *dist_ext_hfrag, + Eterm ref, Eterm token, int normal_kills) { ErtsExitSignalData *xsigd; Eterm *hp, *start_hp, s_reason, s_ref, s_message, s_token, s_from; ErtsMessage *mp; ErlHeapFragment *hfrag; ErlOffHeap *ohp; - Uint hsz, from_sz, reason_sz, ref_sz, token_sz; + Uint hsz, from_sz, reason_sz, ref_sz, token_sz, dist_ext_sz; int seq_trace; #ifdef USE_VM_PROBES Eterm s_utag, utag; Uint utag_sz; #endif + ASSERT((is_value(reason) && dist_ext == NULL) || + (is_non_value(reason) && dist_ext != NULL)); + ASSERT(is_immed(from_tag)); - hsz = sizeof(ErtsExitSignalData)/sizeof(Uint); + hsz = sizeof(ErtsExitSignalData)/sizeof(Eterm); seq_trace = c_p && have_seqtrace(token); if (seq_trace) @@ -977,33 +1009,42 @@ send_gen_exit_signal(Process *c_p, Eterm from_tag, hsz += utag_sz; #endif - token_sz = is_immed(token) ? 0 : size_object(token); + token_sz = size_object(token); hsz += token_sz; - from_sz = is_immed(from) ? 0 : size_object(from); + from_sz = size_object(from); hsz += from_sz; - reason_sz = is_immed(reason) ? 0 : size_object(reason); - hsz += reason_sz; + ref_sz = size_object(ref); + hsz += ref_sz; - switch (op) { - case ERTS_SIG_Q_OP_EXIT: - case ERTS_SIG_Q_OP_EXIT_LINKED: { - /* {'EXIT', From, Reason} */ - hsz += 4; /* 3-tuple */ - ref_sz = 0; - break; - } - case ERTS_SIG_Q_OP_MONITOR_DOWN: { - /* {'DOWN', Ref, process, From, Reason} */ - hsz += 6; /* 5-tuple */ - ref_sz = NC_HEAP_SIZE(ref); - hsz += ref_sz; - break; - } - default: - ERTS_INTERNAL_ERROR("Invalid exit signal op"); - break; + /* The reason was part of the control message, + just use copy it into the xsigd */ + if (is_value(reason)) { + reason_sz = size_object(reason); + hsz += reason_sz; + + switch (op) { + case ERTS_SIG_Q_OP_EXIT: + case ERTS_SIG_Q_OP_EXIT_LINKED: { + /* {'EXIT', From, Reason} */ + hsz += 4; /* 3-tuple */ + break; + } + case ERTS_SIG_Q_OP_MONITOR_DOWN: { + /* {'DOWN', Ref, process, From, Reason} */ + hsz += 6; /* 5-tuple */ + break; + } + default: + ERTS_INTERNAL_ERROR("Invalid exit signal op"); + break; + } + } else if (dist_ext != NULL && dist_ext_hfrag == NULL) { + /* The message was not fragmented so we need to create space + for a single dist_ext element */ + dist_ext_sz = erts_dist_ext_size(dist_ext) / sizeof(Eterm); + hsz += dist_ext_sz; } /* @@ -1015,35 +1056,33 @@ send_gen_exit_signal(Process *c_p, Eterm from_tag, ohp = &hfrag->off_heap; start_hp = hp; - s_token = (is_immed(token) - ? token - : copy_struct(token, token_sz, &hp, ohp)); - - s_reason = (is_immed(reason) - ? reason - : copy_struct(reason, reason_sz, &hp, ohp)); + s_token = copy_struct(token, token_sz, &hp, ohp); + s_from = copy_struct(from, from_sz, &hp, ohp); + s_ref = copy_struct(ref, ref_sz, &hp, ohp); - s_from = (is_immed(from) - ? from - : copy_struct(from, from_sz, &hp, ohp)); + if (is_value(reason)) { + s_reason = copy_struct(reason, reason_sz, &hp, ohp); - if (!ref_sz) - s_ref = NIL; - else - s_ref = STORE_NC(&hp, ohp, ref); - - switch (op) { - case ERTS_SIG_Q_OP_EXIT: - case ERTS_SIG_Q_OP_EXIT_LINKED: - /* {'EXIT', From, Reason} */ - s_message = TUPLE3(hp, am_EXIT, s_from, s_reason); - hp += 4; - break; - case ERTS_SIG_Q_OP_MONITOR_DOWN: - /* {'DOWN', Ref, process, From, Reason} */ - s_message = TUPLE5(hp, am_DOWN, s_ref, am_process, s_from, s_reason); - hp += 6; - break; + switch (op) { + case ERTS_SIG_Q_OP_EXIT: + case ERTS_SIG_Q_OP_EXIT_LINKED: + /* {'EXIT', From, Reason} */ + s_message = TUPLE3(hp, am_EXIT, s_from, s_reason); + hp += 4; + break; + case ERTS_SIG_Q_OP_MONITOR_DOWN: + /* {'DOWN', Ref, process, From, Reason} */ + s_message = TUPLE5(hp, am_DOWN, s_ref, am_process, s_from, s_reason); + hp += 6; + break; + default: + /* This cannot happen, used to silence gcc warning */ + s_message = THE_NON_VALUE; + break; + } + } else { + s_message = THE_NON_VALUE; + s_reason = THE_NON_VALUE; } #ifdef USE_VM_PROBES @@ -1061,11 +1100,13 @@ send_gen_exit_signal(Process *c_p, Eterm from_tag, hfrag->used_size = hp - start_hp; - xsigd = (ErtsExitSignalData *) (char *) hp; + xsigd = (ErtsExitSignalData *) hp; xsigd->message = s_message; xsigd->from = s_from; xsigd->reason = s_reason; + hfrag->next = dist_ext_hfrag; + if (is_nil(s_ref)) xsigd->u.normal_kills = normal_kills; else { @@ -1073,6 +1114,15 @@ send_gen_exit_signal(Process *c_p, Eterm from_tag, xsigd->u.ref = s_ref; } + hp += sizeof(ErtsExitSignalData)/sizeof(Eterm); + + if (dist_ext != NULL && dist_ext_hfrag == NULL && is_non_value(reason)) { + erts_make_dist_ext_copy(dist_ext, (ErtsDistExternal *) hp); + hp += dist_ext_sz; + } + + ASSERT(hp == mp->hfrag.mem + mp->hfrag.alloc_size); + if (seq_trace) do_seq_trace_output(to, s_token, s_message); @@ -1205,7 +1255,19 @@ erts_proc_sig_send_exit(Process *c_p, Eterm from, Eterm to, from_tag = dep->sysname; } send_gen_exit_signal(c_p, from_tag, from, to, ERTS_SIG_Q_OP_EXIT, - reason, NIL, token, normal_kills); + reason, NULL, NULL, NIL, token, normal_kills); +} + +void +erts_proc_sig_send_dist_exit(DistEntry *dep, + Eterm from, Eterm to, + ErtsDistExternal *dist_ext, + ErlHeapFragment *hfrag, + Eterm reason, Eterm token) +{ + send_gen_exit_signal(NULL, dep->sysname, from, to, ERTS_SIG_Q_OP_EXIT, + reason, dist_ext, hfrag, NIL, token, 0); + } void @@ -1219,7 +1281,7 @@ erts_proc_sig_send_link_exit(Process *c_p, Eterm from, ErtsLink *lnk, if (is_not_immed(reason) || is_not_nil(token)) { ASSERT(is_internal_pid(from) || is_internal_port(from)); send_gen_exit_signal(c_p, from, from, to, ERTS_SIG_Q_OP_EXIT_LINKED, - reason, NIL, token, 0); + reason, NULL, NULL, NIL, token, 0); } else { /* Pass signal using old link structure... */ @@ -1274,10 +1336,13 @@ erts_proc_sig_send_unlink(Process *c_p, ErtsLink *lnk) void erts_proc_sig_send_dist_link_exit(DistEntry *dep, Eterm from, Eterm to, + ErtsDistExternal *dist_ext, + ErlHeapFragment *hfrag, Eterm reason, Eterm token) { send_gen_exit_signal(NULL, dep->sysname, from, to, ERTS_SIG_Q_OP_EXIT_LINKED, - reason, NIL, token, 0); + reason, dist_ext, hfrag, NIL, token, 0); + } void @@ -1299,16 +1364,18 @@ erts_proc_sig_send_dist_unlink(DistEntry *dep, Eterm from, Eterm to) void erts_proc_sig_send_dist_monitor_down(DistEntry *dep, Eterm ref, Eterm from, Eterm to, + ErtsDistExternal *dist_ext, + ErlHeapFragment *hfrag, Eterm reason) { Eterm monitored, heap[3]; - if (is_atom(from)) + if (is_atom(from)) monitored = TUPLE2(&heap[0], from, dep->sysname); else monitored = from; send_gen_exit_signal(NULL, dep->sysname, monitored, to, ERTS_SIG_Q_OP_MONITOR_DOWN, - reason, ref, NIL, 0); + reason, dist_ext, hfrag, ref, NIL, 0); } void @@ -1376,10 +1443,10 @@ erts_proc_sig_send_monitor_down(ErtsMonitor *mon, Eterm reason) || is_internal_pid(from_tag) || is_atom(from_tag)); monitored = TUPLE2(&heap[0], name, node); - } + } send_gen_exit_signal(NULL, from_tag, monitored, to, ERTS_SIG_Q_OP_MONITOR_DOWN, - reason, mdp->ref, NIL, 0); + reason, NULL, NULL, mdp->ref, NIL, 0); } erts_monitor_release(mon); } @@ -2037,7 +2104,6 @@ handle_exit_signal(Process *c_p, ErtsSigRecvTracing *tracing, if (type == ERTS_SIG_Q_TYPE_GEN_EXIT) { xsigd = get_exit_signal_data(sig); from = xsigd->from; - reason = xsigd->reason; if (op != ERTS_SIG_Q_OP_EXIT_LINKED) ignore = 0; else { @@ -2062,6 +2128,18 @@ handle_exit_signal(Process *c_p, ErtsSigRecvTracing *tracing, } } + /* This GEN_EXIT was received from another node, decode the exit reason */ + if (ERTS_SIG_IS_GEN_EXIT_EXTERNAL(sig)) + erts_proc_sig_decode_dist(c_p, ERTS_PROC_LOCK_MAIN, sig, 1); + + reason = xsigd->reason; + + if (is_non_value(reason)) { + /* Bad distribution message; remove it from queue... */ + ignore = !0; + destroy = !0; + } + if (!ignore) { if ((op != ERTS_SIG_Q_OP_EXIT || reason != am_kill) @@ -2929,6 +3007,104 @@ handle_sync_suspend(Process *c_p, ErtsMessage *mp) } } +int +erts_proc_sig_decode_dist(Process *proc, ErtsProcLocks proc_locks, + ErtsMessage *msgp, int force_off_heap) +{ + ErtsHeapFactory factory; + ErlHeapFragment *hfrag; + Eterm msg; + Sint need; + ErtsDistExternal *edep; + ErtsExitSignalData *xsigd = NULL; + + edep = erts_proc_sig_get_external(msgp); + if (!ERTS_SIG_IS_EXTERNAL_MSG(msgp)) + xsigd = get_exit_signal_data(msgp); + + if (edep->heap_size >= 0) + need = edep->heap_size; + else { + need = erts_decode_dist_ext_size(edep, 1); + if (need < 0) { + /* bad signal; remove it... */ + return 0; + } + + edep->heap_size = need; + } + + if (ERTS_SIG_IS_NON_MSG(msgp)) { + switch (ERTS_PROC_SIG_OP(ERL_MESSAGE_TERM(msgp))) { + case ERTS_SIG_Q_OP_EXIT: + case ERTS_SIG_Q_OP_EXIT_LINKED: + /* {'EXIT', From, Reason} */ + need += 4; + break; + case ERTS_SIG_Q_OP_MONITOR_DOWN: + /* {'DOWN', Ref, process, From, Reason} */ + need += 6; /* 5-tuple */ + break; + default: + ERTS_INTERNAL_ERROR("Invalid exit signal op"); + break; + } + } + + hfrag = new_message_buffer(need); + erts_factory_heap_frag_init(&factory, hfrag); + + ASSERT(edep->heap_size >= 0); + + msg = erts_decode_dist_ext(&factory, edep, 1); + + if (is_non_value(msg)) { + erts_factory_undo(&factory); + return 0; + } + + if (ERTS_SIG_IS_MSG(msgp)) { + ERL_MESSAGE_TERM(msgp) = msg; + if (msgp->data.heap_frag == &msgp->hfrag) + msgp->data.heap_frag = ERTS_MSG_COMBINED_HFRAG; + } else { + switch (ERTS_PROC_SIG_OP(ERL_MESSAGE_TERM(msgp))) { + case ERTS_SIG_Q_OP_EXIT: + case ERTS_SIG_Q_OP_EXIT_LINKED: + /* {'EXIT', From, Reason} */ + erts_reserve_heap(&factory, 4); + xsigd->message = TUPLE3(factory.hp, am_EXIT, xsigd->from, msg); + factory.hp += 4; + break; + case ERTS_SIG_Q_OP_MONITOR_DOWN: + /* {'DOWN', Ref, process, From, Reason} */ + erts_reserve_heap(&factory, 6); + xsigd->message = TUPLE5(factory.hp, am_DOWN, xsigd->u.ref, am_process, xsigd->from, msg); + factory.hp += 6; + break; + } + xsigd->reason = msg; + } + + erts_free_dist_ext_copy(edep); + + erts_factory_close(&factory); + + hfrag = factory.heap_frags; + while (hfrag->next) + hfrag = hfrag->next; + + if (ERTS_SIG_IS_MSG(msgp) && msgp->data.heap_frag != ERTS_MSG_COMBINED_HFRAG) { + hfrag->next = msgp->data.heap_frag; + msgp->data.heap_frag = factory.heap_frags; + } else { + hfrag->next = msgp->hfrag.next; + msgp->hfrag.next = factory.heap_frags; + } + + return 1; +} + void erts_proc_sig_handle_pending_suspend(Process *c_p) { @@ -3045,7 +3221,7 @@ erts_proc_sig_handle_incoming(Process *c_p, erts_aint32_t *statep, ASSERT(ERTS_SIG_IS_NON_MSG(sig)); tag = ((ErtsSignal *) sig)->common.tag; - + switch (ERTS_PROC_SIG_OP(tag)) { case ERTS_SIG_Q_OP_EXIT: @@ -3091,6 +3267,12 @@ erts_proc_sig_handle_incoming(Process *c_p, erts_aint32_t *statep, break; case ERTS_SIG_Q_TYPE_GEN_EXIT: xsigd = get_exit_signal_data(sig); + + /* This GEN_EXIT was received from another node, decode the exit reason */ + if (ERTS_SIG_IS_GEN_EXIT_EXTERNAL(sig)) + if (!erts_proc_sig_decode_dist(c_p, ERTS_PROC_LOCK_MAIN, sig, 1)) + break; /* Decode failed, just remove signal */ + omon = erts_monitor_tree_lookup(ERTS_P_MONITORS(c_p), xsigd->u.ref); if (omon) { @@ -3590,9 +3772,10 @@ stretch_limit(Process *c_p, ErtsSigRecvTracing *tp, int -erts_proc_sig_handle_exit(Process *c_p, int *redsp) +erts_proc_sig_handle_exit(Process *c_p, Sint *redsp) { - int cnt, limit; + int cnt; + Sint limit; ErtsMessage *sig, ***next_nm_sig; ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE(c_p, 0); @@ -3671,9 +3854,9 @@ erts_proc_sig_handle_exit(Process *c_p, int *redsp) break; case ERTS_SIG_Q_OP_MONITOR: { - ErtsProcExitContext pectxt = {c_p, am_noproc}; + ErtsProcExitContext pectxt = {c_p, am_noproc, NULL, NULL, NIL}; erts_proc_exit_handle_monitor((ErtsMonitor *) sig, - (void *) &pectxt); + (void *) &pectxt, -1); cnt += 4; break; } @@ -3687,7 +3870,7 @@ erts_proc_sig_handle_exit(Process *c_p, int *redsp) case ERTS_SIG_Q_OP_LINK: { ErtsProcExitContext pectxt = {c_p, am_noproc}; - erts_proc_exit_handle_link((ErtsLink *) sig, (void *) &pectxt); + erts_proc_exit_handle_link((ErtsLink *) sig, (void *) &pectxt, -1); break; } @@ -4188,11 +4371,13 @@ handle_msg_tracing(Process *c_p, ErtsSigRecvTracing *tracing, return -1; /* Yield... */ } if (ERTS_SIG_IS_EXTERNAL_MSG(sig)) { - cnt++; - if (!erts_decode_dist_message(c_p, ERTS_PROC_LOCK_MAIN, - sig, 0)) { + cnt += 50; /* Decode is expensive... */ + if (!erts_proc_sig_decode_dist(c_p, ERTS_PROC_LOCK_MAIN, + sig, 0)) { /* Bad dist message; remove it... */ remove_mq_m_sig(c_p, sig, next_sig, next_nm_sig); + sig->next = NULL; + erts_cleanup_messages(sig); sig = *next_sig; continue; } @@ -4264,18 +4449,12 @@ erts_proc_sig_prep_msgq_for_inspection(Process *c_p, if (ERTS_SIG_IS_EXTERNAL_MSG(mp)) { /* decode it... */ - if (mp->data.attached) - erts_decode_dist_message(rp, rp_locks, mp, !0); - - msg = ERL_MESSAGE_TERM(mp); - - if (is_non_value(msg)) { + if (!erts_proc_sig_decode_dist(rp, rp_locks, mp, !0)) { ErtsMessage *bad_mp = mp; /* * Bad distribution message; remove * it from the queue... */ - ASSERT(!mp->data.attached); ASSERT(*mpp == bad_mp); @@ -4287,6 +4466,8 @@ erts_proc_sig_prep_msgq_for_inspection(Process *c_p, erts_cleanup_messages(bad_mp); continue; } + + msg = ERL_MESSAGE_TERM(mp); } ASSERT(is_value(msg)); @@ -4445,12 +4626,21 @@ debug_foreach_sig_fake_oh(Eterm term, } +static void +debug_foreach_sig_external(ErtsMessage *msgp, + void (*ext_func)(ErtsDistExternal *, void *), + void *arg) +{ + ext_func(erts_proc_sig_get_external(msgp), arg); +} + void erts_proc_sig_debug_foreach_sig(Process *c_p, void (*msg_func)(ErtsMessage *, void *), void (*oh_func)(ErlOffHeap *, void *), - void (*mon_func)(ErtsMonitor *, void *), - void (*lnk_func)(ErtsLink *, void *), + ErtsMonitorFunc mon_func, + ErtsLinkFunc lnk_func, + void (*ext_func)(ErtsDistExternal *, void *), void *arg) { ErtsMessage *queue[] = {c_p->sig_qs.first, c_p->sig_qs.cont, c_p->sig_inq.first}; @@ -4459,10 +4649,10 @@ erts_proc_sig_debug_foreach_sig(Process *c_p, for (qix = 0; qix < sizeof(queue)/sizeof(queue[0]); qix++) { ErtsMessage *sig; for (sig = queue[qix]; sig; sig = sig->next) { - - if (ERTS_SIG_IS_MSG(sig)) + + if (ERTS_SIG_IS_MSG(sig)) { msg_func(sig, arg); - else { + } else { Eterm tag; Uint16 type; int op; @@ -4481,18 +4671,21 @@ erts_proc_sig_debug_foreach_sig(Process *c_p, case ERTS_SIG_Q_OP_MONITOR_DOWN: switch (type) { case ERTS_SIG_Q_TYPE_GEN_EXIT: - debug_foreach_sig_heap_frags(&sig->hfrag, oh_func, arg); + if (ERTS_SIG_IS_GEN_EXIT_EXTERNAL(sig)) + debug_foreach_sig_external(sig, ext_func, arg); + else + debug_foreach_sig_heap_frags(&sig->hfrag, oh_func, arg); break; case ERTS_LNK_TYPE_PORT: case ERTS_LNK_TYPE_PROC: case ERTS_LNK_TYPE_DIST_PROC: - lnk_func((ErtsLink *) sig, arg); + lnk_func((ErtsLink *) sig, arg, -1); break; case ERTS_MON_TYPE_PORT: case ERTS_MON_TYPE_PROC: case ERTS_MON_TYPE_DIST_PROC: case ERTS_MON_TYPE_NODE: - mon_func((ErtsMonitor *) sig, arg); + mon_func((ErtsMonitor *) sig, arg, -1); break; default: ERTS_INTERNAL_ERROR("Unexpected sig type"); @@ -4513,7 +4706,7 @@ erts_proc_sig_debug_foreach_sig(Process *c_p, /* Fall through... */ case ERTS_SIG_Q_OP_MONITOR: - mon_func((ErtsMonitor *) sig, arg); + mon_func((ErtsMonitor *) sig, arg, -1); break; case ERTS_SIG_Q_OP_UNLINK: @@ -4525,7 +4718,7 @@ erts_proc_sig_debug_foreach_sig(Process *c_p, /* Fall through... */ case ERTS_SIG_Q_OP_LINK: - lnk_func((ErtsLink *) sig, arg); + lnk_func((ErtsLink *) sig, arg, -1); break; case ERTS_SIG_Q_OP_GROUP_LEADER: { diff --git a/erts/emulator/beam/erl_proc_sig_queue.h b/erts/emulator/beam/erl_proc_sig_queue.h index 6b065a7add..2b055e73bc 100644 --- a/erts/emulator/beam/erl_proc_sig_queue.h +++ b/erts/emulator/beam/erl_proc_sig_queue.h @@ -89,6 +89,7 @@ #endif struct erl_mesg; +struct erl_dist_external; typedef struct { struct erl_mesg *next; @@ -212,6 +213,38 @@ erts_proc_sig_send_exit(Process *c_p, Eterm from, Eterm to, /** * + * @brief Send an exit signal to a process. + * + * This function is used instead of erts_proc_sig_send_link_exit() + * when the signal arrives via the distribution and + * therefore no link structure is available. + * + * @param[in] dep Distribution entry of channel + * that the signal arrived on. + * + * @param[in] from Identifier of sender. + * + * @param[in] to Identifier of receiver. + * + * @param[in] dist_ext The exit reason in external term format + * + * @param[in] hfrag Heap frag with trace token and dist_ext + * iff available, otherwise NULL. + * + * @param[in] reason Exit reason. + * + * @param[in] token Seq trace token. + * + */ +void +erts_proc_sig_send_dist_exit(DistEntry *dep, + Eterm from, Eterm to, + ErtsDistExternal *dist_ext, + ErlHeapFragment *hfrag, + Eterm reason, Eterm token); + +/** + * * @brief Send an exit signal due to broken link to a process. * * @@ -282,7 +315,7 @@ erts_proc_sig_send_unlink(Process *c_p, ErtsLink *lnk); * * This function is used instead of erts_proc_sig_send_link_exit() * when the signal arrives via the distribution and - * no link structure is available. + * therefore no link structure is available. * * @param[in] dep Distribution entry of channel * that the signal arrived on. @@ -291,6 +324,11 @@ erts_proc_sig_send_unlink(Process *c_p, ErtsLink *lnk); * * @param[in] to Identifier of receiver. * + * @param[in] dist_ext The exit reason in external term format + * + * @param[in] hfrag Heap frag with trace token and dist_ext + * iff available, otherwise NULL. + * * @param[in] reason Exit reason. * * @param[in] token Seq trace token. @@ -299,6 +337,8 @@ erts_proc_sig_send_unlink(Process *c_p, ErtsLink *lnk); void erts_proc_sig_send_dist_link_exit(struct dist_entry_ *dep, Eterm from, Eterm to, + ErtsDistExternal *dist_ext, + ErlHeapFragment *hfrag, Eterm reason, Eterm token); /** @@ -307,7 +347,7 @@ erts_proc_sig_send_dist_link_exit(struct dist_entry_ *dep, * * This function is used instead of erts_proc_sig_send_unlink() * when the signal arrives via the distribution and - * no link structure is available. + * therefore no link structure is available. * * @param[in] dep Distribution entry of channel * that the signal arrived on. @@ -380,7 +420,7 @@ erts_proc_sig_send_monitor(ErtsMonitor *mon, Eterm to); * * This function is used instead of erts_proc_sig_send_monitor_down() * when the signal arrives via the distribution and - * no link structure is available. + * therefore no monitor structure is available. * * @param[in] dep Pointer to distribution entry * of channel that the signal @@ -392,12 +432,19 @@ erts_proc_sig_send_monitor(ErtsMonitor *mon, Eterm to); * * @param[in] to Identifier of receiver. * + * @param[in] dist_ext The exit reason in external term format + * + * @param[in] hfrag Heap frag with trace token and dist_ext + * iff available, otherwise NULL. + * * @param[in] reason Exit reason. * */ void erts_proc_sig_send_dist_monitor_down(DistEntry *dep, Eterm ref, Eterm from, Eterm to, + ErtsDistExternal *dist_ext, + ErlHeapFragment *hfrag, Eterm reason); /** @@ -740,7 +787,7 @@ erts_proc_sig_handle_incoming(Process *c_p, erts_aint32_t *statep, * queue. */ int -erts_proc_sig_handle_exit(Process *c_p, int *redsp); +erts_proc_sig_handle_exit(Process *c_p, Sint *redsp); /** * @@ -962,6 +1009,34 @@ void erts_proc_sig_handle_pending_suspend(Process *c_p); /** + * + * @brief Decode the reason term in an external signal + * + * Any distributed signal with a payload only has the control + * message decoded by the dist entry. The final decode of the + * payload is done by the process when it inspects the signal + * by calling this function. + * + * This functions handles both messages and link/monitor exits. + * + * Return true if the decode was successful, false otherwise. + * + * @param[in] c_p Pointer to executing process + * + * @param[in] proc_lock Locks held by process. Should always be MAIN. + * + * @param[in] msgp The signal to decode + * + * @param[in] force_off_heap If the term should be forced to be off-heap + */ +int +erts_proc_sig_decode_dist(Process *proc, ErtsProcLocks proc_locks, + ErtsMessage *msgp, int force_off_heap); + +ErtsDistExternal * +erts_proc_sig_get_external(ErtsMessage *msgp); + +/** * @brief Initialize this functionality */ void erts_proc_sig_queue_init(void); @@ -970,8 +1045,9 @@ void erts_proc_sig_debug_foreach_sig(Process *c_p, void (*msg_func)(ErtsMessage *, void *), void (*oh_func)(ErlOffHeap *, void *), - void (*mon_func)(ErtsMonitor *, void *), - void (*lnk_func)(ErtsLink *, void *), + ErtsMonitorFunc mon_func, + ErtsLinkFunc lnk_func, + void (*ext_func)(ErtsDistExternal *, void *), void *arg); extern Process *erts_dirty_process_signal_handler; diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 6419998702..0a099e69bb 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -735,12 +735,6 @@ erts_pre_init_process(void) #endif } -static void -release_process(void *vproc) -{ - erts_proc_dec_refc((Process *) vproc); -} - /* initialize the scheduler */ void erts_init_process(int ncpu, int proc_tab_size, int legacy_proc_tab) @@ -752,7 +746,7 @@ erts_init_process(int ncpu, int proc_tab_size, int legacy_proc_tab) erts_ptab_init_table(&erts_proc, ERTS_ALC_T_PROC_TABLE, - release_process, + NULL, (ErtsPTabElementCommon *) &erts_invalid_process.common, proc_tab_size, sizeof(Process), @@ -1476,7 +1470,10 @@ proclist_create(Process *p) { ErtsProcList *plp = proclist_alloc(); ensure_later_proc_interval(p->common.u.alive.started_interval); - plp->pid = p->common.id; + if (erts_atomic32_read_nob(&p->state) & ERTS_PSFLG_FREE) + plp->u.p = p; + else + plp->u.pid = p->common.id; plp->started_interval = p->common.u.alive.started_interval; return plp; } @@ -1485,7 +1482,7 @@ static ERTS_INLINE ErtsProcList * proclist_copy(ErtsProcList *plp0) { ErtsProcList *plp1 = proclist_alloc(); - plp1->pid = plp0->pid; + plp1->u.pid = plp0->u.pid; plp1->started_interval = plp0->started_interval; return plp1; } @@ -1520,7 +1517,10 @@ erts_proclist_dump(fmtfn_t to, void *to_arg, ErtsProcList *plp) ErtsProcList *first = plp; while (plp) { - erts_print(to, to_arg, "%T", plp->pid); + if (is_pid(plp->u.pid)) + erts_print(to, to_arg, "%T", plp->u.pid); + else + erts_print(to, to_arg, "%T", plp->u.p->common.id); plp = plp->next; if (plp == first) break; @@ -6526,8 +6526,7 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, n &= ~running_flgs; if ((!!(a & (ERTS_PSFLG_ACTIVE_SYS|ERTS_PSFLG_DIRTY_ACTIVE_SYS)) - | ((a & (ERTS_PSFLG_ACTIVE|ERTS_PSFLG_SUSPENDED)) == ERTS_PSFLG_ACTIVE)) - & !(a & ERTS_PSFLG_FREE)) { + | ((a & (ERTS_PSFLG_ACTIVE|ERTS_PSFLG_SUSPENDED)) == ERTS_PSFLG_ACTIVE))) { enqueue = check_enqueue_in_prio_queue(p, &enq_prio, &n, a); } a = erts_atomic32_cmpxchg_mb(&p->state, n, e); @@ -6562,7 +6561,6 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, else { Process* sched_p; - ASSERT(!(n & ERTS_PSFLG_FREE)); ASSERT(!(n & ERTS_PSFLG_SUSPENDED) || (n & (ERTS_PSFLG_ACTIVE_SYS | ERTS_PSFLG_DIRTY_ACTIVE_SYS))); @@ -6692,8 +6690,8 @@ change_proc_schedule_state(Process *p, enqueue = ERTS_ENQUEUE_NOT; - if (a & ERTS_PSFLG_FREE) - break; /* We don't want to schedule free processes... */ + if ((a & (ERTS_PSFLG_FREE|ERTS_PSFLG_ACTIVE)) == ERTS_PSFLG_FREE) + break; /* If free and not active, do not schedule */ if (clear_state_flags) n &= ~clear_state_flags; @@ -7219,8 +7217,7 @@ schdlr_sspnd_resume_procs(ErtsSchedType sched_type, while (resume->msb.chngrs) { ErtsProcList *plp = resume->msb.chngrs; resume->msb.chngrs = plp->next; - schdlr_sspnd_resume_proc(sched_type, - plp->pid); + schdlr_sspnd_resume_proc(sched_type, plp->u.pid); proclist_destroy(plp); } } @@ -7652,7 +7649,7 @@ suspend_scheduler(ErtsSchedulerData *esdp) else { schdlr_sspnd.changer = am_true; /* change right in transit */ /* resume process that is queued for next change... */ - resume.onln.nxt = plp->pid; + resume.onln.nxt = plp->u.pid; ASSERT(is_internal_pid(resume.onln.nxt)); } } @@ -7872,7 +7869,7 @@ abort_sched_onln_chng_waitq(Process *p) proclist_destroy(plp); plp = erts_proclist_peek_first(schdlr_sspnd.chngq); if (plp) - resume = plp->pid; + resume = plp->u.pid; else schdlr_sspnd.changer = am_false; } @@ -8138,7 +8135,8 @@ done: ErtsSchedSuspendResult erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal, int all) { - int resume_proc, ix, res, have_unlocked_plocks = 0; + ErtsSchedSuspendResult res; + int resume_proc, ix, have_unlocked_plocks = 0; ErtsProcList *plp; ErtsMultiSchedulingBlock *msbp; erts_aint32_t chng_flg; @@ -8371,10 +8369,10 @@ erts_multi_scheduling_blockers(Process *p, int normal) plp1; plp1 = erts_proclist_peek_next(msbp->blckrs, plp1)) { for (plp2 = erts_proclist_peek_first(msbp->blckrs); - plp2->pid != plp1->pid; + plp2->u.pid != plp1->u.pid; plp2 = erts_proclist_peek_next(msbp->blckrs, plp2)); if (plp2 == plp1) { - res = CONS(hp, plp1->pid, res); + res = CONS(hp, plp1->u.pid, res); hp += 2; } /* else: already in result list */ @@ -9043,8 +9041,13 @@ erts_resume_processes(ErtsProcList *list) while (plp) { Process *proc; ErtsProcList *fplp; - ASSERT(is_internal_pid(plp->pid)); - proc = erts_pid2proc(NULL, 0, plp->pid, ERTS_PROC_LOCK_STATUS); + ASSERT(is_internal_pid(plp->u.pid) || is_CP((Eterm)plp->u.p)); + if (is_internal_pid(plp->u.pid)) + proc = erts_pid2proc(NULL, 0, plp->u.pid, ERTS_PROC_LOCK_STATUS); + else { + proc = plp->u.p; + erts_proc_lock(proc, ERTS_PROC_LOCK_STATUS); + } if (proc) { if (erts_proclist_same(plp, proc)) { resume_process(proc, ERTS_PROC_LOCK_STATUS); @@ -9636,7 +9639,8 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) while (1) { erts_aint32_t exp, new; - int run_process; + int run_process, not_running, exiting_on_normal_sched, + not_suspended, not_exiting_on_dirty_sched; new = exp = state; new &= psflg_band_mask; /* @@ -9645,29 +9649,33 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) * scheduler, and not suspended (and not in a * state where suspend should be ignored). */ - run_process = (((!(state & (ERTS_PSFLG_RUNNING - | ERTS_PSFLG_RUNNING_SYS - | ERTS_PSFLG_DIRTY_RUNNING - | ERTS_PSFLG_DIRTY_RUNNING_SYS - | ERTS_PSFLG_FREE))) - | (((state & (ERTS_PSFLG_RUNNING - - | ERTS_PSFLG_FREE - | ERTS_PSFLG_RUNNING_SYS - | ERTS_PSFLG_DIRTY_RUNNING_SYS - | ERTS_PSFLG_EXITING)) - == ERTS_PSFLG_EXITING) - & (!!is_normal_sched)) - ) - & ((state & (ERTS_PSFLG_SUSPENDED - | ERTS_PSFLG_EXITING - | ERTS_PSFLG_FREE - | ERTS_PSFLG_ACTIVE_SYS - | ERTS_PSFLG_DIRTY_ACTIVE_SYS)) - != ERTS_PSFLG_SUSPENDED) - & (!(state & ERTS_PSFLG_EXITING) - | (!!is_normal_sched)) - ); + not_running = !(state & (ERTS_PSFLG_RUNNING + | ERTS_PSFLG_RUNNING_SYS + | ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS + | ERTS_PSFLG_FREE)); + exiting_on_normal_sched = + ((state & (ERTS_PSFLG_RUNNING + | ERTS_PSFLG_ACTIVE + | ERTS_PSFLG_RUNNING_SYS + | ERTS_PSFLG_DIRTY_RUNNING_SYS + | ERTS_PSFLG_EXITING)) + == (ERTS_PSFLG_EXITING|ERTS_PSFLG_ACTIVE)) + & (!!is_normal_sched); + + + not_suspended = ((state & (ERTS_PSFLG_SUSPENDED + | ERTS_PSFLG_EXITING + | ERTS_PSFLG_FREE + | ERTS_PSFLG_ACTIVE_SYS + | ERTS_PSFLG_DIRTY_ACTIVE_SYS)) + != ERTS_PSFLG_SUSPENDED); + + not_exiting_on_dirty_sched = !(state & ERTS_PSFLG_EXITING) | (!!is_normal_sched); + + run_process = (not_running | exiting_on_normal_sched) + & not_suspended + & not_exiting_on_dirty_sched; if (run_process) { if (state & (ERTS_PSFLG_ACTIVE_SYS @@ -9958,7 +9966,7 @@ trace_schedule_in(Process *p, erts_aint32_t state) /* Clear tracer if it has been removed */ if (erts_is_tracer_proc_enabled(p, ERTS_PROC_LOCK_MAIN, &p->common)) { - if (state & ERTS_PSFLG_EXITING) { + if (state & ERTS_PSFLG_EXITING && p->u.terminate) { if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_EXIT)) trace_sched(p, ERTS_PROC_LOCK_MAIN, am_in_exiting); } @@ -9982,12 +9990,9 @@ trace_schedule_out(Process *p, erts_aint32_t state) if (IS_TRACED_FL(p, F_TRACE_CALLS) && !(state & ERTS_PSFLG_FREE)) erts_schedule_time_break(p, ERTS_BP_CALL_TIME_SCHEDULE_OUT); - if ((state & (ERTS_PSFLG_FREE|ERTS_PSFLG_EXITING)) == ERTS_PSFLG_EXITING) { + if (state & ERTS_PSFLG_EXITING && p->u.terminate) { if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_EXIT)) - trace_sched(p, ERTS_PROC_LOCK_MAIN, - ((state & ERTS_PSFLG_FREE) - ? am_out_exited - : am_out_exiting)); + trace_sched(p, ERTS_PROC_LOCK_MAIN, am_out_exiting); } else { if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED) || @@ -12062,12 +12067,91 @@ erts_set_self_exiting(Process *c_p, Eterm reason) add2runq(enqueue, enq_prio, c_p, state, NULL); } -void -erts_proc_exit_handle_monitor(ErtsMonitor *mon, void *vctxt) +static int +erts_proc_exit_handle_dist_monitor(ErtsMonitor *mon, void *vctxt, Sint reds) +{ + ErtsProcExitContext *ctxt = (ErtsProcExitContext *) vctxt; + Process *c_p = ctxt->c_p; + Eterm reason = ctxt->reason; + int code; + ErtsDSigSendContext ctx; + ErtsMonLnkDist *dist; + DistEntry *dep; + Eterm watcher; + ErtsMonitorData *mdp = NULL; + Eterm watched; + + ASSERT(erts_monitor_is_target(mon) && mon->type == ERTS_MON_TYPE_DIST_PROC); + + mdp = erts_monitor_to_data(mon); + + if (mon->flags & ERTS_ML_FLG_NAME) + watched = ((ErtsMonitorDataExtended *) mdp)->u.name; + else + watched = c_p->common.id; + ASSERT(is_internal_pid(watched) || is_atom(watched)); + + watcher = mon->other.item; + ASSERT(is_external_pid(watcher)); + dep = external_pid_dist_entry(watcher); + ASSERT(dep); + dist = ((ErtsMonitorDataExtended *) mdp)->dist; + ASSERT(dist); + + code = erts_dsig_prepare(&ctx, dep, c_p, ERTS_PROC_LOCK_MAIN, + ERTS_DSP_NO_LOCK, 0, 0, 1); + + ctx.reds = (Sint) (reds * TERM_TO_BINARY_LOOP_FACTOR); + + switch (code) { + case ERTS_DSIG_PREP_NOT_ALIVE: + case ERTS_DSIG_PREP_NOT_CONNECTED: + break; + case ERTS_DSIG_PREP_PENDING: + case ERTS_DSIG_PREP_CONNECTED: + if (dist->connection_id != ctx.connection_id) + break; + code = erts_dsig_send_m_exit(&ctx, + watcher, + watched, + mdp->ref, + reason); + switch (code) { + case ERTS_DSIG_SEND_CONTINUE: + erts_set_gc_state(c_p, 0); + ctxt->dist_state = erts_dsend_export_trap_context(c_p, &ctx); + /* fall-through */ + case ERTS_DSIG_SEND_YIELD: + break; + case ERTS_DSIG_SEND_OK: + break; + case ERTS_DSIG_SEND_TOO_LRG: + erts_set_gc_state(c_p, 1); + break; + default: + ASSERT(! "Invalid dsig send exit monitor result"); + break; + } + break; + default: + ASSERT(! "Invalid dsig prep exit monitor result"); + break; + } + if (!erts_monitor_dist_delete(&mdp->origin)) + erts_monitor_release(mon); + else + erts_monitor_release_both(mdp); + return reds - (ctx.reds / TERM_TO_BINARY_LOOP_FACTOR); +} + +int +erts_proc_exit_handle_monitor(ErtsMonitor *mon, void *vctxt, Sint reds) { - Process *c_p = ((ErtsProcExitContext *) vctxt)->c_p; - Eterm reason = ((ErtsProcExitContext *) vctxt)->reason; + ErtsProcExitContext *ctxt = (ErtsProcExitContext *) vctxt; + Process *c_p = ctxt->c_p; + Eterm reason = ctxt->reason; ErtsMonitorData *mdp = NULL; + int res = 1; if (erts_monitor_is_target(mon)) { /* We are being watched... */ @@ -12097,43 +12181,48 @@ erts_proc_exit_handle_monitor(ErtsMonitor *mon, void *vctxt) case ERTS_MON_TYPE_DIST_PROC: { ErtsMonLnkDist *dist; DistEntry *dep; - ErtsDSigData dsd; + ErtsDSigSendContext ctx; int code; Eterm watcher; Eterm watched; - mdp = erts_monitor_to_data(mon); + if (is_immed(reason)) { + mdp = erts_monitor_to_data(mon); - if (mon->flags & ERTS_ML_FLG_NAME) - watched = ((ErtsMonitorDataExtended *) mdp)->u.name; - else - watched = c_p->common.id; - ASSERT(is_internal_pid(watched) || is_atom(watched)); - - watcher = mon->other.item; - ASSERT(is_external_pid(watcher)); - dep = external_pid_dist_entry(watcher); - ASSERT(dep); - dist = ((ErtsMonitorDataExtended *) mdp)->dist; - ASSERT(dist); - code = erts_dsig_prepare(&dsd, dep, NULL, 0, - ERTS_DSP_NO_LOCK, 0, 0); - switch (code) { - case ERTS_DSIG_PREP_CONNECTED: - case ERTS_DSIG_PREP_PENDING: - if (dist->connection_id == dsd.connection_id) { - code = erts_dsig_send_m_exit(&dsd, - watcher, - watched, - mdp->ref, - reason); - ASSERT(code == ERTS_DSIG_SEND_OK); + if (mon->flags & ERTS_ML_FLG_NAME) + watched = ((ErtsMonitorDataExtended *) mdp)->u.name; + else + watched = c_p->common.id; + ASSERT(is_internal_pid(watched) || is_atom(watched)); + + watcher = mon->other.item; + ASSERT(is_external_pid(watcher)); + dep = external_pid_dist_entry(watcher); + ASSERT(dep); + dist = ((ErtsMonitorDataExtended *) mdp)->dist; + ASSERT(dist); + code = erts_dsig_prepare(&ctx, dep, NULL, 0, + ERTS_DSP_NO_LOCK, 1, 1, 0); + switch (code) { + case ERTS_DSIG_PREP_CONNECTED: + case ERTS_DSIG_PREP_PENDING: + if (dist->connection_id == ctx.connection_id) { + code = erts_dsig_send_m_exit(&ctx, + watcher, + watched, + mdp->ref, + reason); + ASSERT(code == ERTS_DSIG_SEND_OK); + } + default: + break; } - default: - break; + if (!erts_monitor_dist_delete(&mdp->origin)) + mdp = NULL; + } else { + erts_monitor_tree_insert(&ctxt->dist_monitors, mon); + return 1; } - if (!erts_monitor_dist_delete(&mdp->origin)) - mdp = NULL; break; } default: @@ -12175,7 +12264,7 @@ erts_proc_exit_handle_monitor(ErtsMonitor *mon, void *vctxt) case ERTS_MON_TYPE_DIST_PROC: { ErtsMonLnkDist *dist; DistEntry *dep; - ErtsDSigData dsd; + ErtsDSigSendContext ctx; int code; Eterm watched; @@ -12192,17 +12281,16 @@ erts_proc_exit_handle_monitor(ErtsMonitor *mon, void *vctxt) ASSERT(is_external_pid(watched)); dep = external_pid_dist_entry(watched); } - code = erts_dsig_prepare(&dsd, dep, NULL, 0, - ERTS_DSP_NO_LOCK, 0, 0); + code = erts_dsig_prepare(&ctx, dep, NULL, 0, + ERTS_DSP_NO_LOCK, 1, 1, 0); switch (code) { case ERTS_DSIG_PREP_CONNECTED: case ERTS_DSIG_PREP_PENDING: - if (dist->connection_id == dsd.connection_id) { - code = erts_dsig_send_demonitor(&dsd, + if (dist->connection_id == ctx.connection_id) { + code = erts_dsig_send_demonitor(&ctx, c_p->common.id, watched, - mdp->ref, - 1); + mdp->ref); ASSERT(code == ERTS_DSIG_SEND_OK); } default: @@ -12210,6 +12298,7 @@ erts_proc_exit_handle_monitor(ErtsMonitor *mon, void *vctxt) } if (!erts_monitor_dist_delete(&mdp->target)) mdp = NULL; + res = 100; break; } default: @@ -12222,11 +12311,84 @@ erts_proc_exit_handle_monitor(ErtsMonitor *mon, void *vctxt) erts_monitor_release_both(mdp); else if (mon) erts_monitor_release(mon); + return res; } -void -erts_proc_exit_handle_link(ErtsLink *lnk, void *vctxt) +static int +erts_proc_exit_handle_dist_link(ErtsLink *lnk, void *vctxt, Sint reds) +{ + ErtsProcExitContext *ctxt = (ErtsProcExitContext *) vctxt; + Process *c_p = ctxt->c_p; + Eterm reason = ctxt->reason; + int code; + ErtsDSigSendContext ctx; + ErtsMonLnkDist *dist; + DistEntry *dep; + ErtsLink *dlnk; + ErtsLinkData *ldp = NULL; + + ASSERT(lnk->type == ERTS_LNK_TYPE_DIST_PROC); + dlnk = erts_link_to_other(lnk, &ldp); + dist = ((ErtsLinkDataExtended *) ldp)->dist; + + ASSERT(is_external_pid(lnk->other.item)); + dep = external_pid_dist_entry(lnk->other.item); + + ASSERT(dep != erts_this_dist_entry); + + if (!erts_link_dist_delete(dlnk)) + ldp = NULL; + + code = erts_dsig_prepare(&ctx, dep, c_p, ERTS_PROC_LOCK_MAIN, + ERTS_DSP_NO_LOCK, 0, 0, 0); + + ctx.reds = (Sint) (reds * TERM_TO_BINARY_LOOP_FACTOR); + + switch (code) { + case ERTS_DSIG_PREP_NOT_ALIVE: + case ERTS_DSIG_PREP_NOT_CONNECTED: + break; + case ERTS_DSIG_PREP_PENDING: + case ERTS_DSIG_PREP_CONNECTED: + if (dist->connection_id != ctx.connection_id) + break; + code = erts_dsig_send_exit_tt(&ctx, + c_p->common.id, + lnk->other.item, + reason, + SEQ_TRACE_TOKEN(c_p)); + switch (code) { + case ERTS_DSIG_SEND_CONTINUE: + erts_set_gc_state(c_p, 0); + ctxt->dist_state = erts_dsend_export_trap_context(c_p, &ctx); + /* fall-through */ + case ERTS_DSIG_SEND_YIELD: + break; + case ERTS_DSIG_SEND_OK: + break; + case ERTS_DSIG_SEND_TOO_LRG: + erts_set_gc_state(c_p, 1); + break; + default: + ASSERT(! "Invalid dsig send exit monitor result"); + break; + } + break; + default: + ASSERT(! "Invalid dsig prep exit monitor result"); + break; + } + if (ldp) + erts_link_release_both(ldp); + else if (lnk) + erts_link_release(lnk); + return reds - (ctx.reds / TERM_TO_BINARY_LOOP_FACTOR); +} + +int +erts_proc_exit_handle_link(ErtsLink *lnk, void *vctxt, Sint reds) { + ErtsProcExitContext *ctxt = (ErtsProcExitContext *) vctxt; Process *c_p = ((ErtsProcExitContext *) vctxt)->c_p; Eterm reason = ((ErtsProcExitContext *) vctxt)->reason; ErtsLinkData *ldp = NULL; @@ -12257,32 +12419,40 @@ erts_proc_exit_handle_link(ErtsLink *lnk, void *vctxt) DistEntry *dep; ErtsMonLnkDist *dist; ErtsLink *dlnk; - ErtsDSigData dsd; + ErtsDSigSendContext ctx; int code; - dlnk = erts_link_to_other(lnk, &ldp); - dist = ((ErtsLinkDataExtended *) ldp)->dist; + if (is_immed(reason)) { + dlnk = erts_link_to_other(lnk, &ldp); + dist = ((ErtsLinkDataExtended *) ldp)->dist; - ASSERT(is_external_pid(lnk->other.item)); - dep = external_pid_dist_entry(lnk->other.item); + ASSERT(is_external_pid(lnk->other.item)); + dep = external_pid_dist_entry(lnk->other.item); - ASSERT(dep != erts_this_dist_entry); + ASSERT(dep != erts_this_dist_entry); - if (!erts_link_dist_delete(dlnk)) - ldp = NULL; + if (!erts_link_dist_delete(dlnk)) + ldp = NULL; - code = erts_dsig_prepare(&dsd, dep, c_p, 0, ERTS_DSP_NO_LOCK, 0, 0); - switch (code) { - case ERTS_DSIG_PREP_CONNECTED: - case ERTS_DSIG_PREP_PENDING: - if (dist->connection_id == dsd.connection_id) { - code = erts_dsig_send_exit_tt(&dsd, - c_p->common.id, - lnk->other.item, - reason, - SEQ_TRACE_TOKEN(c_p)); - ASSERT(code == ERTS_DSIG_SEND_OK); + code = erts_dsig_prepare(&ctx, dep, c_p, 0, ERTS_DSP_NO_LOCK, 1, 1, 0); + switch (code) { + case ERTS_DSIG_PREP_CONNECTED: + case ERTS_DSIG_PREP_PENDING: + if (dist->connection_id == ctx.connection_id) { + code = erts_dsig_send_exit_tt(&ctx, + c_p->common.id, + lnk->other.item, + reason, + SEQ_TRACE_TOKEN(c_p)); + ASSERT(code == ERTS_DSIG_SEND_OK); + } + break; + default: + break; } + } else { + erts_link_tree_insert(&ctxt->dist_links, lnk); + return 1; } break; } @@ -12295,6 +12465,7 @@ erts_proc_exit_handle_link(ErtsLink *lnk, void *vctxt) erts_link_release_both(ldp); else if (lnk) erts_link_release(lnk); + return 1; } /* this function fishishes a process and propagates exit messages - called @@ -12328,11 +12499,8 @@ erts_do_exit_process(Process* p, Eterm reason) set_self_exiting(p, reason, NULL, NULL, NULL); - if (IS_TRACED(p)) { - if (IS_TRACED_FL(p, F_TRACE_CALLS)) - erts_schedule_time_break(p, ERTS_BP_CALL_TIME_SCHEDULE_EXITING); - - } + if (IS_TRACED_FL(p, F_TRACE_CALLS)) + erts_schedule_time_break(p, ERTS_BP_CALL_TIME_SCHEDULE_EXITING); erts_trace_check_exiting(p->common.id); @@ -12347,288 +12515,444 @@ erts_do_exit_process(Process* p, Eterm reason) erts_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR); - if (IS_TRACED_FL(p,F_TRACE_PROCS)) + if (IS_TRACED_FL(p, F_TRACE_PROCS)) trace_proc(p, ERTS_PROC_LOCK_MAIN, p, am_exit, reason); - /* * p->u.initial of this process can *not* be used anymore; * will be overwritten by misc termination data. */ p->u.terminate = NULL; + BUMP_REDS(p, 100); + erts_continue_exit_process(p); } -void -erts_continue_exit_process(Process *p) -{ +enum continue_exit_phase { + ERTS_CONTINUE_EXIT_TIMERS, + ERTS_CONTINUE_EXIT_BLCKD_MSHED, + ERTS_CONTINUE_EXIT_BLCKD_NMSHED, + ERTS_CONTINUE_EXIT_USING_DB, + ERTS_CONTINUE_EXIT_CLEAN_SYS_TASKS, + ERTS_CONTINUE_EXIT_FREE, + ERTS_CONTINUE_EXIT_CLEAN_SYS_TASKS_AFTER, + ERTS_CONTINUE_EXIT_LINKS, + ERTS_CONTINUE_EXIT_MONITORS, + ERTS_CONTINUE_EXIT_LT_MONITORS, + ERTS_CONTINUE_EXIT_HANDLE_PROC_SIG, + ERTS_CONTINUE_EXIT_DIST_LINKS, + ERTS_CONTINUE_EXIT_DIST_MONITORS, + ERTS_CONTINUE_EXIT_DONE, +}; + +struct continue_exit_state { + enum continue_exit_phase phase; ErtsLink *links; ErtsMonitor *monitors; ErtsMonitor *lt_monitors; + Eterm reason; + ErtsProcExitContext pectxt; + DistEntry *dep; + void *yield_state; +}; + +void +erts_continue_exit_process(Process *p) +{ + struct continue_exit_state static_state, *trap_state = &static_state; ErtsProcLocks curr_locks = ERTS_PROC_LOCK_MAIN; - Eterm reason = p->fvalue; - DistEntry *dep = NULL; erts_aint32_t state; int delay_del_proc = 0; - ErtsProcExitContext pectxt; - + Sint reds = ERTS_BIF_REDS_LEFT(p); #ifdef DEBUG int yield_allowed = 1; #endif + if (p->u.terminate) { + trap_state = p->u.terminate; + } else { + trap_state->phase = ERTS_CONTINUE_EXIT_TIMERS; + trap_state->reason = p->fvalue; + trap_state->dep = NULL; + trap_state->yield_state = NULL; + } + ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(p)); ASSERT(ERTS_PROC_IS_EXITING(p)); ASSERT(erts_proc_read_refc(p) > 0); - if (p->bif_timers) { - if (erts_cancel_bif_timers(p, &p->bif_timers, &p->u.terminate)) { - ASSERT(erts_proc_read_refc(p) > 0); - goto yield; - } - ASSERT(erts_proc_read_refc(p) > 0); - p->bif_timers = NULL; - } - - if (p->flags & F_SCHDLR_ONLN_WAITQ) - abort_sched_onln_chng_waitq(p); - - if (p->flags & F_HAVE_BLCKD_MSCHED) { - ErtsSchedSuspendResult ssr; - ssr = erts_block_multi_scheduling(p, ERTS_PROC_LOCK_MAIN, 0, 0, 1); - switch (ssr) { - case ERTS_SCHDLR_SSPND_YIELD_RESTART: - goto yield; - case ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED: - case ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED: - case ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED: - case ERTS_SCHDLR_SSPND_YIELD_DONE_NMSCHED_BLOCKED: - case ERTS_SCHDLR_SSPND_DONE: - case ERTS_SCHDLR_SSPND_YIELD_DONE: - p->flags &= ~F_HAVE_BLCKD_MSCHED; - break; - case ERTS_SCHDLR_SSPND_EINVAL: - default: - erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error: %d\n", - __FILE__, __LINE__, (int) ssr); - } - } - if (p->flags & F_HAVE_BLCKD_NMSCHED) { - ErtsSchedSuspendResult ssr; - ssr = erts_block_multi_scheduling(p, ERTS_PROC_LOCK_MAIN, 0, 1, 1); - switch (ssr) { - case ERTS_SCHDLR_SSPND_YIELD_RESTART: - goto yield; - case ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED: - case ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED: - case ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED: - case ERTS_SCHDLR_SSPND_YIELD_DONE_NMSCHED_BLOCKED: - case ERTS_SCHDLR_SSPND_DONE: - case ERTS_SCHDLR_SSPND_YIELD_DONE: - p->flags &= ~F_HAVE_BLCKD_MSCHED; - break; - case ERTS_SCHDLR_SSPND_EINVAL: - default: - erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error: %d\n", - __FILE__, __LINE__, (int) ssr); - } - } +restart: + switch (trap_state->phase) { + case ERTS_CONTINUE_EXIT_TIMERS: + if (p->bif_timers) { + reds = erts_cancel_bif_timers(p, &p->bif_timers, &trap_state->yield_state, reds); + if (reds <= 0) goto yield; + p->bif_timers = NULL; + } - if (p->flags & F_USING_DB) { - if (erts_db_process_exiting(p, ERTS_PROC_LOCK_MAIN)) - goto yield; - p->flags &= ~F_USING_DB; - } + if (p->flags & F_SCHDLR_ONLN_WAITQ) { + abort_sched_onln_chng_waitq(p); + reds -= 100; + } - erts_set_gc_state(p, 1); - state = erts_atomic32_read_acqb(&p->state); - if ((state & ERTS_PSFLG_SYS_TASKS) || p->dirty_sys_tasks) { - if (cleanup_sys_tasks(p, state, CONTEXT_REDS) >= CONTEXT_REDS/2) - goto yield; - } + trap_state->phase = ERTS_CONTINUE_EXIT_BLCKD_MSHED; + if (reds <= 0) goto yield; + case ERTS_CONTINUE_EXIT_BLCKD_MSHED: + + if (p->flags & F_HAVE_BLCKD_MSCHED) { + ErtsSchedSuspendResult ssr; + ssr = erts_block_multi_scheduling(p, ERTS_PROC_LOCK_MAIN, 0, 0, 1); + switch (ssr) { + case ERTS_SCHDLR_SSPND_DONE: + case ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED: + case ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED: + p->flags &= ~F_HAVE_BLCKD_MSCHED; + break; + default: + erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error: %d\n", + __FILE__, __LINE__, (int) ssr); + } + reds -= 100; + } -#ifdef DEBUG - erts_proc_lock(p, ERTS_PROC_LOCK_STATUS); - ASSERT(ERTS_PROC_GET_DELAYED_GC_TASK_QS(p) == NULL); - ASSERT(p->dirty_sys_tasks == NULL); - erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); -#endif + trap_state->phase = ERTS_CONTINUE_EXIT_BLCKD_NMSHED; + if (reds <= 0) goto yield; + case ERTS_CONTINUE_EXIT_BLCKD_NMSHED: + + if (p->flags & F_HAVE_BLCKD_NMSCHED) { + ErtsSchedSuspendResult ssr; + ssr = erts_block_multi_scheduling(p, ERTS_PROC_LOCK_MAIN, 0, 1, 1); + switch (ssr) { + case ERTS_SCHDLR_SSPND_DONE: + case ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED: + case ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED: + p->flags &= ~F_HAVE_BLCKD_MSCHED; + break; + default: + erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error: %d\n", + __FILE__, __LINE__, (int) ssr); + } + reds -= 100; + } - if (p->flags & F_USING_DDLL) { - erts_ddll_proc_dead(p, ERTS_PROC_LOCK_MAIN); - p->flags &= ~F_USING_DDLL; - } + trap_state->yield_state = NULL; + trap_state->phase = ERTS_CONTINUE_EXIT_USING_DB; + if (reds <= 0) goto yield; + case ERTS_CONTINUE_EXIT_USING_DB: - /* - * The registered name *should* be the last "erlang resource" to - * cleanup. - */ - if (p->common.u.alive.reg) { - (void) erts_unregister_name(p, ERTS_PROC_LOCK_MAIN, NULL, THE_NON_VALUE); - ASSERT(!p->common.u.alive.reg); - } + if (p->flags & F_USING_DB) { + if (erts_db_process_exiting(p, ERTS_PROC_LOCK_MAIN, &trap_state->yield_state)) + goto yield; + p->flags &= ~F_USING_DB; + } - if (IS_TRACED_FL(p, F_TRACE_SCHED_EXIT)) - trace_sched(p, curr_locks, am_out_exited); + erts_set_gc_state(p, 1); - erts_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR); - curr_locks = ERTS_PROC_LOCKS_ALL; + trap_state->phase = ERTS_CONTINUE_EXIT_CLEAN_SYS_TASKS; + case ERTS_CONTINUE_EXIT_CLEAN_SYS_TASKS: + + state = erts_atomic32_read_acqb(&p->state); + if ((state & ERTS_PSFLG_SYS_TASKS) || p->dirty_sys_tasks) { + reds -= cleanup_sys_tasks(p, state, reds); + if (reds <= 0) goto yield; + } + + trap_state->phase = ERTS_CONTINUE_EXIT_FREE; + case ERTS_CONTINUE_EXIT_FREE: - /* - * From this point on we are no longer allowed to yield - * this process. - */ #ifdef DEBUG - yield_allowed = 0; + erts_proc_lock(p, ERTS_PROC_LOCK_STATUS); + ASSERT(ERTS_PROC_GET_DELAYED_GC_TASK_QS(p) == NULL); + ASSERT(p->dirty_sys_tasks == NULL); + erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); #endif - /* - * Note! The monitor and link fields will be overwritten - * by erts_ptab_delete_element() below. - */ - links = ERTS_P_LINKS(p); - monitors = ERTS_P_MONITORS(p); - lt_monitors = ERTS_P_LT_MONITORS(p); + if (p->flags & F_USING_DDLL) { + erts_ddll_proc_dead(p, ERTS_PROC_LOCK_MAIN); + p->flags &= ~F_USING_DDLL; + } - { - /* Do *not* use erts_get_runq_proc() */ - ErtsRunQueue *rq; - rq = erts_get_runq_current(erts_proc_sched_data(p)); + /* + * The registered name *should* be the last "erlang resource" to + * cleanup. + */ + if (p->common.u.alive.reg) { + (void) erts_unregister_name(p, ERTS_PROC_LOCK_MAIN, NULL, THE_NON_VALUE); + ASSERT(!p->common.u.alive.reg); + } - erts_runq_lock(rq); + erts_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR); + curr_locks = ERTS_PROC_LOCKS_ALL; - ASSERT(p->scheduler_data); - ASSERT(p->scheduler_data->current_process == p); - ASSERT(p->scheduler_data->free_process == NULL); + /* + * Note! The monitor and link fields will be overwritten + * by erts_ptab_delete_element() below. + */ + trap_state->links = ERTS_P_LINKS(p); + trap_state->monitors = ERTS_P_MONITORS(p); + trap_state->lt_monitors = ERTS_P_LT_MONITORS(p); - p->scheduler_data->current_process = NULL; - p->scheduler_data->free_process = p; + { + /* Do *not* use erts_get_runq_proc() */ + ErtsRunQueue *rq; + rq = erts_get_runq_current(erts_proc_sched_data(p)); - /* Time of death! */ - erts_ptab_delete_element(&erts_proc, &p->common); + erts_runq_lock(rq); - erts_runq_unlock(rq); - } + ASSERT(p->scheduler_data); + ASSERT(p->scheduler_data->current_process == p); + ASSERT(p->scheduler_data->free_process == NULL); - /* - * All "erlang resources" have to be deallocated before this point, - * e.g. registered name, so monitoring and linked processes can - * be sure that all interesting resources have been deallocated - * when the monitors and/or links hit. - */ + /* Time of death! */ + erts_ptab_delete_element(&erts_proc, &p->common); - { - /* Inactivate and notify free */ - erts_aint32_t n, e, a = erts_atomic32_read_nob(&p->state); - int refc_inced = 0; - while (1) { - n = e = a; - ASSERT(a & ERTS_PSFLG_EXITING); - n |= ERTS_PSFLG_FREE; - n &= ~(ERTS_PSFLG_ACTIVE - | ERTS_PSFLG_ACTIVE_SYS - | ERTS_PSFLG_DIRTY_ACTIVE_SYS); - if ((n & ERTS_PSFLG_IN_RUNQ) && !refc_inced) { - erts_proc_inc_refc(p); - refc_inced = 1; - } - a = erts_atomic32_cmpxchg_mb(&p->state, n, e); - if (a == e) - break; - } + erts_runq_unlock(rq); + } - if (a & (ERTS_PSFLG_DIRTY_RUNNING - | ERTS_PSFLG_DIRTY_RUNNING_SYS)) { - p->flags |= F_DELAYED_DEL_PROC; - delay_del_proc = 1; - /* - * The dirty scheduler decrease refc - * when done with the process... - */ - } + /* + * All "erlang resources" have to be deallocated before this point, + * e.g. registered name, so monitoring and linked processes can + * be sure that all interesting resources have been deallocated + * when the monitors and/or links hit. + */ - if (refc_inced && !(n & ERTS_PSFLG_IN_RUNQ)) - erts_proc_dec_refc(p); - } + { + /* Inactivate and notify free */ + erts_aint32_t n, e, a = erts_atomic32_read_nob(&p->state); + int refc_inced = 0; + while (1) { + n = e = a; + ASSERT(a & ERTS_PSFLG_EXITING); + n |= ERTS_PSFLG_FREE; + if ((n & ERTS_PSFLG_IN_RUNQ) && !refc_inced) { + erts_proc_inc_refc(p); + refc_inced = 1; + } + a = erts_atomic32_cmpxchg_mb(&p->state, n, e); + if (a == e) + break; + } - dep = ((p->flags & F_DISTRIBUTION) - ? ERTS_PROC_SET_DIST_ENTRY(p, NULL) - : NULL); + if (refc_inced && !(n & ERTS_PSFLG_IN_RUNQ)) + erts_proc_dec_refc(p); + } + trap_state->dep = ((p->flags & F_DISTRIBUTION) + ? ERTS_PROC_SET_DIST_ENTRY(p, NULL) + : NULL); + + reds -= 50; - /* - * It might show up signal prio elevation tasks until we - * have entered free state. Cleanup such tasks now. - */ - state = erts_atomic32_read_acqb(&p->state); - if (!(state & ERTS_PSFLG_SYS_TASKS)) - erts_proc_unlock(p, ERTS_PROC_LOCKS_ALL); - else { erts_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR); + curr_locks = ERTS_PROC_LOCK_MAIN; + trap_state->phase = ERTS_CONTINUE_EXIT_CLEAN_SYS_TASKS_AFTER; + case ERTS_CONTINUE_EXIT_CLEAN_SYS_TASKS_AFTER: + /* + * It might show up signal prio elevation tasks until we + * have entered free state. Cleanup such tasks now. + */ - do { - (void) cleanup_sys_tasks(p, state, CONTEXT_REDS); - state = erts_atomic32_read_acqb(&p->state); - } while (state & ERTS_PSFLG_SYS_TASKS); - - erts_proc_unlock(p, ERTS_PROC_LOCK_MAIN); - } + state = erts_atomic32_read_acqb(&p->state); + if ((state & ERTS_PSFLG_SYS_TASKS) || p->dirty_sys_tasks) { + reds -= cleanup_sys_tasks(p, state, reds); + if (reds <= 0) goto yield; + } + + /* Needs to be unlocked for erts_do_net_exits to work?!? */ + // erts_proc_unlock(p, ERTS_PROC_LOCK_MAIN); #ifdef DEBUG - erts_proc_lock(p, ERTS_PROC_LOCK_STATUS); - ASSERT(p->sys_task_qs == NULL); - erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + erts_proc_lock(p, ERTS_PROC_LOCK_STATUS); + ASSERT(p->sys_task_qs == NULL); + erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); #endif - if (dep) { - erts_do_net_exits(dep, (reason == am_kill) ? am_killed : reason); - erts_deref_dist_entry(dep); - } + if (trap_state->dep) { + erts_do_net_exits(trap_state->dep, + (trap_state->reason == am_kill) ? am_killed : trap_state->reason); + erts_deref_dist_entry(trap_state->dep); + } - pectxt.c_p = p; - pectxt.reason = reason; + trap_state->pectxt.c_p = p; + trap_state->pectxt.reason = trap_state->reason; + trap_state->pectxt.dist_links = NULL; + trap_state->pectxt.dist_monitors = NULL; + trap_state->pectxt.dist_state = NIL; + + erts_proc_lock(p, ERTS_PROC_LOCK_MSGQ); + + erts_proc_sig_fetch(p); + + erts_proc_unlock(p, ERTS_PROC_LOCK_MSGQ); + + trap_state->yield_state = NULL; + trap_state->phase = ERTS_CONTINUE_EXIT_LINKS; + if (reds <= 0) goto yield; + case ERTS_CONTINUE_EXIT_LINKS: + + reds = erts_link_tree_foreach_delete_yielding( + &trap_state->links, + erts_proc_exit_handle_link, + (void *) &trap_state->pectxt, + &trap_state->yield_state, + reds); + if (reds <= 0) + goto yield; + + ASSERT(!trap_state->links); + trap_state->yield_state = NULL; + trap_state->phase = ERTS_CONTINUE_EXIT_MONITORS; + case ERTS_CONTINUE_EXIT_MONITORS: + + reds = erts_monitor_tree_foreach_delete_yielding( + &trap_state->monitors, + erts_proc_exit_handle_monitor, + (void *) &trap_state->pectxt, + &trap_state->yield_state, + reds); + if (reds <= 0) + goto yield; + + ASSERT(!trap_state->monitors); + trap_state->yield_state = NULL; + trap_state->phase = ERTS_CONTINUE_EXIT_LT_MONITORS; + case ERTS_CONTINUE_EXIT_LT_MONITORS: + + reds = erts_monitor_list_foreach_delete_yielding( + &trap_state->lt_monitors, + erts_proc_exit_handle_monitor, + (void *) &trap_state->pectxt, + &trap_state->yield_state, + reds); + if (reds <= 0) + goto yield; + + ASSERT(!trap_state->lt_monitors); + trap_state->phase = ERTS_CONTINUE_EXIT_HANDLE_PROC_SIG; + case ERTS_CONTINUE_EXIT_HANDLE_PROC_SIG: { + Sint r = reds; + + if (!erts_proc_sig_handle_exit(p, &r)) + goto yield; + + reds -= r; + + trap_state->phase = ERTS_CONTINUE_EXIT_DIST_LINKS; + } + case ERTS_CONTINUE_EXIT_DIST_LINKS: { + + continue_dist_send: + if (is_not_nil(trap_state->pectxt.dist_state)) { + Binary* bin = erts_magic_ref2bin(trap_state->pectxt.dist_state); + ErtsDSigSendContext* ctx = (ErtsDSigSendContext*) ERTS_MAGIC_BIN_DATA(bin); + Sint initial_reds = (Sint) (ERTS_BIF_REDS_LEFT(p) * TERM_TO_BINARY_LOOP_FACTOR); + int result; + + ctx->reds = initial_reds; + result = erts_dsig_send(ctx); + + /* erts_dsig_send bumps reductions on the process in the ctx */ + reds = ERTS_BIF_REDS_LEFT(p); + + switch (result) { + case ERTS_DSIG_SEND_OK: + case ERTS_DSIG_SEND_TOO_LRG: /*SEND_SYSTEM_LIMIT*/ + case ERTS_DSIG_SEND_YIELD: /*SEND_YIELD_RETURN*/ + break; + case ERTS_DSIG_SEND_CONTINUE: { /*SEND_YIELD_CONTINUE*/ + goto yield; + } + } + erts_set_gc_state(p, 1); + trap_state->pectxt.dist_state = NIL; + if (reds <= 0) + goto yield; + goto restart; + } - erts_proc_lock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_MSGQ); + reds = erts_link_tree_foreach_delete_yielding( + &trap_state->pectxt.dist_links, + erts_proc_exit_handle_dist_link, + (void *) &trap_state->pectxt, + &trap_state->yield_state, + reds); + if (reds <= 0 || is_not_nil(trap_state->pectxt.dist_state)) + goto yield; + trap_state->phase = ERTS_CONTINUE_EXIT_DIST_MONITORS; + } + case ERTS_CONTINUE_EXIT_DIST_MONITORS: { + + if (is_not_nil(trap_state->pectxt.dist_state)) + goto continue_dist_send; + + reds = erts_monitor_tree_foreach_delete_yielding( + &trap_state->pectxt.dist_monitors, + erts_proc_exit_handle_dist_monitor, + (void *) &trap_state->pectxt, + &trap_state->yield_state, + reds); + if (reds <= 0 || is_not_nil(trap_state->pectxt.dist_state)) + goto yield; + + trap_state->phase = ERTS_CONTINUE_EXIT_DONE; + } + case ERTS_CONTINUE_EXIT_DONE: { + erts_aint_t state; + /* + * From this point on we are no longer allowed to yield + * this process. + */ - erts_proc_sig_fetch(p); +#ifdef DEBUG + yield_allowed = 0; +#endif - erts_proc_unlock(p, ERTS_PROC_LOCK_MSGQ); + /* Set state to not active as we don't want this process + to be scheduled in again after this. */ + state = erts_atomic32_read_band_relb(&p->state, + ~(ERTS_PSFLG_ACTIVE + | ERTS_PSFLG_ACTIVE_SYS + | ERTS_PSFLG_DIRTY_ACTIVE_SYS)); - if (links) { - erts_link_tree_foreach_delete(&links, - erts_proc_exit_handle_link, - (void *) &pectxt); - ASSERT(!links); - } + ASSERT(p->scheduler_data); + ASSERT(p->scheduler_data->current_process == p); + ASSERT(p->scheduler_data->free_process == NULL); - if (monitors) { - erts_monitor_tree_foreach_delete(&monitors, - erts_proc_exit_handle_monitor, - (void *) &pectxt); - ASSERT(!monitors); - } + p->scheduler_data->current_process = NULL; + p->scheduler_data->free_process = p; - if (lt_monitors) { - erts_monitor_list_foreach_delete(<_monitors, - erts_proc_exit_handle_monitor, - (void *) &pectxt); - ASSERT(!lt_monitors); - } + if (state & (ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS)) { + p->flags |= F_DELAYED_DEL_PROC; + delay_del_proc = 1; + /* + * The dirty scheduler decrease refc + * when done with the process... + */ + } - /* - * erts_proc_sig_handle_exit() implements yielding. - * However, this function cannot handle it yet... loop - * until done... - */ - while (!0) { - int reds = CONTEXT_REDS; - if (erts_proc_sig_handle_exit(p, &reds)) - break; + erts_schedule_thr_prgr_later_cleanup_op( + (void (*)(void*))erts_proc_dec_refc, + (void *) &p->common, + &p->common.u.release, + sizeof(Process)); + + break; + } } + if (trap_state != &static_state) { + erts_free(ERTS_ALC_T_CONT_EXIT_TRAP, trap_state); + p->u.terminate = NULL; + } + ERTS_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); + if (IS_TRACED_FL(p, F_TRACE_SCHED_EXIT)) + trace_sched(p, curr_locks, am_out_exited); + erts_flush_trace_messages(p, ERTS_PROC_LOCK_MAIN); ERTS_TRACER_CLEAR(&ERTS_TRACER(p)); @@ -12644,9 +12968,26 @@ erts_continue_exit_process(Process *p) ERTS_LC_ASSERT(curr_locks == erts_proc_lc_my_proc_locks(p)); ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN & curr_locks); + ASSERT(erts_proc_read_refc(p) > 0); + + if (trap_state == &static_state) { + trap_state = erts_alloc(ERTS_ALC_T_CONT_EXIT_TRAP, sizeof(*trap_state)); + sys_memcpy(trap_state, &static_state, sizeof(*trap_state)); + p->u.terminate = trap_state; + } + + ASSERT(p->scheduler_data); + ASSERT(p->scheduler_data->current_process == p); + ASSERT(p->scheduler_data->free_process == NULL); + + if (trap_state->phase >= ERTS_CONTINUE_EXIT_FREE) { + p->scheduler_data->current_process = NULL; + p->scheduler_data->free_process = p; + } p->i = (BeamInstr *) beam_continue_exit; + /* Why is this lock take??? */ if (!(curr_locks & ERTS_PROC_LOCK_STATUS)) { erts_proc_lock(p, ERTS_PROC_LOCK_STATUS); curr_locks |= ERTS_PROC_LOCK_STATUS; diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 43937f216c..3b593bce02 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -381,7 +381,10 @@ struct ErtsSchedulerSleepInfo_ { typedef struct ErtsProcList_ ErtsProcList; struct ErtsProcList_ { - Eterm pid; + union { + Eterm pid; + Process *p; + } u; Uint64 started_interval; ErtsProcList* next; ErtsProcList* prev; @@ -1580,7 +1583,7 @@ ERTS_GLB_INLINE int erts_proclist_is_last(ErtsProcList *, ErtsProcList *); ERTS_GLB_INLINE int erts_proclist_same(ErtsProcList *plp, Process *p) { - return (plp->pid == p->common.id + return ((plp->u.pid == p->common.id || plp->u.p == p) && (plp->started_interval == p->common.u.alive.started_interval)); } @@ -1819,9 +1822,12 @@ Eterm erts_process_info(Process *c_p, ErtsHeapFactory *hfact, typedef struct { Process *c_p; Eterm reason; + ErtsLink *dist_links; + ErtsMonitor *dist_monitors; + Eterm dist_state; } ErtsProcExitContext; -void erts_proc_exit_handle_monitor(ErtsMonitor *mon, void *vctxt); -void erts_proc_exit_handle_link(ErtsLink *lnk, void *vctxt); +int erts_proc_exit_handle_monitor(ErtsMonitor *mon, void *vctxt, Sint reds); +int erts_proc_exit_handle_link(ErtsLink *lnk, void *vctxt, Sint reds); Eterm erts_get_process_priority(erts_aint32_t state); Eterm erts_set_process_priority(Process *p, Eterm prio); diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c index 10ea401022..a164ed543e 100644 --- a/erts/emulator/beam/erl_process_dump.c +++ b/erts/emulator/beam/erl_process_dump.c @@ -100,16 +100,18 @@ erts_deep_process_dump(fmtfn_t to, void *to_arg) dump_binaries(to, to_arg, all_binaries); } -static void -monitor_size(ErtsMonitor *mon, void *vsize) +static int +monitor_size(ErtsMonitor *mon, void *vsize, Sint reds) { *((Uint *) vsize) += erts_monitor_size(mon); + return 1; } -static void -link_size(ErtsMonitor *lnk, void *vsize) +static int +link_size(ErtsMonitor *lnk, void *vsize, Sint reds) { *((Uint *) vsize) += erts_link_size(lnk); + return 1; } Uint erts_process_memory(Process *p, int include_sigs_in_transit) @@ -189,11 +191,11 @@ static ERTS_INLINE void dump_msg(fmtfn_t to, void *to_arg, ErtsMessage *mp) { if (ERTS_SIG_IS_MSG((ErtsSignal *) mp)) { - Eterm mesg = ERL_MESSAGE_TERM(mp); - if (is_value(mesg)) - dump_element(to, to_arg, mesg); + Eterm mesg; + if (ERTS_SIG_IS_INTERNAL_MSG(mp)) + dump_element(to, to_arg, ERL_MESSAGE_TERM(mp)); else - dump_dist_ext(to, to_arg, mp->data.dist_ext); + dump_dist_ext(to, to_arg, erts_get_dist_ext(mp->data.heap_frag)); mesg = ERL_MESSAGE_TOKEN(mp); erts_print(to, to_arg, ":"); dump_element(to, to_arg, mesg); @@ -265,6 +267,7 @@ dump_dist_ext(fmtfn_t to, void *to_arg, ErtsDistExternal *edep) else { byte *e; size_t sz; + int i; if (!(edep->flags & ERTS_DIST_EXT_ATOM_TRANS_TAB)) erts_print(to, to_arg, "D0:"); @@ -274,8 +277,8 @@ dump_dist_ext(fmtfn_t to, void *to_arg, ErtsDistExternal *edep) for (i = 0; i < edep->attab.size; i++) dump_element(to, to_arg, edep->attab.atom[i]); } - sz = edep->ext_endp - edep->extp; - e = edep->extp; + sz = edep->data->ext_endp - edep->data->extp; + e = edep->data->extp; if (edep->flags & ERTS_DIST_EXT_DFLAG_HDR) { ASSERT(*e != VERSION_MAGIC); sz++; @@ -286,15 +289,19 @@ dump_dist_ext(fmtfn_t to, void *to_arg, ErtsDistExternal *edep) erts_print(to, to_arg, "E%X:", sz); if (edep->flags & ERTS_DIST_EXT_DFLAG_HDR) { byte sbuf[3]; - int i = 0; + + i = 0; sbuf[i++] = VERSION_MAGIC; - while (i < sizeof(sbuf) && e < edep->ext_endp) { + while (i < sizeof(sbuf) && e < edep->data->ext_endp) { sbuf[i++] = *e++; } erts_print_base64(to, to_arg, sbuf, i); } - erts_print_base64(to, to_arg, e, edep->ext_endp - e); + erts_print_base64(to, to_arg, e, edep->data->ext_endp - e); + for (i = 1; i < edep->data->frag_id; i++) + erts_print_base64(to, to_arg, edep->data[i].extp, + edep->data[i].ext_endp - edep->data[i].extp); } } diff --git a/erts/emulator/beam/erl_ptab.h b/erts/emulator/beam/erl_ptab.h index 94f0247492..c30a684002 100644 --- a/erts/emulator/beam/erl_ptab.h +++ b/erts/emulator/beam/erl_ptab.h @@ -68,8 +68,11 @@ typedef struct { Uint64 started_interval; struct reg_proc *reg; ErtsLink *links; - ErtsMonitor *monitors; + /* Local target monitors, double linked list + contains the remote part of local monitors */ ErtsMonitor *lt_monitors; + /* other monitors, rb tree */ + ErtsMonitor *monitors; } alive; /* --- While being released --- */ diff --git a/erts/emulator/beam/erl_rbtree.h b/erts/emulator/beam/erl_rbtree.h index e50abf5cec..ce401fa7e7 100644 --- a/erts/emulator/beam/erl_rbtree.h +++ b/erts/emulator/beam/erl_rbtree.h @@ -161,7 +161,7 @@ * * - void <ERTS_RBT_PREFIX>_rbt_foreach( * ERTS_RBT_T *tree, - * void (*op)(ERTS_RBT_T *, void *), + * int (*op)(ERTS_RBT_T *, void *), * void *arg); * Operate by calling the operator 'op' on each element. * Order is undefined. @@ -170,7 +170,7 @@ * * - void <ERTS_RBT_PREFIX>_rbt_foreach_destroy( * ERTS_RBT_T *tree, - * void (*op)(ERTS_RBT_T *, void *), + * int (*op)(ERTS_RBT_T *, void *), * void *arg); * Operate by calling the operator 'op' on each element. * Order is undefined. Each element should be destroyed @@ -180,39 +180,46 @@ * * - int <ERTS_RBT_PREFIX>_rbt_foreach_yielding( * ERTS_RBT_T *tree, - * void (*op)(ERTS_RBT_T *, void *), + * int (*op)(ERTS_RBT_T *, void *), * void *arg, * <ERTS_RBT_PREFIX>_rbt_yield_state_t *ystate, - * Sint ylimit); + * Sint reds); * Operate by calling the operator 'op' on each element. * Order is undefined. * - * Yield when 'ylimit' elements has been processed. True is - * returned when yielding, and false is returned when - * the whole tree has been processed. The tree should not be - * modified until all of it has been processed. + * Yield when 'reds' reductions has been processed. The 'op' + * function return the number of reductions that each element + * took to process. The number of reductions remaining is returned, + * meaning that if 0 is returned, there are more elements to be + * processed. If a value greater than 0 is returned the foreach has + * ended. The tree should not be modified until all of it has been + * processed. * * 'arg' is passed as argument to 'op'. * * - int <ERTS_RBT_PREFIX>_rbt_foreach_destroy_yielding( * ERTS_RBT_T *tree, - * void (*op)(ERTS_RBT_T *, void *), + * int (*op)(ERTS_RBT_T *, void *), * void *arg, * <ERTS_RBT_PREFIX>_rbt_yield_state_t *ystate, - * Sint ylimit); + * Sint reds); * Operate by calling the operator 'op' on each element. * Order is undefined. Each element should be destroyed * by 'op'. * - * Yield when 'ylimit' elements has been processed. True is - * returned when yielding, and false is returned when - * the whole tree has been processed. + * Yield when 'reds' reductions has been processed. The 'op' + * function return the number of reductions that each element + * took to process. The number of reductions remaining is returned, + * meaning that if 0 is returned, there are more elements to be + * processed. If a value greater than 0 is returned the foreach has + * ended. The tree should not be modified until all of it has been + * processed. * * 'arg' is passed as argument to 'op'. * * - void <ERTS_RBT_PREFIX>_rbt_foreach_small( * ERTS_RBT_T *tree, - * void (*op)(ERTS_RBT_T *, void *), + * int (*op)(ERTS_RBT_T *, void *), * void *arg); * Operate by calling the operator 'op' on each element from * smallest towards larger elements. @@ -221,7 +228,7 @@ * * - void <ERTS_RBT_PREFIX>_rbt_foreach_large( * ERTS_RBT_T *tree, - * void (*op)(ERTS_RBT_T *, void *), + * int (*op)(ERTS_RBT_T *, void *), * void *arg); * Operate by calling the operator 'op' on each element from * largest towards smaller elements. @@ -230,40 +237,46 @@ * * - int <ERTS_RBT_PREFIX>_rbt_foreach_small_yielding( * ERTS_RBT_T *tree, - * void (*op)(ERTS_RBT_T *, void *), + * int (*op)(ERTS_RBT_T *, void *), * void *arg, * <ERTS_RBT_PREFIX>_rbt_yield_state_t *ystate, - * Sint ylimit); + * Sint reds); * Operate by calling the operator 'op' on each element from * smallest towards larger elements. * - * Yield when 'ylimit' elements has been processed. True is - * returned when yielding, and false is returned when - * the whole tree has been processed. The tree should not be - * modified until all of it has been processed. + * Yield when 'reds' reductions has been processed. The 'op' + * function return the number of reductions that each element + * took to process. The number of reductions remaining is returned, + * meaning that if 0 is returned, there are more elements to be + * processed. If a value greater than 0 is returned the foreach has + * ended. The tree should not be modified until all of it has been + * processed. * * 'arg' is passed as argument to 'op'. * * - int <ERTS_RBT_PREFIX>_rbt_foreach_large_yielding( * ERTS_RBT_T *tree, - * void (*op)(ERTS_RBT_T *, void *), + * int (*op)(ERTS_RBT_T *, void *), * void *arg, * <ERTS_RBT_PREFIX>_rbt_yield_state_t *ystate, - * Sint ylimit); + * Sint reds); * Operate by calling the operator 'op' on each element from * largest towards smaller elements. * - * Yield when 'ylimit' elements has been processed. True is - * returned when yielding, and false is returned when - * the whole tree has been processed. The tree should not be - * modified until all of it has been processed. + * Yield when 'reds' reductions has been processed. The 'op' + * function return the number of reductions that each element + * took to process. The number of reductions remaining is returned, + * meaning that if 0 is returned, there are more elements to be + * processed. If a value greater than 0 is returned the foreach has + * ended. The tree should not be modified until all of it has been + * processed. * * 'arg' is passed as argument to 'op'. * * - void <ERTS_RBT_PREFIX>_rbt_foreach_small_destroy( * ERTS_RBT_T **tree, - * void (*op)(ERTS_RBT_T *, void *), - * void (*destr)(ERTS_RBT_T *, void *), + * int (*op)(ERTS_RBT_T *, void *), + * int (*destr)(ERTS_RBT_T *, void *), * void *arg); * Operate by calling the operator 'op' on each element from * smallest towards larger elements. @@ -277,8 +290,8 @@ * * - void <ERTS_RBT_PREFIX>_rbt_foreach_large_destroy( * ERTS_RBT_T **tree, - * void (*op)(ERTS_RBT_T *, void *), - * void (*destr)(ERTS_RBT_T *, void *), + * int (*op)(ERTS_RBT_T *, void *), + * int (*destr)(ERTS_RBT_T *, void *), * void *arg); * Operate by calling the operator 'op' on each element from * largest towards smaller elements. @@ -292,11 +305,11 @@ * * - int <ERTS_RBT_PREFIX>_rbt_foreach_small_destroy_yielding( * ERTS_RBT_T **tree, - * void (*op)(ERTS_RBT_T *, void *), - * void (*destr)(ERTS_RBT_T *, void *), + * int (*op)(ERTS_RBT_T *, void *), + * int (*destr)(ERTS_RBT_T *, void *), * void *arg, * <ERTS_RBT_PREFIX>_rbt_yield_state_t *ystate, - * Sint ylimit); + * Sint reds); * Operate by calling the operator 'op' on each element from * smallest towards larger elements. * @@ -305,20 +318,23 @@ * Note that elements are often destroyed in another order * than the order that the elements are operated on. * - * Yield when 'ylimit' elements has been processed. True is - * returned when yielding, and false is returned when - * the whole tree has been processed. The tree should not be - * modified until all of it has been processed. + * Yield when 'reds' reductions has been processed. The 'op' and + * 'destr' functions return the number of reductions that each element + * took to process. The number of reductions remaining is returned, + * meaning that if 0 is returned, there are more elements to be + * processed. If a value greater than 0 is returned the foreach has + * ended. The tree should not be modified until all of it has been + * processed. * * 'arg' is passed as argument to 'op' and 'destroy'. * * - int <ERTS_RBT_PREFIX>_rbt_foreach_large_destroy_yielding( * ERTS_RBT_T **tree, - * void (*op)(ERTS_RBT_T *, void *), - * void (*destr)(ERTS_RBT_T *, void *), + * int (*op)(ERTS_RBT_T *, void *), + * int (*destr)(ERTS_RBT_T *, void *), * void *arg, * <ERTS_RBT_PREFIX>_rbt_yield_state_t *ystate, - * Sint ylimit); + * Sint reds); * Operate by calling the operator 'op' on each element from * largest towards smaller elements. * @@ -327,10 +343,13 @@ * Note that elements are often destroyed in another order * than the order that the elements are operated on. * - * Yield when 'ylimit' elements has been processed. True is - * returned when yielding, and false is returned when - * the whole tree has been processed. The tree should not be - * modified until all of it has been processed. + * Yield when 'reds' reductions has been processed. The 'op' and + * 'destr' functions return the number of reductions that each element + * took to process. The number of reductions remaining is returned, + * meaning that if 0 is returned, there are more elements to be + * processed. If a value greater than 0 is returned the foreach has + * ended. The tree should not be modified until all of it has been + * processed. * * 'arg' is passed as argument to 'op' and 'destroy'. * @@ -447,17 +466,6 @@ # define ERTS_RBT_API_INLINE__ ERTS_INLINE #endif -#ifndef ERTS_RBT_YIELD_STAT_INITER -# define ERTS_RBT_YIELD_STAT_INITER {NULL, 0} -#endif -#ifndef ERTS_RBT_YIELD_STAT_INIT -# define ERTS_RBT_YIELD_STAT_INIT(YS) \ - do { \ - (YS)->x = NULL; \ - (YS)->up = 0; \ - } while (0) -#endif - #define ERTS_RBT_CONCAT_MACRO_VALUES___(X, Y) \ X ## Y #define ERTS_RBT_CONCAT_MACRO_VALUES__(X, Y) \ @@ -470,8 +478,38 @@ typedef struct { ERTS_RBT_T *x; int up; +#ifdef DEBUG + int debug_red_adj; +#endif } ERTS_RBT_YIELD_STATE_T__; +#define ERTS_RBT_CALLBACK_FOREACH_FUNC(NAME) int (*NAME)(ERTS_RBT_T *, void *, Sint) + +#ifndef ERTS_RBT_YIELD_STAT_INITER +# ifdef DEBUG +# define ERTS_RBT_YIELD_STAT_INITER {NULL, 0, CONTEXT_REDS} +# else +# define ERTS_RBT_YIELD_STAT_INITER {NULL, 0} +# endif +#endif +#ifndef ERTS_RBT_YIELD_STAT_INIT +# define ERTS_RBT_YIELD_STAT_INIT__(YS) \ + do { \ + (YS)->x = NULL; \ + (YS)->up = 0; \ + } while (0) +# ifdef DEBUG +# define ERTS_RBT_YIELD_STAT_INIT(YS) \ + do { \ + ERTS_RBT_YIELD_STAT_INIT__(YS); \ + (YS)->debug_red_adj = CONTEXT_REDS; \ + } while(0) +# else +# define ERTS_RBT_YIELD_STAT_INIT(YS) ERTS_RBT_YIELD_STAT_INIT__(YS) +# endif +#endif + + #define ERTS_RBT_FUNC__(Name) \ ERTS_RBT_CONCAT_MACRO_VALUES__(ERTS_RBT_PREFIX, _rbt_ ## Name) @@ -1302,11 +1340,11 @@ ERTS_RBT_FUNC__(largest)(ERTS_RBT_T *root) static ERTS_INLINE int ERTS_RBT_FUNC__(foreach_unordered__)(ERTS_RBT_T **root, int destroying, - void (*op)(ERTS_RBT_T *, void *), + ERTS_RBT_CALLBACK_FOREACH_FUNC(op), void *arg, - int yielding, + int yielding, ERTS_RBT_YIELD_STATE_T__ *ystate, - Sint ylimit) + Sint reds) { ERTS_RBT_T *c, *p, *x; @@ -1314,13 +1352,17 @@ ERTS_RBT_FUNC__(foreach_unordered__)(ERTS_RBT_T **root, if (yielding && ystate->x) { x = ystate->x; +#ifdef DEBUG + if (ystate->debug_red_adj > 0) + ystate->debug_red_adj -= 100; +#endif ERTS_RBT_ASSERT(ystate->up); goto restart_up; } else { x = *root; if (!x) - return 0; + return reds; if (destroying) *root = NULL; } @@ -1346,10 +1388,10 @@ ERTS_RBT_FUNC__(foreach_unordered__)(ERTS_RBT_T **root, #ifdef ERTS_RBT_DEBUG int cdir; #endif - if (yielding && ylimit-- <= 0) { + if (yielding && reds <= 0) { ystate->x = x; ystate->up = 1; - return 1; + return 0; } restart_up: @@ -1375,14 +1417,20 @@ ERTS_RBT_FUNC__(foreach_unordered__)(ERTS_RBT_T **root, } #endif - (*op)(x, arg); + reds -= (*op)(x, arg, reds); +#ifdef DEBUG + if (yielding) + reds -= ystate->debug_red_adj; +#endif if (!p) { + /* Done */ if (yielding) { ystate->x = NULL; ystate->up = 0; + return reds <= 0 ? 1 : reds; } - return 0; /* Done */ + return 1; } c = ERTS_RBT_GET_RIGHT(p); @@ -1407,20 +1455,26 @@ static ERTS_INLINE int ERTS_RBT_FUNC__(foreach_ordered__)(ERTS_RBT_T **root, int from_small, int destroying, - void (*op)(ERTS_RBT_T *, void *), - void (*destroy)(ERTS_RBT_T *, void *), + ERTS_RBT_CALLBACK_FOREACH_FUNC(op), + ERTS_RBT_CALLBACK_FOREACH_FUNC(destroy), void *arg, - int yielding, + int yielding, ERTS_RBT_YIELD_STATE_T__ *ystate, - Sint ylimit) + Sint reds) { ERTS_RBT_T *c, *p, *x; ERTS_RBT_ASSERT(!yielding || ystate); ERTS_RBT_ASSERT(!destroying || destroy); + ERTS_RBT_ASSERT(!yielding || yop); + ERTS_RBT_ASSERT(yielding || op); if (yielding && ystate->x) { x = ystate->x; +#ifdef DEBUG + if (ystate->debug_red_adj > 0) + ystate->debug_red_adj -= 100; +#endif if (ystate->up) goto restart_up; else @@ -1429,7 +1483,7 @@ ERTS_RBT_FUNC__(foreach_ordered__)(ERTS_RBT_T **root, else { x = *root; if (!x) - return 0; + return reds; if (destroying) *root = NULL; } @@ -1445,12 +1499,16 @@ ERTS_RBT_FUNC__(foreach_ordered__)(ERTS_RBT_T **root, x = c; } - (*op)(x, arg); + reds -= (*op)(x, arg, reds); +#ifdef DEBUG + if (yielding) + reds -= ystate->debug_red_adj; +#endif - if (yielding && --ylimit <= 0) { + if (yielding && reds <= 0) { ystate->x = x; ystate->up = 0; - return 1; + return 0; } restart_down: @@ -1472,12 +1530,16 @@ ERTS_RBT_FUNC__(foreach_ordered__)(ERTS_RBT_T **root, ? ERTS_RBT_GET_LEFT(p) : ERTS_RBT_GET_RIGHT(p)) == x); - (*op)(p, arg); + reds -= (*op)(p, arg, reds); +#ifdef DEBUG + if (yielding) + reds -= ystate->debug_red_adj; +#endif - if (yielding && --ylimit <= 0) { + if (yielding && reds <= 0) { ystate->x = x; ystate->up = 1; - return 1; + return 0; restart_up: p = ERTS_RBT_GET_PARENT(x); } @@ -1510,15 +1572,20 @@ ERTS_RBT_FUNC__(foreach_ordered__)(ERTS_RBT_T **root, } #endif - (*destroy)(x, arg); + reds -= (*destroy)(x, arg, reds); +#ifdef DEBUG + if (yielding) + reds -= ystate->debug_red_adj; +#endif } if (!p) { if (yielding) { ystate->x = NULL; ystate->up = 0; + return reds <= 0 ? 1 : reds; } - return 0; /* Done */ + return 1; /* Done */ } x = p; } @@ -1531,7 +1598,7 @@ ERTS_RBT_FUNC__(foreach_ordered__)(ERTS_RBT_T **root, static ERTS_RBT_API_INLINE__ void ERTS_RBT_FUNC__(foreach)(ERTS_RBT_T *root, - void (*op)(ERTS_RBT_T *, void *), + ERTS_RBT_CALLBACK_FOREACH_FUNC(op), void *arg) { (void) ERTS_RBT_FUNC__(foreach_unordered__)(&root, 0, op, arg, @@ -1544,7 +1611,7 @@ ERTS_RBT_FUNC__(foreach)(ERTS_RBT_T *root, static ERTS_RBT_API_INLINE__ void ERTS_RBT_FUNC__(foreach_small)(ERTS_RBT_T *root, - void (*op)(ERTS_RBT_T *, void *), + ERTS_RBT_CALLBACK_FOREACH_FUNC(op), void *arg) { (void) ERTS_RBT_FUNC__(foreach_ordered__)(&root, 1, 0, @@ -1558,7 +1625,7 @@ ERTS_RBT_FUNC__(foreach_small)(ERTS_RBT_T *root, static ERTS_RBT_API_INLINE__ void ERTS_RBT_FUNC__(foreach_large)(ERTS_RBT_T *root, - void (*op)(ERTS_RBT_T *, void *), + ERTS_RBT_CALLBACK_FOREACH_FUNC(op), void *arg) { (void) ERTS_RBT_FUNC__(foreach_ordered__)(&root, 0, 0, @@ -1572,13 +1639,13 @@ ERTS_RBT_FUNC__(foreach_large)(ERTS_RBT_T *root, static ERTS_RBT_API_INLINE__ int ERTS_RBT_FUNC__(foreach_yielding)(ERTS_RBT_T *root, - void (*op)(ERTS_RBT_T *, void *), + ERTS_RBT_CALLBACK_FOREACH_FUNC(op), void *arg, ERTS_RBT_YIELD_STATE_T__ *ystate, - Sint ylimit) + Sint reds) { return ERTS_RBT_FUNC__(foreach_unordered__)(&root, 0, op, arg, - 1, ystate, ylimit); + 1, ystate, reds); } #endif /* ERTS_RBT_WANT_FOREACH_YIELDING */ @@ -1587,14 +1654,14 @@ ERTS_RBT_FUNC__(foreach_yielding)(ERTS_RBT_T *root, static ERTS_RBT_API_INLINE__ int ERTS_RBT_FUNC__(foreach_small_yielding)(ERTS_RBT_T *root, - void (*op)(ERTS_RBT_T *, void *), + ERTS_RBT_CALLBACK_FOREACH_FUNC(op), void *arg, ERTS_RBT_YIELD_STATE_T__ *ystate, - Sint ylimit) + Sint reds) { return ERTS_RBT_FUNC__(foreach_ordered__)(&root, 1, 0, op, NULL, arg, - 1, ystate, ylimit); + 1, ystate, reds); } #endif /* ERTS_RBT_WANT_FOREACH_SMALL_YIELDING */ @@ -1603,14 +1670,14 @@ ERTS_RBT_FUNC__(foreach_small_yielding)(ERTS_RBT_T *root, static ERTS_RBT_API_INLINE__ int ERTS_RBT_FUNC__(foreach_large_yielding)(ERTS_RBT_T *root, - void (*op)(ERTS_RBT_T *, void *), + ERTS_RBT_CALLBACK_FOREACH_FUNC(op), void *arg, ERTS_RBT_YIELD_STATE_T__ *ystate, - Sint ylimit) + Sint reds) { return ERTS_RBT_FUNC__(foreach_ordered__)(&root, 0, 0, op, NULL, arg, - 1, ystate, ylimit); + 1, ystate, reds); } #endif /* ERTS_RBT_WANT_FOREACH_LARGE_YIELDING */ @@ -1619,11 +1686,11 @@ ERTS_RBT_FUNC__(foreach_large_yielding)(ERTS_RBT_T *root, static ERTS_RBT_API_INLINE__ void ERTS_RBT_FUNC__(foreach_destroy)(ERTS_RBT_T **root, - void (*op)(ERTS_RBT_T *, void *), + ERTS_RBT_CALLBACK_FOREACH_FUNC(op), void *arg) { (void) ERTS_RBT_FUNC__(foreach_unordered__)(root, 1, op, arg, - 0, NULL, 0); + 0, NULL, 0); } #endif /* ERTS_RBT_WANT_FOREACH_DESTROY */ @@ -1632,8 +1699,8 @@ ERTS_RBT_FUNC__(foreach_destroy)(ERTS_RBT_T **root, static ERTS_RBT_API_INLINE__ void ERTS_RBT_FUNC__(foreach_small_destroy)(ERTS_RBT_T **root, - void (*op)(ERTS_RBT_T *, void *), - void (*destr)(ERTS_RBT_T *, void *), + ERTS_RBT_CALLBACK_FOREACH_FUNC(op), + ERTS_RBT_CALLBACK_FOREACH_FUNC(destr), void *arg) { (void) ERTS_RBT_FUNC__(foreach_ordered__)(root, 1, 1, @@ -1647,8 +1714,8 @@ ERTS_RBT_FUNC__(foreach_small_destroy)(ERTS_RBT_T **root, static ERTS_RBT_API_INLINE__ void ERTS_RBT_FUNC__(foreach_large_destroy)(ERTS_RBT_T **root, - void (*op)(ERTS_RBT_T *, void *), - void (*destr)(ERTS_RBT_T *, void *), + ERTS_RBT_CALLBACK_FOREACH_FUNC(op), + ERTS_RBT_CALLBACK_FOREACH_FUNC(destr), void *arg) { (void) ERTS_RBT_FUNC__(foreach_ordered__)(root, 0, 1, @@ -1662,13 +1729,13 @@ ERTS_RBT_FUNC__(foreach_large_destroy)(ERTS_RBT_T **root, static ERTS_RBT_API_INLINE__ int ERTS_RBT_FUNC__(foreach_destroy_yielding)(ERTS_RBT_T **root, - void (*op)(ERTS_RBT_T *, void *), + ERTS_RBT_CALLBACK_FOREACH_FUNC(op), void *arg, ERTS_RBT_YIELD_STATE_T__ *ystate, - Sint ylimit) + Sint reds) { return ERTS_RBT_FUNC__(foreach_unordered__)(root, 1, op, arg, - 1, ystate, ylimit); + 1, ystate, reds); } #endif /* ERTS_RBT_WANT_FOREACH_DESTROY_YIELDING */ @@ -1677,15 +1744,15 @@ ERTS_RBT_FUNC__(foreach_destroy_yielding)(ERTS_RBT_T **root, static ERTS_RBT_API_INLINE__ int ERTS_RBT_FUNC__(foreach_small_destroy_yielding)(ERTS_RBT_T **root, - void (*op)(ERTS_RBT_T *, void *), - void (*destr)(ERTS_RBT_T *, void *), + ERTS_RBT_CALLBACK_FOREACH_FUNC(op), + ERTS_RBT_CALLBACK_FOREACH_FUNC(destr), void *arg, ERTS_RBT_YIELD_STATE_T__ *ystate, - Sint ylimit) + Sint reds) { return ERTS_RBT_FUNC__(foreach_ordered__)(root, 1, 1, op, destr, arg, - 1, ystate, ylimit); + 1, ystate, reds); } #endif /* ERTS_RBT_WANT_FOREACH_SMALL_DESTROY_YIELDING */ @@ -1694,15 +1761,15 @@ ERTS_RBT_FUNC__(foreach_small_destroy_yielding)(ERTS_RBT_T **root, static ERTS_RBT_API_INLINE__ int ERTS_RBT_FUNC__(foreach_large_destroy_yielding)(ERTS_RBT_T **root, - void (*op)(ERTS_RBT_T *, void *), - void (*destr)(ERTS_RBT_T *, void *), + ERTS_RBT_CALLBACK_FOREACH_FUNC(op), + ERTS_RBT_CALLBACK_FOREACH_FUNC(destr), void *arg, ERTS_RBT_YIELD_STATE_T__ *ystate, - Sint ylimit) + Sint reds) { return ERTS_RBT_FUNC__(foreach_ordered__)(root, 0, 1, op, destr, arg, - 1, ystate, ylimit); + 1, ystate, reds); } #endif /* ERTS_RBT_WANT_FOREACH_LARGE_DESTROY_YIELDING */ @@ -1855,6 +1922,7 @@ ERTS_RBT_FUNC__(hdbg_check_tree)(ERTS_RBT_T *root, ERTS_RBT_T *n) #ifdef ERTS_RBT_UNDEF # undef ERTS_RBT_PREFIX # undef ERTS_RBT_T +# undef ERTS_RBT_CALLBACK_FOREACH_FUNC # undef ERTS_RBT_KEY_T # undef ERTS_RBT_FLAGS_T # undef ERTS_RBT_INIT_EMPTY_TNODE diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c index 29c698e34f..d26ea19494 100644 --- a/erts/emulator/beam/erl_time_sup.c +++ b/erts/emulator/beam/erl_time_sup.c @@ -1911,8 +1911,8 @@ typedef struct { ErtsTimeOffsetMonitorInfo *to_mon_info; } ErtsTimeOffsetMonitorContext; -static void -save_time_offset_monitor(ErtsMonitor *mon, void *vcntxt) +static int +save_time_offset_monitor(ErtsMonitor *mon, void *vcntxt, Sint reds) { ErtsTimeOffsetMonitorContext *cntxt; ErtsMonitorData *mdp = erts_monitor_to_data(mon); @@ -1935,7 +1935,7 @@ save_time_offset_monitor(ErtsMonitor *mon, void *vcntxt) cntxt->to_mon_info[mix].ref = make_internal_ref(&cntxt->to_mon_info[mix].heap[0]); - + return 1; } static void diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 1ded5f031c..73eae614fa 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -262,19 +262,12 @@ erts_finalize_atom_cache_map(ErtsAtomCacheMap *acmp, Uint32 dflags) if (acmp) { int long_atoms = 0; /* !0 if one or more atoms are longer than 255. */ int i; - int sz; - int fix_sz - = 1 /* VERSION_MAGIC */ - + 1 /* DIST_HEADER */ - + 1 /* dist header flags */ - + 1 /* number of internal cache entries */ - ; + int sz = 0; int min_sz; ASSERT(dflags & DFLAG_UTF8_ATOMS); ASSERT(acmp->hdr_sz < 0); /* Make sure cache update instructions fit */ - min_sz = fix_sz+(2+4)*acmp->sz; - sz = fix_sz; + min_sz = (2+4)*acmp->sz; for (i = 0; i < acmp->sz; i++) { Atom *a; Eterm atom; @@ -302,17 +295,28 @@ erts_finalize_atom_cache_map(ErtsAtomCacheMap *acmp, Uint32 dflags) } Uint -erts_encode_ext_dist_header_size(ErtsAtomCacheMap *acmp) +erts_encode_ext_dist_header_size(ErtsAtomCacheMap *acmp, Uint fragments) { if (!acmp) return 0; else { + int fix_sz + = 1 /* VERSION_MAGIC */ + + 1 /* DIST_HEADER */ + + 1 /* dist header flags */ + + 1 /* number of internal cache entries */ + ; ASSERT(acmp->hdr_sz >= 0); - return acmp->hdr_sz; + if (fragments > 1) + fix_sz += 8 /* sequence id */ + + 8 /* number of fragments */ + ; + return fix_sz + acmp->hdr_sz; } } -byte *erts_encode_ext_dist_header_setup(byte *ctl_ext, ErtsAtomCacheMap *acmp) +byte *erts_encode_ext_dist_header_setup(byte *ctl_ext, ErtsAtomCacheMap *acmp, + Uint fragments, Eterm from) { /* Maximum number of atom must be less than the maximum of a 32 bits unsigned integer. Check is done in erl_init.c, erl_start function. */ @@ -346,12 +350,37 @@ byte *erts_encode_ext_dist_header_setup(byte *ctl_ext, ErtsAtomCacheMap *acmp) put_int8(acmp->sz, ep); --ep; put_int8(dist_hdr_flags, ep); - *--ep = DIST_HEADER; - *--ep = VERSION_MAGIC; + if (fragments > 1) { + ASSERT(is_pid(from)); + ep -= 8; + put_int64(fragments, ep); + ep -= 8; + put_int64(from, ep); + *--ep = DIST_FRAG_HEADER; + } else { + *--ep = DIST_HEADER; + } + *--ep = VERSION_MAGIC; return ep; } } +byte *erts_encode_ext_dist_header_fragment(byte **hdrpp, + Uint fragment, + Eterm from) +{ + byte *ep = *hdrpp, *start = ep; + ASSERT(is_pid(from)); + *ep++ = VERSION_MAGIC; + *ep++ = DIST_FRAG_CONT; + put_int64(from, ep); + ep += 8; + put_int64(fragment, ep); + ep += 8; + *hdrpp = ep; + return start; +} + #define PASS_THROUGH 'p' @@ -365,7 +394,8 @@ Sint erts_encode_ext_dist_header_finalize(ErtsDistOutputBuf* ob, int ci, sz; byte dist_hdr_flags; int long_atoms; - register byte *ep = ob->extp; + Uint64 seq_id = 0, frag_id = 0; + register byte *ep = ob->hdrp ? ob->hdrp : ob->extp; ASSERT(dflags & DFLAG_UTF8_ATOMS); /* @@ -416,7 +446,7 @@ Sint erts_encode_ext_dist_header_finalize(ErtsDistOutputBuf* ob, } goto done; } - else if (ep[1] != DIST_HEADER) { + else if (ep[1] != DIST_HEADER && ep[1] != DIST_FRAG_HEADER && ep[1] != DIST_FRAG_CONT) { ASSERT(ep[1] == SMALL_TUPLE_EXT || ep[1] == LARGE_TUPLE_EXT); ASSERT(!(dflags & DFLAG_DIST_HDR_ATOM_CACHE)); /* Node without atom cache, 'pass through' needed */ @@ -424,6 +454,17 @@ Sint erts_encode_ext_dist_header_finalize(ErtsDistOutputBuf* ob, goto done; } + if (ep[1] == DIST_FRAG_CONT) { + ep = ob->extp; + goto done; + } else if (ep[1] == DIST_FRAG_HEADER) { + /* skip the seq id and frag id */ + seq_id = get_int64(&ep[2]); + ep += 8; + frag_id = get_int64(&ep[2]); + ep += 8; + } + dist_hdr_flags = ep[2]; long_atoms = ERTS_DIST_HDR_LONG_ATOMS_FLG & ((int) dist_hdr_flags); @@ -546,11 +587,19 @@ Sint erts_encode_ext_dist_header_finalize(ErtsDistOutputBuf* ob, } --ep; put_int8(ci, ep); - *--ep = DIST_HEADER; + if (seq_id) { + ep -= 8; + put_int64(frag_id, ep); + ep -= 8; + put_int64(seq_id, ep); + *--ep = DIST_FRAG_HEADER; + } else { + *--ep = DIST_HEADER; + } *--ep = VERSION_MAGIC; done: ob->extp = ep; - ASSERT(&ob->data[0] <= ob->extp && ob->extp < ob->ext_endp); + ASSERT((byte*)ob->bin->orig_bytes <= ob->extp && ob->extp < ob->ext_endp); return reds < 0 ? 0 : reds; } @@ -571,7 +620,7 @@ int erts_encode_dist_ext_size(Eterm term, Uint32 flags, ErtsAtomCacheMap *acmp, } } -int erts_encode_dist_ext_size_int(Eterm term, struct erts_dsig_send_context* ctx, Uint* szp) +int erts_encode_dist_ext_size_int(Eterm term, ErtsDSigSendContext *ctx, Uint* szp) { Uint sz; if (encode_size_struct_int(&ctx->u.sc, ctx->acmp, term, ctx->flags, &ctx->reds, &sz)) { @@ -635,56 +684,106 @@ byte* erts_encode_ext_ets(Eterm term, byte *ep, struct erl_off_heap_header** off off_heap); } -ErtsDistExternal * -erts_make_dist_ext_copy(ErtsDistExternal *edep, Uint xsize) + +static Uint +dist_ext_size(ErtsDistExternal *edep) { - size_t align_sz; - size_t dist_ext_sz; - size_t ext_sz; - byte *ep; - ErtsDistExternal *new_edep; + Uint sz = sizeof(ErtsDistExternal); + + ASSERT(edep->data->ext_endp && edep->data->extp); + ASSERT(edep->data->ext_endp >= edep->data->extp); + + if (edep->flags & ERTS_DIST_EXT_ATOM_TRANS_TAB) { + ASSERT(0 <= edep->attab.size \ + && edep->attab.size <= ERTS_ATOM_CACHE_SIZE); + sz -= sizeof(Eterm)*(ERTS_ATOM_CACHE_SIZE - edep->attab.size); + } else { + sz -= sizeof(ErtsAtomTranslationTable); + } + return sz; +} - dist_ext_sz = ERTS_DIST_EXT_SIZE(edep); - ASSERT(edep->ext_endp && edep->extp); - ASSERT(edep->ext_endp >= edep->extp); - ext_sz = edep->ext_endp - edep->extp; +Uint +erts_dist_ext_size(ErtsDistExternal *edep) +{ + Uint sz = dist_ext_size(edep); + sz += edep->data[0].frag_id * sizeof(ErtsDistExternalData); + return sz + ERTS_EXTRA_DATA_ALIGN_SZ(sz); +} - align_sz = ERTS_EXTRA_DATA_ALIGN_SZ(dist_ext_sz + ext_sz); +Uint +erts_dist_ext_data_size(ErtsDistExternal *edep) +{ + Uint sz = 0, i; + for (i = 0; i < edep->data->frag_id; i++) + sz += edep->data[i].ext_endp - edep->data[i].extp; + return sz; +} - new_edep = erts_alloc(ERTS_ALC_T_EXT_TERM_DATA, - dist_ext_sz + ext_sz + align_sz + xsize); +void +erts_dist_ext_frag(ErtsDistExternalData *ede_datap, ErtsDistExternal *edep) +{ + ErtsDistExternalData *new_ede_datap = &edep->data[edep->data->frag_id - ede_datap->frag_id]; + sys_memcpy(new_ede_datap, ede_datap, sizeof(ErtsDistExternalData)); + + /* If the data is not backed by a binary, we create one here to keep + things simple. Only custom distribution drivers should use lists. */ + if (new_ede_datap->binp == NULL) { + size_t ext_sz = ede_datap->ext_endp - ede_datap->extp; + new_ede_datap->binp = erts_bin_nrml_alloc(ext_sz); + sys_memcpy(new_ede_datap->binp->orig_bytes, (void *) ede_datap->extp, ext_sz); + new_ede_datap->extp = (byte*)new_ede_datap->binp->orig_bytes; + new_ede_datap->ext_endp = (byte*)new_ede_datap->binp->orig_bytes + ext_sz; + } else { + erts_refc_inc(&new_ede_datap->binp->intern.refc, 2); + } +} + +void +erts_make_dist_ext_copy(ErtsDistExternal *edep, ErtsDistExternal *new_edep) +{ + size_t dist_ext_sz = dist_ext_size(edep); + byte *ep; ep = (byte *) new_edep; sys_memcpy((void *) ep, (void *) edep, dist_ext_sz); + erts_ref_dist_entry(new_edep->dep); + ep += dist_ext_sz; - if (new_edep->dep) - erts_ref_dist_entry(new_edep->dep); - new_edep->extp = ep; - new_edep->ext_endp = ep + ext_sz; - new_edep->heap_size = -1; - sys_memcpy((void *) ep, (void *) edep->extp, ext_sz); - return new_edep; + + new_edep->data = (ErtsDistExternalData*)ep; + sys_memzero(new_edep->data, sizeof(ErtsDistExternalData) * edep->data->frag_id); + new_edep->data->frag_id = edep->data->frag_id; + erts_dist_ext_frag(edep->data, new_edep); } -int +void +erts_free_dist_ext_copy(ErtsDistExternal *edep) +{ + int i; + erts_deref_dist_entry(edep->dep); + for (i = 0; i < edep->data->frag_id; i++) + if (edep->data[i].binp) + erts_bin_release(edep->data[i].binp); +} + +ErtsPrepDistExtRes erts_prepare_dist_ext(ErtsDistExternal *edep, byte *ext, Uint size, + Binary *binp, DistEntry *dep, Uint32 conn_id, ErtsAtomCache *cache) { register byte *ep; - edep->heap_size = -1; - edep->flags = 0; - edep->dep = dep; - ASSERT(dep); erts_de_rlock(dep); ASSERT(dep->flags & DFLAG_UTF8_ATOMS); + if ((dep->state != ERTS_DE_STATE_CONNECTED && dep->state != ERTS_DE_STATE_PENDING) || dep->connection_id != conn_id) { @@ -697,7 +796,7 @@ erts_prepare_dist_ext(ErtsDistExternal *edep, ext++; size--; } - edep->ext_endp = ext + size; + ep = ext; if (size < 2) @@ -713,16 +812,33 @@ erts_prepare_dist_ext(ErtsDistExternal *edep, goto fail; } + edep->heap_size = -1; + edep->flags = 0; + edep->dep = dep; + edep->connection_id = conn_id; + edep->data->ext_endp = ext+size; + edep->data->binp = binp; + edep->data->seq_id = 0; + edep->data->frag_id = 1; + if (dep->flags & DFLAG_DIST_HDR_ATOM_CACHE) edep->flags |= ERTS_DIST_EXT_DFLAG_HDR; - edep->connection_id = dep->connection_id; - - if (ep[1] != DIST_HEADER) { + if (ep[1] != DIST_HEADER && ep[1] != DIST_FRAG_HEADER && ep[1] != DIST_FRAG_CONT) { if (edep->flags & ERTS_DIST_EXT_DFLAG_HDR) goto bad_hdr; edep->attab.size = 0; - edep->extp = ext; + edep->data->extp = ext; + } + else if (ep[1] == DIST_FRAG_CONT) { + if (!(dep->flags & DFLAG_FRAGMENTS)) + goto bad_hdr; + edep->attab.size = 0; + edep->data->extp = ext + 1 + 1 + 8 + 8; + edep->data->seq_id = get_int64(&ep[2]); + edep->data->frag_id = get_int64(&ep[2+8]); + erts_de_runlock(dep); + return ERTS_PREP_DIST_EXT_FRAG_CONT; } else { int tix; @@ -731,9 +847,17 @@ erts_prepare_dist_ext(ErtsDistExternal *edep, if (!(edep->flags & ERTS_DIST_EXT_DFLAG_HDR)) goto bad_hdr; + if (ep[1] == DIST_FRAG_HEADER) { + if (!(dep->flags & DFLAG_FRAGMENTS)) + goto bad_hdr; + edep->data->seq_id = get_int64(&ep[2]); + edep->data->frag_id = get_int64(&ep[2+8]); + ep += 16; + } + #undef CHKSIZE #define CHKSIZE(SZ) \ - do { if ((SZ) > edep->ext_endp - ep) goto bad_hdr; } while(0) + do { if ((SZ) > edep->data->ext_endp - ep) goto bad_hdr; } while(0) CHKSIZE(1+1+1); ep += 2; @@ -863,7 +987,7 @@ erts_prepare_dist_ext(ErtsDistExternal *edep, #endif } } - edep->extp = ep; + edep->data->extp = ep; #ifdef ERTS_DEBUG_USE_DIST_SEP if (*ep != VERSION_MAGIC) goto bad_hdr; @@ -888,7 +1012,7 @@ erts_prepare_dist_ext(ErtsDistExternal *edep, erts_this_node->sysname, edep->dep->sysname, dist_entry_channel_no(edep->dep)); - for (ep = ext; ep < edep->ext_endp; ep++) + for (ep = ext; ep < edep->data->ext_endp; ep++) erts_dsprintf(dsbufp, ep != ext ? ",%b8u" : "<<%b8u", *ep); erts_dsprintf(dsbufp, ">>"); erts_send_warning_to_logger_nogl(dsbufp); @@ -913,9 +1037,9 @@ bad_dist_ext(ErtsDistExternal *edep) erts_this_node->sysname, dep->sysname, dist_entry_channel_no(dep)); - for (ep = edep->extp; ep < edep->ext_endp; ep++) + for (ep = edep->data->extp; ep < edep->data->ext_endp; ep++) erts_dsprintf(dsbufp, - ep != edep->extp ? ",%b8u" : "<<...,%b8u", + ep != edep->data->extp ? ",%b8u" : "<<...,%b8u", *ep); erts_dsprintf(dsbufp, ">>\n"); erts_dsprintf(dsbufp, "ATOM_CACHE_REF translations: "); @@ -933,30 +1057,32 @@ bad_dist_ext(ErtsDistExternal *edep) } Sint -erts_decode_dist_ext_size(ErtsDistExternal *edep) +erts_decode_dist_ext_size(ErtsDistExternal *edep, int kill_connection) { Sint res; byte *ep; - if (edep->extp >= edep->ext_endp) + + if (edep->data->extp >= edep->data->ext_endp) goto fail; #ifndef ERTS_DEBUG_USE_DIST_SEP if (edep->flags & ERTS_DIST_EXT_DFLAG_HDR) { - if (*edep->extp == VERSION_MAGIC) + if (*edep->data->extp == VERSION_MAGIC) goto fail; - ep = edep->extp; + ep = edep->data->extp; } else #endif { - if (*edep->extp != VERSION_MAGIC) + if (*edep->data->extp != VERSION_MAGIC) goto fail; - ep = edep->extp+1; + ep = edep->data->extp+1; } - res = decoded_size(ep, edep->ext_endp, 0, NULL); + res = decoded_size(ep, edep->data->ext_endp, 0, NULL); if (res >= 0) return res; fail: - bad_dist_ext(edep); + if (kill_connection) + bad_dist_ext(edep); return -1; } @@ -982,12 +1108,15 @@ Sint erts_decode_ext_size_ets(byte *ext, Uint size) */ Eterm erts_decode_dist_ext(ErtsHeapFactory* factory, - ErtsDistExternal *edep) + ErtsDistExternal *edep, + int kill_connection) { Eterm obj; - byte* ep = edep->extp; + byte* ep; + + ep = edep->data->extp; - if (ep >= edep->ext_endp) + if (ep >= edep->data->ext_endp) goto error; #ifndef ERTS_DEBUG_USE_DIST_SEP if (edep->flags & ERTS_DIST_EXT_DFLAG_HDR) { @@ -1005,14 +1134,15 @@ erts_decode_dist_ext(ErtsHeapFactory* factory, if (!ep) goto error; - edep->extp = ep; + edep->data->extp = ep; return obj; error: erts_factory_undo(factory); - bad_dist_ext(edep); + if (kill_connection) + bad_dist_ext(edep); return THE_NON_VALUE; } @@ -1057,6 +1187,7 @@ BIF_RETTYPE erts_debug_dist_ext_to_term_2(BIF_ALIST_2) Eterm res; Sint hsz; ErtsDistExternal ede; + ErtsDistExternalData ede_data; Eterm *tp; Eterm real_bin; Uint offset; @@ -1069,7 +1200,8 @@ BIF_RETTYPE erts_debug_dist_ext_to_term_2(BIF_ALIST_2) ede.flags = ERTS_DIST_EXT_ATOM_TRANS_TAB; ede.dep = NULL; ede.heap_size = -1; - + ede.data = &ede_data; + if (is_not_tuple(BIF_ARG_1)) goto badarg; tp = tuple_val(BIF_ARG_1); @@ -1094,15 +1226,15 @@ BIF_RETTYPE erts_debug_dist_ext_to_term_2(BIF_ALIST_2) if (bitsize != 0) goto badarg; - ede.extp = binary_bytes(real_bin)+offset; - ede.ext_endp = ede.extp + size; + ede.data->extp = binary_bytes(real_bin)+offset; + ede.data->ext_endp = ede.data->extp + size; - hsz = erts_decode_dist_ext_size(&ede); + hsz = erts_decode_dist_ext_size(&ede, 1); if (hsz < 0) goto badarg; erts_factory_proc_prealloc_init(&factory, BIF_P, hsz); - res = erts_decode_dist_ext(&factory, &ede); + res = erts_decode_dist_ext(&factory, &ede, 1); erts_factory_close(&factory); if (is_value(res)) @@ -2348,7 +2480,7 @@ dec_atom(ErtsDistExternal *edep, byte* ep, Eterm* objp) return ep; } -static ERTS_INLINE ErlNode* dec_get_node(Eterm sysname, Uint32 creation) +static ERTS_INLINE ErlNode* dec_get_node(Eterm sysname, Uint32 creation, Eterm book) { if (sysname == INTERNAL_LOCAL_SYSNAME) /* && DFLAG_INTERNAL_TAGS */ return erts_this_node; @@ -2357,7 +2489,7 @@ static ERTS_INLINE ErlNode* dec_get_node(Eterm sysname, Uint32 creation) && (creation == erts_this_node->creation || creation == ORIG_CREATION)) return erts_this_node; - return erts_find_or_insert_node(sysname,creation); + return erts_find_or_insert_node(sysname,creation,book); } static byte* @@ -2403,7 +2535,7 @@ dec_pid(ErtsDistExternal *edep, ErtsHeapFactory* factory, byte* ep, * We are careful to create the node entry only after all * validity tests are done. */ - node = dec_get_node(sysname, cre); + node = dec_get_node(sysname, cre, make_boxed(factory->hp)); if(node == erts_this_node) { *objp = make_internal_pid(data); @@ -3397,7 +3529,7 @@ dec_term_atom_common: cre = get_int32(ep); ep += 4; } - node = dec_get_node(sysname, cre); + node = dec_get_node(sysname, cre, make_boxed(hp)); if(node == erts_this_node) { *objp = make_internal_port(num); } @@ -3477,7 +3609,7 @@ dec_term_atom_common: if (ref_words > ERTS_MAX_REF_NUMBERS) goto error; - node = dec_get_node(sysname, cre); + node = dec_get_node(sysname, cre, make_boxed(hp)); if(node == erts_this_node) { rtp = (ErtsORefThing *) hp; diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h index edac177cc6..396cd9f802 100644 --- a/erts/emulator/beam/external.h +++ b/erts/emulator/beam/external.h @@ -58,6 +58,8 @@ #define SMALL_ATOM_UTF8_EXT 'w' #define DIST_HEADER 'D' +#define DIST_FRAG_HEADER 'E' +#define DIST_FRAG_CONT 'F' #define ATOM_CACHE_REF 'R' #define ATOM_INTERNAL_REF2 'I' #define ATOM_INTERNAL_REF3 'K' @@ -122,13 +124,23 @@ typedef struct { #define ERTS_DIST_CON_ID_MASK ((Uint32) 0x00ffffff) /* also in net_kernel.erl */ -typedef struct { - DistEntry *dep; +struct binary; +typedef struct erl_dist_external_data ErtsDistExternalData; + +struct erl_dist_external_data { + Uint64 seq_id; + Uint64 frag_id; byte *extp; byte *ext_endp; + struct binary *binp; +}; + +typedef struct erl_dist_external { Sint heap_size; - Uint32 connection_id; + DistEntry *dep; Uint32 flags; + Uint32 connection_id; + ErtsDistExternalData *data; ErtsAtomTranslationTable attab; } ErtsDistExternal; @@ -155,8 +167,9 @@ void erts_reset_atom_cache_map(ErtsAtomCacheMap *); void erts_destroy_atom_cache_map(ErtsAtomCacheMap *); void erts_finalize_atom_cache_map(ErtsAtomCacheMap *, Uint32); -Uint erts_encode_ext_dist_header_size(ErtsAtomCacheMap *); -byte *erts_encode_ext_dist_header_setup(byte *, ErtsAtomCacheMap *); +Uint erts_encode_ext_dist_header_size(ErtsAtomCacheMap *, Uint); +byte *erts_encode_ext_dist_header_setup(byte *, ErtsAtomCacheMap *, Uint, Eterm); +byte *erts_encode_ext_dist_header_fragment(byte **, Uint, Eterm); Sint erts_encode_ext_dist_header_finalize(ErtsDistOutputBuf*, DistEntry *, Uint32 dflags, Sint reds); struct erts_dsig_send_context; int erts_encode_dist_ext_size(Eterm, Uint32, ErtsAtomCacheMap*, Uint* szp); @@ -171,20 +184,24 @@ Uint erts_encode_ext_size_ets(Eterm); void erts_encode_ext(Eterm, byte **); byte* erts_encode_ext_ets(Eterm, byte *, struct erl_off_heap_header** ext_off_heap); -ERTS_GLB_INLINE void erts_free_dist_ext_copy(ErtsDistExternal *); -ERTS_GLB_INLINE void *erts_dist_ext_trailer(ErtsDistExternal *); -ErtsDistExternal *erts_make_dist_ext_copy(ErtsDistExternal *, Uint); -void *erts_dist_ext_trailer(ErtsDistExternal *); -void erts_destroy_dist_ext_copy(ErtsDistExternal *); - -#define ERTS_PREP_DIST_EXT_FAILED (-1) -#define ERTS_PREP_DIST_EXT_SUCCESS (0) -#define ERTS_PREP_DIST_EXT_CLOSED (1) - -int erts_prepare_dist_ext(ErtsDistExternal *, byte *, Uint, - DistEntry *, Uint32 conn_id, ErtsAtomCache *); -Sint erts_decode_dist_ext_size(ErtsDistExternal *); -Eterm erts_decode_dist_ext(ErtsHeapFactory* factory, ErtsDistExternal *); +Uint erts_dist_ext_size(ErtsDistExternal *); +Uint erts_dist_ext_data_size(ErtsDistExternal *); +void erts_free_dist_ext_copy(ErtsDistExternal *); +void erts_make_dist_ext_copy(ErtsDistExternal *, ErtsDistExternal *); +void erts_dist_ext_frag(ErtsDistExternalData *, ErtsDistExternal *); +#define erts_get_dist_ext(HFRAG) ((ErtsDistExternal*)((HFRAG)->mem + (HFRAG)->used_size)) + +typedef enum { + ERTS_PREP_DIST_EXT_FAILED, + ERTS_PREP_DIST_EXT_SUCCESS, + ERTS_PREP_DIST_EXT_FRAG_CONT, + ERTS_PREP_DIST_EXT_CLOSED +} ErtsPrepDistExtRes; + +ErtsPrepDistExtRes erts_prepare_dist_ext(ErtsDistExternal *, byte *, Uint, struct binary *, + DistEntry *, Uint32, ErtsAtomCache *); +Sint erts_decode_dist_ext_size(ErtsDistExternal *, int); +Eterm erts_decode_dist_ext(ErtsHeapFactory*, ErtsDistExternal *, int); Sint erts_decode_ext_size(byte*, Uint); Sint erts_decode_ext_size_ets(byte*, Uint); @@ -200,25 +217,4 @@ int erts_debug_max_atom_out_cache_index(void); int erts_debug_atom_to_out_cache_index(Eterm); void transcode_free_ctx(DistEntry* dep); -#if ERTS_GLB_INLINE_INCL_FUNC_DEF - -ERTS_GLB_INLINE void -erts_free_dist_ext_copy(ErtsDistExternal *edep) -{ - if (edep->dep) - erts_deref_dist_entry(edep->dep); - erts_free(ERTS_ALC_T_EXT_TERM_DATA, edep); -} - -ERTS_GLB_INLINE void * -erts_dist_ext_trailer(ErtsDistExternal *edep) -{ - void *res = (void *) (edep->ext_endp - + ERTS_EXTRA_DATA_ALIGN_SZ(edep->ext_endp)); - ASSERT((((UWord) res) % sizeof(Uint)) == 0); - return res; -} - -#endif - #endif /* ERL_EXTERNAL_H__ */ diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 5d0dec7f08..f9bbe4167f 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1096,7 +1096,7 @@ extern int distribution_info(fmtfn_t, void *); extern int is_node_name_atom(Eterm a); extern int erts_net_message(Port *, DistEntry *, Uint32 conn_id, - byte *, ErlDrvSizeT, byte *, ErlDrvSizeT); + byte *, ErlDrvSizeT, Binary *, byte *, ErlDrvSizeT); extern void init_dist(void); extern int stop_dist(void); diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 7322239a73..b961c639f5 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -3644,20 +3644,22 @@ typedef struct { Eterm reason; } ErtsPortExitContext; -static void link_port_exit(ErtsLink *lnk, void *vpectxt) +static int link_port_exit(ErtsLink *lnk, void *vpectxt, Sint reds) { ErtsPortExitContext *pectxt = vpectxt; erts_proc_sig_send_link_exit(NULL, pectxt->port_id, lnk, pectxt->reason, NIL); + return 1; } -static void monitor_port_exit(ErtsMonitor *mon, void *vpectxt) +static int monitor_port_exit(ErtsMonitor *mon, void *vpectxt, Sint reds) { ErtsPortExitContext *pectxt = vpectxt; if (erts_monitor_is_target(mon)) erts_proc_sig_send_monitor_down(mon, pectxt->reason); else erts_proc_sig_send_demonitor(mon); + return 1; } /* 'from' is sending 'this_port' an exit signal, (this_port must be internal). @@ -4836,7 +4838,7 @@ typedef struct { void *arg; } prt_one_lnk_data; -static void prt_one_monitor(ErtsMonitor *mon, void *vprtd) +static int prt_one_monitor(ErtsMonitor *mon, void *vprtd, Sint reds) { ErtsMonitorData *mdp = erts_monitor_to_data(mon); prt_one_lnk_data *prtd = (prt_one_lnk_data *) vprtd; @@ -4844,12 +4846,14 @@ static void prt_one_monitor(ErtsMonitor *mon, void *vprtd) erts_print(prtd->to, prtd->arg, "(%p,%T)", mon->other.ptr, mdp->ref); else erts_print(prtd->to, prtd->arg, "(%T,%T)", mon->other.item, mdp->ref); + return 1; } -static void prt_one_lnk(ErtsLink *lnk, void *vprtd) +static int prt_one_lnk(ErtsLink *lnk, void *vprtd, Sint reds) { prt_one_lnk_data *prtd = (prt_one_lnk_data *) vprtd; erts_print(prtd->to, prtd->arg, "%T", lnk->other.item); + return 1; } static void dump_port_state(fmtfn_t to, void *arg, erts_aint32_t state) @@ -5100,7 +5104,7 @@ erts_port_resume_procs(Port *prt) erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), "%T", prt->common.id); while (plp2 != NULL) { - erts_snprintf(pid_str, sizeof(DTRACE_CHARBUF_NAME(pid_str)), "%T", plp2->pid); + erts_snprintf(pid_str, sizeof(DTRACE_CHARBUF_NAME(pid_str)), "%T", plp2->u.pid); DTRACE2(process_port_unblocked, pid_str, port_str); } } @@ -6177,6 +6181,7 @@ int driver_output_binary(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, dep, conn_id, (byte*) hbuf, hlen, + ErlDrvBinary2Binary(bin), (byte*) (bin->orig_bytes+offs), len); } else @@ -6222,12 +6227,14 @@ int driver_output2(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, dep, conn_id, NULL, 0, + NULL, (byte*) hbuf, hlen); else return erts_net_message(prt, dep, conn_id, (byte*) hbuf, hlen, + NULL, (byte*) buf, len); } else if (state & ERTS_PORT_SFLG_LINEBUF_IO) diff --git a/erts/emulator/beam/msg_instrs.tab b/erts/emulator/beam/msg_instrs.tab index 9bf3aefaca..6f8d1469ef 100644 --- a/erts/emulator/beam/msg_instrs.tab +++ b/erts/emulator/beam/msg_instrs.tab @@ -137,8 +137,8 @@ i_loop_rec(Dest) { if (ERTS_UNLIKELY(ERTS_SIG_IS_EXTERNAL_MSG(msgp))) { FCALLS -= 10; /* FIXME: bump appropriate amount... */ - SWAPOUT; /* erts_decode_dist_message() may write to heap... */ - if (!erts_decode_dist_message(c_p, ERTS_PROC_LOCK_MAIN, msgp, 0)) { + SWAPOUT; /* erts_proc_sig_decode_dist() may write to heap... */ + if (!erts_proc_sig_decode_dist(c_p, ERTS_PROC_LOCK_MAIN, msgp, 0)) { /* * A corrupt distribution message that we weren't able to decode; * remove it... diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 1e7b494a91..36a492f00d 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -2701,7 +2701,8 @@ Sint erts_cmp_compound(Eterm a, Eterm b, int exact, int eq_only) if((AN)->sysname != (BN)->sysname) \ RETURN_NEQ(erts_cmp_atoms((AN)->sysname, (BN)->sysname)); \ ASSERT((AN)->creation != (BN)->creation); \ - RETURN_NEQ(((AN)->creation < (BN)->creation) ? -1 : 1); \ + if ((AN)->creation != 0 && (BN)->creation != 0) \ + RETURN_NEQ(((AN)->creation < (BN)->creation) ? -1 : 1); \ } \ } while (0) @@ -3486,7 +3487,7 @@ store_external_or_ref_(Uint **hpp, ErlOffHeap* oh, Eterm ns) if (is_external_header(*from_hp)) { ExternalThing *etp = (ExternalThing *) from_hp; ASSERT(is_external(ns)); - erts_refc_inc(&etp->node->refc, 2); + erts_ref_node_entry(etp->node, 2, make_boxed(to_hp)); } else if (is_ordinary_ref_thing(from_hp)) return make_internal_ref(to_hp); |