diff options
Diffstat (limited to 'erts/emulator/beam')
25 files changed, 858 insertions, 575 deletions
diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 45b7540aeb..fb223d2a5d 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -208,7 +208,6 @@ atom dirty_nif_finalizer atom disable_trace atom disabled atom discard -atom display_items atom dist atom dist_cmd atom dist_ctrl_put_data @@ -237,6 +236,7 @@ atom eof atom eol atom Eq='=:=' atom Eqeq='==' +atom erl_init atom erl_tracer atom erlang atom erl_signal_server diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 50cbb37f3e..c28a5c57e6 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -2868,6 +2868,7 @@ load_code(LoaderState* stp) break; case op_bs_put_string_WW: case op_i_bs_match_string_xfWW: + case op_i_bs_match_string_yfWW: new_string_patch(stp, ci-1); break; diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 56ac072449..3b45f968cf 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -218,11 +218,8 @@ BIF_RETTYPE link_1(BIF_ALIST_1) * We have (pending) connection. * Setup link and enqueue link signal. */ -#ifdef DEBUG - int inserted = -#endif - erts_link_dist_insert(&ldp->b, dep->mld); - ASSERT(inserted); + int inserted = erts_link_dist_insert(&ldp->b, dep->mld); + ASSERT(inserted); (void)inserted; erts_de_runlock(dep); code = erts_dsig_send_link(&dsd, BIF_P->common.id, BIF_ARG_1); @@ -567,12 +564,8 @@ BIF_RETTYPE monitor_2(BIF_ALIST_2) case ERTS_DSIG_PREP_PENDING: case ERTS_DSIG_PREP_CONNECTED: { -#ifdef DEBUG - int inserted = -#endif - - erts_monitor_dist_insert(&mdp->target, dep->mld); - ASSERT(inserted); + int inserted = erts_monitor_dist_insert(&mdp->target, dep->mld); + ASSERT(inserted); (void)inserted; erts_de_runlock(dep); code = erts_dsig_send_monitor(&dsd, BIF_P->common.id, target, ref); @@ -2745,9 +2738,7 @@ BIF_RETTYPE atom_to_list_1(BIF_ALIST_1) Uint num_chars, num_built, num_eaten; byte* err_pos; Eterm res; -#ifdef DEBUG int ares; -#endif if (is_not_atom(BIF_ARG_1)) BIF_ERROR(BIF_P, BADARG); @@ -2757,11 +2748,9 @@ BIF_RETTYPE atom_to_list_1(BIF_ALIST_1) if (ap->len == 0) BIF_RET(NIL); /* the empty atom */ -#ifdef DEBUG ares = -#endif erts_analyze_utf8(ap->name, ap->len, &err_pos, &num_chars, NULL); - ASSERT(ares == ERTS_UTF8_OK); + ASSERT(ares == ERTS_UTF8_OK); (void)ares; res = erts_utf8_to_list(BIF_P, num_chars, ap->name, ap->len, ap->len, &num_built, &num_eaten, NIL); @@ -4451,13 +4440,6 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); BIF_RET(old_value); - } else if (BIF_ARG_1 == am_display_items) { - int oval = display_items; - if (!is_small(BIF_ARG_2) || (n = signed_val(BIF_ARG_2)) < 0) { - goto error; - } - display_items = n < 32 ? 32 : n; - BIF_RET(make_small(oval)); } else if (BIF_ARG_1 == am_debug_flags) { BIF_RET(am_true); } else if (BIF_ARG_1 == am_backtrace_depth) { @@ -5160,17 +5142,11 @@ BIF_RETTYPE send_to_logger_2(BIF_ALIST_2) else if (len == 0) buf = ""; else { -#ifdef DEBUG ErlDrvSizeT len2; -#endif buf = (byte *) erts_alloc(ERTS_ALC_T_TMP, len+1); -#ifdef DEBUG len2 = -#else - (void) -#endif erts_iolist_to_buf(BIF_ARG_2, buf, len); - ASSERT(len2 == len); + ASSERT(len2 == len); (void)len2; buf[len] = '\0'; switch (BIF_ARG_1) { case am_info: diff --git a/erts/emulator/beam/bs_instrs.tab b/erts/emulator/beam/bs_instrs.tab index 61eb02a7a2..2dde70c2e1 100644 --- a/erts/emulator/beam/bs_instrs.tab +++ b/erts/emulator/beam/bs_instrs.tab @@ -102,6 +102,7 @@ i_bs_get_binary_all2(Fail, Ms, Live, Unit, Dst) { LIGHT_SWAPIN; HEAP_SPACE_VERIFIED(0); ASSERT(is_value(_result)); + $REFRESH_GEN_DEST(); $Dst = _result; } else { HEAP_SPACE_VERIFIED(0); @@ -123,6 +124,7 @@ i_bs_get_binary2(Fail, Ms, Live, Sz, Flags, Dst) { if (is_non_value(_result)) { $FAIL($Fail); } else { + $REFRESH_GEN_DEST(); $Dst = _result; } } @@ -139,6 +141,7 @@ i_bs_get_binary_imm2(Fail, Ms, Live, Sz, Flags, Dst) { if (is_non_value(_result)) { $FAIL($Fail); } else { + $REFRESH_GEN_DEST(); $Dst = _result; } } @@ -161,6 +164,7 @@ i_bs_get_float2(Fail, Ms, Live, Sz, Flags, Dst) { if (is_non_value(_result)) { $FAIL($Fail); } else { + $REFRESH_GEN_DEST(); $Dst = _result; } } @@ -724,26 +728,34 @@ bs_start_match.execute(Fail, Live, Slots, Dst) { $FAIL($Fail); } header = *boxed_val(context); - slots = $Slots; + + /* Reserve a slot for the start position. */ + slots = $Slots + 1; live = $Live; + if (header_is_bin_matchstate(header)) { ErlBinMatchState* ms = (ErlBinMatchState *) boxed_val(context); Uint actual_slots = HEADER_NUM_SLOTS(header); + + /* We're not compatible with contexts created by bs_start_match3. */ + ASSERT(actual_slots >= 1); + ms->save_offset[0] = ms->mb.offset; - if (actual_slots < slots) { - ErlBinMatchState* dst; + if (ERTS_UNLIKELY(actual_slots < slots)) { + ErlBinMatchState* expanded; Uint live = $Live; Uint wordsneeded = ERL_BIN_MATCHSTATE_SIZE(slots); - $GC_TEST_PRESERVE(wordsneeded, live, context); ms = (ErlBinMatchState *) boxed_val(context); - dst = (ErlBinMatchState *) HTOP; - *dst = *ms; + expanded = (ErlBinMatchState *) HTOP; + *expanded = *ms; *HTOP = HEADER_BIN_MATCHSTATE(slots); HTOP += wordsneeded; HEAP_SPACE_VERIFIED(0); - $Dst = make_matchstate(dst); + context = make_matchstate(expanded); + $REFRESH_GEN_DEST(); } + $Dst = context; } else if (is_binary_header(header)) { Eterm result; Uint wordsneeded = ERL_BIN_MATCHSTATE_SIZE(slots); @@ -758,6 +770,7 @@ bs_start_match.execute(Fail, Live, Slots, Dst) { if (is_non_value(result)) { $FAIL($Fail); } + $REFRESH_GEN_DEST(); $Dst = result; } else { $FAIL($Fail); @@ -906,6 +919,7 @@ i_bs_get_integer(Fail, Live, FlagsAndUnit, Ms, Sz, Dst) { } wordsneeded = 1+WSIZE(NBYTES((Uint) size)); $GC_TEST_PRESERVE(wordsneeded, $Live, ms); + $REFRESH_GEN_DEST(); } mb = ms_matchbuffer(ms); LIGHT_SWAPOUT; @@ -939,6 +953,7 @@ i_bs_get_utf8(Ctx, Fail, Dst) { if (is_non_value(result)) { $FAIL($Fail); } + $REFRESH_GEN_DEST(); $Dst = result; } @@ -949,6 +964,7 @@ i_bs_get_utf16(Ctx, Fail, Flags, Dst) { if (is_non_value(result)) { $FAIL($Fail); } + $REFRESH_GEN_DEST(); $Dst = result; } @@ -1029,10 +1045,289 @@ i_bs_match_string(Ctx, Fail, Bits, Ptr) { i_bs_save2(Src, Slot) { ErlBinMatchState* _ms = (ErlBinMatchState*) boxed_val((Eterm) $Src); + ASSERT(HEADER_NUM_SLOTS(_ms->thing_word) > $Slot); _ms->save_offset[$Slot] = _ms->mb.offset; } i_bs_restore2(Src, Slot) { ErlBinMatchState* _ms = (ErlBinMatchState*) boxed_val((Eterm) $Src); + ASSERT(HEADER_NUM_SLOTS(_ms->thing_word) > $Slot); _ms->mb.offset = _ms->save_offset[$Slot]; } + +bs_get_tail(Src, Dst, Live) { + ErlBinMatchBuffer* mb; + Uint size, offs; + ErlSubBin* sb; + Eterm context; + + context = $Src; + + ASSERT(header_is_bin_matchstate(*boxed_val(context))); + + $GC_TEST_PRESERVE(ERL_SUB_BIN_SIZE, $Live, context); + + mb = ms_matchbuffer(context); + + offs = mb->offset; + size = mb->size - offs; + + sb = (ErlSubBin *) HTOP; + HTOP += ERL_SUB_BIN_SIZE; + + sb->thing_word = HEADER_SUB_BIN; + sb->size = BYTE_OFFSET(size); + sb->bitsize = BIT_OFFSET(size); + sb->offs = BYTE_OFFSET(offs); + sb->bitoffs = BIT_OFFSET(offs); + sb->is_writable = 0; + sb->orig = mb->orig; + + $REFRESH_GEN_DEST(); + $Dst = make_binary(sb); +} + + +%if ARCH_64 + +i_bs_start_match3_gp(Src, Live, Fail, Dst, Pos) { + Eterm context, header; + Uint position, live; + + context = $Src; + live = $Live; + + if (!is_boxed(context)) { + $FAIL($Fail); + } + + header = *boxed_val(context); + + if (header_is_bin_matchstate(header)) { + ErlBinMatchBuffer *mb; + + ASSERT(HEADER_NUM_SLOTS(header) == 0); + + mb = ms_matchbuffer(context); + position = mb->offset; + + $Dst = context; + } else if (is_binary_header(header)) { + ErlBinMatchState *ms; + + $GC_TEST_PRESERVE(ERL_BIN_MATCHSTATE_SIZE(0), live, context); + HEAP_TOP(c_p) = HTOP; +#ifdef DEBUG + c_p->stop = E; /* Needed for checking in HeapOnlyAlloc(). */ +#endif + ms = erts_bs_start_match_3(c_p, context); + HTOP = HEAP_TOP(c_p); + HEAP_SPACE_VERIFIED(0); + + if (ms == NULL) { + $FAIL($Fail); + } + + $REFRESH_GEN_DEST(); + $Dst = make_matchstate(ms); + position = ms->mb.offset; + } else { + $FAIL($Fail); + } + + ASSERT(IS_USMALL(0, position)); + $Pos = make_small(position); +} + +i_bs_start_match3(Src, Live, Fail, Dst) { + Eterm context, header; + Uint live; + + context = $Src; + live = $Live; + + if (!is_boxed(context)) { + $FAIL($Fail); + } + + header = *boxed_val(context); + + if (header_is_bin_matchstate(header)) { + ASSERT(HEADER_NUM_SLOTS(header) == 0); + $Dst = context; + } else if (is_binary_header(header)) { + ErlBinMatchState *ms; + + $GC_TEST_PRESERVE(ERL_BIN_MATCHSTATE_SIZE(0), live, context); + HEAP_TOP(c_p) = HTOP; +#ifdef DEBUG + c_p->stop = E; /* Needed for checking in HeapOnlyAlloc(). */ +#endif + ms = erts_bs_start_match_3(c_p, context); + HTOP = HEAP_TOP(c_p); + HEAP_SPACE_VERIFIED(0); + + if (ms == NULL) { + $FAIL($Fail); + } + + $REFRESH_GEN_DEST(); + $Dst = make_matchstate(ms); + } else { + $FAIL($Fail); + } +} + +bs_set_position(Ctx, Pos) { + ErlBinMatchBuffer* mb; + Eterm context; + + context = $Ctx; + ASSERT(header_is_bin_matchstate(*boxed_val(context))); + + mb = ms_matchbuffer(context); + mb->offset = unsigned_val($Pos); +} + +i_bs_get_position(Ctx, Dst) { + ErlBinMatchBuffer* mb; + Eterm context; + + context = $Ctx; + ASSERT(header_is_bin_matchstate(*boxed_val(context))); + + mb = ms_matchbuffer(context); + $Dst = make_small(mb->offset); +} + +%else + +# +# Unlike their 64-bit counterparts, the 32-bit position instructions operate on +# an offset from the "base position" of the context because storing raw +# positions would lead to the creation of far too many bigints. +# +# When a match context is reused we check whether its position fits into an +# immediate, and create a new match context if it does not. This means we only +# have to allocate stuff roughly once every 16MB rather than every time we +# match at a position beyond 16MB. +# + +bs_set_position(Ctx, Pos) { + Eterm context, position; + ErlBinMatchState *ms; + + context = $Ctx; + position = $Pos; + + ASSERT(header_is_bin_matchstate(*boxed_val(context))); + ms = (ErlBinMatchState*)boxed_val(context); + + if (ERTS_LIKELY(is_small(position))) { + ms->mb.offset = ms->save_offset[0] + unsigned_val(position); + } else { + ASSERT(is_big(position)); + ms->mb.offset = ms->save_offset[0] + *BIG_V(big_val(position)); + } +} + +bs_get_position(Ctx, Dst, Live) { + ErlBinMatchState *ms; + Eterm context; + Uint position; + + context = $Ctx; + + ASSERT(header_is_bin_matchstate(*boxed_val(context))); + ms = (ErlBinMatchState*)boxed_val(context); + + position = ms->mb.offset - ms->save_offset[0]; + + if (ERTS_LIKELY(IS_USMALL(0, position))) { + $Dst = make_small(position); + } else { + Eterm *hp; + + $GC_TEST_PRESERVE(BIG_UINT_HEAP_SIZE, $Live, context); + + hp = HTOP; + HTOP += BIG_UINT_HEAP_SIZE; + + *hp = make_pos_bignum_header(1); + BIG_DIGIT(hp, 0) = position; + + $REFRESH_GEN_DEST(); + $Dst = make_big(hp); + } +} + +i_bs_start_match3(Src, Live, Fail, Dst) { + Eterm context, header; + Uint live; + + context = $Src; + live = $Live; + + if (!is_boxed(context)) { + $FAIL($Fail); + } + + header = *boxed_val(context); + + if (header_is_bin_matchstate(header)) { + ErlBinMatchState *current_ms; + Uint position; + + ASSERT(HEADER_NUM_SLOTS(header) == 1); + + current_ms = (ErlBinMatchState*)boxed_val(context); + position = current_ms->mb.offset - current_ms->save_offset[0]; + + if (ERTS_LIKELY(IS_USMALL(0, position))) { + $Dst = context; + } else { + ErlBinMatchState *new_ms; + + $GC_TEST_PRESERVE(ERL_BIN_MATCHSTATE_SIZE(1), live, context); + current_ms = (ErlBinMatchState*)boxed_val(context); + + new_ms = (ErlBinMatchState*)HTOP; + HTOP += ERL_BIN_MATCHSTATE_SIZE(1); + + new_ms->thing_word = HEADER_BIN_MATCHSTATE(1); + new_ms->save_offset[0] = current_ms->mb.offset; + new_ms->mb = current_ms->mb; + + $REFRESH_GEN_DEST(); + $Dst = make_matchstate(new_ms); + } + } else if (is_binary_header(header)) { + Eterm result; + + $GC_TEST_PRESERVE(ERL_BIN_MATCHSTATE_SIZE(1), live, context); + HEAP_TOP(c_p) = HTOP; + +#ifdef DEBUG + c_p->stop = E; /* Needed for checking in HeapOnlyAlloc(). */ +#endif + + /* We intentionally use erts_bs_start_match_2 so that we can use + * save_offset as a base for all saved positions on this context, + * allowing us to avoid bigints for much longer. */ + result = erts_bs_start_match_2(c_p, context, 1); + + HTOP = HEAP_TOP(c_p); + HEAP_SPACE_VERIFIED(0); + + if (is_non_value(result)) { + $FAIL($Fail); + } + + $REFRESH_GEN_DEST(); + $Dst = result; + } else { + $FAIL($Fail); + } +} + +%endif diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index db594a23a0..15642e1669 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -116,11 +116,12 @@ static Export *dist_ctrl_put_data_trap; /* forward declarations */ -static void clear_dist_entry(DistEntry*); static int dsig_send_ctl(ErtsDSigData* dsdp, Eterm ctl, int force_busy); static void send_nodes_mon_msgs(Process *, Eterm, Eterm, Eterm, Eterm); static void init_nodes_monitors(void); static Sint abort_connection(DistEntry* dep, Uint32 conn_id); +static ErtsDistOutputBuf* clear_de_out_queues(DistEntry*); +static void free_de_out_queues(DistEntry*, ErtsDistOutputBuf*); static erts_atomic_t no_caches; static erts_atomic_t no_nodes; @@ -556,7 +557,10 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason) } } else { /* Call from distribution controller (port/process) */ - ErtsMonLnkDist *mld; + ErtsMonLnkDist *mld; + ErtsAtomCache *cache; + ErtsProcList *suspendees; + ErtsDistOutputBuf *obuf; Uint32 flags; erts_atomic_set_mb(&dep->dist_cmd_scheduled, 1); @@ -570,9 +574,7 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason) } if (dep->state == ERTS_DE_STATE_EXITING) { -#ifdef DEBUG ASSERT(erts_atomic32_read_nob(&dep->qflgs) & ERTS_DE_QFLG_EXIT); -#endif } else { dep->state = ERTS_DE_STATE_EXITING; @@ -588,6 +590,22 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason) nodename = dep->sysname; flags = dep->flags; + erts_atomic_set_nob(&dep->input_handler, (erts_aint_t) NIL); + cache = dep->cache; + dep->cache = NULL; + + erts_mtx_lock(&dep->qlock); + + erts_atomic64_set_nob(&dep->in, 0); + erts_atomic64_set_nob(&dep->out, 0); + + obuf = clear_de_out_queues(dep); + suspendees = get_suspended_on_de(dep, ERTS_DE_QFLGS_ALL); + + erts_mtx_unlock(&dep->qlock); + erts_atomic_set_nob(&dep->dist_cmd_scheduled, 0); + dep->send = NULL; + erts_set_dist_entry_not_connected(dep); erts_de_rwunlock(dep); @@ -601,7 +619,13 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason) ? am_connection_closed : reason)); - clear_dist_entry(dep); + erts_resume_processes(suspendees); + + delete_cache(cache); + + free_de_out_queues(dep, obuf); + if (dep->transcode_ctx) + transcode_free_ctx(dep); } dec_no_nodes(); @@ -732,41 +756,6 @@ static void free_de_out_queues(DistEntry* dep, ErtsDistOutputBuf *obuf) } } -static void clear_dist_entry(DistEntry *dep) -{ - ErtsAtomCache *cache; - ErtsProcList *suspendees; - ErtsDistOutputBuf *obuf; - - erts_de_rwlock(dep); - erts_atomic_set_nob(&dep->input_handler, - (erts_aint_t) NIL); - cache = dep->cache; - dep->cache = NULL; - - erts_mtx_lock(&dep->qlock); - - erts_atomic64_set_nob(&dep->in, 0); - erts_atomic64_set_nob(&dep->out, 0); - - obuf = clear_de_out_queues(dep); - dep->state = ERTS_DE_STATE_IDLE; - suspendees = get_suspended_on_de(dep, ERTS_DE_QFLGS_ALL); - - erts_mtx_unlock(&dep->qlock); - erts_atomic_set_nob(&dep->dist_cmd_scheduled, 0); - dep->send = NULL; - erts_de_rwunlock(dep); - - erts_resume_processes(suspendees); - - delete_cache(cache); - - free_de_out_queues(dep, obuf); - if (dep->transcode_ctx) - transcode_free_ctx(dep); -} - int erts_dsend_context_dtor(Binary* ctx_bin) { ErtsSendContext* ctx = ERTS_MAGIC_BIN_DATA(ctx_bin); @@ -861,7 +850,7 @@ erts_dsig_send_m_exit(ErtsDSigData *dsdp, Eterm watcher, Eterm watched, DeclareTmpHeapNoproc(ctl_heap,6); int res; - if (~dsdp->dep->flags & (DFLAG_DIST_MONITOR | DFLAG_DIST_MONITOR_NAME)) { + if (~dsdp->flags & (DFLAG_DIST_MONITOR | DFLAG_DIST_MONITOR_NAME)) { /* * Receiver does not support DOP_MONITOR_P_EXIT (see dsig_send_monitor) */ @@ -889,7 +878,7 @@ erts_dsig_send_monitor(ErtsDSigData *dsdp, Eterm watcher, Eterm watched, DeclareTmpHeapNoproc(ctl_heap,5); int res; - if (~dsdp->dep->flags & (DFLAG_DIST_MONITOR | DFLAG_DIST_MONITOR_NAME)) { + if (~dsdp->flags & (DFLAG_DIST_MONITOR | DFLAG_DIST_MONITOR_NAME)) { /* * Receiver does not support DOP_MONITOR_P. * Just avoid sending it and by doing that reduce this monitor @@ -920,7 +909,7 @@ erts_dsig_send_demonitor(ErtsDSigData *dsdp, Eterm watcher, DeclareTmpHeapNoproc(ctl_heap,5); int res; - if (~dsdp->dep->flags & (DFLAG_DIST_MONITOR | DFLAG_DIST_MONITOR_NAME)) { + if (~dsdp->flags & (DFLAG_DIST_MONITOR | DFLAG_DIST_MONITOR_NAME)) { /* * Receiver does not support DOP_DEMONITOR_P (see dsig_send_monitor) */ @@ -940,7 +929,7 @@ erts_dsig_send_demonitor(ErtsDSigData *dsdp, Eterm watcher, static int can_send_seqtrace_token(ErtsSendContext* ctx, Eterm token) { Eterm label; - if (ctx->dep->flags & DFLAG_BIG_SEQTRACE_LABELS) { + if (ctx->dsd.flags & DFLAG_BIG_SEQTRACE_LABELS) { /* The other end is capable of handling arbitrary seq_trace labels. */ return 1; } @@ -1001,7 +990,7 @@ erts_dsig_send_msg(Eterm remote, Eterm message, ErtsSendContext* ctx) send_token = (token != NIL && can_send_seqtrace_token(ctx, token)); - if (ctx->dep->flags & DFLAG_SEND_SENDER) { + if (ctx->dsd.flags & DFLAG_SEND_SENDER) { dist_op = make_small(send_token ? DOP_SEND_SENDER_TT : DOP_SEND_SENDER); @@ -1218,13 +1207,13 @@ erts_dsig_send_group_leader(ErtsDSigData *dsdp, Eterm leader, Eterm remote) int erts_net_message(Port *prt, DistEntry *dep, + Uint32 conn_id, byte *hbuf, ErlDrvSizeT hlen, byte *buf, ErlDrvSizeT len) { ErtsDistExternal ede; - byte *t; Sint ctl_len; Eterm arg; Eterm from, to; @@ -1242,7 +1231,6 @@ int erts_net_message(Port *prt, Eterm token_size; Uint tuple_arity; int res; - Uint32 connection_id; #ifdef ERTS_DIST_MSG_DBG ErlDrvSizeT orig_len = len; #endif @@ -1258,7 +1246,6 @@ int erts_net_message(Port *prt, return 0; } - ASSERT(hlen == 0); if (len == 0) { /* HANDLE TICK !!! */ @@ -1271,15 +1258,7 @@ int erts_net_message(Port *prt, bw(buf, len); #endif - if (dep->flags & DFLAG_DIST_HDR_ATOM_CACHE) - t = buf; - else { - /* Skip PASS_THROUGH */ - t = buf+1; - len--; - } - - res = erts_prepare_dist_ext(&ede, t, len, dep, dep->cache, &connection_id); + res = erts_prepare_dist_ext(&ede, buf, len, dep, conn_id, dep->cache); switch (res) { case ERTS_PREP_DIST_EXT_CLOSED: @@ -1321,10 +1300,9 @@ int erts_net_message(Port *prt, PURIFY_MSG("data error"); goto decode_error; } - ctl_len = t - buf; #ifdef ERTS_DIST_MSG_DBG - erts_fprintf(stderr, "<<%s CTL: %T\n", len != orig_len ? "P" : " ", arg); + erts_fprintf(stderr, "<< CTL: %T\n", arg); #endif if (is_not_tuple(arg) || @@ -1363,10 +1341,7 @@ int erts_net_message(Port *prt, from, to); ASSERT(ldp->a.other.item == to); ASSERT(eq(ldp->b.other.item, from)); -#ifdef DEBUG - code = -#endif - erts_link_dist_insert(&ldp->a, dep->mld); + code = erts_link_dist_insert(&ldp->a, dep->mld); ASSERT(code); if (erts_proc_sig_send_link(NULL, to, &ldp->b)) @@ -1374,10 +1349,7 @@ int erts_net_message(Port *prt, /* Failed to send signal; cleanup and reply noproc... */ -#ifdef DEBUG - code = -#endif - erts_link_dist_delete(&ldp->a); + code = erts_link_dist_delete(&ldp->a); ASSERT(code); erts_link_release_both(ldp); } @@ -1778,7 +1750,7 @@ decode_error: } data_error: UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE); - erts_kill_dist_connection(dep, connection_id); + erts_kill_dist_connection(dep, conn_id); ERTS_CHK_NO_PROC_LOCKS; return -1; } @@ -1834,7 +1806,7 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx) while (1) { switch (ctx->phase) { case ERTS_DSIG_SEND_PHASE_INIT: - ctx->flags = dsdp->dep->flags; + ctx->flags = dsdp->flags; ctx->c_p = dsdp->proc; if (!ctx->c_p || dsdp->no_suspend) @@ -2095,13 +2067,14 @@ dist_port_command(Port *prt, ErtsDistOutputBuf *obuf) #ifdef USE_VM_PROBES if (DTRACE_ENABLED(dist_output)) { + DistEntry *dep = (DistEntry*) erts_prtsd_get(prt, ERTS_PRTSD_DIST_ENTRY); DTRACE_CHARBUF(port_str, 64); DTRACE_CHARBUF(remote_str, 64); erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), "%T", prt->common.id); erts_snprintf(remote_str, sizeof(DTRACE_CHARBUF_NAME(remote_str)), - "%T", prt->dist_entry->sysname); + "%T", dep->sysname); DTRACE4(dist_output, erts_this_node_sysname, port_str, remote_str, size); } @@ -2157,13 +2130,14 @@ dist_port_commandv(Port *prt, ErtsDistOutputBuf *obuf) #ifdef USE_VM_PROBES if (DTRACE_ENABLED(dist_outputv)) { + DistEntry *dep = (DistEntry*) erts_prtsd_get(prt, ERTS_PRTSD_DIST_ENTRY); DTRACE_CHARBUF(port_str, 64); DTRACE_CHARBUF(remote_str, 64); erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), "%T", prt->common.id); erts_snprintf(remote_str, sizeof(DTRACE_CHARBUF_NAME(remote_str)), - "%T", prt->dist_entry->sysname); + "%T", dep->sysname); DTRACE4(dist_outputv, erts_this_node_sysname, port_str, remote_str, size); } @@ -2201,7 +2175,7 @@ erts_dist_command(Port *prt, int initial_reds) Uint32 flags; Sint qsize, obufsize = 0; ErtsDistOutputQueue oq, foq; - DistEntry *dep = prt->dist_entry; + DistEntry *dep = (DistEntry*) erts_prtsd_get(prt, ERTS_PRTSD_DIST_ENTRY); Uint (*send)(Port *prt, ErtsDistOutputBuf *obuf); erts_aint32_t sched_flags; ErtsSchedulerData *esdp = erts_get_scheduler_data(); @@ -2577,11 +2551,12 @@ dist_ctrl_get_data_notification_1(BIF_ALIST_1) erts_aint32_t qflgs; erts_aint_t qsize; Eterm receiver = NIL; + Uint32 conn_id; if (!dep) BIF_ERROR(BIF_P, EXC_NOTSUP); - if (erts_dhandle_to_dist_entry(BIF_ARG_1) != dep) + if (erts_dhandle_to_dist_entry(BIF_ARG_1, &conn_id) != dep) BIF_ERROR(BIF_P, BADARG); /* @@ -2591,6 +2566,11 @@ dist_ctrl_get_data_notification_1(BIF_ALIST_1) erts_de_rlock(dep); + if (dep->connection_id != conn_id) { + erts_de_runlock(dep); + BIF_ERROR(BIF_P, BADARG); + } + ASSERT(dep->cid == BIF_P->common.id); qflgs = erts_atomic32_read_acqb(&dep->qflgs); @@ -2631,6 +2611,7 @@ dist_ctrl_put_data_2(BIF_ALIST_2) DistEntry *dep; ErlDrvSizeT size; Eterm input_handler; + Uint32 conn_id; if (is_binary(BIF_ARG_2)) size = binary_size(BIF_ARG_2); @@ -2642,7 +2623,7 @@ dist_ctrl_put_data_2(BIF_ALIST_2) else BIF_ERROR(BIF_P, BADARG); - dep = erts_dhandle_to_dist_entry(BIF_ARG_1); + dep = erts_dhandle_to_dist_entry(BIF_ARG_1, &conn_id); if (!dep) BIF_ERROR(BIF_P, BADARG); @@ -2662,7 +2643,7 @@ dist_ctrl_put_data_2(BIF_ALIST_2) erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - (void) erts_net_message(NULL, dep, NULL, 0, data, size); + (void) erts_net_message(NULL, dep, conn_id, NULL, 0, data, size); /* * We ignore any decode failures. On fatal failures the * connection will be taken down by killing the @@ -2686,13 +2667,18 @@ dist_get_stat_1(BIF_ALIST_1) Sint64 read, write, pend; Eterm res, *hp, **hpp; Uint sz, *szp; - DistEntry *dep = erts_dhandle_to_dist_entry(BIF_ARG_1); + Uint32 conn_id; + DistEntry *dep = erts_dhandle_to_dist_entry(BIF_ARG_1, &conn_id); if (!dep) BIF_ERROR(BIF_P, BADARG); erts_de_rlock(dep); + if (dep->connection_id != conn_id) { + erts_de_runlock(dep); + BIF_ERROR(BIF_P, BADARG); + } read = (Sint64) erts_atomic64_read_nob(&dep->in); write = (Sint64) erts_atomic64_read_nob(&dep->out); pend = (Sint64) erts_atomic_read_nob(&dep->qsize); @@ -2723,19 +2709,25 @@ BIF_RETTYPE dist_ctrl_input_handler_2(BIF_ALIST_2) { DistEntry *dep = ERTS_PROC_GET_DIST_ENTRY(BIF_P); + Uint32 conn_id; if (!dep) BIF_ERROR(BIF_P, EXC_NOTSUP); - if (erts_dhandle_to_dist_entry(BIF_ARG_1) != dep) + if (erts_dhandle_to_dist_entry(BIF_ARG_1, &conn_id) != dep) BIF_ERROR(BIF_P, BADARG); if (is_not_internal_pid(BIF_ARG_2)) BIF_ERROR(BIF_P, BADARG); + erts_de_rlock(dep); + if (dep->connection_id != conn_id) { + erts_de_runlock(dep); + BIF_ERROR(BIF_P, BADARG); + } erts_atomic_set_nob(&dep->input_handler, (erts_aint_t) BIF_ARG_2); - + erts_de_runlock(dep); BIF_RET(am_ok); } @@ -2749,15 +2741,21 @@ dist_ctrl_get_data_1(BIF_ALIST_1) Eterm *hp; ProcBin *pb; erts_aint_t qsize; + Uint32 conn_id; if (!dep) BIF_ERROR(BIF_P, EXC_NOTSUP); - if (erts_dhandle_to_dist_entry(BIF_ARG_1) != dep) + if (erts_dhandle_to_dist_entry(BIF_ARG_1, &conn_id) != dep) BIF_ERROR(BIF_P, BADARG); erts_de_rlock(dep); + if (dep->connection_id != conn_id) { + erts_de_runlock(dep); + BIF_ERROR(BIF_P, BADARG); + } + if (dep->state == ERTS_DE_STATE_EXITING) goto return_none; @@ -2844,13 +2842,14 @@ erts_dist_port_not_busy(Port *prt) { #ifdef USE_VM_PROBES if (DTRACE_ENABLED(dist_port_not_busy)) { + DistEntry *dep = (DistEntry*) erts_prtsd_get(prt, ERTS_PRTSD_DIST_ENTRY); DTRACE_CHARBUF(port_str, 64); DTRACE_CHARBUF(remote_str, 64); erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), "%T", prt->common.id); erts_snprintf(remote_str, sizeof(DTRACE_CHARBUF_NAME(remote_str)), - "%T", prt->dist_entry->sysname); + "%T", dep->sysname); DTRACE3(dist_port_not_busy, erts_this_node_sysname, port_str, remote_str); } @@ -2876,10 +2875,10 @@ static void kill_connection(DistEntry *dep) } void -erts_kill_dist_connection(DistEntry *dep, Uint32 connection_id) +erts_kill_dist_connection(DistEntry *dep, Uint32 conn_id) { erts_de_rwlock(dep); - if (connection_id == dep->connection_id + if (conn_id == dep->connection_id && dep->state == ERTS_DE_STATE_CONNECTED) { kill_connection(dep); @@ -3215,23 +3214,6 @@ BIF_RETTYPE erts_internal_create_dist_channel_4(BIF_ALIST_4) else if (!dep) goto system_limit; /* Should never happen!!! */ - erts_de_rlock(dep); - de_locked = -1; - - if (dep->state == ERTS_DE_STATE_EXITING) { - /* Suspend on dist entry waiting for the exit to finish */ - ErtsProcList *plp = erts_proclist_create(BIF_P); - plp->next = NULL; - erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL); - erts_mtx_lock(&dep->qlock); - erts_proclist_store_last(&dep->suspended, plp); - erts_mtx_unlock(&dep->qlock); - goto yield; - } - - erts_de_runlock(dep); - de_locked = 0; - if (is_internal_pid(BIF_ARG_2)) { if (BIF_P->common.id == BIF_ARG_2) { ErtsSetupConnDistCtrl scdc; @@ -3277,7 +3259,7 @@ BIF_RETTYPE erts_internal_create_dist_channel_4(BIF_ALIST_4) hp = HAlloc(BIF_P, 3); } else { - int new; + Uint32 conn_id; pp = erts_id2port_sflgs(BIF_ARG_2, BIF_P, @@ -3286,7 +3268,7 @@ BIF_RETTYPE erts_internal_create_dist_channel_4(BIF_ALIST_4) erts_de_rwlock(dep); de_locked = 1; - if (dep->state == ERTS_DE_STATE_EXITING) + if (dep->state != ERTS_DE_STATE_PENDING) goto badarg; if (!pp || (erts_atomic32_read_nob(&pp->state) @@ -3296,49 +3278,39 @@ BIF_RETTYPE erts_internal_create_dist_channel_4(BIF_ALIST_4) if ((pp->drv_ptr->flags & ERL_DRV_FLAG_SOFT_BUSY) == 0) goto badarg; - if (dep->cid == BIF_ARG_2 && pp->dist_entry == dep) - new = 0; - else { - if (dep->state != ERTS_DE_STATE_PENDING) { - if (dep->state == ERTS_DE_STATE_IDLE) - erts_set_dist_entry_pending(dep); - else - goto badarg; - } - - if (pp->dist_entry || is_not_nil(dep->cid)) - goto badarg; - - erts_atomic32_read_bor_nob(&pp->state, ERTS_PORT_SFLG_DISTRIBUTION); + if (erts_prtsd_get(pp, ERTS_PRTSD_DIST_ENTRY) != NULL + || is_not_nil(dep->cid)) + goto badarg; - pp->dist_entry = dep; + erts_atomic32_read_bor_nob(&pp->state, ERTS_PORT_SFLG_DISTRIBUTION); - ASSERT(pp->drv_ptr->outputv || pp->drv_ptr->output); + erts_prtsd_set(pp, ERTS_PRTSD_DIST_ENTRY, dep); + erts_prtsd_set(pp, ERTS_PRTSD_CONN_ID, (void*)(UWord)dep->connection_id); - dep->send = (pp->drv_ptr->outputv - ? dist_port_commandv - : dist_port_command); - ASSERT(dep->send); + ASSERT(pp->drv_ptr->outputv || pp->drv_ptr->output); - /* - * Dist-ports do not use the "busy port message queue" functionality, but - * instead use "busy dist entry" functionality. - */ - { - ErlDrvSizeT disable = ERL_DRV_BUSY_MSGQ_DISABLED; - erl_drv_busy_msgq_limits(ERTS_Port2ErlDrvPort(pp), &disable, NULL); - } + dep->send = (pp->drv_ptr->outputv + ? dist_port_commandv + : dist_port_command); + ASSERT(dep->send); - setup_connection_epiloge_rwunlock(BIF_P, dep, BIF_ARG_2, flags, version); - de_locked = 0; - new = !0; + /* + * Dist-ports do not use the "busy port message queue" functionality, but + * instead use "busy dist entry" functionality. + */ + { + ErlDrvSizeT disable = ERL_DRV_BUSY_MSGQ_DISABLED; + erl_drv_busy_msgq_limits(ERTS_Port2ErlDrvPort(pp), &disable, NULL); } - hp = HAlloc(BIF_P, 3 + ERTS_MAGIC_REF_THING_SIZE); - res = erts_build_dhandle(&hp, &BIF_P->off_heap, dep); + conn_id = dep->connection_id; + setup_connection_epiloge_rwunlock(BIF_P, dep, BIF_ARG_2, flags, version); + de_locked = 0; + + hp = HAlloc(BIF_P, 3 + ERTS_DHANDLE_SIZE); + res = erts_build_dhandle(&hp, &BIF_P->off_heap, dep, conn_id); res_tag = am_ok; /* Connection up */ - if (new) - dep = NULL; /* inc of refc transferred to port (dist_entry field) */ + dep = NULL; /* inc of refc transferred to port (dist_entry field) */ } ASSERT(is_value(res) && is_value(res_tag)); @@ -3364,12 +3336,6 @@ BIF_RETTYPE erts_internal_create_dist_channel_4(BIF_ALIST_4) return ret; - yield: - ERTS_BIF_PREP_YIELD4(ret, - bif_export[BIF_erts_internal_create_dist_channel_4], - BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, BIF_ARG_4); - goto done; - badarg: ERTS_BIF_PREP_RET(ret, am_badarg); goto done; @@ -3391,8 +3357,7 @@ setup_connection_epiloge_rwunlock(Process *c_p, DistEntry *dep, dep->creation = 0; ASSERT(is_internal_port(ctrlr) || is_internal_pid(ctrlr)); - ASSERT(erts_atomic_read_nob(&dep->qsize) == 0 - || (dep->state == ERTS_DE_STATE_PENDING)); + ASSERT(dep->state == ERTS_DE_STATE_PENDING); if (flags & DFLAG_DIST_HDR_ATOM_CACHE) create_cache(dep); @@ -3438,37 +3403,20 @@ setup_connection_distctrl(Process *c_p, void *arg, int *redsp, ErlHeapFragment * DistEntry *dep = scdcp->dep; int dep_locked = 0; Eterm *hp; - erts_aint32_t state; + Uint32 conn_id; if (redsp) *redsp = 1; - state = erts_atomic32_read_nob(&c_p->state); - - if (state & ERTS_PSFLG_EXITING) - goto badarg; + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); erts_de_rwlock(dep); dep_locked = !0; - if (dep->state == ERTS_DE_STATE_EXITING) + if (dep->state != ERTS_DE_STATE_PENDING) goto badarg; - if (ERTS_PROC_GET_DIST_ENTRY(c_p)) { - if (dep == ERTS_PROC_GET_DIST_ENTRY(c_p) - && (c_p->flags & F_DISTRIBUTION) - && dep->cid == c_p->common.id) { - goto connected; - } - goto badarg; - } - - if (dep->state != ERTS_DE_STATE_PENDING) { - if (dep->state == ERTS_DE_STATE_IDLE) - erts_set_dist_entry_pending(dep); - else - goto badarg; - } + conn_id = dep->connection_id; if (is_not_nil(dep->cid)) goto badarg; @@ -3483,18 +3431,17 @@ setup_connection_distctrl(Process *c_p, void *arg, int *redsp, ErlHeapFragment * setup_connection_epiloge_rwunlock(c_p, dep, c_p->common.id, scdcp->flags, scdcp->version); -connected: /* we take over previous inc in refc of dep */ if (!bpp) /* called directly... */ - return erts_make_dhandle(c_p, dep); + return erts_make_dhandle(c_p, dep, conn_id); erts_free(ERTS_ALC_T_SETUP_CONN_ARG, arg); - *bpp = new_message_buffer(ERTS_MAGIC_REF_THING_SIZE); + *bpp = new_message_buffer(ERTS_DHANDLE_SIZE); hp = (*bpp)->mem; - return erts_build_dhandle(&hp, &(*bpp)->off_heap, dep); + return erts_build_dhandle(&hp, &(*bpp)->off_heap, dep, conn_id); badarg: @@ -3535,34 +3482,30 @@ BIF_RETTYPE erts_internal_new_connection_1(BIF_ALIST_1) erts_de_rwlock(dep); switch (dep->state) { - case ERTS_DE_STATE_PENDING: case ERTS_DE_STATE_CONNECTED: + case ERTS_DE_STATE_EXITING: + case ERTS_DE_STATE_PENDING: conn_id = dep->connection_id; break; case ERTS_DE_STATE_IDLE: erts_set_dist_entry_pending(dep); conn_id = dep->connection_id; break; - case ERTS_DE_STATE_EXITING: - conn_id = (dep->connection_id + 1) & ERTS_DIST_CON_ID_MASK; - break; default: erts_exit(ERTS_ABORT_EXIT, "Invalid dep->state (%d)\n", dep->state); } erts_de_rwunlock(dep); - hp = HAlloc(BIF_P, 3 + ERTS_MAGIC_REF_THING_SIZE); - dhandle = erts_build_dhandle(&hp, &BIF_P->off_heap, dep); + hp = HAlloc(BIF_P, ERTS_DHANDLE_SIZE); + dhandle = erts_build_dhandle(&hp, &BIF_P->off_heap, dep, conn_id); erts_deref_dist_entry(dep); - BIF_RET(TUPLE2(hp, make_small(conn_id), dhandle)); + BIF_RET(dhandle); } -static Sint abort_connection(DistEntry* dep, Uint32 conn_id) +Sint erts_abort_connection_rwunlock(DistEntry* dep) { - erts_de_rwlock(dep); + ERTS_LC_ASSERT(erts_lc_is_de_rwlocked(dep)); - if (dep->connection_id != conn_id) - ; - else if (dep->state == ERTS_DE_STATE_CONNECTED) { + if (dep->state == ERTS_DE_STATE_CONNECTED) { kill_connection(dep); } else if (dep->state == ERTS_DE_STATE_PENDING) { @@ -3591,7 +3534,6 @@ static Sint abort_connection(DistEntry* dep, Uint32 conn_id) dep->send = NULL; erts_set_dist_entry_not_connected(dep); - erts_de_rwunlock(dep); schedule_con_monitor_link_cleanup(mld, THE_NON_VALUE, @@ -3604,42 +3546,37 @@ static Sint abort_connection(DistEntry* dep, Uint32 conn_id) delete_cache(cache); free_de_out_queues(dep, obuf); - - /* - * We wait to make DistEntry idle and accept new connection attempts - * until all is cleared and deallocated. This to get some back pressure - * against repeated failing connection attempts saturating all CPUs - * with cleanup jobs. - */ - erts_de_rwlock(dep); - ASSERT(dep->state == ERTS_DE_STATE_EXITING); - dep->state = ERTS_DE_STATE_IDLE; - erts_de_rwunlock(dep); return reds; } erts_de_rwunlock(dep); return 0; } +static Sint abort_connection(DistEntry *dep, Uint32 conn_id) +{ + erts_de_rwlock(dep); + if (dep->connection_id == conn_id) + return erts_abort_connection_rwunlock(dep); + erts_de_rwunlock(dep); + return 0; +} + BIF_RETTYPE erts_internal_abort_connection_2(BIF_ALIST_2) { DistEntry* dep; - Eterm* tp; + Uint32 conn_id; + Sint reds; - if (is_not_atom(BIF_ARG_1) || is_not_tuple_arity(BIF_ARG_2, 2)) { - BIF_ERROR(BIF_P, BADARG); - } - tp = tuple_val(BIF_ARG_2); - dep = erts_dhandle_to_dist_entry(tp[2]); - if (is_not_small(tp[1]) || dep != erts_find_dist_entry(BIF_ARG_1) + if (is_not_atom(BIF_ARG_1)) + BIF_ERROR(BIF_P, BADARG); + dep = erts_dhandle_to_dist_entry(BIF_ARG_2, &conn_id); + if (!dep || dep != erts_find_dist_entry(BIF_ARG_1) || dep == erts_this_dist_entry) { BIF_ERROR(BIF_P, BADARG); } - if (dep) { - Sint reds = abort_connection(dep, unsigned_val(tp[1])); - BUMP_REDS(BIF_P, reds); - } + reds = abort_connection(dep, conn_id); + BUMP_REDS(BIF_P, reds); BIF_RET(am_true); } @@ -3670,14 +3607,13 @@ int erts_auto_connect(DistEntry* dep, Process *proc, ErtsProcLocks proc_locks) } /* - * Send {auto_connect, Node, ConnId, DHandle} to net_kernel + * Send {auto_connect, Node, DHandle} to net_kernel */ mp = erts_alloc_message_heap(net_kernel, &nk_locks, - 5 + ERTS_MAGIC_REF_THING_SIZE, + 4 + ERTS_DHANDLE_SIZE, &hp, &ohp); - dhandle = erts_build_dhandle(&hp, ohp, dep); - msg = TUPLE4(hp, am_auto_connect, dep->sysname, make_small(conn_id), - dhandle); + dhandle = erts_build_dhandle(&hp, ohp, dep, conn_id); + msg = TUPLE3(hp, am_auto_connect, dep->sysname, dhandle); ERL_MESSAGE_TOKEN(mp) = am_undefined; erts_queue_proc_message(proc, net_kernel, nk_locks, mp, msg); erts_proc_unlock(net_kernel, nk_locks); @@ -3943,28 +3879,22 @@ monitor_node(Process* p, Eterm Node, Eterm Bool, Eterm Options) Node); mdep = (ErtsMonitorDataExtended *) erts_monitor_to_data(mon); if (created) { -#ifdef DEBUG int inserted = -#endif erts_monitor_dist_insert(&mdep->md.target, dep->mld); - ASSERT(inserted); + ASSERT(inserted); (void)inserted; ASSERT(mdep->dist->connection_id == dep->connection_id); } else if (mdep->dist->connection_id != dep->connection_id) { ErtsMonitorDataExtended *mdep2; ErtsMonitor *mon2; -#ifdef DEBUG int inserted; -#endif mdep2 = ((ErtsMonitorDataExtended *) erts_monitor_create(ERTS_MON_TYPE_NODE, NIL, p->common.id, Node, NIL)); mon2 = &mdep2->md.origin; -#ifdef DEBUG inserted = -#endif erts_monitor_dist_insert(&mdep->md.target, dep->mld); - ASSERT(inserted); + ASSERT(inserted); (void)inserted; ASSERT(mdep2->dist->connection_id == dep->connection_id); mdep2->uptr.node_monitors = mdep->uptr.node_monitors; diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h index 55204eb83d..845fab229a 100644 --- a/erts/emulator/beam/dist.h +++ b/erts/emulator/beam/dist.h @@ -136,6 +136,7 @@ typedef struct { Eterm cid; Eterm connection_id; int no_suspend; + Uint32 flags; } ErtsDSigData; #define ERTS_DE_BUSY_LIMIT (1024*1024) @@ -235,6 +236,7 @@ retry: dsdp->cid = dep->cid; dsdp->connection_id = dep->connection_id; dsdp->no_suspend = no_suspend; + dsdp->flags = dep->flags; if (dspl == ERTS_DSP_NO_LOCK) erts_de_runlock(dep); return res; @@ -254,9 +256,9 @@ void erts_schedule_dist_command(Port *prt, DistEntry *dist_entry) ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); ASSERT((erts_atomic32_read_nob(&prt->state) & ERTS_PORT_SFLGS_DEAD) == 0); - ASSERT(prt->dist_entry); - dep = prt->dist_entry; + dep = (DistEntry*) erts_prtsd_get(prt, ERTS_PRTSD_DIST_ENTRY); + ASSERT(dep); id = prt->common.id; } else { @@ -400,5 +402,7 @@ extern void erts_kill_dist_connection(DistEntry *dep, Uint32); extern Uint erts_dist_cache_size(void); +extern Sint erts_abort_connection_rwunlock(DistEntry *dep); + #endif diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 7fada0d548..2a8e7e8858 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -2705,9 +2705,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) goto bld_instruction_counts; } -#ifdef DEBUG ASSERT(endp == hp); -#endif BIF_RET(res); #endif /* #ifndef ERTS_OPCODE_COUNTER_SUPPORT */ diff --git a/erts/emulator/beam/erl_bif_lists.c b/erts/emulator/beam/erl_bif_lists.c index 73d327da3e..395be67a90 100644 --- a/erts/emulator/beam/erl_bif_lists.c +++ b/erts/emulator/beam/erl_bif_lists.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2016. All Rights Reserved. + * Copyright Ericsson AB 1999-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -277,75 +277,121 @@ BIF_RETTYPE lists_member_2(BIF_ALIST_2) BIF_RET2(am_false, CONTEXT_REDS - max_iter/10); } -BIF_RETTYPE lists_reverse_2(BIF_ALIST_2) +static BIF_RETTYPE lists_reverse_alloc(Process *c_p, + Eterm list_in, + Eterm tail_in) { - Eterm list; - Eterm tmp_list; - Eterm result; - Eterm* hp; - Uint n; - int max_iter; - - /* - * Handle legal and illegal non-lists quickly. - */ - if (is_nil(BIF_ARG_1)) { - BIF_RET(BIF_ARG_2); - } else if (is_not_list(BIF_ARG_1)) { - error: - BIF_ERROR(BIF_P, BADARG); + static const Uint CELLS_PER_RED = 40; + + Eterm *heap_top, *heap_end; + Uint cells_left, max_cells; + Eterm list, tail; + Eterm lookahead; + + list = list_in; + tail = tail_in; + + cells_left = max_cells = CELLS_PER_RED * (1 + ERTS_BIF_REDS_LEFT(c_p)); + lookahead = list; + + while (cells_left != 0 && is_list(lookahead)) { + lookahead = CDR(list_val(lookahead)); + cells_left--; } - /* - * First use the rest of the remaning heap space. - */ - list = BIF_ARG_1; - result = BIF_ARG_2; - hp = HEAP_TOP(BIF_P); - n = HeapWordsLeft(BIF_P) / 2; - while (n != 0 && is_list(list)) { - Eterm* pair = list_val(list); - result = CONS(hp, CAR(pair), result); - list = CDR(pair); - hp += 2; - n--; + BUMP_REDS(c_p, (max_cells - cells_left) / CELLS_PER_RED); + + if (is_not_list(lookahead) && is_not_nil(lookahead)) { + BIF_ERROR(c_p, BADARG); } - HEAP_TOP(BIF_P) = hp; + + heap_top = HAlloc(c_p, 2 * (max_cells - cells_left)); + heap_end = heap_top + 2 * (max_cells - cells_left); + + while (heap_top < heap_end) { + Eterm *pair = list_val(list); + + tail = CONS(heap_top, CAR(pair), tail); + list = CDR(pair); + + ASSERT(is_list(list) || is_nil(list)); + + heap_top += 2; + } + if (is_nil(list)) { - BIF_RET(result); + BIF_RET(tail); } - /* - * Calculate length of remaining list (up to a suitable limit). - */ - max_iter = CONTEXT_REDS * 40; - n = 0; - tmp_list = list; - while (max_iter-- > 0 && is_list(tmp_list)) { - tmp_list = CDR(list_val(tmp_list)); - n++; + ASSERT(is_list(tail) && cells_left == 0); + BIF_TRAP2(bif_export[BIF_lists_reverse_2], c_p, list, tail); +} + +static BIF_RETTYPE lists_reverse_onheap(Process *c_p, + Eterm list_in, + Eterm tail_in) +{ + static const Uint CELLS_PER_RED = 60; + + Eterm *heap_top, *heap_end; + Uint cells_left, max_cells; + Eterm list, tail; + + list = list_in; + tail = tail_in; + + cells_left = max_cells = CELLS_PER_RED * (1 + ERTS_BIF_REDS_LEFT(c_p)); + + ASSERT(HEAP_LIMIT(c_p) >= HEAP_TOP(c_p) + 2); + heap_end = HEAP_LIMIT(c_p) - 2; + heap_top = HEAP_TOP(c_p); + + while (heap_top < heap_end && is_list(list)) { + Eterm *pair = list_val(list); + + tail = CONS(heap_top, CAR(pair), tail); + list = CDR(pair); + + heap_top += 2; } - if (is_not_nil(tmp_list) && is_not_list(tmp_list)) { - goto error; + + cells_left -= (heap_top - heap_end) / 2; + BUMP_REDS(c_p, (max_cells - cells_left) / CELLS_PER_RED); + HEAP_TOP(c_p) = heap_top; + + if (is_nil(list)) { + BIF_RET(tail); + } else if (is_list(list)) { + ASSERT(is_list(tail)); + + if (cells_left > CELLS_PER_RED) { + return lists_reverse_alloc(c_p, list, tail); + } + + BUMP_ALL_REDS(c_p); + BIF_TRAP2(bif_export[BIF_lists_reverse_2], c_p, list, tail); } - /* - * Now do one HAlloc() and continue reversing. - */ - hp = HAlloc(BIF_P, 2*n); - while (n != 0 && is_list(list)) { - Eterm* pair = list_val(list); - result = CONS(hp, CAR(pair), result); - list = CDR(pair); - hp += 2; - n--; + BIF_ERROR(c_p, BADARG); +} + +BIF_RETTYPE lists_reverse_2(BIF_ALIST_2) +{ + /* Handle legal and illegal non-lists quickly. */ + if (is_nil(BIF_ARG_1)) { + BIF_RET(BIF_ARG_2); + } else if (is_not_list(BIF_ARG_1)) { + BIF_ERROR(BIF_P, BADARG); } - if (is_nil(list)) { - BIF_RET(result); - } else { - BUMP_ALL_REDS(BIF_P); - BIF_TRAP2(bif_export[BIF_lists_reverse_2], BIF_P, list, result); + + /* We build the reversal on the unused part of the heap if possible to save + * us the trouble of having to figure out the list size. We fall back to + * lists_reverse_alloc when we run out of space. */ + if (HeapWordsLeft(BIF_P) > 8) { + return lists_reverse_onheap(BIF_P, BIF_ARG_1, BIF_ARG_2); } + + return lists_reverse_alloc(BIF_P, BIF_ARG_1, BIF_ARG_2); } BIF_RETTYPE diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c index bbc64eb9aa..e0b9202fe7 100644 --- a/erts/emulator/beam/erl_bif_re.c +++ b/erts/emulator/beam/erl_bif_re.c @@ -532,10 +532,7 @@ re_compile(Process* p, Eterm arg1, Eterm arg2) int options = 0; int pflags = 0; int unicode = 0; -#ifdef DEBUG int buffres; -#endif - if (parse_options(arg2,&options,NULL,&pflags,NULL,NULL,NULL,NULL) < 0) { @@ -556,12 +553,8 @@ re_compile(Process* p, Eterm arg1, Eterm arg2) BIF_ERROR(p,BADARG); } expr = erts_alloc(ERTS_ALC_T_RE_TMP_BUF, slen + 1); -#ifdef DEBUG - buffres = -#endif - erts_iolist_to_buf(arg1, expr, slen); - - ASSERT(buffres >= 0); + buffres = erts_iolist_to_buf(arg1, expr, slen); + ASSERT(buffres >= 0); (void)buffres; expr[slen]='\0'; result = erts_pcre_compile2(expr, options, &errcode, @@ -1052,9 +1045,7 @@ build_capture(Eterm capture_spec[CAPSPEC_SIZE], const pcre *code) tmpb[ap->len] = '\0'; } else { ErlDrvSizeT slen; -#ifdef DEBUG int buffres; -#endif if (erts_iolist_size(val, &slen)) { goto error; @@ -1068,11 +1059,8 @@ build_capture(Eterm capture_spec[CAPSPEC_SIZE], const pcre *code) } } -#ifdef DEBUG - buffres = -#endif - erts_iolist_to_buf(val, tmpb, slen); - ASSERT(buffres >= 0); + buffres = erts_iolist_to_buf(val, tmpb, slen); + ASSERT(buffres >= 0); (void)buffres; tmpb[slen] = '\0'; } build_one_capture(code,&ri,&sallocated,has_dupnames,tmpb); @@ -1145,9 +1133,7 @@ re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) const char *errstr = ""; int errofset = 0; int capture_count; -#ifdef DEBUG int buffres; -#endif if (pflags & PARSE_FLAG_UNICODE && (!is_binary(arg2) || !is_binary(arg1) || @@ -1161,12 +1147,8 @@ re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) expr = erts_alloc(ERTS_ALC_T_RE_TMP_BUF, slen + 1); -#ifdef DEBUG - buffres = -#endif - erts_iolist_to_buf(arg2, expr, slen); - - ASSERT(buffres >= 0); + buffres = erts_iolist_to_buf(arg2, expr, slen); + ASSERT(buffres >= 0); (void)buffres; expr[slen]='\0'; result = erts_pcre_compile2(expr, comp_options, &errcode, @@ -1317,9 +1299,7 @@ re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) restart.subject = (char *) (pb->bytes+offset); restart.flags |= RESTART_FLAG_SUBJECT_IN_BINARY; } else { -#ifdef DEBUG int buffres; -#endif handle_iolist: if (erts_iolist_size(arg1, &slength)) { erts_free(ERTS_ALC_T_RE_SUBJECT, restart.ovector); @@ -1331,11 +1311,8 @@ handle_iolist: } restart.subject = erts_alloc(ERTS_ALC_T_RE_SUBJECT, slength); -#ifdef DEBUG - buffres = -#endif - erts_iolist_to_buf(arg1, restart.subject, slength); - ASSERT(buffres >= 0); + buffres = erts_iolist_to_buf(arg1, restart.subject, slength); + ASSERT(buffres >= 0); (void)buffres; } if (pflags & PARSE_FLAG_REPORT_ERRORS) { @@ -1457,10 +1434,7 @@ re_inspect_2(BIF_ALIST_2) Eterm res; const pcre *code; byte *temp_alloc = NULL; -#ifdef DEBUG - int infores; -#endif - + int infores; if (is_not_tuple(BIF_ARG_1) || (arityval(*tuple_val(BIF_ARG_1)) != 5)) { goto error; @@ -1484,12 +1458,8 @@ re_inspect_2(BIF_ALIST_2) if (erts_pcre_fullinfo(code, NULL, PCRE_INFO_OPTIONS, &options) != 0) goto error; -#ifdef DEBUG - infores = -#endif - erts_pcre_fullinfo(code, NULL, PCRE_INFO_NAMECOUNT, &top); - - ASSERT(infores == 0); + infores = erts_pcre_fullinfo(code, NULL, PCRE_INFO_NAMECOUNT, &top); + ASSERT(infores == 0); (void)infores; if (top <= 0) { hp = HAlloc(BIF_P, 3); @@ -1497,18 +1467,10 @@ re_inspect_2(BIF_ALIST_2) erts_free_aligned_binary_bytes(temp_alloc); BIF_RET(res); } -#ifdef DEBUG - infores = -#endif - erts_pcre_fullinfo(code, NULL, PCRE_INFO_NAMEENTRYSIZE, &entrysize); - + infores = erts_pcre_fullinfo(code, NULL, PCRE_INFO_NAMEENTRYSIZE, &entrysize); ASSERT(infores == 0); -#ifdef DEBUG - infores = -#endif - erts_pcre_fullinfo(code, NULL, PCRE_INFO_NAMETABLE, &nametable); - + infores = erts_pcre_fullinfo(code, NULL, PCRE_INFO_NAMETABLE, &nametable); ASSERT(infores == 0); has_dupnames = ((options & PCRE_DUPNAMES) != 0); diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c index 3a16913473..e82c776e70 100644 --- a/erts/emulator/beam/erl_bits.c +++ b/erts/emulator/beam/erl_bits.c @@ -144,6 +144,42 @@ erts_bs_start_match_2(Process *p, Eterm Binary, Uint Max) return make_matchstate(ms); } +ErlBinMatchState *erts_bs_start_match_3(Process *p, Eterm Binary) +{ + Eterm Orig; + Uint offs; + Uint* hp; + Uint NeededSize; + ErlBinMatchState *ms; + Uint bitoffs; + Uint bitsize; + Uint total_bin_size; + ProcBin* pb; + + ASSERT(is_binary(Binary)); + total_bin_size = binary_size(Binary); + if ((total_bin_size >> (8*sizeof(Uint)-3)) != 0) { + return NULL; + } + + NeededSize = ERL_BIN_MATCHSTATE_SIZE(0); + hp = HeapOnlyAlloc(p, NeededSize); + ms = (ErlBinMatchState *) hp; + ERTS_GET_REAL_BIN(Binary, Orig, offs, bitoffs, bitsize); + pb = (ProcBin *) boxed_val(Orig); + if (pb->thing_word == HEADER_PROC_BIN && pb->flags != 0) { + erts_emasculate_writable_binary(pb); + } + + ms->thing_word = HEADER_BIN_MATCHSTATE(0); + (ms->mb).orig = Orig; + (ms->mb).base = binary_bytes(Orig); + (ms->mb).offset = 8 * offs + bitoffs; + (ms->mb).size = total_bin_size * 8 + (ms->mb).offset + bitsize; + + return ms; +} + #ifdef DEBUG # define CHECK_MATCH_BUFFER(MB) check_match_buffer(MB) diff --git a/erts/emulator/beam/erl_bits.h b/erts/emulator/beam/erl_bits.h index 7beef5cfda..50d353e1fa 100644 --- a/erts/emulator/beam/erl_bits.h +++ b/erts/emulator/beam/erl_bits.h @@ -73,12 +73,16 @@ struct erl_bits_state { typedef struct erl_bin_match_struct{ Eterm thing_word; ErlBinMatchBuffer mb; /* Present match buffer */ - Eterm save_offset[1]; /* Saved offsets */ + Eterm save_offset[1]; /* Saved offsets, only valid for contexts + * created through bs_start_match2. */ } ErlBinMatchState; -#define ERL_BIN_MATCHSTATE_SIZE(_Max) ((sizeof(ErlBinMatchState) + (_Max)*sizeof(Eterm))/sizeof(Eterm)) -#define HEADER_BIN_MATCHSTATE(_Max) _make_header(ERL_BIN_MATCHSTATE_SIZE((_Max))-1, _TAG_HEADER_BIN_MATCHSTATE) -#define HEADER_NUM_SLOTS(hdr) (header_arity(hdr)-sizeof(ErlBinMatchState)/sizeof(Eterm)+1) +#define ERL_BIN_MATCHSTATE_SIZE(_Max) \ + ((offsetof(ErlBinMatchState, save_offset) + (_Max)*sizeof(Eterm))/sizeof(Eterm)) +#define HEADER_BIN_MATCHSTATE(_Max) \ + _make_header(ERL_BIN_MATCHSTATE_SIZE((_Max)) - 1, _TAG_HEADER_BIN_MATCHSTATE) +#define HEADER_NUM_SLOTS(hdr) \ + (header_arity(hdr) - (offsetof(ErlBinMatchState, save_offset) / sizeof(Eterm)) + 1) #define make_matchstate(_Ms) make_boxed((Eterm*)(_Ms)) #define ms_matchbuffer(_Ms) &(((ErlBinMatchState*) boxed_val(_Ms))->mb) @@ -144,6 +148,7 @@ void erts_bits_destroy_state(ERL_BITS_PROTO_0); */ Eterm erts_bs_start_match_2(Process *p, Eterm Bin, Uint Max); +ErlBinMatchState *erts_bs_start_match_3(Process *p, Eterm Bin); Eterm erts_bs_get_integer_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuffer* mb); Eterm erts_bs_get_binary_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuffer* mb); Eterm erts_bs_get_float_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuffer* mb); diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index f1d47326b4..a78623f490 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -3118,9 +3118,7 @@ void* db_store_term_comp(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj) Uint new_sz = offset + db_size_dbterm_comp(tb, obj); byte* basep; DbTerm* newp; -#ifdef DEBUG byte* top; -#endif ASSERT(tb->compress); if (old != 0) { @@ -3142,11 +3140,8 @@ void* db_store_term_comp(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj) } newp->size = size_object(obj); -#ifdef DEBUG - top = -#endif - copy_to_comp(tb, obj, newp, new_sz); - ASSERT(top <= basep + new_sz); + top = copy_to_comp(tb, obj, newp, new_sz); + ASSERT(top <= basep + new_sz); (void)top; /* ToDo: Maybe realloc if ((basep+new_sz) - top) > WASTED_SPACE_LIMIT */ diff --git a/erts/emulator/beam/erl_drv_nif.h b/erts/emulator/beam/erl_drv_nif.h index 31b4817fb1..9ef7c39d41 100644 --- a/erts/emulator/beam/erl_drv_nif.h +++ b/erts/emulator/beam/erl_drv_nif.h @@ -53,7 +53,8 @@ typedef enum { enum ErlNifSelectFlags { ERL_NIF_SELECT_READ = (1 << 0), ERL_NIF_SELECT_WRITE = (1 << 1), - ERL_NIF_SELECT_STOP = (1 << 2) + ERL_NIF_SELECT_STOP = (1 << 2), + ERL_NIF_SELECT_CANCEL = (1 << 3) }; /* diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 5da7b43b9e..9ba3ae226e 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -163,7 +163,6 @@ int erts_initialized = 0; * Configurable parameters. */ -Uint display_items; /* no of items to display in traces etc */ int H_MIN_SIZE; /* The minimum heap grain */ int BIN_VH_MIN_SIZE; /* The minimum binary virtual*/ int H_MAX_SIZE; /* The maximum heap size */ @@ -375,21 +374,19 @@ erl_init(int ncpu, } static Eterm -erl_first_process_otp(char* modname, void* code, unsigned size, int argc, char** argv) +erl_first_process_otp(char* mod_name, int argc, char** argv) { int i; - Eterm start_mod; Eterm args; Eterm res; Eterm* hp; Process parent; ErlSpawnOpts so; - Eterm env; - - start_mod = erts_atom_put((byte *) modname, sys_strlen(modname), ERTS_ATOM_ENC_LATIN1, 1); - if (erts_find_function(start_mod, am_start, 2, + Eterm boot_mod; + + if (erts_find_function(am_erl_init, am_start, 2, erts_active_code_ix()) == NULL) { - erts_exit(ERTS_ERROR_EXIT, "No function %s:start/2\n", modname); + erts_exit(ERTS_ERROR_EXIT, "No function erl_init:start/2\n"); } /* @@ -405,13 +402,13 @@ erl_first_process_otp(char* modname, void* code, unsigned size, int argc, char** args = CONS(hp, new_binary(&parent, (byte*)argv[i], len), args); hp += 2; } - env = new_binary(&parent, code, size); + boot_mod = erts_atom_put((byte *) mod_name, sys_strlen(mod_name), ERTS_ATOM_ENC_LATIN1, 1); args = CONS(hp, args, NIL); hp += 2; - args = CONS(hp, env, args); + args = CONS(hp, boot_mod, args); so.flags = erts_default_spo_flags|SPO_SYSTEM_PROC; - res = erl_create_process(&parent, start_mod, am_start, args, &so); + res = erl_create_process(&parent, am_erl_init, am_start, args, &so); erts_proc_unlock(&parent, ERTS_PROC_LOCK_MAIN); erts_cleanup_empty_process(&parent); return res; @@ -481,7 +478,6 @@ erts_preloaded(Process* p) /* static variables that must not change (use same values at restart) */ static char* program; static char* init = "init"; -static char* boot = "boot"; static int boot_argc; static char** boot_argv; @@ -535,9 +531,6 @@ void erts_usage(void) int this_rel = this_rel_num(); erts_fprintf(stderr, "Usage: %s [flags] [ -- [init_args] ]\n", progname(program)); erts_fprintf(stderr, "The flags are:\n\n"); - - /* erts_fprintf(stderr, "-# number set the number of items to be used in traces etc\n"); */ - erts_fprintf(stderr, "-a size suggested stack size in kilo words for threads\n"); erts_fprintf(stderr, " in the async-thread pool, valid range is [%d-%d]\n", ERTS_ASYNC_THREAD_MIN_STACK_SIZE, @@ -545,13 +538,9 @@ void erts_usage(void) erts_fprintf(stderr, "-A number set number of threads in async thread pool,\n"); erts_fprintf(stderr, " valid range is [0-%d]\n", ERTS_MAX_NO_OF_ASYNC_THREADS); - erts_fprintf(stderr, "-B[c|d|i] c to have Ctrl-c interrupt the Erlang shell,\n"); erts_fprintf(stderr, " d (or no extra option) to disable the break\n"); erts_fprintf(stderr, " handler, i to ignore break signals\n"); - - /* erts_fprintf(stderr, "-b func set the boot function (default boot)\n"); */ - erts_fprintf(stderr, "-c bool enable or disable time correction\n"); erts_fprintf(stderr, "-C mode set time warp mode; valid modes are:\n"); erts_fprintf(stderr, " no_time_warp|single_time_warp|multi_time_warp\n"); @@ -570,7 +559,6 @@ void erts_usage(void) erts_pd_initial_size); erts_fprintf(stderr, "-hmqd val set default message queue data flag for processes,\n"); erts_fprintf(stderr, " valid values are: off_heap | on_heap\n"); - erts_fprintf(stderr, "-IOp number set number of pollsets to be used to poll for I/O,\n"); erts_fprintf(stderr, " This value has to be equal or smaller than the\n"); erts_fprintf(stderr, " number of poll threads. If the current platform\n"); @@ -581,9 +569,7 @@ void erts_usage(void) erts_fprintf(stderr, " number of poll threads."); erts_fprintf(stderr, "-IOPt number set number of threads to be used to poll for I/O\n"); erts_fprintf(stderr, " as a percentage of the number of schedulers."); - - /* erts_fprintf(stderr, "-i module set the boot module (default init)\n"); */ - + erts_fprintf(stderr, "-i module set the boot module (default init)\n"); erts_fprintf(stderr, "-n[s|a|d] Control behavior of signals to ports\n"); erts_fprintf(stderr, " Note that this flag is deprecated!\n"); erts_fprintf(stderr, "-M<X> <Y> memory allocator switches,\n"); @@ -598,7 +584,6 @@ void erts_usage(void) erts_fprintf(stderr, "-R number set compatibility release number,\n"); erts_fprintf(stderr, " valid range [%d-%d]\n", this_rel-2, this_rel); - erts_fprintf(stderr, "-r force ets memory block to be moved on realloc\n"); erts_fprintf(stderr, "-rg amount set reader groups limit\n"); erts_fprintf(stderr, "-sbt type set scheduler bind type, valid types are:\n"); @@ -669,9 +654,7 @@ void erts_usage(void) erts_fprintf(stderr, "-T number set modified timing level, valid range is [0-%d]\n", ERTS_MODIFIED_TIMING_LEVELS-1); erts_fprintf(stderr, "-V print Erlang version\n"); - erts_fprintf(stderr, "-v turn on chatty mode (GCs will be reported etc)\n"); - erts_fprintf(stderr, "-W<i|w|e> set error logger warnings mapping,\n"); erts_fprintf(stderr, " see error_logger documentation for details\n"); erts_fprintf(stderr, "-zdbbl size set the distribution buffer busy limit in kilobytes\n"); @@ -762,7 +745,6 @@ early_init(int *argc, char **argv) /* erts_sched_compact_load = 1; erts_printf_eterm_func = erts_printf_term; - display_items = 200; erts_backtrace_depth = DEFAULT_BACKTRACE_SIZE; erts_async_max_threads = ERTS_DEFAULT_NO_ASYNC_THREADS; erts_async_thread_suggested_stack_size = ERTS_ASYNC_THREAD_MIN_STACK_SIZE; @@ -1269,25 +1251,9 @@ erl_start(int argc, char **argv) /* * NOTE: -M flags are handled (and removed from argv) by - * erts_alloc_init(). - * - * The -d, -m, -S, -t, and -T flags was removed in - * Erlang 5.3/OTP R9C. - * - * -S, and -T has been reused in Erlang 5.5/OTP R11B. - * - * -d has been reused in a patch R12B-4. + * erts_alloc_init(). */ - case '#' : - arg = get_arg(argv[i]+2, argv[i+1], &i); - if ((display_items = atoi(arg)) == 0) { - erts_fprintf(stderr, "bad display items%s\n", arg); - erts_usage(); - } - VERBOSE(DEBUG_SYSTEM, - ("using display items %d\n",display_items)); - break; case 'p': if (!sys_strncmp(argv[i],"-pc",3)) { int printable_chars = ERL_PRINTABLE_CHARACTERS_LATIN1; @@ -1566,11 +1532,6 @@ erl_start(int argc, char **argv) init = get_arg(argv[i]+2, argv[i+1], &i); break; - case 'b': - /* define name of initial function */ - boot = get_arg(argv[i]+2, argv[i+1], &i); - break; - case 'B': if (argv[i][2] == 'i') /* +Bi */ ignore_break = 1; @@ -2256,9 +2217,8 @@ erl_start(int argc, char **argv) erts_initialized = 1; - erts_init_process_id = erl_first_process_otp("otp_ring0", NULL, 0, - boot_argc, boot_argv); - ASSERT(erts_init_process_id != ERTS_INVALID_PID); + erts_init_process_id = erl_first_process_otp(init, boot_argc, boot_argv); + ASSERT(erts_init_process_id != ERTS_INVALID_PID); { /* diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index 4c09496ef1..c5227a0c23 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -160,6 +160,8 @@ typedef int ErlNifEvent; #define ERL_NIF_SELECT_STOP_SCHEDULED (1 << 1) #define ERL_NIF_SELECT_INVALID_EVENT (1 << 2) #define ERL_NIF_SELECT_FAILED (1 << 3) +#define ERL_NIF_SELECT_READ_CANCELLED (1 << 4) +#define ERL_NIF_SELECT_WRITE_CANCELLED (1 << 5) typedef enum { diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index 1f147011a8..f4dc60941a 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -60,6 +60,10 @@ static int references_atoms_need_init = 1; static ErtsMonotonicTime orig_node_tab_delete_delay; static ErtsMonotonicTime node_tab_delete_delay; + +static void report_gc_active_dist_entry(Eterm sysname, enum dist_entry_state); + + /* -- The distribution table ---------------------------------------------- */ #define ErtsBin2DistEntry(B) \ @@ -366,31 +370,43 @@ DistEntry *erts_find_dist_entry(Eterm sysname) } DistEntry * -erts_dhandle_to_dist_entry(Eterm dhandle) +erts_dhandle_to_dist_entry(Eterm dhandle, Uint32 *conn_id) { + Eterm *tpl; Binary *bin; - if (!is_internal_magic_ref(dhandle)) + + if (!is_boxed(dhandle)) + return NULL; + tpl = boxed_val(dhandle); + if (tpl[0] != make_arityval(2) || !is_small(tpl[1]) + || !is_internal_magic_ref(tpl[2])) return NULL; - bin = erts_magic_ref2bin(dhandle); + *conn_id = unsigned_val(tpl[1]); + bin = erts_magic_ref2bin(tpl[2]); if (ERTS_MAGIC_BIN_DESTRUCTOR(bin) != erts_dist_entry_destructor) return NULL; return ErtsBin2DistEntry(bin); } Eterm -erts_build_dhandle(Eterm **hpp, ErlOffHeap* ohp, DistEntry *dep) +erts_build_dhandle(Eterm **hpp, ErlOffHeap* ohp, + DistEntry *dep, Uint32 conn_id) { Binary *bin = ErtsDistEntry2Bin(dep); + Eterm mref, dhandle; ASSERT(bin); ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == erts_dist_entry_destructor); - return erts_mk_magic_ref(hpp, ohp, bin); + mref = erts_mk_magic_ref(hpp, ohp, bin); + dhandle = TUPLE2(*hpp, make_small(conn_id), mref); + *hpp += 3; + return dhandle; } Eterm -erts_make_dhandle(Process *c_p, DistEntry *dep) +erts_make_dhandle(Process *c_p, DistEntry *dep, Uint32 conn_id) { - Eterm *hp = HAlloc(c_p, ERTS_MAGIC_REF_THING_SIZE); - return erts_build_dhandle(&hp, &c_p->off_heap, dep); + Eterm *hp = HAlloc(c_p, ERTS_DHANDLE_SIZE); + return erts_build_dhandle(&hp, &c_p->off_heap, dep, conn_id); } static void start_timer_delete_dist_entry(void *vdep); @@ -451,6 +467,19 @@ static void try_delete_dist_entry(DistEntry* dep) { erts_aint_t refc; + erts_de_rwlock(dep); + if (dep->state != ERTS_DE_STATE_IDLE && de_refc_read(dep,0) == 0) { + Eterm sysname = dep->sysname; + enum dist_entry_state state = dep->state; + + if (dep->state != ERTS_DE_STATE_PENDING) + ERTS_INTERNAL_ERROR("Garbage collecting connected distribution entry"); + erts_abort_connection_rwunlock(dep); + report_gc_active_dist_entry(sysname, state); + } + else + erts_de_rwunlock(dep); + erts_rwmtx_rwlock(&erts_dist_table_rwmtx); /* * Another thread might have looked up this dist entry after @@ -477,6 +506,34 @@ static void try_delete_dist_entry(DistEntry* dep) } } +static void report_gc_active_dist_entry(Eterm sysname, + enum dist_entry_state state) +{ + char *state_str; + erts_dsprintf_buf_t *dsbuf = erts_create_logger_dsbuf(); + switch (state) { + case ERTS_DE_STATE_CONNECTED: + state_str = "connected"; + break; + case ERTS_DE_STATE_PENDING: + state_str = "pending connect"; + break; + case ERTS_DE_STATE_EXITING: + state_str = "exiting"; + break; + case ERTS_DE_STATE_IDLE: + state_str = "idle"; + break; + default: + state_str = "unknown"; + break; + } + erts_dsprintf(dsbuf, "Garbage collecting distribution " + "entry for node %T in state: %s", + sysname, state_str); + erts_send_error_to_logger_nogl(dsbuf); +} + int erts_dist_entry_destructor(Binary *bin) { DistEntry *dep = ErtsBin2DistEntry(bin); @@ -582,7 +639,7 @@ erts_set_dist_entry_not_connected(DistEntry *dep) if(dep->next) dep->next->prev = dep->prev; - dep->state = ERTS_DE_STATE_EXITING; + dep->state = ERTS_DE_STATE_IDLE; dep->flags = 0; dep->prev = NULL; dep->cid = NIL; @@ -1863,8 +1920,9 @@ setup_reference_table(void) if (ohp) insert_offheap(ohp, HEAP_REF, prt->common.id); /* Insert controller */ - if (prt->dist_entry) - insert_dist_entry(prt->dist_entry, + dep = (DistEntry*) erts_prtsd_get(prt, ERTS_PRTSD_DIST_ENTRY); + if (dep) + insert_dist_entry(dep, CTRL_REF, prt->common.id, 0); diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h index 9a792b10b1..c44f1f8991 100644 --- a/erts/emulator/beam/erl_node_tables.h +++ b/erts/emulator/beam/erl_node_tables.h @@ -214,9 +214,10 @@ int erts_lc_is_de_rwlocked(DistEntry *); int erts_lc_is_de_rlocked(DistEntry *); #endif int erts_dist_entry_destructor(Binary *bin); -DistEntry *erts_dhandle_to_dist_entry(Eterm dhandle); -Eterm erts_build_dhandle(Eterm **hpp, ErlOffHeap*, DistEntry*); -Eterm erts_make_dhandle(Process *c_p, DistEntry *dep); +DistEntry *erts_dhandle_to_dist_entry(Eterm dhandle, Uint32* connection_id); +#define ERTS_DHANDLE_SIZE (3+ERTS_MAGIC_REF_THING_SIZE) +Eterm erts_build_dhandle(Eterm **hpp, ErlOffHeap*, DistEntry*, Uint32 conn_id); +Eterm erts_make_dhandle(Process *c_p, DistEntry*, Uint32 conn_id); ERTS_GLB_INLINE void erts_deref_node_entry(ErlNode *np); ERTS_GLB_INLINE void erts_de_rlock(DistEntry *dep); diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h index 9b52b648e5..2be0a5bf74 100644 --- a/erts/emulator/beam/erl_port.h +++ b/erts/emulator/beam/erl_port.h @@ -112,8 +112,10 @@ typedef struct line_buf { /* Buffer used in line oriented I/O */ */ #define ERTS_PRTSD_SCHED_ID 0 +#define ERTS_PRTSD_DIST_ENTRY 1 +#define ERTS_PRTSD_CONN_ID 2 -#define ERTS_PRTSD_SIZE 1 +#define ERTS_PRTSD_SIZE 3 typedef struct { void *data[ERTS_PRTSD_SIZE]; @@ -154,7 +156,6 @@ struct _erl_drv_port { Uint bytes_out; /* Number of bytes written */ ErlPortIOQueue ioq; /* driver accessible i/o queue */ - DistEntry *dist_entry; /* Dist entry used in DISTRIBUTION */ char *name; /* String used in the open */ erts_driver_t* drv_ptr; UWord drv_data; @@ -257,6 +258,8 @@ ERTS_GLB_INLINE void * erts_prtsd_get(Port *prt, int ix) { ErtsPrtSD *psd = (ErtsPrtSD *) erts_atomic_read_nob(&prt->psd); + + ASSERT((unsigned)ix < ERTS_PRTSD_SIZE); if (!psd) return NULL; ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER; @@ -272,6 +275,7 @@ erts_prtsd_set(Port *prt, int ix, void *data) psd = (ErtsPrtSD *) erts_atomic_read_nob(&prt->psd); + ASSERT((unsigned)ix < ERTS_PRTSD_SIZE); if (psd) { #ifdef ETHR_ORDERED_READ_DEPEND ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); @@ -459,7 +463,7 @@ erts_port_unlock(Port *prt) ERTS_INVALID_PORT_OPT((PP), (ID), ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP) #define ERTS_PORT_SCHED_ID(P, ID) \ - ((Uint) (UWord) erts_prtsd_set((P), ERTS_PSD_SCHED_ID, (void *) (UWord) (ID))) + ((Uint) (UWord) erts_prtsd_set((P), ERTS_PRTSD_SCHED_ID, (void *) (UWord) (ID))) extern const Port erts_invalid_port; #define ERTS_PORT_LOCK_BUSY ((Port *) &erts_invalid_port) diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 9f3dfd6c37..a21acb9a57 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -4022,9 +4022,7 @@ schedule_bound_processes(ErtsRunQueue *rq, static ERTS_INLINE void clear_proc_dirty_queue_bit(Process *p, ErtsRunQueue *rq, int prio_bit) { -#ifdef DEBUG erts_aint32_t old; -#endif erts_aint32_t qb = prio_bit; if (rq == ERTS_DIRTY_CPU_RUNQ) qb <<= ERTS_PDSFLGS_IN_CPU_PRQ_MASK_OFFSET; @@ -4032,13 +4030,8 @@ clear_proc_dirty_queue_bit(Process *p, ErtsRunQueue *rq, int prio_bit) ASSERT(rq == ERTS_DIRTY_IO_RUNQ); qb <<= ERTS_PDSFLGS_IN_IO_PRQ_MASK_OFFSET; } -#ifdef DEBUG - old = (int) -#else - (void) -#endif - erts_atomic32_read_band_mb(&p->dirty_state, ~qb); - ASSERT(old & qb); + old = (int) erts_atomic32_read_band_mb(&p->dirty_state, ~qb); + ASSERT(old & qb); (void)old; } @@ -7175,9 +7168,7 @@ msb_scheduler_type_switch(ErtsSchedType sched_type, Uint32 nrml_prio, dcpu_prio, dio_prio; ErtsSchedType exec_type; ErtsRunQueue *exec_rq; -#ifdef DEBUG erts_aint32_t dbg_val; -#endif ASSERT(schdlr_sspnd.msb.ongoing); @@ -7292,16 +7283,12 @@ msb_scheduler_type_switch(ErtsSchedType sched_type, * Suspend this scheduler and wake up scheduler * number one of another type... */ -#ifdef DEBUG dbg_val = -#else - (void) -#endif erts_atomic32_read_bset_mb(&esdp->ssi->flags, (ERTS_SSI_FLG_SUSPENDED | ERTS_SSI_FLG_MSB_EXEC), ERTS_SSI_FLG_SUSPENDED); - ASSERT(dbg_val & ERTS_SSI_FLG_MSB_EXEC); + ASSERT(dbg_val & ERTS_SSI_FLG_MSB_EXEC); (void)dbg_val; switch (exec_type) { case ERTS_SCHED_NORMAL: @@ -7319,11 +7306,7 @@ msb_scheduler_type_switch(ErtsSchedType sched_type, break; } -#ifdef DEBUG dbg_val = -#else - (void) -#endif erts_atomic32_read_bset_mb(&exec_rq->scheduler->ssi->flags, (ERTS_SSI_FLG_SUSPENDED | ERTS_SSI_FLG_MSB_EXEC), @@ -8888,11 +8871,8 @@ erts_suspend(Process* c_p, ErtsProcLocks c_p_locks, Port *busy_port) suspend = 1; if (suspend) { -#ifdef DEBUG - int res = -#endif - suspend_process(c_p, c_p); - ASSERT(res); + int res = suspend_process(c_p, c_p); + ASSERT(res); (void)res; } if (!(c_p_locks & ERTS_PROC_LOCK_STATUS)) @@ -9641,7 +9621,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) if (state & ERTS_PSFLG_RUNNING_SYS) { if (state & (ERTS_PSFLG_SIG_Q|ERTS_PSFLG_SIG_IN_Q)) { int local_only = (!!(p->flags & F_LOCAL_SIGS_ONLY) - & !(state & ERTS_PSFLG_SUSPENDED)); + & !(state & (ERTS_PSFLG_SUSPENDED|ERTS_PSFLGS_DIRTY_WORK))); if (!local_only | !!(state & ERTS_PSFLG_SIG_Q)) { int sig_reds; /* @@ -12490,9 +12470,7 @@ erts_continue_exit_process(Process *p) yield: -#ifdef DEBUG ASSERT(yield_allowed); -#endif ERTS_LC_ASSERT(curr_locks == erts_proc_lc_my_proc_locks(p)); ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN & curr_locks); diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c index d225916ac5..1d6869a7cd 100644 --- a/erts/emulator/beam/erl_unicode.c +++ b/erts/emulator/beam/erl_unicode.c @@ -1358,11 +1358,9 @@ Uint erts_atom_to_string_length(Eterm atom) else { byte* err_pos; Uint num_chars; -#ifdef DEBUG int ares = -#endif erts_analyze_utf8(ap->name, ap->len, &err_pos, &num_chars, NULL); - ASSERT(ares == ERTS_UTF8_OK); + ASSERT(ares == ERTS_UTF8_OK); (void)ares; return num_chars; } diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 904993ceb6..621ba108ba 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -671,30 +671,37 @@ erts_prepare_dist_ext(ErtsDistExternal *edep, byte *ext, Uint size, DistEntry *dep, - ErtsAtomCache *cache, - Uint32 *connection_id) + Uint32 conn_id, + ErtsAtomCache *cache) { -#undef ERTS_EXT_FAIL -#undef ERTS_EXT_HDR_FAIL -#if 1 -#define ERTS_EXT_FAIL goto fail -#define ERTS_EXT_HDR_FAIL goto bad_hdr -#else -#define ERTS_EXT_FAIL abort() -#define ERTS_EXT_HDR_FAIL abort() -#endif + register byte *ep; + + edep->heap_size = -1; + edep->flags = 0; + edep->dep = dep; + + ASSERT(dep); + erts_de_rlock(dep); - register byte *ep = ext; ASSERT(dep->flags & DFLAG_UTF8_ATOMS); - edep->heap_size = -1; - edep->ext_endp = ext+size; + if ((dep->state != ERTS_DE_STATE_CONNECTED && + dep->state != ERTS_DE_STATE_PENDING) + || dep->connection_id != conn_id) { + erts_de_runlock(dep); + return ERTS_PREP_DIST_EXT_CLOSED; + } - if (size < 2) - ERTS_EXT_FAIL; + if (!(dep->flags & DFLAG_DIST_HDR_ATOM_CACHE)) { + /* Skip PASS_THROUGH */ + ext++; + size--; + } + edep->ext_endp = ext + size; + ep = ext; - if (!dep) - ERTS_INTERNAL_ERROR("Invalid use"); + if (size < 2) + goto fail; if (ep[0] != VERSION_MAGIC) { erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); @@ -703,28 +710,17 @@ erts_prepare_dist_ext(ErtsDistExternal *edep, "channel %d\n", dist_entry_channel_no(dep)); erts_send_error_to_logger_nogl(dsbufp); - ERTS_EXT_FAIL; + goto fail; } - edep->flags = 0; - edep->dep = dep; - - erts_de_rlock(dep); - - if (dep->state != ERTS_DE_STATE_CONNECTED && - dep->state != ERTS_DE_STATE_PENDING) { - erts_de_runlock(dep); - return ERTS_PREP_DIST_EXT_CLOSED; - } if (dep->flags & DFLAG_DIST_HDR_ATOM_CACHE) edep->flags |= ERTS_DIST_EXT_DFLAG_HDR; - *connection_id = dep->connection_id; edep->connection_id = dep->connection_id; if (ep[1] != DIST_HEADER) { if (edep->flags & ERTS_DIST_EXT_DFLAG_HDR) - ERTS_EXT_HDR_FAIL; + goto bad_hdr; edep->attab.size = 0; edep->extp = ext; } @@ -733,17 +729,17 @@ erts_prepare_dist_ext(ErtsDistExternal *edep, int no_atoms; if (!(edep->flags & ERTS_DIST_EXT_DFLAG_HDR)) - ERTS_EXT_HDR_FAIL; + goto bad_hdr; #undef CHKSIZE #define CHKSIZE(SZ) \ - do { if ((SZ) > edep->ext_endp - ep) ERTS_EXT_HDR_FAIL; } while(0) + do { if ((SZ) > edep->ext_endp - ep) goto bad_hdr; } while(0) CHKSIZE(1+1+1); ep += 2; no_atoms = (int) get_int8(ep); if (no_atoms < 0 || ERTS_ATOM_CACHE_SIZE < no_atoms) - ERTS_EXT_HDR_FAIL; + goto bad_hdr; ep++; if (no_atoms) { int long_atoms = 0; @@ -821,18 +817,18 @@ erts_prepare_dist_ext(ErtsDistExternal *edep, /* atom already cached */ cix += (int) get_int8(ep); if (cix >= ERTS_ATOM_CACHE_SIZE) - ERTS_EXT_HDR_FAIL; + goto bad_hdr; ep++; atom = cache->in_arr[cix]; if (!is_atom(atom)) - ERTS_EXT_HDR_FAIL; + goto bad_hdr; edep->attab.atom[tix] = atom; } else { /* new cached atom */ cix += (int) get_int8(ep); if (cix >= ERTS_ATOM_CACHE_SIZE) - ERTS_EXT_HDR_FAIL; + goto bad_hdr; ep++; if (long_atoms) { CHKSIZE(2); @@ -850,7 +846,7 @@ erts_prepare_dist_ext(ErtsDistExternal *edep, ERTS_ATOM_ENC_UTF8, 0); if (is_non_value(atom)) - ERTS_EXT_HDR_FAIL; + goto bad_hdr; ep += len; cache->in_arr[cix] = atom; edep->attab.atom[tix] = atom; @@ -870,12 +866,12 @@ erts_prepare_dist_ext(ErtsDistExternal *edep, edep->extp = ep; #ifdef ERTS_DEBUG_USE_DIST_SEP if (*ep != VERSION_MAGIC) - ERTS_EXT_HDR_FAIL; + goto bad_hdr; #endif } #ifdef ERTS_DEBUG_USE_DIST_SEP if (*ep != VERSION_MAGIC) - ERTS_EXT_FAIL; + goto fail; #endif erts_de_runlock(dep); @@ -883,8 +879,6 @@ erts_prepare_dist_ext(ErtsDistExternal *edep, return ERTS_PREP_DIST_EXT_SUCCESS; #undef CHKSIZE -#undef ERTS_EXT_FAIL -#undef ERTS_EXT_HDR_FAIL bad_hdr: { erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); @@ -901,7 +895,7 @@ erts_prepare_dist_ext(ErtsDistExternal *edep, } fail: { erts_de_runlock(dep); - erts_kill_dist_connection(dep, *connection_id); + erts_kill_dist_connection(dep, conn_id); } return ERTS_PREP_DIST_EXT_FAILED; } diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h index bbd9b4bad2..edac177cc6 100644 --- a/erts/emulator/beam/external.h +++ b/erts/emulator/beam/external.h @@ -182,7 +182,7 @@ void erts_destroy_dist_ext_copy(ErtsDistExternal *); #define ERTS_PREP_DIST_EXT_CLOSED (1) int erts_prepare_dist_ext(ErtsDistExternal *, byte *, Uint, - DistEntry *, ErtsAtomCache *, Uint32 *); + DistEntry *, Uint32 conn_id, ErtsAtomCache *); Sint erts_decode_dist_ext_size(ErtsDistExternal *); Eterm erts_decode_dist_ext(ErtsHeapFactory* factory, ErtsDistExternal *); diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 2cf268162d..187d2635ea 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -292,7 +292,6 @@ union erl_off_heap_ptr { /* controls warning mapping in error_logger */ extern Eterm node_cookie; -extern Uint display_items; /* no of items to display in traces etc */ extern int erts_backtrace_depth; extern erts_atomic32_t erts_max_gen_gcs; @@ -1083,7 +1082,7 @@ extern int erts_do_net_exits(DistEntry*, Eterm); extern int distribution_info(fmtfn_t, void *); extern int is_node_name_atom(Eterm a); -extern int erts_net_message(Port *, DistEntry *, +extern int erts_net_message(Port *, DistEntry *, Uint32 conn_id, byte *, ErlDrvSizeT, byte *, ErlDrvSizeT); extern void init_dist(void); diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 133ab485d9..5325480901 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -375,7 +375,6 @@ static Port *create_port(char *name, prt->control_flags = 0; prt->bytes_in = 0; prt->bytes_out = 0; - prt->dist_entry = NULL; ERTS_PORT_INIT_CONNECTED(prt, pid); prt->common.u.alive.reg = NULL; ERTS_PTMR_INIT(prt); @@ -3613,12 +3612,12 @@ terminate_port(Port *prt) erts_cleanup_port_data(prt); + ASSERT(erts_prtsd_get(prt, ERTS_PRTSD_DIST_ENTRY) == NULL); + psd = (ErtsPrtSD *) erts_atomic_read_nob(&prt->psd); if (psd) erts_free(ERTS_ALC_T_PRTSD, psd); - ASSERT(prt->dist_entry == NULL); - kill_port(prt); /* @@ -3759,10 +3758,12 @@ erts_deliver_port_exit(Port *prt, Eterm from, Eterm reason, int send_closed, DRV_MONITOR_UNLOCK_PDL(prt); } - if ((state & ERTS_PORT_SFLG_DISTRIBUTION) && prt->dist_entry) { - erts_do_net_exits(prt->dist_entry, modified_reason); - erts_deref_dist_entry(prt->dist_entry); - prt->dist_entry = NULL; + if (state & ERTS_PORT_SFLG_DISTRIBUTION) { + DistEntry *dep = (DistEntry*) erts_prtsd_get(prt, ERTS_PRTSD_DIST_ENTRY); + ASSERT(dep); + erts_do_net_exits(dep, modified_reason); + erts_deref_dist_entry(dep); + erts_prtsd_set(prt, ERTS_PRTSD_DIST_ENTRY, NULL); erts_atomic32_read_band_relb(&prt->state, ~ERTS_PORT_SFLG_DISTRIBUTION); } @@ -5050,7 +5051,7 @@ set_busy_port(ErlDrvPort dprt, int on) DTRACE1(port_not_busy, port_str); } #endif - if (prt->dist_entry) { + if (erts_prtsd_get(prt, ERTS_PRTSD_DIST_ENTRY) != NULL) { /* * Processes suspended on distribution ports are * normally queued on the dist entry. @@ -6169,9 +6170,12 @@ int driver_output_binary(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, else erts_atomic64_add_nob(&bytes_in, (erts_aint64_t) (hlen + len)); if (state & ERTS_PORT_SFLG_DISTRIBUTION) { - erts_atomic64_inc_nob(&prt->dist_entry->in); + DistEntry* dep = (DistEntry*) erts_prtsd_get(prt, ERTS_PRTSD_DIST_ENTRY); + Uint32 conn_id = (Uint32)(UWord) erts_prtsd_get(prt, ERTS_PRTSD_CONN_ID); + erts_atomic64_inc_nob(&dep->in); return erts_net_message(prt, - prt->dist_entry, + dep, + conn_id, (byte*) hbuf, hlen, (byte*) (bin->orig_bytes+offs), len); } @@ -6210,15 +6214,19 @@ int driver_output2(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, else erts_atomic64_add_nob(&bytes_in, (erts_aint64_t) (hlen + len)); if (state & ERTS_PORT_SFLG_DISTRIBUTION) { - erts_atomic64_inc_nob(&prt->dist_entry->in); + DistEntry *dep = (DistEntry*) erts_prtsd_get(prt, ERTS_PRTSD_DIST_ENTRY); + Uint32 conn_id = (Uint32)(UWord) erts_prtsd_get(prt, ERTS_PRTSD_CONN_ID); + erts_atomic64_inc_nob(&dep->in); if (len == 0) return erts_net_message(prt, - prt->dist_entry, + dep, + conn_id, NULL, 0, (byte*) hbuf, hlen); else return erts_net_message(prt, - prt->dist_entry, + dep, + conn_id, (byte*) hbuf, hlen, (byte*) buf, len); } diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab index d859c4bb24..349034e8ac 100644 --- a/erts/emulator/beam/ops.tab +++ b/erts/emulator/beam/ops.tab @@ -1089,21 +1089,21 @@ func_info M F A => i_func_info u M F A %warm bs_start_match2 Fail=f ica X Y D => jump Fail bs_start_match2 Fail Bin X Y D => i_bs_start_match2 Bin Fail X Y D -i_bs_start_match2 xy f t t x +i_bs_start_match2 xy f t t d bs_save2 Reg Index => gen_bs_save(Reg, Index) -i_bs_save2 x t +i_bs_save2 xy t bs_restore2 Reg Index => gen_bs_restore(Reg, Index) -i_bs_restore2 x t +i_bs_restore2 xy t # Matching integers bs_match_string Fail Ms Bits Val => i_bs_match_string Ms Fail Bits Val -i_bs_match_string x f W W +i_bs_match_string xy f W W # Fetching integers from binaries. -bs_get_integer2 Fail=f Ms=x Live=u Sz=sq Unit=u Flags=u Dst=d => \ +bs_get_integer2 Fail=f Ms=xy Live=u Sz=sq Unit=u Flags=u Dst=d => \ gen_get_integer2(Fail, Ms, Live, Sz, Unit, Flags, Dst) i_bs_get_integer_small_imm Ms Bits Fail Flags Y=y => \ @@ -1112,50 +1112,50 @@ i_bs_get_integer_small_imm Ms Bits Fail Flags Y=y => \ i_bs_get_integer_imm Ms Bits Live Fail Flags Y=y => \ i_bs_get_integer_imm Ms Bits Live Fail Flags x | move x Y -i_bs_get_integer_small_imm x W f? t x -i_bs_get_integer_imm x W t f? t x -i_bs_get_integer f? t t x s xy -i_bs_get_integer_8 x f? xy -i_bs_get_integer_16 x f? xy +i_bs_get_integer_small_imm xy W f? t x +i_bs_get_integer_imm xy W t f? t x +i_bs_get_integer f? t t xy s d +i_bs_get_integer_8 xy f? d +i_bs_get_integer_16 xy f? d %if ARCH_64 -i_bs_get_integer_32 x f? xy +i_bs_get_integer_32 xy f? d %endif # Fetching binaries from binaries. -bs_get_binary2 Fail=f Ms=x Live=u Sz=sq Unit=u Flags=u Dst=d => \ +bs_get_binary2 Fail=f Ms=xy Live=u Sz=sq Unit=u Flags=u Dst=d => \ gen_get_binary2(Fail, Ms, Live, Sz, Unit, Flags, Dst) -i_bs_get_binary_imm2 f? x t W t xy -i_bs_get_binary2 f x t? s t xy -i_bs_get_binary_all2 f? x t t xy -i_bs_get_binary_all_reuse x f? t +i_bs_get_binary_imm2 f? xy t W t d +i_bs_get_binary2 f xy t? s t d +i_bs_get_binary_all2 f? xy t t d +i_bs_get_binary_all_reuse xy f? t # Fetching float from binaries. -bs_get_float2 Fail=f Ms=x Live=u Sz=s Unit=u Flags=u Dst=d => \ +bs_get_float2 Fail=f Ms=xy Live=u Sz=s Unit=u Flags=u Dst=d => \ gen_get_float2(Fail, Ms, Live, Sz, Unit, Flags, Dst) bs_get_float2 Fail=f Ms=x Live=u Sz=q Unit=u Flags=u Dst=d => jump Fail -i_bs_get_float2 f? x t s t xy +i_bs_get_float2 f? xy t s t d # Miscellanous -bs_skip_bits2 Fail=f Ms=x Sz=sq Unit=u Flags=u => \ +bs_skip_bits2 Fail=f Ms=xy Sz=sq Unit=u Flags=u => \ gen_skip_bits2(Fail, Ms, Sz, Unit, Flags) -i_bs_skip_bits_imm2 f? x W -i_bs_skip_bits2 f? x xy t -i_bs_skip_bits_all2 f? x t +i_bs_skip_bits_imm2 f? xy W +i_bs_skip_bits2 f? xy xy t +i_bs_skip_bits_all2 f? xy t -bs_test_tail2 Fail=f Ms=x Bits=u==0 => bs_test_zero_tail2 Fail Ms -bs_test_tail2 Fail=f Ms=x Bits=u => bs_test_tail_imm2 Fail Ms Bits -bs_test_zero_tail2 f? x -bs_test_tail_imm2 f? x W +bs_test_tail2 Fail=f Ms=xy Bits=u==0 => bs_test_zero_tail2 Fail Ms +bs_test_tail2 Fail=f Ms=xy Bits=u => bs_test_tail_imm2 Fail Ms Bits +bs_test_zero_tail2 f? xy +bs_test_tail_imm2 f? xy W bs_test_unit F Ms Unit=u==8 => bs_test_unit8 F Ms -bs_test_unit f? x t -bs_test_unit8 f? x +bs_test_unit f? xy t +bs_test_unit8 f? xy # An y register operand for bs_context_to_binary is rare, # but can happen because of inlining. @@ -1167,23 +1167,55 @@ bs_context_to_binary Y=y => move Y x | bs_context_to_binary x bs_context_to_binary x +# Gets a bitstring from the tail of a context. +bs_get_tail xy d t + +# New bs_start_match variant for contexts with external position storage. +# +# bs_get/set_position is used to save positions into registers instead of +# "slots" in the context itself, which lets us continue matching even after +# we've passed it off to another function. + +%if ARCH_64 +bs_start_match3 Fail Bin Live Ctx | bs_get_position Ctx Pos=x Ignored => \ + i_bs_start_match3_gp Bin Live Fail Ctx Pos +i_bs_start_match3_gp xy t f d x +%endif + +bs_start_match3 Fail=f ica Live Dst => jump Fail +bs_start_match3 Fail Bin Live Dst => i_bs_start_match3 Bin Live Fail Dst + +i_bs_start_match3 xy t f d + +# Match context position instructions. 64-bit assumes that all positions can +# fit into an unsigned small. + +%if ARCH_64 + bs_get_position Src Dst Live => i_bs_get_position Src Dst + i_bs_get_position xy xy + bs_set_position xy xy +%else + bs_get_position xy d t? + bs_set_position xy xy +%endif + # # Utf8/utf16/utf32 support. (R12B-5) # -bs_get_utf8 Fail=f Ms=x u u Dst=d => i_bs_get_utf8 Ms Fail Dst -i_bs_get_utf8 x f? xy +bs_get_utf8 Fail=f Ms=xy u u Dst=d => i_bs_get_utf8 Ms Fail Dst +i_bs_get_utf8 xy f? d -bs_skip_utf8 Fail=f Ms=x u u => i_bs_get_utf8 Ms Fail x +bs_skip_utf8 Fail=f Ms=xy u u => i_bs_get_utf8 Ms Fail x -bs_get_utf16 Fail=f Ms=x u Flags=u Dst=d => i_bs_get_utf16 Ms Fail Flags Dst -bs_skip_utf16 Fail=f Ms=x u Flags=u => i_bs_get_utf16 Ms Fail Flags x +bs_get_utf16 Fail=f Ms=xy u Flags=u Dst=d => i_bs_get_utf16 Ms Fail Flags Dst +bs_skip_utf16 Fail=f Ms=xy u Flags=u => i_bs_get_utf16 Ms Fail Flags x -i_bs_get_utf16 x f? t xy +i_bs_get_utf16 xy f? t d -bs_get_utf32 Fail=f Ms=x Live=u Flags=u Dst=d => \ +bs_get_utf32 Fail=f Ms=xy Live=u Flags=u Dst=d => \ bs_get_integer2 Fail Ms Live i=32 u=1 Flags Dst | \ i_bs_validate_unicode_retract Fail Dst Ms -bs_skip_utf32 Fail=f Ms=x Live=u Flags=u => \ +bs_skip_utf32 Fail=f Ms=xy Live=u Flags=u => \ bs_get_integer2 Fail Ms Live i=32 u=1 Flags x | \ i_bs_validate_unicode_retract Fail x Ms |