diff options
Diffstat (limited to 'erts/emulator/beam')
42 files changed, 1445 insertions, 839 deletions
diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 8c51f788c0..3022c0a99a 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -273,6 +273,10 @@ atom garbage_collecting atom garbage_collection atom garbage_collection_info atom gc_end +atom gc_major_end +atom gc_major_start +atom gc_minor_end +atom gc_minor_start atom gc_start atom Ge='>=' atom generational diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index b10250dc49..40d44dda4c 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -38,7 +38,7 @@ #include "erl_thr_progress.h" static void set_default_trace_pattern(Eterm module); -static Eterm check_process_code(Process* rp, Module* modp, Uint flags, int *redsp); +static Eterm check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls); static void delete_code(Module* modp); static void decrement_refc(BeamCodeHeader*); static int any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size); @@ -467,7 +467,7 @@ check_old_code_1(BIF_ALIST_1) } Eterm -erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp) +erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp, int fcalls) { Module* modp; Eterm res; @@ -483,7 +483,7 @@ erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp) return am_false; erts_rlock_old_code(code_ix); res = (!modp->old.code_hdr ? am_false : - check_process_code(c_p, modp, flags, redsp)); + check_process_code(c_p, modp, flags, redsp, fcalls)); erts_runlock_old_code(code_ix); return res; @@ -506,7 +506,7 @@ BIF_RETTYPE erts_internal_check_process_code_2(BIF_ALIST_2) goto badarg; } - res = erts_check_process_code(BIF_P, BIF_ARG_1, flags, &reds); + res = erts_check_process_code(BIF_P, BIF_ARG_1, flags, &reds, BIF_P->fcalls); ASSERT(is_value(res)); @@ -625,8 +625,8 @@ BIF_RETTYPE call_on_load_function_1(BIF_ALIST_1) { Module* modp = erts_get_module(BIF_ARG_1, erts_active_code_ix()); - if (modp && modp->curr.code_hdr) { - BIF_TRAP_CODE_PTR_0(BIF_P, modp->curr.code_hdr->on_load_function_ptr); + if (modp && modp->old.code_hdr) { + BIF_TRAP_CODE_PTR_0(BIF_P, modp->old.code_hdr->on_load_function_ptr); } else { BIF_ERROR(BIF_P, BADARG); @@ -651,14 +651,14 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) code_ix = erts_active_code_ix(); modp = erts_get_module(BIF_ARG_1, code_ix); - if (!modp || !modp->curr.code_hdr) { + if (!modp || !modp->old.code_hdr) { error: erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_release_code_write_permission(); BIF_ERROR(BIF_P, BADARG); } - if (modp->curr.code_hdr->on_load_function_ptr == NULL) { + if (modp->old.code_hdr->on_load_function_ptr == NULL) { goto error; } if (BIF_ARG_2 != am_false && BIF_ARG_2 != am_true) { @@ -667,44 +667,55 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) if (BIF_ARG_2 == am_true) { int i; + struct erl_module_instance t; + + /* + * Swap old and new code. + */ + t = modp->curr; + modp->curr = modp->old; + modp->old = t; /* * The on_load function succeded. Fix up export entries. */ for (i = 0; i < export_list_size(code_ix); i++) { Export *ep = export_list(i,code_ix); - if (ep != NULL && - ep->code[0] == BIF_ARG_1 && - ep->code[4] != 0) { + if (ep == NULL || ep->code[0] != BIF_ARG_1) { + continue; + } + if (ep->code[4] != 0) { ep->addressv[code_ix] = (void *) ep->code[4]; ep->code[4] = 0; + } else { + if (ep->addressv[code_ix] == ep->code+3 && + ep->code[3] == (BeamInstr) em_apply_bif) { + continue; + } + ep->addressv[code_ix] = ep->code+3; + ep->code[3] = (BeamInstr) em_call_error_handler; } } modp->curr.code_hdr->on_load_function_ptr = NULL; set_default_trace_pattern(BIF_ARG_1); } else if (BIF_ARG_2 == am_false) { - BeamInstr* code; - BeamInstr* end; + int i; /* - * The on_load function failed. Remove the loaded code. - * This is an combination of delete and purge. We purge - * the current code; the old code is not touched. + * The on_load function failed. Remove references to the + * code that is about to be purged from the export entries. */ - erts_total_code_size -= modp->curr.code_length; - code = (BeamInstr*) modp->curr.code_hdr; - end = (BeamInstr *) ((char *)code + modp->curr.code_length); - erts_cleanup_funs_on_purge(code, end); - beam_catches_delmod(modp->curr.catches, code, modp->curr.code_length, - erts_active_code_ix()); - if (modp->curr.code_hdr->literals_start) { - erts_free(ERTS_ALC_T_LITERAL, modp->curr.code_hdr->literals_start); - } - erts_free(ERTS_ALC_T_CODE, modp->curr.code_hdr); - modp->curr.code_hdr = NULL; - modp->curr.code_length = 0; - modp->curr.catches = BEAM_CATCHES_NIL; - erts_remove_from_ranges(code); + + for (i = 0; i < export_list_size(code_ix); i++) { + Export *ep = export_list(i,code_ix); + if (ep == NULL || ep->code[0] != BIF_ARG_1) { + continue; + } + if (ep->code[3] == (BeamInstr) em_apply_bif) { + continue; + } + ep->code[4] = 0; + } } erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); @@ -753,7 +764,7 @@ check_mod_funs(Process *p, ErlOffHeap *off_heap, char *area, size_t area_size) static Eterm -check_process_code(Process* rp, Module* modp, Uint flags, int *redsp) +check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls) { BeamInstr* start; char* literals; @@ -832,7 +843,7 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp) /* * Message queue can contains funs, but (at least currently) no - * constants. If we got references to this module from the message + * literals. If we got references to this module from the message * queue, a GC cannot remove these... */ @@ -853,7 +864,7 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp) for (; hfrag; hfrag = hfrag->next) { if (check_mod_funs(rp, &hfrag->off_heap, mod_start, mod_size)) return am_true; - /* Should not contain any constants... */ + /* Should not contain any literals... */ ASSERT(!any_heap_refs(&hfrag->mem[0], &hfrag->mem[hfrag->used_size], literals, @@ -908,7 +919,7 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp) #ifdef DEBUG /* * Message buffer fragments should not have any references - * to constants, and off heap lists should already have + * to literals, and off heap lists should already have * been moved into process off heap structure. */ for (msgp = rp->msg_frag; msgp; msgp = msgp->next) { @@ -945,7 +956,7 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp) need_gc &= ~done_gc; /* - * Try to get rid of constants by by garbage collecting. + * Try to get rid of literals by by garbage collecting. * Clear both fvalue and ftrace. */ @@ -955,7 +966,7 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp) if (need_gc & ERTS_ORDINARY_GC__) { FLAGS(rp) |= F_NEED_FULLSWEEP; - *redsp += erts_garbage_collect_nobump(rp, 0, rp->arg_reg, rp->arity); + *redsp += erts_garbage_collect_nobump(rp, 0, rp->arg_reg, rp->arity, fcalls); done_gc |= ERTS_ORDINARY_GC__; } if (need_gc & ERTS_LITERAL_GC__) { diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index 2ee98ed7b5..1e30e8d8d1 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -248,7 +248,10 @@ erts_bp_match_export(BpFunctions* f, Eterm mfa[3], int specified) void erts_bp_free_matched_functions(BpFunctions* f) { - Free(f->matching); + if (f->matching) { + Free(f->matching); + } + else ASSERT(f->matched == 0); } void @@ -1432,7 +1435,7 @@ set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags, g = (GenericBp *) pc[-4]; if (g == 0) { int i; - if (count_op == erts_break_reset || count_op == erts_break_stop) { + if (count_op == ERTS_BREAK_RESTART || count_op == ERTS_BREAK_PAUSE) { /* Do not insert a new breakpoint */ return; } @@ -1456,7 +1459,7 @@ set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags, MatchSetUnref(bp->meta_ms); bp_meta_unref(bp->meta_tracer); } else if (common & ERTS_BPF_COUNT) { - if (count_op == erts_break_stop) { + if (count_op == ERTS_BREAK_PAUSE) { bp->flags &= ~ERTS_BPF_COUNT_ACTIVE; } else { bp->flags |= ERTS_BPF_COUNT_ACTIVE; @@ -1468,7 +1471,7 @@ set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags, BpDataTime* bdt = bp->time; Uint i = 0; - if (count_op == erts_break_stop) { + if (count_op == ERTS_BREAK_PAUSE) { bp->flags &= ~ERTS_BPF_TIME_TRACE_ACTIVE; } else { bp->flags |= ERTS_BPF_TIME_TRACE_ACTIVE; diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h index bb171be8a6..9c2fc007a2 100644 --- a/erts/emulator/beam/beam_bp.h +++ b/erts/emulator/beam/beam_bp.h @@ -86,10 +86,10 @@ typedef struct generic_bp { #endif enum erts_break_op{ - erts_break_nop = 0, /* Must be false */ - erts_break_set = !0, /* Must be true */ - erts_break_reset, - erts_break_stop + ERTS_BREAK_NOP = 0, /* Must be false */ + ERTS_BREAK_SET = !0, /* Must be true */ + ERTS_BREAK_RESTART, + ERTS_BREAK_PAUSE }; typedef Uint32 ErtsBpIndex; @@ -173,19 +173,7 @@ void erts_clear_time_trace_bif(BeamInstr *pc); BeamInstr *erts_find_local_func(Eterm mfa[3]); -ERTS_GLB_INLINE Uint erts_bp_sched2ix(void); - #if ERTS_GLB_INLINE_INCL_FUNC_DEF -ERTS_GLB_INLINE Uint erts_bp_sched2ix(void) -{ -#ifdef ERTS_SMP - ErtsSchedulerData *esdp; - esdp = erts_get_scheduler_data(); - return esdp->no - 1; -#else - return 0; -#endif -} extern erts_smp_atomic32_t erts_active_bp_index; extern erts_smp_atomic32_t erts_staging_bp_index; diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 72526bca5e..99eb26bec5 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -308,7 +308,8 @@ void** beam_ops; if (E - HTOP < (needed + (HeapNeed))) { \ SWAPOUT; \ PROCESS_MAIN_CHK_LOCKS(c_p); \ - FCALLS -= erts_garbage_collect_nobump(c_p, needed + (HeapNeed), reg, (M)); \ + FCALLS -= erts_garbage_collect_nobump(c_p, needed + (HeapNeed), \ + reg, (M), FCALLS); \ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ SWAPIN; \ @@ -360,7 +361,7 @@ void** beam_ops; if ((E - HTOP < need) || (MSO(c_p).overhead + (VNh) >= BIN_VHEAP_SZ(c_p))) {\ SWAPOUT; \ PROCESS_MAIN_CHK_LOCKS(c_p); \ - FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live)); \ + FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live), FCALLS); \ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ SWAPIN; \ @@ -381,7 +382,7 @@ void** beam_ops; if (E - HTOP < need) { \ SWAPOUT; \ PROCESS_MAIN_CHK_LOCKS(c_p); \ - FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live));\ + FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live), FCALLS); \ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ SWAPIN; \ @@ -402,7 +403,7 @@ void** beam_ops; SWAPOUT; \ reg[Live] = Extra; \ PROCESS_MAIN_CHK_LOCKS(c_p); \ - FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live)+1); \ + FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live)+1, FCALLS); \ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ Extra = reg[Live]; \ @@ -1196,6 +1197,25 @@ init_emulator(void) #define DTRACE_NIF_RETURN(p, m, f, a) do {} while (0) #endif /* USE_VM_PROBES */ +#ifdef DEBUG +#define ERTS_DBG_CHK_REDS(P, FC) \ + do { \ + if (ERTS_PROC_GET_SAVED_CALLS_BUF((P))) { \ + ASSERT(FC <= 0); \ + ASSERT(ERTS_PROC_GET_SCHDATA(c_p)->virtual_reds \ + <= 0 - (FC)); \ + } \ + else { \ + ASSERT(FC <= CONTEXT_REDS); \ + ASSERT(ERTS_PROC_GET_SCHDATA(c_p)->virtual_reds \ + <= CONTEXT_REDS - (FC)); \ + } \ +} while (0) +#else +#define ERTS_DBG_CHK_REDS(P, FC) +#endif + + /* * process_main() is called twice: * The first call performs some initialisation, including exporting @@ -1290,7 +1310,12 @@ void process_main(void) goto do_schedule1; do_schedule: - reds_used = REDS_IN(c_p) - FCALLS; + ASSERT(c_p->debug_reds_in == REDS_IN(c_p)); + if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) + reds_used = REDS_IN(c_p) - FCALLS; + else + reds_used = REDS_IN(c_p) - (CONTEXT_REDS + FCALLS); + ASSERT(reds_used >= 0); do_schedule1: if (start_time != 0) { @@ -1310,6 +1335,7 @@ void process_main(void) ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); c_p = schedule(c_p, reds_used); + ASSERT(!(c_p->flags & F_HIPE_MODE)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); start_time = 0; #ifdef DEBUG @@ -1348,16 +1374,21 @@ void process_main(void) SET_I(c_p->i); - reds = c_p->fcalls; - if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p) - && (ERTS_TRACE_FLAGS(c_p) & F_SENSITIVE) == 0) { - neg_o_reds = -reds; - FCALLS = REDS_IN(c_p) = 0; + REDS_IN(c_p) = reds = c_p->fcalls; +#ifdef DEBUG + c_p->debug_reds_in = reds; +#endif + + if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) { + neg_o_reds = -CONTEXT_REDS; + FCALLS = neg_o_reds + reds; } else { neg_o_reds = 0; - FCALLS = REDS_IN(c_p) = reds; + FCALLS = reds; } + ERTS_DBG_CHK_REDS(c_p, FCALLS); + next = (BeamInstr *) *I; SWAPIN; ASSERT(VALID_INSTR(next)); @@ -1770,7 +1801,7 @@ void process_main(void) if (E - HTOP < 3) { SWAPOUT; PROCESS_MAIN_CHK_LOCKS(c_p); - FCALLS -= erts_garbage_collect_nobump(c_p, 3, reg+2, 1); + FCALLS -= erts_garbage_collect_nobump(c_p, 3, reg+2, 1, FCALLS); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); SWAPIN; @@ -1865,6 +1896,7 @@ void process_main(void) c_p->flags |= F_DELAY_GC; loop_rec__: + PROCESS_MAIN_CHK_LOCKS(c_p); msgp = PEEK_MESSAGE(c_p); @@ -2007,6 +2039,8 @@ void process_main(void) ERTS_VBUMP_LEAVE_REDS_INTERNAL(c_p, 5, FCALLS); } + ERTS_DBG_CHK_REDS(c_p, FCALLS); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); NextPF(0, next); @@ -2166,7 +2200,7 @@ void process_main(void) PreFetch(0, next); if (IS_TRACED_FL(c_p, F_TRACE_RECEIVE)) { - trace_receive(c_p, am_timeout); + trace_receive(c_p, am_clock_service, am_timeout, NULL); } if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) { save_calls(c_p, &exp_timeout); @@ -2535,6 +2569,7 @@ do { \ GetArg1(2, tmp_reg[0]); bf = (BifFunction) Arg(1); + ERTS_DBG_CHK_REDS(c_p, FCALLS); c_p->fcalls = FCALLS; PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); @@ -2544,6 +2579,7 @@ do { \ PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_HOLE_CHECK(c_p); FCALLS = c_p->fcalls; + ERTS_DBG_CHK_REDS(c_p, FCALLS); if (is_value(result)) { StoreBifResult(3, result); } @@ -2564,6 +2600,7 @@ do { \ GetArg1(1, tmp_reg[0]); bf = (BifFunction) Arg(0); + ERTS_DBG_CHK_REDS(c_p, FCALLS); c_p->fcalls = FCALLS; PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); @@ -2573,6 +2610,7 @@ do { \ PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_HOLE_CHECK(c_p); FCALLS = c_p->fcalls; + ERTS_DBG_CHK_REDS(c_p, FCALLS); if (is_value(result)) { StoreBifResult(2, result); } @@ -2591,6 +2629,7 @@ do { \ GetArg1(2, x(live)); bf = (GcBifFunction) Arg(1); + ERTS_DBG_CHK_REDS(c_p, FCALLS); c_p->fcalls = FCALLS; SWAPOUT; PROCESS_MAIN_CHK_LOCKS(c_p); @@ -2602,6 +2641,7 @@ do { \ SWAPIN; ERTS_HOLE_CHECK(c_p); FCALLS = c_p->fcalls; + ERTS_DBG_CHK_REDS(c_p, FCALLS); if (is_value(result)) { StoreBifResult(4, result); } @@ -2630,6 +2670,7 @@ do { \ */ live++; bf = (GcBifFunction) Arg(1); + ERTS_DBG_CHK_REDS(c_p, FCALLS); c_p->fcalls = FCALLS; SWAPOUT; PROCESS_MAIN_CHK_LOCKS(c_p); @@ -2641,6 +2682,7 @@ do { \ SWAPIN; ERTS_HOLE_CHECK(c_p); FCALLS = c_p->fcalls; + ERTS_DBG_CHK_REDS(c_p, FCALLS); if (is_value(result)) { StoreBifResult(5, result); } @@ -2671,6 +2713,7 @@ do { \ */ live += 2; bf = (GcBifFunction) Arg(1); + ERTS_DBG_CHK_REDS(c_p, FCALLS); c_p->fcalls = FCALLS; SWAPOUT; PROCESS_MAIN_CHK_LOCKS(c_p); @@ -2682,6 +2725,7 @@ do { \ SWAPIN; ERTS_HOLE_CHECK(c_p); FCALLS = c_p->fcalls; + ERTS_DBG_CHK_REDS(c_p, FCALLS); if (is_value(result)) { StoreBifResult(5, result); } @@ -2708,6 +2752,7 @@ do { \ GetArg2(2, tmp_reg[0], tmp_reg[1]); bf = (BifFunction) Arg(1); + ERTS_DBG_CHK_REDS(c_p, FCALLS); c_p->fcalls = FCALLS; PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); @@ -2717,6 +2762,7 @@ do { \ PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_HOLE_CHECK(c_p); FCALLS = c_p->fcalls; + ERTS_DBG_CHK_REDS(c_p, FCALLS); if (is_value(result)) { StoreBifResult(4, result); } @@ -2775,6 +2821,7 @@ do { \ bf = GET_BIF_ADDRESS(Arg(0)); PRE_BIF_SWAPOUT(c_p); + ERTS_DBG_CHK_REDS(c_p, FCALLS); c_p->fcalls = FCALLS - 1; if (FCALLS <= 0) { save_calls(c_p, (Export *) Arg(0)); @@ -2796,6 +2843,7 @@ do { \ PROCESS_MAIN_CHK_LOCKS(c_p); HTOP = HEAP_TOP(c_p); FCALLS = c_p->fcalls; + ERTS_DBG_CHK_REDS(c_p, FCALLS); /* We have to update the cache if we are enabled in order to make sure no book keeping is done after we disabled msacc. We don't always do this as it is quite expensive. */ @@ -3324,8 +3372,13 @@ do { \ * (beacuse the code for the Dispatch() macro becomes shorter that way). */ - reds_used = REDS_IN(c_p) - FCALLS + 1; - + ASSERT(c_p->debug_reds_in == REDS_IN(c_p)); + if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) + reds_used = REDS_IN(c_p) - FCALLS; + else + reds_used = REDS_IN(c_p) - (CONTEXT_REDS + FCALLS); + ASSERT(reds_used >= 0); + /* * Save the argument registers and everything else. */ @@ -3514,6 +3567,7 @@ do { \ DTRACE_BIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); SWAPOUT; + ERTS_DBG_CHK_REDS(c_p, FCALLS - 1); c_p->fcalls = FCALLS - 1; vbf = (BifFunction) Arg(0); PROCESS_MAIN_CHK_LOCKS(c_p); @@ -3549,6 +3603,7 @@ do { \ } SWAPIN; /* There might have been a garbage collection. */ FCALLS = c_p->fcalls; + ERTS_DBG_CHK_REDS(c_p, FCALLS); if (is_value(nif_bif_result)) { r(0) = nif_bif_result; CHECK_TERM(r(0)); @@ -4611,9 +4666,9 @@ do { \ OpCase(i_generic_breakpoint): { BeamInstr real_I; ASSERT(I[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); - SWAPOUT; + HEAVY_SWAPOUT; real_I = erts_generic_breakpoint(c_p, I, reg); - SWAPIN; + HEAVY_SWAPIN; ASSERT(VALID_INSTR(real_I)); Goto(real_I); } @@ -4790,6 +4845,7 @@ do { \ { #define HIPE_MODE_SWITCH(Cmd) \ SWAPOUT; \ + ERTS_DBG_CHK_REDS(c_p, FCALLS); \ c_p->fcalls = FCALLS; \ c_p->def_arg_reg[4] = -neg_o_reds; \ c_p = hipe_mode_switch(c_p, Cmd, reg); \ @@ -4828,6 +4884,9 @@ do { \ #undef HIPE_MODE_SWITCH L_post_hipe_mode_switch: +#ifdef DEBUG + pid = c_p->common.id; /* may have switched process... */ +#endif reg = ERTS_PROC_GET_SCHDATA(c_p)->x_reg_array; freg = ERTS_PROC_GET_SCHDATA(c_p)->f_reg_array; ERL_BITS_RELOAD_STATEP(c_p); @@ -4835,6 +4894,7 @@ do { \ neg_o_reds = -c_p->def_arg_reg[4]; FCALLS = c_p->fcalls; SWAPIN; + ERTS_DBG_CHK_REDS(c_p, FCALLS); switch( c_p->def_arg_reg[3] ) { case HIPE_MODE_SWITCH_RES_RETURN: ASSERT(is_value(reg[0])); @@ -6839,7 +6899,7 @@ erts_current_reductions(Process *current, Process *p) if (current != p) { return 0; } else if (current->fcalls < 0 && ERTS_PROC_GET_SAVED_CALLS_BUF(current)) { - return -current->fcalls; + return current->fcalls + CONTEXT_REDS; } else { return REDS_IN(current) - current->fcalls; } diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 2e21a553ed..95489d9a63 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -32,6 +32,7 @@ #include "bif.h" #include "external.h" #include "beam_load.h" +#include "beam_bp.h" #include "big.h" #include "erl_bits.h" #include "beam_catches.h" @@ -479,9 +480,9 @@ static void free_loader_state(Binary* magic); static ErlHeapFragment* new_literal_fragment(Uint size); static void free_literal_fragment(ErlHeapFragment*); static void loader_state_dtor(Binary* magic); -static Eterm insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, - Eterm group_leader, Eterm module, - BeamCodeHeader* code, Uint size); +static Eterm stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, + Eterm group_leader, Eterm module, + BeamCodeHeader* code, Uint size); static int init_iff_file(LoaderState* stp, byte* code, Uint size); static int scan_iff_file(LoaderState* stp, Uint* chunk_types, Uint num_types, Uint num_mandatory); @@ -513,7 +514,7 @@ static GenOp* gen_get_map_element(LoaderState* stp, GenOpArg Fail, GenOpArg Src, static int freeze_code(LoaderState* stp); -static void final_touch(LoaderState* stp); +static void final_touch(LoaderState* stp, struct erl_module_instance* inst_p); static void short_file(int line, LoaderState* stp, unsigned needed); static void load_printf(int line, LoaderState* context, char *fmt, ...); static int transform_engine(LoaderState* st); @@ -766,8 +767,11 @@ Eterm erts_finish_loading(Binary* magic, Process* c_p, ErtsProcLocks c_p_locks, Eterm* modp) { - Eterm retval; + Eterm retval = NIL; LoaderState* stp = ERTS_MAGIC_BIN_DATA(magic); + Module* mod_tab_p; + struct erl_module_instance* inst_p; + Uint size; /* * No other process may run since we will update the export @@ -776,27 +780,80 @@ erts_finish_loading(Binary* magic, Process* c_p, ERTS_SMP_LC_ASSERT(erts_initialized == 0 || erts_has_code_write_permission() || erts_smp_thr_progress_is_blocking()); - /* * Make current code for the module old and insert the new code * as current. This will fail if there already exists old code * for the module. */ + mod_tab_p = erts_put_module(stp->module); CHKBLK(ERTS_ALC_T_CODE,stp->code); - retval = insert_new_code(c_p, c_p_locks, stp->group_leader, stp->module, - stp->hdr, stp->loaded_size); - if (retval != NIL) { - goto load_error; + if (!stp->on_load) { + /* + * Normal case -- no -on_load() function. + */ + retval = beam_make_current_old(c_p, c_p_locks, stp->module); + ASSERT(retval == NIL); + } else { + ErtsCodeIndex code_ix = erts_staging_code_ix(); + Eterm module = stp->module; + int i; + + /* + * There is an -on_load() function. We will keep the current + * code, but we must turn off any tracing. + */ + + for (i = 0; i < export_list_size(code_ix); i++) { + Export *ep = export_list(i, code_ix); + if (ep == NULL || ep->code[0] != module) { + continue; + } + if (ep->addressv[code_ix] == ep->code+3) { + if (ep->code[3] == (BeamInstr) em_apply_bif) { + continue; + } else if (ep->code[3] == + (BeamInstr) BeamOp(op_i_generic_breakpoint)) { + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); + ASSERT(mod_tab_p->curr.num_traced_exports > 0); + erts_clear_export_break(mod_tab_p, ep->code+3); + ep->addressv[code_ix] = (BeamInstr *) ep->code[4]; + ep->code[4] = 0; + } + ASSERT(ep->code[4] == 0); + } + } + ASSERT(mod_tab_p->curr.num_breakpoints == 0); + ASSERT(mod_tab_p->curr.num_traced_exports == 0); } /* + * Update module table. + */ + + size = stp->loaded_size; + erts_total_code_size += size; + if (stp->on_load) { + inst_p = &mod_tab_p->old; + } else { + inst_p = &mod_tab_p->curr; + } + inst_p->code_hdr = stp->hdr; + inst_p->code_length = size; + + /* + * Update ranges (used for finding a function from a PC value). + */ + + erts_update_ranges((BeamInstr*)inst_p->code_hdr, size); + + /* * Ready for the final touch: fixing the export table entries for * exported and imported functions. This can't fail. */ CHKBLK(ERTS_ALC_T_CODE,stp->code); - final_touch(stp); + final_touch(stp, inst_p); /* * Loading succeded. @@ -820,7 +877,6 @@ erts_finish_loading(Binary* magic, Process* c_p, retval = am_on_load; } - load_error: free_loader_state(magic); return retval; } @@ -1026,9 +1082,9 @@ loader_state_dtor(Binary* magic) } static Eterm -insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, - Eterm group_leader, Eterm module, BeamCodeHeader* code_hdr, - Uint size) +stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, + Eterm group_leader, Eterm module, + BeamCodeHeader* code_hdr, Uint size) { Module* modp; Eterm retval; @@ -4700,14 +4756,13 @@ freeze_code(LoaderState* stp) } static void -final_touch(LoaderState* stp) +final_touch(LoaderState* stp, struct erl_module_instance* inst_p) { int i; int on_load = stp->on_load; unsigned catches; Uint index; BeamInstr* codev = stp->codev; - Module* modp; /* * Allocate catch indices and fix up all catch_yf instructions. @@ -4722,8 +4777,7 @@ final_touch(LoaderState* stp) codev[index+2] = make_catch(catches); index = next; } - modp = erts_put_module(stp->module); - modp->curr.catches = catches; + inst_p->catches = catches; /* * Export functions. @@ -4743,10 +4797,10 @@ final_touch(LoaderState* stp) ep->addressv[erts_staging_code_ix()] = address; } else { /* - * Don't make any of the exported functions - * callable yet. + * on_load: Don't make any of the exported functions + * callable yet. Keep any function in the current + * code callable. */ - ep->addressv[erts_staging_code_ix()] = ep->code+3; ep->code[4] = (BeamInstr) address; } } @@ -6421,7 +6475,8 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) * Insert the module in the module table. */ - rval = insert_new_code(p, 0, p->group_leader, Mod, code_hdr, code_size); + rval = stub_insert_new_code(p, 0, p->group_leader, Mod, + code_hdr, code_size); if (rval != NIL) { goto error; } diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index ed5b2983dd..eed0702688 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -611,7 +611,7 @@ erts_queue_monitor_message(Process *p, ref_copy = copy_struct(ref, ref_size, &hp, ohp); tup = TUPLE5(hp, am_DOWN, ref_copy, type, item_copy, reason_copy); - erts_queue_message(p, p_locksp, msgp, tup); + erts_queue_message(p, *p_locksp, msgp, tup, am_system); } static BIF_RETTYPE @@ -1569,7 +1569,32 @@ static BIF_RETTYPE process_flag_aux(Process *BIF_P, scb->n = 0; } - scb = ERTS_PROC_SET_SAVED_CALLS_BUF(rp, scb); +#ifdef HIPE + if (rp->flags & F_HIPE_MODE) { + ASSERT(!ERTS_PROC_GET_SAVED_CALLS_BUF(rp)); + scb = ERTS_PROC_SET_SUSPENDED_SAVED_CALLS_BUF(rp, scb); + } + else +#endif + { +#ifdef HIPE + ASSERT(!ERTS_PROC_GET_SUSPENDED_SAVED_CALLS_BUF(rp)); +#endif + scb = ERTS_PROC_SET_SAVED_CALLS_BUF(rp, scb); + if (rp == BIF_P && ((scb && i == 0) || (!scb && i != 0))) { + /* Adjust fcalls to match save calls setting... */ + if (i == 0) + BIF_P->fcalls += CONTEXT_REDS; /* disabled it */ + else + BIF_P->fcalls -= CONTEXT_REDS; /* enabled it */ + + /* + * Make sure we reschedule immediately so the + * change take effect at once. + */ + ERTS_VBUMP_ALL_REDS(BIF_P); + } + } if (!scb) old_value = make_small(0); @@ -1578,12 +1603,7 @@ static BIF_RETTYPE process_flag_aux(Process *BIF_P, erts_free(ERTS_ALC_T_CALLS_BUF, (void *) scb); } - /* Make sure the process in question is rescheduled - immediately, if it's us, so the call saving takes effect. */ - if (rp == BIF_P) - BIF_RET2(old_value, CONTEXT_REDS); - else - BIF_RET(old_value); + BIF_RET(old_value); } error: @@ -1736,7 +1756,9 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2) ERTS_TRACE_FLAGS(BIF_P) &= ~F_SENSITIVE; } erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCKS_ALL_MINOR); - BIF_RET(old_value); + /* make sure to bump all reds so that we get + rescheduled immediately so setting takes effect */ + BIF_RET2(old_value, CONTEXT_REDS); } else if (BIF_ARG_1 == am_monitor_nodes) { /* @@ -1775,10 +1797,18 @@ BIF_RETTYPE process_flag_3(BIF_ALIST_3) Process *rp; Eterm res; - if ((rp = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN, - BIF_ARG_1, ERTS_PROC_LOCK_MAIN)) == NULL) { +#ifdef ERTS_SMP + rp = erts_pid2proc_not_running(BIF_P, ERTS_PROC_LOCK_MAIN, + BIF_ARG_1, ERTS_PROC_LOCK_MAIN); + if (rp == ERTS_PROC_LOCK_BUSY) + ERTS_BIF_YIELD3(bif_export[BIF_process_flag_3], BIF_P, + BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); +#else + rp = erts_proc_lookup(BIF_ARG_1); +#endif + + if (!rp) BIF_ERROR(BIF_P, BADARG); - } res = process_flag_aux(BIF_P, rp, BIF_ARG_2, BIF_ARG_3); diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h index 546e3830b9..5d751dd67d 100644 --- a/erts/emulator/beam/bif.h +++ b/erts/emulator/beam/bif.h @@ -52,6 +52,7 @@ extern Export *erts_convert_time_unit_trap; (p)->fcalls = 0; \ else \ (p)->fcalls = -CONTEXT_REDS; \ + ASSERT(ERTS_BIF_REDS_LEFT((p)) == 0); \ } while(0) #define ERTS_VBUMP_ALL_REDS_INTERNAL(p, fcalls) \ diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 58cd31cee9..872f0f9b2a 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -652,6 +652,8 @@ bif erts_debug:size_shared/1 bif erts_debug:copy_shared/1 bif erlang:has_prepared_code_on_load/1 +bif maps:take/2 + # # Obsolete # diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 4f5e80f2e5..d02c6828f9 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -347,8 +347,11 @@ print_process_info(int to, void *to_arg, Process *p) static void print_garb_info(int to, void *to_arg, Process* p) { +#ifdef ERTS_SMP /* ERTS_SMP: A scheduler is probably concurrently doing gc... */ -#ifndef ERTS_SMP + if (!ERTS_IS_CRASH_DUMPING) + return; +#endif erts_print(to, to_arg, "New heap start: %bpX\n", p->heap); erts_print(to, to_arg, "New heap top: %bpX\n", p->htop); erts_print(to, to_arg, "Stack top: %bpX\n", p->stop); @@ -356,7 +359,6 @@ print_garb_info(int to, void *to_arg, Process* p) erts_print(to, to_arg, "Old heap start: %bpX\n", OLD_HEAP(p)); erts_print(to, to_arg, "Old heap top: %bpX\n", OLD_HTOP(p)); erts_print(to, to_arg, "Old heap end: %bpX\n", OLD_HEND(p)); -#endif } void diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index 52fd57e101..09c83f1117 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -397,7 +397,7 @@ static void doit_node_link_net_exits(ErtsLink *lnk, void *vnecp) msgp = erts_alloc_message_heap(rp, &rp_locks, 3, &hp, &ohp); tup = TUPLE2(hp, am_nodedown, name); - erts_queue_message(rp, &rp_locks, msgp, tup); + erts_queue_message(rp, rp_locks, msgp, tup, am_system); } erts_smp_proc_unlock(rp, rp_locks); } @@ -1456,7 +1456,7 @@ int erts_net_message(Port *prt, token = copy_struct(token, token_size, &hp, ohp); } - erts_queue_dist_message(rp, &locks, ede_copy, token); + erts_queue_dist_message(rp, locks, ede_copy, token, from); if (locks) erts_smp_proc_unlock(rp, locks); } @@ -1505,7 +1505,7 @@ int erts_net_message(Port *prt, token = copy_struct(token, token_size, &hp, ohp); } - erts_queue_dist_message(rp, &locks, ede_copy, token); + erts_queue_dist_message(rp, locks, ede_copy, token, tuple[2]); if (locks) erts_smp_proc_unlock(rp, locks); } @@ -1967,7 +1967,7 @@ dist_port_command(Port *prt, ErtsDistOutputBuf *obuf) ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); if (size > (Uint) INT_MAX) - erts_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_DUMP_EXIT, "Absurdly large distribution output data buffer " "(%beu bytes) passed.\n", size); @@ -2007,7 +2007,7 @@ dist_port_commandv(Port *prt, ErtsDistOutputBuf *obuf) ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); if (size > (Uint) INT_MAX) - erts_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_DUMP_EXIT, "Absurdly large distribution output data buffer " "(%beu bytes) passed.\n", size); @@ -3317,7 +3317,7 @@ send_nodes_mon_msg(Process *rp, } ASSERT(hend == hp); - erts_queue_message(rp, rp_locksp, mp, msg); + erts_queue_message(rp, *rp_locksp, mp, msg, am_system); } static void diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index f571515def..c367d4162c 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -3339,7 +3339,7 @@ reply_alloc_info(void *vair) if (hp != hp_end) erts_shrink_message_heap(&mp, rp, hp_start, hp, hp_end, &msg, 1); - erts_queue_message(rp, &rp_locks, mp, msg); + erts_queue_message(rp, rp_locks, mp, msg, am_system); if (air->req_sched == sched_id) rp_locks &= ~ERTS_PROC_LOCK_MAIN; diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index ad62a87326..ba216c7eb4 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -264,7 +264,6 @@ type PRTSD STANDARD SYSTEM port_specific_data type CPUDATA LONG_LIVED SYSTEM cpu_data type TMP_CPU_IDS SHORT_LIVED SYSTEM tmp_cpu_ids type EXT_TERM_DATA SHORT_LIVED PROCESSES external_term_data -type ZLIB STANDARD SYSTEM zlib type CPU_GRPS_MAP LONG_LIVED SYSTEM cpu_groups_map type AUX_WORK_TMO LONG_LIVED SYSTEM aux_work_timeouts type MISC_AUX_WORK_Q LONG_LIVED SYSTEM misc_aux_work_q @@ -297,8 +296,10 @@ type THR_Q_LL LONG_LIVED SYSTEM long_lived_thr_queue +if smp type ASYNC SHORT_LIVED SYSTEM async +type ZLIB STANDARD SYSTEM zlib +else -# sl_alloc is not thread safe in non smp build; therefore, we use driver_alloc +# sl/std_alloc is not thread safe in non smp build; therefore, we use driver_alloc +type ZLIB DRIVER SYSTEM zlib type ASYNC DRIVER SYSTEM async +endif diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index be7c16cc05..2995f2f822 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -860,8 +860,9 @@ erts_alcu_literal_32_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) { void* res; Uint sz = ERTS_SUPERALIGNED_CEILING(*size_p); - ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL - && !allctr->t && allctr->thread_safe); + ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL && + allctr->t == 0); + ERTS_SMP_LC_ASSERT(allctr->thread_safe); res = erts_alcu_mseg_alloc(allctr, &sz, flags); if (res) { @@ -877,8 +878,9 @@ erts_alcu_literal_32_mseg_realloc(Allctr_t *allctr, void *seg, { void* res; Uint new_sz = ERTS_SUPERALIGNED_CEILING(*new_size_p); - ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL - && !allctr->t && allctr->thread_safe); + ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL && + allctr->t == 0); + ERTS_SMP_LC_ASSERT(allctr->thread_safe); if (seg && old_size) clear_literal_range(seg, old_size); @@ -894,8 +896,9 @@ void erts_alcu_literal_32_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, Uint flags) { - ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL - && !allctr->t && allctr->thread_safe); + ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL && + allctr->t == 0); + ERTS_SMP_LC_ASSERT(allctr->thread_safe); erts_alcu_mseg_dealloc(allctr, seg, size, flags); @@ -1007,8 +1010,9 @@ erts_alcu_literal_32_sys_alloc(Allctr_t *allctr, Uint* size_p, int superalign) { void* res; Uint size = ERTS_SUPERALIGNED_CEILING(*size_p); - ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL - && !allctr->t && allctr->thread_safe); + ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL && + allctr->t == 0); + ERTS_SMP_LC_ASSERT(allctr->thread_safe); res = erts_alcu_sys_alloc(allctr, &size, 1); if (res) { @@ -1024,8 +1028,9 @@ erts_alcu_literal_32_sys_realloc(Allctr_t *allctr, void *ptr, Uint* size_p, Uint void* res; Uint size = ERTS_SUPERALIGNED_CEILING(*size_p); - ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL - && !allctr->t && allctr->thread_safe); + ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL && + allctr->t == 0); + ERTS_SMP_LC_ASSERT(allctr->thread_safe); if (ptr && old_size) clear_literal_range(ptr, old_size); @@ -1040,8 +1045,9 @@ erts_alcu_literal_32_sys_realloc(Allctr_t *allctr, void *ptr, Uint* size_p, Uint void erts_alcu_literal_32_sys_dealloc(Allctr_t *allctr, void *ptr, Uint size, int superalign) { - ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL - && !allctr->t && allctr->thread_safe); + ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL && + allctr->t == 0); + ERTS_SMP_LC_ASSERT(allctr->thread_safe); erts_alcu_sys_dealloc(allctr, ptr, size, 1); diff --git a/erts/emulator/beam/erl_async.c b/erts/emulator/beam/erl_async.c index 4d6d699d9f..84254af0c2 100644 --- a/erts/emulator/beam/erl_async.c +++ b/erts/emulator/beam/erl_async.c @@ -283,10 +283,10 @@ static ERTS_INLINE void async_add(ErtsAsync *a, ErtsAsyncQ* q) erts_thr_q_enqueue(&q->thr_q, a); #ifdef USE_LTTNG_VM_TRACEPOINTS - if (LTTNG_ENABLED(aio_pool_add)) { + if (LTTNG_ENABLED(aio_pool_put)) { lttng_decl_portbuf(port_str); lttng_portid_to_str(a->port, port_str); - LTTNG2(aio_pool_add, port_str, -1); + LTTNG2(aio_pool_put, port_str, -1); } #endif #ifdef USE_VM_PROBES diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c index 1e2db38442..ef77201544 100644 --- a/erts/emulator/beam/erl_bif_ddll.c +++ b/erts/emulator/beam/erl_bif_ddll.c @@ -1737,7 +1737,7 @@ static void notify_proc(Process *proc, Eterm ref, Eterm driver_name, Eterm type, hp += REF_THING_SIZE; mess = TUPLE5(hp,type,r,am_driver,driver_name,tag); } - erts_queue_message(proc, &rp_locks, mp, mess); + erts_queue_message(proc, rp_locks, mp, mess, am_system); erts_smp_proc_unlock(proc, rp_locks); ERTS_SMP_CHK_NO_PROC_LOCKS; } diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 95d1fef3d3..85cadafebc 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -1359,9 +1359,10 @@ process_info_aux(Process *BIF_P, total_heap_size += rp->mbuf_sz; - for (mp = rp->msg.first; mp; mp = mp->next) - if (mp->data.attached) - total_heap_size += erts_msg_attached_data_size(mp); + if (rp->flags & F_ON_HEAP_MSGQ) + for (mp = rp->msg.first; mp; mp = mp->next) + if (mp->data.attached) + total_heap_size += erts_msg_attached_data_size(mp); (void) erts_bld_uint(NULL, &hsz, total_heap_size); hp = HAlloc(BIF_P, hsz); diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c index ff2018aa27..b65c0e303f 100644 --- a/erts/emulator/beam/erl_bif_trace.c +++ b/erts/emulator/beam/erl_bif_trace.c @@ -68,6 +68,9 @@ static struct { /* Protected by code write permission */ static Eterm trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist); +static int +erts_set_tracing_event_pattern(Eterm event, Binary*, int on); + #ifdef ERTS_SMP static void smp_bp_finisher(void* arg); #endif @@ -78,6 +81,8 @@ static void new_seq_trace_token(Process* p); /* help func for seq_trace_2*/ static Eterm trace_info_pid(Process* p, Eterm pid_spec, Eterm key); static Eterm trace_info_func(Process* p, Eterm pid_spec, Eterm key); static Eterm trace_info_on_load(Process* p, Eterm key); +static Eterm trace_info_event(Process* p, Eterm event, Eterm key); + static void reset_bif_trace(void); static void setup_bif_trace(void); @@ -85,14 +90,26 @@ static void install_exp_breakpoints(BpFunctions* f); static void uninstall_exp_breakpoints(BpFunctions* f); static void clean_export_entries(BpFunctions* f); +ErtsTracingEvent erts_send_tracing[ERTS_NUM_BP_IX]; +ErtsTracingEvent erts_receive_tracing[ERTS_NUM_BP_IX]; + void erts_bif_trace_init(void) { + int i; + erts_default_trace_pattern_is_on = 0; erts_default_match_spec = NULL; erts_default_meta_match_spec = NULL; erts_default_trace_pattern_flags = erts_trace_pattern_flags_off; erts_default_meta_tracer = erts_tracer_nil; + + for (i=0; i<ERTS_NUM_BP_IX; i++) { + erts_send_tracing[i].on = 1; + erts_send_tracing[i].match_spec = NULL; + erts_receive_tracing[i].on = 1; + erts_receive_tracing[i].match_spec = NULL; + } } /* @@ -137,15 +154,18 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) on = 1; } else if (Pattern == am_restart) { match_prog_set = NULL; - on = erts_break_reset; + on = ERTS_BREAK_RESTART; } else if (Pattern == am_pause) { match_prog_set = NULL; - on = erts_break_stop; - } else if ((match_prog_set = erts_match_set_compile(p, Pattern)) != NULL) { - MatchSetRef(match_prog_set); - on = 1; - } else{ - goto error; + on = ERTS_BREAK_PAUSE; + } else { + match_prog_set = erts_match_set_compile(p, Pattern, MFA); + if (match_prog_set) { + MatchSetRef(match_prog_set); + on = 1; + } else{ + goto error; + } } is_global = 0; @@ -314,6 +334,11 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) matches = erts_set_trace_pattern(p, mfa, specified, match_prog_set, match_prog_set, on, flags, meta_tracer, 0); + } else if (is_atom(MFA)) { + if (is_global || flags.breakpoint || on > ERTS_BREAK_SET) { + goto error; + } + matches = erts_set_tracing_event_pattern(MFA, match_prog_set, on); } error: @@ -791,6 +816,8 @@ Eterm trace_info_2(BIF_ALIST_2) if (What == am_on_load) { res = trace_info_on_load(p, Key); + } else if (What == am_send || What == am_receive) { + res = trace_info_event(p, What, Key); } else if (is_atom(What) || is_pid(What) || is_port(What)) { res = trace_info_pid(p, What, Key); } else if (is_tuple(What)) { @@ -1280,6 +1307,42 @@ trace_info_on_load(Process* p, Eterm key) } } +static Eterm +trace_info_event(Process* p, Eterm event, Eterm key) +{ + ErtsTracingEvent* te; + Eterm retval; + Eterm* hp; + + switch (event) { + case am_send: te = erts_send_tracing; break; + case am_receive: te = erts_receive_tracing; break; + default: + goto error; + } + + if (key != am_match_spec) + goto error; + + te = &te[erts_active_bp_ix()]; + + if (te->on) { + if (!te->match_spec) + retval = am_true; + else + retval = copy_object(MatchSetGetSource(te->match_spec), p); + } + else + retval = am_false; + + hp = HAlloc(p, 3); + return TUPLE2(hp, key, retval); + + error: + BIF_ERROR(p, BADARG); +} + + #undef FUNC_TRACE_NOEXIST #undef FUNC_TRACE_UNTRACED #undef FUNC_TRACE_GLOBAL_TRACE @@ -1307,7 +1370,7 @@ erts_set_trace_pattern(Process*p, Eterm* mfa, int specified, for (i = 0; i < n; i++) { BeamInstr* pc = fp[i].pc; - Export* ep = (Export *)(((char *)(pc-3)) - offsetof(Export, code)); + Export* ep = ErtsContainerStruct(pc, Export, code[3]); if (on && !flags.breakpoint) { /* Turn on global call tracing */ @@ -1468,12 +1531,57 @@ erts_set_trace_pattern(Process*p, Eterm* mfa, int specified, } int +erts_set_tracing_event_pattern(Eterm event, Binary* match_spec, int on) +{ + ErtsBpIndex ix = erts_staging_bp_ix(); + ErtsTracingEvent* st; + + switch (event) { + case am_send: st = &erts_send_tracing[ix]; break; + case am_receive: st = &erts_receive_tracing[ix]; break; + default: return -1; + } + + MatchSetUnref(st->match_spec); + + st->on = on; + st->match_spec = match_spec; + MatchSetRef(match_spec); + + finish_bp.current = 1; /* prepare phase not needed for event trace */ + finish_bp.install = on; + finish_bp.e.matched = 0; + finish_bp.e.matching = NULL; + finish_bp.f.matched = 0; + finish_bp.f.matching = NULL; + +#ifndef ERTS_SMP + while (erts_finish_breakpointing()) { + /* Empty loop body */ + } +#endif + return 1; +} + +static void +consolidate_event_tracing(ErtsTracingEvent te[]) +{ + ErtsTracingEvent* src = &te[erts_active_bp_ix()]; + ErtsTracingEvent* dst = &te[erts_staging_bp_ix()]; + + MatchSetUnref(dst->match_spec); + dst->on = src->on; + dst->match_spec = src->match_spec; + MatchSetRef(dst->match_spec); +} + +int erts_finish_breakpointing(void) { ERTS_SMP_LC_ASSERT(erts_has_code_write_permission()); /* - * Memory barriers will be issued for all processes *before* + * Memory barriers will be issued for all schedulers *before* * each of the stages below. (Unless the other schedulers * are blocked, in which case memory barriers will be issued * when they are awaken.) @@ -1542,6 +1650,8 @@ erts_finish_breakpointing(void) erts_consolidate_bp_data(&finish_bp.f, 1); erts_bp_free_matched_functions(&finish_bp.e); erts_bp_free_matched_functions(&finish_bp.f); + consolidate_event_tracing(erts_send_tracing); + consolidate_event_tracing(erts_receive_tracing); return 0; default: ASSERT(0); @@ -1556,11 +1666,10 @@ install_exp_breakpoints(BpFunctions* f) BpFunction* fp = f->matching; Uint ne = f->matched; Uint i; - Uint offset = offsetof(Export, code) + 3*sizeof(BeamInstr); for (i = 0; i < ne; i++) { BeamInstr* pc = fp[i].pc; - Export* ep = (Export *) (((char *)pc)-offset); + Export* ep = ErtsContainerStruct(pc, Export, code[3]); ep->addressv[code_ix] = pc; } @@ -1573,11 +1682,10 @@ uninstall_exp_breakpoints(BpFunctions* f) BpFunction* fp = f->matching; Uint ne = f->matched; Uint i; - Uint offset = offsetof(Export, code) + 3*sizeof(BeamInstr); for (i = 0; i < ne; i++) { BeamInstr* pc = fp[i].pc; - Export* ep = (Export *) (((char *)pc)-offset); + Export* ep = ErtsContainerStruct(pc, Export, code[3]); if (ep->addressv[code_ix] != pc) { continue; @@ -1594,11 +1702,10 @@ clean_export_entries(BpFunctions* f) BpFunction* fp = f->matching; Uint ne = f->matched; Uint i; - Uint offset = offsetof(Export, code) + 3*sizeof(BeamInstr); for (i = 0; i < ne; i++) { BeamInstr* pc = fp[i].pc; - Export* ep = (Export *) (((char *)pc)-offset); + Export* ep = ErtsContainerStruct(pc, Export, code[3]); if (ep->addressv[code_ix] == pc) { continue; @@ -2277,7 +2384,7 @@ reply_trace_delivered_all(void *vtdarp) #ifdef ERTS_SMP erts_send_sys_msg_proc(rp->common.id, rp->common.id, msg, bp); #else - erts_queue_message(rp, &rp_locks, mp, msg); + erts_queue_message(rp, rp_locks, mp, msg, am_system); #endif erts_free(ERTS_ALC_T_MISC_AUX_WORK, vtdarp); diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 615d23402b..acaca54e9a 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -2833,7 +2833,8 @@ BIF_RETTYPE ets_match_spec_run_r_3(BIF_ALIST_3) BIF_TRAP3(bif_export[BIF_ets_match_spec_run_r_3], BIF_P,lst,BIF_ARG_2,ret); } - res = db_prog_match(BIF_P, mp, CAR(list_val(lst)), NULL, 0, + res = db_prog_match(BIF_P, BIF_P, + mp, CAR(list_val(lst)), NULL, 0, ERTS_PAM_COPY_RESULT, &dummy); if (is_value(res)) { hp = HAlloc(BIF_P, 2); diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index 76b96637ae..95b1cd0148 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -123,6 +123,9 @@ do { \ #define TermWords(t) (((t) / (sizeof(UWord)/sizeof(Eterm))) + !!((t) % (sizeof(UWord)/sizeof(Eterm)))) +#define add_dmc_err(EINFO, STR, VAR, TERM, SEV) \ + vadd_dmc_err(EINFO, SEV, VAR, STR, TERM) + static ERTS_INLINE Process * get_proc(Process *cp, Uint32 cp_locks, Eterm id, Uint32 id_locks) @@ -411,17 +414,27 @@ get_match_pseudo_process(Process *c_p, Uint heap_size) { ErtsMatchPseudoProcess *mpsp; #ifdef ERTS_SMP - mpsp = (ErtsMatchPseudoProcess *) c_p->scheduler_data->match_pseudo_process; - if (mpsp) + ErtsSchedulerData *esdp; + + esdp = c_p ? c_p->scheduler_data : erts_get_scheduler_data(); + + mpsp = esdp ? esdp->match_pseudo_process : + (ErtsMatchPseudoProcess*) erts_smp_tsd_get(match_pseudo_process_key); + + if (mpsp) { + ASSERT(mpsp == erts_smp_tsd_get(match_pseudo_process_key)); + ASSERT(mpsp->process.scheduler_data == esdp); cleanup_match_pseudo_process(mpsp, 0); + } else { ASSERT(erts_smp_tsd_get(match_pseudo_process_key) == NULL); mpsp = create_match_pseudo_process(); - c_p->scheduler_data->match_pseudo_process = (void *) mpsp; + if (esdp) { + esdp->match_pseudo_process = (void *) mpsp; + } + mpsp->process.scheduler_data = esdp; erts_smp_tsd_set(match_pseudo_process_key, (void *) mpsp); } - ASSERT(mpsp == erts_smp_tsd_get(match_pseudo_process_key)); - mpsp->process.scheduler_data = c_p->scheduler_data; #else mpsp = match_pseudo_process; cleanup_match_pseudo_process(mpsp, 0); @@ -889,11 +902,7 @@ void db_match_dis(Binary *prog); #define TRACE /* Nothing */ #define FENCE_PATTERN_SIZE 0 #endif -static void add_dmc_err(DMCErrInfo *err_info, - char *str, - int variable, - Eterm term, - DMCErrorSeverity severity); +static void vadd_dmc_err(DMCErrInfo*, DMCErrorSeverity, int var, const char *str, ...); static Eterm dpm_array_to_list(Process *psp, Eterm *arr, int arity); @@ -989,12 +998,20 @@ Eterm erts_match_set_get_source(Binary *mpsp) } /* This one is for the tracing */ -Binary *erts_match_set_compile(Process *p, Eterm matchexpr) { +Binary *erts_match_set_compile(Process *p, Eterm matchexpr, Eterm MFA) { Binary *bin; Uint sz; Eterm *hp; + Uint flags; + + switch (MFA) { + case am_receive: flags = DCOMP_TRACE; break; + case am_send: flags = DCOMP_TRACE | DCOMP_ALLOW_TRACE_OPS; break; + default: + flags = DCOMP_TRACE | DCOMP_CALL_TRACE | DCOMP_ALLOW_TRACE_OPS; + } - bin = db_match_set_compile(p, matchexpr, DCOMP_TRACE); + bin = db_match_set_compile(p, matchexpr, flags); if (bin != NULL) { MatchProg *prog = Binary2MatchProg(bin); sz = size_object(matchexpr); @@ -1124,8 +1141,8 @@ Eterm db_match_set_lint(Process *p, Eterm matchexpr, Uint flags) int i; if (!is_list(matchexpr)) { - add_dmc_err(err_info, "Match programs are not in a list.", - -1, 0UL, dmcError); + add_dmc_err(err_info, "Match programs are not in a list.", + -1, 0UL, dmcError); goto done; } num_heads = 0; @@ -1133,9 +1150,8 @@ Eterm db_match_set_lint(Process *p, Eterm matchexpr, Uint flags) ++num_heads; if (l != NIL) { /* proper list... */ - add_dmc_err(err_info, "Match programs are not in a proper " - "list.", - -1, 0UL, dmcError); + add_dmc_err(err_info, "Match programs are not in a proper list.", + -1, 0UL, dmcError); goto done; } @@ -1202,30 +1218,37 @@ done: return ret; } -Eterm erts_match_set_run(Process *p, Binary *mpsp, - Eterm *args, int num_args, - enum erts_pam_run_flags in_flags, - Uint32 *return_flags) +/* Returns + * am_false if no match or + * if {message,false} has been called, + * am_true if {message,_} has NOT been called or + * if {message,true} has been called, + * Msg if {message,Msg} has been called. + * + * If return value is_not_immed + * then erts_match_set_release_result_trace() must be called to release it. + */ +Eterm erts_match_set_run_trace(Process *c_p, + Process *self, + Binary *mpsp, + Eterm *args, int num_args, + enum erts_pam_run_flags in_flags, + Uint32 *return_flags) { Eterm ret; - ret = db_prog_match(p, mpsp, NIL, args, num_args, + ret = db_prog_match(c_p, self, mpsp, NIL, args, num_args, in_flags, return_flags); -#if defined(HARDDEBUG) - if (is_non_value(ret)) { - erts_fprintf(stderr, "Failed\n"); - } else { - erts_fprintf(stderr, "Returning : %T\n", ret); + + ASSERT(!(is_non_value(ret) && *return_flags)); + + if (is_non_value(ret) || ret == am_false) { + erts_match_set_release_result(c_p); + return am_false; } -#endif + if (is_immed(ret)) + erts_match_set_release_result(c_p); return ret; - /* Returns - * THE_NON_VALUE if no match - * am_false if {message,false} has been called, - * am_true if {message,_} has not been called or - * if {message,true} has been called, - * Msg if {message,Msg} has been called. - */ } static Eterm erts_match_set_run_ets(Process *p, Binary *mpsp, @@ -1234,7 +1257,8 @@ static Eterm erts_match_set_run_ets(Process *p, Binary *mpsp, { Eterm ret; - ret = db_prog_match(p, mpsp, args, NULL, num_args, + ret = db_prog_match(p, p, + mpsp, args, NULL, num_args, ERTS_PAM_COPY_RESULT, return_flags); #if defined(HARDDEBUG) @@ -1730,7 +1754,9 @@ static Eterm dpm_array_to_list(Process *psp, Eterm *arr, int arity) ** the parameter 'arity' is only used if 'term' is actually an array, ** i.e. 'DCOMP_TRACE' was specified */ -Eterm db_prog_match(Process *c_p, Binary *bprog, +Eterm db_prog_match(Process *c_p, + Process *self, + Binary *bprog, Eterm term, Eterm *termp, int arity, @@ -1743,10 +1769,10 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm *esp; MatchVariable* variables; BeamInstr *cp; - UWord *pc = prog->text; + const UWord *pc = prog->text; Eterm *ehp; Eterm ret; - Uint n = 0; /* To avoid warning. */ + Uint n; int i; unsigned do_catch; ErtsMatchPseudoProcess *mpsp; @@ -1758,46 +1784,28 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm (*bif)(Process*, ...); Eterm bif_args[3]; int fail_label; - int atomic_trace; #ifdef DMC_DEBUG Uint *heap_fence; Uint *stack_fence; Uint save_op; #endif /* DMC_DEBUG */ + ERTS_UNDEF(n,0); + ERTS_UNDEF(current_scheduled,NULL); + + ASSERT(c_p || !(in_flags & ERTS_PAM_COPY_RESULT)); + mpsp = get_match_pseudo_process(c_p, prog->heap_size); psp = &mpsp->process; /* We need to lure the scheduler into believing in the pseudo process, because of floating point exceptions. Do *after* mpsp is set!!! */ - esdp = ERTS_GET_SCHEDULER_DATA_FROM_PROC(c_p); - ASSERT(esdp != NULL); - current_scheduled = esdp->current_process; + esdp = ERTS_GET_SCHEDULER_DATA_FROM_PROC(psp); + if (esdp) + current_scheduled = esdp->current_process; /* SMP: psp->scheduler_data is set by get_match_pseudo_process */ - atomic_trace = 0; -#define BEGIN_ATOMIC_TRACE(p) \ - do { \ - if (! atomic_trace) { \ - erts_refc_inc(&bprog->refc, 2); \ - erts_smp_proc_unlock((p), ERTS_PROC_LOCK_MAIN); \ - erts_smp_thr_progress_block(); \ - atomic_trace = !0; \ - } \ - } while (0) -#define END_ATOMIC_TRACE(p) \ - do { \ - if (atomic_trace) { \ - erts_smp_thr_progress_unblock(); \ - erts_smp_proc_lock((p), ERTS_PROC_LOCK_MAIN); \ - if (erts_refc_dectest(&bprog->refc, 0) == 0) {\ - erts_bin_free(bprog); \ - } \ - atomic_trace = 0; \ - } \ - } while (0) - #ifdef DMC_DEBUG save_op = 0; heap_fence = (Eterm*)((char*) mpsp->u.heap + prog->stack_offset) - 1; @@ -2256,7 +2264,7 @@ restart: pc += n; break; case matchSelf: - *esp++ = c_p->common.id; + *esp++ = self->common.id; break; case matchWaste: --esp; @@ -2266,6 +2274,7 @@ restart: break; case matchProcessDump: { erts_dsprintf_buf_t *dsbufp = erts_create_tmp_dsbuf(0); + ASSERT(c_p == self); print_process_info(ERTS_PRINT_DSBUF, (void *) dsbufp, c_p); *esp++ = new_binary(build_proc, (byte *)dsbufp->str, dsbufp->str_len); @@ -2284,14 +2293,16 @@ restart: *return_flags |= MATCH_SET_EXCEPTION_TRACE; *esp++ = am_true; break; - case matchIsSeqTrace: + case matchIsSeqTrace: + ASSERT(c_p == self); if (have_seqtrace(SEQ_TRACE_TOKEN(c_p))) *esp++ = am_true; else *esp++ = am_false; break; case matchSetSeqToken: - t = erts_seq_trace(c_p, esp[-1], esp[-2], 0); + ASSERT(c_p == self); + t = erts_seq_trace(c_p, esp[-1], esp[-2], 0); if (is_non_value(t)) { esp[-2] = FAIL_TERM; } else { @@ -2299,7 +2310,8 @@ restart: } --esp; break; - case matchSetSeqTokenFake: + case matchSetSeqTokenFake: + ASSERT(c_p == self); t = seq_trace_fake(c_p, esp[-1]); if (is_non_value(t)) { esp[-2] = FAIL_TERM; @@ -2308,7 +2320,8 @@ restart: } --esp; break; - case matchGetSeqToken: + case matchGetSeqToken: + ASSERT(c_p == self); if (have_no_seqtrace(SEQ_TRACE_TOKEN(c_p))) *esp++ = NIL; else { @@ -2332,49 +2345,62 @@ restart: ASSERT(is_immed(ehp[5])); } break; - case matchEnableTrace: + case matchEnableTrace: + ASSERT(c_p == self); if ( (n = erts_trace_flag2bit(esp[-1]))) { - BEGIN_ATOMIC_TRACE(c_p); + erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); set_tracee_flags(c_p, ERTS_TRACER(c_p), 0, n); + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); esp[-1] = am_true; } else { esp[-1] = FAIL_TERM; } break; - case matchEnableTrace2: + case matchEnableTrace2: + ASSERT(c_p == self); n = erts_trace_flag2bit((--esp)[-1]); esp[-1] = FAIL_TERM; if (n) { - BEGIN_ATOMIC_TRACE(c_p); - if ( (tmpp = get_proc(c_p, 0, esp[0], 0))) { + if ( (tmpp = get_proc(c_p, ERTS_PROC_LOCK_MAIN, esp[0], ERTS_PROC_LOCKS_ALL))) { /* Always take over the tracer of the current process */ set_tracee_flags(tmpp, ERTS_TRACER(c_p), 0, n); - esp[-1] = am_true; + if (tmpp == c_p) + erts_smp_proc_unlock(tmpp, ERTS_PROC_LOCKS_ALL_MINOR); + else + erts_smp_proc_unlock(tmpp, ERTS_PROC_LOCKS_ALL); + esp[-1] = am_true; } } break; - case matchDisableTrace: + case matchDisableTrace: + ASSERT(c_p == self); if ( (n = erts_trace_flag2bit(esp[-1]))) { - BEGIN_ATOMIC_TRACE(c_p); + erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); set_tracee_flags(c_p, ERTS_TRACER(c_p), n, 0); + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); esp[-1] = am_true; } else { esp[-1] = FAIL_TERM; } break; - case matchDisableTrace2: + case matchDisableTrace2: + ASSERT(c_p == self); n = erts_trace_flag2bit((--esp)[-1]); esp[-1] = FAIL_TERM; if (n) { - BEGIN_ATOMIC_TRACE(c_p); - if ( (tmpp = get_proc(c_p, 0, esp[0], 0))) { + if ( (tmpp = get_proc(c_p, ERTS_PROC_LOCK_MAIN, esp[0], ERTS_PROC_LOCKS_ALL))) { /* Always take over the tracer of the current process */ set_tracee_flags(tmpp, ERTS_TRACER(c_p), n, 0); - esp[-1] = am_true; + if (tmpp == c_p) + erts_smp_proc_unlock(tmpp, ERTS_PROC_LOCKS_ALL_MINOR); + else + erts_smp_proc_unlock(tmpp, ERTS_PROC_LOCKS_ALL); + esp[-1] = am_true; } } break; - case matchCaller: + case matchCaller: + ASSERT(c_p == self); if (!(c_p->cp) || !(cp = find_function_from_pc(c_p->cp))) { *esp++ = am_undefined; } else { @@ -2386,7 +2412,8 @@ restart: ehp[3] = make_small((Uint) cp[2]); } break; - case matchSilent: + case matchSilent: + ASSERT(c_p == self); --esp; if (in_flags & ERTS_PAM_IGNORE_TRACE_SILENT) break; @@ -2401,7 +2428,8 @@ restart: erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); } break; - case matchTrace2: + case matchTrace2: + ASSERT(c_p == self); { /* disable enable */ Uint d_flags = 0, e_flags = 0; /* process trace flags */ @@ -2433,7 +2461,8 @@ restart: ERTS_TRACER_CLEAR(&tracer); } break; - case matchTrace3: + case matchTrace3: + ASSERT(c_p == self); { /* disable enable */ Uint d_flags = 0, e_flags = 0; /* process trace flags */ @@ -2506,15 +2535,12 @@ success: } #endif - esdp->current_process = current_scheduled; - - END_ATOMIC_TRACE(c_p); + if (esdp) + esdp->current_process = current_scheduled; return ret; #undef FAIL #undef FAIL_TERM -#undef BEGIN_ATOMIC_TRACE -#undef END_ATOMIC_TRACE } @@ -3259,20 +3285,20 @@ int erts_db_is_compiled_ms(Eterm term) ** Utility to add an error */ -static void add_dmc_err(DMCErrInfo *err_info, - char *str, - int variable, - Eterm term, - DMCErrorSeverity severity) +static void vadd_dmc_err(DMCErrInfo *err_info, + DMCErrorSeverity severity, + int variable, + const char *str, + ...) { + DMCError *e; + va_list args; + va_start(args, str); + + /* Linked in in reverse order, to ease the formatting */ - DMCError *e = erts_alloc(ERTS_ALC_T_DB_DMC_ERROR, sizeof(DMCError)); - if (term != 0UL) { - erts_snprintf(e->error_string, DMC_ERR_STR_LEN, str, term); - } else { - strncpy(e->error_string, str, DMC_ERR_STR_LEN); - e->error_string[DMC_ERR_STR_LEN] ='\0'; - } + e = erts_alloc(ERTS_ALC_T_DB_DMC_ERROR, sizeof(DMCError)); + erts_vsnprintf(e->error_string, DMC_ERR_STR_LEN, str, args); e->variable = variable; e->severity = severity; e->next = err_info->first; @@ -3282,8 +3308,11 @@ static void add_dmc_err(DMCErrInfo *err_info, err_info->first = e; if (severity >= dmcError) err_info->error_added = 1; + + va_end(args); } + /* ** Handle one term in the match expression (not the guard) */ @@ -3482,24 +3511,21 @@ static void do_emit_constant(DMCContext *context, DMC_STACK_TYPE(UWord) *text, context->stack_need = context->stack_used; } -#define RETURN_ERROR_X(String, X, Y, ContextP, ConstantF) \ -do { \ -if ((ContextP)->err_info != NULL) { \ - (ConstantF) = 0; \ - add_dmc_err((ContextP)->err_info, String, X, Y, dmcError); \ - return retOk; \ -} else \ - return retFail; \ -} while(0) +#define RETURN_ERROR_X(VAR, ContextP, ConstantF, String, ARG) \ + (((ContextP)->err_info != NULL) \ + ? ((ConstantF) = 0, \ + vadd_dmc_err((ContextP)->err_info, dmcError, VAR, String, ARG), \ + retOk) \ + : retFail) #define RETURN_ERROR(String, ContextP, ConstantF) \ - RETURN_ERROR_X(String, -1, 0UL, ContextP, ConstantF) + return RETURN_ERROR_X(-1, ContextP, ConstantF, String, 0) #define RETURN_VAR_ERROR(String, N, ContextP, ConstantF) \ - RETURN_ERROR_X(String, N, 0UL, ContextP, ConstantF) + return RETURN_ERROR_X(N, ContextP, ConstantF, String, 0) #define RETURN_TERM_ERROR(String, T, ContextP, ConstantF) \ - RETURN_ERROR_X(String, -1, T, ContextP, ConstantF) + return RETURN_ERROR_X(-1, ContextP, ConstantF, String, T) #define WARNING(String, ContextP) \ add_dmc_err((ContextP)->err_info, String, -1, 0UL, dmcWarning) @@ -3765,7 +3791,7 @@ static DMCRet dmc_variable(DMCContext *context, Uint n = db_is_variable(t); if (n >= heap->vars_used || !heap->vars[n].is_bound) { - RETURN_VAR_ERROR("Variable $%d is unbound.", n, context, *constant); + RETURN_VAR_ERROR("Variable $%%d is unbound.", n, context, *constant); } dmc_add_pushv_variant(context, heap, text, n); @@ -4097,7 +4123,30 @@ static DMCRet dmc_exception_trace(DMCContext *context, return retOk; } - +static int check_trace(const char* op, + DMCContext *context, + int *constant, + int need_cflags, + int allow_in_guard, + DMCRet* retp) +{ + if (!(context->cflags & DCOMP_TRACE)) { + *retp = RETURN_ERROR_X(-1, context, *constant, "Special form '%s' " + "used in wrong dialect.", op); + return 0; + } + if ((context->cflags & need_cflags) != need_cflags) { + *retp = RETURN_ERROR_X(-1, context, *constant, "Special form '%s' " + "not allow for this trace event.", op); + return 0; + } + if (context->is_guard && !allow_in_guard) { + *retp = RETURN_ERROR_X(-1, context, *constant, "Special form '%s' " + "called in guard context.", op); + return 0; + } + return 1; +} static DMCRet dmc_is_seq_trace(DMCContext *context, DMCHeap *heap, @@ -4107,12 +4156,11 @@ static DMCRet dmc_is_seq_trace(DMCContext *context, { Eterm *p = tuple_val(t); Uint a = arityval(*p); + DMCRet ret; - if (!(context->cflags & DCOMP_TRACE)) { - RETURN_ERROR("Special form 'is_seq_trace' used in wrong dialect.", - context, - *constant); - } + if (!check_trace("is_seq_trace", context, constant, DCOMP_ALLOW_TRACE_OPS, 1, &ret)) + return ret; + if (a != 1) { RETURN_TERM_ERROR("Special form 'is_seq_trace' called with " "arguments in %T.", t, context, *constant); @@ -4136,16 +4184,8 @@ static DMCRet dmc_set_seq_token(DMCContext *context, DMCRet ret; int c; - - if (!(context->cflags & DCOMP_TRACE)) { - RETURN_ERROR("Special form 'set_seq_token' used in wrong dialect.", - context, - *constant); - } - if (context->is_guard) { - RETURN_ERROR("Special form 'set_seq_token' called in " - "guard context.", context, *constant); - } + if (!check_trace("set_seq_trace", context, constant, DCOMP_ALLOW_TRACE_OPS, 0, &ret)) + return ret; if (a != 3) { RETURN_TERM_ERROR("Special form 'set_seq_token' called with wrong " @@ -4182,16 +4222,11 @@ static DMCRet dmc_get_seq_token(DMCContext *context, { Eterm *p = tuple_val(t); Uint a = arityval(*p); + DMCRet ret; + + if (!check_trace("get_seq_token", context, constant, DCOMP_ALLOW_TRACE_OPS, 0, &ret)) + return ret; - if (!(context->cflags & DCOMP_TRACE)) { - RETURN_ERROR("Special form 'get_seq_token' used in wrong dialect.", - context, - *constant); - } - if (context->is_guard) { - RETURN_ERROR("Special form 'get_seq_token' called in " - "guard context.", context, *constant); - } if (a != 1) { RETURN_TERM_ERROR("Special form 'get_seq_token' called with " "arguments in %T.", t, context, @@ -4255,16 +4290,10 @@ static DMCRet dmc_process_dump(DMCContext *context, { Eterm *p = tuple_val(t); Uint a = arityval(*p); - - if (!(context->cflags & DCOMP_TRACE)) { - RETURN_ERROR("Special form 'process_dump' used in wrong dialect.", - context, - *constant); - } - if (context->is_guard) { - RETURN_ERROR("Special form 'process_dump' called in " - "guard context.", context, *constant); - } + DMCRet ret; + + if (!check_trace("process_dump", context, constant, DCOMP_ALLOW_TRACE_OPS, 0, &ret)) + return ret; if (a != 1) { RETURN_TERM_ERROR("Special form 'process_dump' called with " @@ -4288,17 +4317,8 @@ static DMCRet dmc_enable_trace(DMCContext *context, DMCRet ret; int c; - - if (!(context->cflags & DCOMP_TRACE)) { - RETURN_ERROR("Special form 'enable_trace' used in wrong dialect.", - context, - *constant); - } - if (context->is_guard) { - RETURN_ERROR("Special form 'enable_trace' called in guard context.", - context, - *constant); - } + if (!check_trace("enable_trace", context, constant, DCOMP_ALLOW_TRACE_OPS, 0, &ret)) + return ret; switch (a) { case 2: @@ -4347,18 +4367,9 @@ static DMCRet dmc_disable_trace(DMCContext *context, Uint a = arityval(*p); DMCRet ret; int c; - - if (!(context->cflags & DCOMP_TRACE)) { - RETURN_ERROR("Special form 'disable_trace' used in wrong dialect.", - context, - *constant); - } - if (context->is_guard) { - RETURN_ERROR("Special form 'disable_trace' called in guard context.", - context, - *constant); - } + if (!check_trace("disable_trace", context, constant, DCOMP_ALLOW_TRACE_OPS, 0, &ret)) + return ret; switch (a) { case 2: @@ -4408,17 +4419,8 @@ static DMCRet dmc_trace(DMCContext *context, DMCRet ret; int c; - - if (!(context->cflags & DCOMP_TRACE)) { - RETURN_ERROR("Special form 'trace' used in wrong dialect.", - context, - *constant); - } - if (context->is_guard) { - RETURN_ERROR("Special form 'trace' called in guard context.", - context, - *constant); - } + if (!check_trace("trace", context, constant, DCOMP_ALLOW_TRACE_OPS, 0, &ret)) + return ret; switch (a) { case 3: @@ -4479,16 +4481,11 @@ static DMCRet dmc_caller(DMCContext *context, { Eterm *p = tuple_val(t); Uint a = arityval(*p); + DMCRet ret; - if (!(context->cflags & DCOMP_TRACE)) { - RETURN_ERROR("Special form 'caller' used in wrong dialect.", - context, - *constant); - } - if (context->is_guard) { - RETURN_ERROR("Special form 'caller' called in " - "guard context.", context, *constant); - } + if (!check_trace("caller", context, constant, + (DCOMP_CALL_TRACE|DCOMP_ALLOW_TRACE_OPS), 0, &ret)) + return ret; if (a != 1) { RETURN_TERM_ERROR("Special form 'caller' called with " @@ -4514,15 +4511,8 @@ static DMCRet dmc_silent(DMCContext *context, DMCRet ret; int c; - if (!(context->cflags & DCOMP_TRACE)) { - RETURN_ERROR("Special form 'silent' used in wrong dialect.", - context, - *constant); - } - if (context->is_guard) { - RETURN_ERROR("Special form 'silent' called in " - "guard context.", context, *constant); - } + if (!check_trace("silent", context, constant, DCOMP_ALLOW_TRACE_OPS, 0, &ret)) + return ret; if (a != 2) { RETURN_TERM_ERROR("Special form 'silent' called with wrong " @@ -5062,11 +5052,14 @@ static Eterm match_spec_test(Process *p, Eterm against, Eterm spec, int trace) return THE_NON_VALUE; } if (trace) { - lint_res = db_match_set_lint(p, spec, DCOMP_TRACE | DCOMP_FAKE_DESTRUCTIVE); - mps = db_match_set_compile(p, spec, DCOMP_TRACE | DCOMP_FAKE_DESTRUCTIVE); + const Uint cflags = (DCOMP_TRACE | DCOMP_FAKE_DESTRUCTIVE | + DCOMP_CALL_TRACE | DCOMP_ALLOW_TRACE_OPS); + lint_res = db_match_set_lint(p, spec, cflags); + mps = db_match_set_compile(p, spec, cflags); } else { - lint_res = db_match_set_lint(p, spec, DCOMP_TABLE | DCOMP_FAKE_DESTRUCTIVE); - mps = db_match_set_compile(p, spec, DCOMP_TABLE | DCOMP_FAKE_DESTRUCTIVE); + const Uint cflags = (DCOMP_TABLE | DCOMP_FAKE_DESTRUCTIVE); + lint_res = db_match_set_lint(p, spec, cflags); + mps = db_match_set_compile(p, spec, cflags); } if (mps == NULL) { @@ -5099,7 +5092,8 @@ static Eterm match_spec_test(Process *p, Eterm against, Eterm spec, int trace) } save_cp = p->cp; p->cp = NULL; - res = erts_match_set_run(p, mps, arr, n, + res = erts_match_set_run_trace(p, p, + mps, arr, n, ERTS_PAM_COPY_RESULT|ERTS_PAM_IGNORE_TRACE_SILENT, &ret_flags); p->cp = save_cp; @@ -5182,7 +5176,8 @@ Eterm db_match_dbterm(DbTableCommon* tb, Process* c_p, Binary* bprog, obj = db_alloc_tmp_uncompressed(tb, obj); } - res = db_prog_match(c_p, bprog, make_tuple(obj->tpl), NULL, 0, + res = db_prog_match(c_p, c_p, + bprog, make_tuple(obj->tpl), NULL, 0, ERTS_PAM_COPY_RESULT|ERTS_PAM_CONTIGUOUS_TUPLE, &dummy); if (is_value(res) && hpp!=NULL) { diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index 5d7946a525..60f7067d70 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -425,6 +425,11 @@ typedef struct dmc_err_info { #define DCOMP_FAKE_DESTRUCTIVE ((Uint) 8) /* When this is active, no setting of trace control words or seq_trace tokens will be done. */ +/* Allow lock seizing operations on the tracee and 3rd party processes */ +#define DCOMP_ALLOW_TRACE_OPS ((Uint) 0x10) + +/* This is call trace */ +#define DCOMP_CALL_TRACE ((Uint) 0x20) Binary *db_match_compile(Eterm *matchexpr, Eterm *guards, Eterm *body, int num_matches, @@ -435,7 +440,8 @@ Binary *db_match_compile(Eterm *matchexpr, Eterm *guards, Eterm db_match_dbterm(DbTableCommon* tb, Process* c_p, Binary* bprog, int all, DbTerm* obj, Eterm** hpp, Uint extra); -Eterm db_prog_match(Process *p, Binary *prog, Eterm term, +Eterm db_prog_match(Process *p, Process *self, + Binary *prog, Eterm term, Eterm *termp, int arity, enum erts_pam_run_flags in_flags, Uint32 *return_flags /* Zeroed on enter */); diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index f33ade27f3..bed7e668d7 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -116,7 +116,7 @@ static Eterm *full_sweep_heaps(Process *p, char *oh, Uint oh_size, Eterm *objv, int nobj); static int garbage_collect(Process* p, ErlHeapFragment *live_hf_end, - int need, Eterm* objv, int nobj); + int need, Eterm* objv, int nobj, int fcalls); static int major_collection(Process* p, ErlHeapFragment *live_hf_end, int need, Eterm* objv, int nobj, Uint *recl); static int minor_collection(Process* p, ErlHeapFragment *live_hf_end, @@ -392,15 +392,15 @@ erts_gc_after_bif_call_lhf(Process* p, ErlHeapFragment *live_hf_end, regs = ERTS_PROC_GET_SCHDATA(p)->x_reg_array; } #endif - cost = garbage_collect(p, live_hf_end, 0, regs, p->arity); + cost = garbage_collect(p, live_hf_end, 0, regs, p->arity, p->fcalls); } else { - cost = garbage_collect(p, live_hf_end, 0, regs, arity); + cost = garbage_collect(p, live_hf_end, 0, regs, arity, p->fcalls); } } else { Eterm val[1]; val[0] = result; - cost = garbage_collect(p, live_hf_end, 0, val, 1); + cost = garbage_collect(p, live_hf_end, 0, val, 1, p->fcalls); result = val[0]; } BUMP_REDS(p, cost); @@ -431,7 +431,7 @@ static ERTS_INLINE void reset_active_writer(Process *p) #define ERTS_ABANDON_HEAP_COST 10 static int -delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need) +delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need, int fcalls) { ErlHeapFragment *hfrag; Eterm *orig_heap, *orig_hend, *orig_htop, *orig_stop; @@ -506,12 +506,16 @@ delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need) /* Make sure that we do a proper GC as soon as possible... */ p->flags |= F_FORCE_GC; - reds_left = ERTS_BIF_REDS_LEFT(p); + reds_left = ERTS_REDS_LEFT(p, fcalls); + ASSERT(CONTEXT_REDS - reds_left >= ERTS_PROC_GET_SCHDATA(p)->virtual_reds); + if (reds_left > ERTS_ABANDON_HEAP_COST) { int vreds = reds_left - ERTS_ABANDON_HEAP_COST; - ERTS_VBUMP_REDS(p, vreds); + ERTS_PROC_GET_SCHDATA((p))->virtual_reds += vreds; } - return ERTS_ABANDON_HEAP_COST; + + ASSERT(CONTEXT_REDS >= ERTS_PROC_GET_SCHDATA(p)->virtual_reds); + return reds_left; } static ERTS_FORCE_INLINE Uint @@ -570,7 +574,7 @@ young_gen_usage(Process *p) */ static int garbage_collect(Process* p, ErlHeapFragment *live_hf_end, - int need, Eterm* objv, int nobj) + int need, Eterm* objv, int nobj, int fcalls) { Uint reclaimed_now = 0; int reds; @@ -581,8 +585,11 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end, DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE); #endif + ASSERT(CONTEXT_REDS - ERTS_REDS_LEFT(p, fcalls) + >= ERTS_PROC_GET_SCHDATA(p)->virtual_reds); + if (p->flags & (F_DISABLE_GC|F_DELAY_GC)) - return delay_garbage_collection(p, live_hf_end, need); + return delay_garbage_collection(p, live_hf_end, need, fcalls); if (p->abandoned_heap) live_hf_end = ERTS_INVALID_HFRAG_PTR; @@ -593,10 +600,6 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end, esdp = erts_get_scheduler_data(); - if (IS_TRACED_FL(p, F_TRACE_GC)) { - trace_gc(p, am_gc_start); - } - erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC); if (erts_system_monitor_long_gc != 0) start_time = erts_get_monotonic_time(esdp); @@ -619,18 +622,29 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end, */ if (GEN_GCS(p) < MAX_GEN_GCS(p) && !(FLAGS(p) & F_NEED_FULLSWEEP)) { - DTRACE2(gc_minor_start, pidbuf, need); - reds = minor_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now); - DTRACE2(gc_minor_end, pidbuf, reclaimed_now); - if (reds < 0) - goto do_major_collection; - } - else { - do_major_collection: + if (IS_TRACED_FL(p, F_TRACE_GC)) { + trace_gc(p, am_gc_minor_start, need); + } + DTRACE2(gc_minor_start, pidbuf, need); + reds = minor_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now); + DTRACE2(gc_minor_end, pidbuf, reclaimed_now); + if (IS_TRACED_FL(p, F_TRACE_GC)) { + trace_gc(p, am_gc_minor_end, reclaimed_now); + } + if (reds < 0) + goto do_major_collection; + } else { +do_major_collection: ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_GC_FULL); - DTRACE2(gc_major_start, pidbuf, need); - reds = major_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now); - DTRACE2(gc_major_end, pidbuf, reclaimed_now); + if (IS_TRACED_FL(p, F_TRACE_GC)) { + trace_gc(p, am_gc_major_start, need); + } + DTRACE2(gc_major_start, pidbuf, need); + reds = major_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now); + DTRACE2(gc_major_end, pidbuf, reclaimed_now); + if (IS_TRACED_FL(p, F_TRACE_GC)) { + trace_gc(p, am_gc_major_end, reclaimed_now); + } ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_GC); } @@ -646,10 +660,6 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end, erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC); - if (IS_TRACED_FL(p, F_TRACE_GC)) { - trace_gc(p, am_gc_end); - } - if (erts_system_monitor_long_gc != 0) { ErtsMonotonicTime end_time; Uint gc_time; @@ -700,16 +710,23 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end, } int -erts_garbage_collect_nobump(Process* p, int need, Eterm* objv, int nobj) +erts_garbage_collect_nobump(Process* p, int need, Eterm* objv, int nobj, int fcalls) { - return garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj); + int reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj, fcalls); + int reds_left = ERTS_REDS_LEFT(p, fcalls); + if (reds > reds_left) + reds = reds_left; + ASSERT(CONTEXT_REDS - (reds_left - reds) >= ERTS_PROC_GET_SCHDATA(p)->virtual_reds); + return reds; } void erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) { - int reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj); + int reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj, p->fcalls); BUMP_REDS(p, reds); + ASSERT(CONTEXT_REDS - ERTS_BIF_REDS_LEFT(p) + >= ERTS_PROC_GET_SCHDATA(p)->virtual_reds); } /* @@ -2053,8 +2070,26 @@ copy_one_frag(Eterm** hpp, ErlOffHeap* off_heap, *hp++ = val; break; case TAG_PRIMARY_LIST: +#ifdef SHCOPY_SEND + if (erts_is_literal(val,list_val(val))) { + *hp++ = val; + } else { + *hp++ = offset_ptr(val, offs); + } +#else + *hp++ = offset_ptr(val, offs); +#endif + break; case TAG_PRIMARY_BOXED: - *hp++ = offset_ptr(val, offs); +#ifdef SHCOPY_SEND + if (erts_is_literal(val,boxed_val(val))) { + *hp++ = val; + } else { + *hp++ = offset_ptr(val, offs); + } +#else + *hp++ = offset_ptr(val, offs); +#endif break; case TAG_PRIMARY_HEADER: *hp++ = val; @@ -2903,7 +2938,7 @@ reply_gc_info(void *vgcirp) hpp = &hp; } - erts_queue_message(rp, &rp_locks, mp, msg); + erts_queue_message(rp, rp_locks, mp, msg, am_system); if (gcirp->req_sched == esdp->no) rp_locks &= ~ERTS_PROC_LOCK_MAIN; @@ -2988,10 +3023,30 @@ erts_process_gc_info(Process *p, Uint *sizep, Eterm **hpp) }; Eterm res = THE_NON_VALUE; + ErtsMessage *mp; ERTS_CT_ASSERT(sizeof(values)/sizeof(*values) == sizeof(tags)/sizeof(*tags)); ERTS_CT_ASSERT(sizeof(values)/sizeof(*values) == ERTS_PROCESS_GC_INFO_MAX_TERMS); + if (p->abandoned_heap) { + Eterm *htop, *heap; + ERTS_GET_ORIG_HEAP(p, heap, htop); + values[3] = HIGH_WATER(p) - heap; + values[6] = htop - heap; + } + + if (p->flags & F_ON_HEAP_MSGQ) { + /* If on heap messages in the internal queue are counted + as being part of the heap, so we have to add them to the + am_mbuf_size value. process_info(total_heap_size) should + be the same as adding old_heap_block_size + heap_block_size + + mbuf_size. + */ + for (mp = p->msg.first; mp; mp = mp->next) + if (mp->data.attached) + values[2] += erts_msg_attached_data_size(mp); + } + res = erts_bld_atom_uword_2tup_list(hpp, sizep, sizeof(values)/sizeof(*values), diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h index 40b7c5d12c..8a6ff99990 100644 --- a/erts/emulator/beam/erl_gc.h +++ b/erts/emulator/beam/erl_gc.h @@ -139,7 +139,7 @@ Eterm erts_process_gc_info(struct process*, Uint *, Eterm **); void erts_gc_info(ErtsGCInfo *gcip); void erts_init_gc(void); -int erts_garbage_collect_nobump(struct process*, int, Eterm*, int); +int erts_garbage_collect_nobump(struct process*, int, Eterm*, int, int); void erts_garbage_collect(struct process*, int, Eterm*, int); void erts_garbage_collect_hibernate(struct process* p); Eterm erts_gc_after_bif_call_lhf(struct process* p, ErlHeapFragment *live_hf_end, diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c index 8e201d5711..c418762578 100644 --- a/erts/emulator/beam/erl_hl_timer.c +++ b/erts/emulator/beam/erl_hl_timer.c @@ -1247,8 +1247,8 @@ hlt_bif_timer_timeout(ErtsHLTimer *tmr, Uint32 roflgs) if (!ERTS_PROC_IS_EXITING(proc)) { ErtsMessage *mp = erts_alloc_message(0, NULL); mp->data.heap_frag = tmr->btm.bp; - erts_queue_message(proc, &proc_locks, mp, - tmr->btm.message); + erts_queue_message(proc, proc_locks, mp, + tmr->btm.message, am_clock_service); erts_smp_proc_unlock(proc, ERTS_PROC_LOCKS_MSG_SEND); queued_message = 1; proc_locks &= ~ERTS_PROC_LOCKS_MSG_SEND; @@ -1980,7 +1980,7 @@ access_sched_local_btm(Process *c_p, Eterm pid, ERTS_HLT_ASSERT(hp + (async ? 4 : 3) == hp_end); - erts_queue_message(proc, &proc_locks, mp, msg); + erts_queue_message(proc, proc_locks, mp, msg, am_clock_service); if (c_p) proc_locks &= ~ERTS_PROC_LOCK_MAIN; @@ -2111,7 +2111,7 @@ try_access_sched_remote_btm(ErtsSchedulerData *esdp, msg = TUPLE3(hp, tag, tref, res); - erts_queue_message(c_p, &proc_locks, mp, msg); + erts_queue_message(c_p, proc_locks, mp, msg, am_clock_service); proc_locks &= ~ERTS_PROC_LOCK_MAIN; if (proc_locks) diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index dec9bdfedc..fcd2739ac3 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -206,15 +206,15 @@ int erts_no_line_info = 0; /* -L: Don't load line information */ ErtsModifiedTimings erts_modified_timings[] = { /* 0 */ {make_small(0), CONTEXT_REDS, INPUT_REDUCTIONS}, - /* 1 */ {make_small(0), 2*CONTEXT_REDS, 2*INPUT_REDUCTIONS}, + /* 1 */ {make_small(0), (3*CONTEXT_REDS)/4, 2*INPUT_REDUCTIONS}, /* 2 */ {make_small(0), CONTEXT_REDS/2, INPUT_REDUCTIONS/2}, - /* 3 */ {make_small(0), 3*CONTEXT_REDS, 3*INPUT_REDUCTIONS}, + /* 3 */ {make_small(0), (7*CONTEXT_REDS)/8, 3*INPUT_REDUCTIONS}, /* 4 */ {make_small(0), CONTEXT_REDS/3, 3*INPUT_REDUCTIONS}, - /* 5 */ {make_small(0), 4*CONTEXT_REDS, INPUT_REDUCTIONS/2}, + /* 5 */ {make_small(0), (10*CONTEXT_REDS)/11, INPUT_REDUCTIONS/2}, /* 6 */ {make_small(1), CONTEXT_REDS/4, 2*INPUT_REDUCTIONS}, - /* 7 */ {make_small(1), 5*CONTEXT_REDS, INPUT_REDUCTIONS/3}, + /* 7 */ {make_small(1), (5*CONTEXT_REDS)/7, INPUT_REDUCTIONS/3}, /* 8 */ {make_small(10), CONTEXT_REDS/5, 3*INPUT_REDUCTIONS}, - /* 9 */ {make_small(10), 6*CONTEXT_REDS, INPUT_REDUCTIONS/4} + /* 9 */ {make_small(10), (6*CONTEXT_REDS)/7, INPUT_REDUCTIONS/4} }; #define ERTS_MODIFIED_TIMING_LEVELS \ @@ -578,6 +578,8 @@ void erts_usage(void) VH_DEFAULT_SIZE); erts_fprintf(stderr, "-hpds size initial process dictionary size (default %d)\n", erts_pd_initial_size); + erts_fprintf(stderr, "-hmqd val set default message queue data flag for processes,\n"); + erts_fprintf(stderr, " valid values are: off_heap | on_heap | mixed\n"); /* erts_fprintf(stderr, "-i module set the boot module (default init)\n"); */ @@ -655,8 +657,6 @@ void erts_usage(void) erts_fprintf(stderr, "-W<i|w|e> set error logger warnings mapping,\n"); erts_fprintf(stderr, " see error_logger documentation for details\n"); - erts_fprintf(stderr, "-xmqd val set default message queue data flag for processes,\n"); - erts_fprintf(stderr, " valid values are: off_heap | on_heap | mixed\n"); erts_fprintf(stderr, "-zdbbl size set the distribution buffer busy limit in kilobytes\n"); erts_fprintf(stderr, " valid range is [1-%d]\n", INT_MAX/1024); erts_fprintf(stderr, "-zdntgc time set delayed node table gc in seconds\n"); @@ -1487,6 +1487,7 @@ erl_start(int argc, char **argv) * h|ms - min_heap_size * h|mbs - min_bin_vheap_size * h|pds - erts_pd_initial_size + * h|mqd - message_queue_data * */ if (has_prefix("mbs", sub_param)) { @@ -1512,6 +1513,23 @@ erl_start(int argc, char **argv) } VERBOSE(DEBUG_SYSTEM, ("using initial process dictionary size %d\n", erts_pd_initial_size)); + } else if (has_prefix("mqd", sub_param)) { + arg = get_arg(sub_param+3, argv[i+1], &i); + if (sys_strcmp(arg, "mixed") == 0) + erts_default_spo_flags &= ~(SPO_ON_HEAP_MSGQ|SPO_OFF_HEAP_MSGQ); + else if (sys_strcmp(arg, "on_heap") == 0) { + erts_default_spo_flags &= ~SPO_OFF_HEAP_MSGQ; + erts_default_spo_flags |= SPO_ON_HEAP_MSGQ; + } + else if (sys_strcmp(arg, "off_heap") == 0) { + erts_default_spo_flags &= ~SPO_ON_HEAP_MSGQ; + erts_default_spo_flags |= SPO_OFF_HEAP_MSGQ; + } + else { + erts_fprintf(stderr, + "Invalid message_queue_data flag: %s\n", arg); + erts_usage(); + } } else { /* backward compatibility */ arg = get_arg(argv[i]+2, argv[i+1], &i); @@ -2047,32 +2065,6 @@ erl_start(int argc, char **argv) } break; - case 'x': { - char *sub_param = argv[i]+2; - if (has_prefix("mqd", sub_param)) { - arg = get_arg(sub_param+3, argv[i+1], &i); - if (sys_strcmp(arg, "mixed") == 0) - erts_default_spo_flags &= ~(SPO_ON_HEAP_MSGQ|SPO_OFF_HEAP_MSGQ); - else if (sys_strcmp(arg, "on_heap") == 0) { - erts_default_spo_flags &= ~SPO_OFF_HEAP_MSGQ; - erts_default_spo_flags |= SPO_ON_HEAP_MSGQ; - } - else if (sys_strcmp(arg, "off_heap") == 0) { - erts_default_spo_flags &= ~SPO_ON_HEAP_MSGQ; - erts_default_spo_flags |= SPO_OFF_HEAP_MSGQ; - } - else { - erts_fprintf(stderr, - "Invalid message_queue_data flag: %s\n", arg); - erts_usage(); - } - } else { - erts_fprintf(stderr, "bad -x option %s\n", argv[i]); - erts_usage(); - } - break; - } - case 'z': { char *sub_param = argv[i]+2; diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c index 03a96cb00a..8efc983f04 100644 --- a/erts/emulator/beam/erl_map.c +++ b/erts/emulator/beam/erl_map.c @@ -54,6 +54,7 @@ * - maps:new/0 * - maps:put/3 * - maps:remove/2 + * - maps:take/2 * - maps:to_list/1 * - maps:update/3 * - maps:values/1 @@ -93,7 +94,7 @@ static Uint hashmap_subtree_size(Eterm node); static Eterm hashmap_to_list(Process *p, Eterm map); static Eterm hashmap_keys(Process *p, Eterm map); static Eterm hashmap_values(Process *p, Eterm map); -static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm node); +static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm node, Eterm *value); static Eterm flatmap_from_validated_list(Process *p, Eterm list, Uint size); static Eterm hashmap_from_validated_list(Process *p, Eterm list, Uint size); static Eterm hashmap_from_unsorted_array(ErtsHeapFactory*, hxnode_t *hxns, Uint n, int reject_dupkeys); @@ -1521,10 +1522,45 @@ BIF_RETTYPE maps_put_3(BIF_ALIST_3) { BIF_ERROR(BIF_P, BADMAP); } -/* maps:remove/3 */ +/* maps:take/2 */ -int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res) { +BIF_RETTYPE maps_take_2(BIF_ALIST_2) { + if (is_map(BIF_ARG_2)) { + Eterm res, map, val; + if (erts_maps_take(BIF_P, BIF_ARG_1, BIF_ARG_2, &map, &val)) { + Eterm *hp = HAlloc(BIF_P, 3); + res = make_tuple(hp); + *hp++ = make_arityval(2); + *hp++ = val; + *hp++ = map; + BIF_RET(res); + } + BIF_RET(am_error); + } + BIF_P->fvalue = BIF_ARG_2; + BIF_ERROR(BIF_P, BADMAP); +} + +/* maps:remove/2 */ + +BIF_RETTYPE maps_remove_2(BIF_ALIST_2) { + if (is_map(BIF_ARG_2)) { + Eterm res; + (void) erts_maps_take(BIF_P, BIF_ARG_1, BIF_ARG_2, &res, NULL); + BIF_RET(res); + } + BIF_P->fvalue = BIF_ARG_2; + BIF_ERROR(BIF_P, BADMAP); +} + +/* erts_maps_take + * return 1 if key is found, otherwise 0 + * If the key is not found res (output map) will be map (input map) + */ +int erts_maps_take(Process *p, Eterm key, Eterm map, + Eterm *res, Eterm *value) { Uint32 hx; + Eterm ret; if (is_flatmap(map)) { Sint n; Uint need; @@ -1537,7 +1573,7 @@ int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res) { if (n == 0) { *res = map; - return 1; + return 0; } ks = flatmap_get_keys(mp); @@ -1564,6 +1600,7 @@ int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res) { if (is_immed(key)) { while (1) { if (*ks == key) { + if (value) *value = *vs; goto found_key; } else if (--n) { *mhp++ = *vs++; @@ -1574,6 +1611,7 @@ int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res) { } else { while(1) { if (EQ(*ks, key)) { + if (value) *value = *vs; goto found_key; } else if (--n) { *mhp++ = *vs++; @@ -1589,7 +1627,7 @@ int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res) { HRelease(p, hp_start + need, hp_start); *res = map; - return 1; + return 0; found_key: /* Copy rest of keys and values */ @@ -1601,19 +1639,13 @@ found_key: } ASSERT(is_hashmap(map)); hx = hashmap_make_hash(key); - *res = hashmap_delete(p, hx, key, map); - return 1; -} - -BIF_RETTYPE maps_remove_2(BIF_ALIST_2) { - if (is_map(BIF_ARG_2)) { - Eterm res; - if (erts_maps_remove(BIF_P, BIF_ARG_1, BIF_ARG_2, &res)) { - BIF_RET(res); - } + ret = hashmap_delete(p, hx, key, map, value); + if (is_value(ret)) { + *res = ret; + return 1; } - BIF_P->fvalue = BIF_ARG_2; - BIF_ERROR(BIF_P, BADMAP); + *res = map; + return 0; } int erts_maps_update(Process *p, Eterm key, Eterm value, Eterm map, Eterm *res) { @@ -2322,7 +2354,8 @@ static Eterm hashmap_values(Process* p, Eterm node) { return res; } -static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm map) { +static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, + Eterm map, Eterm *value) { Eterm *hp = NULL, *nhp = NULL, *hp_end = NULL; Eterm *ptr; Eterm hdr, res = map, node = map; @@ -2337,8 +2370,12 @@ static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm map) { switch(primary_tag(node)) { case TAG_PRIMARY_LIST: if (EQ(CAR(list_val(node)), key)) { + if (value) { + *value = CDR(list_val(node)); + } goto unroll; } + res = THE_NON_VALUE; goto not_found; case TAG_PRIMARY_BOXED: ptr = boxed_val(node); @@ -2365,6 +2402,7 @@ static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm map) { n = hashmap_bitcount(hval); } else { /* not occupied */ + res = THE_NON_VALUE; goto not_found; } @@ -2394,6 +2432,7 @@ static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm map) { break; } /* not occupied */ + res = THE_NON_VALUE; goto not_found; default: erts_exit(ERTS_ERROR_EXIT, "bad header tag %ld\r\n", hdr & _HEADER_MAP_SUBTAG_MASK); diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h index 7af9100906..8b5c9582ba 100644 --- a/erts/emulator/beam/erl_map.h +++ b/erts/emulator/beam/erl_map.h @@ -82,6 +82,7 @@ struct ErtsEStack_; Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map); int erts_maps_update(Process *p, Eterm key, Eterm value, Eterm map, Eterm *res); int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res); +int erts_maps_take(Process *p, Eterm key, Eterm map, Eterm *res, Eterm *value); Eterm erts_hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value, Eterm node, int is_update); diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index 9beff52835..9bb6e40a11 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -32,6 +32,7 @@ #include "erl_process.h" #include "erl_binary.h" #include "dtrace-wrapper.h" +#include "beam_bp.h" ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(message_ref, ErtsMessageRef, @@ -251,9 +252,10 @@ erts_realloc_shrink_message(ErtsMessage *mp, Uint sz, Eterm *brefs, Uint brefs_s void erts_queue_dist_message(Process *rcvr, - ErtsProcLocks *rcvr_locks, + ErtsProcLocks rcvr_locks, ErtsDistExternal *dist_ext, - Eterm token) + Eterm token, + Eterm from) { ErtsMessage* mp; #ifdef USE_VM_PROBES @@ -265,7 +267,7 @@ erts_queue_dist_message(Process *rcvr, erts_aint_t state; #endif - ERTS_SMP_LC_ASSERT(*rcvr_locks == erts_proc_lc_my_proc_locks(rcvr)); + ERTS_SMP_LC_ASSERT(rcvr_locks == erts_proc_lc_my_proc_locks(rcvr)); mp = erts_alloc_message(0, NULL); mp->data.dist_ext = dist_ext; @@ -280,10 +282,10 @@ erts_queue_dist_message(Process *rcvr, ERL_MESSAGE_TOKEN(mp) = token; #ifdef ERTS_SMP - if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ)) { + if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ)) { if (erts_smp_proc_trylock(rcvr, ERTS_PROC_LOCK_MSGQ) == EBUSY) { ErtsProcLocks need_locks = ERTS_PROC_LOCK_MSGQ; - if (*rcvr_locks & ERTS_PROC_LOCK_STATUS) { + if (rcvr_locks & ERTS_PROC_LOCK_STATUS) { erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_STATUS); need_locks |= ERTS_PROC_LOCK_STATUS; } @@ -293,7 +295,7 @@ erts_queue_dist_message(Process *rcvr, state = erts_smp_atomic32_read_acqb(&rcvr->state); if (state & (ERTS_PSFLG_PENDING_EXIT|ERTS_PSFLG_EXITING)) { - if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ)) + if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ)) erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ); /* Drop message if receiver is exiting or has a pending exit ... */ erts_cleanup_messages(mp); @@ -301,10 +303,13 @@ erts_queue_dist_message(Process *rcvr, else #endif if (IS_TRACED_FL(rcvr, F_TRACE_RECEIVE)) { + if (from == am_Empty) + from = dist_ext->dep->sysname; + /* Ahh... need to decode it in order to trace it... */ - if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ)) + if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ)) erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ); - if (!erts_decode_dist_message(rcvr, *rcvr_locks, mp, 0)) + if (!erts_decode_dist_message(rcvr, rcvr_locks, mp, 0)) erts_free_message(mp); else { Eterm msg = ERL_MESSAGE_TERM(mp); @@ -324,7 +329,7 @@ erts_queue_dist_message(Process *rcvr, tok_label, tok_lastcnt, tok_serial); } #endif - erts_queue_message(rcvr, rcvr_locks, mp, msg); + erts_queue_message(rcvr, rcvr_locks, mp, msg, from); } } else { @@ -351,12 +356,12 @@ erts_queue_dist_message(Process *rcvr, LINK_MESSAGE(rcvr, mp, &mp->next, 1); - if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ)) + if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ)) erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ); erts_proc_notify_new_message(rcvr, #ifdef ERTS_SMP - *rcvr_locks + rcvr_locks #else 0 #endif @@ -366,14 +371,15 @@ erts_queue_dist_message(Process *rcvr, /* Add messages last in message queue */ static Sint -queue_messages(Process *c_p, - Process* receiver, +queue_messages(Process* receiver, erts_aint32_t *receiver_state, - ErtsProcLocks *receiver_locks, + ErtsProcLocks receiver_locks, ErtsMessage* first, ErtsMessage** last, - Uint len) + Uint len, + Eterm from) { + ErtsTracingEvent* te; Sint res; int locked_msgq = 0; erts_aint32_t state; @@ -386,12 +392,12 @@ queue_messages(Process *c_p, #ifdef ERTS_SMP #ifdef ERTS_ENABLE_LOCK_CHECK ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(receiver) < ERTS_PROC_LOCK_MSGQ || - *receiver_locks == erts_proc_lc_my_proc_locks(receiver)); + receiver_locks == erts_proc_lc_my_proc_locks(receiver)); #endif - if (!(*receiver_locks & ERTS_PROC_LOCK_MSGQ)) { + if (!(receiver_locks & ERTS_PROC_LOCK_MSGQ)) { if (erts_smp_proc_trylock(receiver, ERTS_PROC_LOCK_MSGQ) == EBUSY) { - ErtsProcLocks need_locks = ERTS_PROC_LOCK_MSGQ; + ErtsProcLocks need_locks; if (receiver_state) state = *receiver_state; @@ -400,10 +406,11 @@ queue_messages(Process *c_p, if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) goto exiting; - if (*receiver_locks & ERTS_PROC_LOCK_STATUS) { - erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_STATUS); - need_locks |= ERTS_PROC_LOCK_STATUS; + need_locks = receiver_locks & (ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); + if (need_locks) { + erts_smp_proc_unlock(receiver, need_locks); } + need_locks |= ERTS_PROC_LOCK_MSGQ; erts_smp_proc_lock(receiver, need_locks); } locked_msgq = 1; @@ -426,7 +433,7 @@ queue_messages(Process *c_p, res = receiver->msg.len; #ifdef ERTS_SMP - if (*receiver_locks & ERTS_PROC_LOCK_MAIN) { + if (receiver_locks & ERTS_PROC_LOCK_MAIN) { /* * We move 'in queue' to 'private queue' and place * message at the end of 'private queue' in order @@ -445,7 +452,10 @@ queue_messages(Process *c_p, LINK_MESSAGE(receiver, first, last, len); } - if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE)) { + if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE) + && (te = &erts_receive_tracing[erts_active_bp_ix()], + te->on)) { + ErtsMessage *msg = first; #ifdef USE_VM_PROBES @@ -468,19 +478,18 @@ queue_messages(Process *c_p, tok_label, tok_lastcnt, tok_serial); } #endif - while (msg) { - trace_receive(receiver, ERL_MESSAGE_TERM(msg)); + trace_receive(receiver, from, ERL_MESSAGE_TERM(msg), te); msg = msg->next; } } - - if (locked_msgq) + if (locked_msgq) { erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ); + } #ifdef ERTS_SMP - erts_proc_notify_new_message(receiver, *receiver_locks); + erts_proc_notify_new_message(receiver, receiver_locks); #else erts_proc_notify_new_message(receiver, 0); ERTS_HOLE_CHECK(receiver); @@ -489,31 +498,31 @@ queue_messages(Process *c_p, } static Sint -queue_message(Process *c_p, - Process* receiver, +queue_message(Process* receiver, erts_aint32_t *receiver_state, - ErtsProcLocks *receiver_locks, - ErtsMessage* mp, Eterm msg) + ErtsProcLocks receiver_locks, + ErtsMessage* mp, Eterm msg, Eterm from) { ERL_MESSAGE_TERM(mp) = msg; - return queue_messages(c_p, receiver, receiver_state, receiver_locks, - mp, &mp->next, 1 ); + return queue_messages(receiver, receiver_state, receiver_locks, + mp, &mp->next, 1, from); } Sint -erts_queue_message(Process* receiver, ErtsProcLocks *receiver_locks, - ErtsMessage* mp, Eterm msg) +erts_queue_message(Process* receiver, ErtsProcLocks receiver_locks, + ErtsMessage* mp, Eterm msg, Eterm from) { - return queue_message(NULL, receiver, NULL, receiver_locks, mp, msg); + return queue_message(receiver, NULL, receiver_locks, mp, msg, from); } Sint -erts_queue_messages(Process* receiver, ErtsProcLocks *receiver_locks, - ErtsMessage* first, ErtsMessage** last, Uint len) +erts_queue_messages(Process* receiver, ErtsProcLocks receiver_locks, + ErtsMessage* first, ErtsMessage** last, Uint len, + Eterm from) { - return queue_messages(NULL, receiver, NULL, receiver_locks, - first, last, len); + return queue_messages(receiver, NULL, receiver_locks, + first, last, len, from); } void @@ -830,11 +839,11 @@ erts_send_message(Process* sender, #ifdef USE_VM_PROBES ERL_MESSAGE_DT_UTAG(mp) = utag; #endif - res = queue_message(sender, - receiver, + res = queue_message(receiver, &receiver_state, - receiver_locks, - mp, message); + *receiver_locks, + mp, message, + sender->common.id); BM_SWAP_TIMER(send,system); @@ -891,7 +900,7 @@ erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp, seq_trace_output(token, save, SEQ_TRACE_SEND, to->common.id, NULL); temptoken = copy_struct(token, sz_token, &hp, ohp); ERL_MESSAGE_TOKEN(mp) = temptoken; - erts_queue_message(to, to_locksp, mp, save); + erts_queue_message(to, *to_locksp, mp, save, am_system); } else { sz_from = IS_CONST(from) ? 0 : size_object(from); #ifdef SHCOPY_SEND @@ -913,7 +922,7 @@ erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp, ? from : copy_struct(from, sz_from, &hp, ohp)); save = TUPLE3(hp, am_EXIT, from_copy, mess); - erts_queue_message(to, to_locksp, mp, save); + erts_queue_message(to, *to_locksp, mp, save, am_system); } } diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index 608cf552a2..851ac37fda 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -295,10 +295,10 @@ ErlHeapFragment* new_message_buffer(Uint); ErlHeapFragment* erts_resize_message_buffer(ErlHeapFragment *, Uint, Eterm *, Uint); void free_message_buffer(ErlHeapFragment *); -void erts_queue_dist_message(Process*, ErtsProcLocks*, ErtsDistExternal *, Eterm); -Sint erts_queue_message(Process*, ErtsProcLocks*,ErtsMessage*, Eterm); -Sint erts_queue_messages(Process*, ErtsProcLocks*, - ErtsMessage*, ErtsMessage**, Uint); +void erts_queue_dist_message(Process*, ErtsProcLocks, ErtsDistExternal *, Eterm, Eterm); +Sint erts_queue_message(Process*, ErtsProcLocks,ErtsMessage*, Eterm, Eterm); +Sint erts_queue_messages(Process*, ErtsProcLocks, + ErtsMessage*, ErtsMessage**, Uint, Eterm); void erts_deliver_exit_message(Eterm, Process*, ErtsProcLocks *, Eterm, Eterm); Sint erts_send_message(Process*, Process*, ErtsProcLocks*, Eterm, unsigned); void erts_link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp); diff --git a/erts/emulator/beam/erl_msacc.c b/erts/emulator/beam/erl_msacc.c index d0f305900a..0e625f213b 100644 --- a/erts/emulator/beam/erl_msacc.c +++ b/erts/emulator/beam/erl_msacc.c @@ -257,7 +257,7 @@ static void send_reply(ErtsMsAcc *msacc, ErtsMSAccReq *msaccrp) { if (msacc->unmanaged) erts_mtx_unlock(&msacc->mtx); - erts_queue_message(rp, &rp_locks, msgp, msg); + erts_queue_message(rp, rp_locks, msgp, msg, am_system); if (esdp && msaccrp->req_sched == esdp->no) rp_locks &= ~ERTS_PROC_LOCK_MAIN; diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 73c0eb8eba..a695a028ba 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -366,7 +366,7 @@ int erts_flush_trace_messages(Process *c_p, ErtsProcLocks c_p_locks) rp_locks = 0; if (rp->common.id == c_p->common.id) rp_locks = c_p_locks; - erts_queue_messages(rp, &rp_locks, first, last, len); + erts_queue_messages(rp, rp_locks, first, last, len, c_p->common.id); if (rp->common.id == c_p->common.id) rp_locks &= ~c_p_locks; if (rp_locks) @@ -405,7 +405,10 @@ enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, ErlNifEnv* msg_env, ERL_NIF_TERM msg) { struct enif_msg_environment_t* menv = (struct enif_msg_environment_t*)msg_env; - ErtsProcLocks rp_locks = 0, lc_locks = 0, c_p_locks = ERTS_PROC_LOCK_MAIN; + ErtsProcLocks rp_locks = 0; +#ifdef ERTS_SMP + ErtsProcLocks lc_locks = 0; +#endif Process* rp; Process* c_p; ErtsMessage *mp; @@ -417,7 +420,7 @@ enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, if (env != NULL) { c_p = env->proc; if (receiver == c_p->common.id) { - rp_locks = c_p_locks; + rp_locks = ERTS_PROC_LOCK_MAIN; flush_me = 1; } } @@ -470,14 +473,8 @@ enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, if (c_p && IS_TRACED_FL(c_p, F_TRACE_SEND)) trace_send(c_p, receiver, msg); - -#ifndef ERTS_SMP } -#endif - - erts_queue_message(rp, &rp_locks, mp, msg); #ifdef ERTS_SMP - } else { /* This clause is taken when the nif is called in the context of a traced process. We do not know which locks we have @@ -502,8 +499,6 @@ enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, #ifdef ERTS_ENABLE_LOCK_CHECK lc_locks = erts_proc_lc_my_proc_locks(rp); rp_locks |= lc_locks; - if (receiver == c_p->common.id) - c_p_locks |= lc_locks; #endif if (ERTS_FORCE_ENIF_SEND_DELAY() || msgq || rp_locks & ERTS_PROC_LOCK_MSGQ || @@ -532,19 +527,25 @@ enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, msgq->last = &mp->next; erts_smp_proc_unlock(t_p, ERTS_PROC_LOCK_TRACE); } + goto done; } else { erts_smp_proc_unlock(t_p, ERTS_PROC_LOCK_TRACE); rp_locks &= ~ERTS_PROC_LOCK_TRACE; rp_locks |= ERTS_PROC_LOCK_MSGQ; - erts_queue_message(rp, &rp_locks, mp, msg); } } -#endif +#endif /* ERTS_SMP */ + + erts_queue_message(rp, rp_locks, mp, msg, + c_p ? c_p->common.id : am_undefined); +#ifdef ERTS_SMP +done: if (c_p == rp) rp_locks &= ~ERTS_PROC_LOCK_MAIN; if (rp_locks & ~lc_locks) erts_smp_proc_unlock(rp, rp_locks & ~lc_locks); +#endif if (!scheduler) erts_proc_dec_refc(rp); if (flush_me) { @@ -2443,14 +2444,13 @@ int enif_make_map_remove(ErlNifEnv* env, Eterm key, Eterm *map_out) { - int res; if (!is_map(map_in)) { return 0; } flush_env(env); - res = erts_maps_remove(env->proc, key, map_in, map_out); + (void) erts_maps_take(env->proc, key, map_in, map_out, NULL); cache_env(env); - return res; + return 1; } int enif_map_iterator_create(ErlNifEnv *env, @@ -2781,7 +2781,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) ErlNifEntry* entry = NULL; ErlNifEnv env; int i, err, encoding; - Module* mod; + Module* module_p; Eterm mod_atom; const Atom* mod_atomp; Eterm f_atom; @@ -2791,6 +2791,8 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) int veto; struct erl_module_nif* lib = NULL; int reload_warning = 0; + struct erl_module_instance* this_mi; + struct erl_module_instance* prev_mi; encoding = erts_get_native_filename_encoding(); if (encoding == ERL_FILENAME_WIN_WCHAR) { @@ -2824,21 +2826,29 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) ASSERT(caller != NULL); mod_atom = caller[0]; ASSERT(is_atom(mod_atom)); - mod=erts_get_module(mod_atom, erts_active_code_ix()); - ASSERT(mod != NULL); + module_p = erts_get_module(mod_atom, erts_active_code_ix()); + ASSERT(module_p != NULL); mod_atomp = atom_tab(atom_val(mod_atom)); init_func = erts_static_nif_get_nif_init((char*)mod_atomp->name, mod_atomp->len); if (init_func != NULL) handle = init_func; - if (!in_area(caller, mod->curr.code_hdr, mod->curr.code_length)) { - ASSERT(in_area(caller, mod->old.code_hdr, mod->old.code_length)); + if (in_area(caller, module_p->old.code_hdr, module_p->old.code_length)) { + if (module_p->old.code_hdr->on_load_function_ptr) { + this_mi = &module_p->old; + prev_mi = &module_p->curr; + } else { + ret = load_nif_error(BIF_P, "old_code", "Calling load_nif from old " + "module '%T' not allowed", mod_atom); + goto error; + } + } else { + this_mi = &module_p->curr; + prev_mi = &module_p->old; + } - ret = load_nif_error(BIF_P, "old_code", "Calling load_nif from old " - "module '%T' not allowed", mod_atom); - } - else if (init_func == NULL && + if (init_func == NULL && (err=erts_sys_ddll_open(lib_name, &handle, &errdesc)) != ERL_DE_NO_ERROR) { const char slogan[] = "Failed to load NIF library"; if (strstr(errdesc.str, lib_name) != NULL) { @@ -2887,7 +2897,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) for (i=0; i < entry->num_of_funcs && ret==am_ok; i++) { BeamInstr** code_pp; if (!erts_atom_get(f->name, sys_strlen(f->name), &f_atom, ERTS_ATOM_ENC_LATIN1) - || (code_pp = get_func_pp(mod->curr.code_hdr, f_atom, f->arity))==NULL) { + || (code_pp = get_func_pp(this_mi->code_hdr, f_atom, f->arity))==NULL) { ret = load_nif_error(BIF_P,bad_lib,"Function not found %T:%s/%u", mod_atom, f->name, f->arity); } @@ -2938,9 +2948,9 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) erts_refc_init(&lib->rt_cnt, 0); erts_refc_init(&lib->rt_dtor_cnt, 0); ASSERT(opened_rt_list == NULL); - lib->mod = mod; + lib->mod = module_p; env.mod_nif = lib; - if (mod->curr.nif != NULL) { /*************** Reload ******************/ + if (this_mi->nif != NULL) { /*************** Reload ******************/ /* * Repeated load_nif calls from same Erlang module instance ("reload") * is deprecated and was only ment as a development feature not to @@ -2948,16 +2958,16 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) */ int k, old_incr = 0; ErlNifFunc* old_func; - lib->priv_data = mod->curr.nif->priv_data; + lib->priv_data = this_mi->nif->priv_data; - ASSERT(mod->curr.nif->entry != NULL); + ASSERT(this_mi->nif->entry != NULL); if (entry->reload == NULL) { ret = load_nif_error(BIF_P,reload,"Reload not supported by this NIF library."); goto error; } /* Check that no NIF is removed */ - old_func = mod->curr.nif->entry->funcs; - for (k=0; k < mod->curr.nif->entry->num_of_funcs; k++) { + old_func = this_mi->nif->entry->funcs; + for (k=0; k < this_mi->nif->entry->num_of_funcs; k++) { int incr = 0; ErlNifFunc* f = entry->funcs; for (i=0; i < entry->num_of_funcs; i++) { @@ -2973,7 +2983,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) old_func->name, old_func->arity); goto error; } - old_func = next_func(mod->curr.nif->entry, &old_incr, old_func); + old_func = next_func(this_mi->nif->entry, &old_incr, old_func); } erts_pre_nif(&env, BIF_P, lib, NULL); veto = entry->reload(&env, &lib->priv_data, BIF_ARG_2); @@ -2983,24 +2993,24 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) } else { commit_opened_resource_types(lib); - mod->curr.nif->entry = NULL; /* to prevent 'unload' callback */ - erts_unload_nif(mod->curr.nif); + this_mi->nif->entry = NULL; /* to prevent 'unload' callback */ + erts_unload_nif(this_mi->nif); reload_warning = 1; } } else { lib->priv_data = NULL; - if (mod->old.nif != NULL) { /**************** Upgrade ***************/ - void* prev_old_data = mod->old.nif->priv_data; + if (prev_mi->nif != NULL) { /**************** Upgrade ***************/ + void* prev_old_data = prev_mi->nif->priv_data; if (entry->upgrade == NULL) { ret = load_nif_error(BIF_P, upgrade, "Upgrade not supported by this NIF library."); goto error; } - erts_pre_nif(&env, BIF_P, lib, NULL); - veto = entry->upgrade(&env, &lib->priv_data, &mod->old.nif->priv_data, BIF_ARG_2); + erts_pre_nif(&env, BIF_P, lib, NULL); + veto = entry->upgrade(&env, &lib->priv_data, &prev_mi->nif->priv_data, BIF_ARG_2); erts_post_nif(&env); if (veto) { - mod->old.nif->priv_data = prev_old_data; + prev_mi->nif->priv_data = prev_old_data; ret = load_nif_error(BIF_P, upgrade, "Library upgrade-call unsuccessful."); } else @@ -3025,12 +3035,12 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) int incr = 0; ErlNifFunc* f = entry->funcs; - mod->curr.nif = lib; + this_mi->nif = lib; for (i=0; i < entry->num_of_funcs; i++) { BeamInstr* code_ptr; erts_atom_get(f->name, sys_strlen(f->name), &f_atom, ERTS_ATOM_ENC_LATIN1); - code_ptr = *get_func_pp(mod->curr.code_hdr, f_atom, f->arity); + code_ptr = *get_func_pp(this_mi->code_hdr, f_atom, f->arity); if (code_ptr[1] == 0) { code_ptr[5+0] = (BeamInstr) BeamOp(op_call_nif); @@ -3158,7 +3168,7 @@ Eterm erts_nif_call_function(Process *p, Process *tracee, /* Verify that function is part of this module */ int i; for (i = 0; i < mod->entry->num_of_funcs; i++) - if (fun == mod->entry->funcs+i) + if (fun == &(mod->entry->funcs[i])) break; ASSERT(i < mod->entry->num_of_funcs); if (p) diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 1a66044627..36b0af0c1a 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -1139,7 +1139,7 @@ reply_sched_wall_time(void *vswtrp) hpp = &hp; } - erts_queue_message(rp, &rp_locks, mp, msg); + erts_queue_message(rp, rp_locks, mp, msg, am_system); if (swtrp->req_sched == esdp->no) rp_locks &= ~ERTS_PROC_LOCK_MAIN; @@ -1218,7 +1218,7 @@ reply_system_check(void *vscrp) hpp = &hp; msg = STORE_NC(hpp, ohp, scrp->ref); - erts_queue_message(rp, &rp_locks, mp, msg); + erts_queue_message(rp, rp_locks, mp, msg, am_system); if (scrp->req_sched == esdp->no) rp_locks &= ~ERTS_PROC_LOCK_MAIN; @@ -6688,10 +6688,8 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st) erts_aint32_t e; n = e = a; - if (a & ERTS_PSFLG_FREE) { - res = 0; + if (a & ERTS_PSFLG_FREE) goto cleanup; /* We don't want to schedule free processes... */ - } enqueue = ERTS_ENQUEUE_NOT; n |= ERTS_PSFLG_ACTIVE_SYS; @@ -9240,6 +9238,21 @@ erts_set_process_priority(Process *p, Eterm value) } } +static int +scheduler_gc_proc(Process *c_p, int reds_left) +{ + int fcalls, reds; + if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) + fcalls = reds_left; + else + fcalls = reds_left - CONTEXT_REDS; + reds = erts_garbage_collect_nobump(c_p, 0, c_p->arg_reg, c_p->arity, fcalls); + ASSERT(reds_left >= reds); + return reds; +} + + + /* * schedule() is called from BEAM (process_main()) or HiPE * (hipe_mode_switch()) when the current process is to be @@ -9318,6 +9331,7 @@ Process *schedule(Process *p, int calls) #endif reds = actual_reds = calls - esdp->virtual_reds; + ASSERT(actual_reds >= 0); if (reds < ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST) reds = ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST; esdp->virtual_reds = 0; @@ -9740,7 +9754,7 @@ Process *schedule(Process *p, int calls) esdp->current_process = p; - + calls = 0; reds = context_reds; #ifdef ERTS_SMP @@ -9819,9 +9833,7 @@ Process *schedule(Process *p, int calls) /* Migrate to dirty scheduler... */ sunlock_sched_out_proc: erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); - p->fcalls = reds; goto sched_out_proc; - } } else { @@ -9856,8 +9868,6 @@ Process *schedule(Process *p, int calls) #endif /* ERTS_SMP */ - p->fcalls = reds; - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); if (IS_TRACED(p)) { @@ -9890,14 +9900,15 @@ Process *schedule(Process *p, int calls) * not allowed to execute system tasks. */ if (!(p->flags & F_DELAY_GC)) { - reds -= execute_sys_tasks(p, &state, reds); + int cost = execute_sys_tasks(p, &state, reds); + calls += cost; + reds -= cost; if (reds <= 0 #ifdef ERTS_DIRTY_SCHEDULERS || ERTS_SCHEDULER_IS_DIRTY(esdp) || (state & ERTS_PSFLGS_DIRTY_WORK) #endif ) { - p->fcalls = reds; goto sched_out_proc; } } @@ -9910,8 +9921,9 @@ Process *schedule(Process *p, int calls) if (((state & (ERTS_PSFLG_SUSPENDED | ERTS_PSFLG_ACTIVE)) != ERTS_PSFLG_ACTIVE) - && !(state & ERTS_PSFLG_EXITING)) + && !(state & ERTS_PSFLG_EXITING)) { goto sched_out_proc; + } n = e = state; n &= ~ERTS_PSFLG_RUNNING_SYS; @@ -9930,11 +9942,11 @@ Process *schedule(Process *p, int calls) if (ERTS_IS_GC_DESIRED(p)) { if (!(state & ERTS_PSFLG_EXITING) && !(p->flags & (F_DELAY_GC|F_DISABLE_GC))) { - reds -= erts_garbage_collect_nobump(p, 0, p->arg_reg, p->arity); - if (reds <= 0) { - p->fcalls = reds; + int cost = scheduler_gc_proc(p, reds); + calls += cost; + reds -= cost; + if (reds <= 0) goto sched_out_proc; - } } } @@ -9942,6 +9954,8 @@ Process *schedule(Process *p, int calls) free_proxy_proc(proxy_p); proxy_p = NULL; } + + p->fcalls = reds; ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); @@ -10010,7 +10024,7 @@ notify_sys_task_executed(Process *c_p, ErtsProcSysTask *st, Eterm st_result) ASSERT(hp_start + hsz == hp); #endif - erts_queue_message(rp, &rp_locks, mp, msg); + erts_queue_message(rp, rp_locks, mp, msg, c_p->common.id); if (c_p == rp) rp_locks &= ~ERTS_PROC_LOCK_MAIN; @@ -10211,21 +10225,24 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) else { if (!garbage_collected) { FLAGS(c_p) |= F_NEED_FULLSWEEP; - reds -= erts_garbage_collect_nobump(c_p, - 0, - c_p->arg_reg, - c_p->arity); + reds -= scheduler_gc_proc(c_p, reds); garbage_collected = 1; } st_res = am_true; } break; case ERTS_PSTT_CPC: { + int fcalls; int cpc_reds = 0; + if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) + fcalls = reds; + else + fcalls = reds - CONTEXT_REDS; st_res = erts_check_process_code(c_p, st->arg[0], unsigned_val(st->arg[1]), - &cpc_reds); + &cpc_reds, + fcalls); reds -= cpc_reds; if (is_non_value(st_res)) { /* Needed gc, but gc was disabled */ @@ -10257,6 +10274,9 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) *statep = state; + if (in_reds < reds) + return in_reds; + return in_reds - reds; } @@ -10932,9 +10952,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). INITIALIZE_SHCOPY(info); #endif -#ifdef ERTS_SMP erts_smp_proc_lock(parent, ERTS_PROC_LOCKS_ALL_MINOR); -#endif /* * Check for errors. @@ -11235,6 +11253,8 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). * Schedule process for execution. */ + erts_smp_proc_unlock(parent, locks & ERTS_PROC_LOCKS_ALL_MINOR); + schedule_process(p, state, 0); VERBOSE(DEBUG_PROCESSES, ("Created a new process: %T\n",p->common.id)); @@ -11248,6 +11268,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). DTRACE2(process_spawn, process_name, mfa); } #endif + return res; error: @@ -11481,8 +11502,10 @@ delete_process(Process* p) psd = (ErtsPSD *) erts_smp_atomic_read_nob(&p->psd); - if (psd) + if (psd) { + erts_smp_atomic_set_nob(&p->psd, (erts_aint_t) NULL); /* Reduction counting depends on this... */ erts_free(ERTS_ALC_T_PSD, psd); + } /* Clean binaries and funs */ erts_cleanup_offheap(&p->off_heap); @@ -11743,7 +11766,7 @@ send_exit_message(Process *to, ErtsProcLocks *to_locksp, mp = erts_alloc_message_heap(to, to_locksp, term_size, &hp, &ohp); mess = copy_struct(exit_term, term_size, &hp, ohp); #endif - erts_queue_message(to, to_locksp, mp, mess); + erts_queue_message(to, *to_locksp, mp, mess, am_system); } else { Eterm temp_token; Uint sz_token; @@ -11764,7 +11787,7 @@ send_exit_message(Process *to, ErtsProcLocks *to_locksp, seq_trace_output(token, mess, SEQ_TRACE_SEND, to->common.id, to); temp_token = copy_struct(token, sz_token, &hp, ohp); ERL_MESSAGE_TOKEN(mp) = temp_token; - erts_queue_message(to, to_locksp, mp, mess); + erts_queue_message(to, *to_locksp, mp, mess, am_system); } } @@ -12576,6 +12599,8 @@ erts_continue_exit_process(Process *p) dep = (p->flags & F_DISTRIBUTION) ? erts_this_dist_entry : NULL; scb = ERTS_PROC_SET_SAVED_CALLS_BUF(p, NULL); + if (scb) + p->fcalls += CONTEXT_REDS; /* Reduction counting depends on this... */ pbt = ERTS_PROC_SET_CALL_TIME(p, NULL); nif_export = ERTS_PROC_SET_NIF_TRAP_EXPORT(p, NULL); diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 61acf5924b..59da9c1779 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -805,8 +805,15 @@ erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi) #define ERTS_PSD_CALL_TIME_BP 3 #define ERTS_PSD_DELAYED_GC_TASK_QS 4 #define ERTS_PSD_NIF_TRAP_EXPORT 5 +#ifdef HIPE +#define ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF 6 +#endif +#ifdef HIPE +#define ERTS_PSD_SIZE 7 +#else #define ERTS_PSD_SIZE 6 +#endif typedef struct { void *data[ERTS_PSD_SIZE]; @@ -925,7 +932,6 @@ struct process { Eterm* stop; /* Stack top */ Eterm* heap; /* Heap start */ Eterm* hend; /* Heap end */ - Eterm* abandoned_heap; Uint heap_sz; /* Size of heap in words */ Uint min_heap_size; /* Minimum size of heap (in words). */ Uint min_vheap_size; /* Minimum size of virtual heap (in words). */ @@ -941,6 +947,16 @@ struct process { #endif /* + * Moved to after "struct hipe_process_state hipe", as a temporary fix for + * LLVM hard-coding offsetof(struct process, hipe.nstack) (sic!) + * (see void X86FrameLowering::adjustForHiPEPrologue(...) in + * lib/Target/X86/X86FrameLowering.cpp). + * + * Used to be below "Eterm* hend". + */ + Eterm* abandoned_heap; + + /* * Saved x registers. */ Uint arity; /* Number of live argument registers (only valid @@ -1071,6 +1087,10 @@ struct process { Uint space_verified; /* Avoid HAlloc forcing heap fragments when */ Eterm* space_verified_from; /* we rely on available heap space (TestHeap) */ #endif + +#ifdef DEBUG + Uint debug_reds_in; +#endif }; extern const Process erts_invalid_process; @@ -1342,6 +1362,7 @@ extern int erts_system_profile_ts_type; #define F_DELAY_GC (1 << 16) /* Similar to disable GC (see below) */ #define F_SCHDLR_ONLN_WAITQ (1 << 17) /* Process enqueued waiting to change schedulers online */ #define F_HAVE_BLCKD_NMSCHED (1 << 18) /* Process has blocked normal multi-scheduling */ +#define F_HIPE_MODE (1 << 19) /* * F_DISABLE_GC and F_DELAY_GC are similar. Both will prevent @@ -1999,6 +2020,13 @@ erts_psd_set(Process *p, int ix, void *data) #define ERTS_PROC_SET_NIF_TRAP_EXPORT(P, NTE) \ erts_psd_set((P), ERTS_PSD_NIF_TRAP_EXPORT, (void *) (NTE)) +#ifdef HIPE +#define ERTS_PROC_GET_SUSPENDED_SAVED_CALLS_BUF(P) \ + ((struct saved_calls *) erts_psd_get((P), ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF)) +#define ERTS_PROC_SET_SUSPENDED_SAVED_CALLS_BUF(P, SCB) \ + ((struct saved_calls *) erts_psd_set((P), ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF, (void *) (SCB))) +#endif + ERTS_GLB_INLINE Eterm erts_proc_get_error_handler(Process *p); ERTS_GLB_INLINE Eterm erts_proc_set_error_handler(Process *p, Eterm handler); diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c index 346404fe2a..fadbd704bd 100644 --- a/erts/emulator/beam/erl_time_sup.c +++ b/erts/emulator/beam/erl_time_sup.c @@ -1966,7 +1966,7 @@ send_time_offset_changed_notifications(void *new_offsetp) *patch_refp = ref; ASSERT(hsz == size_object(message_template)); message = copy_struct(message_template, hsz, &hp, ohp); - erts_queue_message(rp, &rp_locks, mp, message); + erts_queue_message(rp, rp_locks, mp, message, am_clock_service); } erts_smp_proc_unlock(rp, rp_locks); } diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index bd88769dfc..1c0fc0a11f 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -237,7 +237,7 @@ write_timestamp(ErtsTraceTimeStamp *tsp, Eterm **hpp) } } -#define PATCH_TS_SIZE(p) patch_ts_size(TFLGS_TS_TYPE(p)) +#ifdef ERTS_SMP static ERTS_INLINE Uint patch_ts_size(int ts_type) @@ -257,6 +257,7 @@ patch_ts_size(int ts_type) return 0; } } +#endif /* ERTS_SMP */ /* * Write a timestamp. The timestamp MUST be the last @@ -359,25 +360,51 @@ void erts_init_trace(void) { *(OHPP) = &(*(BPP))->off_heap, \ (*(BPP))->mem) +enum ErtsTracerOpt { + TRACE_FUN_DEFAULT = 0, + TRACE_FUN_ENABLED = 1, + TRACE_FUN_T_SEND = 2, + TRACE_FUN_T_RECEIVE = 3, + TRACE_FUN_T_CALL = 4, + TRACE_FUN_T_SCHED_PROC = 5, + TRACE_FUN_T_SCHED_PORT = 6, + TRACE_FUN_T_GC = 7, + TRACE_FUN_T_PROCS = 8, + TRACE_FUN_T_PORTS = 9, + TRACE_FUN_E_SEND = 10, + TRACE_FUN_E_RECEIVE = 11, + TRACE_FUN_E_CALL = 12, + TRACE_FUN_E_SCHED_PROC = 13, + TRACE_FUN_E_SCHED_PORT = 14, + TRACE_FUN_E_GC = 15, + TRACE_FUN_E_PROCS = 16, + TRACE_FUN_E_PORTS = 17 +}; + +#define NIF_TRACER_TYPES (18) + + static ERTS_INLINE int send_to_tracer_nif_raw(Process *c_p, Process *tracee, const ErtsTracer tracer, Uint trace_flags, Eterm t_p_id, ErtsTracerNif *tnif, + enum ErtsTracerOpt topt, Eterm tag, Eterm msg, Eterm extra, Eterm pam_result); static ERTS_INLINE int send_to_tracer_nif(Process *c_p, ErtsPTabElementCommon *t_p, - Eterm t_p_id, ErtsTracerNif *tnif, Eterm tag, - Eterm msg, Eterm extra); + Eterm t_p_id, ErtsTracerNif *tnif, + enum ErtsTracerOpt topt, + Eterm tag, Eterm msg, Eterm extra, + Eterm pam_result); static ERTS_INLINE Eterm call_enabled_tracer(Process *c_p, const ErtsTracer tracer, - ErtsTracerNif **tnif_ref, Eterm tag, Eterm t_p_id); + ErtsTracerNif **tnif_ref, + enum ErtsTracerOpt topt, + Eterm tag, Eterm t_p_id); static int -is_tracer_proc_enabled(Process* c_p, ErtsProcLocks c_p_locks, - ErtsPTabElementCommon *t_p, - ErtsTracerNif **tnif_ret, Eterm tag); - -#define SEND_TO_TRACER(c_p, tag, msg) \ - send_to_tracer_nif(c_p, &(c_p)->common, (c_p)->common.id, NULL, tag, \ - msg, THE_NON_VALUE) +is_tracer_enabled(Process* c_p, ErtsProcLocks c_p_locks, + ErtsPTabElementCommon *t_p, + ErtsTracerNif **tnif_ret, + enum ErtsTracerOpt topt, Eterm tag); static Uint active_sched; @@ -433,7 +460,7 @@ erts_set_system_seq_tracer(Process *c_p, ErtsProcLocks c_p_locks, ErtsTracer new if (!ERTS_TRACER_IS_NIL(new)) { Eterm nif_result = call_enabled_tracer( NULL, new, NULL, - am_trace_status, am_undefined); + TRACE_FUN_ENABLED, am_trace_status, am_undefined); switch (nif_result) { case am_trace: break; default: @@ -465,7 +492,8 @@ erts_get_system_seq_tracer(void) erts_smp_rwmtx_runlock(&sys_trace_rwmtx); if (st != erts_tracer_nil && - call_enabled_tracer(NULL, st, NULL, am_trace_status, am_undefined) == am_remove) { + call_enabled_tracer(NULL, st, NULL, TRACE_FUN_ENABLED, + am_trace_status, am_undefined) == am_remove) { erts_set_system_seq_tracer(NULL, 0, erts_tracer_nil); st = erts_tracer_nil; } @@ -484,10 +512,11 @@ get_default_tracing(Uint *flagsp, ErtsTracer *tracerp, if (ERTS_TRACER_IS_NIL(*default_tracer)) { *default_trace_flags &= ~TRACEE_FLAGS; } else { - Eterm nif_result = call_enabled_tracer( - NULL, *default_tracer, NULL, - am_trace_status, am_undefined); - switch (nif_result) { + Eterm nif_res; + nif_res = call_enabled_tracer(NULL, *default_tracer, + NULL, TRACE_FUN_ENABLED, + am_trace_status, am_undefined); + switch (nif_res) { case am_trace: break; default: { ErtsTracer curr_default_tracer = *default_tracer; @@ -629,6 +658,7 @@ do { \ # define GET_NOW(m, s, u) do {get_now(m, s, u);} while (0) #endif + static void write_sys_msg_to_port(Eterm unused_to, Port* trace_port, @@ -709,7 +739,7 @@ profile_send(Eterm from, Eterm message) { else msg = copy_struct(message, sz, &hp, &mp->hfrag.off_heap); - erts_queue_message(profile_p, NULL, mp, msg); + erts_queue_message(profile_p, 0, mp, msg, from); } } @@ -737,7 +767,7 @@ trace_sched_aux(Process *p, ErtsProcLocks locks, Eterm what) break; } - if (!is_tracer_proc_enabled(p, locks, &p->common, &tnif, what)) + if (!is_tracer_enabled(p, locks, &p->common, &tnif, TRACE_FUN_E_SCHED_PROC, what)) return; if (ERTS_PROC_IS_EXITING(p)) @@ -756,8 +786,8 @@ trace_sched_aux(Process *p, ErtsProcLocks locks, Eterm what) hp += 4; } - send_to_tracer_nif(p, &p->common, p->common.id, tnif, - what, tmp, THE_NON_VALUE); + send_to_tracer_nif(p, &p->common, p->common.id, tnif, TRACE_FUN_T_SCHED_PROC, + what, tmp, THE_NON_VALUE, am_true); } /* Send {trace_ts, Pid, What, {Mod, Func, Arity}, Timestamp} @@ -782,9 +812,32 @@ trace_send(Process *p, Eterm to, Eterm msg) { Eterm operation = am_send; ErtsTracerNif *tnif = NULL; + ErtsTracingEvent* te; + Eterm pam_result; ASSERT(ARE_TRACE_FLAGS_ON(p, F_TRACE_SEND)); + te = &erts_send_tracing[erts_active_bp_ix()]; + if (!te->on) { + return; + } + if (te->match_spec) { + Eterm args[2]; + Uint32 return_flags; + args[0] = to; + args[1] = msg; + pam_result = erts_match_set_run_trace(p, p, + te->match_spec, args, 2, + ERTS_PAM_TMP_RESULT, &return_flags); + if (pam_result == am_false) + return; + if (ERTS_TRACE_FLAGS(p) & F_TRACE_SILENT) { + erts_match_set_release_result_trace(p, pam_result); + return; + } + } else + pam_result = am_true; + if (is_internal_pid(to)) { if (!erts_proc_lookup(to)) goto send_to_non_existing_process; @@ -795,21 +848,64 @@ trace_send(Process *p, Eterm to, Eterm msg) operation = am_send_to_non_existing_process; } - if (is_tracer_proc_enabled(p, ERTS_PROC_LOCK_MAIN, &p->common, &tnif, operation)) - send_to_tracer_nif(p, &p->common, p->common.id, tnif, operation, msg, to); + if (is_tracer_enabled(p, ERTS_PROC_LOCK_MAIN, &p->common, &tnif, + TRACE_FUN_E_SEND, operation)) { + send_to_tracer_nif(p, &p->common, p->common.id, tnif, TRACE_FUN_T_SEND, + operation, msg, to, pam_result); + } + erts_match_set_release_result_trace(p, pam_result); } /* Send {trace_ts, Pid, receive, Msg, Timestamp} * or {trace, Pid, receive, Msg} */ void -trace_receive(Process *c_p, Eterm msg) +trace_receive(Process* receiver, + Eterm from, + Eterm msg, ErtsTracingEvent* te) { ErtsTracerNif *tnif = NULL; - if (is_tracer_proc_enabled(NULL, 0, &c_p->common, - &tnif, am_receive)) - send_to_tracer_nif(NULL, &c_p->common, c_p->common.id, - tnif, am_receive, msg, THE_NON_VALUE); + Eterm pam_result; + + if (!te) { + te = &erts_receive_tracing[erts_active_bp_ix()]; + if (!te->on) + return; + } + else ASSERT(te->on); + + if (te->match_spec) { + Eterm args[3]; + Uint32 return_flags; + if (is_pid(from)) { + args[0] = pid_node_name(from); + args[1] = from; + } + else { + ASSERT(is_atom(from)); + args[0] = from; /* node name or other atom (e.g 'system') */ + args[1] = am_undefined; + } + args[2] = msg; + pam_result = erts_match_set_run_trace(NULL, receiver, + te->match_spec, args, 3, + ERTS_PAM_TMP_RESULT, &return_flags); + if (pam_result == am_false) + return; + if (ERTS_TRACE_FLAGS(receiver) & F_TRACE_SILENT) { + erts_match_set_release_result_trace(NULL, pam_result); + return; + } + } else + pam_result = am_true; + + if (is_tracer_enabled(NULL, 0, &receiver->common, &tnif, + TRACE_FUN_E_RECEIVE, am_receive)) { + send_to_tracer_nif(NULL, &receiver->common, receiver->common.id, + tnif, TRACE_FUN_T_RECEIVE, + am_receive, msg, THE_NON_VALUE, pam_result); + } + erts_match_set_release_result_trace(NULL, pam_result); } int @@ -819,8 +915,9 @@ seq_trace_update_send(Process *p) ASSERT((is_tuple(SEQ_TRACE_TOKEN(p)) || is_nil(SEQ_TRACE_TOKEN(p)))); if (have_no_seqtrace(SEQ_TRACE_TOKEN(p)) || (seq_tracer != NIL && - call_enabled_tracer( NULL, seq_tracer, NULL, am_trace_status, - p ? p->common.id : am_undefined) != am_trace) + call_enabled_tracer(NULL, seq_tracer, NULL, + TRACE_FUN_ENABLED, am_trace_status, + p ? p->common.id : am_undefined) != am_trace) #ifdef USE_VM_PROBES || (SEQ_TRACE_TOKEN(p) == am_have_dt_utag) #endif @@ -866,9 +963,10 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type, ASSERT(is_tuple(token) || is_nil(token)); if (token == NIL || (process && ERTS_TRACE_FLAGS(process) & F_SENSITIVE) || ERTS_TRACER_IS_NIL(seq_tracer) || - call_enabled_tracer( - NULL, seq_tracer, NULL, am_trace_status, - process ? process->common.id : am_undefined) != am_trace) { + call_enabled_tracer(NULL, seq_tracer, + NULL, TRACE_FUN_ENABLED, + am_trace_status, + process ? process->common.id : am_undefined) != am_trace) { return; } @@ -897,14 +995,13 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type, msg = TUPLE3(hp, am_EXIT, exitfrom, msg); hp += 4; } - mess = TUPLE5(hp, type_atom, lastcnt_serial, SEQ_TRACE_T_SENDER(token), - receiver, msg); + mess = TUPLE5(hp, type_atom, lastcnt_serial, SEQ_TRACE_T_SENDER(token), receiver, msg); hp += 6; seq_tracer_flags |= ERTS_SEQTFLGS2TFLGS(unsigned_val(SEQ_TRACE_T_FLAGS(token))); send_to_tracer_nif_raw(NULL, process, seq_tracer, seq_tracer_flags, - label, NULL, am_seq_trace, mess, + label, NULL, TRACE_FUN_DEFAULT, am_seq_trace, mess, THE_NON_VALUE, am_true); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); @@ -929,7 +1026,8 @@ erts_trace_return_to(Process *p, BeamInstr *pc) mfa = TUPLE3(hp, code_ptr[0], code_ptr[1], make_small(code_ptr[2])); } - SEND_TO_TRACER(p, am_return_to, mfa); + send_to_tracer_nif(p, &p->common, p->common.id, NULL, TRACE_FUN_T_CALL, + am_return_to, mfa, THE_NON_VALUE, am_true); } @@ -983,7 +1081,7 @@ erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, ErtsTracer *tracer) mfa = TUPLE3(hp, mod, name, make_small(arity)); hp += 4; send_to_tracer_nif_raw(p, NULL, *tracer, *tracee_flags, p->common.id, - NULL, am_return_from, mfa, retval, am_true); + NULL, TRACE_FUN_T_CALL, am_return_from, mfa, retval, am_true); } /* Send {trace_ts, Pid, exception_from, {Mod, Name, Arity}, {Class,Value}, @@ -1038,7 +1136,7 @@ erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value, cv = TUPLE2(hp, class, value); hp += 3; send_to_tracer_nif_raw(p, NULL, *tracer, *tracee_flags, p->common.id, - NULL, am_exception_from, mfa_tuple, cv, am_true); + NULL, TRACE_FUN_T_CALL, am_exception_from, mfa_tuple, cv, am_true); } /* @@ -1088,7 +1186,8 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, * use process flags */ tracee_flags = &ERTS_TRACE_FLAGS(p); - if (!is_tracer_proc_enabled(p, ERTS_PROC_LOCK_MAIN, &p->common, &tnif, am_call)) { + if (!is_tracer_enabled(p, ERTS_PROC_LOCK_MAIN, &p->common, &tnif, + TRACE_FUN_E_CALL, am_call)) { return 0; } } else { @@ -1103,7 +1202,9 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, } meta_flags = F_TRACE_CALLS | F_NOW_TS; tracee_flags = &meta_flags; - switch (call_enabled_tracer(p, *tracer, &tnif, am_call, p->common.id)) { + switch (call_enabled_tracer(p, *tracer, + &tnif, TRACE_FUN_T_CALL, + am_call, p->common.id)) { default: case am_remove: *tracer = erts_tracer_nil; case am_discard: return 0; @@ -1164,20 +1265,14 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, may remove it, and we still want to generate a trace message */ erts_tracer_update(&pre_ms_tracer, *tracer); tracer = &pre_ms_tracer; - pam_result = erts_match_set_run(p, match_spec, args, arity, - ERTS_PAM_TMP_RESULT, &return_flags); - if (is_non_value(pam_result)) { - erts_match_set_release_result(p); - UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); - ERTS_TRACER_CLEAR(&pre_ms_tracer); - return 0; - } + pam_result = erts_match_set_run_trace(p, p, + match_spec, args, arity, + ERTS_PAM_TMP_RESULT, &return_flags); } if (tracee_flags == &meta_flags) { /* Meta trace */ if (pam_result == am_false) { - erts_match_set_release_result(p); UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); ERTS_TRACER_CLEAR(&pre_ms_tracer); return return_flags; @@ -1185,13 +1280,12 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, } else { /* Non-meta trace */ if (*tracee_flags & F_TRACE_SILENT) { - erts_match_set_release_result(p); + erts_match_set_release_result_trace(p, pam_result); UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); ERTS_TRACER_CLEAR(&pre_ms_tracer); return 0; } if (pam_result == am_false) { - erts_match_set_release_result(p); UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); ERTS_TRACER_CLEAR(&pre_ms_tracer); return return_flags; @@ -1227,11 +1321,14 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, * Build the trace tuple and send it to the port. */ send_to_tracer_nif_raw(p, NULL, *tracer, *tracee_flags, p->common.id, - tnif, am_call, mfa_tuple, THE_NON_VALUE, pam_result); - erts_match_set_release_result(p); + tnif, TRACE_FUN_T_CALL, am_call, mfa_tuple, + THE_NON_VALUE, pam_result); - if (match_spec && tracer == &pre_ms_tracer) - ERTS_TRACER_CLEAR(&pre_ms_tracer); + if (match_spec) { + erts_match_set_release_result_trace(p, pam_result); + if (tracer == &pre_ms_tracer) + ERTS_TRACER_CLEAR(&pre_ms_tracer); + } return return_flags; } @@ -1249,9 +1346,10 @@ trace_proc(Process *c_p, ErtsProcLocks c_p_locks, Process *t_p, Eterm what, Eterm data) { ErtsTracerNif *tnif = NULL; - if (is_tracer_proc_enabled(c_p, c_p_locks, &t_p->common, &tnif, what)) - send_to_tracer_nif(c_p, &t_p->common, t_p->common.id, tnif, - what, data, THE_NON_VALUE); + if (is_tracer_enabled(c_p, c_p_locks, &t_p->common, &tnif, + TRACE_FUN_E_PROCS, what)) + send_to_tracer_nif(c_p, &t_p->common, t_p->common.id, tnif, TRACE_FUN_T_PROCS, + what, data, THE_NON_VALUE, am_true); } @@ -1267,31 +1365,34 @@ trace_proc_spawn(Process *p, Eterm what, Eterm pid, Eterm mod, Eterm func, Eterm args) { ErtsTracerNif *tnif = NULL; - if (is_tracer_proc_enabled(p, ERTS_PROC_LOCKS_ALL & + if (is_tracer_enabled(p, ERTS_PROC_LOCKS_ALL & ~(ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE), - &p->common, &tnif, what)) { + &p->common, &tnif, TRACE_FUN_E_PROCS, what)) { Eterm mfa; Eterm* hp; hp = HAlloc(p, 4); mfa = TUPLE3(hp, mod, func, args); hp += 4; - send_to_tracer_nif(p, &p->common, p->common.id, tnif, what, pid, mfa); + send_to_tracer_nif(p, &p->common, p->common.id, tnif, TRACE_FUN_T_PROCS, + what, pid, mfa, am_true); } } void save_calls(Process *p, Export *e) { - struct saved_calls *scb = ERTS_PROC_GET_SAVED_CALLS_BUF(p); - if (scb) { - Export **ct = &scb->ct[0]; - int len = scb->len; - - ct[scb->cur] = e; - if (++scb->cur >= len) - scb->cur = 0; - if (scb->n < len) - scb->n++; + if ((ERTS_TRACE_FLAGS(p) & F_SENSITIVE) == 0) { + struct saved_calls *scb = ERTS_PROC_GET_SAVED_CALLS_BUF(p); + if (scb) { + Export **ct = &scb->ct[0]; + int len = scb->len; + + ct[scb->cur] = e; + if (++scb->cur >= len) + scb->cur = 0; + if (scb->n < len) + scb->n++; + } } } @@ -1308,23 +1409,25 @@ void save_calls(Process *p, Export *e) * are all small (atomic) integers. */ void -trace_gc(Process *p, Eterm what) +trace_gc(Process *p, Eterm what, Uint size) { ErtsTracerNif *tnif = NULL; Eterm* hp; Eterm msg = NIL; - Uint size = 0; + Uint sz = 0; + Eterm tup; - if (is_tracer_proc_enabled( - p, ERTS_PROC_LOCK_MAIN, &p->common, &tnif, what)) { + if (is_tracer_enabled(p, ERTS_PROC_LOCK_MAIN, &p->common, &tnif, TRACE_FUN_E_GC, what)) { - (void) erts_process_gc_info(p, &size, NULL); - hp = HAlloc(p, size); + (void) erts_process_gc_info(p, &sz, NULL); + hp = HAlloc(p, sz + 3 + 2); msg = erts_process_gc_info(p, NULL, &hp); + tup = TUPLE2(hp, am_wordsize, make_small(size)); hp += 3; + msg = CONS(hp, tup, msg); hp += 2; - send_to_tracer_nif(p, &p->common, p->common.id, tnif, what, - msg, THE_NON_VALUE); + send_to_tracer_nif(p, &p->common, p->common.id, tnif, TRACE_FUN_T_GC, + what, msg, am_undefined, am_true); } } @@ -1387,7 +1490,7 @@ monitor_long_schedule_proc(Process *p, BeamInstr *in_fp, BeamInstr *out_fp, Uint { ErtsMessage *mp = erts_alloc_message(0, NULL); mp->data.heap_frag = bp; - erts_queue_message(monitor_p, NULL, mp, msg); + erts_queue_message(monitor_p, 0, mp, msg, am_system); } #endif } @@ -1452,7 +1555,7 @@ monitor_long_schedule_port(Port *pp, ErtsPortTaskType type, Uint time) { ErtsMessage *mp = erts_alloc_message(0, NULL); mp->data.heap_frag = bp; - erts_queue_message(monitor_p, NULL, mp, msg); + erts_queue_message(monitor_p, 0, mp, msg, am_system); } #endif } @@ -1527,7 +1630,7 @@ monitor_long_gc(Process *p, Uint time) { { ErtsMessage *mp = erts_alloc_message(0, NULL); mp->data.heap_frag = bp; - erts_queue_message(monitor_p, NULL, mp, msg); + erts_queue_message(monitor_p, 0, mp, msg, am_system); } #endif } @@ -1602,7 +1705,7 @@ monitor_large_heap(Process *p) { { ErtsMessage *mp = erts_alloc_message(0, NULL); mp->data.heap_frag = bp; - erts_queue_message(monitor_p, NULL, mp, msg); + erts_queue_message(monitor_p, 0, mp, msg, am_system); } #endif } @@ -1634,7 +1737,7 @@ monitor_generic(Process *p, Eterm type, Eterm spec) { { ErtsMessage *mp = erts_alloc_message(0, NULL); mp->data.heap_frag = bp; - erts_queue_message(monitor_p, NULL, mp, msg); + erts_queue_message(monitor_p, 0, mp, msg, am_system); } #endif @@ -1696,50 +1799,15 @@ profile_scheduler(Eterm scheduler_id, Eterm state) { } -void -profile_scheduler_q(Eterm scheduler_id, Eterm state, Eterm no_schedulers, Uint Ms, Uint s, Uint us) { - Eterm *hp, msg, timestamp; - -#ifndef ERTS_SMP -#define LOCAL_HEAP_SIZE (4 + 7) - DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); - UseTmpHeapNoproc(LOCAL_HEAP_SIZE); - - hp = local_heap; -#else - ErlHeapFragment *bp; - Uint hsz; - - hsz = 4 + 7; - - bp = new_message_buffer(hsz); - hp = bp->mem; -#endif - - erts_smp_mtx_lock(&smq_mtx); - - timestamp = TUPLE3(hp, make_small(Ms), make_small(s), make_small(us)); hp += 4; - msg = TUPLE6(hp, am_profile, am_scheduler, scheduler_id, state, no_schedulers, timestamp); hp += 7; -#ifndef ERTS_SMP - profile_send(NIL, msg); - UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); -#undef LOCAL_HEAP_SIZE -#else - enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, NIL, NIL, msg, bp); -#endif - erts_smp_mtx_unlock(&smq_mtx); - -} - /* Port profiling */ void trace_port_open(Port *p, Eterm calling_pid, Eterm drv_name) { ErtsTracerNif *tnif = NULL; ERTS_SMP_CHK_NO_PROC_LOCKS; - if (is_tracer_proc_enabled(NULL, 0, &p->common, &tnif, am_open)) - send_to_tracer_nif(NULL, &p->common, p->common.id, tnif, am_open, - calling_pid, drv_name); + if (is_tracer_enabled(NULL, 0, &p->common, &tnif, TRACE_FUN_E_PORTS, am_open)) + send_to_tracer_nif(NULL, &p->common, p->common.id, tnif, TRACE_FUN_T_PORTS, + am_open, calling_pid, drv_name, am_true); } /* Sends trace message: @@ -1756,9 +1824,9 @@ trace_port(Port *t_p, Eterm what, Eterm data) { ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(t_p) || erts_thr_progress_is_blocking()); ERTS_SMP_CHK_NO_PROC_LOCKS; - if (is_tracer_proc_enabled(NULL, 0, &t_p->common, &tnif, what)) - send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif, - what, data, THE_NON_VALUE); + if (is_tracer_enabled(NULL, 0, &t_p->common, &tnif, TRACE_FUN_E_PORTS, what)) + send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif, TRACE_FUN_T_PORTS, + what, data, THE_NON_VALUE, am_true); } @@ -1802,7 +1870,7 @@ trace_port_receive(Port *t_p, Eterm caller, Eterm what, ...) ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(t_p) || erts_thr_progress_is_blocking()); ERTS_SMP_CHK_NO_PROC_LOCKS; - if (is_tracer_proc_enabled(NULL, 0, &t_p->common, &tnif, am_receive)) { + if (is_tracer_enabled(NULL, 0, &t_p->common, &tnif, TRACE_FUN_E_RECEIVE, am_receive)) { /* We can use a stack heap here, as the nif is called in the context of a port */ #define LOCAL_HEAP_SIZE (3 + 3 + heap_bin_size(ERL_ONHEAP_BIN_LIMIT) + 3) @@ -1894,8 +1962,11 @@ trace_port_receive(Port *t_p, Eterm caller, Eterm what, ...) } data = TUPLE2(hp, caller, data); + hp += 3; + ASSERT(hp <= (local_heap + LOCAL_HEAP_SIZE) || orig_hp); send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif, - am_receive, data, THE_NON_VALUE); + TRACE_FUN_T_RECEIVE, + am_receive, data, THE_NON_VALUE, am_true); if (bptr && erts_refc_dectest(&bptr->refc, 1) == 0) erts_bin_free(bptr); @@ -1916,9 +1987,9 @@ trace_port_send(Port *t_p, Eterm receiver, Eterm msg, int exists) ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(t_p) || erts_thr_progress_is_blocking()); ERTS_SMP_CHK_NO_PROC_LOCKS; - if (is_tracer_proc_enabled(NULL, 0, &t_p->common, &tnif, op)) - send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif, - op, msg, receiver); + if (is_tracer_enabled(NULL, 0, &t_p->common, &tnif, TRACE_FUN_E_SEND, op)) + send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif, TRACE_FUN_T_SEND, + op, msg, receiver, am_true); } void trace_port_send_binary(Port *t_p, Eterm to, Eterm what, char *bin, Sint sz) @@ -1927,7 +1998,7 @@ void trace_port_send_binary(Port *t_p, Eterm to, Eterm what, char *bin, Sint sz) ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(t_p) || erts_thr_progress_is_blocking()); ERTS_SMP_CHK_NO_PROC_LOCKS; - if (is_tracer_proc_enabled(NULL, 0, &t_p->common, &tnif, am_send)) { + if (is_tracer_enabled(NULL, 0, &t_p->common, &tnif, TRACE_FUN_E_SEND, am_send)) { Eterm msg; Binary* bptr = NULL; #define LOCAL_HEAP_SIZE (3 + 3 + heap_bin_size(ERL_ONHEAP_BIN_LIMIT)) @@ -1946,8 +2017,8 @@ void trace_port_send_binary(Port *t_p, Eterm to, Eterm what, char *bin, Sint sz) msg = TUPLE2(hp, t_p->common.id, msg); hp += 3; - send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif, - am_send, msg, to); + send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif, TRACE_FUN_T_SEND, + am_send, msg, to, am_true); if (bptr && erts_refc_dectest(&bptr->refc, 1) == 0) erts_bin_free(bptr); @@ -1975,9 +2046,10 @@ trace_sched_ports_where(Port *t_p, Eterm what, Eterm where) { ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(t_p) || erts_thr_progress_is_blocking()); ERTS_SMP_CHK_NO_PROC_LOCKS; - if (is_tracer_proc_enabled(NULL, 0, &t_p->common, &tnif, what)) + if (is_tracer_enabled(NULL, 0, &t_p->common, &tnif, TRACE_FUN_E_SCHED_PORT, what)) send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, - tnif, what, where, THE_NON_VALUE); + tnif, TRACE_FUN_T_SCHED_PORT, + what, where, THE_NON_VALUE, am_true); } /* Port profiling */ @@ -2417,7 +2489,7 @@ sys_msg_dispatcher_func(void *unused) queue_proc_msg: mp = erts_alloc_message(0, NULL); mp->data.heap_frag = smqp->bp; - erts_queue_message(proc,&proc_locks,mp,smqp->msg); + erts_queue_message(proc,proc_locks,mp,smqp->msg,am_system); #ifdef DEBUG_PRINTOUTS erts_fprintf(stderr, "delivered\n"); #endif @@ -2523,14 +2595,97 @@ init_sys_msg_dispatcher(void) #include "erl_nif.h" +typedef struct { + char *name; + Uint arity; + ErlNifFunc *cb; +} ErtsTracerType; + struct ErtsTracerNif_ { HashBucket hb; Eterm module; struct erl_module_nif* nif_mod; - ErlNifFunc *enabled; - ErlNifFunc *trace; + ErtsTracerType tracers[NIF_TRACER_TYPES]; }; +static void init_tracer_template(ErtsTracerNif *tnif) { + + /* default tracer functions */ + tnif->tracers[TRACE_FUN_DEFAULT].name = "trace"; + tnif->tracers[TRACE_FUN_DEFAULT].arity = 6; + tnif->tracers[TRACE_FUN_DEFAULT].cb = NULL; + + tnif->tracers[TRACE_FUN_ENABLED].name = "enabled"; + tnif->tracers[TRACE_FUN_ENABLED].arity = 3; + tnif->tracers[TRACE_FUN_ENABLED].cb = NULL; + + /* specific tracer functions */ + tnif->tracers[TRACE_FUN_T_SEND].name = "trace_send"; + tnif->tracers[TRACE_FUN_T_SEND].arity = 6; + tnif->tracers[TRACE_FUN_T_SEND].cb = NULL; + + tnif->tracers[TRACE_FUN_T_RECEIVE].name = "trace_receive"; + tnif->tracers[TRACE_FUN_T_RECEIVE].arity = 6; + tnif->tracers[TRACE_FUN_T_RECEIVE].cb = NULL; + + tnif->tracers[TRACE_FUN_T_CALL].name = "trace_call"; + tnif->tracers[TRACE_FUN_T_CALL].arity = 6; + tnif->tracers[TRACE_FUN_T_CALL].cb = NULL; + + tnif->tracers[TRACE_FUN_T_SCHED_PROC].name = "trace_running_procs"; + tnif->tracers[TRACE_FUN_T_SCHED_PROC].arity = 6; + tnif->tracers[TRACE_FUN_T_SCHED_PROC].cb = NULL; + + tnif->tracers[TRACE_FUN_T_SCHED_PORT].name = "trace_running_ports"; + tnif->tracers[TRACE_FUN_T_SCHED_PORT].arity = 6; + tnif->tracers[TRACE_FUN_T_SCHED_PORT].cb = NULL; + + tnif->tracers[TRACE_FUN_T_GC].name = "trace_garbage_collection"; + tnif->tracers[TRACE_FUN_T_GC].arity = 6; + tnif->tracers[TRACE_FUN_T_GC].cb = NULL; + + tnif->tracers[TRACE_FUN_T_PROCS].name = "trace_procs"; + tnif->tracers[TRACE_FUN_T_PROCS].arity = 6; + tnif->tracers[TRACE_FUN_T_PROCS].cb = NULL; + + tnif->tracers[TRACE_FUN_T_PORTS].name = "trace_ports"; + tnif->tracers[TRACE_FUN_T_PORTS].arity = 6; + tnif->tracers[TRACE_FUN_T_PORTS].cb = NULL; + + /* specific enabled functions */ + tnif->tracers[TRACE_FUN_E_SEND].name = "enabled_send"; + tnif->tracers[TRACE_FUN_E_SEND].arity = 3; + tnif->tracers[TRACE_FUN_E_SEND].cb = NULL; + + tnif->tracers[TRACE_FUN_E_RECEIVE].name = "enabled_receive"; + tnif->tracers[TRACE_FUN_E_RECEIVE].arity = 3; + tnif->tracers[TRACE_FUN_E_RECEIVE].cb = NULL; + + tnif->tracers[TRACE_FUN_E_CALL].name = "enabled_call"; + tnif->tracers[TRACE_FUN_E_CALL].arity = 3; + tnif->tracers[TRACE_FUN_E_CALL].cb = NULL; + + tnif->tracers[TRACE_FUN_E_SCHED_PROC].name = "enabled_running_procs"; + tnif->tracers[TRACE_FUN_E_SCHED_PROC].arity = 3; + tnif->tracers[TRACE_FUN_E_SCHED_PROC].cb = NULL; + + tnif->tracers[TRACE_FUN_E_SCHED_PORT].name = "enabled_running_ports"; + tnif->tracers[TRACE_FUN_E_SCHED_PORT].arity = 3; + tnif->tracers[TRACE_FUN_E_SCHED_PORT].cb = NULL; + + tnif->tracers[TRACE_FUN_E_GC].name = "enabled_garbage_collection"; + tnif->tracers[TRACE_FUN_E_GC].arity = 3; + tnif->tracers[TRACE_FUN_E_GC].cb = NULL; + + tnif->tracers[TRACE_FUN_E_PROCS].name = "enabled_procs"; + tnif->tracers[TRACE_FUN_E_PROCS].arity = 3; + tnif->tracers[TRACE_FUN_E_PROCS].cb = NULL; + + tnif->tracers[TRACE_FUN_E_PORTS].name = "enabled_ports"; + tnif->tracers[TRACE_FUN_E_PORTS].arity = 3; + tnif->tracers[TRACE_FUN_E_PORTS].cb = NULL; +} + static Hash *tracer_hash = NULL; static erts_smp_rwmtx_t tracer_mtx; @@ -2543,31 +2698,32 @@ load_tracer_nif(const ErtsTracer tracer) ErlNifFunc *funcs; int num_of_funcs; ErtsTracerNif tnif_tmpl, *tnif; - int i; + ErtsTracerType *tracers; + int i,j; - if (mod && mod->curr.nif != NULL) { - instance = &mod->curr; - } else { + if (!mod || !mod->curr.nif) { return NULL; } - tnif_tmpl.enabled = NULL; - tnif_tmpl.trace = NULL; + instance = &mod->curr; + + init_tracer_template(&tnif_tmpl); tnif_tmpl.nif_mod = instance->nif; tnif_tmpl.module = ERTS_TRACER_MODULE(tracer); + tracers = tnif_tmpl.tracers; num_of_funcs = erts_nif_get_funcs(instance->nif, &funcs); for(i = 0; i < num_of_funcs; i++) { - if (strcmp("enabled",funcs[i].name) == 0 && funcs[i].arity == 3) { - tnif_tmpl.enabled = funcs + i; - } else if (strcmp("trace",funcs[i].name) == 0 && funcs[i].arity == 6) { - tnif_tmpl.trace = funcs + i; + for (j = 0; j < NIF_TRACER_TYPES; j++) { + if (strcmp(tracers[j].name, funcs[i].name) == 0 && tracers[j].arity == funcs[i].arity) { + tracers[j].cb = &(funcs[i]); + break; + } } } - if (tnif_tmpl.enabled == NULL || - tnif_tmpl.trace == NULL ) { + if (tracers[TRACE_FUN_DEFAULT].cb == NULL || tracers[TRACE_FUN_ENABLED].cb == NULL ) { return NULL; } @@ -2660,17 +2816,18 @@ erts_tracer_to_term(Process *p, ErtsTracer tracer) static ERTS_INLINE int send_to_tracer_nif_raw(Process *c_p, Process *tracee, const ErtsTracer tracer, Uint tracee_flags, - Eterm t_p_id, ErtsTracerNif *tnif, Eterm tag, Eterm msg, - Eterm extra, Eterm pam_result) + Eterm t_p_id, ErtsTracerNif *tnif, + enum ErtsTracerOpt topt, + Eterm tag, Eterm msg, Eterm extra, Eterm pam_result) { if (tnif || (tnif = lookup_tracer_nif(tracer)) != NULL) { #define MAP_SIZE 3 - Eterm argv[6], - local_heap[3+MAP_SIZE /* values */+(MAP_SIZE+1 /* keys */)]; + Eterm argv[6], local_heap[3+MAP_SIZE /* values */ + (MAP_SIZE+1 /* keys */)]; flatmap_t *map = (flatmap_t*)(local_heap+(MAP_SIZE+1)); Eterm *map_values = flatmap_get_values(map); - int argc = 6; + topt = (tnif->tracers[topt].cb) ? topt : TRACE_FUN_DEFAULT; + ASSERT(topt < NIF_TRACER_TYPES); argv[0] = tag; argv[1] = ERTS_TRACER_STATE(tracer); @@ -2681,8 +2838,7 @@ send_to_tracer_nif_raw(Process *c_p, Process *tracee, map->thing_word = MAP_HEADER_FLATMAP; map->size = MAP_SIZE; - map->keys = TUPLE3(local_heap, am_match_spec_result, am_scheduler_id, - am_timestamp); + map->keys = TUPLE3(local_heap, am_match_spec_result, am_scheduler_id, am_timestamp); *map_values++ = pam_result; if (tracee_flags & F_TRACE_SCHED_NO) @@ -2705,16 +2861,18 @@ send_to_tracer_nif_raw(Process *c_p, Process *tracee, #undef MAP_SIZE erts_nif_call_function(c_p, tracee ? tracee : c_p, - tnif->nif_mod, tnif->trace, argc, argv); + tnif->nif_mod, + tnif->tracers[topt].cb, + tnif->tracers[topt].arity, + argv); } return 1; } - static ERTS_INLINE int send_to_tracer_nif(Process *c_p, ErtsPTabElementCommon *t_p, - Eterm t_p_id, ErtsTracerNif *tnif, Eterm tag, - Eterm msg, Eterm extra) + Eterm t_p_id, ErtsTracerNif *tnif, enum ErtsTracerOpt topt, + Eterm tag, Eterm msg, Eterm extra, Eterm pam_result) { #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) if (c_p) { @@ -2733,29 +2891,35 @@ send_to_tracer_nif(Process *c_p, ErtsPTabElementCommon *t_p, return send_to_tracer_nif_raw(c_p, is_internal_pid(t_p->id) ? (Process*)t_p : NULL, t_p->tracer, t_p->trace_flags, - t_p_id, tnif, tag, msg, extra, - am_true); + t_p_id, tnif, topt, tag, msg, extra, + pam_result); } static ERTS_INLINE Eterm call_enabled_tracer(Process *c_p, const ErtsTracer tracer, - ErtsTracerNif **tnif_ret, Eterm tag, Eterm t_p_id) -{ + ErtsTracerNif **tnif_ret, + enum ErtsTracerOpt topt, + Eterm tag, Eterm t_p_id) { ErtsTracerNif *tnif = lookup_tracer_nif(tracer); if (tnif) { Eterm argv[] = {tag, ERTS_TRACER_STATE(tracer), t_p_id}; + topt = (tnif->tracers[topt].cb) ? topt : TRACE_FUN_ENABLED; + ASSERT(topt < NIF_TRACER_TYPES); + ASSERT(tnif->tracers[topt].cb != NULL); if (tnif_ret) *tnif_ret = tnif; - return erts_nif_call_function( - c_p, NULL, tnif->nif_mod, tnif->enabled, 3, argv); + return erts_nif_call_function(c_p, NULL, tnif->nif_mod, + tnif->tracers[topt].cb, + tnif->tracers[topt].arity, + argv); } return am_remove; } static int -is_tracer_proc_enabled(Process* c_p, ErtsProcLocks c_p_locks, - ErtsPTabElementCommon *t_p, - ErtsTracerNif **tnif_ret, Eterm tag) -{ +is_tracer_enabled(Process* c_p, ErtsProcLocks c_p_locks, + ErtsPTabElementCommon *t_p, + ErtsTracerNif **tnif_ret, + enum ErtsTracerOpt topt, Eterm tag) { Eterm nif_result; #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) @@ -2773,7 +2937,7 @@ is_tracer_proc_enabled(Process* c_p, ErtsProcLocks c_p_locks, } #endif - nif_result = call_enabled_tracer(c_p, t_p->tracer, tnif_ret, tag, t_p->id); + nif_result = call_enabled_tracer(c_p, t_p->tracer, tnif_ret, topt, tag, t_p->id); switch (nif_result) { case am_discard: return 0; case am_trace: return 1; @@ -2806,14 +2970,13 @@ is_tracer_proc_enabled(Process* c_p, ErtsProcLocks c_p_locks, erts_smp_proc_unlock(c_p, c_p_xlocks); } - return 0; } int erts_is_tracer_proc_enabled(Process* c_p, ErtsProcLocks c_p_locks, ErtsPTabElementCommon *t_p, Eterm type) { - return is_tracer_proc_enabled(c_p, c_p_locks, t_p, NULL, am_trace_status); + return is_tracer_enabled(c_p, c_p_locks, t_p, NULL, TRACE_FUN_ENABLED, am_trace_status); } int erts_is_tracer_enabled(Process *c_p, const ErtsTracer tracer) @@ -2821,7 +2984,7 @@ int erts_is_tracer_enabled(Process *c_p, const ErtsTracer tracer) ErtsTracerNif *tnif = lookup_tracer_nif(tracer); if (tnif) { Eterm nif_result = call_enabled_tracer(c_p, tracer, &tnif, - am_trace_status, + TRACE_FUN_ENABLED, am_trace_status, c_p->common.id); switch (nif_result) { case am_discard: diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h index 177fd373a6..de4beadb7e 100644 --- a/erts/emulator/beam/erl_trace.h +++ b/erts/emulator/beam/erl_trace.h @@ -56,6 +56,15 @@ struct binary; +typedef struct +{ + int on; + struct binary* match_spec; +} ErtsTracingEvent; + +extern ErtsTracingEvent erts_send_tracing[]; +extern ErtsTracingEvent erts_receive_tracing[]; + /* erl_bif_trace.c */ Eterm erl_seq_trace_info(Process *p, Eterm arg1); void erts_system_monitor_clear(Process *c_p); @@ -91,7 +100,7 @@ void erts_send_sys_msg_proc(Eterm, Eterm, Eterm, ErlHeapFragment *); #endif void trace_send(Process*, Eterm, Eterm); -void trace_receive(Process *, Eterm); +void trace_receive(Process*, Eterm, Eterm, ErtsTracingEvent*); Uint32 erts_call_trace(Process *p, BeamInstr mfa[], struct binary *match_spec, Eterm* args, int local, ErtsTracer *tracer); void erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, @@ -103,7 +112,7 @@ void trace_sched(Process*, ErtsProcLocks, Eterm); void trace_proc(Process*, ErtsProcLocks, Process*, Eterm, Eterm); void trace_proc_spawn(Process*, Eterm what, Eterm pid, Eterm mod, Eterm func, Eterm args); void save_calls(Process *p, Export *); -void trace_gc(Process *p, Eterm what); +void trace_gc(Process *p, Eterm what, Uint size); /* port tracing */ void trace_virtual_sched(Process*, ErtsProcLocks, Eterm); void trace_sched_ports(Port *pp, Eterm); diff --git a/erts/emulator/beam/erlang_lttng.h b/erts/emulator/beam/erlang_lttng.h index 43ceeda671..12f68e477b 100644 --- a/erts/emulator/beam/erlang_lttng.h +++ b/erts/emulator/beam/erlang_lttng.h @@ -218,13 +218,13 @@ TRACEPOINT_EVENT( driver_stop, TP_ARGS( char*, pid, - char*, driver, - char*, port + char*, port, + char*, driver ), TP_FIELDS( ctf_string(pid, pid) - ctf_string(driver, driver) ctf_string(port, port) + ctf_string(driver, driver) ) ) @@ -324,7 +324,7 @@ TRACEPOINT_EVENT( TRACEPOINT_EVENT( com_ericsson_otp, - aio_pool_add, + aio_pool_put, TP_ARGS( char*, port, int, length diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 723c25ff77..3c002d43a7 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -4429,22 +4429,32 @@ init_done: SKIP(1+atom_extra_skip); atom_extra_skip = 0; break; - case PID_EXT: case NEW_PID_EXT: + atom_extra_skip = 12; + goto case_PID; + case PID_EXT: atom_extra_skip = 9; + case_PID: /* In case it is an external pid */ heap_size += EXTERNAL_THING_HEAD_SIZE + 1; terms++; break; - case PORT_EXT: case NEW_PORT_EXT: + atom_extra_skip = 8; + goto case_PORT; + case PORT_EXT: atom_extra_skip = 5; + case_PORT: /* In case it is an external port */ heap_size += EXTERNAL_THING_HEAD_SIZE + 1; terms++; break; - case NEW_REFERENCE_EXT: case NEWER_REFERENCE_EXT: + atom_extra_skip = 4; + goto case_NEW_REFERENCE; + case NEW_REFERENCE_EXT: + atom_extra_skip = 1; + case_NEW_REFERENCE: { int id_words; @@ -4455,7 +4465,7 @@ init_done: goto error; ep += 2; - atom_extra_skip = 1 + 4*id_words; + atom_extra_skip += 4*id_words; /* In case it is an external ref */ #if defined(ARCH_64) heap_size += EXTERNAL_THING_HEAD_SIZE + id_words/2 + 1; diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index a7bc990deb..550db95ba8 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -997,7 +997,7 @@ Eterm erl_is_function(Process* p, Eterm arg1, Eterm arg2); #define ERTS_CPC_ALLOW_GC (1 << 0) #define ERTS_CPC_COPY_LITERALS (1 << 1) #define ERTS_CPC_ALL (ERTS_CPC_ALLOW_GC | ERTS_CPC_COPY_LITERALS) -Eterm erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp); +Eterm erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp, int fcalls); typedef struct { Eterm *ptr; @@ -1483,9 +1483,19 @@ do { \ #define MatchSetGetSource(MPSP) erts_match_set_get_source(MPSP) -extern Binary *erts_match_set_compile(Process *p, Eterm matchexpr); +extern Binary *erts_match_set_compile(Process *p, Eterm matchexpr, Eterm MFA); Eterm erts_match_set_lint(Process *p, Eterm matchexpr); extern void erts_match_set_release_result(Process* p); +ERTS_GLB_INLINE void erts_match_set_release_result_trace(Process* p, Eterm); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE +void erts_match_set_release_result_trace(Process* p, Eterm pam_result) +{ + if (is_not_immed(pam_result)) + erts_match_set_release_result(p); +} +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ enum erts_pam_run_flags { ERTS_PAM_TMP_RESULT=1, @@ -1493,10 +1503,12 @@ enum erts_pam_run_flags { ERTS_PAM_CONTIGUOUS_TUPLE=4, ERTS_PAM_IGNORE_TRACE_SILENT=8 }; -extern Eterm erts_match_set_run(Process *p, Binary *mpsp, - Eterm *args, int num_args, - enum erts_pam_run_flags in_flags, - Uint32 *return_flags); +extern Eterm erts_match_set_run_trace(Process *p, + Process *self, + Binary *mpsp, + Eterm *args, int num_args, + enum erts_pam_run_flags in_flags, + Uint32 *return_flags); extern Eterm erts_match_set_get_source(Binary *mpsp); extern void erts_match_prog_foreach_offheap(Binary *b, void (*)(ErlOffHeap *, void *), diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index b14ca77a04..b9f6ab04c6 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -1432,10 +1432,11 @@ finalize_force_imm_drv_call(ErtsTryImmDrvCallState *sp) static ERTS_INLINE void queue_port_sched_op_reply(Process *rp, - ErtsProcLocks *rp_locksp, + ErtsProcLocks rp_locks, ErtsHeapFactory* factory, Uint32 *ref_num, - Eterm msg) + Eterm msg, + Port* prt) { Eterm* hp = erts_produce_heap(factory, ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE, 0); Eterm ref; @@ -1448,11 +1449,12 @@ queue_port_sched_op_reply(Process *rp, erts_factory_trim_and_close(factory, &msg, 1); - erts_queue_message(rp, rp_locksp, factory->message, msg); + erts_queue_message(rp, rp_locks, factory->message, msg, + prt ? prt->common.id : am_undefined); } static void -port_sched_op_reply(Eterm to, Uint32 *ref_num, Eterm msg) +port_sched_op_reply(Eterm to, Uint32 *ref_num, Eterm msg, Port* prt) { Process *rp = erts_proc_lookup_raw(to); if (rp) { @@ -1478,10 +1480,11 @@ port_sched_op_reply(Eterm to, Uint32 *ref_num, Eterm msg) factory.off_heap)); queue_port_sched_op_reply(rp, - &rp_locks, + rp_locks, &factory, ref_num, - msg_copy); + msg_copy, + prt); if (rp_locks) erts_smp_proc_unlock(rp, rp_locks); @@ -1651,7 +1654,7 @@ port_badsig(Port *prt, erts_aint32_t state, int op, state, sigdp->flags & ERTS_P2P_SIG_DATA_FLG_BAD_OUTPUT); if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY) - port_sched_op_reply(sigdp->caller, sigdp->ref, am_badarg); + port_sched_op_reply(sigdp->caller, sigdp->ref, am_badarg, prt); return ERTS_PORT_REDS_BADSIG; } /* port_badsig */ /* bad_port_signal() will @@ -1820,7 +1823,7 @@ port_sig_outputv(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *s } if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY) - port_sched_op_reply(sigdp->caller, sigdp->ref, reply); + port_sched_op_reply(sigdp->caller, sigdp->ref, reply, prt); cleanup_scheduled_outputv(sigdp->u.outputv.evp, sigdp->u.outputv.cbinp); @@ -1928,7 +1931,7 @@ port_sig_output(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *si } if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY) - port_sched_op_reply(sigdp->caller, sigdp->ref, reply); + port_sched_op_reply(sigdp->caller, sigdp->ref, reply, prt); cleanup_scheduled_output(sigdp->u.output.bufp); @@ -2636,7 +2639,7 @@ port_sig_exit(Port *prt, if (sigdp->u.exit.bp) free_message_buffer(sigdp->u.exit.bp); if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY) - port_sched_op_reply(sigdp->caller, sigdp->ref, msg); + port_sched_op_reply(sigdp->caller, sigdp->ref, msg, prt); return ERTS_PORT_REDS_EXIT; } @@ -2829,7 +2832,7 @@ port_sig_connect(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *s msg = am_true; } if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY) - port_sched_op_reply(sigdp->caller, sigdp->ref, msg); + port_sched_op_reply(sigdp->caller, sigdp->ref, msg, prt); return ERTS_PORT_REDS_CONNECT; } @@ -2912,7 +2915,7 @@ port_sig_unlink(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *si if (op == ERTS_PROC2PORT_SIG_EXEC) port_unlink(prt, sigdp->u.unlink.from); if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY) - port_sched_op_reply(sigdp->caller, sigdp->ref, am_true); + port_sched_op_reply(sigdp->caller, sigdp->ref, am_true, prt); return ERTS_PORT_REDS_UNLINK; } @@ -3007,7 +3010,7 @@ port_sig_link(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *sigd port_link_failure(sigdp->u.link.port, sigdp->u.link.to); } if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY) - port_sched_op_reply(sigdp->caller, sigdp->ref, am_true); + port_sched_op_reply(sigdp->caller, sigdp->ref, am_true, prt); return ERTS_PORT_REDS_LINK; } @@ -3064,7 +3067,8 @@ init_ack_send_reply(Port *port, Eterm resp) } port_sched_op_reply(port->async_open_port->to, port->async_open_port->ref, - resp); + resp, + port); erts_free(ERTS_ALC_T_PRTSD, port->async_open_port); port->async_open_port = NULL; @@ -3461,7 +3465,7 @@ deliver_result(Port *prt, Eterm sender, Eterm pid, Eterm res) sz_res + 3, &hp, &ohp); res = copy_struct(res, sz_res, &hp, ohp); tuple = TUPLE2(hp, sender, res); - erts_queue_message(rp, &rp_locks, mp, tuple); + erts_queue_message(rp, rp_locks, mp, tuple, sender); if (rp_locks) erts_smp_proc_unlock(rp, rp_locks); @@ -3562,7 +3566,7 @@ static void deliver_read_message(Port* prt, erts_aint32_t state, Eterm to, trace_port_send(prt, to, tuple, 1); ERL_MESSAGE_TOKEN(mp) = am_undefined; - erts_queue_message(rp, &rp_locks, mp, tuple); + erts_queue_message(rp, rp_locks, mp, tuple, prt->common.id); if (rp_locks) erts_smp_proc_unlock(rp, rp_locks); if (!scheduler) @@ -3734,7 +3738,7 @@ deliver_vec_message(Port* prt, /* Port */ trace_port_send(prt, to, tuple, 1); ERL_MESSAGE_TOKEN(mp) = am_undefined; - erts_queue_message(rp, &rp_locks, mp, tuple); + erts_queue_message(rp, rp_locks, mp, tuple, prt->common.id); erts_smp_proc_unlock(rp, rp_locks); if (!scheduler) erts_proc_dec_refc(rp); @@ -3861,7 +3865,7 @@ terminate_port(Port *prt) lttng_decl_procbuf(proc_str); lttng_pid_to_str(connected_id, proc_str); lttng_port_to_str(prt, port_str); - LTTNG3(driver_stop, proc_str, drv->name, port_str); + LTTNG3(driver_stop, proc_str, port_str, drv->name); } #endif @@ -4388,10 +4392,11 @@ port_sig_control(Port *prt, &factory.hp, factory.off_heap); queue_port_sched_op_reply(rp, - &rp_locks, + rp_locks, &factory, sigdp->ref, - msg); + msg, + prt); if (rp_locks) erts_smp_proc_unlock(rp, rp_locks); @@ -4402,7 +4407,7 @@ port_sig_control(Port *prt, /* failure */ if (sigdp->caller != ERTS_INVALID_PID) - port_sched_op_reply(sigdp->caller, sigdp->ref, am_badarg); + port_sched_op_reply(sigdp->caller, sigdp->ref, am_badarg, prt); done: @@ -4739,10 +4744,11 @@ port_sig_call(Port *prt, msg = TUPLE2(hp, am_ok, msg); queue_port_sched_op_reply(rp, - &rp_locks, + rp_locks, &factory, sigdp->ref, - msg); + msg, + prt); if (rp_locks) erts_smp_proc_unlock(rp, rp_locks); @@ -4754,7 +4760,7 @@ port_sig_call(Port *prt, } } - port_sched_op_reply(sigdp->caller, sigdp->ref, am_badarg); + port_sched_op_reply(sigdp->caller, sigdp->ref, am_badarg, prt); done: @@ -4969,7 +4975,7 @@ port_sig_info(Port *prt, { ASSERT(sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY); if (op != ERTS_PROC2PORT_SIG_EXEC) - port_sched_op_reply(sigdp->caller, sigdp->ref, am_undefined); + port_sched_op_reply(sigdp->caller, sigdp->ref, am_undefined, prt); else { Eterm *hp, *hp_start; Uint hsz; @@ -4995,10 +5001,11 @@ port_sig_info(Port *prt, mp->data.heap_frag = bp; erts_factory_selfcontained_message_init(&factory, mp, hp); queue_port_sched_op_reply(rp, - &rp_locks, + rp_locks, &factory, sigdp->ref, - value); + value, + prt); } if (rp_locks) erts_smp_proc_unlock(rp, rp_locks); @@ -5115,7 +5122,7 @@ reply_io_bytes(void *vreq) msg = TUPLE4(hp, ref, make_small(sched_id), ein, eout); - erts_queue_message(rp, &rp_locks, mp, msg); + erts_queue_message(rp, rp_locks, mp, msg, am_system); if (req->sched_id == sched_id) rp_locks &= ~ERTS_PROC_LOCK_MAIN; @@ -5613,7 +5620,7 @@ void driver_report_exit(ErlDrvPort ix, int status) trace_port_send(prt, pid, tuple, 1); ERL_MESSAGE_TOKEN(mp) = am_undefined; - erts_queue_message(rp, &rp_locks, mp, tuple); + erts_queue_message(rp, rp_locks, mp, tuple, prt->common.id); erts_smp_proc_unlock(rp, rp_locks); if (!scheduler) @@ -6217,15 +6224,20 @@ driver_deliver_term(Port *prt, Eterm to, ErlDrvTermData* data, int len) done: if (res > 0) { + Eterm from = am_undefined; mess = ESTACK_POP(stack); /* get resulting value */ erts_factory_trim_and_close(&factory, &mess, 1); - if (prt && IS_TRACED_FL(prt, F_TRACE_SEND)) - trace_port_send(prt, to, mess, 1); + if (prt) { + if (IS_TRACED_FL(prt, F_TRACE_SEND)) { + trace_port_send(prt, to, mess, 1); + } + from = prt->common.id; + } /* send message */ ERL_MESSAGE_TOKEN(factory.message) = am_undefined; - erts_queue_message(rp, &rp_locks, factory.message, mess); + erts_queue_message(rp, rp_locks, factory.message, mess, from); } else if (res == -2) { /* this clause only happens when we were requested to diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index 748fba15c7..dda3ba565c 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -92,6 +92,9 @@ #define ErtsInArea(ptr,start,nbytes) \ ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes)) +#define ErtsContainerStruct(ptr, type, member) \ + (type *)((char *)(1 ? (ptr) : &((type *)0)->member) - offsetof(type, member)) + #if defined (__WIN32__) # include "erl_win_sys.h" #else diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index b280995488..cedc88e5fe 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -71,41 +71,6 @@ #define HAVE_MALLOPT 0 #endif -/* profile_scheduler mini message queue */ - -typedef struct { - Uint scheduler_id; - Uint no_schedulers; - Uint Ms; - Uint s; - Uint us; - Eterm state; -} profile_sched_msg; - -typedef struct { - profile_sched_msg msg[2]; - Uint n; -} profile_sched_msg_q; - -#ifdef ERTS_SMP - -#if 0 /* Unused */ -static void -dispatch_profile_msg_q(profile_sched_msg_q *psmq) -{ - int i = 0; - profile_sched_msg *msg = NULL; - ASSERT(psmq != NULL); - for (i = 0; i < psmq->n; i++) { - msg = &(psmq->msg[i]); - profile_scheduler_q(make_small(msg->scheduler_id), msg->state, am_undefined, msg->Ms, msg->s, msg->us); - } -} -#endif - -#endif - - Eterm* erts_heap_alloc(Process* p, Uint need, Uint xtra) { @@ -2298,7 +2263,7 @@ static void do_send_logger_message(Eterm *hp, ErlOffHeap *ohp, ErlHeapFragment * { ErtsMessage *mp = erts_alloc_message(0, NULL); mp->data.heap_frag = bp; - erts_queue_message(p, NULL /* only used for smp build */, mp, message); + erts_queue_message(p, 0, mp, message, am_system); } #endif } |