diff options
Diffstat (limited to 'erts/emulator/beam')
168 files changed, 6441 insertions, 4152 deletions
diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c index 099c00bcf6..a5e778e4aa 100644 --- a/erts/emulator/beam/atom.c +++ b/erts/emulator/beam/atom.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2013. All Rights Reserved. + * Copyright Ericsson AB 1996-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/atom.h b/erts/emulator/beam/atom.h index 2c002ca92f..fbd0528009 100644 --- a/erts/emulator/beam/atom.h +++ b/erts/emulator/beam/atom.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2013. All Rights Reserved. + * Copyright Ericsson AB 1996-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 169b071cd7..3022c0a99a 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1996-2013. All Rights Reserved. +# Copyright Ericsson AB 1996-2016. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -43,7 +43,7 @@ atom false true atom Underscore='_' atom Noname='nonode@nohost' atom EOT='$end_of_table' -atom Cookie='' +atom Empty='' # # Used in the Beam emulator loop. (Smaller literals usually means tighter code.) @@ -161,6 +161,7 @@ atom close atom closed atom code atom command +atom commandv atom compact atom compat_rel atom compile @@ -197,6 +198,7 @@ atom dirty_cpu_schedulers_online atom dirty_io atom disable_trace atom disabled +atom discard atom display_items atom dist atom dist_cmd @@ -231,6 +233,7 @@ atom exception_trace atom extended atom Eq='=:=' atom Eqeq='==' +atom erl_tracer atom erlang atom ERROR='ERROR' atom error_handler @@ -243,6 +246,9 @@ atom exact_reductions atom exclusive atom exit_status atom existing +atom existing_processes +atom existing_ports +atom existing atom exiting atom exports atom external @@ -267,6 +273,10 @@ atom garbage_collecting atom garbage_collection atom garbage_collection_info atom gc_end +atom gc_major_end +atom gc_major_start +atom gc_minor_end +atom gc_minor_start atom gc_start atom Ge='>=' atom generational @@ -353,6 +363,7 @@ atom match atom match_limit atom match_limit_recursion atom match_spec +atom match_spec_result atom max atom maximum atom max_tables max_processes @@ -401,6 +412,8 @@ atom net_kernel_terminated atom never_utf atom new atom new_index +atom new_processes +atom new_ports atom new_uniq atom newline atom next @@ -540,6 +553,7 @@ atom scheme atom scientific atom scope atom seconds +atom send_to_non_existing_process atom sensitive atom sequential_tracer atom sequential_trace_token @@ -561,6 +575,7 @@ atom size atom sl_alloc atom spawn_executable atom spawn_driver +atom spawned atom ssl_tls atom stack_size atom start @@ -569,6 +584,7 @@ atom static atom stderr_to_stdout atom stop atom stream +atom strict_monotonic atom strict_monotonic_timestamp atom sunrm atom suspend @@ -596,8 +612,9 @@ atom total_active_tasks atom total_heap_size atom total_run_queue_lengths atom tpkt -atom trace trace_ts traced +atom trace trace_ts traced atom trace_control_word +atom trace_status atom tracer atom trap_exit atom trim diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 0b47fc3586..40d44dda4c 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2013. All Rights Reserved. + * Copyright Ericsson AB 1999-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -38,7 +38,7 @@ #include "erl_thr_progress.h" static void set_default_trace_pattern(Eterm module); -static Eterm check_process_code(Process* rp, Module* modp, Uint flags, int *redsp); +static Eterm check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls); static void delete_code(Module* modp); static void decrement_refc(BeamCodeHeader*); static int any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size); @@ -467,7 +467,7 @@ check_old_code_1(BIF_ALIST_1) } Eterm -erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp) +erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp, int fcalls) { Module* modp; Eterm res; @@ -483,7 +483,7 @@ erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp) return am_false; erts_rlock_old_code(code_ix); res = (!modp->old.code_hdr ? am_false : - check_process_code(c_p, modp, flags, redsp)); + check_process_code(c_p, modp, flags, redsp, fcalls)); erts_runlock_old_code(code_ix); return res; @@ -506,7 +506,7 @@ BIF_RETTYPE erts_internal_check_process_code_2(BIF_ALIST_2) goto badarg; } - res = erts_check_process_code(BIF_P, BIF_ARG_1, flags, &reds); + res = erts_check_process_code(BIF_P, BIF_ARG_1, flags, &reds, BIF_P->fcalls); ASSERT(is_value(res)); @@ -625,8 +625,8 @@ BIF_RETTYPE call_on_load_function_1(BIF_ALIST_1) { Module* modp = erts_get_module(BIF_ARG_1, erts_active_code_ix()); - if (modp && modp->curr.code_hdr) { - BIF_TRAP_CODE_PTR_0(BIF_P, modp->curr.code_hdr->on_load_function_ptr); + if (modp && modp->old.code_hdr) { + BIF_TRAP_CODE_PTR_0(BIF_P, modp->old.code_hdr->on_load_function_ptr); } else { BIF_ERROR(BIF_P, BADARG); @@ -651,14 +651,14 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) code_ix = erts_active_code_ix(); modp = erts_get_module(BIF_ARG_1, code_ix); - if (!modp || !modp->curr.code_hdr) { + if (!modp || !modp->old.code_hdr) { error: erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_release_code_write_permission(); BIF_ERROR(BIF_P, BADARG); } - if (modp->curr.code_hdr->on_load_function_ptr == NULL) { + if (modp->old.code_hdr->on_load_function_ptr == NULL) { goto error; } if (BIF_ARG_2 != am_false && BIF_ARG_2 != am_true) { @@ -667,44 +667,55 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) if (BIF_ARG_2 == am_true) { int i; + struct erl_module_instance t; + + /* + * Swap old and new code. + */ + t = modp->curr; + modp->curr = modp->old; + modp->old = t; /* * The on_load function succeded. Fix up export entries. */ for (i = 0; i < export_list_size(code_ix); i++) { Export *ep = export_list(i,code_ix); - if (ep != NULL && - ep->code[0] == BIF_ARG_1 && - ep->code[4] != 0) { + if (ep == NULL || ep->code[0] != BIF_ARG_1) { + continue; + } + if (ep->code[4] != 0) { ep->addressv[code_ix] = (void *) ep->code[4]; ep->code[4] = 0; + } else { + if (ep->addressv[code_ix] == ep->code+3 && + ep->code[3] == (BeamInstr) em_apply_bif) { + continue; + } + ep->addressv[code_ix] = ep->code+3; + ep->code[3] = (BeamInstr) em_call_error_handler; } } modp->curr.code_hdr->on_load_function_ptr = NULL; set_default_trace_pattern(BIF_ARG_1); } else if (BIF_ARG_2 == am_false) { - BeamInstr* code; - BeamInstr* end; + int i; /* - * The on_load function failed. Remove the loaded code. - * This is an combination of delete and purge. We purge - * the current code; the old code is not touched. + * The on_load function failed. Remove references to the + * code that is about to be purged from the export entries. */ - erts_total_code_size -= modp->curr.code_length; - code = (BeamInstr*) modp->curr.code_hdr; - end = (BeamInstr *) ((char *)code + modp->curr.code_length); - erts_cleanup_funs_on_purge(code, end); - beam_catches_delmod(modp->curr.catches, code, modp->curr.code_length, - erts_active_code_ix()); - if (modp->curr.code_hdr->literals_start) { - erts_free(ERTS_ALC_T_LITERAL, modp->curr.code_hdr->literals_start); - } - erts_free(ERTS_ALC_T_CODE, modp->curr.code_hdr); - modp->curr.code_hdr = NULL; - modp->curr.code_length = 0; - modp->curr.catches = BEAM_CATCHES_NIL; - erts_remove_from_ranges(code); + + for (i = 0; i < export_list_size(code_ix); i++) { + Export *ep = export_list(i,code_ix); + if (ep == NULL || ep->code[0] != BIF_ARG_1) { + continue; + } + if (ep->code[3] == (BeamInstr) em_apply_bif) { + continue; + } + ep->code[4] = 0; + } } erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); @@ -719,13 +730,13 @@ set_default_trace_pattern(Eterm module) Binary *match_spec; Binary *meta_match_spec; struct trace_pattern_flags trace_pattern_flags; - Eterm meta_tracer_pid; + ErtsTracer meta_tracer; erts_get_default_trace_pattern(&trace_pattern_is_on, &match_spec, &meta_match_spec, &trace_pattern_flags, - &meta_tracer_pid); + &meta_tracer); if (trace_pattern_is_on) { Eterm mfa[1]; mfa[0] = module; @@ -733,7 +744,7 @@ set_default_trace_pattern(Eterm module) match_spec, meta_match_spec, 1, trace_pattern_flags, - meta_tracer_pid, 1); + meta_tracer, 1); } } @@ -753,7 +764,7 @@ check_mod_funs(Process *p, ErlOffHeap *off_heap, char *area, size_t area_size) static Eterm -check_process_code(Process* rp, Module* modp, Uint flags, int *redsp) +check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls) { BeamInstr* start; char* literals; @@ -832,7 +843,7 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp) /* * Message queue can contains funs, but (at least currently) no - * constants. If we got references to this module from the message + * literals. If we got references to this module from the message * queue, a GC cannot remove these... */ @@ -853,7 +864,7 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp) for (; hfrag; hfrag = hfrag->next) { if (check_mod_funs(rp, &hfrag->off_heap, mod_start, mod_size)) return am_true; - /* Should not contain any constants... */ + /* Should not contain any literals... */ ASSERT(!any_heap_refs(&hfrag->mem[0], &hfrag->mem[hfrag->used_size], literals, @@ -908,7 +919,7 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp) #ifdef DEBUG /* * Message buffer fragments should not have any references - * to constants, and off heap lists should already have + * to literals, and off heap lists should already have * been moved into process off heap structure. */ for (msgp = rp->msg_frag; msgp; msgp = msgp->next) { @@ -945,7 +956,7 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp) need_gc &= ~done_gc; /* - * Try to get rid of constants by by garbage collecting. + * Try to get rid of literals by by garbage collecting. * Clear both fvalue and ftrace. */ @@ -955,7 +966,7 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp) if (need_gc & ERTS_ORDINARY_GC__) { FLAGS(rp) |= F_NEED_FULLSWEEP; - *redsp += erts_garbage_collect_nobump(rp, 0, rp->arg_reg, rp->arity); + *redsp += erts_garbage_collect_nobump(rp, 0, rp->arg_reg, rp->arity, fcalls); done_gc |= ERTS_ORDINARY_GC__; } if (need_gc & ERTS_LITERAL_GC__) { @@ -1264,6 +1275,7 @@ delete_code(Module* modp) modp->curr.code_length = 0; modp->curr.catches = BEAM_CATCHES_NIL; modp->curr.nif = NULL; + } diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index 5d471d168b..1e30e8d8d1 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2013. All Rights Reserved. + * Copyright Ericsson AB 2000-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -92,15 +92,15 @@ get_mtime(Process *c_p) /* ** Helpers */ -static Eterm do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg, - int local, Binary* ms, Eterm tracer_pid); +static ErtsTracer do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg, + int local, Binary* ms, ErtsTracer tracer); static void set_break(BpFunctions* f, Binary *match_spec, Uint break_flags, - enum erts_break_op count_op, Eterm tracer_pid); + enum erts_break_op count_op, ErtsTracer tracer); static void set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags, enum erts_break_op count_op, - Eterm tracer_pid); + ErtsTracer tracer); static void clear_break(BpFunctions* f, Uint break_flags); static int clear_function_break(BeamInstr *pc, Uint break_flags); @@ -108,7 +108,7 @@ static int clear_function_break(BeamInstr *pc, Uint break_flags); static BpDataTime* get_time_break(BeamInstr *pc); static GenericBpData* check_break(BeamInstr *pc, Uint break_flags); -static void bp_meta_unref(BpMetaPid* bmp); +static void bp_meta_unref(BpMetaTracer* bmt); static void bp_count_unref(BpCount* bcp); static void bp_time_unref(BpDataTime* bdt); static void consolidate_bp_data(Module* modp, BeamInstr* pc, int local); @@ -248,7 +248,10 @@ erts_bp_match_export(BpFunctions* f, Eterm mfa[3], int specified) void erts_bp_free_matched_functions(BpFunctions* f) { - Free(f->matching); + if (f->matching) { + Free(f->matching); + } + else ASSERT(f->matched == 0); } void @@ -302,7 +305,7 @@ consolidate_bp_data(Module* modp, BeamInstr* pc, int local) MatchSetUnref(dst->local_ms); } if (flags & ERTS_BPF_META_TRACE) { - bp_meta_unref(dst->meta_pid); + bp_meta_unref(dst->meta_tracer); MatchSetUnref(dst->meta_ms); } if (flags & ERTS_BPF_COUNT) { @@ -343,8 +346,8 @@ consolidate_bp_data(Module* modp, BeamInstr* pc, int local) MatchSetRef(dst->local_ms); } if (flags & ERTS_BPF_META_TRACE) { - dst->meta_pid = src->meta_pid; - erts_refc_inc(&dst->meta_pid->refc, 1); + dst->meta_tracer = src->meta_tracer; + erts_refc_inc(&dst->meta_tracer->refc, 1); dst->meta_ms = src->meta_ms; MatchSetRef(dst->meta_ms); } @@ -436,13 +439,13 @@ uninstall_breakpoint(BeamInstr* pc) void erts_set_trace_break(BpFunctions* f, Binary *match_spec) { - set_break(f, match_spec, ERTS_BPF_LOCAL_TRACE, 0, am_true); + set_break(f, match_spec, ERTS_BPF_LOCAL_TRACE, 0, erts_tracer_true); } void -erts_set_mtrace_break(BpFunctions* f, Binary *match_spec, Eterm tracer_pid) +erts_set_mtrace_break(BpFunctions* f, Binary *match_spec, ErtsTracer tracer) { - set_break(f, match_spec, ERTS_BPF_META_TRACE, 0, tracer_pid); + set_break(f, match_spec, ERTS_BPF_META_TRACE, 0, tracer); } void @@ -450,13 +453,13 @@ erts_set_call_trace_bif(BeamInstr *pc, Binary *match_spec, int local) { Uint flags = local ? ERTS_BPF_LOCAL_TRACE : ERTS_BPF_GLOBAL_TRACE; - set_function_break(pc, match_spec, flags, 0, NIL); + set_function_break(pc, match_spec, flags, 0, erts_tracer_nil); } void -erts_set_mtrace_bif(BeamInstr *pc, Binary *match_spec, Eterm tracer_pid) +erts_set_mtrace_bif(BeamInstr *pc, Binary *match_spec, ErtsTracer tracer) { - set_function_break(pc, match_spec, ERTS_BPF_META_TRACE, 0, tracer_pid); + set_function_break(pc, match_spec, ERTS_BPF_META_TRACE, 0, tracer); } void @@ -464,7 +467,7 @@ erts_set_time_trace_bif(BeamInstr *pc, enum erts_break_op count_op) { set_function_break(pc, NULL, ERTS_BPF_TIME_TRACE|ERTS_BPF_TIME_TRACE_ACTIVE, - count_op, NIL); + count_op, erts_tracer_nil); } void @@ -474,21 +477,21 @@ erts_clear_time_trace_bif(BeamInstr *pc) { void erts_set_debug_break(BpFunctions* f) { - set_break(f, NULL, ERTS_BPF_DEBUG, 0, NIL); + set_break(f, NULL, ERTS_BPF_DEBUG, 0, erts_tracer_nil); } void erts_set_count_break(BpFunctions* f, enum erts_break_op count_op) { set_break(f, 0, ERTS_BPF_COUNT|ERTS_BPF_COUNT_ACTIVE, - count_op, NIL); + count_op, erts_tracer_nil); } void erts_set_time_break(BpFunctions* f, enum erts_break_op count_op) { set_break(f, 0, ERTS_BPF_TIME_TRACE|ERTS_BPF_TIME_TRACE_ACTIVE, - count_op, NIL); + count_op, erts_tracer_nil); } void @@ -625,19 +628,26 @@ erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg) if (bp_flags & ERTS_BPF_LOCAL_TRACE) { ASSERT((bp_flags & ERTS_BPF_GLOBAL_TRACE) == 0); - (void) do_call_trace(c_p, I, reg, 1, bp->local_ms, am_true); + (void) do_call_trace(c_p, I, reg, 1, bp->local_ms, erts_tracer_true); } else if (bp_flags & ERTS_BPF_GLOBAL_TRACE) { - (void) do_call_trace(c_p, I, reg, 0, bp->local_ms, am_true); + (void) do_call_trace(c_p, I, reg, 0, bp->local_ms, erts_tracer_true); } if (bp_flags & ERTS_BPF_META_TRACE) { - Eterm old_pid; - Eterm new_pid; - - old_pid = (Eterm) erts_smp_atomic_read_nob(&bp->meta_pid->pid); - new_pid = do_call_trace(c_p, I, reg, 1, bp->meta_ms, old_pid); - if (new_pid != old_pid) { - erts_smp_atomic_set_nob(&bp->meta_pid->pid, new_pid); + ErtsTracer old_tracer, new_tracer; + + old_tracer = erts_smp_atomic_read_nob(&bp->meta_tracer->tracer); + + new_tracer = do_call_trace(c_p, I, reg, 1, bp->meta_ms, old_tracer); + if (!ERTS_TRACER_COMPARE(new_tracer, old_tracer)) { + if (old_tracer == erts_smp_atomic_cmpxchg_acqb( + &bp->meta_tracer->tracer, + (erts_aint_t)new_tracer, + (erts_aint_t)old_tracer)) { + ERTS_TRACER_CLEAR(&old_tracer); + } else { + ERTS_TRACER_CLEAR(&new_tracer); + } } } @@ -645,7 +655,8 @@ erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg) erts_smp_atomic_inc_nob(&bp->count->acount); } - if (bp_flags & ERTS_BPF_TIME_TRACE_ACTIVE && erts_is_tracer_proc_valid(c_p)) { + if (bp_flags & ERTS_BPF_TIME_TRACE_ACTIVE + && ERTS_TRACER_PROC_IS_ENABLED(c_p)) { Eterm w; erts_trace_time_call(c_p, I, bp->time); w = (BeamInstr) *c_p->cp; @@ -690,7 +701,7 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) Eterm (*func)(Process*, Eterm*, BeamInstr*); Export* ep = bif_export[bif_index]; Uint32 flags = 0, flags_meta = 0; - Eterm meta_tracer_pid = NIL; + ErtsTracer meta_tracer = erts_tracer_nil; int applying = (I == &(ep->code[3])); /* Yup, the apply code for a bif * is actually in the * export entry */ @@ -718,23 +729,32 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) IS_TRACED_FL(p, F_TRACE_CALLS)) { int local = !!(bp_flags & ERTS_BPF_LOCAL_TRACE); flags = erts_call_trace(p, ep->code, bp->local_ms, args, - local, &ERTS_TRACER_PROC(p)); + local, &ERTS_TRACER(p)); } if (bp_flags & ERTS_BPF_META_TRACE) { - Eterm tpid1, tpid2; + ErtsTracer old_tracer; - tpid1 = tpid2 = - (Eterm) erts_smp_atomic_read_nob(&bp->meta_pid->pid); + meta_tracer = erts_smp_atomic_read_nob(&bp->meta_tracer->tracer); + old_tracer = meta_tracer; flags_meta = erts_call_trace(p, ep->code, bp->meta_ms, args, - 0, &tpid2); - meta_tracer_pid = tpid2; - if (tpid1 != tpid2) { - erts_smp_atomic_set_nob(&bp->meta_pid->pid, tpid2); + 0, &meta_tracer); + + if (!ERTS_TRACER_COMPARE(old_tracer, meta_tracer)) { + ErtsTracer new_tracer = erts_tracer_nil; + erts_tracer_update(&new_tracer, meta_tracer); + if (old_tracer == erts_smp_atomic_cmpxchg_acqb( + &bp->meta_tracer->tracer, + (erts_aint_t)new_tracer, + (erts_aint_t)old_tracer)) { + ERTS_TRACER_CLEAR(&old_tracer); + } else { + ERTS_TRACER_CLEAR(&new_tracer); + } } } if (bp_flags & ERTS_BPF_TIME_TRACE_ACTIVE && IS_TRACED_FL(p, F_TRACE_CALLS) && - erts_is_tracer_proc_valid(p)) { + ERTS_TRACER_PROC_IS_ENABLED(p)) { BeamInstr *pc = (BeamInstr *)ep->code+3; erts_trace_time_call(p, pc, bp->time); } @@ -778,8 +798,6 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) if (reason != TRAP) { Eterm class; Eterm value = p->fvalue; - DeclareTmpHeapNoproc(nocatch,3); - UseTmpHeapNoproc(3); /* Expand error value like in handle_error() */ if (reason & EXF_ARGLIST) { Eterm *tp; @@ -788,7 +806,8 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) value = tp[1]; } if ((reason & EXF_THROWN) && (p->catches <= 0)) { - value = TUPLE2(nocatch, am_nocatch, value); + Eterm *hp = HAlloc(p, 3); + value = TUPLE2(hp, am_nocatch, value); reason = EXC_ERROR; } /* Note: expand_error_value() could theoretically @@ -801,11 +820,11 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) if (flags_meta & MATCH_SET_EXCEPTION_TRACE) { erts_trace_exception(p, ep->code, class, value, - &meta_tracer_pid); + &meta_tracer); } if (flags & MATCH_SET_EXCEPTION_TRACE) { erts_trace_exception(p, ep->code, class, value, - &ERTS_TRACER_PROC(p)); + &ERTS_TRACER(p)); } if ((flags & MATCH_SET_RETURN_TO_TRACE) && p->catches > 0) { /* can only happen if(local)*/ @@ -827,7 +846,6 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) } } } - UnUseTmpHeapNoproc(3); if ((flags_meta|flags) & MATCH_SET_EXCEPTION_TRACE) { erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR); ERTS_TRACE_FLAGS(p) |= F_EXCEPTION_TRACE; @@ -836,11 +854,11 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) } } else { if (flags_meta & MATCH_SET_RX_TRACE) { - erts_trace_return(p, ep->code, result, &meta_tracer_pid); + erts_trace_return(p, ep->code, result, &meta_tracer); } /* MATCH_SET_RETURN_TO_TRACE cannot occur if(meta) */ if (flags & MATCH_SET_RX_TRACE) { - erts_trace_return(p, ep->code, result, &ERTS_TRACER_PROC(p)); + erts_trace_return(p, ep->code, result, &ERTS_TRACER(p)); } if (flags & MATCH_SET_RETURN_TO_TRACE) { /* can only happen if(local)*/ @@ -857,9 +875,9 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) return result; } -static Eterm +static ErtsTracer do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg, - int local, Binary* ms, Eterm tracer_pid) + int local, Binary* ms, ErtsTracer tracer) { Eterm* cpp; int return_to_trace = 0; @@ -899,7 +917,7 @@ do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg, ASSERT(is_CP(*cpp)); } ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); - flags = erts_call_trace(c_p, I-3, ms, reg, local, &tracer_pid); + flags = erts_call_trace(c_p, I-3, ms, reg, local, &tracer); ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); if (cpp) { c_p->cp = cp_save; @@ -910,7 +928,7 @@ do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg, need += 1; } if (flags & MATCH_SET_RX_TRACE) { - need += 3; + need += 3 + size_object(tracer); } if (need) { ASSERT(c_p->htop <= E && E <= c_p->hend); @@ -926,14 +944,15 @@ do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg, E[0] = make_cp(c_p->cp); c_p->cp = beam_return_to_trace; } - if (flags & MATCH_SET_RX_TRACE) { + if (flags & MATCH_SET_RX_TRACE) + { E -= 3; + c_p->stop = E; ASSERT(c_p->htop <= E && E <= c_p->hend); ASSERT(is_CP((Eterm) (UWord) (I - 3))); - ASSERT(am_true == tracer_pid || - is_internal_pid(tracer_pid) || is_internal_port(tracer_pid)); + ASSERT(IS_TRACER_VALID(tracer)); E[2] = make_cp(c_p->cp); - E[1] = tracer_pid; + E[1] = copy_object(tracer, c_p); E[0] = make_cp(I - 3); /* We ARE at the beginning of an instruction, the funcinfo is above i. */ @@ -942,9 +961,9 @@ do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg, erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); ERTS_TRACE_FLAGS(c_p) |= F_EXCEPTION_TRACE; erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); - } - c_p->stop = E; - return tracer_pid; + } else + c_p->stop = E; + return tracer; } void @@ -974,7 +993,7 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt) if (pbt == 0) { /* First call of process to instrumented function */ pbt = Alloc(sizeof(process_breakpoint_time_t)); - (void) ERTS_PROC_SET_CALL_TIME(c_p, ERTS_PROC_LOCK_MAIN, pbt); + (void) ERTS_PROC_SET_CALL_TIME(c_p, pbt); } else { ASSERT(pbt->pc); /* add time to previous code */ @@ -1098,9 +1117,9 @@ erts_is_trace_break(BeamInstr *pc, Binary **match_spec_ret, int local) return 0; } -int +int erts_is_mtrace_break(BeamInstr *pc, Binary **match_spec_ret, - Eterm *tracer_pid_ret) + ErtsTracer *tracer_ret) { GenericBpData* bp = check_break(pc, ERTS_BPF_META_TRACE); @@ -1108,9 +1127,8 @@ erts_is_mtrace_break(BeamInstr *pc, Binary **match_spec_ret, if (match_spec_ret) { *match_spec_ret = bp->meta_ms; } - if (tracer_pid_ret) { - *tracer_pid_ret = - (Eterm) erts_smp_atomic_read_nob(&bp->meta_pid->pid); + if (tracer_ret) { + *tracer_ret = erts_smp_atomic_read_nob(&bp->meta_tracer->tracer); } return 1; } @@ -1391,7 +1409,7 @@ void erts_schedule_time_break(Process *p, Uint schedule) { static void set_break(BpFunctions* f, Binary *match_spec, Uint break_flags, - enum erts_break_op count_op, Eterm tracer_pid) + enum erts_break_op count_op, ErtsTracer tracer) { Uint i; Uint n; @@ -1400,13 +1418,13 @@ set_break(BpFunctions* f, Binary *match_spec, Uint break_flags, for (i = 0; i < n; i++) { BeamInstr* pc = f->matching[i].pc; set_function_break(pc, match_spec, break_flags, - count_op, tracer_pid); + count_op, tracer); } } static void set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags, - enum erts_break_op count_op, Eterm tracer_pid) + enum erts_break_op count_op, ErtsTracer tracer) { GenericBp* g; GenericBpData* bp; @@ -1417,7 +1435,7 @@ set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags, g = (GenericBp *) pc[-4]; if (g == 0) { int i; - if (count_op == erts_break_reset || count_op == erts_break_stop) { + if (count_op == ERTS_BREAK_RESTART || count_op == ERTS_BREAK_PAUSE) { /* Do not insert a new breakpoint */ return; } @@ -1439,9 +1457,9 @@ set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags, MatchSetUnref(bp->local_ms); } else if (common & ERTS_BPF_META_TRACE) { MatchSetUnref(bp->meta_ms); - bp_meta_unref(bp->meta_pid); + bp_meta_unref(bp->meta_tracer); } else if (common & ERTS_BPF_COUNT) { - if (count_op == erts_break_stop) { + if (count_op == ERTS_BREAK_PAUSE) { bp->flags &= ~ERTS_BPF_COUNT_ACTIVE; } else { bp->flags |= ERTS_BPF_COUNT_ACTIVE; @@ -1453,7 +1471,7 @@ set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags, BpDataTime* bdt = bp->time; Uint i = 0; - if (count_op == erts_break_stop) { + if (count_op == ERTS_BREAK_PAUSE) { bp->flags &= ~ERTS_BPF_TIME_TRACE_ACTIVE; } else { bp->flags |= ERTS_BPF_TIME_TRACE_ACTIVE; @@ -1474,13 +1492,15 @@ set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags, MatchSetRef(match_spec); bp->local_ms = match_spec; } else if (break_flags & ERTS_BPF_META_TRACE) { - BpMetaPid* bmp; + BpMetaTracer* bmt; + ErtsTracer meta_tracer = erts_tracer_nil; MatchSetRef(match_spec); bp->meta_ms = match_spec; - bmp = Alloc(sizeof(BpMetaPid)); - erts_refc_init(&bmp->refc, 1); - erts_smp_atomic_init_nob(&bmp->pid, tracer_pid); - bp->meta_pid = bmp; + bmt = Alloc(sizeof(BpMetaTracer)); + erts_refc_init(&bmt->refc, 1); + erts_tracer_update(&meta_tracer, tracer); /* copy tracer */ + erts_smp_atomic_init_nob(&bmt->tracer, (erts_aint_t)meta_tracer); + bp->meta_tracer = bmt; } else if (break_flags & ERTS_BPF_COUNT) { BpCount* bcp; @@ -1544,7 +1564,7 @@ clear_function_break(BeamInstr *pc, Uint break_flags) } if (common & ERTS_BPF_META_TRACE) { MatchSetUnref(bp->meta_ms); - bp_meta_unref(bp->meta_pid); + bp_meta_unref(bp->meta_tracer); } if (common & ERTS_BPF_COUNT) { ASSERT((bp->flags & ERTS_BPF_COUNT_ACTIVE) == 0); @@ -1560,10 +1580,12 @@ clear_function_break(BeamInstr *pc, Uint break_flags) } static void -bp_meta_unref(BpMetaPid* bmp) +bp_meta_unref(BpMetaTracer* bmt) { - if (erts_refc_dectest(&bmp->refc, 0) <= 0) { - Free(bmp); + if (erts_refc_dectest(&bmt->refc, 0) <= 0) { + ErtsTracer trc = erts_smp_atomic_read_nob(&bmt->tracer); + ERTS_TRACER_CLEAR(&trc); + Free(bmt); } } @@ -1598,9 +1620,7 @@ bp_time_unref(BpDataTime* bdt) h_p = erts_pid2proc(NULL, 0, item->pid, ERTS_PROC_LOCK_MAIN); if (h_p) { - pbt = ERTS_PROC_SET_CALL_TIME(h_p, - ERTS_PROC_LOCK_MAIN, - NULL); + pbt = ERTS_PROC_SET_CALL_TIME(h_p, NULL); if (pbt) { Free(pbt); } diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h index 2b89d6fc71..9c2fc007a2 100644 --- a/erts/emulator/beam/beam_bp.h +++ b/erts/emulator/beam/beam_bp.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2013. All Rights Reserved. + * Copyright Ericsson AB 2000-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -55,15 +55,15 @@ typedef struct { } BpCount; typedef struct { - erts_smp_atomic_t pid; + erts_smp_atomic_t tracer; erts_refc_t refc; -} BpMetaPid; +} BpMetaTracer; typedef struct generic_bp_data { Uint flags; Binary* local_ms; /* Match spec for local call trace */ Binary* meta_ms; /* Match spec for meta trace */ - BpMetaPid* meta_pid; /* Meta trace pid */ + BpMetaTracer* meta_tracer; /* Meta tracer */ BpCount* count; /* For call count */ BpDataTime* time; /* For time trace */ } GenericBpData; @@ -86,10 +86,10 @@ typedef struct generic_bp { #endif enum erts_break_op{ - erts_break_nop = 0, /* Must be false */ - erts_break_set = !0, /* Must be true */ - erts_break_reset, - erts_break_stop + ERTS_BREAK_NOP = 0, /* Must be false */ + ERTS_BREAK_SET = !0, /* Must be true */ + ERTS_BREAK_RESTART, + ERTS_BREAK_PAUSE }; typedef Uint32 ErtsBpIndex; @@ -132,10 +132,10 @@ void erts_set_call_trace_bif(BeamInstr *pc, Binary *match_spec, int local); void erts_clear_call_trace_bif(BeamInstr *pc, int local); void erts_set_mtrace_break(BpFunctions *f, Binary *match_spec, - Eterm tracer_pid); + ErtsTracer tracer); void erts_clear_mtrace_break(BpFunctions *f); void erts_set_mtrace_bif(BeamInstr *pc, Binary *match_spec, - Eterm tracer_pid); + ErtsTracer tracer); void erts_clear_mtrace_bif(BeamInstr *pc); void erts_set_debug_break(BpFunctions *f); @@ -150,13 +150,13 @@ void erts_clear_export_break(Module *modp, BeamInstr* pc); BeamInstr erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg); BeamInstr erts_trace_break(Process *p, BeamInstr *pc, Eterm *args, - Uint32 *ret_flags, Eterm *tracer_pid); + Uint32 *ret_flags, ErtsTracer *tracer); int erts_is_trace_break(BeamInstr *pc, Binary **match_spec_ret, int local); int erts_is_mtrace_break(BeamInstr *pc, Binary **match_spec_ret, - Eterm *tracer_pid_rte); + ErtsTracer *tracer_ret); int erts_is_mtrace_bif(BeamInstr *pc, Binary **match_spec_ret, - Eterm *tracer_pid_ret); + ErtsTracer *tracer_ret); int erts_is_native_break(BeamInstr *pc); int erts_is_count_break(BeamInstr *pc, Uint *count_ret); int erts_is_time_break(Process *p, BeamInstr *pc, Eterm *call_time); @@ -173,19 +173,7 @@ void erts_clear_time_trace_bif(BeamInstr *pc); BeamInstr *erts_find_local_func(Eterm mfa[3]); -ERTS_GLB_INLINE Uint erts_bp_sched2ix(void); - #if ERTS_GLB_INLINE_INCL_FUNC_DEF -ERTS_GLB_INLINE Uint erts_bp_sched2ix(void) -{ -#ifdef ERTS_SMP - ErtsSchedulerData *esdp; - esdp = erts_get_scheduler_data(); - return esdp->no - 1; -#else - return 0; -#endif -} extern erts_smp_atomic32_t erts_active_bp_index; extern erts_smp_atomic32_t erts_staging_bp_index; diff --git a/erts/emulator/beam/beam_catches.c b/erts/emulator/beam/beam_catches.c index 7a1f4901aa..cd592c7e5e 100644 --- a/erts/emulator/beam/beam_catches.c +++ b/erts/emulator/beam/beam_catches.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2013. All Rights Reserved. + * Copyright Ericsson AB 2000-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/beam_catches.h b/erts/emulator/beam/beam_catches.h index 59ee64d033..8eb2165ac9 100644 --- a/erts/emulator/beam/beam_catches.h +++ b/erts/emulator/beam/beam_catches.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2013. All Rights Reserved. + * Copyright Ericsson AB 2000-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index e37bd4d78c..a4ad3e7886 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2013. All Rights Reserved. + * Copyright Ericsson AB 1998-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -431,7 +431,7 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) packed >>= 10; break; case '0': /* Tight shift */ - *ap++ = packed & (BEAM_TIGHT_MASK / sizeof(Eterm)); + *ap++ = packed & BEAM_TIGHT_MASK; packed >>= BEAM_TIGHT_SHIFT; break; case '6': /* Shift 16 steps */ diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index a390422040..99eb26bec5 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2014. All Rights Reserved. + * Copyright Ericsson AB 1996-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -308,7 +308,8 @@ void** beam_ops; if (E - HTOP < (needed + (HeapNeed))) { \ SWAPOUT; \ PROCESS_MAIN_CHK_LOCKS(c_p); \ - FCALLS -= erts_garbage_collect_nobump(c_p, needed + (HeapNeed), reg, (M)); \ + FCALLS -= erts_garbage_collect_nobump(c_p, needed + (HeapNeed), \ + reg, (M), FCALLS); \ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ SWAPIN; \ @@ -360,7 +361,7 @@ void** beam_ops; if ((E - HTOP < need) || (MSO(c_p).overhead + (VNh) >= BIN_VHEAP_SZ(c_p))) {\ SWAPOUT; \ PROCESS_MAIN_CHK_LOCKS(c_p); \ - FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live)); \ + FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live), FCALLS); \ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ SWAPIN; \ @@ -381,7 +382,7 @@ void** beam_ops; if (E - HTOP < need) { \ SWAPOUT; \ PROCESS_MAIN_CHK_LOCKS(c_p); \ - FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live));\ + FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live), FCALLS); \ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ SWAPIN; \ @@ -402,7 +403,7 @@ void** beam_ops; SWAPOUT; \ reg[Live] = Extra; \ PROCESS_MAIN_CHK_LOCKS(c_p); \ - FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live)+1); \ + FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live)+1, FCALLS); \ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ Extra = reg[Live]; \ @@ -702,6 +703,17 @@ void** beam_ops; dst[1] = E2; \ } while (0) +#define GetTupleElement2Y(Src, Element, D1, D2) \ + do { \ + Eterm* src; \ + Eterm E1, E2; \ + src = ADD_BYTE_OFFSET(tuple_val(Src), (Element)); \ + E1 = src[0]; \ + E2 = src[1]; \ + D1 = E1; \ + D2 = E2; \ + } while (0) + #define GetTupleElement3(Src, Element, Dest) \ do { \ Eterm* src; \ @@ -1185,6 +1197,25 @@ init_emulator(void) #define DTRACE_NIF_RETURN(p, m, f, a) do {} while (0) #endif /* USE_VM_PROBES */ +#ifdef DEBUG +#define ERTS_DBG_CHK_REDS(P, FC) \ + do { \ + if (ERTS_PROC_GET_SAVED_CALLS_BUF((P))) { \ + ASSERT(FC <= 0); \ + ASSERT(ERTS_PROC_GET_SCHDATA(c_p)->virtual_reds \ + <= 0 - (FC)); \ + } \ + else { \ + ASSERT(FC <= CONTEXT_REDS); \ + ASSERT(ERTS_PROC_GET_SCHDATA(c_p)->virtual_reds \ + <= CONTEXT_REDS - (FC)); \ + } \ +} while (0) +#else +#define ERTS_DBG_CHK_REDS(P, FC) +#endif + + /* * process_main() is called twice: * The first call performs some initialisation, including exporting @@ -1279,7 +1310,12 @@ void process_main(void) goto do_schedule1; do_schedule: - reds_used = REDS_IN(c_p) - FCALLS; + ASSERT(c_p->debug_reds_in == REDS_IN(c_p)); + if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) + reds_used = REDS_IN(c_p) - FCALLS; + else + reds_used = REDS_IN(c_p) - (CONTEXT_REDS + FCALLS); + ASSERT(reds_used >= 0); do_schedule1: if (start_time != 0) { @@ -1299,6 +1335,7 @@ void process_main(void) ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); c_p = schedule(c_p, reds_used); + ASSERT(!(c_p->flags & F_HIPE_MODE)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); start_time = 0; #ifdef DEBUG @@ -1337,16 +1374,21 @@ void process_main(void) SET_I(c_p->i); - reds = c_p->fcalls; - if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p) - && (ERTS_TRACE_FLAGS(c_p) & F_SENSITIVE) == 0) { - neg_o_reds = -reds; - FCALLS = REDS_IN(c_p) = 0; + REDS_IN(c_p) = reds = c_p->fcalls; +#ifdef DEBUG + c_p->debug_reds_in = reds; +#endif + + if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) { + neg_o_reds = -CONTEXT_REDS; + FCALLS = neg_o_reds + reds; } else { neg_o_reds = 0; - FCALLS = REDS_IN(c_p) = reds; + FCALLS = reds; } + ERTS_DBG_CHK_REDS(c_p, FCALLS); + next = (BeamInstr *) *I; SWAPIN; ASSERT(VALID_INSTR(next)); @@ -1759,7 +1801,7 @@ void process_main(void) if (E - HTOP < 3) { SWAPOUT; PROCESS_MAIN_CHK_LOCKS(c_p); - FCALLS -= erts_garbage_collect_nobump(c_p, 3, reg+2, 1); + FCALLS -= erts_garbage_collect_nobump(c_p, 3, reg+2, 1, FCALLS); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); SWAPIN; @@ -1854,6 +1896,7 @@ void process_main(void) c_p->flags |= F_DELAY_GC; loop_rec__: + PROCESS_MAIN_CHK_LOCKS(c_p); msgp = PEEK_MESSAGE(c_p); @@ -1996,6 +2039,8 @@ void process_main(void) ERTS_VBUMP_LEAVE_REDS_INTERNAL(c_p, 5, FCALLS); } + ERTS_DBG_CHK_REDS(c_p, FCALLS); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); NextPF(0, next); @@ -2155,7 +2200,7 @@ void process_main(void) PreFetch(0, next); if (IS_TRACED_FL(c_p, F_TRACE_RECEIVE)) { - trace_receive(c_p, am_timeout); + trace_receive(c_p, am_clock_service, am_timeout, NULL); } if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) { save_calls(c_p, &exp_timeout); @@ -2524,6 +2569,7 @@ do { \ GetArg1(2, tmp_reg[0]); bf = (BifFunction) Arg(1); + ERTS_DBG_CHK_REDS(c_p, FCALLS); c_p->fcalls = FCALLS; PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); @@ -2533,6 +2579,7 @@ do { \ PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_HOLE_CHECK(c_p); FCALLS = c_p->fcalls; + ERTS_DBG_CHK_REDS(c_p, FCALLS); if (is_value(result)) { StoreBifResult(3, result); } @@ -2553,6 +2600,7 @@ do { \ GetArg1(1, tmp_reg[0]); bf = (BifFunction) Arg(0); + ERTS_DBG_CHK_REDS(c_p, FCALLS); c_p->fcalls = FCALLS; PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); @@ -2562,6 +2610,7 @@ do { \ PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_HOLE_CHECK(c_p); FCALLS = c_p->fcalls; + ERTS_DBG_CHK_REDS(c_p, FCALLS); if (is_value(result)) { StoreBifResult(2, result); } @@ -2580,6 +2629,7 @@ do { \ GetArg1(2, x(live)); bf = (GcBifFunction) Arg(1); + ERTS_DBG_CHK_REDS(c_p, FCALLS); c_p->fcalls = FCALLS; SWAPOUT; PROCESS_MAIN_CHK_LOCKS(c_p); @@ -2591,6 +2641,7 @@ do { \ SWAPIN; ERTS_HOLE_CHECK(c_p); FCALLS = c_p->fcalls; + ERTS_DBG_CHK_REDS(c_p, FCALLS); if (is_value(result)) { StoreBifResult(4, result); } @@ -2619,6 +2670,7 @@ do { \ */ live++; bf = (GcBifFunction) Arg(1); + ERTS_DBG_CHK_REDS(c_p, FCALLS); c_p->fcalls = FCALLS; SWAPOUT; PROCESS_MAIN_CHK_LOCKS(c_p); @@ -2630,6 +2682,7 @@ do { \ SWAPIN; ERTS_HOLE_CHECK(c_p); FCALLS = c_p->fcalls; + ERTS_DBG_CHK_REDS(c_p, FCALLS); if (is_value(result)) { StoreBifResult(5, result); } @@ -2660,6 +2713,7 @@ do { \ */ live += 2; bf = (GcBifFunction) Arg(1); + ERTS_DBG_CHK_REDS(c_p, FCALLS); c_p->fcalls = FCALLS; SWAPOUT; PROCESS_MAIN_CHK_LOCKS(c_p); @@ -2671,6 +2725,7 @@ do { \ SWAPIN; ERTS_HOLE_CHECK(c_p); FCALLS = c_p->fcalls; + ERTS_DBG_CHK_REDS(c_p, FCALLS); if (is_value(result)) { StoreBifResult(5, result); } @@ -2697,6 +2752,7 @@ do { \ GetArg2(2, tmp_reg[0], tmp_reg[1]); bf = (BifFunction) Arg(1); + ERTS_DBG_CHK_REDS(c_p, FCALLS); c_p->fcalls = FCALLS; PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); @@ -2706,6 +2762,7 @@ do { \ PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_HOLE_CHECK(c_p); FCALLS = c_p->fcalls; + ERTS_DBG_CHK_REDS(c_p, FCALLS); if (is_value(result)) { StoreBifResult(4, result); } @@ -2764,6 +2821,7 @@ do { \ bf = GET_BIF_ADDRESS(Arg(0)); PRE_BIF_SWAPOUT(c_p); + ERTS_DBG_CHK_REDS(c_p, FCALLS); c_p->fcalls = FCALLS - 1; if (FCALLS <= 0) { save_calls(c_p, (Export *) Arg(0)); @@ -2785,6 +2843,7 @@ do { \ PROCESS_MAIN_CHK_LOCKS(c_p); HTOP = HEAP_TOP(c_p); FCALLS = c_p->fcalls; + ERTS_DBG_CHK_REDS(c_p, FCALLS); /* We have to update the cache if we are enabled in order to make sure no book keeping is done after we disabled msacc. We don't always do this as it is quite expensive. */ @@ -3313,8 +3372,13 @@ do { \ * (beacuse the code for the Dispatch() macro becomes shorter that way). */ - reds_used = REDS_IN(c_p) - FCALLS + 1; - + ASSERT(c_p->debug_reds_in == REDS_IN(c_p)); + if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) + reds_used = REDS_IN(c_p) - FCALLS; + else + reds_used = REDS_IN(c_p) - (CONTEXT_REDS + FCALLS); + ASSERT(reds_used >= 0); + /* * Save the argument registers and everything else. */ @@ -3346,7 +3410,7 @@ do { \ OpCase(normal_exit): { SWAPOUT; c_p->freason = EXC_NORMAL; - c_p->arity = 0; /* In case this process will ever be garbed again. */ + c_p->arity = 0; /* In case this process will never be garbed again. */ ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); erts_do_exit_process(c_p, am_normal); ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); @@ -3360,48 +3424,18 @@ do { \ goto do_schedule; } - OpCase(raise_ss): { - /* This was not done very well in R10-0; then, we passed the tag in - the first argument and hoped that the existing c_p->ftrace was - still correct. But the ftrace-object already includes the tag - (or rather, the freason). Now, we pass the original ftrace in - the first argument. We also handle atom tags in the first - argument for backwards compatibility. - */ - Eterm raise_val1; - Eterm raise_val2; - GetArg2(0, raise_val1, raise_val2); - c_p->fvalue = raise_val2; - if (c_p->freason == EXC_NULL) { - /* a safety check for the R10-0 case; should not happen */ - c_p->ftrace = NIL; - c_p->freason = EXC_ERROR; - } - /* for R10-0 code, keep existing c_p->ftrace and hope it's correct */ - switch (raise_val1) { - case am_throw: - c_p->freason = EXC_THROWN & ~EXF_SAVETRACE; - break; - case am_error: - c_p->freason = EXC_ERROR & ~EXF_SAVETRACE; - break; - case am_exit: - c_p->freason = EXC_EXIT & ~EXF_SAVETRACE; - break; - default: - {/* R10-1 and later - XXX note: should do sanity check on given trace if it can be - passed from a user! Currently only expecting generated calls. - */ - struct StackTrace *s; - c_p->ftrace = raise_val1; - s = get_trace_from_exc(raise_val1); - if (s == NULL) { - c_p->freason = EXC_ERROR; - } else { - c_p->freason = PRIMARY_EXCEPTION(s->freason); - } - } + OpCase(i_raise): { + Eterm raise_trace = x(2); + Eterm raise_value = x(1); + struct StackTrace *s; + + c_p->fvalue = raise_value; + c_p->ftrace = raise_trace; + s = get_trace_from_exc(raise_trace); + if (s == NULL) { + c_p->freason = EXC_ERROR; + } else { + c_p->freason = PRIMARY_EXCEPTION(s->freason); } goto find_func_info; } @@ -3490,7 +3524,7 @@ do { \ typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]); NifF* fp = vbf = (NifF*) I[1]; struct enif_environment_t env; - erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2]); + erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2], NULL); live_hf_end = c_p->mbuf; nif_bif_result = (*fp)(&env, bif_nif_arity, reg); if (env.exception_thrown) @@ -3533,6 +3567,7 @@ do { \ DTRACE_BIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); SWAPOUT; + ERTS_DBG_CHK_REDS(c_p, FCALLS - 1); c_p->fcalls = FCALLS - 1; vbf = (BifFunction) Arg(0); PROCESS_MAIN_CHK_LOCKS(c_p); @@ -3568,6 +3603,7 @@ do { \ } SWAPIN; /* There might have been a garbage collection. */ FCALLS = c_p->fcalls; + ERTS_DBG_CHK_REDS(c_p, FCALLS); if (is_value(nif_bif_result)) { r(0) = nif_bif_result; CHECK_TERM(r(0)); @@ -4102,7 +4138,7 @@ do { \ StoreBifResult(1, result); } - OpCase(i_bs_put_utf16_jIs): { + OpCase(bs_put_utf16_jIs): { Eterm arg; GetArg1(2, arg); @@ -4618,7 +4654,7 @@ do { \ SWAPOUT; /* Needed for shared heap */ ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); - erts_trace_return(c_p, code, r(0), E+1/*Process tracer*/); + erts_trace_return(c_p, code, r(0), ERTS_TRACER_FROM_ETERM(E+1)/* tracer */); ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); SWAPIN; c_p->cp = NULL; @@ -4630,9 +4666,9 @@ do { \ OpCase(i_generic_breakpoint): { BeamInstr real_I; ASSERT(I[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); - SWAPOUT; + HEAVY_SWAPOUT; real_I = erts_generic_breakpoint(c_p, I, reg); - SWAPIN; + HEAVY_SWAPIN; ASSERT(VALID_INSTR(real_I)); Goto(real_I); } @@ -4809,6 +4845,7 @@ do { \ { #define HIPE_MODE_SWITCH(Cmd) \ SWAPOUT; \ + ERTS_DBG_CHK_REDS(c_p, FCALLS); \ c_p->fcalls = FCALLS; \ c_p->def_arg_reg[4] = -neg_o_reds; \ c_p = hipe_mode_switch(c_p, Cmd, reg); \ @@ -4847,6 +4884,9 @@ do { \ #undef HIPE_MODE_SWITCH L_post_hipe_mode_switch: +#ifdef DEBUG + pid = c_p->common.id; /* may have switched process... */ +#endif reg = ERTS_PROC_GET_SCHDATA(c_p)->x_reg_array; freg = ERTS_PROC_GET_SCHDATA(c_p)->f_reg_array; ERL_BITS_RELOAD_STATEP(c_p); @@ -4854,6 +4894,7 @@ do { \ neg_o_reds = -c_p->def_arg_reg[4]; FCALLS = c_p->fcalls; SWAPIN; + ERTS_DBG_CHK_REDS(c_p, FCALLS); switch( c_p->def_arg_reg[3] ) { case HIPE_MODE_SWITCH_RES_RETURN: ASSERT(is_value(reg[0])); @@ -5258,7 +5299,8 @@ next_catch(Process* c_p, Eterm *reg) { BeamInstr *cpp = c_p->cp; if (cpp == beam_exception_trace) { erts_trace_exception(c_p, cp_val(ptr[0]), - reg[1], reg[2], ptr+1); + reg[1], reg[2], + ERTS_TRACER_FROM_ETERM(ptr+1)); /* Skip return_trace parameters */ ptr += 2; } else if (cpp == beam_return_trace) { @@ -5285,7 +5327,8 @@ next_catch(Process* c_p, Eterm *reg) { } if (cp_val(*prev) == beam_exception_trace) { erts_trace_exception(c_p, cp_val(ptr[0]), - reg[1], reg[2], ptr+1); + reg[1], reg[2], + ERTS_TRACER_FROM_ETERM(ptr+1)); } /* Skip return_trace parameters */ ptr += 2; @@ -6079,7 +6122,7 @@ erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* re return -1; } #else /* ERTS_SMP */ - ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p); + ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p); if (!c_p->msg.len) #endif erts_smp_atomic32_read_band_relb(&c_p->state, ~ERTS_PSFLG_ACTIVE); @@ -6856,7 +6899,7 @@ erts_current_reductions(Process *current, Process *p) if (current != p) { return 0; } else if (current->fcalls < 0 && ERTS_PROC_GET_SAVED_CALLS_BUF(current)) { - return -current->fcalls; + return current->fcalls + CONTEXT_REDS; } else { return REDS_IN(current) - current->fcalls; } diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index f115df935f..95489d9a63 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2014. All Rights Reserved. + * Copyright Ericsson AB 1996-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -32,6 +32,7 @@ #include "bif.h" #include "external.h" #include "beam_load.h" +#include "beam_bp.h" #include "big.h" #include "erl_bits.h" #include "beam_catches.h" @@ -479,9 +480,9 @@ static void free_loader_state(Binary* magic); static ErlHeapFragment* new_literal_fragment(Uint size); static void free_literal_fragment(ErlHeapFragment*); static void loader_state_dtor(Binary* magic); -static Eterm insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, - Eterm group_leader, Eterm module, - BeamCodeHeader* code, Uint size); +static Eterm stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, + Eterm group_leader, Eterm module, + BeamCodeHeader* code, Uint size); static int init_iff_file(LoaderState* stp, byte* code, Uint size); static int scan_iff_file(LoaderState* stp, Uint* chunk_types, Uint num_types, Uint num_mandatory); @@ -513,7 +514,7 @@ static GenOp* gen_get_map_element(LoaderState* stp, GenOpArg Fail, GenOpArg Src, static int freeze_code(LoaderState* stp); -static void final_touch(LoaderState* stp); +static void final_touch(LoaderState* stp, struct erl_module_instance* inst_p); static void short_file(int line, LoaderState* stp, unsigned needed); static void load_printf(int line, LoaderState* context, char *fmt, ...); static int transform_engine(LoaderState* st); @@ -766,8 +767,11 @@ Eterm erts_finish_loading(Binary* magic, Process* c_p, ErtsProcLocks c_p_locks, Eterm* modp) { - Eterm retval; + Eterm retval = NIL; LoaderState* stp = ERTS_MAGIC_BIN_DATA(magic); + Module* mod_tab_p; + struct erl_module_instance* inst_p; + Uint size; /* * No other process may run since we will update the export @@ -776,27 +780,80 @@ erts_finish_loading(Binary* magic, Process* c_p, ERTS_SMP_LC_ASSERT(erts_initialized == 0 || erts_has_code_write_permission() || erts_smp_thr_progress_is_blocking()); - /* * Make current code for the module old and insert the new code * as current. This will fail if there already exists old code * for the module. */ + mod_tab_p = erts_put_module(stp->module); CHKBLK(ERTS_ALC_T_CODE,stp->code); - retval = insert_new_code(c_p, c_p_locks, stp->group_leader, stp->module, - stp->hdr, stp->loaded_size); - if (retval != NIL) { - goto load_error; + if (!stp->on_load) { + /* + * Normal case -- no -on_load() function. + */ + retval = beam_make_current_old(c_p, c_p_locks, stp->module); + ASSERT(retval == NIL); + } else { + ErtsCodeIndex code_ix = erts_staging_code_ix(); + Eterm module = stp->module; + int i; + + /* + * There is an -on_load() function. We will keep the current + * code, but we must turn off any tracing. + */ + + for (i = 0; i < export_list_size(code_ix); i++) { + Export *ep = export_list(i, code_ix); + if (ep == NULL || ep->code[0] != module) { + continue; + } + if (ep->addressv[code_ix] == ep->code+3) { + if (ep->code[3] == (BeamInstr) em_apply_bif) { + continue; + } else if (ep->code[3] == + (BeamInstr) BeamOp(op_i_generic_breakpoint)) { + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); + ASSERT(mod_tab_p->curr.num_traced_exports > 0); + erts_clear_export_break(mod_tab_p, ep->code+3); + ep->addressv[code_ix] = (BeamInstr *) ep->code[4]; + ep->code[4] = 0; + } + ASSERT(ep->code[4] == 0); + } + } + ASSERT(mod_tab_p->curr.num_breakpoints == 0); + ASSERT(mod_tab_p->curr.num_traced_exports == 0); } /* + * Update module table. + */ + + size = stp->loaded_size; + erts_total_code_size += size; + if (stp->on_load) { + inst_p = &mod_tab_p->old; + } else { + inst_p = &mod_tab_p->curr; + } + inst_p->code_hdr = stp->hdr; + inst_p->code_length = size; + + /* + * Update ranges (used for finding a function from a PC value). + */ + + erts_update_ranges((BeamInstr*)inst_p->code_hdr, size); + + /* * Ready for the final touch: fixing the export table entries for * exported and imported functions. This can't fail. */ CHKBLK(ERTS_ALC_T_CODE,stp->code); - final_touch(stp); + final_touch(stp, inst_p); /* * Loading succeded. @@ -820,7 +877,6 @@ erts_finish_loading(Binary* magic, Process* c_p, retval = am_on_load; } - load_error: free_loader_state(magic); return retval; } @@ -1026,9 +1082,9 @@ loader_state_dtor(Binary* magic) } static Eterm -insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, - Eterm group_leader, Eterm module, BeamCodeHeader* code_hdr, - Uint size) +stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, + Eterm group_leader, Eterm module, + BeamCodeHeader* code_hdr, Uint size) { Module* modp; Eterm retval; @@ -1548,7 +1604,7 @@ read_literal_table(LoaderState* stp) erts_factory_heap_frag_init(&factory, new_literal_fragment(heap_size)); factory.alloc_type = ERTS_ALC_T_PREPARED_CODE; - val = erts_decode_ext(&factory, &p); + val = erts_decode_ext(&factory, &p, 0); if (is_non_value(val)) { LoadError1(stp, "literal %d: bad external format", i); @@ -1559,7 +1615,7 @@ read_literal_table(LoaderState* stp) } else { erts_factory_dummy_init(&factory); - val = erts_decode_ext(&factory, &p); + val = erts_decode_ext(&factory, &p, 0); if (is_non_value(val)) { LoadError1(stp, "literal %d: bad external format", i); } @@ -2028,42 +2084,47 @@ load_code(LoaderState* stp) ASSERT(arity == last_op->arity); do_transform: - if (stp->genop == NULL) { - last_op_next = NULL; - goto get_next_instr; - } - + ASSERT(stp->genop != NULL); if (gen_opc[stp->genop->op].transform != -1) { - int need; - tmp_op = stp->genop; - - for (need = gen_opc[stp->genop->op].min_window-1; need > 0; need--) { - if (tmp_op == NULL) { - goto get_next_instr; - } - tmp_op = tmp_op->next; + if (stp->genop->next == NULL) { + /* + * Simple heuristic: Most transformations requires + * at least two instructions, so make sure that + * there are. That will reduce the number of + * TE_SHORT_WINDOWs. + */ + goto get_next_instr; } switch (transform_engine(stp)) { case TE_FAIL: - last_op_next = NULL; - last_op = NULL; + /* + * No transformation found. stp->genop != NULL and + * last_op_next is still valid. Go ahead and load + * the instruction. + */ break; case TE_OK: + /* + * Some transformation was applied. last_op_next is + * no longer valid and stp->genop may be NULL. + * Try to transform again. + */ + if (stp->genop == NULL) { + last_op_next = &stp->genop; + goto get_next_instr; + } last_op_next = NULL; - last_op = NULL; goto do_transform; case TE_SHORT_WINDOW: - last_op_next = NULL; - last_op = NULL; + /* + * No transformation applied. stp->genop != NULL and + * last_op_next is still valid. Fetch a new instruction + * before trying the transformation again. + */ goto get_next_instr; } } - if (stp->genop == NULL) { - last_op_next = NULL; - goto get_next_instr; - } - /* * From the collected generic instruction, find the specific * instruction. @@ -2584,7 +2645,10 @@ load_code(LoaderState* stp) { GenOp* next = stp->genop->next; FREE_GENOP(stp, stp->genop); - stp->genop = next; + if ((stp->genop = next) == NULL) { + last_op_next = &stp->genop; + goto get_next_instr; + } goto do_transform; } } @@ -2728,10 +2792,10 @@ mixed_types(LoaderState* stp, GenOpArg Size, GenOpArg* Rest) } static int -same_label(LoaderState* stp, GenOpArg Target, GenOpArg Label) +is_killed_apply(LoaderState* stp, GenOpArg Reg, GenOpArg Live) { - return Target.type = TAG_f && Label.type == TAG_u && - Target.val == Label.val; + return Reg.type == TAG_x && Live.type == TAG_u && + Live.val+2 <= Reg.val; } static int @@ -4692,14 +4756,13 @@ freeze_code(LoaderState* stp) } static void -final_touch(LoaderState* stp) +final_touch(LoaderState* stp, struct erl_module_instance* inst_p) { int i; int on_load = stp->on_load; unsigned catches; Uint index; BeamInstr* codev = stp->codev; - Module* modp; /* * Allocate catch indices and fix up all catch_yf instructions. @@ -4714,8 +4777,7 @@ final_touch(LoaderState* stp) codev[index+2] = make_catch(catches); index = next; } - modp = erts_put_module(stp->module); - modp->curr.catches = catches; + inst_p->catches = catches; /* * Export functions. @@ -4735,10 +4797,10 @@ final_touch(LoaderState* stp) ep->addressv[erts_staging_code_ix()] = address; } else { /* - * Don't make any of the exported functions - * callable yet. + * on_load: Don't make any of the exported functions + * callable yet. Keep any function in the current + * code callable. */ - ep->addressv[erts_staging_code_ix()] = ep->code+3; ep->code[4] = (BeamInstr) address; } } @@ -4798,31 +4860,25 @@ transform_engine(LoaderState* st) Uint op; int ap; /* Current argument. */ Uint* restart; /* Where to restart if current match fails. */ - GenOpArg def_vars[TE_MAX_VARS]; /* Default buffer for variables. */ - GenOpArg* var = def_vars; - int num_vars = 0; + GenOpArg var[TE_MAX_VARS]; /* Buffer for variables. */ + GenOpArg* rest_args = NULL; + int num_rest_args = 0; int i; /* General index. */ Uint mask; GenOp* instr; + GenOp* first = st->genop; + GenOp* keep = NULL; Uint* pc; - int rval; static Uint restart_fail[1] = {TOP_fail}; - ASSERT(gen_opc[st->genop->op].transform != -1); - pc = op_transform + gen_opc[st->genop->op].transform; - restart = pc; + ASSERT(gen_opc[first->op].transform != -1); + restart = op_transform + gen_opc[first->op].transform; restart: - if (var != def_vars) { - erts_free(ERTS_ALC_T_LOADER_TMP, (void *) var); - var = def_vars; - } ASSERT(restart != NULL); pc = restart; ASSERT(*pc < NUM_TOPS); /* Valid instruction? */ - instr = st->genop; - -#define RETURN(r) rval = (r); goto do_return; + instr = first; #ifdef DEBUG restart = NULL; @@ -4840,7 +4896,7 @@ transform_engine(LoaderState* st) * We'll need at least one more instruction to decide whether * this combination matches or not. */ - RETURN(TE_SHORT_WINDOW); + return TE_SHORT_WINDOW; } if (*pc++ != instr->op) goto restart; @@ -5002,19 +5058,9 @@ transform_engine(LoaderState* st) #if defined(TOP_rest_args) case TOP_rest_args: { - int n = *pc++; int formal_arity = gen_opc[instr->op].arity; - int j = formal_arity; - - num_vars = n + (instr->arity - formal_arity); - var = erts_alloc(ERTS_ALC_T_LOADER_TMP, - num_vars * sizeof(GenOpArg)); - for (i = 0; i < n; i++) { - var[i] = def_vars[i]; - } - while (i < num_vars) { - var[i++] = instr->a[j++]; - } + num_rest_args = instr->arity - formal_arity; + rest_args = instr->a + formal_arity; } break; #endif @@ -5023,21 +5069,22 @@ transform_engine(LoaderState* st) break; case TOP_commit: instr = instr->next; /* The next_instr was optimized away. */ - - /* - * The left-hand side of this transformation matched. - * Delete all matched instructions. - */ - while (st->genop != instr) { - GenOp* next = st->genop->next; - FREE_GENOP(st, st->genop); - st->genop = next; - } + keep = instr; + st->genop = instr; #ifdef DEBUG instr = 0; #endif break; - +#if defined(TOP_keep) + case TOP_keep: + /* Keep the current instruction unchanged. */ + keep = instr; + st->genop = instr; +#ifdef DEBUG + instr = 0; +#endif + break; +#endif #if defined(TOP_call_end) case TOP_call_end: { @@ -5062,22 +5109,19 @@ transform_engine(LoaderState* st) lastp = &((*lastp)->next); } - instr = instr->next; /* The next_instr was optimized away. */ - - /* - * The left-hand side of this transformation matched. - * Delete all matched instructions. - */ - while (st->genop != instr) { - GenOp* next = st->genop->next; - FREE_GENOP(st, st->genop); - st->genop = next; - } - *lastp = st->genop; + keep = instr->next; /* The next_instr was optimized away. */ + *lastp = keep; st->genop = new_instr; } - RETURN(TE_OK); + /* FALLTHROUGH */ #endif + case TOP_end: + while (first != keep) { + GenOp* next = first->next; + FREE_GENOP(st, first); + first = next; + } + return TE_OK; case TOP_new_instr: /* * Note that the instructions are generated in reverse order. @@ -5089,6 +5133,12 @@ transform_engine(LoaderState* st) instr->arity = gen_opc[op].arity; ap = 0; break; +#ifdef TOP_rename + case TOP_rename: + instr->op = op = *pc++; + instr->arity = gen_opc[op].arity; + return TE_OK; +#endif case TOP_store_type: i = *pc++; instr->a[ap].type = i; @@ -5108,14 +5158,10 @@ transform_engine(LoaderState* st) #if defined(TOP_store_rest_args) case TOP_store_rest_args: { - int n = *pc++; - int num_extra = num_vars - n; - - ASSERT(n <= num_vars); - GENOP_ARITY(instr, instr->arity+num_extra); + GENOP_ARITY(instr, instr->arity+num_rest_args); memcpy(instr->a, instr->def_args, ap*sizeof(GenOpArg)); - memcpy(instr->a+ap, var+n, num_extra*sizeof(GenOpArg)); - ap += num_extra; + memcpy(instr->a+ap, rest_args, num_rest_args*sizeof(GenOpArg)); + ap += num_rest_args; } break; #endif @@ -5127,21 +5173,12 @@ transform_engine(LoaderState* st) case TOP_try_me_else_fail: restart = restart_fail; break; - case TOP_end: - RETURN(TE_OK); case TOP_fail: - RETURN(TE_FAIL); + return TE_FAIL; default: ASSERT(0); } } -#undef RETURN - - do_return: - if (var != def_vars) { - erts_free(ERTS_ALC_T_LOADER_TMP, (void *) var); - } - return rval; } static void @@ -5719,7 +5756,7 @@ attributes_for_module(Process* p, /* Process whose heap to use. */ if (ext != NULL) { ErtsHeapFactory factory; erts_factory_proc_prealloc_init(&factory, p, code_hdr->attr_size_on_heap); - result = erts_decode_ext(&factory, &ext); + result = erts_decode_ext(&factory, &ext, 0); if (is_value(result)) { erts_factory_close(&factory); } @@ -5742,7 +5779,7 @@ compilation_info_for_module(Process* p, /* Process whose heap to use. */ if (ext != NULL) { ErtsHeapFactory factory; erts_factory_proc_prealloc_init(&factory, p, code_hdr->compile_size_on_heap); - result = erts_decode_ext(&factory, &ext); + result = erts_decode_ext(&factory, &ext, 0); if (is_value(result)) { erts_factory_close(&factory); } @@ -6438,7 +6475,8 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) * Insert the module in the module table. */ - rval = insert_new_code(p, 0, p->group_leader, Mod, code_hdr, code_size); + rval = stub_insert_new_code(p, 0, p->group_leader, Mod, + code_hdr, code_size); if (rval != NIL) { goto error; } diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h index 22ab71c868..fd2dd97fee 100644 --- a/erts/emulator/beam/beam_load.h +++ b/erts/emulator/beam/beam_load.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2013. All Rights Reserved. + * Copyright Ericsson AB 1999-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -33,7 +33,6 @@ typedef struct gen_op_entry { int specific; int num_specific; int transform; - int min_window; } GenOpEntry; extern GenOpEntry gen_opc[]; diff --git a/erts/emulator/beam/beam_ranges.c b/erts/emulator/beam/beam_ranges.c index 54c337ee72..55342a38c6 100644 --- a/erts/emulator/beam/beam_ranges.c +++ b/erts/emulator/beam/beam_ranges.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2012. All Rights Reserved. + * Copyright Ericsson AB 2012-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/benchmark.c b/erts/emulator/beam/benchmark.c index 44f5c760c2..c8409784ef 100644 --- a/erts/emulator/beam/benchmark.c +++ b/erts/emulator/beam/benchmark.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2012. All Rights Reserved. + * Copyright Ericsson AB 2002-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/benchmark.h b/erts/emulator/beam/benchmark.h index 0fb0b93f12..0272896f4f 100644 --- a/erts/emulator/beam/benchmark.h +++ b/erts/emulator/beam/benchmark.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2012. All Rights Reserved. + * Copyright Ericsson AB 2002-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 37bb28c6f8..eed0702688 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2014. All Rights Reserved. + * Copyright Ericsson AB 1996-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -123,15 +123,12 @@ static int insert_internal_link(Process* p, Eterm rpid) erts_add_link(&ERTS_P_LINKS(p), LINK_PID, rp->common.id); erts_add_link(&ERTS_P_LINKS(rp), LINK_PID, p->common.id); - ASSERT(is_nil(ERTS_TRACER_PROC(p)) - || is_internal_pid(ERTS_TRACER_PROC(p)) - || is_internal_port(ERTS_TRACER_PROC(p))); + ASSERT(IS_TRACER_VALID(ERTS_TRACER(p))); if (IS_TRACED(p)) { if (ERTS_TRACE_FLAGS(p) & (F_TRACE_SOL|F_TRACE_SOL1)) { ERTS_TRACE_FLAGS(rp) |= (ERTS_TRACE_FLAGS(p) & TRACEE_FLAGS); - ERTS_TRACER_PROC(rp) = ERTS_TRACER_PROC(p); /* maybe steal */ - + erts_tracer_replace(&rp->common, ERTS_TRACER(p)); if (ERTS_TRACE_FLAGS(p) & F_TRACE_SOL1) { /* maybe override */ ERTS_TRACE_FLAGS(rp) &= ~(F_TRACE_SOL1 | F_TRACE_SOL); ERTS_TRACE_FLAGS(p) &= ~(F_TRACE_SOL1 | F_TRACE_SOL); @@ -140,7 +137,8 @@ static int insert_internal_link(Process* p, Eterm rpid) } } if (IS_TRACED_FL(rp, F_TRACE_PROCS)) - trace_proc(p, rp, am_getting_linked, p->common.id); + trace_proc(p, p == rp ? rp_locks : ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK, + rp, am_getting_linked, p->common.id); if (p == rp) erts_smp_proc_unlock(p, rp_locks & ~ERTS_PROC_LOCK_MAIN); @@ -159,7 +157,7 @@ BIF_RETTYPE link_1(BIF_ALIST_1) DistEntry *dep; if (IS_TRACED_FL(BIF_P, F_TRACE_PROCS)) { - trace_proc(BIF_P, BIF_P, am_link, BIF_ARG_1); + trace_proc(BIF_P, ERTS_PROC_LOCK_MAIN, BIF_P, am_link, BIF_ARG_1); } /* check that the pid or port which is our argument is OK */ @@ -613,7 +611,7 @@ erts_queue_monitor_message(Process *p, ref_copy = copy_struct(ref, ref_size, &hp, ohp); tup = TUPLE5(hp, am_DOWN, ref_copy, type, item_copy, reason_copy); - erts_queue_message(p, p_locksp, msgp, tup, NIL); + erts_queue_message(p, *p_locksp, msgp, tup, am_system); } static BIF_RETTYPE @@ -1001,6 +999,7 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1) Process *rp; DistEntry *dep; ErtsLink *l = NULL, *rl = NULL; + ErtsProcLocks cp_locks = ERTS_PROC_LOCK_MAIN; /* * SMP specific note concerning incoming exit signals: @@ -1015,7 +1014,7 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1) */ if (IS_TRACED_FL(BIF_P, F_TRACE_PROCS)) { - trace_proc(BIF_P, BIF_P, am_unlink, BIF_ARG_1); + trace_proc(BIF_P, cp_locks, BIF_P, am_unlink, BIF_ARG_1); } if (is_internal_port(BIF_ARG_1)) { @@ -1120,10 +1119,10 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1) erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS); + cp_locks |= ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS; + /* get process struct */ - rp = erts_pid2proc_opt(BIF_P, (ERTS_PROC_LOCK_MAIN - | ERTS_PROC_LOCK_LINK - | ERTS_PROC_LOCK_STATUS), + rp = erts_pid2proc_opt(BIF_P, cp_locks, BIF_ARG_1, ERTS_PROC_LOCK_LINK, ERTS_P2P_FLG_ALLOW_OTHER_X); @@ -1149,14 +1148,17 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1) erts_destroy_link(rl); if (IS_TRACED_FL(rp, F_TRACE_PROCS) && rl != NULL) { - trace_proc(BIF_P, rp, am_getting_unlinked, BIF_P->common.id); + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_STATUS); + cp_locks &= ~ERTS_PROC_LOCK_STATUS; + trace_proc(BIF_P, (ERTS_PROC_LOCK_MAIN | ERTS_PROC_LOCK_LINK), + rp, am_getting_unlinked, BIF_P->common.id); } if (rp != BIF_P) erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); } - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS); + erts_smp_proc_unlock(BIF_P, cp_locks & ~ERTS_PROC_LOCK_MAIN); BIF_RET(am_true); @@ -1567,7 +1569,32 @@ static BIF_RETTYPE process_flag_aux(Process *BIF_P, scb->n = 0; } - scb = ERTS_PROC_SET_SAVED_CALLS_BUF(rp, ERTS_PROC_LOCK_MAIN, scb); +#ifdef HIPE + if (rp->flags & F_HIPE_MODE) { + ASSERT(!ERTS_PROC_GET_SAVED_CALLS_BUF(rp)); + scb = ERTS_PROC_SET_SUSPENDED_SAVED_CALLS_BUF(rp, scb); + } + else +#endif + { +#ifdef HIPE + ASSERT(!ERTS_PROC_GET_SUSPENDED_SAVED_CALLS_BUF(rp)); +#endif + scb = ERTS_PROC_SET_SAVED_CALLS_BUF(rp, scb); + if (rp == BIF_P && ((scb && i == 0) || (!scb && i != 0))) { + /* Adjust fcalls to match save calls setting... */ + if (i == 0) + BIF_P->fcalls += CONTEXT_REDS; /* disabled it */ + else + BIF_P->fcalls -= CONTEXT_REDS; /* enabled it */ + + /* + * Make sure we reschedule immediately so the + * change take effect at once. + */ + ERTS_VBUMP_ALL_REDS(BIF_P); + } + } if (!scb) old_value = make_small(0); @@ -1576,12 +1603,7 @@ static BIF_RETTYPE process_flag_aux(Process *BIF_P, erts_free(ERTS_ALC_T_CALLS_BUF, (void *) scb); } - /* Make sure the process in question is rescheduled - immediately, if it's us, so the call saving takes effect. */ - if (rp == BIF_P) - BIF_RET2(old_value, CONTEXT_REDS); - else - BIF_RET(old_value); + BIF_RET(old_value); } error: @@ -1595,9 +1617,7 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2) if (is_not_atom(BIF_ARG_2)) { goto error; } - old_value = erts_proc_set_error_handler(BIF_P, - ERTS_PROC_LOCK_MAIN, - BIF_ARG_2); + old_value = erts_proc_set_error_handler(BIF_P, BIF_ARG_2); BIF_RET(old_value); } else if (BIF_ARG_1 == am_priority) { @@ -1622,14 +1642,17 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2) * true. For more info, see implementation of * erts_send_exit_signal(). */ + erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCKS_XSIG_SEND); if (trap_exit) state = erts_smp_atomic32_read_bor_mb(&BIF_P->state, ERTS_PSFLG_TRAP_EXIT); else state = erts_smp_atomic32_read_band_mb(&BIF_P->state, ~ERTS_PSFLG_TRAP_EXIT); + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCKS_XSIG_SEND); + #ifdef ERTS_SMP - if (ERTS_PROC_PENDING_EXIT(BIF_P)) { + if (state & ERTS_PSFLG_PENDING_EXIT) { erts_handle_pending_exit(BIF_P, ERTS_PROC_LOCK_MAIN); ERTS_BIF_EXITED(BIF_P); } @@ -1733,7 +1756,9 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2) ERTS_TRACE_FLAGS(BIF_P) &= ~F_SENSITIVE; } erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCKS_ALL_MINOR); - BIF_RET(old_value); + /* make sure to bump all reds so that we get + rescheduled immediately so setting takes effect */ + BIF_RET2(old_value, CONTEXT_REDS); } else if (BIF_ARG_1 == am_monitor_nodes) { /* @@ -1772,10 +1797,18 @@ BIF_RETTYPE process_flag_3(BIF_ALIST_3) Process *rp; Eterm res; - if ((rp = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN, - BIF_ARG_1, ERTS_PROC_LOCK_MAIN)) == NULL) { +#ifdef ERTS_SMP + rp = erts_pid2proc_not_running(BIF_P, ERTS_PROC_LOCK_MAIN, + BIF_ARG_1, ERTS_PROC_LOCK_MAIN); + if (rp == ERTS_PROC_LOCK_BUSY) + ERTS_BIF_YIELD3(bif_export[BIF_process_flag_3], BIF_P, + BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); +#else + rp = erts_proc_lookup(BIF_ARG_1); +#endif + + if (!rp) BIF_ERROR(BIF_P, BADARG); - } res = process_flag_aux(BIF_P, rp, BIF_ARG_2, BIF_ARG_3); @@ -1908,7 +1941,7 @@ static Sint remote_send(Process *p, DistEntry *dep, } if (res >= 0) { - if (IS_TRACED(p)) + if (IS_TRACED_FL(p, F_TRACE_SEND)) trace_send(p, full_to, msg); if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) save_calls(p, &exp_send); @@ -1927,7 +1960,7 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext* ctx) Eterm* tp; if (is_internal_pid(to)) { - if (IS_TRACED(p)) + if (IS_TRACED_FL(p, F_TRACE_SEND)) trace_send(p, to, msg); if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) save_calls(p, &exp_send); @@ -1956,7 +1989,7 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext* ctx) rp = erts_proc_lookup_raw(id); if (rp) { - if (IS_TRACED(p)) + if (IS_TRACED_FL(p, F_TRACE_SEND)) trace_send(p, to, msg); if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) save_calls(p, &exp_send); @@ -1972,7 +2005,7 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext* ctx) goto port_common; } - if (IS_TRACED(p)) + if (IS_TRACED_FL(p, F_TRACE_SEND)) trace_send(p, to, msg); if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) save_calls(p, &exp_send); @@ -2003,11 +2036,20 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext* ctx) port_common: ret_val = 0; - + if (pt) { int ps_flags = ctx->suspend ? 0 : ERTS_PORT_SIG_FLG_NOSUSPEND; *refp = NIL; + if (IS_TRACED_FL(p, F_TRACE_SEND)) /* trace once only !! */ + trace_send(p, portid, msg); + + if (have_seqtrace(SEQ_TRACE_TOKEN(p))) { + seq_trace_update_send(p); + seq_trace_output(SEQ_TRACE_TOKEN(p), msg, + SEQ_TRACE_SEND, portid, p); + } + switch (erts_port_command(p, ps_flags, pt, msg, refp)) { case ERTS_PORT_OP_CALLER_EXIT: /* We are exiting... */ @@ -2040,18 +2082,10 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext* ctx) break; } } - - if (IS_TRACED(p)) /* trace once only !! */ - trace_send(p, portid, msg); + if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) save_calls(p, &exp_send); - - if (have_seqtrace(SEQ_TRACE_TOKEN(p))) { - seq_trace_update_send(p); - seq_trace_output(SEQ_TRACE_TOKEN(p), msg, - SEQ_TRACE_SEND, portid, p); - } - + if (ERTS_PROC_IS_EXITING(p)) { KILL_CATCHES(p); /* Must exit */ return SEND_USER_ERROR; @@ -2074,7 +2108,7 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext* ctx) if (dep == erts_this_dist_entry) { Eterm id; erts_deref_dist_entry(dep); - if (IS_TRACED(p)) + if (IS_TRACED_FL(p, F_TRACE_SEND)) trace_send(p, to, msg); if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) save_calls(p, &exp_send); @@ -2105,7 +2139,7 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext* ctx) } return ret; } else { - if (IS_TRACED(p)) /* XXX Is this really neccessary ??? */ + if (IS_TRACED_FL(p, F_TRACE_SEND)) trace_send(p, to, msg); if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) save_calls(p, &exp_send); @@ -4336,12 +4370,34 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) } else if (BIF_ARG_1 == am_trace_control_word) { BIF_RET(db_set_trace_control_word(BIF_P, BIF_ARG_2)); } else if (BIF_ARG_1 == am_sequential_tracer) { - Eterm old_value = erts_set_system_seq_tracer(BIF_P, - ERTS_PROC_LOCK_MAIN, - BIF_ARG_2); - if (old_value != THE_NON_VALUE) { - BIF_RET(old_value); - } + ErtsTracer new_seq_tracer, old_seq_tracer; + Eterm ret; + + if (BIF_ARG_2 == am_false) + new_seq_tracer = erts_tracer_nil; + else + new_seq_tracer = erts_term_to_tracer(THE_NON_VALUE, BIF_ARG_2); + + if (new_seq_tracer == THE_NON_VALUE) + goto error; + + old_seq_tracer = erts_set_system_seq_tracer(BIF_P, + ERTS_PROC_LOCK_MAIN, + new_seq_tracer); + + ERTS_TRACER_CLEAR(&new_seq_tracer); + + if (old_seq_tracer == THE_NON_VALUE) + goto error; + + if (ERTS_TRACER_IS_NIL(old_seq_tracer)) + BIF_RET(am_false); + + ret = erts_tracer_to_term(BIF_P, old_seq_tracer); + + ERTS_TRACER_CLEAR(&old_seq_tracer); + + BIF_RET(ret); } else if (BIF_ARG_1 == make_small(1)) { int i, max; ErtsMessage* mp; @@ -4678,7 +4734,7 @@ skip_current_msgq(Process *c_p) res = 0; } else { - ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p); + ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p); c_p->msg.save = c_p->msg.last; res = 1; } diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h index 46af552b39..5d751dd67d 100644 --- a/erts/emulator/beam/bif.h +++ b/erts/emulator/beam/bif.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2013. All Rights Reserved. + * Copyright Ericsson AB 1996-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -52,6 +52,7 @@ extern Export *erts_convert_time_unit_trap; (p)->fcalls = 0; \ else \ (p)->fcalls = -CONTEXT_REDS; \ + ASSERT(ERTS_BIF_REDS_LEFT((p)) == 0); \ } while(0) #define ERTS_VBUMP_ALL_REDS_INTERNAL(p, fcalls) \ diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index a6670c10e8..872f0f9b2a 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1996-2013. All Rights Reserved. +# Copyright Ericsson AB 1996-2016. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -181,9 +181,8 @@ bif erlang:port_set_data/2 bif erlang:port_get_data/1 # Tracing & debugging. -bif erlang:trace_pattern/2 -bif erlang:trace_pattern/3 -bif erlang:trace/3 +bif erts_internal:trace_pattern/3 +bif erts_internal:trace/3 bif erlang:trace_info/2 bif erlang:trace_delivered/1 bif erlang:seq_trace/2 @@ -653,6 +652,8 @@ bif erts_debug:size_shared/1 bif erts_debug:copy_shared/1 bif erlang:has_prepared_code_on_load/1 +bif maps:take/2 + # # Obsolete # diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c index c738a9ecf5..4baee7900b 100644 --- a/erts/emulator/beam/big.c +++ b/erts/emulator/beam/big.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2014. All Rights Reserved. + * Copyright Ericsson AB 1996-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h index ee51e5d313..464acd67f6 100644 --- a/erts/emulator/beam/big.h +++ b/erts/emulator/beam/big.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2014. All Rights Reserved. + * Copyright Ericsson AB 1996-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c index 3fd284b589..cfd8fbe2f5 100644 --- a/erts/emulator/beam/binary.c +++ b/erts/emulator/beam/binary.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2013. All Rights Reserved. + * Copyright Ericsson AB 1996-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 8c039ef132..d02c6828f9 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2013. All Rights Reserved. + * Copyright Ericsson AB 1996-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -347,8 +347,11 @@ print_process_info(int to, void *to_arg, Process *p) static void print_garb_info(int to, void *to_arg, Process* p) { +#ifdef ERTS_SMP /* ERTS_SMP: A scheduler is probably concurrently doing gc... */ -#ifndef ERTS_SMP + if (!ERTS_IS_CRASH_DUMPING) + return; +#endif erts_print(to, to_arg, "New heap start: %bpX\n", p->heap); erts_print(to, to_arg, "New heap top: %bpX\n", p->htop); erts_print(to, to_arg, "Stack top: %bpX\n", p->stop); @@ -356,7 +359,6 @@ print_garb_info(int to, void *to_arg, Process* p) erts_print(to, to_arg, "Old heap start: %bpX\n", OLD_HEAP(p)); erts_print(to, to_arg, "Old heap top: %bpX\n", OLD_HTOP(p)); erts_print(to, to_arg, "Old heap end: %bpX\n", OLD_HEND(p)); -#endif } void @@ -684,7 +686,7 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) crash dump. */ erts_thr_progress_fatal_error_block(&tpd_buf); -#ifdef ERTS_THR_HAVE_SIG_FUNCS +#ifdef ERTS_SYS_SUSPEND_SIGNAL /* * We suspend all scheduler threads so that we can dump some * data about the currently running processes and scheduler data. @@ -818,7 +820,7 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) #ifdef ERTS_SMP -#if defined(ERTS_THR_HAVE_SIG_FUNCS) +#ifdef ERTS_SYS_SUSPEND_SIGNAL /* We resume all schedulers so that we are in a known safe state when we write the rest of the crash dump */ diff --git a/erts/emulator/beam/code_ix.c b/erts/emulator/beam/code_ix.c index 1b0968c55c..ec6267711b 100644 --- a/erts/emulator/beam/code_ix.c +++ b/erts/emulator/beam/code_ix.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2012. All Rights Reserved. + * Copyright Ericsson AB 2012-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -94,6 +94,7 @@ void erts_commit_staging_code_ix(void) ix = (ix + 1) % ERTS_NUM_CODE_IX; erts_smp_atomic32_set_nob(&the_staging_code_index, ix); export_staging_unlock(); + erts_tracer_nif_clear(); CIX_TRACE("activate"); } diff --git a/erts/emulator/beam/code_ix.h b/erts/emulator/beam/code_ix.h index 7a66211a5b..584a605771 100644 --- a/erts/emulator/beam/code_ix.h +++ b/erts/emulator/beam/code_ix.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2012-2013. All Rights Reserved. + * Copyright Ericsson AB 2012-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index b38ebe066b..ccc4cbad43 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2012. All Rights Reserved. + * Copyright Ericsson AB 1996-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index fa385f105d..09c83f1117 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2014. All Rights Reserved. + * Copyright Ericsson AB 1996-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -337,7 +337,7 @@ static void doit_link_net_exits_sub(ErtsLink *sublnk, void *vlnecp) erts_destroy_link(rlnk); if (xres >= 0 && IS_TRACED_FL(rp, F_TRACE_PROCS)) { /* We didn't exit the process and it is traced */ - trace_proc(NULL, rp, am_getting_unlinked, sublnk->pid); + trace_proc(NULL, 0, rp, am_getting_unlinked, sublnk->pid); } } erts_smp_proc_unlock(rp, rp_locks); @@ -397,7 +397,7 @@ static void doit_node_link_net_exits(ErtsLink *lnk, void *vnecp) msgp = erts_alloc_message_heap(rp, &rp_locks, 3, &hp, &ohp); tup = TUPLE2(hp, am_nodedown, name); - erts_queue_message(rp, &rp_locks, msgp, tup, NIL); + erts_queue_message(rp, rp_locks, msgp, tup, am_system); } erts_smp_proc_unlock(rp, rp_locks); } @@ -906,9 +906,9 @@ erts_dsig_send_msg(Eterm remote, Eterm message, ErtsSendContext* ctx) if (token != NIL) ctl = TUPLE4(&ctx->ctl_heap[0], - make_small(DOP_SEND_TT), am_Cookie, remote, token); + make_small(DOP_SEND_TT), am_Empty, remote, token); else - ctl = TUPLE3(&ctx->ctl_heap[0], make_small(DOP_SEND), am_Cookie, remote); + ctl = TUPLE3(&ctx->ctl_heap[0], make_small(DOP_SEND), am_Empty, remote); DTRACE6(message_send, sender_name, receiver_name, msize, tok_label, tok_lastcnt, tok_serial); DTRACE7(message_send_remote, sender_name, node_name, receiver_name, @@ -963,10 +963,10 @@ erts_dsig_send_reg_msg(Eterm remote_name, Eterm message, if (token != NIL) ctl = TUPLE5(&ctx->ctl_heap[0], make_small(DOP_REG_SEND_TT), - sender->common.id, am_Cookie, remote_name, token); + sender->common.id, am_Empty, remote_name, token); else ctl = TUPLE4(&ctx->ctl_heap[0], make_small(DOP_REG_SEND), - sender->common.id, am_Cookie, remote_name); + sender->common.id, am_Empty, remote_name); DTRACE6(message_send, sender_name, receiver_name, msize, tok_label, tok_lastcnt, tok_serial); DTRACE7(message_send_remote, sender_name, node_name, receiver_name, @@ -1275,7 +1275,7 @@ int erts_net_message(Port *prt, erts_smp_de_links_unlock(dep); if (IS_TRACED_FL(rp, F_TRACE_PROCS)) - trace_proc(NULL, rp, am_getting_linked, from); + trace_proc(NULL, 0, rp, am_getting_linked, from); erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); break; @@ -1300,7 +1300,7 @@ int erts_net_message(Port *prt, lnk = erts_remove_link(&ERTS_P_LINKS(rp), from); if (IS_TRACED_FL(rp, F_TRACE_PROCS) && lnk != NULL) { - trace_proc(NULL, rp, am_getting_unlinked, from); + trace_proc(NULL, 0, rp, am_getting_unlinked, from); } erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); @@ -1456,7 +1456,7 @@ int erts_net_message(Port *prt, token = copy_struct(token, token_size, &hp, ohp); } - erts_queue_dist_message(rp, &locks, ede_copy, token); + erts_queue_dist_message(rp, locks, ede_copy, token, from); if (locks) erts_smp_proc_unlock(rp, locks); } @@ -1505,7 +1505,7 @@ int erts_net_message(Port *prt, token = copy_struct(token, token_size, &hp, ohp); } - erts_queue_dist_message(rp, &locks, ede_copy, token); + erts_queue_dist_message(rp, locks, ede_copy, token, tuple[2]); if (locks) erts_smp_proc_unlock(rp, locks); } @@ -1628,7 +1628,11 @@ int erts_net_message(Port *prt, ERTS_XSIG_FLG_IGN_KILL); if (xres >= 0 && IS_TRACED_FL(rp, F_TRACE_PROCS)) { /* We didn't exit the process and it is traced */ - trace_proc(NULL, rp, am_getting_unlinked, from); + if (rp_locks & ERTS_PROC_LOCKS_XSIG_SEND) { + erts_smp_proc_unlock(rp, ERTS_PROC_LOCKS_XSIG_SEND); + rp_locks &= ~ERTS_PROC_LOCKS_XSIG_SEND; + } + trace_proc(NULL, 0, rp, am_getting_unlinked, from); } } erts_smp_proc_unlock(rp, rp_locks); @@ -1719,7 +1723,7 @@ decode_error: } data_error: UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE); - erts_deliver_port_exit(prt, dep->cid, am_killed, 0); + erts_deliver_port_exit(prt, dep->cid, am_killed, 0, 1); ERTS_SMP_CHK_NO_PROC_LOCKS; return -1; } @@ -1963,7 +1967,7 @@ dist_port_command(Port *prt, ErtsDistOutputBuf *obuf) ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); if (size > (Uint) INT_MAX) - erts_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_DUMP_EXIT, "Absurdly large distribution output data buffer " "(%beu bytes) passed.\n", size); @@ -2003,7 +2007,7 @@ dist_port_commandv(Port *prt, ErtsDistOutputBuf *obuf) ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); if (size > (Uint) INT_MAX) - erts_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_DUMP_EXIT, "Absurdly large distribution output data buffer " "(%beu bytes) passed.\n", size); @@ -2089,7 +2093,7 @@ erts_dist_command(Port *prt, int reds_limit) erts_smp_de_runlock(dep); if (status & ERTS_DE_SFLG_EXITING) { - erts_deliver_port_exit(prt, prt->common.id, am_killed, 0); + erts_deliver_port_exit(prt, prt->common.id, am_killed, 0, 1); erts_deref_dist_entry(dep); return reds + ERTS_PORT_REDS_DIST_CMD_EXIT; } @@ -3313,7 +3317,7 @@ send_nodes_mon_msg(Process *rp, } ASSERT(hend == hp); - erts_queue_message(rp, rp_locksp, mp, msg, NIL); + erts_queue_message(rp, *rp_locksp, mp, msg, am_system); } static void diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h index fb777d9ac1..e82b416286 100644 --- a/erts/emulator/beam/dist.h +++ b/erts/emulator/beam/dist.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2014. All Rights Reserved. + * Copyright Ericsson AB 1996-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -43,6 +43,7 @@ #define DFLAG_INTERNAL_TAGS 0x8000 #define DFLAG_UTF8_ATOMS 0x10000 #define DFLAG_MAP_TAG 0x20000 +#define DFLAG_BIG_CREATION 0x40000 /* All flags that should be enabled when term_to_binary/1 is used. */ #define TERM_TO_BINARY_DFLAGS (DFLAG_EXTENDED_REFERENCES \ @@ -51,7 +52,8 @@ | DFLAG_EXTENDED_PIDS_PORTS \ | DFLAG_EXPORT_PTR_TAG \ | DFLAG_BIT_BINARIES \ - | DFLAG_MAP_TAG) + | DFLAG_MAP_TAG \ + | DFLAG_BIG_CREATION) /* opcodes used in distribution messages */ #define DOP_LINK 1 diff --git a/erts/emulator/beam/dtrace-wrapper.h b/erts/emulator/beam/dtrace-wrapper.h index d343dc5ab0..6f70d5961e 100644 --- a/erts/emulator/beam/dtrace-wrapper.h +++ b/erts/emulator/beam/dtrace-wrapper.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Dustin Sallings, Michal Ptaszek, Scott Lystig Fritchie 2011-2012. + * Copyright Dustin Sallings, Michal Ptaszek, Scott Lystig Fritchie 2011-2016. * All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/erts/emulator/beam/elib_memmove.c b/erts/emulator/beam/elib_memmove.c index d4ca30158e..2f45f69026 100644 --- a/erts/emulator/beam/elib_memmove.c +++ b/erts/emulator/beam/elib_memmove.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1997-2009. All Rights Reserved. + * Copyright Ericsson AB 1997-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_afit_alloc.c b/erts/emulator/beam/erl_afit_alloc.c index 4b0541c10e..eda3ad870a 100644 --- a/erts/emulator/beam/erl_afit_alloc.c +++ b/erts/emulator/beam/erl_afit_alloc.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2003-2013. All Rights Reserved. + * Copyright Ericsson AB 2003-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_afit_alloc.h b/erts/emulator/beam/erl_afit_alloc.h index ef050ff50e..74258e284a 100644 --- a/erts/emulator/beam/erl_afit_alloc.h +++ b/erts/emulator/beam/erl_afit_alloc.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2003-2013. All Rights Reserved. + * Copyright Ericsson AB 2003-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 490e0c0915..c367d4162c 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2013. All Rights Reserved. + * Copyright Ericsson AB 2002-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -30,6 +30,7 @@ #endif #define ERTS_ALLOC_C__ #define ERTS_ALC_INTERNAL__ +#define ERTS_WANT_MEM_MAPPERS #include "sys.h" #define ERL_THREADS_EMU_INTERNAL__ #include "erl_threads.h" @@ -131,8 +132,16 @@ static ErtsAllocatorState_t ets_alloc_state; static ErtsAllocatorState_t driver_alloc_state; static ErtsAllocatorState_t fix_alloc_state; static ErtsAllocatorState_t literal_alloc_state; +#ifdef ERTS_ALC_A_EXEC +static ErtsAllocatorState_t exec_alloc_state; +#endif static ErtsAllocatorState_t test_alloc_state; +#define ERTS_ALC_INFO_A_ALLOC_UTIL (ERTS_ALC_A_MAX + 1) +#define ERTS_ALC_INFO_A_MSEG_ALLOC (ERTS_ALC_A_MAX + 2) +#define ERTS_ALC_INFO_A_ERTS_MMAP (ERTS_ALC_A_MAX + 3) +#define ERTS_ALC_INFO_A_MAX ERTS_ALC_INFO_A_ERTS_MMAP + typedef struct { erts_smp_atomic32_t refc; int only_sz; @@ -141,13 +150,9 @@ typedef struct { Process *proc; Eterm ref; Eterm ref_heap[REF_THING_SIZE]; - int allocs[ERTS_ALC_A_MAX-ERTS_ALC_A_MIN+1+2]; + int allocs[ERTS_ALC_INFO_A_MAX - ERTS_ALC_A_MIN + 1 + 1]; } ErtsAllocInfoReq; -#define ERTS_ALC_INFO_A_ALLOC_UTIL (ERTS_ALC_A_MAX + 1) -#define ERTS_ALC_INFO_A_MSEG_ALLOC (ERTS_ALC_A_MAX + 2) -#define ERTS_ALC_INFO_A_MAX ERTS_ALC_INFO_A_MSEG_ALLOC - ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(aireq, ErtsAllocInfoReq, 5, @@ -216,6 +221,7 @@ typedef struct { struct au_init driver_alloc; struct au_init fix_alloc; struct au_init literal_alloc; + struct au_init exec_alloc; struct au_init test_alloc; } erts_alc_hndl_args_init_t; @@ -307,9 +313,9 @@ set_default_literal_alloc_opts(struct au_init *ip) ip->init.util.name_prefix = "literal_"; ip->init.util.alloc_no = ERTS_ALC_A_LITERAL; #ifndef SMALL_MEMORY - ip->init.util.mmbcs = 2*1024*1024; /* Main carrier size */ + ip->init.util.mmbcs = 1024*1024; /* Main carrier size */ #else - ip->init.util.mmbcs = 1*1024*1024; /* Main carrier size */ + ip->init.util.mmbcs = 256*1024; /* Main carrier size */ #endif ip->init.util.ts = ERTS_ALC_MTA_LITERAL; ip->init.util.asbcst = 0; @@ -329,15 +335,48 @@ set_default_literal_alloc_opts(struct au_init *ip) ip->init.util.sys_dealloc = &erts_alcu_literal_32_sys_dealloc; #elif defined(ARCH_64) # ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION - ip->init.util.mseg_alloc = &erts_alcu_literal_64_mseg_alloc; - ip->init.util.mseg_realloc = &erts_alcu_literal_64_mseg_realloc; - ip->init.util.mseg_dealloc = &erts_alcu_literal_64_mseg_dealloc; + ip->init.util.mseg_alloc = &erts_alcu_mmapper_mseg_alloc; + ip->init.util.mseg_realloc = &erts_alcu_mmapper_mseg_realloc; + ip->init.util.mseg_dealloc = &erts_alcu_mmapper_mseg_dealloc; + ip->init.util.mseg_mmapper = &erts_literal_mmapper; # endif #else # error Unknown architecture #endif } +#ifdef ERTS_ALC_A_EXEC +static void +set_default_exec_alloc_opts(struct au_init *ip) +{ + SET_DEFAULT_ALLOC_OPTS(ip); + ip->enable = 1; + ip->thr_spec = 0; + ip->disable_allowed = 0; + ip->thr_spec_allowed = 0; + ip->carrier_migration_allowed = 0; + ip->atype = BESTFIT; + ip->init.bf.ao = 1; + ip->init.util.ramv = 0; + ip->init.util.mmsbc = 0; + ip->init.util.sbct = ~((UWord) 0); + ip->init.util.name_prefix = "exec_"; + ip->init.util.alloc_no = ERTS_ALC_A_EXEC; + ip->init.util.mmbcs = 0; /* No main carrier */ + ip->init.util.ts = ERTS_ALC_MTA_EXEC; + ip->init.util.asbcst = 0; + ip->init.util.rsbcst = 0; + ip->init.util.rsbcmt = 0; + ip->init.util.rmbcmt = 0; + ip->init.util.acul = 0; + + ip->init.util.mseg_alloc = &erts_alcu_mmapper_mseg_alloc; + ip->init.util.mseg_realloc = &erts_alcu_mmapper_mseg_realloc; + ip->init.util.mseg_dealloc = &erts_alcu_mmapper_mseg_dealloc; + ip->init.util.mseg_mmapper = &erts_exec_mmapper; +} +#endif /* ERTS_ALC_A_EXEC */ + static void set_default_temp_alloc_opts(struct au_init *ip) { @@ -659,6 +698,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) set_default_fix_alloc_opts(&init.fix_alloc, fix_type_sizes); set_default_literal_alloc_opts(&init.literal_alloc); +#ifdef ERTS_ALC_A_EXEC + set_default_exec_alloc_opts(&init.exec_alloc); +#endif set_default_test_alloc_opts(&init.test_alloc); if (argc && argv) @@ -688,6 +730,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) init.driver_alloc.thr_spec = 0; init.fix_alloc.thr_spec = 0; init.literal_alloc.thr_spec = 0; +#ifdef ERTS_ALC_A_EXEC + init.exec_alloc.thr_spec = 0; +#endif #endif /* Make adjustments for carrier migration support */ @@ -701,6 +746,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) adjust_carrier_migration_support(&init.driver_alloc); adjust_carrier_migration_support(&init.fix_alloc); adjust_carrier_migration_support(&init.literal_alloc); +#ifdef ERTS_ALC_A_EXEC + adjust_carrier_migration_support(&init.exec_alloc); +#endif if (init.erts_alloc_config) { /* Adjust flags that erts_alloc_config won't like */ @@ -716,6 +764,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) init.driver_alloc.thr_spec = 0; init.fix_alloc.thr_spec = 0; init.literal_alloc.thr_spec = 0; +#ifdef ERTS_ALC_A_EXEC + init.exec_alloc.thr_spec = 0; +#endif /* No carrier migration */ init.temp_alloc.init.util.acul = 0; @@ -728,6 +779,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) init.driver_alloc.init.util.acul = 0; init.fix_alloc.init.util.acul = 0; init.literal_alloc.init.util.acul = 0; +#ifdef ERTS_ALC_A_EXEC + init.exec_alloc.init.util.acul = 0; +#endif } #ifdef ERTS_SMP @@ -745,6 +799,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) adjust_tpref(&init.driver_alloc, erts_no_schedulers); adjust_tpref(&init.fix_alloc, erts_no_schedulers); adjust_tpref(&init.literal_alloc, erts_no_schedulers); +#ifdef ERTS_ALC_A_EXEC + adjust_tpref(&init.exec_alloc, erts_no_schedulers); +#endif #else /* No thread specific if not smp */ @@ -764,6 +821,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) refuse_af_strategy(&init.driver_alloc); refuse_af_strategy(&init.fix_alloc); refuse_af_strategy(&init.literal_alloc); +#ifdef ERTS_ALC_A_EXEC + refuse_af_strategy(&init.exec_alloc); +#endif #ifdef ERTS_SMP if (!init.temp_alloc.thr_spec) @@ -808,6 +868,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) set_au_allocator(ERTS_ALC_A_DRIVER, &init.driver_alloc, ncpu); set_au_allocator(ERTS_ALC_A_FIXED_SIZE, &init.fix_alloc, ncpu); set_au_allocator(ERTS_ALC_A_LITERAL, &init.literal_alloc, ncpu); +#ifdef ERTS_ALC_A_EXEC + set_au_allocator(ERTS_ALC_A_EXEC, &init.exec_alloc, ncpu); +#endif set_au_allocator(ERTS_ALC_A_TEST, &init.test_alloc, ncpu); for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { @@ -864,7 +927,11 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) start_au_allocator(ERTS_ALC_A_LITERAL, &init.literal_alloc, &literal_alloc_state); - +#ifdef ERTS_ALC_A_EXEC + start_au_allocator(ERTS_ALC_A_EXEC, + &init.exec_alloc, + &exec_alloc_state); +#endif start_au_allocator(ERTS_ALC_A_TEST, &init.test_alloc, &test_alloc_state); @@ -1499,6 +1566,16 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) else handle_au_arg(&init->literal_alloc, &argv[i][3], argv, &i, 0); break; + case 'X': + if (has_prefix("scs", argv[i]+3)) { +#ifdef ERTS_ALC_A_EXEC + init->mseg.exec_mmap.scs = +#endif + get_mb_value(argv[i]+6, argv, &i); + } + else + handle_au_arg(&init->exec_alloc, &argv[i][3], argv, &i, 0); + break; case 'D': handle_au_arg(&init->std_alloc, &argv[i][3], argv, &i, 0); break; @@ -1949,7 +2026,7 @@ erts_alc_fatal_error(int error, int func, ErtsAlcType_t n, ...) va_start(argp, n); size = va_arg(argp, Uint); va_end(argp); - erts_exit(1, + erts_exit(ERTS_DUMP_EXIT, "%s: Cannot %s %lu bytes of memory (of type \"%s\").\n", allctr_str, op, size, t_str); break; @@ -2764,10 +2841,18 @@ erts_allocator_info(int to, void *arg) int i; for (i = 0; i <= max; i++) { erts_print(to, arg, "=allocator:mseg_alloc[%d]\n", i); - erts_mseg_info(i, &to, arg, 0, NULL, NULL); + erts_mseg_info(i, &to, arg, 0, 0, NULL, NULL); } - erts_print(to, arg, "=allocator:mseg_alloc.erts_mmap\n"); + erts_print(to, arg, "=allocator:erts_mmap.default_mmap\n"); erts_mmap_info(&erts_dflt_mmapper, &to, arg, NULL, NULL, &emis); +#if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) + erts_print(to, arg, "=allocator:erts_mmap.literal_mmap\n"); + erts_mmap_info(&erts_literal_mmapper, &to, arg, NULL, NULL, &emis); +#endif +#ifdef ERTS_ALC_A_EXEC + erts_print(to, arg, "=allocator:erts_mmap.exec_mmap\n"); + erts_mmap_info(&erts_exec_mmapper, &to, arg, NULL, NULL, &emis); +#endif } #endif @@ -2878,6 +2963,11 @@ erts_allocator_options(void *proc) atoms[length] = am_atom_put("alloc_util", 10); terms[length++] = erts_alcu_au_info_options(NULL, NULL, hpp, szp); +#if HAVE_ERTS_MMAP + atoms[length] = ERTS_MAKE_AM("erts_mmap"); + terms[length++] = erts_mmap_info_options(&erts_dflt_mmapper, NULL, NULL, + NULL, hpp, szp); +#endif { Eterm o[3], v[3]; o[0] = am_atom_put("m", 1); @@ -2914,7 +3004,12 @@ erts_allocator_options(void *proc) #if ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC terms[length++] = am_atom_put("sys_aligned_alloc", 17); #endif - +#if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) + terms[length++] = ERTS_MAKE_AM("literal_mmap"); +#endif +#ifdef ERTS_ALC_A_EXEC + terms[length++] = ERTS_MAKE_AM("exec_mmap"); +#endif features = length ? erts_bld_list(hpp, szp, length, terms) : NIL; #if defined(__GLIBC__) @@ -2999,7 +3094,15 @@ reply_alloc_info(void *vair) Uint sz, *szp; ErlOffHeap *ohp = NULL; ErtsMessage *mp = NULL; - struct erts_mmap_info_struct emis; +#if HAVE_ERTS_MMAP + struct erts_mmap_info_struct mmap_info_dflt; +# if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) + struct erts_mmap_info_struct mmap_info_literal; +# endif +# ifdef ERTS_ALC_A_EXEC + struct erts_mmap_info_struct mmap_info_exec; +# endif +#endif int i; Eterm (*info_func)(Allctr_t *, int, @@ -3105,26 +3208,53 @@ reply_alloc_info(void *vair) make_small(0), ainfo); break; + case ERTS_ALC_INFO_A_ERTS_MMAP: + alloc_atom = erts_bld_atom(hpp, szp, "erts_mmap"); +#if HAVE_ERTS_MMAP + ainfo = (air->only_sz ? NIL : + erts_mmap_info(&erts_dflt_mmapper, NULL, NULL, + hpp, szp, &mmap_info_dflt)); + ainfo = erts_bld_tuple3(hpp, szp, + alloc_atom, + erts_bld_atom(hpp,szp,"default_mmap"), + ainfo); +# if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) + ai_list = erts_bld_cons(hpp, szp, + ainfo, ai_list); + ainfo = (air->only_sz ? NIL : + erts_mmap_info(&erts_literal_mmapper, NULL, NULL, + hpp, szp, &mmap_info_literal)); + ainfo = erts_bld_tuple3(hpp, szp, + alloc_atom, + erts_bld_atom(hpp,szp,"literal_mmap"), + ainfo); +# endif +# ifdef ERTS_ALC_A_EXEC + ai_list = erts_bld_cons(hpp, szp, + ainfo, ai_list); + ainfo = (air->only_sz ? NIL : + erts_mmap_info(&erts_exec_mmapper, NULL, NULL, + hpp, szp, &mmap_info_exec)); + ainfo = erts_bld_tuple3(hpp, szp, + alloc_atom, + erts_bld_atom(hpp,szp,"exec_mmap"), + ainfo); +# endif +#else /* !HAVE_ERTS_MMAP */ + ainfo = erts_bld_tuple2(hpp, szp, alloc_atom, + am_false); +#endif + break; case ERTS_ALC_INFO_A_MSEG_ALLOC: alloc_atom = erts_bld_atom(hpp, szp, "mseg_alloc"); #if HAVE_ERTS_MSEG - ainfo = (air->only_sz - ? NIL - : erts_mseg_info(0, NULL, NULL, hpp != NULL, - hpp, szp)); + ainfo = erts_mseg_info(0, NULL, NULL, hpp != NULL, + air->only_sz, hpp, szp); ainfo = erts_bld_tuple3(hpp, szp, alloc_atom, make_small(0), ainfo); - ai_list = erts_bld_cons(hpp, szp, - ainfo, ai_list); - ainfo = (air->only_sz ? NIL : - erts_mmap_info(&erts_dflt_mmapper, NULL, NULL, hpp, szp, &emis)); - ainfo = erts_bld_tuple3(hpp, szp, - alloc_atom, - erts_bld_atom(hpp,szp,"erts_mmap"), - ainfo); #else ainfo = erts_bld_tuple2(hpp, szp, alloc_atom, am_false); @@ -3156,15 +3286,14 @@ reply_alloc_info(void *vair) } switch (ai) { case ERTS_ALC_A_SYSTEM: - case ERTS_ALC_INFO_A_ALLOC_UTIL: + case ERTS_ALC_INFO_A_ALLOC_UTIL: + case ERTS_ALC_INFO_A_ERTS_MMAP: break; case ERTS_ALC_INFO_A_MSEG_ALLOC: #if HAVE_ERTS_MSEG && defined(ERTS_SMP) alloc_atom = erts_bld_atom(hpp, szp, "mseg_alloc"); - ainfo = (air->only_sz - ? NIL - : erts_mseg_info(sched_id, NULL, NULL, - hpp != NULL, hpp, szp)); + ainfo = erts_mseg_info(sched_id, NULL, NULL, + hpp != NULL, air->only_sz, hpp, szp); ainfo = erts_bld_tuple(hpp, szp, 3, alloc_atom, make_small(sched_id), @@ -3210,7 +3339,7 @@ reply_alloc_info(void *vair) if (hp != hp_end) erts_shrink_message_heap(&mp, rp, hp_start, hp, hp_end, &msg, 1); - erts_queue_message(rp, &rp_locks, mp, msg, NIL); + erts_queue_message(rp, rp_locks, mp, msg, am_system); if (air->req_sched == sched_id) rp_locks &= ~ERTS_PROC_LOCK_MAIN; @@ -3230,7 +3359,7 @@ erts_request_alloc_info(struct process *c_p, int internal) { ErtsAllocInfoReq *air = aireq_alloc(); - Eterm req_ai[ERTS_ALC_A_MAX+1+2] = {0}; + Eterm req_ai[ERTS_ALC_INFO_A_MAX+1] = {0}; Eterm alist; Eterm *hp; int airix = 0, ai; @@ -3266,6 +3395,10 @@ erts_request_alloc_info(struct process *c_p, ai = ERTS_ALC_INFO_A_MSEG_ALLOC; goto save_alloc; } + if (erts_is_atom_str("erts_mmap", alloc, 0)) { + ai = ERTS_ALC_INFO_A_ERTS_MMAP; + goto save_alloc; + } if (erts_is_atom_str("alloc_util", alloc, 0)) { ai = ERTS_ALC_INFO_A_ALLOC_UTIL; save_alloc: diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h index 71e4713624..925a081a02 100644 --- a/erts/emulator/beam/erl_alloc.h +++ b/erts/emulator/beam/erl_alloc.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2013. All Rights Reserved. + * Copyright Ericsson AB 2002-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -235,9 +235,9 @@ void *erts_alloc(ErtsAlcType_t type, Uint size) void *res; ERTS_MSACC_PUSH_AND_SET_STATE_X(ERTS_MSACC_STATE_ALLOC); res = (*erts_allctrs[ERTS_ALC_T2A(type)].alloc)( - ERTS_ALC_T2N(type), - erts_allctrs[ERTS_ALC_T2A(type)].extra, - size); + ERTS_ALC_T2N(type), + erts_allctrs[ERTS_ALC_T2A(type)].extra, + size); if (!res) erts_alloc_n_enomem(ERTS_ALC_T2N(type), size); ERTS_MSACC_POP_STATE_X(); @@ -564,5 +564,3 @@ NAME##_free(TYPE *p) \ #undef ERTS_ALC_ATTRIBUTES #endif /* #ifndef ERL_ALLOC_H__ */ - - diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 2932adca84..ba216c7eb4 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2003-2014. All Rights Reserved. +# Copyright Ericsson AB 2003-2016. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -87,6 +87,9 @@ allocator EHEAP true eheap_alloc allocator ETS true ets_alloc allocator FIXED_SIZE true fix_alloc allocator LITERAL true literal_alloc ++if exec_alloc +allocator EXEC true exec_alloc ++endif +else # Non smp build @@ -98,6 +101,9 @@ allocator EHEAP false eheap_alloc allocator ETS false ets_alloc allocator FIXED_SIZE false fix_alloc allocator LITERAL false literal_alloc ++if exec_alloc +allocator EXEC false exec_alloc ++endif +endif @@ -258,7 +264,6 @@ type PRTSD STANDARD SYSTEM port_specific_data type CPUDATA LONG_LIVED SYSTEM cpu_data type TMP_CPU_IDS SHORT_LIVED SYSTEM tmp_cpu_ids type EXT_TERM_DATA SHORT_LIVED PROCESSES external_term_data -type ZLIB STANDARD SYSTEM zlib type CPU_GRPS_MAP LONG_LIVED SYSTEM cpu_groups_map type AUX_WORK_TMO LONG_LIVED SYSTEM aux_work_timeouts type MISC_AUX_WORK_Q LONG_LIVED SYSTEM misc_aux_work_q @@ -270,6 +275,9 @@ type PROC_SYS_TSK SHORT_LIVED PROCESSES proc_sys_task type PROC_SYS_TSK_QS SHORT_LIVED PROCESSES proc_sys_task_queues type NEW_TIME_OFFSET SHORT_LIVED SYSTEM new_time_offset type IOB_REQ SHORT_LIVED SYSTEM io_bytes_request +type TRACER_NIF LONG_LIVED SYSTEM tracer_nif +type TRACE_MSG_QUEUE SHORT_LIVED SYSTEM trace_message_queue +type SCHED_ASYNC_JOB SHORT_LIVED SYSTEM async_calls +if threads_no_smp # Need thread safe allocs, but std_alloc and fix_alloc are not; @@ -288,8 +296,10 @@ type THR_Q_LL LONG_LIVED SYSTEM long_lived_thr_queue +if smp type ASYNC SHORT_LIVED SYSTEM async +type ZLIB STANDARD SYSTEM zlib +else -# sl_alloc is not thread safe in non smp build; therefore, we use driver_alloc +# sl/std_alloc is not thread safe in non smp build; therefore, we use driver_alloc +type ZLIB DRIVER SYSTEM zlib type ASYNC DRIVER SYSTEM async +endif @@ -335,14 +345,22 @@ type SL_MPATHS SHORT_LIVED SYSTEM sl_migration_paths # Currently most hipe code use this type. type HIPE SYSTEM SYSTEM hipe_data ++if exec_alloc +type HIPE_EXEC EXEC CODE hipe_code +endif ++endif + + + +if heap_frag_elim_test type SSB SHORT_LIVED PROCESSES ssb +endif +type DEBUG SHORT_LIVED SYSTEM debugging + type DDLL_PROCESS STANDARD SYSTEM ddll_processes type MONITOR_LH STANDARD PROCESSES monitor_lh type NLINK_LH STANDARD PROCESSES nlink_lh diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 5e7dd7cce8..2995f2f822 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2013. All Rights Reserved. + * Copyright Ericsson AB 2002-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -52,6 +52,7 @@ #ifdef ERTS_ENABLE_LOCK_COUNT #include "erl_lock_count.h" #endif +#include "lttng-wrapper.h" #if defined(ERTS_ALLOC_UTIL_HARD_DEBUG) && defined(__GNUC__) #warning "* * * * * * * * * *" @@ -858,28 +859,36 @@ void* erts_alcu_literal_32_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) { void* res; - ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL - && !allctr->t && allctr->thread_safe); + Uint sz = ERTS_SUPERALIGNED_CEILING(*size_p); + ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL && + allctr->t == 0); + ERTS_SMP_LC_ASSERT(allctr->thread_safe); - res = erts_alcu_mseg_alloc(allctr, size_p, flags); - if (res) - set_literal_range(res, *size_p); + res = erts_alcu_mseg_alloc(allctr, &sz, flags); + if (res) { + set_literal_range(res, sz); + *size_p = sz; + } return res; } void* erts_alcu_literal_32_mseg_realloc(Allctr_t *allctr, void *seg, - Uint old_size, Uint *new_size_p) + Uint old_size, Uint *new_size_p) { void* res; - ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL - && !allctr->t && allctr->thread_safe); + Uint new_sz = ERTS_SUPERALIGNED_CEILING(*new_size_p); + ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL && + allctr->t == 0); + ERTS_SMP_LC_ASSERT(allctr->thread_safe); if (seg && old_size) clear_literal_range(seg, old_size); - res = erts_alcu_mseg_realloc(allctr, seg, old_size, new_size_p); - if (res) - set_literal_range(res, *new_size_p); + res = erts_alcu_mseg_realloc(allctr, seg, old_size, &new_sz); + if (res) { + set_literal_range(res, new_sz); + *new_size_p = new_sz; + } return res; } @@ -887,8 +896,9 @@ void erts_alcu_literal_32_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, Uint flags) { - ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL - && !allctr->t && allctr->thread_safe); + ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL && + allctr->t == 0); + ERTS_SMP_LC_ASSERT(allctr->thread_safe); erts_alcu_mseg_dealloc(allctr, seg, size, flags); @@ -897,8 +907,9 @@ erts_alcu_literal_32_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, #elif defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) +/* Used by literal allocator that has its own mmapper (super carrier) */ void* -erts_alcu_literal_64_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) +erts_alcu_mmapper_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) { void* res; UWord size = (UWord) *size_p; @@ -906,33 +917,33 @@ erts_alcu_literal_64_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) if (flags & ERTS_MSEG_FLG_2POW) mmap_flags |= ERTS_MMAPFLG_SUPERALIGNED; - res = erts_mmap(&erts_literal_mmapper, mmap_flags, &size); + res = erts_mmap(allctr->mseg_mmapper, mmap_flags, &size); *size_p = (Uint)size; INC_CC(allctr->calls.mseg_alloc); return res; } void* -erts_alcu_literal_64_mseg_realloc(Allctr_t *allctr, void *seg, +erts_alcu_mmapper_mseg_realloc(Allctr_t *allctr, void *seg, Uint old_size, Uint *new_size_p) { void *res; UWord new_size = (UWord) *new_size_p; - res = erts_mremap(&erts_literal_mmapper, ERTS_MSEG_FLG_NONE, seg, old_size, &new_size); + res = erts_mremap(allctr->mseg_mmapper, ERTS_MSEG_FLG_NONE, seg, old_size, &new_size); *new_size_p = (Uint) new_size; INC_CC(allctr->calls.mseg_realloc); return res; } void -erts_alcu_literal_64_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, +erts_alcu_mmapper_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, Uint flags) { Uint32 mmap_flags = ERTS_MMAPFLG_SUPERCARRIER_ONLY; if (flags & ERTS_MSEG_FLG_2POW) mmap_flags |= ERTS_MMAPFLG_SUPERALIGNED; - erts_munmap(&erts_literal_mmapper, mmap_flags, seg, (UWord)size); + erts_munmap(allctr->mseg_mmapper, mmap_flags, seg, (UWord)size); INC_CC(allctr->calls.mseg_dealloc); } #endif /* ARCH_64 && ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION */ @@ -940,9 +951,10 @@ erts_alcu_literal_64_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, #endif /* HAVE_ERTS_MSEG */ void* -erts_alcu_sys_alloc(Allctr_t *allctr, Uint size, int superalign) +erts_alcu_sys_alloc(Allctr_t *allctr, Uint* size_p, int superalign) { void *res; + const Uint size = *size_p; #if ERTS_SA_MB_CARRIERS && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC if (superalign) res = erts_sys_aligned_alloc(ERTS_SACRR_UNIT_SZ, size); @@ -956,9 +968,10 @@ erts_alcu_sys_alloc(Allctr_t *allctr, Uint size, int superalign) } void* -erts_alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int superalign) +erts_alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint *size_p, Uint old_size, int superalign) { void *res; + const Uint size = *size_p; #if ERTS_SA_MB_CARRIERS && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC if (superalign) @@ -993,38 +1006,48 @@ erts_alcu_sys_dealloc(Allctr_t *allctr, void *ptr, Uint size, int superalign) #ifdef ARCH_32 void* -erts_alcu_literal_32_sys_alloc(Allctr_t *allctr, Uint size, int superalign) +erts_alcu_literal_32_sys_alloc(Allctr_t *allctr, Uint* size_p, int superalign) { void* res; - ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL - && !allctr->t && allctr->thread_safe); + Uint size = ERTS_SUPERALIGNED_CEILING(*size_p); + ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL && + allctr->t == 0); + ERTS_SMP_LC_ASSERT(allctr->thread_safe); - res = erts_alcu_sys_alloc(allctr, size, 1); - if (res) + res = erts_alcu_sys_alloc(allctr, &size, 1); + if (res) { set_literal_range(res, size); + *size_p = size; + } return res; } void* -erts_alcu_literal_32_sys_realloc(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int superalign) +erts_alcu_literal_32_sys_realloc(Allctr_t *allctr, void *ptr, Uint* size_p, Uint old_size, int superalign) { void* res; - ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL - && !allctr->t && allctr->thread_safe); + Uint size = ERTS_SUPERALIGNED_CEILING(*size_p); + + ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL && + allctr->t == 0); + ERTS_SMP_LC_ASSERT(allctr->thread_safe); if (ptr && old_size) clear_literal_range(ptr, old_size); - res = erts_alcu_sys_realloc(allctr, ptr, size, old_size, 1); - if (res) + res = erts_alcu_sys_realloc(allctr, ptr, &size, old_size, 1); + if (res) { set_literal_range(res, size); + *size_p = size; + } return res; } void erts_alcu_literal_32_sys_dealloc(Allctr_t *allctr, void *ptr, Uint size, int superalign) { - ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL - && !allctr->t && allctr->thread_safe); + ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL && + allctr->t == 0); + ERTS_SMP_LC_ASSERT(allctr->thread_safe); erts_alcu_sys_dealloc(allctr, ptr, size, 1); @@ -3125,6 +3148,7 @@ cpool_insert(Allctr_t *allctr, Carrier_t *crr) erts_smp_atomic_set_wb(&crr->allctr, ((erts_aint_t) allctr)|ERTS_CRR_ALCTR_FLG_IN_POOL); + LTTNG3(carrier_pool_put, ERTS_ALC_A2AD(allctr->alloc_no), allctr->ix, CARRIER_SZ(crr)); } static void @@ -3240,6 +3264,7 @@ cpool_fetch(Allctr_t *allctr, UWord size) first_old_traitor = allctr->cpool.traitor_list.next; cpool_entrance = NULL; + LTTNG3(carrier_pool_get, ERTS_ALC_A2AD(allctr->alloc_no), allctr->ix, (unsigned long)size); /* * Search my own pooled_list, * i.e my abandoned carriers that were in the pool last time I checked. @@ -3861,12 +3886,12 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) ? UNIT_CEILING(bcrr_sz) : SYS_ALLOC_CARRIER_CEILING(bcrr_sz)); - crr = (Carrier_t *) allctr->sys_alloc(allctr, crr_sz, flags & CFLG_MBC); + crr = (Carrier_t *) allctr->sys_alloc(allctr, &crr_sz, flags & CFLG_MBC); if (!crr) { if (crr_sz > UNIT_CEILING(bcrr_sz)) { crr_sz = UNIT_CEILING(bcrr_sz); - crr = (Carrier_t *) allctr->sys_alloc(allctr, crr_sz, flags & CFLG_MBC); + crr = (Carrier_t *) allctr->sys_alloc(allctr, &crr_sz, flags & CFLG_MBC); } if (!crr) { #if HAVE_ERTS_MSEG @@ -3925,6 +3950,21 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) } +#ifdef USE_LTTNG_VM_TRACEPOINTS + if (LTTNG_ENABLED(carrier_create)) { + lttng_decl_carrier_stats(mbc_stats); + lttng_decl_carrier_stats(sbc_stats); + LTTNG_CARRIER_STATS_TO_LTTNG_STATS(&(allctr->mbcs), mbc_stats); + LTTNG_CARRIER_STATS_TO_LTTNG_STATS(&(allctr->sbcs), sbc_stats); + LTTNG5(carrier_create, + ERTS_ALC_A2AD(allctr->alloc_no), + allctr->ix, + crr_sz, + mbc_stats, + sbc_stats); + } +#endif + DEBUG_SAVE_ALIGNMENT(crr); return blk; } @@ -4009,7 +4049,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) new_crr = (Carrier_t *) allctr->sys_realloc(allctr, (void *) old_crr, - new_crr_sz, + &new_crr_sz, old_crr_sz, 0); if (new_crr) { @@ -4030,7 +4070,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) new_crr_sz = UNIT_CEILING(new_crr_sz); new_crr = (Carrier_t *) allctr->sys_realloc(allctr, (void *) old_crr, - new_crr_sz, + &new_crr_sz, old_crr_sz, 0); if (new_crr) @@ -4148,6 +4188,21 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp) allctr->remove_mbc(allctr, crr); } +#ifdef USE_LTTNG_VM_TRACEPOINTS + if (LTTNG_ENABLED(carrier_destroy)) { + lttng_decl_carrier_stats(mbc_stats); + lttng_decl_carrier_stats(sbc_stats); + LTTNG_CARRIER_STATS_TO_LTTNG_STATS(&(allctr->mbcs), mbc_stats); + LTTNG_CARRIER_STATS_TO_LTTNG_STATS(&(allctr->sbcs), sbc_stats); + LTTNG5(carrier_destroy, + ERTS_ALC_A2AD(allctr->alloc_no), + allctr->ix, + crr_sz, + mbc_stats, + sbc_stats); + } +#endif + #ifdef ERTS_SMP schedule_dealloc_carrier(allctr, crr); #else @@ -6094,6 +6149,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->mseg_alloc = init->mseg_alloc; allctr->mseg_realloc = init->mseg_realloc; allctr->mseg_dealloc = init->mseg_dealloc; + allctr->mseg_mmapper = init->mseg_mmapper; } else { ASSERT(!init->mseg_realloc && !init->mseg_dealloc); diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index b7d717ed23..f50f09907a 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2013. All Rights Reserved. + * Copyright Ericsson AB 2002-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -30,6 +30,7 @@ #endif #include "erl_mseg.h" +#include "lttng-wrapper.h" #define ERTS_AU_PREF_ALLOC_BITS 11 #define ERTS_AU_MAX_PREF_ALLOC_INSTANCES (1 << ERTS_AU_PREF_ALLOC_BITS) @@ -71,9 +72,10 @@ typedef struct { void* (*mseg_alloc)(Allctr_t*, Uint *size_p, Uint flags); void* (*mseg_realloc)(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p); void (*mseg_dealloc)(Allctr_t*, void *seg, Uint size, Uint flags); + ErtsMemMapper *mseg_mmapper; #endif - void* (*sys_alloc)(Allctr_t *allctr, Uint size, int superalign); - void* (*sys_realloc)(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int superalign); + void* (*sys_alloc)(Allctr_t *allctr, Uint *size_p, int superalign); + void* (*sys_realloc)(Allctr_t *allctr, void *ptr, Uint *size_p, Uint old_size, int superalign); void (*sys_dealloc)(Allctr_t *allctr, void *ptr, Uint size, int superalign); } AllctrInit_t; @@ -204,18 +206,18 @@ void* erts_alcu_literal_32_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uin void erts_alcu_literal_32_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags); # elif defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) -void* erts_alcu_literal_64_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags); -void* erts_alcu_literal_64_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p); -void erts_alcu_literal_64_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags); +void* erts_alcu_mmapper_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags); +void* erts_alcu_mmapper_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p); +void erts_alcu_mmapper_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags); # endif #endif /* HAVE_ERTS_MSEG */ -void* erts_alcu_sys_alloc(Allctr_t*, Uint size, int superalign); -void* erts_alcu_sys_realloc(Allctr_t*, void *ptr, Uint size, Uint old_size, int superalign); +void* erts_alcu_sys_alloc(Allctr_t*, Uint *size_p, int superalign); +void* erts_alcu_sys_realloc(Allctr_t*, void *ptr, Uint *size_p, Uint old_size, int superalign); void erts_alcu_sys_dealloc(Allctr_t*, void *ptr, Uint size, int superalign); #ifdef ARCH_32 -void* erts_alcu_literal_32_sys_alloc(Allctr_t*, Uint size, int superalign); -void* erts_alcu_literal_32_sys_realloc(Allctr_t*, void *ptr, Uint size, Uint old_size, int superalign); +void* erts_alcu_literal_32_sys_alloc(Allctr_t*, Uint *size_p, int superalign); +void* erts_alcu_literal_32_sys_realloc(Allctr_t*, void *ptr, Uint *size_p, Uint old_size, int superalign); void erts_alcu_literal_32_sys_dealloc(Allctr_t*, void *ptr, Uint size, int superalign); #endif @@ -417,6 +419,18 @@ typedef struct { } blocks; } CarriersStats_t; +#ifdef USE_LTTNG_VM_TRACEPOINTS +#define LTTNG_CARRIER_STATS_TO_LTTNG_STATS(CSP, LSP) \ + do { \ + (LSP)->carriers.size = (CSP)->curr.norm.mseg.size \ + + (CSP)->curr.norm.sys_alloc.size; \ + (LSP)->carriers.no = (CSP)->curr.norm.mseg.no \ + + (CSP)->curr.norm.sys_alloc.no; \ + (LSP)->blocks.size = (CSP)->blocks.curr.size; \ + (LSP)->blocks.no = (CSP)->blocks.curr.no; \ + } while (0) +#endif + #ifdef ERTS_SMP typedef union ErtsAllctrDDBlock_t_ ErtsAllctrDDBlock_t; @@ -588,9 +602,10 @@ struct Allctr_t_ { void* (*mseg_alloc)(Allctr_t*, Uint *size_p, Uint flags); void* (*mseg_realloc)(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p); void (*mseg_dealloc)(Allctr_t*, void *seg, Uint size, Uint flags); + ErtsMemMapper *mseg_mmapper; #endif - void* (*sys_alloc)(Allctr_t *allctr, Uint size, int superalign); - void* (*sys_realloc)(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int superalign); + void* (*sys_alloc)(Allctr_t *allctr, Uint *size_p, int superalign); + void* (*sys_realloc)(Allctr_t *allctr, void *ptr, Uint *size_p, Uint old_size, int superalign); void (*sys_dealloc)(Allctr_t *allctr, void *ptr, Uint size, int superalign); void (*init_atoms) (void); diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c index 5cd067ff54..fbe4724047 100644 --- a/erts/emulator/beam/erl_ao_firstfit_alloc.c +++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2003-2013. All Rights Reserved. + * Copyright Ericsson AB 2003-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.h b/erts/emulator/beam/erl_ao_firstfit_alloc.h index 4200f20622..7349c6ab19 100644 --- a/erts/emulator/beam/erl_ao_firstfit_alloc.h +++ b/erts/emulator/beam/erl_ao_firstfit_alloc.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2003-2013. All Rights Reserved. + * Copyright Ericsson AB 2003-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_arith.c b/erts/emulator/beam/erl_arith.c index 3671025d22..861532f241 100644 --- a/erts/emulator/beam/erl_arith.c +++ b/erts/emulator/beam/erl_arith.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2010. All Rights Reserved. + * Copyright Ericsson AB 1999-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_async.c b/erts/emulator/beam/erl_async.c index cdeeb5281b..84254af0c2 100644 --- a/erts/emulator/beam/erl_async.c +++ b/erts/emulator/beam/erl_async.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2013. All Rights Reserved. + * Copyright Ericsson AB 2000-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,6 +28,7 @@ #include "erl_thr_queue.h" #include "erl_async.h" #include "dtrace-wrapper.h" +#include "lttng-wrapper.h" #define ERTS_MAX_ASYNC_READY_CALLS_IN_SEQ 20 @@ -281,6 +282,13 @@ static ERTS_INLINE void async_add(ErtsAsync *a, ErtsAsyncQ* q) #endif erts_thr_q_enqueue(&q->thr_q, a); +#ifdef USE_LTTNG_VM_TRACEPOINTS + if (LTTNG_ENABLED(aio_pool_put)) { + lttng_decl_portbuf(port_str); + lttng_portid_to_str(a->port, port_str); + LTTNG2(aio_pool_put, port_str, -1); + } +#endif #ifdef USE_VM_PROBES if (DTRACE_ENABLED(aio_pool_add)) { DTRACE_CHARBUF(port_str, 16); @@ -317,6 +325,14 @@ static ERTS_INLINE ErtsAsync *async_get(ErtsThrQ_t *q, if (saved_fin_deq) erts_thr_q_append_finalize_dequeue_data(&a->q.fin_deq, &fin_deq); #endif +#ifdef USE_LTTNG_VM_TRACEPOINTS + if (LTTNG_ENABLED(aio_pool_get)) { + lttng_decl_portbuf(port_str); + int length = erts_thr_q_length_dirty(q); + lttng_portid_to_str(a->port, port_str); + LTTNG2(aio_pool_get, port_str, length); + } +#endif #ifdef USE_VM_PROBES if (DTRACE_ENABLED(aio_pool_get)) { DTRACE_CHARBUF(port_str, 16); diff --git a/erts/emulator/beam/erl_async.h b/erts/emulator/beam/erl_async.h index 65538bcef0..473c7686e5 100644 --- a/erts/emulator/beam/erl_async.h +++ b/erts/emulator/beam/erl_async.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2011. All Rights Reserved. + * Copyright Ericsson AB 2011-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_bestfit_alloc.c b/erts/emulator/beam/erl_bestfit_alloc.c index f39a18ac88..379cee39a1 100644 --- a/erts/emulator/beam/erl_bestfit_alloc.c +++ b/erts/emulator/beam/erl_bestfit_alloc.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2003-2013. All Rights Reserved. + * Copyright Ericsson AB 2003-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_bestfit_alloc.h b/erts/emulator/beam/erl_bestfit_alloc.h index b315518b88..3a5f51f5dc 100644 --- a/erts/emulator/beam/erl_bestfit_alloc.h +++ b/erts/emulator/beam/erl_bestfit_alloc.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2003-2013. All Rights Reserved. + * Copyright Ericsson AB 2003-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c index aec72bd61a..6e10980b6b 100644 --- a/erts/emulator/beam/erl_bif_binary.c +++ b/erts/emulator/beam/erl_bif_binary.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2010-2013. All Rights Reserved. + * Copyright Ericsson AB 2010-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_bif_chksum.c b/erts/emulator/beam/erl_bif_chksum.c index e3074d6309..9417803e14 100644 --- a/erts/emulator/beam/erl_bif_chksum.c +++ b/erts/emulator/beam/erl_bif_chksum.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2008-2013. All Rights Reserved. + * Copyright Ericsson AB 2008-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c index f006856b6a..ef77201544 100644 --- a/erts/emulator/beam/erl_bif_ddll.c +++ b/erts/emulator/beam/erl_bif_ddll.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2006-2013. All Rights Reserved. + * Copyright Ericsson AB 2006-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -48,6 +48,7 @@ #include "erl_version.h" #include "erl_bif_unique.h" #include "dtrace-wrapper.h" +#include "lttng-wrapper.h" #ifdef ERTS_SMP #define DDLL_SMP 1 @@ -1619,6 +1620,7 @@ static int do_unload_driver_entry(DE_Handle *dh, Eterm *save_name) if (q->finish) { int fpe_was_unmasked = erts_block_fpe(); DTRACE1(driver_finish, q->name); + LTTNG1(driver_finish, q->name); (*(q->finish))(); erts_unblock_fpe(fpe_was_unmasked); } @@ -1735,7 +1737,7 @@ static void notify_proc(Process *proc, Eterm ref, Eterm driver_name, Eterm type, hp += REF_THING_SIZE; mess = TUPLE5(hp,type,r,am_driver,driver_name,tag); } - erts_queue_message(proc, &rp_locks, mp, mess, am_undefined); + erts_queue_message(proc, rp_locks, mp, mess, am_system); erts_smp_proc_unlock(proc, rp_locks); ERTS_SMP_CHK_NO_PROC_LOCKS; } diff --git a/erts/emulator/beam/erl_bif_guard.c b/erts/emulator/beam/erl_bif_guard.c index 4a9a6a5fcd..b42d2dc28b 100644 --- a/erts/emulator/beam/erl_bif_guard.c +++ b/erts/emulator/beam/erl_bif_guard.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2006-2011. All Rights Reserved. + * Copyright Ericsson AB 2006-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 8c748c9bf7..85cadafebc 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2013. All Rights Reserved. + * Copyright Ericsson AB 1999-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,6 +22,7 @@ # include "config.h" #endif +#define ERTS_WANT_MEM_MAPPERS #include "sys.h" #include "erl_vm.h" #include "global.h" @@ -126,6 +127,9 @@ static char erts_system_version[] = ("Erlang/OTP " ERLANG_OTP_RELEASE #ifdef ERTS_FRMPTR " [frame-pointer]" #endif +#ifdef USE_LTTNG + " [lttng]" +#endif #ifdef USE_DTRACE " [dtrace]" #endif @@ -843,6 +847,7 @@ process_info_list(Process *c_p, Eterm pid, Eterm list, int always_wrap, if (unlock_locks) erts_smp_proc_unlock(rp, unlock_locks); + } /* @@ -1354,9 +1359,10 @@ process_info_aux(Process *BIF_P, total_heap_size += rp->mbuf_sz; - for (mp = rp->msg.first; mp; mp = mp->next) - if (mp->data.attached) - total_heap_size += erts_msg_attached_data_size(mp); + if (rp->flags & F_ON_HEAP_MSGQ) + for (mp = rp->msg.first; mp; mp = mp->next) + if (mp->data.attached) + total_heap_size += erts_msg_attached_data_size(mp); (void) erts_bld_uint(NULL, &hsz, total_heap_size); hp = HAlloc(BIF_P, hsz); @@ -1480,7 +1486,7 @@ process_info_aux(Process *BIF_P, } case am_last_calls: { - struct saved_calls *scb = ERTS_PROC_GET_SAVED_CALLS_BUF(BIF_P); + struct saved_calls *scb = ERTS_PROC_GET_SAVED_CALLS_BUF(rp); if (!scb) { hp = HAlloc(BIF_P, 3); res = am_false; @@ -2159,8 +2165,8 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) res = build_snifs_term(&hp, NULL, NIL); BIF_RET(res); } else if (BIF_ARG_1 == am_sequential_tracer) { - val = erts_get_system_seq_tracer(); - ASSERT(is_internal_pid(val) || is_internal_port(val) || val==am_false); + ErtsTracer seq_tracer = erts_get_system_seq_tracer(); + val = erts_tracer_to_term(BIF_P, seq_tracer); hp = HAlloc(BIF_P, 3); res = TUPLE2(hp, am_sequential_tracer, val); BIF_RET(res); @@ -2748,6 +2754,9 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) #elif defined(USE_SYSTEMTAP) DECL_AM(systemtap); BIF_RET(AM_systemtap); +#elif defined(USE_LTTNG) + DECL_AM(lttng); + BIF_RET(AM_lttng); #else BIF_RET(am_none); #endif @@ -3546,7 +3555,7 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) BIF_RET(res); } else if (ERTS_IS_ATOM_STR("mmap", BIF_ARG_1)) { - BIF_RET(erts_mmap_debug_info(&erts_dflt_mmapper, BIF_P)); + BIF_RET(erts_mmap_debug_info(BIF_P)); } else if (ERTS_IS_ATOM_STR("unique_monotonic_integer_state", BIF_ARG_1)) { BIF_RET(erts_debug_get_unique_monotonic_integer_state(BIF_P)); diff --git a/erts/emulator/beam/erl_bif_lists.c b/erts/emulator/beam/erl_bif_lists.c index fe64e76575..73d327da3e 100644 --- a/erts/emulator/beam/erl_bif_lists.c +++ b/erts/emulator/beam/erl_bif_lists.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2011. All Rights Reserved. + * Copyright Ericsson AB 1999-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -40,32 +40,93 @@ static BIF_RETTYPE append(Process* p, Eterm A, Eterm B) Eterm list; Eterm copy; Eterm last; - size_t need; - Eterm* hp; + Eterm* hp = NULL; Sint i; - if ((i = erts_list_length(A)) < 0) { - BIF_ERROR(p, BADARG); + list = A; + + if (is_nil(list)) { + BIF_RET(B); } - if (i == 0) { - BIF_RET(B); - } else if (is_nil(B)) { - BIF_RET(A); + + if (is_not_list(list)) { + BIF_ERROR(p, BADARG); } - need = 2*i; - hp = HAlloc(p, need); - list = A; + /* optimistic append on heap first */ + + if ((i = HeapWordsLeft(p) / 2) < 4) { + goto list_tail; + } + + hp = HEAP_TOP(p); copy = last = CONS(hp, CAR(list_val(list)), make_list(hp+2)); list = CDR(list_val(list)); - hp += 2; + hp += 2; + i -= 2; /* don't use the last 2 words (extra i--;) */ + + while(i-- && is_list(list)) { + Eterm* listp = list_val(list); + last = CONS(hp, CAR(listp), make_list(hp+2)); + list = CDR(listp); + hp += 2; + } + + /* A is proper and B is NIL return A as-is, don't update HTOP */ + + if (is_nil(list) && is_nil(B)) { + BIF_RET(A); + } + + if (is_nil(list)) { + HEAP_TOP(p) = hp; + CDR(list_val(last)) = B; + BIF_RET(copy); + } + +list_tail: + + if ((i = erts_list_length(list)) < 0) { + BIF_ERROR(p, BADARG); + } + + /* remaining list was proper and B is NIL */ + if (is_nil(B)) { + BIF_RET(A); + } + + if (hp) { + /* Note: fall through case, already written + * on the heap. + * The last 2 words of the heap is not written yet + */ + Eterm *hp_save = hp; + ASSERT(i != 0); + HEAP_TOP(p) = hp + 2; + if (i == 1) { + hp[0] = CAR(list_val(list)); + hp[1] = B; + BIF_RET(copy); + } + hp = HAlloc(p, 2*(i - 1)); + last = CONS(hp_save, CAR(list_val(list)), make_list(hp)); + } else { + hp = HAlloc(p, 2*i); + copy = last = CONS(hp, CAR(list_val(list)), make_list(hp+2)); + hp += 2; + } + + list = CDR(list_val(list)); i--; + + ASSERT(i > -1); while(i--) { - Eterm* listp = list_val(list); - last = CONS(hp, CAR(listp), make_list(hp+2)); - list = CDR(listp); - hp += 2; + Eterm* listp = list_val(list); + last = CONS(hp, CAR(listp), make_list(hp+2)); + list = CDR(listp); + hp += 2; } + CDR(list_val(last)) = B; BIF_RET(copy); } diff --git a/erts/emulator/beam/erl_bif_op.c b/erts/emulator/beam/erl_bif_op.c index 0f20ded1d6..aecb8bf0c1 100644 --- a/erts/emulator/beam/erl_bif_op.c +++ b/erts/emulator/beam/erl_bif_op.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2013. All Rights Reserved. + * Copyright Ericsson AB 1999-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_bif_os.c b/erts/emulator/beam/erl_bif_os.c index 2333ca0851..46777d3aa5 100644 --- a/erts/emulator/beam/erl_bif_os.c +++ b/erts/emulator/beam/erl_bif_os.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2012. All Rights Reserved. + * Copyright Ericsson AB 1999-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c index 86bcf43678..37f4e1de49 100644 --- a/erts/emulator/beam/erl_bif_port.c +++ b/erts/emulator/beam/erl_bif_port.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2013. All Rights Reserved. + * Copyright Ericsson AB 2001-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -79,6 +79,7 @@ BIF_RETTYPE erts_internal_open_port_2(BIF_ALIST_2) } if (port->drv_ptr->flags & ERL_DRV_FLAG_USE_INIT_ACK) { + /* Copied from erl_port_task.c */ port->async_open_port = erts_alloc(ERTS_ALC_T_PRTSD, sizeof(*port->async_open_port)); @@ -109,6 +110,10 @@ BIF_RETTYPE erts_internal_open_port_2(BIF_ALIST_2) erts_add_link(&ERTS_P_LINKS(port), LINK_PID, BIF_P->common.id); erts_add_link(&ERTS_P_LINKS(BIF_P), LINK_PID, port->common.id); + if (IS_TRACED_FL(BIF_P, F_TRACE_PROCS)) + trace_proc(BIF_P, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK, BIF_P, + am_link, port->common.id); + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK); erts_port_release(port); @@ -339,8 +344,7 @@ BIF_RETTYPE erts_internal_port_close_1(BIF_ALIST_1) if (!prt) BIF_RET(am_badarg); - - switch (erts_port_exit(BIF_P, 0, prt, prt->common.id, am_normal, &ref)) { + switch (erts_port_exit(BIF_P, 0, prt, BIF_P->common.id, am_normal, &ref)) { case ERTS_PORT_OP_CALLER_EXIT: case ERTS_PORT_OP_BADARG: case ERTS_PORT_OP_DROPPED: @@ -373,7 +377,7 @@ BIF_RETTYPE erts_internal_port_connect_2(BIF_ALIST_2) ref = NIL; #endif - switch (erts_port_connect(BIF_P, 0, prt, prt->common.id, BIF_ARG_2, &ref)) { + switch (erts_port_connect(BIF_P, 0, prt, BIF_P->common.id, BIF_ARG_2, &ref)) { case ERTS_PORT_OP_CALLER_EXIT: case ERTS_PORT_OP_BADARG: case ERTS_PORT_OP_DROPPED: @@ -911,7 +915,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump) } if (IS_TRACED_FL(p, F_TRACE_SCHED_PROCS)) { - trace_virtual_sched(p, am_out); + trace_sched(p, ERTS_PROC_LOCK_MAIN, am_out); } @@ -928,21 +932,22 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump) DTRACE3(port_open, process_str, name_buf, port_str); } #endif + + if (port && IS_TRACED_FL(port, F_TRACE_PORTS)) + trace_port(port, am_getting_linked, p->common.id); + erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); + if (IS_TRACED_FL(p, F_TRACE_SCHED_PROCS)) { + trace_sched(p, ERTS_PROC_LOCK_MAIN, am_in); + } + if (!port) { DEBUGF(("open_driver returned (%d:%d)\n", err_typep ? *err_typep : 4711, err_nump ? *err_nump : 4711)); - if (IS_TRACED_FL(p, F_TRACE_SCHED_PROCS)) { - trace_virtual_sched(p, am_in); - } goto do_return; } - - if (IS_TRACED_FL(p, F_TRACE_SCHED_PROCS)) { - trace_virtual_sched(p, am_in); - } if (linebuf && port->linebuf == NULL){ port->linebuf = allocate_linebuf(linebuf); @@ -985,6 +990,8 @@ static char **convert_args(Eterm l) } n = erts_list_length(l); + if (n < 0) + return NULL; /* We require at least one element in argv[0] + NULL at end */ pp = erts_alloc(ERTS_ALC_T_TMP, (n + 2) * sizeof(char **)); pp[i++] = erts_default_arg0; diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c index 7f7cd376ac..f4daecd2a4 100644 --- a/erts/emulator/beam/erl_bif_re.c +++ b/erts/emulator/beam/erl_bif_re.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2008-2013. All Rights Reserved. + * Copyright Ericsson AB 2008-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c index 71a7079b09..b65c0e303f 100644 --- a/erts/emulator/beam/erl_bif_trace.c +++ b/erts/emulator/beam/erl_bif_trace.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2013. All Rights Reserved. + * Copyright Ericsson AB 1999-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -52,7 +52,7 @@ static int erts_default_trace_pattern_is_on; static Binary *erts_default_match_spec; static Binary *erts_default_meta_match_spec; static struct trace_pattern_flags erts_default_trace_pattern_flags; -static Eterm erts_default_meta_tracer_pid; +static ErtsTracer erts_default_meta_tracer; static struct { /* Protected by code write permission */ int current; @@ -68,6 +68,9 @@ static struct { /* Protected by code write permission */ static Eterm trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist); +static int +erts_set_tracing_event_pattern(Eterm event, Binary*, int on); + #ifdef ERTS_SMP static void smp_bp_finisher(void* arg); #endif @@ -75,11 +78,11 @@ static BIF_RETTYPE system_monitor(Process *p, Eterm monitor_pid, Eterm list); static void new_seq_trace_token(Process* p); /* help func for seq_trace_2*/ -static int already_traced(Process *p, Process *tracee_p, Eterm tracer); -static int port_already_traced(Process *p, Port *tracee_port, Eterm tracer); static Eterm trace_info_pid(Process* p, Eterm pid_spec, Eterm key); static Eterm trace_info_func(Process* p, Eterm pid_spec, Eterm key); static Eterm trace_info_on_load(Process* p, Eterm key); +static Eterm trace_info_event(Process* p, Eterm event, Eterm key); + static void reset_bif_trace(void); static void setup_bif_trace(void); @@ -87,28 +90,34 @@ static void install_exp_breakpoints(BpFunctions* f); static void uninstall_exp_breakpoints(BpFunctions* f); static void clean_export_entries(BpFunctions* f); +ErtsTracingEvent erts_send_tracing[ERTS_NUM_BP_IX]; +ErtsTracingEvent erts_receive_tracing[ERTS_NUM_BP_IX]; + void erts_bif_trace_init(void) { + int i; + erts_default_trace_pattern_is_on = 0; erts_default_match_spec = NULL; erts_default_meta_match_spec = NULL; erts_default_trace_pattern_flags = erts_trace_pattern_flags_off; - erts_default_meta_tracer_pid = NIL; + erts_default_meta_tracer = erts_tracer_nil; + + for (i=0; i<ERTS_NUM_BP_IX; i++) { + erts_send_tracing[i].on = 1; + erts_send_tracing[i].match_spec = NULL; + erts_receive_tracing[i].on = 1; + erts_receive_tracing[i].match_spec = NULL; + } } /* * Turn on/off call tracing for the given function(s). */ - -Eterm -trace_pattern_2(BIF_ALIST_2) -{ - return trace_pattern(BIF_P, BIF_ARG_1, BIF_ARG_2, NIL); -} Eterm -trace_pattern_3(BIF_ALIST_3) +erts_internal_trace_pattern_3(BIF_ALIST_3) { return trace_pattern(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); } @@ -125,11 +134,10 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) Eterm l; struct trace_pattern_flags flags = erts_trace_pattern_flags_off; int is_global; - Process *meta_tracer_proc = p; - Eterm meta_tracer_pid = p->common.id; + ErtsTracer meta_tracer = erts_tracer_nil; if (!erts_try_seize_code_write_permission(p)) { - ERTS_BIF_YIELD3(bif_export[BIF_trace_pattern_3], p, MFA, Pattern, flaglist); + ERTS_BIF_YIELD3(bif_export[BIF_erts_internal_trace_pattern_3], p, MFA, Pattern, flaglist); } finish_bp.current = -1; @@ -146,45 +154,28 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) on = 1; } else if (Pattern == am_restart) { match_prog_set = NULL; - on = erts_break_reset; + on = ERTS_BREAK_RESTART; } else if (Pattern == am_pause) { match_prog_set = NULL; - on = erts_break_stop; - } else if ((match_prog_set = erts_match_set_compile(p, Pattern)) != NULL) { - MatchSetRef(match_prog_set); - on = 1; - } else{ - goto error; + on = ERTS_BREAK_PAUSE; + } else { + match_prog_set = erts_match_set_compile(p, Pattern, MFA); + if (match_prog_set) { + MatchSetRef(match_prog_set); + on = 1; + } else{ + goto error; + } } is_global = 0; for(l = flaglist; is_list(l); l = CDR(list_val(l))) { if (is_tuple(CAR(list_val(l)))) { - Eterm *tp = tuple_val(CAR(list_val(l))); - - if (arityval(tp[0]) != 2 || tp[1] != am_meta) { - goto error; - } - meta_tracer_pid = tp[2]; - if (is_internal_pid(meta_tracer_pid)) { - meta_tracer_proc = erts_pid2proc(NULL, 0, meta_tracer_pid, 0); - if (!meta_tracer_proc) { - goto error; - } - } else if (is_internal_port(meta_tracer_pid)) { - Port *meta_tracer_port; - meta_tracer_proc = NULL; - meta_tracer_port = (erts_port_lookup( - meta_tracer_pid, - ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP)); - if (!meta_tracer_port) - goto error; - } else { - goto error; - } - if (is_global) { - goto error; - } + meta_tracer = erts_term_to_tracer(am_meta, CAR(list_val(l))); + if (meta_tracer == THE_NON_VALUE) { + meta_tracer = erts_tracer_nil; + goto error; + } flags.breakpoint = 1; flags.meta = 1; } else { @@ -202,6 +193,8 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) } flags.breakpoint = 1; flags.meta = 1; + if (ERTS_TRACER_IS_NIL(meta_tracer)) + meta_tracer = erts_term_to_tracer(THE_NON_VALUE, p->common.id); break; case am_global: if (flags.breakpoint) { @@ -252,14 +245,11 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) MatchSetUnref(erts_default_meta_match_spec); erts_default_meta_match_spec = match_prog_set; MatchSetRef(erts_default_meta_match_spec); - erts_default_meta_tracer_pid = meta_tracer_pid; - if (meta_tracer_proc) { - ERTS_TRACE_FLAGS(meta_tracer_proc) |= F_TRACER; - } + erts_tracer_update(&erts_default_meta_tracer, meta_tracer); } else if (! flags.breakpoint) { MatchSetUnref(erts_default_meta_match_spec); erts_default_meta_match_spec = NULL; - erts_default_meta_tracer_pid = NIL; + ERTS_TRACER_CLEAR(&erts_default_meta_tracer); } if (erts_default_trace_pattern_flags.breakpoint && flags.breakpoint) { @@ -340,20 +330,23 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) if (is_small(mfa[2])) { mfa[2] = signed_val(mfa[2]); } - - if (meta_tracer_proc) { - ERTS_TRACE_FLAGS(meta_tracer_proc) |= F_TRACER; - } matches = erts_set_trace_pattern(p, mfa, specified, match_prog_set, match_prog_set, - on, flags, meta_tracer_pid, 0); + on, flags, meta_tracer, 0); + } else if (is_atom(MFA)) { + if (is_global || flags.breakpoint || on > ERTS_BREAK_SET) { + goto error; + } + matches = erts_set_tracing_event_pattern(MFA, match_prog_set, on); } error: MatchSetUnref(match_prog_set); UnUseTmpHeap(3,p); + ERTS_TRACER_CLEAR(&meta_tracer); + #ifdef ERTS_SMP if (finish_bp.current >= 0) { ASSERT(matches >= 0); @@ -404,7 +397,7 @@ erts_get_default_trace_pattern(int *trace_pattern_is_on, Binary **match_spec, Binary **meta_match_spec, struct trace_pattern_flags *trace_pattern_flags, - Eterm *meta_tracer_pid) + ErtsTracer *meta_tracer) { ERTS_SMP_LC_ASSERT(erts_has_code_write_permission() || erts_smp_thr_progress_is_blocking()); @@ -416,8 +409,8 @@ erts_get_default_trace_pattern(int *trace_pattern_is_on, *meta_match_spec = erts_default_meta_match_spec; if (trace_pattern_flags) *trace_pattern_flags = erts_default_trace_pattern_flags; - if (meta_tracer_pid) - *meta_tracer_pid = erts_default_meta_tracer_pid; + if (meta_tracer) + *meta_tracer = erts_default_meta_tracer; } int erts_is_default_trace_enabled(void) @@ -465,12 +458,12 @@ erts_trace_flag2bit(Eterm flag) ** occurred in the argument list. */ int -erts_trace_flags(Eterm List, - Uint *pMask, Eterm *pTracer, int *pCpuTimestamp) +erts_trace_flags(Eterm List, + Uint *pMask, ErtsTracer *pTracer, int *pCpuTimestamp) { Eterm list = List; Uint mask = 0; - Eterm tracer = NIL; + ErtsTracer tracer = erts_tracer_nil; int cpu_timestamp = 0; while (is_list(list)) { @@ -483,33 +476,73 @@ erts_trace_flags(Eterm List, cpu_timestamp = !0; #endif } else if (is_tuple(item)) { - Eterm* tp = tuple_val(item); - - if (arityval(tp[0]) != 2 || tp[1] != am_tracer) goto error; - if (is_internal_pid(tp[2]) || is_internal_port(tp[2])) { - tracer = tp[2]; - } else goto error; + tracer = erts_term_to_tracer(am_tracer, item); + if (tracer == THE_NON_VALUE) + goto error; } else goto error; list = CDR(list_val(list)); } if (is_not_nil(list)) goto error; - if (pMask && mask) *pMask = mask; - if (pTracer && tracer != NIL) *pTracer = tracer; - if (pCpuTimestamp && cpu_timestamp) *pCpuTimestamp = cpu_timestamp; + if (pMask && mask) *pMask = mask; + if (pTracer && !ERTS_TRACER_IS_NIL(tracer)) *pTracer = tracer; + if (pCpuTimestamp && cpu_timestamp) *pCpuTimestamp = cpu_timestamp; return !0; error: return 0; } -Eterm trace_3(BIF_ALIST_3) +static ERTS_INLINE int +start_trace(Process *c_p, ErtsTracer tracer, + ErtsPTabElementCommon *common, + int on, int mask) +{ + /* We can use the common part of both port+proc without checking what it is + In the code below port is used for both proc and port */ + Port *port = (Port*)common; + + /* + * SMP build assumes that either system is blocked or: + * * main lock is held on c_p + * * all locks are held on port common + */ + + if (!ERTS_TRACER_IS_NIL(tracer)) { + if ((ERTS_TRACE_FLAGS(port) & TRACEE_FLAGS) + && !ERTS_TRACER_COMPARE(ERTS_TRACER(port), tracer)) { + /* This tracee is already being traced, and not by the + * tracer to be */ + if (erts_is_tracer_proc_enabled(c_p, ERTS_PROC_LOCKS_ALL, + common, am_trace_status)) { + /* The tracer is still in use */ + return 1; + } + /* Current tracer now invalid */ + } + } + + if (on) + ERTS_TRACE_FLAGS(port) |= mask; + else + ERTS_TRACE_FLAGS(port) &= ~mask; + + if ((ERTS_TRACE_FLAGS(port) & TRACEE_FLAGS) == 0) { + tracer = erts_tracer_nil; + erts_tracer_replace(common, erts_tracer_nil); + } else if (!ERTS_TRACER_IS_NIL(tracer)) + erts_tracer_replace(common, tracer); + + return 0; +} + +Eterm erts_internal_trace_3(BIF_ALIST_3) { Process* p = BIF_P; Eterm pid_spec = BIF_ARG_1; Eterm how = BIF_ARG_2; Eterm list = BIF_ARG_3; int on; - Eterm tracer = NIL; + ErtsTracer tracer = erts_tracer_nil; int matches = 0; Uint mask = 0; int cpu_ts = 0; @@ -522,41 +555,24 @@ Eterm trace_3(BIF_ALIST_3) } if (!erts_try_seize_code_write_permission(BIF_P)) { - ERTS_BIF_YIELD3(bif_export[BIF_trace_3], BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); + ERTS_BIF_YIELD3(bif_export[BIF_erts_internal_trace_3], + BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); } - if (is_nil(tracer) || is_internal_pid(tracer)) { - Process *tracer_proc = erts_pid2proc(p, - ERTS_PROC_LOCK_MAIN, - is_nil(tracer) ? p->common.id : tracer, - ERTS_PROC_LOCKS_ALL); - if (!tracer_proc) - goto error; - ERTS_TRACE_FLAGS(tracer_proc) |= F_TRACER; - erts_smp_proc_unlock(tracer_proc, - (tracer_proc == p - ? ERTS_PROC_LOCKS_ALL_MINOR - : ERTS_PROC_LOCKS_ALL)); - } else if (is_internal_port(tracer)) { - Port *tracer_port = erts_id2port_sflgs(tracer, - p, - ERTS_PROC_LOCK_MAIN, - ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP); - if (!tracer_port) - goto error; - ERTS_TRACE_FLAGS(tracer_port) |= F_TRACER; - erts_port_release(tracer_port); - } else - goto error; - switch (how) { case am_false: on = 0; break; case am_true: on = 1; - if (is_nil(tracer)) - tracer = p->common.id; + if (ERTS_TRACER_IS_NIL(tracer)) + tracer = erts_term_to_tracer(am_tracer, p->common.id); + + if (tracer == THE_NON_VALUE) { + tracer = erts_tracer_nil; + goto error; + } + break; default: goto error; @@ -575,34 +591,20 @@ Eterm trace_3(BIF_ALIST_3) } #endif - if (pid_spec == tracer) - goto error; - tracee_port = erts_id2port_sflgs(pid_spec, p, ERTS_PROC_LOCK_MAIN, ERTS_PORT_SFLGS_INVALID_LOOKUP); + if (!tracee_port) goto error; - - if (tracer != NIL && port_already_traced(p, tracee_port, tracer)) { + + if (start_trace(p, tracer, &tracee_port->common, on, mask)) { erts_port_release(tracee_port); goto already_traced; - } - - if (on) - ERTS_TRACE_FLAGS(tracee_port) |= mask; - else - ERTS_TRACE_FLAGS(tracee_port) &= ~mask; - - if (!ERTS_TRACE_FLAGS(tracee_port)) - ERTS_TRACER_PROC(tracee_port) = NIL; - else if (tracer != NIL) - ERTS_TRACER_PROC(tracee_port) = tracer; - - erts_port_release(tracee_port); - - matches = 1; + } + erts_port_release(tracee_port); + matches = 1; } else if (is_pid(pid_spec)) { Process *tracee_p; @@ -615,33 +617,19 @@ Eterm trace_3(BIF_ALIST_3) * and not about to be tracing. */ - if (pid_spec == tracer) - goto error; - tracee_p = erts_pid2proc(p, ERTS_PROC_LOCK_MAIN, pid_spec, ERTS_PROC_LOCKS_ALL); if (!tracee_p) goto error; - if (tracer != NIL && already_traced(p, tracee_p, tracer)) { + if (start_trace(tracee_p, tracer, &tracee_p->common, on, mask)) { erts_smp_proc_unlock(tracee_p, (tracee_p == p ? ERTS_PROC_LOCKS_ALL_MINOR : ERTS_PROC_LOCKS_ALL)); goto already_traced; - } - - if (on) - ERTS_TRACE_FLAGS(tracee_p) |= mask; - else - ERTS_TRACE_FLAGS(tracee_p) &= ~mask; - - if ((ERTS_TRACE_FLAGS(tracee_p) & TRACEE_FLAGS) == 0) - ERTS_TRACER_PROC(tracee_p) = NIL; - else if (tracer != NIL) - ERTS_TRACER_PROC(tracee_p) = tracer; - - erts_smp_proc_unlock(tracee_p, + } + erts_smp_proc_unlock(tracee_p, (tracee_p == p ? ERTS_PROC_LOCKS_ALL_MINOR : ERTS_PROC_LOCKS_ALL)); @@ -692,18 +680,27 @@ Eterm trace_3(BIF_ALIST_3) } #endif - if (pid_spec == am_all || pid_spec == am_existing) { + if (pid_spec == am_all || pid_spec == am_existing || + pid_spec == am_ports || pid_spec == am_processes || + pid_spec == am_existing_ports || pid_spec == am_existing_processes + ) { int i; int procs = 0; int ports = 0; int mods = 0; if (mask & (ERTS_PROC_TRACEE_FLAGS & ~ERTS_TRACEE_MODIFIER_FLAGS)) - procs = 1; + procs = pid_spec != am_ports && pid_spec != am_existing_ports; if (mask & (ERTS_PORT_TRACEE_FLAGS & ~ERTS_TRACEE_MODIFIER_FLAGS)) - ports = 1; - if (mask & ERTS_TRACEE_MODIFIER_FLAGS) - mods = 1; + ports = pid_spec != am_processes && pid_spec != am_existing_processes; + if (mask & ERTS_TRACEE_MODIFIER_FLAGS) { + if (pid_spec == am_ports || pid_spec == am_existing_ports) + ports = 1; + else if (pid_spec == am_processes || pid_spec == am_existing_processes) + procs = 1; + else + mods = 1; + } #ifdef ERTS_SMP erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN); @@ -719,22 +716,7 @@ Eterm trace_3(BIF_ALIST_3) Process* tracee_p = erts_pix2proc(i); if (! tracee_p) continue; - if (tracer != NIL) { - if (tracee_p->common.id == tracer) - continue; - if (already_traced(NULL, tracee_p, tracer)) - continue; - } - if (on) { - ERTS_TRACE_FLAGS(tracee_p) |= mask; - } else { - ERTS_TRACE_FLAGS(tracee_p) &= ~mask; - } - if(!(ERTS_TRACE_FLAGS(tracee_p) & TRACEE_FLAGS)) { - ERTS_TRACER_PROC(tracee_p) = NIL; - } else if (tracer != NIL) { - ERTS_TRACER_PROC(tracee_p) = tracer; - } + start_trace(p, tracer, &tracee_p->common, on, mask); matches++; } } @@ -749,33 +731,26 @@ Eterm trace_3(BIF_ALIST_3) state = erts_atomic32_read_nob(&tracee_port->state); if (state & ERTS_PORT_SFLGS_DEAD) continue; - if (tracer != NIL) { - if (tracee_port->common.id == tracer) - continue; - if (port_already_traced(NULL, tracee_port, tracer)) - continue; - } - - if (on) ERTS_TRACE_FLAGS(tracee_port) |= mask; - else ERTS_TRACE_FLAGS(tracee_port) &= ~mask; - - if (!(ERTS_TRACE_FLAGS(tracee_port) & TRACEE_FLAGS)) { - ERTS_TRACER_PROC(tracee_port) = NIL; - } else if (tracer != NIL) { - ERTS_TRACER_PROC(tracee_port) = tracer; - } - /* matches are not counted for ports since it would violate compatibility */ - /* This could be a reason to modify this function or make a new one. */ + start_trace(p, tracer, &tracee_port->common, on, mask); + matches++; } } } - if (pid_spec == am_all || pid_spec == am_new) { - Uint def_flags = mask; - Eterm def_tracer = tracer; + if (pid_spec == am_all || pid_spec == am_new + || pid_spec == am_ports || pid_spec == am_processes + || pid_spec == am_new_ports || pid_spec == am_new_processes + ) { ok = 1; - erts_change_default_tracing(on, &def_flags, &def_tracer); + if (mask & ERTS_PROC_TRACEE_FLAGS && + pid_spec != am_ports && pid_spec != am_new_ports) + erts_change_default_proc_tracing( + on, mask & ERTS_PROC_TRACEE_FLAGS, tracer); + if (mask & ERTS_PORT_TRACEE_FLAGS && + pid_spec != am_processes && pid_spec != am_new_processes) + erts_change_default_port_tracing( + on, mask & ERTS_PORT_TRACEE_FLAGS, tracer); #ifdef HAVE_ERTS_NOW_CPU if (cpu_ts && !on) { @@ -801,6 +776,7 @@ Eterm trace_3(BIF_ALIST_3) } #endif erts_release_code_write_permission(); + ERTS_TRACER_CLEAR(&tracer); BIF_RET(make_small(matches)); @@ -810,6 +786,8 @@ Eterm trace_3(BIF_ALIST_3) error: + ERTS_TRACER_CLEAR(&tracer); + #ifdef ERTS_SMP if (system_blocked) { erts_smp_thr_progress_unblock(); @@ -821,88 +799,6 @@ Eterm trace_3(BIF_ALIST_3) BIF_ERROR(p, BADARG); } -/* Check that the process to be traced is not already traced - * by a valid other tracer than the tracer to be. - */ -static int port_already_traced(Process *c_p, Port *tracee_port, Eterm tracer) -{ - /* - * SMP build assumes that either system is blocked or: - * * main lock is held on c_p - * * all locks are held on port tracee_p - */ - if ((ERTS_TRACE_FLAGS(tracee_port) & TRACEE_FLAGS) - && ERTS_TRACER_PROC(tracee_port) != tracer) { - /* This tracee is already being traced, and not by the - * tracer to be */ - if (is_internal_port(ERTS_TRACER_PROC(tracee_port))) { - if (!erts_is_valid_tracer_port(ERTS_TRACER_PROC(tracee_port))) { - /* Current trace port now invalid - * - discard it and approve the new. */ - goto remove_tracer; - } else - return 1; - } - else if(is_internal_pid(ERTS_TRACER_PROC(tracee_port))) { - Process *tracer_p = erts_proc_lookup(ERTS_TRACER_PROC(tracee_port)); - if (!tracer_p) { - /* Current trace process now invalid - * - discard it and approve the new. */ - goto remove_tracer; - } else - return 1; - } - else { - remove_tracer: - ERTS_TRACE_FLAGS(tracee_port) &= ~TRACEE_FLAGS; - ERTS_TRACER_PROC(tracee_port) = NIL; - } - } - return 0; -} - -/* Check that the process to be traced is not already traced - * by a valid other tracer than the tracer to be. - */ -static int already_traced(Process *c_p, Process *tracee_p, Eterm tracer) -{ - /* - * SMP build assumes that either system is blocked or: - * * main lock is held on c_p - * * all locks multiple are held on tracee_p - */ - if ((ERTS_TRACE_FLAGS(tracee_p) & TRACEE_FLAGS) - && ERTS_TRACER_PROC(tracee_p) != tracer) { - /* This tracee is already being traced, and not by the - * tracer to be */ - if (is_internal_port(ERTS_TRACER_PROC(tracee_p))) { - if (!erts_is_valid_tracer_port(ERTS_TRACER_PROC(tracee_p))) { - /* Current trace port now invalid - * - discard it and approve the new. */ - goto remove_tracer; - } else - return 1; - } - else if(is_internal_pid(ERTS_TRACER_PROC(tracee_p))) { - Process *tracer_p; - - tracer_p = erts_proc_lookup(ERTS_TRACER_PROC(tracee_p)); - if (!tracer_p) { - /* Current trace process now invalid - * - discard it and approve the new. */ - goto remove_tracer; - } else - return 1; - } - else { - remove_tracer: - ERTS_TRACE_FLAGS(tracee_p) &= ~TRACEE_FLAGS; - ERTS_TRACER_PROC(tracee_p) = NIL; - } - } - return 0; -} - /* * Return information about a process or an external function being traced. */ @@ -920,7 +816,9 @@ Eterm trace_info_2(BIF_ALIST_2) if (What == am_on_load) { res = trace_info_on_load(p, Key); - } else if (is_atom(What) || is_pid(What)) { + } else if (What == am_send || What == am_receive) { + res = trace_info_event(p, What, Key); + } else if (is_atom(What) || is_pid(What) || is_port(What)) { res = trace_info_pid(p, What, Key); } else if (is_tuple(What)) { res = trace_info_func(p, What, Key); @@ -936,41 +834,52 @@ static Eterm trace_info_pid(Process* p, Eterm pid_spec, Eterm key) { Eterm tracer; - Uint trace_flags; + Uint trace_flags = am_false; Eterm* hp; - if (pid_spec == am_new) { - erts_get_default_tracing(&trace_flags, &tracer); + if (pid_spec == am_new || pid_spec == am_new_processes) { + ErtsTracer def_tracer; + erts_get_default_proc_tracing(&trace_flags, &def_tracer); + tracer = erts_tracer_to_term(p, def_tracer); + ERTS_TRACER_CLEAR(&def_tracer); + } else if (pid_spec == am_new_ports) { + ErtsTracer def_tracer; + erts_get_default_port_tracing(&trace_flags, &def_tracer); + tracer = erts_tracer_to_term(p, def_tracer); + ERTS_TRACER_CLEAR(&def_tracer); + } else if (is_internal_port(pid_spec)) { + Port *tracee; + tracee = erts_id2port_sflgs(pid_spec, p, ERTS_PROC_LOCK_MAIN, + ERTS_PORT_SFLGS_INVALID_LOOKUP); + + if (!tracee) + return am_undefined; + + if (!ERTS_TRACER_IS_NIL(ERTS_TRACER(tracee))) + erts_is_tracer_proc_enabled(NULL, 0, &tracee->common, am_trace_status); + + tracer = erts_tracer_to_term(p, ERTS_TRACER(tracee)); + trace_flags = ERTS_TRACE_FLAGS(tracee); + + erts_port_release(tracee); + } else if (is_internal_pid(pid_spec)) { Process *tracee; tracee = erts_pid2proc(p, ERTS_PROC_LOCK_MAIN, - pid_spec, ERTS_PROC_LOCKS_ALL); + pid_spec, ERTS_PROC_LOCK_MAIN); - if (!tracee) { + if (!tracee) return am_undefined; - } else { - tracer = ERTS_TRACER_PROC(tracee); - trace_flags = ERTS_TRACE_FLAGS(tracee); - } - if (is_internal_pid(tracer)) { - if (!erts_proc_lookup(tracer)) { - reset_tracer: - ERTS_TRACE_FLAGS(tracee) &= ~TRACEE_FLAGS; - trace_flags = ERTS_TRACE_FLAGS(tracee); - tracer = ERTS_TRACER_PROC(tracee) = NIL; - } - } - else if (is_internal_port(tracer)) { - if (!erts_is_valid_tracer_port(tracer)) - goto reset_tracer; - } -#ifdef ERTS_SMP - erts_smp_proc_unlock(tracee, - (tracee == p - ? ERTS_PROC_LOCKS_ALL_MINOR - : ERTS_PROC_LOCKS_ALL)); -#endif + if (!ERTS_TRACER_IS_NIL(ERTS_TRACER(tracee))) + erts_is_tracer_proc_enabled(tracee, ERTS_PROC_LOCK_MAIN, + &tracee->common, am_trace_status); + + tracer = erts_tracer_to_term(p, ERTS_TRACER(tracee)); + trace_flags = ERTS_TRACE_FLAGS(tracee); + + if (tracee != p) + erts_smp_proc_unlock(tracee, ERTS_PROC_LOCK_MAIN); } else if (is_external_pid(pid_spec) && external_pid_dist_entry(pid_spec) == erts_this_dist_entry) { return am_undefined; @@ -1024,8 +933,10 @@ trace_info_pid(Process* p, Eterm pid_spec, Eterm key) HRelease(p,limit,hp+3); return TUPLE2(hp, key, flag_list); } else if (key == am_tracer) { - hp = HAlloc(p, 3); - return TUPLE2(hp, key, tracer); /* Local pid or port */ + if (tracer == am_false) + tracer = NIL; + hp = HAlloc(p, 3); + return TUPLE2(hp, key, tracer); } else { goto error; } @@ -1054,11 +965,11 @@ trace_info_pid(Process* p, Eterm pid_spec, Eterm key) */ static int function_is_traced(Process *p, Eterm mfa[3], - Binary **ms, /* out */ - Binary **ms_meta, /* out */ - Eterm *tracer_pid_meta, /* out */ - Uint *count, /* out */ - Eterm *call_time) /* out */ + Binary **ms, /* out */ + Binary **ms_meta, /* out */ + ErtsTracer *tracer_pid_meta, /* out */ + Uint *count, /* out */ + Eterm *call_time) /* out */ { Export e; Export* ep; @@ -1123,7 +1034,7 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key) Eterm traced = am_false; Eterm match_spec = am_false; Eterm retval = am_false; - Eterm meta = am_false; + ErtsTracer meta = erts_tracer_nil; Eterm call_time = NIL; int r; @@ -1193,7 +1104,10 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key) retval = match_spec; break; case am_meta: - retval = meta; + retval = erts_tracer_to_term(p, meta); + if (retval == am_false) + /* backwards compatibility */ + retval = NIL; break; case am_meta_match_spec: if (r & FUNC_TRACE_META_TRACE) { @@ -1216,7 +1130,8 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key) } break; case am_all: { - Eterm match_spec_meta = am_false, c = am_false, t, ct = am_false; + Eterm match_spec_meta = am_false, c = am_false, t, ct = am_false, + m = am_false; if (ms) { match_spec = MatchSetGetSource(ms); @@ -1235,6 +1150,9 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key) if (r & FUNC_TRACE_TIME_TRACE) { ct = call_time; } + + m = erts_tracer_to_term(p, meta); + hp = HAlloc(p, (3+2)*6); retval = NIL; t = TUPLE2(hp, am_call_count, c); hp += 3; @@ -1243,7 +1161,7 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key) retval = CONS(hp, t, retval); hp += 2; t = TUPLE2(hp, am_meta_match_spec, match_spec_meta); hp += 3; retval = CONS(hp, t, retval); hp += 2; - t = TUPLE2(hp, am_meta, meta); hp += 3; + t = TUPLE2(hp, am_meta, m); hp += 3; retval = CONS(hp, t, retval); hp += 2; t = TUPLE2(hp, am_match_spec, match_spec); hp += 3; retval = CONS(hp, t, retval); hp += 2; @@ -1306,7 +1224,8 @@ trace_info_on_load(Process* p, Eterm key) case am_meta: hp = HAlloc(p, 3); if (erts_default_trace_pattern_flags.meta) { - return TUPLE2(hp, key, erts_default_meta_tracer_pid); + ASSERT(!ERTS_TRACER_IS_NIL(erts_default_meta_tracer)); + return TUPLE2(hp, key, erts_tracer_to_term(p, erts_default_meta_tracer)); } else { return TUPLE2(hp, key, am_false); } @@ -1345,7 +1264,7 @@ trace_info_on_load(Process* p, Eterm key) } case am_all: { - Eterm match_spec = am_false, meta_match_spec = am_false, r = NIL, t; + Eterm match_spec = am_false, meta_match_spec = am_false, r = NIL, t, m; if (erts_default_trace_pattern_flags.local || (! erts_default_trace_pattern_flags.breakpoint)) { @@ -1363,6 +1282,8 @@ trace_info_on_load(Process* p, Eterm key) MatchSetGetSource(erts_default_meta_match_spec); meta_match_spec = copy_object(meta_match_spec, p); } + m = (erts_default_trace_pattern_flags.meta + ? erts_tracer_to_term(p, erts_default_meta_tracer) : am_false); hp = HAlloc(p, (3+2)*5 + 3); t = TUPLE2(hp, am_call_count, (erts_default_trace_pattern_flags.call_count @@ -1370,9 +1291,7 @@ trace_info_on_load(Process* p, Eterm key) r = CONS(hp, t, r); hp += 2; t = TUPLE2(hp, am_meta_match_spec, meta_match_spec); hp += 3; r = CONS(hp, t, r); hp += 2; - t = TUPLE2(hp, am_meta, - (erts_default_trace_pattern_flags.meta - ? erts_default_meta_tracer_pid : am_false)); hp += 3; + t = TUPLE2(hp, am_meta, m); hp += 3; r = CONS(hp, t, r); hp += 2; t = TUPLE2(hp, am_match_spec, match_spec); hp += 3; r = CONS(hp, t, r); hp += 2; @@ -1388,6 +1307,42 @@ trace_info_on_load(Process* p, Eterm key) } } +static Eterm +trace_info_event(Process* p, Eterm event, Eterm key) +{ + ErtsTracingEvent* te; + Eterm retval; + Eterm* hp; + + switch (event) { + case am_send: te = erts_send_tracing; break; + case am_receive: te = erts_receive_tracing; break; + default: + goto error; + } + + if (key != am_match_spec) + goto error; + + te = &te[erts_active_bp_ix()]; + + if (te->on) { + if (!te->match_spec) + retval = am_true; + else + retval = copy_object(MatchSetGetSource(te->match_spec), p); + } + else + retval = am_false; + + hp = HAlloc(p, 3); + return TUPLE2(hp, key, retval); + + error: + BIF_ERROR(p, BADARG); +} + + #undef FUNC_TRACE_NOEXIST #undef FUNC_TRACE_UNTRACED #undef FUNC_TRACE_GLOBAL_TRACE @@ -1397,7 +1352,7 @@ int erts_set_trace_pattern(Process*p, Eterm* mfa, int specified, Binary* match_prog_set, Binary *meta_match_prog_set, int on, struct trace_pattern_flags flags, - Eterm meta_tracer_pid, int is_blocking) + ErtsTracer meta_tracer, int is_blocking) { const ErtsCodeIndex code_ix = erts_active_code_ix(); int matches = 0; @@ -1415,7 +1370,7 @@ erts_set_trace_pattern(Process*p, Eterm* mfa, int specified, for (i = 0; i < n; i++) { BeamInstr* pc = fp[i].pc; - Export* ep = (Export *)(((char *)(pc-3)) - offsetof(Export, code)); + Export* ep = ErtsContainerStruct(pc, Export, code[3]); if (on && !flags.breakpoint) { /* Turn on global call tracing */ @@ -1487,7 +1442,7 @@ erts_set_trace_pattern(Process*p, Eterm* mfa, int specified, } if (flags.meta) { erts_set_mtrace_bif(pc, meta_match_prog_set, - meta_tracer_pid); + meta_tracer); m = 1; } if (flags.call_time) { @@ -1527,7 +1482,7 @@ erts_set_trace_pattern(Process*p, Eterm* mfa, int specified, } if (flags.meta) { erts_set_mtrace_break(&finish_bp.f, meta_match_prog_set, - meta_tracer_pid); + meta_tracer); } if (flags.call_count) { erts_set_count_break(&finish_bp.f, on); @@ -1576,12 +1531,57 @@ erts_set_trace_pattern(Process*p, Eterm* mfa, int specified, } int +erts_set_tracing_event_pattern(Eterm event, Binary* match_spec, int on) +{ + ErtsBpIndex ix = erts_staging_bp_ix(); + ErtsTracingEvent* st; + + switch (event) { + case am_send: st = &erts_send_tracing[ix]; break; + case am_receive: st = &erts_receive_tracing[ix]; break; + default: return -1; + } + + MatchSetUnref(st->match_spec); + + st->on = on; + st->match_spec = match_spec; + MatchSetRef(match_spec); + + finish_bp.current = 1; /* prepare phase not needed for event trace */ + finish_bp.install = on; + finish_bp.e.matched = 0; + finish_bp.e.matching = NULL; + finish_bp.f.matched = 0; + finish_bp.f.matching = NULL; + +#ifndef ERTS_SMP + while (erts_finish_breakpointing()) { + /* Empty loop body */ + } +#endif + return 1; +} + +static void +consolidate_event_tracing(ErtsTracingEvent te[]) +{ + ErtsTracingEvent* src = &te[erts_active_bp_ix()]; + ErtsTracingEvent* dst = &te[erts_staging_bp_ix()]; + + MatchSetUnref(dst->match_spec); + dst->on = src->on; + dst->match_spec = src->match_spec; + MatchSetRef(dst->match_spec); +} + +int erts_finish_breakpointing(void) { ERTS_SMP_LC_ASSERT(erts_has_code_write_permission()); /* - * Memory barriers will be issued for all processes *before* + * Memory barriers will be issued for all schedulers *before* * each of the stages below. (Unless the other schedulers * are blocked, in which case memory barriers will be issued * when they are awaken.) @@ -1650,6 +1650,8 @@ erts_finish_breakpointing(void) erts_consolidate_bp_data(&finish_bp.f, 1); erts_bp_free_matched_functions(&finish_bp.e); erts_bp_free_matched_functions(&finish_bp.f); + consolidate_event_tracing(erts_send_tracing); + consolidate_event_tracing(erts_receive_tracing); return 0; default: ASSERT(0); @@ -1664,11 +1666,10 @@ install_exp_breakpoints(BpFunctions* f) BpFunction* fp = f->matching; Uint ne = f->matched; Uint i; - Uint offset = offsetof(Export, code) + 3*sizeof(BeamInstr); for (i = 0; i < ne; i++) { BeamInstr* pc = fp[i].pc; - Export* ep = (Export *) (((char *)pc)-offset); + Export* ep = ErtsContainerStruct(pc, Export, code[3]); ep->addressv[code_ix] = pc; } @@ -1681,11 +1682,10 @@ uninstall_exp_breakpoints(BpFunctions* f) BpFunction* fp = f->matching; Uint ne = f->matched; Uint i; - Uint offset = offsetof(Export, code) + 3*sizeof(BeamInstr); for (i = 0; i < ne; i++) { BeamInstr* pc = fp[i].pc; - Export* ep = (Export *) (((char *)pc)-offset); + Export* ep = ErtsContainerStruct(pc, Export, code[3]); if (ep->addressv[code_ix] != pc) { continue; @@ -1702,11 +1702,10 @@ clean_export_entries(BpFunctions* f) BpFunction* fp = f->matching; Uint ne = f->matched; Uint i; - Uint offset = offsetof(Export, code) + 3*sizeof(BeamInstr); for (i = 0; i < ne; i++) { BeamInstr* pc = fp[i].pc; - Export* ep = (Export *) (((char *)pc)-offset); + Export* ep = ErtsContainerStruct(pc, Export, code[3]); if (ep->addressv[code_ix] == pc) { continue; @@ -2336,50 +2335,86 @@ BIF_RETTYPE system_profile_2(BIF_ALIST_2) } /* End: Trace for System Profiling */ -BIF_RETTYPE -trace_delivered_1(BIF_ALIST_1) +/* Trace delivered send an aux work message to all schedulers + and when all schedulers have acknowledged that they have seen + the message the message is sent to the requesting process. + + IMPORTANT: We have to make sure that the all messages sent + using enif_send have been delivered before we send the message + to the caller. + + There used to be a separate implementation for when only a pid + is passed in, but since this is not performance critical code + we now use the same approach for both. +*/ + +typedef struct { + Process *proc; + Eterm ref; + Eterm ref_heap[REF_THING_SIZE]; + Eterm target; + erts_smp_atomic32_t refc; +} ErtsTraceDeliveredAll; + +static void +reply_trace_delivered_all(void *vtdarp) { - DECL_AM(trace_delivered); -#ifdef ERTS_SMP - ErlHeapFragment *bp; -#else - ErtsProcLocks locks = 0; -#endif - Eterm *hp; - Eterm msg, ref, msg_ref; - Process *p; - if (BIF_ARG_1 == am_all) { - p = NULL; - } else if (! (p = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN, - BIF_ARG_1, ERTS_PROC_LOCKS_ALL))) { - if (is_not_internal_pid(BIF_ARG_1)) { - BIF_ERROR(BIF_P, BADARG); - } - } - - ref = erts_make_ref(BIF_P); + ErtsTraceDeliveredAll *tdarp = (ErtsTraceDeliveredAll *) vtdarp; + if (erts_smp_atomic32_dec_read_nob(&tdarp->refc) == 0) { + Eterm ref_copy, msg; + Process *rp = tdarp->proc; + Eterm *hp = NULL; + ErlOffHeap *ohp; #ifdef ERTS_SMP - bp = new_message_buffer(REF_THING_SIZE + 4); - hp = &bp->mem[0]; - msg_ref = STORE_NC(&hp, &bp->off_heap, ref); + ErlHeapFragment *bp; + bp = new_message_buffer(4 + NC_HEAP_SIZE(tdarp->ref)); + hp = &bp->mem[0]; + ohp = &bp->off_heap; #else - hp = HAlloc(BIF_P, 4); - msg_ref = ref; + ErtsProcLocks rp_locks = 0; + ErtsMessage *mp; + mp = erts_alloc_message_heap( + rp, &rp_locks, 4 + NC_HEAP_SIZE(tdarp->ref), &hp, &ohp); #endif - msg = TUPLE3(hp, AM_trace_delivered, BIF_ARG_1, msg_ref); + ref_copy = STORE_NC(&hp, ohp, tdarp->ref); + msg = TUPLE3(hp, am_trace_delivered, tdarp->target, ref_copy); #ifdef ERTS_SMP - erts_send_sys_msg_proc(BIF_P->common.id, BIF_P->common.id, msg, bp); - if (p) - erts_smp_proc_unlock(p, - (BIF_P == p - ? ERTS_PROC_LOCKS_ALL_MINOR - : ERTS_PROC_LOCKS_ALL)); + erts_send_sys_msg_proc(rp->common.id, rp->common.id, msg, bp); #else - erts_send_message(BIF_P, BIF_P, &locks, msg, ERTS_SND_FLG_NO_SEQ_TRACE); + erts_queue_message(rp, rp_locks, mp, msg, am_system); #endif - BIF_RET(ref); + erts_free(ERTS_ALC_T_MISC_AUX_WORK, vtdarp); + erts_proc_dec_refc(rp); + } +} + +BIF_RETTYPE +trace_delivered_1(BIF_ALIST_1) +{ + + if (BIF_ARG_1 == am_all || is_internal_pid(BIF_ARG_1)) { + Eterm *hp, ref; + ErtsTraceDeliveredAll *tdarp = + erts_alloc(ERTS_ALC_T_MISC_AUX_WORK, sizeof(ErtsTraceDeliveredAll)); + + tdarp->proc = BIF_P; + ref = erts_make_ref(BIF_P); + hp = &tdarp->ref_heap[0]; + tdarp->ref = STORE_NC(&hp, NULL, ref); + tdarp->target = BIF_ARG_1; + erts_smp_atomic32_init_nob(&tdarp->refc, + (erts_aint32_t) erts_no_schedulers); + erts_proc_add_refc(BIF_P, 1); + erts_schedule_multi_misc_aux_work(0, + erts_no_schedulers, + reply_trace_delivered_all, + (void *) tdarp); + BIF_RET(ref); + } else { + BIF_ERROR(BIF_P, BADARG); + } } diff --git a/erts/emulator/beam/erl_bif_unique.c b/erts/emulator/beam/erl_bif_unique.c index c4a39b8897..1e57e9fa53 100644 --- a/erts/emulator/beam/erl_bif_unique.c +++ b/erts/emulator/beam/erl_bif_unique.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2014. All Rights Reserved. + * Copyright Ericsson AB 2014-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -266,17 +266,19 @@ static ERTS_INLINE Eterm unique_integer_bif(Process *c_p, int positive) } Uint -erts_raw_unique_integer_heap_size(Uint64 val[ERTS_UNIQUE_INT_RAW_VALUES]) +erts_raw_unique_integer_heap_size(Uint64 val[ERTS_UNIQUE_INT_RAW_VALUES], + int positive) { Uint sz; - bld_unique_integer_term(NULL, &sz, val[0], val[1], 0); + bld_unique_integer_term(NULL, &sz, val[0], val[1], positive); return sz; } Eterm -erts_raw_make_unique_integer(Eterm **hpp, Uint64 val[ERTS_UNIQUE_INT_RAW_VALUES]) +erts_raw_make_unique_integer(Eterm **hpp, Uint64 val[ERTS_UNIQUE_INT_RAW_VALUES], + int positive) { - return bld_unique_integer_term(hpp, NULL, val[0], val[1], 0); + return bld_unique_integer_term(hpp, NULL, val[0], val[1], positive); } void @@ -426,16 +428,16 @@ erts_raw_get_unique_monotonic_integer(void) } Uint -erts_raw_unique_monotonic_integer_heap_size(Sint64 raw) +erts_raw_unique_monotonic_integer_heap_size(Sint64 raw, int positive) { - return get_unique_monotonic_integer_heap_size(raw, 0); + return get_unique_monotonic_integer_heap_size(raw, positive); } Eterm -erts_raw_make_unique_monotonic_integer_value(Eterm **hpp, Sint64 raw) +erts_raw_make_unique_monotonic_integer_value(Eterm **hpp, Sint64 raw, int positive) { - Uint hsz = get_unique_monotonic_integer_heap_size(raw, 0); - Eterm res = make_unique_monotonic_integer_value(*hpp, hsz, raw, 0); + Uint hsz = get_unique_monotonic_integer_heap_size(raw, positive); + Eterm res = make_unique_monotonic_integer_value(*hpp, hsz, raw, positive); *hpp += hsz; return res; } diff --git a/erts/emulator/beam/erl_bif_unique.h b/erts/emulator/beam/erl_bif_unique.h index 37d5d91c39..c6481864d0 100644 --- a/erts/emulator/beam/erl_bif_unique.h +++ b/erts/emulator/beam/erl_bif_unique.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2014. All Rights Reserved. + * Copyright Ericsson AB 2014-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -41,8 +41,9 @@ void erts_make_ref_in_array(Uint32 ref[ERTS_MAX_REF_NUMBERS]); * not necessarily correspond to the end result. */ Sint64 erts_raw_get_unique_monotonic_integer(void); -Uint erts_raw_unique_monotonic_integer_heap_size(Sint64 raw); -Eterm erts_raw_make_unique_monotonic_integer_value(Eterm **hpp, Sint64 raw); +Uint erts_raw_unique_monotonic_integer_heap_size(Sint64 raw, int positive); +Eterm erts_raw_make_unique_monotonic_integer_value(Eterm **hpp, Sint64 raw, + int positive); Sint64 erts_get_min_unique_monotonic_integer(void); @@ -53,8 +54,11 @@ Eterm erts_debug_get_unique_monotonic_integer_state(Process *c_p); #define ERTS_UNIQUE_INT_RAW_VALUES 2 #define ERTS_MAX_UNIQUE_INT_HEAP_SIZE ERTS_UINT64_ARRAY_TO_BIG_MAX_HEAP_SZ(2) -Uint erts_raw_unique_integer_heap_size(Uint64 val[ERTS_UNIQUE_INT_RAW_VALUES]); -Eterm erts_raw_make_unique_integer(Eterm **hpp, Uint64 val[ERTS_UNIQUE_INT_RAW_VALUES]); +Uint erts_raw_unique_integer_heap_size(Uint64 val[ERTS_UNIQUE_INT_RAW_VALUES], + int positive); +Eterm erts_raw_make_unique_integer(Eterm **hpp, + Uint64 val[ERTS_UNIQUE_INT_RAW_VALUES], + int postive); void erts_raw_get_unique_integer(Uint64 val[ERTS_UNIQUE_INT_RAW_VALUES]); Sint64 erts_get_min_unique_integer(void); diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h index e181b5555d..fdc5efea98 100644 --- a/erts/emulator/beam/erl_binary.h +++ b/erts/emulator/beam/erl_binary.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2013. All Rights Reserved. + * Copyright Ericsson AB 2000-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c index 11d83686a3..6bf52fb303 100644 --- a/erts/emulator/beam/erl_bits.c +++ b/erts/emulator/beam/erl_bits.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2013. All Rights Reserved. + * Copyright Ericsson AB 1999-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_bits.h b/erts/emulator/beam/erl_bits.h index 8b7807fbd9..1c2a090f07 100644 --- a/erts/emulator/beam/erl_bits.h +++ b/erts/emulator/beam/erl_bits.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2011. All Rights Reserved. + * Copyright Ericsson AB 1999-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_cpu_topology.c b/erts/emulator/beam/erl_cpu_topology.c index c263eebfcc..28aaeeb479 100644 --- a/erts/emulator/beam/erl_cpu_topology.c +++ b/erts/emulator/beam/erl_cpu_topology.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2010-2013. All Rights Reserved. + * Copyright Ericsson AB 2010-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_cpu_topology.h b/erts/emulator/beam/erl_cpu_topology.h index f3fbfc6da0..45324ac4a0 100644 --- a/erts/emulator/beam/erl_cpu_topology.h +++ b/erts/emulator/beam/erl_cpu_topology.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2010-2013. All Rights Reserved. + * Copyright Ericsson AB 2010-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 441dbdde4f..acaca54e9a 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2014. All Rights Reserved. + * Copyright Ericsson AB 1996-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -2833,7 +2833,8 @@ BIF_RETTYPE ets_match_spec_run_r_3(BIF_ALIST_3) BIF_TRAP3(bif_export[BIF_ets_match_spec_run_r_3], BIF_P,lst,BIF_ARG_2,ret); } - res = db_prog_match(BIF_P, mp, CAR(list_val(lst)), NULL, 0, + res = db_prog_match(BIF_P, BIF_P, + mp, CAR(list_val(lst)), NULL, 0, ERTS_PAM_COPY_RESULT, &dummy); if (is_value(res)) { hp = HAlloc(BIF_P, 2); diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h index a589af784c..1d26c49652 100644 --- a/erts/emulator/beam/erl_db.h +++ b/erts/emulator/beam/erl_db.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2013. All Rights Reserved. + * Copyright Ericsson AB 1996-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 240b79fc60..74979f984a 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2012. All Rights Reserved. + * Copyright Ericsson AB 1998-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h index 66d8ec71d9..e654363cd5 100644 --- a/erts/emulator/beam/erl_db_hash.h +++ b/erts/emulator/beam/erl_db_hash.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2014. All Rights Reserved. + * Copyright Ericsson AB 1998-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index 0f642f0867..02d211a4bb 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2013. All Rights Reserved. + * Copyright Ericsson AB 1998-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_db_tree.h b/erts/emulator/beam/erl_db_tree.h index 6098387f5d..72749ead1e 100644 --- a/erts/emulator/beam/erl_db_tree.h +++ b/erts/emulator/beam/erl_db_tree.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2009. All Rights Reserved. + * Copyright Ericsson AB 1998-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index 973be58420..95b1cd0148 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2014. All Rights Reserved. + * Copyright Ericsson AB 1998-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -123,6 +123,9 @@ do { \ #define TermWords(t) (((t) / (sizeof(UWord)/sizeof(Eterm))) + !!((t) % (sizeof(UWord)/sizeof(Eterm)))) +#define add_dmc_err(EINFO, STR, VAR, TERM, SEV) \ + vadd_dmc_err(EINFO, SEV, VAR, STR, TERM) + static ERTS_INLINE Process * get_proc(Process *cp, Uint32 cp_locks, Eterm id, Uint32 id_locks) @@ -135,21 +138,22 @@ get_proc(Process *cp, Uint32 cp_locks, Eterm id, Uint32 id_locks) static Eterm -set_tracee_flags(Process *tracee_p, Eterm tracer, Uint d_flags, Uint e_flags) { +set_tracee_flags(Process *tracee_p, ErtsTracer tracer, + Uint d_flags, Uint e_flags) { Eterm ret; Uint flags; - if (tracer == NIL) { + if (ERTS_TRACER_IS_NIL(tracer)) { flags = ERTS_TRACE_FLAGS(tracee_p) & ~TRACEE_FLAGS; } else { flags = ((ERTS_TRACE_FLAGS(tracee_p) & ~d_flags) | e_flags); - if (! flags) tracer = NIL; + if (! flags) tracer = erts_tracer_nil; } - ret = ((ERTS_TRACER_PROC(tracee_p) != tracer + ret = ((!ERTS_TRACER_COMPARE(ERTS_TRACER(tracee_p),tracer) || ERTS_TRACE_FLAGS(tracee_p) != flags) ? am_true : am_false); - ERTS_TRACER_PROC(tracee_p) = tracer; + erts_tracer_replace(&tracee_p->common, tracer); ERTS_TRACE_FLAGS(tracee_p) = flags; return ret; } @@ -163,40 +167,16 @@ set_tracee_flags(Process *tracee_p, Eterm tracer, Uint d_flags, Uint e_flags) { ** returns fail_term on failure. Fails if tracer pid or port is invalid. */ static Eterm -set_match_trace(Process *tracee_p, Eterm fail_term, Eterm tracer, +set_match_trace(Process *tracee_p, Eterm fail_term, ErtsTracer tracer, Uint d_flags, Uint e_flags) { - Eterm ret = fail_term; - Process *tracer_p; - - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCKS_ALL == - erts_proc_lc_my_proc_locks(tracee_p)); - - if (is_internal_pid(tracer) - && (tracer_p = - erts_pid2proc(tracee_p, ERTS_PROC_LOCKS_ALL, - tracer, ERTS_PROC_LOCKS_ALL))) { - if (tracee_p != tracer_p) { - ret = set_tracee_flags(tracee_p, tracer, d_flags, e_flags); - ERTS_TRACE_FLAGS(tracer_p) |= (ERTS_TRACE_FLAGS(tracee_p) - ? F_TRACER - : 0); - erts_smp_proc_unlock(tracer_p, ERTS_PROC_LOCKS_ALL); - } - } else if (is_internal_port(tracer)) { - Port *tracer_port = - erts_id2port_sflgs(tracer, - tracee_p, - ERTS_PROC_LOCKS_ALL, - ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP); - if (tracer_port) { - ret = set_tracee_flags(tracee_p, tracer, d_flags, e_flags); - erts_port_release(tracer_port); - } - } else { - ASSERT(is_nil(tracer)); - ret = set_tracee_flags(tracee_p, tracer, d_flags, e_flags); - } - return ret; + + ERTS_SMP_LC_ASSERT( + ERTS_PROC_LOCKS_ALL == erts_proc_lc_my_proc_locks(tracee_p) + || erts_thr_progress_is_blocking()); + + if (ERTS_TRACER_IS_NIL(tracer) || erts_is_tracer_enabled(tracee_p, tracer)) + return set_tracee_flags(tracee_p, tracer, d_flags, e_flags); + return fail_term; } /* @@ -434,17 +414,27 @@ get_match_pseudo_process(Process *c_p, Uint heap_size) { ErtsMatchPseudoProcess *mpsp; #ifdef ERTS_SMP - mpsp = (ErtsMatchPseudoProcess *) c_p->scheduler_data->match_pseudo_process; - if (mpsp) + ErtsSchedulerData *esdp; + + esdp = c_p ? c_p->scheduler_data : erts_get_scheduler_data(); + + mpsp = esdp ? esdp->match_pseudo_process : + (ErtsMatchPseudoProcess*) erts_smp_tsd_get(match_pseudo_process_key); + + if (mpsp) { + ASSERT(mpsp == erts_smp_tsd_get(match_pseudo_process_key)); + ASSERT(mpsp->process.scheduler_data == esdp); cleanup_match_pseudo_process(mpsp, 0); + } else { ASSERT(erts_smp_tsd_get(match_pseudo_process_key) == NULL); mpsp = create_match_pseudo_process(); - c_p->scheduler_data->match_pseudo_process = (void *) mpsp; + if (esdp) { + esdp->match_pseudo_process = (void *) mpsp; + } + mpsp->process.scheduler_data = esdp; erts_smp_tsd_set(match_pseudo_process_key, (void *) mpsp); } - ASSERT(mpsp == erts_smp_tsd_get(match_pseudo_process_key)); - mpsp->process.scheduler_data = c_p->scheduler_data; #else mpsp = match_pseudo_process; cleanup_match_pseudo_process(mpsp, 0); @@ -912,11 +902,7 @@ void db_match_dis(Binary *prog); #define TRACE /* Nothing */ #define FENCE_PATTERN_SIZE 0 #endif -static void add_dmc_err(DMCErrInfo *err_info, - char *str, - int variable, - Eterm term, - DMCErrorSeverity severity); +static void vadd_dmc_err(DMCErrInfo*, DMCErrorSeverity, int var, const char *str, ...); static Eterm dpm_array_to_list(Process *psp, Eterm *arr, int arity); @@ -1012,12 +998,20 @@ Eterm erts_match_set_get_source(Binary *mpsp) } /* This one is for the tracing */ -Binary *erts_match_set_compile(Process *p, Eterm matchexpr) { +Binary *erts_match_set_compile(Process *p, Eterm matchexpr, Eterm MFA) { Binary *bin; Uint sz; Eterm *hp; + Uint flags; + + switch (MFA) { + case am_receive: flags = DCOMP_TRACE; break; + case am_send: flags = DCOMP_TRACE | DCOMP_ALLOW_TRACE_OPS; break; + default: + flags = DCOMP_TRACE | DCOMP_CALL_TRACE | DCOMP_ALLOW_TRACE_OPS; + } - bin = db_match_set_compile(p, matchexpr, DCOMP_TRACE); + bin = db_match_set_compile(p, matchexpr, flags); if (bin != NULL) { MatchProg *prog = Binary2MatchProg(bin); sz = size_object(matchexpr); @@ -1147,8 +1141,8 @@ Eterm db_match_set_lint(Process *p, Eterm matchexpr, Uint flags) int i; if (!is_list(matchexpr)) { - add_dmc_err(err_info, "Match programs are not in a list.", - -1, 0UL, dmcError); + add_dmc_err(err_info, "Match programs are not in a list.", + -1, 0UL, dmcError); goto done; } num_heads = 0; @@ -1156,9 +1150,8 @@ Eterm db_match_set_lint(Process *p, Eterm matchexpr, Uint flags) ++num_heads; if (l != NIL) { /* proper list... */ - add_dmc_err(err_info, "Match programs are not in a proper " - "list.", - -1, 0UL, dmcError); + add_dmc_err(err_info, "Match programs are not in a proper list.", + -1, 0UL, dmcError); goto done; } @@ -1225,30 +1218,37 @@ done: return ret; } -Eterm erts_match_set_run(Process *p, Binary *mpsp, - Eterm *args, int num_args, - enum erts_pam_run_flags in_flags, - Uint32 *return_flags) +/* Returns + * am_false if no match or + * if {message,false} has been called, + * am_true if {message,_} has NOT been called or + * if {message,true} has been called, + * Msg if {message,Msg} has been called. + * + * If return value is_not_immed + * then erts_match_set_release_result_trace() must be called to release it. + */ +Eterm erts_match_set_run_trace(Process *c_p, + Process *self, + Binary *mpsp, + Eterm *args, int num_args, + enum erts_pam_run_flags in_flags, + Uint32 *return_flags) { Eterm ret; - ret = db_prog_match(p, mpsp, NIL, args, num_args, + ret = db_prog_match(c_p, self, mpsp, NIL, args, num_args, in_flags, return_flags); -#if defined(HARDDEBUG) - if (is_non_value(ret)) { - erts_fprintf(stderr, "Failed\n"); - } else { - erts_fprintf(stderr, "Returning : %T\n", ret); + + ASSERT(!(is_non_value(ret) && *return_flags)); + + if (is_non_value(ret) || ret == am_false) { + erts_match_set_release_result(c_p); + return am_false; } -#endif + if (is_immed(ret)) + erts_match_set_release_result(c_p); return ret; - /* Returns - * THE_NON_VALUE if no match - * am_false if {message,false} has been called, - * am_true if {message,_} has not been called or - * if {message,true} has been called, - * Msg if {message,Msg} has been called. - */ } static Eterm erts_match_set_run_ets(Process *p, Binary *mpsp, @@ -1257,7 +1257,8 @@ static Eterm erts_match_set_run_ets(Process *p, Binary *mpsp, { Eterm ret; - ret = db_prog_match(p, mpsp, args, NULL, num_args, + ret = db_prog_match(p, p, + mpsp, args, NULL, num_args, ERTS_PAM_COPY_RESULT, return_flags); #if defined(HARDDEBUG) @@ -1753,7 +1754,9 @@ static Eterm dpm_array_to_list(Process *psp, Eterm *arr, int arity) ** the parameter 'arity' is only used if 'term' is actually an array, ** i.e. 'DCOMP_TRACE' was specified */ -Eterm db_prog_match(Process *c_p, Binary *bprog, +Eterm db_prog_match(Process *c_p, + Process *self, + Binary *bprog, Eterm term, Eterm *termp, int arity, @@ -1766,10 +1769,10 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm *esp; MatchVariable* variables; BeamInstr *cp; - UWord *pc = prog->text; + const UWord *pc = prog->text; Eterm *ehp; Eterm ret; - Uint n = 0; /* To avoid warning. */ + Uint n; int i; unsigned do_catch; ErtsMatchPseudoProcess *mpsp; @@ -1781,46 +1784,28 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm (*bif)(Process*, ...); Eterm bif_args[3]; int fail_label; - int atomic_trace; #ifdef DMC_DEBUG Uint *heap_fence; Uint *stack_fence; Uint save_op; #endif /* DMC_DEBUG */ + ERTS_UNDEF(n,0); + ERTS_UNDEF(current_scheduled,NULL); + + ASSERT(c_p || !(in_flags & ERTS_PAM_COPY_RESULT)); + mpsp = get_match_pseudo_process(c_p, prog->heap_size); psp = &mpsp->process; /* We need to lure the scheduler into believing in the pseudo process, because of floating point exceptions. Do *after* mpsp is set!!! */ - esdp = ERTS_GET_SCHEDULER_DATA_FROM_PROC(c_p); - ASSERT(esdp != NULL); - current_scheduled = esdp->current_process; + esdp = ERTS_GET_SCHEDULER_DATA_FROM_PROC(psp); + if (esdp) + current_scheduled = esdp->current_process; /* SMP: psp->scheduler_data is set by get_match_pseudo_process */ - atomic_trace = 0; -#define BEGIN_ATOMIC_TRACE(p) \ - do { \ - if (! atomic_trace) { \ - erts_refc_inc(&bprog->refc, 2); \ - erts_smp_proc_unlock((p), ERTS_PROC_LOCK_MAIN); \ - erts_smp_thr_progress_block(); \ - atomic_trace = !0; \ - } \ - } while (0) -#define END_ATOMIC_TRACE(p) \ - do { \ - if (atomic_trace) { \ - erts_smp_thr_progress_unblock(); \ - erts_smp_proc_lock((p), ERTS_PROC_LOCK_MAIN); \ - if (erts_refc_dectest(&bprog->refc, 0) == 0) {\ - erts_bin_free(bprog); \ - } \ - atomic_trace = 0; \ - } \ - } while (0) - #ifdef DMC_DEBUG save_op = 0; heap_fence = (Eterm*)((char*) mpsp->u.heap + prog->stack_offset) - 1; @@ -2279,7 +2264,7 @@ restart: pc += n; break; case matchSelf: - *esp++ = c_p->common.id; + *esp++ = self->common.id; break; case matchWaste: --esp; @@ -2289,6 +2274,7 @@ restart: break; case matchProcessDump: { erts_dsprintf_buf_t *dsbufp = erts_create_tmp_dsbuf(0); + ASSERT(c_p == self); print_process_info(ERTS_PRINT_DSBUF, (void *) dsbufp, c_p); *esp++ = new_binary(build_proc, (byte *)dsbufp->str, dsbufp->str_len); @@ -2307,14 +2293,16 @@ restart: *return_flags |= MATCH_SET_EXCEPTION_TRACE; *esp++ = am_true; break; - case matchIsSeqTrace: + case matchIsSeqTrace: + ASSERT(c_p == self); if (have_seqtrace(SEQ_TRACE_TOKEN(c_p))) *esp++ = am_true; else *esp++ = am_false; break; case matchSetSeqToken: - t = erts_seq_trace(c_p, esp[-1], esp[-2], 0); + ASSERT(c_p == self); + t = erts_seq_trace(c_p, esp[-1], esp[-2], 0); if (is_non_value(t)) { esp[-2] = FAIL_TERM; } else { @@ -2322,7 +2310,8 @@ restart: } --esp; break; - case matchSetSeqTokenFake: + case matchSetSeqTokenFake: + ASSERT(c_p == self); t = seq_trace_fake(c_p, esp[-1]); if (is_non_value(t)) { esp[-2] = FAIL_TERM; @@ -2331,7 +2320,8 @@ restart: } --esp; break; - case matchGetSeqToken: + case matchGetSeqToken: + ASSERT(c_p == self); if (have_no_seqtrace(SEQ_TRACE_TOKEN(c_p))) *esp++ = NIL; else { @@ -2355,49 +2345,62 @@ restart: ASSERT(is_immed(ehp[5])); } break; - case matchEnableTrace: + case matchEnableTrace: + ASSERT(c_p == self); if ( (n = erts_trace_flag2bit(esp[-1]))) { - BEGIN_ATOMIC_TRACE(c_p); - set_tracee_flags(c_p, ERTS_TRACER_PROC(c_p), 0, n); + erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); + set_tracee_flags(c_p, ERTS_TRACER(c_p), 0, n); + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); esp[-1] = am_true; } else { esp[-1] = FAIL_TERM; } break; - case matchEnableTrace2: + case matchEnableTrace2: + ASSERT(c_p == self); n = erts_trace_flag2bit((--esp)[-1]); esp[-1] = FAIL_TERM; if (n) { - BEGIN_ATOMIC_TRACE(c_p); - if ( (tmpp = get_proc(c_p, 0, esp[0], 0))) { + if ( (tmpp = get_proc(c_p, ERTS_PROC_LOCK_MAIN, esp[0], ERTS_PROC_LOCKS_ALL))) { /* Always take over the tracer of the current process */ - set_tracee_flags(tmpp, ERTS_TRACER_PROC(c_p), 0, n); - esp[-1] = am_true; + set_tracee_flags(tmpp, ERTS_TRACER(c_p), 0, n); + if (tmpp == c_p) + erts_smp_proc_unlock(tmpp, ERTS_PROC_LOCKS_ALL_MINOR); + else + erts_smp_proc_unlock(tmpp, ERTS_PROC_LOCKS_ALL); + esp[-1] = am_true; } } break; - case matchDisableTrace: + case matchDisableTrace: + ASSERT(c_p == self); if ( (n = erts_trace_flag2bit(esp[-1]))) { - BEGIN_ATOMIC_TRACE(c_p); - set_tracee_flags(c_p, ERTS_TRACER_PROC(c_p), n, 0); + erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); + set_tracee_flags(c_p, ERTS_TRACER(c_p), n, 0); + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); esp[-1] = am_true; } else { esp[-1] = FAIL_TERM; } break; - case matchDisableTrace2: + case matchDisableTrace2: + ASSERT(c_p == self); n = erts_trace_flag2bit((--esp)[-1]); esp[-1] = FAIL_TERM; if (n) { - BEGIN_ATOMIC_TRACE(c_p); - if ( (tmpp = get_proc(c_p, 0, esp[0], 0))) { + if ( (tmpp = get_proc(c_p, ERTS_PROC_LOCK_MAIN, esp[0], ERTS_PROC_LOCKS_ALL))) { /* Always take over the tracer of the current process */ - set_tracee_flags(tmpp, ERTS_TRACER_PROC(c_p), n, 0); - esp[-1] = am_true; + set_tracee_flags(tmpp, ERTS_TRACER(c_p), n, 0); + if (tmpp == c_p) + erts_smp_proc_unlock(tmpp, ERTS_PROC_LOCKS_ALL_MINOR); + else + erts_smp_proc_unlock(tmpp, ERTS_PROC_LOCKS_ALL); + esp[-1] = am_true; } } break; - case matchCaller: + case matchCaller: + ASSERT(c_p == self); if (!(c_p->cp) || !(cp = find_function_from_pc(c_p->cp))) { *esp++ = am_undefined; } else { @@ -2409,7 +2412,8 @@ restart: ehp[3] = make_small((Uint) cp[2]); } break; - case matchSilent: + case matchSilent: + ASSERT(c_p == self); --esp; if (in_flags & ERTS_PAM_IGNORE_TRACE_SILENT) break; @@ -2424,11 +2428,12 @@ restart: erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); } break; - case matchTrace2: + case matchTrace2: + ASSERT(c_p == self); { /* disable enable */ Uint d_flags = 0, e_flags = 0; /* process trace flags */ - Eterm tracer = ERTS_TRACER_PROC(c_p); + ErtsTracer tracer = erts_tracer_nil; /* XXX Atomicity note: Not fully atomic. Default tracer * is sampled from current process but applied to * tracee and tracer later after releasing main @@ -2440,29 +2445,35 @@ restart: * {trace,[],[{{tracer,Tracer}}]} is much, much older. */ int cputs = 0; + erts_tracer_update(&tracer, ERTS_TRACER(c_p)); if (! erts_trace_flags(esp[-1], &d_flags, &tracer, &cputs) || ! erts_trace_flags(esp[-2], &e_flags, &tracer, &cputs) || cputs ) { (--esp)[-1] = FAIL_TERM; + ERTS_TRACER_CLEAR(&tracer); break; } erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); (--esp)[-1] = set_match_trace(c_p, FAIL_TERM, tracer, d_flags, e_flags); erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); + ERTS_TRACER_CLEAR(&tracer); } break; - case matchTrace3: + case matchTrace3: + ASSERT(c_p == self); { /* disable enable */ Uint d_flags = 0, e_flags = 0; /* process trace flags */ - Eterm tracer = ERTS_TRACER_PROC(c_p); + ErtsTracer tracer = erts_tracer_nil; /* XXX Atomicity note. Not fully atomic. See above. * Above it could possibly be solved, but not here. */ int cputs = 0; Eterm tracee = (--esp)[0]; + + erts_tracer_update(&tracer, ERTS_TRACER(c_p)); if (! erts_trace_flags(esp[-1], &d_flags, &tracer, &cputs) || ! erts_trace_flags(esp[-2], &e_flags, &tracer, &cputs) || @@ -2470,6 +2481,7 @@ restart: ! (tmpp = get_proc(c_p, ERTS_PROC_LOCK_MAIN, tracee, ERTS_PROC_LOCKS_ALL))) { (--esp)[-1] = FAIL_TERM; + ERTS_TRACER_CLEAR(&tracer); break; } if (tmpp == c_p) { @@ -2483,6 +2495,7 @@ restart: erts_smp_proc_unlock(tmpp, ERTS_PROC_LOCKS_ALL); erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); } + ERTS_TRACER_CLEAR(&tracer); } break; case matchCatch: /* Match success, now build result */ @@ -2522,15 +2535,12 @@ success: } #endif - esdp->current_process = current_scheduled; - - END_ATOMIC_TRACE(c_p); + if (esdp) + esdp->current_process = current_scheduled; return ret; #undef FAIL #undef FAIL_TERM -#undef BEGIN_ATOMIC_TRACE -#undef END_ATOMIC_TRACE } @@ -3275,20 +3285,20 @@ int erts_db_is_compiled_ms(Eterm term) ** Utility to add an error */ -static void add_dmc_err(DMCErrInfo *err_info, - char *str, - int variable, - Eterm term, - DMCErrorSeverity severity) +static void vadd_dmc_err(DMCErrInfo *err_info, + DMCErrorSeverity severity, + int variable, + const char *str, + ...) { + DMCError *e; + va_list args; + va_start(args, str); + + /* Linked in in reverse order, to ease the formatting */ - DMCError *e = erts_alloc(ERTS_ALC_T_DB_DMC_ERROR, sizeof(DMCError)); - if (term != 0UL) { - erts_snprintf(e->error_string, DMC_ERR_STR_LEN, str, term); - } else { - strncpy(e->error_string, str, DMC_ERR_STR_LEN); - e->error_string[DMC_ERR_STR_LEN] ='\0'; - } + e = erts_alloc(ERTS_ALC_T_DB_DMC_ERROR, sizeof(DMCError)); + erts_vsnprintf(e->error_string, DMC_ERR_STR_LEN, str, args); e->variable = variable; e->severity = severity; e->next = err_info->first; @@ -3298,8 +3308,11 @@ static void add_dmc_err(DMCErrInfo *err_info, err_info->first = e; if (severity >= dmcError) err_info->error_added = 1; + + va_end(args); } + /* ** Handle one term in the match expression (not the guard) */ @@ -3498,24 +3511,21 @@ static void do_emit_constant(DMCContext *context, DMC_STACK_TYPE(UWord) *text, context->stack_need = context->stack_used; } -#define RETURN_ERROR_X(String, X, Y, ContextP, ConstantF) \ -do { \ -if ((ContextP)->err_info != NULL) { \ - (ConstantF) = 0; \ - add_dmc_err((ContextP)->err_info, String, X, Y, dmcError); \ - return retOk; \ -} else \ - return retFail; \ -} while(0) +#define RETURN_ERROR_X(VAR, ContextP, ConstantF, String, ARG) \ + (((ContextP)->err_info != NULL) \ + ? ((ConstantF) = 0, \ + vadd_dmc_err((ContextP)->err_info, dmcError, VAR, String, ARG), \ + retOk) \ + : retFail) #define RETURN_ERROR(String, ContextP, ConstantF) \ - RETURN_ERROR_X(String, -1, 0UL, ContextP, ConstantF) + return RETURN_ERROR_X(-1, ContextP, ConstantF, String, 0) #define RETURN_VAR_ERROR(String, N, ContextP, ConstantF) \ - RETURN_ERROR_X(String, N, 0UL, ContextP, ConstantF) + return RETURN_ERROR_X(N, ContextP, ConstantF, String, 0) #define RETURN_TERM_ERROR(String, T, ContextP, ConstantF) \ - RETURN_ERROR_X(String, -1, T, ContextP, ConstantF) + return RETURN_ERROR_X(-1, ContextP, ConstantF, String, T) #define WARNING(String, ContextP) \ add_dmc_err((ContextP)->err_info, String, -1, 0UL, dmcWarning) @@ -3781,7 +3791,7 @@ static DMCRet dmc_variable(DMCContext *context, Uint n = db_is_variable(t); if (n >= heap->vars_used || !heap->vars[n].is_bound) { - RETURN_VAR_ERROR("Variable $%d is unbound.", n, context, *constant); + RETURN_VAR_ERROR("Variable $%%d is unbound.", n, context, *constant); } dmc_add_pushv_variant(context, heap, text, n); @@ -4113,7 +4123,30 @@ static DMCRet dmc_exception_trace(DMCContext *context, return retOk; } - +static int check_trace(const char* op, + DMCContext *context, + int *constant, + int need_cflags, + int allow_in_guard, + DMCRet* retp) +{ + if (!(context->cflags & DCOMP_TRACE)) { + *retp = RETURN_ERROR_X(-1, context, *constant, "Special form '%s' " + "used in wrong dialect.", op); + return 0; + } + if ((context->cflags & need_cflags) != need_cflags) { + *retp = RETURN_ERROR_X(-1, context, *constant, "Special form '%s' " + "not allow for this trace event.", op); + return 0; + } + if (context->is_guard && !allow_in_guard) { + *retp = RETURN_ERROR_X(-1, context, *constant, "Special form '%s' " + "called in guard context.", op); + return 0; + } + return 1; +} static DMCRet dmc_is_seq_trace(DMCContext *context, DMCHeap *heap, @@ -4123,12 +4156,11 @@ static DMCRet dmc_is_seq_trace(DMCContext *context, { Eterm *p = tuple_val(t); Uint a = arityval(*p); + DMCRet ret; - if (!(context->cflags & DCOMP_TRACE)) { - RETURN_ERROR("Special form 'is_seq_trace' used in wrong dialect.", - context, - *constant); - } + if (!check_trace("is_seq_trace", context, constant, DCOMP_ALLOW_TRACE_OPS, 1, &ret)) + return ret; + if (a != 1) { RETURN_TERM_ERROR("Special form 'is_seq_trace' called with " "arguments in %T.", t, context, *constant); @@ -4152,16 +4184,8 @@ static DMCRet dmc_set_seq_token(DMCContext *context, DMCRet ret; int c; - - if (!(context->cflags & DCOMP_TRACE)) { - RETURN_ERROR("Special form 'set_seq_token' used in wrong dialect.", - context, - *constant); - } - if (context->is_guard) { - RETURN_ERROR("Special form 'set_seq_token' called in " - "guard context.", context, *constant); - } + if (!check_trace("set_seq_trace", context, constant, DCOMP_ALLOW_TRACE_OPS, 0, &ret)) + return ret; if (a != 3) { RETURN_TERM_ERROR("Special form 'set_seq_token' called with wrong " @@ -4198,16 +4222,11 @@ static DMCRet dmc_get_seq_token(DMCContext *context, { Eterm *p = tuple_val(t); Uint a = arityval(*p); + DMCRet ret; + + if (!check_trace("get_seq_token", context, constant, DCOMP_ALLOW_TRACE_OPS, 0, &ret)) + return ret; - if (!(context->cflags & DCOMP_TRACE)) { - RETURN_ERROR("Special form 'get_seq_token' used in wrong dialect.", - context, - *constant); - } - if (context->is_guard) { - RETURN_ERROR("Special form 'get_seq_token' called in " - "guard context.", context, *constant); - } if (a != 1) { RETURN_TERM_ERROR("Special form 'get_seq_token' called with " "arguments in %T.", t, context, @@ -4271,16 +4290,10 @@ static DMCRet dmc_process_dump(DMCContext *context, { Eterm *p = tuple_val(t); Uint a = arityval(*p); - - if (!(context->cflags & DCOMP_TRACE)) { - RETURN_ERROR("Special form 'process_dump' used in wrong dialect.", - context, - *constant); - } - if (context->is_guard) { - RETURN_ERROR("Special form 'process_dump' called in " - "guard context.", context, *constant); - } + DMCRet ret; + + if (!check_trace("process_dump", context, constant, DCOMP_ALLOW_TRACE_OPS, 0, &ret)) + return ret; if (a != 1) { RETURN_TERM_ERROR("Special form 'process_dump' called with " @@ -4304,17 +4317,8 @@ static DMCRet dmc_enable_trace(DMCContext *context, DMCRet ret; int c; - - if (!(context->cflags & DCOMP_TRACE)) { - RETURN_ERROR("Special form 'enable_trace' used in wrong dialect.", - context, - *constant); - } - if (context->is_guard) { - RETURN_ERROR("Special form 'enable_trace' called in guard context.", - context, - *constant); - } + if (!check_trace("enable_trace", context, constant, DCOMP_ALLOW_TRACE_OPS, 0, &ret)) + return ret; switch (a) { case 2: @@ -4363,18 +4367,9 @@ static DMCRet dmc_disable_trace(DMCContext *context, Uint a = arityval(*p); DMCRet ret; int c; - - if (!(context->cflags & DCOMP_TRACE)) { - RETURN_ERROR("Special form 'disable_trace' used in wrong dialect.", - context, - *constant); - } - if (context->is_guard) { - RETURN_ERROR("Special form 'disable_trace' called in guard context.", - context, - *constant); - } + if (!check_trace("disable_trace", context, constant, DCOMP_ALLOW_TRACE_OPS, 0, &ret)) + return ret; switch (a) { case 2: @@ -4424,17 +4419,8 @@ static DMCRet dmc_trace(DMCContext *context, DMCRet ret; int c; - - if (!(context->cflags & DCOMP_TRACE)) { - RETURN_ERROR("Special form 'trace' used in wrong dialect.", - context, - *constant); - } - if (context->is_guard) { - RETURN_ERROR("Special form 'trace' called in guard context.", - context, - *constant); - } + if (!check_trace("trace", context, constant, DCOMP_ALLOW_TRACE_OPS, 0, &ret)) + return ret; switch (a) { case 3: @@ -4495,16 +4481,11 @@ static DMCRet dmc_caller(DMCContext *context, { Eterm *p = tuple_val(t); Uint a = arityval(*p); + DMCRet ret; - if (!(context->cflags & DCOMP_TRACE)) { - RETURN_ERROR("Special form 'caller' used in wrong dialect.", - context, - *constant); - } - if (context->is_guard) { - RETURN_ERROR("Special form 'caller' called in " - "guard context.", context, *constant); - } + if (!check_trace("caller", context, constant, + (DCOMP_CALL_TRACE|DCOMP_ALLOW_TRACE_OPS), 0, &ret)) + return ret; if (a != 1) { RETURN_TERM_ERROR("Special form 'caller' called with " @@ -4530,15 +4511,8 @@ static DMCRet dmc_silent(DMCContext *context, DMCRet ret; int c; - if (!(context->cflags & DCOMP_TRACE)) { - RETURN_ERROR("Special form 'silent' used in wrong dialect.", - context, - *constant); - } - if (context->is_guard) { - RETURN_ERROR("Special form 'silent' called in " - "guard context.", context, *constant); - } + if (!check_trace("silent", context, constant, DCOMP_ALLOW_TRACE_OPS, 0, &ret)) + return ret; if (a != 2) { RETURN_TERM_ERROR("Special form 'silent' called with wrong " @@ -5078,13 +5052,16 @@ static Eterm match_spec_test(Process *p, Eterm against, Eterm spec, int trace) return THE_NON_VALUE; } if (trace) { - lint_res = db_match_set_lint(p, spec, DCOMP_TRACE | DCOMP_FAKE_DESTRUCTIVE); - mps = db_match_set_compile(p, spec, DCOMP_TRACE | DCOMP_FAKE_DESTRUCTIVE); + const Uint cflags = (DCOMP_TRACE | DCOMP_FAKE_DESTRUCTIVE | + DCOMP_CALL_TRACE | DCOMP_ALLOW_TRACE_OPS); + lint_res = db_match_set_lint(p, spec, cflags); + mps = db_match_set_compile(p, spec, cflags); } else { - lint_res = db_match_set_lint(p, spec, DCOMP_TABLE | DCOMP_FAKE_DESTRUCTIVE); - mps = db_match_set_compile(p, spec, DCOMP_TABLE | DCOMP_FAKE_DESTRUCTIVE); + const Uint cflags = (DCOMP_TABLE | DCOMP_FAKE_DESTRUCTIVE); + lint_res = db_match_set_lint(p, spec, cflags); + mps = db_match_set_compile(p, spec, cflags); } - + if (mps == NULL) { hp = HAlloc(p,3); ret = TUPLE2(hp, am_error, lint_res); @@ -5115,7 +5092,8 @@ static Eterm match_spec_test(Process *p, Eterm against, Eterm spec, int trace) } save_cp = p->cp; p->cp = NULL; - res = erts_match_set_run(p, mps, arr, n, + res = erts_match_set_run_trace(p, p, + mps, arr, n, ERTS_PAM_COPY_RESULT|ERTS_PAM_IGNORE_TRACE_SILENT, &ret_flags); p->cp = save_cp; @@ -5198,7 +5176,8 @@ Eterm db_match_dbterm(DbTableCommon* tb, Process* c_p, Binary* bprog, obj = db_alloc_tmp_uncompressed(tb, obj); } - res = db_prog_match(c_p, bprog, make_tuple(obj->tpl), NULL, 0, + res = db_prog_match(c_p, c_p, + bprog, make_tuple(obj->tpl), NULL, 0, ERTS_PAM_COPY_RESULT|ERTS_PAM_CONTIGUOUS_TUPLE, &dummy); if (is_value(res) && hpp!=NULL) { diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index 911ed37aef..60f7067d70 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2013. All Rights Reserved. + * Copyright Ericsson AB 1998-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -425,6 +425,11 @@ typedef struct dmc_err_info { #define DCOMP_FAKE_DESTRUCTIVE ((Uint) 8) /* When this is active, no setting of trace control words or seq_trace tokens will be done. */ +/* Allow lock seizing operations on the tracee and 3rd party processes */ +#define DCOMP_ALLOW_TRACE_OPS ((Uint) 0x10) + +/* This is call trace */ +#define DCOMP_CALL_TRACE ((Uint) 0x20) Binary *db_match_compile(Eterm *matchexpr, Eterm *guards, Eterm *body, int num_matches, @@ -435,7 +440,8 @@ Binary *db_match_compile(Eterm *matchexpr, Eterm *guards, Eterm db_match_dbterm(DbTableCommon* tb, Process* c_p, Binary* bprog, int all, DbTerm* obj, Eterm** hpp, Uint extra); -Eterm db_prog_match(Process *p, Binary *prog, Eterm term, +Eterm db_prog_match(Process *p, Process *self, + Binary *prog, Eterm term, Eterm *termp, int arity, enum erts_pam_run_flags in_flags, Uint32 *return_flags /* Zeroed on enter */); diff --git a/erts/emulator/beam/erl_debug.c b/erts/emulator/beam/erl_debug.c index 2902c98864..3e3bfa03a2 100644 --- a/erts/emulator/beam/erl_debug.c +++ b/erts/emulator/beam/erl_debug.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2013. All Rights Reserved. + * Copyright Ericsson AB 1998-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h index 0234b9b0e4..97a69140c3 100644 --- a/erts/emulator/beam/erl_driver.h +++ b/erts/emulator/beam/erl_driver.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2014. All Rights Reserved. + * Copyright Ericsson AB 1999-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_drv_nif.h b/erts/emulator/beam/erl_drv_nif.h index f6b946ae82..2700b62854 100644 --- a/erts/emulator/beam/erl_drv_nif.h +++ b/erts/emulator/beam/erl_drv_nif.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2010. All Rights Reserved. + * Copyright Ericsson AB 2010-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c index 184f8e8931..92edce5176 100644 --- a/erts/emulator/beam/erl_drv_thread.c +++ b/erts/emulator/beam/erl_drv_thread.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2007-2011. All Rights Reserved. + * Copyright Ericsson AB 2007-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c index ed555aa92f..6ce1376c81 100644 --- a/erts/emulator/beam/erl_fun.c +++ b/erts/emulator/beam/erl_fun.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2010. All Rights Reserved. + * Copyright Ericsson AB 2000-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_fun.h b/erts/emulator/beam/erl_fun.h index 0024b1ff71..8c4deea7a0 100644 --- a/erts/emulator/beam/erl_fun.h +++ b/erts/emulator/beam/erl_fun.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2012. All Rights Reserved. + * Copyright Ericsson AB 2000-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 0d125c5598..bed7e668d7 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2014. All Rights Reserved. + * Copyright Ericsson AB 2002-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -116,7 +116,7 @@ static Eterm *full_sweep_heaps(Process *p, char *oh, Uint oh_size, Eterm *objv, int nobj); static int garbage_collect(Process* p, ErlHeapFragment *live_hf_end, - int need, Eterm* objv, int nobj); + int need, Eterm* objv, int nobj, int fcalls); static int major_collection(Process* p, ErlHeapFragment *live_hf_end, int need, Eterm* objv, int nobj, Uint *recl); static int minor_collection(Process* p, ErlHeapFragment *live_hf_end, @@ -392,15 +392,15 @@ erts_gc_after_bif_call_lhf(Process* p, ErlHeapFragment *live_hf_end, regs = ERTS_PROC_GET_SCHDATA(p)->x_reg_array; } #endif - cost = garbage_collect(p, live_hf_end, 0, regs, p->arity); + cost = garbage_collect(p, live_hf_end, 0, regs, p->arity, p->fcalls); } else { - cost = garbage_collect(p, live_hf_end, 0, regs, arity); + cost = garbage_collect(p, live_hf_end, 0, regs, arity, p->fcalls); } } else { Eterm val[1]; val[0] = result; - cost = garbage_collect(p, live_hf_end, 0, val, 1); + cost = garbage_collect(p, live_hf_end, 0, val, 1, p->fcalls); result = val[0]; } BUMP_REDS(p, cost); @@ -431,7 +431,7 @@ static ERTS_INLINE void reset_active_writer(Process *p) #define ERTS_ABANDON_HEAP_COST 10 static int -delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need) +delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need, int fcalls) { ErlHeapFragment *hfrag; Eterm *orig_heap, *orig_hend, *orig_htop, *orig_stop; @@ -506,12 +506,16 @@ delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need) /* Make sure that we do a proper GC as soon as possible... */ p->flags |= F_FORCE_GC; - reds_left = ERTS_BIF_REDS_LEFT(p); + reds_left = ERTS_REDS_LEFT(p, fcalls); + ASSERT(CONTEXT_REDS - reds_left >= ERTS_PROC_GET_SCHDATA(p)->virtual_reds); + if (reds_left > ERTS_ABANDON_HEAP_COST) { int vreds = reds_left - ERTS_ABANDON_HEAP_COST; - ERTS_VBUMP_REDS(p, vreds); + ERTS_PROC_GET_SCHDATA((p))->virtual_reds += vreds; } - return ERTS_ABANDON_HEAP_COST; + + ASSERT(CONTEXT_REDS >= ERTS_PROC_GET_SCHDATA(p)->virtual_reds); + return reds_left; } static ERTS_FORCE_INLINE Uint @@ -570,7 +574,7 @@ young_gen_usage(Process *p) */ static int garbage_collect(Process* p, ErlHeapFragment *live_hf_end, - int need, Eterm* objv, int nobj) + int need, Eterm* objv, int nobj, int fcalls) { Uint reclaimed_now = 0; int reds; @@ -581,8 +585,11 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end, DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE); #endif + ASSERT(CONTEXT_REDS - ERTS_REDS_LEFT(p, fcalls) + >= ERTS_PROC_GET_SCHDATA(p)->virtual_reds); + if (p->flags & (F_DISABLE_GC|F_DELAY_GC)) - return delay_garbage_collection(p, live_hf_end, need); + return delay_garbage_collection(p, live_hf_end, need, fcalls); if (p->abandoned_heap) live_hf_end = ERTS_INVALID_HFRAG_PTR; @@ -593,10 +600,6 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end, esdp = erts_get_scheduler_data(); - if (IS_TRACED_FL(p, F_TRACE_GC)) { - trace_gc(p, am_gc_start); - } - erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC); if (erts_system_monitor_long_gc != 0) start_time = erts_get_monotonic_time(esdp); @@ -619,18 +622,29 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end, */ if (GEN_GCS(p) < MAX_GEN_GCS(p) && !(FLAGS(p) & F_NEED_FULLSWEEP)) { - DTRACE2(gc_minor_start, pidbuf, need); - reds = minor_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now); - DTRACE2(gc_minor_end, pidbuf, reclaimed_now); - if (reds < 0) - goto do_major_collection; - } - else { - do_major_collection: + if (IS_TRACED_FL(p, F_TRACE_GC)) { + trace_gc(p, am_gc_minor_start, need); + } + DTRACE2(gc_minor_start, pidbuf, need); + reds = minor_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now); + DTRACE2(gc_minor_end, pidbuf, reclaimed_now); + if (IS_TRACED_FL(p, F_TRACE_GC)) { + trace_gc(p, am_gc_minor_end, reclaimed_now); + } + if (reds < 0) + goto do_major_collection; + } else { +do_major_collection: ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_GC_FULL); - DTRACE2(gc_major_start, pidbuf, need); - reds = major_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now); - DTRACE2(gc_major_end, pidbuf, reclaimed_now); + if (IS_TRACED_FL(p, F_TRACE_GC)) { + trace_gc(p, am_gc_major_start, need); + } + DTRACE2(gc_major_start, pidbuf, need); + reds = major_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now); + DTRACE2(gc_major_end, pidbuf, reclaimed_now); + if (IS_TRACED_FL(p, F_TRACE_GC)) { + trace_gc(p, am_gc_major_end, reclaimed_now); + } ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_GC); } @@ -646,10 +660,6 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end, erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC); - if (IS_TRACED_FL(p, F_TRACE_GC)) { - trace_gc(p, am_gc_end); - } - if (erts_system_monitor_long_gc != 0) { ErtsMonotonicTime end_time; Uint gc_time; @@ -700,16 +710,23 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end, } int -erts_garbage_collect_nobump(Process* p, int need, Eterm* objv, int nobj) +erts_garbage_collect_nobump(Process* p, int need, Eterm* objv, int nobj, int fcalls) { - return garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj); + int reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj, fcalls); + int reds_left = ERTS_REDS_LEFT(p, fcalls); + if (reds > reds_left) + reds = reds_left; + ASSERT(CONTEXT_REDS - (reds_left - reds) >= ERTS_PROC_GET_SCHDATA(p)->virtual_reds); + return reds; } void erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) { - int reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj); + int reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj, p->fcalls); BUMP_REDS(p, reds); + ASSERT(CONTEXT_REDS - ERTS_BIF_REDS_LEFT(p) + >= ERTS_PROC_GET_SCHDATA(p)->virtual_reds); } /* @@ -2053,8 +2070,26 @@ copy_one_frag(Eterm** hpp, ErlOffHeap* off_heap, *hp++ = val; break; case TAG_PRIMARY_LIST: +#ifdef SHCOPY_SEND + if (erts_is_literal(val,list_val(val))) { + *hp++ = val; + } else { + *hp++ = offset_ptr(val, offs); + } +#else + *hp++ = offset_ptr(val, offs); +#endif + break; case TAG_PRIMARY_BOXED: - *hp++ = offset_ptr(val, offs); +#ifdef SHCOPY_SEND + if (erts_is_literal(val,boxed_val(val))) { + *hp++ = val; + } else { + *hp++ = offset_ptr(val, offs); + } +#else + *hp++ = offset_ptr(val, offs); +#endif break; case TAG_PRIMARY_HEADER: *hp++ = val; @@ -2233,9 +2268,7 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset) n++; } #endif - ASSERT(is_nil(ERTS_TRACER_PROC(p)) || - is_internal_pid(ERTS_TRACER_PROC(p)) || - is_internal_port(ERTS_TRACER_PROC(p))); + ASSERT(IS_TRACER_VALID(ERTS_TRACER(p))); ASSERT(is_pid(follow_moved(p->group_leader, (Eterm) 0))); if (is_not_immed(p->group_leader)) { @@ -2905,7 +2938,7 @@ reply_gc_info(void *vgcirp) hpp = &hp; } - erts_queue_message(rp, &rp_locks, mp, msg, NIL); + erts_queue_message(rp, rp_locks, mp, msg, am_system); if (gcirp->req_sched == esdp->no) rp_locks &= ~ERTS_PROC_LOCK_MAIN; @@ -2990,10 +3023,30 @@ erts_process_gc_info(Process *p, Uint *sizep, Eterm **hpp) }; Eterm res = THE_NON_VALUE; + ErtsMessage *mp; ERTS_CT_ASSERT(sizeof(values)/sizeof(*values) == sizeof(tags)/sizeof(*tags)); ERTS_CT_ASSERT(sizeof(values)/sizeof(*values) == ERTS_PROCESS_GC_INFO_MAX_TERMS); + if (p->abandoned_heap) { + Eterm *htop, *heap; + ERTS_GET_ORIG_HEAP(p, heap, htop); + values[3] = HIGH_WATER(p) - heap; + values[6] = htop - heap; + } + + if (p->flags & F_ON_HEAP_MSGQ) { + /* If on heap messages in the internal queue are counted + as being part of the heap, so we have to add them to the + am_mbuf_size value. process_info(total_heap_size) should + be the same as adding old_heap_block_size + heap_block_size + + mbuf_size. + */ + for (mp = p->msg.first; mp; mp = mp->next) + if (mp->data.attached) + values[2] += erts_msg_attached_data_size(mp); + } + res = erts_bld_atom_uword_2tup_list(hpp, sizep, sizeof(values)/sizeof(*values), diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h index 2cedd9361f..8a6ff99990 100644 --- a/erts/emulator/beam/erl_gc.h +++ b/erts/emulator/beam/erl_gc.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2007-2011. All Rights Reserved. + * Copyright Ericsson AB 2007-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -139,7 +139,7 @@ Eterm erts_process_gc_info(struct process*, Uint *, Eterm **); void erts_gc_info(ErtsGCInfo *gcip); void erts_init_gc(void); -int erts_garbage_collect_nobump(struct process*, int, Eterm*, int); +int erts_garbage_collect_nobump(struct process*, int, Eterm*, int, int); void erts_garbage_collect(struct process*, int, Eterm*, int); void erts_garbage_collect_hibernate(struct process* p); Eterm erts_gc_after_bif_call_lhf(struct process* p, ErlHeapFragment *live_hf_end, diff --git a/erts/emulator/beam/erl_goodfit_alloc.c b/erts/emulator/beam/erl_goodfit_alloc.c index 9b4aad9d91..223ba193da 100644 --- a/erts/emulator/beam/erl_goodfit_alloc.c +++ b/erts/emulator/beam/erl_goodfit_alloc.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2003-2013. All Rights Reserved. + * Copyright Ericsson AB 2003-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_goodfit_alloc.h b/erts/emulator/beam/erl_goodfit_alloc.h index ababdbd0a1..76dd558234 100644 --- a/erts/emulator/beam/erl_goodfit_alloc.h +++ b/erts/emulator/beam/erl_goodfit_alloc.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2003-2013. All Rights Reserved. + * Copyright Ericsson AB 2003-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c index 491d2f8c84..c418762578 100644 --- a/erts/emulator/beam/erl_hl_timer.c +++ b/erts/emulator/beam/erl_hl_timer.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2015. All Rights Reserved. + * Copyright Ericsson AB 2015-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -1247,8 +1247,8 @@ hlt_bif_timer_timeout(ErtsHLTimer *tmr, Uint32 roflgs) if (!ERTS_PROC_IS_EXITING(proc)) { ErtsMessage *mp = erts_alloc_message(0, NULL); mp->data.heap_frag = tmr->btm.bp; - erts_queue_message(proc, &proc_locks, mp, - tmr->btm.message, NIL); + erts_queue_message(proc, proc_locks, mp, + tmr->btm.message, am_clock_service); erts_smp_proc_unlock(proc, ERTS_PROC_LOCKS_MSG_SEND); queued_message = 1; proc_locks &= ~ERTS_PROC_LOCKS_MSG_SEND; @@ -1980,7 +1980,7 @@ access_sched_local_btm(Process *c_p, Eterm pid, ERTS_HLT_ASSERT(hp + (async ? 4 : 3) == hp_end); - erts_queue_message(proc, &proc_locks, mp, msg, NIL); + erts_queue_message(proc, proc_locks, mp, msg, am_clock_service); if (c_p) proc_locks &= ~ERTS_PROC_LOCK_MAIN; @@ -2111,7 +2111,7 @@ try_access_sched_remote_btm(ErtsSchedulerData *esdp, msg = TUPLE3(hp, tag, tref, res); - erts_queue_message(c_p, &proc_locks, mp, msg, NIL); + erts_queue_message(c_p, proc_locks, mp, msg, am_clock_service); proc_locks &= ~ERTS_PROC_LOCK_MAIN; if (proc_locks) diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 86c4eeb484..fcd2739ac3 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1997-2013. All Rights Reserved. + * Copyright Ericsson AB 1997-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -206,15 +206,15 @@ int erts_no_line_info = 0; /* -L: Don't load line information */ ErtsModifiedTimings erts_modified_timings[] = { /* 0 */ {make_small(0), CONTEXT_REDS, INPUT_REDUCTIONS}, - /* 1 */ {make_small(0), 2*CONTEXT_REDS, 2*INPUT_REDUCTIONS}, + /* 1 */ {make_small(0), (3*CONTEXT_REDS)/4, 2*INPUT_REDUCTIONS}, /* 2 */ {make_small(0), CONTEXT_REDS/2, INPUT_REDUCTIONS/2}, - /* 3 */ {make_small(0), 3*CONTEXT_REDS, 3*INPUT_REDUCTIONS}, + /* 3 */ {make_small(0), (7*CONTEXT_REDS)/8, 3*INPUT_REDUCTIONS}, /* 4 */ {make_small(0), CONTEXT_REDS/3, 3*INPUT_REDUCTIONS}, - /* 5 */ {make_small(0), 4*CONTEXT_REDS, INPUT_REDUCTIONS/2}, + /* 5 */ {make_small(0), (10*CONTEXT_REDS)/11, INPUT_REDUCTIONS/2}, /* 6 */ {make_small(1), CONTEXT_REDS/4, 2*INPUT_REDUCTIONS}, - /* 7 */ {make_small(1), 5*CONTEXT_REDS, INPUT_REDUCTIONS/3}, + /* 7 */ {make_small(1), (5*CONTEXT_REDS)/7, INPUT_REDUCTIONS/3}, /* 8 */ {make_small(10), CONTEXT_REDS/5, 3*INPUT_REDUCTIONS}, - /* 9 */ {make_small(10), 6*CONTEXT_REDS, INPUT_REDUCTIONS/4} + /* 9 */ {make_small(10), (6*CONTEXT_REDS)/7, INPUT_REDUCTIONS/4} }; #define ERTS_MODIFIED_TIMING_LEVELS \ @@ -578,6 +578,8 @@ void erts_usage(void) VH_DEFAULT_SIZE); erts_fprintf(stderr, "-hpds size initial process dictionary size (default %d)\n", erts_pd_initial_size); + erts_fprintf(stderr, "-hmqd val set default message queue data flag for processes,\n"); + erts_fprintf(stderr, " valid values are: off_heap | on_heap | mixed\n"); /* erts_fprintf(stderr, "-i module set the boot module (default init)\n"); */ @@ -655,8 +657,6 @@ void erts_usage(void) erts_fprintf(stderr, "-W<i|w|e> set error logger warnings mapping,\n"); erts_fprintf(stderr, " see error_logger documentation for details\n"); - erts_fprintf(stderr, "-xmqd val set default message queue data flag for processes,\n"); - erts_fprintf(stderr, " valid values are: off_heap | on_heap | mixed\n"); erts_fprintf(stderr, "-zdbbl size set the distribution buffer busy limit in kilobytes\n"); erts_fprintf(stderr, " valid range is [1-%d]\n", INT_MAX/1024); erts_fprintf(stderr, "-zdntgc time set delayed node table gc in seconds\n"); @@ -1487,6 +1487,7 @@ erl_start(int argc, char **argv) * h|ms - min_heap_size * h|mbs - min_bin_vheap_size * h|pds - erts_pd_initial_size + * h|mqd - message_queue_data * */ if (has_prefix("mbs", sub_param)) { @@ -1512,6 +1513,23 @@ erl_start(int argc, char **argv) } VERBOSE(DEBUG_SYSTEM, ("using initial process dictionary size %d\n", erts_pd_initial_size)); + } else if (has_prefix("mqd", sub_param)) { + arg = get_arg(sub_param+3, argv[i+1], &i); + if (sys_strcmp(arg, "mixed") == 0) + erts_default_spo_flags &= ~(SPO_ON_HEAP_MSGQ|SPO_OFF_HEAP_MSGQ); + else if (sys_strcmp(arg, "on_heap") == 0) { + erts_default_spo_flags &= ~SPO_OFF_HEAP_MSGQ; + erts_default_spo_flags |= SPO_ON_HEAP_MSGQ; + } + else if (sys_strcmp(arg, "off_heap") == 0) { + erts_default_spo_flags &= ~SPO_ON_HEAP_MSGQ; + erts_default_spo_flags |= SPO_OFF_HEAP_MSGQ; + } + else { + erts_fprintf(stderr, + "Invalid message_queue_data flag: %s\n", arg); + erts_usage(); + } } else { /* backward compatibility */ arg = get_arg(argv[i]+2, argv[i+1], &i); @@ -2047,32 +2065,6 @@ erl_start(int argc, char **argv) } break; - case 'x': { - char *sub_param = argv[i]+2; - if (has_prefix("mqd", sub_param)) { - arg = get_arg(sub_param+3, argv[i+1], &i); - if (sys_strcmp(arg, "mixed") == 0) - erts_default_spo_flags &= ~(SPO_ON_HEAP_MSGQ|SPO_OFF_HEAP_MSGQ); - else if (sys_strcmp(arg, "on_heap") == 0) { - erts_default_spo_flags &= ~SPO_OFF_HEAP_MSGQ; - erts_default_spo_flags |= SPO_ON_HEAP_MSGQ; - } - else if (sys_strcmp(arg, "off_heap") == 0) { - erts_default_spo_flags &= ~SPO_ON_HEAP_MSGQ; - erts_default_spo_flags |= SPO_OFF_HEAP_MSGQ; - } - else { - erts_fprintf(stderr, - "Invalid message_queue_data flag: %s\n", arg); - erts_usage(); - } - } else { - erts_fprintf(stderr, "bad -x option %s\n", argv[i]); - erts_usage(); - } - break; - } - case 'z': { char *sub_param = argv[i]+2; @@ -2188,6 +2180,7 @@ erl_start(int argc, char **argv) init_break_handler(); if (replace_intr) erts_replace_intr(); + sys_init_suspend_handler(); #endif boot_argc = argc - i; /* Number of arguments to init */ diff --git a/erts/emulator/beam/erl_instrument.c b/erts/emulator/beam/erl_instrument.c index 12a72ad839..f84c63e7a4 100644 --- a/erts/emulator/beam/erl_instrument.c +++ b/erts/emulator/beam/erl_instrument.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2003-2013. All Rights Reserved. + * Copyright Ericsson AB 2003-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_instrument.h b/erts/emulator/beam/erl_instrument.h index cb3b1920d3..1f04c91d5e 100644 --- a/erts/emulator/beam/erl_instrument.h +++ b/erts/emulator/beam/erl_instrument.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2003-2009. All Rights Reserved. + * Copyright Ericsson AB 2003-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 598bf84c0b..39c0617143 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2013. All Rights Reserved. + * Copyright Ericsson AB 2005-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -97,6 +97,7 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "dist_entry_links", "address" }, { "code_write_permission", NULL }, { "proc_status", "pid" }, + { "proc_trace", "pid" }, { "ports_snapshot", NULL }, { "meta_name_tab", "address" }, { "meta_main_tab_slot", "address" }, @@ -148,6 +149,7 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "dist_entry_out_queue", "address" }, { "port_sched_lock", "port_id" }, { "sys_msg_q", NULL }, + { "tracer_mtx", NULL }, { "port_table", NULL }, #endif { "mtrace_op", NULL }, @@ -160,9 +162,6 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "proclist_pre_alloc_lock", "address" }, { "xports_list_pre_alloc_lock", "address" }, { "inet_buffer_stack_lock", NULL }, - { "gc_info", NULL }, - { "io_wake", NULL }, - { "timer_wheel", NULL }, { "system_block", NULL }, { "timeofday", NULL }, { "get_time", NULL }, diff --git a/erts/emulator/beam/erl_lock_check.h b/erts/emulator/beam/erl_lock_check.h index 66251ef4e8..18296d1fec 100644 --- a/erts/emulator/beam/erl_lock_check.h +++ b/erts/emulator/beam/erl_lock_check.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2012. All Rights Reserved. + * Copyright Ericsson AB 2005-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_lock_count.c b/erts/emulator/beam/erl_lock_count.c index bd00480ba2..a00e0f0fff 100644 --- a/erts/emulator/beam/erl_lock_count.c +++ b/erts/emulator/beam/erl_lock_count.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2008-2012. All Rights Reserved. + * Copyright Ericsson AB 2008-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_lock_count.h b/erts/emulator/beam/erl_lock_count.h index 4cc6a5c695..3e8dcefe69 100644 --- a/erts/emulator/beam/erl_lock_count.h +++ b/erts/emulator/beam/erl_lock_count.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2008-2012. All Rights Reserved. + * Copyright Ericsson AB 2008-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c index 4b6931f848..8efc983f04 100644 --- a/erts/emulator/beam/erl_map.c +++ b/erts/emulator/beam/erl_map.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2014. All Rights Reserved. + * Copyright Ericsson AB 2014-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -54,6 +54,7 @@ * - maps:new/0 * - maps:put/3 * - maps:remove/2 + * - maps:take/2 * - maps:to_list/1 * - maps:update/3 * - maps:values/1 @@ -93,7 +94,7 @@ static Uint hashmap_subtree_size(Eterm node); static Eterm hashmap_to_list(Process *p, Eterm map); static Eterm hashmap_keys(Process *p, Eterm map); static Eterm hashmap_values(Process *p, Eterm map); -static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm node); +static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm node, Eterm *value); static Eterm flatmap_from_validated_list(Process *p, Eterm list, Uint size); static Eterm hashmap_from_validated_list(Process *p, Eterm list, Uint size); static Eterm hashmap_from_unsorted_array(ErtsHeapFactory*, hxnode_t *hxns, Uint n, int reject_dupkeys); @@ -196,13 +197,13 @@ erts_maps_get(Eterm key, Eterm map) return &vs[i]; } } - } - - for (i = 0; i < n; i++) { - if (EQ(ks[i], key)) { - return &vs[i]; - } - } + } else { + for (i = 0; i < n; i++) { + if (EQ(ks[i], key)) { + return &vs[i]; + } + } + } return NULL; } ASSERT(is_hashmap(map)); @@ -1521,10 +1522,45 @@ BIF_RETTYPE maps_put_3(BIF_ALIST_3) { BIF_ERROR(BIF_P, BADMAP); } -/* maps:remove/3 */ +/* maps:take/2 */ -int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res) { +BIF_RETTYPE maps_take_2(BIF_ALIST_2) { + if (is_map(BIF_ARG_2)) { + Eterm res, map, val; + if (erts_maps_take(BIF_P, BIF_ARG_1, BIF_ARG_2, &map, &val)) { + Eterm *hp = HAlloc(BIF_P, 3); + res = make_tuple(hp); + *hp++ = make_arityval(2); + *hp++ = val; + *hp++ = map; + BIF_RET(res); + } + BIF_RET(am_error); + } + BIF_P->fvalue = BIF_ARG_2; + BIF_ERROR(BIF_P, BADMAP); +} + +/* maps:remove/2 */ + +BIF_RETTYPE maps_remove_2(BIF_ALIST_2) { + if (is_map(BIF_ARG_2)) { + Eterm res; + (void) erts_maps_take(BIF_P, BIF_ARG_1, BIF_ARG_2, &res, NULL); + BIF_RET(res); + } + BIF_P->fvalue = BIF_ARG_2; + BIF_ERROR(BIF_P, BADMAP); +} + +/* erts_maps_take + * return 1 if key is found, otherwise 0 + * If the key is not found res (output map) will be map (input map) + */ +int erts_maps_take(Process *p, Eterm key, Eterm map, + Eterm *res, Eterm *value) { Uint32 hx; + Eterm ret; if (is_flatmap(map)) { Sint n; Uint need; @@ -1537,7 +1573,7 @@ int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res) { if (n == 0) { *res = map; - return 1; + return 0; } ks = flatmap_get_keys(mp); @@ -1564,6 +1600,7 @@ int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res) { if (is_immed(key)) { while (1) { if (*ks == key) { + if (value) *value = *vs; goto found_key; } else if (--n) { *mhp++ = *vs++; @@ -1574,6 +1611,7 @@ int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res) { } else { while(1) { if (EQ(*ks, key)) { + if (value) *value = *vs; goto found_key; } else if (--n) { *mhp++ = *vs++; @@ -1589,7 +1627,7 @@ int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res) { HRelease(p, hp_start + need, hp_start); *res = map; - return 1; + return 0; found_key: /* Copy rest of keys and values */ @@ -1601,19 +1639,13 @@ found_key: } ASSERT(is_hashmap(map)); hx = hashmap_make_hash(key); - *res = hashmap_delete(p, hx, key, map); - return 1; -} - -BIF_RETTYPE maps_remove_2(BIF_ALIST_2) { - if (is_map(BIF_ARG_2)) { - Eterm res; - if (erts_maps_remove(BIF_P, BIF_ARG_1, BIF_ARG_2, &res)) { - BIF_RET(res); - } + ret = hashmap_delete(p, hx, key, map, value); + if (is_value(ret)) { + *res = ret; + return 1; } - BIF_P->fvalue = BIF_ARG_2; - BIF_ERROR(BIF_P, BADMAP); + *res = map; + return 0; } int erts_maps_update(Process *p, Eterm key, Eterm value, Eterm map, Eterm *res) { @@ -2322,7 +2354,8 @@ static Eterm hashmap_values(Process* p, Eterm node) { return res; } -static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm map) { +static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, + Eterm map, Eterm *value) { Eterm *hp = NULL, *nhp = NULL, *hp_end = NULL; Eterm *ptr; Eterm hdr, res = map, node = map; @@ -2337,8 +2370,12 @@ static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm map) { switch(primary_tag(node)) { case TAG_PRIMARY_LIST: if (EQ(CAR(list_val(node)), key)) { + if (value) { + *value = CDR(list_val(node)); + } goto unroll; } + res = THE_NON_VALUE; goto not_found; case TAG_PRIMARY_BOXED: ptr = boxed_val(node); @@ -2365,6 +2402,7 @@ static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm map) { n = hashmap_bitcount(hval); } else { /* not occupied */ + res = THE_NON_VALUE; goto not_found; } @@ -2394,6 +2432,7 @@ static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm map) { break; } /* not occupied */ + res = THE_NON_VALUE; goto not_found; default: erts_exit(ERTS_ERROR_EXIT, "bad header tag %ld\r\n", hdr & _HEADER_MAP_SUBTAG_MASK); diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h index 052fa99f03..8b5c9582ba 100644 --- a/erts/emulator/beam/erl_map.h +++ b/erts/emulator/beam/erl_map.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2014. All Rights Reserved. + * Copyright Ericsson AB 2014-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -66,7 +66,7 @@ typedef struct flatmap_s { /* erl_term.h stuff */ -#define flatmap_get_values(x) (((Eterm *)(x)) + 3) +#define flatmap_get_values(x) (((Eterm *)(x)) + sizeof(flatmap_t)/sizeof(Eterm)) #define flatmap_get_keys(x) (((Eterm *)tuple_val(((flatmap_t *)(x))->keys)) + 1) #define flatmap_get_size(x) (((flatmap_t*)(x))->size) @@ -82,6 +82,7 @@ struct ErtsEStack_; Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map); int erts_maps_update(Process *p, Eterm key, Eterm value, Eterm map, Eterm *res); int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res); +int erts_maps_take(Process *p, Eterm key, Eterm map, Eterm *res, Eterm *value); Eterm erts_hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value, Eterm node, int is_update); diff --git a/erts/emulator/beam/erl_math.c b/erts/emulator/beam/erl_math.c index b46cc37495..fc0aaed18a 100644 --- a/erts/emulator/beam/erl_math.c +++ b/erts/emulator/beam/erl_math.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1997-2009. All Rights Reserved. + * Copyright Ericsson AB 1997-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index 88efb2c59f..9bb6e40a11 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1997-2012. All Rights Reserved. + * Copyright Ericsson AB 1997-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -32,6 +32,7 @@ #include "erl_process.h" #include "erl_binary.h" #include "dtrace-wrapper.h" +#include "beam_bp.h" ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(message_ref, ErtsMessageRef, @@ -251,9 +252,10 @@ erts_realloc_shrink_message(ErtsMessage *mp, Uint sz, Eterm *brefs, Uint brefs_s void erts_queue_dist_message(Process *rcvr, - ErtsProcLocks *rcvr_locks, + ErtsProcLocks rcvr_locks, ErtsDistExternal *dist_ext, - Eterm token) + Eterm token, + Eterm from) { ErtsMessage* mp; #ifdef USE_VM_PROBES @@ -265,7 +267,7 @@ erts_queue_dist_message(Process *rcvr, erts_aint_t state; #endif - ERTS_SMP_LC_ASSERT(*rcvr_locks == erts_proc_lc_my_proc_locks(rcvr)); + ERTS_SMP_LC_ASSERT(rcvr_locks == erts_proc_lc_my_proc_locks(rcvr)); mp = erts_alloc_message(0, NULL); mp->data.dist_ext = dist_ext; @@ -280,10 +282,10 @@ erts_queue_dist_message(Process *rcvr, ERL_MESSAGE_TOKEN(mp) = token; #ifdef ERTS_SMP - if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ)) { + if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ)) { if (erts_smp_proc_trylock(rcvr, ERTS_PROC_LOCK_MSGQ) == EBUSY) { ErtsProcLocks need_locks = ERTS_PROC_LOCK_MSGQ; - if (*rcvr_locks & ERTS_PROC_LOCK_STATUS) { + if (rcvr_locks & ERTS_PROC_LOCK_STATUS) { erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_STATUS); need_locks |= ERTS_PROC_LOCK_STATUS; } @@ -293,7 +295,7 @@ erts_queue_dist_message(Process *rcvr, state = erts_smp_atomic32_read_acqb(&rcvr->state); if (state & (ERTS_PSFLG_PENDING_EXIT|ERTS_PSFLG_EXITING)) { - if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ)) + if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ)) erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ); /* Drop message if receiver is exiting or has a pending exit ... */ erts_cleanup_messages(mp); @@ -301,10 +303,13 @@ erts_queue_dist_message(Process *rcvr, else #endif if (IS_TRACED_FL(rcvr, F_TRACE_RECEIVE)) { + if (from == am_Empty) + from = dist_ext->dep->sysname; + /* Ahh... need to decode it in order to trace it... */ - if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ)) + if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ)) erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ); - if (!erts_decode_dist_message(rcvr, *rcvr_locks, mp, 0)) + if (!erts_decode_dist_message(rcvr, rcvr_locks, mp, 0)) erts_free_message(mp); else { Eterm msg = ERL_MESSAGE_TERM(mp); @@ -324,7 +329,7 @@ erts_queue_dist_message(Process *rcvr, tok_label, tok_lastcnt, tok_serial); } #endif - erts_queue_message(rcvr, rcvr_locks, mp, msg, token); + erts_queue_message(rcvr, rcvr_locks, mp, msg, from); } } else { @@ -349,14 +354,14 @@ erts_queue_dist_message(Process *rcvr, } #endif - LINK_MESSAGE(rcvr, mp); + LINK_MESSAGE(rcvr, mp, &mp->next, 1); - if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ)) + if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ)) erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ); erts_proc_notify_new_message(rcvr, #ifdef ERTS_SMP - *rcvr_locks + rcvr_locks #else 0 #endif @@ -364,31 +369,35 @@ erts_queue_dist_message(Process *rcvr, } } -/* Add a message last in message queue */ +/* Add messages last in message queue */ static Sint -queue_message(Process *c_p, - Process* receiver, - erts_aint32_t *receiver_state, - ErtsProcLocks *receiver_locks, - ErtsMessage* mp, - Eterm message, - Eterm seq_trace_token -#ifdef USE_VM_PROBES - , Eterm dt_utag -#endif - ) +queue_messages(Process* receiver, + erts_aint32_t *receiver_state, + ErtsProcLocks receiver_locks, + ErtsMessage* first, + ErtsMessage** last, + Uint len, + Eterm from) { + ErtsTracingEvent* te; Sint res; int locked_msgq = 0; erts_aint32_t state; - ERTS_SMP_LC_ASSERT(*receiver_locks == erts_proc_lc_my_proc_locks(receiver)); + ASSERT(is_value(ERL_MESSAGE_TERM(first))); + ASSERT(ERL_MESSAGE_TOKEN(first) == am_undefined || + ERL_MESSAGE_TOKEN(first) == NIL || + is_tuple(ERL_MESSAGE_TOKEN(first))); #ifdef ERTS_SMP +#ifdef ERTS_ENABLE_LOCK_CHECK + ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(receiver) < ERTS_PROC_LOCK_MSGQ || + receiver_locks == erts_proc_lc_my_proc_locks(receiver)); +#endif - if (!(*receiver_locks & ERTS_PROC_LOCK_MSGQ)) { + if (!(receiver_locks & ERTS_PROC_LOCK_MSGQ)) { if (erts_smp_proc_trylock(receiver, ERTS_PROC_LOCK_MSGQ) == EBUSY) { - ErtsProcLocks need_locks = ERTS_PROC_LOCK_MSGQ; + ErtsProcLocks need_locks; if (receiver_state) state = *receiver_state; @@ -397,10 +406,11 @@ queue_message(Process *c_p, if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) goto exiting; - if (*receiver_locks & ERTS_PROC_LOCK_STATUS) { - erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_STATUS); - need_locks |= ERTS_PROC_LOCK_STATUS; + need_locks = receiver_locks & (ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); + if (need_locks) { + erts_smp_proc_unlock(receiver, need_locks); } + need_locks |= ERTS_PROC_LOCK_MSGQ; erts_smp_proc_lock(receiver, need_locks); } locked_msgq = 1; @@ -417,19 +427,13 @@ queue_message(Process *c_p, /* Drop message if receiver is exiting or has a pending exit... */ if (locked_msgq) erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ); - erts_cleanup_messages(mp); + erts_cleanup_messages(first); return 0; } - ERL_MESSAGE_TERM(mp) = message; - ERL_MESSAGE_TOKEN(mp) = seq_trace_token; -#ifdef USE_VM_PROBES - ERL_MESSAGE_DT_UTAG(mp) = dt_utag; -#endif - res = receiver->msg.len; #ifdef ERTS_SMP - if (*receiver_locks & ERTS_PROC_LOCK_MAIN) { + if (receiver_locks & ERTS_PROC_LOCK_MAIN) { /* * We move 'in queue' to 'private queue' and place * message at the end of 'private queue' in order @@ -440,75 +444,85 @@ queue_message(Process *c_p, */ res += receiver->msg_inq.len; ERTS_SMP_MSGQ_MV_INQ2PRIVQ(receiver); - LINK_MESSAGE_PRIVQ(receiver, mp); + LINK_MESSAGE_PRIVQ(receiver, first, last, len); } else #endif { - LINK_MESSAGE(receiver, mp); + LINK_MESSAGE(receiver, first, last, len); } + if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE) + && (te = &erts_receive_tracing[erts_active_bp_ix()], + te->on)) { + + ErtsMessage *msg = first; + #ifdef USE_VM_PROBES - if (DTRACE_ENABLED(message_queued)) { - DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE); - Sint tok_label = 0; - Sint tok_lastcnt = 0; - Sint tok_serial = 0; - - dtrace_proc_str(receiver, receiver_name); - if (seq_trace_token != NIL && is_tuple(seq_trace_token)) { - tok_label = signed_val(SEQ_TRACE_T_LABEL(seq_trace_token)); - tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(seq_trace_token)); - tok_serial = signed_val(SEQ_TRACE_T_SERIAL(seq_trace_token)); + if (DTRACE_ENABLED(message_queued)) { + DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE); + Sint tok_label = 0; + Sint tok_lastcnt = 0; + Sint tok_serial = 0; + Eterm seq_trace_token = ERL_MESSAGE_TOKEN(msg); + + dtrace_proc_str(receiver, receiver_name); + if (seq_trace_token != NIL && is_tuple(seq_trace_token)) { + tok_label = signed_val(SEQ_TRACE_T_LABEL(seq_trace_token)); + tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(seq_trace_token)); + tok_serial = signed_val(SEQ_TRACE_T_SERIAL(seq_trace_token)); + } + DTRACE6(message_queued, + receiver_name, size_object(ERL_MESSAGE_TERM(msg)), + receiver->msg.len, + tok_label, tok_lastcnt, tok_serial); } - DTRACE6(message_queued, - receiver_name, size_object(message), receiver->msg.len, - tok_label, tok_lastcnt, tok_serial); - } #endif + while (msg) { + trace_receive(receiver, from, ERL_MESSAGE_TERM(msg), te); + msg = msg->next; + } - if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE)) - trace_receive(receiver, message); - - if (locked_msgq) + } + if (locked_msgq) { erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ); + } - erts_proc_notify_new_message(receiver, #ifdef ERTS_SMP - *receiver_locks + erts_proc_notify_new_message(receiver, receiver_locks); #else - 0 -#endif - ); - -#ifndef ERTS_SMP + erts_proc_notify_new_message(receiver, 0); ERTS_HOLE_CHECK(receiver); #endif return res; } -void -#ifdef USE_VM_PROBES -erts_queue_message_probe(Process* receiver, ErtsProcLocks *receiver_locks, - ErtsMessage* mp, - Eterm message, Eterm seq_trace_token, Eterm dt_utag) -#else -erts_queue_message(Process* receiver, ErtsProcLocks *receiver_locks, - ErtsMessage* mp, - Eterm message, Eterm seq_trace_token) -#endif +static Sint +queue_message(Process* receiver, + erts_aint32_t *receiver_state, + ErtsProcLocks receiver_locks, + ErtsMessage* mp, Eterm msg, Eterm from) { - queue_message(NULL, - receiver, - NULL, - receiver_locks, - mp, - message, - seq_trace_token -#ifdef USE_VM_PROBES - , dt_utag -#endif - ); + ERL_MESSAGE_TERM(mp) = msg; + return queue_messages(receiver, receiver_state, receiver_locks, + mp, &mp->next, 1, from); +} + +Sint +erts_queue_message(Process* receiver, ErtsProcLocks receiver_locks, + ErtsMessage* mp, Eterm msg, Eterm from) +{ + return queue_message(receiver, NULL, receiver_locks, mp, msg, from); +} + + +Sint +erts_queue_messages(Process* receiver, ErtsProcLocks receiver_locks, + ErtsMessage* first, ErtsMessage** last, Uint len, + Eterm from) +{ + return queue_messages(receiver, NULL, receiver_locks, + first, last, len, from); } void @@ -591,7 +605,7 @@ erts_try_alloc_message_on_heap(Process *pp, #if defined(ERTS_SMP) *plp & ERTS_PROC_LOCK_MAIN #else - 1 + pp #endif ) { #ifdef ERTS_SMP @@ -621,7 +635,7 @@ erts_try_alloc_message_on_heap(Process *pp, *on_heap_p = !0; } #ifdef ERTS_SMP - else if (erts_smp_proc_trylock(pp, ERTS_PROC_LOCK_MAIN) == 0) { + else if (pp && erts_smp_proc_trylock(pp, ERTS_PROC_LOCK_MAIN) == 0) { locked_main = 1; *psp = erts_smp_atomic32_read_nob(&pp->state); *plp |= ERTS_PROC_LOCK_MAIN; @@ -766,17 +780,7 @@ erts_send_message(Process* sender, utag = DT_UTAG(sender); else utag = copy_struct(DT_UTAG(sender), dt_utag_size, &hp, ohp); -#ifdef DTRACE_TAG_HARDDEBUG - erts_fprintf(stderr, - "Dtrace -> (%T) Spreading tag (%T) with " - "message %T!\r\n",sender->common.id, utag, message); -#endif } -#endif - BM_MESSAGE_COPIED(msize); - BM_SWAP_TIMER(copy,send); - -#ifdef USE_VM_PROBES if (DTRACE_ENABLED(message_send)) { if (have_seqtrace(stoken)) { tok_label = signed_val(SEQ_TRACE_T_LABEL(stoken)); @@ -787,6 +791,9 @@ erts_send_message(Process* sender, msize, tok_label, tok_lastcnt, tok_serial); } #endif + BM_MESSAGE_COPIED(msize); + BM_SWAP_TIMER(copy,send); + } else { Eterm *hp; @@ -822,21 +829,21 @@ erts_send_message(Process* sender, BM_MESSAGE_COPIED(msz); BM_SWAP_TIMER(copy,send); } +#ifdef USE_VM_PROBES DTRACE6(message_send, sender_name, receiver_name, msize, tok_label, tok_lastcnt, tok_serial); +#endif } - res = queue_message(sender, - receiver, - &receiver_state, - receiver_locks, - mp, - message, - token + ERL_MESSAGE_TOKEN(mp) = token; #ifdef USE_VM_PROBES - , utag + ERL_MESSAGE_DT_UTAG(mp) = utag; #endif - ); + res = queue_message(receiver, + &receiver_state, + *receiver_locks, + mp, message, + sender->common.id); BM_SWAP_TIMER(send,system); @@ -892,7 +899,8 @@ erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp, /* the trace token must in this case be updated by the caller */ seq_trace_output(token, save, SEQ_TRACE_SEND, to->common.id, NULL); temptoken = copy_struct(token, sz_token, &hp, ohp); - erts_queue_message(to, to_locksp, mp, save, temptoken); + ERL_MESSAGE_TOKEN(mp) = temptoken; + erts_queue_message(to, *to_locksp, mp, save, am_system); } else { sz_from = IS_CONST(from) ? 0 : size_object(from); #ifdef SHCOPY_SEND @@ -914,7 +922,7 @@ erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp, ? from : copy_struct(from, sz_from, &hp, ohp)); save = TUPLE3(hp, am_EXIT, from_copy, mess); - erts_queue_message(to, to_locksp, mp, save, NIL); + erts_queue_message(to, *to_locksp, mp, save, am_system); } } @@ -1489,7 +1497,7 @@ erts_factory_message_create(ErtsHeapFactory* factory, int on_heap; erts_aint32_t state; - state = erts_smp_atomic32_read_nob(&proc->state); + state = proc ? erts_smp_atomic32_read_nob(&proc->state) : 0; if (state & ERTS_PSFLG_OFF_HEAP_MSGQ) { msgp = erts_alloc_message(sz, &hp); diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index 60035d15ae..851ac37fda 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1997-2012. All Rights Reserved. + * Copyright Ericsson AB 1997-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -182,48 +182,64 @@ typedef struct { Sint len; /* queue length */ } ErlMessageInQueue; +typedef struct erl_trace_message_queue__ { + struct erl_trace_message_queue__ *next; /* point to the next receiver */ + Eterm receiver; + ErtsMessage* first; + ErtsMessage** last; /* point to the last next pointer */ + Sint len; /* queue length */ +} ErlTraceMessageQueue; + #endif /* Get "current" message */ #define PEEK_MESSAGE(p) (*(p)->msg.save) +#ifdef USE_VM_PROBES +#define LINK_MESSAGE_DTAG(mp, dt) ERL_MESSAGE_DT_UTAG(mp) = dt +#else +#define LINK_MESSAGE_DTAG(mp, dt) +#endif -/* Add message last in private message queue */ -#define LINK_MESSAGE_PRIVQ(p, mp) do { \ - *(p)->msg.last = (mp); \ - (p)->msg.last = &(mp)->next; \ - (p)->msg.len++; \ -} while (0) - +#define LINK_MESSAGE_IMPL(p, first_msg, last_msg, num_msgs, where) do { \ + *(p)->where.last = (first_msg); \ + (p)->where.last = (last_msg); \ + (p)->where.len += (num_msgs); \ + } while(0) #ifdef ERTS_SMP -/* Move in message queue to end of private message queue */ -#define ERTS_SMP_MSGQ_MV_INQ2PRIVQ(P) \ -do { \ - if ((P)->msg_inq.first) { \ - *(P)->msg.last = (P)->msg_inq.first; \ - (P)->msg.last = (P)->msg_inq.last; \ - (P)->msg.len += (P)->msg_inq.len; \ - (P)->msg_inq.first = NULL; \ - (P)->msg_inq.last = &(P)->msg_inq.first; \ - (P)->msg_inq.len = 0; \ - } \ -} while (0) - -/* Add message last in message queue */ -#define LINK_MESSAGE(p, mp) do { \ - *(p)->msg_inq.last = (mp); \ - (p)->msg_inq.last = &(mp)->next; \ - (p)->msg_inq.len++; \ -} while(0) +/* Add message last in private message queue */ +#define LINK_MESSAGE_PRIVQ(p, first_msg, last_msg, len) \ + do { \ + LINK_MESSAGE_IMPL(p, first_msg, last_msg, len, msg); \ + } while (0) + +/* Add message last_msg in message queue */ +#define LINK_MESSAGE(p, first_msg, last_msg, len) \ + LINK_MESSAGE_IMPL(p, first_msg, last_msg, len, msg_inq) + +#define ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p) \ + do { \ + if (p->msg_inq.first) { \ + *p->msg.last = p->msg_inq.first; \ + p->msg.last = p->msg_inq.last; \ + p->msg.len += p->msg_inq.len; \ + p->msg_inq.first = NULL; \ + p->msg_inq.last = &p->msg_inq.first; \ + p->msg_inq.len = 0; \ + } \ + } while (0) #else -#define ERTS_SMP_MSGQ_MV_INQ2PRIVQ(P) +#define ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p) -/* Add message last in message queue */ -#define LINK_MESSAGE(p, mp) LINK_MESSAGE_PRIVQ((p), (mp)) +/* Add message last_msg in message queue */ +#define LINK_MESSAGE(p, first_msg, last_msg, len) \ + do { \ + LINK_MESSAGE_IMPL(p, first_msg, last_msg, len, msg); \ + } while(0) #endif @@ -259,23 +275,30 @@ do { \ (HEAP_FRAG_P)->off_heap.overhead = 0; \ } while (0) +#ifdef USE_VM_PROBES +#define ERL_MESSAGE_DT_UTAG_INIT(MP) ERL_MESSAGE_DT_UTAG(MP) = NIL +#else +#define ERL_MESSAGE_DT_UTAG_INIT(MP) do{ } while (0) +#endif + +#define ERTS_INIT_MESSAGE(MP) \ + do { \ + (MP)->next = NULL; \ + ERL_MESSAGE_TERM(MP) = THE_NON_VALUE; \ + ERL_MESSAGE_TOKEN(MP) = NIL; \ + ERL_MESSAGE_DT_UTAG_INIT(MP); \ + MP->data.attached = NULL; \ + } while (0) + void init_message(void); ErlHeapFragment* new_message_buffer(Uint); ErlHeapFragment* erts_resize_message_buffer(ErlHeapFragment *, Uint, Eterm *, Uint); void free_message_buffer(ErlHeapFragment *); -void erts_queue_dist_message(Process*, ErtsProcLocks*, ErtsDistExternal *, Eterm); -#ifdef USE_VM_PROBES -void erts_queue_message_probe(Process*, ErtsProcLocks*, ErtsMessage*, - Eterm message, Eterm seq_trace_token, Eterm dt_utag); -#define erts_queue_message(RP,RL,BP,Msg,SEQ) \ - erts_queue_message_probe((RP),(RL),(BP),(Msg),(SEQ),NIL) -#else -void erts_queue_message(Process*, ErtsProcLocks*, ErtsMessage*, - Eterm message, Eterm seq_trace_token); -#define erts_queue_message_probe(RP,RL,BP,Msg,SEQ,TAG) \ - erts_queue_message((RP),(RL),(BP),(Msg),(SEQ)) -#endif +void erts_queue_dist_message(Process*, ErtsProcLocks, ErtsDistExternal *, Eterm, Eterm); +Sint erts_queue_message(Process*, ErtsProcLocks,ErtsMessage*, Eterm, Eterm); +Sint erts_queue_messages(Process*, ErtsProcLocks, + ErtsMessage*, ErtsMessage**, Uint, Eterm); void erts_deliver_exit_message(Eterm, Process*, ErtsProcLocks *, Eterm, Eterm); Sint erts_send_message(Process*, Process*, ErtsProcLocks*, Eterm, unsigned); void erts_link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp); @@ -354,9 +377,7 @@ ERTS_GLB_FORCE_INLINE ErtsMessage *erts_alloc_message(Uint sz, Eterm **hpp) if (sz == 0) { mp = erts_alloc_message_ref(); - mp->next = NULL; - ERL_MESSAGE_TERM(mp) = NIL; - mp->data.attached = NULL; + ERTS_INIT_MESSAGE(mp); if (hpp) *hpp = NULL; return mp; @@ -365,8 +386,7 @@ ERTS_GLB_FORCE_INLINE ErtsMessage *erts_alloc_message(Uint sz, Eterm **hpp) mp = erts_alloc(ERTS_ALC_T_MSG, sizeof(ErtsMessage) + (sz - 1)*sizeof(Eterm)); - mp->next = NULL; - ERL_MESSAGE_TERM(mp) = NIL; + ERTS_INIT_MESSAGE(mp); mp->data.attached = ERTS_MSG_COMBINED_HFRAG; ERTS_INIT_HEAP_FRAG(&mp->hfrag, sz, sz); diff --git a/erts/emulator/beam/erl_monitors.c b/erts/emulator/beam/erl_monitors.c index bd899fa2f9..910598690d 100644 --- a/erts/emulator/beam/erl_monitors.c +++ b/erts/emulator/beam/erl_monitors.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2004-2013. All Rights Reserved. + * Copyright Ericsson AB 2004-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_monitors.h b/erts/emulator/beam/erl_monitors.h index 901ed8940b..9e2beedea3 100644 --- a/erts/emulator/beam/erl_monitors.h +++ b/erts/emulator/beam/erl_monitors.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2004-2013. All Rights Reserved. + * Copyright Ericsson AB 2004-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_msacc.c b/erts/emulator/beam/erl_msacc.c index 71e3fd8b6e..0e625f213b 100644 --- a/erts/emulator/beam/erl_msacc.c +++ b/erts/emulator/beam/erl_msacc.c @@ -257,7 +257,7 @@ static void send_reply(ErtsMsAcc *msacc, ErtsMSAccReq *msaccrp) { if (msacc->unmanaged) erts_mtx_unlock(&msacc->mtx); - erts_queue_message(rp, &rp_locks, msgp, msg, NIL); + erts_queue_message(rp, rp_locks, msgp, msg, am_system); if (esdp && msaccrp->req_sched == esdp->no) rp_locks &= ~ERTS_PROC_LOCK_MAIN; diff --git a/erts/emulator/beam/erl_mtrace.c b/erts/emulator/beam/erl_mtrace.c index fdaf02f37a..e275867928 100644 --- a/erts/emulator/beam/erl_mtrace.c +++ b/erts/emulator/beam/erl_mtrace.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2003-2013. All Rights Reserved. + * Copyright Ericsson AB 2003-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_mtrace.h b/erts/emulator/beam/erl_mtrace.h index b34eab3c4a..776c70a819 100644 --- a/erts/emulator/beam/erl_mtrace.h +++ b/erts/emulator/beam/erl_mtrace.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2003-2009. All Rights Reserved. + * Copyright Ericsson AB 2003-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 8580bc2689..a695a028ba 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2009-2014. All Rights Reserved. + * Copyright Ericsson AB 2009-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -92,11 +92,12 @@ static ERTS_INLINE Eterm* alloc_heap(ErlNifEnv* env, unsigned need) } static Eterm* alloc_heap_heavy(ErlNifEnv* env, unsigned need, Eterm* hp) -{ +{ env->hp = hp; - if (env->heap_frag == NULL) { + if (env->heap_frag == NULL) { ASSERT(HEAP_LIMIT(env->proc) == env->hp_end); - HEAP_TOP(env->proc) = env->hp; + ASSERT(env->hp + need > env->hp_end); + HEAP_TOP(env->proc) = env->hp; } else { env->heap_frag->used_size = hp - env->heap_frag->mem; @@ -120,7 +121,8 @@ static ERTS_INLINE void ensure_heap(ErlNifEnv* env, unsigned may_need) } #endif -void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif) +void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif, + Process* tracee) { env->mod_nif = mod_nif; env->proc = p; @@ -130,17 +132,7 @@ void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif) env->fpe_was_unmasked = erts_block_fpe(); env->tmp_obj_list = NULL; env->exception_thrown = 0; -} - -static void pre_nif_noproc(ErlNifEnv* env, struct erl_module_nif* mod_nif) -{ - env->mod_nif = mod_nif; - env->proc = NULL; - env->hp = NULL; - env->hp_end = NULL; - env->heap_frag = NULL; - env->fpe_was_unmasked = erts_block_fpe(); - env->tmp_obj_list = NULL; + env->tracee = tracee; } /* Temporary object header, auto-deallocated when NIF returns @@ -180,13 +172,6 @@ void erts_post_nif(ErlNifEnv* env) free_tmp_objs(env); } -static void post_nif_noproc(ErlNifEnv* env) -{ - erts_unblock_fpe(env->fpe_was_unmasked); - free_tmp_objs(env); -} - - /* Flush out our cached heap pointers to allow an ordinary HAlloc */ static void flush_env(ErlNifEnv* env) @@ -209,6 +194,7 @@ static void flush_env(ErlNifEnv* env) */ static void cache_env(ErlNifEnv* env) { + env->heap_frag = MBUF(env->proc); if (env->heap_frag == NULL) { ASSERT(env->hp_end == HEAP_LIMIT(env->proc)); ASSERT(env->hp <= HEAP_TOP(env->proc)); @@ -216,10 +202,6 @@ static void cache_env(ErlNifEnv* env) env->hp = HEAP_TOP(env->proc); } else { - ASSERT(env->hp_end != HEAP_LIMIT(env->proc)); - ASSERT(env->hp_end - env->hp <= env->heap_frag->alloc_size); - env->heap_frag = MBUF(env->proc); - ASSERT(env->heap_frag != NULL); env->hp = env->heap_frag->mem + env->heap_frag->used_size; env->hp_end = env->heap_frag->mem + env->heap_frag->alloc_size; } @@ -250,18 +232,20 @@ struct enif_msg_environment_t Process phony_proc; }; -ErlNifEnv* enif_alloc_env(void) +static ERTS_INLINE void +setup_nif_env(struct enif_msg_environment_t* msg_env, + struct erl_module_nif* mod, + Process* tracee) { - struct enif_msg_environment_t* msg_env = - erts_alloc_fnf(ERTS_ALC_T_NIF, sizeof(struct enif_msg_environment_t)); Eterm* phony_heap = (Eterm*) msg_env; /* dummy non-NULL ptr */ - - msg_env->env.hp = phony_heap; + + msg_env->env.hp = phony_heap; msg_env->env.hp_end = phony_heap; msg_env->env.heap_frag = NULL; - msg_env->env.mod_nif = NULL; + msg_env->env.mod_nif = mod; msg_env->env.tmp_obj_list = NULL; msg_env->env.proc = &msg_env->phony_proc; + msg_env->env.exception_thrown = 0; memset(&msg_env->phony_proc, 0, sizeof(Process)); HEAP_START(&msg_env->phony_proc) = phony_heap; HEAP_TOP(&msg_env->phony_proc) = phony_heap; @@ -273,6 +257,14 @@ ErlNifEnv* enif_alloc_env(void) msg_env->phony_proc.space_verified = 0; msg_env->phony_proc.space_verified_from = NULL; #endif + msg_env->env.tracee = tracee; +} + +ErlNifEnv* enif_alloc_env(void) +{ + struct enif_msg_environment_t* msg_env = + erts_alloc_fnf(ERTS_ALC_T_NIF, sizeof(struct enif_msg_environment_t)); + setup_nif_env(msg_env, NULL, NULL); return &msg_env->env; } void enif_free_env(ErlNifEnv* env) @@ -281,6 +273,20 @@ void enif_free_env(ErlNifEnv* env) erts_free(ERTS_ALC_T_NIF, env); } +static ERTS_INLINE void pre_nif_noproc(struct enif_msg_environment_t* msg_env, + struct erl_module_nif* mod, + Process* tracee) +{ + setup_nif_env(msg_env, mod, tracee); + msg_env->env.fpe_was_unmasked = erts_block_fpe(); +} + +static ERTS_INLINE void post_nif_noproc(struct enif_msg_environment_t* msg_env) +{ + erts_unblock_fpe(msg_env->env.fpe_was_unmasked); + enif_clear_env(&msg_env->env); +} + static ERTS_INLINE void clear_offheap(ErlOffHeap* oh) { oh->first = NULL; @@ -307,18 +313,109 @@ void enif_clear_env(ErlNifEnv* env) ASSERT(!is_offheap(&MSO(p))); free_tmp_objs(env); } -int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, - ErlNifEnv* msg_env, ERL_NIF_TERM msg) + +#ifdef ERTS_SMP +#ifdef DEBUG +static int enif_send_delay = 0; +#define ERTS_FORCE_ENIF_SEND_DELAY() (enif_send_delay++ % 2 == 0) +#else +#ifdef ERTS_PROC_LOCK_OWN_IMPL +#define ERTS_FORCE_ENIF_SEND_DELAY() 0 +#else +/* + * We always schedule messages if we do not use our own + * process lock implementation, as if we try to do a trylock on + * a lock that might already be locked by the same thread. + * And what happens then with different mutex implementations + * is not always guaranteed. + */ +#define ERTS_FORCE_ENIF_SEND_DELAY() 1 +#endif +#endif + +int erts_flush_trace_messages(Process *c_p, ErtsProcLocks c_p_locks) +{ + ErlTraceMessageQueue *msgq, **last_msgq; + int reds = 0; + + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_TRACE); + + msgq = c_p->trace_msg_q; + + if (!msgq) + goto error; + + do { + Process* rp; + ErtsProcLocks rp_locks; + ErtsMessage *first, **last; + Uint len; + + first = msgq->first; + last = msgq->last; + len = msgq->len; + msgq->first = NULL; + msgq->last = &msgq->first; + msgq->len = 0; + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_TRACE); + + ASSERT(len != 0); + + rp = erts_proc_lookup(msgq->receiver); + if (rp) { + rp_locks = 0; + if (rp->common.id == c_p->common.id) + rp_locks = c_p_locks; + erts_queue_messages(rp, rp_locks, first, last, len, c_p->common.id); + if (rp->common.id == c_p->common.id) + rp_locks &= ~c_p_locks; + if (rp_locks) + erts_smp_proc_unlock(rp, rp_locks); + reds += len; + } else { + erts_cleanup_messages(first); + } + reds += 1; + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_TRACE); + msgq = msgq->next; + } while (msgq); + + last_msgq = &c_p->trace_msg_q; + + while (*last_msgq) { + msgq = *last_msgq; + if (msgq->len == 0) { + *last_msgq = msgq->next; + erts_free(ERTS_ALC_T_TRACE_MSG_QUEUE, msgq); + } else { + last_msgq = &msgq->next; + } + } + +error: + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_TRACE); + + return reds; +} + +#endif + +int +enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, + ErlNifEnv* msg_env, ERL_NIF_TERM msg) { struct enif_msg_environment_t* menv = (struct enif_msg_environment_t*)msg_env; ErtsProcLocks rp_locks = 0; +#ifdef ERTS_SMP + ErtsProcLocks lc_locks = 0; +#endif Process* rp; Process* c_p; ErtsMessage *mp; - ErlHeapFragment* frags; Eterm receiver = to_pid->pid; int flush_me = 0; - int scheduler = erts_get_scheduler_id() != 0; + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + int scheduler = esdp ? esdp->no : 0; if (env != NULL) { c_p = env->proc; @@ -333,40 +430,122 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, #else erts_exit(ERTS_ABORT_EXIT,"enif_send: env==NULL on non-SMP VM"); #endif - } + } rp = (scheduler ? erts_proc_lookup(receiver) : erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN, receiver, rp_locks, ERTS_P2P_FLG_INC_REFC)); + if (rp == NULL) { ASSERT(env == NULL || receiver != c_p->common.id); return 0; } - flush_env(msg_env); - frags = menv->env.heap_frag; - ASSERT(frags == MBUF(&menv->phony_proc)); - if (frags != NULL) { - /* Move all offheap's from phony proc to the first fragment. - Quick and dirty... */ - ASSERT(!is_offheap(&frags->off_heap)); - frags->off_heap = MSO(&menv->phony_proc); - clear_offheap(&MSO(&menv->phony_proc)); - menv->env.heap_frag = NULL; - MBUF(&menv->phony_proc) = NULL; - } - ASSERT(!is_offheap(&MSO(&menv->phony_proc))); - - if (flush_me) { - flush_env(env); /* Needed for ERTS_HOLE_CHECK */ - } - mp = erts_alloc_message(0, NULL); - mp->data.heap_frag = frags; - erts_queue_message(rp, &rp_locks, mp, msg, am_undefined); + if (menv) { + flush_env(msg_env); + mp = erts_alloc_message(0, NULL); + mp->data.heap_frag = menv->env.heap_frag; + ASSERT(mp->data.heap_frag == MBUF(&menv->phony_proc)); + if (mp->data.heap_frag != NULL) { + /* Move all offheap's from phony proc to the first fragment. + Quick and dirty... */ + ASSERT(!is_offheap(&mp->data.heap_frag->off_heap)); + mp->data.heap_frag->off_heap = MSO(&menv->phony_proc); + clear_offheap(&MSO(&menv->phony_proc)); + menv->env.heap_frag = NULL; + MBUF(&menv->phony_proc) = NULL; + } + } else { + Uint sz = size_object(msg); + Eterm *hp; + mp = erts_alloc_message(sz, &hp); + msg = copy_struct(msg, sz, &hp, &mp->hfrag.off_heap); + ASSERT(hp == mp->hfrag.mem+mp->hfrag.used_size); + } + + ERL_MESSAGE_TERM(mp) = msg; + + if (flush_me) { + flush_env(env); /* Needed for ERTS_HOLE_CHECK */ + } + + if (!env || !env->tracee) { + + if (c_p && IS_TRACED_FL(c_p, F_TRACE_SEND)) + trace_send(c_p, receiver, msg); + } +#ifdef ERTS_SMP + else { + /* This clause is taken when the nif is called in the context + of a traced process. We do not know which locks we have + so we have to do a try lock and if that fails we enqueue + the message in a special trace message output queue of the + tracee */ + ErlTraceMessageQueue *msgq; + Process *t_p = env->tracee; + + + erts_smp_proc_lock(t_p, ERTS_PROC_LOCK_TRACE); + + msgq = t_p->trace_msg_q; + + while (msgq != NULL) { + if (msgq->receiver == receiver) { + break; + } + msgq = msgq->next; + } + +#ifdef ERTS_ENABLE_LOCK_CHECK + lc_locks = erts_proc_lc_my_proc_locks(rp); + rp_locks |= lc_locks; +#endif + if (ERTS_FORCE_ENIF_SEND_DELAY() || msgq || + rp_locks & ERTS_PROC_LOCK_MSGQ || + erts_smp_proc_trylock(rp, ERTS_PROC_LOCK_MSGQ) == EBUSY) { + + if (!msgq) { + + msgq = erts_alloc(ERTS_ALC_T_TRACE_MSG_QUEUE, + sizeof(ErlTraceMessageQueue)); + msgq->receiver = receiver; + msgq->first = mp; + msgq->last = &mp->next; + msgq->len = 1; + + /* Insert in linked list */ + msgq->next = t_p->trace_msg_q; + t_p->trace_msg_q = msgq; + + erts_smp_proc_unlock(t_p, ERTS_PROC_LOCK_TRACE); + + erts_schedule_flush_trace_messages(t_p->common.id); + + } else { + msgq->len++; + *msgq->last = mp; + msgq->last = &mp->next; + erts_smp_proc_unlock(t_p, ERTS_PROC_LOCK_TRACE); + } + goto done; + } else { + erts_smp_proc_unlock(t_p, ERTS_PROC_LOCK_TRACE); + rp_locks &= ~ERTS_PROC_LOCK_TRACE; + rp_locks |= ERTS_PROC_LOCK_MSGQ; + } + } +#endif /* ERTS_SMP */ + + erts_queue_message(rp, rp_locks, mp, msg, + c_p ? c_p->common.id : am_undefined); + +#ifdef ERTS_SMP +done: if (c_p == rp) rp_locks &= ~ERTS_PROC_LOCK_MAIN; - if (rp_locks) - erts_smp_proc_unlock(rp, rp_locks); + if (rp_locks & ~lc_locks) + erts_smp_proc_unlock(rp, rp_locks & ~lc_locks); +#endif if (!scheduler) erts_proc_dec_refc(rp); if (flush_me) { @@ -375,6 +554,32 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, return 1; } +int +enif_port_command(ErlNifEnv *env, const ErlNifPort* to_port, + ErlNifEnv *msg_env, ERL_NIF_TERM msg) +{ + + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + int scheduler = esdp ? esdp->no : 0; + Port *prt; + + if (scheduler == 0 || !env) + return 0; + + prt = erts_port_lookup(to_port->port_id, + (erts_port_synchronous_ops + ? ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP + : ERTS_PORT_SFLGS_INVALID_LOOKUP)); + + if (!prt) + return 0; + + if (IS_TRACED_FL(prt, F_TRACE_RECEIVE)) + trace_port_receive(prt, env->proc->common.id, am_command, msg); + + return erts_port_output_async(prt, env->proc->common.id, msg); +} + ERL_NIF_TERM enif_make_copy(ErlNifEnv* dst_env, ERL_NIF_TERM src_term) { Uint sz; @@ -404,12 +609,28 @@ static int is_offheap(const ErlOffHeap* oh) ErlNifPid* enif_self(ErlNifEnv* caller_env, ErlNifPid* pid) { + if (caller_env->proc->common.id == ERTS_INVALID_PID) + return NULL; pid->pid = caller_env->proc->common.id; return pid; } + int enif_get_local_pid(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifPid* pid) { - return is_internal_pid(term) ? (pid->pid=term, 1) : 0; + if (is_internal_pid(term)) { + pid->pid=term; + return 1; + } + return 0; +} + +int enif_get_local_port(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifPort* port) +{ + if (is_internal_port(term)) { + port->port_id=term; + return 1; + } + return 0; } int enif_is_atom(ErlNifEnv* env, ERL_NIF_TERM term) @@ -622,6 +843,68 @@ unsigned char* enif_make_new_binary(ErlNifEnv* env, size_t size, return binary_bytes(*termp); } +int enif_term_to_binary(ErlNifEnv *dst_env, ERL_NIF_TERM term, + ErlNifBinary *bin) +{ + Sint size; + byte *bp; + Binary* refbin; + + size = erts_encode_ext_size(term); + if (!enif_alloc_binary(size, bin)) + return 0; + + refbin = bin->ref_bin; + + bp = bin->data; + + erts_encode_ext(term, &bp); + + bin->size = bp - bin->data; + refbin->orig_size = bin->size; + + ASSERT(bin->data + bin->size == bp); + + return 1; +} + +size_t enif_binary_to_term(ErlNifEnv *dst_env, + const unsigned char* data, + size_t data_sz, + ERL_NIF_TERM *term, + ErlNifBinaryToTerm opts) +{ + Sint size; + ErtsHeapFactory factory; + byte *bp = (byte*) data; + + ERTS_CT_ASSERT(ERL_NIF_BIN2TERM_SAFE == ERTS_DIST_EXT_BTT_SAFE); + + if (opts & ~ERL_NIF_BIN2TERM_SAFE) { + return 0; + } + if ((size = erts_decode_ext_size(bp, data_sz)) < 0) + return 0; + + if (size > 0) { + flush_env(dst_env); + erts_factory_proc_prealloc_init(&factory, dst_env->proc, size); + } else { + erts_factory_dummy_init(&factory); + } + + *term = erts_decode_ext(&factory, &bp, (Uint32)opts); + + if (is_non_value(*term)) { + return 0; + } + erts_factory_close(&factory); + cache_env(dst_env); + + ASSERT(bp > data); + return bp - data; +} + int enif_is_identical(Eterm lhs, Eterm rhs) { return EQ(lhs,rhs); @@ -1158,6 +1441,103 @@ int enif_make_reverse_list(ErlNifEnv* env, ERL_NIF_TERM term, ERL_NIF_TERM *list return 1; } +int enif_is_process_alive(ErlNifEnv* env, ErlNifPid *proc) +{ + ErtsProcLocks rp_locks = 0; /* We don't need any locks, + just to check if it is alive */ + Eterm target = proc->pid; + Process* rp; + Process* c_p; + int scheduler = erts_get_scheduler_id() != 0; + + if (env != NULL) { + c_p = env->proc; + if (target == c_p->common.id) { + /* We are alive! */ + return 1; + } + } + else { +#ifdef ERTS_SMP + c_p = NULL; +#else + erts_exit(ERTS_ABORT_EXIT,"enif_is_process_alive: " + "env==NULL on non-SMP VM"); +#endif + } + + rp = (scheduler + ? erts_proc_lookup(target) + : erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN, + target, rp_locks, ERTS_P2P_FLG_INC_REFC)); + if (rp == NULL) { + ASSERT(env == NULL || target != c_p->common.id); + return 0; + } else { + if (!scheduler) + erts_proc_dec_refc(rp); + return 1; + } +} + +int enif_is_port_alive(ErlNifEnv *env, ErlNifPort *port) +{ + /* only allowed if called from scheduler */ + if (erts_get_scheduler_id() == 0) + erts_exit(ERTS_ABORT_EXIT,"enif_is_port_alive: called from non-scheduler"); + + return erts_port_lookup( + port->port_id, + (erts_port_synchronous_ops + ? ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP + : ERTS_PORT_SFLGS_INVALID_LOOKUP)) != NULL; +} + +ERL_NIF_TERM +enif_now_time(ErlNifEnv *env) +{ + Uint mega, sec, micro; + Eterm *hp; + get_now(&mega, &sec, µ); + hp = alloc_heap(env, 4); + return TUPLE3(hp, make_small(mega), make_small(sec), make_small(micro)); +} + +ERL_NIF_TERM +enif_cpu_time(ErlNifEnv *env) +{ +#ifdef HAVE_ERTS_NOW_CPU + Uint mega, sec, micro; + Eterm *hp; + erts_get_now_cpu(&mega, &sec, µ); + hp = alloc_heap(env, 4); + return TUPLE3(hp, make_small(mega), make_small(sec), make_small(micro)); +#else + return enif_make_badarg(env); +#endif +} + +ERL_NIF_TERM +enif_make_unique_integer(ErlNifEnv *env, ErlNifUniqueInteger properties) +{ + int monotonic = properties & ERL_NIF_UNIQUE_MONOTONIC; + int positive = properties & ERL_NIF_UNIQUE_POSITIVE; + Eterm *hp; + Uint hsz; + + if (monotonic) { + Sint64 raw_unique = erts_raw_get_unique_monotonic_integer(); + hsz = erts_raw_unique_monotonic_integer_heap_size(raw_unique, positive); + hp = alloc_heap(env, hsz); + return erts_raw_make_unique_monotonic_integer_value(&hp, raw_unique, positive); + } else { + Uint64 raw_unique[ERTS_UNIQUE_INT_RAW_VALUES]; + erts_raw_get_unique_integer(raw_unique); + hsz = erts_raw_unique_integer_heap_size(raw_unique, positive); + hp = alloc_heap(env, hsz); + return erts_raw_make_unique_integer(&hp, raw_unique, positive); + } +} ErlNifMutex* enif_mutex_create(char *name) { return erl_drv_mutex_create(name); } void enif_mutex_destroy(ErlNifMutex *mtx) { erl_drv_mutex_destroy(mtx); } @@ -1284,10 +1664,10 @@ static void close_lib(struct erl_module_nif* lib) ASSERT(erts_refc_read(&lib->rt_dtor_cnt,0) == 0); if (lib->entry != NULL && lib->entry->unload != NULL) { - ErlNifEnv env; - pre_nif_noproc(&env, lib); - lib->entry->unload(&env, lib->priv_data); - post_nif_noproc(&env); + struct enif_msg_environment_t msg_env; + pre_nif_noproc(&msg_env, lib, NULL); + lib->entry->unload(&msg_env.env, lib->priv_data); + post_nif_noproc(&msg_env); } if (!erts_is_static_nif(lib->handle)) erts_sys_ddll_close(lib->handle); @@ -1433,10 +1813,10 @@ static void nif_resource_dtor(Binary* bin) ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == &nif_resource_dtor); if (type->dtor != NULL) { - ErlNifEnv env; - pre_nif_noproc(&env, type->owner); - type->dtor(&env,resource->data); - post_nif_noproc(&env); + struct enif_msg_environment_t msg_env; + pre_nif_noproc(&msg_env, type->owner, NULL); + type->dtor(&msg_env.env, resource->data); + post_nif_noproc(&msg_env); } if (erts_refc_dectest(&type->refc, 0) == 0) { ASSERT(type->next == NULL); @@ -1653,7 +2033,7 @@ allocate_nif_sched_data(Process* proc, int argc) ep->exp.addressv[i] = &ep->exp.code[3]; } ep->exp.code[3] = (BeamInstr) em_call_nif; - (void) ERTS_PROC_SET_NIF_TRAP_EXPORT(proc, ERTS_PROC_LOCK_MAIN, ep); + (void) ERTS_PROC_SET_NIF_TRAP_EXPORT(proc, ep); return ep; } @@ -2064,14 +2444,13 @@ int enif_make_map_remove(ErlNifEnv* env, Eterm key, Eterm *map_out) { - int res; if (!is_map(map_in)) { return 0; } flush_env(env); - res = erts_maps_remove(env->proc, key, map_in, map_out); + (void) erts_maps_take(env->proc, key, map_in, map_out, NULL); cache_env(env); - return res; + return 1; } int enif_map_iterator_create(ErlNifEnv *env, @@ -2402,7 +2781,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) ErlNifEntry* entry = NULL; ErlNifEnv env; int i, err, encoding; - Module* mod; + Module* module_p; Eterm mod_atom; const Atom* mod_atomp; Eterm f_atom; @@ -2412,6 +2791,8 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) int veto; struct erl_module_nif* lib = NULL; int reload_warning = 0; + struct erl_module_instance* this_mi; + struct erl_module_instance* prev_mi; encoding = erts_get_native_filename_encoding(); if (encoding == ERL_FILENAME_WIN_WCHAR) { @@ -2445,21 +2826,29 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) ASSERT(caller != NULL); mod_atom = caller[0]; ASSERT(is_atom(mod_atom)); - mod=erts_get_module(mod_atom, erts_active_code_ix()); - ASSERT(mod != NULL); + module_p = erts_get_module(mod_atom, erts_active_code_ix()); + ASSERT(module_p != NULL); mod_atomp = atom_tab(atom_val(mod_atom)); init_func = erts_static_nif_get_nif_init((char*)mod_atomp->name, mod_atomp->len); if (init_func != NULL) handle = init_func; - if (!in_area(caller, mod->curr.code_hdr, mod->curr.code_length)) { - ASSERT(in_area(caller, mod->old.code_hdr, mod->old.code_length)); + if (in_area(caller, module_p->old.code_hdr, module_p->old.code_length)) { + if (module_p->old.code_hdr->on_load_function_ptr) { + this_mi = &module_p->old; + prev_mi = &module_p->curr; + } else { + ret = load_nif_error(BIF_P, "old_code", "Calling load_nif from old " + "module '%T' not allowed", mod_atom); + goto error; + } + } else { + this_mi = &module_p->curr; + prev_mi = &module_p->old; + } - ret = load_nif_error(BIF_P, "old_code", "Calling load_nif from old " - "module '%T' not allowed", mod_atom); - } - else if (init_func == NULL && + if (init_func == NULL && (err=erts_sys_ddll_open(lib_name, &handle, &errdesc)) != ERL_DE_NO_ERROR) { const char slogan[] = "Failed to load NIF library"; if (strstr(errdesc.str, lib_name) != NULL) { @@ -2508,7 +2897,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) for (i=0; i < entry->num_of_funcs && ret==am_ok; i++) { BeamInstr** code_pp; if (!erts_atom_get(f->name, sys_strlen(f->name), &f_atom, ERTS_ATOM_ENC_LATIN1) - || (code_pp = get_func_pp(mod->curr.code_hdr, f_atom, f->arity))==NULL) { + || (code_pp = get_func_pp(this_mi->code_hdr, f_atom, f->arity))==NULL) { ret = load_nif_error(BIF_P,bad_lib,"Function not found %T:%s/%u", mod_atom, f->name, f->arity); } @@ -2559,9 +2948,9 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) erts_refc_init(&lib->rt_cnt, 0); erts_refc_init(&lib->rt_dtor_cnt, 0); ASSERT(opened_rt_list == NULL); - lib->mod = mod; + lib->mod = module_p; env.mod_nif = lib; - if (mod->curr.nif != NULL) { /*************** Reload ******************/ + if (this_mi->nif != NULL) { /*************** Reload ******************/ /* * Repeated load_nif calls from same Erlang module instance ("reload") * is deprecated and was only ment as a development feature not to @@ -2569,16 +2958,16 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) */ int k, old_incr = 0; ErlNifFunc* old_func; - lib->priv_data = mod->curr.nif->priv_data; + lib->priv_data = this_mi->nif->priv_data; - ASSERT(mod->curr.nif->entry != NULL); + ASSERT(this_mi->nif->entry != NULL); if (entry->reload == NULL) { ret = load_nif_error(BIF_P,reload,"Reload not supported by this NIF library."); goto error; } /* Check that no NIF is removed */ - old_func = mod->curr.nif->entry->funcs; - for (k=0; k < mod->curr.nif->entry->num_of_funcs; k++) { + old_func = this_mi->nif->entry->funcs; + for (k=0; k < this_mi->nif->entry->num_of_funcs; k++) { int incr = 0; ErlNifFunc* f = entry->funcs; for (i=0; i < entry->num_of_funcs; i++) { @@ -2594,9 +2983,9 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) old_func->name, old_func->arity); goto error; } - old_func = next_func(mod->curr.nif->entry, &old_incr, old_func); + old_func = next_func(this_mi->nif->entry, &old_incr, old_func); } - erts_pre_nif(&env, BIF_P, lib); + erts_pre_nif(&env, BIF_P, lib, NULL); veto = entry->reload(&env, &lib->priv_data, BIF_ARG_2); erts_post_nif(&env); if (veto) { @@ -2604,31 +2993,31 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) } else { commit_opened_resource_types(lib); - mod->curr.nif->entry = NULL; /* to prevent 'unload' callback */ - erts_unload_nif(mod->curr.nif); + this_mi->nif->entry = NULL; /* to prevent 'unload' callback */ + erts_unload_nif(this_mi->nif); reload_warning = 1; } } else { lib->priv_data = NULL; - if (mod->old.nif != NULL) { /**************** Upgrade ***************/ - void* prev_old_data = mod->old.nif->priv_data; + if (prev_mi->nif != NULL) { /**************** Upgrade ***************/ + void* prev_old_data = prev_mi->nif->priv_data; if (entry->upgrade == NULL) { ret = load_nif_error(BIF_P, upgrade, "Upgrade not supported by this NIF library."); goto error; } - erts_pre_nif(&env, BIF_P, lib); - veto = entry->upgrade(&env, &lib->priv_data, &mod->old.nif->priv_data, BIF_ARG_2); + erts_pre_nif(&env, BIF_P, lib, NULL); + veto = entry->upgrade(&env, &lib->priv_data, &prev_mi->nif->priv_data, BIF_ARG_2); erts_post_nif(&env); if (veto) { - mod->old.nif->priv_data = prev_old_data; + prev_mi->nif->priv_data = prev_old_data; ret = load_nif_error(BIF_P, upgrade, "Library upgrade-call unsuccessful."); } else commit_opened_resource_types(lib); } else if (entry->load != NULL) { /********* Initial load ***********/ - erts_pre_nif(&env, BIF_P, lib); + erts_pre_nif(&env, BIF_P, lib, NULL); veto = entry->load(&env, &lib->priv_data, BIF_ARG_2); erts_post_nif(&env); if (veto) { @@ -2646,12 +3035,12 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) int incr = 0; ErlNifFunc* f = entry->funcs; - mod->curr.nif = lib; + this_mi->nif = lib; for (i=0; i < entry->num_of_funcs; i++) { BeamInstr* code_ptr; erts_atom_get(f->name, sys_strlen(f->name), &f_atom, ERTS_ATOM_ENC_LATIN1); - code_ptr = *get_func_pp(mod->curr.code_hdr, f_atom, f->arity); + code_ptr = *get_func_pp(this_mi->code_hdr, f_atom, f->arity); if (code_ptr[1] == 0) { code_ptr[5+0] = (BeamInstr) BeamOp(op_call_nif); @@ -2715,6 +3104,9 @@ erts_unload_nif(struct erl_module_nif* lib) ASSERT(erts_smp_thr_progress_is_blocking()); ASSERT(lib != NULL); ASSERT(lib->mod != NULL); + + erts_tracer_nif_clear(); + for (rt = resource_type_list.next; rt != &resource_type_list; rt = next) { @@ -2757,6 +3149,76 @@ void erl_nif_init() resource_type_list.owner = NULL; resource_type_list.module = THE_NON_VALUE; resource_type_list.name = THE_NON_VALUE; + +} + +int erts_nif_get_funcs(struct erl_module_nif* mod, + ErlNifFunc **funcs) +{ + *funcs = mod->entry->funcs; + return mod->entry->num_of_funcs; +} + +Eterm erts_nif_call_function(Process *p, Process *tracee, + struct erl_module_nif* mod, + ErlNifFunc *fun, int argc, Eterm *argv) +{ + Eterm nif_result; +#ifdef DEBUG + /* Verify that function is part of this module */ + int i; + for (i = 0; i < mod->entry->num_of_funcs; i++) + if (fun == &(mod->entry->funcs[i])) + break; + ASSERT(i < mod->entry->num_of_funcs); + if (p) + ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(p) & ERTS_PROC_LOCK_MAIN + || erts_smp_thr_progress_is_blocking()); +#endif + if (p) { + /* This is almost a normal nif call like in beam_emu, + except that any heap fragment created in the nif will be + discarded without checking if anything in it is live. + This is because we cannot do a GC here as we don't know + the number of live registers that have to be preserved. + This means that any heap part of the returned term may + not be used outside this function. */ + struct enif_environment_t env; + ErlHeapFragment *orig_hf = MBUF(p); + ErlOffHeap orig_oh = MSO(p); + ASSERT(is_internal_pid(p->common.id)); + MBUF(p) = NULL; + clear_offheap(&MSO(p)); + + erts_pre_nif(&env, p, mod, tracee); + nif_result = (*fun->fptr)(&env, argc, argv); + if (env.exception_thrown) + nif_result = THE_NON_VALUE; + erts_post_nif(&env); + + /* Free any offheap and heap fragments created in nif */ + if (MSO(p).first) { + erts_cleanup_offheap(&MSO(p)); + clear_offheap(&MSO(p)); + } + if (MBUF(p)) + free_message_buffer(MBUF(p)); + + /* restore original heap fragment list */ + MBUF(p) = orig_hf; + MSO(p) = orig_oh; + } else { + /* Nif call was done without a process context, + so we create a phony one. */ + struct enif_msg_environment_t msg_env; + pre_nif_noproc(&msg_env, mod, tracee); + nif_result = (*fun->fptr)(&msg_env.env, argc, argv); + if (msg_env.env.exception_thrown) + nif_result = THE_NON_VALUE; + post_nif_noproc(&msg_env); + } + + return nif_result; } #ifdef USE_VM_PROBES diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index c9c8b1d0af..3964f7f679 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2009-2014. All Rights Reserved. + * Copyright Ericsson AB 2009-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -87,16 +87,16 @@ typedef ErlNifSInt64 ErlNifTime; #define ERL_NIF_TIME_ERROR ((ErlNifSInt64) ERTS_NAPI_TIME_ERROR__) typedef enum { - ERL_NIF_SEC = ERTS_NAPI_SEC__, - ERL_NIF_MSEC = ERTS_NAPI_MSEC__, - ERL_NIF_USEC = ERTS_NAPI_USEC__, - ERL_NIF_NSEC = ERTS_NAPI_NSEC__ + ERL_NIF_SEC = ERTS_NAPI_SEC__, + ERL_NIF_MSEC = ERTS_NAPI_MSEC__, + ERL_NIF_USEC = ERTS_NAPI_USEC__, + ERL_NIF_NSEC = ERTS_NAPI_NSEC__ } ErlNifTimeUnit; struct enif_environment_t; typedef struct enif_environment_t ErlNifEnv; -typedef struct +typedef struct enif_func_t { const char* name; unsigned arity; @@ -150,7 +150,12 @@ typedef enum typedef struct { ERL_NIF_TERM pid; /* internal, may change */ -}ErlNifPid; +} ErlNifPid; + +typedef struct +{ + ERL_NIF_TERM port_id; /* internal, may change */ +}ErlNifPort; typedef ErlDrvSysInfo ErlNifSysInfo; @@ -197,6 +202,15 @@ typedef enum { ERL_NIF_MAP_ITERATOR_TAIL = ERL_NIF_MAP_ITERATOR_LAST } ErlNifMapIteratorEntry; +typedef enum { + ERL_NIF_UNIQUE_POSITIVE = (1 << 0), + ERL_NIF_UNIQUE_MONOTONIC = (1 << 1) +} ErlNifUniqueInteger; + +typedef enum { + ERL_NIF_BIN2TERM_SAFE = 0x20000000 +} ErlNifBinaryToTerm; + #if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) # define ERL_NIF_API_FUNC_DECL(RET_TYPE, NAME, ARGS) RET_TYPE (*NAME) ARGS typedef struct { diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h index 1448a508a2..a5acd86551 100644 --- a/erts/emulator/beam/erl_nif_api_funcs.h +++ b/erts/emulator/beam/erl_nif_api_funcs.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2009-2014. All Rights Reserved. + * Copyright Ericsson AB 2009-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -163,6 +163,15 @@ ERL_NIF_API_FUNC_DECL(int,enif_getenv,(const char* key, char* value, size_t* val ERL_NIF_API_FUNC_DECL(ErlNifTime, enif_monotonic_time, (ErlNifTimeUnit)); ERL_NIF_API_FUNC_DECL(ErlNifTime, enif_time_offset, (ErlNifTimeUnit)); ERL_NIF_API_FUNC_DECL(ErlNifTime, enif_convert_time_unit, (ErlNifTime, ErlNifTimeUnit, ErlNifTimeUnit)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM, enif_now_time, (ErlNifEnv *env)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM, enif_cpu_time, (ErlNifEnv *env)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM, enif_make_unique_integer, (ErlNifEnv *env, ErlNifUniqueInteger properties)); +ERL_NIF_API_FUNC_DECL(int, enif_is_process_alive, (ErlNifEnv *env, ErlNifPid *pid)); +ERL_NIF_API_FUNC_DECL(int, enif_is_port_alive, (ErlNifEnv *env, ErlNifPort *port_id)); +ERL_NIF_API_FUNC_DECL(int, enif_get_local_port, (ErlNifEnv* env, ERL_NIF_TERM, ErlNifPort* port_id)); +ERL_NIF_API_FUNC_DECL(int, enif_term_to_binary, (ErlNifEnv *env, ERL_NIF_TERM term, ErlNifBinary *bin)); +ERL_NIF_API_FUNC_DECL(size_t, enif_binary_to_term, (ErlNifEnv *env, const unsigned char* data, size_t sz, ERL_NIF_TERM *term, unsigned int opts)); +ERL_NIF_API_FUNC_DECL(int, enif_port_command, (ErlNifEnv *env, const ErlNifPort* to_port, ErlNifEnv *msg_env, ERL_NIF_TERM msg)); /* ** ADD NEW ENTRIES HERE (before this comment) !!! @@ -318,6 +327,15 @@ ERL_NIF_API_FUNC_DECL(int,enif_is_on_dirty_scheduler,(ErlNifEnv*)); # define enif_monotonic_time ERL_NIF_API_FUNC_MACRO(enif_monotonic_time) # define enif_time_offset ERL_NIF_API_FUNC_MACRO(enif_time_offset) # define enif_convert_time_unit ERL_NIF_API_FUNC_MACRO(enif_convert_time_unit) +# define enif_now_time ERL_NIF_API_FUNC_MACRO(enif_now_time) +# define enif_cpu_time ERL_NIF_API_FUNC_MACRO(enif_cpu_time) +# define enif_make_unique_integer ERL_NIF_API_FUNC_MACRO(enif_make_unique_integer) +# define enif_is_process_alive ERL_NIF_API_FUNC_MACRO(enif_is_process_alive) +# define enif_is_port_alive ERL_NIF_API_FUNC_MACRO(enif_is_port_alive) +# define enif_get_local_port ERL_NIF_API_FUNC_MACRO(enif_get_local_port) +# define enif_term_to_binary ERL_NIF_API_FUNC_MACRO(enif_term_to_binary) +# define enif_binary_to_term ERL_NIF_API_FUNC_MACRO(enif_binary_to_term) +# define enif_port_command ERL_NIF_API_FUNC_MACRO(enif_port_command) /* ** ADD NEW ENTRIES HERE (before this comment) diff --git a/erts/emulator/beam/erl_node_container_utils.h b/erts/emulator/beam/erl_node_container_utils.h index b6a3531e28..0c76c6fe7d 100644 --- a/erts/emulator/beam/erl_node_container_utils.h +++ b/erts/emulator/beam/erl_node_container_utils.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2013. All Rights Reserved. + * Copyright Ericsson AB 2001-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index 8617f42d7b..646f786651 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2013. All Rights Reserved. + * Copyright Ericsson AB 2001-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -497,31 +497,7 @@ node_table_hash(void *venp) Uint32 cre = ((ErlNode *) venp)->creation; HashValue h = atom_tab(atom_val(((ErlNode *) venp)->sysname))->slot.bucket.hvalue; - h *= PRIME0; - h += cre & 0xff; - -#if MAX_CREATION >= (1 << 8) - h *= PRIME1; - h += (cre >> 8) & 0xff; -#endif - -#if MAX_CREATION >= (1 << 16) - h *= PRIME2; - h += (cre >> 16) & 0xff; -#endif - -#if MAX_CREATION >= (1 << 24) - h *= PRIME3; - h += (cre >> 24) & 0xff; -#endif - -#if 0 -/* XXX Problems in older versions of GCC */ - #if MAX_CREATION >= (1UL << 32) - #error "MAX_CREATION larger than size of expected creation storage (Uint32)" - #endif -#endif - return h; + return (h + cre) * PRIME0; } static int @@ -599,7 +575,7 @@ erts_node_table_info(int to, void *to_arg) } -ErlNode *erts_find_or_insert_node(Eterm sysname, Uint creation) +ErlNode *erts_find_or_insert_node(Eterm sysname, Uint32 creation) { ErlNode *res; ErlNode ne; diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h index fb2f2a5407..7a4434acbf 100644 --- a/erts/emulator/beam/erl_node_tables.h +++ b/erts/emulator/beam/erl_node_tables.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2012. All Rights Reserved. + * Copyright Ericsson AB 2001-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -182,7 +182,7 @@ Uint erts_dist_table_size(void); void erts_dist_table_info(int, void *); void erts_set_dist_entry_not_connected(DistEntry *); void erts_set_dist_entry_connected(DistEntry *, Eterm, Uint); -ErlNode *erts_find_or_insert_node(Eterm, Uint); +ErlNode *erts_find_or_insert_node(Eterm, Uint32); void erts_schedule_delete_node(ErlNode *); void erts_set_this_node(Eterm, Uint); Uint erts_node_table_size(void); diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h index fa97707a87..c2588e718d 100644 --- a/erts/emulator/beam/erl_port.h +++ b/erts/emulator/beam/erl_port.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2012-2013. All Rights Reserved. + * Copyright Ericsson AB 2012-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -185,7 +185,7 @@ struct _erl_drv_port { int control_flags; /* Flags for port_control() */ ErlDrvPDL port_data_lock; - ErtsPrtSD *psd; /* Port specific data */ + erts_smp_atomic_t psd; /* Port specific data */ int reds; /* Only used while executing driver callbacks */ struct { @@ -252,22 +252,51 @@ ERTS_GLB_INLINE void *erts_prtsd_set(Port *p, int ix, void *new); ERTS_GLB_INLINE void * erts_prtsd_get(Port *prt, int ix) { - return prt->psd ? prt->psd->data[ix] : NULL; + ErtsPrtSD *psd = (ErtsPrtSD *) erts_smp_atomic_read_nob(&prt->psd); + if (!psd) + return NULL; + ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER; + return psd->data[ix]; } ERTS_GLB_INLINE void * erts_prtsd_set(Port *prt, int ix, void *data) { - if (prt->psd) { - void *old = prt->psd->data[ix]; - prt->psd->data[ix] = data; + ErtsPrtSD *psd, *new_psd; + void *old; + int i; + + psd = (ErtsPrtSD *) erts_smp_atomic_read_nob(&prt->psd); + + if (psd) { +#ifdef ERTS_SMP +#ifdef ETHR_ORDERED_READ_DEPEND + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreStore); +#endif +#endif + old = psd->data[ix]; + psd->data[ix] = data; return old; } - else { - prt->psd = erts_alloc(ERTS_ALC_T_PRTSD, sizeof(ErtsPrtSD)); - prt->psd->data[ix] = data; + + if (!data) return NULL; - } + + new_psd = erts_alloc(ERTS_ALC_T_PRTSD, sizeof(ErtsPrtSD)); + for (i = 0; i < ERTS_PRTSD_SIZE; i++) + new_psd->data[i] = NULL; + psd = (ErtsPrtSD *) erts_smp_atomic_cmpxchg_mb(&prt->psd, + (erts_aint_t) new_psd, + (erts_aint_t) NULL); + if (psd) + erts_free(ERTS_ALC_T_PRTSD, new_psd); + else + psd = new_psd; + old = psd->data[ix]; + psd->data[ix] = data; + return old; } #endif @@ -915,7 +944,7 @@ erts_schedule_proc2port_signal(Process *, ErtsPortTaskHandle *, ErtsProc2PortSigCallback); -int erts_deliver_port_exit(Port *, Eterm, Eterm, int); +int erts_deliver_port_exit(Port *, Eterm, Eterm, int, int); /* * Port signal flags @@ -949,6 +978,8 @@ ErtsPortOpResult erts_port_control(Process *, Port *, unsigned int, Eterm, Eterm ErtsPortOpResult erts_port_call(Process *, Port *, unsigned int, Eterm, Eterm *); ErtsPortOpResult erts_port_info(Process *, Port *, Eterm, Eterm *); +int erts_port_output_async(Port *, Eterm, Eterm); + /* * Signals from ports to ports. Used by sys drivers. */ diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c index e85395e062..3102e44c11 100644 --- a/erts/emulator/beam/erl_port_task.c +++ b/erts/emulator/beam/erl_port_task.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2006-2013. All Rights Reserved. + * Copyright Ericsson AB 2006-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -35,6 +35,7 @@ #include "dist.h" #include "erl_check_io.h" #include "dtrace-wrapper.h" +#include "lttng-wrapper.h" #include <stdarg.h> /* @@ -69,6 +70,18 @@ static void chk_task_queues(Port *pp, ErtsPortTask *execq, int processing_busy_q #else #define DTRACE_DRIVER(PROBE_NAME, PP) do {} while(0) #endif +#ifdef USE_LTTNG_VM_TRACEPOINTS +#define LTTNG_DRIVER(TRACEPOINT, PP) \ + if (LTTNG_ENABLED(TRACEPOINT)) { \ + lttng_decl_portbuf(port_str); \ + lttng_decl_procbuf(proc_str); \ + lttng_pid_to_str(ERTS_PORT_GET_CONNECTED(PP), proc_str); \ + lttng_port_to_str((PP), port_str); \ + LTTNG3(TRACEPOINT, proc_str, port_str, (PP)->name); \ + } +#else +#define LTTNG_DRIVER(TRACEPOINT, PP) do {} while(0) +#endif #define ERTS_SMP_LC_VERIFY_RQ(RQ, PP) \ do { \ @@ -180,10 +193,9 @@ p2p_sig_data_to_task(ErtsProc2PortSigData *sigdp) return ptp; } -ErtsProc2PortSigData * -erts_port_task_alloc_p2p_sig_data(void) +static ERTS_INLINE ErtsProc2PortSigData * +p2p_sig_data_init(ErtsPortTask *ptp) { - ErtsPortTask *ptp = port_task_alloc(); ptp->type = ERTS_PORT_TASK_PROC_SIG; ptp->u.alive.flags = ERTS_PT_FLG_SIG_DEP; @@ -194,6 +206,31 @@ erts_port_task_alloc_p2p_sig_data(void) return &ptp->u.alive.td.psig.data; } +ErtsProc2PortSigData * +erts_port_task_alloc_p2p_sig_data(void) +{ + ErtsPortTask *ptp = port_task_alloc(); + + return p2p_sig_data_init(ptp); +} + +ErtsProc2PortSigData * +erts_port_task_alloc_p2p_sig_data_extra(size_t extra, void **extra_ptr) +{ + ErtsPortTask *ptp = erts_alloc(ERTS_ALC_T_PORT_TASK, + sizeof(ErtsPortTask) + extra); + + *extra_ptr = ptp+1; + + return p2p_sig_data_init(ptp); +} + +void +erts_port_task_free_p2p_sig_data(ErtsProc2PortSigData *sigdp) +{ + schedule_port_task_free(p2p_sig_data_to_task(sigdp)); +} + static ERTS_INLINE Eterm task_caller(ErtsPortTask *ptp) { @@ -1728,6 +1765,9 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) reds = ERTS_PORT_REDS_TIMEOUT; if (!(state & ERTS_PORT_SFLGS_DEAD)) { DTRACE_DRIVER(driver_timeout, pp); + LTTNG_DRIVER(driver_timeout, pp); + if (IS_TRACED_FL(pp, F_TRACE_RECEIVE)) + trace_port(pp, am_receive, am_timeout); (*pp->drv_ptr->timeout)((ErlDrvData) pp->drv_data); } } @@ -1736,6 +1776,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) reds = ERTS_PORT_REDS_INPUT; ASSERT((state & ERTS_PORT_SFLGS_DEAD) == 0); DTRACE_DRIVER(driver_ready_input, pp); + LTTNG_DRIVER(driver_ready_input, pp); /* NOTE some windows drivers use ->ready_input for input and output */ (*pp->drv_ptr->ready_input)((ErlDrvData) pp->drv_data, @@ -1747,6 +1788,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) reds = ERTS_PORT_REDS_OUTPUT; ASSERT((state & ERTS_PORT_SFLGS_DEAD) == 0); DTRACE_DRIVER(driver_ready_output, pp); + LTTNG_DRIVER(driver_ready_output, pp); (*pp->drv_ptr->ready_output)((ErlDrvData) pp->drv_data, ptp->u.alive.td.io.event); reset_executed_io_task_handle(ptp); @@ -1756,6 +1798,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) reds = ERTS_PORT_REDS_EVENT; ASSERT((state & ERTS_PORT_SFLGS_DEAD) == 0); DTRACE_DRIVER(driver_event, pp); + LTTNG_DRIVER(driver_event, pp); (*pp->drv_ptr->event)((ErlDrvData) pp->drv_data, ptp->u.alive.td.io.event, ptp->u.alive.td.io.event_data); diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h index 335f7a77d5..2a6bd165a3 100644 --- a/erts/emulator/beam/erl_port_task.h +++ b/erts/emulator/beam/erl_port_task.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2006-2013. All Rights Reserved. + * Copyright Ericsson AB 2006-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -269,6 +269,8 @@ int erts_port_task_schedule(Eterm, void erts_port_task_free_port(Port *); int erts_port_is_scheduled(Port *); ErtsProc2PortSigData *erts_port_task_alloc_p2p_sig_data(void); +ErtsProc2PortSigData *erts_port_task_alloc_p2p_sig_data_extra(size_t extra, void **extra_ptr); +void erts_port_task_free_p2p_sig_data(ErtsProc2PortSigData *sigdp); #ifdef ERTS_SMP void erts_enqueue_port(ErtsRunQueue *rq, Port *pp); diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c index 45ba4371dc..1a579704a8 100644 --- a/erts/emulator/beam/erl_printf_term.c +++ b/erts/emulator/beam/erl_printf_term.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2014. All Rights Reserved. + * Copyright Ericsson AB 2005-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_printf_term.h b/erts/emulator/beam/erl_printf_term.h index 4618457f27..8a30286fd8 100644 --- a/erts/emulator/beam/erl_printf_term.h +++ b/erts/emulator/beam/erl_printf_term.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2013. All Rights Reserved. + * Copyright Ericsson AB 2005-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index a706ebf595..36b0af0c1a 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2014. All Rights Reserved. + * Copyright Ericsson AB 1996-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -43,6 +43,7 @@ #include "erl_thr_queue.h" #include "erl_async.h" #include "dtrace-wrapper.h" +#include "lttng-wrapper.h" #include "erl_ptab.h" #include "erl_bif_unique.h" #define ERTS_WANT_TIMER_WHEEL_API @@ -443,7 +444,8 @@ int erts_system_profile_ts_type = ERTS_TRACE_FLG_NOW_TIMESTAMP; typedef enum { ERTS_PSTT_GC, /* Garbage Collect */ ERTS_PSTT_CPC, /* Check Process Code */ - ERTS_PSTT_COHMQ /* Change off heap message queue */ + ERTS_PSTT_COHMQ, /* Change off heap message queue */ + ERTS_PSTT_FTMQ /* Flush trace msg queue */ } ErtsProcSysTaskType; #define ERTS_MAX_PROC_SYS_TASK_ARGS 2 @@ -687,45 +689,36 @@ erts_pre_init_process(void) = "DEBUG_WAIT_COMPLETED"; #ifdef ERTS_ENABLE_LOCK_CHECK - { - int ix; - erts_psd_required_locks[ERTS_PSD_ERROR_HANDLER].get_locks - = ERTS_PSD_ERROR_HANDLER_BUF_GET_LOCKS; - erts_psd_required_locks[ERTS_PSD_ERROR_HANDLER].set_locks - = ERTS_PSD_ERROR_HANDLER_BUF_SET_LOCKS; + erts_psd_required_locks[ERTS_PSD_ERROR_HANDLER].get_locks + = ERTS_PSD_ERROR_HANDLER_BUF_GET_LOCKS; + erts_psd_required_locks[ERTS_PSD_ERROR_HANDLER].set_locks + = ERTS_PSD_ERROR_HANDLER_BUF_SET_LOCKS; - erts_psd_required_locks[ERTS_PSD_SAVED_CALLS_BUF].get_locks - = ERTS_PSD_SAVED_CALLS_BUF_GET_LOCKS; - erts_psd_required_locks[ERTS_PSD_SAVED_CALLS_BUF].set_locks - = ERTS_PSD_SAVED_CALLS_BUF_SET_LOCKS; + erts_psd_required_locks[ERTS_PSD_SAVED_CALLS_BUF].get_locks + = ERTS_PSD_SAVED_CALLS_BUF_GET_LOCKS; + erts_psd_required_locks[ERTS_PSD_SAVED_CALLS_BUF].set_locks + = ERTS_PSD_SAVED_CALLS_BUF_SET_LOCKS; - erts_psd_required_locks[ERTS_PSD_SCHED_ID].get_locks - = ERTS_PSD_SCHED_ID_GET_LOCKS; - erts_psd_required_locks[ERTS_PSD_SCHED_ID].set_locks - = ERTS_PSD_SCHED_ID_SET_LOCKS; + erts_psd_required_locks[ERTS_PSD_SCHED_ID].get_locks + = ERTS_PSD_SCHED_ID_GET_LOCKS; + erts_psd_required_locks[ERTS_PSD_SCHED_ID].set_locks + = ERTS_PSD_SCHED_ID_SET_LOCKS; - erts_psd_required_locks[ERTS_PSD_CALL_TIME_BP].get_locks - = ERTS_PSD_CALL_TIME_BP_GET_LOCKS; - erts_psd_required_locks[ERTS_PSD_CALL_TIME_BP].set_locks - = ERTS_PSD_CALL_TIME_BP_SET_LOCKS; + erts_psd_required_locks[ERTS_PSD_CALL_TIME_BP].get_locks + = ERTS_PSD_CALL_TIME_BP_GET_LOCKS; + erts_psd_required_locks[ERTS_PSD_CALL_TIME_BP].set_locks + = ERTS_PSD_CALL_TIME_BP_SET_LOCKS; - erts_psd_required_locks[ERTS_PSD_DELAYED_GC_TASK_QS].get_locks - = ERTS_PSD_DELAYED_GC_TASK_QS_GET_LOCKS; - erts_psd_required_locks[ERTS_PSD_DELAYED_GC_TASK_QS].set_locks - = ERTS_PSD_DELAYED_GC_TASK_QS_SET_LOCKS; + erts_psd_required_locks[ERTS_PSD_DELAYED_GC_TASK_QS].get_locks + = ERTS_PSD_DELAYED_GC_TASK_QS_GET_LOCKS; + erts_psd_required_locks[ERTS_PSD_DELAYED_GC_TASK_QS].set_locks + = ERTS_PSD_DELAYED_GC_TASK_QS_SET_LOCKS; - erts_psd_required_locks[ERTS_PSD_NIF_TRAP_EXPORT].get_locks - = ERTS_PSD_NIF_TRAP_EXPORT_GET_LOCKS; - erts_psd_required_locks[ERTS_PSD_NIF_TRAP_EXPORT].set_locks - = ERTS_PSD_NIF_TRAP_EXPORT_SET_LOCKS; - - /* Check that we have locks for all entries */ - for (ix = 0; ix < ERTS_PSD_SIZE; ix++) { - ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].get_locks); - ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].set_locks); - } - } + erts_psd_required_locks[ERTS_PSD_NIF_TRAP_EXPORT].get_locks + = ERTS_PSD_NIF_TRAP_EXPORT_GET_LOCKS; + erts_psd_required_locks[ERTS_PSD_NIF_TRAP_EXPORT].set_locks + = ERTS_PSD_NIF_TRAP_EXPORT_SET_LOCKS; #endif } @@ -1146,7 +1139,7 @@ reply_sched_wall_time(void *vswtrp) hpp = &hp; } - erts_queue_message(rp, &rp_locks, mp, msg, NIL); + erts_queue_message(rp, rp_locks, mp, msg, am_system); if (swtrp->req_sched == esdp->no) rp_locks &= ~ERTS_PROC_LOCK_MAIN; @@ -1225,7 +1218,7 @@ reply_system_check(void *vscrp) hpp = &hp; msg = STORE_NC(hpp, ohp, scrp->ref); - erts_queue_message(rp, &rp_locks, mp, msg, NIL); + erts_queue_message(rp, rp_locks, mp, msg, am_system); if (scrp->req_sched == esdp->no) rp_locks &= ~ERTS_PROC_LOCK_MAIN; @@ -1314,46 +1307,25 @@ erts_proclist_destroy(ErtsProcList *plp) } void * -erts_psd_set_init(Process *p, ErtsProcLocks plocks, int ix, void *data) +erts_psd_set_init(Process *p, int ix, void *data) { void *old; - ErtsProcLocks xplocks; - int refc = 0; - ErtsPSD *psd = erts_alloc(ERTS_ALC_T_PSD, sizeof(ErtsPSD)); + ErtsPSD *psd, *new_psd; int i; - for (i = 0; i < ERTS_PSD_SIZE; i++) - psd->data[i] = NULL; - ERTS_SMP_LC_ASSERT(plocks); - ERTS_SMP_LC_ASSERT(plocks == erts_proc_lc_my_proc_locks(p)); + new_psd = erts_alloc(ERTS_ALC_T_PSD, sizeof(ErtsPSD)); + for (i = 0; i < ERTS_PSD_SIZE; i++) + new_psd->data[i] = NULL; - xplocks = ERTS_PROC_LOCKS_ALL; - xplocks &= ~plocks; - if (xplocks && erts_smp_proc_trylock(p, xplocks) == EBUSY) { - if (xplocks & ERTS_PROC_LOCK_MAIN) { - erts_proc_inc_refc(p); - erts_smp_proc_unlock(p, plocks); - erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL); - refc = 1; - } - else { - if (plocks & ERTS_PROC_LOCKS_ALL_MINOR) - erts_smp_proc_unlock(p, plocks & ERTS_PROC_LOCKS_ALL_MINOR); - erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR); - } - } - if (!p->psd) - p->psd = psd; - if (xplocks) - erts_smp_proc_unlock(p, xplocks); - if (refc) - erts_proc_dec_refc(p); - ASSERT(p->psd); - if (p->psd != psd) - erts_free(ERTS_ALC_T_PSD, psd); - old = p->psd->data[ix]; - p->psd->data[ix] = data; - ERTS_SMP_LC_ASSERT(plocks == erts_proc_lc_my_proc_locks(p)); + psd = (ErtsPSD *) erts_smp_atomic_cmpxchg_mb(&p->psd, + (erts_aint_t) new_psd, + (erts_aint_t) NULL); + if (psd) + erts_free(ERTS_ALC_T_PSD, new_psd); + else + psd = new_psd; + old = psd->data[ix]; + psd->data[ix] = data; return old; } @@ -2211,8 +2183,7 @@ setup_thr_debug_wait_completed(void *vproc) if (debug_wait_completed_flags & ERTS_DEBUG_WAIT_COMPLETED_DEALLOCATIONS) { erts_alloc_fix_alloc_shrink(awdp->sched_id, 0); wait_flags |= (ERTS_SSI_AUX_WORK_DD - | ERTS_SSI_AUX_WORK_DD_THR_PRGR - | ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP); + | ERTS_SSI_AUX_WORK_DD_THR_PRGR); #ifdef ERTS_SMP aux_work_flags |= ERTS_SSI_AUX_WORK_DD; #endif @@ -2220,8 +2191,7 @@ setup_thr_debug_wait_completed(void *vproc) if (debug_wait_completed_flags & ERTS_DEBUG_WAIT_COMPLETED_TIMER_CANCELLATIONS) { wait_flags |= (ERTS_SSI_AUX_WORK_CNCLD_TMRS - | ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR - | ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP); + | ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR); #ifdef ERTS_SMP if (awdp->esdp && !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)) aux_work_flags |= ERTS_SSI_AUX_WORK_CNCLD_TMRS; @@ -2235,26 +2205,42 @@ setup_thr_debug_wait_completed(void *vproc) awdp->debug.wait_completed.arg = vproc; } -static void -prep_setup_thr_debug_wait_completed(void *vproc) +struct debug_lop { + ErtsThrPrgrLaterOp lop; + Process *proc; +}; + +static void later_thr_debug_wait_completed(void *vlop) { + struct debug_lop *lop = vlop; erts_aint32_t count = (erts_aint32_t) erts_no_schedulers; #ifdef ERTS_SMP count += 1; /* aux thread */ #endif if (erts_atomic32_dec_read_mb(&debug_wait_completed_count) == count) { - /* scheduler threads */ - erts_schedule_multi_misc_aux_work(0, - erts_no_schedulers, - setup_thr_debug_wait_completed, - vproc); + /* scheduler threads */ + erts_schedule_multi_misc_aux_work(0, + erts_no_schedulers, + setup_thr_debug_wait_completed, + lop->proc); #ifdef ERTS_SMP - /* aux_thread */ - erts_schedule_misc_aux_work(0, - setup_thr_debug_wait_completed, - vproc); + /* aux_thread */ + erts_schedule_misc_aux_work(0, + setup_thr_debug_wait_completed, + lop->proc); #endif } + erts_free(ERTS_ALC_T_DEBUG, lop); +} + + +static void +init_thr_debug_wait_completed(void *vproc) +{ + struct debug_lop* lop = erts_alloc(ERTS_ALC_T_DEBUG, + sizeof(struct debug_lop)); + lop->proc = vproc; + erts_schedule_thr_prgr_later_op(later_thr_debug_wait_completed, lop, &lop->lop); } @@ -2264,7 +2250,7 @@ erts_debug_wait_completed(Process *c_p, int flags) /* Only one process at a time can do this */ erts_aint32_t count = (erts_aint32_t) (2*erts_no_schedulers); #ifdef ERTS_SMP - count += 2; /* aux thread */ + count += 1; /* aux thread */ #endif if (0 == erts_atomic32_cmpxchg_mb(&debug_wait_completed_count, count, @@ -2272,17 +2258,12 @@ erts_debug_wait_completed(Process *c_p, int flags) debug_wait_completed_flags = flags; erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL); erts_proc_inc_refc(c_p); - /* scheduler threads */ + + /* First flush later-ops on all scheduler threads */ erts_schedule_multi_misc_aux_work(0, erts_no_schedulers, - prep_setup_thr_debug_wait_completed, + init_thr_debug_wait_completed, (void *) c_p); -#ifdef ERTS_SMP - /* aux_thread */ - erts_schedule_misc_aux_work(0, - prep_setup_thr_debug_wait_completed, - (void *) c_p); -#endif return 1; } return 0; @@ -2331,7 +2312,7 @@ handle_reap_ports(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) ERTS_PORT_SFLG_HALT); erts_smp_atomic32_inc_nob(&erts_halt_progress); if (!(state & (ERTS_PORT_SFLG_EXITING|ERTS_PORT_SFLG_CLOSING))) - erts_deliver_port_exit(prt, prt->common.id, am_killed, 0); + erts_deliver_port_exit(prt, prt->common.id, am_killed, 0, 1); } erts_port_release(prt); @@ -3238,6 +3219,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_CHECK_IO); ASSERT(!erts_port_task_have_outstanding_io_tasks()); + LTTNG2(scheduler_poll, esdp->no, 1); erl_sys_schedule(1); /* Might give us something to do */ ERTS_MSACC_POP_STATE_M(); @@ -3361,6 +3343,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) ASSERT(!erts_port_task_have_outstanding_io_tasks()); ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_CHECK_IO); + LTTNG2(scheduler_poll, esdp->no, 0); erl_sys_schedule(0); @@ -6705,10 +6688,8 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st) erts_aint32_t e; n = e = a; - if (a & ERTS_PSFLG_FREE) { - res = 0; + if (a & ERTS_PSFLG_FREE) goto cleanup; /* We don't want to schedule free processes... */ - } enqueue = ERTS_ENQUEUE_NOT; n |= ERTS_PSFLG_ACTIVE_SYS; @@ -9257,6 +9238,21 @@ erts_set_process_priority(Process *p, Eterm value) } } +static int +scheduler_gc_proc(Process *c_p, int reds_left) +{ + int fcalls, reds; + if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) + fcalls = reds_left; + else + fcalls = reds_left - CONTEXT_REDS; + reds = erts_garbage_collect_nobump(c_p, 0, c_p->arg_reg, c_p->arity, fcalls); + ASSERT(reds_left >= reds); + return reds; +} + + + /* * schedule() is called from BEAM (process_main()) or HiPE * (hipe_mode_switch()) when the current process is to be @@ -9324,8 +9320,6 @@ Process *schedule(Process *p, int calls) } else { sched_out_proc: - ASSERT(!(p->flags & F_DELAY_GC)); - #ifdef ERTS_SMP ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); esdp = p->scheduler_data; @@ -9335,7 +9329,9 @@ Process *schedule(Process *p, int calls) esdp = erts_scheduler_data; ASSERT(esdp->current_process == p); #endif + reds = actual_reds = calls - esdp->virtual_reds; + ASSERT(actual_reds >= 0); if (reds < ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST) reds = ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST; esdp->virtual_reds = 0; @@ -9347,27 +9343,39 @@ Process *schedule(Process *p, int calls) p->reds += actual_reds; - erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); +#ifdef ERTS_SMP + erts_smp_proc_lock(p, ERTS_PROC_LOCK_TRACE); + if (p->trace_msg_q) { + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_TRACE); + erts_schedule_flush_trace_messages(p->common.id); + } else + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_TRACE); +#endif - state = erts_smp_atomic32_read_acqb(&p->state); + state = erts_smp_atomic32_read_nob(&p->state); if (IS_TRACED(p)) { if (IS_TRACED_FL(p, F_TRACE_CALLS) && !(state & ERTS_PSFLG_FREE)) erts_schedule_time_break(p, ERTS_BP_CALL_TIME_SCHEDULE_OUT); - if (state & (ERTS_PSFLG_FREE|ERTS_PSFLG_EXITING)) { + if ((state & (ERTS_PSFLG_FREE|ERTS_PSFLG_EXITING)) == ERTS_PSFLG_EXITING) { if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_EXIT)) - trace_sched(p, ((state & ERTS_PSFLG_FREE) - ? am_out_exited - : am_out_exiting)); + trace_sched(p, ERTS_PROC_LOCK_MAIN, + ((state & ERTS_PSFLG_FREE) + ? am_out_exited + : am_out_exiting)); } else { - if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED)) - trace_sched(p, am_out); - else if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_PROCS)) - trace_virtual_sched(p, am_out); + if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED) || + ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_PROCS)) + trace_sched(p, ERTS_PROC_LOCK_MAIN, am_out); } } + erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + + /* have to re-read state after taking lock */ + state = erts_smp_atomic32_read_nob(&p->state); + #ifdef ERTS_SMP if (state & ERTS_PSFLG_PENDING_EXIT) erts_handle_pending_exit(p, (ERTS_PROC_LOCK_MAIN @@ -9581,7 +9589,10 @@ Process *schedule(Process *p, int calls) erts_sys_schedule_interrupt(0); #endif erts_smp_runq_unlock(rq); - ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_CHECK_IO); + + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_CHECK_IO); + LTTNG2(scheduler_poll, esdp->no, 1); + erl_sys_schedule(1); ERTS_MSACC_POP_STATE_M(); @@ -9724,12 +9735,14 @@ Process *schedule(Process *p, int calls) | ERTS_PSFLG_ACTIVE_SYS | ERTS_PSFLG_DIRTY_ACTIVE_SYS)) == ERTS_PSFLG_SUSPENDED)) { - if (state & ERTS_PSFLG_FREE) - erts_proc_dec_refc(p); if (proxy_p) { free_proxy_proc(proxy_p); proxy_p = NULL; } + else if (state & ERTS_PSFLG_FREE) { + /* free and not queued by proxy */ + erts_proc_dec_refc(p); + } goto pick_next_process; } state = new; @@ -9741,7 +9754,7 @@ Process *schedule(Process *p, int calls) esdp->current_process = p; - + calls = 0; reds = context_reds; #ifdef ERTS_SMP @@ -9789,10 +9802,7 @@ Process *schedule(Process *p, int calls) if (erts_sched_stat.enabled) { int prio; - UWord old = ERTS_PROC_SCHED_ID(p, - (ERTS_PROC_LOCK_MAIN - | ERTS_PROC_LOCK_STATUS), - (UWord) esdp->no); + UWord old = ERTS_PROC_SCHED_ID(p, (UWord) esdp->no); int migrated = old && old != esdp->no; #ifdef ERTS_DIRTY_SCHEDULERS @@ -9823,9 +9833,7 @@ Process *schedule(Process *p, int calls) /* Migrate to dirty scheduler... */ sunlock_sched_out_proc: erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); - p->fcalls = reds; goto sched_out_proc; - } } else { @@ -9860,41 +9868,49 @@ Process *schedule(Process *p, int calls) #endif /* ERTS_SMP */ - p->fcalls = reds; + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); if (IS_TRACED(p)) { if (state & ERTS_PSFLG_EXITING) { if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_EXIT)) - trace_sched(p, am_in_exiting); + trace_sched(p, ERTS_PROC_LOCK_MAIN, am_in_exiting); } else { - if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED)) - trace_sched(p, am_in); - else if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_PROCS)) - trace_virtual_sched(p, am_in); + if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED) || + ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_PROCS)) + trace_sched(p, ERTS_PROC_LOCK_MAIN, am_in); } if (IS_TRACED_FL(p, F_TRACE_CALLS)) { erts_schedule_time_break(p, ERTS_BP_CALL_TIME_SCHEDULE_IN); } } - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); #ifdef ERTS_SMP - if (is_not_nil(ERTS_TRACER_PROC(p))) - erts_check_my_tracer_proc(p); + /* Clears tracer if it has been removed */ + (void)ERTS_TRACER_PROC_IS_ENABLED(p); #endif if (state & ERTS_PSFLG_RUNNING_SYS) { - reds -= execute_sys_tasks(p, &state, reds); - if (reds <= 0 + /* + * GC is normally never delayed when a process + * is scheduled out, but might be when executing + * hand written beam assembly in + * prim_eval:'receive'. If GC is delayed we are + * not allowed to execute system tasks. + */ + if (!(p->flags & F_DELAY_GC)) { + int cost = execute_sys_tasks(p, &state, reds); + calls += cost; + reds -= cost; + if (reds <= 0 #ifdef ERTS_DIRTY_SCHEDULERS - || ERTS_SCHEDULER_IS_DIRTY(esdp) - || (state & ERTS_PSFLGS_DIRTY_WORK) + || ERTS_SCHEDULER_IS_DIRTY(esdp) + || (state & ERTS_PSFLGS_DIRTY_WORK) #endif - ) { - p->fcalls = reds; - goto sched_out_proc; + ) { + goto sched_out_proc; + } } ASSERT(state & ERTS_PSFLG_RUNNING_SYS); @@ -9905,8 +9921,9 @@ Process *schedule(Process *p, int calls) if (((state & (ERTS_PSFLG_SUSPENDED | ERTS_PSFLG_ACTIVE)) != ERTS_PSFLG_ACTIVE) - && !(state & ERTS_PSFLG_EXITING)) + && !(state & ERTS_PSFLG_EXITING)) { goto sched_out_proc; + } n = e = state; n &= ~ERTS_PSFLG_RUNNING_SYS; @@ -9924,12 +9941,12 @@ Process *schedule(Process *p, int calls) } if (ERTS_IS_GC_DESIRED(p)) { - if (!(state & ERTS_PSFLG_EXITING) && !(p->flags & F_DISABLE_GC)) { - reds -= erts_garbage_collect_nobump(p, 0, p->arg_reg, p->arity); - if (reds <= 0) { - p->fcalls = reds; + if (!(state & ERTS_PSFLG_EXITING) && !(p->flags & (F_DELAY_GC|F_DISABLE_GC))) { + int cost = scheduler_gc_proc(p, reds); + calls += cost; + reds -= cost; + if (reds <= 0) goto sched_out_proc; - } } } @@ -9937,6 +9954,8 @@ Process *schedule(Process *p, int calls) free_proxy_proc(proxy_p); proxy_p = NULL; } + + p->fcalls = reds; ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); @@ -10005,7 +10024,7 @@ notify_sys_task_executed(Process *c_p, ErtsProcSysTask *st, Eterm st_result) ASSERT(hp_start + hsz == hp); #endif - erts_queue_message(rp, &rp_locks, mp, msg, NIL); + erts_queue_message(rp, rp_locks, mp, msg, c_p->common.id); if (c_p == rp) rp_locks &= ~ERTS_PROC_LOCK_MAIN; @@ -10173,8 +10192,7 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) { int garbage_collected = 0; erts_aint32_t state = *statep; - int max_reds = in_reds; - int reds = 0; + int reds = in_reds; int qmask = 0; ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCK_MAIN); @@ -10202,35 +10220,47 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) if (c_p->flags & F_DISABLE_GC) { save_gc_task(c_p, st, st_prio); st = NULL; - reds++; + reds--; } else { if (!garbage_collected) { FLAGS(c_p) |= F_NEED_FULLSWEEP; - reds += erts_garbage_collect_nobump(c_p, - 0, - c_p->arg_reg, - c_p->arity); + reds -= scheduler_gc_proc(c_p, reds); garbage_collected = 1; } st_res = am_true; } break; - case ERTS_PSTT_CPC: + case ERTS_PSTT_CPC: { + int fcalls; + int cpc_reds = 0; + if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) + fcalls = reds; + else + fcalls = reds - CONTEXT_REDS; st_res = erts_check_process_code(c_p, st->arg[0], unsigned_val(st->arg[1]), - &reds); + &cpc_reds, + fcalls); + reds -= cpc_reds; if (is_non_value(st_res)) { /* Needed gc, but gc was disabled */ save_gc_task(c_p, st, st_prio); st = NULL; } break; + } case ERTS_PSTT_COHMQ: - reds += erts_complete_off_heap_message_queue_change(c_p); + reds -= erts_complete_off_heap_message_queue_change(c_p); st_res = am_true; break; +#ifdef ERTS_SMP + case ERTS_PSTT_FTMQ: + reds -= erts_flush_trace_messages(c_p, ERTS_PROC_LOCK_MAIN); + st_res = am_true; + break; +#endif default: ERTS_INTERNAL_ERROR("Invalid process sys task type"); st_res = am_false; @@ -10240,11 +10270,14 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) reds += notify_sys_task_executed(c_p, st, st_res); state = erts_smp_atomic32_read_acqb(&c_p->state); - } while (qmask && reds < max_reds); + } while (qmask && reds > 0); *statep = state; - return reds; + if (in_reds < reds) + return in_reds; + + return in_reds - reds; } static int @@ -10276,6 +10309,12 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds) case ERTS_PSTT_COHMQ: st_res = am_false; break; +#ifdef ERTS_SMP + case ERTS_PSTT_FTMQ: + reds -= erts_flush_trace_messages(c_p, ERTS_PROC_LOCK_MAIN); + st_res = am_true; + break; +#endif default: ERTS_INTERNAL_ERROR("Invalid process sys task type"); st_res = am_false; @@ -10411,8 +10450,8 @@ badarg: BIF_ERROR(BIF_P, BADARG); } -void -erts_schedule_complete_off_heap_message_queue_change(Eterm pid) +static void +erts_schedule_generic_sys_task(Eterm pid, ErtsProcSysTaskType type) { Process *rp = erts_proc_lookup(pid); if (rp) { @@ -10422,7 +10461,7 @@ erts_schedule_complete_off_heap_message_queue_change(Eterm pid) st = erts_alloc(ERTS_ALC_T_PROC_SYS_TSK, ERTS_PROC_SYS_TASK_SIZE(0)); - st->type = ERTS_PSTT_COHMQ; + st->type = type; st->requester = NIL; st->reply_tag = NIL; st->req_id = NIL; @@ -10437,6 +10476,18 @@ erts_schedule_complete_off_heap_message_queue_change(Eterm pid) } } +void +erts_schedule_complete_off_heap_message_queue_change(Eterm pid) +{ + erts_schedule_generic_sys_task(pid, ERTS_PSTT_COHMQ); +} + +void +erts_schedule_flush_trace_messages(Eterm pid) +{ + erts_schedule_generic_sys_task(pid, ERTS_PSTT_FTMQ); +} + static void save_gc_task(Process *c_p, ErtsProcSysTask *st, int prio) { @@ -10456,7 +10507,7 @@ save_gc_task(Process *c_p, ErtsProcSysTask *st, int prio) qs->q[PRIORITY_NORMAL] = NULL; qs->q[PRIORITY_LOW] = NULL; qs->q[prio] = st; - (void) ERTS_PROC_SET_DELAYED_GC_TASK_QS(c_p, ERTS_PROC_LOCK_MAIN, qs); + (void) ERTS_PROC_SET_DELAYED_GC_TASK_QS(c_p, qs); } else { if (!qs->q[prio]) { @@ -10603,7 +10654,7 @@ erts_set_gc_state(Process *c_p, int enable) erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS); - (void) ERTS_PROC_SET_DELAYED_GC_TASK_QS(c_p, ERTS_PROC_LOCK_MAIN, NULL); + (void) ERTS_PROC_SET_DELAYED_GC_TASK_QS(c_p, NULL); if (dgc_tsk_qs) proc_sys_task_queues_free(dgc_tsk_qs); @@ -10895,14 +10946,13 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). Eterm res = THE_NON_VALUE; erts_aint32_t state = 0; erts_aint32_t prio = (erts_aint32_t) PRIORITY_NORMAL; + ErtsProcLocks locks = ERTS_PROC_LOCKS_ALL; #ifdef SHCOPY_SPAWN erts_shcopy_t info; INITIALIZE_SHCOPY(info); #endif -#ifdef ERTS_SMP erts_smp_proc_lock(parent, ERTS_PROC_LOCKS_ALL_MINOR); -#endif /* * Check for errors. @@ -11071,7 +11121,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). : STORE_NC(&p->htop, &p->off_heap, parent->group_leader); } - erts_get_default_tracing(&ERTS_TRACE_FLAGS(p), &ERTS_TRACER_PROC(p)); + erts_get_default_proc_tracing(&ERTS_TRACE_FLAGS(p), &ERTS_TRACER(p)); p->msg.first = NULL; p->msg.last = &p->msg.first; @@ -11089,7 +11139,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->mbuf = NULL; p->msg_frag = NULL; p->mbuf_sz = 0; - p->psd = NULL; + erts_smp_atomic_init_nob(&p->psd, (erts_aint_t) NULL); p->dictionary = NULL; p->seq_trace_lastcnt = 0; p->seq_trace_clock = 0; @@ -11107,21 +11157,60 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->last_old_htop = NULL; #endif +#ifdef ERTS_SMP + p->trace_msg_q = NULL; + p->scheduler_data = NULL; + p->suspendee = NIL; + p->pending_suspenders = NULL; + p->pending_exit.reason = THE_NON_VALUE; + p->pending_exit.bp = NULL; +#endif + +#if !defined(NO_FPE_SIGNALS) || defined(HIPE) + p->fp_exception = 0; +#endif + if (IS_TRACED(parent)) { if (ERTS_TRACE_FLAGS(parent) & F_TRACE_SOS) { ERTS_TRACE_FLAGS(p) |= (ERTS_TRACE_FLAGS(parent) & TRACEE_FLAGS); - ERTS_TRACER_PROC(p) = ERTS_TRACER_PROC(parent); - } - if (ARE_TRACE_FLAGS_ON(parent, F_TRACE_PROCS)) { - trace_proc_spawn(parent, p->common.id, mod, func, args); + erts_tracer_replace(&p->common, ERTS_TRACER(parent)); } - if (ERTS_TRACE_FLAGS(parent) & F_TRACE_SOS1) { + if (ERTS_TRACE_FLAGS(parent) & F_TRACE_SOS1) { /* Overrides TRACE_CHILDREN */ ERTS_TRACE_FLAGS(p) |= (ERTS_TRACE_FLAGS(parent) & TRACEE_FLAGS); - ERTS_TRACER_PROC(p) = ERTS_TRACER_PROC(parent); + erts_tracer_replace(&p->common, ERTS_TRACER(parent)); ERTS_TRACE_FLAGS(p) &= ~(F_TRACE_SOS1 | F_TRACE_SOS); ERTS_TRACE_FLAGS(parent) &= ~(F_TRACE_SOS1 | F_TRACE_SOS); } + if (so->flags & SPO_LINK && ERTS_TRACE_FLAGS(parent) & (F_TRACE_SOL|F_TRACE_SOL1)) { + ERTS_TRACE_FLAGS(p) |= (ERTS_TRACE_FLAGS(parent)&TRACEE_FLAGS); + erts_tracer_replace(&p->common, ERTS_TRACER(parent)); + if (ERTS_TRACE_FLAGS(parent) & F_TRACE_SOL1) {/*maybe override*/ + ERTS_TRACE_FLAGS(p) &= ~(F_TRACE_SOL1 | F_TRACE_SOL); + ERTS_TRACE_FLAGS(parent) &= ~(F_TRACE_SOL1 | F_TRACE_SOL); + } + } + if (ARE_TRACE_FLAGS_ON(parent, F_TRACE_PROCS)) { + locks &= ~(ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); + erts_smp_proc_unlock(parent, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); + trace_proc_spawn(parent, am_spawn, p->common.id, mod, func, args); + if (so->flags & SPO_LINK) + trace_proc(parent, locks, parent, am_link, p->common.id); + } + } + + if (IS_TRACED_FL(p, F_TRACE_PROCS)) { + if ((locks & (ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE)) + == (ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE)) { + /* This happens when parent was not traced, but child is */ + locks &= ~(ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); + erts_smp_proc_unlock(parent, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); + } + trace_proc_spawn(p, am_spawned, parent->common.id, mod, func, args); + if (so->flags & SPO_LINK) + trace_proc(p, locks, p, am_getting_linked, parent->common.id); } /* @@ -11132,10 +11221,6 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). #ifdef DEBUG int ret; #endif - if (IS_TRACED_FL(parent, F_TRACE_PROCS)) { - trace_proc(parent, parent, am_link, p->common.id); - } - #ifdef DEBUG ret = erts_add_link(&ERTS_P_LINKS(parent), LINK_PID, p->common.id); ASSERT(ret == 0); @@ -11146,17 +11231,6 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). erts_add_link(&ERTS_P_LINKS(p), LINK_PID, parent->common.id); #endif - if (IS_TRACED(parent)) { - if (ERTS_TRACE_FLAGS(parent) & (F_TRACE_SOL|F_TRACE_SOL1)) { - ERTS_TRACE_FLAGS(p) |= (ERTS_TRACE_FLAGS(parent)&TRACEE_FLAGS); - ERTS_TRACER_PROC(p) = ERTS_TRACER_PROC(parent); /*maybe steal*/ - - if (ERTS_TRACE_FLAGS(parent) & F_TRACE_SOL1) {/*maybe override*/ - ERTS_TRACE_FLAGS(p) &= ~(F_TRACE_SOL1 | F_TRACE_SOL); - ERTS_TRACE_FLAGS(parent) &= ~(F_TRACE_SOL1 | F_TRACE_SOL); - } - } - } } /* @@ -11171,19 +11245,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). so->mref = mref; } -#ifdef ERTS_SMP - p->scheduler_data = NULL; - p->suspendee = NIL; - p->pending_suspenders = NULL; - p->pending_exit.reason = THE_NON_VALUE; - p->pending_exit.bp = NULL; -#endif - -#if !defined(NO_FPE_SIGNALS) || defined(HIPE) - p->fp_exception = 0; -#endif - - erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL); + erts_smp_proc_unlock(p, locks); res = p->common.id; @@ -11191,6 +11253,8 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). * Schedule process for execution. */ + erts_smp_proc_unlock(parent, locks & ERTS_PROC_LOCKS_ALL_MINOR); + schedule_process(p, state, 0); VERBOSE(DEBUG_PROCESSES, ("Created a new process: %T\n",p->common.id)); @@ -11204,10 +11268,11 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). DTRACE2(process_spawn, process_name, mfa); } #endif + return res; error: - erts_smp_proc_unlock(parent, ERTS_PROC_LOCKS_ALL_MINOR); + erts_smp_proc_unlock(parent, locks & ERTS_PROC_LOCKS_ALL_MINOR); return res; } @@ -11232,7 +11297,7 @@ void erts_init_empty_process(Process *p) p->rcount = 0; p->common.id = ERTS_INVALID_PID; p->reds = 0; - ERTS_TRACER_PROC(p) = NIL; + ERTS_TRACER(p) = erts_tracer_nil; ERTS_TRACE_FLAGS(p) = F_INITIAL_TRACE_FLAGS; p->group_leader = ERTS_INVALID_PID; p->flags = 0; @@ -11258,7 +11323,7 @@ void erts_init_empty_process(Process *p) p->mbuf = NULL; p->msg_frag = NULL; p->mbuf_sz = 0; - p->psd = NULL; + erts_smp_atomic_init_nob(&p->psd, (erts_aint_t) NULL); ERTS_P_MONITORS(p) = NULL; ERTS_P_LINKS(p) = NULL; /* List of links */ p->nodes_monitors = NULL; @@ -11352,7 +11417,7 @@ erts_debug_verify_clean_empty_process(Process* p) ASSERT(p->live_hf_end == ERTS_INVALID_HFRAG_PTR); ASSERT(p->heap == NULL); ASSERT(p->common.id == ERTS_INVALID_PID); - ASSERT(ERTS_TRACER_PROC(p) == NIL); + ASSERT(ERTS_TRACER_IS_NIL(ERTS_TRACER(p))); ASSERT(ERTS_TRACE_FLAGS(p) == F_INITIAL_TRACE_FLAGS); ASSERT(p->group_leader == ERTS_INVALID_PID); ASSERT(p->next == NULL); @@ -11428,14 +11493,19 @@ static void delete_process(Process* p) { Eterm *heap; + ErtsPSD *psd; VERBOSE(DEBUG_PROCESSES, ("Removing process: %T\n",p->common.id)); VERBOSE(DEBUG_SHCOPY, ("[pid=%T] delete process: %p %p %p %p\n", p->common.id, HEAP_START(p), HEAP_END(p), OLD_HEAP(p), OLD_HEND(p))); /* Cleanup psd */ - if (p->psd) - erts_free(ERTS_ALC_T_PSD, p->psd); + psd = (ErtsPSD *) erts_smp_atomic_read_nob(&p->psd); + + if (psd) { + erts_smp_atomic_set_nob(&p->psd, (erts_aint_t) NULL); /* Reduction counting depends on this... */ + erts_free(ERTS_ALC_T_PSD, psd); + } /* Clean binaries and funs */ erts_cleanup_offheap(&p->off_heap); @@ -11696,7 +11766,7 @@ send_exit_message(Process *to, ErtsProcLocks *to_locksp, mp = erts_alloc_message_heap(to, to_locksp, term_size, &hp, &ohp); mess = copy_struct(exit_term, term_size, &hp, ohp); #endif - erts_queue_message(to, to_locksp, mp, mess, NIL); + erts_queue_message(to, *to_locksp, mp, mess, am_system); } else { Eterm temp_token; Uint sz_token; @@ -11714,9 +11784,10 @@ send_exit_message(Process *to, ErtsProcLocks *to_locksp, mess = copy_struct(exit_term, term_size, &hp, ohp); #endif /* the trace token must in this case be updated by the caller */ - seq_trace_output(token, mess, SEQ_TRACE_SEND, to->common.id, NULL); + seq_trace_output(token, mess, SEQ_TRACE_SEND, to->common.id, to); temp_token = copy_struct(token, sz_token, &hp, ohp); - erts_queue_message(to, to_locksp, mp, mess, temp_token); + ERL_MESSAGE_TOKEN(mp) = temp_token; + erts_queue_message(to, *to_locksp, mp, mess, am_system); } } @@ -11825,6 +11896,9 @@ send_exit_signal(Process *c_p, /* current process if and only if ((state & ERTS_PSFLG_TRAP_EXIT) && (reason != am_kill || (flags & ERTS_XSIG_FLG_IGN_KILL))) { + /* have to release the status lock in order to send the exit message */ + erts_smp_proc_unlock(rp, *rp_locks & ERTS_PROC_LOCKS_XSIG_SEND); + *rp_locks &= ~ERTS_PROC_LOCKS_XSIG_SEND; if (have_seqtrace(token) && token_update) seq_trace_update_send(token_update); if (is_value(exit_tuple)) @@ -12122,6 +12196,7 @@ static void doit_exit_link(ErtsLink *lnk, void *vpcontext) DistEntry *dep; Process *rp; + switch(lnk->type) { case LINK_PID: if(is_internal_port(item)) { @@ -12169,7 +12244,11 @@ static void doit_exit_link(ErtsLink *lnk, void *vpcontext) if (xres >= 0 && IS_TRACED_FL(rp, F_TRACE_PROCS)) { /* We didn't exit the process and it is traced */ if (IS_TRACED_FL(rp, F_TRACE_PROCS)) { - trace_proc(p, rp, am_getting_unlinked, p->common.id); + if (rp_locks & ERTS_PROC_LOCKS_XSIG_SEND) { + erts_smp_proc_unlock(rp, ERTS_PROC_LOCKS_XSIG_SEND); + rp_locks &= ~ERTS_PROC_LOCKS_XSIG_SEND; + } + trace_proc(NULL, 0, rp, am_getting_unlinked, p->common.id); } } } @@ -12286,8 +12365,6 @@ erts_do_exit_process(Process* p, Eterm reason) if (IS_TRACED_FL(p, F_TRACE_CALLS)) erts_schedule_time_break(p, ERTS_BP_CALL_TIME_SCHEDULE_EXITING); - if (IS_TRACED_FL(p,F_TRACE_PROCS)) - trace_proc(p, p, am_exit, reason); } erts_trace_check_exiting(p->common.id); @@ -12303,6 +12380,10 @@ erts_do_exit_process(Process* p, Eterm reason) erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR); + if (IS_TRACED_FL(p,F_TRACE_PROCS)) + trace_proc(p, ERTS_PROC_LOCK_MAIN, p, am_exit, reason); + + /* * p->u.initial of this process can *not* be used anymore; * will be overwritten by misc termination data. @@ -12442,6 +12523,9 @@ erts_continue_exit_process(Process *p) ASSERT(!p->common.u.alive.reg); } + if (IS_TRACED_FL(p, F_TRACE_SCHED_EXIT)) + trace_sched(p, curr_locks, am_out_exited); + erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR); curr_locks = ERTS_PROC_LOCKS_ALL; @@ -12514,9 +12598,11 @@ erts_continue_exit_process(Process *p) } dep = (p->flags & F_DISTRIBUTION) ? erts_this_dist_entry : NULL; - scb = ERTS_PROC_SET_SAVED_CALLS_BUF(p, ERTS_PROC_LOCKS_ALL, NULL); - pbt = ERTS_PROC_SET_CALL_TIME(p, ERTS_PROC_LOCKS_ALL, NULL); - nif_export = ERTS_PROC_SET_NIF_TRAP_EXPORT(p, ERTS_PROC_LOCKS_ALL, NULL); + scb = ERTS_PROC_SET_SAVED_CALLS_BUF(p, NULL); + if (scb) + p->fcalls += CONTEXT_REDS; /* Reduction counting depends on this... */ + pbt = ERTS_PROC_SET_CALL_TIME(p, NULL); + nif_export = ERTS_PROC_SET_NIF_TRAP_EXPORT(p, NULL); erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL); #ifdef BM_COUNTERS @@ -12565,6 +12651,12 @@ erts_continue_exit_process(Process *p) if (nif_export) erts_destroy_nif_export(nif_export); +#ifdef ERTS_SMP + erts_flush_trace_messages(p, 0); +#endif + + ERTS_TRACER_CLEAR(&ERTS_TRACER(p)); + delete_process(p); #ifdef ERTS_SMP diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index c88bd7056c..59da9c1779 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2014. All Rights Reserved. + * Copyright Ericsson AB 1996-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -805,8 +805,15 @@ erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi) #define ERTS_PSD_CALL_TIME_BP 3 #define ERTS_PSD_DELAYED_GC_TASK_QS 4 #define ERTS_PSD_NIF_TRAP_EXPORT 5 +#ifdef HIPE +#define ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF 6 +#endif +#ifdef HIPE +#define ERTS_PSD_SIZE 7 +#else #define ERTS_PSD_SIZE 6 +#endif typedef struct { void *data[ERTS_PSD_SIZE]; @@ -818,8 +825,8 @@ typedef struct { #define ERTS_PSD_ERROR_HANDLER_BUF_GET_LOCKS ERTS_PROC_LOCK_MAIN #define ERTS_PSD_ERROR_HANDLER_BUF_SET_LOCKS ERTS_PROC_LOCK_MAIN -#define ERTS_PSD_SAVED_CALLS_BUF_GET_LOCKS ERTS_PROC_LOCK_MAIN -#define ERTS_PSD_SAVED_CALLS_BUF_SET_LOCKS ERTS_PROC_LOCK_MAIN +#define ERTS_PSD_SAVED_CALLS_BUF_GET_LOCKS ((ErtsProcLocks) 0) +#define ERTS_PSD_SAVED_CALLS_BUF_SET_LOCKS ((ErtsProcLocks) 0) #define ERTS_PSD_SCHED_ID_GET_LOCKS ERTS_PROC_LOCK_STATUS #define ERTS_PSD_SCHED_ID_SET_LOCKS ERTS_PROC_LOCK_STATUS @@ -830,8 +837,8 @@ typedef struct { #define ERTS_PSD_DELAYED_GC_TASK_QS_GET_LOCKS ERTS_PROC_LOCK_MAIN #define ERTS_PSD_DELAYED_GC_TASK_QS_SET_LOCKS ERTS_PROC_LOCK_MAIN -#define ERTS_PSD_NIF_TRAP_EXPORT_GET_LOCKS ERTS_PROC_LOCK_MAIN -#define ERTS_PSD_NIF_TRAP_EXPORT_SET_LOCKS ERTS_PROC_LOCK_MAIN +#define ERTS_PSD_NIF_TRAP_EXPORT_GET_LOCKS ((ErtsProcLocks) 0) +#define ERTS_PSD_NIF_TRAP_EXPORT_SET_LOCKS ((ErtsProcLocks) 0) typedef struct { ErtsProcLocks get_locks; @@ -925,7 +932,6 @@ struct process { Eterm* stop; /* Stack top */ Eterm* heap; /* Heap start */ Eterm* hend; /* Heap end */ - Eterm* abandoned_heap; Uint heap_sz; /* Size of heap in words */ Uint min_heap_size; /* Minimum size of heap (in words). */ Uint min_vheap_size; /* Minimum size of virtual heap (in words). */ @@ -941,6 +947,16 @@ struct process { #endif /* + * Moved to after "struct hipe_process_state hipe", as a temporary fix for + * LLVM hard-coding offsetof(struct process, hipe.nstack) (sic!) + * (see void X86FrameLowering::adjustForHiPEPrologue(...) in + * lib/Target/X86/X86FrameLowering.cpp). + * + * Used to be below "Eterm* hend". + */ + Eterm* abandoned_heap; + + /* * Saved x registers. */ Uint arity; /* Number of live argument registers (only valid @@ -1026,7 +1042,7 @@ struct process { ErlHeapFragment* live_hf_end; ErtsMessage *msg_frag; /* Pointer to message fragment list */ Uint mbuf_sz; /* Total size of heap fragments and message fragments */ - ErtsPSD *psd; /* Rarely used process specific data */ + erts_smp_atomic_t psd; /* Rarely used process specific data */ Uint64 bin_vheap_sz; /* Virtual heap block size for binaries */ Uint64 bin_old_vheap_sz; /* Virtual old heap block size for binaries */ @@ -1041,6 +1057,7 @@ struct process { #ifdef ERTS_SMP ErlMessageInQueue msg_inq; + ErlTraceMessageQueue *trace_msg_q; ErtsPendExit pending_exit; erts_proc_lock_t lock; ErtsSchedulerData *scheduler_data; @@ -1070,6 +1087,10 @@ struct process { Uint space_verified; /* Avoid HAlloc forcing heap fragments when */ Eterm* space_verified_from; /* we rely on available heap space (TestHeap) */ #endif + +#ifdef DEBUG + Uint debug_reds_in; +#endif }; extern const Process erts_invalid_process; @@ -1341,6 +1362,7 @@ extern int erts_system_profile_ts_type; #define F_DELAY_GC (1 << 16) /* Similar to disable GC (see below) */ #define F_SCHDLR_ONLN_WAITQ (1 << 17) /* Process enqueued waiting to change schedulers online */ #define F_HAVE_BLCKD_NMSCHED (1 << 18) /* Process has blocked normal multi-scheduling */ +#define F_HIPE_MODE (1 << 19) /* * F_DISABLE_GC and F_DELAY_GC are similar. Both will prevent @@ -1358,7 +1380,6 @@ extern int erts_system_profile_ts_type; #define F_TRACE_FLAG(N) (1 << (ERTS_TRACE_TS_TYPE_BITS + (N))) /* process trace_flags */ - #define F_NOW_TS (ERTS_TRACE_FLG_NOW_TIMESTAMP \ << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT) #define F_STRICT_MON_TS (ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP \ @@ -1380,8 +1401,7 @@ extern int erts_system_profile_ts_type; #define F_TRACE_ARITY_ONLY F_TRACE_FLAG(12) #define F_TRACE_RETURN_TO F_TRACE_FLAG(13) /* Return_to trace when breakpoint tracing */ #define F_TRACE_SILENT F_TRACE_FLAG(14) /* No call trace msg suppress */ -#define F_TRACER F_TRACE_FLAG(15) /* May be (has been) tracer */ -#define F_EXCEPTION_TRACE F_TRACE_FLAG(16) /* May have exception trace on stack */ +#define F_EXCEPTION_TRACE F_TRACE_FLAG(15) /* May have exception trace on stack */ /* port trace flags, currently the same as process trace flags */ #define F_TRACE_SCHED_PORTS F_TRACE_FLAG(17) /* Trace of port scheduling */ @@ -1410,12 +1430,14 @@ extern int erts_system_profile_ts_type; | F_TRACE_SCHED_PORTS | F_TRACE_SCHED_NO \ | F_TRACE_SCHED_EXIT) + #define ERTS_TRACEE_MODIFIER_FLAGS \ - (F_TRACE_SILENT | F_TIMESTAMP_MASK | F_TRACE_SCHED_NO) -#define ERTS_PORT_TRACEE_FLAGS \ - (ERTS_TRACEE_MODIFIER_FLAGS | F_TRACE_PORTS | F_TRACE_SCHED_PORTS) + (F_TRACE_SILENT | F_TIMESTAMP_MASK | F_TRACE_SCHED_NO \ + | F_TRACE_RECEIVE | F_TRACE_SEND) +#define ERTS_PORT_TRACEE_FLAGS \ + (ERTS_TRACEE_MODIFIER_FLAGS | F_TRACE_PORTS | F_TRACE_SCHED_PORTS) #define ERTS_PROC_TRACEE_FLAGS \ - ((TRACEE_FLAGS & ~ERTS_PORT_TRACEE_FLAGS) | ERTS_TRACEE_MODIFIER_FLAGS) + ((TRACEE_FLAGS & ~ERTS_PORT_TRACEE_FLAGS) | ERTS_TRACEE_MODIFIER_FLAGS) #define SEQ_TRACE_FLAG(N) (1 << (ERTS_TRACE_TS_TYPE_BITS + (N))) @@ -1717,6 +1739,8 @@ void erts_schedule_thr_prgr_later_cleanup_op(void (*)(void *), ErtsThrPrgrLaterOp *, UWord); void erts_schedule_complete_off_heap_message_queue_change(Eterm pid); +void erts_schedule_flush_trace_messages(Eterm pid); +int erts_flush_trace_messages(Process *c_p, ErtsProcLocks locks); #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) int erts_dbg_check_halloc_lock(Process *p); @@ -1903,18 +1927,19 @@ do { \ #define ERTS_SMP_LC_CHK_RUNQ_LOCK(RQ, L) #endif -void *erts_psd_set_init(Process *p, ErtsProcLocks plocks, int ix, void *data); +void *erts_psd_set_init(Process *p, int ix, void *data); ERTS_GLB_INLINE void * erts_psd_get(Process *p, int ix); ERTS_GLB_INLINE void * -erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *new); +erts_psd_set(Process *p, int ix, void *new); #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE void * erts_psd_get(Process *p, int ix) { + ErtsPSD *psd; #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) ErtsProcLocks locks = erts_proc_lc_my_proc_locks(p); if (ERTS_LC_PSD_ANY_LOCK == erts_psd_required_locks[ix].get_locks) @@ -1925,17 +1950,19 @@ erts_psd_get(Process *p, int ix) || erts_thr_progress_is_blocking()); } #endif + + psd = (ErtsPSD *) erts_smp_atomic_read_nob(&p->psd); ASSERT(0 <= ix && ix < ERTS_PSD_SIZE); - return p->psd ? p->psd->data[ix] : NULL; + if (!psd) + return NULL; + ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER; + return psd->data[ix]; } - -/* - * NOTE: erts_psd_set() might release and reacquire locks on 'p'. - */ ERTS_GLB_INLINE void * -erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *data) +erts_psd_set(Process *p, int ix, void *data) { + ErtsPSD *psd; #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) ErtsProcLocks locks = erts_proc_lc_my_proc_locks(p); if (ERTS_LC_PSD_ANY_LOCK == erts_psd_required_locks[ix].set_locks) @@ -1946,50 +1973,62 @@ erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *data) || erts_thr_progress_is_blocking()); } #endif + psd = (ErtsPSD *) erts_smp_atomic_read_nob(&p->psd); ASSERT(0 <= ix && ix < ERTS_PSD_SIZE); - if (p->psd) { - void *old = p->psd->data[ix]; - p->psd->data[ix] = data; + if (psd) { + void *old; +#ifdef ERTS_SMP +#ifdef ETHR_ORDERED_READ_DEPEND + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreStore); +#endif +#endif + old = psd->data[ix]; + psd->data[ix] = data; return old; } - else { - if (!data) - return NULL; - else - return erts_psd_set_init(p, plocks, ix, data); - } + + if (!data) + return NULL; + + return erts_psd_set_init(p, ix, data); } #endif -#define ERTS_PROC_SCHED_ID(P, L, ID) \ - ((UWord) erts_psd_set((P), (L), ERTS_PSD_SCHED_ID, (void *) (ID))) +#define ERTS_PROC_SCHED_ID(P, ID) \ + ((UWord) erts_psd_set((P), ERTS_PSD_SCHED_ID, (void *) (ID))) #define ERTS_PROC_GET_SAVED_CALLS_BUF(P) \ ((struct saved_calls *) erts_psd_get((P), ERTS_PSD_SAVED_CALLS_BUF)) -#define ERTS_PROC_SET_SAVED_CALLS_BUF(P, L, SCB) \ - ((struct saved_calls *) erts_psd_set((P), (L), ERTS_PSD_SAVED_CALLS_BUF, (void *) (SCB))) +#define ERTS_PROC_SET_SAVED_CALLS_BUF(P, SCB) \ + ((struct saved_calls *) erts_psd_set((P), ERTS_PSD_SAVED_CALLS_BUF, (void *) (SCB))) #define ERTS_PROC_GET_CALL_TIME(P) \ ((process_breakpoint_time_t *) erts_psd_get((P), ERTS_PSD_CALL_TIME_BP)) -#define ERTS_PROC_SET_CALL_TIME(P, L, PBT) \ - ((process_breakpoint_time_t *) erts_psd_set((P), (L), ERTS_PSD_CALL_TIME_BP, (void *) (PBT))) +#define ERTS_PROC_SET_CALL_TIME(P, PBT) \ + ((process_breakpoint_time_t *) erts_psd_set((P), ERTS_PSD_CALL_TIME_BP, (void *) (PBT))) #define ERTS_PROC_GET_DELAYED_GC_TASK_QS(P) \ ((ErtsProcSysTaskQs *) erts_psd_get((P), ERTS_PSD_DELAYED_GC_TASK_QS)) -#define ERTS_PROC_SET_DELAYED_GC_TASK_QS(P, L, PBT) \ - ((ErtsProcSysTaskQs *) erts_psd_set((P), (L), ERTS_PSD_DELAYED_GC_TASK_QS, (void *) (PBT))) +#define ERTS_PROC_SET_DELAYED_GC_TASK_QS(P, PBT) \ + ((ErtsProcSysTaskQs *) erts_psd_set((P), ERTS_PSD_DELAYED_GC_TASK_QS, (void *) (PBT))) #define ERTS_PROC_GET_NIF_TRAP_EXPORT(P) \ erts_psd_get((P), ERTS_PSD_NIF_TRAP_EXPORT) -#define ERTS_PROC_SET_NIF_TRAP_EXPORT(P, L, NTE) \ - erts_psd_set((P), (L), ERTS_PSD_NIF_TRAP_EXPORT, (void *) (NTE)) +#define ERTS_PROC_SET_NIF_TRAP_EXPORT(P, NTE) \ + erts_psd_set((P), ERTS_PSD_NIF_TRAP_EXPORT, (void *) (NTE)) +#ifdef HIPE +#define ERTS_PROC_GET_SUSPENDED_SAVED_CALLS_BUF(P) \ + ((struct saved_calls *) erts_psd_get((P), ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF)) +#define ERTS_PROC_SET_SUSPENDED_SAVED_CALLS_BUF(P, SCB) \ + ((struct saved_calls *) erts_psd_set((P), ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF, (void *) (SCB))) +#endif ERTS_GLB_INLINE Eterm erts_proc_get_error_handler(Process *p); -ERTS_GLB_INLINE Eterm erts_proc_set_error_handler(Process *p, - ErtsProcLocks plocks, - Eterm handler); +ERTS_GLB_INLINE Eterm erts_proc_set_error_handler(Process *p, Eterm handler); #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE Eterm @@ -2005,13 +2044,13 @@ erts_proc_get_error_handler(Process *p) } ERTS_GLB_INLINE Eterm -erts_proc_set_error_handler(Process *p, ErtsProcLocks plocks, Eterm handler) +erts_proc_set_error_handler(Process *p, Eterm handler) { void *old_val; void *new_val; ASSERT(is_atom(handler)); new_val = (handler == am_error_handler) ? NULL : (void *) (UWord) handler; - old_val = erts_psd_set(p, plocks, ERTS_PSD_ERROR_HANDLER, new_val); + old_val = erts_psd_set(p, ERTS_PSD_ERROR_HANDLER, new_val); if (!old_val) return am_error_handler; else { @@ -2317,14 +2356,17 @@ erts_alloc_message_heap_state(Process *pp, ErlOffHeap **ohpp) { int on_heap; + ErtsMessage *mp; if ((*psp) & ERTS_PSFLG_OFF_HEAP_MSGQ) { - ErtsMessage *mp = erts_alloc_message(sz, hpp); + mp = erts_alloc_message(sz, hpp); *ohpp = sz == 0 ? NULL : &mp->hfrag.off_heap; return mp; } - return erts_try_alloc_message_on_heap(pp, psp, plp, sz, hpp, ohpp, &on_heap); + mp = erts_try_alloc_message_on_heap(pp, psp, plp, sz, hpp, ohpp, &on_heap); + ASSERT(pp || !on_heap); + return mp; } ERTS_GLB_INLINE ErtsMessage * @@ -2334,7 +2376,7 @@ erts_alloc_message_heap(Process *pp, Eterm **hpp, ErlOffHeap **ohpp) { - erts_aint32_t state = erts_smp_atomic32_read_nob(&pp->state); + erts_aint32_t state = pp ? erts_smp_atomic32_read_nob(&pp->state) : 0; return erts_alloc_message_heap_state(pp, &state, plp, sz, hpp, ohpp); } diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c index 487b944fc0..d8c2eaba94 100644 --- a/erts/emulator/beam/erl_process_dict.c +++ b/erts/emulator/beam/erl_process_dict.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2013. All Rights Reserved. + * Copyright Ericsson AB 1999-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_process_dict.h b/erts/emulator/beam/erl_process_dict.h index dac214c8a1..387562058c 100644 --- a/erts/emulator/beam/erl_process_dict.h +++ b/erts/emulator/beam/erl_process_dict.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2009. All Rights Reserved. + * Copyright Ericsson AB 1999-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c index 39c36ee7a9..fa76773cac 100644 --- a/erts/emulator/beam/erl_process_dump.c +++ b/erts/emulator/beam/erl_process_dump.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2003-2013. All Rights Reserved. + * Copyright Ericsson AB 2003-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -103,7 +103,7 @@ Uint erts_process_memory(Process *p, int incl_msg_inq) { size += p->arity * sizeof(p->arg_reg[0]); } - if (p->psd) + if (erts_smp_atomic_read_nob(&p->psd) != (erts_aint_t) NULL) size += sizeof(ErtsPSD); scb = ERTS_PROC_GET_SAVED_CALLS_BUF(p); diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c index 0bee2c848c..a69185bc5c 100644 --- a/erts/emulator/beam/erl_process_lock.c +++ b/erts/emulator/beam/erl_process_lock.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2007-2012. All Rights Reserved. + * Copyright Ericsson AB 2007-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -106,6 +106,7 @@ static struct { Sint16 proc_lock_msgq; Sint16 proc_lock_btm; Sint16 proc_lock_status; + Sint16 proc_lock_trace; } lc_id; #endif @@ -152,6 +153,7 @@ erts_init_proc_lock(int cpus) lc_id.proc_lock_msgq = erts_lc_get_lock_order_id("proc_msgq"); lc_id.proc_lock_btm = erts_lc_get_lock_order_id("proc_btm"); lc_id.proc_lock_status = erts_lc_get_lock_order_id("proc_status"); + lc_id.proc_lock_trace = erts_lc_get_lock_order_id("proc_trace"); #endif } @@ -1057,6 +1059,11 @@ erts_proc_lock_init(Process *p) #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_trylock(1, &p->lock.status.lc); #endif + erts_mtx_init_x(&p->lock.trace, "proc_trace", p->common.id, do_lock_count); + ethr_mutex_lock(&p->lock.trace.mtx); +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_lc_trylock(1, &p->lock.trace.lc); +#endif #endif #ifdef ERTS_PROC_LOCK_DEBUG for (i = 0; i <= ERTS_PROC_LOCK_MAX_BIT; i++) @@ -1078,6 +1085,7 @@ erts_proc_lock_fin(Process *p) erts_mtx_destroy(&p->lock.msgq); erts_mtx_destroy(&p->lock.btm); erts_mtx_destroy(&p->lock.status); + erts_mtx_destroy(&p->lock.trace); #endif #if defined(ERTS_ENABLE_LOCK_COUNT) && defined(ERTS_SMP) erts_lcnt_proc_lock_destroy(p); @@ -1100,26 +1108,29 @@ void erts_lcnt_enable_proc_lock_count(int enable) { void erts_lcnt_proc_lock_init(Process *p) { if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) { erts_lcnt_init_lock_empty(&(p->lock.lcnt_main)); + erts_lcnt_init_lock_empty(&(p->lock.lcnt_link)); erts_lcnt_init_lock_empty(&(p->lock.lcnt_msgq)); erts_lcnt_init_lock_empty(&(p->lock.lcnt_btm)); - erts_lcnt_init_lock_empty(&(p->lock.lcnt_link)); erts_lcnt_init_lock_empty(&(p->lock.lcnt_status)); + erts_lcnt_init_lock_empty(&(p->lock.lcnt_trace)); } else { /* now the common case */ Eterm pid = (p->common.id != ERTS_INVALID_PID) ? p->common.id : NIL; erts_lcnt_init_lock_x(&(p->lock.lcnt_main), "proc_main", ERTS_LCNT_LT_PROCLOCK, pid); + erts_lcnt_init_lock_x(&(p->lock.lcnt_link), "proc_link", ERTS_LCNT_LT_PROCLOCK, pid); erts_lcnt_init_lock_x(&(p->lock.lcnt_msgq), "proc_msgq", ERTS_LCNT_LT_PROCLOCK, pid); erts_lcnt_init_lock_x(&(p->lock.lcnt_btm), "proc_btm", ERTS_LCNT_LT_PROCLOCK, pid); - erts_lcnt_init_lock_x(&(p->lock.lcnt_link), "proc_link", ERTS_LCNT_LT_PROCLOCK, pid); erts_lcnt_init_lock_x(&(p->lock.lcnt_status),"proc_status",ERTS_LCNT_LT_PROCLOCK, pid); + erts_lcnt_init_lock_x(&(p->lock.lcnt_trace), "proc_trace", ERTS_LCNT_LT_PROCLOCK, pid); } /* the lock names should really be aligned to four characters */ } /* logic reversed */ void erts_lcnt_proc_lock_destroy(Process *p) { erts_lcnt_destroy_lock(&(p->lock.lcnt_main)); + erts_lcnt_destroy_lock(&(p->lock.lcnt_link)); erts_lcnt_destroy_lock(&(p->lock.lcnt_msgq)); erts_lcnt_destroy_lock(&(p->lock.lcnt_btm)); - erts_lcnt_destroy_lock(&(p->lock.lcnt_link)); erts_lcnt_destroy_lock(&(p->lock.lcnt_status)); + erts_lcnt_destroy_lock(&(p->lock.lcnt_trace)); } static void lcnt_enable_proc_lock_count(Process *proc, int enable) { @@ -1138,10 +1149,11 @@ static void lcnt_enable_proc_lock_count(Process *proc, int enable) { void erts_lcnt_proc_lock(erts_proc_lock_t *lock, ErtsProcLocks locks) { if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return; if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_lock(&(lock->lcnt_main)); } + if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_lock(&(lock->lcnt_link)); } if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_lock(&(lock->lcnt_msgq)); } if (locks & ERTS_PROC_LOCK_BTM) { erts_lcnt_lock(&(lock->lcnt_btm)); } - if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_lock(&(lock->lcnt_link)); } if (locks & ERTS_PROC_LOCK_STATUS) { erts_lcnt_lock(&(lock->lcnt_status)); } + if (locks & ERTS_PROC_LOCK_TRACE) { erts_lcnt_lock(&(lock->lcnt_trace)); } } void erts_lcnt_proc_lock_post_x(erts_proc_lock_t *lock, ErtsProcLocks locks, @@ -1150,44 +1162,50 @@ void erts_lcnt_proc_lock_post_x(erts_proc_lock_t *lock, ErtsProcLocks locks, if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_lock_post_x(&(lock->lcnt_main), file, line); } + if (locks & ERTS_PROC_LOCK_LINK) { + erts_lcnt_lock_post_x(&(lock->lcnt_link), file, line); + } if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_lock_post_x(&(lock->lcnt_msgq), file, line); } if (locks & ERTS_PROC_LOCK_BTM) { erts_lcnt_lock_post_x(&(lock->lcnt_btm), file, line); } - if (locks & ERTS_PROC_LOCK_LINK) { - erts_lcnt_lock_post_x(&(lock->lcnt_link), file, line); - } if (locks & ERTS_PROC_LOCK_STATUS) { erts_lcnt_lock_post_x(&(lock->lcnt_status), file, line); } + if (locks & ERTS_PROC_LOCK_TRACE) { + erts_lcnt_lock_post_x(&(lock->lcnt_trace), file, line); + } } void erts_lcnt_proc_lock_unaquire(erts_proc_lock_t *lock, ErtsProcLocks locks) { if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return; if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_lock_unaquire(&(lock->lcnt_main)); } + if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_lock_unaquire(&(lock->lcnt_link)); } if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_lock_unaquire(&(lock->lcnt_msgq)); } if (locks & ERTS_PROC_LOCK_BTM) { erts_lcnt_lock_unaquire(&(lock->lcnt_btm)); } - if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_lock_unaquire(&(lock->lcnt_link)); } if (locks & ERTS_PROC_LOCK_STATUS) { erts_lcnt_lock_unaquire(&(lock->lcnt_status)); } + if (locks & ERTS_PROC_LOCK_TRACE) { erts_lcnt_lock_unaquire(&(lock->lcnt_trace)); } } void erts_lcnt_proc_unlock(erts_proc_lock_t *lock, ErtsProcLocks locks) { if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return; if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_unlock(&(lock->lcnt_main)); } + if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_unlock(&(lock->lcnt_link)); } if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_unlock(&(lock->lcnt_msgq)); } if (locks & ERTS_PROC_LOCK_BTM) { erts_lcnt_unlock(&(lock->lcnt_btm)); } - if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_unlock(&(lock->lcnt_link)); } if (locks & ERTS_PROC_LOCK_STATUS) { erts_lcnt_unlock(&(lock->lcnt_status)); } + if (locks & ERTS_PROC_LOCK_TRACE) { erts_lcnt_unlock(&(lock->lcnt_trace)); } } void erts_lcnt_proc_trylock(erts_proc_lock_t *lock, ErtsProcLocks locks, int res) { if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return; if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_trylock(&(lock->lcnt_main), res); } + if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_trylock(&(lock->lcnt_link), res); } if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_trylock(&(lock->lcnt_msgq), res); } if (locks & ERTS_PROC_LOCK_BTM) { erts_lcnt_trylock(&(lock->lcnt_btm), res); } - if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_trylock(&(lock->lcnt_link), res); } if (locks & ERTS_PROC_LOCK_STATUS) { erts_lcnt_trylock(&(lock->lcnt_status), res); } + if (locks & ERTS_PROC_LOCK_TRACE) { erts_lcnt_trylock(&(lock->lcnt_trace), res); } } /* reversed logic */ #endif /* ERTS_ENABLE_LOCK_COUNT */ @@ -1224,6 +1242,10 @@ erts_proc_lc_lock(Process *p, ErtsProcLocks locks, char *file, unsigned int line lck.id = lc_id.proc_lock_status; erts_lc_lock_x(&lck,file,line); } + if (locks & ERTS_PROC_LOCK_TRACE) { + lck.id = lc_id.proc_lock_trace; + erts_lc_lock_x(&lck,file,line); + } } void @@ -1253,6 +1275,10 @@ erts_proc_lc_trylock(Process *p, ErtsProcLocks locks, int locked, lck.id = lc_id.proc_lock_status; erts_lc_trylock_x(locked, &lck, file, line); } + if (locks & ERTS_PROC_LOCK_TRACE) { + lck.id = lc_id.proc_lock_trace; + erts_lc_trylock_x(locked, &lck, file, line); + } } void @@ -1261,6 +1287,10 @@ erts_proc_lc_unlock(Process *p, ErtsProcLocks locks) erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1, p->common.id, ERTS_LC_FLG_LT_PROCLOCK); + if (locks & ERTS_PROC_LOCK_TRACE) { + lck.id = lc_id.proc_lock_trace; + erts_lc_unlock(&lck); + } if (locks & ERTS_PROC_LOCK_STATUS) { lck.id = lc_id.proc_lock_status; erts_lc_unlock(&lck); @@ -1292,6 +1322,10 @@ erts_proc_lc_might_unlock(Process *p, ErtsProcLocks locks) erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1, p->common.id, ERTS_LC_FLG_LT_PROCLOCK); + if (locks & ERTS_PROC_LOCK_TRACE) { + lck.id = lc_id.proc_lock_trace; + erts_lc_might_unlock(&lck); + } if (locks & ERTS_PROC_LOCK_STATUS) { lck.id = lc_id.proc_lock_status; erts_lc_might_unlock(&lck); @@ -1323,6 +1357,8 @@ erts_proc_lc_might_unlock(Process *p, ErtsProcLocks locks) erts_lc_might_unlock(&p->lock.btm.lc); if (locks & ERTS_PROC_LOCK_STATUS) erts_lc_might_unlock(&p->lock.status.lc); + if (locks & ERTS_PROC_LOCK_TRACE) + erts_lc_might_unlock(&p->lock.trace.lc); #endif } @@ -1354,6 +1390,10 @@ erts_proc_lc_require_lock(Process *p, ErtsProcLocks locks, char *file, lck.id = lc_id.proc_lock_status; erts_lc_require_lock(&lck, file, line); } + if (locks & ERTS_PROC_LOCK_TRACE) { + lck.id = lc_id.proc_lock_trace; + erts_lc_require_lock(&lck, file, line); + } #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL if (locks & ERTS_PROC_LOCK_MAIN) erts_lc_require_lock(&p->lock.main.lc, file, line); @@ -1365,6 +1405,8 @@ erts_proc_lc_require_lock(Process *p, ErtsProcLocks locks, char *file, erts_lc_require_lock(&p->lock.btm.lc, file, line); if (locks & ERTS_PROC_LOCK_STATUS) erts_lc_require_lock(&p->lock.status.lc, file, line); + if (locks & ERTS_PROC_LOCK_TRACE) + erts_lc_require_lock(&p->lock.trace.lc, file, line); #endif } @@ -1375,6 +1417,10 @@ erts_proc_lc_unrequire_lock(Process *p, ErtsProcLocks locks) erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1, p->common.id, ERTS_LC_FLG_LT_PROCLOCK); + if (locks & ERTS_PROC_LOCK_TRACE) { + lck.id = lc_id.proc_lock_trace; + erts_lc_unrequire_lock(&lck); + } if (locks & ERTS_PROC_LOCK_STATUS) { lck.id = lc_id.proc_lock_status; erts_lc_unrequire_lock(&lck); @@ -1406,6 +1452,8 @@ erts_proc_lc_unrequire_lock(Process *p, ErtsProcLocks locks) erts_lc_unrequire_lock(&p->lock.btm.lc); if (locks & ERTS_PROC_LOCK_STATUS) erts_lc_unrequire_lock(&p->lock.status.lc); + if (locks & ERTS_PROC_LOCK_TRACE) + erts_lc_unrequire_lock(&p->lock.trace.lc); #endif } @@ -1429,6 +1477,8 @@ erts_proc_lc_trylock_force_busy(Process *p, ErtsProcLocks locks) lck.id = lc_id.proc_lock_btm; else if (locks & ERTS_PROC_LOCK_STATUS) lck.id = lc_id.proc_lock_status; + else if (locks & ERTS_PROC_LOCK_TRACE) + lck.id = lc_id.proc_lock_trace; else erts_lc_fail("Unknown proc lock found"); @@ -1441,14 +1491,7 @@ erts_proc_lc_trylock_force_busy(Process *p, ErtsProcLocks locks) void erts_proc_lc_chk_only_proc_main(Process *p) { -#if ERTS_PROC_LOCK_OWN_IMPL - erts_lc_lock_t proc_main = ERTS_LC_LOCK_INIT(lc_id.proc_lock_main, - p->common.id, - ERTS_LC_FLG_LT_PROCLOCK); - erts_lc_check_exact(&proc_main, 1); -#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL - erts_lc_check_exact(&p->lock.main.lc, 1); -#endif + erts_proc_lc_chk_only_proc(p, ERTS_PROC_LOCK_MAIN); } #if ERTS_PROC_LOCK_OWN_IMPL @@ -1456,14 +1499,67 @@ void erts_proc_lc_chk_only_proc_main(Process *p) ERTS_LC_LOCK_INIT(-1, THE_NON_VALUE, ERTS_LC_FLG_LT_PROCLOCK) #endif /* ERTS_PROC_LOCK_OWN_IMPL */ +void erts_proc_lc_chk_only_proc(Process *p, ErtsProcLocks locks) +{ + int have_locks_len = 0; +#if ERTS_PROC_LOCK_OWN_IMPL + erts_lc_lock_t have_locks[6] = {ERTS_PROC_LC_EMPTY_LOCK_INIT, + ERTS_PROC_LC_EMPTY_LOCK_INIT, + ERTS_PROC_LC_EMPTY_LOCK_INIT, + ERTS_PROC_LC_EMPTY_LOCK_INIT, + ERTS_PROC_LC_EMPTY_LOCK_INIT, + ERTS_PROC_LC_EMPTY_LOCK_INIT}; + if (locks & ERTS_PROC_LOCK_MAIN) { + have_locks[have_locks_len].id = lc_id.proc_lock_main; + have_locks[have_locks_len++].extra = p->common.id; + } + if (locks & ERTS_PROC_LOCK_LINK) { + have_locks[have_locks_len].id = lc_id.proc_lock_link; + have_locks[have_locks_len++].extra = p->common.id; + } + if (locks & ERTS_PROC_LOCK_MSGQ) { + have_locks[have_locks_len].id = lc_id.proc_lock_msgq; + have_locks[have_locks_len++].extra = p->common.id; + } + if (locks & ERTS_PROC_LOCK_BTM) { + have_locks[have_locks_len].id = lc_id.proc_lock_btm; + have_locks[have_locks_len++].extra = p->common.id; + } + if (locks & ERTS_PROC_LOCK_STATUS) { + have_locks[have_locks_len].id = lc_id.proc_lock_status; + have_locks[have_locks_len++].extra = p->common.id; + } + if (locks & ERTS_PROC_LOCK_TRACE) { + have_locks[have_locks_len].id = lc_id.proc_lock_trace; + have_locks[have_locks_len++].extra = p->common.id; + } +#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL + erts_lc_lock_t have_locks[6]; + if (locks & ERTS_PROC_LOCK_MAIN) + have_locks[have_locks_len++] = p->lock.main.lc; + if (locks & ERTS_PROC_LOCK_LINK) + have_locks[have_locks_len++] = p->lock.link.lc; + if (locks & ERTS_PROC_LOCK_MSGQ) + have_locks[have_locks_len++] = p->lock.msgq.lc; + if (locks & ERTS_PROC_LOCK_BTM) + have_locks[have_locks_len++] = p->lock.btm.lc; + if (locks & ERTS_PROC_LOCK_STATUS) + have_locks[have_locks_len++] = p->lock.status.lc; + if (locks & ERTS_PROC_LOCK_TRACE) + have_locks[have_locks_len++] = p->lock.trace.lc; +#endif + erts_lc_check_exact(have_locks, have_locks_len); +} + void erts_proc_lc_chk_have_proc_locks(Process *p, ErtsProcLocks locks) { int have_locks_len = 0; #if ERTS_PROC_LOCK_OWN_IMPL - erts_lc_lock_t have_locks[5] = {ERTS_PROC_LC_EMPTY_LOCK_INIT, + erts_lc_lock_t have_locks[6] = {ERTS_PROC_LC_EMPTY_LOCK_INIT, ERTS_PROC_LC_EMPTY_LOCK_INIT, ERTS_PROC_LC_EMPTY_LOCK_INIT, + ERTS_PROC_LC_EMPTY_LOCK_INIT, ERTS_PROC_LC_EMPTY_LOCK_INIT, ERTS_PROC_LC_EMPTY_LOCK_INIT}; if (locks & ERTS_PROC_LOCK_MAIN) { @@ -1486,8 +1582,12 @@ erts_proc_lc_chk_have_proc_locks(Process *p, ErtsProcLocks locks) have_locks[have_locks_len].id = lc_id.proc_lock_status; have_locks[have_locks_len++].extra = p->common.id; } + if (locks & ERTS_PROC_LOCK_TRACE) { + have_locks[have_locks_len].id = lc_id.proc_lock_trace; + have_locks[have_locks_len++].extra = p->common.id; + } #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL - erts_lc_lock_t have_locks[5]; + erts_lc_lock_t have_locks[6]; if (locks & ERTS_PROC_LOCK_MAIN) have_locks[have_locks_len++] = p->lock.main.lc; if (locks & ERTS_PROC_LOCK_LINK) @@ -1498,6 +1598,8 @@ erts_proc_lc_chk_have_proc_locks(Process *p, ErtsProcLocks locks) have_locks[have_locks_len++] = p->lock.btm.lc; if (locks & ERTS_PROC_LOCK_STATUS) have_locks[have_locks_len++] = p->lock.status.lc; + if (locks & ERTS_PROC_LOCK_TRACE) + have_locks[have_locks_len++] = p->lock.trace.lc; #endif erts_lc_check(have_locks, have_locks_len, NULL, 0); } @@ -1508,12 +1610,14 @@ erts_proc_lc_chk_proc_locks(Process *p, ErtsProcLocks locks) int have_locks_len = 0; int have_not_locks_len = 0; #if ERTS_PROC_LOCK_OWN_IMPL - erts_lc_lock_t have_locks[5] = {ERTS_PROC_LC_EMPTY_LOCK_INIT, + erts_lc_lock_t have_locks[6] = {ERTS_PROC_LC_EMPTY_LOCK_INIT, ERTS_PROC_LC_EMPTY_LOCK_INIT, + ERTS_PROC_LC_EMPTY_LOCK_INIT, ERTS_PROC_LC_EMPTY_LOCK_INIT, ERTS_PROC_LC_EMPTY_LOCK_INIT}; - erts_lc_lock_t have_not_locks[5] = {ERTS_PROC_LC_EMPTY_LOCK_INIT, + erts_lc_lock_t have_not_locks[6] = {ERTS_PROC_LC_EMPTY_LOCK_INIT, ERTS_PROC_LC_EMPTY_LOCK_INIT, + ERTS_PROC_LC_EMPTY_LOCK_INIT, ERTS_PROC_LC_EMPTY_LOCK_INIT, ERTS_PROC_LC_EMPTY_LOCK_INIT}; @@ -1557,9 +1661,17 @@ erts_proc_lc_chk_proc_locks(Process *p, ErtsProcLocks locks) have_not_locks[have_not_locks_len].id = lc_id.proc_lock_status; have_not_locks[have_not_locks_len++].extra = p->common.id; } + if (locks & ERTS_PROC_LOCK_TRACE) { + have_locks[have_locks_len].id = lc_id.proc_lock_trace; + have_locks[have_locks_len++].extra = p->common.id; + } + else { + have_not_locks[have_not_locks_len].id = lc_id.proc_lock_trace; + have_not_locks[have_not_locks_len++].extra = p->common.id; + } #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL - erts_lc_lock_t have_locks[5]; - erts_lc_lock_t have_not_locks[5]; + erts_lc_lock_t have_locks[6]; + erts_lc_lock_t have_not_locks[6]; if (locks & ERTS_PROC_LOCK_MAIN) have_locks[have_locks_len++] = p->lock.main.lc; @@ -1581,6 +1693,10 @@ erts_proc_lc_chk_proc_locks(Process *p, ErtsProcLocks locks) have_locks[have_locks_len++] = p->lock.status.lc; else have_not_locks[have_not_locks_len++] = p->lock.status.lc; + if (locks & ERTS_PROC_LOCK_TRACE) + have_locks[have_locks_len++] = p->lock.trace.lc; + else + have_not_locks[have_not_locks_len++] = p->lock.trace.lc; #endif erts_lc_check(have_locks, have_locks_len, @@ -1590,10 +1706,10 @@ erts_proc_lc_chk_proc_locks(Process *p, ErtsProcLocks locks) ErtsProcLocks erts_proc_lc_my_proc_locks(Process *p) { - int resv[5]; + int resv[6]; ErtsProcLocks res = 0; #if ERTS_PROC_LOCK_OWN_IMPL - erts_lc_lock_t locks[5] = {ERTS_LC_LOCK_INIT(lc_id.proc_lock_main, + erts_lc_lock_t locks[6] = {ERTS_LC_LOCK_INIT(lc_id.proc_lock_main, p->common.id, ERTS_LC_FLG_LT_PROCLOCK), ERTS_LC_LOCK_INIT(lc_id.proc_lock_link, @@ -1607,16 +1723,20 @@ erts_proc_lc_my_proc_locks(Process *p) ERTS_LC_FLG_LT_PROCLOCK), ERTS_LC_LOCK_INIT(lc_id.proc_lock_status, p->common.id, + ERTS_LC_FLG_LT_PROCLOCK), + ERTS_LC_LOCK_INIT(lc_id.proc_lock_trace, + p->common.id, ERTS_LC_FLG_LT_PROCLOCK)}; #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL - erts_lc_lock_t locks[5] = {p->lock.main.lc, + erts_lc_lock_t locks[6] = {p->lock.main.lc, p->lock.link.lc, p->lock.msgq.lc, p->lock.btm.lc, - p->lock.status.lc}; + p->lock.status.lc, + p->lock.trace.lc}; #endif - erts_lc_have_locks(resv, locks, 5); + erts_lc_have_locks(resv, locks, 6); if (resv[0]) res |= ERTS_PROC_LOCK_MAIN; if (resv[1]) @@ -1627,6 +1747,8 @@ erts_proc_lc_my_proc_locks(Process *p) res |= ERTS_PROC_LOCK_BTM; if (resv[4]) res |= ERTS_PROC_LOCK_STATUS; + if (resv[5]) + res |= ERTS_PROC_LOCK_TRACE; return res; } @@ -1634,14 +1756,15 @@ erts_proc_lc_my_proc_locks(Process *p) void erts_proc_lc_chk_no_proc_locks(char *file, int line) { - int resv[5]; - int ids[5] = {lc_id.proc_lock_main, + int resv[6]; + int ids[6] = {lc_id.proc_lock_main, lc_id.proc_lock_link, lc_id.proc_lock_msgq, lc_id.proc_lock_btm, - lc_id.proc_lock_status}; - erts_lc_have_lock_ids(resv, ids, 5); - if (!ERTS_IS_CRASH_DUMPING && (resv[0] || resv[1] || resv[2] || resv[3] || resv[4])) { + lc_id.proc_lock_status, + lc_id.proc_lock_trace}; + erts_lc_have_lock_ids(resv, ids, 6); + if (!ERTS_IS_CRASH_DUMPING && (resv[0] || resv[1] || resv[2] || resv[3] || resv[4] || resv[5])) { erts_lc_fail("%s:%d: Thread has process locks locked when expected " "not to have any process locks locked", file, line); diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h index 9c59301086..2cccf0697a 100644 --- a/erts/emulator/beam/erl_process_lock.h +++ b/erts/emulator/beam/erl_process_lock.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2007-2012. All Rights Reserved. + * Copyright Ericsson AB 2007-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -66,7 +66,7 @@ #endif -#define ERTS_PROC_LOCK_MAX_BIT 4 +#define ERTS_PROC_LOCK_MAX_BIT 5 typedef erts_aint32_t ErtsProcLocks; @@ -84,6 +84,7 @@ typedef struct erts_proc_lock_t_ { erts_lcnt_lock_t lcnt_msgq; erts_lcnt_lock_t lcnt_btm; erts_lcnt_lock_t lcnt_status; + erts_lcnt_lock_t lcnt_trace; #endif #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL erts_mtx_t main; @@ -91,6 +92,7 @@ typedef struct erts_proc_lock_t_ { erts_mtx_t msgq; erts_mtx_t btm; erts_mtx_t status; + erts_mtx_t trace; #else # error "no implementation" #endif @@ -137,9 +139,18 @@ typedef struct erts_proc_lock_t_ { * Protects the following fields in the process structure: * * pending_suspenders * * suspendee + * * sys_tasks * * ... */ -#define ERTS_PROC_LOCK_STATUS (((ErtsProcLocks) 1) << ERTS_PROC_LOCK_MAX_BIT) +#define ERTS_PROC_LOCK_STATUS (((ErtsProcLocks) 1) << 4) + +/* + * Trace message lock: + * Protects the order in which messages are sent + * from trace nifs. This lock is taken inside enif_send. + * + */ +#define ERTS_PROC_LOCK_TRACE (((ErtsProcLocks) 1) << ERTS_PROC_LOCK_MAX_BIT) /* * Special fields: @@ -154,8 +165,8 @@ typedef struct erts_proc_lock_t_ { * all process locks are held, and are allowed to be read if * at least one process lock (whichever one doesn't matter) * is held: - * * tracer_proc - * * tracer_flags + * * common.tracer + * * common.trace_flags * * The following fields are only allowed to be accessed if * both the schedule queue lock and at least one process lock @@ -198,17 +209,15 @@ typedef struct erts_proc_lock_t_ { /* ERTS_PROC_LOCKS_* are combinations of process locks */ -#define ERTS_PROC_LOCKS_MSG_RECEIVE (ERTS_PROC_LOCK_MSGQ \ - | ERTS_PROC_LOCK_STATUS) -#define ERTS_PROC_LOCKS_MSG_SEND (ERTS_PROC_LOCK_MSGQ \ - | ERTS_PROC_LOCK_STATUS) +#define ERTS_PROC_LOCKS_MSG_RECEIVE ERTS_PROC_LOCK_MSGQ +#define ERTS_PROC_LOCKS_MSG_SEND ERTS_PROC_LOCK_MSGQ #define ERTS_PROC_LOCKS_XSIG_SEND ERTS_PROC_LOCK_STATUS #define ERTS_PROC_LOCKS_ALL \ ((((ErtsProcLocks) 1) << (ERTS_PROC_LOCK_MAX_BIT + 1)) - 1) #define ERTS_PROC_LOCKS_ALL_MINOR (ERTS_PROC_LOCKS_ALL \ - & ~ERTS_PROC_LOCK_MAIN) + & ~ERTS_PROC_LOCK_MAIN) #define ERTS_PIX_LOCKS_BITS 10 @@ -260,6 +269,7 @@ void erts_proc_lc_might_unlock(Process *p, ErtsProcLocks locks); void erts_proc_lc_chk_have_proc_locks(Process *p, ErtsProcLocks locks); void erts_proc_lc_chk_proc_locks(Process *p, ErtsProcLocks locks); void erts_proc_lc_chk_only_proc_main(Process *p); +void erts_proc_lc_chk_only_proc(Process *p, ErtsProcLocks locks); void erts_proc_lc_chk_no_proc_locks(char *file, int line); ErtsProcLocks erts_proc_lc_my_proc_locks(Process *p); int erts_proc_lc_trylock_force_busy(Process *p, ErtsProcLocks locks); @@ -477,9 +487,15 @@ erts_smp_proc_raw_trylock__(Process *p, ErtsProcLocks locks) if (locks & ERTS_PROC_LOCK_STATUS) if (erts_mtx_trylock(&p->lock.status) == EBUSY) goto busy_status; + if (locks & ERTS_PROC_LOCK_TRACE) + if (erts_mtx_trylock(&p->lock.trace) == EBUSY) + goto busy_trace; return 0; +busy_trace: + if (locks & ERTS_PROC_LOCK_TRACE) + erts_mtx_unlock(&p->lock.trace); busy_status: if (locks & ERTS_PROC_LOCK_BTM) erts_mtx_unlock(&p->lock.btm); @@ -568,6 +584,8 @@ erts_smp_proc_lock__(Process *p, erts_mtx_lock(&p->lock.btm); if (locks & ERTS_PROC_LOCK_STATUS) erts_mtx_lock(&p->lock.status); + if (locks & ERTS_PROC_LOCK_TRACE) + erts_mtx_lock(&p->lock.trace); #ifdef ERTS_PROC_LOCK_DEBUG erts_proc_lock_op_debug(p, locks, 1); @@ -653,6 +671,8 @@ erts_smp_proc_unlock__(Process *p, erts_proc_lock_op_debug(p, locks, 0); #endif + if (locks & ERTS_PROC_LOCK_TRACE) + erts_mtx_unlock(&p->lock.trace); if (locks & ERTS_PROC_LOCK_STATUS) erts_mtx_unlock(&p->lock.status); if (locks & ERTS_PROC_LOCK_BTM) diff --git a/erts/emulator/beam/erl_ptab.c b/erts/emulator/beam/erl_ptab.c index 9ed175fe01..22830a19c4 100644 --- a/erts/emulator/beam/erl_ptab.c +++ b/erts/emulator/beam/erl_ptab.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2012-2013. All Rights Reserved. + * Copyright Ericsson AB 2012-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_ptab.h b/erts/emulator/beam/erl_ptab.h index 8fd961e3ce..a5931ffc25 100644 --- a/erts/emulator/beam/erl_ptab.h +++ b/erts/emulator/beam/erl_ptab.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2012-2013. All Rights Reserved. + * Copyright Ericsson AB 2012-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -37,14 +37,16 @@ #include "erl_alloc.h" #include "erl_monitors.h" -#define ERTS_TRACER_PROC(P) ((P)->common.tracer_proc) +#define ERTS_TRACER(P) ((P)->common.tracer) +#define ERTS_TRACER_MODULE(T) (CAR(list_val(T))) +#define ERTS_TRACER_STATE(T) (CDR(list_val(T))) #define ERTS_TRACE_FLAGS(P) ((P)->common.trace_flags) #define ERTS_P_LINKS(P) ((P)->common.u.alive.links) #define ERTS_P_MONITORS(P) ((P)->common.u.alive.monitors) #define IS_TRACED(p) \ - (ERTS_TRACER_PROC((p)) != NIL) + (ERTS_TRACER(p) != NIL) #define ARE_TRACE_FLAGS_ON(p,tf) \ ((ERTS_TRACE_FLAGS((p)) & (tf|F_SENSITIVE)) == (tf)) #define IS_TRACED_FL(p,tf) \ @@ -56,7 +58,7 @@ typedef struct { erts_atomic_t atmc; Sint sint; } refc; - Eterm tracer_proc; + ErtsTracer tracer; Uint trace_flags; erts_smp_atomic_t timer; union { diff --git a/erts/emulator/beam/erl_sched_spec_pre_alloc.c b/erts/emulator/beam/erl_sched_spec_pre_alloc.c index caec24bc03..cab4bd73db 100644 --- a/erts/emulator/beam/erl_sched_spec_pre_alloc.c +++ b/erts/emulator/beam/erl_sched_spec_pre_alloc.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2011-2012. All Rights Reserved. + * Copyright Ericsson AB 2011-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_sched_spec_pre_alloc.h b/erts/emulator/beam/erl_sched_spec_pre_alloc.h index 4d07b0f674..7808d7d438 100644 --- a/erts/emulator/beam/erl_sched_spec_pre_alloc.h +++ b/erts/emulator/beam/erl_sched_spec_pre_alloc.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2011-2012. All Rights Reserved. + * Copyright Ericsson AB 2011-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h index 5fc5e989a6..713ed50b86 100644 --- a/erts/emulator/beam/erl_smp.h +++ b/erts/emulator/beam/erl_smp.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2013. All Rights Reserved. + * Copyright Ericsson AB 2005-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_sock.h b/erts/emulator/beam/erl_sock.h index 7be6062115..3429a52d7e 100644 --- a/erts/emulator/beam/erl_sock.h +++ b/erts/emulator/beam/erl_sock.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2003-2009. All Rights Reserved. + * Copyright Ericsson AB 2003-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_sys_driver.h b/erts/emulator/beam/erl_sys_driver.h index 4031eef0aa..d46e88cb05 100644 --- a/erts/emulator/beam/erl_sys_driver.h +++ b/erts/emulator/beam/erl_sys_driver.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2013. All Rights Reserved. + * Copyright Ericsson AB 2001-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_term.c b/erts/emulator/beam/erl_term.c index d2343912b4..7d857ad326 100644 --- a/erts/emulator/beam/erl_term.c +++ b/erts/emulator/beam/erl_term.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2013. All Rights Reserved. + * Copyright Ericsson AB 2000-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -59,96 +59,6 @@ erts_set_literal_tag(Eterm *term, Eterm *hp_start, Eterm hsz) #endif } -__decl_noreturn static void __noreturn -et_abort(const char *expr, const char *file, unsigned line) -{ -#ifdef EXIT_ON_ET_ABORT - static int have_been_called = 0; - - if (have_been_called) { - abort(); - } else { - /* - * Prevent infinite loop. - */ - have_been_called = 1; - erts_exit(ERTS_ERROR_EXIT, "TYPE ASSERTION FAILED, file %s, line %u: %s\n", file, line, expr); - } -#else - erts_fprintf(stderr, "TYPE ASSERTION FAILED, file %s, line %u: %s\n", file, line, expr); - abort(); -#endif -} - -#if ET_DEBUG -#define ET_ASSERT(expr,file,line) \ -do { \ - if (!(expr)) \ - et_abort(#expr, file, line); \ -} while(0) -#else -#define ET_ASSERT(expr,file,line) do { } while(0) -#endif - -#if ET_DEBUG -unsigned tag_val_def_debug(Wterm x, const char *file, unsigned line) -#else -unsigned tag_val_def(Wterm x) -#define file __FILE__ -#define line __LINE__ -#endif -{ - static char msg[32]; - - switch (x & _TAG_PRIMARY_MASK) { - case TAG_PRIMARY_LIST: - ET_ASSERT(_list_precond(x),file,line); - return LIST_DEF; - case TAG_PRIMARY_BOXED: { - Eterm hdr = *boxed_val(x); - ET_ASSERT(is_header(hdr),file,line); - switch ((hdr & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE) { - case (_TAG_HEADER_ARITYVAL >> _TAG_PRIMARY_SIZE): return TUPLE_DEF; - case (_TAG_HEADER_POS_BIG >> _TAG_PRIMARY_SIZE): return BIG_DEF; - case (_TAG_HEADER_NEG_BIG >> _TAG_PRIMARY_SIZE): return BIG_DEF; - case (_TAG_HEADER_REF >> _TAG_PRIMARY_SIZE): return REF_DEF; - case (_TAG_HEADER_FLOAT >> _TAG_PRIMARY_SIZE): return FLOAT_DEF; - case (_TAG_HEADER_EXPORT >> _TAG_PRIMARY_SIZE): return EXPORT_DEF; - case (_TAG_HEADER_FUN >> _TAG_PRIMARY_SIZE): return FUN_DEF; - case (_TAG_HEADER_EXTERNAL_PID >> _TAG_PRIMARY_SIZE): return EXTERNAL_PID_DEF; - case (_TAG_HEADER_EXTERNAL_PORT >> _TAG_PRIMARY_SIZE): return EXTERNAL_PORT_DEF; - case (_TAG_HEADER_EXTERNAL_REF >> _TAG_PRIMARY_SIZE): return EXTERNAL_REF_DEF; - case (_TAG_HEADER_MAP >> _TAG_PRIMARY_SIZE): return MAP_DEF; - case (_TAG_HEADER_REFC_BIN >> _TAG_PRIMARY_SIZE): return BINARY_DEF; - case (_TAG_HEADER_HEAP_BIN >> _TAG_PRIMARY_SIZE): return BINARY_DEF; - case (_TAG_HEADER_SUB_BIN >> _TAG_PRIMARY_SIZE): return BINARY_DEF; - case (_TAG_HEADER_BIN_MATCHSTATE >> _TAG_PRIMARY_SIZE): return MATCHSTATE_DEF; - } - - break; - } - case TAG_PRIMARY_IMMED1: { - switch ((x & _TAG_IMMED1_MASK) >> _TAG_PRIMARY_SIZE) { - case (_TAG_IMMED1_PID >> _TAG_PRIMARY_SIZE): return PID_DEF; - case (_TAG_IMMED1_PORT >> _TAG_PRIMARY_SIZE): return PORT_DEF; - case (_TAG_IMMED1_IMMED2 >> _TAG_PRIMARY_SIZE): { - switch ((x & _TAG_IMMED2_MASK) >> _TAG_IMMED1_SIZE) { - case (_TAG_IMMED2_ATOM >> _TAG_IMMED1_SIZE): return ATOM_DEF; - case (_TAG_IMMED2_NIL >> _TAG_IMMED1_SIZE): return NIL_DEF; - } - break; - } - case (_TAG_IMMED1_SMALL >> _TAG_PRIMARY_SIZE): return SMALL_DEF; - } - break; - } - } - erts_snprintf(msg, sizeof(msg), "tag_val_def: %#lx", (unsigned long) x); - et_abort(msg, file, line); -#undef file -#undef line -} - /* * XXX: define NUMBER_CODE() here when new representation is used */ diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h index 2b28762db5..c3234ee349 100644 --- a/erts/emulator/beam/erl_term.h +++ b/erts/emulator/beam/erl_term.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2014. All Rights Reserved. + * Copyright Ericsson AB 2000-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -558,14 +558,6 @@ _ET_DECLARE_CHECKED(Eterm*,tuple_val,Wterm) #define _GETBITS(X,Pos,Size) (((X) >> (Pos)) & ~(~((Uint) 0) << (Size))) -/* - * Creation in node specific data (pids, ports, refs) - */ - -#define _CRE_SIZE 2 - -/* MAX value for the creation field in pid, port and reference */ -#define MAX_CREATION (1 << _CRE_SIZE) /* * PID layout (internal pids): @@ -579,7 +571,7 @@ _ET_DECLARE_CHECKED(Eterm*,tuple_val,Wterm) * * n : number * - * Old pid layout: + * Very old pid layout: * * |3 3 2 2 2 2 2 2|2 2 2 2 1 1 1 1|1 1 1 1 1 1 | | * |1 0 9 8 7 6 5 4|3 2 1 0 9 8 7 6|5 4 3 2 1 0 9 8|7 6 5 4 3 2 1 0| @@ -1129,11 +1121,11 @@ _ET_DECLARE_CHECKED(Uint,loader_y_reg_index,Uint) #define FIRST_VACANT_TAG_DEF 0x12 #if ET_DEBUG -extern unsigned tag_val_def_debug(Wterm, const char*, unsigned); -#define tag_val_def(x) tag_val_def_debug((x),__FILE__,__LINE__) +ERTS_GLB_INLINE unsigned tag_val_def(Wterm, const char*, unsigned); #else -extern unsigned tag_val_def(Wterm); +ERTS_GLB_INLINE unsigned tag_val_def(Wterm); #endif + #define not_eq_tags(X,Y) (tag_val_def((X)) ^ tag_val_def((Y))) #define NUMBER_CODE(x,y) ((tag_val_def(x) << 5) | tag_val_def(y)) @@ -1152,5 +1144,80 @@ extern unsigned tag_val_def(Wterm); void erts_set_literal_tag(Eterm *term, Eterm *hp_start, Eterm hsz); +#if ET_DEBUG +#define ET_ASSERT(expr,file,line) \ +do { \ + if (!(expr)) \ + erl_assert_error("TYPE ASSERTION: " #expr, __FUNCTION__, file, line); \ +} while(0) +#else +#define ET_ASSERT(expr,file,line) do { } while(0) +#endif + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +#if ET_DEBUG +ERTS_GLB_INLINE unsigned tag_val_def(Wterm x, const char *file, unsigned line) +#else +ERTS_GLB_INLINE unsigned tag_val_def(Wterm x) +#define file __FILE__ +#define line __LINE__ +#endif +{ + static char *msg = "tag_val_def error"; + + switch (x & _TAG_PRIMARY_MASK) { + case TAG_PRIMARY_LIST: + ET_ASSERT(_list_precond(x),file,line); + return LIST_DEF; + case TAG_PRIMARY_BOXED: { + Eterm hdr = *boxed_val(x); + ET_ASSERT(is_header(hdr),file,line); + switch ((hdr & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE) { + case (_TAG_HEADER_ARITYVAL >> _TAG_PRIMARY_SIZE): return TUPLE_DEF; + case (_TAG_HEADER_POS_BIG >> _TAG_PRIMARY_SIZE): return BIG_DEF; + case (_TAG_HEADER_NEG_BIG >> _TAG_PRIMARY_SIZE): return BIG_DEF; + case (_TAG_HEADER_REF >> _TAG_PRIMARY_SIZE): return REF_DEF; + case (_TAG_HEADER_FLOAT >> _TAG_PRIMARY_SIZE): return FLOAT_DEF; + case (_TAG_HEADER_EXPORT >> _TAG_PRIMARY_SIZE): return EXPORT_DEF; + case (_TAG_HEADER_FUN >> _TAG_PRIMARY_SIZE): return FUN_DEF; + case (_TAG_HEADER_EXTERNAL_PID >> _TAG_PRIMARY_SIZE): return EXTERNAL_PID_DEF; + case (_TAG_HEADER_EXTERNAL_PORT >> _TAG_PRIMARY_SIZE): return EXTERNAL_PORT_DEF; + case (_TAG_HEADER_EXTERNAL_REF >> _TAG_PRIMARY_SIZE): return EXTERNAL_REF_DEF; + case (_TAG_HEADER_MAP >> _TAG_PRIMARY_SIZE): return MAP_DEF; + case (_TAG_HEADER_REFC_BIN >> _TAG_PRIMARY_SIZE): return BINARY_DEF; + case (_TAG_HEADER_HEAP_BIN >> _TAG_PRIMARY_SIZE): return BINARY_DEF; + case (_TAG_HEADER_SUB_BIN >> _TAG_PRIMARY_SIZE): return BINARY_DEF; + case (_TAG_HEADER_BIN_MATCHSTATE >> _TAG_PRIMARY_SIZE): return MATCHSTATE_DEF; + } + + break; + } + case TAG_PRIMARY_IMMED1: { + switch ((x & _TAG_IMMED1_MASK) >> _TAG_PRIMARY_SIZE) { + case (_TAG_IMMED1_PID >> _TAG_PRIMARY_SIZE): return PID_DEF; + case (_TAG_IMMED1_PORT >> _TAG_PRIMARY_SIZE): return PORT_DEF; + case (_TAG_IMMED1_IMMED2 >> _TAG_PRIMARY_SIZE): { + switch ((x & _TAG_IMMED2_MASK) >> _TAG_IMMED1_SIZE) { + case (_TAG_IMMED2_ATOM >> _TAG_IMMED1_SIZE): return ATOM_DEF; + case (_TAG_IMMED2_NIL >> _TAG_IMMED1_SIZE): return NIL_DEF; + } + break; + } + case (_TAG_IMMED1_SMALL >> _TAG_PRIMARY_SIZE): return SMALL_DEF; + } + break; + } + } + erl_assert_error(msg, __FUNCTION__, file, line); +#undef file +#undef line +} +#endif + +#if ET_DEBUG +#define tag_val_def(X) tag_val_def(X, __FILE__, __LINE__) +#endif + #endif /* __ERL_TERM_H */ diff --git a/erts/emulator/beam/erl_thr_progress.c b/erts/emulator/beam/erl_thr_progress.c index 7b06fd840f..542541165b 100644 --- a/erts/emulator/beam/erl_thr_progress.c +++ b/erts/emulator/beam/erl_thr_progress.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2011-2013. All Rights Reserved. + * Copyright Ericsson AB 2011-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_thr_progress.h b/erts/emulator/beam/erl_thr_progress.h index b89cc4c267..b2894ba1fe 100644 --- a/erts/emulator/beam/erl_thr_progress.h +++ b/erts/emulator/beam/erl_thr_progress.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2011-2013. All Rights Reserved. + * Copyright Ericsson AB 2011-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_thr_queue.c b/erts/emulator/beam/erl_thr_queue.c index 7ff456b915..f56d0828dd 100644 --- a/erts/emulator/beam/erl_thr_queue.c +++ b/erts/emulator/beam/erl_thr_queue.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2011-2013. All Rights Reserved. + * Copyright Ericsson AB 2011-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -780,3 +780,35 @@ erts_thr_q_dequeue(ErtsThrQ_t *q) return res; #endif } + +#ifdef USE_LTTNG_VM_TRACEPOINTS +int +erts_thr_q_length_dirty(ErtsThrQ_t *q) +{ + int n = 0; +#ifndef USE_THREADS + void *res; + ErtsThrQElement_t *tmp; + + for (tmp = q->first; tmp != NULL; tmp = tmp->next) { + n++; + } +#else + ErtsThrQElement_t *e; + erts_aint_t inext; + + e = ErtsThrQDirtyReadEl(&q->head.head); + inext = erts_atomic_read_acqb(&e->next); + + while (inext != ERTS_AINT_NULL) { + e = (ErtsThrQElement_t *) inext; + if (e != &q->tail.data.marker) { + /* don't count marker */ + n++; + } + inext = erts_atomic_read_acqb(&e->next); + } +#endif + return n; +} +#endif diff --git a/erts/emulator/beam/erl_thr_queue.h b/erts/emulator/beam/erl_thr_queue.h index 27a6d03224..705a67af4c 100644 --- a/erts/emulator/beam/erl_thr_queue.h +++ b/erts/emulator/beam/erl_thr_queue.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2011-2013. All Rights Reserved. + * Copyright Ericsson AB 2011-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -190,6 +190,10 @@ void erts_thr_q_append_finalize_dequeue_data(ErtsThrQFinDeQ_t *, int erts_thr_q_finalize_dequeue(ErtsThrQFinDeQ_t *); void erts_thr_q_finalize_dequeue_state_init(ErtsThrQFinDeQ_t *); +#ifdef USE_LTTNG_VM_TRACEPOINTS +int erts_thr_q_length_dirty(ErtsThrQ_t *); +#endif + #ifdef ERTS_SMP ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_q_need_thr_progress(ErtsThrQ_t *q); #endif diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h index e689547d1d..eccd49f2a9 100644 --- a/erts/emulator/beam/erl_threads.h +++ b/erts/emulator/beam/erl_threads.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2013. All Rights Reserved. + * Copyright Ericsson AB 2001-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h index 5242063550..a1c4220633 100644 --- a/erts/emulator/beam/erl_time.h +++ b/erts/emulator/beam/erl_time.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2006-2011. All Rights Reserved. + * Copyright Ericsson AB 2006-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c index f3f528eaf6..fadbd704bd 100644 --- a/erts/emulator/beam/erl_time_sup.c +++ b/erts/emulator/beam/erl_time_sup.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2015. All Rights Reserved. + * Copyright Ericsson AB 1999-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -1966,7 +1966,7 @@ send_time_offset_changed_notifications(void *new_offsetp) *patch_refp = ref; ASSERT(hsz == size_object(message_template)); message = copy_struct(message_template, hsz, &hp, ohp); - erts_queue_message(rp, &rp_locks, mp, message, NIL); + erts_queue_message(rp, rp_locks, mp, message, am_clock_service); } erts_smp_proc_unlock(rp, rp_locks); } diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index 41876b8c62..1c0fc0a11f 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2014. All Rights Reserved. + * Copyright Ericsson AB 1999-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +20,19 @@ /* * Support functions for tracing. + * + * Ideas for future speed improvements in tracing framework: + * * Move ErtsTracerNif into ErtsTracer + * + Removes need for locking + * + Removes hash lookup overhead + * + Use a refc on the ErtsTracerNif to know when it can + * be freed. We don't want to allocate a separate + * ErtsTracerNif for each module used. + * * Optimize GenericBp for cache locality by reusing equivalent + * GenericBp and GenericBpData in multiple tracer points. + * + Possibly we want to use specialized instructions for different + * types of trace so that the knowledge of which struct is used + * can be in the instruction. */ #ifdef HAVE_CONFIG_H @@ -39,6 +52,7 @@ #include "erl_bits.h" #include "erl_thr_progress.h" #include "erl_bif_unique.h" +#include "erl_map.h" #if 0 #define DEBUG_PRINTOUTS @@ -46,17 +60,15 @@ #undef DEBUG_PRINTOUTS #endif -extern BeamInstr beam_return_to_trace[1]; /* OpCode(i_return_to_trace) */ -extern BeamInstr beam_return_trace[1]; /* OpCode(i_return_trace) */ -extern BeamInstr beam_return_time_trace[1]; /* OpCode(i_return_time_trace) */ - /* Pseudo export entries. Never filled in with data, only used to yield unique pointers of the correct type. */ Export exp_send, exp_receive, exp_timeout; -static Eterm system_seq_tracer; -static Uint default_trace_flags; -static Eterm default_tracer; +static ErtsTracer system_seq_tracer; +static Uint default_proc_trace_flags; +static ErtsTracer default_proc_tracer; +static Uint default_port_trace_flags; +static ErtsTracer default_port_tracer; static Eterm system_monitor; static Eterm system_profile; @@ -70,8 +82,6 @@ static erts_smp_rwmtx_t sys_trace_rwmtx; enum ErtsSysMsgType { SYS_MSG_TYPE_UNDEFINED, - SYS_MSG_TYPE_TRACE, - SYS_MSG_TYPE_SEQTRACE, SYS_MSG_TYPE_SYSMON, SYS_MSG_TYPE_ERRLGR, SYS_MSG_TYPE_PROC_MSG, @@ -176,7 +186,7 @@ take_timestamp(ErtsTraceTimeStamp *tsp, int ts_type) hsz += 3; /* 2-tuple */ raw_unique = erts_raw_get_unique_monotonic_integer(); tsp->u.monotonic.raw_unique = raw_unique; - hsz += erts_raw_unique_monotonic_integer_heap_size(raw_unique); + hsz += erts_raw_unique_monotonic_integer_heap_size(raw_unique, 0); } return hsz; } @@ -216,8 +226,7 @@ write_timestamp(ErtsTraceTimeStamp *tsp, Eterm **hpp) return emtime; raw = tsp->u.monotonic.raw_unique; - unique = erts_raw_make_unique_monotonic_integer_value(hpp, - raw); + unique = erts_raw_make_unique_monotonic_integer_value(hpp, raw, 0); res = TUPLE2(*hpp, emtime, unique); *hpp += 3; return res; @@ -228,7 +237,7 @@ write_timestamp(ErtsTraceTimeStamp *tsp, Eterm **hpp) } } -#define PATCH_TS_SIZE(p) patch_ts_size(TFLGS_TS_TYPE(p)) +#ifdef ERTS_SMP static ERTS_INLINE Uint patch_ts_size(int ts_type) @@ -248,6 +257,7 @@ patch_ts_size(int ts_type) return 0; } } +#endif /* ERTS_SMP */ /* * Write a timestamp. The timestamp MUST be the last @@ -299,43 +309,6 @@ write_ts(int ts_type, Eterm *hp, ErlHeapFragment *bp, Process *tracer) return res; } -/* - * Patch a timestamp into a tuple. The tuple MUST be the last thing - * built on the heap before the call, and the timestamp MUST be - * the last thing after the call. This since patch_ts() might adjust - * the size of the used area. - */ - -#define PATCH_TS__(Type, Tuple, Hp, Bp, Tracer) \ - do { \ - int ts_type__ = (Type); \ - if (ts_type__) \ - patch_ts(ts_type__, (Tuple), (Hp), (Bp), (Tracer)); \ - } while (0) - -#ifdef ERTS_SMP -#define PATCH_TS(Type, Tuple, Hp, Bp, Tracer) \ - PATCH_TS__((Type), (Tuple), (Hp), (Bp), NULL) -#else -#define PATCH_TS(Type, Tuple, Hp, Bp, Tracer) \ - PATCH_TS__((Type), (Tuple), (Hp), (Bp), (Tracer)) -#endif - -static ERTS_INLINE void -patch_ts(int ts_type, Eterm tuple, Eterm* hp, ErlHeapFragment *bp, Process *tracer) -{ - Eterm *tptr = tuple_val(tuple); - int arity = arityval(*tptr); - - ASSERT(ts_type); - ASSERT((tptr+arity+1) == hp); - - tptr[0] = make_arityval(arity+1); - tptr[1] = am_trace_ts; - - *hp = write_ts(ts_type, hp+1, bp, tracer); -} - #ifdef ERTS_SMP static void enqueue_sys_msg_unlocked(enum ErtsSysMsgType type, Eterm from, @@ -350,6 +323,14 @@ static void enqueue_sys_msg(enum ErtsSysMsgType type, static void init_sys_msg_dispatcher(void); #endif +static void init_tracer_nif(void); +static int tracer_cmp_fun(void*, void*); +static HashValue tracer_hash_fun(void*); +static void *tracer_alloc_fun(void*); +static void tracer_free_fun(void*); + +typedef struct ErtsTracerNif_ ErtsTracerNif; + void erts_init_trace(void) { erts_smp_rwmtx_opt_t rwmtx_opts = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; rwmtx_opts.type = ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ; @@ -363,80 +344,67 @@ void erts_init_trace(void) { erts_bif_trace_init(); erts_system_monitor_clear(NULL); erts_system_profile_clear(NULL); - default_trace_flags = F_INITIAL_TRACE_FLAGS; - default_tracer = NIL; - system_seq_tracer = am_false; + default_proc_trace_flags = F_INITIAL_TRACE_FLAGS; + default_proc_tracer = erts_tracer_nil; + default_port_trace_flags = F_INITIAL_TRACE_FLAGS; + default_port_tracer = erts_tracer_nil; + system_seq_tracer = erts_tracer_nil; #ifdef ERTS_SMP init_sys_msg_dispatcher(); #endif + init_tracer_nif(); } -static Eterm system_seq_tracer; - #define ERTS_ALLOC_SYSMSG_HEAP(SZ, BPP, OHPP, UNUSED) \ (*(BPP) = new_message_buffer((SZ)), \ *(OHPP) = &(*(BPP))->off_heap, \ (*(BPP))->mem) -#ifdef ERTS_SMP -#define ERTS_ENQ_TRACE_MSG(FPID, TPID, MSG, BP) \ -do { \ - ERTS_LC_ASSERT(erts_smp_lc_mtx_is_locked(&smq_mtx)); \ - enqueue_sys_msg_unlocked(SYS_MSG_TYPE_TRACE, (FPID), (TPID), (MSG), (BP)); \ -} while(0) -#else -#define ERTS_ENQ_TRACE_MSG(FPID, TPROC, MSG, BP) \ - do { \ - ErtsMessage *mp__ = erts_alloc_message(0, NULL); \ - mp__->data.heap_frag = (BP); \ - erts_queue_message((TPROC), NULL, mp__, (MSG), NIL); \ - } while (0) -#endif +enum ErtsTracerOpt { + TRACE_FUN_DEFAULT = 0, + TRACE_FUN_ENABLED = 1, + TRACE_FUN_T_SEND = 2, + TRACE_FUN_T_RECEIVE = 3, + TRACE_FUN_T_CALL = 4, + TRACE_FUN_T_SCHED_PROC = 5, + TRACE_FUN_T_SCHED_PORT = 6, + TRACE_FUN_T_GC = 7, + TRACE_FUN_T_PROCS = 8, + TRACE_FUN_T_PORTS = 9, + TRACE_FUN_E_SEND = 10, + TRACE_FUN_E_RECEIVE = 11, + TRACE_FUN_E_CALL = 12, + TRACE_FUN_E_SCHED_PROC = 13, + TRACE_FUN_E_SCHED_PORT = 14, + TRACE_FUN_E_GC = 15, + TRACE_FUN_E_PROCS = 16, + TRACE_FUN_E_PORTS = 17 +}; -/* - * NOTE that the ERTS_GET_TRACER_REF() returns from the function (!!!) - * using it, and resets the parameters used if the tracer is invalid, i.e., - * use it with extreme care! - */ -#ifdef ERTS_SMP -#define ERTS_NULL_TRACER_REF NIL -#define ERTS_TRACER_REF_TYPE Eterm - /* In the smp case, we never find the tracer invalid here (the sys - message dispatcher thread takes care of that). */ -#define ERTS_GET_TRACER_REF(RES, TPID, TRACEE_FLGS) \ -do { (RES) = (TPID); } while(0) -int -erts_is_tracer_proc_valid(Process* p) -{ - return 1; -} -#else -#define ERTS_NULL_TRACER_REF NULL -#define ERTS_TRACER_REF_TYPE Process * -#define ERTS_GET_TRACER_REF(RES, TPID, TRACEE_FLGS) \ -do { \ - (RES) = erts_proc_lookup((TPID)); \ - if (!(RES) || !(ERTS_TRACE_FLAGS((RES)) & F_TRACER)) { \ - (TPID) = NIL; \ - (TRACEE_FLGS) &= ~TRACEE_FLAGS; \ - return; \ - } \ -} while (0) -int -erts_is_tracer_proc_valid(Process* p) -{ - Process* tracer; +#define NIF_TRACER_TYPES (18) - tracer = erts_proc_lookup(ERTS_TRACER_PROC(p)); - if (tracer && ERTS_TRACE_FLAGS(tracer) & F_TRACER) { - return 1; - } else { - ERTS_TRACER_PROC(p) = NIL; - ERTS_TRACE_FLAGS(p) = ~TRACEE_FLAGS; - return 0; - } -} -#endif + +static ERTS_INLINE int +send_to_tracer_nif_raw(Process *c_p, Process *tracee, const ErtsTracer tracer, + Uint trace_flags, Eterm t_p_id, ErtsTracerNif *tnif, + enum ErtsTracerOpt topt, + Eterm tag, Eterm msg, Eterm extra, Eterm pam_result); +static ERTS_INLINE int +send_to_tracer_nif(Process *c_p, ErtsPTabElementCommon *t_p, + Eterm t_p_id, ErtsTracerNif *tnif, + enum ErtsTracerOpt topt, + Eterm tag, Eterm msg, Eterm extra, + Eterm pam_result); +static ERTS_INLINE Eterm +call_enabled_tracer(Process *c_p, const ErtsTracer tracer, + ErtsTracerNif **tnif_ref, + enum ErtsTracerOpt topt, + Eterm tag, Eterm t_p_id); +static int +is_tracer_enabled(Process* c_p, ErtsProcLocks c_p_locks, + ErtsPTabElementCommon *t_p, + ErtsTracerNif **tnif_ret, + enum ErtsTracerOpt topt, Eterm tag); static Uint active_sched; @@ -451,19 +419,6 @@ static void exiting_reset(Eterm exiting) { erts_smp_rwmtx_rwlock(&sys_trace_rwmtx); - if (exiting == default_tracer) { - default_tracer = NIL; - default_trace_flags &= TRACEE_FLAGS; -#ifdef DEBUG - default_trace_flags |= F_INITIAL_TRACE_FLAGS; -#endif - } - if (exiting == system_seq_tracer) { -#ifdef DEBUG_PRINTOUTS - erts_fprintf(stderr, "seq tracer %T exited\n", exiting); -#endif - system_seq_tracer = am_false; - } if (exiting == system_monitor) { #ifdef ERTS_SMP system_monitor = NIL; @@ -488,11 +443,7 @@ erts_trace_check_exiting(Eterm exiting) { int reset = 0; erts_smp_rwmtx_rlock(&sys_trace_rwmtx); - if (exiting == default_tracer) - reset = 1; - else if (exiting == system_seq_tracer) - reset = 1; - else if (exiting == system_monitor) + if (exiting == system_monitor) reset = 1; else if (exiting == system_profile) reset = 1; @@ -501,23 +452,26 @@ erts_trace_check_exiting(Eterm exiting) exiting_reset(exiting); } -static ERTS_INLINE int -is_valid_tracer(Eterm tracer) +ErtsTracer +erts_set_system_seq_tracer(Process *c_p, ErtsProcLocks c_p_locks, ErtsTracer new) { - return erts_proc_lookup(tracer) || erts_is_valid_tracer_port(tracer); -} + ErtsTracer old; -Eterm -erts_set_system_seq_tracer(Process *c_p, ErtsProcLocks c_p_locks, Eterm new) -{ - Eterm old; - - if (new != am_false && !is_valid_tracer(new)) - return THE_NON_VALUE; + if (!ERTS_TRACER_IS_NIL(new)) { + Eterm nif_result = call_enabled_tracer( + NULL, new, NULL, + TRACE_FUN_ENABLED, am_trace_status, am_undefined); + switch (nif_result) { + case am_trace: break; + default: + return THE_NON_VALUE; + } + } erts_smp_rwmtx_rwlock(&sys_trace_rwmtx); old = system_seq_tracer; - system_seq_tracer = new; + system_seq_tracer = erts_tracer_nil; + erts_tracer_update(&system_seq_tracer, new); #ifdef DEBUG_PRINTOUTS erts_fprintf(stderr, "set seq tracer new=%T old=%T\n", new, old); @@ -526,66 +480,134 @@ erts_set_system_seq_tracer(Process *c_p, ErtsProcLocks c_p_locks, Eterm new) return old; } -Eterm +ErtsTracer erts_get_system_seq_tracer(void) { - Eterm st; + ErtsTracer st; erts_smp_rwmtx_rlock(&sys_trace_rwmtx); st = system_seq_tracer; #ifdef DEBUG_PRINTOUTS erts_fprintf(stderr, "get seq tracer %T\n", st); #endif erts_smp_rwmtx_runlock(&sys_trace_rwmtx); + + if (st != erts_tracer_nil && + call_enabled_tracer(NULL, st, NULL, TRACE_FUN_ENABLED, + am_trace_status, am_undefined) == am_remove) { + erts_set_system_seq_tracer(NULL, 0, erts_tracer_nil); + st = erts_tracer_nil; + } + return st; } static ERTS_INLINE void -get_default_tracing(Uint *flagsp, Eterm *tracerp) -{ - if (!(default_trace_flags & TRACEE_FLAGS)) - default_tracer = NIL; - - if (is_nil(default_tracer)) { - default_trace_flags &= ~TRACEE_FLAGS; - } else if (is_internal_pid(default_tracer)) { - if (!erts_proc_lookup(default_tracer)) { - reset_tracer: - default_trace_flags &= ~TRACEE_FLAGS; - default_tracer = NIL; - } +get_default_tracing(Uint *flagsp, ErtsTracer *tracerp, + Uint *default_trace_flags, + ErtsTracer *default_tracer) +{ + if (!(*default_trace_flags & TRACEE_FLAGS)) + ERTS_TRACER_CLEAR(default_tracer); + + if (ERTS_TRACER_IS_NIL(*default_tracer)) { + *default_trace_flags &= ~TRACEE_FLAGS; } else { - ASSERT(is_internal_port(default_tracer)); - if (!erts_is_valid_tracer_port(default_tracer)) - goto reset_tracer; + Eterm nif_res; + nif_res = call_enabled_tracer(NULL, *default_tracer, + NULL, TRACE_FUN_ENABLED, + am_trace_status, am_undefined); + switch (nif_res) { + case am_trace: break; + default: { + ErtsTracer curr_default_tracer = *default_tracer; + if (tracerp) { + /* we only have a rlock, so we have to unlock and then rwlock */ + erts_smp_rwmtx_runlock(&sys_trace_rwmtx); + erts_smp_rwmtx_rwlock(&sys_trace_rwmtx); + } + /* check if someone else changed default tracer + while we got the write lock, if so we don't do + anything. */ + if (curr_default_tracer == *default_tracer) { + *default_trace_flags &= ~TRACEE_FLAGS; + ERTS_TRACER_CLEAR(default_tracer); + } + if (tracerp) { + erts_smp_rwmtx_rwunlock(&sys_trace_rwmtx); + erts_smp_rwmtx_rlock(&sys_trace_rwmtx); + } + } + } } if (flagsp) - *flagsp = default_trace_flags; - if (tracerp) - *tracerp = default_tracer; + *flagsp = *default_trace_flags; + if (tracerp) { + erts_tracer_update(tracerp,*default_tracer); + } +} + +static ERTS_INLINE void +erts_change_default_tracing(int setflags, Uint flags, + const ErtsTracer tracer, + Uint *default_trace_flags, + ErtsTracer *default_tracer) +{ + if (setflags) + *default_trace_flags |= flags; + else + *default_trace_flags &= ~flags; + + erts_tracer_update(default_tracer, tracer); + + get_default_tracing(NULL, NULL, default_trace_flags, default_tracer); } void -erts_change_default_tracing(int setflags, Uint *flagsp, Eterm *tracerp) +erts_change_default_proc_tracing(int setflags, Uint flagsp, + const ErtsTracer tracer) { erts_smp_rwmtx_rwlock(&sys_trace_rwmtx); - if (flagsp) { - if (setflags) - default_trace_flags |= *flagsp; - else - default_trace_flags &= ~(*flagsp); - } - if (tracerp) - default_tracer = *tracerp; - get_default_tracing(flagsp, tracerp); + erts_change_default_tracing( + setflags, flagsp, tracer, + &default_proc_trace_flags, + &default_proc_tracer); + erts_smp_rwmtx_rwunlock(&sys_trace_rwmtx); +} + +void +erts_change_default_port_tracing(int setflags, Uint flagsp, + const ErtsTracer tracer) +{ + erts_smp_rwmtx_rwlock(&sys_trace_rwmtx); + erts_change_default_tracing( + setflags, flagsp, tracer, + &default_port_trace_flags, + &default_port_tracer); erts_smp_rwmtx_rwunlock(&sys_trace_rwmtx); } void -erts_get_default_tracing(Uint *flagsp, Eterm *tracerp) +erts_get_default_proc_tracing(Uint *flagsp, ErtsTracer *tracerp) +{ + erts_smp_rwmtx_rlock(&sys_trace_rwmtx); + *tracerp = erts_tracer_nil; /* initialize */ + get_default_tracing( + flagsp, tracerp, + &default_proc_trace_flags, + &default_proc_tracer); + erts_smp_rwmtx_runlock(&sys_trace_rwmtx); +} + +void +erts_get_default_port_tracing(Uint *flagsp, ErtsTracer *tracerp) { erts_smp_rwmtx_rlock(&sys_trace_rwmtx); - get_default_tracing(flagsp, tracerp); + *tracerp = erts_tracer_nil; /* initialize */ + get_default_tracing( + flagsp, tracerp, + &default_port_trace_flags, + &default_port_tracer); erts_smp_rwmtx_runlock(&sys_trace_rwmtx); } @@ -623,29 +645,22 @@ erts_get_system_profile(void) { return profile; } -#ifdef ERTS_SMP -static void -do_send_to_port(Eterm to, - Port* unused_port, - Eterm from, - enum ErtsSysMsgType type, - Eterm message) -{ - Uint sz = size_object(message); - ErlHeapFragment *bp = new_message_buffer(sz); - Uint *hp = bp->mem; - Eterm msg = copy_struct(message, sz, &hp, &bp->off_heap); - enqueue_sys_msg_unlocked(type, from, to, msg, bp); -} - -#define WRITE_SYS_MSG_TO_PORT write_sys_msg_to_port +#ifdef HAVE_ERTS_NOW_CPU +# define GET_NOW(m, s, u) \ +do { \ + if (erts_cpu_timestamp) \ + erts_get_now_cpu(m, s, u); \ + else \ + get_now(m, s, u); \ +} while (0) #else -#define WRITE_SYS_MSG_TO_PORT do_send_to_port +# define GET_NOW(m, s, u) do {get_now(m, s, u);} while (0) #endif + static void -WRITE_SYS_MSG_TO_PORT(Eterm unused_to, +write_sys_msg_to_port(Eterm unused_to, Port* trace_port, Eterm unused_from, enum ErtsSysMsgType unused_type, @@ -672,150 +687,6 @@ WRITE_SYS_MSG_TO_PORT(Eterm unused_to, erts_free(ERTS_ALC_T_TMP, (void *) buffer); } - -#ifndef ERTS_SMP -/* Send {trace_ts, Pid, out, 0, Timestamp} - * followed by {trace_ts, Pid, in, 0, NewTimestamp} - * - * 'NewTimestamp' through patch_ts(). - */ -static void -do_send_schedfix_to_port(Port *trace_port, Eterm pid, Eterm timestamp, int ts_type) { -#define LOCAL_HEAP_SIZE (5+5+ERTS_TRACE_PATCH_TS_MAX_SIZE) - DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); - Eterm message; - Eterm *hp; - Eterm mfarity; - - UseTmpHeapNoproc(LOCAL_HEAP_SIZE); - ASSERT(is_pid(pid)); - ASSERT(is_tuple(timestamp)); - ASSERT(*tuple_val(timestamp) == make_arityval(3)); - - hp = local_heap; - mfarity = make_small(0); - message = TUPLE5(hp, am_trace_ts, pid, am_out, mfarity, timestamp); - /* Note, hp is deliberately NOT incremented since it will be reused */ - - do_send_to_port(trace_port->common.id, - trace_port, - pid, - SYS_MSG_TYPE_UNDEFINED, - message); - - - message = TUPLE5(hp, am_trace_ts, pid, am_in, mfarity, - NIL /* Will be overwritten by timestamp */); - hp += 6; - hp[-1] = write_ts(ts_type, hp, NULL, NULL); - - do_send_to_port(trace_port->common.id, - trace_port, - pid, - SYS_MSG_TYPE_UNDEFINED, - message); - UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); -#undef LOCAL_HEAP_SIZE -} -#endif - -/* If (c_p != NULL), a fake schedule out/in message pair will be sent, - * if the driver so requests. - * It is assumed that 'message' is not an 'out' message. - * - * 'c_p' is the currently executing process, "tracee" is the traced process - * which 'message' concerns => if (*tracee_flags & F_TIMESTAMP_MASK), - * 'message' must contain a timestamp. - */ -static void -send_to_port(Process *c_p, Eterm message, - Eterm *tracer_pid, Uint *tracee_flags) { - Port* trace_port; -#ifndef ERTS_SMP - int ts_type; -#define LOCAL_HEAP_SIZE ERTS_TRACE_PATCH_TS_MAX_SIZE - Eterm ts; - DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); -#endif - - ASSERT(is_internal_port(*tracer_pid)); -#ifdef ERTS_SMP - if (is_not_internal_port(*tracer_pid)) - return; - - trace_port = NULL; -#else - - trace_port = erts_id2port_sflgs(*tracer_pid, - NULL, - 0, - ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP); - - if (!trace_port) { - *tracee_flags &= ~TRACEE_FLAGS; - *tracer_pid = NIL; - return; - } - - /* - * Make a fake schedule only if the current process is traced - * with 'running' and 'timestamp'. - */ - - if ( c_p == NULL || - (! IS_TRACED_FL(c_p, F_TRACE_SCHED | F_TIMESTAMP_MASK))) { -#endif - do_send_to_port(*tracer_pid, - trace_port, - c_p ? c_p->common.id : NIL, - SYS_MSG_TYPE_TRACE, - message); -#ifndef ERTS_SMP - erts_port_release(trace_port); - return; - } - - /* - * Note that the process being traced for some type of trace messages - * (e.g. getting_linked) need not be the current process. That other - * process might not have timestamps enabled. - */ - UseTmpHeapNoproc(LOCAL_HEAP_SIZE); - - /* A fake schedule might be needed. - * Create a dummy trace message with timestamp to be - * passed to do_send_schedfix_to_port(). - */ - ts_type = TFLGS_TS_TYPE(c_p); - ts = write_ts(ts_type, local_heap, NULL, NULL); - - trace_port->control_flags &= ~PORT_CONTROL_FLAG_HEAVY; - do_send_to_port(*tracer_pid, - trace_port, - c_p ? c_p->common.id : NIL, - SYS_MSG_TYPE_TRACE, - message); - - if (trace_port->control_flags & PORT_CONTROL_FLAG_HEAVY) { - /* The driver has just informed us that the last write took a - * non-neglectible amount of time. - * - * We need to fake some trace messages to compensate for the time the - * current process had to sacrifice for the writing of the previous - * trace message. We pretend that the process got scheduled out - * just after writning the real trace message, and now gets scheduled - * in again. - */ - do_send_schedfix_to_port(trace_port, c_p->common.id, ts, ts_type); - } - - erts_port_release(trace_port); - - UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); -#undef LOCAL_HEAP_SIZE -#endif -} - #ifndef ERTS_SMP /* Profile send * Checks if profiler is port or process @@ -844,11 +715,11 @@ profile_send(Eterm from, Eterm message) { 0, ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP); if (profiler_port) { - do_send_to_port(profiler, - profiler_port, - NIL, /* or current process->common.id */ - SYS_MSG_TYPE_SYSPROF, - message); + write_sys_msg_to_port(profiler, + profiler_port, + NIL, /* or current process->common.id */ + SYS_MSG_TYPE_SYSPROF, + message); erts_port_release(profiler_port); } @@ -868,154 +739,26 @@ profile_send(Eterm from, Eterm message) { else msg = copy_struct(message, sz, &hp, &mp->hfrag.off_heap); - erts_queue_message(profile_p, NULL, mp, msg, NIL); + erts_queue_message(profile_p, 0, mp, msg, from); } } #endif - -/* A fake schedule out/in message pair will be sent, - * if the driver so requests. - * - * 'c_p' is the currently executing process, may be NULL. - */ static void -seq_trace_send_to_port(Process *c_p, - Eterm seq_tracer, - Eterm message) +trace_sched_aux(Process *p, ErtsProcLocks locks, Eterm what) { - Port* trace_port; -#ifndef ERTS_SMP - int ts_type; - Eterm ts; -#define LOCAL_HEAP_SIZE ERTS_TRACE_PATCH_TS_MAX_SIZE - DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); - UseTmpHeapNoproc(LOCAL_HEAP_SIZE); -#endif - - ASSERT(is_internal_port(seq_tracer)); -#ifdef ERTS_SMP - if (is_not_internal_port(seq_tracer)) - return; - - trace_port = NULL; -#else - trace_port = erts_id2port_sflgs(seq_tracer, - NULL, - 0, - ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP); - if (!trace_port) { - system_seq_tracer = am_false; -#ifndef ERTS_SMP - UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); -#endif - return; - } - - if (c_p == NULL - || (! IS_TRACED_FL(c_p, F_TRACE_SCHED | F_TIMESTAMP_MASK))) { -#endif - do_send_to_port(seq_tracer, - trace_port, - c_p ? c_p->common.id : NIL, - SYS_MSG_TYPE_SEQTRACE, - message); - -#ifndef ERTS_SMP - erts_port_release(trace_port); - UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); - return; - } - /* Make a fake schedule only if the current process is traced - * with 'running' and 'timestamp'. - */ - - /* A fake schedule might be needed. - * Create a dummy trace message with timestamp to be - * passed to do_send_schedfix_to_port(). - */ - ts_type = TFLGS_TS_TYPE(c_p); - ts = write_ts(ts_type, local_heap, NULL, NULL); - - trace_port->control_flags &= ~PORT_CONTROL_FLAG_HEAVY; - do_send_to_port(seq_tracer, - trace_port, - c_p ? c_p->common.id : NIL, - SYS_MSG_TYPE_SEQTRACE, - message); - - if (trace_port->control_flags & PORT_CONTROL_FLAG_HEAVY) { - /* The driver has just informed us that the last write took a - * non-neglectible amount of time. - * - * We need to fake some trace messages to compensate for the time the - * current process had to sacrifice for the writing of the previous - * trace message. We pretend that the process got scheduled out - * just after writing the real trace message, and now gets scheduled - * in again. - */ - do_send_schedfix_to_port(trace_port, c_p->common.id, ts, ts_type); - } - - erts_port_release(trace_port); - - UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); -#undef LOCAL_HEAP_SIZE -#endif -} - -static ERTS_INLINE void -send_to_tracer(Process *tracee, - ERTS_TRACER_REF_TYPE tracer_ref, - Eterm msg, - Eterm **hpp, - ErlHeapFragment *bp, - int no_fake_sched) -{ - ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(tracee)); - - erts_smp_mtx_lock(&smq_mtx); - - if (is_internal_pid(ERTS_TRACER_PROC(tracee))) { - PATCH_TS(TFLGS_TS_TYPE(tracee), msg, *hpp, bp, tracer_ref); - ERTS_ENQ_TRACE_MSG(tracee->common.id, tracer_ref, msg, bp); - } - else { - ASSERT(is_internal_port(ERTS_TRACER_PROC(tracee))); - PATCH_TS(TFLGS_TS_TYPE(tracee), msg, *hpp, NULL, NULL); - send_to_port(no_fake_sched ? NULL : tracee, - msg, - &ERTS_TRACER_PROC(tracee), - &ERTS_TRACE_FLAGS(tracee)); - } - - erts_smp_mtx_unlock(&smq_mtx); + Eterm tmp, *hp; + int curr_func; + ErtsTracerNif *tnif = NULL; -} - -static void -trace_sched_aux(Process *p, Eterm what, int never_fake_sched) -{ -#define LOCAL_HEAP_SIZE (5+4+1+ERTS_TRACE_PATCH_TS_MAX_SIZE) - DeclareTmpHeap(local_heap,LOCAL_HEAP_SIZE,p); - Eterm tmp, mess, *hp; - ErlHeapFragment *bp = NULL; - ErlOffHeap *off_heap; - ERTS_TRACER_REF_TYPE tracer_ref = ERTS_NULL_TRACER_REF; - int sched_no, curr_func, to_port, no_fake_sched; - - if (is_nil(ERTS_TRACER_PROC(p))) + if (ERTS_TRACER_IS_NIL(ERTS_TRACER(p))) return; - no_fake_sched = never_fake_sched; - switch (what) { case am_out: case am_out_exiting: case am_out_exited: - no_fake_sched = 1; - break; case am_in: case am_in_exiting: break; @@ -1024,16 +767,8 @@ trace_sched_aux(Process *p, Eterm what, int never_fake_sched) break; } - sched_no = IS_TRACED_FL(p, F_TRACE_SCHED_NO); - to_port = is_internal_port(ERTS_TRACER_PROC(p)); - - if (!to_port) { - ASSERT(is_internal_pid(ERTS_TRACER_PROC(p))); - - ERTS_GET_TRACER_REF(tracer_ref, - ERTS_TRACER_PROC(p), - ERTS_TRACE_FLAGS(p)); - } + if (!is_tracer_enabled(p, locks, &p->common, &tnif, TRACE_FUN_E_SCHED_PROC, what)) + return; if (ERTS_PROC_IS_EXITING(p)) curr_func = 0; @@ -1043,44 +778,16 @@ trace_sched_aux(Process *p, Eterm what, int never_fake_sched) curr_func = p->current != NULL; } - UseTmpHeap(LOCAL_HEAP_SIZE,p); - - if (to_port) - hp = local_heap; - else { - Uint size = 5; - if (curr_func) - size += 4; - if (sched_no) - size += 1; - size += PATCH_TS_SIZE(p); - hp = ERTS_ALLOC_SYSMSG_HEAP(size, &bp, &off_heap, tracer_ref); - } - if (!curr_func) { tmp = make_small(0); } else { + hp = HAlloc(p, 4); tmp = TUPLE3(hp,p->current[0],p->current[1],make_small(p->current[2])); hp += 4; } - if (!sched_no) { - mess = TUPLE4(hp, am_trace, p->common.id, what, tmp); - hp += 5; - } - else { -#ifdef ERTS_SMP - Eterm sched_id = make_small(p->scheduler_data->no); -#else - Eterm sched_id = make_small(1); -#endif - mess = TUPLE5(hp, am_trace, p->common.id, what, sched_id, tmp); - hp += 6; - } - - send_to_tracer(p, tracer_ref, mess, &hp, bp, no_fake_sched); - UnUseTmpHeap(LOCAL_HEAP_SIZE,p); -#undef LOCAL_HEAP_SIZE + send_to_tracer_nif(p, &p->common, p->common.id, tnif, TRACE_FUN_T_SCHED_PROC, + what, tmp, THE_NON_VALUE, am_true); } /* Send {trace_ts, Pid, What, {Mod, Func, Arity}, Timestamp} @@ -1090,9 +797,9 @@ trace_sched_aux(Process *p, Eterm what, int never_fake_sched) * 'out_exiting', or 'out_exited'. */ void -trace_sched(Process *p, Eterm what) +trace_sched(Process *p, ErtsProcLocks locks, Eterm what) { - trace_sched_aux(p, what, 0); + trace_sched_aux(p, locks, what); } /* Send {trace_ts, Pid, Send, Msg, DestPid, Timestamp} @@ -1103,140 +810,118 @@ trace_sched(Process *p, Eterm what) void trace_send(Process *p, Eterm to, Eterm msg) { - Eterm operation; - unsigned sz_msg; - unsigned sz_to; - Eterm* hp; - Eterm mess; - - if (!ARE_TRACE_FLAGS_ON(p, F_TRACE_SEND)) { + Eterm operation = am_send; + ErtsTracerNif *tnif = NULL; + ErtsTracingEvent* te; + Eterm pam_result; + + ASSERT(ARE_TRACE_FLAGS_ON(p, F_TRACE_SEND)); + + te = &erts_send_tracing[erts_active_bp_ix()]; + if (!te->on) { return; } + if (te->match_spec) { + Eterm args[2]; + Uint32 return_flags; + args[0] = to; + args[1] = msg; + pam_result = erts_match_set_run_trace(p, p, + te->match_spec, args, 2, + ERTS_PAM_TMP_RESULT, &return_flags); + if (pam_result == am_false) + return; + if (ERTS_TRACE_FLAGS(p) & F_TRACE_SILENT) { + erts_match_set_release_result_trace(p, pam_result); + return; + } + } else + pam_result = am_true; - operation = am_send; if (is_internal_pid(to)) { if (!erts_proc_lookup(to)) goto send_to_non_existing_process; } else if(is_external_pid(to) && external_pid_dist_entry(to) == erts_this_dist_entry) { - char *s; send_to_non_existing_process: - s = "send_to_non_existing_process"; - operation = am_atom_put(s, sys_strlen(s)); + operation = am_send_to_non_existing_process; } - if (is_internal_port(ERTS_TRACER_PROC(p))) { -#define LOCAL_HEAP_SIZE (6 + ERTS_TRACE_PATCH_TS_MAX_SIZE) - DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); - UseTmpHeapNoproc(LOCAL_HEAP_SIZE); - - hp = local_heap; - mess = TUPLE5(hp, am_trace, p->common.id, operation, msg, to); - hp += 6; - erts_smp_mtx_lock(&smq_mtx); - PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, NULL, NULL); - send_to_port(p, mess, &ERTS_TRACER_PROC(p), &ERTS_TRACE_FLAGS(p)); - UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); -#undef LOCAL_HEAP_SIZE - erts_smp_mtx_unlock(&smq_mtx); - } else { - Uint need; - ErlHeapFragment *bp; - ErlOffHeap *off_heap; - ERTS_TRACER_REF_TYPE tracer_ref; - - ASSERT(is_internal_pid(ERTS_TRACER_PROC(p))); - - ERTS_GET_TRACER_REF(tracer_ref, - ERTS_TRACER_PROC(p), - ERTS_TRACE_FLAGS(p)); - - sz_msg = size_object(msg); - sz_to = size_object(to); - need = sz_msg + sz_to + 6 + PATCH_TS_SIZE(p); - - hp = ERTS_ALLOC_SYSMSG_HEAP(need, &bp, &off_heap, tracer_ref); - - to = copy_struct(to, - sz_to, - &hp, - off_heap); - msg = copy_struct(msg, - sz_msg, - &hp, - off_heap); - mess = TUPLE5(hp, am_trace, p->common.id, operation, msg, to); - hp += 6; - - erts_smp_mtx_lock(&smq_mtx); - - PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, bp, tracer_ref); - ERTS_ENQ_TRACE_MSG(p->common.id, tracer_ref, mess, bp); - erts_smp_mtx_unlock(&smq_mtx); + if (is_tracer_enabled(p, ERTS_PROC_LOCK_MAIN, &p->common, &tnif, + TRACE_FUN_E_SEND, operation)) { + send_to_tracer_nif(p, &p->common, p->common.id, tnif, TRACE_FUN_T_SEND, + operation, msg, to, pam_result); } + erts_match_set_release_result_trace(p, pam_result); } /* Send {trace_ts, Pid, receive, Msg, Timestamp} * or {trace, Pid, receive, Msg} */ void -trace_receive(Process *rp, Eterm msg) +trace_receive(Process* receiver, + Eterm from, + Eterm msg, ErtsTracingEvent* te) { - Eterm mess; - size_t sz_msg; - Eterm* hp; - - if (is_internal_port(ERTS_TRACER_PROC(rp))) { -#define LOCAL_HEAP_SIZE (5+ERTS_TRACE_PATCH_TS_MAX_SIZE) - DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); - UseTmpHeapNoproc(LOCAL_HEAP_SIZE); - - hp = local_heap; - mess = TUPLE4(hp, am_trace, rp->common.id, am_receive, msg); - hp += 5; - erts_smp_mtx_lock(&smq_mtx); - PATCH_TS(TFLGS_TS_TYPE(rp), mess, hp, NULL, NULL); - send_to_port(rp, mess, &ERTS_TRACER_PROC(rp), &ERTS_TRACE_FLAGS(rp)); - UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); -#undef LOCAL_HEAP_SIZE - erts_smp_mtx_unlock(&smq_mtx); - } else { - Uint hsz; - ErlHeapFragment *bp; - ErlOffHeap *off_heap; - ERTS_TRACER_REF_TYPE tracer_ref; - - ASSERT(is_internal_pid(ERTS_TRACER_PROC(rp))); - - ERTS_GET_TRACER_REF(tracer_ref, - ERTS_TRACER_PROC(rp), - ERTS_TRACE_FLAGS(rp)); - - sz_msg = size_object(msg); - - hsz = sz_msg + 5 + PATCH_TS_SIZE(rp); - - hp = ERTS_ALLOC_SYSMSG_HEAP(hsz, &bp, &off_heap, tracer_ref); - - msg = copy_struct(msg, sz_msg, &hp, off_heap); - mess = TUPLE4(hp, am_trace, rp->common.id, am_receive, msg); - hp += 5; - - erts_smp_mtx_lock(&smq_mtx); - - PATCH_TS(TFLGS_TS_TYPE(rp), mess, hp, bp, tracer_ref); - ERTS_ENQ_TRACE_MSG(rp->common.id, tracer_ref, mess, bp); - erts_smp_mtx_unlock(&smq_mtx); - } + ErtsTracerNif *tnif = NULL; + Eterm pam_result; + + if (!te) { + te = &erts_receive_tracing[erts_active_bp_ix()]; + if (!te->on) + return; + } + else ASSERT(te->on); + + if (te->match_spec) { + Eterm args[3]; + Uint32 return_flags; + if (is_pid(from)) { + args[0] = pid_node_name(from); + args[1] = from; + } + else { + ASSERT(is_atom(from)); + args[0] = from; /* node name or other atom (e.g 'system') */ + args[1] = am_undefined; + } + args[2] = msg; + pam_result = erts_match_set_run_trace(NULL, receiver, + te->match_spec, args, 3, + ERTS_PAM_TMP_RESULT, &return_flags); + if (pam_result == am_false) + return; + if (ERTS_TRACE_FLAGS(receiver) & F_TRACE_SILENT) { + erts_match_set_release_result_trace(NULL, pam_result); + return; + } + } else + pam_result = am_true; + + if (is_tracer_enabled(NULL, 0, &receiver->common, &tnif, + TRACE_FUN_E_RECEIVE, am_receive)) { + send_to_tracer_nif(NULL, &receiver->common, receiver->common.id, + tnif, TRACE_FUN_T_RECEIVE, + am_receive, msg, THE_NON_VALUE, pam_result); + } + erts_match_set_release_result_trace(NULL, pam_result); } int seq_trace_update_send(Process *p) { - Eterm seq_tracer = erts_get_system_seq_tracer(); + ErtsTracer seq_tracer = erts_get_system_seq_tracer(); ASSERT((is_tuple(SEQ_TRACE_TOKEN(p)) || is_nil(SEQ_TRACE_TOKEN(p)))); - if ((p->common.id == seq_tracer) || have_no_seqtrace(SEQ_TRACE_TOKEN(p))) { + if (have_no_seqtrace(SEQ_TRACE_TOKEN(p)) || + (seq_tracer != NIL && + call_enabled_tracer(NULL, seq_tracer, NULL, + TRACE_FUN_ENABLED, am_trace_status, + p ? p->common.id : am_undefined) != am_trace) +#ifdef USE_VM_PROBES + || (SEQ_TRACE_TOKEN(p) == am_have_dt_utag) +#endif + ) { return 0; } SEQ_TRACE_TOKEN_SENDER(p) = p->common.id; @@ -1264,20 +949,29 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type, Eterm receiver, Process *process, Eterm exitfrom) { Eterm mess; - ErlHeapFragment* bp; Eterm* hp; Eterm label; Eterm lastcnt_serial; Eterm type_atom; - int sz_exit; - Eterm seq_tracer; - int ts_type; + ErtsTracer seq_tracer; + int seq_tracer_flags = 0; +#define LOCAL_HEAP_SIZE (64) + DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); seq_tracer = erts_get_system_seq_tracer(); ASSERT(is_tuple(token) || is_nil(token)); - if (SEQ_TRACE_T_SENDER(token) == seq_tracer || token == NIL || - (process && ERTS_TRACE_FLAGS(process) & F_SENSITIVE)) { + if (token == NIL || (process && ERTS_TRACE_FLAGS(process) & F_SENSITIVE) || + ERTS_TRACER_IS_NIL(seq_tracer) || + call_enabled_tracer(NULL, seq_tracer, + NULL, TRACE_FUN_ENABLED, + am_trace_status, + process ? process->common.id : am_undefined) != am_trace) { + return; + } + + if ((unsigned_val(SEQ_TRACE_T_FLAGS(token)) & type) == 0) { + /* No flags set, nothing to do */ return; } @@ -1290,151 +984,28 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type, return; /* To avoid warning */ } - if ((unsigned_val(SEQ_TRACE_T_FLAGS(token)) & type) == 0) { - /* No flags set, nothing to do */ - return; - } + UseTmpHeapNoproc(LOCAL_HEAP_SIZE); - if (seq_tracer == am_false) { - return; /* no need to send anything */ + hp = local_heap; + label = SEQ_TRACE_T_LABEL(token); + lastcnt_serial = TUPLE2(hp, SEQ_TRACE_T_LASTCNT(token), + SEQ_TRACE_T_SERIAL(token)); + hp += 3; + if (exitfrom != NIL) { + msg = TUPLE3(hp, am_EXIT, exitfrom, msg); + hp += 4; } + mess = TUPLE5(hp, type_atom, lastcnt_serial, SEQ_TRACE_T_SENDER(token), receiver, msg); + hp += 6; - ts_type = ERTS_SEQTFLGS2TSTYPE(unsigned_val(SEQ_TRACE_T_FLAGS(token))); + seq_tracer_flags |= ERTS_SEQTFLGS2TFLGS(unsigned_val(SEQ_TRACE_T_FLAGS(token))); - if (is_internal_port(seq_tracer)) { -#define LOCAL_HEAP_SIZE (60 + ERTS_TRACE_PATCH_TS_MAX_SIZE) - DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); - UseTmpHeapNoproc(LOCAL_HEAP_SIZE); + send_to_tracer_nif_raw(NULL, process, seq_tracer, seq_tracer_flags, + label, NULL, TRACE_FUN_DEFAULT, am_seq_trace, mess, + THE_NON_VALUE, am_true); - hp = local_heap; - label = SEQ_TRACE_T_LABEL(token); - lastcnt_serial = TUPLE2(hp, SEQ_TRACE_T_LASTCNT(token), - SEQ_TRACE_T_SERIAL(token)); - hp += 3; - if (exitfrom != NIL) { - msg = TUPLE3(hp, am_EXIT, exitfrom, msg); - hp += 4; - } - mess = TUPLE5(hp, type_atom, lastcnt_serial, SEQ_TRACE_T_SENDER(token), - receiver, msg); - hp += 6; - - erts_smp_mtx_lock(&smq_mtx); - if (!ts_type) { - mess = TUPLE3(hp, am_seq_trace, label, mess); - seq_trace_send_to_port(NULL, seq_tracer, mess); - } else { - mess = TUPLE4(hp, am_seq_trace, label, mess, - NIL /* Will be overwritten by timestamp */); - hp += 5; - hp[-1] = write_ts(ts_type, hp, NULL, NULL); - seq_trace_send_to_port(process, seq_tracer, mess); - } - UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); #undef LOCAL_HEAP_SIZE - erts_smp_mtx_unlock(&smq_mtx); - } else { -#ifndef ERTS_SMP - Process* tracer; -#endif - Eterm sender_copy; - Eterm receiver_copy; - Eterm m2; - Uint sz_label, sz_lastcnt_serial, sz_msg, sz_ts, sz_sender, - sz_exitfrom, sz_receiver; - - ASSERT(is_internal_pid(seq_tracer)); - -#ifndef ERTS_SMP - - tracer = erts_proc_lookup(seq_tracer); - if (!tracer) { - system_seq_tracer = am_false; - return; /* no need to send anything */ - } -#endif - if (receiver == seq_tracer) { - return; /* no need to send anything */ - } - - sz_label = size_object(SEQ_TRACE_T_LABEL(token)); - sz_sender = size_object(SEQ_TRACE_T_SENDER(token)); - sz_receiver = size_object(receiver); - sz_lastcnt_serial = 3; /* TUPLE2 */ - sz_msg = size_object(msg); - - sz_ts = patch_ts_size(ts_type); - if (exitfrom != NIL) { - sz_exit = 4; /* create {'EXIT',exitfrom,msg} */ - sz_exitfrom = size_object(exitfrom); - } - else { - sz_exit = 0; - sz_exitfrom = 0; - } - bp = new_message_buffer(4 /* TUPLE3 */ + sz_ts + 6 /* TUPLE5 */ - + sz_lastcnt_serial + sz_label + sz_msg - + sz_exit + sz_exitfrom - + sz_sender + sz_receiver); - hp = bp->mem; - label = copy_struct(SEQ_TRACE_T_LABEL(token), sz_label, &hp, &bp->off_heap); - lastcnt_serial = TUPLE2(hp,SEQ_TRACE_T_LASTCNT(token),SEQ_TRACE_T_SERIAL(token)); - hp += 3; - m2 = copy_struct(msg, sz_msg, &hp, &bp->off_heap); - if (sz_exit) { - Eterm exitfrom_copy = copy_struct(exitfrom, - sz_exitfrom, - &hp, - &bp->off_heap); - m2 = TUPLE3(hp, am_EXIT, exitfrom_copy, m2); - hp += 4; - } - sender_copy = copy_struct(SEQ_TRACE_T_SENDER(token), - sz_sender, - &hp, - &bp->off_heap); - receiver_copy = copy_struct(receiver, - sz_receiver, - &hp, - &bp->off_heap); - mess = TUPLE5(hp, - type_atom, - lastcnt_serial, - sender_copy, - receiver_copy, - m2); - hp += 6; - - erts_smp_mtx_lock(&smq_mtx); - - if (!ts_type) - mess = TUPLE3(hp, am_seq_trace, label, mess); - else { - mess = TUPLE4(hp, am_seq_trace, label, mess, - NIL /* Will be overwritten by timestamp */); - hp += 5; - /* Write timestamp in element 6 of the 'msg' tuple */ - hp[-1] = write_ts(ts_type, hp, bp, -#ifndef ERTS_SMP - tracer -#else - NULL -#endif - ); - } - -#ifdef ERTS_SMP - enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SEQTRACE, NIL, NIL, mess, bp); - erts_smp_mtx_unlock(&smq_mtx); -#else - /* trace_token must be NIL here */ - { - ErtsMessage *mp = erts_alloc_message(0, NULL); - mp->data.heap_frag = bp; - erts_queue_message(tracer, NULL, mp, mess, NIL); - } -#endif - } } /* Send {trace_ts, Pid, return_to, {Mod, Func, Arity}, Timestamp} @@ -1443,63 +1014,20 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type, void erts_trace_return_to(Process *p, BeamInstr *pc) { -#define LOCAL_HEAP_SIZE (4+5+ERTS_TRACE_PATCH_TS_MAX_SIZE) - Eterm* hp; Eterm mfa; - Eterm mess; - DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); BeamInstr *code_ptr = find_function_from_pc(pc); - UseTmpHeapNoproc(LOCAL_HEAP_SIZE); - - hp = local_heap; - if (!code_ptr) { mfa = am_undefined; } else { + Eterm *hp = HAlloc(p, 4); mfa = TUPLE3(hp, code_ptr[0], code_ptr[1], make_small(code_ptr[2])); - hp += 4; } - - mess = TUPLE4(hp, am_trace, p->common.id, am_return_to, mfa); - hp += 5; - - erts_smp_mtx_lock(&smq_mtx); - - PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, NULL, NULL); - - if (is_internal_port(ERTS_TRACER_PROC(p))) { - send_to_port(p, mess, &ERTS_TRACER_PROC(p), &ERTS_TRACE_FLAGS(p)); - } else { - ErlHeapFragment *bp; - ErlOffHeap *off_heap; - ERTS_TRACER_REF_TYPE tracer_ref; - unsigned size; - - /* - * Find the tracer. - */ - ASSERT(is_internal_pid(ERTS_TRACER_PROC(p))); - - ERTS_GET_TRACER_REF(tracer_ref, - ERTS_TRACER_PROC(p), - ERTS_TRACE_FLAGS(p)); - size = size_object(mess); - - hp = ERTS_ALLOC_SYSMSG_HEAP(size, &bp, &off_heap, tracer_ref); - - /* - * Copy the trace message into the buffer and enqueue it. - */ - mess = copy_struct(mess, size, &hp, off_heap); - ERTS_ENQ_TRACE_MSG(p->common.id, tracer_ref, mess, bp); - } - UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); -#undef LOCAL_HEAP_SIZE - erts_smp_mtx_unlock(&smq_mtx); + send_to_tracer_nif(p, &p->common, p->common.id, NULL, TRACE_FUN_T_CALL, + am_return_to, mfa, THE_NON_VALUE, am_true); } @@ -1507,114 +1035,53 @@ erts_trace_return_to(Process *p, BeamInstr *pc) * or {trace, Pid, return_from, {Mod, Name, Arity}, Retval} */ void -erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid) +erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, ErtsTracer *tracer) { Eterm* hp; - Eterm mfa; - Eterm mess; - Eterm mod, name; + Eterm mfa, mod, name; int arity; Uint meta_flags, *tracee_flags; - int ts_type; -#ifdef ERTS_SMP - Eterm tracee; -#endif - - ASSERT(tracer_pid); - if (*tracer_pid == am_true) { + + ASSERT(tracer); + if (ERTS_TRACER_COMPARE(*tracer, erts_tracer_true)) { /* Breakpoint trace enabled without specifying tracer => * use process tracer and flags */ - tracer_pid = &ERTS_TRACER_PROC(p); + tracer = &ERTS_TRACER(p); } - if (is_nil(*tracer_pid)) { + if (ERTS_TRACER_IS_NIL(*tracer)) { /* Trace disabled */ return; } - ASSERT(is_internal_pid(*tracer_pid) || is_internal_port(*tracer_pid)); - if (*tracer_pid == p->common.id) { - /* Do not generate trace messages to oneself */ - return; - } - if (tracer_pid == &ERTS_TRACER_PROC(p)) { + ASSERT(IS_TRACER_VALID(*tracer)); + if (tracer == &ERTS_TRACER(p)) { /* Tracer specified in process structure => * non-breakpoint trace => * use process flags */ tracee_flags = &ERTS_TRACE_FLAGS(p); -#ifdef ERTS_SMP - tracee = p->common.id; -#endif + if (! (*tracee_flags & F_TRACE_CALLS)) { + return; + } } else { /* Tracer not specified in process structure => * tracer specified in breakpoint => * meta trace => * use fixed flag set instead of process flags - */ + */ meta_flags = F_TRACE_CALLS | F_NOW_TS; tracee_flags = &meta_flags; -#ifdef ERTS_SMP - tracee = NIL; -#endif - } - if (! (*tracee_flags & F_TRACE_CALLS)) { - return; } - + mod = fi[0]; name = fi[1]; arity = fi[2]; - - ts_type = ERTS_TFLGS2TSTYPE(*tracee_flags); - - if (is_internal_port(*tracer_pid)) { -#define LOCAL_HEAP_SIZE (4+6+ERTS_TRACE_PATCH_TS_MAX_SIZE) - DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); - UseTmpHeapNoproc(LOCAL_HEAP_SIZE); - hp = local_heap; - mfa = TUPLE3(hp, mod, name, make_small(arity)); - hp += 4; - mess = TUPLE5(hp, am_trace, p->common.id, am_return_from, mfa, retval); - hp += 6; - erts_smp_mtx_lock(&smq_mtx); - PATCH_TS(ts_type, mess, hp, NULL, NULL); - send_to_port(p, mess, tracer_pid, tracee_flags); - UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); -#undef LOCAL_HEAP_SIZE - erts_smp_mtx_unlock(&smq_mtx); - } else { - ErlHeapFragment *bp; - ErlOffHeap *off_heap; - ERTS_TRACER_REF_TYPE tracer_ref; - unsigned size; - unsigned retval_size; - - ASSERT(is_internal_pid(*tracer_pid)); - - ERTS_GET_TRACER_REF(tracer_ref, *tracer_pid, *tracee_flags); - retval_size = size_object(retval); - size = 6 + 4 + retval_size + patch_ts_size(ts_type); - - hp = ERTS_ALLOC_SYSMSG_HEAP(size, &bp, &off_heap, tracer_ref); - - /* - * Build the trace tuple and put it into receive queue of the tracer process. - */ - - mfa = TUPLE3(hp, mod, name, make_small(arity)); - hp += 4; - retval = copy_struct(retval, retval_size, &hp, off_heap); - mess = TUPLE5(hp, am_trace, p->common.id, am_return_from, mfa, retval); - hp += 6; - - erts_smp_mtx_lock(&smq_mtx); - - PATCH_TS(ts_type, mess, hp, bp, tracer_ref); - - ERTS_ENQ_TRACE_MSG(tracee, tracer_ref, mess, bp); - erts_smp_mtx_unlock(&smq_mtx); - } + hp = HAlloc(p, 4); + mfa = TUPLE3(hp, mod, name, make_small(arity)); + hp += 4; + send_to_tracer_nif_raw(p, NULL, *tracer, *tracee_flags, p->common.id, + NULL, TRACE_FUN_T_CALL, am_return_from, mfa, retval, am_true); } /* Send {trace_ts, Pid, exception_from, {Mod, Name, Arity}, {Class,Value}, @@ -1626,116 +1093,50 @@ erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid) */ void erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value, - Eterm *tracer_pid) + ErtsTracer *tracer) { Eterm* hp; - Eterm mfa_tuple; - Eterm cv; - Eterm mess; + Eterm mfa_tuple, cv; Uint meta_flags, *tracee_flags; - int ts_type; -#ifdef ERTS_SMP - Eterm tracee; -#endif - - ASSERT(tracer_pid); - if (*tracer_pid == am_true) { + + ASSERT(tracer); + if (ERTS_TRACER_COMPARE(*tracer, erts_tracer_true)) { /* Breakpoint trace enabled without specifying tracer => * use process tracer and flags */ - tracer_pid = &ERTS_TRACER_PROC(p); + tracer = &ERTS_TRACER(p); } - if (is_nil(*tracer_pid)) { + if (ERTS_TRACER_IS_NIL(*tracer)) { /* Trace disabled */ return; } - ASSERT(is_internal_pid(*tracer_pid) || is_internal_port(*tracer_pid)); - if (*tracer_pid == p->common.id) { - /* Do not generate trace messages to oneself */ - return; - } - if (tracer_pid == &ERTS_TRACER_PROC(p)) { + ASSERT(IS_TRACER_VALID(*tracer)); + if (tracer == &ERTS_TRACER(p)) { /* Tracer specified in process structure => * non-breakpoint trace => * use process flags */ tracee_flags = &ERTS_TRACE_FLAGS(p); -#ifdef ERTS_SMP - tracee = p->common.id; -#endif - if (! (*tracee_flags & F_TRACE_CALLS)) { - return; - } + if (! (*tracee_flags & F_TRACE_CALLS)) { + return; + } } else { /* Tracer not specified in process structure => * tracer specified in breakpoint => * meta trace => * use fixed flag set instead of process flags - */ + */ meta_flags = F_TRACE_CALLS | F_NOW_TS; tracee_flags = &meta_flags; -#ifdef ERTS_SMP - tracee = NIL; -#endif } - - ts_type = ERTS_TFLGS2TSTYPE(*tracee_flags); - - if (is_internal_port(*tracer_pid)) { -#define LOCAL_HEAP_SIZE (4+3+6+ERTS_TRACE_PATCH_TS_MAX_SIZE) - DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); - UseTmpHeapNoproc(LOCAL_HEAP_SIZE); - - hp = local_heap; - mfa_tuple = TUPLE3(hp, (Eterm) mfa[0], (Eterm) mfa[1], make_small((Eterm)mfa[2])); - hp += 4; - cv = TUPLE2(hp, class, value); - hp += 3; - mess = TUPLE5(hp, am_trace, p->common.id, am_exception_from, mfa_tuple, cv); - hp += 6; - ASSERT((hp - local_heap) <= LOCAL_HEAP_SIZE); - erts_smp_mtx_lock(&smq_mtx); - PATCH_TS(ts_type, mess, hp, NULL, NULL); - send_to_port(p, mess, tracer_pid, tracee_flags); - UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); -#undef LOCAL_HEAP_SIZE - erts_smp_mtx_unlock(&smq_mtx); - } else { - ErlHeapFragment *bp; - ErlOffHeap *off_heap; - ERTS_TRACER_REF_TYPE tracer_ref; - unsigned size; - unsigned value_size; - - ASSERT(is_internal_pid(*tracer_pid)); - - ERTS_GET_TRACER_REF(tracer_ref, *tracer_pid, *tracee_flags); - - value_size = size_object(value); - size = 6 + 4 + 3 + value_size + patch_ts_size(ts_type); - - hp = ERTS_ALLOC_SYSMSG_HEAP(size, &bp, &off_heap, tracer_ref); - - /* - * Build the trace tuple and put it into receive queue of the tracer process. - */ - - mfa_tuple = TUPLE3(hp, (Eterm) mfa[0], (Eterm) mfa[1], make_small((Eterm) mfa[2])); - hp += 4; - value = copy_struct(value, value_size, &hp, off_heap); - cv = TUPLE2(hp, class, value); - hp += 3; - mess = TUPLE5(hp, am_trace, p->common.id, - am_exception_from, mfa_tuple, cv); - hp += 6; - - erts_smp_mtx_lock(&smq_mtx); - PATCH_TS(ts_type, mess, hp, bp, tracer_ref); - - ERTS_ENQ_TRACE_MSG(tracee, tracer_ref, mess, bp); - erts_smp_mtx_unlock(&smq_mtx); - } + hp = HAlloc(p, 7);; + mfa_tuple = TUPLE3(hp, (Eterm) mfa[0], (Eterm) mfa[1], make_small((Eterm)mfa[2])); + hp += 4; + cv = TUPLE2(hp, class, value); + hp += 3; + send_to_tracer_nif_raw(p, NULL, *tracer, *tracee_flags, p->common.id, + NULL, TRACE_FUN_T_CALL, am_exception_from, mfa_tuple, cv, am_true); } /* @@ -1754,7 +1155,7 @@ erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value, */ Uint32 erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, - Eterm* args, int local, Eterm *tracer_pid) + Eterm* args, int local, ErtsTracer *tracer) { Eterm* hp; Eterm mfa_tuple; @@ -1762,55 +1163,53 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, int i; Uint32 return_flags; Eterm pam_result = am_true; - Eterm mess; Uint meta_flags, *tracee_flags; - int ts_type; -#ifdef ERTS_SMP - Eterm tracee; -#endif + ErtsTracerNif *tnif = NULL; Eterm transformed_args[MAX_ARG]; - DeclareTypedTmpHeap(ErlSubBin,sub_bin_heap,p); + ErtsTracer pre_ms_tracer = erts_tracer_nil; - ASSERT(tracer_pid); - if (*tracer_pid == am_true) { - /* Breakpoint trace enabled without specifying tracer => + ASSERT(tracer); + if (ERTS_TRACER_COMPARE(*tracer, erts_tracer_true)) { + /* Breakpoint trace enabled without specifying tracer => * use process tracer and flags */ - tracer_pid = &ERTS_TRACER_PROC(p); - } - if (is_nil(*tracer_pid)) { - /* Trace disabled */ - return 0; + tracer = &ERTS_TRACER(p); } - ASSERT(is_internal_pid(*tracer_pid) || is_internal_port(*tracer_pid)); - if (*tracer_pid == p->common.id) { - /* Do not generate trace messages to oneself */ + if (ERTS_TRACER_IS_NIL(*tracer)) { + /* Trace disabled */ return 0; } - if (tracer_pid == &ERTS_TRACER_PROC(p)) { + ASSERT(IS_TRACER_VALID(*tracer)); + if (tracer == &ERTS_TRACER(p)) { /* Tracer specified in process structure => * non-breakpoint trace => * use process flags */ tracee_flags = &ERTS_TRACE_FLAGS(p); -#ifdef ERTS_SMP - tracee = p->common.id; -#endif + if (!is_tracer_enabled(p, ERTS_PROC_LOCK_MAIN, &p->common, &tnif, + TRACE_FUN_E_CALL, am_call)) { + return 0; + } } else { /* Tracer not specified in process structure => * tracer specified in breakpoint => * meta trace => * use fixed flag set instead of process flags - */ - if (ERTS_TRACE_FLAGS(p) & F_SENSITIVE) { - /* No trace messages for sensitive processes. */ - return 0; - } + */ + if (ERTS_TRACE_FLAGS(p) & F_SENSITIVE) { + /* No trace messages for sensitive processes. */ + return 0; + } meta_flags = F_TRACE_CALLS | F_NOW_TS; tracee_flags = &meta_flags; -#ifdef ERTS_SMP - tracee = NIL; -#endif + switch (call_enabled_tracer(p, *tracer, + &tnif, TRACE_FUN_T_CALL, + am_call, p->common.id)) { + default: + case am_remove: *tracer = erts_tracer_nil; + case am_discard: return 0; + case am_trace: break; + } } /* @@ -1821,18 +1220,13 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, * temporarily convert any match contexts to sub binaries. */ arity = (Eterm) mfa[2]; - UseTmpHeap(ERL_SUB_BIN_SIZE,p); -#ifdef DEBUG - sub_bin_heap->thing_word = 0; -#endif for (i = 0; i < arity; i++) { Eterm arg = args[i]; if (is_boxed(arg) && header_is_bin_matchstate(*boxed_val(arg))) { ErlBinMatchState* ms = (ErlBinMatchState *) boxed_val(arg); ErlBinMatchBuffer* mb = &ms->mb; Uint bit_size; - - ASSERT(sub_bin_heap->thing_word == 0); /* At most one of match context */ + ErlSubBin *sub_bin_heap = (ErlSubBin *)HAlloc(p, ERL_SUB_BIN_SIZE); bit_size = mb->size - mb->offset; sub_bin_heap->thing_word = HEADER_SUB_BIN; @@ -1849,275 +1243,94 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, } args = transformed_args; - ts_type = ERTS_TFLGS2TSTYPE(*tracee_flags); - - if (is_internal_port(*tracer_pid)) { - Eterm local_heap[64+ERTS_TRACE_PATCH_TS_MAX_SIZE+MAX_ARG]; - hp = local_heap; - - if (!erts_is_valid_tracer_port(*tracer_pid)) { -#ifdef ERTS_SMP - ASSERT(is_nil(tracee) || tracer_pid == &ERTS_TRACER_PROC(p)); - if (is_not_nil(tracee)) - erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR); -#endif - *tracee_flags &= ~TRACEE_FLAGS; - *tracer_pid = NIL; -#ifdef ERTS_SMP - if (is_not_nil(tracee)) - erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR); -#endif - UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); - return 0; - } - - /* - * If there is a PAM program, run it. Return if it fails. - * - * Some precedence rules: - * - * - No proc flags, e.g 'silent' or 'return_to' - * has any effect on meta trace. - * - The 'silent' process trace flag silences all call - * related messages, e.g 'call', 'return_to' and 'return_from'. - * - The {message,_} PAM function does not affect {return_trace}. - * - The {message,false} PAM function shall give the same - * 'call' trace message as no PAM match. - * - The {message,true} PAM function shall give the same - * 'call' trace message as a nonexistent PAM program. - */ - - /* BEGIN this code should be the same for port and pid trace */ - return_flags = 0; - if (match_spec) { - pam_result = erts_match_set_run(p, match_spec, args, arity, - ERTS_PAM_TMP_RESULT, &return_flags); - if (is_non_value(pam_result)) { - erts_match_set_release_result(p); - UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); - return 0; - } - } - if (tracee_flags == &meta_flags) { - /* Meta trace */ - if (pam_result == am_false) { - erts_match_set_release_result(p); - UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); - return return_flags; - } - } else { - /* Non-meta trace */ - if (*tracee_flags & F_TRACE_SILENT) { - erts_match_set_release_result(p); - UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); - return 0; - } - if (pam_result == am_false) { - erts_match_set_release_result(p); - UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); - return return_flags; - } - if (local && (*tracee_flags & F_TRACE_RETURN_TO)) { - return_flags |= MATCH_SET_RETURN_TO_TRACE; - } - } - /* END this code should be the same for port and pid trace */ - - /* - * Build the the {M,F,A} tuple in the local heap. - * (A is arguments or arity.) - */ - - if (*tracee_flags & F_TRACE_ARITY_ONLY) { - mfa_tuple = make_small(arity); - } else { - mfa_tuple = NIL; - for (i = arity-1; i >= 0; i--) { - mfa_tuple = CONS(hp, args[i], mfa_tuple); - hp += 2; - } - } - mfa_tuple = TUPLE3(hp, (Eterm) mfa[0], (Eterm) mfa[1], mfa_tuple); - hp += 4; - - /* - * Build the trace tuple and send it to the port. - */ - - mess = TUPLE4(hp, am_trace, p->common.id, am_call, mfa_tuple); - hp += 5; - if (pam_result != am_true) { - hp[-5] = make_arityval(5); - *hp++ = pam_result; - } - erts_smp_mtx_lock(&smq_mtx); - PATCH_TS(ts_type, mess, hp, NULL, NULL); - send_to_port(p, mess, tracer_pid, tracee_flags); - erts_smp_mtx_unlock(&smq_mtx); - erts_match_set_release_result(p); - UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); - return *tracer_pid == NIL ? 0 : return_flags; + /* + * If there is a PAM program, run it. Return if it fails. + * + * Some precedence rules: + * + * - No proc flags, e.g 'silent' or 'return_to' + * has any effect on meta trace. + * - The 'silent' process trace flag silences all call + * related messages, e.g 'call', 'return_to' and 'return_from'. + * - The {message,_} PAM function does not affect {return_trace}. + * - The {message,false} PAM function shall give the same + * 'call' trace message as no PAM match. + * - The {message,true} PAM function shall give the same + * 'call' trace message as a nonexistent PAM program. + */ + return_flags = 0; + if (match_spec) { + /* we have to make a copy of the tracer here as the match spec + may remove it, and we still want to generate a trace message */ + erts_tracer_update(&pre_ms_tracer, *tracer); + tracer = &pre_ms_tracer; + pam_result = erts_match_set_run_trace(p, p, + match_spec, args, arity, + ERTS_PAM_TMP_RESULT, &return_flags); + } + + if (tracee_flags == &meta_flags) { + /* Meta trace */ + if (pam_result == am_false) { + UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); + ERTS_TRACER_CLEAR(&pre_ms_tracer); + return return_flags; + } } else { - ErlHeapFragment *bp; - ErlOffHeap *off_heap; - Process *tracer; - ERTS_TRACER_REF_TYPE tracer_ref; -#ifdef ERTS_SMP - Eterm tpid; -#endif - unsigned size; - unsigned sizes[MAX_ARG]; - unsigned pam_result_size = 0; - int invalid_tracer; - - ASSERT(is_internal_pid(*tracer_pid)); - - tracer = erts_pid2proc(p, ERTS_PROC_LOCK_MAIN, - *tracer_pid, ERTS_PROC_LOCK_STATUS); - if (!tracer) - invalid_tracer = 1; - else { - invalid_tracer = !(ERTS_TRACE_FLAGS(tracer) & F_TRACER); - erts_smp_proc_unlock(tracer, ERTS_PROC_LOCK_STATUS); - } - - if (invalid_tracer) { -#ifdef ERTS_SMP - ASSERT(is_nil(tracee) - || tracer_pid == &ERTS_TRACER_PROC(p)); - if (is_not_nil(tracee)) - erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR); -#endif - *tracee_flags &= ~TRACEE_FLAGS; - *tracer_pid = NIL; -#ifdef ERTS_SMP - if (is_not_nil(tracee)) - erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR); -#endif - UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); - return 0; - } + /* Non-meta trace */ + if (*tracee_flags & F_TRACE_SILENT) { + erts_match_set_release_result_trace(p, pam_result); + UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); + ERTS_TRACER_CLEAR(&pre_ms_tracer); + return 0; + } + if (pam_result == am_false) { + UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); + ERTS_TRACER_CLEAR(&pre_ms_tracer); + return return_flags; + } + if (local && (*tracee_flags & F_TRACE_RETURN_TO)) { + return_flags |= MATCH_SET_RETURN_TO_TRACE; + } + } + + ASSERT(!ERTS_TRACER_IS_NIL(*tracer)); -#ifdef ERTS_SMP - tpid = *tracer_pid; /* Need to save tracer pid, - since *tracer_pid might - be reset by erts_match_set_run() */ - tracer_ref = tpid; -#else - tracer_ref = tracer; -#endif - - /* - * If there is a PAM program, run it. Return if it fails. - * - * See the rules above in the port trace code. - */ - - /* BEGIN this code should be the same for port and pid trace */ - return_flags = 0; - if (match_spec) { - pam_result = erts_match_set_run(p, match_spec, args, arity, - ERTS_PAM_TMP_RESULT, &return_flags); - if (is_non_value(pam_result)) { - erts_match_set_release_result(p); - UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); - return 0; - } - } - if (tracee_flags == &meta_flags) { - /* Meta trace */ - if (pam_result == am_false) { - erts_match_set_release_result(p); - UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); - return return_flags; - } - } else { - /* Non-meta trace */ - if (*tracee_flags & F_TRACE_SILENT) { - erts_match_set_release_result(p); - UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); - return 0; - } - if (pam_result == am_false) { - erts_match_set_release_result(p); - UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); - return return_flags; - } - if (local && (*tracee_flags & F_TRACE_RETURN_TO)) { - return_flags |= MATCH_SET_RETURN_TO_TRACE; - } - } - /* END this code should be the same for port and pid trace */ - - /* - * Calculate number of words needed on heap. - */ - - size = 4 + 5; /* Trace tuple + MFA tuple. */ - if (! (*tracee_flags & F_TRACE_ARITY_ONLY)) { - size += 2*arity; - for (i = arity-1; i >= 0; i--) { - sizes[i] = size_object(args[i]); - size += sizes[i]; - } - } - size += patch_ts_size(ts_type); - if (pam_result != am_true) { - pam_result_size = size_object(pam_result); - size += 1 + pam_result_size; - /* One element in trace tuple + term size. */ - } - - hp = ERTS_ALLOC_SYSMSG_HEAP(size, &bp, &off_heap, tracer_ref); - - /* - * Build the the {M,F,A} tuple in the message buffer. - * (A is arguments or arity.) - */ - - if (*tracee_flags & F_TRACE_ARITY_ONLY) { - mfa_tuple = make_small(arity); - } else { - mfa_tuple = NIL; - for (i = arity-1; i >= 0; i--) { - Eterm term = copy_struct(args[i], sizes[i], &hp, off_heap); - mfa_tuple = CONS(hp, term, mfa_tuple); - hp += 2; - } - } - mfa_tuple = TUPLE3(hp, (Eterm) mfa[0], (Eterm) mfa[1], mfa_tuple); - hp += 4; - - /* - * Copy the PAM result (if any) onto the heap. - */ - - if (pam_result != am_true) { - pam_result = copy_struct(pam_result, pam_result_size, &hp, off_heap); - } + /* + * Build the the {M,F,A} tuple in the local heap. + * (A is arguments or arity.) + */ - erts_match_set_release_result(p); - /* - * Build the trace tuple and enqueue it. - */ - - mess = TUPLE4(hp, am_trace, p->common.id, am_call, mfa_tuple); - hp += 5; - if (pam_result != am_true) { - hp[-5] = make_arityval(5); - *hp++ = pam_result; - } + if (*tracee_flags & F_TRACE_ARITY_ONLY) { + hp = HAlloc(p, 4); + mfa_tuple = make_small(arity); + } else { + hp = HAlloc(p, 4 + arity * 2); + mfa_tuple = NIL; + for (i = arity-1; i >= 0; i--) { + mfa_tuple = CONS(hp, args[i], mfa_tuple); + hp += 2; + } + } + mfa_tuple = TUPLE3(hp, (Eterm) mfa[0], (Eterm) mfa[1], mfa_tuple); + hp += 4; - erts_smp_mtx_lock(&smq_mtx); + /* + * Build the trace tuple and send it to the port. + */ + send_to_tracer_nif_raw(p, NULL, *tracer, *tracee_flags, p->common.id, + tnif, TRACE_FUN_T_CALL, am_call, mfa_tuple, + THE_NON_VALUE, pam_result); - PATCH_TS(ts_type, mess, hp, bp, tracer_ref); - ERTS_ENQ_TRACE_MSG(tracee, tracer_ref, mess, bp); - erts_smp_mtx_unlock(&smq_mtx); - UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); - return return_flags; + if (match_spec) { + erts_match_set_release_result_trace(p, pam_result); + if (tracer == &pre_ms_tracer) + ERTS_TRACER_CLEAR(&pre_ms_tracer); } + + return return_flags; } /* Sends trace message: @@ -2129,69 +1342,14 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, * 't_p' is the traced process. */ void -trace_proc(Process *c_p, Process *t_p, Eterm what, Eterm data) +trace_proc(Process *c_p, ErtsProcLocks c_p_locks, + Process *t_p, Eterm what, Eterm data) { - Eterm mess; - Eterm* hp; - int need; - - ERTS_SMP_LC_ASSERT((erts_proc_lc_my_proc_locks(t_p) != 0) - || erts_thr_progress_is_blocking()); - if (is_internal_port(ERTS_TRACER_PROC(t_p))) { -#define LOCAL_HEAP_SIZE (5+5) - DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); - UseTmpHeapNoproc(LOCAL_HEAP_SIZE); - - - hp = local_heap; - mess = TUPLE4(hp, am_trace, t_p->common.id, what, data); - hp += 5; - erts_smp_mtx_lock(&smq_mtx); - PATCH_TS(TFLGS_TS_TYPE(t_p), mess, hp, NULL, NULL); - send_to_port( -#ifndef ERTS_SMP - /* No fake schedule out and in again after an exit */ - what == am_exit ? NULL : c_p, -#else - /* Fake schedule out and in are never sent when smp enabled */ - c_p, -#endif - mess, - &ERTS_TRACER_PROC(t_p), - &ERTS_TRACE_FLAGS(t_p)); - UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); -#undef LOCAL_HEAP_SIZE - erts_smp_mtx_unlock(&smq_mtx); - } else { - Eterm tmp; - ErlHeapFragment *bp; - ErlOffHeap *off_heap; - ERTS_TRACER_REF_TYPE tracer_ref; - size_t sz_data; - - ASSERT(is_internal_pid(ERTS_TRACER_PROC(t_p))); - - ERTS_GET_TRACER_REF(tracer_ref, - ERTS_TRACER_PROC(t_p), - ERTS_TRACE_FLAGS(t_p)); - - sz_data = size_object(data); - - need = sz_data + 5 + PATCH_TS_SIZE(t_p); - - hp = ERTS_ALLOC_SYSMSG_HEAP(need, &bp, &off_heap, tracer_ref); - - tmp = copy_struct(data, sz_data, &hp, off_heap); - mess = TUPLE4(hp, am_trace, t_p->common.id, what, tmp); - hp += 5; - - erts_smp_mtx_lock(&smq_mtx); - - PATCH_TS(TFLGS_TS_TYPE(t_p), mess, hp, bp, tracer_ref); - - ERTS_ENQ_TRACE_MSG(t_p->common.id, tracer_ref, mess, bp); - erts_smp_mtx_unlock(&smq_mtx); - } + ErtsTracerNif *tnif = NULL; + if (is_tracer_enabled(c_p, c_p_locks, &t_p->common, &tnif, + TRACE_FUN_E_PROCS, what)) + send_to_tracer_nif(c_p, &t_p->common, t_p->common.id, tnif, TRACE_FUN_T_PROCS, + what, data, THE_NON_VALUE, am_true); } @@ -2203,77 +1361,38 @@ trace_proc(Process *c_p, Process *t_p, Eterm what, Eterm data) * and 'args' may be a deep term. */ void -trace_proc_spawn(Process *p, Eterm pid, +trace_proc_spawn(Process *p, Eterm what, Eterm pid, Eterm mod, Eterm func, Eterm args) { - Eterm mfa; - Eterm mess; - Eterm* hp; - - if (is_internal_port(ERTS_TRACER_PROC(p))) { -#define LOCAL_HEAP_SIZE (4+6+5) - DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); - UseTmpHeapNoproc(LOCAL_HEAP_SIZE); - - hp = local_heap; - mfa = TUPLE3(hp, mod, func, args); - hp += 4; - mess = TUPLE5(hp, am_trace, p->common.id, am_spawn, pid, mfa); - hp += 6; - erts_smp_mtx_lock(&smq_mtx); - PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, NULL, NULL); - send_to_port(p, mess, &ERTS_TRACER_PROC(p), &ERTS_TRACE_FLAGS(p)); - UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); -#undef LOCAL_HEAP_SIZE - erts_smp_mtx_unlock(&smq_mtx); - } else { - Eterm tmp; - ErlHeapFragment *bp; - ErlOffHeap *off_heap; - ERTS_TRACER_REF_TYPE tracer_ref; - size_t sz_args, sz_pid; - Uint need; - - ASSERT(is_internal_pid(ERTS_TRACER_PROC(p))); - - ERTS_GET_TRACER_REF(tracer_ref, - ERTS_TRACER_PROC(p), - ERTS_TRACE_FLAGS(p)); - - sz_args = size_object(args); - sz_pid = size_object(pid); - need = sz_args + 4 + 6 + PATCH_TS_SIZE(p); - - hp = ERTS_ALLOC_SYSMSG_HEAP(need, &bp, &off_heap, tracer_ref); - - tmp = copy_struct(args, sz_args, &hp, off_heap); - mfa = TUPLE3(hp, mod, func, tmp); - hp += 4; - tmp = copy_struct(pid, sz_pid, &hp, off_heap); - mess = TUPLE5(hp, am_trace, p->common.id, am_spawn, tmp, mfa); - hp += 6; + ErtsTracerNif *tnif = NULL; + if (is_tracer_enabled(p, ERTS_PROC_LOCKS_ALL & + ~(ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE), + &p->common, &tnif, TRACE_FUN_E_PROCS, what)) { + Eterm mfa; + Eterm* hp; - erts_smp_mtx_lock(&smq_mtx); - - PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, bp, tracer_ref); - - ERTS_ENQ_TRACE_MSG(p->common.id, tracer_ref, mess, bp); - erts_smp_mtx_unlock(&smq_mtx); + hp = HAlloc(p, 4); + mfa = TUPLE3(hp, mod, func, args); + hp += 4; + send_to_tracer_nif(p, &p->common, p->common.id, tnif, TRACE_FUN_T_PROCS, + what, pid, mfa, am_true); } } void save_calls(Process *p, Export *e) { - struct saved_calls *scb = ERTS_PROC_GET_SAVED_CALLS_BUF(p); - if (scb) { - Export **ct = &scb->ct[0]; - int len = scb->len; - - ct[scb->cur] = e; - if (++scb->cur >= len) - scb->cur = 0; - if (scb->n < len) - scb->n++; + if ((ERTS_TRACE_FLAGS(p) & F_SENSITIVE) == 0) { + struct saved_calls *scb = ERTS_PROC_GET_SAVED_CALLS_BUF(p); + if (scb) { + Export **ct = &scb->ct[0]; + int len = scb->len; + + ct[scb->cur] = e; + if (++scb->cur >= len) + scb->cur = 0; + if (scb->n < len) + scb->n++; + } } } @@ -2290,68 +1409,26 @@ void save_calls(Process *p, Export *e) * are all small (atomic) integers. */ void -trace_gc(Process *p, Eterm what) +trace_gc(Process *p, Eterm what, Uint size) { - ErlHeapFragment *bp = NULL; - ErlOffHeap *off_heap; - ERTS_TRACER_REF_TYPE tracer_ref = ERTS_NULL_TRACER_REF; /* Initialized - to eliminate - compiler - warning */ + ErtsTracerNif *tnif = NULL; Eterm* hp; Eterm msg = NIL; - Uint size; - -#define LOCAL_HEAP_SIZE \ - (ERTS_PROCESS_GC_INFO_MAX_SIZE) + \ - 5/*4-tuple */ + ERTS_TRACE_PATCH_TS_MAX_SIZE - DeclareTmpHeap(local_heap,LOCAL_HEAP_SIZE,p); - - UseTmpHeap(LOCAL_HEAP_SIZE,p); - - if (is_internal_port(ERTS_TRACER_PROC(p))) { - hp = local_heap; -#ifdef DEBUG - size = 0; - (void) erts_process_gc_info(p, &size, NULL); - - size += 5/*4-tuple*/ + PATCH_TS_SIZE(p); -#endif - } else { - ASSERT(is_internal_pid(ERTS_TRACER_PROC(p))); - - ERTS_GET_TRACER_REF(tracer_ref, - ERTS_TRACER_PROC(p), - ERTS_TRACE_FLAGS(p)); - - size = 0; - (void) erts_process_gc_info(p, &size, NULL); - - size += 5/*4-tuple*/ + PATCH_TS_SIZE(p); - - hp = ERTS_ALLOC_SYSMSG_HEAP(size, &bp, &off_heap, tracer_ref); - } - - ASSERT(size <= LOCAL_HEAP_SIZE); + Uint sz = 0; + Eterm tup; - msg = erts_process_gc_info(p, NULL, &hp); + if (is_tracer_enabled(p, ERTS_PROC_LOCK_MAIN, &p->common, &tnif, TRACE_FUN_E_GC, what)) { - msg = TUPLE4(hp, am_trace, p->common.id, what, msg); - hp += 5; + (void) erts_process_gc_info(p, &sz, NULL); + hp = HAlloc(p, sz + 3 + 2); - erts_smp_mtx_lock(&smq_mtx); + msg = erts_process_gc_info(p, NULL, &hp); + tup = TUPLE2(hp, am_wordsize, make_small(size)); hp += 3; + msg = CONS(hp, tup, msg); hp += 2; - if (is_internal_port(ERTS_TRACER_PROC(p))) { - PATCH_TS(TFLGS_TS_TYPE(p), msg, hp, NULL, NULL); - send_to_port(p, msg, &ERTS_TRACER_PROC(p), &ERTS_TRACE_FLAGS(p)); + send_to_tracer_nif(p, &p->common, p->common.id, tnif, TRACE_FUN_T_GC, + what, msg, am_undefined, am_true); } - else { - PATCH_TS(TFLGS_TS_TYPE(p), msg, hp, bp, tracer_ref); - ERTS_ENQ_TRACE_MSG(p->common.id, tracer_ref, msg, bp); - } - erts_smp_mtx_unlock(&smq_mtx); - UnUseTmpHeap(LOCAL_HEAP_SIZE,p); -#undef LOCAL_HEAP_SIZE } void @@ -2413,7 +1490,7 @@ monitor_long_schedule_proc(Process *p, BeamInstr *in_fp, BeamInstr *out_fp, Uint { ErtsMessage *mp = erts_alloc_message(0, NULL); mp->data.heap_frag = bp; - erts_queue_message(monitor_p, NULL, mp, msg, NIL); + erts_queue_message(monitor_p, 0, mp, msg, am_system); } #endif } @@ -2478,7 +1555,7 @@ monitor_long_schedule_port(Port *pp, ErtsPortTaskType type, Uint time) { ErtsMessage *mp = erts_alloc_message(0, NULL); mp->data.heap_frag = bp; - erts_queue_message(monitor_p, NULL, mp, msg, NIL); + erts_queue_message(monitor_p, 0, mp, msg, am_system); } #endif } @@ -2553,7 +1630,7 @@ monitor_long_gc(Process *p, Uint time) { { ErtsMessage *mp = erts_alloc_message(0, NULL); mp->data.heap_frag = bp; - erts_queue_message(monitor_p, NULL, mp, msg, NIL); + erts_queue_message(monitor_p, 0, mp, msg, am_system); } #endif } @@ -2628,7 +1705,7 @@ monitor_large_heap(Process *p) { { ErtsMessage *mp = erts_alloc_message(0, NULL); mp->data.heap_frag = bp; - erts_queue_message(monitor_p, NULL, mp, msg, NIL); + erts_queue_message(monitor_p, 0, mp, msg, am_system); } #endif } @@ -2660,7 +1737,7 @@ monitor_generic(Process *p, Eterm type, Eterm spec) { { ErtsMessage *mp = erts_alloc_message(0, NULL); mp->data.heap_frag = bp; - erts_queue_message(monitor_p, NULL, mp, msg, NIL); + erts_queue_message(monitor_p, 0, mp, msg, am_system); } #endif @@ -2722,106 +1799,15 @@ profile_scheduler(Eterm scheduler_id, Eterm state) { } -void -profile_scheduler_q(Eterm scheduler_id, Eterm state, Eterm no_schedulers, Uint Ms, Uint s, Uint us) { - Eterm *hp, msg, timestamp; - -#ifndef ERTS_SMP -#define LOCAL_HEAP_SIZE (4 + 7) - DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); - UseTmpHeapNoproc(LOCAL_HEAP_SIZE); - - hp = local_heap; -#else - ErlHeapFragment *bp; - Uint hsz; - - hsz = 4 + 7; - - bp = new_message_buffer(hsz); - hp = bp->mem; -#endif - - erts_smp_mtx_lock(&smq_mtx); - - timestamp = TUPLE3(hp, make_small(Ms), make_small(s), make_small(us)); hp += 4; - msg = TUPLE6(hp, am_profile, am_scheduler, scheduler_id, state, no_schedulers, timestamp); hp += 7; -#ifndef ERTS_SMP - profile_send(NIL, msg); - UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); -#undef LOCAL_HEAP_SIZE -#else - enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, NIL, NIL, msg, bp); -#endif - erts_smp_mtx_unlock(&smq_mtx); - -} - - -/* Send {trace_ts, Pid, What, {Mod, Func, Arity}, Timestamp} - * or {trace, Pid, What, {Mod, Func, Arity}} - * - * where 'What' is supposed to be 'in' or 'out'. - * - * Virtual scheduling do not fake scheduling for ports. - */ - - -void trace_virtual_sched(Process *p, Eterm what) -{ - trace_sched_aux(p, what, 1); -} - /* Port profiling */ void trace_port_open(Port *p, Eterm calling_pid, Eterm drv_name) { - Eterm mess; - Eterm* hp; - - if (is_internal_port(ERTS_TRACER_PROC(p))) { -#define LOCAL_HEAP_SIZE (6+ERTS_TRACE_PATCH_TS_MAX_SIZE) - DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); - UseTmpHeapNoproc(LOCAL_HEAP_SIZE); - - hp = local_heap; - - mess = TUPLE5(hp, am_trace, calling_pid, am_open, p->common.id, drv_name); - hp += 6; - erts_smp_mtx_lock(&smq_mtx); - PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, NULL, NULL); - /* No fake schedule */ - send_to_port(NULL, mess, &ERTS_TRACER_PROC(p), &ERTS_TRACE_FLAGS(p)); - UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); -#undef LOCAL_HEAP_SIZE - erts_smp_mtx_unlock(&smq_mtx); - } else { - ErlHeapFragment *bp; - ErlOffHeap *off_heap; - size_t sz_data; - ERTS_TRACER_REF_TYPE tracer_ref; - - ASSERT(is_internal_pid(ERTS_TRACER_PROC(p))); - - sz_data = 6 + PATCH_TS_SIZE(p); - - ERTS_GET_TRACER_REF(tracer_ref, - ERTS_TRACER_PROC(p), - ERTS_TRACE_FLAGS(p)); - - hp = ERTS_ALLOC_SYSMSG_HEAP(sz_data, &bp, &off_heap, tracer_ref); - - mess = TUPLE5(hp, am_trace, calling_pid, am_open, p->common.id, drv_name); - hp += 6; - - erts_smp_mtx_lock(&smq_mtx); - - PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, bp, tracer_ref); - - ERTS_ENQ_TRACE_MSG(p->common.id, tracer_ref, mess, bp); - erts_smp_mtx_unlock(&smq_mtx); - } - + ErtsTracerNif *tnif = NULL; + ERTS_SMP_CHK_NO_PROC_LOCKS; + if (is_tracer_enabled(NULL, 0, &p->common, &tnif, TRACE_FUN_E_PORTS, am_open)) + send_to_tracer_nif(NULL, &p->common, p->common.id, tnif, TRACE_FUN_T_PORTS, + am_open, calling_pid, drv_name, am_true); } /* Sends trace message: @@ -2833,52 +1819,211 @@ trace_port_open(Port *p, Eterm calling_pid, Eterm drv_name) { */ void trace_port(Port *t_p, Eterm what, Eterm data) { - Eterm mess; - Eterm* hp; + ErtsTracerNif *tnif = NULL; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(t_p) || erts_thr_progress_is_blocking()); + ERTS_SMP_CHK_NO_PROC_LOCKS; + if (is_tracer_enabled(NULL, 0, &t_p->common, &tnif, TRACE_FUN_E_PORTS, what)) + send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif, TRACE_FUN_T_PORTS, + what, data, THE_NON_VALUE, am_true); +} - if (is_internal_port(ERTS_TRACER_PROC(t_p))) { -#define LOCAL_HEAP_SIZE (5+ERTS_TRACE_PATCH_TS_MAX_SIZE) - DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); - UseTmpHeapNoproc(LOCAL_HEAP_SIZE); - hp = local_heap; - mess = TUPLE4(hp, am_trace, t_p->common.id, what, data); - hp += 5; - erts_smp_mtx_lock(&smq_mtx); - PATCH_TS(TFLGS_TS_TYPE(t_p), mess, hp, NULL, NULL); - /* No fake schedule */ - send_to_port(NULL,mess,&ERTS_TRACER_PROC(t_p),&ERTS_TRACE_FLAGS(t_p)); - UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); -#undef LOCAL_HEAP_SIZE - erts_smp_mtx_unlock(&smq_mtx); +static Eterm +trace_port_tmp_binary(char *bin, Sint sz, Binary **bptrp, Eterm **hp) +{ + if (sz <= ERL_ONHEAP_BIN_LIMIT) { + ErlHeapBin *hb = (ErlHeapBin *)*hp; + hb->thing_word = header_heap_bin(sz); + hb->size = sz; + sys_memcpy(hb->data, bin, sz); + *hp += heap_bin_size(sz); + return make_binary(hb); } else { - ErlHeapFragment *bp; - ErlOffHeap *off_heap; - size_t sz_data; - ERTS_TRACER_REF_TYPE tracer_ref; + ProcBin* pb = (ProcBin *)*hp; + Binary *bptr = erts_bin_nrml_alloc(sz); + erts_refc_init(&bptr->refc, 1); + sys_memcpy(bptr->orig_bytes, bin, sz); + pb->thing_word = HEADER_PROC_BIN; + pb->size = sz; + pb->next = NULL; + pb->val = bptr; + pb->bytes = (byte*) bptr->orig_bytes; + pb->flags = 0; + *bptrp = bptr; + *hp += PROC_BIN_SIZE; + return make_binary(pb); + } +} + +/* Sends trace message: + * {trace, PortPid, 'receive', {pid(), {command, iolist()}}} + * {trace, PortPid, 'receive', {pid(), {control, pid()}}} + * {trace, PortPid, 'receive', {pid(), exit}} + * + */ +void +trace_port_receive(Port *t_p, Eterm caller, Eterm what, ...) +{ + ErtsTracerNif *tnif = NULL; + ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(t_p) + || erts_thr_progress_is_blocking()); + ERTS_SMP_CHK_NO_PROC_LOCKS; + if (is_tracer_enabled(NULL, 0, &t_p->common, &tnif, TRACE_FUN_E_RECEIVE, am_receive)) { + /* We can use a stack heap here, as the nif is called in the + context of a port */ +#define LOCAL_HEAP_SIZE (3 + 3 + heap_bin_size(ERL_ONHEAP_BIN_LIMIT) + 3) + DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); + + Eterm *hp, data, *orig_hp = NULL; + Binary *bptr = NULL; + va_list args; + UseTmpHeapNoproc(LOCAL_HEAP_SIZE); + hp = local_heap; + + if (what == am_close) { + data = what; + } else { + Eterm arg; + va_start(args, what); + if (what == am_command) { + char *bin = va_arg(args, char *); + Sint sz = va_arg(args, Sint); + va_end(args); + arg = trace_port_tmp_binary(bin, sz, &bptr, &hp); + } else if (what == am_call || what == am_control) { + unsigned int command = va_arg(args, unsigned int); + char *bin = va_arg(args, char *); + Sint sz = va_arg(args, Sint); + Eterm cmd; + va_end(args); + arg = trace_port_tmp_binary(bin, sz, &bptr, &hp); +#if defined(ARCH_32) + if (!IS_USMALL(0, command)) { + *hp = make_pos_bignum_header(1); + BIG_DIGIT(hp, 0) = (Uint)command; + cmd = make_big(hp); + hp += 2; + } else +#endif + { + cmd = make_small((Sint)command); + } + arg = TUPLE2(hp, cmd, arg); + hp += 3; + } else if (what == am_commandv) { + ErlIOVec *evp = va_arg(args, ErlIOVec*); + int i; + va_end(args); + if ((6 + evp->vsize * (2+PROC_BIN_SIZE+ERL_SUB_BIN_SIZE)) > LOCAL_HEAP_SIZE) { + hp = erts_alloc(ERTS_ALC_T_TMP, + (6 + evp->vsize * (2+PROC_BIN_SIZE+ERL_SUB_BIN_SIZE)) * sizeof(Eterm)); + orig_hp = hp; + } + arg = NIL; + /* Convert each element in the ErlIOVec to a sub bin that points + to a procbin. We don't have to increment the proc bin refc as + the port task keeps the reference alive. */ + for (i = evp->vsize-1; i >= 0; i--) { + if (evp->iov[i].iov_len) { + ProcBin* pb = (ProcBin*)hp; + ErlSubBin *sb; + ASSERT(evp->binv[i]); + pb->thing_word = HEADER_PROC_BIN; + pb->val = ErlDrvBinary2Binary(evp->binv[i]); + pb->size = pb->val->orig_size; + pb->next = NULL; + pb->bytes = (byte*) pb->val->orig_bytes; + pb->flags = 0; + hp += PROC_BIN_SIZE; + + sb = (ErlSubBin*) hp; + sb->thing_word = HEADER_SUB_BIN; + sb->size = evp->iov[i].iov_len; + sb->offs = (byte*)(evp->iov[i].iov_base) - pb->bytes; + sb->orig = make_binary(pb); + sb->bitoffs = 0; + sb->bitsize = 0; + sb->is_writable = 0; + hp += ERL_SUB_BIN_SIZE; + + arg = CONS(hp, make_binary(sb), arg); + hp += 2; + } + } + what = am_command; + } else { + arg = va_arg(args, Eterm); + va_end(args); + } + data = TUPLE2(hp, what, arg); + hp += 3; + } + + data = TUPLE2(hp, caller, data); + hp += 3; + ASSERT(hp <= (local_heap + LOCAL_HEAP_SIZE) || orig_hp); + send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif, + TRACE_FUN_T_RECEIVE, + am_receive, data, THE_NON_VALUE, am_true); + + if (bptr && erts_refc_dectest(&bptr->refc, 1) == 0) + erts_bin_free(bptr); + + if (orig_hp) + erts_free(ERTS_ALC_T_TMP, orig_hp); + + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); + } +#undef LOCAL_HEAP_SIZE +} - ASSERT(is_internal_pid(ERTS_TRACER_PROC(t_p))); +void +trace_port_send(Port *t_p, Eterm receiver, Eterm msg, int exists) +{ + ErtsTracerNif *tnif = NULL; + Eterm op = exists ? am_send : am_send_to_non_existing_process; + ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(t_p) + || erts_thr_progress_is_blocking()); + ERTS_SMP_CHK_NO_PROC_LOCKS; + if (is_tracer_enabled(NULL, 0, &t_p->common, &tnif, TRACE_FUN_E_SEND, op)) + send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif, TRACE_FUN_T_SEND, + op, msg, receiver, am_true); +} - sz_data = 5 + PATCH_TS_SIZE(t_p); +void trace_port_send_binary(Port *t_p, Eterm to, Eterm what, char *bin, Sint sz) +{ + ErtsTracerNif *tnif = NULL; + ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(t_p) + || erts_thr_progress_is_blocking()); + ERTS_SMP_CHK_NO_PROC_LOCKS; + if (is_tracer_enabled(NULL, 0, &t_p->common, &tnif, TRACE_FUN_E_SEND, am_send)) { + Eterm msg; + Binary* bptr = NULL; +#define LOCAL_HEAP_SIZE (3 + 3 + heap_bin_size(ERL_ONHEAP_BIN_LIMIT)) + DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); - ERTS_GET_TRACER_REF(tracer_ref, - ERTS_TRACER_PROC(t_p), - ERTS_TRACE_FLAGS(t_p)); + Eterm *hp; - hp = ERTS_ALLOC_SYSMSG_HEAP(sz_data, &bp, &off_heap, tracer_ref); + ERTS_CT_ASSERT(heap_bin_size(ERL_ONHEAP_BIN_LIMIT) >= PROC_BIN_SIZE); + UseTmpHeapNoproc(LOCAL_HEAP_SIZE); + hp = local_heap; - mess = TUPLE4(hp, am_trace, t_p->common.id, what, data); - hp += 5; + msg = trace_port_tmp_binary(bin, sz, &bptr, &hp); - erts_smp_mtx_lock(&smq_mtx); + msg = TUPLE2(hp, what, msg); + hp += 3; + msg = TUPLE2(hp, t_p->common.id, msg); + hp += 3; - PATCH_TS(TFLGS_TS_TYPE(t_p), mess, hp, bp, tracer_ref); + send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif, TRACE_FUN_T_SEND, + am_send, msg, to, am_true); + if (bptr && erts_refc_dectest(&bptr->refc, 1) == 0) + erts_bin_free(bptr); - ERTS_ENQ_TRACE_MSG(t_p->common.id, tracer_ref, mess, bp); - erts_smp_mtx_unlock(&smq_mtx); + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); +#undef LOCAL_HEAP_SIZE } } @@ -2892,83 +2037,19 @@ trace_port(Port *t_p, Eterm what, Eterm data) { void trace_sched_ports(Port *p, Eterm what) { - trace_sched_ports_where(p,what, make_small(0)); + trace_sched_ports_where(p, what, make_small(0)); } -void -trace_sched_ports_where(Port *p, Eterm what, Eterm where) { - Eterm mess; - Eterm* hp; - int ws = 5; - Eterm sched_id = am_undefined; - - if (is_internal_port(ERTS_TRACER_PROC(p))) { -#define LOCAL_HEAP_SIZE (6+ERTS_TRACE_PATCH_TS_MAX_SIZE) - DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); - UseTmpHeapNoproc(LOCAL_HEAP_SIZE); - - hp = local_heap; - - if (IS_TRACED_FL(p, F_TRACE_SCHED_NO)) { -#ifdef ERTS_SMP - ErtsSchedulerData *esd = erts_get_scheduler_data(); - if (esd) sched_id = make_small(esd->no); - else sched_id = am_undefined; -#else - sched_id = make_small(1); -#endif - mess = TUPLE5(hp, am_trace, p->common.id, what, sched_id, where); - ws = 6; - } else { - mess = TUPLE4(hp, am_trace, p->common.id, what, where); - ws = 5; - } - hp += ws; - - erts_smp_mtx_lock(&smq_mtx); - - PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, NULL, NULL); - - /* No fake scheduling */ - send_to_port(NULL, mess, &ERTS_TRACER_PROC(p), &ERTS_TRACE_FLAGS(p)); - UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); -#undef LOCAL_HEAP_SIZE - erts_smp_mtx_unlock(&smq_mtx); - } else { - ErlHeapFragment *bp; - ErlOffHeap *off_heap; - ERTS_TRACER_REF_TYPE tracer_ref; - - ASSERT(is_internal_pid(ERTS_TRACER_PROC(p))); - - if (IS_TRACED_FL(p, F_TRACE_SCHED_NO)) ws = 6; /* Make place for scheduler id */ - - ERTS_GET_TRACER_REF(tracer_ref, - ERTS_TRACER_PROC(p), - ERTS_TRACE_FLAGS(p)); - - hp = ERTS_ALLOC_SYSMSG_HEAP(ws+PATCH_TS_SIZE(p), &bp, &off_heap, tracer_ref); - - if (IS_TRACED_FL(p, F_TRACE_SCHED_NO)) { -#ifdef ERTS_SMP - ErtsSchedulerData *esd = erts_get_scheduler_data(); - if (esd) sched_id = make_small(esd->no); - else sched_id = am_undefined; -#else - sched_id = make_small(1); -#endif - mess = TUPLE5(hp, am_trace, p->common.id, what, sched_id, where); - } else { - mess = TUPLE4(hp, am_trace, p->common.id, what, where); - } - hp += ws; - - erts_smp_mtx_lock(&smq_mtx); - - PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, bp, tracer_ref); - ERTS_ENQ_TRACE_MSG(p->common.id, tracer_ref, mess, bp); - erts_smp_mtx_unlock(&smq_mtx); - } +void +trace_sched_ports_where(Port *t_p, Eterm what, Eterm where) { + ErtsTracerNif *tnif = NULL; + ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(t_p) + || erts_thr_progress_is_blocking()); + ERTS_SMP_CHK_NO_PROC_LOCKS; + if (is_tracer_enabled(NULL, 0, &t_p->common, &tnif, TRACE_FUN_E_SCHED_PORT, what)) + send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, + tnif, TRACE_FUN_T_SCHED_PORT, + what, where, THE_NON_VALUE, am_true); } /* Port profiling */ @@ -3021,6 +2102,7 @@ profile_runnable_proc(Process *p, Eterm status){ Eterm *hp, msg; Eterm where = am_undefined; ErlHeapFragment *bp = NULL; + int use_current = 1; #ifndef ERTS_SMP #define LOCAL_HEAP_SIZE (4 + 6 + ERTS_TRACE_PATCH_TS_MAX_SIZE) @@ -3033,12 +2115,19 @@ profile_runnable_proc(Process *p, Eterm status){ Uint hsz = 4 + 6 + patch_ts_size(erts_system_profile_ts_type)-1; #endif - if (!p->current) { - p->current = find_function_from_pc(p->i); + if (ERTS_PROC_IS_EXITING(p)) { + use_current = 0; + /* could probably set 'where' to 'exiting' here, + * though it's not documented as such */ + } else { + if (!p->current) { + p->current = find_function_from_pc(p->i); + } + use_current = p->current != NULL; } #ifdef ERTS_SMP - if (!p->current) { + if (!use_current) { hsz -= 4; } @@ -3046,7 +2135,7 @@ profile_runnable_proc(Process *p, Eterm status){ hp = bp->mem; #endif - if (p->current) { + if (use_current) { where = TUPLE3(hp, p->current[0], p->current[1], make_small(p->current[2])); hp += 4; } else { where = make_small(0); @@ -3076,28 +2165,6 @@ profile_runnable_proc(Process *p, Eterm status){ #ifdef ERTS_SMP -void -erts_check_my_tracer_proc(Process *p) -{ - if (is_internal_pid(ERTS_TRACER_PROC(p))) { - Process *tracer = erts_pid2proc(p, - ERTS_PROC_LOCK_MAIN, - ERTS_TRACER_PROC(p), - ERTS_PROC_LOCK_STATUS); - int invalid_tracer = (!tracer - || !(ERTS_TRACE_FLAGS(tracer) & F_TRACER)); - if (tracer) - erts_smp_proc_unlock(tracer, ERTS_PROC_LOCK_STATUS); - if (invalid_tracer) { - erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR); - ERTS_TRACE_FLAGS(p) &= ~TRACEE_FLAGS; - ERTS_TRACER_PROC(p) = NIL; - erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR); - } - } -} - - typedef struct ErtsSysMsgQ_ ErtsSysMsgQ; struct ErtsSysMsgQ_ { ErtsSysMsgQ *next; @@ -3175,12 +2242,6 @@ static void print_msg_type(ErtsSysMsgQ *smqp) { switch (smqp->type) { - case SYS_MSG_TYPE_TRACE: - erts_fprintf(stderr, "TRACE "); - break; - case SYS_MSG_TYPE_SEQTRACE: - erts_fprintf(stderr, "SEQTRACE "); - break; case SYS_MSG_TYPE_SYSMON: erts_fprintf(stderr, "SYSMON "); break; @@ -3191,8 +2252,8 @@ print_msg_type(ErtsSysMsgQ *smqp) erts_fprintf(stderr, "ERRLGR "); break; case SYS_MSG_TYPE_PROC_MSG: - erts_fprintf(stderr, "PROC_MSG "); - break; + erts_fprintf(stderr, "PROC_MSG "); + break; default: erts_fprintf(stderr, "??? "); break; @@ -3204,17 +2265,6 @@ static void sys_msg_disp_failure(ErtsSysMsgQ *smqp, Eterm receiver) { switch (smqp->type) { - case SYS_MSG_TYPE_TRACE: - /* Invalid tracer_proc's are removed when processes - are scheduled in. */ - break; - case SYS_MSG_TYPE_SEQTRACE: - /* Reset seq_tracer if it hasn't changed */ - erts_smp_rwmtx_rwlock(&sys_trace_rwmtx); - if (system_seq_tracer == receiver) - system_seq_tracer = am_false; - erts_smp_rwmtx_rwunlock(&sys_trace_rwmtx); - break; case SYS_MSG_TYPE_SYSMON: if (receiver == NIL && !erts_system_monitor_long_gc @@ -3275,7 +2325,7 @@ sys_msg_disp_failure(ErtsSysMsgQ *smqp, Eterm receiver) break; } case SYS_MSG_TYPE_PROC_MSG: - break; + break; default: ASSERT(0); } @@ -3393,13 +2443,9 @@ sys_msg_dispatcher_func(void *unused) print_msg_type(smqp); #endif switch (smqp->type) { - case SYS_MSG_TYPE_TRACE: - case SYS_MSG_TYPE_PROC_MSG: - receiver = smqp->to; - break; - case SYS_MSG_TYPE_SEQTRACE: - receiver = erts_get_system_seq_tracer(); - break; + case SYS_MSG_TYPE_PROC_MSG: + receiver = smqp->to; + break; case SYS_MSG_TYPE_SYSMON: receiver = erts_get_system_monitor(); if (smqp->from == receiver) { @@ -3434,16 +2480,8 @@ sys_msg_dispatcher_func(void *unused) if (is_internal_pid(receiver)) { proc = erts_pid2proc(NULL, 0, receiver, proc_locks); - if (!proc - || (smqp->type == SYS_MSG_TYPE_TRACE - && !(ERTS_TRACE_FLAGS(proc) & F_TRACER))) { + if (!proc) { /* Bad tracer */ -#ifdef DEBUG_PRINTOUTS - if (smqp->type == SYS_MSG_TYPE_TRACE && proc) - erts_fprintf(stderr, - "<tracer alive but missing " - "F_TRACER flag> "); -#endif goto failure; } else { @@ -3451,7 +2489,7 @@ sys_msg_dispatcher_func(void *unused) queue_proc_msg: mp = erts_alloc_message(0, NULL); mp->data.heap_frag = smqp->bp; - erts_queue_message(proc,&proc_locks,mp,smqp->msg,NIL); + erts_queue_message(proc,proc_locks,mp,smqp->msg,am_system); #ifdef DEBUG_PRINTOUTS erts_fprintf(stderr, "delivered\n"); #endif @@ -3517,12 +2555,6 @@ erts_foreach_sys_msg_in_q(void (*func)(Eterm, for (sm = sys_message_queue; sm; sm = sm->next) { Eterm to; switch (sm->type) { - case SYS_MSG_TYPE_TRACE: - to = sm->to; - break; - case SYS_MSG_TYPE_SEQTRACE: - to = erts_get_system_seq_tracer(); - break; case SYS_MSG_TYPE_SYSMON: to = erts_get_system_monitor(); break; @@ -3560,3 +2592,589 @@ init_sys_msg_dispatcher(void) } #endif + +#include "erl_nif.h" + +typedef struct { + char *name; + Uint arity; + ErlNifFunc *cb; +} ErtsTracerType; + +struct ErtsTracerNif_ { + HashBucket hb; + Eterm module; + struct erl_module_nif* nif_mod; + ErtsTracerType tracers[NIF_TRACER_TYPES]; +}; + +static void init_tracer_template(ErtsTracerNif *tnif) { + + /* default tracer functions */ + tnif->tracers[TRACE_FUN_DEFAULT].name = "trace"; + tnif->tracers[TRACE_FUN_DEFAULT].arity = 6; + tnif->tracers[TRACE_FUN_DEFAULT].cb = NULL; + + tnif->tracers[TRACE_FUN_ENABLED].name = "enabled"; + tnif->tracers[TRACE_FUN_ENABLED].arity = 3; + tnif->tracers[TRACE_FUN_ENABLED].cb = NULL; + + /* specific tracer functions */ + tnif->tracers[TRACE_FUN_T_SEND].name = "trace_send"; + tnif->tracers[TRACE_FUN_T_SEND].arity = 6; + tnif->tracers[TRACE_FUN_T_SEND].cb = NULL; + + tnif->tracers[TRACE_FUN_T_RECEIVE].name = "trace_receive"; + tnif->tracers[TRACE_FUN_T_RECEIVE].arity = 6; + tnif->tracers[TRACE_FUN_T_RECEIVE].cb = NULL; + + tnif->tracers[TRACE_FUN_T_CALL].name = "trace_call"; + tnif->tracers[TRACE_FUN_T_CALL].arity = 6; + tnif->tracers[TRACE_FUN_T_CALL].cb = NULL; + + tnif->tracers[TRACE_FUN_T_SCHED_PROC].name = "trace_running_procs"; + tnif->tracers[TRACE_FUN_T_SCHED_PROC].arity = 6; + tnif->tracers[TRACE_FUN_T_SCHED_PROC].cb = NULL; + + tnif->tracers[TRACE_FUN_T_SCHED_PORT].name = "trace_running_ports"; + tnif->tracers[TRACE_FUN_T_SCHED_PORT].arity = 6; + tnif->tracers[TRACE_FUN_T_SCHED_PORT].cb = NULL; + + tnif->tracers[TRACE_FUN_T_GC].name = "trace_garbage_collection"; + tnif->tracers[TRACE_FUN_T_GC].arity = 6; + tnif->tracers[TRACE_FUN_T_GC].cb = NULL; + + tnif->tracers[TRACE_FUN_T_PROCS].name = "trace_procs"; + tnif->tracers[TRACE_FUN_T_PROCS].arity = 6; + tnif->tracers[TRACE_FUN_T_PROCS].cb = NULL; + + tnif->tracers[TRACE_FUN_T_PORTS].name = "trace_ports"; + tnif->tracers[TRACE_FUN_T_PORTS].arity = 6; + tnif->tracers[TRACE_FUN_T_PORTS].cb = NULL; + + /* specific enabled functions */ + tnif->tracers[TRACE_FUN_E_SEND].name = "enabled_send"; + tnif->tracers[TRACE_FUN_E_SEND].arity = 3; + tnif->tracers[TRACE_FUN_E_SEND].cb = NULL; + + tnif->tracers[TRACE_FUN_E_RECEIVE].name = "enabled_receive"; + tnif->tracers[TRACE_FUN_E_RECEIVE].arity = 3; + tnif->tracers[TRACE_FUN_E_RECEIVE].cb = NULL; + + tnif->tracers[TRACE_FUN_E_CALL].name = "enabled_call"; + tnif->tracers[TRACE_FUN_E_CALL].arity = 3; + tnif->tracers[TRACE_FUN_E_CALL].cb = NULL; + + tnif->tracers[TRACE_FUN_E_SCHED_PROC].name = "enabled_running_procs"; + tnif->tracers[TRACE_FUN_E_SCHED_PROC].arity = 3; + tnif->tracers[TRACE_FUN_E_SCHED_PROC].cb = NULL; + + tnif->tracers[TRACE_FUN_E_SCHED_PORT].name = "enabled_running_ports"; + tnif->tracers[TRACE_FUN_E_SCHED_PORT].arity = 3; + tnif->tracers[TRACE_FUN_E_SCHED_PORT].cb = NULL; + + tnif->tracers[TRACE_FUN_E_GC].name = "enabled_garbage_collection"; + tnif->tracers[TRACE_FUN_E_GC].arity = 3; + tnif->tracers[TRACE_FUN_E_GC].cb = NULL; + + tnif->tracers[TRACE_FUN_E_PROCS].name = "enabled_procs"; + tnif->tracers[TRACE_FUN_E_PROCS].arity = 3; + tnif->tracers[TRACE_FUN_E_PROCS].cb = NULL; + + tnif->tracers[TRACE_FUN_E_PORTS].name = "enabled_ports"; + tnif->tracers[TRACE_FUN_E_PORTS].arity = 3; + tnif->tracers[TRACE_FUN_E_PORTS].cb = NULL; +} + +static Hash *tracer_hash = NULL; +static erts_smp_rwmtx_t tracer_mtx; + +static ErtsTracerNif * +load_tracer_nif(const ErtsTracer tracer) +{ + Module* mod = erts_get_module(ERTS_TRACER_MODULE(tracer), + erts_active_code_ix()); + struct erl_module_instance *instance; + ErlNifFunc *funcs; + int num_of_funcs; + ErtsTracerNif tnif_tmpl, *tnif; + ErtsTracerType *tracers; + int i,j; + + if (!mod || !mod->curr.nif) { + return NULL; + } + + instance = &mod->curr; + + init_tracer_template(&tnif_tmpl); + tnif_tmpl.nif_mod = instance->nif; + tnif_tmpl.module = ERTS_TRACER_MODULE(tracer); + tracers = tnif_tmpl.tracers; + + num_of_funcs = erts_nif_get_funcs(instance->nif, &funcs); + + for(i = 0; i < num_of_funcs; i++) { + for (j = 0; j < NIF_TRACER_TYPES; j++) { + if (strcmp(tracers[j].name, funcs[i].name) == 0 && tracers[j].arity == funcs[i].arity) { + tracers[j].cb = &(funcs[i]); + break; + } + } + } + + if (tracers[TRACE_FUN_DEFAULT].cb == NULL || tracers[TRACE_FUN_ENABLED].cb == NULL ) { + return NULL; + } + + erts_smp_rwmtx_rwlock(&tracer_mtx); + tnif = hash_put(tracer_hash, &tnif_tmpl); + erts_smp_rwmtx_rwunlock(&tracer_mtx); + + return tnif; +} + +static ERTS_INLINE ErtsTracerNif * +lookup_tracer_nif(const ErtsTracer tracer) +{ + ErtsTracerNif tnif_tmpl; + ErtsTracerNif *tnif; + tnif_tmpl.module = ERTS_TRACER_MODULE(tracer); + erts_smp_rwmtx_rlock(&tracer_mtx); + if ((tnif = hash_get(tracer_hash, &tnif_tmpl)) == NULL) { + erts_smp_rwmtx_runlock(&tracer_mtx); + tnif = load_tracer_nif(tracer); + ASSERT(!tnif || tnif->nif_mod); + return tnif; + } + erts_smp_rwmtx_runlock(&tracer_mtx); + ASSERT(tnif->nif_mod); + return tnif; +} + +/* This function converts an Erlang tracer term to ErtsTracer. + It returns THE_NON_VALUE if an invalid tracer term was given. + Accepted input is: + pid() || port() || {prefix, pid()} || {prefix, port()} || + {prefix, atom(), term()} || {atom(), term()} + */ +ErtsTracer +erts_term_to_tracer(Eterm prefix, Eterm t) +{ + ErtsTracer tracer = erts_tracer_nil; + ASSERT(is_atom(prefix) || prefix == THE_NON_VALUE); + if (!is_nil(t)) { + Eterm module = am_erl_tracer, state = THE_NON_VALUE; + Eterm hp[2]; + if (is_tuple(t)) { + Eterm *tp = tuple_val(t); + if (prefix != THE_NON_VALUE) { + if (arityval(tp[0]) == 2 && tp[1] == prefix) + t = tp[2]; + else if (arityval(tp[0]) == 3 && tp[1] == prefix && is_atom(tp[2])) { + module = tp[2]; + state = tp[3]; + } + } else { + if (arityval(tp[0]) == 2 && is_atom(tp[2])) { + module = tp[1]; + state = tp[2]; + } + } + } + if (state == THE_NON_VALUE && (is_internal_pid(t) || is_internal_port(t))) + state = t; + if (state == THE_NON_VALUE) + return THE_NON_VALUE; + erts_tracer_update(&tracer, CONS(hp, module, state)); + } + if (!lookup_tracer_nif(tracer)) { + ASSERT(ERTS_TRACER_MODULE(tracer) != am_erl_tracer); + ERTS_TRACER_CLEAR(&tracer); + return THE_NON_VALUE; + } + return tracer; +} + +Eterm +erts_tracer_to_term(Process *p, ErtsTracer tracer) +{ + if (ERTS_TRACER_IS_NIL(tracer)) + return am_false; + if (ERTS_TRACER_MODULE(tracer) == am_erl_tracer) + /* Have to manage these specifically in order to be + backwards compatible */ + return ERTS_TRACER_STATE(tracer); + else { + Eterm *hp = HAlloc(p, 3); + return TUPLE2(hp, ERTS_TRACER_MODULE(tracer), + copy_object(ERTS_TRACER_STATE(tracer), p)); + } +} + + +static ERTS_INLINE int +send_to_tracer_nif_raw(Process *c_p, Process *tracee, + const ErtsTracer tracer, Uint tracee_flags, + Eterm t_p_id, ErtsTracerNif *tnif, + enum ErtsTracerOpt topt, + Eterm tag, Eterm msg, Eterm extra, Eterm pam_result) +{ + if (tnif || (tnif = lookup_tracer_nif(tracer)) != NULL) { +#define MAP_SIZE 3 + Eterm argv[6], local_heap[3+MAP_SIZE /* values */ + (MAP_SIZE+1 /* keys */)]; + flatmap_t *map = (flatmap_t*)(local_heap+(MAP_SIZE+1)); + Eterm *map_values = flatmap_get_values(map); + + topt = (tnif->tracers[topt].cb) ? topt : TRACE_FUN_DEFAULT; + ASSERT(topt < NIF_TRACER_TYPES); + + argv[0] = tag; + argv[1] = ERTS_TRACER_STATE(tracer); + argv[2] = t_p_id; + argv[3] = msg; + argv[4] = extra == THE_NON_VALUE ? am_undefined : extra; + argv[5] = make_flatmap(map); + + map->thing_word = MAP_HEADER_FLATMAP; + map->size = MAP_SIZE; + map->keys = TUPLE3(local_heap, am_match_spec_result, am_scheduler_id, am_timestamp); + + *map_values++ = pam_result; + if (tracee_flags & F_TRACE_SCHED_NO) + *map_values++ = make_small(erts_get_scheduler_id()); + else + *map_values++ = am_undefined; + if (tracee_flags & F_NOW_TS) +#ifdef HAVE_ERTS_NOW_CPU + if (erts_cpu_timestamp) + *map_values++ = am_cpu_timestamp; + else +#endif + *map_values++ = am_timestamp; + else if (tracee_flags & F_STRICT_MON_TS) + *map_values++ = am_strict_monotonic; + else if (tracee_flags & F_MON_TS) + *map_values++ = am_monotonic; + else + *map_values++ = am_undefined; + +#undef MAP_SIZE + erts_nif_call_function(c_p, tracee ? tracee : c_p, + tnif->nif_mod, + tnif->tracers[topt].cb, + tnif->tracers[topt].arity, + argv); + } + return 1; +} + +static ERTS_INLINE int +send_to_tracer_nif(Process *c_p, ErtsPTabElementCommon *t_p, + Eterm t_p_id, ErtsTracerNif *tnif, enum ErtsTracerOpt topt, + Eterm tag, Eterm msg, Eterm extra, Eterm pam_result) +{ +#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) + if (c_p) { + /* We have to hold the main lock of the currently executing process */ + erts_proc_lc_chk_have_proc_locks(c_p, ERTS_PROC_LOCK_MAIN); + } + if (is_internal_pid(t_p->id)) { + /* We have to have at least one lock */ + ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks((Process*)t_p) & ERTS_PROC_LOCKS_ALL); + } else { + ASSERT(is_internal_port(t_p->id)); + ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked((Port*)t_p)); + } +#endif + + return send_to_tracer_nif_raw(c_p, + is_internal_pid(t_p->id) ? (Process*)t_p : NULL, + t_p->tracer, t_p->trace_flags, + t_p_id, tnif, topt, tag, msg, extra, + pam_result); +} + +static ERTS_INLINE Eterm +call_enabled_tracer(Process *c_p, const ErtsTracer tracer, + ErtsTracerNif **tnif_ret, + enum ErtsTracerOpt topt, + Eterm tag, Eterm t_p_id) { + ErtsTracerNif *tnif = lookup_tracer_nif(tracer); + if (tnif) { + Eterm argv[] = {tag, ERTS_TRACER_STATE(tracer), t_p_id}; + topt = (tnif->tracers[topt].cb) ? topt : TRACE_FUN_ENABLED; + ASSERT(topt < NIF_TRACER_TYPES); + ASSERT(tnif->tracers[topt].cb != NULL); + if (tnif_ret) *tnif_ret = tnif; + return erts_nif_call_function(c_p, NULL, tnif->nif_mod, + tnif->tracers[topt].cb, + tnif->tracers[topt].arity, + argv); + } + return am_remove; +} + +static int +is_tracer_enabled(Process* c_p, ErtsProcLocks c_p_locks, + ErtsPTabElementCommon *t_p, + ErtsTracerNif **tnif_ret, + enum ErtsTracerOpt topt, Eterm tag) { + Eterm nif_result; + +#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) + if (c_p) + ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == c_p_locks + || erts_thr_progress_is_blocking()); + if (is_internal_pid(t_p->id)) { + /* We have to have at least one lock */ + ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks((Process*)t_p) & ERTS_PROC_LOCKS_ALL + || erts_thr_progress_is_blocking()); + } else { + ASSERT(is_internal_port(t_p->id)); + ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked((Port*)t_p) + || erts_thr_progress_is_blocking()); + } +#endif + + nif_result = call_enabled_tracer(c_p, t_p->tracer, tnif_ret, topt, tag, t_p->id); + switch (nif_result) { + case am_discard: return 0; + case am_trace: return 1; + case THE_NON_VALUE: + case am_remove: break; + default: + /* only am_remove should be returned, but if + something else is returned we fall-through + and remove the tracer. */ + ASSERT(0); + } + + /* Only remove tracer on self() and ports */ + if (is_internal_port(t_p->id) || (c_p && c_p->common.id == t_p->id)) { + ErtsProcLocks c_p_xlocks = 0; + if (is_internal_pid(t_p->id)) { + ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) & ERTS_PROC_LOCK_MAIN); + if (c_p_locks != ERTS_PROC_LOCKS_ALL) { + c_p_xlocks = ~c_p_locks & ERTS_PROC_LOCKS_ALL; + if (erts_smp_proc_trylock(c_p, c_p_xlocks) == EBUSY) { + erts_smp_proc_unlock(c_p, c_p_locks & ~ERTS_PROC_LOCK_MAIN); + erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); + } + } + } + erts_tracer_replace(t_p, erts_tracer_nil); + t_p->trace_flags &= ~TRACEE_FLAGS; + + if (c_p_xlocks) + erts_smp_proc_unlock(c_p, c_p_xlocks); + } + + return 0; +} + +int erts_is_tracer_proc_enabled(Process* c_p, ErtsProcLocks c_p_locks, + ErtsPTabElementCommon *t_p, Eterm type) +{ + return is_tracer_enabled(c_p, c_p_locks, t_p, NULL, TRACE_FUN_ENABLED, am_trace_status); +} + +int erts_is_tracer_enabled(Process *c_p, const ErtsTracer tracer) +{ + ErtsTracerNif *tnif = lookup_tracer_nif(tracer); + if (tnif) { + Eterm nif_result = call_enabled_tracer(c_p, tracer, &tnif, + TRACE_FUN_ENABLED, am_trace_status, + c_p->common.id); + switch (nif_result) { + case am_discard: + case am_trace: return 1; + default: + break; + } + } + return 0; +} + +void erts_tracer_replace(ErtsPTabElementCommon *t_p, const ErtsTracer tracer) +{ +#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) + if (is_internal_pid(t_p->id) && !erts_thr_progress_is_blocking()) { + erts_proc_lc_chk_have_proc_locks((Process*)t_p, ERTS_PROC_LOCKS_ALL); + } else if (is_internal_port(t_p->id)) { + ERTS_LC_ASSERT(erts_lc_is_port_locked((Port*)t_p) + || erts_thr_progress_is_blocking()); + } +#endif + if (ERTS_TRACER_COMPARE(t_p->tracer, tracer)) + return; + + erts_tracer_update(&t_p->tracer, tracer); +} + +static void free_tracer(void *p) +{ + ErtsTracer tracer = (ErtsTracer)p; + + if (is_immed(ERTS_TRACER_STATE(tracer))) { + erts_free(ERTS_ALC_T_HEAP_FRAG, ptr_val(tracer)); + } else { + ErlHeapFragment *hf = (void*)((char*)(ptr_val(tracer)) - offsetof(ErlHeapFragment, mem)); + free_message_buffer(hf); + } +} + +/* un-define erts_tracer_update before implementation */ +#ifdef erts_tracer_update +#undef erts_tracer_update +#endif + +/* + * ErtsTracer is either NIL, 'true' or [Mod | State] + * + * - If State is immediate then the memory for + * the cons cell is just two words + sizeof(ErtsThrPrgrLaterOp) large. + * - If State is a complex term then the cons cell + * is allocated in an ErlHeapFragment where the cons + * ptr points to the mem field. So in order to get the + * ptr to the fragment you do this: + * (char*)(ptr_val(tracer)) - offsetof(ErlHeapFragment, mem) + * Normally you shouldn't have to care about this though + * as erts_tracer_update takes care of it for you. + * + * When ErtsTracer is stored in the stack as part of a + * return trace, the cons cell is stored on the heap of + * the process. + * + * The cons cell is not always stored on the heap as: + * 1) for port/meta tracing there is no heap + * 2) we would need the main lock in order to + * read the tracer which is undesirable. + * + * One way to optimize this (memory wise) is to keep an refc and only bump + * the refc when *tracer is NIL. + */ +void +erts_tracer_update(ErtsTracer *tracer, const ErtsTracer new_tracer) +{ + ErlHeapFragment *hf; + + if (is_not_nil(*tracer)) { + Uint offs = 2; + UWord size = 2 * sizeof(Eterm) + sizeof(ErtsThrPrgrLaterOp); + ASSERT(is_list(*tracer)); + if (is_not_immed(ERTS_TRACER_STATE(*tracer))) { + hf = (void*)(((char*)(ptr_val(*tracer)) - offsetof(ErlHeapFragment, mem))); + offs = hf->used_size; + size = hf->alloc_size * sizeof(Eterm) + sizeof(ErlHeapFragment); + ASSERT(offs == size_object(*tracer)); + } + /* We schedule the free:ing of the tracer until after a thread progress + has been made so that we know that no schedulers have any references + to it. Because we do this, it is possible to release all locks of a + process/port and still use the ErtsTracer of that port/process + without having to worry if it is free'd. + */ + erts_schedule_thr_prgr_later_cleanup_op( + free_tracer, (void*)(*tracer), + (ErtsThrPrgrLaterOp*)(ptr_val(*tracer) + offs), + size); + } + + if (is_nil(new_tracer)) { + *tracer = new_tracer; + } else if (is_immed(ERTS_TRACER_STATE(new_tracer))) { + /* If tracer state is an immediate we only allocate a 2 Eterm heap. + Not sure if it is worth it, we save 4 words (sizeof(ErlHeapFragment)) + per tracer. */ + Eterm *hp = erts_alloc(ERTS_ALC_T_HEAP_FRAG, + 2*sizeof(Eterm) + sizeof(ErtsThrPrgrLaterOp)); + *tracer = CONS(hp, ERTS_TRACER_MODULE(new_tracer), + ERTS_TRACER_STATE(new_tracer)); + } else { + Eterm *hp, tracer_state = ERTS_TRACER_STATE(new_tracer), + tracer_module = ERTS_TRACER_MODULE(new_tracer); + Uint sz = size_object(tracer_state); + hf = new_message_buffer(sz + 2 /* cons cell */ + (sizeof(ErtsThrPrgrLaterOp)+sizeof(Eterm)-1)/sizeof(Eterm)); + hp = hf->mem + 2; + hf->used_size -= (sizeof(ErtsThrPrgrLaterOp)+sizeof(Eterm)-1)/sizeof(Eterm); + *tracer = copy_struct(tracer_state, sz, &hp, &hf->off_heap); + *tracer = CONS(hf->mem, tracer_module, *tracer); + ASSERT((void*)(((char*)(ptr_val(*tracer)) - offsetof(ErlHeapFragment, mem))) == hf); + } +} + +static void init_tracer_nif() +{ + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ; + rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; + erts_smp_rwmtx_init_opt(&tracer_mtx, &rwmtx_opt, "tracer_mtx"); + + erts_tracer_nif_clear(); + +} + +int erts_tracer_nif_clear() +{ + + erts_smp_rwmtx_rlock(&tracer_mtx); + if (!tracer_hash || tracer_hash->nobjs) { + + HashFunctions hf; + hf.hash = tracer_hash_fun; + hf.cmp = tracer_cmp_fun; + hf.alloc = tracer_alloc_fun; + hf.free = tracer_free_fun; + hf.meta_alloc = (HMALLOC_FUN) erts_alloc; + hf.meta_free = (HMFREE_FUN) erts_free; + hf.meta_print = (HMPRINT_FUN) erts_print; + + erts_smp_rwmtx_runlock(&tracer_mtx); + erts_smp_rwmtx_rwlock(&tracer_mtx); + + if (tracer_hash) + hash_delete(tracer_hash); + + tracer_hash = hash_new(ERTS_ALC_T_TRACER_NIF, "tracer_hash", 10, hf); + + erts_smp_rwmtx_rwunlock(&tracer_mtx); + return 1; + } + + erts_smp_rwmtx_runlock(&tracer_mtx); + return 0; +} + +static int tracer_cmp_fun(void* a, void* b) +{ + return ((ErtsTracerNif*)a)->module != ((ErtsTracerNif*)b)->module; +} + +static HashValue tracer_hash_fun(void* obj) +{ + return make_internal_hash(((ErtsTracerNif*)obj)->module); +} + +static void *tracer_alloc_fun(void* tmpl) +{ + ErtsTracerNif *obj = erts_alloc(ERTS_ALC_T_TRACER_NIF, + sizeof(ErtsTracerNif) + + sizeof(ErtsThrPrgrLaterOp)); + memcpy(obj, tmpl, sizeof(*obj)); + return obj; +} + +static void tracer_free_fun_cb(void* obj) +{ + erts_free(ERTS_ALC_T_TRACER_NIF, obj); +} + +static void tracer_free_fun(void* obj) +{ + ErtsTracerNif *tnif = obj; + erts_schedule_thr_prgr_later_op( + tracer_free_fun_cb, obj, + (ErtsThrPrgrLaterOp*)(tnif + 1)); + +} diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h index a0058264d7..de4beadb7e 100644 --- a/erts/emulator/beam/erl_trace.h +++ b/erts/emulator/beam/erl_trace.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2012-2013. All Rights Reserved. + * Copyright Ericsson AB 2012-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -46,6 +46,8 @@ #define ERTS_SEQTFLGS2TSTYPE(SEQTFLGS) \ ((int) (((SEQTFLGS) >> ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT) \ & ERTS_TRACE_TS_TYPE_MASK)) +#define ERTS_SEQTFLGS2TFLGS(SEQTFLGS) \ + (ERTS_SEQTFLGS2TSTYPE(SEQTFLGS) << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT) #endif /* ERL_TRACE_H__FLAGS__ */ @@ -54,6 +56,15 @@ struct binary; +typedef struct +{ + int on; + struct binary* match_spec; +} ErtsTracingEvent; + +extern ErtsTracingEvent erts_send_tracing[]; +extern ErtsTracingEvent erts_receive_tracing[]; + /* erl_bif_trace.c */ Eterm erl_seq_trace_info(Process *p, Eterm arg1); void erts_system_monitor_clear(Process *c_p); @@ -62,15 +73,19 @@ void erts_system_profile_clear(Process *c_p); /* erl_trace.c */ void erts_init_trace(void); void erts_trace_check_exiting(Eterm exiting); -Eterm erts_set_system_seq_tracer(Process *c_p, - ErtsProcLocks c_p_locks, - Eterm new); -Eterm erts_get_system_seq_tracer(void); -void erts_change_default_tracing(int setflags, Uint *flagsp, Eterm *tracerp); -void erts_get_default_tracing(Uint *flagsp, Eterm *tracerp); +ErtsTracer erts_set_system_seq_tracer(Process *c_p, + ErtsProcLocks c_p_locks, + ErtsTracer new); +ErtsTracer erts_get_system_seq_tracer(void); +void erts_change_default_proc_tracing(int setflags, Uint flagsp, + const ErtsTracer tracerp); +void erts_get_default_proc_tracing(Uint *flagsp, ErtsTracer *tracerp); +void erts_change_default_port_tracing(int setflags, Uint flagsp, + const ErtsTracer tracerp); +void erts_get_default_port_tracing(Uint *flagsp, ErtsTracer *tracerp); void erts_set_system_monitor(Eterm monitor); Eterm erts_get_system_monitor(void); -int erts_is_tracer_proc_valid(Process* p); +int erts_is_tracer_valid(Process* p); #ifdef ERTS_SMP void erts_check_my_tracer_proc(Process *); @@ -81,28 +96,32 @@ void erts_foreach_sys_msg_in_q(void (*func)(Eterm, Eterm, ErlHeapFragment *)); void erts_queue_error_logger_message(Eterm, Eterm, ErlHeapFragment *); +void erts_send_sys_msg_proc(Eterm, Eterm, Eterm, ErlHeapFragment *); #endif -void erts_send_sys_msg_proc(Eterm, Eterm, Eterm, ErlHeapFragment *); void trace_send(Process*, Eterm, Eterm); -void trace_receive(Process*, Eterm); -Uint32 erts_call_trace(Process *p, BeamInstr mfa[], struct binary *match_spec, Eterm* args, - int local, Eterm *tracer_pid); -void erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid); +void trace_receive(Process*, Eterm, Eterm, ErtsTracingEvent*); +Uint32 erts_call_trace(Process *p, BeamInstr mfa[], struct binary *match_spec, + Eterm* args, int local, ErtsTracer *tracer); +void erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, + ErtsTracer *tracer); void erts_trace_exception(Process* p, BeamInstr mfa[], Eterm class, Eterm value, - Eterm *tracer); + ErtsTracer *tracer); void erts_trace_return_to(Process *p, BeamInstr *pc); -void trace_sched(Process*, Eterm); -void trace_proc(Process*, Process*, Eterm, Eterm); -void trace_proc_spawn(Process*, Eterm pid, Eterm mod, Eterm func, Eterm args); +void trace_sched(Process*, ErtsProcLocks, Eterm); +void trace_proc(Process*, ErtsProcLocks, Process*, Eterm, Eterm); +void trace_proc_spawn(Process*, Eterm what, Eterm pid, Eterm mod, Eterm func, Eterm args); void save_calls(Process *p, Export *); -void trace_gc(Process *p, Eterm what); +void trace_gc(Process *p, Eterm what, Uint size); /* port tracing */ -void trace_virtual_sched(Process*, Eterm); +void trace_virtual_sched(Process*, ErtsProcLocks, Eterm); void trace_sched_ports(Port *pp, Eterm); void trace_sched_ports_where(Port *pp, Eterm, Eterm); void trace_port(Port *, Eterm what, Eterm data); void trace_port_open(Port *, Eterm calling_pid, Eterm drv_name); +void trace_port_receive(Port *, Eterm calling_pid, Eterm tag, ...); +void trace_port_send(Port *, Eterm to, Eterm msg, int exists); +void trace_port_send_binary(Port *, Eterm to, Eterm what, char *bin, Sint sz); /* system_profile */ void erts_set_system_profile(Eterm profile); @@ -121,7 +140,7 @@ void monitor_large_heap(Process *p); void monitor_generic(Process *p, Eterm type, Eterm spec); Uint erts_trace_flag2bit(Eterm flag); int erts_trace_flags(Eterm List, - Uint *pMask, Eterm *pTracer, int *pCpuTimestamp); + Uint *pMask, ErtsTracer *pTracer, int *pCpuTimestamp); Eterm erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr *I); #ifdef ERTS_SMP @@ -161,15 +180,53 @@ int erts_set_trace_pattern(Process*p, Eterm* mfa, int specified, struct binary* match_prog_set, struct binary *meta_match_prog_set, int on, struct trace_pattern_flags, - Eterm meta_tracer_pid, int is_blocking); + ErtsTracer meta_tracer, int is_blocking); void erts_get_default_trace_pattern(int *trace_pattern_is_on, struct binary **match_spec, struct binary **meta_match_spec, struct trace_pattern_flags *trace_pattern_flags, - Eterm *meta_tracer_pid); + ErtsTracer *meta_tracer); int erts_is_default_trace_enabled(void); void erts_bif_trace_init(void); int erts_finish_breakpointing(void); +/* Nif tracer functions */ +int erts_is_tracer_proc_enabled(Process *c_p, ErtsProcLocks c_p_locks, + ErtsPTabElementCommon *t_p, Eterm type); +int erts_is_tracer_enabled(Process *c_p, const ErtsTracer tracer); +Eterm erts_tracer_to_term(Process *p, ErtsTracer tracer); +ErtsTracer erts_term_to_tracer(Eterm prefix, Eterm term); +void erts_tracer_replace(ErtsPTabElementCommon *t_p, + const ErtsTracer new_tracer); +void erts_tracer_update(ErtsTracer *tracer, const ErtsTracer new_tracer); +int erts_tracer_nif_clear(void); + +#define erts_tracer_update(t,n) do { if (*(t) != (n)) erts_tracer_update(t,n); } while(0) +#define ERTS_TRACER_CLEAR(t) erts_tracer_update(t, erts_tracer_nil) + +static const ErtsTracer +ERTS_DECLARE_DUMMY(erts_tracer_true) = am_true; + +static const ErtsTracer +ERTS_DECLARE_DUMMY(erts_tracer_nil) = NIL; + +#define ERTS_TRACER_COMPARE(t1, t2) \ + (EQ((t1), (t2))) + +#define ERTS_TRACER_IS_NIL(t1) ERTS_TRACER_COMPARE(t1, erts_tracer_nil) + +#define IS_TRACER_VALID(tracer) \ + (ERTS_TRACER_COMPARE(tracer,erts_tracer_true) \ + || ERTS_TRACER_IS_NIL(tracer) \ + || (is_list(tracer) && is_atom(CAR(list_val(tracer))))) + +#define ERTS_TRACER_FROM_ETERM(termp) \ + ((ErtsTracer*)(termp)) + +#define ERTS_TRACER_PROC_IS_ENABLED(PROC) \ + (!ERTS_TRACER_IS_NIL(ERTS_TRACER(PROC)) \ + && erts_is_tracer_proc_enabled(PROC, ERTS_PROC_LOCK_MAIN, \ + &(PROC)->common, am_trace_status)) + #endif /* ERL_TRACE_H__ */ diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c index 36d85b0a22..3c3536c021 100644 --- a/erts/emulator/beam/erl_unicode.c +++ b/erts/emulator/beam/erl_unicode.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2008-2013. All Rights Reserved. + * Copyright Ericsson AB 2008-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_unicode.h b/erts/emulator/beam/erl_unicode.h index 4c25d89b7c..e01eaa787e 100644 --- a/erts/emulator/beam/erl_unicode.h +++ b/erts/emulator/beam/erl_unicode.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2008-2009. All Rights Reserved. + * Copyright Ericsson AB 2008-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_unicode_normalize.h b/erts/emulator/beam/erl_unicode_normalize.h index 16c62db50e..21e2a52544 100644 --- a/erts/emulator/beam/erl_unicode_normalize.h +++ b/erts/emulator/beam/erl_unicode_normalize.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2010. All Rights Reserved. + * Copyright Ericsson AB 1999-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h index 640969daf0..81800752f0 100644 --- a/erts/emulator/beam/erl_utils.h +++ b/erts/emulator/beam/erl_utils.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2012-2014. All Rights Reserved. + * Copyright Ericsson AB 2012-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h index 98f27a1725..f3c54de214 100644 --- a/erts/emulator/beam/erl_vm.h +++ b/erts/emulator/beam/erl_vm.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2014. All Rights Reserved. + * Copyright Ericsson AB 1996-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -165,7 +165,6 @@ extern int erts_atom_table_size;/* Atom table size */ extern int erts_pd_initial_size;/* Initial Process dictionary table size */ #define ORIG_CREATION 0 -#define INTERNAL_CREATION 255 /* macros for extracting bytes from uint16's */ diff --git a/erts/emulator/beam/erl_zlib.c b/erts/emulator/beam/erl_zlib.c index e44e86c39c..944ff2e35f 100644 --- a/erts/emulator/beam/erl_zlib.c +++ b/erts/emulator/beam/erl_zlib.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2009-2013. All Rights Reserved. + * Copyright Ericsson AB 2009-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_zlib.h b/erts/emulator/beam/erl_zlib.h index 2ca1e3735e..c83c6f291f 100644 --- a/erts/emulator/beam/erl_zlib.h +++ b/erts/emulator/beam/erl_zlib.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2009-2013. All Rights Reserved. + * Copyright Ericsson AB 2009-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erlang_dtrace.d b/erts/emulator/beam/erlang_dtrace.d index c682f76e3a..237889e0f5 100644 --- a/erts/emulator/beam/erlang_dtrace.d +++ b/erts/emulator/beam/erlang_dtrace.d @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Dustin Sallings, Michal Ptaszek, Scott Lystig Fritchie 2011-2012. + * Copyright Dustin Sallings, Michal Ptaszek, Scott Lystig Fritchie 2011-2016. * All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -700,6 +700,35 @@ provider erlang { */ probe efile_drv__return(int, int, char *, int, int, int); + +/* + * The set of probes called by the erlang tracer nif backend. In order + * to receive events on these you both have to enable tracing in erlang + * using the trace bifs and also from dtrace/systemtap. + */ + + + /** + * A trace message of type `event` was triggered by process `p`. + * + * + * @param p the PID (string form) of the process + * @param event the event that was triggered (i.e. call or spawn) + * @param state the state of the tracer nif as a string + * @param arg1 first argument to the trace event + * @param arg2 second argument to the trace event + */ + probe trace(char *p, char *event, char *state, char *arg1, char *arg2); + + /** + * A sequence trace message of type `label` was triggered. + * + * @param state the state of the tracer nif as a string + * @param label the seq trace label + * @param seq_info the seq trace info tuple as a string + */ + probe trace_seq(char *state, char *label, char *seq_info); + /* * NOTE: * For formatting int64_t arguments within a D script, see: diff --git a/erts/emulator/beam/erlang_lttng.c b/erts/emulator/beam/erlang_lttng.c new file mode 100644 index 0000000000..fce40eedc1 --- /dev/null +++ b/erts/emulator/beam/erlang_lttng.c @@ -0,0 +1,32 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 1996-2016. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#ifdef USE_LTTNG +#define TRACEPOINT_CREATE_PROBES +/* + * The header containing our TRACEPOINT_EVENTs. + */ +#define TRACEPOINT_DEFINE +#include "erlang_lttng.h" +#endif /* USE_LTTNG */ diff --git a/erts/emulator/beam/erlang_lttng.h b/erts/emulator/beam/erlang_lttng.h new file mode 100644 index 0000000000..12f68e477b --- /dev/null +++ b/erts/emulator/beam/erlang_lttng.h @@ -0,0 +1,424 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 1996-2016. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifdef USE_LTTNG +#undef TRACEPOINT_PROVIDER +#define TRACEPOINT_PROVIDER com_ericsson_otp + +#undef TRACEPOINT_INCLUDE +#define TRACEPOINT_INCLUDE "erlang_lttng.h" + +#if !defined(__ERLANG_LTTNG_H__) || defined(TRACEPOINT_HEADER_MULTI_READ) +#define __ERLANG_LTTNG_H__ + +#include <lttng/tracepoint.h> + +/* Schedulers */ + +TRACEPOINT_EVENT( + com_ericsson_otp, + scheduler_poll, + TP_ARGS( + int, id, + int, runnable + ), + TP_FIELDS( + ctf_integer(int, scheduler, id) + ctf_integer(int, runnable, runnable) + ) +) + +#ifndef LTTNG_CARRIER_STATS +#define LTTNG_CARRIER_STATS +typedef struct { + unsigned long no; + unsigned long size; +} lttng_stat_values_t; + +typedef struct { + lttng_stat_values_t carriers; + lttng_stat_values_t blocks; +} lttng_carrier_stats_t; +#endif + + +/* Port and Driver Scheduling */ + +TRACEPOINT_EVENT( + com_ericsson_otp, + driver_start, + TP_ARGS( + char*, pid, + char*, driver, + char*, port + ), + TP_FIELDS( + ctf_string(pid, pid) + ctf_string(driver, driver) + ctf_string(port, port) + ) +) + +TRACEPOINT_EVENT( + com_ericsson_otp, + driver_init, + TP_ARGS( + char*, driver, + int, major, + int, minor, + int, flags + ), + TP_FIELDS( + ctf_string(driver, driver) + ctf_integer(int, major, major) + ctf_integer(int, minor, minor) + ctf_integer(int, flags, flags) + ) +) + +TRACEPOINT_EVENT( + com_ericsson_otp, + driver_outputv, + TP_ARGS( + char*, pid, + char*, port, + char*, driver, + size_t, bytes + ), + TP_FIELDS( + ctf_string(pid, pid) + ctf_string(port, port) + ctf_string(driver, driver) + ctf_integer(size_t, bytes, bytes) + ) +) + +TRACEPOINT_EVENT( + com_ericsson_otp, + driver_output, + TP_ARGS( + char*, pid, + char*, port, + char*, driver, + size_t, bytes + ), + TP_FIELDS( + ctf_string(pid, pid) + ctf_string(port, port) + ctf_string(driver, driver) + ctf_integer(size_t, bytes, bytes) + ) +) + +TRACEPOINT_EVENT( + com_ericsson_otp, + driver_ready_input, + TP_ARGS( + char*, pid, + char*, port, + char*, driver + ), + TP_FIELDS( + ctf_string(pid, pid) + ctf_string(port, port) + ctf_string(driver, driver) + ) +) + +TRACEPOINT_EVENT( + com_ericsson_otp, + driver_ready_output, + TP_ARGS( + char*, pid, + char*, port, + char*, driver + ), + TP_FIELDS( + ctf_string(pid, pid) + ctf_string(port, port) + ctf_string(driver, driver) + ) +) + +TRACEPOINT_EVENT( + com_ericsson_otp, + driver_event, + TP_ARGS( + char*, pid, + char*, port, + char*, driver + ), + TP_FIELDS( + ctf_string(pid, pid) + ctf_string(port, port) + ctf_string(driver, driver) + ) +) + +TRACEPOINT_EVENT( + com_ericsson_otp, + driver_timeout, + TP_ARGS( + char*, pid, + char*, port, + char*, driver + ), + TP_FIELDS( + ctf_string(pid, pid) + ctf_string(port, port) + ctf_string(driver, driver) + ) +) + +TRACEPOINT_EVENT( + com_ericsson_otp, + driver_stop_select, + TP_ARGS( + char*, driver + ), + TP_FIELDS( + ctf_string(driver, driver) + ) +) + +TRACEPOINT_EVENT( + com_ericsson_otp, + driver_flush, + TP_ARGS( + char*, pid, + char*, port, + char*, driver + ), + TP_FIELDS( + ctf_string(pid, pid) + ctf_string(port, port) + ctf_string(driver, driver) + ) +) + +TRACEPOINT_EVENT( + com_ericsson_otp, + driver_stop, + TP_ARGS( + char*, pid, + char*, port, + char*, driver + ), + TP_FIELDS( + ctf_string(pid, pid) + ctf_string(port, port) + ctf_string(driver, driver) + ) +) + +TRACEPOINT_EVENT( + com_ericsson_otp, + driver_process_exit, + TP_ARGS( + char*, pid, + char*, port, + char*, driver + ), + TP_FIELDS( + ctf_string(pid, pid) + ctf_string(port, port) + ctf_string(driver, driver) + ) +) + +TRACEPOINT_EVENT( + com_ericsson_otp, + driver_ready_async, + TP_ARGS( + char*, pid, + char*, port, + char*, driver + ), + TP_FIELDS( + ctf_string(pid, pid) + ctf_string(port, port) + ctf_string(driver, driver) + ) +) + +TRACEPOINT_EVENT( + com_ericsson_otp, + driver_finish, + TP_ARGS( + char*, driver + ), + TP_FIELDS( + ctf_string(driver, driver) + ) +) + +TRACEPOINT_EVENT( + com_ericsson_otp, + driver_call, + TP_ARGS( + char*, pid, + char*, port, + char*, driver, + unsigned int, command, + size_t, bytes + ), + TP_FIELDS( + ctf_string(pid, pid) + ctf_string(port, port) + ctf_string(driver, driver) + ctf_integer(unsigned int, command, command) + ctf_integer(size_t, bytes, bytes) + ) +) + +TRACEPOINT_EVENT( + com_ericsson_otp, + driver_control, + TP_ARGS( + char*, pid, + char*, port, + char*, driver, + unsigned int, command, + size_t, bytes + ), + TP_FIELDS( + ctf_string(pid, pid) + ctf_string(port, port) + ctf_string(driver, driver) + ctf_integer(unsigned int, command, command) + ctf_integer(size_t, bytes, bytes) + ) +) + +/* Async pool */ + +TRACEPOINT_EVENT( + com_ericsson_otp, + aio_pool_get, + TP_ARGS( + char*, port, + int, length + ), + TP_FIELDS( + ctf_string(port, port) + ctf_integer(int, length, length) + ) +) + +TRACEPOINT_EVENT( + com_ericsson_otp, + aio_pool_put, + TP_ARGS( + char*, port, + int, length + ), + TP_FIELDS( + ctf_string(port, port) + ctf_integer(int, length, length) + ) +) + + +/* Memory Allocator */ + +TRACEPOINT_EVENT( + com_ericsson_otp, + carrier_create, + TP_ARGS( + const char*, type, + int, instance, + unsigned long, size, + lttng_carrier_stats_t *, mbcs, + lttng_carrier_stats_t *, sbcs + ), + TP_FIELDS( + ctf_string(type, type) + ctf_integer(int, instance, instance) + ctf_integer(unsigned long, size, size) + ctf_integer(unsigned long, mbc_carriers, mbcs->carriers.no) + ctf_integer(unsigned long, mbc_carriers_size, mbcs->carriers.size) + ctf_integer(unsigned long, mbc_blocks, mbcs->blocks.no) + ctf_integer(unsigned long, mbc_blocks_size, mbcs->blocks.size) + ctf_integer(unsigned long, sbc_carriers, sbcs->carriers.no) + ctf_integer(unsigned long, sbc_carriers_size, sbcs->carriers.size) + ctf_integer(unsigned long, sbc_blocks, sbcs->blocks.no) + ctf_integer(unsigned long, sbc_blocks_size, sbcs->blocks.size) + ) +) + + +TRACEPOINT_EVENT( + com_ericsson_otp, + carrier_destroy, + TP_ARGS( + const char*, type, + int, instance, + unsigned long, size, + lttng_carrier_stats_t *, mbcs, + lttng_carrier_stats_t *, sbcs + ), + TP_FIELDS( + ctf_string(type, type) + ctf_integer(int, instance, instance) + ctf_integer(unsigned long, size, size) + ctf_integer(unsigned long, mbc_carriers, mbcs->carriers.no) + ctf_integer(unsigned long, mbc_carriers_size, mbcs->carriers.size) + ctf_integer(unsigned long, mbc_blocks, mbcs->blocks.no) + ctf_integer(unsigned long, mbc_blocks_size, mbcs->blocks.size) + ctf_integer(unsigned long, sbc_carriers, sbcs->carriers.no) + ctf_integer(unsigned long, sbc_carriers_size, sbcs->carriers.size) + ctf_integer(unsigned long, sbc_blocks, sbcs->blocks.no) + ctf_integer(unsigned long, sbc_blocks_size, sbcs->blocks.size) + ) +) + +TRACEPOINT_EVENT( + com_ericsson_otp, + carrier_pool_put, + TP_ARGS( + const char*, name, + int, instance, + unsigned long, size + ), + TP_FIELDS( + ctf_string(type, name) + ctf_integer(int, instance, instance) + ctf_integer(unsigned long, size, size) + ) +) + +TRACEPOINT_EVENT( + com_ericsson_otp, + carrier_pool_get, + TP_ARGS( + const char*, name, + int, instance, + unsigned long, size + ), + TP_FIELDS( + ctf_string(type, name) + ctf_integer(int, instance, instance) + ctf_integer(unsigned long, size, size) + ) +) + +#endif /* __ERLANG_LTTNG_H__ */ +#include <lttng/tracepoint-event.h> +#endif /* USE_LTTNG */ diff --git a/erts/emulator/beam/error.h b/erts/emulator/beam/error.h index cba8672c68..6c33b12dd0 100644 --- a/erts/emulator/beam/error.h +++ b/erts/emulator/beam/error.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c index 581efe6eec..02c24557c1 100644 --- a/erts/emulator/beam/export.c +++ b/erts/emulator/beam/export.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2013. All Rights Reserved. + * Copyright Ericsson AB 1996-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/export.h b/erts/emulator/beam/export.h index a8bc9d2f66..8c81cbd410 100644 --- a/erts/emulator/beam/export.h +++ b/erts/emulator/beam/export.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2013. All Rights Reserved. + * Copyright Ericsson AB 1996-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index aac1490f0c..3c002d43a7 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2014. All Rights Reserved. + * Copyright Ericsson AB 1996-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -51,7 +51,18 @@ #define MAX_STRING_LEN 0xffff -#define is_valid_creation(Cre) ((unsigned)(Cre) < MAX_CREATION || (Cre) == INTERNAL_CREATION) +/* MAX value for the creation field in pid, port and reference + for the local node and for the current external format. + + Larger creation values than this are allowed in external pid, port and refs + encoded with NEW_PID_EXT, NEW_PORT_EXT and NEWER_REFERENCE_EXT. + The point here is to prepare for future upgrade to 32-bit creation. + OTP-19 (erts-8.0) can handle big creation values from other (newer) nodes, + but do not use big creation values for the local node yet, + as we still may have to communicate with older nodes. +*/ +#define ERTS_MAX_LOCAL_CREATION (3) +#define is_valid_creation(Cre) ((unsigned)(Cre) <= ERTS_MAX_LOCAL_CREATION) #undef ERTS_DEBUG_USE_DIST_SEP #ifdef DEBUG @@ -97,7 +108,7 @@ static byte* enc_pid(ErtsAtomCacheMap *, Eterm, byte*, Uint32); struct B2TContext_t; static byte* dec_term(ErtsDistExternal*, ErtsHeapFactory*, byte*, Eterm*, struct B2TContext_t*); static byte* dec_atom(ErtsDistExternal *, byte*, Eterm*); -static byte* dec_pid(ErtsDistExternal *, ErtsHeapFactory*, byte*, Eterm*); +static byte* dec_pid(ErtsDistExternal *, ErtsHeapFactory*, byte*, Eterm*, byte tag); static Sint decoded_size(byte *ep, byte* endp, int internal_tags, struct B2TContext_t*); static BIF_RETTYPE term_to_binary_trap_1(BIF_ALIST_1); @@ -967,19 +978,24 @@ erts_decode_dist_ext(ErtsHeapFactory* factory, return THE_NON_VALUE; } -Eterm erts_decode_ext(ErtsHeapFactory* factory, byte **ext) +Eterm erts_decode_ext(ErtsHeapFactory* factory, byte **ext, Uint32 flags) { + ErtsDistExternal ede, *edep; Eterm obj; byte *ep = *ext; if (*ep++ != VERSION_MAGIC) { erts_factory_undo(factory); return THE_NON_VALUE; } - ep = dec_term(NULL, factory, ep, &obj, NULL); + if (flags) { + ASSERT(flags == ERTS_DIST_EXT_BTT_SAFE); + ede.flags = flags; /* a dummy struct just for the flags */ + edep = &ede; + } else { + edep = NULL; + } + ep = dec_term(edep, factory, ep, &obj, NULL); if (!ep) { -#ifdef DEBUG - bin_write(ERTS_PRINT_STDERR,NULL,*ext,500); -#endif return THE_NON_VALUE; } *ext = ep; @@ -2147,12 +2163,13 @@ static byte* enc_pid(ErtsAtomCacheMap *acmp, Eterm pid, byte* ep, Uint32 dflags) { Uint on, os; + Eterm sysname = ((is_internal_pid(pid) && (dflags & DFLAG_INTERNAL_TAGS)) + ? am_Empty : pid_node_name(pid)); + Uint32 creation = pid_creation(pid); + byte* tagp = ep++; - *ep++ = PID_EXT; /* insert atom here containing host and sysname */ - ep = enc_atom(acmp, pid_node_name(pid), ep, dflags); - - /* two bytes for each number and serial */ + ep = enc_atom(acmp, sysname, ep, dflags); on = pid_number(pid); os = pid_serial(pid); @@ -2161,8 +2178,15 @@ enc_pid(ErtsAtomCacheMap *acmp, Eterm pid, byte* ep, Uint32 dflags) ep += 4; put_int32(os, ep); ep += 4; - *ep++ = (is_internal_pid(pid) && (dflags & DFLAG_INTERNAL_TAGS)) ? - INTERNAL_CREATION : pid_creation(pid); + if (creation <= ERTS_MAX_LOCAL_CREATION) { + *tagp = PID_EXT; + *ep++ = creation; + } else { + ASSERT(is_external_pid(pid)); + *tagp = NEW_PID_EXT; + put_int32(creation, ep); + ep += 4; + } return ep; } @@ -2242,27 +2266,27 @@ dec_atom(ErtsDistExternal *edep, byte* ep, Eterm* objp) return ep; } -static ERTS_INLINE ErlNode* dec_get_node(Eterm sysname, Uint creation) +static ERTS_INLINE ErlNode* dec_get_node(Eterm sysname, Uint32 creation) { - switch (creation) { - case INTERNAL_CREATION: + if (sysname == am_Empty) /* && DFLAG_INTERNAL_TAGS */ return erts_this_node; - case ORIG_CREATION: - if (sysname == erts_this_node->sysname) { - creation = erts_this_node->creation; - } - } + + if (sysname == erts_this_node->sysname + && (creation == erts_this_node->creation || creation == ORIG_CREATION)) + return erts_this_node; + return erts_find_or_insert_node(sysname,creation); } static byte* -dec_pid(ErtsDistExternal *edep, ErtsHeapFactory* factory, byte* ep, Eterm* objp) +dec_pid(ErtsDistExternal *edep, ErtsHeapFactory* factory, byte* ep, + Eterm* objp, byte tag) { Eterm sysname; Uint data; Uint num; Uint ser; - Uint cre; + Uint32 cre; ErlNode *node; *objp = NIL; /* In case we fail, don't leave a hole in the heap */ @@ -2278,12 +2302,19 @@ dec_pid(ErtsDistExternal *edep, ErtsHeapFactory* factory, byte* ep, Eterm* objp) ep += 4; if (ser > ERTS_MAX_PID_SERIAL) return NULL; - cre = get_int8(ep); - ep += 1; - if (!is_valid_creation(cre)) { - return NULL; + if (tag == PID_EXT) { + cre = get_int8(ep); + ep += 1; + if (!is_valid_creation(cre)) { + return NULL; + } + } else { + ASSERT(tag == NEW_PID_EXT); + cre = get_int32(ep); + ep += 4; } + data = make_pid_data(ser, num); /* @@ -2523,16 +2554,26 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, case REF_DEF: case EXTERNAL_REF_DEF: { Uint32 *ref_num; + Eterm sysname = (((dflags & DFLAG_INTERNAL_TAGS) && is_internal_ref(obj)) + ? am_Empty : ref_node_name(obj)); + Uint32 creation = ref_creation(obj); + byte* tagp = ep++; ASSERT(dflags & DFLAG_EXTENDED_REFERENCES); - *ep++ = NEW_REFERENCE_EXT; i = ref_no_of_numbers(obj); put_int16(i, ep); ep += 2; - ep = enc_atom(acmp,ref_node_name(obj),ep,dflags); - *ep++ = ((dflags & DFLAG_INTERNAL_TAGS) && is_internal_ref(obj)) ? - INTERNAL_CREATION : ref_creation(obj); + ep = enc_atom(acmp, sysname, ep, dflags); + if (creation <= ERTS_MAX_LOCAL_CREATION) { + *tagp = NEW_REFERENCE_EXT; + *ep++ = creation; + } else { + ASSERT(is_external_ref(obj)); + *tagp = NEWER_REFERENCE_EXT; + put_int32(creation, ep); + ep += 4; + } ref_num = ref_numbers(obj); for (j = 0; j < i; j++) { put_int32(ref_num[j], ep); @@ -2541,17 +2582,27 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, break; } case PORT_DEF: - case EXTERNAL_PORT_DEF: + case EXTERNAL_PORT_DEF: { + Eterm sysname = (((dflags & DFLAG_INTERNAL_TAGS) && is_internal_port(obj)) + ? am_Empty : port_node_name(obj)); + Uint32 creation = port_creation(obj); + byte* tagp = ep++; - *ep++ = PORT_EXT; - ep = enc_atom(acmp,port_node_name(obj),ep,dflags); + ep = enc_atom(acmp, sysname, ep, dflags); j = port_number(obj); put_int32(j, ep); ep += 4; - *ep++ = ((dflags & DFLAG_INTERNAL_TAGS) && is_internal_port(obj)) ? - INTERNAL_CREATION : port_creation(obj); + if (creation <= ERTS_MAX_LOCAL_CREATION) { + *tagp = PORT_EXT; + *ep++ = creation; + } else { + ASSERT(is_external_port(obj)); + *tagp = NEW_PORT_EXT; + put_int32(creation, ep); + ep += 4; + } break; - + } case LIST_DEF: { int is_str; @@ -3255,20 +3306,23 @@ dec_term_atom_common: hp += FLOAT_SIZE_OBJECT; break; } - case PID_EXT: + case PID_EXT: + case NEW_PID_EXT: factory->hp = hp; - ep = dec_pid(edep, factory, ep, objp); + ep = dec_pid(edep, factory, ep, objp, ep[-1]); hp = factory->hp; if (ep == NULL) { goto error; } break; - case PORT_EXT: + case PORT_EXT: + case NEW_PORT_EXT: { Eterm sysname; ErlNode *node; Uint num; - Uint cre; + Uint32 cre; + byte tag = ep[-1]; if ((ep = dec_atom(edep, ep, &sysname)) == NULL) { goto error; @@ -3277,12 +3331,17 @@ dec_term_atom_common: goto error; } ep += 4; - cre = get_int8(ep); - ep++; - if (!is_valid_creation(cre)) { - goto error; - } - + if (tag == PORT_EXT) { + cre = get_int8(ep); + ep++; + if (!is_valid_creation(cre)) { + goto error; + } + } + else { + cre = get_int32(ep); + ep += 4; + } node = dec_get_node(sysname, cre); if(node == erts_this_node) { *objp = make_internal_port(num); @@ -3307,7 +3366,7 @@ dec_term_atom_common: Eterm sysname; ErlNode *node; int i; - Uint cre; + Uint32 cre; Uint32 *ref_num; Uint32 r0; Uint ref_words; @@ -3331,9 +3390,6 @@ dec_term_atom_common: ref_words = get_int16(ep); ep += 2; - if (ref_words > ERTS_MAX_REF_NUMBERS) - goto error; - if ((ep = dec_atom(edep, ep, &sysname)) == NULL) goto error; @@ -3346,8 +3402,23 @@ dec_term_atom_common: ep += 4; if (r0 >= MAX_REFERENCE) goto error; + goto ref_ext_common; + + case NEWER_REFERENCE_EXT: + ref_words = get_int16(ep); + ep += 2; + + if ((ep = dec_atom(edep, ep, &sysname)) == NULL) + goto error; + + cre = get_int32(ep); + ep += 4; + r0 = get_int32(ep); /* allow full word */ + ep += 4; ref_ext_common: + if (ref_words > ERTS_MAX_REF_NUMBERS) + goto error; node = dec_get_node(sysname, cre); if(node == erts_this_node) { @@ -3701,9 +3772,9 @@ dec_term_atom_common: *objp = make_fun(funp); /* Creator pid */ - if (*ep != PID_EXT - || (ep = dec_pid(edep, factory, ++ep, - &funp->creator))==NULL) { + if ((*ep != PID_EXT && *ep != NEW_PID_EXT) + || (ep = dec_pid(edep, factory, ep+1, + &funp->creator, *ep))==NULL) { goto error; } @@ -3887,9 +3958,13 @@ error: * Must unlink all off-heap objects that may have been * linked into the process. */ - if (factory->hp < hp) { /* Sometimes we used hp and sometimes factory->hp */ - factory->hp = hp; /* the largest must be the freshest */ + if (factory->mode != FACTORY_CLOSED) { + if (factory->hp < hp) { /* Sometimes we used hp and sometimes factory->hp */ + factory->hp = hp; /* the largest must be the freshest */ + } } + else ASSERT(factory->hp == hp); + error_hamt: erts_factory_undo(factory); PSTACK_DESTROY(hamt_array); @@ -4005,20 +4080,29 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, else result += 1 + 4 + 1 + i; /* tag,size,sign,digits */ break; + case EXTERNAL_PID_DEF: + if (external_pid_creation(obj) > ERTS_MAX_LOCAL_CREATION) + result += 3; + /*fall through*/ case PID_DEF: - case EXTERNAL_PID_DEF: result += (1 + encode_size_struct2(acmp, pid_node_name(obj), dflags) + 4 + 4 + 1); break; + case EXTERNAL_REF_DEF: + if (external_ref_creation(obj) > ERTS_MAX_LOCAL_CREATION) + result += 3; + /*fall through*/ case REF_DEF: - case EXTERNAL_REF_DEF: ASSERT(dflags & DFLAG_EXTENDED_REFERENCES); i = ref_no_of_numbers(obj); result += (1 + 2 + encode_size_struct2(acmp, ref_node_name(obj), dflags) + 1 + 4*i); break; - case PORT_DEF: - case EXTERNAL_PORT_DEF: + case EXTERNAL_PORT_DEF: + if (external_port_creation(obj) > ERTS_MAX_LOCAL_CREATION) + result += 3; + /*fall through*/ + case PORT_DEF: result += (1 + encode_size_struct2(acmp, port_node_name(obj), dflags) + 4 + 1); break; @@ -4345,19 +4429,32 @@ init_done: SKIP(1+atom_extra_skip); atom_extra_skip = 0; break; - case PID_EXT: + case NEW_PID_EXT: + atom_extra_skip = 12; + goto case_PID; + case PID_EXT: atom_extra_skip = 9; + case_PID: /* In case it is an external pid */ heap_size += EXTERNAL_THING_HEAD_SIZE + 1; terms++; break; - case PORT_EXT: + case NEW_PORT_EXT: + atom_extra_skip = 8; + goto case_PORT; + case PORT_EXT: atom_extra_skip = 5; + case_PORT: /* In case it is an external port */ heap_size += EXTERNAL_THING_HEAD_SIZE + 1; terms++; break; - case NEW_REFERENCE_EXT: + case NEWER_REFERENCE_EXT: + atom_extra_skip = 4; + goto case_NEW_REFERENCE; + case NEW_REFERENCE_EXT: + atom_extra_skip = 1; + case_NEW_REFERENCE: { int id_words; @@ -4368,7 +4465,7 @@ init_done: goto error; ep += 2; - atom_extra_skip = 1 + 4*id_words; + atom_extra_skip += 4*id_words; /* In case it is an external ref */ #if defined(ARCH_64) heap_size += EXTERNAL_THING_HEAD_SIZE + id_words/2 + 1; diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h index d12051c6b4..f00426cc16 100644 --- a/erts/emulator/beam/external.h +++ b/erts/emulator/beam/external.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2014. All Rights Reserved. + * Copyright Ericsson AB 1996-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,8 +18,6 @@ * %CopyrightEnd% */ -/* Same order as the ordering of terms in erlang */ - /* Since there are 255 different External tag values to choose from There is no reason to not be extravagant. Hence, the different tags for large/small tuple e.t.c @@ -37,9 +35,12 @@ #define SMALL_ATOM_EXT 's' #define REFERENCE_EXT 'e' #define NEW_REFERENCE_EXT 'r' +#define NEWER_REFERENCE_EXT 'Z' #define PORT_EXT 'f' +#define NEW_PORT_EXT 'Y' #define NEW_FLOAT_EXT 'F' #define PID_EXT 'g' +#define NEW_PID_EXT 'X' #define SMALL_TUPLE_EXT 'h' #define LARGE_TUPLE_EXT 'i' #define NIL_EXT 'j' @@ -191,7 +192,7 @@ Eterm erts_decode_dist_ext(ErtsHeapFactory* factory, ErtsDistExternal *); Sint erts_decode_ext_size(byte*, Uint); Sint erts_decode_ext_size_ets(byte*, Uint); -Eterm erts_decode_ext(ErtsHeapFactory*, byte**); +Eterm erts_decode_ext(ErtsHeapFactory*, byte**, Uint32 flags); Eterm erts_decode_ext_ets(ErtsHeapFactory*, byte*); Eterm erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags); diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index d7edf451be..550db95ba8 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2014. All Rights Reserved. + * Copyright Ericsson AB 1996-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -44,6 +44,8 @@ #include "erl_port.h" #include "erl_gc.h" +struct enif_func_t; + struct enif_environment_t /* ErlNifEnv */ { struct erl_module_nif* mod_nif; @@ -54,14 +56,21 @@ struct enif_environment_t /* ErlNifEnv */ int fpe_was_unmasked; struct enif_tmp_obj_t* tmp_obj_list; int exception_thrown; /* boolean */ + Process *tracee; }; extern void erts_pre_nif(struct enif_environment_t*, Process*, - struct erl_module_nif*); + struct erl_module_nif*, Process* tracee); extern void erts_post_nif(struct enif_environment_t* env); extern Eterm erts_nif_taints(Process* p); extern void erts_print_nif_taints(int to, void* to_arg); void erts_unload_nif(struct erl_module_nif* nif); extern void erl_nif_init(void); +extern int erts_nif_get_funcs(struct erl_module_nif*, + struct enif_func_t **funcs); +extern Eterm erts_nif_call_function(Process *p, Process *tracee, + struct erl_module_nif*, + struct enif_func_t *, + int argc, Eterm *argv); /* Driver handle (wrapper for old plain handle) */ #define ERL_DE_OK 0 @@ -988,7 +997,7 @@ Eterm erl_is_function(Process* p, Eterm arg1, Eterm arg2); #define ERTS_CPC_ALLOW_GC (1 << 0) #define ERTS_CPC_COPY_LITERALS (1 << 1) #define ERTS_CPC_ALL (ERTS_CPC_ALLOW_GC | ERTS_CPC_COPY_LITERALS) -Eterm erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp); +Eterm erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp, int fcalls); typedef struct { Eterm *ptr; @@ -1474,9 +1483,19 @@ do { \ #define MatchSetGetSource(MPSP) erts_match_set_get_source(MPSP) -extern Binary *erts_match_set_compile(Process *p, Eterm matchexpr); +extern Binary *erts_match_set_compile(Process *p, Eterm matchexpr, Eterm MFA); Eterm erts_match_set_lint(Process *p, Eterm matchexpr); extern void erts_match_set_release_result(Process* p); +ERTS_GLB_INLINE void erts_match_set_release_result_trace(Process* p, Eterm); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE +void erts_match_set_release_result_trace(Process* p, Eterm pam_result) +{ + if (is_not_immed(pam_result)) + erts_match_set_release_result(p); +} +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ enum erts_pam_run_flags { ERTS_PAM_TMP_RESULT=1, @@ -1484,10 +1503,12 @@ enum erts_pam_run_flags { ERTS_PAM_CONTIGUOUS_TUPLE=4, ERTS_PAM_IGNORE_TRACE_SILENT=8 }; -extern Eterm erts_match_set_run(Process *p, Binary *mpsp, - Eterm *args, int num_args, - enum erts_pam_run_flags in_flags, - Uint32 *return_flags); +extern Eterm erts_match_set_run_trace(Process *p, + Process *self, + Binary *mpsp, + Eterm *args, int num_args, + enum erts_pam_run_flags in_flags, + Uint32 *return_flags); extern Eterm erts_match_set_get_source(Binary *mpsp); extern void erts_match_prog_foreach_offheap(Binary *b, void (*)(ErlOffHeap *, void *), @@ -1532,10 +1553,15 @@ ERTS_GLB_INLINE void dtrace_fun_decode(Process *process, ERTS_GLB_INLINE void dtrace_pid_str(Eterm pid, char *process_buf) { - erts_snprintf(process_buf, DTRACE_TERM_BUF_SIZE, "<%lu.%lu.%lu>", - pid_channel_no(pid), - pid_number(pid), - pid_serial(pid)); + if (is_pid(pid)) + erts_snprintf(process_buf, DTRACE_TERM_BUF_SIZE, "<%lu.%lu.%lu>", + pid_channel_no(pid), + pid_number(pid), + pid_serial(pid)); + else if (is_port(pid)) + erts_snprintf(process_buf, DTRACE_TERM_BUF_SIZE, "#Port<%lu.%lu>", + port_channel_no(pid), + port_number(pid)); } ERTS_GLB_INLINE void @@ -1547,9 +1573,7 @@ dtrace_proc_str(Process *process, char *process_buf) ERTS_GLB_INLINE void dtrace_port_str(Port *port, char *port_buf) { - erts_snprintf(port_buf, DTRACE_TERM_BUF_SIZE, "#Port<%lu.%lu>", - port_channel_no(port->common.id), - port_number(port->common.id)); + dtrace_pid_str(port->common.id, port_buf); } ERTS_GLB_INLINE void diff --git a/erts/emulator/beam/hash.c b/erts/emulator/beam/hash.c index 5a0b93f693..e255b961f1 100644 --- a/erts/emulator/beam/hash.c +++ b/erts/emulator/beam/hash.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. + * Copyright Ericsson AB 1996-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/hash.h b/erts/emulator/beam/hash.h index e94aaa0a84..9f773d8faa 100644 --- a/erts/emulator/beam/hash.h +++ b/erts/emulator/beam/hash.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. + * Copyright Ericsson AB 1996-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/index.c b/erts/emulator/beam/index.c index 5f6ea14732..26d6c04ea0 100644 --- a/erts/emulator/beam/index.c +++ b/erts/emulator/beam/index.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2012. All Rights Reserved. + * Copyright Ericsson AB 1996-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/index.h b/erts/emulator/beam/index.h index 99b2bdfab0..218779c33b 100644 --- a/erts/emulator/beam/index.h +++ b/erts/emulator/beam/index.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2013. All Rights Reserved. + * Copyright Ericsson AB 1996-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 797e4cdadc..b9f6ab04c6 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2014. All Rights Reserved. + * Copyright Ericsson AB 1996-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -47,6 +47,7 @@ #define ERTS_WANT_EXTERNAL_TAGS #include "external.h" #include "dtrace-wrapper.h" +#include "lttng-wrapper.h" #include "erl_map.h" #include "erl_bif_unique.h" #include "erl_hl_timer.h" @@ -87,7 +88,7 @@ int erts_port_parallelism = 0; static erts_atomic64_t bytes_in; static erts_atomic64_t bytes_out; -static void deliver_result(Eterm sender, Eterm pid, Eterm res); +static void deliver_result(Port *p, Eterm sender, Eterm pid, Eterm res); static int init_driver(erts_driver_t *, ErlDrvEntry *, DE_Handle *); static void terminate_port(Port *p); static void pdl_init(void); @@ -211,6 +212,7 @@ static ERTS_INLINE void kill_port(Port *pp) { ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(pp)); + ERTS_TRACER_CLEAR(&ERTS_TRACER(pp)); erts_ptab_delete_element(&erts_port, &pp->common); /* Time of death */ erts_port_task_free_port(pp); /* In non-smp case the port structure may have been deallocated now */ @@ -397,13 +399,13 @@ static Port *create_port(char *name, prt->common.u.alive.reg = NULL; ERTS_PTMR_INIT(prt); erts_port_task_handle_init(&prt->timeout_task); - prt->psd = NULL; + erts_smp_atomic_init_nob(&prt->psd, (erts_aint_t) NULL); prt->async_open_port = NULL; prt->drv_data = (SWord) 0; prt->os_pid = -1; /* Set default tracing */ - erts_get_default_tracing(&ERTS_TRACE_FLAGS(prt), &ERTS_TRACER_PROC(prt)); + erts_get_default_port_tracing(&ERTS_TRACE_FLAGS(prt), &ERTS_TRACER(prt)); ERTS_CT_ASSERT(offsetof(Port,common) == 0); @@ -708,7 +710,7 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */ if (driver->start) { ERTS_MSACC_PUSH_STATE_M(); if (IS_TRACED_FL(port, F_TRACE_SCHED_PORTS)) { - trace_sched_ports_where(port, am_in, am_start); + trace_sched_ports_where(port, am_in, am_open); } port->caller = pid; #ifdef USE_VM_PROBES @@ -717,7 +719,19 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */ DTRACE3(driver_start, process_str, driver->name, port_str); } #endif + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_PORT); + +#ifdef USE_LTTNG_VM_TRACEPOINTS + if (LTTNG_ENABLED(driver_start)) { + lttng_decl_portbuf(port_str); + lttng_decl_procbuf(proc_str); + lttng_pid_to_str(pid, proc_str); + lttng_port_to_str(port, port_str); + LTTNG3(driver_start, proc_str, driver->name, port_str); + } +#endif + fpe_was_unmasked = erts_block_fpe(); drv_data = (*driver->start)(ERTS_Port2ErlDrvPort(port), name, opts); if (((SWord) drv_data) == -1) @@ -740,7 +754,7 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */ ERTS_MSACC_POP_STATE_M(); port->caller = NIL; if (IS_TRACED_FL(port, F_TRACE_SCHED_PORTS)) { - trace_sched_ports_where(port, am_out, am_start); + trace_sched_ports_where(port, am_out, am_open); } #ifdef ERTS_SMP if (port->xports) @@ -1287,7 +1301,7 @@ try_imm_drv_call(ErtsTryImmDrvCallState *sp) reds_left_in = CONTEXT_REDS/10; else { if (IS_TRACED_FL(c_p, F_TRACE_SCHED_PROCS)) - trace_virtual_sched(c_p, am_out); + trace_sched(c_p, ERTS_PROC_LOCK_MAIN, am_out); /* * No status lock held while sending runnable * proc trace messages. It is however not needed @@ -1371,7 +1385,7 @@ finalize_imm_drv_call(ErtsTryImmDrvCallState *sp) } if (IS_TRACED_FL(c_p, F_TRACE_SCHED_PROCS)) - trace_virtual_sched(c_p, am_in); + trace_sched(c_p, ERTS_PROC_LOCK_MAIN, am_in); /* * No status lock held while sending runnable * proc trace messages. It is however not needed @@ -1418,10 +1432,11 @@ finalize_force_imm_drv_call(ErtsTryImmDrvCallState *sp) static ERTS_INLINE void queue_port_sched_op_reply(Process *rp, - ErtsProcLocks *rp_locksp, + ErtsProcLocks rp_locks, ErtsHeapFactory* factory, Uint32 *ref_num, - Eterm msg) + Eterm msg, + Port* prt) { Eterm* hp = erts_produce_heap(factory, ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE, 0); Eterm ref; @@ -1434,11 +1449,12 @@ queue_port_sched_op_reply(Process *rp, erts_factory_trim_and_close(factory, &msg, 1); - erts_queue_message(rp, rp_locksp, factory->message, msg, NIL); + erts_queue_message(rp, rp_locks, factory->message, msg, + prt ? prt->common.id : am_undefined); } static void -port_sched_op_reply(Eterm to, Uint32 *ref_num, Eterm msg) +port_sched_op_reply(Eterm to, Uint32 *ref_num, Eterm msg, Port* prt) { Process *rp = erts_proc_lookup_raw(to); if (rp) { @@ -1464,10 +1480,11 @@ port_sched_op_reply(Eterm to, Uint32 *ref_num, Eterm msg) factory.off_heap)); queue_port_sched_op_reply(rp, - &rp_locks, + rp_locks, &factory, ref_num, - msg_copy); + msg_copy, + prt); if (rp_locks) erts_smp_proc_unlock(rp, rp_locks); @@ -1522,9 +1539,8 @@ erts_schedule_proc2port_signal(Process *c_p, ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p); c_p->msg.save = c_p->msg.last; - erts_smp_proc_unlock(c_p, - (ERTS_PROC_LOCK_MAIN - | ERTS_PROC_LOCKS_MSG_RECEIVE)); + erts_smp_proc_unlock(c_p, (ERTS_PROC_LOCKS_MSG_RECEIVE + | ERTS_PROC_LOCK_MAIN)); } @@ -1542,8 +1558,19 @@ erts_schedule_proc2port_signal(Process *c_p, erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); if (sched_res != 0) { - if (refp) + if (refp) { + /* + * We need to restore the message queue save + * pointer to the beginning of the message queue + * since the caller now wont wait for a message + * containing the reference created above... + */ + ASSERT(c_p); + erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); + JOIN_MESSAGE(c_p); + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); *refp = NIL; + } return ERTS_PORT_OP_DROPPED; } return ERTS_PORT_OP_SCHEDULED; @@ -1627,7 +1654,7 @@ port_badsig(Port *prt, erts_aint32_t state, int op, state, sigdp->flags & ERTS_P2P_SIG_DATA_FLG_BAD_OUTPUT); if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY) - port_sched_op_reply(sigdp->caller, sigdp->ref, am_badarg); + port_sched_op_reply(sigdp->caller, sigdp->ref, am_badarg, prt); return ERTS_PORT_REDS_BADSIG; } /* port_badsig */ /* bad_port_signal() will @@ -1718,12 +1745,25 @@ call_driver_outputv(int bang_op, ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt) || ERTS_IS_CRASH_DUMPING); + + if (IS_TRACED_FL(prt, F_TRACE_RECEIVE)) + trace_port_receive(prt, caller, am_commandv, evp); + #ifdef USE_VM_PROBES if (DTRACE_ENABLED(driver_outputv)) { DTRACE_FORMAT_COMMON_PID_AND_PORT(caller, prt); DTRACE4(driver_outputv, process_str, port_str, prt->name, size); } #endif +#ifdef USE_LTTNG_VM_TRACEPOINTS + if (LTTNG_ENABLED(driver_outputv)) { + lttng_decl_portbuf(port_str); + lttng_decl_procbuf(proc_str); + lttng_pid_to_str(caller, proc_str); + lttng_port_to_str(prt, port_str); + LTTNG4(driver_outputv, proc_str, port_str, prt->name, size); + } +#endif prt->caller = caller; (*drv->outputv)((ErlDrvData) prt->drv_data, evp); @@ -1749,7 +1789,6 @@ cleanup_scheduled_outputv(ErlIOVec *ev, ErlDrvBinary *cbinp) driver_free_binary(ev->binv[i]); if (cbinp) driver_free_binary(cbinp); - erts_free(ERTS_ALC_T_DRV_CMD_DATA, ev); } static int @@ -1784,7 +1823,7 @@ port_sig_outputv(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *s } if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY) - port_sched_op_reply(sigdp->caller, sigdp->ref, reply); + port_sched_op_reply(sigdp->caller, sigdp->ref, reply, prt); cleanup_scheduled_outputv(sigdp->u.outputv.evp, sigdp->u.outputv.cbinp); @@ -1826,6 +1865,18 @@ call_driver_output(int bang_op, DTRACE4(driver_output, process_str, port_str, prt->name, size); } #endif +#ifdef USE_LTTNG_VM_TRACEPOINTS + if (LTTNG_ENABLED(driver_output)) { + lttng_decl_portbuf(port_str); + lttng_decl_procbuf(proc_str); + lttng_pid_to_str(caller, proc_str); + lttng_port_to_str(prt, port_str); + LTTNG4(driver_output, proc_str, port_str, prt->name, size); + } +#endif + + if (IS_TRACED_FL(prt, F_TRACE_RECEIVE)) + trace_port_receive(prt, caller, am_command, bufp, size); prt->caller = caller; (*drv->output)((ErlDrvData) prt->drv_data, bufp, size); @@ -1880,13 +1931,195 @@ port_sig_output(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *si } if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY) - port_sched_op_reply(sigdp->caller, sigdp->ref, reply); + port_sched_op_reply(sigdp->caller, sigdp->ref, reply, prt); cleanup_scheduled_output(sigdp->u.output.bufp); return ERTS_PORT_REDS_CMD_OUTPUT; } + +/* + * This erts_port_output will always create a port task. + * The call is treated as a port_command call, i.e. no + * badsig i generated if the input in invalid. However + * an error_logger message is generated. + */ +int +erts_port_output_async(Port *prt, Eterm from, Eterm list) +{ + + ErtsPortOpResult res; + ErtsProc2PortSigData *sigdp; + erts_driver_t *drv = prt->drv_ptr; + size_t size; + int task_flags; + ErtsProc2PortSigCallback port_sig_callback; + ErlDrvBinary *cbin = NULL; + ErlIOVec *evp = NULL; + char *buf = NULL; + ErtsPortTaskHandle *ns_pthp; + + if (drv->outputv) { + ErlIOVec ev; + SysIOVec* ivp; + ErlDrvBinary** bvp; + int vsize; + Uint csize; + Uint pvsize; + Uint pcsize; + size_t iov_offset, binv_offset, alloc_size; + Uint blimit = 0; + char *ptr; + int i; + + Eterm* bptr = NULL; + Uint offset; + + if (is_binary(list)) { + /* We optimize for when we get a procbin without offset */ + Eterm real_bin; + int bitoffs; + int bitsize; + ERTS_GET_REAL_BIN(list, real_bin, offset, bitoffs, bitsize); + bptr = binary_val(real_bin); + if (*bptr == HEADER_PROC_BIN && bitoffs == 0) { + size = binary_size(list); + vsize = 1; + } else + bptr = NULL; + } + + if (!bptr) { + if (io_list_vec_len(list, &vsize, &csize, &pvsize, &pcsize, &size)) + goto bad_value; + + /* To pack or not to pack (small binaries) ...? */ + if (vsize >= SMALL_WRITE_VEC) { + /* Do pack */ + vsize = pvsize + 1; + csize = pcsize; + blimit = ERL_SMALL_IO_BIN_LIMIT; + } + cbin = driver_alloc_binary(csize); + if (!cbin) + erts_alloc_enomem(ERTS_ALC_T_DRV_BINARY, ERTS_SIZEOF_Binary(csize)); + } + + + iov_offset = ERTS_ALC_DATA_ALIGN_SIZE(sizeof(ErlIOVec)); + binv_offset = iov_offset; + binv_offset += ERTS_ALC_DATA_ALIGN_SIZE((vsize+1)*sizeof(SysIOVec)); + alloc_size = binv_offset; + alloc_size += (vsize+1)*sizeof(ErlDrvBinary *); + + sigdp = erts_port_task_alloc_p2p_sig_data_extra(alloc_size, (void**)&ptr); + + evp = (ErlIOVec *) ptr; + ivp = evp->iov = (SysIOVec *) (ptr + iov_offset); + bvp = evp->binv = (ErlDrvBinary **) (ptr + binv_offset); + + ivp[0].iov_base = NULL; + ivp[0].iov_len = 0; + bvp[0] = NULL; + + if (bptr) { + ProcBin* pb = (ProcBin *) bptr; + + ivp[1].iov_base = pb->bytes+offset; + ivp[1].iov_len = size; + bvp[1] = Binary2ErlDrvBinary(pb->val); + + evp->vsize = 1; + } else { + + evp->vsize = io_list_to_vec(list, ivp+1, bvp+1, cbin, blimit); + if (evp->vsize < 0) { + if (evp != &ev) + erts_free(ERTS_ALC_T_DRV_CMD_DATA, evp); + driver_free_binary(cbin); + goto bad_value; + } + } +#if 0 + /* This assertion may say something useful, but it can + be falsified during the emulator test suites. */ + ASSERT(evp->vsize == vsize); +#endif + evp->vsize++; + evp->size = size; /* total size */ + + /* Need to increase refc on all binaries */ + for (i = 1; i < evp->vsize; i++) + if (bvp[i]) + driver_binary_inc_refc(bvp[i]); + + sigdp->flags = ERTS_P2P_SIG_TYPE_OUTPUTV; + sigdp->u.outputv.from = from; + sigdp->u.outputv.evp = evp; + sigdp->u.outputv.cbinp = cbin; + port_sig_callback = port_sig_outputv; + } else { + ErlDrvSizeT ERTS_DECLARE_DUMMY(r); + + /* + * Apperently there exist code that write 1 byte to + * much in buffer. Where it resides I don't know, but + * we can live with one byte extra allocated... + */ + + if (erts_iolist_size(list, &size)) + goto bad_value; + + buf = erts_alloc(ERTS_ALC_T_DRV_CMD_DATA, size + 1); + + r = erts_iolist_to_buf(list, buf, size); + ASSERT(ERTS_IOLIST_TO_BUF_SUCCEEDED(r)); + + sigdp = erts_port_task_alloc_p2p_sig_data(); + sigdp->flags = ERTS_P2P_SIG_TYPE_OUTPUT; + sigdp->u.output.from = from; + sigdp->u.output.bufp = buf; + sigdp->u.output.size = size; + port_sig_callback = port_sig_output; + } + sigdp->flags = 0; + ns_pthp = NULL; + task_flags = 0; + + res = erts_schedule_proc2port_signal(NULL, + prt, + ERTS_INVALID_PID, + NULL, + sigdp, + task_flags, + ns_pthp, + port_sig_callback); + + if (res != ERTS_PORT_OP_SCHEDULED) { + if (drv->outputv) + cleanup_scheduled_outputv(evp, cbin); + else + cleanup_scheduled_output(buf); + return 1; + } + return 1; + +bad_value: + + /* + * We call badsig directly here as this function is called with + * the main lock of the calling process still held. + * At the moment this operation is always not a bang_op, so + * only an error_logger message should be generated, no badsig. + */ + + badsig_received(0, prt, erts_atomic32_read_nob(&prt->state), 1); + + return 0; + +} + ErtsPortOpResult erts_port_output(Process *c_p, int flags, @@ -1896,7 +2129,7 @@ erts_port_output(Process *c_p, Eterm *refp) { ErtsPortOpResult res; - ErtsProc2PortSigData *sigdp; + ErtsProc2PortSigData *sigdp = NULL; erts_driver_t *drv = prt->drv_ptr; size_t size; int try_call; @@ -1949,7 +2182,6 @@ erts_port_output(Process *c_p, DTRACE4(port_command, process_str, port_str, prt->name, "command"); } #endif - if (drv->outputv) { ErlIOVec ev; SysIOVec iv[SMALL_WRITE_VEC]; @@ -1978,10 +2210,13 @@ erts_port_output(Process *c_p, evp = &ev; } else { - char *ptr = erts_alloc((try_call - ? ERTS_ALC_T_TMP - : ERTS_ALC_T_DRV_CMD_DATA), alloc_size); - + char *ptr; + if (try_call) { + ptr = erts_alloc(ERTS_ALC_T_TMP, alloc_size); + } else { + sigdp = erts_port_task_alloc_p2p_sig_data_extra( + alloc_size, (void**)&ptr); + } evp = (ErlIOVec *) ptr; ivp = evp->iov = (SysIOVec *) (ptr + iov_offset); bvp = evp->binv = (ErlDrvBinary **) (ptr + binv_offset); @@ -2010,9 +2245,12 @@ erts_port_output(Process *c_p, bvp[0] = NULL; evp->vsize = io_list_to_vec(list, ivp+1, bvp+1, cbin, blimit); if (evp->vsize < 0) { - if (evp != &ev) - erts_free(try_call ? ERTS_ALC_T_TMP : ERTS_ALC_T_DRV_CMD_DATA, - evp); + if (evp != &ev) { + if (try_call) + erts_free(ERTS_ALC_T_TMP, evp); + else + erts_port_task_free_p2p_sig_data(sigdp); + } driver_free_binary(cbin); goto bad_value; } @@ -2064,8 +2302,10 @@ erts_port_output(Process *c_p, /* Fall through... */ case ERTS_TRY_IMM_DRV_CALL_INVALID_PORT: driver_free_binary(cbin); - if (evp != &ev) + if (evp != &ev) { + ASSERT(!sigdp); erts_free(ERTS_ALC_T_TMP, evp); + } if (try_call_res != ERTS_TRY_IMM_DRV_CALL_OK) return ERTS_PORT_OP_DROPPED; if (c_p) @@ -2076,8 +2316,10 @@ erts_port_output(Process *c_p, if (async_nosuspend && (sched_flags & (busy_flgs|ERTS_PTS_FLG_EXIT))) { driver_free_binary(cbin); - if (evp != &ev) + if (evp != &ev) { + ASSERT(!sigdp); erts_free(ERTS_ALC_T_TMP, evp); + } return ((sched_flags & ERTS_PTS_FLG_EXIT) ? ERTS_PORT_OP_DROPPED : ERTS_PORT_OP_BUSY); @@ -2092,9 +2334,16 @@ erts_port_output(Process *c_p, if (bvp[i]) driver_binary_inc_refc(bvp[i]); - new_evp = erts_alloc(ERTS_ALC_T_DRV_CMD_DATA, alloc_size); + /* The port task and iovec is allocated in the + same structure as an optimization. This + is especially important in erts_port_output_async + of when !try_call */ + ASSERT(sigdp == NULL); + sigdp = erts_port_task_alloc_p2p_sig_data_extra( + alloc_size, (void**)&new_evp); if (evp != &ev) { + /* Copy from TMP alloc to port task */ sys_memcpy((void *) new_evp, (void *) evp, alloc_size); new_evp->iov = (SysIOVec *) (((char *) new_evp) + iov_offset); @@ -2142,7 +2391,6 @@ erts_port_output(Process *c_p, evp = new_evp; } - sigdp = erts_port_task_alloc_p2p_sig_data(); sigdp->flags = ERTS_P2P_SIG_TYPE_OUTPUTV; sigdp->u.outputv.from = from; sigdp->u.outputv.evp = evp; @@ -2351,7 +2599,10 @@ call_deliver_port_exit(int bang_op, return ERTS_PORT_OP_DROPPED; } - if (!erts_deliver_port_exit(prt, from, reason, bang_op)) + if (IS_TRACED_FL(prt, F_TRACE_RECEIVE)) + trace_port_receive(prt, from, am_close); + + if (!erts_deliver_port_exit(prt, from, reason, bang_op, broken_link)) return ERTS_PORT_OP_DROPPED; #ifdef USE_VM_PROBES @@ -2388,7 +2639,7 @@ port_sig_exit(Port *prt, if (sigdp->u.exit.bp) free_message_buffer(sigdp->u.exit.bp); if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY) - port_sched_op_reply(sigdp->caller, sigdp->ref, msg); + port_sched_op_reply(sigdp->caller, sigdp->ref, msg, prt); return ERTS_PORT_REDS_EXIT; } @@ -2422,13 +2673,13 @@ erts_port_exit(Process *c_p, ERTS_PORT_SFLGS_INVALID_LOOKUP, 0, !refp, - am_exit); + am_close); switch (try_imm_drv_call(&try_call_state)) { case ERTS_TRY_IMM_DRV_CALL_OK: { res = call_deliver_port_exit(flags & ERTS_PORT_SIG_FLG_BANG_OP, - from, + c_p ? c_p->common.id : from, prt, try_call_state.state, reason, @@ -2507,8 +2758,11 @@ set_port_connected(int bang_op, return ERTS_PORT_OP_DROPPED; } + if (IS_TRACED_FL(prt, F_TRACE_RECEIVE)) + trace_port_receive(prt, from, am_connect, connect); + ERTS_PORT_SET_CONNECTED(prt, connect); - deliver_result(prt->common.id, from, am_connected); + deliver_result(prt, prt->common.id, from, am_connected); #ifdef USE_VM_PROBES if(DTRACE_ENABLED(port_command)) { @@ -2530,10 +2784,22 @@ set_port_connected(int bang_op, erts_add_link(&ERTS_P_LINKS(rp), LINK_PID, prt->common.id); erts_add_link(&ERTS_P_LINKS(prt), LINK_PID, connect); + if (IS_TRACED_FL(rp, F_TRACE_PROCS)) + trace_proc(NULL, 0, rp, am_getting_linked, prt->common.id); + ERTS_PORT_SET_CONNECTED(prt, connect); erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); + if (IS_TRACED_FL(prt, F_TRACE_PORTS)) + trace_port(prt, am_getting_linked, connect); + if (IS_TRACED_FL(prt, F_TRACE_RECEIVE)) + trace_port_receive(prt, from, am_connect, connect); + if (IS_TRACED_FL(prt, F_TRACE_SEND)) { + Eterm hp[3]; + trace_port_send(prt, from, TUPLE2(hp, prt->common.id, am_connected), 1); + } + #ifdef USE_VM_PROBES if (DTRACE_ENABLED(port_connect)) { DTRACE_CHARBUF(process_str, DTRACE_TERM_BUF_SIZE); @@ -2566,7 +2832,7 @@ port_sig_connect(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *s msg = am_true; } if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY) - port_sched_op_reply(sigdp->caller, sigdp->ref, msg); + port_sched_op_reply(sigdp->caller, sigdp->ref, msg, prt); return ERTS_PORT_REDS_CONNECT; } @@ -2636,8 +2902,11 @@ static void port_unlink(Port *prt, Eterm from) { ErtsLink *lnk = erts_remove_link(&ERTS_P_LINKS(prt), from); - if (lnk) + if (lnk) { + if (IS_TRACED_FL(prt, F_TRACE_PORTS)) + trace_port(prt, am_getting_unlinked, from); erts_destroy_link(lnk); + } } static int @@ -2646,7 +2915,7 @@ port_sig_unlink(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *si if (op == ERTS_PROC2PORT_SIG_EXEC) port_unlink(prt, sigdp->u.unlink.from); if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY) - port_sched_op_reply(sigdp->caller, sigdp->ref, am_true); + port_sched_op_reply(sigdp->caller, sigdp->ref, am_true, prt); return ERTS_PORT_REDS_UNLINK; } @@ -2707,10 +2976,10 @@ port_link_failure(Eterm port_id, Eterm linker) NIL, NULL, 0); - if (xres >= 0 && IS_TRACED_FL(rp, F_TRACE_PROCS)) { + if (xres >= 0) { /* We didn't exit the process and it is traced */ if (IS_TRACED_FL(rp, F_TRACE_PROCS)) - trace_proc(NULL, rp, am_getting_unlinked, port_id); + trace_proc(NULL, 0, rp, am_getting_unlinked, port_id); } if (rp_locks) erts_smp_proc_unlock(rp, rp_locks); @@ -2721,10 +2990,15 @@ port_link_failure(Eterm port_id, Eterm linker) static void port_link(Port *prt, erts_aint32_t state, Eterm to) { - if (!(state & ERTS_PORT_SFLGS_INVALID_LOOKUP)) + if (IS_TRACED_FL(prt, F_TRACE_PORTS)) + trace_port(prt, am_getting_linked, to); + if (!(state & ERTS_PORT_SFLGS_INVALID_LOOKUP)) { erts_add_link(&ERTS_P_LINKS(prt), LINK_PID, to); - else + } else { port_link_failure(prt->common.id, to); + if (IS_TRACED_FL(prt, F_TRACE_PORTS)) + trace_port(prt, am_unlink, to); + } } static int @@ -2732,10 +3006,11 @@ port_sig_link(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *sigd { if (op == ERTS_PROC2PORT_SIG_EXEC) port_link(prt, state, sigdp->u.link.to); - else + else { port_link_failure(sigdp->u.link.port, sigdp->u.link.to); + } if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY) - port_sched_op_reply(sigdp->caller, sigdp->ref, am_true); + port_sched_op_reply(sigdp->caller, sigdp->ref, am_true, prt); return ERTS_PORT_REDS_LINK; } @@ -2792,7 +3067,8 @@ init_ack_send_reply(Port *port, Eterm resp) } port_sched_op_reply(port->async_open_port->to, port->async_open_port->ref, - resp); + resp, + port); erts_free(ERTS_ALC_T_PRTSD, port->async_open_port); port->async_open_port = NULL; @@ -3153,7 +3429,7 @@ static int read_linebuf(LineBufContext *bp) } static void -deliver_result(Eterm sender, Eterm pid, Eterm res) +deliver_result(Port *prt, Eterm sender, Eterm pid, Eterm res) { Process *rp; ErtsProcLocks rp_locks = 0; @@ -3161,12 +3437,22 @@ deliver_result(Eterm sender, Eterm pid, Eterm res) ERTS_SMP_CHK_NO_PROC_LOCKS; + ASSERT(!prt || prt->common.id == sender); +#ifdef ERTS_SMP + ASSERT(!prt || erts_lc_is_port_locked(prt)); +#endif + ASSERT(is_internal_port(sender) && is_internal_pid(pid)); rp = (scheduler ? erts_proc_lookup(pid) : erts_pid2proc_opt(NULL, 0, pid, 0, ERTS_P2P_FLG_INC_REFC)); + if (prt && IS_TRACED_FL(prt, F_TRACE_SEND)) { + Eterm hp[3]; + trace_port_send(prt, pid, TUPLE2(hp, sender, res), !!rp); + } + if (rp) { Eterm tuple; ErtsMessage *mp; @@ -3179,7 +3465,7 @@ deliver_result(Eterm sender, Eterm pid, Eterm res) sz_res + 3, &hp, &ohp); res = copy_struct(res, sz_res, &hp, ohp); tuple = TUPLE2(hp, sender, res); - erts_queue_message(rp, &rp_locks, mp, tuple, NIL); + erts_queue_message(rp, rp_locks, mp, tuple, sender); if (rp_locks) erts_smp_proc_unlock(rp, rp_locks); @@ -3211,6 +3497,7 @@ static void deliver_read_message(Port* prt, erts_aint32_t state, Eterm to, ErlOffHeap *ohp; ErtsProcLocks rp_locks = 0; int scheduler = erts_get_scheduler_id() != 0; + int trace_send = IS_TRACED_FL(prt, F_TRACE_SEND); ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); ERTS_SMP_CHK_NO_PROC_LOCKS; @@ -3233,7 +3520,7 @@ static void deliver_read_message(Port* prt, erts_aint32_t state, Eterm to, if (!rp) return; - mp = erts_alloc_message_heap(rp, &rp_locks, need, &hp, &ohp); + mp = erts_alloc_message_heap(trace_send ? NULL : rp, &rp_locks, need, &hp, &ohp); listp = NIL; if ((state & ERTS_PORT_SFLG_BINARY_IO) == 0) { @@ -3275,7 +3562,11 @@ static void deliver_read_message(Port* prt, erts_aint32_t state, Eterm to, tuple = TUPLE2(hp, prt->common.id, tuple); hp += 3; - erts_queue_message(rp, &rp_locks, mp, tuple, am_undefined); + if (trace_send) + trace_port_send(prt, to, tuple, 1); + + ERL_MESSAGE_TOKEN(mp) = am_undefined; + erts_queue_message(rp, rp_locks, mp, tuple, prt->common.id); if (rp_locks) erts_smp_proc_unlock(rp, rp_locks); if (!scheduler) @@ -3354,6 +3645,7 @@ deliver_vec_message(Port* prt, /* Port */ ErtsProcLocks rp_locks = 0; int scheduler = erts_get_scheduler_id() != 0; erts_aint32_t state; + int trace_send = IS_TRACED_FL(prt, F_TRACE_SEND); ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); ERTS_SMP_CHK_NO_PROC_LOCKS; @@ -3381,7 +3673,7 @@ deliver_vec_message(Port* prt, /* Port */ need += (hlen+csize)*2; } - mp = erts_alloc_message_heap(rp, &rp_locks, need, &hp, &ohp); + mp = erts_alloc_message_heap(trace_send ? NULL : rp, &rp_locks, need, &hp, &ohp); listp = NIL; iov += vsize; @@ -3442,7 +3734,11 @@ deliver_vec_message(Port* prt, /* Port */ tuple = TUPLE2(hp, prt->common.id, tuple); hp += 3; - erts_queue_message(rp, &rp_locks, mp, tuple, am_undefined); + if (IS_TRACED_FL(prt, F_TRACE_SEND)) + trace_port_send(prt, to, tuple, 1); + + ERL_MESSAGE_TOKEN(mp) = am_undefined; + erts_queue_message(rp, rp_locks, mp, tuple, prt->common.id); erts_smp_proc_unlock(rp, rp_locks); if (!scheduler) erts_proc_dec_refc(rp); @@ -3489,6 +3785,17 @@ static void flush_port(Port *p) DTRACE3(driver_flush, process_str, port_str, p->name); } #endif +#ifdef USE_LTTNG_VM_TRACEPOINTS + if (LTTNG_ENABLED(driver_flush)) { + lttng_decl_portbuf(port_str); + lttng_decl_procbuf(proc_str); + lttng_pid_to_str(ERTS_PORT_GET_CONNECTED(p), proc_str); + lttng_port_to_str(p, port_str); + LTTNG3(driver_flush, proc_str, port_str, p->name); + } +#endif + + if (IS_TRACED_FL(p, F_TRACE_SCHED_PORTS)) { trace_sched_ports_where(p, am_in, am_flush); } @@ -3520,6 +3827,7 @@ terminate_port(Port *prt) Eterm connected_id = NIL /* Initialize to silence compiler */; erts_driver_t *drv; erts_aint32_t state; + ErtsPrtSD *psd; ERTS_SMP_CHK_NO_PROC_LOCKS; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); @@ -3551,6 +3859,16 @@ terminate_port(Port *prt) DTRACE3(driver_stop, process_str, drv->name, port_str); } #endif +#ifdef USE_LTTNG_VM_TRACEPOINTS + if (LTTNG_ENABLED(driver_stop)) { + lttng_decl_portbuf(port_str); + lttng_decl_procbuf(proc_str); + lttng_pid_to_str(connected_id, proc_str); + lttng_port_to_str(prt, port_str); + LTTNG3(driver_stop, proc_str, port_str, drv->name); + } +#endif + (*drv->stop)((ErlDrvData)prt->drv_data); erts_unblock_fpe(fpe_was_unmasked); ERTS_MSACC_POP_STATE_M(); @@ -3560,6 +3878,11 @@ terminate_port(Port *prt) ASSERT(!prt->xports); #endif } + + if (is_internal_port(send_closed_port_id) + && IS_TRACED_FL(prt, F_TRACE_SEND)) + trace_port_send(prt, connected_id, am_closed, 1); + if(drv->handle != NULL) { erts_smp_rwmtx_rlock(&erts_driver_list_lock); erts_ddll_decrement_port_count(drv->handle); @@ -3573,8 +3896,9 @@ terminate_port(Port *prt) erts_cleanup_port_data(prt); - if (prt->psd) - erts_free(ERTS_ALC_T_PRTSD, prt->psd); + psd = (ErtsPrtSD *) erts_smp_atomic_read_nob(&prt->psd); + if (psd) + erts_free(ERTS_ALC_T_PRTSD, psd); ASSERT(prt->dist_entry == NULL); @@ -3590,7 +3914,7 @@ terminate_port(Port *prt) erts_flush_async_exit(erts_halt_code, ""); } if (is_internal_port(send_closed_port_id)) - deliver_result(send_closed_port_id, connected_id, am_closed); + deliver_result(NULL, send_closed_port_id, connected_id, am_closed); } void @@ -3623,7 +3947,7 @@ static void sweep_one_monitor(ErtsMonitor *mon, void *vpsc) typedef struct { - Eterm port; + Port *port; Eterm reason; } SweepContext; @@ -3632,10 +3956,13 @@ static void sweep_one_link(ErtsLink *lnk, void *vpsc) SweepContext *psc = vpsc; DistEntry *dep; Process *rp; - + Eterm port_id = psc->port->common.id; ASSERT(lnk->type == LINK_PID); - + + if (IS_TRACED_FL(psc->port, F_TRACE_PORTS)) + trace_port(psc->port, am_unlink, lnk->pid); + if (is_external_pid(lnk->pid)) { dep = external_pid_dist_entry(lnk->pid); if(dep != erts_this_dist_entry) { @@ -3648,9 +3975,9 @@ static void sweep_one_link(ErtsLink *lnk, void *vpsc) case ERTS_DSIG_PREP_NOT_CONNECTED: break; case ERTS_DSIG_PREP_CONNECTED: - erts_remove_dist_link(&dld, psc->port, lnk->pid, dep); + erts_remove_dist_link(&dld, port_id, lnk->pid, dep); erts_destroy_dist_link(&dld); - code = erts_dsig_send_exit(&dsd, psc->port, lnk->pid, + code = erts_dsig_send_exit(&dsd, port_id, lnk->pid, psc->reason); ASSERT(code == ERTS_DSIG_SEND_OK); break; @@ -3664,23 +3991,25 @@ static void sweep_one_link(ErtsLink *lnk, void *vpsc) ASSERT(is_internal_pid(lnk->pid)); rp = erts_pid2proc(NULL, 0, lnk->pid, rp_locks); if (rp) { - ErtsLink *rlnk = erts_remove_link(&ERTS_P_LINKS(rp), psc->port); + ErtsLink *rlnk = erts_remove_link(&ERTS_P_LINKS(rp), port_id); if (rlnk) { int xres = erts_send_exit_signal(NULL, - psc->port, + port_id, rp, &rp_locks, psc->reason, NIL, NULL, 0); - if (xres >= 0 && IS_TRACED_FL(rp, F_TRACE_PROCS)) { + if (xres >= 0) { + if (rp_locks & ERTS_PROC_LOCKS_XSIG_SEND) { + erts_smp_proc_unlock(rp, ERTS_PROC_LOCKS_XSIG_SEND); + rp_locks &= ~ERTS_PROC_LOCKS_XSIG_SEND; + } /* We didn't exit the process and it is traced */ - if (IS_TRACED_FL(rp, F_TRACE_PROCS)) { - trace_proc(NULL, rp, am_getting_unlinked, - psc->port); - } + if (IS_TRACED_FL(rp, F_TRACE_PROCS)) + trace_proc(NULL, 0, rp, am_getting_unlinked, port_id); } erts_destroy_link(rlnk); } @@ -3702,7 +4031,8 @@ static void sweep_one_link(ErtsLink *lnk, void *vpsc) */ int -erts_deliver_port_exit(Port *p, Eterm from, Eterm reason, int send_closed) +erts_deliver_port_exit(Port *p, Eterm from, Eterm reason, int send_closed, + int drop_normal) { ErtsLink *lnk; Eterm rreason; @@ -3732,8 +4062,10 @@ erts_deliver_port_exit(Port *p, Eterm from, Eterm reason, int send_closed) | ERTS_PORT_SFLG_CLOSING)) return 0; - if (reason == am_normal && from != ERTS_PORT_GET_CONNECTED(p) && from != p->common.id) + if (reason == am_normal && from != ERTS_PORT_GET_CONNECTED(p) + && from != p->common.id && drop_normal) { return 0; + } set_state_flags = ERTS_PORT_SFLG_EXITING; if (send_closed) @@ -3744,9 +4076,8 @@ erts_deliver_port_exit(Port *p, Eterm from, Eterm reason, int send_closed) state = erts_atomic32_read_bor_mb(&p->state, set_state_flags); state |= set_state_flags; - if (IS_TRACED_FL(p, F_TRACE_PORTS)) { + if (IS_TRACED_FL(p, F_TRACE_PORTS)) trace_port(p, am_closed, reason); - } erts_trace_check_exiting(p->common.id); @@ -3756,7 +4087,7 @@ erts_deliver_port_exit(Port *p, Eterm from, Eterm reason, int send_closed) (void) erts_unregister_name(NULL, 0, p, p->common.u.alive.reg->name); { - SweepContext sc = {p->common.id, rreason}; + SweepContext sc = {p, rreason}; lnk = ERTS_P_LINKS(p); ERTS_P_LINKS(p) = NULL; erts_sweep_links(lnk, &sweep_one_link, &sc); @@ -3878,9 +4209,22 @@ call_driver_control(Eterm caller, command, size); } #endif - + + if (IS_TRACED_FL(prt, F_TRACE_RECEIVE)) + trace_port_receive(prt, caller, am_control, command, bufp, size); + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_PORT); +#ifdef USE_LTTNG_VM_TRACEPOINTS + if (LTTNG_ENABLED(driver_control)) { + lttng_decl_procbuf(proc_str); + lttng_decl_portbuf(port_str); + lttng_pid_to_str(caller, proc_str); + lttng_port_to_str(prt, port_str); + LTTNG5(driver_control, proc_str, port_str, prt->name, command, size); + } +#endif + prt->caller = caller; cres = prt->drv_ptr->control((ErlDrvData) prt->drv_data, command, @@ -3895,6 +4239,9 @@ call_driver_control(Eterm caller, if (cres < 0) return ERTS_PORT_OP_BADARG; + if (IS_TRACED_FL(prt, F_TRACE_SEND)) + trace_port_send_binary(prt, caller, am_control, *resp_bufp, cres); + *from_size = (ErlDrvSizeT) cres; return ERTS_PORT_OP_DONE; @@ -4045,10 +4392,11 @@ port_sig_control(Port *prt, &factory.hp, factory.off_heap); queue_port_sched_op_reply(rp, - &rp_locks, + rp_locks, &factory, sigdp->ref, - msg); + msg, + prt); if (rp_locks) erts_smp_proc_unlock(rp, rp_locks); @@ -4059,7 +4407,7 @@ port_sig_control(Port *prt, /* failure */ if (sigdp->caller != ERTS_INVALID_PID) - port_sched_op_reply(sigdp->caller, sigdp->ref, am_badarg); + port_sched_op_reply(sigdp->caller, sigdp->ref, am_badarg, prt); done: @@ -4294,6 +4642,18 @@ call_driver_call(Eterm caller, DTRACE5(driver_call, process_str, port_str, prt->name, command, size); } #endif +#ifdef USE_LTTNG_VM_TRACEPOINTS + if (LTTNG_ENABLED(driver_call)) { + lttng_decl_procbuf(proc_str); + lttng_decl_portbuf(port_str); + lttng_pid_to_str(caller,proc_str); + lttng_port_to_str(prt, port_str); + LTTNG5(driver_call, proc_str, port_str, prt->name, command, size); + } +#endif + + if (IS_TRACED_FL(prt, F_TRACE_RECEIVE)) + trace_port_receive(prt, caller, am_call, command, bufp, size); ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_PORT); @@ -4313,6 +4673,9 @@ call_driver_call(Eterm caller, || ((byte) (*resp_bufp)[0]) != VERSION_MAGIC) return ERTS_PORT_OP_BADARG; + if (IS_TRACED_FL(prt, F_TRACE_SEND)) + trace_port_send_binary(prt, caller, am_call, *resp_bufp, cres); + *from_size = (ErlDrvSizeT) cres; return ERTS_PORT_OP_DONE; @@ -4373,7 +4736,7 @@ port_sig_call(Port *prt, (void) erts_factory_message_create(&factory, rp, &rp_locks, hsz); endp = (byte *) resp_bufp; - msg = erts_decode_ext(&factory, &endp); + msg = erts_decode_ext(&factory, &endp, 0); if (is_value(msg)) { hp = erts_produce_heap(&factory, 3, @@ -4381,10 +4744,11 @@ port_sig_call(Port *prt, msg = TUPLE2(hp, am_ok, msg); queue_port_sched_op_reply(rp, - &rp_locks, + rp_locks, &factory, sigdp->ref, - msg); + msg, + prt); if (rp_locks) erts_smp_proc_unlock(rp, rp_locks); @@ -4396,7 +4760,7 @@ port_sig_call(Port *prt, } } - port_sched_op_reply(sigdp->caller, sigdp->ref, am_badarg); + port_sched_op_reply(sigdp->caller, sigdp->ref, am_badarg, prt); done: @@ -4492,7 +4856,7 @@ erts_port_call(Process* c_p, hsz += 3; erts_factory_proc_prealloc_init(&factory, c_p, hsz); endp = (byte *) resp_bufp; - term = erts_decode_ext(&factory, &endp); + term = erts_decode_ext(&factory, &endp, 0); if (term == THE_NON_VALUE) return ERTS_PORT_OP_BADARG; hp = erts_produce_heap(&factory,3,0); @@ -4611,7 +4975,7 @@ port_sig_info(Port *prt, { ASSERT(sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY); if (op != ERTS_PROC2PORT_SIG_EXEC) - port_sched_op_reply(sigdp->caller, sigdp->ref, am_undefined); + port_sched_op_reply(sigdp->caller, sigdp->ref, am_undefined, prt); else { Eterm *hp, *hp_start; Uint hsz; @@ -4637,10 +5001,11 @@ port_sig_info(Port *prt, mp->data.heap_frag = bp; erts_factory_selfcontained_message_init(&factory, mp, hp); queue_port_sched_op_reply(rp, - &rp_locks, + rp_locks, &factory, sigdp->ref, - value); + value, + prt); } if (rp_locks) erts_smp_proc_unlock(rp, rp_locks); @@ -4756,7 +5121,8 @@ reply_io_bytes(void *vreq) eout = erts_bld_uint64(&hp, NULL, out); msg = TUPLE4(hp, ref, make_small(sched_id), ein, eout); - erts_queue_message(rp, &rp_locks, mp, msg, NIL); + + erts_queue_message(rp, rp_locks, mp, msg, am_system); if (req->sched_id == sched_id) rp_locks &= ~ERTS_PROC_LOCK_MAIN; @@ -5056,6 +5422,15 @@ int async_ready(Port *p, void* data) DTRACE3(driver_ready_async, process_str, port_str, p->name); } #endif +#ifdef USE_LTTNG_VM_TRACEPOINTS + if (LTTNG_ENABLED(driver_ready_async)) { + lttng_decl_portbuf(port_str); + lttng_decl_procbuf(proc_str); + lttng_pid_to_str(ERTS_PORT_GET_CONNECTED(p), proc_str); + lttng_port_to_str(p, port_str); + LTTNG3(driver_ready_async, proc_str, port_str, p->name); + } +#endif (*p->drv_ptr->ready_async)((ErlDrvData)p->drv_data, data); need_free = 0; ERTS_MSACC_POP_STATE_M(); @@ -5218,6 +5593,7 @@ void driver_report_exit(ErlDrvPort ix, int status) ErtsProcLocks rp_locks = 0; int scheduler = erts_get_scheduler_id() != 0; Port* prt = erts_drvport2port(ix); + int trace_send = IS_TRACED_FL(prt, F_TRACE_SEND); if (prt == ERTS_INVALID_ERL_DRV_PORT) return; @@ -5234,13 +5610,17 @@ void driver_report_exit(ErlDrvPort ix, int status) if (!rp) return; - mp = erts_alloc_message_heap(rp, &rp_locks, 3+3, &hp, &ohp); + mp = erts_alloc_message_heap(trace_send ? NULL : rp, &rp_locks, 3+3, &hp, &ohp); tuple = TUPLE2(hp, am_exit_status, make_small(status)); hp += 3; tuple = TUPLE2(hp, prt->common.id, tuple); - erts_queue_message(rp, &rp_locks, mp, tuple, am_undefined); + if (IS_TRACED_FL(prt, F_TRACE_SEND)) + trace_port_send(prt, pid, tuple, 1); + + ERL_MESSAGE_TOKEN(mp) = am_undefined; + erts_queue_message(rp, rp_locks, mp, tuple, prt->common.id); erts_smp_proc_unlock(rp, rp_locks); if (!scheduler) @@ -5334,13 +5714,13 @@ cleanup_b2t_states(struct b2t_states__ *b2tsp) */ static int -driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) +driver_deliver_term(Port *prt, Eterm to, ErlDrvTermData* data, int len) { #define HEAP_EXTRA 200 #define ERTS_DDT_FAIL do { res = -1; goto done; } while (0) Uint need = 0; int depth = 0; - int res; + int res = 0; ErlDrvTermData* ptr; ErlDrvTermData* ptr_end; DECLARE_ESTACK(stack); @@ -5559,11 +5939,26 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) ? erts_proc_lookup(to) : erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_INC_REFC)); if (!rp) { - res = 0; - goto done; - } + if (!prt || !IS_TRACED_FL(prt, F_TRACE_SEND)) + goto done; + if (!erts_is_tracer_proc_enabled(NULL, 0, &prt->common, am_send)) + goto done; + + res = -2; - (void) erts_factory_message_create(&factory, rp, &rp_locks, need); + /* We allocate a temporary heap to be used to create + the message that may be sent using tracing */ + erts_factory_tmp_init(&factory, erts_alloc(ERTS_ALC_T_DRIVER, need*sizeof(Eterm)), + need, ERTS_ALC_T_DRIVER); + + } else { + /* We force the creation of a heap fragment (rp == NULL) when send + tracing so that we don't have the main lock of the process while + tracing */ + Process *trace_rp = prt && IS_TRACED_FL(prt, F_TRACE_SEND) ? NULL : rp; + (void) erts_factory_message_create(&factory, trace_rp, &rp_locks, need); + res = 1; + } /* * Interpret the instructions and build the term. @@ -5826,22 +6221,45 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) ESTACK_PUSH(stack, mess); } - res = 1; - done: if (res > 0) { + Eterm from = am_undefined; mess = ESTACK_POP(stack); /* get resulting value */ erts_factory_trim_and_close(&factory, &mess, 1); + + if (prt) { + if (IS_TRACED_FL(prt, F_TRACE_SEND)) { + trace_port_send(prt, to, mess, 1); + } + from = prt->common.id; + } + /* send message */ - erts_queue_message(rp, &rp_locks, factory.message, mess, am_undefined); + ERL_MESSAGE_TOKEN(factory.message) = am_undefined; + erts_queue_message(rp, rp_locks, factory.message, mess, from); + } + else if (res == -2) { + /* this clause only happens when we were requested to + generate a send trace, but the process to send to + did not exist any more */ + mess = ESTACK_POP(stack); /* get resulting value */ + + trace_port_send(prt, to, mess, 0); + + erts_factory_trim_and_close(&factory, &mess, 1); + erts_free(ERTS_ALC_T_DRIVER, factory.hp_start); + res = 0; } else { if (b2t.ix > b2t.used) b2t.used = b2t.ix; for (b2t.ix = 0; b2t.ix < b2t.used; b2t.ix++) erts_binary2term_abort(&b2t.state[b2t.ix]); - erts_factory_undo(&factory); + if (factory.mode != FACTORY_CLOSED) { + ERL_MESSAGE_TERM(factory.message) = am_undefined; + erts_factory_undo(&factory); + } } if (rp) { if (rp_locks) @@ -5857,7 +6275,8 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) } static ERTS_INLINE int -deliver_term_check_port(ErlDrvTermData port_id, Eterm *connected_p) +deliver_term_check_port(ErlDrvTermData port_id, Eterm *connected_p, + Port **trace_prt) { #ifdef ERTS_SMP ErtsThrPrgrDelayHandle dhndl = erts_thr_progress_unmanaged_delay(); @@ -5885,8 +6304,11 @@ deliver_term_check_port(ErlDrvTermData port_id, Eterm *connected_p) if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED) { erts_thr_progress_unmanaged_continue(dhndl); ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); - } + } else #endif + { + *trace_prt = prt; + } ERTS_SMP_LC_ASSERT(dhndl == ERTS_THR_PRGR_DHANDLE_MANAGED ? erts_lc_is_port_locked(prt) : !erts_lc_is_port_locked(prt)); @@ -5897,10 +6319,11 @@ int erl_drv_output_term(ErlDrvTermData port_id, ErlDrvTermData* data, int len) { /* May be called from arbitrary thread */ Eterm connected; - int res = deliver_term_check_port(port_id, &connected); + Port *prt = NULL; + int res = deliver_term_check_port(port_id, &connected, &prt); if (res <= 0) return res; - return driver_deliver_term(connected, data, len); + return driver_deliver_term(prt, connected, data, len); } /* @@ -5924,7 +6347,7 @@ driver_output_term(ErlDrvPort drvport, ErlDrvTermData* data, int len) if (state & ERTS_PORT_SFLG_CLOSING) return 0; - return driver_deliver_term(ERTS_PORT_GET_CONNECTED(prt), data, len); + return driver_deliver_term(prt, ERTS_PORT_GET_CONNECTED(prt), data, len); } int erl_drv_send_term(ErlDrvTermData port_id, @@ -5933,10 +6356,11 @@ int erl_drv_send_term(ErlDrvTermData port_id, int len) { /* May be called from arbitrary thread */ - int res = deliver_term_check_port(port_id, NULL); + Port *prt = NULL; + int res = deliver_term_check_port(port_id, NULL, &prt); if (res <= 0) return res; - return driver_deliver_term(to, data, len); + return driver_deliver_term(prt, to, data, len); } /* @@ -5955,20 +6379,21 @@ driver_send_term(ErlDrvPort drvport, * to make this access safe without using a less efficient * internal data representation for ErlDrvPort. */ + Port* prt = NULL; ERTS_SMP_CHK_NO_PROC_LOCKS; #ifdef ERTS_SMP if (erts_thr_progress_is_managed_thread()) #endif { erts_aint32_t state; - Port* prt = erts_drvport2port_state(drvport, &state); + prt = erts_drvport2port_state(drvport, &state); if (prt == ERTS_INVALID_ERL_DRV_PORT) return -1; /* invalid (dead) */ ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); if (state & ERTS_PORT_SFLG_CLOSING) return 0; } - return driver_deliver_term(to, data, len); + return driver_deliver_term(prt, to, data, len); } @@ -7102,6 +7527,15 @@ void erts_fire_port_monitor(Port *prt, Eterm ref) DTRACE3(driver_process_exit, process_str, port_str, prt->name); } #endif +#ifdef USE_LTTNG_VM_TRACEPOINTS + if (LTTNG_ENABLED(driver_process_exit)) { + lttng_decl_portbuf(port_str); + lttng_decl_procbuf(proc_str); + lttng_pid_to_str(ERTS_PORT_GET_CONNECTED(prt), proc_str); + lttng_port_to_str(prt, port_str); + LTTNG3(driver_process_exit, proc_str, port_str, prt->name); + } +#endif fpe_was_unmasked = erts_block_fpe(); (*callback)((ErlDrvData) (prt->drv_data), &drv_monitor); erts_unblock_fpe(fpe_was_unmasked); @@ -7135,7 +7569,7 @@ driver_failure_term(ErlDrvPort ix, Eterm term, int eof) if (state & ERTS_PORT_SFLG_CLOSING) { terminate_port(prt); } else if (eof && (state & ERTS_PORT_SFLG_SOFT_EOF)) { - deliver_result(prt->common.id, ERTS_PORT_GET_CONNECTED(prt), am_eof); + deliver_result(prt, prt->common.id, ERTS_PORT_GET_CONNECTED(prt), am_eof); } else { /* XXX UGLY WORK AROUND, Let erts_deliver_port_exit() terminate the port */ if (prt->port_data_lock) @@ -7143,7 +7577,7 @@ driver_failure_term(ErlDrvPort ix, Eterm term, int eof) prt->ioq.size = 0; if (prt->port_data_lock) driver_pdl_unlock(prt->port_data_lock); - erts_deliver_port_exit(prt, prt->common.id, eof ? am_normal : term, 0); + erts_deliver_port_exit(prt, prt->common.id, eof ? am_normal : term, 0, 0); } return 0; } @@ -7579,6 +8013,8 @@ init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle) int fpe_was_unmasked = erts_block_fpe(); DTRACE4(driver_init, drv->name, drv->version.major, drv->version.minor, drv->flags); + LTTNG4(driver_init, drv->name, drv->version.major, drv->version.minor, + drv->flags); res = (*de->init)(); erts_unblock_fpe(fpe_was_unmasked); return res; diff --git a/erts/emulator/beam/lttng-wrapper.h b/erts/emulator/beam/lttng-wrapper.h new file mode 100644 index 0000000000..294872c365 --- /dev/null +++ b/erts/emulator/beam/lttng-wrapper.h @@ -0,0 +1,107 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 1996-2016. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifndef __LTTNG_WRAPPER_H__ +#define __LTTNG_WRAPPER_H__ + +#ifdef USE_LTTNG + +#include "erlang_lttng.h" +#define USE_LTTNG_VM_TRACEPOINTS + +#define LTTNG_BUFFER_SZ (256) +#define LTTNG_PROC_BUFFER_SZ (16) +#define LTTNG_PORT_BUFFER_SZ (20) +#define LTTNG_MFA_BUFFER_SZ (256) + +#define lttng_decl_procbuf(Name) \ + char Name[LTTNG_PROC_BUFFER_SZ] + +#define lttng_decl_portbuf(Name) \ + char Name[LTTNG_PORT_BUFFER_SZ] + +#define lttng_decl_mfabuf(Name) \ + char Name[LTTNG_MFA_BUFFER_SZ] + +#define lttng_decl_carrier_stats(Name) \ + lttng_carrier_stats_t Name##_STATSTRUCT, *Name = &Name##_STATSTRUCT + +#define lttng_pid_to_str(pid, name) \ + erts_snprintf(name, LTTNG_PROC_BUFFER_SZ, "%T", (pid)) + +#define lttng_portid_to_str(pid, name) \ + erts_snprintf(name, LTTNG_PORT_BUFFER_SZ, "%T", (pid)) + +#define lttng_proc_to_str(p, name) \ + lttng_pid_to_str(((p) ? (p)->common.id : ERTS_INVALID_PID), name) + +#define lttng_port_to_str(p, name) \ + lttng_portid_to_str(((p) ? (p)->common.id : ERTS_INVALID_PORT), name) + +#define lttng_mfa_to_str(m,f,a, Name) \ + erts_snprintf(Name, LTTNG_MFA_BUFFER_SZ, "%T:%T/%lu", (Eterm)(m), (Eterm)(f), (Uint)(a)) + +#define lttng_proc_to_mfa_str(p, Name) \ + do { \ + if (ERTS_PROC_IS_EXITING((p))) { \ + strcpy(Name, "<exiting>"); \ + } else { \ + BeamInstr *_fptr = find_function_from_pc((p)->i); \ + if (_fptr) { \ + lttng_mfa_to_str(_fptr[0],_fptr[1],_fptr[2], Name); \ + } else { \ + strcpy(Name, "<unknown>"); \ + } \ + } \ + } while(0) + +/* ErtsRunQueue->ErtsSchedulerData->Uint */ +#define lttng_rq_to_id(RQ) \ + (RQ)->scheduler->no + +#define LTTNG_ENABLED(Name) \ + tracepoint_enabled(com_ericsson_otp, Name) + +/* include a special LTTNG_DO for do_tracepoint ? */ +#define LTTNG1(Name, Arg1) \ + tracepoint(com_ericsson_otp, Name, (Arg1)) + +#define LTTNG2(Name, Arg1, Arg2) \ + tracepoint(com_ericsson_otp, Name, (Arg1), (Arg2)) + +#define LTTNG3(Name, Arg1, Arg2, Arg3) \ + tracepoint(com_ericsson_otp, Name, (Arg1), (Arg2), (Arg3)) + +#define LTTNG4(Name, Arg1, Arg2, Arg3, Arg4) \ + tracepoint(com_ericsson_otp, Name, (Arg1), (Arg2), (Arg3), (Arg4)) + +#define LTTNG5(Name, Arg1, Arg2, Arg3, Arg4, Arg5) \ + tracepoint(com_ericsson_otp, Name, (Arg1), (Arg2), (Arg3), (Arg4), (Arg5)) + +#else /* USE_LTTNG */ + +#define LTTNG1(Name, Arg1) do {} while(0) +#define LTTNG2(Name, Arg1, Arg2) do {} while(0) +#define LTTNG3(Name, Arg1, Arg2, Arg3) do {} while(0) +#define LTTNG4(Name, Arg1, Arg2, Arg3, Arg4) do {} while(0) +#define LTTNG5(Name, Arg1, Arg2, Arg3, Arg4, Arg5) do {} while(0) + +#endif /* USE_LTTNG */ +#endif /* __LTTNG_WRAPPER_H__ */ diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c index f5c7b177d3..3eb11f1693 100644 --- a/erts/emulator/beam/module.c +++ b/erts/emulator/beam/module.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2013. All Rights Reserved. + * Copyright Ericsson AB 1996-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h index b7468b0926..4e12731d85 100644 --- a/erts/emulator/beam/module.h +++ b/erts/emulator/beam/module.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2013. All Rights Reserved. + * Copyright Ericsson AB 1996-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab index 9e53b4bfcc..879daaca0a 100644 --- a/erts/emulator/beam/ops.tab +++ b/erts/emulator/beam/ops.tab @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1997-2013. All Rights Reserved. +# Copyright Ericsson AB 1997-2016. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -181,11 +181,6 @@ i_jump_on_val_zero y f I i_jump_on_val x f I I i_jump_on_val y f I I -jump Target | label Lbl | same_label(Target, Lbl) => label Lbl - -is_ne_exact L1 S1 S2 | jump Fail | label L2 | same_label(L1, L2) => \ - is_eq_exact Fail S1 S2 | label L2 - %macro: get_list GetList -pack get_list x x x get_list x x y @@ -234,6 +229,9 @@ i_get_tuple_element y P y %macro: i_get_tuple_element2 GetTupleElement2 -pack i_get_tuple_element2 x P x +%macro: i_get_tuple_element2y GetTupleElement2Y -pack +i_get_tuple_element2y x P y y + %macro: i_get_tuple_element3 GetTupleElement3 -pack i_get_tuple_element3 x P x @@ -256,7 +254,14 @@ case_end x badmatch x if_end -raise s s + +# Operands for raise/2 are almost always in x(2) and x(1). +# Optimize for that case. +raise x==2 x==1 => i_raise +raise Trace=y Value=y => move Trace x=2 | move Value x=1 | i_raise +raise Trace Value => move Trace x=3 | move Value x=1 | move x=3 x=2 | i_raise + +i_raise # Internal now, but could be useful to make known to the compiler. badarg j @@ -303,7 +308,7 @@ move_window5 x x x x x y # Swap registers. move R1=x Tmp=x | move R2=xy R1 | move Tmp R2 => swap_temp R1 R2 Tmp -swap_temp R1 R2 Tmp | line Loc | apply Live | is_killed(Tmp, Live) => \ +swap_temp R1 R2 Tmp | line Loc | apply Live | is_killed_apply(Tmp, Live) => \ swap R1 R2 | line Loc | apply Live swap_temp R1 R2 Tmp | line Loc | call Live Addr | is_killed(Tmp, Live) => \ @@ -527,9 +532,9 @@ i_put_tuple x I i_put_tuple y I # -# The instruction "put_list Const [] Dst" will not be generated by -# the current BEAM compiler. But until R15A, play it safe by handling -# that instruction with the following transformation. +# The instruction "put_list Const [] Dst" were generated in rare +# circumstances up to and including OTP 18. Starting with OTP 19, +# AFAIK, it should never be generated. # put_list Const=c n Dst => move Const x | put_list x n Dst @@ -647,6 +652,9 @@ get_tuple_element Reg=x P1 D1=x | get_tuple_element Reg=x P2 D2=x | \ get_tuple_element Reg=x P1 D1=x | get_tuple_element Reg=x P2 D2=x | \ succ(P1, P2) | succ(D1, D2) => i_get_tuple_element2 Reg P1 D1 +get_tuple_element Reg=x P1 D1=y | get_tuple_element Reg=x P2 D2=y | \ + succ(P1, P2) => i_get_tuple_element2y Reg P1 D1 D2 + get_tuple_element Reg P Dst => i_get_tuple_element Reg P Dst is_integer Fail=f i => @@ -1355,9 +1363,7 @@ bs_put_utf8 Fail u Src=s => i_bs_put_utf8 Fail Src i_bs_put_utf8 j s -bs_put_utf16 Fail Flags=u Src=s => i_bs_put_utf16 Fail Flags Src - -i_bs_put_utf16 j I s +bs_put_utf16 j I s bs_put_utf32 Fail=j Flags=u Src=s => \ i_bs_validate_unicode Fail Src | bs_put_integer Fail i=32 u=1 Flags Src @@ -1539,7 +1545,6 @@ gen_minus p Live Reg=d Int=i Dst | negation_is_small(Int) => \ # GCing arithmetic instructions. # -gen_plus Fail Live Y=y X=x Dst => i_plus Fail Live X Y Dst gen_plus Fail Live S1 S2 Dst => i_plus Fail Live S1 S2 Dst gen_minus Fail Live S1 S2 Dst => i_minus Fail Live S1 S2 Dst diff --git a/erts/emulator/beam/packet_parser.c b/erts/emulator/beam/packet_parser.c index a737a86f14..f14910bc72 100644 --- a/erts/emulator/beam/packet_parser.c +++ b/erts/emulator/beam/packet_parser.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2008-2013. All Rights Reserved. + * Copyright Ericsson AB 2008-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/packet_parser.h b/erts/emulator/beam/packet_parser.h index 717d905fad..358d650804 100644 --- a/erts/emulator/beam/packet_parser.h +++ b/erts/emulator/beam/packet_parser.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2008-2009. All Rights Reserved. + * Copyright Ericsson AB 2008-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/register.c b/erts/emulator/beam/register.c index ebc35b0c4d..77f79fcea4 100644 --- a/erts/emulator/beam/register.c +++ b/erts/emulator/beam/register.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2013. All Rights Reserved. + * Copyright Ericsson AB 1996-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -226,7 +226,8 @@ int erts_register_name(Process *c_p, Eterm name, Eterm id) rp = (RegProc*) hash_put(&process_reg, (void*) &r); if (proc && rp->p == proc) { if (IS_TRACED_FL(proc, F_TRACE_PROCS)) { - trace_proc(c_p, proc, am_register, name); + trace_proc(proc, ERTS_PROC_LOCK_MAIN, + proc, am_register, name); } proc->common.u.alive.reg = rp; } @@ -470,8 +471,8 @@ int erts_unregister_name(Process *c_p, int res = 0; RegProc r, *rp; Port *port = c_prt; + ErtsProcLocks current_c_p_locks = 0; #ifdef ERTS_SMP - ErtsProcLocks current_c_p_locks; /* * SMP note: If 'c_prt != NULL' and 'c_prt->reg->name == name', @@ -537,8 +538,12 @@ int erts_unregister_name(Process *c_p, ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(port)); rp->pt->common.u.alive.reg = NULL; - + if (IS_TRACED_FL(port, F_TRACE_PORTS)) { + if (current_c_p_locks) { + erts_smp_proc_unlock(c_p, current_c_p_locks); + current_c_p_locks = 0; + } trace_port(port, am_unregister, r.name); } @@ -555,7 +560,8 @@ int erts_unregister_name(Process *c_p, #endif rp->p->common.u.alive.reg = NULL; if (IS_TRACED_FL(rp->p, F_TRACE_PROCS)) { - trace_proc(c_p, rp->p, am_unregister, r.name); + trace_proc(rp->p, (c_p == rp->p) ? c_p_locks : ERTS_PROC_LOCK_MAIN, + rp->p, am_unregister, r.name); } #ifdef ERTS_SMP if (rp->p != c_p) { diff --git a/erts/emulator/beam/register.h b/erts/emulator/beam/register.h index 144536f34b..88ab7b7bf1 100644 --- a/erts/emulator/beam/register.h +++ b/erts/emulator/beam/register.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2012. All Rights Reserved. + * Copyright Ericsson AB 1996-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/safe_hash.c b/erts/emulator/beam/safe_hash.c index 3f039c8dfd..30b26a7296 100644 --- a/erts/emulator/beam/safe_hash.c +++ b/erts/emulator/beam/safe_hash.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2008-2011. All Rights Reserved. + * Copyright Ericsson AB 2008-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/safe_hash.h b/erts/emulator/beam/safe_hash.h index a11370813c..6910b33004 100644 --- a/erts/emulator/beam/safe_hash.h +++ b/erts/emulator/beam/safe_hash.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2008-2009. All Rights Reserved. + * Copyright Ericsson AB 2008-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index 03b9088adc..dda3ba565c 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2013. All Rights Reserved. + * Copyright Ericsson AB 1996-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -92,6 +92,9 @@ #define ErtsInArea(ptr,start,nbytes) \ ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes)) +#define ErtsContainerStruct(ptr, type, member) \ + (type *)((char *)(1 ? (ptr) : &((type *)0)->member) - offsetof(type, member)) + #if defined (__WIN32__) # include "erl_win_sys.h" #else @@ -138,10 +141,10 @@ typedef ERTS_SYS_FD_TYPE ErtsSysFdType; #endif #if ERTS_AT_LEAST_GCC_VSN__(2, 96, 0) -#ifndef __llvm__ -# define ERTS_WRITE_UNLIKELY(X) X __attribute__ ((section ("ERTS_LOW_WRITE") )) -#else +#if (defined(__APPLE__) && defined(__MACH__)) || defined(__DARWIN__) # define ERTS_WRITE_UNLIKELY(X) X __attribute__ ((section ("__DATA,ERTS_LOW_WRITE") )) +#else +# define ERTS_WRITE_UNLIKELY(X) X __attribute__ ((section ("ERTS_LOW_WRITE") )) #endif #else # define ERTS_WRITE_UNLIKELY(X) X @@ -644,6 +647,15 @@ typedef struct preload { unsigned char* code; /* Code pointer */ } Preload; +/* + * ErtsTracer is either NIL, 'true' or [Mod | State] + * + * If set to NIL, it means no tracer. + * If set to 'true' it means the current process' tracer. + * If set to [Mod | State], there is a tracer. + * See erts_tracer_update for more details + */ +typedef Eterm ErtsTracer; /* * This structure contains options to all built in drivers. diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c index 0ce4f443f3..6f15082130 100644 --- a/erts/emulator/beam/time.c +++ b/erts/emulator/beam/time.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2013. All Rights Reserved. + * Copyright Ericsson AB 1996-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 66fcfcf6ce..cedc88e5fe 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2014. All Rights Reserved. + * Copyright Ericsson AB 1996-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -71,41 +71,6 @@ #define HAVE_MALLOPT 0 #endif -/* profile_scheduler mini message queue */ - -typedef struct { - Uint scheduler_id; - Uint no_schedulers; - Uint Ms; - Uint s; - Uint us; - Eterm state; -} profile_sched_msg; - -typedef struct { - profile_sched_msg msg[2]; - Uint n; -} profile_sched_msg_q; - -#ifdef ERTS_SMP - -#if 0 /* Unused */ -static void -dispatch_profile_msg_q(profile_sched_msg_q *psmq) -{ - int i = 0; - profile_sched_msg *msg = NULL; - ASSERT(psmq != NULL); - for (i = 0; i < psmq->n; i++) { - msg = &(psmq->msg[i]); - profile_scheduler_q(make_small(msg->scheduler_id), msg->state, am_undefined, msg->Ms, msg->s, msg->us); - } -} -#endif - -#endif - - Eterm* erts_heap_alloc(Process* p, Uint need, Uint xtra) { @@ -2298,7 +2263,7 @@ static void do_send_logger_message(Eterm *hp, ErlOffHeap *ohp, ErlHeapFragment * { ErtsMessage *mp = erts_alloc_message(0, NULL); mp->data.heap_frag = bp; - erts_queue_message(p, NULL /* only used for smp build */, mp, message, NIL); + erts_queue_message(p, 0, mp, message, am_system); } #endif } diff --git a/erts/emulator/beam/version.h b/erts/emulator/beam/version.h index 004725eaf5..0fa775fe8c 100644 --- a/erts/emulator/beam/version.h +++ b/erts/emulator/beam/version.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. + * Copyright Ericsson AB 1996-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. |